Merge pull request #423 from Blobscan/perf/db/add-blob-block-columns

feat: add block hash, number and timestamp field to `BlobsOnTransactions` model

.changeset/smooth-llamas-doubt.md

@@ -0,0 +1,6 @@
+---
+"@blobscan/api": minor
+"@blobscan/db": minor
+---
+
+Introduced block number and timestamp fields to the `BlobsOnTransactions` model, enabling faster and more efficient sorting and filtering when retrieving multiple blobs.

clis/stats-aggregation-cli/src/runner.ts

@@ -10,5 +10,6 @@ main()
 
     return process.exit(1);
   })
-
+
+  // eslint-disable-next-line @typescript-eslint/no-misused-promises
   .finally(gracefulShutdown);

packages/api/src/middlewares/withExpands.ts

@@ -47,14 +47,12 @@ const expandedBlockSelect = Prisma.validator<Prisma.BlockSelect>()({
   blobGasPrice: true,
   blobGasUsed: true,
   excessBlobGas: true,
-  hash: true,
-  timestamp: true,
   slot: true,
 });
 
 export type ExpandedBlock = Partial<DBBlock>;
 
-export type ExpandedTransaction = MakeFieldsMandatory<DBTransaction, "hash"> &
+export type ExpandedTransaction = Partial<DBTransaction> &
   Partial<DerivedTxBlobGasFields>;
 
 export type ExpandedBlob = MakeFieldsMandatory<

packages/api/src/routers/blob/common/selects.ts

@@ -1,10 +1,7 @@
 import { Prisma } from "@blobscan/db";
 
 import type { Expands } from "../../../middlewares/withExpands";
-import {
-  dataStorageReferencesSelect,
-  transactionReferenceSelect,
-} from "../../../utils";
+import { dataStorageReferencesSelect } from "../../../utils";
 
 export const baseBlobSelect = Prisma.validator<Prisma.BlobSelect>()({
   commitment: true,
@@ -16,27 +13,27 @@ export const baseBlobSelect = Prisma.validator<Prisma.BlobSelect>()({
   },
 });
 
-function createTransactionSelect(expands: Expands) {
-  return Prisma.validator<Prisma.TransactionSelect>()({
-    ...(expands.transaction?.select ?? {}),
-    ...transactionReferenceSelect,
-    rollup: true,
-    blockHash: true,
-    blockNumber: true,
-    blockTimestamp: true,
-    block: expands.block,
-  });
-}
-
 export function createBlobSelect(expands: Expands) {
   return Prisma.validator<Prisma.BlobSelect>()({
     ...baseBlobSelect,
     transactions: {
       select: {
+        blockHash: true,
+        blockNumber: true,
+        blockTimestamp: true,
         index: true,
+        txHash: true,
         transaction: {
-          select: createTransactionSelect(expands),
+          select: {
+            rollup: true,
+            ...(expands.transaction?.select ?? {}),
+          },
         },
+        ...(expands.block
+          ? {
+              block: expands.block,
+            }
+          : {}),
       },
     },
   });
@@ -45,11 +42,20 @@ export function createBlobSelect(expands: Expands) {
 export function createBlobsOnTransactionsSelect(expands: Expands) {
   return Prisma.validator<Prisma.BlobsOnTransactionsSelect>()({
     index: true,
+    blobHash: true,
+    blockHash: true,
+    blockNumber: true,
+    blockTimestamp: true,
+    txHash: true,
     blob: {
       select: baseBlobSelect,
     },
     transaction: {
-      select: createTransactionSelect(expands),
+      select: {
+        rollup: true,
+        ...(expands.transaction?.select ?? {}),
+      },
     },
+    ...(expands.block ? { block: expands.block } : {}),
   });
 }

packages/api/src/routers/blob/common/serializers.ts

@@ -2,7 +2,6 @@ import type {
   Blob as DBBlob,
   BlobsOnTransactions as DBBlobsOnTransactions,
   BlobDataStorageReference as DBBlobDataStorageReference,
-  Transaction as DBTransaction,
 } from "@blobscan/db";
 import { z } from "@blobscan/zod";
 
@@ -12,14 +11,17 @@ import {
   serializedExpandedBlockSchema,
   serializedExpandedTransactionSchema,
 } from "../../../middlewares/withExpands";
-import type { ExpandedBlock } from "../../../middlewares/withExpands";
+import type {
+  ExpandedBlock,
+  ExpandedTransaction,
+} from "../../../middlewares/withExpands";
 import {
   blobIndexSchema,
   blockNumberSchema,
   serializedBlobDataStorageReferenceSchema,
-  isEmptyObject,
   serializeBlobDataStorageReferences,
   serializeDate,
+  isEmptyObject,
 } from "../../../utils";
 
 type BaseBlob = Pick<
@@ -32,21 +34,18 @@ type BaseBlob = Pick<
   >[];
 };
 
-type Transaction = Omit<DBTransaction, "insertedAt" | "updatedAt"> & {
-  block?: ExpandedBlock;
-};
-
-type Blob = BaseBlob & {
+type QueriedBlob = BaseBlob & {
   data: string;
-  transactions: {
-    index: number;
-    transaction: Transaction;
-  }[];
+  transactions: (Omit<DBBlobsOnTransactions, "blobHash"> & {
+    block?: ExpandedBlock;
+    transaction?: ExpandedTransaction;
+  })[];
 };
 
-type BlobOnTransaction = Pick<DBBlobsOnTransactions, "index"> & {
+type QueriedBlobOnTransaction = DBBlobsOnTransactions & {
   blob: BaseBlob;
-  transaction: Transaction;
+  block?: ExpandedBlock;
+  transaction: ExpandedTransaction;
 };
 
 export type SerializedBlobDataStorageReference = z.infer<
@@ -119,50 +118,70 @@ export function serializeBaseBlob({
 }
 
 export function serializeBlobOnTransaction(
-  blobOnTransaction: BlobOnTransaction
+  blobOnTransaction: QueriedBlobOnTransaction
 ): SerializedBlobOnTransaction {
-  const { blob, transaction, index } = blobOnTransaction;
-
-  const { hash, blockHash, blockTimestamp, blockNumber, block } = transaction;
-
-  const expandedBlock = block ? serializeExpandedBlock(block) : undefined;
-  const expandedTransaction = serializeExpandedTransaction(transaction);
-
-  return {
-    ...serializeBaseBlob(blob),
+  const {
+    blob,
+    blockHash,
+    blockNumber,
+    blockTimestamp,
     index,
+    txHash,
+    block,
+    transaction,
+  } = blobOnTransaction;
+  const serializedBlob: SerializedBlobOnTransaction = {
+    ...serializeBaseBlob(blob),
     blockHash,
     blockNumber,
     blockTimestamp: serializeDate(blockTimestamp),
-    txHash: hash,
-    ...(expandedBlock ? { block: expandedBlock } : {}),
-    ...(isEmptyObject(expandedTransaction)
-      ? {}
-      : { transaction: expandedTransaction }),
+    txHash,
+    index,
   };
+
+  if (block) {
+    serializedBlob.block = serializeExpandedBlock(block);
+  }
+
+  if (transaction && !isEmptyObject(transaction)) {
+    const expandedTransaction = serializeExpandedTransaction(transaction);
+
+    if (!isEmptyObject(expandedTransaction)) {
+      serializedBlob.transaction = expandedTransaction;
+    }
+  }
+
+  return serializedBlob;
 }
 
-export function serializeBlob(blob: Blob): SerializedBlob {
+export function serializeBlob(blob: QueriedBlob): SerializedBlob {
   const { transactions, ...baseBlob } = blob;
 
   return {
     ...serializeBaseBlob(baseBlob),
     data: blob.data,
     transactions: transactions
-      .sort((a, b) => a.transaction.hash.localeCompare(b.transaction.hash))
-      .map(({ index, transaction }) => {
-        const { block, hash, blockHash, blockNumber, blockTimestamp } =
-          transaction;
-
-        return {
-          index,
-          hash,
+      .sort((a, b) => a.txHash.localeCompare(b.txHash))
+      .map(
+        ({
           blockHash,
           blockNumber,
           blockTimestamp,
-          ...serializeExpandedTransaction(transaction),
-          ...(block ? { block: serializeExpandedBlock(block) } : {}),
-        };
-      }),
+          index,
+          txHash,
+          block,
+          transaction,
+        }) => {
+          return {
+            index,
+            hash: txHash,
+            blockHash,
+            blockNumber,
+            blockTimestamp,
+            ...(transaction ? serializeExpandedTransaction(transaction) : {}),
+            ...(block ? { block: serializeExpandedBlock(block) } : {}),
+          };
+        }
+      ),
   };
 }

packages/api/src/routers/blob/getAll.ts

@@ -49,23 +49,19 @@ export const getAll = publicProcedure
       ctx.prisma.blobsOnTransactions.findMany({
         select: createBlobsOnTransactionsSelect(ctx.expands),
         where: {
+          blockNumber: filters.blockNumber,
+          blockTimestamp: filters.blockTimestamp,
+          block: {
+            slot: filters.blockSlot,
+            transactionForks: filters.blockType,
+          },
           transaction: {
-            blockNumber: filters.blockNumber,
-            blockTimestamp: filters.blockTimestamp,
             rollup: filters.transactionRollup,
             OR: filters.transactionAddresses,
-            block: {
-              slot: filters.blockSlot,
-              transactionForks: filters.blockType,
-            },
           },
         },
         orderBy: [
-          {
-            transaction: {
-              blockNumber: filters.sort,
-            },
-          },
+          { blockNumber: filters.sort },
           {
             txHash: filters.sort,
           },

packages/api/src/routers/indexer/indexData.ts

@@ -7,6 +7,7 @@ import { jwtAuthedProcedure } from "../../procedures";
 import { INDEXER_PATH } from "./common";
 import {
   createDBBlobs,
+  createDBBlobsOnTransactions,
   createDBBlock,
   createDBTransactions,
 } from "./indexData.utils";
@@ -116,6 +117,7 @@ export const indexData = jwtAuthedProcedure
       const dbTxs = createDBTransactions(input);
       const dbBlock = createDBBlock(input, dbTxs);
       const dbBlobs = createDBBlobs(input);
+      const dbBlobsOnTransactions = createDBBlobsOnTransactions(input);
 
       operations.push(
         prisma.block.upsert({
@@ -144,14 +146,7 @@ export const indexData = jwtAuthedProcedure
       }
 
       operations.push(
-        prisma.blobsOnTransactions.createMany({
-          data: input.blobs.map((blob) => ({
-            blobHash: blob.versionedHash,
-            txHash: blob.txHash,
-            index: blob.index,
-          })),
-          skipDuplicates: true,
-        })
+        prisma.blobsOnTransactions.upsertMany(dbBlobsOnTransactions)
       );
 
       // 3. Execute all database operations in a single transaction

packages/api/src/routers/indexer/indexData.utils.ts

@@ -1,5 +1,6 @@
 import type {
   Blob,
+  BlobsOnTransactions,
   Block,
   Transaction,
   WithoutTimestampFields,
@@ -56,6 +57,10 @@ function bigIntToDecimal(bigint: bigint) {
   return new Prisma.Decimal(bigint.toString());
 }
 
+function timestampToDate(timestamp: number) {
+  return new Date(timestamp * 1000);
+}
+
 function fakeExponential(
   factor: bigint,
   numerator: bigint,
@@ -139,7 +144,7 @@ export function createDBTransactions({
       return {
         blockHash: block.hash,
         blockNumber: block.number,
-        blockTimestamp: new Date(block.timestamp * 1000),
+        blockTimestamp: timestampToDate(block.timestamp),
         hash,
         fromId: from,
         toId: to,
@@ -168,7 +173,7 @@ export function createDBBlock(
   return {
     number,
     hash,
-    timestamp: new Date(timestamp * 1000),
+    timestamp: timestampToDate(timestamp),
     slot,
     blobGasUsed: bigIntToDecimal(blobGasUsed),
     blobGasPrice: bigIntToDecimal(blobGasPrice),
@@ -204,3 +209,17 @@ export function createDBBlobs({
     }
   );
 }
+
+export function createDBBlobsOnTransactions({
+  block,
+  blobs,
+}: IndexDataFormattedInput): BlobsOnTransactions[] {
+  return blobs.map(({ versionedHash, txHash, index }) => ({
+    blobHash: versionedHash,
+    blockHash: block.hash,
+    blockNumber: block.number,
+    blockTimestamp: timestampToDate(block.timestamp),
+    txHash: txHash,
+    index,
+  }));
+}