[GraphQL/TransactionBlock] Scan Limits (#18413)

## Description Implement learnings from GraphQL performance benchmarks: - Implement transaction block pagination as a two step process: First fetch the relevant transaction sequence numbers, then fetch their contents. - Every "atomic" filter on transaction blocks is served by a single `tx_` table, with two indices on it, both of which are prepped to perform index-only scans. - The primary index is used to apply the filter directly. - The secondary index applies the filter after limiting results to one sender. - Compound filters are served by joining multiple atomic filters together. - The "scan limit" concept is introduced to limit the amount of work done when dealing with compound filters (see below). ### Scan Limits - If a filter is compound, a scan limit must be provided, and controls how many transactions are considered as candidates when building a page of results. - There is an upperbound on the scan limit, currently 100M, which is enough for over a week of transactions at 100TPS. - When scan limits are enabled, pagination behaviour changes: Pages can be returned with fewer results than the page size (including no results), but still have a previous or next page, because there were no valid candidates in the area scanned but there is more room to scan on either side. - The start and end cursor for the page may no longer point to an element in the results, because they point to the first and last candidate transaction. ## Test plan ``` sui$ cargo build -p sui-indexer sui$ cargo nextest run -p sui-graphql-rpc sui$ cargo nextest run -p sui-graphql-e2e-tests --features pg_integration ``` --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [x] GraphQL: Introduce `scanLimit` for paginating `TransactionBlocks`. Queries that include multiple complex filters (filters on the function called, affected objects, recipient), need to include a scan limit which controls the number of transactions that are looked at as candidates. - [ ] CLI: - [ ] Rust SDK: --------- Co-authored-by: Ashok Menon <[email protected]>
MystenLabs · Sep 16, 2024 · 18c89f6 · 18c89f6
1 parent 012f185
commit 18c89f6
Show file tree

Hide file tree

Showing 60 changed files with 8,523 additions and 465 deletions.
diff --git a/crates/sui-graphql-e2e-tests/tests/consistency/balances.exp b/crates/sui-graphql-e2e-tests/tests/consistency/balances.exp
@@ -59,7 +59,7 @@ task 15, line 76:
 Checkpoint created: 7
 
 task 16, lines 78-99:
-//# run-graphql --cursors {"c":2,"t":1,"tc":1}
+//# run-graphql --cursors {"c":2,"t":1,"i":false}
 Response: {
   "data": {
     "transactionBlocks": {
@@ -95,7 +95,7 @@ Response: {
 }
 
 task 17, lines 101-122:
-//# run-graphql --cursors {"c":3,"t":1,"tc":1}
+//# run-graphql --cursors {"c":3,"t":1,"i":false}
 Response: {
   "data": {
     "transactionBlocks": {
@@ -131,7 +131,7 @@ Response: {
 }
 
 task 18, lines 124-145:
-//# run-graphql --cursors {"c":4,"t":1,"tc":1}
+//# run-graphql --cursors {"c":4,"t":1,"i":false}
 Response: {
   "data": {
     "transactionBlocks": {
@@ -175,7 +175,7 @@ task 20, line 149:
 Checkpoint created: 8
 
 task 21, lines 151-172:
-//# run-graphql --cursors {"c":2,"t":1,"tc":1}
+//# run-graphql --cursors {"c":2,"t":1,"i":false}
 Response: {
   "data": {
     "transactionBlocks": {
@@ -211,7 +211,7 @@ Response: {
 }
 
 task 22, lines 174-195:
-//# run-graphql --cursors {"c":3,"t":1,"tc":1}
+//# run-graphql --cursors {"c":3,"t":1,"i":false}
 Response: {
   "data": {
     "transactionBlocks": {
@@ -247,7 +247,7 @@ Response: {
 }
 
 task 23, lines 197-218:
-//# run-graphql --cursors {"c":4,"t":1,"tc":1}
+//# run-graphql --cursors {"c":4,"t":1,"i":false}
 Response: {
   "data": {
     "transactionBlocks": {
@@ -291,7 +291,7 @@ task 25, line 222:
 Checkpoint created: 9
 
 task 26, lines 224-245:
-//# run-graphql --cursors {"c":2,"t":1,"tc":1}
+//# run-graphql --cursors {"c":2,"t":1,"i":false}
 Response: {
   "data": {
     "transactionBlocks": {
@@ -326,7 +326,7 @@ Response: {
 }
 
 task 27, lines 247-268:
-//# run-graphql --cursors {"c":3,"t":1,"tc":1}
+//# run-graphql --cursors {"c":3,"t":1,"i":false}
 Response: {
   "data": {
     "transactionBlocks": {
@@ -362,7 +362,7 @@ Response: {
 }
 
 task 28, lines 270-291:
-//# run-graphql --cursors {"c":4,"t":1,"tc":1}
+//# run-graphql --cursors {"c":4,"t":1,"i":false}
 Response: {
   "data": {
     "transactionBlocks": {
@@ -406,7 +406,7 @@ task 30, line 296:
 Checkpoint created: 10
 
 task 31, lines 298-319:
-//# run-graphql --cursors {"c":2,"t":1,"tc":1}
+//# run-graphql --cursors {"c":2,"t":1,"i":false}
 Response: {
   "data": {
     "transactionBlocks": {
@@ -441,7 +441,7 @@ Response: {
 }
 
 task 32, lines 321-342:
-//# run-graphql --cursors {"c":3,"t":1,"tc":1}
+//# run-graphql --cursors {"c":3,"t":1,"i":false}
 Response: {
   "data": {
     "transactionBlocks": {
@@ -476,7 +476,7 @@ Response: {
 }
 
 task 33, lines 344-365:
-//# run-graphql --cursors {"c":4,"t":1,"tc":1}
+//# run-graphql --cursors {"c":4,"t":1,"i":false}
 Response: {
   "data": {
     "transactionBlocks": {

diff --git a/crates/sui-graphql-e2e-tests/tests/consistency/balances.move b/crates/sui-graphql-e2e-tests/tests/consistency/balances.move
@@ -75,7 +75,7 @@ module P0::fake {
 
 //# create-checkpoint
 
-//# run-graphql --cursors {"c":2,"t":1,"tc":1}
+//# run-graphql --cursors {"c":2,"t":1,"i":false}
 # Emulating viewing transaction blocks at checkpoint 2. Fake coin balance should be 700.
 {
   transactionBlocks(first: 1, after: "@{cursor_0}", filter: {signAddress: "@{A}"}) {
@@ -98,7 +98,7 @@ module P0::fake {
   }
 }
 
-//# run-graphql --cursors {"c":3,"t":1,"tc":1}
+//# run-graphql --cursors {"c":3,"t":1,"i":false}
 # Emulating viewing transaction blocks at checkpoint 3. Fake coin balance should be 500.
 {
   transactionBlocks(first: 1, after: "@{cursor_0}", filter: {signAddress: "@{A}"}) {
@@ -121,7 +121,7 @@ module P0::fake {
   }
 }
 
-//# run-graphql --cursors {"c":4,"t":1,"tc":1}
+//# run-graphql --cursors {"c":4,"t":1,"i":false}
 # Emulating viewing transaction blocks at checkpoint 4. Fake coin balance should be 400.
 {
   transactionBlocks(first: 1, after: "@{cursor_0}", filter: {signAddress: "@{A}"}) {
@@ -148,7 +148,7 @@ module P0::fake {
 
 //# create-checkpoint
 
-//# run-graphql --cursors {"c":2,"t":1,"tc":1}
+//# run-graphql --cursors {"c":2,"t":1,"i":false}
 # Emulating viewing transaction blocks at checkpoint 2. Fake coin balance should be 700.
 {
   transactionBlocks(first: 1, after: "@{cursor_0}", filter: {signAddress: "@{A}"}) {
@@ -171,7 +171,7 @@ module P0::fake {
   }
 }
 
-//# run-graphql --cursors {"c":3,"t":1,"tc":1}
+//# run-graphql --cursors {"c":3,"t":1,"i":false}
 # Emulating viewing transaction blocks at checkpoint 3. Fake coin balance should be 500.
 {
   transactionBlocks(first: 1, after: "@{cursor_0}", filter: {signAddress: "@{A}"}) {
@@ -194,7 +194,7 @@ module P0::fake {
   }
 }
 
-//# run-graphql --cursors {"c":4,"t":1,"tc":1}
+//# run-graphql --cursors {"c":4,"t":1,"i":false}
 # Emulating viewing transaction blocks at checkpoint 4. Fake coin balance should be 400.
 {
   transactionBlocks(first: 1, after: "@{cursor_0}", filter: {signAddress: "@{A}"}) {
@@ -221,7 +221,7 @@ module P0::fake {
 
 //# create-checkpoint
 
-//# run-graphql --cursors {"c":2,"t":1,"tc":1}
+//# run-graphql --cursors {"c":2,"t":1,"i":false}
 # Outside available range
 {
   transactionBlocks(first: 1, after: "@{cursor_0}", filter: {signAddress: "@{A}"}) {
@@ -244,7 +244,7 @@ module P0::fake {
   }
 }
 
-//# run-graphql --cursors {"c":3,"t":1,"tc":1}
+//# run-graphql --cursors {"c":3,"t":1,"i":false}
 # Emulating viewing transaction blocks at checkpoint 3. Fake coin balance should be 500.
 {
   transactionBlocks(first: 1, after: "@{cursor_0}", filter: {signAddress: "@{A}"}) {
@@ -267,7 +267,7 @@ module P0::fake {
   }
 }
 
-//# run-graphql --cursors {"c":4,"t":1,"tc":1}
+//# run-graphql --cursors {"c":4,"t":1,"i":false}
 # Emulating viewing transaction blocks at checkpoint 4. Fake coin balance should be 400.
 {
   transactionBlocks(first: 1, after: "@{cursor_0}", filter: {signAddress: "@{A}"}) {
@@ -295,7 +295,7 @@ module P0::fake {
 
 //# create-checkpoint
 
-//# run-graphql --cursors {"c":2,"t":1,"tc":1}
+//# run-graphql --cursors {"c":2,"t":1,"i":false}
 # Outside available range
 {
   transactionBlocks(first: 1, after: "@{cursor_0}", filter: {signAddress: "@{A}"}) {
@@ -318,7 +318,7 @@ module P0::fake {
   }
 }
 
-//# run-graphql --cursors {"c":3,"t":1,"tc":1}
+//# run-graphql --cursors {"c":3,"t":1,"i":false}
 # Outside available range
 {
   transactionBlocks(first: 1, after: "@{cursor_0}", filter: {signAddress: "@{A}"}) {
@@ -341,7 +341,7 @@ module P0::fake {
   }
 }
 
-//# run-graphql --cursors {"c":4,"t":1,"tc":1}
+//# run-graphql --cursors {"c":4,"t":1,"i":false}
 # Outside available range
 {
   transactionBlocks(first: 1, after: "@{cursor_0}", filter: {signAddress: "@{A}"}) {

diff --git a/crates/sui-graphql-e2e-tests/tests/consistency/checkpoints/transaction_blocks.exp b/crates/sui-graphql-e2e-tests/tests/consistency/checkpoints/transaction_blocks.exp
@@ -92,7 +92,7 @@ Response: {
           "transactionBlocks": {
             "edges": [
               {
-                "cursor": "eyJjIjozLCJ0IjoyLCJ0YyI6MX0",
+                "cursor": "eyJjIjozLCJ0IjoyLCJpIjpmYWxzZX0",
                 "node": {
                   "digest": "Cwqr9jTgQjajoYaqcjzAaQGcQEyCg8XxoN7smGCLiBrs",
                   "sender": {
@@ -107,7 +107,7 @@ Response: {
                 }
               },
               {
-                "cursor": "eyJjIjozLCJ0IjozLCJ0YyI6MX0",
+                "cursor": "eyJjIjozLCJ0IjozLCJpIjpmYWxzZX0",
                 "node": {
                   "digest": "H1WU8uXMGaENQs54EpoHGpV1iMYdH8P5scd1d16s9ECB",
                   "sender": {
@@ -122,7 +122,7 @@ Response: {
                 }
               },
               {
-                "cursor": "eyJjIjozLCJ0Ijo0LCJ0YyI6MX0",
+                "cursor": "eyJjIjozLCJ0Ijo0LCJpIjpmYWxzZX0",
                 "node": {
                   "digest": "4vJbSYKwEJb5sYU2jiayqsZNRnBywD8y6sd3RQoMppF9",
                   "sender": {
@@ -137,7 +137,7 @@ Response: {
                 }
               },
               {
-                "cursor": "eyJjIjozLCJ0Ijo1LCJ0YyI6MX0",
+                "cursor": "eyJjIjozLCJ0Ijo1LCJpIjpmYWxzZX0",
                 "node": {
                   "digest": "4W23PZz7dHVxoZ2VMCWU9j38Jxy7tLkqcFBcJUB3aCSB",
                   "sender": {
@@ -159,7 +159,7 @@ Response: {
           "transactionBlocks": {
             "edges": [
               {
-                "cursor": "eyJjIjozLCJ0Ijo2LCJ0YyI6Mn0",
+                "cursor": "eyJjIjozLCJ0Ijo2LCJpIjpmYWxzZX0",
                 "node": {
                   "digest": "JLAF7P6DumC8rgzT1Ygp2QgTwpHE2FUqQbVXL6cGEEQ",
                   "sender": {
@@ -174,7 +174,7 @@ Response: {
                 }
               },
               {
-                "cursor": "eyJjIjozLCJ0Ijo3LCJ0YyI6Mn0",
+                "cursor": "eyJjIjozLCJ0Ijo3LCJpIjpmYWxzZX0",
                 "node": {
                   "digest": "BVMVdn7DDpTbCjtYwWFekcFA9sNeMgDh1wTNWRrngZxh",
                   "sender": {
@@ -189,7 +189,7 @@ Response: {
                 }
               },
               {
-                "cursor": "eyJjIjozLCJ0Ijo4LCJ0YyI6Mn0",
+                "cursor": "eyJjIjozLCJ0Ijo4LCJpIjpmYWxzZX0",
                 "node": {
                   "digest": "4J5tno4AoU4NPS2NgEseAZK7cpLDh6KJduVtbtwzmHk5",
                   "sender": {
@@ -211,7 +211,7 @@ Response: {
           "transactionBlocks": {
             "edges": [
               {
-                "cursor": "eyJjIjozLCJ0Ijo5LCJ0YyI6M30",
+                "cursor": "eyJjIjozLCJ0Ijo5LCJpIjpmYWxzZX0",
                 "node": {
                   "digest": "5BCS9sencxEJRJHBBPeGhx3rWutYoGSuLFCmnMAaYcDm",
                   "sender": {
@@ -226,7 +226,7 @@ Response: {
                 }
               },
               {
-                "cursor": "eyJjIjozLCJ0IjoxMCwidGMiOjN9",
+                "cursor": "eyJjIjozLCJ0IjoxMCwiaSI6ZmFsc2V9",
                 "node": {
                   "digest": "HQYJnLLcGf4DwgTkpqF4zHbQsLHwc1s4WbQ3Xr5BBaxh",
                   "sender": {