From 7474c74e56cdd940fb46d0008aa8df8853f3fa43 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Sun, 5 May 2024 17:55:45 +0100 Subject: [PATCH 1/2] [Indexer] Create `objects_version` table. (#17542) ## Description This table maps an object's ID and version to a checkpoint sequence number, in a table partitioned by the first byte of the object ID. This speeds up look ups into `objects_history` by offering a path for a first look-up to the correct partition in that table for a given object's ID and version. This PR introduces the table, and the logic to populate it in the indexer. ## Test plan ``` sui$ cargo nextest run -p sui-indexer sui$ cargo nextest run -p sui-graphql-rpc sui$ cargo nextest run -p sui-graphql-e2e-tests --features pg_integration ``` A future PR will make use of this table from GraphQL, which will test it further. ## Stack - #17686 - #17687 - #17688 - #17689 - #17691 - #17694 - #17695 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [ ] GraphQL: - [ ] CLI: - [ ] Rust SDK: --- .../2024-05-05-155158_obj_indices/down.sql | 1 + .../2024-05-05-155158_obj_indices/up.sql | 9 +++++ .../pg/2024-05-05-155158_obj_indices/down.sql | 1 + .../pg/2024-05-05-155158_obj_indices/up.sql | 31 ++++++++++++++ crates/sui-indexer/src/models/mod.rs | 1 + crates/sui-indexer/src/models/obj_indices.rs | 40 +++++++++++++++++++ crates/sui-indexer/src/schema/mod.rs | 3 ++ crates/sui-indexer/src/schema/mysql.rs | 8 ++++ crates/sui-indexer/src/schema/pg.rs | 9 +++++ .../sui-indexer/src/store/pg_indexer_store.rs | 15 +++++-- 10 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 crates/sui-indexer/migrations/mysql/2024-05-05-155158_obj_indices/down.sql create mode 100644 crates/sui-indexer/migrations/mysql/2024-05-05-155158_obj_indices/up.sql create mode 100644 crates/sui-indexer/migrations/pg/2024-05-05-155158_obj_indices/down.sql create mode 100644 crates/sui-indexer/migrations/pg/2024-05-05-155158_obj_indices/up.sql create mode 100644 crates/sui-indexer/src/models/obj_indices.rs diff --git a/crates/sui-indexer/migrations/mysql/2024-05-05-155158_obj_indices/down.sql b/crates/sui-indexer/migrations/mysql/2024-05-05-155158_obj_indices/down.sql new file mode 100644 index 0000000000000..7a3a7670f24c2 --- /dev/null +++ b/crates/sui-indexer/migrations/mysql/2024-05-05-155158_obj_indices/down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS objects_version; diff --git a/crates/sui-indexer/migrations/mysql/2024-05-05-155158_obj_indices/up.sql b/crates/sui-indexer/migrations/mysql/2024-05-05-155158_obj_indices/up.sql new file mode 100644 index 0000000000000..e501b71a073c0 --- /dev/null +++ b/crates/sui-indexer/migrations/mysql/2024-05-05-155158_obj_indices/up.sql @@ -0,0 +1,9 @@ +-- The Postgres version of this table is partitioned by the first byte +-- of object_id, but this kind of partition is not easily supported in +-- MySQL, so this variant is unpartitioned for now. +CREATE TABLE objects_version ( + object_id BLOB NOT NULL, + object_version BIGINT NOT NULL, + cp_sequence_number BIGINT NOT NULL, + PRIMARY KEY (object_id(32), object_version) +) diff --git a/crates/sui-indexer/migrations/pg/2024-05-05-155158_obj_indices/down.sql b/crates/sui-indexer/migrations/pg/2024-05-05-155158_obj_indices/down.sql new file mode 100644 index 0000000000000..7a3a7670f24c2 --- /dev/null +++ b/crates/sui-indexer/migrations/pg/2024-05-05-155158_obj_indices/down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS objects_version; diff --git a/crates/sui-indexer/migrations/pg/2024-05-05-155158_obj_indices/up.sql b/crates/sui-indexer/migrations/pg/2024-05-05-155158_obj_indices/up.sql new file mode 100644 index 0000000000000..666e5a2423319 --- /dev/null +++ b/crates/sui-indexer/migrations/pg/2024-05-05-155158_obj_indices/up.sql @@ -0,0 +1,31 @@ +-- Indexing table mapping an object's ID and version to its checkpoint +-- sequence number, partitioned by the first byte of its Object ID. +CREATE TABLE objects_version ( + object_id bytea NOT NULL, + object_version bigint NOT NULL, + cp_sequence_number bigint NOT NULL, + PRIMARY KEY (object_id, object_version) +) PARTITION BY RANGE (object_id); + +-- Create a partition for each first byte value. +DO $$ +DECLARE + lo text; + hi text; +BEGIN + FOR i IN 0..254 LOOP + lo := LPAD(TO_HEX(i), 2, '0'); + hi := LPAD(TO_HEX(i + 1), 2, '0'); + EXECUTE FORMAT($F$ + CREATE TABLE objects_version_%1$s PARTITION OF objects_version FOR VALUES + FROM (E'\\x%1$s00000000000000000000000000000000000000000000000000000000000000') + TO (E'\\x%2$s00000000000000000000000000000000000000000000000000000000000000'); + $F$, lo, hi); + END LOOP; +END; +$$ LANGUAGE plpgsql; + +-- Special case for the last partition, because of the upper bound. +CREATE TABLE objects_version_ff PARTITION OF objects_version FOR VALUES +FROM (E'\\xff00000000000000000000000000000000000000000000000000000000000000') +TO (MAXVALUE); diff --git a/crates/sui-indexer/src/models/mod.rs b/crates/sui-indexer/src/models/mod.rs index 3b8233ec45021..775d067ea510c 100644 --- a/crates/sui-indexer/src/models/mod.rs +++ b/crates/sui-indexer/src/models/mod.rs @@ -5,6 +5,7 @@ pub mod checkpoints; pub mod display; pub mod epoch; pub mod events; +pub mod obj_indices; pub mod objects; pub mod packages; pub mod transactions; diff --git a/crates/sui-indexer/src/models/obj_indices.rs b/crates/sui-indexer/src/models/obj_indices.rs new file mode 100644 index 0000000000000..7e5298008834c --- /dev/null +++ b/crates/sui-indexer/src/models/obj_indices.rs @@ -0,0 +1,40 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use diesel::prelude::*; + +use crate::schema::objects_version; + +use super::objects::StoredDeletedObject; +use super::objects::StoredObject; + +/// Model types related to tables that support efficient execution of queries on the `objects`, +/// `objects_history` and `objects_snapshot` tables. + +#[derive(Queryable, Insertable, Debug, Identifiable, Clone, QueryableByName)] +#[diesel(table_name = objects_version, primary_key(object_id, object_version))] +pub struct StoredObjectVersion { + pub object_id: Vec, + pub object_version: i64, + pub cp_sequence_number: i64, +} + +impl From<&StoredObject> for StoredObjectVersion { + fn from(o: &StoredObject) -> Self { + Self { + object_id: o.object_id.clone(), + object_version: o.object_version, + cp_sequence_number: o.checkpoint_sequence_number, + } + } +} + +impl From<&StoredDeletedObject> for StoredObjectVersion { + fn from(o: &StoredDeletedObject) -> Self { + Self { + object_id: o.object_id.clone(), + object_version: o.object_version, + cp_sequence_number: o.checkpoint_sequence_number, + } + } +} diff --git a/crates/sui-indexer/src/schema/mod.rs b/crates/sui-indexer/src/schema/mod.rs index ab9d362b5788a..4e2d01fb5683e 100644 --- a/crates/sui-indexer/src/schema/mod.rs +++ b/crates/sui-indexer/src/schema/mod.rs @@ -19,6 +19,7 @@ mod inner { pub use crate::schema::pg::objects; pub use crate::schema::pg::objects_history; pub use crate::schema::pg::objects_snapshot; + pub use crate::schema::pg::objects_version; pub use crate::schema::pg::packages; pub use crate::schema::pg::transactions; pub use crate::schema::pg::tx_calls; @@ -39,6 +40,7 @@ mod inner { pub use crate::schema::mysql::objects; pub use crate::schema::mysql::objects_history; pub use crate::schema::mysql::objects_snapshot; + pub use crate::schema::mysql::objects_version; pub use crate::schema::mysql::packages; pub use crate::schema::mysql::transactions; pub use crate::schema::mysql::tx_calls; @@ -56,6 +58,7 @@ pub use inner::events; pub use inner::objects; pub use inner::objects_history; pub use inner::objects_snapshot; +pub use inner::objects_version; pub use inner::packages; pub use inner::transactions; pub use inner::tx_calls; diff --git a/crates/sui-indexer/src/schema/mysql.rs b/crates/sui-indexer/src/schema/mysql.rs index dfb7c1b75362c..df05e1429dc07 100644 --- a/crates/sui-indexer/src/schema/mysql.rs +++ b/crates/sui-indexer/src/schema/mysql.rs @@ -142,6 +142,14 @@ diesel::table! { } } +diesel::table! { + objects_version (object_id, object_version) { + object_id -> Blob, + object_version -> Bigint, + cp_sequence_number -> Bigint, + } +} + diesel::table! { packages (package_id) { package_id -> Blob, diff --git a/crates/sui-indexer/src/schema/pg.rs b/crates/sui-indexer/src/schema/pg.rs index d1539303765f4..22493a0c5dacc 100644 --- a/crates/sui-indexer/src/schema/pg.rs +++ b/crates/sui-indexer/src/schema/pg.rs @@ -183,6 +183,14 @@ diesel::table! { } } +diesel::table! { + objects_version (object_id, object_version) { + object_id -> Bytea, + object_version -> Int8, + cp_sequence_number -> Int8, + } +} + diesel::table! { packages (package_id) { package_id -> Bytea, @@ -282,6 +290,7 @@ diesel::allow_tables_to_appear_in_same_query!( objects_history, objects_history_partition_0, objects_snapshot, + objects_version, packages, transactions, transactions_partition_0, diff --git a/crates/sui-indexer/src/store/pg_indexer_store.rs b/crates/sui-indexer/src/store/pg_indexer_store.rs index 16a685b0f0a92..b0b11768c9686 100644 --- a/crates/sui-indexer/src/store/pg_indexer_store.rs +++ b/crates/sui-indexer/src/store/pg_indexer_store.rs @@ -31,6 +31,7 @@ use crate::models::checkpoints::StoredCheckpoint; use crate::models::display::StoredDisplay; use crate::models::epoch::StoredEpochInfo; use crate::models::events::StoredEvent; +use crate::models::obj_indices::StoredObjectVersion; use crate::models::objects::{ StoredDeletedHistoryObject, StoredDeletedObject, StoredHistoryObject, StoredObject, StoredObjectSnapshot, @@ -38,9 +39,9 @@ use crate::models::objects::{ use crate::models::packages::StoredPackage; use crate::models::transactions::StoredTransaction; use crate::schema::{ - checkpoints, display, epochs, events, objects, objects_history, objects_snapshot, packages, - transactions, tx_calls, tx_changed_objects, tx_digests, tx_input_objects, tx_recipients, - tx_senders, + checkpoints, display, epochs, events, objects, objects_history, objects_snapshot, + objects_version, packages, transactions, tx_calls, tx_changed_objects, tx_digests, + tx_input_objects, tx_recipients, tx_senders, }; use crate::types::{IndexedCheckpoint, IndexedEvent, IndexedPackage, IndexedTransaction, TxIndex}; use crate::{ @@ -437,13 +438,16 @@ impl PgIndexerStore { .checkpoint_db_commit_latency_objects_history_chunks .start_timer(); let mut mutated_objects: Vec = vec![]; + let mut object_versions: Vec = vec![]; let mut deleted_object_ids: Vec = vec![]; for object in objects { match object { ObjectChangeToCommit::MutatedObject(stored_object) => { + object_versions.push(StoredObjectVersion::from(&stored_object)); mutated_objects.push(stored_object.into()); } ObjectChangeToCommit::DeletedObject(stored_deleted_object) => { + object_versions.push(StoredObjectVersion::from(&stored_deleted_object)); deleted_object_ids.push(stored_deleted_object.into()); } } @@ -462,6 +466,11 @@ impl PgIndexerStore { ); } + for object_version_chunk in object_versions.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) + { + insert_or_ignore_into!(objects_version::table, object_version_chunk, conn); + } + for deleted_objects_chunk in deleted_object_ids.chunks(PG_COMMIT_CHUNK_SIZE_INTRA_DB_TX) { From fe4808abd1b0273da88fb3946ec7336df0d46b6e Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Sun, 28 Apr 2024 17:15:06 +0100 Subject: [PATCH 2/2] [Indexer] Index Package original ID, version, cp sequence number (#17726) ## Description Recreating #17690 which was accidentally closed. Adding data to the `packages` table, to support the following GraphQL queries: ```graphql type Query { # Fetch all packages created strictly `afterCheckpoint` and strictly # before `beforeCheckpoint`. packages( afterCheckpoint: Int, beforeCheckpoint: Int, first: Int, after: String, last: Int, before: String, ): MovePackageConnection # Fetch all packages in the same family as the package at `address` # with versions strictly after `afterVersion` and strictly before # `beforeVersion`. packageVersions( address: SuiAddress!, afterVersion: Int, beforeVersion: Int, first: Int, after: String, last: Int, before: String, ): MovePackageConnection # Fetch a package by its address, and optionally supplying a # version. If the version is supplied, returns the package whose # Original ID matches the Original ID of the package at `address`, # but whose version is `version`, otherwise just fetches the package # directly. package(address: SuiAddress!, version: Int): MovePackage } type MovePackage { # Return the package whose Original ID matches this package's but # whose version matches `version`. If `version` is not supplied, # defaults to the latest version. atVersion(version: Int): MovePackage # Fetch all packages in the same family as this package, with # versions strictly after `afterVersion` and strictly before # `beforeVersion`. versions( afterVersion: Int, beforeVersion: Int, first: Int, after: String, last: Int, before: String, ): MovePackageConnection } ``` These queries are important for writing tools that perform whole-chain package analyses, and also for the .move Registry. ## Test plan Make sure nothing is broken: ``` sui$ cargo nextest run -p sui-indexer ``` Tests of new features will be included in a stacked change that uses these tables and indices in GraphQL. ## Stack - #17686 - #17687 - #17688 - #17689 - #17691 - #17694 - #17695 - #17542 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [x] Indexer: Adds the following fields: `packages.original_id`, `packages.package_version`, `packages.checkpoint_sequence_number` to support queries about package upgrades. - [ ] JSON-RPC: - [ ] GraphQL: - [ ] CLI: - [ ] Rust SDK: --- .../mysql/2024-04-24-180249_packages/up.sql | 13 ++++++++++--- .../pg/2023-08-19-060729_packages/up.sql | 14 +++++++++++--- crates/sui-indexer/src/models/packages.rs | 6 ++++++ crates/sui-indexer/src/schema/mysql.rs | 3 +++ crates/sui-indexer/src/schema/pg.rs | 3 +++ 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/crates/sui-indexer/migrations/mysql/2024-04-24-180249_packages/up.sql b/crates/sui-indexer/migrations/mysql/2024-04-24-180249_packages/up.sql index f3fe2539038fc..7ee89206f254f 100644 --- a/crates/sui-indexer/migrations/mysql/2024-04-24-180249_packages/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-04-24-180249_packages/up.sql @@ -1,7 +1,14 @@ CREATE TABLE packages ( - package_id blob NOT NULL, + package_id BLOB NOT NULL, + original_id BLOB NOT NULL, + package_version BIGINT NOT NULL, -- bcs serialized MovePackage - move_package MEDIUMBLOB NOT NULL, - CONSTRAINT packages_pk PRIMARY KEY (package_id(255)) + move_package MEDIUMBLOB NOT NULL, + checkpoint_sequence_number BIGINT NOT NULL, + CONSTRAINT packages_pk PRIMARY KEY (package_id(32), original_id(32), package_version), + CONSTRAINT packages_unique_package_id UNIQUE (package_id(32)) ); + +CREATE INDEX packages_cp_id_version ON packages (checkpoint_sequence_number, original_id(32), package_version); +CREATE INDEX packages_id_version_cp ON packages (original_id(32), package_version, checkpoint_sequence_number); diff --git a/crates/sui-indexer/migrations/pg/2023-08-19-060729_packages/up.sql b/crates/sui-indexer/migrations/pg/2023-08-19-060729_packages/up.sql index a95489af4dc41..f08a5549608eb 100644 --- a/crates/sui-indexer/migrations/pg/2023-08-19-060729_packages/up.sql +++ b/crates/sui-indexer/migrations/pg/2023-08-19-060729_packages/up.sql @@ -1,6 +1,14 @@ -CREATE TABLE packages +CREATE TABLE packages ( - package_id bytea PRIMARY KEY, + package_id bytea NOT NULL, + original_id bytea NOT NULL, + package_version bigint NOT NULL, -- bcs serialized MovePackage - move_package bytea NOT NULL + move_package bytea NOT NULL, + checkpoint_sequence_number bigint NOT NULL, + CONSTRAINT packages_pkey PRIMARY KEY (package_id, original_id, package_version), + CONSTRAINT packages_unique_package_id UNIQUE (package_id) ); + +CREATE INDEX packages_cp_id_version ON packages (checkpoint_sequence_number, original_id, package_version); +CREATE INDEX packages_id_version_cp ON packages (original_id, package_version, checkpoint_sequence_number); diff --git a/crates/sui-indexer/src/models/packages.rs b/crates/sui-indexer/src/models/packages.rs index 1d042bf51387b..bfb45c71e602d 100644 --- a/crates/sui-indexer/src/models/packages.rs +++ b/crates/sui-indexer/src/models/packages.rs @@ -10,14 +10,20 @@ use diesel::prelude::*; #[diesel(table_name = packages, primary_key(package_id))] pub struct StoredPackage { pub package_id: Vec, + pub original_id: Vec, + pub package_version: i64, pub move_package: Vec, + pub checkpoint_sequence_number: i64, } impl From for StoredPackage { fn from(p: IndexedPackage) -> Self { Self { package_id: p.package_id.to_vec(), + original_id: p.move_package.original_package_id().to_vec(), + package_version: p.move_package.version().value() as i64, move_package: bcs::to_bytes(&p.move_package).unwrap(), + checkpoint_sequence_number: p.checkpoint_sequence_number as i64, } } } diff --git a/crates/sui-indexer/src/schema/mysql.rs b/crates/sui-indexer/src/schema/mysql.rs index df05e1429dc07..912e7a44c506e 100644 --- a/crates/sui-indexer/src/schema/mysql.rs +++ b/crates/sui-indexer/src/schema/mysql.rs @@ -153,7 +153,10 @@ diesel::table! { diesel::table! { packages (package_id) { package_id -> Blob, + original_id -> Blob, + package_version -> Bigint, move_package -> Mediumblob, + checkpoint_sequence_number -> Bigint, } } diff --git a/crates/sui-indexer/src/schema/pg.rs b/crates/sui-indexer/src/schema/pg.rs index 22493a0c5dacc..52feeb83c313c 100644 --- a/crates/sui-indexer/src/schema/pg.rs +++ b/crates/sui-indexer/src/schema/pg.rs @@ -194,7 +194,10 @@ diesel::table! { diesel::table! { packages (package_id) { package_id -> Bytea, + original_id -> Bytea, + package_version -> Int8, move_package -> Bytea, + checkpoint_sequence_number -> Int8, } }