feat: add parent() method to kvapi::Key to describe the hierarchica…

…l structure of meta-service data (#14716) Keys in meta-service have hierarchical structure. For example a database is the parent of a table. The `kvapi::Key` should expose a structure that enables the traversal from a `Tenant` to all associated data when exporting metadata for that tenant.
databendlabs · Feb 23, 2024 · 5d06510 · 5d06510
1 parent f52351b
commit 5d06510
Show file tree

Hide file tree

Showing 14 changed files with 234 additions and 0 deletions.
diff --git a/src/meta/api/src/id_generator.rs b/src/meta/api/src/id_generator.rs
@@ -101,6 +101,10 @@ impl kvapi::Key for IdGenerator {
 
     type ValueType = Infallible;
 
+    fn parent(&self) -> Option<String> {
+        None
+    }
+
     fn to_string_key(&self) -> String {
         kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
             .push_raw(&self.resource)

diff --git a/src/meta/app/src/background/background_job.rs b/src/meta/app/src/background/background_job.rs
@@ -383,13 +383,19 @@ mod kvapi_key_impl {
     use crate::background::background_job::BackgroundJobId;
     use crate::background::background_job::BackgroundJobIdent;
     use crate::background::BackgroundJobInfo;
+    use crate::tenant::Tenant;
 
     /// <prefix>/<tenant>/<background_job_ident> -> <id>
     impl kvapi::Key for BackgroundJobIdent {
         const PREFIX: &'static str = "__fd_background_job";
 
         type ValueType = BackgroundJobId;
 
+        /// It belongs to a tenant
+        fn parent(&self) -> Option<String> {
+            Some(Tenant::new(&self.tenant).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)
@@ -413,6 +419,10 @@ mod kvapi_key_impl {
 
         type ValueType = BackgroundJobInfo;
 
+        fn parent(&self) -> Option<String> {
+            None
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.id)

diff --git a/src/meta/app/src/background/background_task.rs b/src/meta/app/src/background/background_task.rs
@@ -220,6 +220,7 @@ mod kvapi_key_impl {
 
     use crate::background::background_task::BackgroundTaskIdent;
     use crate::background::BackgroundTaskInfo;
+    use crate::tenant::Tenant;
 
     // task is named by id, and will not encounter renaming issue.
     /// <prefix>/<tenant>/<background_task_ident> -> info
@@ -228,6 +229,11 @@ mod kvapi_key_impl {
 
         type ValueType = BackgroundTaskInfo;
 
+        /// It belongs to a tenant
+        fn parent(&self) -> Option<String> {
+            Some(Tenant::new(&self.tenant).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)

diff --git a/src/meta/app/src/data_mask/mod.rs b/src/meta/app/src/data_mask/mod.rs
@@ -128,13 +128,19 @@ mod kvapi_key_impl {
     use super::MaskpolicyTableIdListKey;
     use crate::data_mask::DatamaskMeta;
     use crate::data_mask::MaskpolicyTableIdList;
+    use crate::tenant::Tenant;
 
     /// __fd_database/<tenant>/<name> -> <data_mask_id>
     impl kvapi::Key for DatamaskNameIdent {
         const PREFIX: &'static str = "__fd_datamask";
 
         type ValueType = DatamaskId;
 
+        /// It belongs to a tenant
+        fn parent(&self) -> Option<String> {
+            Some(Tenant::new(&self.tenant).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)
@@ -159,6 +165,10 @@ mod kvapi_key_impl {
 
         type ValueType = DatamaskMeta;
 
+        fn parent(&self) -> Option<String> {
+            None
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.id)
@@ -180,6 +190,11 @@ mod kvapi_key_impl {
 
         type ValueType = MaskpolicyTableIdList;
 
+        /// It belongs to a tenant
+        fn parent(&self) -> Option<String> {
+            Some(Tenant::new(&self.tenant).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)

diff --git a/src/meta/app/src/principal/user_defined_function.rs b/src/meta/app/src/principal/user_defined_function.rs
@@ -154,12 +154,18 @@ mod kv_api_impl {
 
     use super::UdfName;
     use crate::principal::UserDefinedFunction;
+    use crate::tenant::Tenant;
 
     impl kvapi::Key for UdfName {
         const PREFIX: &'static str = "__fd_udfs";
 
         type ValueType = UserDefinedFunction;
 
+        /// It belongs to a tenant
+        fn parent(&self) -> Option<String> {
+            Some(Tenant::new(&self.tenant).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)

diff --git a/src/meta/app/src/schema/catalog.rs b/src/meta/app/src/schema/catalog.rs
@@ -125,6 +125,12 @@ pub struct CatalogId {
     pub catalog_id: u64,
 }
 
+impl CatalogId {
+    pub fn new(catalog_id: u64) -> CatalogId {
+        CatalogId { catalog_id }
+    }
+}
+
 impl Display for CatalogNameIdent {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "'{}'/'{}'", self.tenant, self.catalog_name)
@@ -236,13 +242,19 @@ mod kvapi_key_impl {
     use super::CatalogIdToName;
     use super::CatalogNameIdent;
     use crate::schema::CatalogMeta;
+    use crate::tenant::Tenant;
 
     /// __fd_catalog/<tenant>/<catalog_name> -> <catalog_id>
     impl kvapi::Key for CatalogNameIdent {
         const PREFIX: &'static str = "__fd_catalog";
 
         type ValueType = CatalogId;
 
+        /// It belongs to a tenant
+        fn parent(&self) -> Option<String> {
+            Some(Tenant::new(&self.tenant).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)
@@ -270,6 +282,10 @@ mod kvapi_key_impl {
 
         type ValueType = CatalogMeta;
 
+        fn parent(&self) -> Option<String> {
+            None
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.catalog_id)
@@ -292,6 +308,10 @@ mod kvapi_key_impl {
 
         type ValueType = CatalogNameIdent;
 
+        fn parent(&self) -> Option<String> {
+            Some(CatalogId::new(self.catalog_id).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.catalog_id)

diff --git a/src/meta/app/src/schema/database.rs b/src/meta/app/src/schema/database.rs
@@ -67,6 +67,12 @@ pub struct DatabaseId {
     pub db_id: u64,
 }
 
+impl DatabaseId {
+    pub fn new(db_id: u64) -> Self {
+        DatabaseId { db_id }
+    }
+}
+
 impl From<u64> for DatabaseId {
     fn from(db_id: u64) -> Self {
         DatabaseId { db_id }
@@ -345,13 +351,18 @@ mod kvapi_key_impl {
     use crate::schema::DatabaseNameIdent;
     use crate::schema::DbIdList;
     use crate::schema::DbIdListKey;
+    use crate::tenant::Tenant;
 
     /// __fd_database/<tenant>/<db_name> -> <db_id>
     impl kvapi::Key for DatabaseNameIdent {
         const PREFIX: &'static str = "__fd_database";
 
         type ValueType = DatabaseId;
 
+        fn parent(&self) -> Option<String> {
+            Some(Tenant::new(&self.tenant).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)
@@ -376,6 +387,10 @@ mod kvapi_key_impl {
 
         type ValueType = DatabaseMeta;
 
+        fn parent(&self) -> Option<String> {
+            None
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.db_id)
@@ -398,6 +413,10 @@ mod kvapi_key_impl {
 
         type ValueType = DatabaseNameIdent;
 
+        fn parent(&self) -> Option<String> {
+            Some(DatabaseId::new(self.db_id).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.db_id)
@@ -420,6 +439,10 @@ mod kvapi_key_impl {
 
         type ValueType = DbIdList;
 
+        fn parent(&self) -> Option<String> {
+            Some(Tenant::new(&self.tenant).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)

diff --git a/src/meta/app/src/schema/index.rs b/src/meta/app/src/schema/index.rs
@@ -63,6 +63,12 @@ pub struct IndexId {
     pub index_id: u64,
 }
 
+impl IndexId {
+    pub fn new(index_id: u64) -> IndexId {
+        IndexId { index_id }
+    }
+}
+
 impl Display for IndexId {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", self.index_id)
@@ -242,13 +248,18 @@ mod kvapi_key_impl {
     use crate::schema::IndexIdToName;
     use crate::schema::IndexMeta;
     use crate::schema::IndexNameIdent;
+    use crate::tenant::Tenant;
 
     /// <prefix>/<tenant>/<index_name> -> <index_id>
     impl kvapi::Key for IndexNameIdent {
         const PREFIX: &'static str = "__fd_index";
 
         type ValueType = IndexId;
 
+        fn parent(&self) -> Option<String> {
+            Some(Tenant::new(&self.tenant).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)
@@ -273,6 +284,10 @@ mod kvapi_key_impl {
 
         type ValueType = IndexMeta;
 
+        fn parent(&self) -> Option<String> {
+            None
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.index_id)
@@ -295,6 +310,10 @@ mod kvapi_key_impl {
 
         type ValueType = IndexNameIdent;
 
+        fn parent(&self) -> Option<String> {
+            Some(IndexId::new(self.index_id).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.index_id)

diff --git a/src/meta/app/src/schema/lock.rs b/src/meta/app/src/schema/lock.rs
@@ -182,6 +182,7 @@ mod kvapi_key_impl {
     use databend_common_meta_kvapi::kvapi;
 
     use crate::schema::LockMeta;
+    use crate::schema::TableId;
     use crate::schema::TableLockKey;
 
     /// __fd_table_lock/table_id/revision -> LockMeta
@@ -190,6 +191,10 @@ mod kvapi_key_impl {
 
         type ValueType = LockMeta;
 
+        fn parent(&self) -> Option<String> {
+            Some(TableId::new(self.table_id).to_string_key())
+        }
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.table_id)