feat: alter fulltext options (#4952)

* feat(WIP): alter fulltext index Co-Authored-By: irenjj <[email protected]> * feat: alter column fulltext option Co-Authored-By: irenjj <[email protected]> * chore: fmt * test: add unit and integration tests Co-Authored-By: irenjj <[email protected]> * test: update sqlness test * chore: new line * chore: lock file update * chore: apply review comments * test: update sqlness test * test: update sqlness test * fix: convert * chore: apply review comments * fix: toml fmt * fix: tests * test: add test for mito * chore: error message * fix: test * fix: test * fix: wrong comment * chore: change proto rev * chore: apply review comments * chore: apply review comments * chore: fmt --------- Co-authored-by: irenjj <[email protected]>
GreptimeTeam · Nov 12, 2024 · 84aa5b7 · 84aa5b7
1 parent cbf21e5
commit 84aa5b7
Show file tree

Hide file tree

Showing 30 changed files with 1,043 additions and 122 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -121,7 +121,7 @@ etcd-client = { version = "0.13" }
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "255f87a3318ace3f88a67f76995a0e14910983f4" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "67bb1d52bc1241972c368657e658592b1be7ead3" }
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"

diff --git a/src/api/src/v1/column_def.rs b/src/api/src/v1/column_def.rs
@@ -15,9 +15,10 @@
 use std::collections::HashMap;
 
 use datatypes::schema::{
-    ColumnDefaultConstraint, ColumnSchema, FulltextOptions, COMMENT_KEY, FULLTEXT_KEY,
-    INVERTED_INDEX_KEY,
+    ColumnDefaultConstraint, ColumnSchema, FulltextAnalyzer, FulltextOptions, COMMENT_KEY,
+    FULLTEXT_KEY, INVERTED_INDEX_KEY,
 };
+use greptime_proto::v1::Analyzer;
 use snafu::ResultExt;
 
 use crate::error::{self, Result};
@@ -104,6 +105,14 @@ pub fn options_from_fulltext(fulltext: &FulltextOptions) -> Result<Option<Column
     Ok((!options.options.is_empty()).then_some(options))
 }
 
+/// Tries to construct a `FulltextAnalyzer` from the given analyzer.
+pub fn as_fulltext_option(analyzer: Analyzer) -> FulltextAnalyzer {
+    match analyzer {
+        Analyzer::English => FulltextAnalyzer::English,
+        Analyzer::Chinese => FulltextAnalyzer::Chinese,
+    }
+}
+
 #[cfg(test)]
 mod tests {
 

diff --git a/src/common/grpc-expr/src/alter.rs b/src/common/grpc-expr/src/alter.rs
@@ -15,20 +15,22 @@
 use api::helper::ColumnDataTypeWrapper;
 use api::v1::add_column_location::LocationType;
 use api::v1::alter_expr::Kind;
+use api::v1::column_def::as_fulltext_option;
 use api::v1::{
-    column_def, AddColumnLocation as Location, AlterExpr, ChangeColumnTypes, CreateTableExpr,
-    DropColumns, RenameTable, SemanticType,
+    column_def, AddColumnLocation as Location, AlterExpr, Analyzer, ChangeColumnTypes,
+    CreateTableExpr, DropColumns, RenameTable, SemanticType,
 };
 use common_query::AddColumnLocation;
-use datatypes::schema::{ColumnSchema, RawSchema};
+use datatypes::schema::{ColumnSchema, FulltextOptions, RawSchema};
 use snafu::{ensure, OptionExt, ResultExt};
 use store_api::region_request::ChangeOption;
 use table::metadata::TableId;
 use table::requests::{AddColumnRequest, AlterKind, AlterTableRequest, ChangeColumnTypeRequest};
 
 use crate::error::{
-    InvalidChangeTableOptionRequestSnafu, InvalidColumnDefSnafu, MissingFieldSnafu,
-    MissingTimestampColumnSnafu, Result, UnknownLocationTypeSnafu,
+    InvalidChangeFulltextOptionRequestSnafu, InvalidChangeTableOptionRequestSnafu,
+    InvalidColumnDefSnafu, MissingFieldSnafu, MissingTimestampColumnSnafu, Result,
+    UnknownLocationTypeSnafu,
 };
 
 const LOCATION_TYPE_FIRST: i32 = LocationType::First as i32;
@@ -102,6 +104,17 @@ pub fn alter_expr_to_request(table_id: TableId, expr: AlterExpr) -> Result<Alter
                 .collect::<std::result::Result<Vec<_>, _>>()
                 .context(InvalidChangeTableOptionRequestSnafu)?,
         },
+        Kind::ChangeColumnFulltext(c) => AlterKind::ChangeColumnFulltext {
+            column_name: c.column_name,
+            options: FulltextOptions {
+                enable: c.enable,
+                analyzer: as_fulltext_option(
+                    Analyzer::try_from(c.analyzer)
+                        .context(InvalidChangeFulltextOptionRequestSnafu)?,
+                ),
+                case_sensitive: c.case_sensitive,
+            },
+        },
     };
 
     let request = AlterTableRequest {

diff --git a/src/common/grpc-expr/src/error.rs b/src/common/grpc-expr/src/error.rs
@@ -125,6 +125,14 @@ pub enum Error {
         #[snafu(source)]
         error: MetadataError,
     },
+
+    #[snafu(display("Invalid change fulltext option request"))]
+    InvalidChangeFulltextOptionRequest {
+        #[snafu(implicit)]
+        location: Location,
+        #[snafu(source)]
+        error: prost::DecodeError,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -148,7 +156,8 @@ impl ErrorExt for Error {
             Error::UnknownColumnDataType { .. } | Error::InvalidFulltextColumnType { .. } => {
                 StatusCode::InvalidArguments
             }
-            Error::InvalidChangeTableOptionRequest { .. } => StatusCode::InvalidArguments,
+            Error::InvalidChangeTableOptionRequest { .. }
+            | Error::InvalidChangeFulltextOptionRequest { .. } => StatusCode::InvalidArguments,
         }
     }
 

diff --git a/src/common/meta/src/ddl/alter_table/region_request.rs b/src/common/meta/src/ddl/alter_table/region_request.rs
@@ -107,6 +107,9 @@ fn create_proto_alter_kind(
         }
         Kind::RenameTable(_) => Ok(None),
         Kind::ChangeTableOptions(v) => Ok(Some(alter_request::Kind::ChangeTableOptions(v.clone()))),
+        Kind::ChangeColumnFulltext(v) => {
+            Ok(Some(alter_request::Kind::ChangeColumnFulltext(v.clone())))
+        }
     }
 }
 

diff --git a/src/common/meta/src/ddl/alter_table/update_metadata.rs b/src/common/meta/src/ddl/alter_table/update_metadata.rs
@@ -53,7 +53,8 @@ impl AlterTableProcedure {
             }
             AlterKind::DropColumns { .. }
             | AlterKind::ChangeColumnTypes { .. }
-            | AlterKind::ChangeTableOptions { .. } => {}
+            | AlterKind::ChangeTableOptions { .. }
+            | AlterKind::ChangeColumnFulltext { .. } => {}
         }
 
         Ok(new_info)

diff --git a/src/datatypes/Cargo.toml b/src/datatypes/Cargo.toml
@@ -33,3 +33,5 @@ paste = "1.0"
 serde.workspace = true
 serde_json.workspace = true
 snafu.workspace = true
+sqlparser.workspace = true
+sqlparser_derive = "0.1"
diff --git a/src/datatypes/src/error.rs b/src/datatypes/src/error.rs
@@ -212,6 +212,13 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Invalid fulltext option: {}", msg))]
+    InvalidFulltextOption {
+        msg: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 impl ErrorExt for Error {
@@ -230,7 +237,8 @@ impl ErrorExt for Error {
             | DuplicateMeta { .. }
             | InvalidTimestampPrecision { .. }
             | InvalidPrecisionOrScale { .. }
-            | InvalidJson { .. } => StatusCode::InvalidArguments,
+            | InvalidJson { .. }
+            | InvalidFulltextOption { .. } => StatusCode::InvalidArguments,
 
             ValueExceedsPrecision { .. }
             | CastType { .. }

diff --git a/src/datatypes/src/schema.rs b/src/datatypes/src/schema.rs
@@ -27,8 +27,10 @@ use snafu::{ensure, ResultExt};
 use crate::error::{self, DuplicateColumnSnafu, Error, ProjectArrowSchemaSnafu, Result};
 use crate::prelude::DataType;
 pub use crate::schema::column_schema::{
-    ColumnSchema, FulltextAnalyzer, FulltextOptions, Metadata, COMMENT_KEY, FULLTEXT_KEY,
-    INVERTED_INDEX_KEY, TIME_INDEX_KEY,
+    ColumnSchema, FulltextAnalyzer, FulltextOptions, Metadata,
+    COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE, COLUMN_FULLTEXT_OPT_KEY_ANALYZER,
+    COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE, COMMENT_KEY, FULLTEXT_KEY, INVERTED_INDEX_KEY,
+    TIME_INDEX_KEY,
 };
 pub use crate::schema::constraint::ColumnDefaultConstraint;
 pub use crate::schema::raw::RawSchema;

diff --git a/src/datatypes/src/schema/column_schema.rs b/src/datatypes/src/schema/column_schema.rs
@@ -18,9 +18,10 @@ use std::fmt;
 use arrow::datatypes::Field;
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, ResultExt};
+use sqlparser_derive::{Visit, VisitMut};
 
 use crate::data_type::{ConcreteDataType, DataType};
-use crate::error::{self, Error, Result};
+use crate::error::{self, Error, InvalidFulltextOptionSnafu, Result};
 use crate::schema::constraint::ColumnDefaultConstraint;
 use crate::schema::TYPE_KEY;
 use crate::types::JSON_TYPE_NAME;
@@ -38,6 +39,12 @@ const DEFAULT_CONSTRAINT_KEY: &str = "greptime:default_constraint";
 pub const FULLTEXT_KEY: &str = "greptime:fulltext";
 /// Key used to store whether the column has inverted index in arrow field's metadata.
 pub const INVERTED_INDEX_KEY: &str = "greptime:inverted_index";
+
+/// Keys used in fulltext options
+pub const COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE: &str = "enable";
+pub const COLUMN_FULLTEXT_OPT_KEY_ANALYZER: &str = "analyzer";
+pub const COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE: &str = "case_sensitive";
+
 /// Schema of a column, used as an immutable struct.
 #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct ColumnSchema {
@@ -283,6 +290,14 @@ impl ColumnSchema {
         );
         Ok(self)
     }
+
+    pub fn set_fulltext_options(&mut self, options: &FulltextOptions) -> Result<()> {
+        self.metadata.insert(
+            FULLTEXT_KEY.to_string(),
+            serde_json::to_string(options).context(error::SerializeSnafu)?,
+        );
+        Ok(())
+    }
 }
 
 impl TryFrom<&Field> for ColumnSchema {
@@ -347,7 +362,7 @@ impl TryFrom<&ColumnSchema> for Field {
 }
 
 /// Fulltext options for a column.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Visit, VisitMut)]
 #[serde(rename_all = "kebab-case")]
 pub struct FulltextOptions {
     /// Whether the fulltext index is enabled.
@@ -360,8 +375,71 @@ pub struct FulltextOptions {
     pub case_sensitive: bool,
 }
 
+impl fmt::Display for FulltextOptions {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "enable={}", self.enable)?;
+        if self.enable {
+            write!(f, ", analyzer={}", self.analyzer)?;
+            write!(f, ", case_sensitive={}", self.case_sensitive)?;
+        }
+        Ok(())
+    }
+}
+
+impl TryFrom<HashMap<String, String>> for FulltextOptions {
+    type Error = Error;
+
+    fn try_from(options: HashMap<String, String>) -> Result<Self> {
+        let mut fulltext_options = FulltextOptions {
+            enable: true,
+            ..Default::default()
+        };
+
+        if let Some(enable) = options.get(COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE) {
+            match enable.to_ascii_lowercase().as_str() {
+                "true" => fulltext_options.enable = true,
+                "false" => fulltext_options.enable = false,
+                _ => {
+                    return InvalidFulltextOptionSnafu {
+                        msg: format!("{enable}, expected: 'true' | 'false'"),
+                    }
+                    .fail();
+                }
+            }
+        };
+
+        if let Some(analyzer) = options.get(COLUMN_FULLTEXT_OPT_KEY_ANALYZER) {
+            match analyzer.to_ascii_lowercase().as_str() {
+                "english" => fulltext_options.analyzer = FulltextAnalyzer::English,
+                "chinese" => fulltext_options.analyzer = FulltextAnalyzer::Chinese,
+                _ => {
+                    return InvalidFulltextOptionSnafu {
+                        msg: format!("{analyzer}, expected: 'English' | 'Chinese'"),
+                    }
+                    .fail();
+                }
+            }
+        };
+
+        if let Some(case_sensitive) = options.get(COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE) {
+            match case_sensitive.to_ascii_lowercase().as_str() {
+                "true" => fulltext_options.case_sensitive = true,
+                "false" => fulltext_options.case_sensitive = false,
+                _ => {
+                    return InvalidFulltextOptionSnafu {
+                        msg: format!("{case_sensitive}, expected: 'true' | 'false'"),
+                    }
+                    .fail();
+                }
+            }
+        }
+
+        Ok(fulltext_options)
+    }
+}
+
 /// Fulltext analyzer.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Visit, VisitMut)]
 pub enum FulltextAnalyzer {
     #[default]
     English,