feat: support horaedb submit compaction task to remote

Cargo.toml

@@ -24,7 +24,7 @@ license = "Apache-2.0"
 [workspace]
 resolver = "2"
 # In alphabetical order
-members = [
+members = [ 
     "horaectl",
     "integration_tests",
     "integration_tests/sdk/rust",
@@ -33,7 +33,8 @@ members = [
     "src/catalog",
     "src/catalog_impls",
     "src/cluster",
-    "src/common_types",
+    "src/common_types", 
+    "src/compaction_client",
     "src/components/alloc_tracker",
     "src/components/arena",
     "src/components/arrow_ext",
@@ -101,7 +102,8 @@ thiserror = "1"
 bytes_ext = { path = "src/components/bytes_ext" }
 catalog = { path = "src/catalog" }
 catalog_impls = { path = "src/catalog_impls" }
-horaedbproto = { git = "https://github.com/apache/incubator-horaedb-proto.git", rev = "a5874d9fedee32ab1292252c4eb6defc4f6e245a" }
+# TODO(leslie): modify it when the related pr in incubator-horaedb-proto is merged.
+horaedbproto = { git = "https://github.com/LeslieKid/incubator-horaedb-proto.git", rev = "84f72744efc6e5f2e3a3ad8bbb24b807e383c492" }
 codec = { path = "src/components/codec" }
 chrono = "0.4"
 clap = { version = "4.5.1", features = ["derive"] }
@@ -110,6 +112,7 @@ cluster = { path = "src/cluster" }
 criterion = "0.5"
 horaedb-client = "1.0.2"
 common_types = { path = "src/common_types" }
+compaction_client = { path = "src/compaction_client" }
 datafusion = { git = "https://github.com/CeresDB/arrow-datafusion.git", rev = "e21b03154" }
 datafusion-proto = { git = "https://github.com/CeresDB/arrow-datafusion.git", rev = "e21b03154" }
 derive_builder = "0.12"

docs/example-cluster-3.toml

@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[node]
+addr = "127.0.0.1"
+
+[logger]
+level = "info"
+
+[server]
+bind_addr = "0.0.0.0"
+http_port = 5442
+grpc_port = 8833
+mysql_port = 23307
+deploy_mode = "Cluster"
+
+[analytic]
+compaction_offload = true
+
+[analytic.storage]
+mem_cache_capacity = '1G'
+mem_cache_partition_bits = 0
+
+[analytic.storage.object_store]
+type = "Local"
+data_dir = "/tmp/horaedb0"
+
+[analytic.wal]
+type = "RocksDB"
+data_dir = "/tmp/horaedb0"
+
+[cluster_deployment]
+mode = "WithMeta"
+cmd_channel_buffer_size = 10
+
+[cluster_deployment.meta_client]
+# Only support "defaultCluster" currently.
+cluster_name = "defaultCluster"
+meta_addr = "http://127.0.0.1:2379"
+lease = "10s"
+timeout = "5s"
+
+[cluster_deployment.etcd_client]
+server_addrs = ['127.0.0.1:2379']
+
+[limiter]
+write_block_list = ['mytable1']
+read_block_list = ['mytable1']

docs/example-cluster-4.toml

@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[node]
+addr = "127.0.0.1"
+
+[logger]
+level = "debug"
+
+[server]
+bind_addr = "0.0.0.0"
+http_port = 5443
+grpc_port = 8834
+mysql_port = 33307
+postgresql_port = 25433
+deploy_mode = "Cluster"
+
+[analytic.storage]
+mem_cache_capacity = '1G'
+mem_cache_partition_bits = 0
+
+[analytic.storage.object_store]
+type = "Local"
+data_dir = "/tmp/horaedb1"
+
+[analytic.wal]
+type = "RocksDB"
+data_dir = "/tmp/horaedb1"
+
+[cluster_deployment]
+mode = "WithMeta"
+cmd_channel_buffer_size = 10
+node_type = "CompactionServer"
+
+[cluster_deployment.meta_client]
+# Only support "defaultCluster" currently.
+cluster_name = "defaultCluster"
+meta_addr = "http://127.0.0.1:2379"
+lease = "10s"
+timeout = "5s"
+
+[cluster_deployment.etcd_client]
+server_addrs = ['127.0.0.1:2379']
+
+[limiter]
+write_block_list = ['mytable1']
+read_block_list = ['mytable1']

src/analytic_engine/Cargo.toml

@@ -48,8 +48,10 @@ async-trait = { workspace = true }
 atomic_enum = { workspace = true }
 base64 = { workspace = true }
 bytes_ext = { workspace = true }
+cluster = { workspace = true }
 codec = { workspace = true }
 common_types = { workspace = true }
+compaction_client = { workspace = true }
 datafusion = { workspace = true }
 future_ext = { workspace = true }
 futures = { workspace = true }
@@ -65,6 +67,7 @@ logger = { workspace = true }
 lru = { workspace = true }
 macros = { workspace = true }
 message_queue = { workspace = true }
+meta_client = { workspace = true }
 metric_ext = { workspace = true }
 object_store = { workspace = true }
 parquet = { workspace = true }