[Enhancement] Make some system tables query from leader fe

Signed-off-by: wyb <[email protected]>
StarRocks · Oct 11, 2024 · 916ee30 · 916ee30
1 parent 437544d
commit 916ee30
Show file tree

Hide file tree

Showing 55 changed files with 198 additions and 69 deletions.
diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/SystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/SystemTable.java
@@ -14,11 +14,27 @@
 
 package com.starrocks.catalog.system;
 
+import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Lists;
 import com.starrocks.analysis.DescriptorTable.ReferencedPartitionInfo;
 import com.starrocks.catalog.Column;
 import com.starrocks.catalog.Table;
 import com.starrocks.catalog.Type;
+import com.starrocks.catalog.system.information.BeConfigsSystemTable;
+import com.starrocks.catalog.system.information.BeTabletsSystemTable;
+import com.starrocks.catalog.system.information.FeTabletSchedulesSystemTable;
+import com.starrocks.catalog.system.information.LoadTrackingLogsSystemTable;
+import com.starrocks.catalog.system.information.LoadsSystemTable;
+import com.starrocks.catalog.system.information.MaterializedViewsSystemTable;
+import com.starrocks.catalog.system.information.PartitionsMetaSystemTable;
+import com.starrocks.catalog.system.information.PipesSystemTable;
+import com.starrocks.catalog.system.information.RoutineLoadJobsSystemTable;
+import com.starrocks.catalog.system.information.StreamLoadsSystemTable;
+import com.starrocks.catalog.system.information.TablesConfigSystemTable;
+import com.starrocks.catalog.system.information.TaskRunsSystemTable;
+import com.starrocks.catalog.system.information.TasksSystemTable;
+import com.starrocks.catalog.system.information.TemporaryTablesTable;
+import com.starrocks.catalog.system.information.ViewsSystemTable;
 import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
 import com.starrocks.thrift.TSchemaTable;
 import com.starrocks.thrift.TSchemaTableType;
@@ -41,11 +57,31 @@ public class SystemTable extends Table {
     public static final int NAME_CHAR_LEN = 2048;
     public static final int MAX_FIELD_VARCHAR_LENGTH = 65535;
 
+    // some metadata may be inaccurate in the follower fe, because they may be not persisted in leader fe,
+    // such as routine load job state changed from NEED_SCHEDULE to RUNNING.
+    private static final ImmutableSortedSet<String> QUERY_FROM_LEADER_TABLES =
+            ImmutableSortedSet.orderedBy(String.CASE_INSENSITIVE_ORDER)
+                    .add(FeTabletSchedulesSystemTable.NAME)
+                    .add(LoadTrackingLogsSystemTable.NAME)
+                    .add(LoadsSystemTable.NAME)
+                    .add(MaterializedViewsSystemTable.NAME)
+                    .add(PartitionsMetaSystemTable.NAME)
+                    .add(PipesSystemTable.NAME)
+                    .add(RoutineLoadJobsSystemTable.NAME)
+                    .add(StreamLoadsSystemTable.NAME)
+                    .add(TablesConfigSystemTable.NAME)
+                    .add(TaskRunsSystemTable.NAME)
+                    .add(TasksSystemTable.NAME)
+                    .add(TemporaryTablesTable.NAME)
+                    .add(ViewsSystemTable.NAME)
+                    .build();
+
     private final TSchemaTableType schemaTableType;
 
     private final String catalogName;
 
-    public SystemTable(long id, String name, TableType type, List<Column> baseSchema, TSchemaTableType schemaTableType) {
+    public SystemTable(long id, String name, TableType type, List<Column> baseSchema,
+                       TSchemaTableType schemaTableType) {
         this(DEFAULT_INTERNAL_CATALOG_NAME, id, name, type, baseSchema, schemaTableType);
     }
 
@@ -67,12 +103,12 @@ public static boolean isFeSchemaTable(String name) {
 
     public boolean requireOperatePrivilege() {
         return (SystemTable.isBeSchemaTable(getName()) || SystemTable.isFeSchemaTable(getName())) &&
-                !getName().equals("be_tablets") && !getName().equals("fe_tablet_schedules");
+                !getName().equals(BeTabletsSystemTable.NAME) && !getName().equals(FeTabletSchedulesSystemTable.NAME);
     }
 
     @Override
     public boolean supportsUpdate() {
-        return name.equals("be_configs");
+        return name.equals(BeConfigsSystemTable.NAME);
     }
 
     @Override
@@ -139,10 +175,15 @@ public boolean supportFeEvaluation() {
 
     /**
      * Evaluate the system table query with specified predicate
+     *
      * @param predicate can only be conjuncts
      * @return All columns and rows according to the schema of this table
      */
     public List<List<ScalarOperator>> evaluate(ScalarOperator predicate) {
         throw new NotImplementedException("not supported");
     }
+
+    public static boolean needQueryFromLeader(String tableName) {
+        return QUERY_FROM_LEADER_TABLES.contains(tableName);
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeBvarsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeBvarsSystemTable.java
@@ -24,9 +24,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class BeBvarsSystemTable {
+    private static final String NAME = "be_bvars";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.BE_BVARS_ID,
-                "be_bvars",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT), false)

diff --git a/...in/java/com/starrocks/catalog/system/information/BeCloudNativeCompactionsSystemTable.java b/...in/java/com/starrocks/catalog/system/information/BeCloudNativeCompactionsSystemTable.java
@@ -23,9 +23,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class BeCloudNativeCompactionsSystemTable {
+    private static final String NAME = "be_cloud_native_compactions";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.BE_CLOUD_NATIVE_COMPACTIONS,
-                "be_cloud_native_compactions",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT))

diff --git a/...core/src/main/java/com/starrocks/catalog/system/information/BeCompactionsSystemTable.java b/...core/src/main/java/com/starrocks/catalog/system/information/BeCompactionsSystemTable.java
@@ -23,9 +23,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class BeCompactionsSystemTable {
+    private static final String NAME = "be_compactions";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.BE_COMPACTIONS_ID,
-                "be_compactions",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT))

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeConfigsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeConfigsSystemTable.java
@@ -24,9 +24,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class BeConfigsSystemTable {
+    public static final String NAME = "be_configs";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.BE_CONFIGS_ID,
-                "be_configs",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT))

diff --git a/...-core/src/main/java/com/starrocks/catalog/system/information/BeDataCacheMetricsTable.java b/...-core/src/main/java/com/starrocks/catalog/system/information/BeDataCacheMetricsTable.java
@@ -31,6 +31,7 @@
 import static com.starrocks.catalog.system.SystemTable.NAME_CHAR_LEN;
 
 public class BeDataCacheMetricsTable {
+    private static final String NAME = "be_datacache_metrics";
 
     public static SystemTable create() {
         ArrayList<StructField> dirSpacesFields = new ArrayList<>();
@@ -42,7 +43,7 @@ public static SystemTable create() {
         MapType usedBytesDetailType =
                 new MapType(ScalarType.createType(PrimitiveType.INT), ScalarType.createType(PrimitiveType.BIGINT));
 
-        return new SystemTable(SystemId.BE_DATACACHE_METRICS, "be_datacache_metrics", Table.TableType.SCHEMA,
+        return new SystemTable(SystemId.BE_DATACACHE_METRICS, NAME, Table.TableType.SCHEMA,
                 SystemTable.builder()
                         .column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT))
                         .column("STATUS", ScalarType.createVarchar(NAME_CHAR_LEN))

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeLogsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeLogsSystemTable.java
@@ -24,9 +24,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class BeLogsSystemTable {
+    private static final String NAME = "be_logs";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.BE_LOGS_ID,
-                "be_logs",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT))

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeMetricsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeMetricsSystemTable.java
@@ -24,9 +24,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class BeMetricsSystemTable {
+    private static final String NAME = "be_metrics";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.BE_METRICS_ID,
-                "be_metrics",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT))

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeTabletsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeTabletsSystemTable.java
@@ -24,9 +24,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class BeTabletsSystemTable {
+    public static final String NAME = "be_tablets";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.BE_TABLETS_ID,
-                "be_tablets",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT))

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeThreadsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeThreadsSystemTable.java
@@ -24,9 +24,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class BeThreadsSystemTable {
+    private static final String NAME = "be_threads";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.BE_THREADS_ID,
-                "be_threads",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT))

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeTxnsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/BeTxnsSystemTable.java
@@ -24,9 +24,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class BeTxnsSystemTable {
+    private static final String NAME = "be_txns";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.BE_TXNS_ID,
-                "be_txns",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT))

diff --git a/...core/src/main/java/com/starrocks/catalog/system/information/CharacterSetsSystemTable.java b/...core/src/main/java/com/starrocks/catalog/system/information/CharacterSetsSystemTable.java
@@ -23,11 +23,13 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class CharacterSetsSystemTable {
+    private static final String NAME = "character_sets";
+
     public static SystemTable create(String catalogName) {
         return new SystemTable(
                 catalogName,
                 SystemId.CHARACTER_SETS_ID,
-                "character_sets",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("CHARACTER_SET_NAME", ScalarType.createVarchar(512))

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/CollationsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/CollationsSystemTable.java
@@ -23,11 +23,13 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class CollationsSystemTable {
+    private static final String NAME = "collations";
+
     public static SystemTable create(String catalogName) {
         return new SystemTable(
                 catalogName,
                 SystemId.COLLATIONS_ID,
-                "collations",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("COLLATION_NAME", ScalarType.createVarchar(512))

diff --git a/...e/src/main/java/com/starrocks/catalog/system/information/ColumnPrivilegesSystemTable.java b/...e/src/main/java/com/starrocks/catalog/system/information/ColumnPrivilegesSystemTable.java
@@ -23,11 +23,13 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class ColumnPrivilegesSystemTable {
+    private static final String NAME = "column_privileges";
+
     public static SystemTable create(String catalogName) {
         return new SystemTable(
                 catalogName,
                 SystemId.COLUMN_PRIVILEGES_ID,
-                "column_privileges",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("GRANTEE", ScalarType.createVarchar(NAME_CHAR_LEN))

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ColumnsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/ColumnsSystemTable.java
@@ -23,11 +23,13 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class ColumnsSystemTable {
+    private static final String NAME = "columns";
+
     public static SystemTable create(String catalogName) {
         return new SystemTable(
                 catalogName,
                 SystemId.COLUMNS_ID,
-                "columns",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("TABLE_CATALOG", ScalarType.createVarchar(512))

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/EnginesSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/EnginesSystemTable.java
@@ -22,11 +22,13 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class EnginesSystemTable {
+    private static final String NAME = "engines";
+
     public static SystemTable create(String catalogName) {
         return new SystemTable(
                 catalogName,
                 SystemId.ENGINES_ID,
-                "engines",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("ENGINE", ScalarType.createVarchar(64))

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/EventsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/EventsSystemTable.java
@@ -24,11 +24,13 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class EventsSystemTable {
+    private static final String NAME = "events";
+
     public static SystemTable create(String catalogName) {
         return new SystemTable(
                 catalogName,
                 SystemId.EVENTS_ID,
-                "events",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("EVENT_CATALOG", ScalarType.createVarchar(64))

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/FeMetricsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/FeMetricsSystemTable.java
@@ -24,9 +24,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class FeMetricsSystemTable {
+    public static final String NAME = "fe_metrics";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.FE_METRICS_ID,
-                "fe_metrics",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("FE_ID", ScalarType.createVarchar(NAME_CHAR_LEN))

diff --git a/.../src/main/java/com/starrocks/catalog/system/information/FeTabletSchedulesSystemTable.java b/.../src/main/java/com/starrocks/catalog/system/information/FeTabletSchedulesSystemTable.java
@@ -24,9 +24,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class FeTabletSchedulesSystemTable {
+    public static final String NAME = "fe_tablet_schedules";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.FE_SCHEDULES_ID,
-                "fe_tablet_schedules",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("TABLE_ID", ScalarType.createType(PrimitiveType.BIGINT))

diff --git a/...re/src/main/java/com/starrocks/catalog/system/information/GlobalVariablesSystemTable.java b/...re/src/main/java/com/starrocks/catalog/system/information/GlobalVariablesSystemTable.java
@@ -22,9 +22,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class GlobalVariablesSystemTable {
+    private static final String NAME = "global_variables";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.GLOBAL_VARIABLES_ID,
-                "global_variables",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("VARIABLE_NAME", ScalarType.createVarchar(64))

diff --git a/...ore/src/main/java/com/starrocks/catalog/system/information/KeyColumnUsageSystemTable.java b/...ore/src/main/java/com/starrocks/catalog/system/information/KeyColumnUsageSystemTable.java
@@ -24,11 +24,13 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class KeyColumnUsageSystemTable {
+    private static final String NAME = "key_column_usage";
+
     public static SystemTable create(String catalogName) {
         return new SystemTable(
                 catalogName,
                 SystemId.KEY_COLUMN_USAGE_ID,
-                "key_column_usage",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("CONSTRAINT_CATALOG", ScalarType.createVarchar(NAME_CHAR_LEN))

diff --git a/...e/src/main/java/com/starrocks/catalog/system/information/LoadTrackingLogsSystemTable.java b/...e/src/main/java/com/starrocks/catalog/system/information/LoadTrackingLogsSystemTable.java
@@ -25,9 +25,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class LoadTrackingLogsSystemTable {
+    public static final String NAME = "load_tracking_logs";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.LOAD_TRACKING_LOGS_ID,
-                "load_tracking_logs",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("ID", ScalarType.createType(PrimitiveType.BIGINT))

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/LoadsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/LoadsSystemTable.java
@@ -24,9 +24,11 @@
 import static com.starrocks.catalog.system.SystemTable.builder;
 
 public class LoadsSystemTable {
+    public static final String NAME = "loads";
+
     public static SystemTable create() {
         return new SystemTable(SystemId.LOADS_ID,
-                "loads",
+                NAME,
                 Table.TableType.SCHEMA,
                 builder()
                         .column("ID", ScalarType.createType(PrimitiveType.BIGINT))