From 4504e8c2a173a2de11ba545c2bcad1d1866b6fab Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Thu, 10 Oct 2024 15:28:51 +0800 Subject: [PATCH 1/4] Add task's defination into task run status Signed-off-by: shuming.li --- .../src/main/java/com/starrocks/scheduler/TaskRun.java | 4 ++++ .../com/starrocks/scheduler/persist/TaskRunStatus.java | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/TaskRun.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/TaskRun.java index 5b5984cb7d5af..103ec3e6388b6 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/TaskRun.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/TaskRun.java @@ -340,6 +340,10 @@ public TaskRunStatus initStatus(String queryId, Long createTime) { status.setDbName(task.getDbName()); status.setPostRun(task.getPostRun()); status.setExpireTime(created + Config.task_runs_ttl_second * 1000L); + // NOTE: definition will cause a lot of repeats and cost a lot of metadata memory resources, + // since history task runs has been stored in sr's internal table, we can save it in the + // task run status. + status.setDefinition(task.getDefinition()); status.getMvTaskRunExtraMessage().setExecuteOption(this.executeOption); LOG.info("init task status, task:{}, query_id:{}, create_time:{}", task.getName(), queryId, status.getCreateTime()); diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/persist/TaskRunStatus.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/persist/TaskRunStatus.java index 8fecfd974dd5c..5cc965955ff3e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/persist/TaskRunStatus.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/persist/TaskRunStatus.java @@ -425,6 +425,13 @@ public boolean match(TGetTasksParams params) { return true; } + public String getDefinition() { + return definition; + } + + public void setDefinition(String definition) { + this.definition = definition; + } public static TaskRunStatus read(DataInput in) throws IOException { String json = Text.readString(in); From d45b299f4cbab891480fc7108da4be5e6a6431be Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Thu, 10 Oct 2024 17:17:53 +0800 Subject: [PATCH 2/4] fix bugs Signed-off-by: shuming.li --- .../starrocks/scheduler/TaskManagerTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/fe/fe-core/src/test/java/com/starrocks/scheduler/TaskManagerTest.java b/fe/fe-core/src/test/java/com/starrocks/scheduler/TaskManagerTest.java index efd4ce5351924..64e69d09caf9c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/scheduler/TaskManagerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/scheduler/TaskManagerTest.java @@ -799,4 +799,21 @@ public ConnectContext getRunCtx() { taskRunManager.killTaskRun(1L, true); Assert.assertEquals(0, taskRunScheduler.getRunningTaskCount()); } + + @Test + public void testTaskRunDefinition() { + Task task = new Task("test"); + task.setDefinition("select 1"); + long taskId = 1; + TaskRun taskRun = TaskRunBuilder + .newBuilder(task) + .setExecuteOption(DEFAULT_MERGE_OPTION) + .build(); + long now = System.currentTimeMillis(); + taskRun.setTaskId(taskId); + taskRun.initStatus("1", now + 10); + taskRun.getStatus().setPriority(0); + TaskRunStatus taskRunStatus = taskRun.getStatus(); + Assert.assertEquals(taskRunStatus.getDefinition(), "select 1"); + } } \ No newline at end of file From 7c8cf6ba53b7d12927c88fdc1138c33ea98ae862 Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Fri, 11 Oct 2024 09:51:31 +0800 Subject: [PATCH 3/4] fix bugs Signed-off-by: shuming.li --- .../information/TaskRunsSystemTable.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/TaskRunsSystemTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/TaskRunsSystemTable.java index 360fca40526d7..2bb4b9699f35b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/TaskRunsSystemTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/system/information/TaskRunsSystemTable.java @@ -47,6 +47,7 @@ import org.apache.commons.lang3.NotImplementedException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.util.Strings; import org.apache.thrift.meta_data.FieldValueMetaData; import org.apache.thrift.protocol.TType; @@ -211,14 +212,18 @@ public static TGetTaskRunInfoResult query(TGetTasksParams params) { info.setState(status.getState().toString()); info.setCatalog(status.getCatalogName()); info.setDatabase(ClusterNamespace.getNameFromFullName(status.getDbName())); - try { - // NOTE: use task's definition to display task-run's definition here - Task task = taskManager.getTaskWithoutLock(taskName); - if (task != null) { - info.setDefinition(task.getDefinition()); + if (!Strings.isEmpty(status.getDefinition())) { + info.setDefinition(status.getDefinition()); + } else { + try { + // NOTE: use task's definition to display task-run's definition here + Task task = taskManager.getTaskWithoutLock(taskName); + if (task != null) { + info.setDefinition(task.getDefinition()); + } + } catch (Exception e) { + LOG.warn("Get taskName {} definition failed: {}", taskName, e); } - } catch (Exception e) { - LOG.warn("Get taskName {} definition failed: {}", taskName, e); } info.setError_code(status.getErrorCode()); info.setError_message(status.getErrorMessage()); From 117cef595cfc15ea0c616c1fdc8e773bdc7c903c Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Fri, 11 Oct 2024 10:37:15 +0800 Subject: [PATCH 4/4] fix bugs Signed-off-by: shuming.li --- .../com/starrocks/backup/RestoreJobMaterializedViewTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java index 9e3e434526ca1..9409d77c759c8 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobMaterializedViewTest.java @@ -510,7 +510,6 @@ public void testMVRestoreMVWithBaseTable2() { assertMVActiveEquals(MATERIALIZED_VIEW_NAME, true); } - @Test @Ignore public void testMVRestoreMVWithBaseTable3() { new MockUp() { @@ -530,7 +529,6 @@ public Table getTable(String catalogName, String dbName, String tblName) { assertMVActiveEquals(MATERIALIZED_VIEW_NAME, true); } - @Test @Ignore public void testMVRestoreMVWithBaseTable4() { new MockUp() {