From 95a1a71546009d9c1c58f8afb9bd14460e5c44a5 Mon Sep 17 00:00:00 2001 From: liuliaozhong Date: Wed, 24 Apr 2024 14:12:07 +0800 Subject: [PATCH 01/12] =?UTF-8?q?feat:=20=E6=9C=AC=E5=9C=B0=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E4=B8=8A=E4=BC=A0=E7=B1=BB=E5=9E=8B=E9=99=90=E5=88=B6?= =?UTF-8?q?=EF=BC=8C=E9=9C=80=E8=A6=81=E6=94=AF=E6=8C=81.xx.xx=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=20#2917?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/web/request/globalsetting/FileUploadSettingReq.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/FileUploadSettingReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/FileUploadSettingReq.java index 85252c67c2..289139e529 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/FileUploadSettingReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/FileUploadSettingReq.java @@ -62,7 +62,7 @@ public class FileUploadSettingReq { private Integer restrictMode; @ApiModelProperty("后缀列表") - private List<@Pattern(regexp = "^\\.[A-Za-z0-9_-]{1,24}$", + private List<@Pattern(regexp = "^(\\.[A-Za-z0-9_-]{1,24})+$", message = "{validation.constraints.InvalidUploadFileSuffix.message}") String> suffixList; } From b531b7c0ed06b4bb734676903c6c986be943e658 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 9 May 2024 16:15:44 +0800 Subject: [PATCH 02/12] =?UTF-8?q?doc:=203.9.2=20=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E6=97=A5=E5=BF=97=20#2964?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- versionLogs/en/V3.9.2_2024-05-09.md | 28 ++++++++++++++++++++++++++ versionLogs/zh_CN/V3.9.2_2024-05-09.md | 28 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 versionLogs/en/V3.9.2_2024-05-09.md create mode 100644 versionLogs/zh_CN/V3.9.2_2024-05-09.md diff --git a/versionLogs/en/V3.9.2_2024-05-09.md b/versionLogs/en/V3.9.2_2024-05-09.md new file mode 100644 index 0000000000..e0755d795f --- /dev/null +++ b/versionLogs/en/V3.9.2_2024-05-09.md @@ -0,0 +1,28 @@ +# V3.9.2 Release Note + + + + +### Feature +- [ Feature ] Added support for container execution [details](http://github.com/TencentBlueKing/bk-job/issues/2725) +- [ Feature ] Added MacOS host statistics data to the operation analysis page [details](http://github.com/TencentBlueKing/bk-job/issues/2842) + + +### Improved +- [ Improved ] Optimized the creation of IP white lists, which will no longer automatically select any "target business" [details](http://github.com/TencentBlueKing/bk-job/issues/2892) +- [ Improved ] Added a shortcut to "Load in previous execution parameters" on the parameter filling page of job plan [details](http://github.com/TencentBlueKing/bk-job/issues/2824) +- [ Improved ] Added an operation entry for "Relaunch" on the task details page [details](http://github.com/TencentBlueKing/bk-job/issues/2823) +- [ Improved ] To reduce resource usage, when a business does not exist, the CRON tasks in JOB will be automatically closed after a delay [details](http://github.com/TencentBlueKing/bk-job/issues/621) +- [ Improved ] Optimized the storage dependency of the Cron scheduling engine to improve service availability [details](http://github.com/TencentBlueKing/bk-job/issues/2809) + + +### Fixed +- [ Fixed ] Solved the problem of data conflict caused by reading duplicate execution history data during archiving [details](http://github.com/TencentBlueKing/bk-job/issues/2962) +- [ Fixed ] Solved the problem of manual input using Chinese commas in the host selector causing abnormal parsing [details](http://github.com/TencentBlueKing/bk-job/issues/2945) +- [ Fixed ] Solved the problem of the search box for IP white lists lacking the "Action on" option [details](http://github.com/TencentBlueKing/bk-job/issues/2872) +- [ Fixed ] Solved the problem of mismatched access paths and page navigation menu selection states [details](http://github.com/TencentBlueKing/bk-job/issues/2834) +- [ Fixed ] Solved the problem of inability to save personalized script editing [details](http://github.com/TencentBlueKing/bk-job/issues/2838) +- [ Fixed ] Solved the problem of error thrown when searching detection records using interception ID without entering a number [details](http://github.com/TencentBlueKing/bk-job/issues/2837) +- [ Fixed ] Solved the problem of job execution status showing success, but the output execution log still showing as executing [details](http://github.com/TencentBlueKing/bk-job/issues/2849) +- [ Fixed ] Supplemented the association between editing and usage permissions that were missing after creating credentials [details](http://github.com/TencentBlueKing/bk-job/issues/2457) +- [ Fixed ] Solved the problem of occasional failure in local file distribution due to temporary file cleaning [details](http://github.com/TencentBlueKing/bk-job/issues/2771) \ No newline at end of file diff --git a/versionLogs/zh_CN/V3.9.2_2024-05-09.md b/versionLogs/zh_CN/V3.9.2_2024-05-09.md new file mode 100644 index 0000000000..24e40a5c12 --- /dev/null +++ b/versionLogs/zh_CN/V3.9.2_2024-05-09.md @@ -0,0 +1,28 @@ +# V3.9.2 版本更新日志 + + + + +### 新增 +- [ 新增 ] 支持容器执行 [详情](http://github.com/TencentBlueKing/bk-job/issues/2725) +- [ 新增 ] 运营分析页面新增MacOS主机统计数据 [详情](http://github.com/TencentBlueKing/bk-job/issues/2842) + + +### 优化 +- [ 优化 ] 优化新建IP白名单时，不自动勾选任何“目标业务” [详情](http://github.com/TencentBlueKing/bk-job/issues/2892) +- [ 优化 ] 在作业执行方案的填参页增加“填入上一次执行参数”的快捷入口 [详情](http://github.com/TencentBlueKing/bk-job/issues/2824) +- [ 优化 ] 在任务详情页中增加“去重做”的操作入口 [详情](http://github.com/TencentBlueKing/bk-job/issues/2823) +- [ 优化 ] 当业务不存在后，为了减少资源占用，对于在JOB的Cron任务会延时自动关闭 [详情](http://github.com/TencentBlueKing/bk-job/issues/621) +- [ 优化 ] 优化定时调度引擎的存储依赖，提升服务可用性 [详情](http://github.com/TencentBlueKing/bk-job/issues/2809) + + +### 修复 +- [ 修复 ] 修复了执行历史归档读取数据重复导致归档数据写入冲突的问题 [详情](http://github.com/TencentBlueKing/bk-job/issues/2962) +- [ 修复 ] 解决主机选择器的手动输入使用中文顿号会导致无法正常解析的问题 [详情](http://github.com/TencentBlueKing/bk-job/issues/2945) +- [ 修复 ] 修复了IP白名单的搜索框缺少“生效范围”选项的问题 [详情](http://github.com/TencentBlueKing/bk-job/issues/2872) +- [ 修复 ] 解决了访问路径和页面导航菜单选中态不匹配的问题 [详情](http://github.com/TencentBlueKing/bk-job/issues/2834) +- [ 修复 ] 解决了个性化脚本编辑无法保存的问题 [详情](http://github.com/TencentBlueKing/bk-job/issues/2838) +- [ 修复 ] 解决了检测记录使用拦截ID不输入数字搜索会抛错的问题 [详情](http://github.com/TencentBlueKing/bk-job/issues/2837) +- [ 修复 ] 解决了作业执行状态显示成功，但输出的执行日志仍然显示为执行中的问题 [详情](http://github.com/TencentBlueKing/bk-job/issues/2849) +- [ 修复 ] 补充了创建凭证后缺少编辑与使用权限的关联 [详情](http://github.com/TencentBlueKing/bk-job/issues/2457) +- [ 修复 ] 解决了因临时文件清理可能导致本地文件分发小概率失败的问题 [详情](http://github.com/TencentBlueKing/bk-job/issues/2771) \ No newline at end of file From 0b59cf54cf034e96a69edfc9a213ca389489a823 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Mon, 26 Feb 2024 16:13:46 +0800 Subject: [PATCH 03/12] =?UTF-8?q?fix:=20=E4=B8=B4=E6=97=B6=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E6=B8=85=E7=90=86=E5=AF=BC=E8=87=B4=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=88=86=E5=8F=91=E5=B0=8F=E6=A6=82=E7=8E=87?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=20#2771?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1.分发时更新临时文件最后修改时间避免被清理便于复用； 2.优化清理策略：将分发过程产生的无用上级目录一并清理，避免inode耗尽。 --- .../bk/job/common/util/file/FileUtil.java | 7 +- .../prepare/local/LocalFileDownloadTask.java | 6 ++ .../execute/task/LocalTmpFileCleanTask.java | 85 +++++++++++++++++-- 3 files changed, 89 insertions(+), 9 deletions(-) diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/file/FileUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/file/FileUtil.java index 6446fa6583..3e8a752cc5 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/file/FileUtil.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/file/FileUtil.java @@ -313,11 +313,12 @@ private static void closeStreams( * 删除空目录 * * @param directory 目录文件 + * @return 是否删除了目录 */ - public static void deleteEmptyDirectory(File directory) { + public static boolean deleteEmptyDirectory(File directory) { if (directory == null) { log.warn("Directory is null!"); - return; + return false; } if (isEmpty(directory.toPath())) { @@ -327,8 +328,10 @@ public static void deleteEmptyDirectory(File directory) { } else { log.warn("Delete directory {} failed!", directory.getPath()); } + return delete; } else { log.debug("Directory {} not empty!", directory.getPath()); + return false; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFileDownloadTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFileDownloadTask.java index e1c52ad5d9..061d39df2b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFileDownloadTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFileDownloadTask.java @@ -105,6 +105,12 @@ private Boolean doCall() { localPath = PathUtil.joinFilePath(localPath, filePath); // 如果本地文件还未下载就已存在并且Md5正确，直接完成准备阶段 if (currentLocalFileValid(localPath, nodeDTO)) { + // 将最后修改时间设置为当前时间，避免在分发过程中被清理 + File localFile = new File(localPath); + boolean lastModifyTimeSet = localFile.setLastModified(System.currentTimeMillis()); + if (!lastModifyTimeSet) { + log.warn("Fail to set lastModifyTime for {}", localPath); + } return true; } Pair pair = artifactoryClient.getFileInputStream( diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/LocalTmpFileCleanTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/LocalTmpFileCleanTask.java index a3a2694f9b..11e4aba3d6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/LocalTmpFileCleanTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/LocalTmpFileCleanTask.java @@ -25,6 +25,8 @@ package com.tencent.bk.job.execute.task; import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.file.FileUtil; import com.tencent.bk.job.common.util.file.PathUtil; import com.tencent.bk.job.execute.config.FileDistributeConfig; import com.tencent.bk.job.execute.config.LocalFileConfigForExecute; @@ -57,7 +59,7 @@ public void execute() { // 使用制品库作为存储后端时才清理下载产生的临时文件 if (JobConstants.FILE_STORAGE_BACKEND_ARTIFACTORY .equals(localFileConfigForExecute.getStorageBackend())) { - cleanLocalTmpFile(); + cleanLocalTmpFileAndDir(); } else { // 使用本地NFS作为存储后端时在job-manage中筛选被引用文件并清理NFS，此处无需清理 log.info( @@ -68,10 +70,10 @@ public void execute() { } /** - * 清理下载到本地的临时文件 + * 清理下载到本地的临时文件及目录 */ - private void cleanLocalTmpFile() { - log.info("begin to cleanLocalTmpFile downloaded from artifactory"); + private void cleanLocalTmpFileAndDir() { + log.info("begin to cleanLocalTmpFileAndDir downloaded from artifactory"); // 单个任务最长执行时间为1天，清理一天之前的所有临时文件 Date thresholdDate = new Date( System.currentTimeMillis() - 3600 * 24 * 1000 @@ -89,19 +91,88 @@ private void cleanLocalTmpFile() { File[] files = localFileDir.listFiles(); if (files != null && files.length > 0) { - Iterator fileIterator = FileUtils.iterateFiles( + Iterator fileIterator = FileUtils.iterateFilesAndDirs( localFileDir, new AgeFileFilter(thresholdDate), TrueFileFilter.TRUE ); while (fileIterator.hasNext()) { File aFile = fileIterator.next(); - log.info("Delete local tmp file {}", aFile.getPath()); - FileUtils.deleteQuietly(aFile); + // 清理存量遗留的空目录 + if (aFile.isDirectory()) { + clearEmptyDirAndParent(aFile); + continue; + } + // 清理文件 + boolean fileDeleted = FileUtils.deleteQuietly(aFile); + if (!fileDeleted) { + log.warn("Fail to delete tmp file {}, ignore", aFile.getAbsolutePath()); + continue; + } + // 同时清理两级无用的父目录 + File parentDirFile = aFile.getParentFile(); + boolean parentDirDeleted = false; + boolean grandParentDirDeleted = false; + if (parentDirFile != null) { + parentDirDeleted = deleteEmptyDirIfNotDisTributeRoot(parentDirFile); + File grandParentDirFile = parentDirFile.getParentFile(); + if (grandParentDirFile != null) { + grandParentDirDeleted = deleteEmptyDirIfNotDisTributeRoot(grandParentDirFile); + } + } + log.info( + "Local tmp file {} deleted, parentDirDeleted={}, grandParentDirDeleted={}", + aFile.getPath(), + parentDirDeleted, + grandParentDirDeleted + ); } } else { log.warn("Local tmp file directory is empty: {}", localFileDirPath); } log.info("cleanLocalTmpFile finished"); } + + /** + * 清理非分发根目录的空目录及其父目录 + */ + private void clearEmptyDirAndParent(File dirFile) { + boolean dirDeleted = deleteEmptyDirIfNotDisTributeRoot(dirFile); + if (!dirDeleted) { + return; + } + File parentDirFile = dirFile.getParentFile(); + boolean parentDirDeleted = false; + if (parentDirFile != null) { + parentDirDeleted = deleteEmptyDirIfNotDisTributeRoot(parentDirFile); + } + log.info("Local tmp dir {} deleted, parentDirDeleted={}", dirFile.getAbsolutePath(), parentDirDeleted); + } + + /** + * 清理非分发根目录的空目录 + * + * @param dirFile 目录文件 + * @return 是否删除了空目录 + */ + private boolean deleteEmptyDirIfNotDisTributeRoot(File dirFile) { + if (!isJobDisTributeRootPath(dirFile.getAbsolutePath())) { + return FileUtil.deleteEmptyDirectory(dirFile); + } + return false; + } + + /** + * 判断路径是否指向分发根目录（忽略反斜杠后缀） + * + * @param path 路径 + * @return 路径是否指向分发根目录 + */ + private boolean isJobDisTributeRootPath(String path) { + if (path == null) { + return false; + } + String jobDisTributeRootPath = StringUtil.removeSuffix(fileDistributeConfig.getJobDistributeRootPath(), "/"); + return jobDisTributeRootPath.equals(StringUtil.removeSuffix(path, "/")); + } } From 604e0497595e5e0e73f0b8ffa44666adaac56ff5 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Thu, 9 May 2024 21:43:54 +0800 Subject: [PATCH 04/12] =?UTF-8?q?perf:=20=E6=9C=AC=E5=9C=B0=E4=B8=B4?= =?UTF-8?q?=E6=97=B6=E6=96=87=E4=BB=B6=E6=B8=85=E7=90=86=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E4=BC=98=E5=8C=96=20#2951?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 支持对多种日期时间格式的解析 --- .../bk/job/common/util/date/DateUtils.java | 45 +++++++++++++++++++ .../job/common/util/date/DateUtilsTest.java | 38 ++++++++++++++++ .../manage/task/UserUploadFileCleanTask.java | 12 +++-- 3 files changed, 91 insertions(+), 4 deletions(-) diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java index 5b07c8660b..4a8f70e727 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java @@ -24,6 +24,7 @@ package com.tencent.bk.job.common.util.date; +import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.slf4j.helpers.MessageFormatter; @@ -41,8 +42,13 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Calendar; import java.util.Date; +import java.util.Formatter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * 时间处理 @@ -215,6 +221,45 @@ public static LocalDateTime convertFromMillSeconds(long millSeconds) { ZoneOffset.systemDefault().getRules().getOffset(Instant.now())); } + /** + * 尝试使用多种格式来解析字符串中的时间，只要有任意一种格式匹配即可成功解析 + * + * @param dateTime 日期时间字符串 + * @param patterns 多个可能的日期时间格式 + * @return 本地日期时间对象 + */ + public static LocalDateTime convertFromStringDateByPatterns(String dateTime, String... patterns) { + if (patterns.length == 0) { + throw new IllegalArgumentException("patterns must not be empty"); + } + LocalDateTime localDateTime = null; + Map exceptionMap = new HashMap<>(); + for (String pattern : patterns) { + try { + localDateTime = convertFromStringDate(dateTime, pattern); + } catch (Exception e) { + exceptionMap.put(pattern, e); + } + } + if (localDateTime == null) { + String patternsInvalidMsg = MessageFormatter.format( + "Fail to convertFromStringDateByPatterns: dateTime={}, patterns={}, exceptions: ", + dateTime, + patterns + ).getMessage(); + log.warn(patternsInvalidMsg); + exceptionMap.forEach((pattern, e) -> { + String patternInvalidMsg = MessageFormatter.format( + "pattern: {}, exception: ", + pattern + ).getMessage(); + log.warn(patternInvalidMsg, e); + }); + throw new IllegalArgumentException(patternsInvalidMsg); + } + return localDateTime; + } + public static LocalDateTime convertFromStringDate(String dateTime, String pattern) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); return LocalDateTime.parse(dateTime, formatter); diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/date/DateUtilsTest.java b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/date/DateUtilsTest.java index 61b60aa927..378bd6bdad 100644 --- a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/date/DateUtilsTest.java +++ b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/date/DateUtilsTest.java @@ -31,6 +31,7 @@ import java.time.ZonedDateTime; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class DateUtilsTest { @@ -74,4 +75,41 @@ void getUTCDayEndTimestamp() { // 2023-07-14 00:00:00 UTC assertThat(endTime4).isEqualTo(1689292800000L); } + + @Test + void convertFromStringDateByPatterns() { + String[] patterns = new String[]{ + "yyyy-MM-dd'T'HH:mm:ss.SSS", + "yyyy-MM-dd'T'HH:mm:ss.SS", + "yyyy-MM-dd'T'HH:mm:ss.S", + "yyyy-MM-dd'T'HH:mm:ss.", + "yyyy-MM-dd'T'HH:mm:ss" + }; + LocalDateTime localDateTime = DateUtils.convertFromStringDateByPatterns( + "2022-04-21T10:55:54.655", patterns + ); + assertThat(localDateTime).isNotNull(); + localDateTime = DateUtils.convertFromStringDateByPatterns( + "2022-04-21T10:55:54.65", patterns + ); + assertThat(localDateTime).isNotNull(); + localDateTime = DateUtils.convertFromStringDateByPatterns( + "2022-04-21T10:55:54.6", patterns + ); + assertThat(localDateTime).isNotNull(); + localDateTime = DateUtils.convertFromStringDateByPatterns( + "2022-04-21T10:55:54.", patterns + ); + assertThat(localDateTime).isNotNull(); + localDateTime = DateUtils.convertFromStringDateByPatterns( + "2022-04-21T10:55:54", patterns + ); + assertThat(localDateTime).isNotNull(); + assertThatThrownBy(() -> DateUtils.convertFromStringDateByPatterns( + "2022-04-21T10:55:54.65" + )).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> DateUtils.convertFromStringDateByPatterns( + "2022-04-21 10:55:54.65" + )).isInstanceOf(IllegalArgumentException.class); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java index 88fdd3141b..3893675311 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java @@ -153,12 +153,16 @@ private int deleteExpiredNodes(List nodeList, Set skipPath) { continue; } // 从制品库删除有效日期前创建的节点 - LocalDateTime endNodeLastDate = DateUtils.convertFromStringDate( - endNode.getLastModifiedDate(), "yyyy-MM-dd'T'HH:mm:ss.SSS" + LocalDateTime endNodeLastDate = DateUtils.convertFromStringDateByPatterns( + endNode.getLastModifiedDate(), + "yyyy-MM-dd'T'HH:mm:ss.SSS", + "yyyy-MM-dd'T'HH:mm:ss.SS", + "yyyy-MM-dd'T'HH:mm:ss.S", + "yyyy-MM-dd'T'HH:mm:ss.", + "yyyy-MM-dd'T'HH:mm:ss" ); if (endNodeLastDate - .plusDays(localFileConfigForManage.getExpireDays()) - .compareTo(LocalDateTime.now()) < 0) { + .plusDays(localFileConfigForManage.getExpireDays()).isBefore(LocalDateTime.now())) { artifactoryClient.deleteNode( artifactoryConfig.getArtifactoryJobProject(), localFileConfigForManage.getLocalUploadRepo(), From b89f27a8ee8ec28bdabaa24eda949b067d81ce76 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Fri, 10 May 2024 15:21:15 +0800 Subject: [PATCH 05/12] =?UTF-8?q?perf:=20=E6=9C=AC=E5=9C=B0=E4=B8=B4?= =?UTF-8?q?=E6=97=B6=E6=96=87=E4=BB=B6=E6=B8=85=E7=90=86=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E4=BC=98=E5=8C=96=20#2951?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1.优化制品库中本地文件清理逻辑，支持目录清理与子目录递归检测清理； 2.优化带心跳Redis锁的异常打印。 --- .../artifactory/sdk/ArtifactoryClient.java | 6 +- .../redis/util/RedisKeyHeartBeatThread.java | 33 ++- .../manage/task/UserUploadFileCleanTask.java | 259 ++++++++++++------ 3 files changed, 205 insertions(+), 93 deletions(-) diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java index 1f749b7802..696c674d72 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java @@ -433,12 +433,12 @@ public NodeDTO queryNodeDetail(String projectId, String repoName, String fullPat return resp.getData(); } - public Boolean deleteProject(String projectId) { + public boolean deleteProject(String projectId) { log.info("deleteProject:{}", projectId); throw new NotImplementedException("Not support feature", ErrorCode.NOT_SUPPORT_FEATURE); } - public Boolean deleteRepo(String projectId, String repoName, Boolean forced) { + public boolean deleteRepo(String projectId, String repoName, Boolean forced) { DeleteRepoReq req = new DeleteRepoReq(); req.setProjectId(projectId); req.setRepoName(repoName); @@ -449,7 +449,7 @@ public Boolean deleteRepo(String projectId, String repoName, Boolean forced) { return resp.getCode() == 0; } - public Boolean deleteNode(String projectId, String repoName, String fullPath) { + public boolean deleteNode(String projectId, String repoName, String fullPath) { DeleteNodeReq req = new DeleteNodeReq(); req.setProjectId(projectId); req.setRepoName(repoName); diff --git a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/RedisKeyHeartBeatThread.java b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/RedisKeyHeartBeatThread.java index 05222fce1c..d561b32ad4 100644 --- a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/RedisKeyHeartBeatThread.java +++ b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/RedisKeyHeartBeatThread.java @@ -26,8 +26,11 @@ import com.tencent.bk.job.common.util.ThreadUtils; +import io.lettuce.core.RedisCommandInterruptedException; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.data.redis.RedisSystemException; import org.springframework.data.redis.core.RedisTemplate; import java.util.concurrent.TimeUnit; @@ -70,7 +73,16 @@ public void run() { ThreadUtils.sleep(periodMillis, false); } } catch (Throwable t) { - log.error("RedisKeyHeartBeatThread {} quit unexpectedly:", this.getName(), t); + String msg = MessageFormatter.format( + "RedisKeyHeartBeatThread {} quit unexpectedly:", + this.getName() + ).getMessage(); + // 主动终止线程产生的异常只打印调试级别日志 + if (!causeByStopAtOnceInterrupt(t)) { + log.error(msg, t); + } else { + log.debug(msg, t); + } } finally { deleteRedisKeySafely(); } @@ -81,7 +93,24 @@ private void deleteRedisKeySafely() { Boolean result = redisTemplate.delete(redisKey); log.debug("delete redis key:{}, result={}", redisKey, result); } catch (Throwable e) { - log.error("Delete redis key fail", e); + // 主动终止线程产生的异常只打印调试级别日志 + if (!causeByStopAtOnceInterrupt(e)) { + log.error("Delete redis key fail", e); + } else { + log.debug("Delete redis key fail", e); + } + } + } + + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + private boolean causeByStopAtOnceInterrupt(Throwable t) { + if (runFlag) { + return false; + } + if (t instanceof RedisSystemException) { + Throwable cause = t.getCause(); + return cause instanceof RedisCommandInterruptedException; } + return false; } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java index 3893675311..f4eca41a6c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java @@ -30,22 +30,25 @@ import com.tencent.bk.job.common.artifactory.model.dto.PageData; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.redis.util.LockUtils; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLock; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import com.tencent.bk.job.common.redis.util.LockResult; import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.file.FileUtil; +import com.tencent.bk.job.common.util.ip.IpUtils; import com.tencent.bk.job.manage.config.LocalFileConfigForManage; import com.tencent.bk.job.manage.config.StorageSystemConfig; import com.tencent.bk.job.manage.service.plan.TaskPlanService; import com.tencent.bk.job.manage.service.template.TaskTemplateService; +import lombok.AllArgsConstructor; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.AgeFileFilter; import org.apache.commons.io.filefilter.TrueFileFilter; -import org.slf4j.helpers.FormattingTuple; -import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; import java.io.File; @@ -54,7 +57,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -import java.util.UUID; /** * @since 25/1/2021 21:19 @@ -63,6 +65,7 @@ @Component public class UserUploadFileCleanTask { + private static final String machineIp = IpUtils.getFirstMachineIP(); private static final String FILE_CLEAN_TASK_LOCK_KEY = "user:upload:file:clean"; private final String uploadPath; private final File uploadDirectory; @@ -71,6 +74,7 @@ public class UserUploadFileCleanTask { private final ArtifactoryConfig artifactoryConfig; private final LocalFileConfigForManage localFileConfigForManage; private final ArtifactoryClient artifactoryClient; + private final RedisTemplate redisTemplate; public UserUploadFileCleanTask( StorageSystemConfig storageSystemConfig, @@ -78,7 +82,8 @@ public UserUploadFileCleanTask( TaskPlanService taskPlanService, ArtifactoryConfig artifactoryConfig, LocalFileConfigForManage localFileConfigForManage, - @Qualifier("jobArtifactoryClient") ArtifactoryClient artifactoryClient + @Qualifier("jobArtifactoryClient") ArtifactoryClient artifactoryClient, + RedisTemplate redisTemplate ) { this.uploadPath = storageSystemConfig.getJobStorageRootPath() + "/localupload/"; this.taskTemplateService = taskTemplateService; @@ -87,22 +92,34 @@ public UserUploadFileCleanTask( this.localFileConfigForManage = localFileConfigForManage; this.artifactoryClient = artifactoryClient; this.uploadDirectory = new File(uploadPath); + this.redisTemplate = redisTemplate; } public void execute() { - String executeId = UUID.randomUUID().toString(); - + HeartBeatRedisLockConfig config = HeartBeatRedisLockConfig.getDefault(); + config.setHeartBeatThreadName("UserUploadFileCleanRedisKeyHeartBeatThread"); + HeartBeatRedisLock lock = new HeartBeatRedisLock( + redisTemplate, + FILE_CLEAN_TASK_LOCK_KEY, + machineIp, + config + ); + LockResult lockResult = lock.lock(); + if (!lockResult.isLockGotten()) { + log.info( + "lock {} gotten by another machine: {}, return", + FILE_CLEAN_TASK_LOCK_KEY, + lockResult.getLockValue() + ); + return; + } try { - if (LockUtils.tryGetDistributedLock(FILE_CLEAN_TASK_LOCK_KEY, executeId, 3600_000L)) { - Set skipFile = loadFileListFromDb(); - processFile(skipFile); - } else { - log.info("Some one else is running this task! Skip!"); - } + Set skipFile = loadFileListFromDb(); + processFile(skipFile); } catch (Exception e) { log.error("Error while running user upload file clean task!", e); } finally { - LockUtils.releaseDistributedLock(FILE_CLEAN_TASK_LOCK_KEY, executeId); + lockResult.tryToRelease(); } } @@ -120,98 +137,164 @@ private void processFile(Set skipFile) { } } - private NodeDTO getEndNode(NodeDTO nodeDTO) { - if (nodeDTO == null) return null; - PageData nodePage = artifactoryClient.listNode( - artifactoryConfig.getArtifactoryJobProject(), - localFileConfigForManage.getLocalUploadRepo(), - nodeDTO.getFullPath(), - 0, - 1 - ); - if (nodePage == null || nodePage.getRecords() == null || nodePage.getRecords().isEmpty()) { - return nodeDTO; + @Getter + @AllArgsConstructor + static class DeleteNodeResult { + // 删除的节点总数，含子节点 + int deletedNodeNum; + // 当前节点是否被删除 + boolean currentNodeDeleted; + + public DeleteNodeResult(int deletedNodeNum) { + this.deletedNodeNum = deletedNodeNum; + this.currentNodeDeleted = false; } - return getEndNode(nodePage.getRecords().get(0)); } /** - * 从制品库中删除nodeList内指定的过期节点 + * 检查并删除过期的节点及其子节点 * - * @param nodeList 需要检查的节点列表 + * @param node 目标节点 * @param skipPath 需要跳过的路径 - * @return 删除的节点数 + * @return 节点删除结果 */ - private int deleteExpiredNodes(List nodeList, Set skipPath) { - int deletedNodeNum = 0; - for (NodeDTO nodeDTO : nodeList) { - try { - NodeDTO endNode = getEndNode(nodeDTO); - String endNodePath = StringUtil.removePrefix(endNode.getFullPath(), "/"); - if (skipPath.contains(endNodePath)) { - log.info("Skip artifactory file {}", endNodePath); - continue; - } - // 从制品库删除有效日期前创建的节点 - LocalDateTime endNodeLastDate = DateUtils.convertFromStringDateByPatterns( - endNode.getLastModifiedDate(), - "yyyy-MM-dd'T'HH:mm:ss.SSS", - "yyyy-MM-dd'T'HH:mm:ss.SS", - "yyyy-MM-dd'T'HH:mm:ss.S", - "yyyy-MM-dd'T'HH:mm:ss.", - "yyyy-MM-dd'T'HH:mm:ss" - ); - if (endNodeLastDate - .plusDays(localFileConfigForManage.getExpireDays()).isBefore(LocalDateTime.now())) { - artifactoryClient.deleteNode( - artifactoryConfig.getArtifactoryJobProject(), - localFileConfigForManage.getLocalUploadRepo(), - nodeDTO.getFullPath() - ); - log.info("localFile {} in artifactory deleted", endNode.getFullPath()); - deletedNodeNum += 1; - } - } catch (Throwable t) { - FormattingTuple msg = MessageFormatter.format( - "Fail to check and process expire node {}", - nodeDTO.getFullPath() - ); - log.warn(msg.getMessage(), t); - } + private DeleteNodeResult checkAndDeleteExpiredNodeAndChild(NodeDTO node, Set skipPath) { + if (node == null) { + return new DeleteNodeResult(0); + } + String nodePath = StringUtil.removePrefix(node.getFullPath(), "/"); + if (skipPath.contains(nodePath)) { + log.info("Skip artifactory node {}", nodePath); + return new DeleteNodeResult(0); + } + boolean isFolder = node.getFolder() != null && node.getFolder(); + if (!isFolder) { + // 文件节点，直接检查并删除 + return checkAndDeleteFileNode(node); + } else { + // 目录节点，需要递归遍历检查并删除 + return checkAndDeleteDirNode(node, skipPath); } - return deletedNodeNum; } - /** - * 清理蓝鲸制品库中的临时文件 - * - * @param skipFile 需要跳过的文件 - */ - private void processArtifactoryFile(Set skipFile) { - int start = 0; + private DeleteNodeResult checkAndDeleteDirNode(NodeDTO node, Set skipPath) { + int deletedNum = 0; + // 1.先处理子节点 + PageData nodePage; + int pageNumber = 0; int pageSize = 100; - int deletedNodeNum; - List nodeList; do { - PageData nodePage = artifactoryClient.listNode( + nodePage = artifactoryClient.listNode( artifactoryConfig.getArtifactoryJobProject(), localFileConfigForManage.getLocalUploadRepo(), - "/", - start, + node.getFullPath(), + pageNumber, pageSize ); - if (nodePage == null) { - log.warn("nodePage is null"); - return; + if (isEmpty(nodePage)) { + break; + } + List subNodeList = nodePage.getRecords(); + int subNodeDeletedNum = 0; + for (NodeDTO subNode : subNodeList) { + DeleteNodeResult result = checkAndDeleteExpiredNodeAndChild(subNode, skipPath); + deletedNum += result.getDeletedNodeNum(); + if (result.currentNodeDeleted) { + subNodeDeletedNum += 1; + } + } + if (subNodeDeletedNum == 0) { + // 所有子节点本身都没有被删除，页码数才增加，否则当前页需要重新检查处理 + pageNumber += 1; } - nodeList = nodePage.getRecords(); - if (CollectionUtils.isNotEmpty(nodeList)) { - deletedNodeNum = deleteExpiredNodes(nodeList, skipFile); - start += pageSize - deletedNodeNum; + } while (!isEmpty(nodePage)); + // 2.根目录不删除 + if ("/".equals(node.getFullPath().trim())) { + return new DeleteNodeResult(deletedNum); + } + // 3.非根目录再判断是否为空目录，若为空也删除 + if (isDirNodeEmpty(node)) { + if (deleteNode(node)) { + deletedNum += 1; + return new DeleteNodeResult(deletedNum, true); } else { - deletedNodeNum = 0; + log.warn("Fail to delete empty dirNode:{}", node.getFullPath()); } - } while (deletedNodeNum != 0 || (CollectionUtils.isNotEmpty(nodeList))); + } + return new DeleteNodeResult(deletedNum); + } + + private boolean isDirNodeEmpty(NodeDTO dirNode) { + PageData nodePage = artifactoryClient.listNode( + artifactoryConfig.getArtifactoryJobProject(), + localFileConfigForManage.getLocalUploadRepo(), + dirNode.getFullPath(), + 0, + 1 + ); + return nodePage != null && (nodePage.getRecords() == null || nodePage.getRecords().isEmpty()); + } + + private DeleteNodeResult checkAndDeleteFileNode(NodeDTO node) { + if (isExpired(node)) { + if (deleteNode(node)) { + return new DeleteNodeResult(1, true); + } else { + return new DeleteNodeResult(0); + } + } + return new DeleteNodeResult(0); + } + + private boolean isEmpty(PageData nodePage) { + return nodePage == null || nodePage.getRecords() == null || nodePage.getRecords().isEmpty(); + } + + private boolean isExpired(NodeDTO nodeDTO) { + LocalDateTime endNodeLastDate = DateUtils.convertFromStringDateByPatterns( + nodeDTO.getLastModifiedDate(), + "yyyy-MM-dd'T'HH:mm:ss.SSS", + "yyyy-MM-dd'T'HH:mm:ss.SS", + "yyyy-MM-dd'T'HH:mm:ss.S", + "yyyy-MM-dd'T'HH:mm:ss.", + "yyyy-MM-dd'T'HH:mm:ss" + ); + boolean result = endNodeLastDate + .plusDays(localFileConfigForManage.getExpireDays()).isBefore(LocalDateTime.now()); + log.debug( + "check node {}, lastModify={}, expired={}", + nodeDTO.getFullPath(), + nodeDTO.getLastModifiedDate(), + result + ); + return result; + } + + private boolean deleteNode(NodeDTO nodeDTO) { + boolean deleted = artifactoryClient.deleteNode( + artifactoryConfig.getArtifactoryJobProject(), + localFileConfigForManage.getLocalUploadRepo(), + nodeDTO.getFullPath() + ); + log.info("Delete localUpload node {} in artifactory, result={}", nodeDTO.getFullPath(), deleted); + return deleted; + } + + /** + * 清理蓝鲸制品库中的临时文件 + * + * @param skipFile 需要跳过的文件 + */ + private void processArtifactoryFile(Set skipFile) { + NodeDTO localUploadNode = artifactoryClient.queryNodeDetail( + artifactoryConfig.getArtifactoryJobProject(), + localFileConfigForManage.getLocalUploadRepo(), + "/" + ); + DeleteNodeResult result = checkAndDeleteExpiredNodeAndChild(localUploadNode, skipFile); + if (result.deletedNodeNum > 0) { + log.info("processArtifactoryFile finished, deletedNodeNum={}", result.deletedNodeNum); + } } /** From d0132886eed85693c5fd6bb869c081597f8f709a Mon Sep 17 00:00:00 2001 From: jsonwan Date: Fri, 10 May 2024 18:16:19 +0800 Subject: [PATCH 06/12] =?UTF-8?q?perf:=20=E6=9C=AC=E5=9C=B0=E4=B8=B4?= =?UTF-8?q?=E6=97=B6=E6=96=87=E4=BB=B6=E6=B8=85=E7=90=86=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E4=BC=98=E5=8C=96=20#2951?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1.支持配置本地文件过期时间与是否删除。 --- .../config/LocalFileConfigForManage.java | 3 ++ .../manage/task/UserUploadFileCleanTask.java | 46 +++++++++++-------- .../bk-job/templates/configmap-common.yaml | 2 + .../kubernetes/charts/bk-job/values.yaml | 4 ++ 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/LocalFileConfigForManage.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/LocalFileConfigForManage.java index 67d6671a67..3ec536d727 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/LocalFileConfigForManage.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/LocalFileConfigForManage.java @@ -38,6 +38,9 @@ public class LocalFileConfigForManage { @Value("${local-file.artifactory.repo:localupload}") private String localUploadRepo; + @Value("${local-file.expire-delete:true}") + private boolean expireDelete = true; + @Value("${local-file.expire-days:7}") private Integer expireDays; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java index f4eca41a6c..ff2cb1e0bc 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java @@ -181,7 +181,7 @@ private DeleteNodeResult checkAndDeleteDirNode(NodeDTO node, Set skipPat int deletedNum = 0; // 1.先处理子节点 PageData nodePage; - int pageNumber = 0; + int pageNumber = 1; int pageSize = 100; do { nodePage = artifactoryClient.listNode( @@ -216,6 +216,7 @@ private DeleteNodeResult checkAndDeleteDirNode(NodeDTO node, Set skipPat if (isDirNodeEmpty(node)) { if (deleteNode(node)) { deletedNum += 1; + log.info("Delete empty dirNode: {}", node.getFullPath()); return new DeleteNodeResult(deletedNum, true); } else { log.warn("Fail to delete empty dirNode:{}", node.getFullPath()); @@ -229,7 +230,7 @@ private boolean isDirNodeEmpty(NodeDTO dirNode) { artifactoryConfig.getArtifactoryJobProject(), localFileConfigForManage.getLocalUploadRepo(), dirNode.getFullPath(), - 0, + 1, 1 ); return nodePage != null && (nodePage.getRecords() == null || nodePage.getRecords().isEmpty()); @@ -271,13 +272,18 @@ private boolean isExpired(NodeDTO nodeDTO) { } private boolean deleteNode(NodeDTO nodeDTO) { - boolean deleted = artifactoryClient.deleteNode( - artifactoryConfig.getArtifactoryJobProject(), - localFileConfigForManage.getLocalUploadRepo(), - nodeDTO.getFullPath() - ); - log.info("Delete localUpload node {} in artifactory, result={}", nodeDTO.getFullPath(), deleted); - return deleted; + if (localFileConfigForManage.isExpireDelete()) { + boolean deleted = artifactoryClient.deleteNode( + artifactoryConfig.getArtifactoryJobProject(), + localFileConfigForManage.getLocalUploadRepo(), + nodeDTO.getFullPath() + ); + log.info("Delete localUpload node {} in artifactory, result={}", nodeDTO.getFullPath(), deleted); + return deleted; + } else { + log.info("Fake Delete localUpload node {} in artifactory, result={}", nodeDTO.getFullPath(), false); + return false; + } } /** @@ -314,17 +320,19 @@ private void processLocalFile(Set skipFile) { if (log.isDebugEnabled()) { log.debug("Skip file {}", aFile.getPath()); } - continue; } else { - log.info("Delete file {}", aFile.getPath()); - FileUtils.deleteQuietly(aFile); - } - - try { - FileUtil.deleteEmptyDirectory(aFile.getParentFile()); - FileUtil.deleteEmptyDirectory(aFile.getParentFile().getParentFile()); - } catch (Exception e) { - log.warn("Error while delete empty parent of file {}", aFile.getPath()); + if (localFileConfigForManage.isExpireDelete()) { + log.info("Delete file {}", aFile.getPath()); + FileUtils.deleteQuietly(aFile); + try { + FileUtil.deleteEmptyDirectory(aFile.getParentFile()); + FileUtil.deleteEmptyDirectory(aFile.getParentFile().getParentFile()); + } catch (Exception e) { + log.warn("Error while delete empty parent of file {}", aFile.getPath()); + } + } else { + log.info("Fake Delete file {}", aFile.getPath()); + } } } } diff --git a/support-files/kubernetes/charts/bk-job/templates/configmap-common.yaml b/support-files/kubernetes/charts/bk-job/templates/configmap-common.yaml index 8fc8ffa0fb..fa36f1e22d 100644 --- a/support-files/kubernetes/charts/bk-job/templates/configmap-common.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/configmap-common.yaml @@ -147,6 +147,8 @@ data: download: concurrency: {{ .Values.localFile.artifactory.download.concurrency }} repo: {{ .Values.localFile.artifactory.repo }} + expire-delete: {{ .Values.localFile.artifactory.repo }} + expire-days: {{ .Values.localFile.expireDays }} swagger: url: {{ .Values.swagger.url }} management: diff --git a/support-files/kubernetes/charts/bk-job/values.yaml b/support-files/kubernetes/charts/bk-job/values.yaml index 5e812cd80a..18ba2d40f7 100644 --- a/support-files/kubernetes/charts/bk-job/values.yaml +++ b/support-files/kubernetes/charts/bk-job/values.yaml @@ -552,6 +552,10 @@ localFile: concurrency: 10 # 制品库中作业平台使用的本地文件仓库代码，初始化时由作业平台通过Admin账号自动创建 repo: localupload + # 本地文件过期后是否删除 + expireDelete: true + # 本地文件过期时间（天） + expireDays: 7 ## Swagger配置 swagger: From 4aaa1217376ab0b0369f8b773289c8d8952c2992 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Fri, 10 May 2024 19:55:14 +0800 Subject: [PATCH 07/12] =?UTF-8?q?perf:=20=E6=9C=AC=E5=9C=B0=E4=B8=B4?= =?UTF-8?q?=E6=97=B6=E6=96=87=E4=BB=B6=E6=B8=85=E7=90=86=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E4=BC=98=E5=8C=96=20#2951?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1.支持配置本地文件过期时间与是否删除。 --- .../kubernetes/charts/bk-job/templates/configmap-common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support-files/kubernetes/charts/bk-job/templates/configmap-common.yaml b/support-files/kubernetes/charts/bk-job/templates/configmap-common.yaml index fa36f1e22d..95e4df913b 100644 --- a/support-files/kubernetes/charts/bk-job/templates/configmap-common.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/configmap-common.yaml @@ -147,7 +147,7 @@ data: download: concurrency: {{ .Values.localFile.artifactory.download.concurrency }} repo: {{ .Values.localFile.artifactory.repo }} - expire-delete: {{ .Values.localFile.artifactory.repo }} + expire-delete: {{ .Values.localFile.expireDelete }} expire-days: {{ .Values.localFile.expireDays }} swagger: url: {{ .Values.swagger.url }} From 7d819d0d2db104d6fabc136c1ab721ecca65111b Mon Sep 17 00:00:00 2001 From: jsonwan Date: Fri, 10 May 2024 20:44:29 +0800 Subject: [PATCH 08/12] =?UTF-8?q?perf:=20=E6=9C=AC=E5=9C=B0=E4=B8=B4?= =?UTF-8?q?=E6=97=B6=E6=96=87=E4=BB=B6=E6=B8=85=E7=90=86=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E4=BC=98=E5=8C=96=20#2951?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1.空目录按照创建时间判断是否删除； 2.单个节点处理失败后跳过。 --- .../manage/task/UserUploadFileCleanTask.java | 74 ++++++++++++++----- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java index ff2cb1e0bc..d9ba029361 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java @@ -47,6 +47,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.AgeFileFilter; import org.apache.commons.io.filefilter.TrueFileFilter; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; @@ -197,10 +198,19 @@ private DeleteNodeResult checkAndDeleteDirNode(NodeDTO node, Set skipPat List subNodeList = nodePage.getRecords(); int subNodeDeletedNum = 0; for (NodeDTO subNode : subNodeList) { - DeleteNodeResult result = checkAndDeleteExpiredNodeAndChild(subNode, skipPath); - deletedNum += result.getDeletedNodeNum(); - if (result.currentNodeDeleted) { - subNodeDeletedNum += 1; + // 单个节点处理失败后直接跳过，继续处理后续节点。 + try { + DeleteNodeResult result = checkAndDeleteExpiredNodeAndChild(subNode, skipPath); + deletedNum += result.getDeletedNodeNum(); + if (result.currentNodeDeleted) { + subNodeDeletedNum += 1; + } + } catch (Exception e) { + String msg = MessageFormatter.format( + "Fail to checkAndDelete node:{}", + subNode.getFullPath() + ).getMessage(); + log.warn(msg, e); } } if (subNodeDeletedNum == 0) { @@ -212,8 +222,9 @@ private DeleteNodeResult checkAndDeleteDirNode(NodeDTO node, Set skipPat if ("/".equals(node.getFullPath().trim())) { return new DeleteNodeResult(deletedNum); } - // 3.非根目录再判断是否为空目录，若为空也删除 - if (isDirNodeEmpty(node)) { + // 3.非根目录再判断是否为空目录，若为空且创建日期过期了也删除 + // 此处的目录过期需要用创建时间判断，因为子目录/文件删除会导致父目录的最后修改时间更新 + if (isNodeCreatedTimeExpired(node) && isDirNodeEmpty(node)) { if (deleteNode(node)) { deletedNum += 1; log.info("Delete empty dirNode: {}", node.getFullPath()); @@ -237,7 +248,7 @@ private boolean isDirNodeEmpty(NodeDTO dirNode) { } private DeleteNodeResult checkAndDeleteFileNode(NodeDTO node) { - if (isExpired(node)) { + if (isNodeLastModifyTimeExpired(node)) { if (deleteNode(node)) { return new DeleteNodeResult(1, true); } else { @@ -251,17 +262,9 @@ private boolean isEmpty(PageData nodePage) { return nodePage == null || nodePage.getRecords() == null || nodePage.getRecords().isEmpty(); } - private boolean isExpired(NodeDTO nodeDTO) { - LocalDateTime endNodeLastDate = DateUtils.convertFromStringDateByPatterns( - nodeDTO.getLastModifiedDate(), - "yyyy-MM-dd'T'HH:mm:ss.SSS", - "yyyy-MM-dd'T'HH:mm:ss.SS", - "yyyy-MM-dd'T'HH:mm:ss.S", - "yyyy-MM-dd'T'HH:mm:ss.", - "yyyy-MM-dd'T'HH:mm:ss" - ); - boolean result = endNodeLastDate - .plusDays(localFileConfigForManage.getExpireDays()).isBefore(LocalDateTime.now()); + private boolean isNodeLastModifyTimeExpired(NodeDTO nodeDTO) { + LocalDateTime lastModifyDateTime = parseLastModifyDateTime(nodeDTO); + boolean result = isExpired(lastModifyDateTime); log.debug( "check node {}, lastModify={}, expired={}", nodeDTO.getFullPath(), @@ -271,6 +274,41 @@ private boolean isExpired(NodeDTO nodeDTO) { return result; } + private boolean isNodeCreatedTimeExpired(NodeDTO nodeDTO) { + LocalDateTime createdDateTime = parseCreatedDateTime(nodeDTO); + boolean result = isExpired(createdDateTime); + log.debug( + "check node {}, createdDateTime={}, expired={}", + nodeDTO.getFullPath(), + nodeDTO.getCreatedDate(), + result + ); + return result; + } + + private LocalDateTime parseLastModifyDateTime(NodeDTO nodeDTO) { + return parseDateTimeFromStr(nodeDTO.getLastModifiedDate()); + } + + private LocalDateTime parseCreatedDateTime(NodeDTO nodeDTO) { + return parseDateTimeFromStr(nodeDTO.getCreatedDate()); + } + + private LocalDateTime parseDateTimeFromStr(String dateTimeStr) { + return DateUtils.convertFromStringDateByPatterns( + dateTimeStr, + "yyyy-MM-dd'T'HH:mm:ss.SSS", + "yyyy-MM-dd'T'HH:mm:ss.SS", + "yyyy-MM-dd'T'HH:mm:ss.S", + "yyyy-MM-dd'T'HH:mm:ss.", + "yyyy-MM-dd'T'HH:mm:ss" + ); + } + + private boolean isExpired(LocalDateTime dateTime) { + return dateTime.plusDays(localFileConfigForManage.getExpireDays()).isBefore(LocalDateTime.now()); + } + private boolean deleteNode(NodeDTO nodeDTO) { if (localFileConfigForManage.isExpireDelete()) { boolean deleted = artifactoryClient.deleteNode( From cf28ba2392df337d06011be374b806c853e724a1 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Thu, 29 Feb 2024 14:28:24 +0800 Subject: [PATCH 09/12] =?UTF-8?q?perf:=20ESB=E6=8E=A5=E5=8F=A3get=5Fstep?= =?UTF-8?q?=5Fstatus=E6=94=AF=E6=8C=81=E8=BF=94=E5=9B=9E=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E4=B8=AD=E6=AD=A5=E9=AA=A4=E7=9A=84=E5=AE=9E=E6=97=B6=E6=80=BB?= =?UTF-8?q?=E8=80=97=E6=97=B6=20#2788?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1.使用外层数据结构中计算得到的总耗时； 2.完善测试用例。 --- .../api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java | 6 +++--- .../testcase/EsbGetStepInstanceStatusV3ResourceAPITest.java | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java index 31b2cc815b..3bb321cbc0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java @@ -79,9 +79,9 @@ private EsbStepInstanceStatusV3DTO buildEsbStepInstanceStatusV3DTO(StepExecution stepInst.setType(stepInstance.getExecuteType()); stepInst.setStatus(stepInstance.getStatus().getValue()); stepInst.setCreateTime(stepInstance.getCreateTime()); - stepInst.setStartTime(stepInstance.getStartTime()); - stepInst.setEndTime(stepInstance.getEndTime()); - stepInst.setTotalTime(stepInstance.getTotalTime()); + stepInst.setStartTime(executionResult.getStartTime()); + stepInst.setEndTime(executionResult.getEndTime()); + stepInst.setTotalTime(executionResult.getTotalTime()); List stepResultGroupList = new ArrayList<>(); List resultGroups = executionResult.getResultGroups(); diff --git a/tests/openapi/src/test/java/com/tencent/bk/job/api/v3/testcase/EsbGetStepInstanceStatusV3ResourceAPITest.java b/tests/openapi/src/test/java/com/tencent/bk/job/api/v3/testcase/EsbGetStepInstanceStatusV3ResourceAPITest.java index c27f779ef1..4667a0007f 100644 --- a/tests/openapi/src/test/java/com/tencent/bk/job/api/v3/testcase/EsbGetStepInstanceStatusV3ResourceAPITest.java +++ b/tests/openapi/src/test/java/com/tencent/bk/job/api/v3/testcase/EsbGetStepInstanceStatusV3ResourceAPITest.java @@ -13,6 +13,7 @@ import static io.restassured.RestAssured.given; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.notNullValue; @@ -50,6 +51,7 @@ void testGetScriptStepInstanceStatus() { .body("data.execute_count", equalTo(0)) .body("data.type", equalTo(1)) .body("data.status", notNullValue()) + .body("data.total_time", greaterThan(0)) .body("data.step_result_group_list", hasSize(1)); } @@ -73,6 +75,7 @@ void testGetFileStepInstanceStatus() { .body("data.execute_count", equalTo(0)) .body("data.type", equalTo(2)) .body("data.status", notNullValue()) + .body("data.total_time", greaterThan(0)) .body("data.step_result_group_list", hasSize(1)); } From 5452232a9aefcb11be0b4dd5a481a7065f2e5c60 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Thu, 29 Feb 2024 16:10:33 +0800 Subject: [PATCH 10/12] =?UTF-8?q?perf:=20ESB=E6=8E=A5=E5=8F=A3get=5Fstep?= =?UTF-8?q?=5Fstatus=E6=94=AF=E6=8C=81=E8=BF=94=E5=9B=9E=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E4=B8=AD=E6=AD=A5=E9=AA=A4=E7=9A=84=E5=AE=9E=E6=97=B6=E6=80=BB?= =?UTF-8?q?=E8=80=97=E6=97=B6=20#2788?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1.使用外层数据结构中计算得到的总耗时； 2.完善测试用例。 --- .../v3/model/EsbStepInstanceStatusV3DTO.java | 153 ++++++++++++++++++ ...etStepInstanceStatusV3ResourceAPITest.java | 54 ++++--- 2 files changed, 187 insertions(+), 20 deletions(-) create mode 100644 tests/openapi/src/test/java/com/tencent/bk/job/api/v3/model/EsbStepInstanceStatusV3DTO.java diff --git a/tests/openapi/src/test/java/com/tencent/bk/job/api/v3/model/EsbStepInstanceStatusV3DTO.java b/tests/openapi/src/test/java/com/tencent/bk/job/api/v3/model/EsbStepInstanceStatusV3DTO.java new file mode 100644 index 0000000000..df86852f7a --- /dev/null +++ b/tests/openapi/src/test/java/com/tencent/bk/job/api/v3/model/EsbStepInstanceStatusV3DTO.java @@ -0,0 +1,153 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.api.v3.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +@Data +public class EsbStepInstanceStatusV3DTO { + + /** + * 步骤实例id + */ + @JsonProperty("step_instance_id") + private Long id; + + /** + * 执行次数 + */ + @JsonProperty("execute_count") + private Integer executeCount; + /** + * 名称 + */ + @JsonProperty("name") + private String name; + + /** + * 步骤类型：1.脚本步骤; 2.文件步骤; 4.SQL步骤 + */ + @JsonProperty("type") + private Integer type; + /** + * 状态： 1.未执行、2.正在执行、3.执行完成且成功、4.执行失败 + */ + @JsonProperty("status") + private Integer status; + /** + * 创建时间 + */ + @JsonProperty("create_time") + private Long createTime; + /** + * 开始时间 + */ + @JsonProperty("start_time") + private Long startTime; + /** + * 结束时间 + */ + @JsonProperty("end_time") + private Long endTime; + /** + * 总耗时，单位：毫秒 + */ + @JsonProperty("total_time") + private Long totalTime; + + @JsonProperty("step_result_group_list") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + private List stepResultGroupList; + + @Setter + @Getter + public static class StepResultGroup { + @JsonProperty("result_type") + private Integer resultType; + + @JsonProperty("result_type_desc") + private String resultTypeDesc; + + private String tag; + + @JsonProperty("host_size") + private Integer hostSize; + + @JsonProperty("host_result_list") + private List hostResultList; + } + + @Setter + @Getter + public static class HostResult { + @JsonProperty("bk_host_id") + private Long hostId; + + private String ip; + + private String ipv6; + + @JsonProperty("bk_agent_id") + private String agentId; + + @JsonProperty("bk_cloud_id") + private Long cloudAreaId; + + @JsonProperty("bk_cloud_name") + private String cloudAreaName; + + private Integer status; + + @JsonProperty("status_desc") + private String statusDesc; + + private String tag; + + @JsonProperty("exit_code") + private Integer exitCode; + + /** + * 开始时间 + */ + @JsonProperty("start_time") + private Long startTime; + /** + * 结束时间 + */ + @JsonProperty("end_time") + private Long endTime; + /** + * 总耗时，单位：毫秒 + */ + @JsonProperty("total_time") + private Long totalTime; + } +} diff --git a/tests/openapi/src/test/java/com/tencent/bk/job/api/v3/testcase/EsbGetStepInstanceStatusV3ResourceAPITest.java b/tests/openapi/src/test/java/com/tencent/bk/job/api/v3/testcase/EsbGetStepInstanceStatusV3ResourceAPITest.java index 4667a0007f..c6d2c7b842 100644 --- a/tests/openapi/src/test/java/com/tencent/bk/job/api/v3/testcase/EsbGetStepInstanceStatusV3ResourceAPITest.java +++ b/tests/openapi/src/test/java/com/tencent/bk/job/api/v3/testcase/EsbGetStepInstanceStatusV3ResourceAPITest.java @@ -1,21 +1,21 @@ package com.tencent.bk.job.api.v3.testcase; import com.tencent.bk.job.api.constant.ErrorCode; +import com.tencent.bk.job.api.model.EsbResp; import com.tencent.bk.job.api.props.TestProps; import com.tencent.bk.job.api.util.ApiUtil; import com.tencent.bk.job.api.util.Operations; import com.tencent.bk.job.api.v3.constants.APIV3Urls; import com.tencent.bk.job.api.v3.model.EsbJobExecuteV3DTO; +import com.tencent.bk.job.api.v3.model.EsbStepInstanceStatusV3DTO; +import io.restassured.common.mapper.TypeRef; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import static io.restassured.RestAssured.given; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.notNullValue; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; /** * 获取步骤状态 API 测试 @@ -36,7 +36,7 @@ void testGetScriptStepInstanceStatus() { // 1.触发一次执行 EsbJobExecuteV3DTO esbJobExecuteV3DTO = Operations.fastExecuteScriptTask(); // 2.获取步骤详情 - given() + EsbStepInstanceStatusV3DTO stepInstanceStatusV3DTO = given() .spec(ApiUtil.requestSpec(TestProps.DEFAULT_TEST_USER)) .param("bk_scope_type", "biz") .param("bk_scope_id", TestProps.DEFAULT_BIZ) @@ -46,13 +46,20 @@ void testGetScriptStepInstanceStatus() { .get(APIV3Urls.GET_STEP_INSTANCE_STATUS) .then() .spec(ApiUtil.successResponseSpec()) - .body("data", notNullValue()) - .body("data.step_instance_id", equalTo(esbJobExecuteV3DTO.getStepInstanceId())) - .body("data.execute_count", equalTo(0)) - .body("data.type", equalTo(1)) - .body("data.status", notNullValue()) - .body("data.total_time", greaterThan(0)) - .body("data.step_result_group_list", hasSize(1)); + .extract() + .body() + .as(new TypeRef>() { + }) + .getData(); + assertThat(stepInstanceStatusV3DTO).isNotNull(); + assertThat(stepInstanceStatusV3DTO.getId()).isEqualTo(esbJobExecuteV3DTO.getStepInstanceId()); + assertThat(stepInstanceStatusV3DTO.getExecuteCount()).isEqualTo(0); + assertThat(stepInstanceStatusV3DTO.getType()).isEqualTo(1); + assertThat(stepInstanceStatusV3DTO.getStatus()).isNotNull(); + assertThat(stepInstanceStatusV3DTO.getStartTime()).isGreaterThan(0L); + assertThat(stepInstanceStatusV3DTO.getTotalTime()).isGreaterThan(0L); + assertThat(stepInstanceStatusV3DTO.getStepResultGroupList()).isNotNull(); + assertThat(stepInstanceStatusV3DTO.getStepResultGroupList().size()).isEqualTo(1); } @Test @@ -61,7 +68,7 @@ void testGetFileStepInstanceStatus() { // 1.触发一次执行 EsbJobExecuteV3DTO esbJobExecuteV3DTO = Operations.fastTransferFileTask(); // 2.获取步骤详情 - given() + EsbStepInstanceStatusV3DTO stepInstanceStatusV3DTO = given() .spec(ApiUtil.requestSpec(TestProps.DEFAULT_TEST_USER)) .param("bk_scope_type", "biz") .param("bk_scope_id", TestProps.DEFAULT_BIZ) @@ -70,13 +77,20 @@ void testGetFileStepInstanceStatus() { .get(APIV3Urls.GET_STEP_INSTANCE_STATUS) .then() .spec(ApiUtil.successResponseSpec()) - .body("data", notNullValue()) - .body("data.step_instance_id", equalTo(esbJobExecuteV3DTO.getStepInstanceId())) - .body("data.execute_count", equalTo(0)) - .body("data.type", equalTo(2)) - .body("data.status", notNullValue()) - .body("data.total_time", greaterThan(0)) - .body("data.step_result_group_list", hasSize(1)); + .extract() + .body() + .as(new TypeRef>() { + }) + .getData(); + assertThat(stepInstanceStatusV3DTO).isNotNull(); + assertThat(stepInstanceStatusV3DTO.getId()).isEqualTo(esbJobExecuteV3DTO.getStepInstanceId()); + assertThat(stepInstanceStatusV3DTO.getExecuteCount()).isEqualTo(0); + assertThat(stepInstanceStatusV3DTO.getType()).isEqualTo(2); + assertThat(stepInstanceStatusV3DTO.getStatus()).isNotNull(); + assertThat(stepInstanceStatusV3DTO.getStartTime()).isGreaterThan(0L); + assertThat(stepInstanceStatusV3DTO.getTotalTime()).isGreaterThan(0L); + assertThat(stepInstanceStatusV3DTO.getStepResultGroupList()).isNotNull(); + assertThat(stepInstanceStatusV3DTO.getStepResultGroupList().size()).isEqualTo(1); } @Test From ac414684828043d5fee2dd927461fa514c5a80e7 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Sat, 11 May 2024 12:18:39 +0800 Subject: [PATCH 11/12] =?UTF-8?q?perf:=20=E6=9C=AC=E5=9C=B0=E4=B8=B4?= =?UTF-8?q?=E6=97=B6=E6=96=87=E4=BB=B6=E6=B8=85=E7=90=86=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E4=BC=98=E5=8C=96=20#2951?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 处理review意见 --- .../bk/job/common/util/date/DateUtils.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java index 4a8f70e727..8c0fd9fd09 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java @@ -232,32 +232,32 @@ public static LocalDateTime convertFromStringDateByPatterns(String dateTime, Str if (patterns.length == 0) { throw new IllegalArgumentException("patterns must not be empty"); } - LocalDateTime localDateTime = null; + LocalDateTime localDateTime; Map exceptionMap = new HashMap<>(); for (String pattern : patterns) { try { localDateTime = convertFromStringDate(dateTime, pattern); + if (localDateTime != null) { + return localDateTime; + } } catch (Exception e) { exceptionMap.put(pattern, e); } } - if (localDateTime == null) { - String patternsInvalidMsg = MessageFormatter.format( - "Fail to convertFromStringDateByPatterns: dateTime={}, patterns={}, exceptions: ", - dateTime, - patterns + String patternsInvalidMsg = MessageFormatter.format( + "Fail to convertFromStringDateByPatterns: dateTime={}, patterns={}, exceptions: ", + dateTime, + patterns + ).getMessage(); + log.warn(patternsInvalidMsg); + exceptionMap.forEach((pattern, e) -> { + String patternInvalidMsg = MessageFormatter.format( + "pattern: {}, exception: ", + pattern ).getMessage(); - log.warn(patternsInvalidMsg); - exceptionMap.forEach((pattern, e) -> { - String patternInvalidMsg = MessageFormatter.format( - "pattern: {}, exception: ", - pattern - ).getMessage(); - log.warn(patternInvalidMsg, e); - }); - throw new IllegalArgumentException(patternsInvalidMsg); - } - return localDateTime; + log.warn(patternInvalidMsg, e); + }); + throw new IllegalArgumentException(patternsInvalidMsg); } public static LocalDateTime convertFromStringDate(String dateTime, String pattern) { From 37b75c3fc9bc032616b4d5cd897bf1cab676e732 Mon Sep 17 00:00:00 2001 From: hLinx <327159425@qq.com> Date: Mon, 13 May 2024 16:16:45 +0800 Subject: [PATCH 12/12] =?UTF-8?q?feat:=20=E7=99=BB=E5=BD=95=E5=B0=8F?= =?UTF-8?q?=E7=AA=97=E4=BF=AE=E6=94=B9=E4=B8=BA=E4=BD=BF=E7=94=A8window.op?= =?UTF-8?q?en=20#2973=20#=20Reviewed,=20transaction=20id:=207746?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/frontend/package.json | 1 + src/frontend/src/utils/request/middleware/response.js | 6 ++++-- src/frontend/static/login_success.html | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/frontend/package.json b/src/frontend/package.json index 1b403ecaac..ef561a5757 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -15,6 +15,7 @@ "license": "ISC", "dependencies": { "@blueking/ip-selector": "0.3.0-beta.21", + "@blueking/login-modal": "1.0.4", "@blueking/notice-component-vue2": "2.0.0-beta.2", "@blueking/paas-login": "0.0.11", "@blueking/user-selector": "1.0.12", diff --git a/src/frontend/src/utils/request/middleware/response.js b/src/frontend/src/utils/request/middleware/response.js index 6f1bd79654..8d6b0ce92b 100644 --- a/src/frontend/src/utils/request/middleware/response.js +++ b/src/frontend/src/utils/request/middleware/response.js @@ -26,9 +26,9 @@ import AuthResultModel from '@model/auth-result'; import EventBus from '@utils/event-bus'; +import { showLoginModal } from '@blueking/login-modal' import { - loginDialog, messageError, permissionDialog, systemPermission, @@ -93,7 +93,9 @@ export default (interceptors) => { // 未登录 case 401: if (hasLogined) { - loginDialog(error.message); + showLoginModal({ + loginUrl: `${error.message}&c_url=${decodeURIComponent(`${window.location.origin}/static/login_success.html`)}` + }) } else { window.location.href = `${error.message}&c_url=${decodeURIComponent(window.location.href)}`; } diff --git a/src/frontend/static/login_success.html b/src/frontend/static/login_success.html index 2af6d86cfe..8a3046ce07 100644 --- a/src/frontend/static/login_success.html +++ b/src/frontend/static/login_success.html @@ -33,7 +33,8 @@