From 70563e01a2840ad79400924546477340b3181efc Mon Sep 17 00:00:00 2001 From: liuliaozhong Date: Wed, 8 May 2024 20:28:11 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20mongodb=E6=95=B0=E6=8D=AE=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E8=87=AA=E5=8A=A8=E6=B8=85=E7=90=86=20#2783?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service-job-backup/build.gradle | 1 + .../archive/AbstractMongoDBArchivist.java | 264 ++++++++++++++++++ .../archive/JobExecuteArchiveManage.java | 38 ++- .../archive/impl/MongoDBFileLogArchivist.java | 54 ++++ .../impl/MongoDBScriptLogArchivist.java | 54 ++++ .../backup/config/ArchiveConfiguration.java | 9 +- .../config/ExecuteMongoDBConfiguration.java | 74 +++++ .../job/backup/model/dto/ArchiveSummary.java | 4 + .../templates/job-backup/configmap.yaml | 6 +- .../#etc#job#job-backup#job-backup.yml | 3 + 10 files changed, 501 insertions(+), 6 deletions(-) create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractMongoDBArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/MongoDBFileLogArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/MongoDBScriptLogArchivist.java create mode 100644 src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteMongoDBConfiguration.java diff --git a/src/backend/job-backup/service-job-backup/build.gradle b/src/backend/job-backup/service-job-backup/build.gradle index 19c66dab8b..346d4887ba 100644 --- a/src/backend/job-backup/service-job-backup/build.gradle +++ b/src/backend/job-backup/service-job-backup/build.gradle @@ -38,6 +38,7 @@ dependencies { implementation("org.springframework.boot:spring-boot-starter-web") implementation("org.springframework.boot:spring-boot-starter-jdbc") implementation("org.springframework.boot:spring-boot-starter-jooq") + implementation("org.springframework.boot:spring-boot-starter-data-mongodb") implementation "org.apache.commons:commons-collections4" implementation "commons-io:commons-io" implementation "ch.qos.logback:logback-core" diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractMongoDBArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractMongoDBArchivist.java new file mode 100644 index 0000000000..56ad8603a3 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractMongoDBArchivist.java @@ -0,0 +1,264 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.config.ArchiveDBProperties; +import com.tencent.bk.job.backup.constant.ArchiveModeEnum; +import com.tencent.bk.job.backup.constant.MongoDBLogTypeEnum; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import com.tencent.bk.job.backup.model.dto.ArchiveSummary; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.data.mongodb.core.MongoTemplate; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +/** + * mongodb归档基础实现 + */ +@Slf4j +public abstract class AbstractMongoDBArchivist { + protected ArchiveDBProperties archiveDBProperties; + protected CountDownLatch countDownLatch; + protected ArchiveSummary archiveSummary; + protected boolean isAcquireLock; + protected ArchiveTaskLock archiveTaskLock; + protected MongoDBLogTypeEnum mongoDBLogTypeEnum; + protected AtomicInteger deleteCounter; + protected AtomicInteger backupCounter; + protected AtomicInteger failCounter; + + protected final ArchiveErrorTaskCounter archiveErrorTaskCounter; + protected final MongoTemplate mongoTemplate; + protected final String SCRIPT_LOG_PREFIX = "job_log_script"; + protected final String FILE_LOG_PREFIX = "job_log_file"; + protected final String COLLECTION_NAME_SEPARATOR = "_"; + protected final String COLLECTION_NAME_DATE_FORMATTER = "yyyy_MM_dd"; + + public AbstractMongoDBArchivist(MongoTemplate mongoTemplate, + ArchiveDBProperties archiveDBProperties, + ArchiveTaskLock archiveTaskLock, + CountDownLatch countDownLatch, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + MongoDBLogTypeEnum mongoDBLogTypeEnum) { + this.mongoTemplate = mongoTemplate; + this.archiveDBProperties = archiveDBProperties; + this.countDownLatch = countDownLatch; + this.archiveSummary = new ArchiveSummary(); + this.archiveTaskLock = archiveTaskLock; + this.archiveErrorTaskCounter = archiveErrorTaskCounter; + this.mongoDBLogTypeEnum = mongoDBLogTypeEnum; + deleteCounter = new AtomicInteger(0); + backupCounter = new AtomicInteger(0); + failCounter = new AtomicInteger(0); + } + + public void archive() { + if (!archiveDBProperties.isEnabled()) { + archiveSummary.setSkip(true); + log.info("[{}] Archive is disabled, skip archive", getLogTypeStr()); + return; + } + archiveSummary.setEnabled(true); + log.info("[{}] Start archive, keep days:{}", getLogTypeStr(), archiveDBProperties.getKeepDays()); + List targetCollectionNames = listArchiveCollectionName(archiveDBProperties.getKeepDays()); + log.debug("[{}] archive, target collections:{}", getLogTypeStr(), targetCollectionNames); + + long startTime = System.currentTimeMillis(); + boolean success = true; + for (String collectionName : targetCollectionNames) { + if (!acquireLock(collectionName)) { + continue; + } + backupAndDelete(collectionName, isBackupEnable()); + } + + if (targetCollectionNames.size() > 0 + && targetCollectionNames.size() == failCounter.get()) { + archiveErrorTaskCounter.increment(); + success = false; + } + + long archiveCost = System.currentTimeMillis() - startTime; + log.info("[{}] Archive finished, totalCollectionSize:{},deleteCollectionSize:{}, backupCollectionSize:{}," + + "cost: {}ms", + getLogTypeStr(), + targetCollectionNames.size(), + deleteCounter.get(), + backupCounter.get(), + archiveCost + ); + setArchiveSummary(getLogTypeStr(), + archiveDBProperties.getMode(), + archiveCost, + deleteCounter.get(), + backupCounter.get(), + targetCollectionNames.size(), + success + ); + storeArchiveSummary(); + + countDownLatch.countDown(); + } + + private void backupAndDelete(String collectionName, boolean backupEnabled) { + long startTime = System.currentTimeMillis(); + boolean success = true; + try { + if (backupEnabled) { + backupRecords(collectionName); + } + deleteRecord(collectionName); + } catch (Exception e) { + success = false; + failCounter.incrementAndGet(); + String msg = MessageFormatter.format( + "[{}] archive error, collectionName:{}", + getLogTypeStr(), + collectionName + ).getMessage(); + log.error(msg, e); + } finally { + long archiveCost = System.currentTimeMillis() - startTime; + log.info( + "Archive {} finished, result: {}, cost: {}ms", + collectionName, + success ? "success" : "fail", + archiveCost + ); + if (this.isAcquireLock) { + archiveTaskLock.unlock(collectionName); + } + } + } + + protected void backupRecords(String collectionName) { + // 暂不支持备份 + } + + private void deleteRecord(String collectionName) { + mongoTemplate.dropCollection(collectionName); + deleteCounter.incrementAndGet(); + } + + /** + * 获取满足归档条件的集合名称 + */ + private List listArchiveCollectionName(int keepDays) { + Set allCollectionNames = mongoTemplate.getCollectionNames(); + List targetCollectionNames = filterCollectionNames(allCollectionNames, keepDays); + return targetCollectionNames; + } + + /** + * 从所有集合名称中过滤出要归档的集合名称，返回集合名称列表，并按日期排序 + */ + private List filterCollectionNames(Set collectionNames, int keepDays) { + LocalDate endDate = LocalDate.now().minusDays(keepDays); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(COLLECTION_NAME_DATE_FORMATTER); + String endDateString = endDate.format(formatter); + log.info("[{}] archive,the deadline is {}", getLogTypeStr(), endDateString); + return collectionNames.stream() + .filter(collectionName -> { + if (collectionName.startsWith(getCollectionNamePrefix())) { + String dateString = collectionName.substring(getCollectionNamePrefix().length()); + LocalDate collectionDate = LocalDate.parse(dateString, formatter); + return collectionDate.isBefore(endDate); + } + return false; + }) + .sorted((collectionName1, collectionName2) -> { + LocalDate date1 = extractDateFromCollectionName(collectionName1); + LocalDate date2 = extractDateFromCollectionName(collectionName2); + return date1.compareTo(date2); + }) + .collect(Collectors.toList()); + } + + private String getCollectionNamePrefix() { + Integer type = mongoDBLogTypeEnum.getValue(); + String prefix = ""; + if (MongoDBLogTypeEnum.SCRIPT.getValue().equals(type)) { + prefix = SCRIPT_LOG_PREFIX + COLLECTION_NAME_SEPARATOR; + } else if (MongoDBLogTypeEnum.FILE.getValue().equals(type)) { + prefix = FILE_LOG_PREFIX + COLLECTION_NAME_SEPARATOR; + } + return prefix; + } + + private LocalDate extractDateFromCollectionName(String collectionName) { + String dateString = collectionName.substring(getCollectionNamePrefix().length()); + return LocalDate.parse(dateString, DateTimeFormatter.ofPattern(COLLECTION_NAME_DATE_FORMATTER)); + } + + private String getLogTypeStr() { + Integer type = mongoDBLogTypeEnum.getValue(); + String str = "script_log"; + if (MongoDBLogTypeEnum.FILE.getValue().equals(type)) { + str = "file_log"; + } + return str; + } + + private boolean acquireLock(String collectionName) { + this.isAcquireLock = archiveTaskLock.lock(collectionName); + if (!isAcquireLock) { + log.info("[{}] Acquire lock fail", collectionName); + } + return isAcquireLock; + } + + private boolean isBackupEnable() { + return archiveDBProperties.isEnabled() + && ArchiveModeEnum.BACKUP_THEN_DELETE == ArchiveModeEnum.valOf(archiveDBProperties.getMode()); + } + + private void setArchiveSummary(String tableName, + String archiveMode, + long archiveCost, + long deleteCollectionSize, + long backupCollectionSize, + long totalCollectionSize, + boolean success) { + archiveSummary.setArchiveCost(archiveCost); + archiveSummary.setArchiveMode(archiveMode); + archiveSummary.setSuccess(success); + archiveSummary.setTableName(tableName); + archiveSummary.setDeleteCollectionSize(deleteCollectionSize); + archiveSummary.setBackupCollectionSize(backupCollectionSize); + archiveSummary.setTotalCollectionSize(totalCollectionSize); + } + + private void storeArchiveSummary() { + ArchiveSummaryHolder.getInstance().addArchiveSummary(this.archiveSummary); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java index 738d205de4..9c889d9d54 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java @@ -30,6 +30,7 @@ import com.tencent.bk.job.backup.archive.impl.GseScriptAgentTaskArchivist; import com.tencent.bk.job.backup.archive.impl.GseScriptExecuteObjTaskArchivist; import com.tencent.bk.job.backup.archive.impl.GseTaskArchivist; +import com.tencent.bk.job.backup.archive.impl.MongoDBScriptLogArchivist; import com.tencent.bk.job.backup.archive.impl.OperationLogArchivist; import com.tencent.bk.job.backup.archive.impl.RollingConfigArchivist; import com.tencent.bk.job.backup.archive.impl.StepInstanceArchivist; @@ -42,6 +43,7 @@ import com.tencent.bk.job.backup.archive.impl.TaskInstanceHostArchivist; import com.tencent.bk.job.backup.archive.impl.TaskInstanceVariableArchivist; import com.tencent.bk.job.backup.config.ArchiveDBProperties; +import com.tencent.bk.job.backup.constant.MongoDBLogTypeEnum; import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; import com.tencent.bk.job.backup.dao.ExecuteRecordDAO; import com.tencent.bk.job.backup.dao.impl.FileSourceTaskLogRecordDAO; @@ -68,6 +70,7 @@ import org.joda.time.DateTime; import org.slf4j.helpers.MessageFormatter; import org.springframework.context.SmartLifecycle; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.scheduling.annotation.Scheduled; import java.util.concurrent.CountDownLatch; @@ -99,6 +102,7 @@ public class JobExecuteArchiveManage implements SmartLifecycle { private final ExecuteArchiveDAO executeArchiveDAO; private final ArchiveTaskLock archiveTaskLock; private final ArchiveErrorTaskCounter archiveErrorTaskCounter; + private final MongoTemplate mongoTemplate; /** @@ -128,7 +132,8 @@ public JobExecuteArchiveManage(TaskInstanceRecordDAO taskInstanceRecordDAO, ArchiveDBProperties archiveDBProperties, ExecutorService archiveExecutor, ArchiveTaskLock archiveTaskLock, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { + ArchiveErrorTaskCounter archiveErrorTaskCounter, + MongoTemplate mongoTemplate) { log.info("Init JobExecuteArchiveManage! archiveConfig: {}", archiveDBProperties); this.archiveDBProperties = archiveDBProperties; this.archiveProgressService = archiveProgressService; @@ -153,6 +158,7 @@ public JobExecuteArchiveManage(TaskInstanceRecordDAO taskInstanceRecordDAO, this.executeArchiveDAO = executeArchiveDAO; this.archiveTaskLock = archiveTaskLock; this.archiveErrorTaskCounter = archiveErrorTaskCounter; + this.mongoTemplate = mongoTemplate; } @Scheduled(cron = "${job.backup.archive.execute.cron:0 0 4 * * *}") @@ -271,7 +277,7 @@ private long getLastArchiveId(ExecuteRecordDAO executeRecordDAO) { private void archive(long maxNeedArchiveTaskInstanceId, long maxNeedArchiveStepInstanceId) throws InterruptedException { - CountDownLatch countDownLatch = new CountDownLatch(17); + CountDownLatch countDownLatch = new CountDownLatch(19); log.info("Submitting archive task..."); // task_instance @@ -308,6 +314,10 @@ private void archive(long maxNeedArchiveTaskInstanceId, long maxNeedArchiveStepI addGseScriptExecuteObjTaskArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); // gse_file_execute_obj_task addGseFileExecuteObjTaskArchiveTask(maxNeedArchiveTaskInstanceId, countDownLatch); + // mongodb job_log_script_task + addMongoDBScriptLogArchiveTask(countDownLatch); + // mongodb job_log_file_task + addMongoDBFileLogArchiveTask(countDownLatch); log.info("Archive task submitted. Waiting for complete..."); countDownLatch.await(); @@ -568,5 +578,29 @@ private void addTaskInstanceHostArchiveTask(Long maxNeedArchiveTaskInstanceId, archiveErrorTaskCounter ).archive()); } + + private void addMongoDBScriptLogArchiveTask(CountDownLatch countDownLatch) { + archiveExecutor.execute(() -> + new MongoDBScriptLogArchivist( + mongoTemplate, + archiveDBProperties, + archiveTaskLock, + countDownLatch, + archiveErrorTaskCounter, + MongoDBLogTypeEnum.SCRIPT + ).archive()); + } + + private void addMongoDBFileLogArchiveTask(CountDownLatch countDownLatch) { + archiveExecutor.execute(() -> + new MongoDBScriptLogArchivist( + mongoTemplate, + archiveDBProperties, + archiveTaskLock, + countDownLatch, + archiveErrorTaskCounter, + MongoDBLogTypeEnum.FILE + ).archive()); + } } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/MongoDBFileLogArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/MongoDBFileLogArchivist.java new file mode 100644 index 0000000000..5bbbcaf73d --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/MongoDBFileLogArchivist.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.AbstractMongoDBArchivist; +import com.tencent.bk.job.backup.archive.ArchiveTaskLock; +import com.tencent.bk.job.backup.config.ArchiveDBProperties; +import com.tencent.bk.job.backup.constant.MongoDBLogTypeEnum; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import org.springframework.data.mongodb.core.MongoTemplate; + +import java.util.concurrent.CountDownLatch; + +/** + * mongodb 文件日志归档 + */ +public class MongoDBFileLogArchivist extends AbstractMongoDBArchivist { + + public MongoDBFileLogArchivist(MongoTemplate mongoTemplate, + ArchiveDBProperties archiveDBProperties, + ArchiveTaskLock archiveTaskLock, + CountDownLatch countDownLatch, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + MongoDBLogTypeEnum mongoDBLogTypeEnum) { + super(mongoTemplate, + archiveDBProperties, + archiveTaskLock, + countDownLatch, + archiveErrorTaskCounter, + mongoDBLogTypeEnum); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/MongoDBScriptLogArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/MongoDBScriptLogArchivist.java new file mode 100644 index 0000000000..bad8ead658 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/MongoDBScriptLogArchivist.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.AbstractMongoDBArchivist; +import com.tencent.bk.job.backup.archive.ArchiveTaskLock; +import com.tencent.bk.job.backup.config.ArchiveDBProperties; +import com.tencent.bk.job.backup.constant.MongoDBLogTypeEnum; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import org.springframework.data.mongodb.core.MongoTemplate; + +import java.util.concurrent.CountDownLatch; + +/** + * mongodb 脚本日志归档 + */ +public class MongoDBScriptLogArchivist extends AbstractMongoDBArchivist { + + public MongoDBScriptLogArchivist(MongoTemplate mongoTemplate, + ArchiveDBProperties archiveDBProperties, + ArchiveTaskLock archiveTaskLock, + CountDownLatch countDownLatch, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + MongoDBLogTypeEnum mongoDBLogTypeEnum) { + super(mongoTemplate, + archiveDBProperties, + archiveTaskLock, + countDownLatch, + archiveErrorTaskCounter, + mongoDBLogTypeEnum); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java index dcc1541178..d456abfc7b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java @@ -57,6 +57,7 @@ import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.scheduling.annotation.EnableScheduling; @@ -69,7 +70,7 @@ @EnableScheduling @Slf4j @EnableConfigurationProperties(ArchiveDBProperties.class) -@Import({ExecuteDbConfiguration.class, ExecuteBackupDbConfiguration.class}) +@Import({ExecuteDbConfiguration.class, ExecuteBackupDbConfiguration.class, ExecuteMongoDBConfiguration.class}) public class ArchiveConfiguration { /** @@ -244,7 +245,8 @@ public JobExecuteArchiveManage jobExecuteArchiveManage( @Qualifier("archiveExecutor") ExecutorService archiveExecutor, ArchiveDBProperties archiveDBProperties, ArchiveTaskLock archiveTaskLock, - ArchiveErrorTaskCounter archiveErrorTaskCounter) { + ArchiveErrorTaskCounter archiveErrorTaskCounter, + MongoTemplate mongoTemplate) { log.info("Init JobExecuteArchiveManage"); return new JobExecuteArchiveManage( @@ -270,6 +272,7 @@ public JobExecuteArchiveManage jobExecuteArchiveManage( archiveDBProperties, archiveExecutor, archiveTaskLock, - archiveErrorTaskCounter); + archiveErrorTaskCounter, + mongoTemplate); } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteMongoDBConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteMongoDBConfiguration.java new file mode 100644 index 0000000000..b2d8344faf --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteMongoDBConfiguration.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.config; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.tencent.bk.job.common.service.constants.DeployModeEnum; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.AllNestedConditions; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.mongodb.core.MongoTemplate; + +@Configuration("executeMongoDBConfiguration") +@Conditional(ExecuteMongoDBConfiguration.ExecuteMongoDBInitCondition.class) +public class ExecuteMongoDBConfiguration { + + @Value("${spring.datasource.job-execute-mongodb.uri:}") + private String mongoUri; + + @Value("${spring.datasource.job-execute-mongodb.database:joblog}") + private String database; + + @Bean + public MongoClient mongoClient() { + return MongoClients.create(mongoUri); + } + + @Bean + public MongoTemplate mongoTemplate(MongoClient mongoClient) { + return new MongoTemplate(mongoClient, database); + } + + static class ExecuteMongoDBInitCondition extends AllNestedConditions { + public ExecuteMongoDBInitCondition() { + super(ConfigurationPhase.PARSE_CONFIGURATION); + } + + @ConditionalOnProperty(value = "job.backup.archive.execute.enabled", havingValue = "true") + class ArchiveEnableCondition { + + } + + @ConditionalOnProperty(value = "deploy.mode", havingValue = DeployModeEnum.Constants.STANDARD, + matchIfMissing = true) + class StandardDeployModeCondition { + + } + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveSummary.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveSummary.java index 5ba69b5932..0a8ee31c53 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveSummary.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveSummary.java @@ -51,6 +51,10 @@ public class ArchiveSummary { private Long lastDeletedId; private Long deleteRecordSize; + private Long deleteCollectionSize; + private Long backupCollectionSize; + private Long totalCollectionSize; + public ArchiveSummary(String tableName) { this.tableName = tableName; } diff --git a/support-files/kubernetes/charts/bk-job/templates/job-backup/configmap.yaml b/support-files/kubernetes/charts/bk-job/templates/job-backup/configmap.yaml index f60b615cce..fd5a23d0bd 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-backup/configmap.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-backup/configmap.yaml @@ -63,7 +63,11 @@ data: idle-timeout: 6000 poolName: "job-execute" validationTimeout: 5000 - {{- end }} + {{- end }} + {{- if and .Values.backupConfig.archive.execute.enabled }} + job-execute-mongodb: + uri: {{ include "job.mongodb.connect.uri" . | quote }} + {{- end }} {{- if and .Values.backupConfig.archive.execute.enabled (eq .Values.backupConfig.archive.execute.mode "backupThenDelete") }} job-execute-archive: driver-class-name: {{ include "job.jdbcMysqlDriverClass" . }} diff --git a/support-files/templates/#etc#job#job-backup#job-backup.yml b/support-files/templates/#etc#job#job-backup#job-backup.yml index cd3395cda8..67c67f5473 100644 --- a/support-files/templates/#etc#job#job-backup#job-backup.yml +++ b/support-files/templates/#etc#job#job-backup#job-backup.yml @@ -38,6 +38,9 @@ spring: idle-timeout: 6000 poolName: "job-execute" validationTimeout: 5000 + # 执行日志mongodb数据源 + job-execute-mongodb: + uri: __BK_JOB_LOGSVR_MONGODB_URI__ # 执行数据归档DB配置，若需开启数据归档请去除注释并添加相应环境变量进行配置 #job-execute-archive: #driver-class-name: io.opentelemetry.instrumentation.jdbc.OpenTelemetryDriver