Skip to content

Commit

Permalink
Merge pull request #1396 from Tencent/master
Browse files Browse the repository at this point in the history
Merge: master->3.6.x
  • Loading branch information
jsonwan authored Sep 27, 2022
2 parents bd009ae + 2ef75f2 commit 1ca774e
Show file tree
Hide file tree
Showing 96 changed files with 1,372 additions and 1,023 deletions.
15 changes: 15 additions & 0 deletions .ci/codecc.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
version: v2.0

resources:
repositories:
- repository: ci_templates/public/codecc
name: codecc

stages:
- name: "stage1"
check-out:
gates:
- template: commonGate.yml@codecc
jobs:
template:
- name: openScan.yml@codecc
1 change: 0 additions & 1 deletion package.sh
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,6 @@ fi
cp -r support-files/bk-cmdb/ release/job/support-files/
cp -r support-files/bkiam/ release/job/support-files/
cp -r support-files/dependJarInfo/ release/job/support-files/
cp support-files/javaagent/* release/job/backend/
# Package dependJarLists
if [[ -d "support-files/dependJarLists/" ]]; then
cp -r support-files/dependJarLists/ release/job/support-files/
Expand Down
4 changes: 4 additions & 0 deletions src/backend/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ ext {
set('jodaTimeVersion', "2.10.5")
set('bcprovVersion', "1.64")
set('reflectionsVersion', "0.9.12")
set('cronUtilsVersion', "9.1.6")
set('otelJdbcVersion', "1.9.2-alpha")
if (System.getProperty("bkjobVersion")) {
set('bkjobVersion', System.getProperty("bkjobVersion"))
println "bkjobVersoin:" + bkjobVersion
Expand Down Expand Up @@ -245,6 +247,8 @@ subprojects {
dependency "org.bouncycastle:bcprov-jdk15on:$bcprovVersion"
// https://github.com/ronmamo/reflections
dependency "org.reflections:reflections:$reflectionsVersion"
dependency "com.cronutils:cron-utils:$cronUtilsVersion"
dependency "io.opentelemetry.instrumentation:opentelemetry-jdbc:$otelJdbcVersion"
dependencySet(group: "org.jooq", version: "$jooqVersion") {
entry "jooq"
entry "jooq-codegen"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@

package com.tencent.bk.job.common.cc.config;

import com.tencent.bk.job.common.cc.sdk.BizCmdbClient;
import com.tencent.bk.job.common.redis.util.RedisSlideWindowFlowController;
import com.tencent.bk.job.common.util.StringUtil;
import lombok.extern.slf4j.Slf4j;
Expand Down Expand Up @@ -85,7 +84,6 @@ public void initCMDBGlobalFlowController() {
map, cmdbConfig.getFlowControlDefaultLimit(), cmdbConfig.getFlowControlPrecision());
cmdbGlobalFlowController.init(redisTemplate, map, cmdbConfig.getFlowControlDefaultLimit(),
cmdbConfig.getFlowControlPrecision());
BizCmdbClient.setGlobalFlowController(cmdbGlobalFlowController);
} catch (Exception e) {
log.error("Fail to init globalFlowController", e);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
/*
* Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
*
* Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
*
* License for BK-JOB蓝鲸智云作业平台:
* --------------------------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
* to permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
* THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
* CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/

package com.tencent.bk.job.common.cc.config;

import com.tencent.bk.job.common.cc.sdk.BizCmdbClient;
import com.tencent.bk.job.common.esb.config.EsbConfig;
import com.tencent.bk.job.common.esb.constants.EsbLang;
import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient;
import com.tencent.bk.job.common.util.FlowController;
import io.micrometer.core.instrument.MeterRegistry;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;

import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

@Slf4j
@Configuration
public class BizCmdbClientAutoConfig {

@Bean("cmdbThreadPoolExecutor")
public ThreadPoolExecutor cmdbThreadPoolExecutor(CmdbConfig cmdbConfig) {
int cmdbQueryThreadsNum = cmdbConfig.getCmdbQueryThreadsNum();
return new ThreadPoolExecutor(
cmdbQueryThreadsNum,
cmdbQueryThreadsNum,
180L,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(cmdbQueryThreadsNum * 4), (r, executor) -> {
//使用请求的线程直接拉取数据
log.error("cmdb request runnable rejected, use current thread({}), plz add more threads",
Thread.currentThread().getName());
r.run();
});
}

@Bean("cmdbLongTermThreadPoolExecutor")
public ThreadPoolExecutor cmdbLongTermThreadPoolExecutor(CmdbConfig cmdbConfig) {
int longTermCmdbQueryThreadsNum = cmdbConfig.getFindHostRelationLongTermConcurrency();
return new ThreadPoolExecutor(
longTermCmdbQueryThreadsNum,
longTermCmdbQueryThreadsNum,
180L,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(longTermCmdbQueryThreadsNum * 4), (r, executor) -> {
//使用请求的线程直接拉取数据
log.warn("cmdb long term request runnable rejected, use current thread({}), plz add more threads",
Thread.currentThread().getName());
r.run();
});
}

@Bean
@Primary
public BizCmdbClient bizCmdbClient(EsbConfig esbConfig,
CmdbConfig cmdbConfig,
ThreadPoolExecutor cmdbThreadPoolExecutor,
ThreadPoolExecutor cmdbLongTermThreadPoolExecutor,
@Autowired(required = false) QueryAgentStatusClient queryAgentStatusClient,
MeterRegistry meterRegistry,
@Autowired(required = false) FlowController flowController) {
return new BizCmdbClient(
esbConfig,
cmdbConfig,
EsbLang.EN,
cmdbThreadPoolExecutor,
cmdbLongTermThreadPoolExecutor,
queryAgentStatusClient,
flowController,
meterRegistry
);
}

@Bean("cnBizCmdbClient")
public BizCmdbClient cnBizCmdbClient(EsbConfig esbConfig,
CmdbConfig cmdbConfig,
ThreadPoolExecutor cmdbThreadPoolExecutor,
ThreadPoolExecutor cmdbLongTermThreadPoolExecutor,
@Autowired(required = false) QueryAgentStatusClient queryAgentStatusClient,
MeterRegistry meterRegistry,
@Autowired(required = false) FlowController flowController) {
return new BizCmdbClient(
esbConfig,
cmdbConfig,
EsbLang.CN,
cmdbThreadPoolExecutor,
cmdbLongTermThreadPoolExecutor,
queryAgentStatusClient,
flowController,
meterRegistry
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -159,13 +159,13 @@ public class BizCmdbClient extends AbstractEsbSdkClient implements IBizCmdbClien
private static final ConcurrentHashMap<Long, Pair<InstanceTopologyDTO, Long>> bizInternalTopoMap =
new ConcurrentHashMap<>();
private static final ConcurrentHashMap<Long, ReentrantLock> bizInternalTopoLockMap = new ConcurrentHashMap<>();
public static ThreadPoolExecutor threadPoolExecutor = null;
public static ThreadPoolExecutor longTermThreadPoolExecutor = null;
public static CmdbConfig cmdbConfig = null;
private final ThreadPoolExecutor threadPoolExecutor;
private final ThreadPoolExecutor longTermThreadPoolExecutor;
private final CmdbConfig cmdbConfig;
/**
* 对整个应用中所有的CMDB调用进行限流
*/
private static FlowController globalFlowController = null;
private final FlowController globalFlowController;

static {
interfaceNameMap.put(SEARCH_BIZ_INST_TOPO, "search_biz_inst_topo");
Expand All @@ -187,66 +187,37 @@ public class BizCmdbClient extends AbstractEsbSdkClient implements IBizCmdbClien

protected String defaultSupplierAccount;
protected String defaultUin = "admin";
private QueryAgentStatusClient queryAgentStatusClient;
private final QueryAgentStatusClient queryAgentStatusClient;
private final MeterRegistry meterRegistry;
private final LoadingCache<Long, InstanceTopologyDTO> bizInstCompleteTopologyCache = CacheBuilder.newBuilder()
.maximumSize(1000).expireAfterWrite(30, TimeUnit.SECONDS).
build(new CacheLoader<Long, InstanceTopologyDTO>() {
@Override
public InstanceTopologyDTO load(Long bizId) {
public InstanceTopologyDTO load(@SuppressWarnings("NullableProblems") Long bizId) {
return getBizInstCompleteTopology(bizId);
}
}
);

public BizCmdbClient(EsbConfig esbConfig, CmdbConfig cmdbConfig, QueryAgentStatusClient queryAgentStatusClient,
public BizCmdbClient(EsbConfig esbConfig,
CmdbConfig cmdbConfig,
String lang,
ThreadPoolExecutor threadPoolExecutor,
ThreadPoolExecutor longTermThreadPoolExecutor,
QueryAgentStatusClient queryAgentStatusClient,
FlowController flowController,
MeterRegistry meterRegistry) {
this(esbConfig, cmdbConfig, null, queryAgentStatusClient, meterRegistry);
}

public BizCmdbClient(EsbConfig esbConfig, CmdbConfig cmdbConfig, String lang,
QueryAgentStatusClient queryAgentStatusClient, MeterRegistry meterRegistry) {
super(esbConfig.getEsbUrl(), esbConfig.getAppCode(), esbConfig.getAppSecret(), lang,
esbConfig.isUseEsbTestEnv());
this.cmdbConfig = cmdbConfig;
this.defaultSupplierAccount = cmdbConfig.getDefaultSupplierAccount();
this.threadPoolExecutor = threadPoolExecutor;
this.longTermThreadPoolExecutor = longTermThreadPoolExecutor;
this.queryAgentStatusClient = queryAgentStatusClient;
this.globalFlowController = flowController;
this.meterRegistry = meterRegistry;
}

public static void setGlobalFlowController(FlowController flowController) {
globalFlowController = flowController;
}

private static void initThreadPoolExecutor(int cmdbQueryThreadsNum, int longTermCmdbQueryThreadsNum) {
threadPoolExecutor = new ThreadPoolExecutor(cmdbQueryThreadsNum, cmdbQueryThreadsNum, 180L, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(cmdbQueryThreadsNum * 4), (r, executor) -> {
//使用请求的线程直接拉取数据
log.error("cmdb request runnable rejected, use current thread({}), plz add more threads",
Thread.currentThread().getName());
r.run();
});
longTermThreadPoolExecutor = new ThreadPoolExecutor(longTermCmdbQueryThreadsNum, longTermCmdbQueryThreadsNum,
180L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(longTermCmdbQueryThreadsNum * 4), (r, executor) -> {
//使用请求的线程直接拉取数据
log.warn("cmdb long term request runnable rejected, use current thread({}), plz add more threads",
Thread.currentThread().getName());
r.run();
});
}

public static void init() {
initThreadPoolExecutor(cmdbConfig.getCmdbQueryThreadsNum(),
cmdbConfig.getFindHostRelationLongTermConcurrency());
}

public static void setCcConfig(CmdbConfig cmdbConfig) {
BizCmdbClient.cmdbConfig = cmdbConfig;
}

public void setQueryAgentStatusClient(QueryAgentStatusClient queryAgentStatusClient) {
this.queryAgentStatusClient = queryAgentStatusClient;
}

@Override
public InstanceTopologyDTO getBizInstCompleteTopology(long bizId) {
InstanceTopologyDTO completeTopologyDTO;
Expand Down Expand Up @@ -555,15 +526,15 @@ private List<ApplicationHostDTO> findModuleHostRelationConcurrently(long bizId,
FindModuleHostRelationTask task = new FindModuleHostRelationTask(resultQueue,
genFindModuleHostRelationReq(bizId, moduleIdList, start, limit),
JobContextUtil.getRequestId());
Future<?> future;
if (totalCount > 10000) {
//主机数太多,防止将CMDB拉挂了
Future<?> future = longTermThreadPoolExecutor.submit(task);
futures.add(future);
future = longTermThreadPoolExecutor.submit(task);
} else {
// 默认采用多个并发线程拉取
Future<?> future = threadPoolExecutor.submit(task);
futures.add(future);
future = threadPoolExecutor.submit(task);
}
futures.add(future);
totalCount -= limit;
}
futures.forEach(it -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,53 +24,13 @@

package com.tencent.bk.job.common.cc.sdk;

import com.google.common.collect.Maps;
import com.tencent.bk.job.common.cc.config.CmdbConfig;
import com.tencent.bk.job.common.esb.config.EsbConfig;
import com.tencent.bk.job.common.esb.constants.EsbLang;
import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient;
import com.tencent.bk.job.common.i18n.locale.LocaleUtils;
import com.tencent.bk.job.common.util.ApplicationContextRegister;
import io.micrometer.core.instrument.MeterRegistry;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.DependsOn;

import java.util.Map;

@Slf4j
@DependsOn({"applicationContextRegister", "cmdbConfigSetter"})
public class CmdbClientFactory {

private static final Map<String, IBizCmdbClient> CMDB_CLIENT_MAPS = Maps.newHashMap();

static {
EsbConfig esbConfig = null;
CmdbConfig cmdbConfig = null;
QueryAgentStatusClient queryAgentStatusClient = null;
MeterRegistry meterRegistry = null;
try {
esbConfig = ApplicationContextRegister.getBean(EsbConfig.class);
cmdbConfig = ApplicationContextRegister.getBean(CmdbConfig.class);
queryAgentStatusClient = ApplicationContextRegister.getBean(QueryAgentStatusClient.class);
meterRegistry = ApplicationContextRegister.getBean(MeterRegistry.class);
} catch (Throwable e) {
log.error("Error while initialize bk config!", e);
throw e;
}
CMDB_CLIENT_MAPS.put(LocaleUtils.LANG_ZH_CN,
new BizCmdbClient(esbConfig, cmdbConfig, EsbLang.CN, queryAgentStatusClient, meterRegistry)
);
CMDB_CLIENT_MAPS.put(LocaleUtils.LANG_EN,
new BizCmdbClient(esbConfig, cmdbConfig, EsbLang.EN, queryAgentStatusClient, meterRegistry)
);
CMDB_CLIENT_MAPS.put(LocaleUtils.LANG_ZH,
new BizCmdbClient(esbConfig, cmdbConfig, EsbLang.CN, queryAgentStatusClient, meterRegistry)
);
CMDB_CLIENT_MAPS.put(LocaleUtils.LANG_EN_US,
new BizCmdbClient(esbConfig, cmdbConfig, EsbLang.EN, queryAgentStatusClient, meterRegistry)
);
}

public static IBizCmdbClient getCmdbClient() {
return getCmdbClient(LocaleUtils.LANG_EN_US);
}
Expand All @@ -79,10 +39,12 @@ public static IBizCmdbClient getCmdbClient(String language) {
if (language == null) {
language = LocaleUtils.LANG_EN_US;
}
IBizCmdbClient icmdbClient = CMDB_CLIENT_MAPS.get(language);
if (icmdbClient == null) {
icmdbClient = CMDB_CLIENT_MAPS.get(LocaleUtils.LANG_EN_US);
switch (language) {
case LocaleUtils.LANG_ZH:
case LocaleUtils.LANG_ZH_CN:
return ApplicationContextRegister.getBean("cnBizCmdbClient", IBizCmdbClient.class);
default:
return ApplicationContextRegister.getBean(IBizCmdbClient.class);
}
return icmdbClient;
}
}
Loading

0 comments on commit 1ca774e

Please sign in to comment.