Merge pull request #2669 from jsonwan/github_perf/log

perf: job-manage错误日志优化 #2656
TencentBlueKing · Dec 14, 2023 · ac0dc06 · ac0dc06
2 parents 51c5fc8 + 3539aeb
commit ac0dc06
Show file tree

Hide file tree

Showing 6 changed files with 92 additions and 30 deletions.
diff --git a/...sdk/src/main/java/com/tencent/bk/job/common/gse/service/AbstractAgentStateClientImpl.java b/...sdk/src/main/java/com/tencent/bk/job/common/gse/service/AbstractAgentStateClientImpl.java
@@ -31,6 +31,7 @@
 import com.tencent.bk.job.common.util.ConcurrencyUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.helpers.FormattingTuple;
 import org.slf4j.helpers.MessageFormatter;
 import org.springframework.util.StopWatch;
@@ -59,6 +60,9 @@ public AbstractAgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig,
     }
 
     protected AgentState getAgentState(String agentId) {
+        if (StringUtils.isBlank(agentId)) {
+            return null;
+        }
         ListAgentStateReq req = new ListAgentStateReq();
         req.setAgentIdList(Collections.singletonList(agentId));
         List<AgentState> agentStateList = gseClient.listAgentState(req);

diff --git a/...sdk/src/main/java/com/tencent/bk/job/common/gse/service/PreferV2AgentStateClientImpl.java b/...sdk/src/main/java/com/tencent/bk/job/common/gse/service/PreferV2AgentStateClientImpl.java
@@ -61,6 +61,9 @@ public String getEffectiveAgentId(HostAgentStateQuery hostAgentStateQuery) {
     @Override
     public AgentState getAgentState(HostAgentStateQuery hostAgentStateQuery) {
         String finalAgentId = getEffectiveAgentId(hostAgentStateQuery);
+        if (StringUtils.isBlank(finalAgentId)) {
+            return null;
+        }
         return getAgentState(finalAgentId);
     }
 

diff --git a/...c/main/java/com/tencent/bk/job/common/gse/service/UseV2ByFeatureAgentStateClientImpl.java b/...c/main/java/com/tencent/bk/job/common/gse/service/UseV2ByFeatureAgentStateClientImpl.java
@@ -77,7 +77,10 @@ public String getEffectiveAgentId(HostAgentStateQuery hostAgentStateQuery) {
     @Override
     public AgentState getAgentState(HostAgentStateQuery hostAgentStateQuery) {
         // 填充需要的字段
-        getEffectiveAgentId(hostAgentStateQuery);
+        String effectiveAgentId = getEffectiveAgentId(hostAgentStateQuery);
+        if (StringUtils.isBlank(effectiveAgentId)) {
+            return null;
+        }
         if (needToUseGseV2(hostAgentStateQuery)) {
             return gseV2AgentStateClient.getAgentState(hostAgentStateQuery);
         }

diff --git a/...service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbUserInfoDAO.java b/...service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbUserInfoDAO.java
@@ -41,4 +41,6 @@ public interface EsbUserInfoDAO {
 
     List<EsbUserInfoDTO> listEsbUserInfo(Collection<String> userNames);
 
+    List<String> listExistUserName(Collection<String> userNames);
+
 }
diff --git a/...ob-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbUserInfoDAOImpl.java b/...ob-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbUserInfoDAOImpl.java
@@ -31,6 +31,7 @@
 import lombok.val;
 import org.jooq.Condition;
 import org.jooq.DSLContext;
+import org.jooq.Record1;
 import org.jooq.Result;
 import org.jooq.conf.ParamType;
 import org.jooq.types.ULong;
@@ -39,9 +40,11 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Repository;
+import org.springframework.util.CollectionUtils;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 
 @Repository
@@ -118,6 +121,20 @@ public List<EsbUserInfoDTO> listEsbUserInfo(Collection<String> userNames) {
         return listEsbUserInfoByConditions(dslContext, conditions, null);
     }
 
+    @Override
+    public List<String> listExistUserName(Collection<String> userNames) {
+        if (CollectionUtils.isEmpty(userNames)) {
+            return Collections.emptyList();
+        }
+        List<Condition> conditions = new ArrayList<>();
+        conditions.add(defaultTable.USERNAME.in(userNames));
+        val baseQuery = dslContext.select(defaultTable.USERNAME)
+            .from(defaultTable)
+            .where(conditions);
+        Result<Record1<String>> records = baseQuery.fetch();
+        return records.map(record -> record.get(defaultTable.USERNAME));
+    }
+
     private List<EsbUserInfoDTO> listEsbUserInfoByConditions(DSLContext dslContext, List<Condition> conditions,
                                                              Long limit) {
         val baseQuery = dslContext.selectFrom(defaultTable)

diff --git a/...ob-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java b/...ob-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java
@@ -24,15 +24,16 @@
 
 package com.tencent.bk.job.manage.service.impl.notify;
 
+import com.tencent.bk.job.common.util.ThreadUtils;
 import com.tencent.bk.job.manage.dao.notify.EsbUserInfoDAO;
-import com.tencent.bk.job.manage.model.dto.notify.EsbUserInfoDTO;
 import com.tencent.bk.job.manage.service.impl.WatchableSendMsgService;
 import lombok.Builder;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.slf4j.helpers.FormattingTuple;
 import org.slf4j.helpers.MessageFormatter;
 
+import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
@@ -54,6 +55,8 @@ public class SendNotifyTask implements Runnable {
     private final String title;
     private final String content;
 
+    private Set<String> validReceivers;
+
     public void bindService(WatchableSendMsgService watchableSendMsgService,
                             EsbUserInfoDAO esbUserInfoDAO) {
         this.watchableSendMsgService = watchableSendMsgService;
@@ -62,8 +65,13 @@ public void bindService(WatchableSendMsgService watchableSendMsgService,
 
     @Override
     public void run() {
-        if (CollectionUtils.isEmpty(receivers)) {
-            logInvalidReceivers();
+        pickValidReceivers();
+        Collection<String> removedReceivers = CollectionUtils.subtract(receivers, validReceivers);
+        if (CollectionUtils.isNotEmpty(removedReceivers)) {
+            log.info("Invalid user removed:{}, ignore to send notify to them", removedReceivers);
+        }
+        if (CollectionUtils.isEmpty(validReceivers)) {
+            logValidReceiversEmpty();
             return;
         }
         try {
@@ -78,8 +86,13 @@ public void run() {
         }
     }
 
-    private void logInvalidReceivers() {
-        log.warn("receivers is null or empty, skip, msgType={},title={}", msgType, title);
+    private void pickValidReceivers() {
+        List<String> existUserNameList = esbUserInfoDAO.listExistUserName(receivers);
+        validReceivers = new HashSet<>(existUserNameList);
+    }
+
+    private void logValidReceiversEmpty() {
+        log.info("valid receivers is null or empty, skip, msgType={},title={}", msgType, title);
     }
 
     private boolean sendMsgWithRetry() {
@@ -93,47 +106,67 @@ private boolean sendMsgWithRetry() {
                     createTimeMillis,
                     msgType,
                     sender,
-                    receivers,
+                    validReceivers,
                     title,
                     content
                 );
                 result = true;
             } catch (Exception e) {
-                log.error("Fail to sendMsg", e);
+                if (count < NOTIFY_MAX_RETRY_COUNT) {
+                    long sleepMills = count * 1000;
+                    String msg = MessageFormatter.format(
+                        "Fail to sendMsg, sleep {}ms and retry {}",
+                        sleepMills,
+                        count
+                    ).getMessage();
+                    log.warn(msg, e);
+                    ThreadUtils.sleep(sleepMills);
+                } else {
+                    log.error("Fail to sendMsg", e);
+                }
             }
         }
         return result;
     }
 
     private void logSendSuccess() {
-        log.info("Success to send notify:({},{},{})", String.join(",", receivers), msgType, title);
+        log.info("Success to send notify:({},{},{})", String.join(",", validReceivers), msgType, title);
     }
 
     private void handleSendFail(Exception e) {
-        List<EsbUserInfoDTO> validUsers = esbUserInfoDAO.listEsbUserInfo(receivers);
-        if (validUsers.isEmpty()) {
-            // 收信人已全部离职/某些渠道不支持平台用户
-            logIgnoreToSend();
-            return;
-        }
-        FormattingTuple msg = MessageFormatter.arrayFormat(
-            "Fail to send notify:({},{},{},{})",
-            new Object[]{
-                String.join(",", receivers),
-                msgType,
-                title,
-                content
-            }
-        );
-        if (e != null) {
-            log.error(msg.getMessage(), e);
+        int titleMaxLength = 32;
+        int contentMaxLength = 200;
+        String msg;
+        if (log.isDebugEnabled()) {
+            msg = MessageFormatter.arrayFormat(
+                "Fail to send notify:({},{},{},{})",
+                new Object[]{
+                    String.join(",", validReceivers),
+                    msgType,
+                    title,
+                    content
+                }
+            ).getMessage();
+            log.debug(msg, e);
         } else {
-            log.error(msg.getMessage());
+            msg = MessageFormatter.arrayFormat(
+                "Fail to send notify:({},{},{},{})",
+                new Object[]{
+                    String.join(",", validReceivers),
+                    msgType,
+                    buildLogContent(title, titleMaxLength),
+                    buildLogContent(content, contentMaxLength)
+                }
+            ).getMessage();
+            log.error(msg, e);
         }
     }
 
-    private void logIgnoreToSend() {
-        log.info("Ignore to send notify:({},{},{})", String.join(",", receivers), msgType, title);
+    private String buildLogContent(String content, int maxLength) {
+        if (content != null && content.length() > maxLength) {
+            return content.substring(0, maxLength) + "...";
+        }
+        return content;
     }
 
 }