Skip to content

Commit

Permalink
Merge pull request #1734 from jsonwan/3.6.x
Browse files Browse the repository at this point in the history 
bugfix: 任务执行详情页面导出日志时不显示文件大小 #1731
  • Loading branch information
@jsonwan
jsonwan authored Feb 9, 2023
2 parents a9cb2dc + d8d9087 commit 298a05f
Showing 1 changed file with 113 additions and 60 deletions.
173 changes: 113 additions & 60 deletions ...b-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,69 +181,122 @@ private String getExportJobKey(Long appId, Long stepInstanceId, Long hostId, Str
return key;
}

private void doPackage(LogExportJobInfoDTO exportJobInfo, long stepInstanceId, Long hostId,
String ip, int executeCount, String logFileDir, String logFileName) {
private void doPackage(LogExportJobInfoDTO exportJobInfo,
long stepInstanceId,
Long hostId,
String cloudIp,
int executeCount,
String logFileDir,
String logFileName) {
StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId);
boolean isGetByHost = hostId != null || StringUtils.isNotBlank(ip);
boolean isGetByHost = hostId != null || StringUtils.isNotBlank(cloudIp);
File logFile = new File(logFileDir + logFileName);

StopWatch watch = new StopWatch("exportJobLog");
watch.start("listJobIps");
watch.start("getGseAgentTasks");
List<AgentTaskDTO> gseAgentTasks = getGseAgentTasks(stepInstance, executeCount, hostId, cloudIp);
watch.stop();

if (gseAgentTasks == null || gseAgentTasks.isEmpty()) {
log.warn("Gse task ips are empty! stepInstanceId={}", stepInstanceId);
markJobFailed(exportJobInfo);
return;
}

watch.start("getLogContentAndWriteToFile");
if (!getLogContentAndWriteToFile(stepInstance, gseAgentTasks, logFile, isGetByHost, exportJobInfo)) {
log.warn("Fail to getLogContentAndWriteToFile");
return;
}
watch.stop();

watch.start("zipLogFileAndSaveToBackend");
zipLogFileAndSaveToBackend(logFile, exportJobInfo, logFileName);
watch.stop();

if (watch.getTotalTimeMillis() > 10000L) {
log.info("Export job execution log is slow, cost: {}", watch.prettyPrint());
}
}

/**
* 根据hostId或CloudIP获取日志记录信息，hostId与CloudIP均为空则获取所有目标机器日志记录信息
*
* @param stepInstance 步骤实例
* @param executeCount 重试次数
* @param hostId 要获取日志记录的主机ID
* @param cloudIp 要获取日志记录的CloudIP
* @return 日志记录信息列表
*/
private List<AgentTaskDTO> getGseAgentTasks(StepInstanceBaseDTO stepInstance,
int executeCount,
Long hostId,
String cloudIp) {
List<AgentTaskDTO> gseAgentTasks = new ArrayList<>();
boolean isGetByHost = hostId != null || StringUtils.isNotBlank(cloudIp);
if (isGetByHost) {
HostDTO host = HostDTO.fromHostIdAndCloudIp(hostId, ip);
HostDTO host = HostDTO.fromHostIdAndCloudIp(hostId, cloudIp);
AgentTaskDTO agentTask = scriptAgentTaskService.getAgentTaskByHost(stepInstance, executeCount, null,
host);
if (agentTask != null) {
gseAgentTasks.add(agentTask);
}
} else {
gseAgentTasks = scriptAgentTaskService.listAgentTasks(stepInstanceId, executeCount, null);
}
watch.stop();

if (gseAgentTasks == null || gseAgentTasks.isEmpty()) {
log.warn("Gse task ips are empty! stepInstanceId={}", stepInstanceId);
markJobFailed(exportJobInfo);
return;
gseAgentTasks = scriptAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, null);
}
return gseAgentTasks;
}

Collection<LogBatchQuery> querys = buildLogBatchQuery(stepInstanceId, gseAgentTasks);
private boolean getLogContentAndWriteToFile(StepInstanceBaseDTO stepInstance,
List<AgentTaskDTO> gseAgentTasks,
File logFile,
boolean isGetByHost,
LogExportJobInfoDTO exportJobInfo) {
Collection<LogBatchQuery> querys = buildLogBatchQuery(stepInstance.getId(), gseAgentTasks);

watch.start("getLogContent");
String jobCreateDate = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS,
"yyyy_MM_dd", ZoneId.of("UTC"));
try (PrintWriter out = new PrintWriter(logFile, "UTF-8")) {
for (LogBatchQuery query : querys) {
for (List<HostDTO> hosts : query.getHostBatches()) {
List<ScriptHostLogContent> scriptHostLogContentList =
logService.batchGetScriptHostLogContent(jobCreateDate, stepInstanceId, query.getExecuteCount(),
null, hosts);
for (ScriptHostLogContent scriptHostLogContent : scriptHostLogContentList) {
if (scriptHostLogContent != null && StringUtils.isNotEmpty(scriptHostLogContent.getContent())) {
String[] logList = scriptHostLogContent.getContent().split("\n");
for (String log : logList) {
if (isGetByHost) {
out.println(log);
} else {
out.println(scriptHostLogContent.getIp() + " | " + log);
}
}
}

}
writeOneBatchIpLogs(out, jobCreateDate, stepInstance, query, hosts, isGetByHost);
}
}
out.flush();
return true;
} catch (Exception e) {
log.warn("Export execution log fail", e);
FileUtils.deleteQuietly(logFile);
markJobFailed(exportJobInfo);
return;
return false;
}
watch.stop();
}

private void writeOneBatchIpLogs(PrintWriter out,
String jobCreateDate,
StepInstanceBaseDTO stepInstance,
LogBatchQuery query,
List<HostDTO> hosts,
boolean isGetByHost) {
List<ScriptHostLogContent> scriptHostLogContentList =
logService.batchGetScriptHostLogContent(jobCreateDate, stepInstance.getId(),
query.getExecuteCount(),
null, hosts);
for (ScriptHostLogContent scriptHostLogContent : scriptHostLogContentList) {
if (scriptHostLogContent != null && StringUtils.isNotEmpty(scriptHostLogContent.getContent())) {
String[] logList = scriptHostLogContent.getContent().split("\n");
for (String log : logList) {
if (isGetByHost) {
out.println(log);
} else {
out.println(scriptHostLogContent.getIp() + " | " + log);
}
}
}
}
}

watch.start("zipLogFile");
private void zipLogFileAndSaveToBackend(File logFile, LogExportJobInfoDTO exportJobInfo, String logFileName) {
try {
File zipFile = ZipUtil.zip(logFile.getAbsolutePath());
if (zipFile == null) {
Expand All @@ -253,39 +306,39 @@ private void doPackage(LogExportJobInfoDTO exportJobInfo, long stepInstanceId, L
} else {
FileUtils.deleteQuietly(logFile);
}
// 将zip文件上传至制品库
if (JobConstants.FILE_STORAGE_BACKEND_ARTIFACTORY.equals(logExportConfig.getStorageBackend())) {
try {
artifactoryClient.uploadGenericFile(
artifactoryConfig.getArtifactoryJobProject(),
logExportConfig.getLogExportRepo(),
zipFile.getName(),
zipFile
);
FileUtils.deleteQuietly(zipFile);
} catch (Exception e) {
String msg = MessageFormatter.format(
"Fail to upload {} to artifactory",
zipFile.getAbsolutePath()
).getMessage();
log.error(msg, e);
markJobFailed(exportJobInfo);
return;
}
}
long zipFileLength = zipFile.length();
uploadZipFileToArtifactoryIfNeeded(zipFile, exportJobInfo);
exportJobInfo.setStatus(LogExportStatusEnum.SUCCESS);
exportJobInfo.setZipFileName(logFileName + ".zip");
exportJobInfo.setFileSize(zipFile.length());
exportJobInfo.setFileSize(zipFileLength);
saveExportInfo(exportJobInfo);
} catch (Exception e) {
log.warn("Zip log file fail, fileName={}", logFile.getName());
log.warn("zipLogFileAndSaveToBackend fail, fileName={}", logFile.getName());
markJobFailed(exportJobInfo);
return;
}
watch.stop();
}

if (watch.getTotalTimeMillis() > 10000L) {
log.info("Export job execution log is slow, cost: {}", watch.prettyPrint());
private void uploadZipFileToArtifactoryIfNeeded(File zipFile, LogExportJobInfoDTO exportJobInfo) {
if (!JobConstants.FILE_STORAGE_BACKEND_ARTIFACTORY.equals(logExportConfig.getStorageBackend())) {
return;
}
// 将zip文件上传至制品库
try {
artifactoryClient.uploadGenericFile(
artifactoryConfig.getArtifactoryJobProject(),
logExportConfig.getLogExportRepo(),
zipFile.getName(),
zipFile
);
FileUtils.deleteQuietly(zipFile);
} catch (Exception e) {
String msg = MessageFormatter.format(
"Fail to upload {} to artifactory",
zipFile.getAbsolutePath()
).getMessage();
log.error(msg, e);
markJobFailed(exportJobInfo);
throw e;
}
}

Expand Down

0 comments on commit 298a05f

Please sign in to comment.