Skip to content

Commit

Permalink
Merge pull request #1988 from liuliaozhong/3.3.x_issue812
Browse files Browse the repository at this point in the history
fix: fast_transfer_file API 分发文件,如果源文件中的文件名包含空格,会报错 #812
  • Loading branch information
wangyu096 authored Apr 24, 2023
2 parents baf1a1a + b9a0192 commit 7d99751
Show file tree
Hide file tree
Showing 7 changed files with 143 additions and 107 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package com.tencent.bk.job.common.util;

import org.apache.commons.lang3.StringUtils;

import java.util.regex.Pattern;

/**
* 文件路径合法性校验工具类
*/
public class FilePathValidateUtil {
// 传统DOS正则表达式
private static final String CONVENTIONAL_DOS_PATH_REGEX = "(^[A-Za-z]:\\\\([^\\\\])(([^\\\\/:*?\"<>|])*\\\\?)*)|" +
"(^[A-Za-z]:[\\\\])";
// Linux路径正则表达式
private static final String LINUX_PATH_REGEX = "^/(((../)*|(./)*)|(\\.?[^.].*/{0,1}))+";

// 传统DOS Pattern
private static final Pattern CONVENTIONAL_DOS_PATH_PATTERN = Pattern.compile(CONVENTIONAL_DOS_PATH_REGEX);
// Linux路径Pattern
private static final Pattern LINUX_PATH_PATTERN = Pattern.compile(LINUX_PATH_REGEX);

/**
* 验证文件系统绝对路径的合法性
* @param path 绝对路径
* @return boolean true合法,false非法
*/
public static boolean validateFileSystemAbsolutePath(String path) {
if (StringUtils.isBlank(path)) {
return false;
}
if (isLinuxAbsolutePath(path)) {
return validateLinuxFileSystemAbsolutePath(path);
} else {
return validateWindowsFileSystemAbsolutePath(path);
}
}

/**
* 判断是否Linux绝对路径
*
* @param path 文件路径
* @return boolean
*/
private static boolean isLinuxAbsolutePath(String path) {
if (path.startsWith("/")) {
return true;
}
return false;
}

/**
* 1 传统DOS路径
* 标准的DOS路径可由以下三部分组成:
* 1)卷号或驱动器号,后跟卷分隔符(:)。
* 2)目录名称。目录分隔符用来分隔嵌套目录层次结构中的子目录。
* 3)文件名。目录分隔符用来分隔文件路径和文件名。
* @param path
* @return boolean
*/
private static boolean validateWindowsFileSystemAbsolutePath(String path) {
// 传统DOS
if (CONVENTIONAL_DOS_PATH_PATTERN.matcher(path).matches()) {
return true;
}
return false;
}

/**
* 文件或目录名,除了/以外,所有的字符都合法
*
* @param path
* @return boolean
*/
private static boolean validateLinuxFileSystemAbsolutePath(String path) {
if (LINUX_PATH_PATTERN.matcher(path).matches()) {
return true;
}
return false;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package com.tencent.bk.job.common.util;

import org.junit.jupiter.api.Test;

import static org.assertj.core.api.AssertionsForClassTypes.assertThat;

public class FilePathValidateUtilTest {
@Test
void testFileSystemAbsolutePath(){
// 传统DOS路径
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\Documents\\abc.txt")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("c:\\Documents\\abc.txt")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\Documents\\嘉 abc.txt")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath(":\\abc.txt")).isFalse();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:")).isFalse();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\\\")).isFalse();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\logs\\..\\access.log")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\.config\\conf")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\user\\abc>a")).isFalse();

// linux路径
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/data/test_2022-04-12.apk")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/data/test_2022 04 12.apk")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/tmp/")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/tmp/.conf/abc")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/tmp/test/../test.log")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("data/test_2022-04-12.apk")).isFalse();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("///")).isTrue(); // 根目录
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/tmp////")).isTrue(); // /tmp/
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/tmp//test/")).isTrue();// /tmp/test/
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("///")).isTrue();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import com.tencent.bk.job.common.iam.exception.InSufficientPermissionException;
import com.tencent.bk.job.common.iam.service.AuthService;
import com.tencent.bk.job.common.model.ValidateResult;
import com.tencent.bk.job.common.util.FilePathValidateUtil;
import com.tencent.bk.job.common.util.date.DateUtils;
import com.tencent.bk.job.execute.api.esb.v2.EsbFastPushFileResource;
import com.tencent.bk.job.execute.common.constants.RunStatusEnum;
Expand All @@ -56,8 +57,6 @@
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@RestController
@Slf4j
Expand Down Expand Up @@ -119,7 +118,7 @@ private ValidateResult checkFastPushFileRequest(EsbFastPushFileRequest request)
log.warn("Fast transfer file, appId invalid!appId is empty or invalid!");
return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "bk_biz_id");
}
if (!validateFileSystemPath(request.getTargetPath())) {
if (!FilePathValidateUtil.validateFileSystemAbsolutePath(request.getTargetPath())) {
log.warn("Fast transfer file, target path is invalid!path={}", request.getTargetPath());
return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "file_target_path");
}
Expand Down Expand Up @@ -161,7 +160,7 @@ private ValidateResult validateFileSource(EsbFastPushFileRequest request) {
return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "file_source.files");
}
for (String file : files) {
if (!validateFileSystemPath(file)) {
if (!FilePathValidateUtil.validateFileSystemAbsolutePath(file)) {
log.warn("Invalid path:{}", file);
return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "file_source.files");
}
Expand All @@ -182,32 +181,6 @@ private ValidateResult validateFileSource(EsbFastPushFileRequest request) {
return ValidateResult.pass();
}

private boolean validateFileSystemPath(String path) {
if (StringUtils.isBlank(path)) {
return false;
}
if (path.indexOf(' ') != -1) {
return false;
}
Pattern p1 = Pattern.compile("(//|\\\\)+");
Matcher m1 = p1.matcher(path);
if (m1.matches()) {
return false;
}

Pattern p2 = Pattern.compile("^[a-zA-Z]:(/|\\\\).*");//windows
Matcher m2 = p2.matcher(path);

if (!m2.matches()) { //非windows
if (path.charAt(0) == '/') {
return !path.contains("\\\\");
} else {
return false;
}
}
return true;
}

private TaskInstanceDTO buildFastFileTaskInstance(EsbFastPushFileRequest request) {
TaskInstanceDTO taskInstance = new TaskInstanceDTO();
taskInstance.setType(TaskTypeEnum.FILE.getValue());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,10 @@
import com.tencent.bk.job.common.iam.service.AuthService;
import com.tencent.bk.job.common.model.ServiceResponse;
import com.tencent.bk.job.common.model.ValidateResult;
import com.tencent.bk.job.common.util.FilePathValidateUtil;
import com.tencent.bk.job.common.util.date.DateUtils;
import com.tencent.bk.job.execute.client.FileSourceResourceClient;
import com.tencent.bk.job.execute.common.constants.FileTransferModeEnum;
import com.tencent.bk.job.execute.common.constants.RunStatusEnum;
import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum;
import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum;
import com.tencent.bk.job.execute.common.constants.TaskTypeEnum;
import com.tencent.bk.job.execute.common.constants.*;
import com.tencent.bk.job.execute.model.FileDetailDTO;
import com.tencent.bk.job.execute.model.FileSourceDTO;
import com.tencent.bk.job.execute.model.StepInstanceDTO;
Expand All @@ -59,8 +56,6 @@
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@RestController
@Slf4j
Expand Down Expand Up @@ -137,7 +132,7 @@ private ValidateResult checkFileSource(EsbFileSourceV3DTO fileSource) {
if ((fileType == null
|| TaskFileTypeEnum.SERVER.getType() == fileType
|| TaskFileTypeEnum.LOCAL.getType() == fileType)
&& !validateFileSystemPath(file)) {
&& !FilePathValidateUtil.validateFileSystemAbsolutePath(file)) {
log.warn("Invalid path:{}", file);
return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "file_source.file_list");
}
Expand Down Expand Up @@ -175,7 +170,7 @@ private ValidateResult checkFastTransferFileRequest(EsbFastTransferFileV3Request
log.warn("Fast transfer file, appId invalid!appId is empty or invalid!");
return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "bk_biz_id");
}
if (!validateFileSystemPath(request.getTargetPath())) {
if (!FilePathValidateUtil.validateFileSystemAbsolutePath(request.getTargetPath())) {
log.warn("Fast transfer file, target path is invalid!path={}", request.getTargetPath());
return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "file_target_path");
}
Expand Down Expand Up @@ -207,32 +202,6 @@ private ValidateResult checkFastTransferFileRequest(EsbFastTransferFileV3Request
return ValidateResult.pass();
}

private boolean validateFileSystemPath(String path) {
if (StringUtils.isBlank(path)) {
return false;
}
if (path.indexOf(' ') != -1) {
return false;
}
Pattern p1 = Pattern.compile("(//|\\\\)+");
Matcher m1 = p1.matcher(path);
if (m1.matches()) {
return false;
}

Pattern p2 = Pattern.compile("^[a-zA-Z]:(/|\\\\).*");//windows
Matcher m2 = p2.matcher(path);

if (!m2.matches()) { //非windows
if (path.charAt(0) == '/') {
return !path.contains("\\\\");
} else {
return false;
}
}
return true;
}

private TaskInstanceDTO buildFastFileTaskInstance(EsbFastTransferFileV3Request request) {
TaskInstanceDTO taskInstance = new TaskInstanceDTO();
taskInstance.setType(TaskTypeEnum.FILE.getValue());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,8 @@
import com.tencent.bk.job.common.iam.service.WebAuthService;
import com.tencent.bk.job.common.model.ServiceResponse;
import com.tencent.bk.job.common.model.dto.IpDTO;
import com.tencent.bk.job.common.util.check.IlegalCharChecker;
import com.tencent.bk.job.common.util.check.MaxLengthChecker;
import com.tencent.bk.job.common.util.check.NotEmptyChecker;
import com.tencent.bk.job.common.util.check.StringCheckHelper;
import com.tencent.bk.job.common.util.check.TrimChecker;
import com.tencent.bk.job.common.util.FilePathValidateUtil;
import com.tencent.bk.job.common.util.check.*;
import com.tencent.bk.job.common.util.check.exception.StringCheckException;
import com.tencent.bk.job.common.util.date.DateUtils;
import com.tencent.bk.job.execute.api.web.WebExecuteTaskResource;
Expand All @@ -47,29 +44,9 @@
import com.tencent.bk.job.execute.common.constants.TaskTypeEnum;
import com.tencent.bk.job.execute.constants.StepOperationEnum;
import com.tencent.bk.job.execute.engine.model.TaskVariableDTO;
import com.tencent.bk.job.execute.model.DynamicServerGroupDTO;
import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO;
import com.tencent.bk.job.execute.model.FileDetailDTO;
import com.tencent.bk.job.execute.model.FileSourceDTO;
import com.tencent.bk.job.execute.model.ServersDTO;
import com.tencent.bk.job.execute.model.StepInstanceDTO;
import com.tencent.bk.job.execute.model.StepOperationDTO;
import com.tencent.bk.job.execute.model.TaskExecuteParam;
import com.tencent.bk.job.execute.model.TaskInstanceDTO;
import com.tencent.bk.job.execute.model.web.request.RedoTaskRequest;
import com.tencent.bk.job.execute.model.web.request.WebFastExecuteScriptRequest;
import com.tencent.bk.job.execute.model.web.request.WebFastPushFileRequest;
import com.tencent.bk.job.execute.model.web.request.WebStepOperation;
import com.tencent.bk.job.execute.model.web.request.WebTaskExecuteRequest;
import com.tencent.bk.job.execute.model.web.vo.ExecuteFileDestinationInfoVO;
import com.tencent.bk.job.execute.model.web.vo.ExecuteFileSourceInfoVO;
import com.tencent.bk.job.execute.model.web.vo.ExecuteHostVO;
import com.tencent.bk.job.execute.model.web.vo.ExecuteServersVO;
import com.tencent.bk.job.execute.model.web.vo.ExecuteTargetVO;
import com.tencent.bk.job.execute.model.web.vo.ExecuteVariableVO;
import com.tencent.bk.job.execute.model.web.vo.StepExecuteVO;
import com.tencent.bk.job.execute.model.web.vo.StepOperationVO;
import com.tencent.bk.job.execute.model.web.vo.TaskExecuteVO;
import com.tencent.bk.job.execute.model.*;
import com.tencent.bk.job.execute.model.web.request.*;
import com.tencent.bk.job.execute.model.web.vo.*;
import com.tencent.bk.job.execute.service.TaskExecuteService;
import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum;
import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum;
Expand All @@ -84,12 +61,7 @@
import java.util.ArrayList;
import java.util.List;

import static com.tencent.bk.job.common.constant.TaskVariableTypeEnum.ASSOCIATIVE_ARRAY;
import static com.tencent.bk.job.common.constant.TaskVariableTypeEnum.CIPHER;
import static com.tencent.bk.job.common.constant.TaskVariableTypeEnum.HOST_LIST;
import static com.tencent.bk.job.common.constant.TaskVariableTypeEnum.INDEX_ARRAY;
import static com.tencent.bk.job.common.constant.TaskVariableTypeEnum.NAMESPACE;
import static com.tencent.bk.job.common.constant.TaskVariableTypeEnum.STRING;
import static com.tencent.bk.job.common.constant.TaskVariableTypeEnum.*;

@RestController
@Slf4j
Expand Down Expand Up @@ -459,10 +431,16 @@ private boolean checkFastPushFileRequest(WebFastPushFileRequest request) {
log.warn("Fast send file, account is empty!");
return false;
}
for (String file : fileSource.getFileLocation()) {
if (!FilePathValidateUtil.validateFileSystemAbsolutePath(file)) {
log.warn("Fast send file, fileLocation is null or illegal!");
return false;
}
}
}
}
if (StringUtils.isBlank(fileDestination.getPath())) {
log.warn("Fast send file, targetPath is empty");
if (!FilePathValidateUtil.validateFileSystemAbsolutePath(fileDestination.getPath())) {
log.warn("Fast send file, fileDestinationPath is null or illegal!");
return false;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@
package com.tencent.bk.job.manage.model.web.vo.task;

import com.tencent.bk.job.common.model.vo.TaskTargetVO;
import com.tencent.bk.job.common.util.FilePathValidateUtil;
import com.tencent.bk.job.common.util.JobContextUtil;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;

import java.util.List;
Expand Down Expand Up @@ -80,8 +80,8 @@ public boolean validate(boolean isCreate) {
return false;
}
for (String file : fileLocation) {
if (StringUtils.isBlank(file)) {
JobContextUtil.addDebugMessage("Invalid file location!");
if (!FilePathValidateUtil.validateFileSystemAbsolutePath(file)) {
JobContextUtil.addDebugMessage("fileLocation is null or illegal!");
return false;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@
import com.tencent.bk.job.common.constant.ErrorCode;
import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum;
import com.tencent.bk.job.common.exception.ServiceException;
import com.tencent.bk.job.common.util.FilePathValidateUtil;
import com.tencent.bk.job.common.util.JobContextUtil;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;

import java.util.List;
Expand Down Expand Up @@ -115,8 +115,8 @@ public boolean validate(boolean isCreate) {
JobContextUtil.addDebugMessage("Empty destination info!");
return false;
}
if (StringUtils.isBlank(fileDestination.getPath())) {
JobContextUtil.addDebugMessage("Empty destination location!");
if (!FilePathValidateUtil.validateFileSystemAbsolutePath(fileDestination.getPath())) {
JobContextUtil.addDebugMessage("fileDestinationPath is null or illegal!");
return false;
}
if (fileDestination.getAccount() == null || fileDestination.getAccount() <= 0) {
Expand Down

0 comments on commit 7d99751

Please sign in to comment.