Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fast_transfer_file API 分发文件,如果源文件中的文件名包含空格,会报错 #812 #1649

Open
wants to merge 5 commits into
base: 3.5.x
Choose a base branch
from
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package com.tencent.bk.job.common.util;

import org.apache.commons.lang3.StringUtils;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
* 文件路径合法性校验工具类
*/
public class FilePathValidateUtil {
/**
* 验证文件系统绝对路径的合法性
* @param path 绝对路径
* @return boolean true合法,false非法
*/
public static boolean validateFileSystemAbsolutePath(String path) {
if (StringUtils.isBlank(path)) {
return false;
}
if (isLinuxAbsolutePath(path)) {
return validateLinuxFileSystemAbsolutePath(path);
} else {
return validateWindowsFileSystemAbsolutePath(path);
}
}

/**
* 判断是否Linux绝对路径
*
* @param path 文件路径
* @return boolean
*/
private static boolean isLinuxAbsolutePath(String path) {
if (path.startsWith("/")) {
return true;
}
return false;
}

/**
* 1 DOS设备路径:
* 设备路径说明符(\\.\ 或 \\?\),它将路径标识为DOS设备路径
* 2 UNC路径
* 以\\开头的服务器名或主机名,路径必须始终是完全限定的
* 组成:\\服务器名\共享名\可选目录名\可选文件名
* 3 传统DOS路径
* 标准的DOS路径可由以下三部分组成:
* 1)卷号或驱动器号,后跟卷分隔符(:)。
* 2)目录名称。目录分隔符用来分隔嵌套目录层次结构中的子目录。
* 3)文件名。目录分隔符用来分隔文件路径和文件名。
*
* @param path
* @return boolean
*/
private static boolean validateWindowsFileSystemAbsolutePath(String path) {
// DOS设备
String pattern = "^\\\\\\\\.\\\\.+|\\\\\\\\\\?\\\\.+";
Pattern r = Pattern.compile(pattern);
Matcher m = r.matcher(path);
if (m.matches()) {
return true;
}
// UNC
pattern = "^\\\\\\\\[^\\\\]+\\\\[^\\\\]+\\\\.*";
r = Pattern.compile(pattern);
m = r.matcher(path);
if (m.matches()) {
return true;
}
// 传统DOS
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

传统DOS是最常见的,所以可以把这块的检查放到最前面,其他校验代码就不需要执行

pattern = "^[A-Za-z]:\\\\[^\\\\].*";
r = Pattern.compile(pattern);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pattern不建议调用没调用一次方法就compile一次,浪费性能。可以使用private static final Pattern 先预处理

m = r.matcher(path);
if (m.matches()) {
return true;
}
return false;
}

/**
* 文件或目录名,除了/以外,所有的字符都合法
*
* @param path
* @return boolean
*/
private static boolean validateLinuxFileSystemAbsolutePath(String path) {
String pattern = "^/([^/].*/{0,1})+";
Pattern r = Pattern.compile(pattern);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pattern不建议调用没调用一次方法就compile一次,浪费性能。可以使用private static final Pattern 先预处理

Matcher m = r.matcher(path);
if (m.matches()) {
return true;
}
return false;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package com.tencent.bk.job.common.util;

import org.junit.jupiter.api.Test;

import static org.assertj.core.api.AssertionsForClassTypes.assertThat;

public class FilePathValidateUtilTest {
@Test
void testFileSystemAbsolutePath(){
// DOS设备路径
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("\\\\.\\C:\\Test\\Foo.txt")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("\\\\?\\C:\\Test\\Foo.txt")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("\\\\Test\\Foo.txt")).isFalse();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("\\\\?\\C:\\Te st\\嘉Foo.txt")).isTrue();

// UNC路径
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("\\\\system07\\C$\\")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("\\\\Server2\\Share\\Test\\Foo.txt")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("\\\\Server201\\C$\\Test1\\Fo o.txt")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("\\\\system07")).isFalse();

// 传统DOS路径
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\Documents\\abc.txt")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("c:\\Documents\\abc.txt")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\Documents\\嘉 abc.txt")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath(":\\abc.txt")).isFalse();

// linux路径
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/data/test_2022-04-12.apk")).isTrue();
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

需要补充更多的测试用例,如一些比较常见的:

  1. 根目录 /
  2. /tmp/
  3. /tmp/test/../test.log

assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/data/test_2022 04 12.apk")).isTrue();
assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("data/test_2022-04-12.apk")).isFalse();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import com.tencent.bk.job.common.model.ValidateResult;
import com.tencent.bk.job.common.service.AppScopeMappingService;
import com.tencent.bk.job.common.util.ArrayUtil;
import com.tencent.bk.job.common.util.FilePathValidateUtil;
import com.tencent.bk.job.common.util.date.DateUtils;
import com.tencent.bk.job.common.web.metrics.CustomTimed;
import com.tencent.bk.job.execute.api.esb.v2.EsbFastPushFileResource;
Expand Down Expand Up @@ -124,7 +125,7 @@ public EsbResp<EsbJobExecuteDTO> fastPushFile(EsbFastPushFileRequest request) {
}

private ValidateResult checkFastPushFileRequest(EsbFastPushFileRequest request) {
if (!validateFileSystemPath(request.getTargetPath())) {
if (!FilePathValidateUtil.validateFileSystemAbsolutePath(request.getTargetPath())) {
log.warn("Fast transfer file, target path is invalid!path={}", request.getTargetPath());
return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "file_target_path");
}
Expand Down Expand Up @@ -162,7 +163,7 @@ private ValidateResult validateFileSource(EsbFastPushFileRequest request) {
return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "file_source.files");
}
for (String file : files) {
if (!validateFileSystemPath(file)) {
if (!FilePathValidateUtil.validateFileSystemAbsolutePath(file)) {
log.warn("Invalid path:{}", file);
return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "file_source.files");
}
Expand All @@ -183,32 +184,6 @@ private ValidateResult validateFileSource(EsbFastPushFileRequest request) {
return ValidateResult.pass();
}

private boolean validateFileSystemPath(String path) {
if (StringUtils.isBlank(path)) {
return false;
}
if (path.indexOf(' ') != -1) {
return false;
}
Pattern p1 = Pattern.compile("(//|\\\\)+");
Matcher m1 = p1.matcher(path);
if (m1.matches()) {
return false;
}

Pattern p2 = Pattern.compile("^[a-zA-Z]:(/|\\\\).*");//windows
Matcher m2 = p2.matcher(path);

if (!m2.matches()) { //非windows
if (path.charAt(0) == '/') {
return !path.contains("\\\\");
} else {
return false;
}
}
return true;
}

private TaskInstanceDTO buildFastFileTaskInstance(EsbFastPushFileRequest request) {
TaskInstanceDTO taskInstance = new TaskInstanceDTO();
taskInstance.setType(TaskTypeEnum.FILE.getValue());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import com.tencent.bk.job.common.model.InternalResponse;
import com.tencent.bk.job.common.model.ValidateResult;
import com.tencent.bk.job.common.service.AppScopeMappingService;
import com.tencent.bk.job.common.util.FilePathValidateUtil;
import com.tencent.bk.job.common.util.date.DateUtils;
import com.tencent.bk.job.common.web.metrics.CustomTimed;
import com.tencent.bk.job.execute.client.FileSourceResourceClient;
Expand Down Expand Up @@ -144,7 +145,7 @@ private ValidateResult checkFileSource(EsbFileSourceV3DTO fileSource) {
for (String file : files) {
if ((fileType == null
|| TaskFileTypeEnum.SERVER.getType() == fileType)
&& !validateFileSystemPath(file)) {
&& !FilePathValidateUtil.validateFileSystemAbsolutePath(file)) {
log.warn("Invalid path:{}", file);
return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "file_source.file_list");
}
Expand Down Expand Up @@ -178,7 +179,7 @@ private ValidateResult checkFileSource(EsbFileSourceV3DTO fileSource) {
}

private ValidateResult checkFastTransferFileRequest(EsbFastTransferFileV3Request request) {
if (!validateFileSystemPath(request.getTargetPath())) {
if (!FilePathValidateUtil.validateFileSystemAbsolutePath(request.getTargetPath())) {
log.warn("Fast transfer file, target path is invalid!path={}", request.getTargetPath());
return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "file_target_path");
}
Expand All @@ -205,32 +206,6 @@ private ValidateResult checkFastTransferFileRequest(EsbFastTransferFileV3Request
return ValidateResult.pass();
}

private boolean validateFileSystemPath(String path) {
if (StringUtils.isBlank(path)) {
return false;
}
if (path.indexOf(' ') != -1) {
return false;
}
Pattern p1 = Pattern.compile("(//|\\\\)+");
Matcher m1 = p1.matcher(path);
if (m1.matches()) {
return false;
}

Pattern p2 = Pattern.compile("^[a-zA-Z]:(/|\\\\).*");//windows
Matcher m2 = p2.matcher(path);

if (!m2.matches()) { //非windows
if (path.charAt(0) == '/') {
return !path.contains("\\\\");
} else {
return false;
}
}
return true;
}

private TaskInstanceDTO buildFastFileTaskInstance(EsbFastTransferFileV3Request request) {
TaskInstanceDTO taskInstance = new TaskInstanceDTO();
taskInstance.setType(TaskTypeEnum.FILE.getValue());
Expand Down