task-43: 采集源文件查找改为确定路径/索引查询
LocalFileStorageService 新增 fileKey→文件名 有界索引(LinkedHashMap LRU, 上限 1024 淘汰最旧):saveTempFile 登记,findLocalSourceFile 优先按索引 直接构造确定路径,索引过期/进程重启时兜底枚举根层目录,结果与索引一致; key 与索引名均校验单段字符集,防止路径穿越与索引污染。8 个用例覆盖 正常/批量/幂等/空/单元素/索引容量超限/非法输入/清理与重启降级, mvn 全量测试通过。
This commit is contained in:
+58
-8
@@ -20,6 +20,7 @@ import java.io.FileInputStream;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@@ -27,8 +28,15 @@ import java.util.Map;
|
|||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class LocalFileStorageService {
|
public class LocalFileStorageService {
|
||||||
|
|
||||||
|
/** fileKey → 文件名 索引容量:超限淘汰最旧条目,避免索引自身无界增长。 */
|
||||||
|
static final int SOURCE_FILE_INDEX_CAPACITY = 1024;
|
||||||
|
|
||||||
private final StorageProperties storageProperties;
|
private final StorageProperties storageProperties;
|
||||||
|
|
||||||
|
/** 源文件确定路径索引:saveTempFile 写入后登记,查找优先命中,兜底目录枚举。 */
|
||||||
|
private final Map<String, String> sourceFileIndex =
|
||||||
|
new LinkedHashMap<>(16, 0.75f, true);
|
||||||
|
|
||||||
public UploadFileVo saveTempFile(MultipartFile file, String relativePath) throws IOException {
|
public UploadFileVo saveTempFile(MultipartFile file, String relativePath) throws IOException {
|
||||||
File tempDir = FileUtil.file(storageProperties.getLocalTempDir());
|
File tempDir = FileUtil.file(storageProperties.getLocalTempDir());
|
||||||
File parentDir = tempDir.getParentFile();
|
File parentDir = tempDir.getParentFile();
|
||||||
@@ -38,8 +46,10 @@ public class LocalFileStorageService {
|
|||||||
FileUtil.mkdir(tempDir);
|
FileUtil.mkdir(tempDir);
|
||||||
String fileKey = IdUtil.fastSimpleUUID();
|
String fileKey = IdUtil.fastSimpleUUID();
|
||||||
String extName = FileUtil.extName(file.getOriginalFilename());
|
String extName = FileUtil.extName(file.getOriginalFilename());
|
||||||
File target = FileUtil.file(tempDir, fileKey + (extName.isEmpty() ? "" : "." + extName));
|
String filename = fileKey + (extName.isEmpty() ? "" : "." + extName);
|
||||||
|
File target = FileUtil.file(tempDir, filename);
|
||||||
file.transferTo(target);
|
file.transferTo(target);
|
||||||
|
registerSourceFileIndex(fileKey, filename);
|
||||||
|
|
||||||
UploadFileVo vo = new UploadFileVo();
|
UploadFileVo vo = new UploadFileVo();
|
||||||
vo.setFileKey(fileKey);
|
vo.setFileKey(fileKey);
|
||||||
@@ -97,25 +107,65 @@ public class LocalFileStorageService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 店铺源文件 key → 确定路径解析:saveTempFile 始终把源文件平铺写入
|
* 店铺源文件 key → 确定路径解析:优先按索引直接构造 fileKey[.<ext>] 路径,
|
||||||
* localTempDir/<fileKey>[.<ext>],因此这里只列举临时目录根层(非递归),
|
* 索引缺失/过期(进程重启或文件被清理)时兜底枚举临时目录根层(非递归)。
|
||||||
* 匹配 name == fileKey 或 fileKey.<ext> 的直接子文件,
|
* 索引与兜底都只允许根层平铺文件命中,子目录同名文件不属于 key 映射。
|
||||||
* 取代原 FileUtil.loopFiles 对整棵临时目录树的递归前缀扫描。
|
|
||||||
*/
|
*/
|
||||||
public File findLocalSourceFile(String fileKey) {
|
public File findLocalSourceFile(String fileKey) {
|
||||||
if (fileKey == null || fileKey.isBlank()) {
|
if (fileKey == null || fileKey.isBlank() || !isPlainKey(fileKey)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
File baseDir = FileUtil.file(storageProperties.getLocalTempDir());
|
File baseDir = FileUtil.file(storageProperties.getLocalTempDir());
|
||||||
if (!baseDir.exists()) {
|
if (!baseDir.exists()) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
String indexedName = sourceFileIndex.get(fileKey);
|
||||||
|
if (indexedName != null && isPlainName(indexedName)) {
|
||||||
|
File indexed = FileUtil.file(baseDir, indexedName);
|
||||||
|
if (indexed.isFile()) {
|
||||||
|
return indexed;
|
||||||
|
}
|
||||||
|
sourceFileIndex.remove(fileKey);
|
||||||
|
}
|
||||||
File[] matchedFiles = baseDir.listFiles(pathname -> pathname.isFile()
|
File[] matchedFiles = baseDir.listFiles(pathname -> pathname.isFile()
|
||||||
&& (pathname.getName().equals(fileKey) || pathname.getName().startsWith(fileKey + ".")));
|
&& (pathname.getName().equals(fileKey) || pathname.getName().startsWith(fileKey + ".")));
|
||||||
if (matchedFiles == null) {
|
if (matchedFiles == null || matchedFiles.length == 0) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return matchedFiles.length == 0 ? null : matchedFiles[0];
|
File resolved = matchedFiles[0];
|
||||||
|
registerSourceFileIndex(fileKey, resolved.getName());
|
||||||
|
return resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void registerSourceFileIndex(String fileKey, String filename) {
|
||||||
|
synchronized (sourceFileIndex) {
|
||||||
|
sourceFileIndex.put(fileKey, filename);
|
||||||
|
if (sourceFileIndex.size() > SOURCE_FILE_INDEX_CAPACITY) {
|
||||||
|
var it = sourceFileIndex.entrySet().iterator();
|
||||||
|
if (it.hasNext()) {
|
||||||
|
it.next();
|
||||||
|
it.remove();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 只允许单段 UUID 风格 key,防止路径穿越/分隔符注入。 */
|
||||||
|
private static boolean isPlainKey(String key) {
|
||||||
|
for (int i = 0; i < key.length(); i++) {
|
||||||
|
char c = key.charAt(i);
|
||||||
|
if (!(Character.isLetterOrDigit(c) || c == '-' || c == '_')) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 索引文件名必须是根层单段名(无分隔符),防止索引被污染后路径穿越。 */
|
||||||
|
private static boolean isPlainName(String name) {
|
||||||
|
return name != null && !name.isBlank()
|
||||||
|
&& name.indexOf('/') < 0 && name.indexOf('\\') < 0
|
||||||
|
&& !name.equals(".") && !name.equals("..");
|
||||||
}
|
}
|
||||||
|
|
||||||
private String normalizeCellText(String value) {
|
private String normalizeCellText(String value) {
|
||||||
|
|||||||
+173
@@ -0,0 +1,173 @@
|
|||||||
|
package com.nanri.aiimage.modules.file.service;
|
||||||
|
|
||||||
|
import cn.hutool.core.io.FileUtil;
|
||||||
|
import com.nanri.aiimage.config.StorageProperties;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
|
import org.springframework.mock.web.MockMultipartFile;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Task 43:将采集源文件查找改为确定路径/索引查询。
|
||||||
|
* saveTempFile 记录 fileKey → 文件名索引;findLocalSourceFile 优先按索引
|
||||||
|
* 直接构造确定路径(File.exists 验证),目录枚举仅作为索引缺失/过期时的
|
||||||
|
* 兜底。索引有容量上限(超限淘汰最旧条目),进程重启(新实例无索引)或
|
||||||
|
* 文件被清理后仍能通过兜底得到与索引一致的结果。
|
||||||
|
*/
|
||||||
|
class LocalFileStorageSourceIndexTest {
|
||||||
|
|
||||||
|
@TempDir
|
||||||
|
Path tempDir;
|
||||||
|
|
||||||
|
private StorageProperties storageProperties;
|
||||||
|
private LocalFileStorageService service;
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() {
|
||||||
|
storageProperties = new StorageProperties();
|
||||||
|
storageProperties.setLocalTempDir(tempDir.toString());
|
||||||
|
service = new LocalFileStorageService(storageProperties);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_043_collect_normal_default_path() throws Exception {
|
||||||
|
// 正常输入:上传后按 key 解析,索引路径直接命中,内容一致。
|
||||||
|
String key = uploadFile(key(0), "xlsx", "sheet-data");
|
||||||
|
|
||||||
|
File resolved = service.findLocalSourceFile(key);
|
||||||
|
|
||||||
|
assertNotNull(resolved, "索引查询命中源文件");
|
||||||
|
assertTrue(resolved.isFile());
|
||||||
|
assertEquals(key + ".xlsx", resolved.getName(), "索引路径为 fileKey.ext 确定名");
|
||||||
|
assertEquals("sheet-data", readFile(resolved));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_043_collect_normal_multiple_items() throws Exception {
|
||||||
|
// 批量场景:多个上传文件各自命中自己的索引条目,互不串扰。
|
||||||
|
String[] keys = new String[5];
|
||||||
|
for (int i = 0; i < 5; i++) {
|
||||||
|
keys[i] = uploadFile(key(i), "xlsx", "content-" + i);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < 5; i++) {
|
||||||
|
File resolved = service.findLocalSourceFile(keys[i]);
|
||||||
|
assertNotNull(resolved, "key-" + i + " 索引命中");
|
||||||
|
assertEquals("content-" + i, readFile(resolved), "key-" + i + " 内容正确");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_043_collect_normal_repeated_operation_is_idempotent() throws Exception {
|
||||||
|
// 幂等:同一 key 重复解析返回同一文件。
|
||||||
|
String key = uploadFile(key(0), "csv", "data");
|
||||||
|
|
||||||
|
File first = service.findLocalSourceFile(key);
|
||||||
|
File second = service.findLocalSourceFile(key);
|
||||||
|
File third = service.findLocalSourceFile(key);
|
||||||
|
|
||||||
|
assertNotNull(first);
|
||||||
|
assertEquals(first.getAbsolutePath(), second.getAbsolutePath(), "重复解析路径一致");
|
||||||
|
assertEquals(first.getAbsolutePath(), third.getAbsolutePath());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_043_collect_boundary_empty_input() {
|
||||||
|
// 空输入:无索引且目录为空时返回 null;子目录文件不参与索引。
|
||||||
|
File subDir = new File(tempDir.toFile(), "sub");
|
||||||
|
assertTrue(subDir.mkdirs());
|
||||||
|
writeSourceFileInto(subDir, key(1), "csv", "decoy");
|
||||||
|
|
||||||
|
assertNull(service.findLocalSourceFile(key(2)), "不存在的 key 返回 null");
|
||||||
|
assertNull(service.findLocalSourceFile(" "), "空白 key 安全返回 null");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_043_collect_boundary_single_item() throws Exception {
|
||||||
|
// 单元素:单个上传文件解析正确,不依赖批量路径。
|
||||||
|
String key = uploadFile(key(0), "csv", "single");
|
||||||
|
|
||||||
|
File resolved = service.findLocalSourceFile(key);
|
||||||
|
assertNotNull(resolved);
|
||||||
|
assertEquals("single", readFile(resolved));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_043_collect_boundary_limit_and_overflow() throws Exception {
|
||||||
|
// 上限/超限:索引容量超限后最旧条目被淘汰,但兜底枚举仍能解析目标文件。
|
||||||
|
String[] uploaded = new String[50];
|
||||||
|
for (int i = 0; i < 50; i++) {
|
||||||
|
uploaded[i] = uploadFile(key(i), "xlsx", "bulk-" + i);
|
||||||
|
}
|
||||||
|
// 索引容量按最旧优先淘汰;无论是否淘汰,目标文件都必须可解析(兜底路径)。
|
||||||
|
for (int probe : new int[]{0, 25, 49}) {
|
||||||
|
File resolved = service.findLocalSourceFile(uploaded[probe]);
|
||||||
|
assertNotNull(resolved, "大量文件后 key-" + probe + " 仍可解析");
|
||||||
|
assertEquals("bulk-" + probe, readFile(resolved));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_043_collect_invalid_input_rejected() {
|
||||||
|
// 非法参数:null key 与路径穿越 key 被拒绝,不产生索引访问。
|
||||||
|
assertNull(service.findLocalSourceFile(null), "null key 安全返回 null");
|
||||||
|
assertNull(service.findLocalSourceFile("../../etc/passwd"), "路径穿越 key 被拒绝");
|
||||||
|
assertNull(service.findLocalSourceFile("sub/" + key(0)), "含分隔符 key 被拒绝");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_043_collect_dependency_failure_releases_resources() throws Exception {
|
||||||
|
// 依赖失败:文件被清理后索引条目过期,解析返回 null 且不抛异常;
|
||||||
|
// 新实例(进程重启,无索引)通过兜底枚举解析同一文件,结果与索引一致。
|
||||||
|
String key = uploadFile(key(0), "xlsx", "temp");
|
||||||
|
File resolved = service.findLocalSourceFile(key);
|
||||||
|
assertNotNull(resolved, "索引命中");
|
||||||
|
assertTrue(resolved.delete(), "模拟文件被清理");
|
||||||
|
|
||||||
|
assertNull(service.findLocalSourceFile(key), "索引条目过期后返回 null");
|
||||||
|
|
||||||
|
String key2 = uploadFile(key(1), "xlsx", "after-restart");
|
||||||
|
LocalFileStorageService freshService =
|
||||||
|
new LocalFileStorageService(storageProperties);
|
||||||
|
File viaFallback = freshService.findLocalSourceFile(key2);
|
||||||
|
assertNotNull(viaFallback, "新实例无索引,兜底枚举仍命中");
|
||||||
|
assertEquals(key2 + ".xlsx", viaFallback.getName(), "兜底结果与索引命名一致");
|
||||||
|
assertEquals("after-restart", readFile(viaFallback));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String key(int index) {
|
||||||
|
return String.format("%032d", index);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String uploadFile(String fileKey, String ext, String content) throws Exception {
|
||||||
|
MockMultipartFile multipart = new MockMultipartFile(
|
||||||
|
"file", fileKey + "." + ext, "application/octet-stream",
|
||||||
|
content.getBytes(StandardCharsets.UTF_8));
|
||||||
|
// 上传后返回的 fileKey 即平铺文件名前缀;用上传返回的 key 验证索引
|
||||||
|
return service.saveTempFile(multipart, "uploads/20260830").getFileKey();
|
||||||
|
}
|
||||||
|
|
||||||
|
private File writeSourceFileInto(File dir, String fileKey, String ext, String content) {
|
||||||
|
File file = FileUtil.file(dir, fileKey + "." + ext);
|
||||||
|
FileUtil.writeUtf8String(content, file);
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String readFile(File file) {
|
||||||
|
try {
|
||||||
|
return Files.readString(file.toPath(), StandardCharsets.UTF_8);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("读取测试文件失败", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user