super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-158: 结果存在文件缺失巡检（resultFileUrl 对应 OSS 文件缺失检出、空白 URL 不告警、只读、可重复）+ 8 条测试

Browse Source
This commit is contained in:
黄自达
2026-09-02 07:04:06 +08:00
parent 9144bce33f
commit 9998571770
2 changed files with 224 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ResultFileMissingInspector.java
+80
View File
@@ -0,0 +1,80 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Predicate;
/**
* 结果存在文件缺失巡检（task-158）。
*
* 只读巡检：file_result 的 resultFileUrl 非空但对应文件在 OSS 不存在的清单输出
* 报表；空白 URL 行视为"未生成文件"状态、不告警（历史/失败任务无文件属正常）；
* 绝不修改任何状态。可重复执行。
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class ResultFileMissingInspector {
private final FileResultMapper fileResultMapper;
private final OssStorageService ossStorageService;
public record MissingFileEntry(Long resultId, Long taskId, String moduleType, String resultFileUrl) {
}
public record MissingFileReport(List<MissingFileEntry> entries) {
public boolean isEmpty() {
return entries == null || entries.isEmpty();
}
}
public MissingFileReport inspectResultsMissingFile(int limit) {
return inspectResultsMissingFile(limit, ossStorageService::objectExists);
}
MissingFileReport inspectResultsMissingFile(int limit, Predicate<String> urlExists) {
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.isNotNull(FileResultEntity::getResultFileUrl)
.last("limit " + Math.max(1, Math.min(limit, 500))));
if (results == null || results.isEmpty()) {
return new MissingFileReport(List.of());
}
List<MissingFileEntry> entries = new ArrayList<>();
for (FileResultEntity result : results) {
String url = result.getResultFileUrl();
if (url == null || url.isBlank()) {
continue;
}
boolean exists;
try {
exists = urlExists.test(url);
} catch (Exception ex) {
log.warn("result file existence check failed resultId={} url={} err={}",
result.getId(), url, ex.getMessage());
continue;
}
if (!exists) {
entries.add(new MissingFileEntry(
result.getId(), result.getTaskId(), result.getModuleType(), url));
}
}
MissingFileReport report = new MissingFileReport(List.copyOf(entries));
if (!report.isEmpty()) {
log.info("result-missing-file report: count={}", report.entries().size());
for (MissingFileEntry entry : report.entries()) {
log.info("result-missing-file: resultId={} taskId={} moduleType={} url={}",
entry.resultId(), entry.taskId(), entry.moduleType(), entry.resultFileUrl());
}
}
return report;
}
}
backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ResultFileMissingInspectorTest.java
+144
View File
@@ -0,0 +1,144 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* task-158：结果存在文件缺失巡检契约（plan 09）。
* resultFileUrl 对应文件在 OSS 不存在 → 检出；存在不报；空白 URL 不告警；
* 只读；可重复。
*/
@ExtendWith(MockitoExtension.class)
class ResultFileMissingInspectorTest {
private static final Long RESULT_ID = 6001L;
private static final Long TASK_ID = 6002L;
@Mock private FileResultMapper fileResultMapper;
@Mock private OssStorageService ossStorageService;
private ResultFileMissingInspector inspector;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
}
@BeforeEach
void setUp() {
inspector = new ResultFileMissingInspector(fileResultMapper, ossStorageService);
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
private FileResultEntity result(Long id, String url) {
FileResultEntity result = new FileResultEntity();
result.setId(id);
result.setTaskId(TASK_ID);
result.setModuleType("SIMILAR_ASIN");
result.setResultFileUrl(url);
return result;
}
@Test
void missingFileDetected() {
when(fileResultMapper.selectList(any())).thenReturn(List.of(result(RESULT_ID, "oss://result/missing.xlsx")));
var report = inspector.inspectResultsMissingFile(50, url -> false);
assertEquals(1, report.entries().size());
assertEquals(RESULT_ID, report.entries().getFirst().resultId());
}
@Test
void existingFileNotReported() {
when(fileResultMapper.selectList(any())).thenReturn(List.of(result(RESULT_ID, "oss://result/ok.xlsx")));
var report = inspector.inspectResultsMissingFile(50, url -> true);
assertTrue(report.isEmpty());
}
@Test
void blankUrlHandledAsNormal() {
when(fileResultMapper.selectList(any())).thenReturn(List.of(result(RESULT_ID, " ")));
var report = inspector.inspectResultsMissingFile(50, url -> false);
assertTrue(report.isEmpty(), "空白 URL 视为未生成文件状态，不告警");
}
@Test
void ossCheckUsedByDefault() {
when(fileResultMapper.selectList(any())).thenReturn(List.of(result(RESULT_ID, "oss://result/x.xlsx")));
when(ossStorageService.objectExists("oss://result/x.xlsx")).thenReturn(true);
var report = inspector.inspectResultsMissingFile(50);
assertTrue(report.isEmpty(), "默认走 OSS 存在性校验");
}
@Test
void emptyReportWhenNoResults() {
var report = inspector.inspectResultsMissingFile(50, url -> false);
assertTrue(report.isEmpty());
}
@Test
void readOnlyDoesNotModifyAnything() {
when(fileResultMapper.selectList(any())).thenReturn(List.of(result(RESULT_ID, "oss://result/m.xlsx")));
inspector.inspectResultsMissingFile(50, url -> false);
verify(fileResultMapper, never()).update(any(), any());
verify(fileResultMapper, never()).delete(any());
verify(fileResultMapper, never()).updateById(any(FileResultEntity.class));
}
@Test
void reportIncludesDetails() {
when(fileResultMapper.selectList(any())).thenReturn(List.of(result(RESULT_ID, "oss://result/m.xlsx")));
var report = inspector.inspectResultsMissingFile(50, url -> false);
ResultFileMissingInspector.MissingFileEntry entry = report.entries().getFirst();
assertEquals(RESULT_ID, entry.resultId());
assertEquals(TASK_ID, entry.taskId());
assertEquals("SIMILAR_ASIN", entry.moduleType());
assertEquals("oss://result/m.xlsx", entry.resultFileUrl());
assertNotNull(entry.resultFileUrl());
}
@Test
void rerunIsSafeAndStable() {
when(fileResultMapper.selectList(any())).thenReturn(List.of(result(RESULT_ID, "oss://result/m.xlsx")));
var first = inspector.inspectResultsMissingFile(50, url -> false);
var second = inspector.inspectResultsMissingFile(50, url -> false);
assertEquals(first.entries().size(), second.entries().size());
assertEquals(first.entries().getFirst().resultId(), second.entries().getFirst().resultId());
}
}