fix(断点续传): 外观/结果作业/去重三处中断语义补齐,重启不再静默丢行

- 外观专利行标改两态(in-flight 15min / processed 24h):只有 CAS 写回 payload 成功
  才转「已处理」,异常、覆盖率不足、CAS 打满等未落库出口一律撤销标记 —— 修「发 LLM
  请求前就盖章,进程被杀的那一批永不再补检」
- 结果文件作业开机自回收:按 owner_instance_id 只收本机名下 RUNNING 且心跳超 90s 的行;
  卡死阈值 30 → 5 分钟,重启接回从最长半小时压到 5 分钟
- 外观结果表缺检行显式写「未取得检测结果」并判失败,不再留空靠 0.5 覆盖率阈值蒙过
- 去重进度改 NodeSharedStore 跨节点共享,runId 用已落库的 task_no:轮询打到另一节点或
  发版重启后仍能读到真实进度、已完成文件仍可下载;共享态缺失时按 task_no 从库重建;
  逐文件心跳防 StaleTaskRepair 把长任务误判死;开机把本机残留 RUNNING 判失败并给提示
This commit is contained in:
2026-09-27 22:26:09 +08:00
parent ed7dd9381b
commit a58dc8a680
17 changed files with 1662 additions and 84 deletions
@@ -0,0 +1,273 @@
package com.nanri.aiimage.modules.appearancepatent.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.common.service.ResultDownloadResolver;
import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InOrder;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils;
import org.springframework.transaction.PlatformTransactionManager;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 「行什么时候算检测完」的时序约束。
*
* <p>合批之后一次请求带走 5–10 行,一次中断就是整批白做。原实现在收集候选时(发请求**之前**)
* 就把行标成已处理,所以发版重启留下的不是"稍后补检",而是"永不再检"。
* 这里钉住两条:申领发生在收集阶段、完成标记只允许出现在载荷 CAS 落库成功之后。
*/
@ExtendWith(MockitoExtension.class)
class AppearancePatentRowMarkTimingTest {
private static final Long TASK_ID = 29436L;
private static final Long JOB_ID = 20788L;
private static final String SCOPE_HASH = "scope-hash-1";
private static final int CHUNK_INDEX = 1;
private static final String POINTER = "ptr-chunk-1.json";
private static final String UNRESOLVED_ROWS =
"[{\"rowToken\":\"r1\",\"asin\":\"B0A0000001\",\"title\":\"RUNNING SHOE\",\"url\":\"http://img/1.jpg\",\"groupKey\":\"g1\"},"
+ "{\"rowToken\":\"r2\",\"asin\":\"B0A0000002\",\"title\":\"COTTON BAG\",\"url\":\"http://img/2.jpg\",\"groupKey\":\"g2\"}]";
private static final String RESOLVED_ROWS =
"[{\"rowToken\":\"r1\",\"asin\":\"B0A0000001\",\"title\":\"RUNNING SHOE\",\"url\":\"http://img/1.jpg\",\"groupKey\":\"g1\",\"titleRisk\":\"无侵权\",\"appearanceRisk\":\"无侵权\",\"conclusion\":\"无侵权\"},"
+ "{\"rowToken\":\"r2\",\"asin\":\"B0A0000002\",\"title\":\"COTTON BAG\",\"url\":\"http://img/2.jpg\",\"groupKey\":\"g2\",\"titleRisk\":\"侵权\",\"appearanceRisk\":\"侵权\",\"conclusion\":\"侵权\"}]";
@Mock private LocalFileStorageService localFileStorageService;
@Mock private OssStorageService ossStorageService;
@Mock private ResultDownloadResolver resultDownloadResolver;
@Mock private StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private AppearancePatentLlmClient llmClient;
@Mock private AppearancePatentTaskCacheService taskCacheService;
@Mock private AppearancePatentProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private PlatformTransactionManager transactionManager;
@Mock private DistributedJobLockService distributedJobLockService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private InstanceMetadata instanceMetadata;
@Mock private TaskProgressLightAssembler taskProgressLightAssembler;
@Mock private UserApiSecretService userApiSecretService;
@InjectMocks private AppearancePatentTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.getLlmBatchSize()).thenReturn(2);
lenient().when(properties.getFlushPendingMinutes()).thenReturn(1);
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
lenient().when(userApiSecretService.findPlainValue(any(), anyString())).thenReturn("");
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER), anyString())).thenReturn(UNRESOLVED_ROWS);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk()));
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(chunk());
}
/** 收集阶段只申领、不完成:这是「先干活后标记」和旧行为唯一看得见的区别。 */
@Test
@SuppressWarnings("unchecked")
void collectionClaimsRowsWithoutCompletingThem() {
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
List<AppearancePatentResultRowDto> candidates = (List<AppearancePatentResultRowDto>) ReflectionTestUtils
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()));
assertEquals(2, candidates.size());
verify(taskCacheService).beginRowInspection(TASK_ID, "r1");
verify(taskCacheService).beginRowInspection(TASK_ID, "r2");
verify(taskCacheService, never()).completeRowInspection(anyLong(), any());
}
/** 已经被别的入口申领走的行不再进来,否则会双发 LLM、双扣次数。 */
@Test
@SuppressWarnings("unchecked")
void rowsHeldByAnotherEntryAreSkipped() {
when(taskCacheService.beginRowInspection(eq(TASK_ID), eq("r1"))).thenReturn(true);
when(taskCacheService.beginRowInspection(eq(TASK_ID), eq("r2"))).thenReturn(false);
List<AppearancePatentResultRowDto> candidates = (List<AppearancePatentResultRowDto>) ReflectionTestUtils
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()));
assertEquals(1, candidates.size());
assertEquals("r1", candidates.get(0).getRowToken());
}
/** 落库 CAS 成功之后才允许打完成标记——顺序错了就等于回到"发版丢行"的老 bug。 */
@Test
void completionMarkComesAfterPayloadCasUpdate() throws Exception {
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
when(llmClient.inspectRows(any(), any(), any())).thenReturn(rows(RESOLVED_ROWS));
stubMergeWrite(1);
submit();
InOrder order = inOrder(taskChunkMapper, taskCacheService);
order.verify(taskChunkMapper).update(any(), any());
order.verify(taskCacheService).completeRowInspection(eq(TASK_ID), eq(List.of("r1", "r2")));
}
/**
* LLM 整批失败:失败原因照样落库(用户看得见),但**不打完成标记**,而是撤销处理中标记——
* assemble job 的 5 次重试几分钟内就跑完了,等 TTL 过期的话这批行在整个重试窗口里都申领不到。
* 旧行为是"永远记着这次失败",再不会重检。
*/
@Test
void failedLlmBatchLeavesRowsRetryable() throws Exception {
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
when(llmClient.inspectRows(any(), any(), any())).thenThrow(new IllegalStateException("LLM 网关无应答"));
when(llmClient.markRowsFailed(any(), anyString())).thenReturn(rows(UNRESOLVED_ROWS));
stubMergeWrite(1);
submit();
verify(taskCacheService, never()).completeRowInspection(anyLong(), any());
verify(taskCacheService).releaseRowInspection(TASK_ID, List.of("r1", "r2"));
}
/** 成功批次不能撤销处理中标记:撤销是给"下一轮立刻重试"用的,成功行重跑就是重复扣次数。 */
@Test
void successfulBatchDoesNotReleaseInFlightMarks() throws Exception {
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
when(llmClient.inspectRows(any(), any(), any())).thenReturn(rows(RESOLVED_ROWS));
stubMergeWrite(1);
submit();
verify(taskCacheService, never()).releaseRowInspection(anyLong(), any());
}
/** 结果拿到了但载荷写不进去(CAS 重试耗尽):同样不能算完成,否则结果是内存里的幻影。 */
@Test
void exhaustedCasDoesNotCompleteRows() throws Exception {
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
when(llmClient.inspectRows(any(), any(), any())).thenReturn(rows(RESOLVED_ROWS));
stubMergeWrite(0);
assertThrows(IllegalStateException.class, this::submit);
verify(taskCacheService, never()).completeRowInspection(anyLong(), any());
verify(taskCacheService).releaseRowInspection(TASK_ID, List.of("r1", "r2"));
}
/** 发送前续一次处理中标记,避免一批跑得比 TTL 长时被并发入口抢走。 */
@Test
void inFlightMarkIsRenewedRightBeforeSending() throws Exception {
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
when(llmClient.inspectRows(any(), any(), any())).thenReturn(rows(RESOLVED_ROWS));
stubMergeWrite(1);
submit();
ArgumentCaptor<List<String>> keys = ArgumentCaptor.forClass(List.class);
InOrder order = inOrder(taskCacheService, llmClient);
order.verify(taskCacheService).touchRowInspection(eq(TASK_ID), keys.capture());
order.verify(llmClient).inspectRows(any(), any(), any());
assertEquals(List.of("r1", "r2"), keys.getValue());
}
// ===== 辅助 =====
private void submit() {
ReflectionTestUtils.invokeMethod(service, "submitLlmBatches",
task(), new FileResultEntity(), job(), List.of(chunk()), Map.of());
}
private void stubMergeWrite(int updatedRows) {
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
anyString(), anyLong(), anyString(), any(), anyString())).thenReturn("sibling-1");
when(taskChunkMapper.update(any(), any())).thenReturn(updatedRows);
lenient().when(taskChunkMapper.selectById(anyLong())).thenReturn(chunk());
}
private List<AppearancePatentResultRowDto> rows(String json) throws Exception {
return List.of(new ObjectMapper().readValue(json, AppearancePatentResultRowDto[].class));
}
private FileTaskEntity task() {
FileTaskEntity task = new FileTaskEntity();
task.setId(TASK_ID);
task.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
task.setUserId(7L);
return task;
}
private com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity job() {
com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity job =
new com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity();
job.setId(JOB_ID);
job.setTaskId(TASK_ID);
return job;
}
private TaskChunkEntity chunk() {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(11L);
chunk.setTaskId(TASK_ID);
chunk.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
chunk.setScopeHash(SCOPE_HASH);
chunk.setChunkIndex(CHUNK_INDEX);
chunk.setPayloadJson(POINTER);
chunk.setPayloadHash("hash-old");
return chunk;
}
}
@@ -0,0 +1,155 @@
package com.nanri.aiimage.modules.appearancepatent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import java.time.Duration;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 外观专利行级检测标记的两态语义。
*
* <p>历史实现只有一个标记,且在**发 LLM 请求之前**就打({@code collectPendingLlmCandidates}),
* TTL 24 小时、无撤销路径。于是进程被杀(发版、崩溃)或载荷落库失败时,那批行既没有结果、
* 又被判定为「已处理」,续跑时直接跳过——线上表现为少数几行永远空白而任务报成功。
* 现在拆成两态:**处理中**(短 TTL,谁在跑这行)与**已完成**(长 TTL,只在结果落库成功后打)。
*/
@ExtendWith(MockitoExtension.class)
class AppearancePatentTaskCacheServiceRowMarkTest {
private static final Long TASK_ID = 29436L;
private static final String ROW_KEY = "r1";
@Mock private StringRedisTemplate redisTemplate;
@Mock private ValueOperations<String, String> valueOperations;
private AppearancePatentTaskCacheService cache;
@BeforeEach
void setUp() {
cache = new AppearancePatentTaskCacheService(redisTemplate, new ObjectMapper());
lenient().when(redisTemplate.opsForValue()).thenReturn(valueOperations);
}
/** 没任何标记时,第一次申领必须拿到这行。 */
@Test
void beginClaimsRowWhenNoMarkExists() {
when(valueOperations.setIfAbsent(anyString(), anyString(), any(Duration.class))).thenReturn(true);
assertTrue(cache.beginRowInspection(TASK_ID, ROW_KEY));
}
/** 处理中标记还在(另一个入口正在跑这行)时不能重复申领,否则会双发 LLM、双扣次数。 */
@Test
void beginRejectsRowStillInFlight() {
when(valueOperations.setIfAbsent(anyString(), anyString(), any(Duration.class))).thenReturn(false);
assertFalse(cache.beginRowInspection(TASK_ID, ROW_KEY));
}
/** 已完成的行连申领都不该发起:不能白占一个处理中名额。 */
@Test
void beginRejectsCompletedRowWithoutTouchingInFlightMark() {
when(redisTemplate.hasKey(anyString())).thenReturn(true);
assertFalse(cache.beginRowInspection(TASK_ID, ROW_KEY));
verify(valueOperations, never()).setIfAbsent(anyString(), anyString(), any(Duration.class));
}
/**
* 处理中标记必须是分钟级:进程被杀后这些行要能重新成为候选。
* 上限取 30 分钟——线上实测单批 10 行最慢 220 秒,再长就是拿丢行换并发安全。
*/
@Test
void inFlightMarkExpiresWithinMinutes() {
when(valueOperations.setIfAbsent(anyString(), anyString(), any(Duration.class))).thenReturn(true);
cache.beginRowInspection(TASK_ID, ROW_KEY);
ArgumentCaptor<Duration> ttl = ArgumentCaptor.forClass(Duration.class);
verify(valueOperations).setIfAbsent(anyString(), anyString(), ttl.capture());
assertTrue(ttl.getValue().toSeconds() > 0, "处理中标记必须带 TTL,否则一次中断就永久丢行");
assertTrue(ttl.getValue().toMinutes() <= 30, "处理中标记不得超过 30 分钟,实测单批最慢约 220 秒");
}
/** 完成标记只在落库成功后打:长 TTL,并且要把处理中名额让出来。 */
@Test
void completeMarksLongTtlAndReleasesInFlight() {
cache.completeRowInspection(TASK_ID, List.of(ROW_KEY));
ArgumentCaptor<String> completed = ArgumentCaptor.forClass(String.class);
ArgumentCaptor<Duration> ttl = ArgumentCaptor.forClass(Duration.class);
verify(valueOperations).set(completed.capture(), anyString(), ttl.capture());
assertEquals(Duration.ofHours(24), ttl.getValue());
ArgumentCaptor<String> deleted = ArgumentCaptor.forClass(String.class);
verify(redisTemplate).delete(deleted.capture());
assertNotEquals(completed.getValue(), deleted.getValue(), "清除的必须是处理中键,不能把刚打的完成标记删掉");
}
/** 两态必须落在不同键上:同键的话「完成」会把短 TTL 覆盖成长 TTL,或反过来互相清除。 */
@Test
void inFlightAndCompletedMarksUseDifferentKeys() {
when(valueOperations.setIfAbsent(anyString(), anyString(), any(Duration.class))).thenReturn(true);
cache.beginRowInspection(TASK_ID, ROW_KEY);
cache.completeRowInspection(TASK_ID, List.of(ROW_KEY));
ArgumentCaptor<String> inFlight = ArgumentCaptor.forClass(String.class);
verify(valueOperations).setIfAbsent(inFlight.capture(), anyString(), any(Duration.class));
ArgumentCaptor<String> completed = ArgumentCaptor.forClass(String.class);
verify(valueOperations).set(completed.capture(), anyString(), any(Duration.class));
assertNotEquals(inFlight.getValue(), completed.getValue());
}
/** 发送前续期只延长处理中,绝不碰完成标记。 */
@Test
void touchRenewsOnlyInFlightMark() {
cache.touchRowInspection(TASK_ID, List.of(ROW_KEY));
verify(redisTemplate).expire(anyString(), any(Duration.class));
verify(valueOperations, never()).set(anyString(), anyString(), any(Duration.class));
}
/** Redis 降级时必须放行:持久化层的 hasResolvedLlmFields 才是最终判据,不能因为 Redis 挂了就整单停摆。 */
@Test
void beginAllowsRowThroughWhenRedisFails() {
when(redisTemplate.hasKey(anyString())).thenThrow(new IllegalStateException("redis down"));
assertTrue(cache.beginRowInspection(TASK_ID, ROW_KEY));
}
/**
* 检测失败要立刻撤销处理中标记,而不是等 TTL 过期。
*
* <p>assemble job 的 5 次重试是在几分钟内跑完的,若失败后仍占着 15 分钟的处理中名额,
* 这几行在重试窗口内永远申领不到——"能重检"就变成了"更快地彻底失败"。
*/
@Test
void releaseDeletesInFlightMarkWithoutCompleting() {
cache.releaseRowInspection(TASK_ID, List.of(ROW_KEY));
ArgumentCaptor<String> deleted = ArgumentCaptor.forClass(String.class);
verify(redisTemplate).delete(deleted.capture());
assertFalse(deleted.getValue().contains("processed-rows"), "撤销只能删处理中键,删完成标记会让已检测的行重跑并重复计费");
verify(valueOperations, never()).set(anyString(), anyString(), any(Duration.class));
}
}
@@ -209,8 +209,8 @@ class AppearancePatentTaskServiceDelegationTest {
try (XSSFWorkbook wb = new XSSFWorkbook(out)) {
Row data = wb.getSheet("外观专利检测结果").getRow(1);
assertEquals("", data.getCell(9).getStringCellValue(), "无结果行 LLM 列留空");
assertEquals("", data.getCell(12).getStringCellValue(), "无结果行状态留空");
assertEquals("未取得检测结果", data.getCell(9).getStringCellValue(), "无结果行判定列显式标注(空白与没风险无法区分)");
assertEquals("失败", data.getCell(12).getStringCellValue(), "无结果行状态判失败");
assertEquals("brand-A", data.getCell(4).getStringCellValue(), "品牌仍从解析行 values 回退");
assertEquals("", data.getCell(5).getStringCellValue(), "无结果行价格留空(resolvePrice 仅读结果行,与现状一致)");
}
@@ -367,7 +367,8 @@ class AppearancePatentTaskServiceDelegationTest {
row.setConclusion("");
assertEquals("LLM http 503: This model is currently overloaded",
AppearancePatentSheetBuilder.userFacingLlmCellValue(row, row.getTitleRisk()));
assertEquals("失败", AppearancePatentSheetBuilder.userFacingStatus(row),
assertEquals("失败", AppearancePatentSheetBuilder.resolveResultStatus(
AppearancePatentSheetBuilder.userFacingConclusion(row)),
"LLM 技术失败没得出判定 → 状态失败(旧口径按结论非空误判为成功)");
}
@@ -135,6 +135,12 @@ class AppearancePatentSheetBuilderTest {
}
}
/**
* 没有结果的行必须写明"没结果",不能留空。
*
* <p>覆盖率只要求过半(llm-coverage-min-ratio),所以少数缺失行照样交付;历史上这些格子是
* 空字符串,用户在表格里看到的"空白"和"没风险"长得一模一样——静默漏判。
*/
@Test
void test_sheet_builder_no_result_row_uses_parsed_fallback() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
@@ -145,8 +151,72 @@ class AppearancePatentSheetBuilderTest {
assertEquals("title-a", data.getCell(6).getStringCellValue(), "无结果行回退解析行标题");
assertEquals("http://img/a.jpg", data.getCell(7).getStringCellValue(), "回退解析行 URL");
assertEquals("SKU-1", data.getCell(8).getStringCellValue(), "回退解析行 sku");
assertEquals("", data.getCell(9).getStringCellValue(), "无结果行 LLM 列留空");
assertEquals("", data.getCell(12).getStringCellValue(), "无结果行状态留空");
assertEquals("未取得检测结果", data.getCell(9).getStringCellValue(), "无结果行标题维度显式标注");
assertEquals("未取得检测结果", data.getCell(10).getStringCellValue(), "无结果行外观维度显式标注");
assertEquals("未取得检测结果", data.getCell(11).getStringCellValue(), "无结果行结论显式标注");
assertEquals("失败", data.getCell(12).getStringCellValue(), "无结果行状态判失败,不能留空");
Row reason = wb.getSheet("原因").getRow(1);
assertEquals("B01A", reason.getCell(0).getStringCellValue());
assertTrue(reason.getCell(1).getStringCellValue().contains("未取得检测结果"),
"原因 sheet 也要说明这行为什么没有原因: " + reason.getCell(1).getStringCellValue());
}
}
/** 行存在但三个维度都没给出可用判定(客户端兜底成空串):同样要显式标注,且不能盖掉已有错误信息。 */
@Test
void test_sheet_builder_undeliverable_row_shows_missing_marker() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
AppearancePatentParsedRowVo p = parsedRow("2_1", "B01A", "US", "t", "", "");
AppearancePatentResultRowDto r = resultRow("2_1", "B01A", "US");
r.setTitleRisk("");
r.setAppearanceRisk("");
r.setConclusion("");
r.setError("LLM 网关无应答");
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of(
AppearancePatentSheetBuilder.rowKey(r), r));
Row data = wb.getSheet("外观专利检测结果").getRow(1);
assertEquals("LLM 网关无应答", data.getCell(9).getStringCellValue(), "有错误信息时展示错误信息");
assertEquals("失败", data.getCell(12).getStringCellValue(), "结论列没内容时状态仍判失败");
}
}
/** 结论被兜底成非技术性文案(如"外观识别异常")时也属于没结果,不能空着交付。 */
@Test
void test_sheet_builder_blank_verdict_row_is_marked() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
AppearancePatentParsedRowVo p = parsedRow("2_1", "B01A", "US", "t", "", "");
AppearancePatentResultRowDto r = resultRow("2_1", "B01A", "US");
r.setTitleRisk("");
r.setAppearanceRisk("");
r.setConclusion("");
r.setError("");
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of(
AppearancePatentSheetBuilder.rowKey(r), r));
Row data = wb.getSheet("外观专利检测结果").getRow(1);
assertEquals("未取得检测结果", data.getCell(11).getStringCellValue(), "空白结论改写成显式标注");
assertEquals("失败", data.getCell(12).getStringCellValue());
}
}
/** 有完整判定的行不受影响:显式标注只能出现在真的没结果的行上。 */
@Test
void test_sheet_builder_resolved_row_keeps_verdicts() throws Exception {
try (Workbook wb = new XSSFWorkbook()) {
AppearancePatentParsedRowVo p = parsedRow("2_1", "B01A", "US", "t", "", "");
AppearancePatentResultRowDto r = resultRow("2_1", "B01A", "US");
AppearancePatentSheetBuilder.buildResultSheet(wb, List.of(p), Map.of(
AppearancePatentSheetBuilder.rowKey(r), r));
Row data = wb.getSheet("外观专利检测结果").getRow(1);
assertEquals("无风险", data.getCell(9).getStringCellValue());
assertEquals("无风险", data.getCell(10).getStringCellValue());
assertEquals("已侵权", data.getCell(11).getStringCellValue());
assertEquals("成功", data.getCell(12).getStringCellValue());
assertEquals("", wb.getSheet("原因").getRow(1).getCell(1).getStringCellValue(),
"有结果的行不写缺失说明");
}
}
@@ -0,0 +1,436 @@
package com.nanri.aiimage.modules.dedupe.service;
import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeSourceFileDto;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunProgressVo;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.test.util.ReflectionTestUtils;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 去重任务的进度可见性(⑤)。
*
* <p>去重是服务端内存态任务:runId 是随机 UUID、进度只存在提交它的那个进程的 Map 里。
* 生产是双实例 + 无粘滞保证,轮询落到另一实例就返回 not_found,前端直接报
* 「去重任务不存在或已过期」—— 文件其实已经处理完并进了历史表。发版重启同理。
*
* <p>本测试锁定三件事:runId 必须能解析到落库的任务行;跨节点/重启后进度要能从
* 共享状态或数据库重建;重建出来的状态必须是真话(含已完成文件的下载地址)。
*/
@ExtendWith(MockitoExtension.class)
class DedupeRunProgressResumeTest {
private static final String PROGRESS_KEY_PREFIX = "aiimage:dedupe:run:progress:";
private static final String RUN_TASK_NO = "DEDUPE-1234567890";
@Mock
private FileTaskMapper fileTaskMapper;
@Mock
private FileResultMapper fileResultMapper;
@Mock
private StorageProperties storageProperties;
@Mock
private OssStorageService ossStorageService;
@Mock
private DedupeTotalDataService dedupeTotalDataService;
@Mock
private StringRedisTemplate redisTemplate;
@Mock
private ValueOperations<String, String> valueOperations;
private final ObjectMapper objectMapper = new ObjectMapper();
/** 模拟 Redis:两个 service 实例共用它就是"另一个节点"。 */
private final Map<String, String> remoteState = new HashMap<>();
/** 脱离 Spring 时 MyBatis-Plus 没有 lambda 列缓存,Lambda 条件构造器要先初始化表信息。 */
@org.junit.jupiter.api.BeforeAll
static void initTableInfo() {
com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo(
new org.apache.ibatis.builder.MapperBuilderAssistant(
new com.baomidou.mybatisplus.core.MybatisConfiguration(), ""),
FileTaskEntity.class);
com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo(
new org.apache.ibatis.builder.MapperBuilderAssistant(
new com.baomidou.mybatisplus.core.MybatisConfiguration(), ""),
FileResultEntity.class);
}
private DedupeRunService newService() {
lenient().when(redisTemplate.opsForValue()).thenReturn(valueOperations);
lenient().when(valueOperations.get(anyString())).thenAnswer(invocation -> remoteState.get(invocation.getArgument(0)));
lenient().doAnswer(invocation -> {
remoteState.put(invocation.getArgument(0), invocation.getArgument(1));
return null;
}).when(valueOperations).set(anyString(), anyString(), any(Duration.class));
lenient().when(redisTemplate.delete(anyString())).thenAnswer(invocation -> {
remoteState.remove(invocation.getArgument(0, String.class));
return true;
});
return new DedupeRunService(fileTaskMapper, fileResultMapper, storageProperties,
ossStorageService, dedupeTotalDataService, redisTemplate, objectMapper,
new InstanceMetadata("server-110"));
}
private DedupeRunRequest request(int fileCount) {
DedupeRunRequest request = new DedupeRunRequest();
request.setUserId(9L);
request.setSelectedColumns(List.of("id", "ASIN"));
request.setKeepUnderscoreIds(true);
List<DedupeSourceFileDto> files = new java.util.ArrayList<>();
for (int i = 0; i < fileCount; i++) {
DedupeSourceFileDto dto = new DedupeSourceFileDto();
dto.setFileKey("key-" + i);
dto.setOriginalFilename("file-" + i + ".xlsx");
files.add(dto);
}
request.setFiles(files);
return request;
}
private FileTaskEntity task(String status, String errorMessage) {
FileTaskEntity task = new FileTaskEntity();
task.setId(7L);
task.setTaskNo(RUN_TASK_NO);
task.setModuleType("DEDUPE");
task.setStatus(status);
task.setSourceFileCount(3);
task.setErrorMessage(errorMessage);
task.setCreatedAt(LocalDateTime.of(2026, 9, 27, 10, 0));
return task;
}
private FileResultEntity result(Long id, String sourceName, String objectKey, boolean success, String error) {
FileResultEntity entity = new FileResultEntity();
entity.setId(id);
entity.setTaskId(7L);
entity.setModuleType("DEDUPE");
entity.setSourceFilename(sourceName);
entity.setResultFilename(objectKey == null ? null : sourceName + "_cleaned.xlsx");
entity.setResultFileUrl(objectKey);
entity.setResultFileSize(objectKey == null ? null : 1024L);
entity.setSuccess(success ? 1 : 0);
entity.setErrorMessage(error);
entity.setCreatedAt(LocalDateTime.of(2026, 9, 27, 10, 5));
return entity;
}
// ---------- runId 必须可解析 ----------
@Test
@DisplayName("runId 用落库的 task_no,并把归属实例一起写进任务行")
void runIdIsThePersistedTaskNoAndOwnerIsRecorded() {
DedupeRunService service = newService();
// 异步 worker 是否来得及读这个目录取决于线程时序,只做兜底、不参与断言
lenient().when(storageProperties.getLocalTempDir()).thenReturn("D:/definitely-not-a-real-dedupe-dir/xx");
DedupeRunProgressVo submitted = service.submitRun(request(1));
ArgumentCaptor<FileTaskEntity> captor = ArgumentCaptor.forClass(FileTaskEntity.class);
verify(fileTaskMapper, atLeastOnce()).insert(captor.capture());
FileTaskEntity persisted = captor.getValue();
assertEquals(persisted.getTaskNo(), submitted.getRunId(),
"runId 必须是已落库且唯一的 task_no,否则重启/换节点后无法解析到任务行");
assertEquals("server-110", persisted.getOwnerInstanceId(),
"归属实例要落库:开机回收只能收自己名下中断的任务,否则会把另一实例正在跑的任务判死");
}
// ---------- 跨节点:共享状态 ----------
@Test
@DisplayName("另一节点写入的进度快照要能直接读到,不再报 not_found")
void progressWrittenByAnotherNodeIsReadable() throws Exception {
DedupeRunProgressVo remote = new DedupeRunProgressVo();
remote.setRunId(RUN_TASK_NO);
remote.setStatus("success");
remote.setTotal(3);
remote.setProcessedCount(3);
remote.setSuccessCount(2);
remote.setFailedCount(1);
remote.setFinished(true);
remote.setResult(new com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunVo());
remote.getResult().setItems(List.of());
remoteState.put(PROGRESS_KEY_PREFIX + RUN_TASK_NO, objectMapper.writeValueAsString(remote));
DedupeRunProgressVo progress = newService().getProgress(RUN_TASK_NO);
assertEquals("success", progress.getStatus(), "本节点内存没有这条 run,但共享状态有 → 不能返回 not_found");
assertTrue(progress.isFinished());
assertEquals(2, progress.getSuccessCount());
}
@Test
@DisplayName("共享状态说 running 而库里已终态时,以库为准并清掉过期快照")
void terminalDatabaseStateOvertakesStaleRunningSnapshot() throws Exception {
DedupeRunProgressVo stale = new DedupeRunProgressVo();
stale.setRunId(RUN_TASK_NO);
stale.setStatus("running");
stale.setTotal(3);
stale.setProcessedCount(1);
stale.setSuccessCount(1);
stale.setFailedCount(0);
stale.setFinished(false);
remoteState.put(PROGRESS_KEY_PREFIX + RUN_TASK_NO, objectMapper.writeValueAsString(stale));
when(fileTaskMapper.selectOne(any())).thenReturn(task("SUCCESS", null));
when(fileResultMapper.selectList(any())).thenReturn(List.of(
result(1L, "a.xlsx", "dedupe/a.xlsx", true, null),
result(2L, "b.xlsx", "dedupe/b.xlsx", true, null),
result(3L, "c.xlsx", null, false, "表头缺少所选列")));
when(ossStorageService.generateFreshDownloadUrl(anyString())).thenReturn("https://dl/a");
DedupeRunProgressVo progress = newService().getProgress(RUN_TASK_NO);
assertEquals("success", progress.getStatus(), "库已 SUCCESS,快照还 running 说明快照过期(提交节点已重启/被回收)");
assertFalse(remoteState.containsKey(PROGRESS_KEY_PREFIX + RUN_TASK_NO),
"过期快照要删掉,否则下一轮还是读到旧状态");
}
// ---------- 重启/过期后:从库重建 ----------
@Test
@DisplayName("进度快照丢失后用任务行与逐文件结果行重建,已完成文件仍可下载")
void progressIsRebuiltFromDatabaseWhenSharedStateExpired() {
DedupeRunService service = newService();
when(fileTaskMapper.selectOne(any())).thenReturn(task("SUCCESS", null));
when(fileResultMapper.selectList(any())).thenReturn(List.of(
result(1L, "a.xlsx", "dedupe/a.xlsx", true, null),
result(2L, "b.xlsx", "dedupe/b.xlsx", true, null),
result(3L, "c.xlsx", null, false, "表头缺少所选列")));
when(ossStorageService.generateFreshDownloadUrl("dedupe/a.xlsx")).thenReturn("https://dl/a.xlsx");
when(ossStorageService.generateFreshDownloadUrl("dedupe/b.xlsx")).thenReturn("https://dl/b.xlsx");
DedupeRunProgressVo progress = service.getProgress(RUN_TASK_NO);
assertEquals("success", progress.getStatus());
assertEquals(RUN_TASK_NO, progress.getRunId());
assertTrue(progress.isFinished());
assertEquals(3, progress.getTotal());
assertEquals(3, progress.getProcessedCount(), "processed 必须等于已落库结果行数");
assertEquals(2, progress.getSuccessCount());
assertEquals(1, progress.getFailedCount());
assertNotNull(progress.getResult());
assertEquals(3, progress.getResult().getItems().size(), "结果行就是交付物,重建后必须还能拿到 3 个文件的处置");
assertEquals("https://dl/a.xlsx", progress.getResult().getItems().get(0).getDownloadUrl());
assertFalse(progress.getResult().getItems().get(2).isSuccess());
assertEquals("表头缺少所选列", progress.getResult().getItems().get(2).getError());
}
@Test
@DisplayName("库里仍在跑:按已落库结果行报真实进度,而不是 not_found")
void runningTaskWithoutSharedStateReportsRealCounts() {
DedupeRunService service = newService();
when(fileTaskMapper.selectOne(any())).thenReturn(task("RUNNING", null));
when(fileResultMapper.selectList(any())).thenReturn(List.of(
result(1L, "a.xlsx", "dedupe/a.xlsx", true, null)));
DedupeRunProgressVo progress = service.getProgress(RUN_TASK_NO);
assertEquals("running", progress.getStatus());
assertFalse(progress.isFinished());
assertEquals(1, progress.getProcessedCount());
assertEquals(3, progress.getTotal());
}
@Test
@DisplayName("库里已失败:回写落库的失败原因")
void failedTaskWithoutSharedStateReportsItsStoredError() {
DedupeRunService service = newService();
when(fileTaskMapper.selectOne(any())).thenReturn(task("FAILED", "上传文件不存在,请重新上传"));
when(fileResultMapper.selectList(any())).thenReturn(List.of(
result(1L, "a.xlsx", null, false, "上传文件不存在,请重新上传")));
DedupeRunProgressVo progress = service.getProgress(RUN_TASK_NO);
assertEquals("failed", progress.getStatus());
assertTrue(progress.isFinished());
assertEquals("上传文件不存在,请重新上传", progress.getError());
assertEquals(1, progress.getFailedCount());
}
@Test
@DisplayName("库里也没有这条 run 时,仍然如实返回 not_found")
void unknownRunIdStillReportsNotFound() {
DedupeRunService service = newService();
when(fileTaskMapper.selectOne(any())).thenReturn(null);
DedupeRunProgressVo progress = service.getProgress("DEDUPE-999");
assertEquals("not_found", progress.getStatus());
assertEquals("任务不存在或已过期", progress.getError());
}
@Test
@DisplayName("重建只查 DEDUPE 模块的行,避免同名 task_no 串到别的模块")
void reconstructionIsScopedToDedupeModule() {
DedupeRunService service = newService();
when(fileTaskMapper.selectOne(any())).thenReturn(null);
service.getProgress(RUN_TASK_NO);
ArgumentCaptor<Wrapper<FileTaskEntity>> captor = ArgumentCaptor.forClass(Wrapper.class);
verify(fileTaskMapper).selectOne(captor.capture());
String sql = captor.getValue().getSqlSegment();
assertTrue(sql.contains("module_type"), "按 runId 反查必须限定 module_type:" + sql);
assertTrue(sql.contains("task_no"), "按已落库的 uk_task_no 唯一索引反查:" + sql);
}
// ---------- 发版重启中断的在跑任务 ----------
@Test
@DisplayName("开机把本机遗留的 RUNNING 去重任务判失败,不再空挂 2 小时")
void bootReclaimFailsOwnInterruptedRuns() {
DedupeRunService service = newService();
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task("RUNNING", null)));
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
service.reclaimInterruptedRunsOnBoot();
ArgumentCaptor<LambdaUpdateWrapper<FileTaskEntity>> captor =
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
verify(fileTaskMapper).update(isNull(), captor.capture());
LambdaUpdateWrapper<FileTaskEntity> update = captor.getValue();
assertTrue(update.getSqlSet().contains("status="), "要写终态:" + update.getSqlSet());
assertTrue(update.getSqlSet().contains("error_message="), "要写中断原因:" + update.getSqlSet());
Map<String, Object> params = paramValues(update);
assertTrue(params.containsValue("FAILED"), "本机刚起来,自己名下的 RUNNING 必然是中断残留:" + params);
assertTrue(params.values().stream().anyMatch(v -> v instanceof String && ((String) v).contains("重启")),
"原因要点明是服务重启,用户才知道要重新提交:" + params);
}
@Test
@DisplayName("开机回收只查本机归属的去重任务")
void bootReclaimOnlyTargetsOwnDedupeRuns() {
DedupeRunService service = newService();
when(fileTaskMapper.selectList(any())).thenReturn(List.of());
service.reclaimInterruptedRunsOnBoot();
ArgumentCaptor<Wrapper<FileTaskEntity>> captor = ArgumentCaptor.forClass(Wrapper.class);
verify(fileTaskMapper).selectList(captor.capture());
String sql = captor.getValue().getSqlSegment();
Map<String, Object> params = paramValues(captor.getValue());
assertTrue(sql.contains("owner_instance_id"), "必须按归属实例过滤,否则会把另一实例正在跑的任务判死:" + sql);
assertTrue(sql.contains("module_type"), "只管去重模块:" + sql);
assertTrue(params.containsValue("server-110"), "查的是本机自己:" + params);
verify(fileTaskMapper, org.mockito.Mockito.never()).update(isNull(), any());
}
@Test
@DisplayName("开机回收挂在 ApplicationReadyEvent 上,且异常不能阻断启动")
void bootReclaimIsAnnotatedAndCannotBreakStartup() {
DedupeRunService service = newService();
org.springframework.context.event.EventListener annotation = findListener();
assertTrue(java.util.Arrays.asList(annotation.value()).contains(
org.springframework.boot.context.event.ApplicationReadyEvent.class),
"要在应用就绪后执行,太早查库会连不上数据源");
when(fileTaskMapper.selectList(any())).thenThrow(new RuntimeException("db down"));
service.reclaimInterruptedRunsOnBoot();
}
// ---------- 心跳:别让 2 小时判死误杀真活任务 ----------
@Test
@DisplayName("每个文件处理完刷新任务行心跳,使巡检的 2 小时判死不误杀长任务")
void heartbeatTouchesOnlyUpdatedAtOfTheTaskRow() {
DedupeRunService service = newService();
service.touchRunHeartbeat(7L);
ArgumentCaptor<LambdaUpdateWrapper<FileTaskEntity>> captor =
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
verify(fileTaskMapper).update(isNull(), captor.capture());
LambdaUpdateWrapper<FileTaskEntity> update = captor.getValue();
assertTrue(update.getSqlSet().contains("updated_at"), "心跳就是刷 updated_at:" + update.getSqlSet());
assertFalse(update.getSqlSet().contains("status="), "心跳不许改状态:" + update.getSqlSet());
assertTrue(update.getSqlSegment().contains("id"), "按主键定位这一条任务:" + update.getSqlSegment());
assertTrue(paramValues(update).containsValue(7L), "更新的是本任务:" + paramValues(update));
}
@Test
@DisplayName("taskId 缺失时心跳静默跳过(异步线程里没有任务 id 不能抛)")
void heartbeatIgnoresMissingTaskId() {
DedupeRunService service = newService();
service.touchRunHeartbeat(null);
verify(fileTaskMapper, org.mockito.Mockito.never()).update(isNull(), any());
}
// ---------- helpers ----------
private org.springframework.context.event.EventListener findListener() {
for (java.lang.reflect.Method method : DedupeRunService.class.getDeclaredMethods()) {
if ("reclaimInterruptedRunsOnBoot".equals(method.getName())) {
org.springframework.context.event.EventListener annotation =
method.getAnnotation(org.springframework.context.event.EventListener.class);
assertNotNull(annotation, "开机回收必须由事件触发,不能靠 @PostConstruct 抢在数据源就绪前");
return annotation;
}
}
throw new AssertionError("缺少 reclaimInterruptedRunsOnBoot 方法");
}
/** LambdaQueryWrapper/UpdateWrapper 的参数表要等 SQL 片段渲染后才会填充。 */
private Map<String, Object> paramValues(Object wrapper) {
if (wrapper instanceof com.baomidou.mybatisplus.core.conditions.AbstractWrapper<?, ?, ?> query) {
if (query instanceof LambdaUpdateWrapper<?> update) {
update.getSqlSet();
}
query.getCustomSqlSegment();
return new HashMap<>(query.getParamNameValuePairs());
}
throw new IllegalStateException("不是 MyBatis-Plus wrapper: " + wrapper);
}
@Test
@DisplayName("共享状态写失败不影响提交(Redis 挂了退化为本地内存,与原行为一致)")
void submitStillWorksWhenSharedStateWriteFails() {
when(redisTemplate.opsForValue()).thenThrow(new RuntimeException("redis down"))
.thenReturn(valueOperations);
// 异步 worker 是否来得及读这个目录取决于线程时序,只做兜底、不参与断言
lenient().when(storageProperties.getLocalTempDir()).thenReturn("D:/definitely-not-a-real-dedupe-dir/xx");
DedupeRunService service = new DedupeRunService(fileTaskMapper, fileResultMapper, storageProperties,
ossStorageService, dedupeTotalDataService, redisTemplate, objectMapper,
new InstanceMetadata("server-110"));
DedupeRunProgressVo submitted = service.submitRun(request(1));
assertNotNull(submitted.getRunId());
assertEquals("running", submitted.getStatus());
}
}
@@ -150,7 +150,8 @@ class DedupeRunServiceCleanRulesTest {
writeInput(input, rows);
DedupeRunService service = new DedupeRunService(
fileTaskMapper, fileResultMapper, storageProperties, ossStorageService, dedupeTotalDataService);
fileTaskMapper, fileResultMapper, storageProperties, ossStorageService, dedupeTotalDataService,
null, null, new com.nanri.aiimage.config.InstanceMetadata("test-instance"));
when(dedupeTotalDataService.findExistingComparableValues(any())).thenReturn(Set.of());
when(dedupeTotalDataService.normalizeComparableValueOrBlank(any()))
.thenAnswer(invocation -> {
@@ -0,0 +1,182 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.context.ApplicationEventPublisher;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Objects;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 开机自回收:本机 owner 名下 RUNNING 且已停止心跳的作业,在进程起来时立刻退回 PENDING。
*
* <p>发版/崩溃杀掉的 job 会一直挂在 RUNNING,等 60 秒一轮的 stuck 扫描按时间判死才能复位
* (阈值历史上是 30 分钟)。也就是说每次发版都会把在跑的结果文件作业冻结掉一整个阈值窗口,
* 而这批 job 其实早已无人持有。开机回收把这个等待从「阈值」压到「进程起来那一刻」。
*
* <p>与 stuck 扫描的关键区别:**不累加 retry_count**。发版不是这个作业的错,
* 每发一次版扣掉它 5 次重试中的一次,反复发版会把正常任务判死。
*/
@ExtendWith(MockitoExtension.class)
class TaskFileJobBootReclaimTest {
private static final String OWNER = "java-server";
@Mock private TaskFileJobMapper taskFileJobMapper;
@Mock private ApplicationEventPublisher applicationEventPublisher;
private TaskFileJobService service;
@BeforeAll
static void initializeTableInfo() {
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
TaskFileJobEntity.class);
}
@BeforeEach
void setUp() {
service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
}
private static TaskFileJobEntity abandonedJob(Long id) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(id);
job.setTaskId(1000L + id);
job.setResultId(2000L + id);
job.setModuleType("APPEARANCE_PATENT");
job.setScopeKey("task:" + id + ":owner:" + OWNER);
job.setOwner(OWNER);
job.setJobType("ASSEMBLE_RESULT");
job.setStatus("RUNNING");
job.setRetryCount(1);
job.setUpdatedAt(LocalDateTime.now().minusMinutes(3));
return job;
}
/** 心跳已停的自家 RUNNING 作业退回 PENDING 并补发派发事件,且不得消耗 retry_count。 */
@Test
void staleOwnedRunningJobsGoBackToPendingWithoutBurningRetry() {
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class)))
.thenReturn(List.of(abandonedJob(1L), abandonedJob(2L)));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1, 1);
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(abandonedJob(1L), abandonedJob(2L)));
int reclaimed = service.reclaimOwnedRunningJobs(OWNER, 90, 20);
assertEquals(2, reclaimed);
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> updates =
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
verify(taskFileJobMapper, times(2)).update(isNull(), updates.capture());
for (LambdaUpdateWrapper<TaskFileJobEntity> update : updates.getAllValues()) {
String sqlSet = update.getSqlSet();
assertTrue(sqlSet.contains("status="), "必须把状态改回 PENDING: " + sqlSet);
assertFalse(sqlSet.contains("retry_count"), "开机回收不得消耗重试次数: " + sqlSet);
}
verify(applicationEventPublisher, times(2)).publishEvent(any(TaskFileJobDispatchEvent.class));
}
/** 只回收「心跳已停」的:阈值必须是 now-staleSeconds,否则会抢另一个仍在跑同 owner 的进程的作业。 */
@Test
void onlyJobsStalerThanTheThresholdAreCandidates() {
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
LocalDateTime before = LocalDateTime.now();
service.reclaimOwnedRunningJobs(OWNER, 90, 20);
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
verify(taskFileJobMapper).selectList(query.capture());
String segment = query.getValue().getCustomSqlSegment();
assertTrue(segment.contains("owner = "), "必须只回收本机 owner: " + segment);
assertTrue(segment.contains("status"), "必须只回收 RUNNING: " + segment);
assertTrue(segment.contains("updated_at"), "必须按心跳时间设阈值: " + segment);
LocalDateTime cutoff = query.getValue().getParamNameValuePairs().values().stream()
.filter(v -> v instanceof LocalDateTime)
.map(v -> (LocalDateTime) v)
.findFirst()
.orElse(null);
assertTrue(cutoff != null && !cutoff.isAfter(before.minusSeconds(90)),
"阈值不得晚于 now-90s,否则会抢仍在心跳的作业: " + cutoff);
}
/** 没有 owner 就没有"本机"可言:不查库、不改写,交给通用 stuck 扫描。 */
@Test
void blankOwnerReclaimsNothing() {
assertEquals(0, service.reclaimOwnedRunningJobs(" ", 90, 20));
assertEquals(0, service.reclaimOwnedRunningJobs(null, 90, 20));
verify(taskFileJobMapper, times(0)).selectList(any(LambdaQueryWrapper.class));
verify(taskFileJobMapper, times(0)).update(isNull(), any(LambdaUpdateWrapper.class));
}
/** CAS 没命中(已被其它路径复位)时不算回收成功,也不发派发事件。 */
@Test
void lostCasNeitherCountsNorDispatches() {
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(abandonedJob(3L)));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
assertEquals(0, service.reclaimOwnedRunningJobs(OWNER, 90, 20));
verify(applicationEventPublisher, times(0)).publishEvent(any(TaskFileJobDispatchEvent.class));
}
/** 开机一次取太多会长时间持锁:limit 沿用既有夹紧。 */
@Test
void limitIsClamped() {
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
service.reclaimOwnedRunningJobs(OWNER, 90, 0);
service.reclaimOwnedRunningJobs(OWNER, 90, 5000);
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
verify(taskFileJobMapper, times(2)).selectList(query.capture());
List<String> segments = query.getAllValues().stream()
.map(LambdaQueryWrapper::getCustomSqlSegment).toList();
assertTrue(segments.get(0).contains("limit 1"), "limit=0 回退到 1: " + segments.get(0));
assertTrue(segments.get(1).contains("limit 100"), "超大 limit 钳制到 100: " + segments.get(1));
}
/** staleSeconds 给得太小时取不到任何作业:至少留出一个心跳周期的余量。 */
@Test
void staleThresholdNeverBelowHeartbeatInterval() {
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
service.reclaimOwnedRunningJobs(OWNER, 1, 20);
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
verify(taskFileJobMapper).selectList(query.capture());
// 参数表要先把 SQL 段渲染出来才落值(MyBatis-Plus 惰性填充),否则读到空 map
query.getValue().getCustomSqlSegment();
LocalDateTime cutoff = query.getValue().getParamNameValuePairs().values().stream()
.filter(Objects::nonNull)
.filter(v -> v instanceof LocalDateTime)
.map(v -> (LocalDateTime) v)
.findFirst()
.orElse(null);
assertTrue(cutoff != null && cutoff.isBefore(LocalDateTime.now().minusSeconds(30)),
"阈值过短会把仍在正常心跳的作业抢走: " + cutoff);
}
}
@@ -30,6 +30,8 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
import org.junit.jupiter.api.Test;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import java.lang.reflect.Field;
import java.nio.charset.StandardCharsets;
@@ -39,8 +41,11 @@ import java.nio.file.Paths;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -269,8 +274,13 @@ class TaskResultFileJobWorkerStuckScanTest {
Path source = Paths.get("src", "main", "java", "com", "nanri", "aiimage", "modules",
"task", "service", "TaskResultFileJobWorker.java");
String content = new String(java.nio.file.Files.readAllBytes(source), java.nio.charset.StandardCharsets.UTF_8);
assertTrue(content.contains("stuck-timeout-minutes:30"), "@Value 默认卡死超时 30 分钟");
assertTrue(content.contains("stuck-timeout-minutes:5"), "@Value 默认卡死超时 5 分钟");
assertTrue(content.contains("batch-size:20"), "@Value 默认批次 20");
Path yml = Paths.get("src", "main", "resources", "application.yml");
String config = new String(Files.readAllBytes(yml), StandardCharsets.UTF_8);
// 运行期真正生效的是 yml 的兜底值,只改 @Value 默认等于没改
assertTrue(config.contains("STUCK_TIMEOUT_MINUTES:5}"), "application.yml 卡死超时 5 分钟: " + config);
assertTrue(config.contains("BOOT_RECLAIM_STALE_SECONDS:90}"), "开机回收阈值可在 env 覆盖");
}
@Test
@@ -288,6 +298,52 @@ class TaskResultFileJobWorkerStuckScanTest {
verify(taskFileJobService).markFailureFinalized(9L, "文件生成任务运行超时,已达到最大重试次数");
}
/** 开机回收必须只针对"本机 instanceId 名下"的遗留作业,否则会抢另一台实例正在跑的 job。 */
@Test
void bootReclaimTargetsOwnAbandonedJobs() throws Exception {
TaskResultFileJobWorker worker = buildWorker();
when(instanceMetadata.getInstanceId()).thenReturn("java-server");
setInt(worker, "batchSize", 20);
setLong(worker, "bootReclaimStaleSeconds", 90L);
worker.reclaimOwnedJobsOnBoot();
verify(taskFileJobService).reclaimOwnedRunningJobs("java-server", 90L, 20);
}
/** 回收只是加速恢复,DB 抽风时绝不能把应用启动一起带下水。 */
@Test
void bootReclaimFailureDoesNotBreakStartup() throws Exception {
TaskResultFileJobWorker worker = buildWorker();
when(instanceMetadata.getInstanceId()).thenReturn("java-server");
when(taskFileJobService.reclaimOwnedRunningJobs(anyString(), anyLong(), anyInt()))
.thenThrow(new IllegalStateException("db down"));
worker.reclaimOwnedJobsOnBoot();
verify(taskFileJobService).reclaimOwnedRunningJobs("java-server", 0L, 0);
}
/**
* 必须挂在 ApplicationReadyEvent 上:早了(Bean 还没齐)会 NPE,晚了不如挂事件;
* 挂在 ContextRefreshed 之类的事件上则会随上下文刷新重复执行。
*/
@Test
void bootReclaimRunsOnApplicationReady() throws Exception {
EventListener listener = TaskResultFileJobWorker.class
.getMethod("reclaimOwnedJobsOnBoot")
.getAnnotation(EventListener.class);
assertNotNull(listener, "开机回收必须由事件驱动,否则没人调用");
assertTrue(List.of(listener.value()).contains(ApplicationReadyEvent.class),
"只在应用就绪后执行: " + List.of(listener.value()));
}
private static void setLong(TaskResultFileJobWorker worker, String fieldName, long value) throws Exception {
Field field = TaskResultFileJobWorker.class.getDeclaredField(fieldName);
field.setAccessible(true);
field.setLong(worker, value);
}
private static void setInt(TaskResultFileJobWorker worker, String fieldName, int value) throws Exception {
Field field = TaskResultFileJobWorker.class.getDeclaredField(fieldName);
field.setAccessible(true);