fix(断点续传): 外观/结果作业/去重三处中断语义补齐,重启不再静默丢行
- 外观专利行标改两态(in-flight 15min / processed 24h):只有 CAS 写回 payload 成功 才转「已处理」,异常、覆盖率不足、CAS 打满等未落库出口一律撤销标记 —— 修「发 LLM 请求前就盖章,进程被杀的那一批永不再补检」 - 结果文件作业开机自回收:按 owner_instance_id 只收本机名下 RUNNING 且心跳超 90s 的行; 卡死阈值 30 → 5 分钟,重启接回从最长半小时压到 5 分钟 - 外观结果表缺检行显式写「未取得检测结果」并判失败,不再留空靠 0.5 覆盖率阈值蒙过 - 去重进度改 NodeSharedStore 跨节点共享,runId 用已落库的 task_no:轮询打到另一节点或 发版重启后仍能读到真实进度、已完成文件仍可下载;共享态缺失时按 task_no 从库重建; 逐文件心跳防 StaleTaskRepair 把长任务误判死;开机把本机残留 RUNNING 判失败并给提示
This commit is contained in:
+182
@@ -0,0 +1,182 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 开机自回收:本机 owner 名下 RUNNING 且已停止心跳的作业,在进程起来时立刻退回 PENDING。
|
||||
*
|
||||
* <p>发版/崩溃杀掉的 job 会一直挂在 RUNNING,等 60 秒一轮的 stuck 扫描按时间判死才能复位
|
||||
* (阈值历史上是 30 分钟)。也就是说每次发版都会把在跑的结果文件作业冻结掉一整个阈值窗口,
|
||||
* 而这批 job 其实早已无人持有。开机回收把这个等待从「阈值」压到「进程起来那一刻」。
|
||||
*
|
||||
* <p>与 stuck 扫描的关键区别:**不累加 retry_count**。发版不是这个作业的错,
|
||||
* 每发一次版扣掉它 5 次重试中的一次,反复发版会把正常任务判死。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class TaskFileJobBootReclaimTest {
|
||||
|
||||
private static final String OWNER = "java-server";
|
||||
|
||||
@Mock private TaskFileJobMapper taskFileJobMapper;
|
||||
@Mock private ApplicationEventPublisher applicationEventPublisher;
|
||||
|
||||
private TaskFileJobService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeTableInfo() {
|
||||
TableInfoHelper.initTableInfo(
|
||||
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
TaskFileJobEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
}
|
||||
|
||||
private static TaskFileJobEntity abandonedJob(Long id) {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(id);
|
||||
job.setTaskId(1000L + id);
|
||||
job.setResultId(2000L + id);
|
||||
job.setModuleType("APPEARANCE_PATENT");
|
||||
job.setScopeKey("task:" + id + ":owner:" + OWNER);
|
||||
job.setOwner(OWNER);
|
||||
job.setJobType("ASSEMBLE_RESULT");
|
||||
job.setStatus("RUNNING");
|
||||
job.setRetryCount(1);
|
||||
job.setUpdatedAt(LocalDateTime.now().minusMinutes(3));
|
||||
return job;
|
||||
}
|
||||
|
||||
/** 心跳已停的自家 RUNNING 作业退回 PENDING 并补发派发事件,且不得消耗 retry_count。 */
|
||||
@Test
|
||||
void staleOwnedRunningJobsGoBackToPendingWithoutBurningRetry() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class)))
|
||||
.thenReturn(List.of(abandonedJob(1L), abandonedJob(2L)));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1, 1);
|
||||
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(abandonedJob(1L), abandonedJob(2L)));
|
||||
|
||||
int reclaimed = service.reclaimOwnedRunningJobs(OWNER, 90, 20);
|
||||
|
||||
assertEquals(2, reclaimed);
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> updates =
|
||||
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(taskFileJobMapper, times(2)).update(isNull(), updates.capture());
|
||||
for (LambdaUpdateWrapper<TaskFileJobEntity> update : updates.getAllValues()) {
|
||||
String sqlSet = update.getSqlSet();
|
||||
assertTrue(sqlSet.contains("status="), "必须把状态改回 PENDING: " + sqlSet);
|
||||
assertFalse(sqlSet.contains("retry_count"), "开机回收不得消耗重试次数: " + sqlSet);
|
||||
}
|
||||
verify(applicationEventPublisher, times(2)).publishEvent(any(TaskFileJobDispatchEvent.class));
|
||||
}
|
||||
|
||||
/** 只回收「心跳已停」的:阈值必须是 now-staleSeconds,否则会抢另一个仍在跑同 owner 的进程的作业。 */
|
||||
@Test
|
||||
void onlyJobsStalerThanTheThresholdAreCandidates() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
LocalDateTime before = LocalDateTime.now();
|
||||
|
||||
service.reclaimOwnedRunningJobs(OWNER, 90, 20);
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||
verify(taskFileJobMapper).selectList(query.capture());
|
||||
String segment = query.getValue().getCustomSqlSegment();
|
||||
assertTrue(segment.contains("owner = "), "必须只回收本机 owner: " + segment);
|
||||
assertTrue(segment.contains("status"), "必须只回收 RUNNING: " + segment);
|
||||
assertTrue(segment.contains("updated_at"), "必须按心跳时间设阈值: " + segment);
|
||||
LocalDateTime cutoff = query.getValue().getParamNameValuePairs().values().stream()
|
||||
.filter(v -> v instanceof LocalDateTime)
|
||||
.map(v -> (LocalDateTime) v)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
assertTrue(cutoff != null && !cutoff.isAfter(before.minusSeconds(90)),
|
||||
"阈值不得晚于 now-90s,否则会抢仍在心跳的作业: " + cutoff);
|
||||
}
|
||||
|
||||
/** 没有 owner 就没有"本机"可言:不查库、不改写,交给通用 stuck 扫描。 */
|
||||
@Test
|
||||
void blankOwnerReclaimsNothing() {
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(" ", 90, 20));
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(null, 90, 20));
|
||||
|
||||
verify(taskFileJobMapper, times(0)).selectList(any(LambdaQueryWrapper.class));
|
||||
verify(taskFileJobMapper, times(0)).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
}
|
||||
|
||||
/** CAS 没命中(已被其它路径复位)时不算回收成功,也不发派发事件。 */
|
||||
@Test
|
||||
void lostCasNeitherCountsNorDispatches() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(abandonedJob(3L)));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
|
||||
|
||||
assertEquals(0, service.reclaimOwnedRunningJobs(OWNER, 90, 20));
|
||||
|
||||
verify(applicationEventPublisher, times(0)).publishEvent(any(TaskFileJobDispatchEvent.class));
|
||||
}
|
||||
|
||||
/** 开机一次取太多会长时间持锁:limit 沿用既有夹紧。 */
|
||||
@Test
|
||||
void limitIsClamped() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
|
||||
service.reclaimOwnedRunningJobs(OWNER, 90, 0);
|
||||
service.reclaimOwnedRunningJobs(OWNER, 90, 5000);
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||
verify(taskFileJobMapper, times(2)).selectList(query.capture());
|
||||
List<String> segments = query.getAllValues().stream()
|
||||
.map(LambdaQueryWrapper::getCustomSqlSegment).toList();
|
||||
assertTrue(segments.get(0).contains("limit 1"), "limit=0 回退到 1: " + segments.get(0));
|
||||
assertTrue(segments.get(1).contains("limit 100"), "超大 limit 钳制到 100: " + segments.get(1));
|
||||
}
|
||||
|
||||
/** staleSeconds 给得太小时取不到任何作业:至少留出一个心跳周期的余量。 */
|
||||
@Test
|
||||
void staleThresholdNeverBelowHeartbeatInterval() {
|
||||
when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of());
|
||||
|
||||
service.reclaimOwnedRunningJobs(OWNER, 1, 20);
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<TaskFileJobEntity>> query = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||
verify(taskFileJobMapper).selectList(query.capture());
|
||||
// 参数表要先把 SQL 段渲染出来才落值(MyBatis-Plus 惰性填充),否则读到空 map
|
||||
query.getValue().getCustomSqlSegment();
|
||||
LocalDateTime cutoff = query.getValue().getParamNameValuePairs().values().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(v -> v instanceof LocalDateTime)
|
||||
.map(v -> (LocalDateTime) v)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
assertTrue(cutoff != null && cutoff.isBefore(LocalDateTime.now().minusSeconds(30)),
|
||||
"阈值过短会把仍在正常心跳的作业抢走: " + cutoff);
|
||||
}
|
||||
}
|
||||
+57
-1
@@ -30,6 +30,8 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -39,8 +41,11 @@ import java.nio.file.Paths;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -269,8 +274,13 @@ class TaskResultFileJobWorkerStuckScanTest {
|
||||
Path source = Paths.get("src", "main", "java", "com", "nanri", "aiimage", "modules",
|
||||
"task", "service", "TaskResultFileJobWorker.java");
|
||||
String content = new String(java.nio.file.Files.readAllBytes(source), java.nio.charset.StandardCharsets.UTF_8);
|
||||
assertTrue(content.contains("stuck-timeout-minutes:30"), "@Value 默认卡死超时 30 分钟");
|
||||
assertTrue(content.contains("stuck-timeout-minutes:5"), "@Value 默认卡死超时 5 分钟");
|
||||
assertTrue(content.contains("batch-size:20"), "@Value 默认批次 20");
|
||||
Path yml = Paths.get("src", "main", "resources", "application.yml");
|
||||
String config = new String(Files.readAllBytes(yml), StandardCharsets.UTF_8);
|
||||
// 运行期真正生效的是 yml 的兜底值,只改 @Value 默认等于没改
|
||||
assertTrue(config.contains("STUCK_TIMEOUT_MINUTES:5}"), "application.yml 卡死超时 5 分钟: " + config);
|
||||
assertTrue(config.contains("BOOT_RECLAIM_STALE_SECONDS:90}"), "开机回收阈值可在 env 覆盖");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -288,6 +298,52 @@ class TaskResultFileJobWorkerStuckScanTest {
|
||||
verify(taskFileJobService).markFailureFinalized(9L, "文件生成任务运行超时,已达到最大重试次数");
|
||||
}
|
||||
|
||||
/** 开机回收必须只针对"本机 instanceId 名下"的遗留作业,否则会抢另一台实例正在跑的 job。 */
|
||||
@Test
|
||||
void bootReclaimTargetsOwnAbandonedJobs() throws Exception {
|
||||
TaskResultFileJobWorker worker = buildWorker();
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("java-server");
|
||||
setInt(worker, "batchSize", 20);
|
||||
setLong(worker, "bootReclaimStaleSeconds", 90L);
|
||||
|
||||
worker.reclaimOwnedJobsOnBoot();
|
||||
|
||||
verify(taskFileJobService).reclaimOwnedRunningJobs("java-server", 90L, 20);
|
||||
}
|
||||
|
||||
/** 回收只是加速恢复,DB 抽风时绝不能把应用启动一起带下水。 */
|
||||
@Test
|
||||
void bootReclaimFailureDoesNotBreakStartup() throws Exception {
|
||||
TaskResultFileJobWorker worker = buildWorker();
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("java-server");
|
||||
when(taskFileJobService.reclaimOwnedRunningJobs(anyString(), anyLong(), anyInt()))
|
||||
.thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
worker.reclaimOwnedJobsOnBoot();
|
||||
|
||||
verify(taskFileJobService).reclaimOwnedRunningJobs("java-server", 0L, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* 必须挂在 ApplicationReadyEvent 上:早了(Bean 还没齐)会 NPE,晚了不如挂事件;
|
||||
* 挂在 ContextRefreshed 之类的事件上则会随上下文刷新重复执行。
|
||||
*/
|
||||
@Test
|
||||
void bootReclaimRunsOnApplicationReady() throws Exception {
|
||||
EventListener listener = TaskResultFileJobWorker.class
|
||||
.getMethod("reclaimOwnedJobsOnBoot")
|
||||
.getAnnotation(EventListener.class);
|
||||
assertNotNull(listener, "开机回收必须由事件驱动,否则没人调用");
|
||||
assertTrue(List.of(listener.value()).contains(ApplicationReadyEvent.class),
|
||||
"只在应用就绪后执行: " + List.of(listener.value()));
|
||||
}
|
||||
|
||||
private static void setLong(TaskResultFileJobWorker worker, String fieldName, long value) throws Exception {
|
||||
Field field = TaskResultFileJobWorker.class.getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
field.setLong(worker, value);
|
||||
}
|
||||
|
||||
private static void setInt(TaskResultFileJobWorker worker, String fieldName, int value) throws Exception {
|
||||
Field field = TaskResultFileJobWorker.class.getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
|
||||
Reference in New Issue
Block a user