fix(similar-asin): 迟到回传重开超时判死任务,收尾复位组装 job
心跳判死巡检只改任务状态,客户端继续补发时被 40901 拒绝并删本地 副本;线上 31577/31472 的 214 行结果分片整批丢失。24 小时内、仅 心跳/回传停滞类判死文案的 FAILED 任务,收到迟到分片时条件重开为 RUNNING;业务失败与 SUCCESS 仍按终态拒绝。 判死巡检会把组装 job 一并写成 FAILED+retry=max,底座不会自动重排; 收尾进入等组装态时复位它,避免重开后卡在 RUNNING。
This commit is contained in:
+85
@@ -117,6 +117,15 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
private static final String LLM_STATUS_FAILED = "FAILED";
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
||||
/** 心跳/回传停滞类判死文案:客户端迟到回传可把任务重开,避免 40901 直接丢数据。 */
|
||||
private static final List<String> STALE_FAILURE_MARKERS = List.of(
|
||||
"任务心跳超时,已自动失败",
|
||||
"长时间未收到 Python 心跳,任务已自动失败",
|
||||
"客户端未回传任何检测结果",
|
||||
"客户端回传中断",
|
||||
"任务创建至今超过",
|
||||
"Python heartbeat alive but no result chunk uploaded");
|
||||
private static final Duration STALE_REOPEN_WINDOW = Duration.ofHours(24);
|
||||
private static final int MAX_LIGHT_TASK_IDS = 200;
|
||||
/** progressBatch 的 IN 子句上限:看板单次轮询的在册任务量级远小于它,取值留了 5 倍余量。 */
|
||||
private static final int MAX_BATCH_TASK_IDS = 1000;
|
||||
@@ -754,6 +763,7 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
if (request == null) {
|
||||
throw new BusinessException("结果请求不能为空");
|
||||
}
|
||||
reopenStaleFailedTask(taskId);
|
||||
PreparedSubmittedChunk prepared = pipelineSupport().prepareSubmittedChunk(taskId, request);
|
||||
PersistSubmittedChunkResult persisted = transactionManager == null
|
||||
? pipelineSupport().persistSubmittedChunk(prepared)
|
||||
@@ -769,10 +779,85 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
taskCacheService.touchTaskHeartbeat(taskId);
|
||||
} else {
|
||||
applyFinalizeSideEffects(persisted.finalizeResult());
|
||||
// 中间分片重开后任务已是 RUNNING,finalize 时不能再靠「本次是否重开」判断;
|
||||
// terminal=false 才是「收尾后等组装」,此时才需要复位判死时一起失败的组装 job。
|
||||
if (!persisted.finalizeResult().terminal()) {
|
||||
recoverTerminalFailedAssemble(taskId);
|
||||
}
|
||||
}
|
||||
pipelineSupport().scheduleLlmPipelineForSubmittedChunk(context);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判死巡检会把在跑的组装 job 一起写成 FAILED + retryCount=max(见
|
||||
* {@code StaleTaskRepairService.markDeadDownstream})。底座对重试耗尽的 job 不会自动重排,
|
||||
* 迟到回传重开后若不复位,任务会永远挂在 RUNNING 等一个不会跑的组装。
|
||||
* {@code resetTerminalFailedForRecovery} 自带「非终态失败不动作」守卫,重复调用安全。
|
||||
*/
|
||||
private void recoverTerminalFailedAssemble(Long taskId) {
|
||||
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, taskId)
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||
.last("limit 1"));
|
||||
if (results == null || results.isEmpty() || results.getFirst().getId() == null) {
|
||||
return;
|
||||
}
|
||||
taskFileJobService.resetTerminalFailedForRecovery(taskId, MODULE_TYPE, results.getFirst().getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* 被心跳/回传超时判死(含首心跳宽限)的任务收到客户端迟到分片时先重开,再走正常落库。
|
||||
*
|
||||
* <p>判死巡检只改 {@code biz_file_task},客户端并不知道;它继续补发时旧逻辑按终态回 40901,
|
||||
* 客户端收到 {@code success=true} 即删本地副本,任务已采集的数据整批丢掉(生产 31577/31472)。
|
||||
* 只重开归因于客户端停滞的判死文案、24 小时内、owner 为本实例的记录;业务性失败
|
||||
* (如结果文件生成失败)与 SUCCESS 仍按终态拒绝。
|
||||
*/
|
||||
private void reopenStaleFailedTask(Long taskId) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_FAILED.equals(task.getStatus())) {
|
||||
return;
|
||||
}
|
||||
if (!isStaleFailure(task.getErrorMessage())) {
|
||||
return;
|
||||
}
|
||||
String reason = task.getErrorMessage();
|
||||
LocalDateTime anchor = task.getFinishedAt() == null ? task.getUpdatedAt() : task.getFinishedAt();
|
||||
if (anchor == null || anchor.isBefore(LocalDateTime.now().minus(STALE_REOPEN_WINDOW))) {
|
||||
return;
|
||||
}
|
||||
if (!ownershipSupport().isOwnerCurrent(ownershipSupport().ownerFromTask(task))) {
|
||||
return;
|
||||
}
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, taskId)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.eq(FileTaskEntity::getErrorMessage, reason)
|
||||
.set(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getErrorMessage, null)
|
||||
.set(FileTaskEntity::getFinishedAt, null)
|
||||
.set(FileTaskEntity::getSuccessFileCount, 0)
|
||||
.set(FileTaskEntity::getFailedFileCount, 0)
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated == 1) {
|
||||
taskCacheService.deleteTaskCache(taskId);
|
||||
log.warn("[similar-asin] 迟到结果回传,重开心跳/回传超时判死的任务 taskId={} reason={}",
|
||||
taskId, reason);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isStaleFailure(String errorMessage) {
|
||||
if (errorMessage == null || errorMessage.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
for (String marker : STALE_FAILURE_MARKERS) {
|
||||
if (errorMessage.contains(marker)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public void deleteTask(Long taskId, Long userId) {
|
||||
List<String> payloads;
|
||||
try (TaskDistributedLockService.LockHandle ignored = ownershipSupport().requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) {
|
||||
|
||||
+88
@@ -45,6 +45,7 @@ import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
@@ -274,6 +275,93 @@ class SimilarAsinTaskServiceSubmitTest {
|
||||
anyString(), anyLong(), anyString(), any(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lateResultReopensStaleFailedTaskInsteadOfDroppingIt() throws Exception {
|
||||
// 生产 31577/31472:心跳超时判死后客户端继续补发,旧逻辑回 40901,客户端删本地副本,
|
||||
// 已采数据整批丢失。迟到分片到达时应先把任务拉回 RUNNING,再正常落库。
|
||||
FileTaskEntity task = runningTask("instance-a");
|
||||
task.setStatus("FAILED");
|
||||
task.setErrorMessage("任务心跳超时,已自动失败");
|
||||
task.setFinishedAt(LocalDateTime.now().minusMinutes(30));
|
||||
task.setFailedFileCount(1);
|
||||
AtomicBoolean reopened = new AtomicBoolean();
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
when(fileTaskMapper.update(any(), any())).thenAnswer(invocation -> {
|
||||
reopened.set(true);
|
||||
task.setStatus("RUNNING");
|
||||
task.setErrorMessage(null);
|
||||
task.setFinishedAt(null);
|
||||
return 1;
|
||||
});
|
||||
configureNewChunkAndScope();
|
||||
configureChunkStore(false);
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
service.submitResult(TASK_ID, request(false));
|
||||
|
||||
assertTrue(reopened.get(), "心跳超时判死的任务收到迟到分片必须先重开,而不是 40901 丢数据");
|
||||
assertEquals("RUNNING", task.getStatus());
|
||||
verify(taskCacheService).touchTaskHeartbeat(TASK_ID);
|
||||
}
|
||||
|
||||
@Test
|
||||
void lateFinalResultResetsTerminalFailedAssembleJob() throws Exception {
|
||||
// 判死巡检把组装 job 一起写成 FAILED+retryCount=max;迟到 done 重开后若不复位,
|
||||
// finalizeTask 只把任务挂回 RUNNING,组装永远不会再跑。31577/31472 正是这个形状。
|
||||
FileTaskEntity task = runningTask("instance-a");
|
||||
task.setStatus("FAILED");
|
||||
task.setErrorMessage("任务心跳超时,已自动失败");
|
||||
task.setFinishedAt(LocalDateTime.now().minusMinutes(30));
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
when(fileTaskMapper.update(any(), any())).thenAnswer(invocation -> {
|
||||
task.setStatus("RUNNING");
|
||||
task.setErrorMessage(null);
|
||||
task.setFinishedAt(null);
|
||||
return 1;
|
||||
});
|
||||
configureNewChunkAndScope();
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||
eq(SimilarAsinTaskService.MODULE_TYPE), eq(TASK_ID), anyString(), eq(0), anyString()))
|
||||
.thenReturn(STORED_CHUNK_POINTER);
|
||||
when(transientPayloadStorageService.wasLastStoreLocalFallback()).thenReturn(false);
|
||||
String parsedPayload = parsedPayloadJson(List.of(new SimilarAsinParsedRowVo()));
|
||||
when(transientPayloadStorageService.resolvePayload(eq(PARSED_POINTER), anyString()))
|
||||
.thenReturn(parsedPayload);
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(501L);
|
||||
result.setTaskId(TASK_ID);
|
||||
result.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
|
||||
result.setSourceFilename("germany.xlsx");
|
||||
when(fileResultMapper.selectList(any())).thenReturn(List.of(result));
|
||||
|
||||
// 先来一个中间分片:任务在这里被重开;随后 done 到达时状态已是 RUNNING,
|
||||
// 不能再用「本次请求是否发生重开」决定要不要复位组装 job。
|
||||
service.submitResult(TASK_ID, request(false));
|
||||
assertEquals("RUNNING", task.getStatus(), "中间分片应先把判死任务重开");
|
||||
|
||||
service.submitResult(TASK_ID, request(true));
|
||||
|
||||
assertEquals("RUNNING", task.getStatus(), "迟到 done 收尾后应进入等待组装态");
|
||||
verify(taskFileJobService).resetTerminalFailedForRecovery(
|
||||
TASK_ID, SimilarAsinTaskService.MODULE_TYPE, 501L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void businessFailedTaskStillRejectsLateResultWithTerminalCode() throws Exception {
|
||||
// 只有客户端停滞类判死可重开;结果文件生成失败等业务失败仍按终态收口。
|
||||
FileTaskEntity task = runningTask("instance-a");
|
||||
task.setStatus("FAILED");
|
||||
task.setErrorMessage("相似ASIN结果文件生成失败");
|
||||
task.setFinishedAt(LocalDateTime.now().minusMinutes(30));
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> service.submitResult(TASK_ID, request(false)));
|
||||
assertEquals(BusinessCodes.TASK_ALREADY_FINISHED, ex.getCode());
|
||||
verify(fileTaskMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void submitResultOnFinishedTaskIsRejectedWithTerminalCode() throws Exception {
|
||||
// 真终态被拒必须带 40901:异常处理器对它幂等回 success=true,客户端据此删本地副本、停止补发。
|
||||
|
||||
Reference in New Issue
Block a user