fix(task): 终态任务保留失败兜底出部分结果
B28 的终态守卫把父任务已失败的结果作业直接标终态;terminal_callback_at 一打, stuck 扫描就再也捞不到它,fallbackAssembleOnFailure 不再执行,失败任务永远没有结果文件。 改为判死但保留兜底回调,由归属实例收口;已终态任务的补检直接放行,按已有分片出部分结果。
This commit is contained in:
+12
-3
@@ -159,6 +159,10 @@ public class AppearancePatentDeliverySupport {
|
||||
* 「因为慢而抛满重试」不再等于「永远补不了」。
|
||||
*
|
||||
* <p>预算耗尽、或重排没接上时仍按已有结果出表:「起码失败了也可以下载」那条保证不变。
|
||||
*
|
||||
* <p>父任务已终态时不再排补检:B28 的终态守卫不会让结果作业再跑,重排只会被下一轮
|
||||
* claim 立刻打回 FAILED,白转一圈(现网 32217/32220 锁续 86 分钟就是这条循环的旧形态)。
|
||||
* 终态任务直接按已有分片出部分结果,缺口由行账文案标在表里。
|
||||
*/
|
||||
public boolean scheduleFallbackRecheck(FileTaskEntity task, FileResultEntity result,
|
||||
AppearancePatentRowOutcome.Facts facts) {
|
||||
@@ -167,15 +171,20 @@ public class AppearancePatentDeliverySupport {
|
||||
|| facts == null || maxRounds <= 0) {
|
||||
return false;
|
||||
}
|
||||
if (facts.missingRows() <= 0 || facts.missingKeys().isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
if (TaskStatuses.FileTask.TERMINAL.contains(task.getStatus())) {
|
||||
log.warn("[appearance-patent] 任务已终态,补检不再执行,按已有结果交付 taskId={} status={} missingRows={} keys={}",
|
||||
task.getId(), task.getStatus(), facts.missingRows(), facts.missingKeys().size());
|
||||
return false;
|
||||
}
|
||||
long done = taskCacheService.recheckRounds(task.getId());
|
||||
if (done >= maxRounds) {
|
||||
log.warn("[appearance-patent] 补检预算已耗尽,按已有结果交付 taskId={} rounds={}/{}",
|
||||
task.getId(), done, maxRounds);
|
||||
return false;
|
||||
}
|
||||
if (facts.missingRows() <= 0 || facts.missingKeys().isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
taskCacheService.releaseRowInspection(task.getId(), facts.missingKeys(), true);
|
||||
TaskFileJobEntity reset = taskFileJobService.resetTerminalFailedForRecovery(
|
||||
task.getId(), moduleType, result.getId());
|
||||
|
||||
+27
-5
@@ -239,10 +239,16 @@ public class TaskFileJobService {
|
||||
return status != null && TaskStatuses.FileTask.TERMINAL.contains(status);
|
||||
}
|
||||
|
||||
/** 终态父任务的 job 直接拉满重试并打终态回调:不执行、不再被下一轮扫到。 */
|
||||
/**
|
||||
* 终态父任务的 job 直接拉满重试:不执行、不再被下一轮扫到。
|
||||
*
|
||||
* <p>终态回调要留给卡死扫描的失败兜底({@code fallbackAssembleOnFailure}):外观专利等模块
|
||||
* 靠它用已回传分片出部分结果文件,「起码失败了也可以下载」才不会被 B28 的终态守卫顺手掐掉。
|
||||
* 这里若直接打 {@code terminal_callback_at},兜底扫描再也捞不到这行,用户只剩红字、连半张表都没有。
|
||||
*/
|
||||
private void skipJobOfTerminalTask(TaskFileJobEntity job, String stage) {
|
||||
String message = "父任务已终态,结果作业不再执行";
|
||||
markFailedPermanent(job, message);
|
||||
markFailedPermanentAwaitingFallback(job, message);
|
||||
log.info("[task-file-job] skipped because parent task is terminal jobId={} taskId={} moduleType={} stage={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), stage);
|
||||
}
|
||||
@@ -801,15 +807,31 @@ public class TaskFileJobService {
|
||||
* 适用于 task/result 已经不存在等无法通过重试恢复的场景。
|
||||
*/
|
||||
public void markFailedPermanent(TaskFileJobEntity job, String message) {
|
||||
markFailedPermanent(job, message, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* 父任务已终态时的作业收口:同样判死并拉满重试,但**不打 terminal_callback_at**。
|
||||
*
|
||||
* <p>不打标不是漏收尾:卡死扫描专门捞「FAILED + retry 到顶 + terminal_callback_at 为空」的行,
|
||||
* 由它调用模块的失败兜底出部分结果(外观专利任务 31302/31354 的「起码失败了也可以下载」)。
|
||||
* 这里一旦打标,兜底路径就被永久跳过,失败任务的 result_file_url 永远为空,前端连下载按钮都没有。
|
||||
*/
|
||||
public void markFailedPermanentAwaitingFallback(TaskFileJobEntity job, String message) {
|
||||
markFailedPermanent(job, message, false);
|
||||
}
|
||||
|
||||
private void markFailedPermanent(TaskFileJobEntity job, String message, boolean finalizeCallback) {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, LocalDateTime.now()));
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, finalizeCallback ? now : null));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+12
-3
@@ -102,6 +102,14 @@ public class TaskResultFileJobWorker {
|
||||
result.resetCount(), result.exhaustedJobs().size(), stuckTimeoutMinutes);
|
||||
}
|
||||
for (TaskFileJobEntity job : result.exhaustedJobs()) {
|
||||
// 终态兜底出表必须落在归属实例:fallbackAssembleResultFile 在非归属机会直接返回 false,
|
||||
// 但 finalizeRetryExhausted 随后仍会打 terminal_callback_at,归属机的兜底就永远跑不到了。
|
||||
if (isOwnerScopedJob(job) && !isOwnedByCurrentInstance(job)) {
|
||||
log.info("[task-file-job] skip exhausted owner-scoped job of another instance jobId={} taskId={} moduleType={} owner={} current={} stage=SKIP_OWNER",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(),
|
||||
ownerFromScopeKey(job.getScopeKey()), currentInstanceId());
|
||||
continue;
|
||||
}
|
||||
String message = job.getErrorMessage() == null || job.getErrorMessage().isBlank()
|
||||
? "文件生成任务运行超时,已达到最大重试次数"
|
||||
: job.getErrorMessage();
|
||||
@@ -215,7 +223,8 @@ public class TaskResultFileJobWorker {
|
||||
}
|
||||
// 终态后不再 touchRunning:否则 job 的 updated_at 一直被刷新,
|
||||
// job 级 stuck 扫描永远看不到这行,成为不可回收的 RUNNING 遗骸。
|
||||
taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行");
|
||||
// 判死但不打 terminal_callback_at:留给兜底扫描出部分结果,失败任务仍可下载。
|
||||
taskFileJobService.markFailedPermanentAwaitingFallback(job, "父任务已终态,结果作业不再执行");
|
||||
log.info("[task-file-job] parent task terminal; stopped renewal and failed job jobId={} taskId={} moduleType={} stage=TERMINAL_STOP_RENEW",
|
||||
job.getId(), job.getTaskId(), job.getModuleType());
|
||||
}
|
||||
@@ -231,7 +240,7 @@ public class TaskResultFileJobWorker {
|
||||
|
||||
private void processClaimedWithHeartbeat(TaskFileJobEntity job, TaskFileJobEntity claim) {
|
||||
if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) {
|
||||
taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行");
|
||||
taskFileJobService.markFailedPermanentAwaitingFallback(job, "父任务已终态,结果作业不再执行");
|
||||
log.info("[task-file-job] skipped because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_SKIP",
|
||||
job.getId(), job.getTaskId(), job.getModuleType());
|
||||
return;
|
||||
@@ -279,7 +288,7 @@ public class TaskResultFileJobWorker {
|
||||
}
|
||||
if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) {
|
||||
try (lockHandle) {
|
||||
taskFileJobService.markFailedPermanent(job, "父任务已终态,结果作业不再执行");
|
||||
taskFileJobService.markFailedPermanentAwaitingFallback(job, "父任务已终态,结果作业不再执行");
|
||||
}
|
||||
log.info("[task-file-job] skipped after acquiring lock because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_AFTER_LOCK",
|
||||
job.getId(), job.getTaskId(), job.getModuleType());
|
||||
|
||||
+15
-2
@@ -371,7 +371,7 @@ class AppearancePatentDeliveryGateTest {
|
||||
when(taskFileJobService.resetTerminalFailedForRecovery(TASK_ID, "APPEARANCE_PATENT", 700L))
|
||||
.thenReturn(pendingJob(500L));
|
||||
|
||||
assertTrue(scheduleFallbackRecheck(task("FAILED"), result(700L), facts));
|
||||
assertTrue(scheduleFallbackRecheck(task("RUNNING"), result(700L), facts));
|
||||
verify(taskCacheService).releaseRowInspection(eq(TASK_ID), anyList(), eq(true));
|
||||
verify(taskCacheService).countRecheckRound(TASK_ID);
|
||||
}
|
||||
@@ -382,7 +382,20 @@ class AppearancePatentDeliveryGateTest {
|
||||
AppearancePatentRowOutcome.Facts facts = summarize(rows, submittedRows(rows, 1, true));
|
||||
when(taskCacheService.recheckRounds(TASK_ID)).thenReturn(2L);
|
||||
|
||||
assertFalse(scheduleFallbackRecheck(task("RUNNING"), result(700L), facts));
|
||||
verify(taskFileJobService, never()).resetTerminalFailedForRecovery(anyLong(), any(), anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void terminalTaskWithGapShipsPartialInsteadOfRechecking() throws Exception {
|
||||
// B28 起终态任务的结果作业不会再被 claim;再排补检只会被下一轮终态守卫立刻打回 FAILED。
|
||||
// 终态任务直接按已有分片出部分结果,缺口由行账文案标在表里。
|
||||
List<AppearancePatentParsedRowVo> rows = sheetRows(2);
|
||||
AppearancePatentRowOutcome.Facts facts = summarize(rows, submittedRows(rows, 1, true));
|
||||
|
||||
assertFalse(scheduleFallbackRecheck(task("FAILED"), result(700L), facts));
|
||||
verify(taskCacheService, never()).recheckRounds(anyLong());
|
||||
verify(taskCacheService, never()).releaseRowInspection(anyLong(), anyList(), eq(true));
|
||||
verify(taskFileJobService, never()).resetTerminalFailedForRecovery(anyLong(), any(), anyLong());
|
||||
}
|
||||
|
||||
@@ -406,7 +419,7 @@ class AppearancePatentDeliveryGateTest {
|
||||
when(taskFileJobService.resetTerminalFailedForRecovery(TASK_ID, "APPEARANCE_PATENT", 700L))
|
||||
.thenReturn(job(500L));
|
||||
|
||||
assertFalse(scheduleFallbackRecheck(task("FAILED"), result(700L), facts));
|
||||
assertFalse(scheduleFallbackRecheck(task("RUNNING"), result(700L), facts));
|
||||
verify(taskCacheService, never()).countRecheckRound(anyLong());
|
||||
}
|
||||
|
||||
|
||||
+5
@@ -92,6 +92,9 @@ class TaskFileJobTerminalTaskGuardTest {
|
||||
assertTrue(hasParam(wrapper, "FAILED"), "必须把作业拉成 FAILED,下一轮扫描才捞不到");
|
||||
assertTrue(hasParam(wrapper, 5), "retry_count 必须一次拉满到上限");
|
||||
assertFalse(hasParam(wrapper, "RUNNING"), "不得再翻转成 RUNNING");
|
||||
assertTrue(wrapper.getSqlSet().contains("terminal_callback_at"),
|
||||
"terminal_callback_at 必须显式清空:留给兜底扫描出部分结果,失败任务仍可下载");
|
||||
assertTrue(hasParam(wrapper, null), "terminal_callback_at 置空后兜底扫描才捞得到这一行");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -118,6 +121,8 @@ class TaskFileJobTerminalTaskGuardTest {
|
||||
verify(taskFileJobMapper, times(2)).update(isNull(), captor.capture());
|
||||
assertTrue(hasParam(captor.getAllValues().get(0), "FAILED"), "第一笔是终态收尾");
|
||||
assertTrue(hasParam(captor.getAllValues().get(1), "RUNNING"), "第二笔才是正常 claim");
|
||||
assertTrue(captor.getAllValues().get(0).getSqlSet().contains("terminal_callback_at"),
|
||||
"终态作业必须留给兜底扫描,不能提前打终态回调");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+23
-2
@@ -355,7 +355,7 @@ class TaskResultFileJobWorkerStuckScanTest {
|
||||
@Test
|
||||
void terminalParentHeartbeatStopsTouchingAndFailsJob() throws Exception {
|
||||
// 父任务在 worker 执行中变终态:心跳不得再 touchRunning(否则 stuck 扫描永远看不到),
|
||||
// 并主动把 job 收口为 FAILED(终态回调由 markFailedPermanent 打标)。
|
||||
// 并主动把 job 收口为 FAILED;终态回调留给兜底扫描出部分结果,失败任务仍可下载。
|
||||
TaskResultFileJobWorker worker = buildWorker();
|
||||
setLong(worker, "heartbeatIntervalMillis", 10L);
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
@@ -373,13 +373,34 @@ class TaskResultFileJobWorkerStuckScanTest {
|
||||
try {
|
||||
// heartbeat 间隔下限 1000ms,首轮在 1s 后才跑;给足调度余量
|
||||
verify(taskFileJobService, timeout(3000))
|
||||
.markFailedPermanent(eq(job), contains("父任务已终态"));
|
||||
.markFailedPermanentAwaitingFallback(eq(job), contains("父任务已终态"));
|
||||
verify(taskFileJobService, after(150).never()).touchRunning(77L);
|
||||
} finally {
|
||||
future.cancel(true);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 终态兜底出表只在归属实例做:非归属实例若抢先 markFailureFinalized,归属机的
|
||||
* fallbackAssembleOnFailure 就永远跑不到,失败任务的结果文件也就永远不生成。
|
||||
*/
|
||||
@Test
|
||||
void exhaustedOwnerScopedJobOfAnotherInstanceIsLeftToItsOwner() throws Exception {
|
||||
TaskResultFileJobWorker worker = buildWorker();
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
TaskFileJobEntity job = exhaustedJob(9L, 19L, "父任务已终态,结果作业不再执行");
|
||||
job.setModuleType("APPEARANCE_PATENT");
|
||||
job.setScopeKey("appearance-patent:19:owner:instance-b");
|
||||
TaskFileJobService.StuckJobResetResult result =
|
||||
new TaskFileJobService.StuckJobResetResult(0, List.of(job));
|
||||
when(taskFileJobService.resetStuckRunningJobsDetailed(0, 0)).thenReturn(result);
|
||||
|
||||
worker.resetStuckJobs();
|
||||
|
||||
verify(appearance, never()).fallbackAssembleResultFile(any());
|
||||
verify(taskFileJobService, never()).markFailureFinalized(any(), anyString());
|
||||
}
|
||||
|
||||
private static void setLong(TaskResultFileJobWorker worker, String fieldName, long value) throws Exception {
|
||||
Field field = TaskResultFileJobWorker.class.getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
|
||||
Reference in New Issue
Block a user