fix(task): 失败重试也复位已交付出表作业,判死文案按「客户端是否交完」分档
This commit is contained in:
@@ -19,6 +19,8 @@ import java.util.List;
|
||||
* @param lastHeartbeatAt 最后一次客户端心跳时间,Redis 无记录时为 null
|
||||
* @param lastProgress 客户端最后一次上报的位置/阶段文本,老客户端或无位置信息时为 null
|
||||
* @param fileJobExhausted 结果文件作业是否已重试耗尽(true 时责任不在客户端回传)
|
||||
* @param submissionComplete 客户端是否已交完全部结果(末分片 done=true/scope 已封口)。
|
||||
* true 时责任在出表侧:采集早就干完了,是结果文件没生成出来
|
||||
* @param timeoutMinutes 判死阈值(分钟)
|
||||
*/
|
||||
public record StaleTaskDeathReason(long chunkCount,
|
||||
@@ -26,17 +28,51 @@ public record StaleTaskDeathReason(long chunkCount,
|
||||
LocalDateTime lastHeartbeatAt,
|
||||
String lastProgress,
|
||||
boolean fileJobExhausted,
|
||||
boolean submissionComplete,
|
||||
int timeoutMinutes) {
|
||||
|
||||
/**
|
||||
* 没有「客户端是否交完」信号的模块用这一档:等价于 {@code submissionComplete=false},
|
||||
* 文案与加这个字段之前逐字相同。
|
||||
*
|
||||
* <p>留在重载里而不是让调用方自己写 {@code false},是为了不动相似ASIN那一支——
|
||||
* {@code SimilarAsinTaskService} 是在册的超长文件(2066 行,可读棘轮只许减),
|
||||
* 多一行参数就把它顶到 2067 并当场判红。外观专利有真实现场(2026-11-10 33828/33758),
|
||||
* 那条链路已经接上真信号;相似ASIN这一格要接,得等它先拆出判死文案那一段。
|
||||
*/
|
||||
public StaleTaskDeathReason(long chunkCount,
|
||||
LocalDateTime lastChunkAt,
|
||||
LocalDateTime lastHeartbeatAt,
|
||||
String lastProgress,
|
||||
boolean fileJobExhausted,
|
||||
int timeoutMinutes) {
|
||||
this(chunkCount, lastChunkAt, lastHeartbeatAt, lastProgress, fileJobExhausted, false, timeoutMinutes);
|
||||
}
|
||||
|
||||
private static final DateTimeFormatter TIME = DateTimeFormatter.ofPattern("MM-dd HH:mm");
|
||||
|
||||
public String describe() {
|
||||
if (chunkCount > 0 && fileJobExhausted) {
|
||||
return "已收到 " + chunkCount + " 个结果分片,但结果文件生成多次失败,任务已自动结束";
|
||||
}
|
||||
String headline = chunkCount <= 0L
|
||||
? "客户端未回传任何检测结果,超过 " + Math.max(1, timeoutMinutes) + " 分钟无进展"
|
||||
: "客户端回传中断";
|
||||
String headline;
|
||||
if (submissionComplete) {
|
||||
// 2026-11-10 现网 33828/33758 的形状:采集侧末分片 done=true 已上传成功
|
||||
// (worker 日志「任务处理完成 99/99,抓取失败 0 行」),出表作业却停在上一代的 SUCCESS 上
|
||||
// 没被复位,于是永不出表、心跳停了 30 分钟被判死。此时说「客户端回传中断」与事实相反——
|
||||
// 客户端早就传完了,卡住的是结果文件生成。
|
||||
headline = "客户端已上传完全部结果,但结果文件未能生成";
|
||||
} else if (chunkCount <= 0L) {
|
||||
headline = "客户端未回传任何检测结果,超过 " + Math.max(1, timeoutMinutes) + " 分钟无进展";
|
||||
if (lastProgress == null || lastProgress.isBlank()) {
|
||||
// 有位置文本时客户端至少开过工(如「排队等待空闲槽位」),这半句只加在「零证据」那一格:
|
||||
// 2026-11-10 慧赢2 的 33827 就是没人领到(采集机 02:50→09:30 全程无日志)却写成
|
||||
// 「无进展」的,用户据此以为是自己数据有问题。
|
||||
headline += "(全程无处理位置上报,可能一直在等待采集槽位)";
|
||||
}
|
||||
} else {
|
||||
headline = "客户端回传中断";
|
||||
}
|
||||
List<String> evidence = new ArrayList<>();
|
||||
if (chunkCount > 0L) {
|
||||
evidence.add("已收到 " + chunkCount + " 个分片,最后一次 " + at(lastChunkAt));
|
||||
|
||||
+1
-1
@@ -389,7 +389,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
return new AppearancePatentStaleTaskScanner(MODULE_TYPE, TASK_LOCK_TTL,
|
||||
fileTaskMapper, taskChunkMapper, taskScopeStateMapper, taskCacheService,
|
||||
taskHeartbeatPositionService, taskFileJobService, taskDistributedLockService,
|
||||
properties, taskOwnership());
|
||||
properties, taskOwnership(), this::isResultSubmissionComplete);
|
||||
}
|
||||
|
||||
public void finalizeStaleTasks() {
|
||||
|
||||
+9
-2
@@ -24,6 +24,7 @@ import java.time.ZoneId;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
/**
|
||||
* 外观专利的陈旧任务扫描与二次判死(A 聚类),从 {@code AppearancePatentTaskService}
|
||||
@@ -67,6 +68,8 @@ public class AppearancePatentStaleTaskScanner {
|
||||
private final TaskDistributedLockService taskDistributedLockService;
|
||||
private final AppearancePatentProperties properties;
|
||||
private final AppearancePatentTaskOwnership ownership;
|
||||
/** 客户端是否已交完全部结果(末分片 done=true);判死文案据此分开「采集没传完」与「采集完了没出表」。 */
|
||||
private final Predicate<Long> submissionComplete;
|
||||
|
||||
public AppearancePatentStaleTaskScanner(String moduleType,
|
||||
Duration taskLockTtl,
|
||||
@@ -78,7 +81,8 @@ public class AppearancePatentStaleTaskScanner {
|
||||
TaskFileJobService taskFileJobService,
|
||||
TaskDistributedLockService taskDistributedLockService,
|
||||
AppearancePatentProperties properties,
|
||||
AppearancePatentTaskOwnership ownership) {
|
||||
AppearancePatentTaskOwnership ownership,
|
||||
Predicate<Long> submissionComplete) {
|
||||
this.moduleType = moduleType;
|
||||
this.taskLockTtl = taskLockTtl;
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
@@ -90,6 +94,7 @@ public class AppearancePatentStaleTaskScanner {
|
||||
this.taskDistributedLockService = taskDistributedLockService;
|
||||
this.properties = properties;
|
||||
this.ownership = ownership;
|
||||
this.submissionComplete = submissionComplete;
|
||||
}
|
||||
|
||||
public void sweep(StaleDeathFinalizer finalizer) {
|
||||
@@ -144,7 +149,8 @@ public class AppearancePatentStaleTaskScanner {
|
||||
}
|
||||
|
||||
/**
|
||||
* 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、还是文件生成侧失败。
|
||||
* 判死文案按证据分述:客户端从未回传(多半是排队等槽位)、回传中断、客户端已交完但出表没跑、
|
||||
* 还是文件生成侧失败。
|
||||
* 三条路径历史上共用一句 "Python interrupted before uploading final appearance patent result",
|
||||
* 把「没开跑」说成「跑挂了」。
|
||||
*/
|
||||
@@ -158,6 +164,7 @@ public class AppearancePatentStaleTaskScanner {
|
||||
: null,
|
||||
taskHeartbeatPositionService.describe(taskId),
|
||||
taskFileJobService.hasExhaustedAssembleJob(taskId, moduleType),
|
||||
submissionComplete.test(taskId),
|
||||
properties.getStaleTimeoutMinutes()).describe();
|
||||
}
|
||||
|
||||
|
||||
+7
-1
@@ -94,8 +94,14 @@ public class TaskInPlaceRetryService {
|
||||
// 中断已交付过部分结果:先清直链再复位作业,顺序反了会出现「作业已 PENDING、
|
||||
// 已交付闸还短着」的一轮空转。直链清掉后下载按钮收起,新一轮组装完成再挂新文件。
|
||||
clearDeliveredResultUrl(task.getId(), task.getModuleType());
|
||||
resetDeliveredAssembleJob(task.getId(), task.getModuleType());
|
||||
}
|
||||
// 复位「上一代已出过表」的作业:失败代与中断代**都要做**,差别只在直链清不清。
|
||||
// 只给中断代做会留下一个永不出表的死循环——那行 SUCCESS 留在原地时,
|
||||
// enqueueAssembleResult 每次都原样拿回它(底座的 .ne SUCCESS 是刻意的:没人要求重做时
|
||||
// 不许替换用户可能已下载的成品),于是采集侧把结果全部传完也轮不到出表、心跳停了
|
||||
// 30 分钟再被陈旧扫描判死一次,且文案写成「客户端回传中断」。2026-11-10 现网三单同型:
|
||||
// 33828 二代/三代(worker 两次「任务处理完成 99/99,抓取失败 0 行」)、33758(657/657)。
|
||||
resetDeliveredAssembleJob(task.getId(), task.getModuleType());
|
||||
clearResultError(task.getId(), task.getModuleType());
|
||||
resetAssembleJob(task.getId(), task.getModuleType());
|
||||
log.info("[task-retry] 已在原任务上复位重试 taskId={} moduleType={} 代数={} 上次状态={} 上次失败={}",
|
||||
|
||||
+36
-7
@@ -13,7 +13,7 @@ class StaleTaskDeathReasonTest {
|
||||
|
||||
@Test
|
||||
void neverUploadedSaysNoResultUploadedNotThatPythonCrashed() {
|
||||
String reason = new StaleTaskDeathReason(0L, null, T, null, false, 30).describe();
|
||||
String reason = new StaleTaskDeathReason(0L, null, T, null, false, false, 30).describe();
|
||||
assertTrue(reason.contains("未回传任何检测结果"), reason);
|
||||
assertTrue(reason.contains("超过 30 分钟无进展"), reason);
|
||||
assertTrue(reason.contains("最后心跳 10-01 12:34"), reason);
|
||||
@@ -23,21 +23,23 @@ class StaleTaskDeathReasonTest {
|
||||
|
||||
@Test
|
||||
void queuedTaskIsToldApartFromACrashedOneByTheReportedPhase() {
|
||||
String reason = new StaleTaskDeathReason(0L, null, T, "排队等待空闲槽位", false, 30).describe();
|
||||
String reason = new StaleTaskDeathReason(0L, null, T, "排队等待空闲槽位", false, false, 30).describe();
|
||||
assertTrue(reason.contains("最后上报 排队等待空闲槽位"), reason);
|
||||
assertFalse(reason.contains("已收到"), reason);
|
||||
// 有位置文本=客户端开过工,不得再挂「等待采集槽位」那半句
|
||||
assertFalse(reason.contains("等待采集槽位"), reason);
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingHeartbeatAndBlankPhaseDegradeQuietly() {
|
||||
String reason = new StaleTaskDeathReason(0L, null, null, " ", false, 30).describe();
|
||||
String reason = new StaleTaskDeathReason(0L, null, null, " ", false, false, 30).describe();
|
||||
assertTrue(reason.contains("最后心跳 时间未知"), reason);
|
||||
assertFalse(reason.contains("最后上报"), reason);
|
||||
}
|
||||
|
||||
@Test
|
||||
void interruptedUploadReportsChunksAndLastChunkTime() {
|
||||
String reason = new StaleTaskDeathReason(7L, T, T, "第 120/500 组", false, 30).describe();
|
||||
String reason = new StaleTaskDeathReason(7L, T, T, "第 120/500 组", false, false, 30).describe();
|
||||
assertTrue(reason.contains("客户端回传中断"), reason);
|
||||
assertTrue(reason.contains("已收到 7 个分片,最后一次 10-01 12:34"), reason);
|
||||
assertTrue(reason.contains("最后上报 第 120/500 组"), reason);
|
||||
@@ -46,21 +48,48 @@ class StaleTaskDeathReasonTest {
|
||||
|
||||
@Test
|
||||
void exhaustedFileJobBlamesFileGenerationInsteadOfTheClient() {
|
||||
String reason = new StaleTaskDeathReason(7L, T, T, "第 120/500 组", true, 30).describe();
|
||||
String reason = new StaleTaskDeathReason(7L, T, T, "第 120/500 组", true, false, 30).describe();
|
||||
assertTrue(reason.contains("结果文件生成多次失败"), reason);
|
||||
assertFalse(reason.contains("回传中断"), reason);
|
||||
}
|
||||
|
||||
@Test
|
||||
void exhaustedFileJobWithoutAnyChunkStillReportsClientSilence() {
|
||||
String reason = new StaleTaskDeathReason(0L, null, T, null, true, 30).describe();
|
||||
String reason = new StaleTaskDeathReason(0L, null, T, null, true, false, 30).describe();
|
||||
assertTrue(reason.contains("未回传任何检测结果"), reason);
|
||||
assertFalse(reason.contains("结果文件生成"), reason);
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonPositiveTimeoutNeverPrintsZeroMinutes() {
|
||||
assertTrue(new StaleTaskDeathReason(0L, null, null, null, false, 0)
|
||||
assertTrue(new StaleTaskDeathReason(0L, null, null, null, false, false, 0)
|
||||
.describe().contains("超过 1 分钟"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 2026-11-10 现网 33828/33758 的形状:采集侧末分片 done=true 已上传成功
|
||||
* (worker 日志「任务处理完成,抓取失败 0 行」),出表作业却停在上一代的 SUCCESS 上没被复位,
|
||||
* 于是永不出表、心跳停了 30 分钟被判死。原文案「客户端回传中断」把责任判给了客户端,
|
||||
* 与其实际行为相反,排查方向被带偏(用户以为是采集没传完)。
|
||||
*/
|
||||
@Test
|
||||
void submissionCompletedTaskBlamesFileGenerationNotTheClient() {
|
||||
String reason = new StaleTaskDeathReason(12L, T, null, "Linux后台抓取: 98/99", false, true, 30).describe();
|
||||
assertTrue(reason.contains("已上传完全部结果"), reason);
|
||||
assertTrue(reason.contains("结果文件未能生成"), reason);
|
||||
assertTrue(reason.contains("已收到 12 个分片,最后一次 10-01 12:34"), reason);
|
||||
assertFalse(reason.contains("回传中断"), reason);
|
||||
assertFalse(reason.contains("未回传任何检测结果"), reason);
|
||||
}
|
||||
|
||||
/**
|
||||
* 2026-11-10 慧赢2 的 33827:建单后 30 分钟没有任何采集槽位认领(采集机该窗全程无日志),
|
||||
* 却被写成「无进展」,用户无从判断是自己的数据问题还是排队。零位置文本时补一句成因提示,
|
||||
* 且措辞只到「可能」——没认领这件事本身无法从这两列证明。
|
||||
*/
|
||||
@Test
|
||||
void neverUploadedWithoutAnyPositionHintsQueuedForCollectorSlot() {
|
||||
String reason = new StaleTaskDeathReason(0L, null, T, null, false, false, 30).describe();
|
||||
assertTrue(reason.contains("可能一直在等待采集槽位"), reason);
|
||||
}
|
||||
}
|
||||
|
||||
+46
@@ -156,6 +156,52 @@ class TaskInPlaceRetryServiceTest {
|
||||
verify(handler, never()).onTaskResumed(anyLong());
|
||||
}
|
||||
|
||||
/**
|
||||
* 2026-11-10 现网 33828/33758:上一代**已经出过表**(组装作业停在 SUCCESS)的任务判失败后,
|
||||
* 重试若不复位那行作业,enqueueAssembleResult 每次原样拿回它、永不出表——采集侧把结果全部传完
|
||||
* (worker 日志「任务处理完成 99/99」)也轮不到出表,30 分钟没心跳再被判一次,
|
||||
* 还写成「客户端回传中断」。失败代与中断代的差别只在直链清不清,作业复位两边都要做。
|
||||
*/
|
||||
@Test
|
||||
void 失败任务重试也要复位上一代已交付的出表作业() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
FileResultMapper results = mock(FileResultMapper.class);
|
||||
TaskFileJobService jobs = mock(TaskFileJobService.class);
|
||||
TaskFileJobMapper jobMapper = mock(TaskFileJobMapper.class);
|
||||
when(mapper.update(isNull(), any())).thenReturn(1);
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(36758L);
|
||||
when(results.selectOne(any())).thenReturn(result);
|
||||
when(jobMapper.selectOne(any())).thenReturn(deliveredJob());
|
||||
|
||||
boolean ok = service(mapper, results, jobs, jobMapper,
|
||||
mock(ResultFileJobHandlerRegistry.class)).retryInPlace(failedTask());
|
||||
|
||||
assertTrue(ok);
|
||||
// 已交付作业被复位成 PENDING(下一轮才排得上出表)
|
||||
verify(jobMapper).update(isNull(), any());
|
||||
// 失败保留直链:只清旧错误文案,不清下载直链(用户手上那份还能下)
|
||||
verify(results, times(1)).update(isNull(), any());
|
||||
verify(jobs).resetTerminalFailedForRecovery(eq(32967L), eq("APPEARANCE_PATENT"), eq(36758L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 失败任务没有已交付作业时不动物业行() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
FileResultMapper results = mock(FileResultMapper.class);
|
||||
TaskFileJobMapper jobMapper = mock(TaskFileJobMapper.class);
|
||||
when(mapper.update(isNull(), any())).thenReturn(1);
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(36758L);
|
||||
when(results.selectOne(any())).thenReturn(result);
|
||||
when(jobMapper.selectOne(any())).thenReturn(null);
|
||||
|
||||
service(mapper, results, mock(TaskFileJobService.class), jobMapper,
|
||||
mock(ResultFileJobHandlerRegistry.class)).retryInPlace(failedTask());
|
||||
|
||||
verify(jobMapper, never()).update(isNull(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 成功任务不可继续() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
|
||||
Reference in New Issue
Block a user