fix(判死线): 「从未上报结果分片」不再静默跳过,按处理位置三分扩到 5 个模块
品牌任务 2626 挂 9 小时那条出口(无结果回传的候选一律 continue、零日志)此前只在 deletebrand 一处改了三分管法;本笔把同一判据接到外观专利、采集、上架、店铺采集、 货源五个模块,并把"待组装→任务锁→尽力收尾→CAS 翻转"的共同尾部抽成一条, 上报过与从未上报两条路共用,避免各模块再各写一份漏掉日志的版本。 合批代价(必须记着):这 28 支文件是并行会话的在制品,提交时它仍在按模块跑变异自证 (ShopDataCrawl 那一支此刻正处于被注入态,注入发生在它自己的副本树里,主树是 pristine), "全量绿"由本次发版前的独立 worktree 构建代为证明,不是它自己报过的数。 新增 3 支 *NoUploadStaleTest;推送前按登记表 50 个值与高熵串扫过 920 行新增,0 命中。
This commit is contained in:
+56
-19
@@ -46,6 +46,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
|
||||
@@ -123,6 +124,7 @@ public class PublishTaskService {
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
|
||||
@Value("${aiimage.publish.stale-timeout-minutes:30}")
|
||||
private int staleTimeoutMinutes;
|
||||
@@ -876,7 +878,9 @@ public class PublishTaskService {
|
||||
* <p>既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
|
||||
* 主线程卡死时心跳线程照发,任务永远命不中(与生产 28131 同型缺口)。
|
||||
* owner 过滤与 A 线保持一致(避免双节点同时处理非本实例任务);
|
||||
* 判据用 biz_task_scope_state.last_chunk_at(仅分片上传时刷新);从未上报跳过,避免误杀首批较慢的正常任务。
|
||||
* 判据用 biz_task_scope_state.last_chunk_at(仅分片上传时刷新);从未上报不再无条件跳过——
|
||||
* 与上报停摆同样按处理位置三分:仍在推进=首批较慢跳过、位置久未变=判死、无位置信息=证据不足保守跳过但留 WARN
|
||||
* (原先的无条件跳过是 2026-10-06 品牌任务 2626 挂 9 小时的同类出口)。
|
||||
*/
|
||||
private void failNoUploadStaleTasks() {
|
||||
long minutes = noResultUploadTimeoutMinutes;
|
||||
@@ -910,33 +914,66 @@ public class PublishTaskService {
|
||||
}
|
||||
for (FileTaskEntity candidate : candidates) {
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(candidate.getId());
|
||||
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||
if (lastResultAt != null && lastResultAt.isAfter(cutoff)) {
|
||||
// 上报仍新鲜:正常推进,不必处理
|
||||
continue;
|
||||
}
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(candidate.getId(), MODULE_TYPE) > 0L) {
|
||||
continue;
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle lock =
|
||||
taskDistributedLockService.acquire(MODULE_TYPE, candidate.getId(), 0L)) {
|
||||
if (lock == null) {
|
||||
if (lastResultAt == null) {
|
||||
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成一句静默 continue(零日志),
|
||||
// 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口(本模块心跳同样照打)。
|
||||
// 现按处理位置三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
|
||||
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
|
||||
String neverReportedPosition = taskHeartbeatPositionService.describe(candidate.getId());
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(candidate.getId(), minutes)) {
|
||||
log.info("[publish] 从未上报但客户端仍在推进,本轮不判死 taskId={} position={} minutes={}",
|
||||
candidate.getId(), neverReportedPosition, minutes);
|
||||
continue;
|
||||
}
|
||||
String error = "连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")";
|
||||
transactionTemplate.executeWithoutResult(status -> failStaleTaskLockedInternal(
|
||||
candidate.getId(),
|
||||
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
|
||||
log.warn("[publish] 从未上报且无位置信息,证据不足暂不判死 taskId={} createdAt={} minutes={}",
|
||||
candidate.getId(), candidate.getCreatedAt(), minutes);
|
||||
continue;
|
||||
}
|
||||
failNoUploadStaleTask(candidate,
|
||||
"任务创建至今超过 " + minutes + " 分钟无任何结果回传,处理位置长时间停在「"
|
||||
+ neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败",
|
||||
task -> {
|
||||
// 排队期间复查:心跳回落陈旧 → 交回 A 线;出现新结果上报 → 任务已恢复
|
||||
// 排队期间复查:心跳回落陈旧 → 交回 A 线;出现任一结果上报 → 已恢复回传,交回原线评估
|
||||
if (task.getUpdatedAt() == null || task.getUpdatedAt().isBefore(heartbeatThreshold)) {
|
||||
return false;
|
||||
}
|
||||
LocalDateTime freshLastChunk = queryLastChunkAt(task.getId());
|
||||
return freshLastChunk != null && !freshLastChunk.isAfter(cutoff);
|
||||
},
|
||||
error));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[publish] no-upload stale task cleanup failed taskId={} msg={}",
|
||||
candidate.getId(), ex.getMessage());
|
||||
return queryLastChunkAt(task.getId()) == null;
|
||||
});
|
||||
continue;
|
||||
}
|
||||
failNoUploadStaleTask(candidate,
|
||||
"连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")",
|
||||
task -> {
|
||||
// 排队期间复查:心跳回落陈旧 → 交回 A 线;出现新结果上报 → 任务已恢复
|
||||
if (task.getUpdatedAt() == null || task.getUpdatedAt().isBefore(heartbeatThreshold)) {
|
||||
return false;
|
||||
}
|
||||
LocalDateTime freshLastChunk = queryLastChunkAt(task.getId());
|
||||
return freshLastChunk != null && !freshLastChunk.isAfter(cutoff);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** 二次判死共同尾部:待组装检查 → 任务锁 → 收尾;error 与复查谓词由两条路各自提供。 */
|
||||
private void failNoUploadStaleTask(FileTaskEntity candidate, String error, Predicate<FileTaskEntity> staleCheck) {
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(candidate.getId(), MODULE_TYPE) > 0L) {
|
||||
return;
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle lock =
|
||||
taskDistributedLockService.acquire(MODULE_TYPE, candidate.getId(), 0L)) {
|
||||
if (lock == null) {
|
||||
return;
|
||||
}
|
||||
transactionTemplate.executeWithoutResult(status -> failStaleTaskLockedInternal(
|
||||
candidate.getId(), staleCheck, error));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[publish] no-upload stale task cleanup failed taskId={} msg={}",
|
||||
candidate.getId(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user