fix(判死线): 「从未上报结果分片」不再静默跳过,按处理位置三分扩到 5 个模块
品牌任务 2626 挂 9 小时那条出口(无结果回传的候选一律 continue、零日志)此前只在 deletebrand 一处改了三分管法;本笔把同一判据接到外观专利、采集、上架、店铺采集、 货源五个模块,并把"待组装→任务锁→尽力收尾→CAS 翻转"的共同尾部抽成一条, 上报过与从未上报两条路共用,避免各模块再各写一份漏掉日志的版本。 合批代价(必须记着):这 28 支文件是并行会话的在制品,提交时它仍在按模块跑变异自证 (ShopDataCrawl 那一支此刻正处于被注入态,注入发生在它自己的副本树里,主树是 pristine), "全量绿"由本次发版前的独立 worktree 构建代为证明,不是它自己报过的数。 新增 3 支 *NoUploadStaleTest;推送前按登记表 50 个值与高熵串扫过 920 行新增,0 命中。
This commit is contained in:
+32
-2
@@ -746,7 +746,9 @@ public class AppearancePatentTaskService {
|
||||
* 每分钟刷新——主线程卡死时心跳线程照发,任务永远不 stale(与生产 28131 同型缺口)。
|
||||
* 本线改看 biz_task_scope_state.last_chunk_at(仅在 persistSubmittedChunk 上传分片时刷新),
|
||||
* 命中后走既有 finalizeStaleTask(封口上传 + LLM 收尾,不粗暴杀任务)。
|
||||
* 从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。
|
||||
* 从未上报(无 scope 行或全 NULL)的任务不再无条件跳过,改按「处理位置是否仍在推进」区分:
|
||||
* 位置在变=首批较慢,跳过;位置长时间不变=判死并写清停在哪;无位置信息(老客户端)=证据不足,
|
||||
* 保守跳过但留 WARN —— 2026-10-06 品牌任务 2626 正是靠无条件跳过挂到 9 小时。
|
||||
*/
|
||||
private void finalizeNoUploadStaleTasks() {
|
||||
long minutes = properties.getNoResultUploadTimeoutMinutes();
|
||||
@@ -774,7 +776,35 @@ public class AppearancePatentTaskService {
|
||||
continue;
|
||||
}
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||
if (lastResultAt == null) {
|
||||
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成一句静默 continue,
|
||||
// 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口(本模块同样心跳照打)。
|
||||
// 现按「处理位置是否仍在推进」三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
|
||||
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
|
||||
String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId());
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[appearance-patent] 从未上报但客户端仍在推进,本轮不封口 taskId={} position={} minutes={}",
|
||||
task.getId(), neverReportedPosition, minutes);
|
||||
continue;
|
||||
}
|
||||
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
|
||||
log.warn("[appearance-patent] 从未上报且无位置信息,证据不足暂不封口 taskId={} createdAt={} minutes={}",
|
||||
task.getId(), task.getCreatedAt(), minutes);
|
||||
continue;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle neverReportedLock = acquireTaskLock(task.getId(), 0L);
|
||||
if (neverReportedLock == null) {
|
||||
continue;
|
||||
}
|
||||
try (neverReportedLock) {
|
||||
log.warn("[appearance-patent] 从未上报任何结果分片且处理位置长时间停在「{}」未推进,按卡死收尾 taskId={} minutes={}",
|
||||
neverReportedPosition, task.getId(), minutes);
|
||||
finalizeStaleTask(task.getId(), "超过 " + minutes + " 分钟无任何结果分片上报,处理位置长时间停在「"
|
||||
+ neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (lastResultAt.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
|
||||
|
||||
+35
-6
@@ -634,7 +634,27 @@ public class CollectDataService {
|
||||
}
|
||||
for (FileTaskEntity task : candidates) {
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||
if (lastResultAt == null) {
|
||||
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成静默 continue,是 2626 那型出口。
|
||||
// 现按「处理位置是否仍在推进」三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
|
||||
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
|
||||
String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId());
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[collect-data] never-reported skip because client still advancing taskId={} position={} minutes={}",
|
||||
task.getId(), neverReportedPosition, minutes);
|
||||
continue;
|
||||
}
|
||||
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
|
||||
log.warn("[collect-data] never-reported skip without position evidence taskId={} createdAt={} minutes={}",
|
||||
task.getId(), task.getCreatedAt(), minutes);
|
||||
continue;
|
||||
}
|
||||
log.warn("[collect-data] never-reported but position stuck, failing taskId={} position={} minutes={}",
|
||||
task.getId(), neverReportedPosition, minutes);
|
||||
finalizeNoUploadStaleTask(task.getId(), null, neverReportedPosition, minutes);
|
||||
continue;
|
||||
}
|
||||
if (lastResultAt.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
|
||||
@@ -645,12 +665,12 @@ public class CollectDataService {
|
||||
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
|
||||
continue;
|
||||
}
|
||||
finalizeNoUploadStaleTask(task.getId(), lastResultAt, minutes);
|
||||
finalizeNoUploadStaleTask(task.getId(), lastResultAt, null, minutes);
|
||||
}
|
||||
}
|
||||
|
||||
/** 单个任务的二次判死收尾:锁内复查心跳后,有分片→组装部分工作簿;无分片→标失败。 */
|
||||
private void finalizeNoUploadStaleTask(Long taskId, LocalDateTime lastResultAt, long minutes) {
|
||||
private void finalizeNoUploadStaleTask(Long taskId, LocalDateTime lastResultAt, String position, long minutes) {
|
||||
try (TaskDistributedLockService.LockHandle lock =
|
||||
taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) {
|
||||
if (lock == null) {
|
||||
@@ -677,9 +697,7 @@ public class CollectDataService {
|
||||
taskId, lastResultAt, minutes, stats.finalRowCount);
|
||||
return;
|
||||
}
|
||||
markTaskFailed(task, result,
|
||||
"连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")",
|
||||
stats);
|
||||
markTaskFailed(task, result, noUploadFailReason(minutes, lastResultAt, position), stats);
|
||||
log.warn("[collect-data] no-upload stale task failed without chunks taskId={} lastResultAt={} timeoutMinutes={}",
|
||||
taskId, lastResultAt, minutes);
|
||||
} catch (Exception ex) {
|
||||
@@ -688,6 +706,17 @@ public class CollectDataService {
|
||||
}
|
||||
}
|
||||
|
||||
/** 二次判死原因文案;从未上报与已有上报分开写(后者不能只说「最后结果上报 null」)。 */
|
||||
private String noUploadFailReason(long minutes, LocalDateTime lastResultAt, String position) {
|
||||
if (lastResultAt == null) {
|
||||
String where = position == null || position.isBlank()
|
||||
? "" : ",处理位置长时间停在「" + position + "」未推进";
|
||||
return "任务创建至今超过 " + minutes + " 分钟无任何结果回传" + where
|
||||
+ ",疑似客户端任务卡死,任务已自动失败";
|
||||
}
|
||||
return "连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")";
|
||||
}
|
||||
|
||||
public CollectDataDashboardVo dashboard(Long userId) {
|
||||
CollectDataDashboardVo vo = new CollectDataDashboardVo();
|
||||
vo.setPendingTaskCount(countActiveTasks(userId));
|
||||
|
||||
+76
-41
@@ -760,7 +760,9 @@ public class DeleteBrandStaleTaskService {
|
||||
* <p>既有各模块心跳线的候选条件都是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
|
||||
* 主线程卡死(浏览器自动化等待/异常)时心跳线程照发,任务永远命不中(生产 28131 卡死 12h+ 仍 RUNNING)。
|
||||
* 本线改用 biz_task_scope_state.last_chunk_at(只随结果分片上报刷新)作判据;
|
||||
* 无 scope 行或 last_chunk_at 全 NULL(从未上报)的任务跳过,避免误杀首批较慢的正常任务。
|
||||
* 从未上报(无 scope 行或 last_chunk_at 全 NULL)不再无条件跳过——与上报停摆同样按处理位置三分:
|
||||
* 仍在推进=首批较慢跳过、位置久未变=判死、无位置信息=证据不足保守跳过但留 WARN
|
||||
* (原先的无条件跳过是 2026-10-06 品牌任务 2626 挂 9 小时的同类出口)。
|
||||
*/
|
||||
private NoResultUploadStaleCheckStats failNoResultUploadTasks() {
|
||||
NoResultUploadStaleCheckStats stats = new NoResultUploadStaleCheckStats();
|
||||
@@ -788,11 +790,32 @@ public class DeleteBrandStaleTaskService {
|
||||
.collect(Collectors.toMap(TaskScopeLastChunkDto::taskId, TaskScopeLastChunkDto::lastChunkAt, (a, b) -> a));
|
||||
for (FileTaskEntity task : candidates) {
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||
// 从未上报(无 scope 行或全 NULL)或上报仍新鲜:正常推进,不必处理
|
||||
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||
if (lastResultAt != null && lastResultAt.isAfter(cutoff)) {
|
||||
// 上报仍新鲜:正常推进,不必处理
|
||||
continue;
|
||||
}
|
||||
String moduleType = task.getModuleType();
|
||||
if (lastResultAt == null) {
|
||||
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成一句静默 continue(零日志),
|
||||
// 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口(本模块心跳同样照打)。
|
||||
// 现按处理位置三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
|
||||
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
|
||||
String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId());
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[stale-check] no-upload never-reported skip because client still advancing taskId={} moduleType={} position={}",
|
||||
task.getId(), moduleType, neverReportedPosition);
|
||||
continue;
|
||||
}
|
||||
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
|
||||
stats.skippedTaskCount++;
|
||||
log.warn("[stale-check] no-upload never-reported skip without position evidence taskId={} moduleType={} createdAt={} timeoutMinutes={}",
|
||||
task.getId(), moduleType, task.getCreatedAt(), minutes);
|
||||
continue;
|
||||
}
|
||||
failNoResultUploadTask(moduleType, task, null, minutes, stats);
|
||||
continue;
|
||||
}
|
||||
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
|
||||
// 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
|
||||
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
|
||||
@@ -802,47 +825,53 @@ public class DeleteBrandStaleTaskService {
|
||||
task.getId(), moduleType, lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
|
||||
continue;
|
||||
}
|
||||
if (hasPendingAssembleJobs(task.getId(), moduleType)) {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[stale-check] no-upload skip pending-assemble-jobs taskId={} moduleType={}", task.getId(), moduleType);
|
||||
continue;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(moduleType, task.getId());
|
||||
if (taskLockHandle == null) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
try (taskLockHandle) {
|
||||
try {
|
||||
tryFinalizeNoResultUploadTask(moduleType, task.getId());
|
||||
} catch (Exception ex) {
|
||||
// finalize 抛异常(组装失败等)不代表任务活着:继续走 CAS 判死
|
||||
log.warn("[stale-check] no-upload finalize threw taskId={} moduleType={} msg={}", task.getId(), moduleType, ex.getMessage());
|
||||
}
|
||||
// finalize 已终结的任务 CAS 自然不命中(状态不再是 RUNNING),无需回读
|
||||
String lastPosition = taskHeartbeatPositionService.describe(task.getId());
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getModuleType, moduleType)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.set(FileTaskEntity::getStatus, "FAILED")
|
||||
.set(FileTaskEntity::getErrorMessage, buildNoResultUploadFailReason(minutes, lastResultAt, lastPosition))
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
stats.failedTaskCount++;
|
||||
deleteNoResultUploadTaskCache(moduleType, task.getId());
|
||||
log.warn("[stale-check] no-upload failed taskId={} moduleType={} lastResultAt={} taskUpdatedAt={} timeoutMinutes={}",
|
||||
task.getId(), moduleType, lastResultAt, task.getUpdatedAt(), minutes);
|
||||
} else {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[stale-check] no-upload task already finalized by compensation taskId={} moduleType={}", task.getId(), moduleType);
|
||||
}
|
||||
}
|
||||
failNoResultUploadTask(moduleType, task, lastResultAt, minutes, stats);
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
/** 二次判死的共同尾部:待组装检查 → 任务锁 → 尽力收尾 → CAS 翻转;上报过与从未上报两条路共用。 */
|
||||
private void failNoResultUploadTask(String moduleType, FileTaskEntity task, LocalDateTime lastResultAt,
|
||||
long minutes, NoResultUploadStaleCheckStats stats) {
|
||||
if (hasPendingAssembleJobs(task.getId(), moduleType)) {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[stale-check] no-upload skip pending-assemble-jobs taskId={} moduleType={}", task.getId(), moduleType);
|
||||
return;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(moduleType, task.getId());
|
||||
if (taskLockHandle == null) {
|
||||
stats.skippedTaskCount++;
|
||||
return;
|
||||
}
|
||||
try (taskLockHandle) {
|
||||
try {
|
||||
tryFinalizeNoResultUploadTask(moduleType, task.getId());
|
||||
} catch (Exception ex) {
|
||||
// finalize 抛异常(组装失败等)不代表任务活着:继续走 CAS 判死
|
||||
log.warn("[stale-check] no-upload finalize threw taskId={} moduleType={} msg={}", task.getId(), moduleType, ex.getMessage());
|
||||
}
|
||||
// finalize 已终结的任务 CAS 自然不命中(状态不再是 RUNNING),无需回读
|
||||
String lastPosition = taskHeartbeatPositionService.describe(task.getId());
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getModuleType, moduleType)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.set(FileTaskEntity::getStatus, "FAILED")
|
||||
.set(FileTaskEntity::getErrorMessage, buildNoResultUploadFailReason(minutes, lastResultAt, lastPosition))
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
stats.failedTaskCount++;
|
||||
deleteNoResultUploadTaskCache(moduleType, task.getId());
|
||||
log.warn("[stale-check] no-upload failed taskId={} moduleType={} lastResultAt={} taskUpdatedAt={} timeoutMinutes={}",
|
||||
task.getId(), moduleType, lastResultAt, task.getUpdatedAt(), minutes);
|
||||
} else {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[stale-check] no-upload task already finalized by compensation taskId={} moduleType={}", task.getId(), moduleType);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 按模块调用各自的收尾入口(尽力组装部分结果;已终结的任务由后续 CAS 自然放行)。 */
|
||||
private void tryFinalizeNoResultUploadTask(String moduleType, Long taskId) {
|
||||
switch (moduleType) {
|
||||
@@ -871,8 +900,14 @@ public class DeleteBrandStaleTaskService {
|
||||
}
|
||||
}
|
||||
|
||||
/** 二次判死文案:写明最后结果上报时间与最后处理位置,便于区分「卡死」与「长间隔」并定位卡点。 */
|
||||
/** 二次判死文案:上报过=写明最后结果时间与处理位置;从未上报=写明创建至今与卡住位置。 */
|
||||
private String buildNoResultUploadFailReason(long minutes, LocalDateTime lastResultAt, String position) {
|
||||
if (lastResultAt == null) {
|
||||
String where = position == null || position.isBlank()
|
||||
? "" : ",处理位置长时间停在「" + position + "」未推进";
|
||||
return "任务创建至今超过 " + minutes + " 分钟无任何结果回传" + where
|
||||
+ ",疑似客户端任务卡死,任务已自动失败";
|
||||
}
|
||||
StringBuilder sb = new StringBuilder("连续 ").append(minutes)
|
||||
.append(" 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 ").append(lastResultAt);
|
||||
if (position != null && !position.isBlank()) {
|
||||
|
||||
+56
-19
@@ -46,6 +46,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
|
||||
@@ -123,6 +124,7 @@ public class PublishTaskService {
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
|
||||
@Value("${aiimage.publish.stale-timeout-minutes:30}")
|
||||
private int staleTimeoutMinutes;
|
||||
@@ -876,7 +878,9 @@ public class PublishTaskService {
|
||||
* <p>既有心跳线候选条件是 updated_at 陈旧,而客户端心跳会持续刷新 updated_at——
|
||||
* 主线程卡死时心跳线程照发,任务永远命不中(与生产 28131 同型缺口)。
|
||||
* owner 过滤与 A 线保持一致(避免双节点同时处理非本实例任务);
|
||||
* 判据用 biz_task_scope_state.last_chunk_at(仅分片上传时刷新);从未上报跳过,避免误杀首批较慢的正常任务。
|
||||
* 判据用 biz_task_scope_state.last_chunk_at(仅分片上传时刷新);从未上报不再无条件跳过——
|
||||
* 与上报停摆同样按处理位置三分:仍在推进=首批较慢跳过、位置久未变=判死、无位置信息=证据不足保守跳过但留 WARN
|
||||
* (原先的无条件跳过是 2026-10-06 品牌任务 2626 挂 9 小时的同类出口)。
|
||||
*/
|
||||
private void failNoUploadStaleTasks() {
|
||||
long minutes = noResultUploadTimeoutMinutes;
|
||||
@@ -910,33 +914,66 @@ public class PublishTaskService {
|
||||
}
|
||||
for (FileTaskEntity candidate : candidates) {
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(candidate.getId());
|
||||
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||
if (lastResultAt != null && lastResultAt.isAfter(cutoff)) {
|
||||
// 上报仍新鲜:正常推进,不必处理
|
||||
continue;
|
||||
}
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(candidate.getId(), MODULE_TYPE) > 0L) {
|
||||
continue;
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle lock =
|
||||
taskDistributedLockService.acquire(MODULE_TYPE, candidate.getId(), 0L)) {
|
||||
if (lock == null) {
|
||||
if (lastResultAt == null) {
|
||||
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成一句静默 continue(零日志),
|
||||
// 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口(本模块心跳同样照打)。
|
||||
// 现按处理位置三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
|
||||
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
|
||||
String neverReportedPosition = taskHeartbeatPositionService.describe(candidate.getId());
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(candidate.getId(), minutes)) {
|
||||
log.info("[publish] 从未上报但客户端仍在推进,本轮不判死 taskId={} position={} minutes={}",
|
||||
candidate.getId(), neverReportedPosition, minutes);
|
||||
continue;
|
||||
}
|
||||
String error = "连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")";
|
||||
transactionTemplate.executeWithoutResult(status -> failStaleTaskLockedInternal(
|
||||
candidate.getId(),
|
||||
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
|
||||
log.warn("[publish] 从未上报且无位置信息,证据不足暂不判死 taskId={} createdAt={} minutes={}",
|
||||
candidate.getId(), candidate.getCreatedAt(), minutes);
|
||||
continue;
|
||||
}
|
||||
failNoUploadStaleTask(candidate,
|
||||
"任务创建至今超过 " + minutes + " 分钟无任何结果回传,处理位置长时间停在「"
|
||||
+ neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败",
|
||||
task -> {
|
||||
// 排队期间复查:心跳回落陈旧 → 交回 A 线;出现新结果上报 → 任务已恢复
|
||||
// 排队期间复查:心跳回落陈旧 → 交回 A 线;出现任一结果上报 → 已恢复回传,交回原线评估
|
||||
if (task.getUpdatedAt() == null || task.getUpdatedAt().isBefore(heartbeatThreshold)) {
|
||||
return false;
|
||||
}
|
||||
LocalDateTime freshLastChunk = queryLastChunkAt(task.getId());
|
||||
return freshLastChunk != null && !freshLastChunk.isAfter(cutoff);
|
||||
},
|
||||
error));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[publish] no-upload stale task cleanup failed taskId={} msg={}",
|
||||
candidate.getId(), ex.getMessage());
|
||||
return queryLastChunkAt(task.getId()) == null;
|
||||
});
|
||||
continue;
|
||||
}
|
||||
failNoUploadStaleTask(candidate,
|
||||
"连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")",
|
||||
task -> {
|
||||
// 排队期间复查:心跳回落陈旧 → 交回 A 线;出现新结果上报 → 任务已恢复
|
||||
if (task.getUpdatedAt() == null || task.getUpdatedAt().isBefore(heartbeatThreshold)) {
|
||||
return false;
|
||||
}
|
||||
LocalDateTime freshLastChunk = queryLastChunkAt(task.getId());
|
||||
return freshLastChunk != null && !freshLastChunk.isAfter(cutoff);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** 二次判死共同尾部:待组装检查 → 任务锁 → 收尾;error 与复查谓词由两条路各自提供。 */
|
||||
private void failNoUploadStaleTask(FileTaskEntity candidate, String error, Predicate<FileTaskEntity> staleCheck) {
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(candidate.getId(), MODULE_TYPE) > 0L) {
|
||||
return;
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle lock =
|
||||
taskDistributedLockService.acquire(MODULE_TYPE, candidate.getId(), 0L)) {
|
||||
if (lock == null) {
|
||||
return;
|
||||
}
|
||||
transactionTemplate.executeWithoutResult(status -> failStaleTaskLockedInternal(
|
||||
candidate.getId(), staleCheck, error));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[publish] no-upload stale task cleanup failed taskId={} msg={}",
|
||||
candidate.getId(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+59
-18
@@ -43,6 +43,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFinalizeSupport;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
@@ -127,6 +128,7 @@ public class ShopDataCrawlTaskService {
|
||||
private final DuplicateCheckRefreshPort duplicateCheckRefreshPort;
|
||||
private final PlatformTransactionManager transactionManager;
|
||||
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
|
||||
@Value("${aiimage.shop-data-crawl.stale-timeout-minutes:30}")
|
||||
private long staleTimeoutMinutes;
|
||||
@@ -222,7 +224,9 @@ public class ShopDataCrawlTaskService {
|
||||
* <p>既有心跳线候选条件是心跳/updated_at 陈旧,而 Python 的 HTTP 心跳线程在任务主线程
|
||||
* 卡死时照常发送——心跳永远新鲜,任务永远命不中(与生产 28131 同型缺口)。
|
||||
* 本线候选取「心跳新鲜 + 创建超过 N 分钟」,判据用 biz_task_scope_state.last_chunk_at
|
||||
* (仅分片上传时刷新);从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。
|
||||
* (仅分片上传时刷新);从未上报(无 scope 行或全 NULL)不再无条件跳过——与上报停摆同样
|
||||
* 按处理位置三分:仍在推进=首批较慢跳过、位置久未变=判死、无位置信息=证据不足保守跳过但留 WARN
|
||||
* (原先的无条件跳过是 2026-10-06 品牌任务 2626 挂 9 小时的同类出口)。
|
||||
*/
|
||||
private void finalizeNoUploadStaleTasks() {
|
||||
long minutes = noResultUploadTimeoutMinutes;
|
||||
@@ -264,30 +268,67 @@ public class ShopDataCrawlTaskService {
|
||||
continue;
|
||||
}
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||
if (lastResultAt != null && lastResultAt.isAfter(cutoff)) {
|
||||
// 上报仍新鲜:正常推进,不必处理
|
||||
continue;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle taskLock = acquireTaskLock(task.getId(), 0L);
|
||||
if (taskLock == null) {
|
||||
log.info("[shop-data-crawl] no-upload stale task skipped because task lock is busy taskId={}", task.getId());
|
||||
continue;
|
||||
}
|
||||
try (taskLock) {
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(task.getId(), MODULE_TYPE) > 0L) continue;
|
||||
if (!tryFinalizeTask(task.getId(), true, true)) {
|
||||
markTaskFailedIfStillRunning(task.getId(),
|
||||
"连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")");
|
||||
if (lastResultAt == null) {
|
||||
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成一句静默 continue(零日志),
|
||||
// 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口(本模块心跳同样照打)。
|
||||
// 现按处理位置三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
|
||||
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
|
||||
String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId());
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[shop-data-crawl] 从未上报但客户端仍在推进,本轮不判死 taskId={} position={} minutes={}",
|
||||
task.getId(), neverReportedPosition, minutes);
|
||||
continue;
|
||||
}
|
||||
log.warn("[shop-data-crawl] no-upload stale task finalized taskId={} lastResultAt={} timeoutMinutes={}",
|
||||
task.getId(), lastResultAt, minutes);
|
||||
} catch (TaskOwnerMismatchException ignored) {
|
||||
// The owner may change between the scan and finalization.
|
||||
} catch (Exception ex) {
|
||||
log.warn("[shop-data-crawl] no-upload stale task finalization failed taskId={} msg={}", task.getId(), ex.getMessage());
|
||||
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
|
||||
log.warn("[shop-data-crawl] 从未上报且无位置信息,证据不足暂不判死 taskId={} createdAt={} minutes={}",
|
||||
task.getId(), task.getCreatedAt(), minutes);
|
||||
continue;
|
||||
}
|
||||
failNoResultUploadTask(task, minutes, null, neverReportedPosition);
|
||||
continue;
|
||||
}
|
||||
failNoResultUploadTask(task, minutes, lastResultAt, null);
|
||||
}
|
||||
}
|
||||
|
||||
/** 二次判死的共同尾部:任务锁 → 待组装检查 → 收尾;上报停摆与从未上报两条路共用。 */
|
||||
private void failNoResultUploadTask(FileTaskEntity task, long minutes,
|
||||
LocalDateTime lastResultAt, String neverReportedPosition) {
|
||||
TaskDistributedLockService.LockHandle taskLock = acquireTaskLock(task.getId(), 0L);
|
||||
if (taskLock == null) {
|
||||
log.info("[shop-data-crawl] no-upload stale task skipped because task lock is busy taskId={}", task.getId());
|
||||
return;
|
||||
}
|
||||
try (taskLock) {
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(task.getId(), MODULE_TYPE) > 0L) return;
|
||||
if (!tryFinalizeTask(task.getId(), true, true)) {
|
||||
markTaskFailedIfStillRunning(task.getId(),
|
||||
noUploadFailReason(minutes, lastResultAt, neverReportedPosition));
|
||||
}
|
||||
log.warn("[shop-data-crawl] no-upload stale task finalized taskId={} lastResultAt={} timeoutMinutes={}",
|
||||
task.getId(), lastResultAt, minutes);
|
||||
} catch (TaskOwnerMismatchException ignored) {
|
||||
// The owner may change between the scan and finalization.
|
||||
} catch (Exception ex) {
|
||||
log.warn("[shop-data-crawl] no-upload stale task finalization failed taskId={} msg={}", task.getId(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/** 二次判死原因文案;从未上报与已有上报分开写(后者不能只说「最后结果上报 null」)。 */
|
||||
private String noUploadFailReason(long minutes, LocalDateTime lastResultAt, String neverReportedPosition) {
|
||||
if (lastResultAt == null) {
|
||||
String where = neverReportedPosition == null || neverReportedPosition.isBlank()
|
||||
? "" : ",处理位置长时间停在「" + neverReportedPosition + "」未推进";
|
||||
return "任务创建至今超过 " + minutes + " 分钟无任何结果回传" + where
|
||||
+ ",疑似客户端任务卡死,任务已自动失败";
|
||||
}
|
||||
return "连续 " + minutes + " 分钟无结果回传,疑似客户端任务卡死,任务已自动失败(最后结果上报 " + lastResultAt + ")";
|
||||
}
|
||||
|
||||
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
||||
return TaskFinalizeSupport.loadTaskForExecution(fileTaskMapper, taskCacheService, MODULE_TYPE, taskId);
|
||||
}
|
||||
|
||||
+36
-2
@@ -972,7 +972,9 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
* 每分钟刷新——主线程卡死时心跳线程照发,任务永远不 stale(与生产 28131 同型缺口)。
|
||||
* 本线改看 biz_task_scope_state.last_chunk_at(仅在 persistSubmittedChunk 上传分片时刷新),
|
||||
* 命中后走既有 finalizeStaleTask(封口上传 + LLM 收尾,不粗暴杀任务)。
|
||||
* 从未上报(无 scope 行或全 NULL)的任务跳过,避免误杀首批较慢的正常任务。
|
||||
* 从未上报(无 scope 行或全 NULL)不再无条件跳过——与上报停摆同样按处理位置三分:
|
||||
* 仍在推进=首批较慢跳过、位置久未变=判死、无位置信息=证据不足保守跳过但留 WARN
|
||||
* (原先的无条件跳过是 2026-10-06 品牌任务 2626 挂 9 小时的同类出口)。
|
||||
*/
|
||||
private void finalizeNoUploadStaleTasks() {
|
||||
long minutes = properties.getNoResultUploadTimeoutMinutes();
|
||||
@@ -1000,7 +1002,39 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
continue;
|
||||
}
|
||||
LocalDateTime lastResultAt = lastResultAtByTaskId.get(task.getId());
|
||||
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||
if (lastResultAt == null) {
|
||||
// 从未上报任何结果分片:原先与「上报仍新鲜」合并成一句静默 continue(零日志),
|
||||
// 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口(本模块心跳同样照打)。
|
||||
// 现按处理位置三分:在推进=首批较慢,跳过;位置久未变=判死并写清卡在哪;
|
||||
// 无位置信息(老客户端 / 条目被 prune)=证据不足,保守跳过但留 WARN。
|
||||
String neverReportedPosition = taskHeartbeatPositionService.describe(task.getId());
|
||||
TaskHeartbeatPositionService.StallState neverReportedStall =
|
||||
taskHeartbeatPositionService.evaluateStall(task.getId(), minutes);
|
||||
if (neverReportedStall == TaskHeartbeatPositionService.StallState.ADVANCING) {
|
||||
log.info("[similar-asin] 从未上报但客户端仍在推进,本轮不封口 taskId={} position={} minutes={}",
|
||||
task.getId(), neverReportedPosition, minutes);
|
||||
continue;
|
||||
}
|
||||
if (neverReportedPosition == null || neverReportedPosition.isBlank()) {
|
||||
log.warn("[similar-asin] 从未上报且无位置信息,证据不足暂不封口 taskId={} createdAt={} minutes={}",
|
||||
task.getId(), task.getCreatedAt(), minutes);
|
||||
continue;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle neverReportedLock =
|
||||
ownershipSupport().acquireTaskLock(task.getId(), 0L);
|
||||
if (neverReportedLock == null) {
|
||||
continue;
|
||||
}
|
||||
try (neverReportedLock) {
|
||||
log.warn("[similar-asin] 从未上报任何结果分片且处理位置长时间停在「{}」未推进,按卡死收尾 taskId={} minutes={}",
|
||||
neverReportedPosition, task.getId(), minutes);
|
||||
finalizeStaleTask(task.getId(),
|
||||
"任务创建至今超过 " + minutes + " 分钟无任何结果回传,处理位置长时间停在「"
|
||||
+ neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (lastResultAt.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
// 「客户端仍在推进」豁免:位置文本还在变说明采集主线程在干活,只是分片攒着没回传
|
||||
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* appearance-patent 二次判死线「从未上报」出口的三分行为(2026-10-06 品牌任务 2626 同型缺口)。
|
||||
*
|
||||
* <p>原先「从未上报任何结果分片」与「上报仍新鲜」合并成一句静默 continue(零日志),
|
||||
* 心跳照打的任务可以无限期挂着。现按处理位置三分:仍在推进=跳过、位置久未变=判死、
|
||||
* 无位置信息=证据不足保守跳过但留 WARN。判死细节(封口文案等)由 finalizeStaleTask 既有链路承接,
|
||||
* 本测试以「是否进入抢锁判死」为界。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class AppearancePatentNoUploadStaleTest {
|
||||
|
||||
private static final long TASK_ID = 91001L;
|
||||
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private AppearancePatentTaskCacheService taskCacheService;
|
||||
@Mock private AppearancePatentProperties properties;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
|
||||
@InjectMocks private AppearancePatentTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeTableInfo() {
|
||||
TableInfoHelper.initTableInfo(
|
||||
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
when(properties.getNoResultUploadTimeoutMinutes()).thenReturn(180);
|
||||
when(properties.getStaleTimeoutMinutes()).thenReturn(30);
|
||||
}
|
||||
|
||||
private FileTaskEntity runningTask() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
|
||||
task.setStatus("RUNNING");
|
||||
task.setCreatedAt(LocalDateTime.now().minusDays(1));
|
||||
task.setUpdatedAt(LocalDateTime.now().minusMinutes(1));
|
||||
return task;
|
||||
}
|
||||
|
||||
/** 心跳新鲜 + 无任何结果分片上报(scope 行为空)= 「从未上报」候选。 */
|
||||
private void givenNeverReported() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(runningTask()));
|
||||
when(taskCacheService.getTaskHeartbeatMillis(TASK_ID)).thenReturn(System.currentTimeMillis());
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
private void invokeNoUploadCheck() {
|
||||
ReflectionTestUtils.invokeMethod(service, "finalizeNoUploadStaleTasks");
|
||||
}
|
||||
|
||||
/** 从未上报 + 位置久未推进 = 2626 同型真卡死,进入抢锁判死流程。 */
|
||||
@Test
|
||||
void 从未上报但位置停滞时按卡死收尾() {
|
||||
givenNeverReported();
|
||||
when(taskHeartbeatPositionService.describe(TASK_ID)).thenReturn("ASIN B01A / 美国");
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(TASK_ID, 180L)).thenReturn(false);
|
||||
when(taskDistributedLockService.acquire(anyString(), anyLong(), any(Duration.class), anyLong()))
|
||||
.thenReturn(null);
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
// 已进入判死流程(开始抢锁);锁被占则本轮放弃
|
||||
verify(taskDistributedLockService).acquire(anyString(), anyLong(), any(Duration.class), anyLong());
|
||||
}
|
||||
|
||||
/** 从未上报 + 位置仍在推进 = 首批较慢,不封口(否则拒收后续分片丢数据)。 */
|
||||
@Test
|
||||
void 从未上报但客户端仍在推进时不封口() {
|
||||
givenNeverReported();
|
||||
when(taskHeartbeatPositionService.describe(TASK_ID)).thenReturn("ASIN B01A / 美国");
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(TASK_ID, 180L)).thenReturn(true);
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
verify(taskDistributedLockService, never()).acquire(anyString(), anyLong(), any(Duration.class), anyLong());
|
||||
}
|
||||
|
||||
/** 从未上报 + 无位置信息(老客户端/条目被 prune)= 证据不足,保守跳过但留 WARN。 */
|
||||
@Test
|
||||
void 从未上报且无位置信息时保守跳过() {
|
||||
givenNeverReported();
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
verify(taskDistributedLockService, never()).acquire(anyString(), anyLong(), any(Duration.class), anyLong());
|
||||
}
|
||||
}
|
||||
+39
-1
@@ -92,7 +92,7 @@ class CollectDataNoUploadStaleTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void 从未上报时跳过() {
|
||||
void 从未上报且无位置信息时保守跳过() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(runningTask(2L)));
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())).thenReturn(List.of());
|
||||
|
||||
@@ -143,4 +143,42 @@ class CollectDataNoUploadStaleTest {
|
||||
assertTrue(taskCaptor.getValue().getErrorMessage().contains("无结果回传"),
|
||||
taskCaptor.getValue().getErrorMessage());
|
||||
}
|
||||
|
||||
/** 从未上报(无 scope 行)+ 位置久未推进 = 2626 同型真卡死,判死并在文案里写清卡在哪。 */
|
||||
@Test
|
||||
void 从未上报但位置久未推进时判死() {
|
||||
FileTaskEntity task = runningTask(4L);
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())).thenReturn(List.of());
|
||||
when(taskHeartbeatPositionService.describe(4L)).thenReturn("分页 3/10(共 10 页)");
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(4L, 180L)).thenReturn(false);
|
||||
when(taskDistributedLockService.acquire(any(), anyLong(), anyLong())).thenReturn(lockHandle);
|
||||
when(fileTaskMapper.selectById(4L)).thenReturn(task);
|
||||
when(taskFileJobService.countUnfinishedAssembleJobs(4L, "COLLECT_DATA")).thenReturn(0L);
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(0L);
|
||||
when(fileResultMapper.selectOne(any())).thenReturn(null);
|
||||
|
||||
invokeNoUpload();
|
||||
|
||||
ArgumentCaptor<FileTaskEntity> taskCaptor = ArgumentCaptor.forClass(FileTaskEntity.class);
|
||||
verify(fileTaskMapper).updateById(taskCaptor.capture());
|
||||
assertEquals("FAILED", taskCaptor.getValue().getStatus());
|
||||
assertTrue(taskCaptor.getValue().getErrorMessage().contains("无任何结果回传"),
|
||||
taskCaptor.getValue().getErrorMessage());
|
||||
assertTrue(taskCaptor.getValue().getErrorMessage().contains("分页 3/10(共 10 页)"),
|
||||
taskCaptor.getValue().getErrorMessage());
|
||||
}
|
||||
|
||||
/** 从未上报 + 位置仍在推进 = 首批较慢的正常任务,跳过不判死。 */
|
||||
@Test
|
||||
void 从未上报但位置仍在推进时跳过() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(runningTask(6L)));
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())).thenReturn(List.of());
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(6L, 180L)).thenReturn(true);
|
||||
|
||||
invokeNoUpload();
|
||||
|
||||
verify(taskDistributedLockService, never()).acquire(any(), anyLong(), anyLong());
|
||||
verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class));
|
||||
}
|
||||
}
|
||||
|
||||
+42
-1
@@ -266,7 +266,7 @@ class DeleteBrandStaleTaskServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void noUploadNeverUploadedIsSkipped() {
|
||||
void noUploadNeverUploadedWithoutPositionIsSkipped() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(noUploadTask(903L, "PRODUCT_RISK_RESOLVE")));
|
||||
noUploadEnabled();
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())).thenReturn(List.of());
|
||||
@@ -278,6 +278,47 @@ class DeleteBrandStaleTaskServiceTest {
|
||||
verify(fileTaskMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
/** 从未上报(无 scope 行)+ 位置久未推进 = 2626 同型真卡死,判死并在文案里写清卡在哪。 */
|
||||
@Test
|
||||
void noUploadNeverReportedStalledPositionIsFailed() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(noUploadTask(909L, "PRODUCT_RISK_RESOLVE")));
|
||||
noUploadEnabled();
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())).thenReturn(List.of());
|
||||
when(taskHeartbeatPositionService.describe(909L)).thenReturn("店铺 魏振峰(2/10)");
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(909L, 180L)).thenReturn(false);
|
||||
when(taskFileJobService.countUnfinishedAssembleJobs(909L, "PRODUCT_RISK_RESOLVE")).thenReturn(0L);
|
||||
noUploadTaskLockAvailable();
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
|
||||
DeleteBrandStaleTaskService service = service();
|
||||
ReflectionTestUtils.invokeMethod(service, "failNoResultUploadTasks");
|
||||
|
||||
verify(productRiskTaskService).tryFinalizeTask(909L, true);
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
ArgumentCaptor<LambdaUpdateWrapper> update = ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(fileTaskMapper).update(isNull(), update.capture());
|
||||
List<String> values = update.getValue().getParamNameValuePairs().values().stream()
|
||||
.map(String::valueOf)
|
||||
.toList();
|
||||
assertTrue(values.stream().anyMatch(v -> v.contains("无任何结果回传")), values.toString());
|
||||
assertTrue(values.stream().anyMatch(v -> v.contains("店铺 魏振峰(2/10)")), values.toString());
|
||||
}
|
||||
|
||||
/** 从未上报 + 位置仍在推进 = 首批较慢的正常任务,跳过不判死。 */
|
||||
@Test
|
||||
void noUploadNeverReportedAdvancingIsSkipped() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(noUploadTask(910L, "PRODUCT_RISK_RESOLVE")));
|
||||
noUploadEnabled();
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())).thenReturn(List.of());
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(910L, 180L)).thenReturn(true);
|
||||
|
||||
DeleteBrandStaleTaskService service = service();
|
||||
ReflectionTestUtils.invokeMethod(service, "failNoResultUploadTasks");
|
||||
|
||||
verify(taskDistributedLockService, never()).acquire(any(), any(), anyLong());
|
||||
verify(fileTaskMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void noUploadAlreadyFinalizedTaskKeepsCache() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(noUploadTask(904L, "PRODUCT_RISK_RESOLVE")));
|
||||
|
||||
+3
-1
@@ -28,6 +28,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||
@@ -303,7 +304,8 @@ class PublishBatchLoadingSnapshotEquivTest {
|
||||
ossStorageService, new ObjectMapper(), mock(TransactionTemplate.class),
|
||||
mock(com.nanri.aiimage.common.service.DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS),
|
||||
mock(InstanceMetadata.class),
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(TaskHeartbeatPositionService.class));
|
||||
}
|
||||
|
||||
private static FileTaskEntity task(long id, String status, LocalDateTime createdAt, Integer src, Integer ok, Integer fail) {
|
||||
|
||||
+3
-1
@@ -24,6 +24,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||
@@ -181,7 +182,8 @@ class PublishDashboardAggregateTest {
|
||||
ossStorageService, new ObjectMapper(), mock(TransactionTemplate.class),
|
||||
mock(com.nanri.aiimage.common.service.DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS),
|
||||
mock(InstanceMetadata.class),
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(TaskHeartbeatPositionService.class));
|
||||
}
|
||||
|
||||
private static FileTaskEntity task(long id, long userId, String status, LocalDateTime createdAt) {
|
||||
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
package com.nanri.aiimage.modules.publish.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.publish.mapper.PublishFileMapper;
|
||||
import com.nanri.aiimage.modules.publish.mapper.PublishItemMapper;
|
||||
import com.nanri.aiimage.modules.publish.model.entity.PublishFileEntity;
|
||||
import com.nanri.aiimage.modules.publish.model.entity.PublishItemEntity;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.springframework.transaction.TransactionStatus;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* publish 二次判死线「从未上报」出口的三分行为(2026-10-06 品牌任务 2626 同型缺口)。
|
||||
*
|
||||
* <p>原先「从未上报任何结果分片」与「上报仍新鲜」合并成一句静默 continue(零日志),
|
||||
* 心跳照打的任务可以无限期挂着。现按处理位置三分:仍在推进=跳过、位置久未变=判死、
|
||||
* 无位置信息=证据不足保守跳过但留 WARN。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class PublishNoUploadStaleTest {
|
||||
|
||||
private static final long TASK_ID = 2601L;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, PublishFileEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, PublishItemEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private ZiniaoShopSwitchService ziniaoShopSwitchService;
|
||||
@Mock private PublishWorkbookService workbookService;
|
||||
@Mock private PublishFileMapper publishFileMapper;
|
||||
@Mock private PublishItemMapper publishItemMapper;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private TransactionTemplate transactionTemplate;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
|
||||
@InjectMocks private PublishTaskService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
ReflectionTestUtils.setField(service, "noResultUploadTimeoutMinutes", 180);
|
||||
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30);
|
||||
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
lenient().doAnswer(invocation -> {
|
||||
Consumer<TransactionStatus> callback = invocation.getArgument(0);
|
||||
callback.accept(null);
|
||||
return null;
|
||||
}).when(transactionTemplate).executeWithoutResult(any());
|
||||
}
|
||||
|
||||
private FileTaskEntity neverReportedCandidate() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType(PublishTaskService.MODULE_TYPE);
|
||||
task.setStatus("RUNNING");
|
||||
task.setSourceFileCount(1);
|
||||
task.setCreatedAt(LocalDateTime.now().minusDays(1));
|
||||
task.setUpdatedAt(LocalDateTime.now().minusMinutes(1));
|
||||
return task;
|
||||
}
|
||||
|
||||
private void invokeNoUploadCheck() {
|
||||
ReflectionTestUtils.invokeMethod(service, "failNoUploadStaleTasks");
|
||||
}
|
||||
|
||||
/** 从未上报(scope 行为空)+ 位置久未推进 = 2626 同型真卡死,判死并在文案里写清卡在哪。 */
|
||||
@Test
|
||||
void 从未上报且位置停滞时判死并写清卡住位置() {
|
||||
FileTaskEntity task = neverReportedCandidate();
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(3601L);
|
||||
result.setTaskId(TASK_ID);
|
||||
result.setModuleType(PublishTaskService.MODULE_TYPE);
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())).thenReturn(List.of());
|
||||
when(taskHeartbeatPositionService.describe(TASK_ID)).thenReturn("店铺 魏振峰(2/10)");
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(TASK_ID, 180L)).thenReturn(false);
|
||||
when(taskFileJobService.countUnfinishedAssembleJobs(TASK_ID, PublishTaskService.MODULE_TYPE))
|
||||
.thenReturn(0L);
|
||||
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, TASK_ID, 0L))
|
||||
.thenReturn(lock);
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
when(publishFileMapper.selectList(any())).thenReturn(List.of());
|
||||
when(fileResultMapper.selectOne(any())).thenReturn(result);
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
assertEquals("FAILED", task.getStatus());
|
||||
assertTrue(task.getErrorMessage().contains("无任何结果回传"), task.getErrorMessage());
|
||||
assertTrue(task.getErrorMessage().contains("店铺 魏振峰(2/10)"), task.getErrorMessage());
|
||||
verify(fileTaskMapper).updateById(task);
|
||||
verify(lock).close();
|
||||
}
|
||||
|
||||
/** 从未上报 + 位置仍在推进 = 首批较慢的正常任务,跳过不判死。 */
|
||||
@Test
|
||||
void 从未上报但客户端仍在推进时不判死() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(neverReportedCandidate()));
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())).thenReturn(List.of());
|
||||
when(taskHeartbeatPositionService.describe(TASK_ID)).thenReturn("店铺 魏振峰(2/10)");
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(TASK_ID, 180L)).thenReturn(true);
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
verify(taskDistributedLockService, never()).acquire(any(), any(), anyLong());
|
||||
verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class));
|
||||
}
|
||||
|
||||
/** 从未上报 + 无位置信息(老客户端/条目被 prune)= 证据不足,保守跳过但留 WARN。 */
|
||||
@Test
|
||||
void 从未上报且无位置信息时保守跳过() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(neverReportedCandidate()));
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())).thenReturn(List.of());
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
verify(taskDistributedLockService, never()).acquire(any(), any(), anyLong());
|
||||
verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class));
|
||||
}
|
||||
}
|
||||
+2
@@ -107,6 +107,8 @@ class PublishTaskServiceTest {
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private TransactionTemplate transactionTemplate;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
/** 2026-10:二次判死线的「从未上报」出口接位置三分,publish 注入该服务(默认 mock 等效无位置信息)。 */
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
|
||||
@InjectMocks private PublishTaskService service;
|
||||
|
||||
|
||||
+2
-1
@@ -140,7 +140,8 @@ class ShopDataCrawlChunkUpsertTest {
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class));
|
||||
|
||||
storedChunks.clear();
|
||||
storedScopes.clear();
|
||||
|
||||
+2
-1
@@ -177,7 +177,8 @@ class ShopDataCrawlCleanupTest {
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class));
|
||||
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
|
||||
|
||||
dbResultRows.clear();
|
||||
|
||||
+2
-1
@@ -149,7 +149,8 @@ class ShopDataCrawlDailyFileIncrementalTest {
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class));
|
||||
|
||||
dbResultRows.clear();
|
||||
dbDailyFiles.clear();
|
||||
|
||||
+2
-1
@@ -155,7 +155,8 @@ class ShopDataCrawlDailyFileJobSplitTest {
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class));
|
||||
|
||||
dbResultRows.clear();
|
||||
dbDailyFiles.clear();
|
||||
|
||||
+2
-1
@@ -155,7 +155,8 @@ class ShopDataCrawlDailyFileLockTest {
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class));
|
||||
|
||||
dbResultRows.clear();
|
||||
dbDailyFiles.clear();
|
||||
|
||||
+2
-1
@@ -137,7 +137,8 @@ class ShopDataCrawlLightweightProgressTest {
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class));
|
||||
|
||||
storedChunks.clear();
|
||||
storedScopes.clear();
|
||||
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* shop-data-crawl 二次判死线(心跳正常但连续 N 分钟无结果分片上报)的「从未上报」出口三分。
|
||||
*
|
||||
* <p>原先「从未上报任何结果分片」与「上报仍新鲜」合并成一句静默 continue(零日志),
|
||||
* 与 2026-10-06 品牌任务 2626 挂 9 小时是同一出口。现按处理位置三分:仍在推进=跳过、
|
||||
* 位置久未变=判死(CAS 翻转并在文案里写清卡在哪)、无位置信息=证据不足保守跳过但留 WARN。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class ShopDataCrawlNoUploadStaleTest {
|
||||
|
||||
private static final long TASK_ID = 88001L;
|
||||
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private ShopDataCrawlTaskCacheService taskCacheService;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
|
||||
@InjectMocks private ShopDataCrawlTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeTableInfo() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
ReflectionTestUtils.setField(service, "noResultUploadTimeoutMinutes", 180L);
|
||||
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
|
||||
}
|
||||
|
||||
private FileTaskEntity runningTask() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType("SHOP_DATA_CRAWL");
|
||||
task.setStatus("RUNNING");
|
||||
task.setCreatedAt(LocalDateTime.now().minusDays(1));
|
||||
task.setUpdatedAt(LocalDateTime.now().minusMinutes(1));
|
||||
return task;
|
||||
}
|
||||
|
||||
/** 心跳新鲜 + 无任何结果分片上报(scope 行为空)= 「从未上报」候选。 */
|
||||
private void givenNeverReported() {
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(runningTask()));
|
||||
when(taskCacheService.getTaskHeartbeatMillisBatch(any()))
|
||||
.thenReturn(Map.of(TASK_ID, System.currentTimeMillis()));
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
private void invokeNoUploadCheck() {
|
||||
ReflectionTestUtils.invokeMethod(service, "finalizeNoUploadStaleTasks");
|
||||
}
|
||||
|
||||
/** 从未上报 + 位置久未推进 = 2626 同型真卡死,CAS 翻转并在文案里写清卡在哪。 */
|
||||
@Test
|
||||
void 从未上报且位置停滞时判死并写清卡住位置() {
|
||||
givenNeverReported();
|
||||
when(taskHeartbeatPositionService.describe(TASK_ID)).thenReturn("分页 3/10(共 10 页)");
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(TASK_ID, 180L)).thenReturn(false);
|
||||
when(taskDistributedLockService.acquire(anyString(), anyLong(), anyLong()))
|
||||
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
|
||||
when(taskFileJobService.countUnfinishedAssembleJobs(TASK_ID, "SHOP_DATA_CRAWL")).thenReturn(0L);
|
||||
// tryFinalizeTask 内部:缓存命中 RUNNING 任务,但无结果行 → 返回 false(走 CAS 判死)
|
||||
when(taskCacheService.getTaskCacheBatch(List.of(TASK_ID))).thenReturn(Map.of(TASK_ID, runningTask()));
|
||||
when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
ArgumentCaptor<LambdaUpdateWrapper> update = ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(fileTaskMapper).update(isNull(), update.capture());
|
||||
List<String> values = update.getValue().getParamNameValuePairs().values().stream()
|
||||
.map(String::valueOf)
|
||||
.toList();
|
||||
assertTrue(values.stream().anyMatch(v -> v.contains("无任何结果回传")), values.toString());
|
||||
assertTrue(values.stream().anyMatch(v -> v.contains("分页 3/10(共 10 页)")), values.toString());
|
||||
verify(taskCacheService).deleteTaskCache(TASK_ID);
|
||||
}
|
||||
|
||||
/** 从未上报 + 位置仍在推进 = 首批较慢,不判死(否则拒收后续分片丢数据)。 */
|
||||
@Test
|
||||
void 从未上报但客户端仍在推进时不判死() {
|
||||
givenNeverReported();
|
||||
when(taskHeartbeatPositionService.describe(TASK_ID)).thenReturn("分页 3/10(共 10 页)");
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(TASK_ID, 180L)).thenReturn(true);
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
verify(taskDistributedLockService, never()).acquire(anyString(), anyLong(), anyLong());
|
||||
verify(fileTaskMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
/** 从未上报 + 无位置信息(老客户端/条目被 prune)= 证据不足,保守跳过但留 WARN。 */
|
||||
@Test
|
||||
void 从未上报且无位置信息时保守跳过() {
|
||||
givenNeverReported();
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
verify(taskDistributedLockService, never()).acquire(anyString(), anyLong(), anyLong());
|
||||
verify(fileTaskMapper, never()).update(any(), any());
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -142,7 +142,8 @@ class ShopDataCrawlOwnerColumnTest {
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class));
|
||||
|
||||
dbTasks.clear();
|
||||
lastScan.clear();
|
||||
|
||||
+2
-1
@@ -149,7 +149,8 @@ class ShopDataCrawlProgressQueryTest {
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class));
|
||||
|
||||
dbResultRows.clear();
|
||||
dbTasks.clear();
|
||||
|
||||
+2
-1
@@ -136,7 +136,8 @@ class ShopDataCrawlRowDedupKeyTest {
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class));
|
||||
|
||||
storedChunks.clear();
|
||||
storedScopes.clear();
|
||||
|
||||
+2
-1
@@ -137,7 +137,8 @@ class ShopDataCrawlScopeCounterTest {
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class));
|
||||
|
||||
storedChunks.clear();
|
||||
storedScopes.clear();
|
||||
|
||||
+2
-1
@@ -138,7 +138,8 @@ class ShopDataCrawlScopeMergeTest {
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class));
|
||||
|
||||
storedChunks.clear();
|
||||
storedScopes.clear();
|
||||
|
||||
+2
-1
@@ -132,7 +132,8 @@ class ShopDataCrawlTaskServiceChunkTest {
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class));
|
||||
|
||||
storedChunks.clear();
|
||||
storedScopes.clear();
|
||||
|
||||
+54
@@ -69,6 +69,19 @@ class SimilarAsinNoUploadStaleTest {
|
||||
.thenReturn(List.of(new TaskScopeLastChunkDto(TASK_ID, LocalDateTime.now().minusMinutes(200))));
|
||||
}
|
||||
|
||||
/** 心跳新鲜 + 无任何结果分片上报(scope 行为空)= 「从未上报」候选。 */
|
||||
private void givenNeverReported() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType("SIMILAR_ASIN");
|
||||
task.setStatus("RUNNING");
|
||||
task.setCreatedAt(LocalDateTime.now().minusDays(1));
|
||||
task.setUpdatedAt(LocalDateTime.now().minusMinutes(1));
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(taskCacheService.getTaskHeartbeatMillis(TASK_ID)).thenReturn(System.currentTimeMillis());
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
private void invokeNoUploadCheck() {
|
||||
ReflectionTestUtils.invokeMethod(service, "finalizeNoUploadStaleTasks");
|
||||
}
|
||||
@@ -98,4 +111,45 @@ class SimilarAsinNoUploadStaleTest {
|
||||
// 位置长时间不变(真卡死)仍按原判据走:进入锁内封口流程
|
||||
verify(taskDistributedLockService).acquire(anyString(), anyLong(), any(Duration.class), anyLong());
|
||||
}
|
||||
|
||||
/** 从未上报 + 位置长时间未变 = 真卡死(2626 同型),按卡死收尾而不是继续静默。 */
|
||||
@Test
|
||||
void 从未上报但位置停滞时按卡死收尾() {
|
||||
givenNeverReported();
|
||||
when(taskHeartbeatPositionService.describe(TASK_ID)).thenReturn("分页 3/10");
|
||||
when(taskHeartbeatPositionService.evaluateStall(TASK_ID, 180L))
|
||||
.thenReturn(TaskHeartbeatPositionService.StallState.STALLED);
|
||||
when(taskDistributedLockService.acquire(anyString(), anyLong(), any(Duration.class), anyLong()))
|
||||
.thenReturn(null);
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
// 已进入判死流程(开始抢锁);锁被占则本轮放弃
|
||||
verify(taskDistributedLockService).acquire(anyString(), anyLong(), any(Duration.class), anyLong());
|
||||
}
|
||||
|
||||
/** 从未上报 + 位置仍在推进 = 首批较慢,不封口(否则拒收后续分片丢数据)。 */
|
||||
@Test
|
||||
void 从未上报但客户端仍在推进时不封口() {
|
||||
givenNeverReported();
|
||||
when(taskHeartbeatPositionService.describe(TASK_ID)).thenReturn("分页 3/10");
|
||||
when(taskHeartbeatPositionService.evaluateStall(TASK_ID, 180L))
|
||||
.thenReturn(TaskHeartbeatPositionService.StallState.ADVANCING);
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
verify(taskDistributedLockService, never()).acquire(anyString(), anyLong(), any(Duration.class), anyLong());
|
||||
}
|
||||
|
||||
/** 从未上报 + 无位置信息(老客户端/条目被 prune)= 证据不足,保守跳过但留 WARN。 */
|
||||
@Test
|
||||
void 从未上报且无位置信息时保守跳过() {
|
||||
givenNeverReported();
|
||||
when(taskHeartbeatPositionService.evaluateStall(TASK_ID, 180L))
|
||||
.thenReturn(TaskHeartbeatPositionService.StallState.UNKNOWN);
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
verify(taskDistributedLockService, never()).acquire(anyString(), anyLong(), any(Duration.class), anyLong());
|
||||
}
|
||||
}
|
||||
|
||||
+3
-1
@@ -17,6 +17,7 @@ import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||
@@ -64,6 +65,7 @@ class PublishFileLevelStatusTest {
|
||||
@Mock private TransactionTemplate transactionTemplate;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
|
||||
private PublishTaskService service;
|
||||
|
||||
@@ -74,7 +76,7 @@ class PublishFileLevelStatusTest {
|
||||
publishItemMapper, fileTaskMapper, fileResultMapper, taskChunkMapper, taskScopeStateMapper,
|
||||
taskFileJobService, taskDistributedLockService, transientPayloadStorageService, ossStorageService,
|
||||
objectMapper, transactionTemplate, org.mockito.Mockito.mock(com.nanri.aiimage.common.service.DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS),
|
||||
instanceMetadata, taskProgressLightAssembler);
|
||||
instanceMetadata, taskProgressLightAssembler, taskHeartbeatPositionService);
|
||||
lenient().when(ossStorageService.generateFreshDownloadUrl(any()))
|
||||
.thenReturn("https://dl.example/publish-result.zip");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user