fix(外观专利): 失败兜底出部分结果文件+WAIT_LLM 停放作业心跳续租(31302/31354)
线上两例死法修复,目标是「任务失败了也能下载」: - 31354:组装作业在 WAIT_LLM 停放期执行线程已退出、Worker 执行期心跳 已取消,job 级 stuck 扫描按 updated_at 判陈旧,5 分钟阈值×5 次重试 耗尽后误判「文件生成任务运行超时」。客户端心跳经 AppearancePatentTaskHeartbeatSpi#onHeartbeat 按 dbJobTouchIntervalMillis(默认 60s,远小于 5 分钟阈值)节流续租 (touchRunningAssembleJobsIfStale,与 SimilarAsin 同口径)。 - 31302:覆盖率闸门判红重试 5 次判死后,1196 行成品零交付且不能下载。 接通 fallbackAssembleOnFailure 兜底钩子:作业判死后用已有分片出 部分结果文件(validateCompleteLlmCoverage 加 allowPartial 旁路), 写上 result.resultFileUrl 后前端下载按钮即恢复;兜底幂等(已有 结果文件短 路)、不动任务终态、与补传恢复重跑兼容。 - 棘轮尺 STRICT_TOTAL 1003→1004、NOT_RETHROW 561→562 抬格并逐处 记名(兜底自吞 catch 属接口契约要求的 fail-open 形状)。 全量 mvn -o test 4492 支 0 失败(9 skipped 为既有跳过)。
This commit is contained in:
@@ -72,6 +72,18 @@ public class AppearancePatentProperties {
|
||||
*/
|
||||
private int flushPendingMinutes = 1;
|
||||
|
||||
/**
|
||||
* 组装作业在 WAIT_LLM 停放期间的租约续命间隔(毫秒)。
|
||||
*
|
||||
* <p>组装作业等下一轮分片回传时 process 返回 false、执行线程退出、Worker 的执行期心跳
|
||||
* 随之取消,作业停在 RUNNING 只能靠客户端心跳经 {@code onHeartbeat} 触发
|
||||
* {@code touchRunningAssembleJobsIfStale} 按本间隔节流续命(与 similar-asin 同一套口径)。
|
||||
* 取值必须小于 job 级 stuck 扫描阈值(result-file-job.stuck-timeout-minutes,默认 5 分钟),
|
||||
* 否则停放作业在两跳续命之间就会被判陈旧、5 次重试耗尽后被误判成
|
||||
* 「文件生成任务运行超时,已达到最大重试次数」(线上任务 31354 的死法)。
|
||||
*/
|
||||
private long dbJobTouchIntervalMillis = 60000L;
|
||||
|
||||
/** 单个外观专利源文件最多解析的有效数据行数,防止 POI 用户模型撑爆堆。 */
|
||||
private int maxParseRows = 50000;
|
||||
|
||||
|
||||
+9
-1
@@ -6,7 +6,8 @@ import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
|
||||
/**
|
||||
* APPEARANCE_PATENT 结果文件 Job Handler(04 注册表)。
|
||||
* process 返回 Service 的 boolean(false=等待 LLM 异步结果,Worker 保持运行心跳);
|
||||
* process 返回 Service 的 boolean(false=停放等待下一轮分片回传触发;此时执行线程已退出、
|
||||
* Worker 的执行期心跳已取消,停放租约由客户端心跳经 AppearancePatentTaskHeartbeatSpi#onHeartbeat 续命);
|
||||
* cleanup 委托 cleanupResultFileJob;支持异步 offload;owner scoped。
|
||||
*/
|
||||
public class AppearancePatentResultFileJobHandler implements ResultFileJobHandler {
|
||||
@@ -32,6 +33,13 @@ public class AppearancePatentResultFileJobHandler implements ResultFileJobHandle
|
||||
appearancePatentTaskService.cleanupResultFileJob(job);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean fallbackAssembleOnFailure(TaskFileJobEntity job, String message) {
|
||||
// 作业重试耗尽(含 WAIT_LLM 停放被 stuck 棘轮判死)时:用已有分片/结论尽力出
|
||||
// 部分结果文件供下载,别让已跑完的检测一张表都不剩(线上任务 31302/31354)。
|
||||
return appearancePatentTaskService.fallbackAssembleResultFile(job);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean supportsAsyncOffload() {
|
||||
return true;
|
||||
|
||||
+27
@@ -1,6 +1,9 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -10,12 +13,21 @@ import org.springframework.stereotype.Service;
|
||||
*
|
||||
* <p>心跳逻辑(缓存刷新 / 任务缓存回写)从 task 模块收回本模块,task 侧只依赖 SPI 接口,
|
||||
* 消除 task → 业务模块的编译期依赖。
|
||||
*
|
||||
* <p>onHeartbeat 顺带给结果文件组装作业续租(与 {@code SimilarAsinTaskHeartbeatSpi} 同一套口径):
|
||||
* 组装作业在 WAIT_LLM 停放期间(process 返回 false 等下一轮分片回传触发)执行线程已退出、
|
||||
* Worker 的执行期心跳已随之取消,作业停在 RUNNING 无人续命,会被 job 级 stuck 扫描按
|
||||
* updated_at 判陈旧、5 次重试耗尽后误判成「文件生成任务运行超时」(线上任务 31354)。
|
||||
* 客户端在任务执行期持续上报心跳,经这里续租即可覆盖整个停放窗口;
|
||||
* 客户端也停了的场景由 stale 收尾链路与失败兜底(fallbackAssembleResultFile)接住。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class AppearancePatentTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final AppearancePatentTaskCacheService cacheService;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final AppearancePatentProperties appearancePatentProperties;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
@@ -26,4 +38,19 @@ public class AppearancePatentTaskHeartbeatSpi implements TaskModuleHeartbeatSpi
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
cacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onHeartbeat(FileTaskEntity task) {
|
||||
if (task == null || task.getId() == null) {
|
||||
return;
|
||||
}
|
||||
// touchRunningAssembleJobsIfStale 的 SQL 自带「距上次刷新超过间隔才写」的节流,
|
||||
// 心跳再密也不会把库写爆,无需在这里另做节流。
|
||||
taskFileJobService.touchRunningAssembleJobsIfStale(
|
||||
task.getId(), moduleType(), jobTouchIntervalMillis());
|
||||
}
|
||||
|
||||
private long jobTouchIntervalMillis() {
|
||||
return Math.max(1_000L, appearancePatentProperties.getDbJobTouchIntervalMillis());
|
||||
}
|
||||
}
|
||||
|
||||
+82
-3
@@ -1889,6 +1889,67 @@ public class AppearancePatentTaskService {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 失败兜底:组装作业重试耗尽(WAIT_LLM 停放被 stuck 棘轮判死、或组装异常连吃 5 次重试)后调用,
|
||||
* 用已回传的分片与已拿到结论的行尽力生成可下载的部分结果文件。
|
||||
*
|
||||
* <p>用户线上点名要的保证:「起码失败了也可以下载」。判死时已跑完的检测结论仍挂在分片里,
|
||||
* 不出表就一张都不剩(线上任务 31302:1196 行成品困在判死作业里零交付)。
|
||||
*
|
||||
* <p>覆盖率闸门按「允许部分结果」放行(缺口只记行账);任务状态在这里**不收尾**——判死时
|
||||
* 任务可能仍在跑、客户端仍在心跳(stuck 棘轮只判作业、不判任务),提前置终态会把活任务误杀;
|
||||
* 任务终态仍由 stale 收尾链路 / 分片回传链路按既有口径落。文件挂上 result(result_file_url)
|
||||
* 后前端下载按钮即出现,与任务/作业状态无关({@code AppearancePatentHistoryAssembler#attachFileJobState})。
|
||||
*
|
||||
* <p>幂等:结果文件已存在(result_file_url 非空)直接返回 true,不重做组装——stuck 扫描对
|
||||
* 终态失败作业每轮都会重入兜底钩子,靠这里挡住重复组装。与补传恢复(maybeRecoverTerminalFailedAssemble)
|
||||
* 的重跑并发时两边都只写合法结果文件,后写者覆盖,无中间态。
|
||||
*
|
||||
* <p>实现自吞异常(兜底失败只记日志,不影响作业已终态失败的事实),返回是否有可下载结果。
|
||||
*/
|
||||
public boolean fallbackAssembleResultFile(TaskFileJobEntity job) {
|
||||
if (job == null || job.getTaskId() == null || job.getResultId() == null) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
|
||||
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
|
||||
log.warn("[appearance-patent] fallback assemble skipped because result record missing taskId={} resultId={}",
|
||||
job.getTaskId(), job.getResultId());
|
||||
return false;
|
||||
}
|
||||
if (result.getResultFileUrl() != null && !result.getResultFileUrl().isBlank()) {
|
||||
log.info("[appearance-patent] fallback assemble skipped because result file already exists taskId={} resultId={} jobStatus={}",
|
||||
job.getTaskId(), job.getResultId(), job.getStatus());
|
||||
return true;
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
log.warn("[appearance-patent] fallback assemble skipped because task missing taskId={} resultId={}",
|
||||
job.getTaskId(), job.getResultId());
|
||||
return false;
|
||||
}
|
||||
// 与 processResultFileJobLocked 同一判法:非归属实例让给归属机(stuck 扫描两台都在跑,
|
||||
// 不挡的话每台都会每 60 秒重入一次兜底钩子)。
|
||||
job.setScopeKey(buildTaskOwnerScopeKey(task));
|
||||
if (!isJobOwnedByCurrentInstance(job)) {
|
||||
log.info("[appearance-patent] fallback assemble skipped because owner is another instance taskId={} resultId={}",
|
||||
job.getTaskId(), job.getResultId());
|
||||
return false;
|
||||
}
|
||||
AppearancePatentRowOutcome.Facts facts = assembleResultWorkbook(task, result, true);
|
||||
fileResultMapper.updateById(result);
|
||||
log.warn("[appearance-patent] fallback assemble produced partial result taskId={} resultId={} jobStatus={} expectedRows={} judgedRows={} missingRows={} noDataRows={}",
|
||||
job.getTaskId(), job.getResultId(), job.getStatus(),
|
||||
facts.expectedRows(), facts.judgedRows(), facts.missingRows(), facts.noDataRows());
|
||||
return true;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent] fallback assemble failed taskId={} resultId={} msg={}",
|
||||
job.getTaskId(), job.getResultId(), ex.getMessage(), ex);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** 返回真正落库的行键;没匹配到分片行的键不会出现在里面,调用方据此决定要不要撤销已检测标记。 */
|
||||
private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
|
||||
List<AppearancePatentResultRowDto> llmRows,
|
||||
@@ -2189,6 +2250,16 @@ public class AppearancePatentTaskService {
|
||||
}
|
||||
|
||||
private AppearancePatentRowOutcome.Facts assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) {
|
||||
return assembleResultWorkbook(task, result, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param allowPartial 供失败兜底路径({@link #fallbackAssembleResultFile})使用:
|
||||
* 覆盖率闸门按「允许部分结果」放行——作业已判死,缺口不再有补齐窗口,
|
||||
* 此刻拦下成品只会让用户连半张表都拿不到;缺口仍由行账锚点记数。
|
||||
*/
|
||||
private AppearancePatentRowOutcome.Facts assembleResultWorkbook(FileTaskEntity task, FileResultEntity result,
|
||||
boolean allowPartial) {
|
||||
AppearancePatentParsedPayloadDto parsed = readParsedPayload(task);
|
||||
List<AppearancePatentParsedRowVo> parsedRows = parsed.getAllItems() == null
|
||||
? List.of() : parsed.getAllItems();
|
||||
@@ -2214,7 +2285,7 @@ public class AppearancePatentTaskService {
|
||||
// 覆盖率先于「本文件无结果就跳过」判定:整源文件的 LLM 结果全丢(分片被跳过、
|
||||
// 结果清空)时覆盖率是 0%,必须在这里判掉,不能让 zip 少一个文件还报任务成功。
|
||||
deliveryFacts = deliveryFacts.plus(
|
||||
validateCompleteLlmCoverage(task.getId(), item.rows(), resultMap));
|
||||
validateCompleteLlmCoverage(task.getId(), item.rows(), resultMap, allowPartial));
|
||||
List<AppearancePatentParsedRowVo> receivedRows = filterReceivedParsedRows(item.rows(), resultMap);
|
||||
if (receivedRows.isEmpty()) {
|
||||
resultMap.clear();
|
||||
@@ -2417,6 +2488,13 @@ public class AppearancePatentTaskService {
|
||||
private AppearancePatentRowOutcome.Facts validateCompleteLlmCoverage(Long taskId,
|
||||
List<AppearancePatentParsedRowVo> parsedRows,
|
||||
Map<String, AppearancePatentResultRowDto> resultMap) {
|
||||
return validateCompleteLlmCoverage(taskId, parsedRows, resultMap, false);
|
||||
}
|
||||
|
||||
private AppearancePatentRowOutcome.Facts validateCompleteLlmCoverage(Long taskId,
|
||||
List<AppearancePatentParsedRowVo> parsedRows,
|
||||
Map<String, AppearancePatentResultRowDto> resultMap,
|
||||
boolean allowPartial) {
|
||||
if (parsedRows == null || parsedRows.isEmpty()) {
|
||||
return AppearancePatentRowOutcome.Facts.EMPTY;
|
||||
}
|
||||
@@ -2432,8 +2510,9 @@ public class AppearancePatentTaskService {
|
||||
}
|
||||
double minCoverage = properties.getLlmCoverageMinRatio();
|
||||
// 已判失败的任务走的是"按已有分片出部分结果"这条既有口径,这里再抛会把成品彻底拿掉;
|
||||
// 但它不再静默:上面的行账锚点必打,缺口由调用方写进任务错误信息,用户看得到「N 行没检测」。
|
||||
if (!taskAlreadyFailed && minCoverage > 0 && facts.expectedRows() > 0
|
||||
// 失败兜底路径(allowPartial=true,作业已判死)同理放行。
|
||||
// 两者都不再静默:上面的行账锚点必打,缺口由调用方写进任务错误信息,用户看得到「N 行没检测」。
|
||||
if (!taskAlreadyFailed && !allowPartial && minCoverage > 0 && facts.expectedRows() > 0
|
||||
&& facts.coverage() + 1e-9 < minCoverage) {
|
||||
throw new BusinessException("LLM 结果不完整:仅 " + facts.judgedRows() + "/" + facts.expectedRows()
|
||||
+ " 行拿到检测结论(低于 " + Math.round(minCoverage * 100) + "% 覆盖率下限)"
|
||||
|
||||
@@ -46,8 +46,9 @@ class CatchSilenceRatchetTest {
|
||||
|
||||
// ==== 验收基线(用户 2026-10-08 12:0x 实测,HEAD 274ad017,单位:处)====
|
||||
// 10-17 心跳证据两处 fail-open 抬 559 -> 561,逐处出处见 STRICT_NOT_RETHROW 上方注释。
|
||||
// 10-18 外观专利失败兜底自吞抬 561 -> 562,逐处出处同下。
|
||||
static final int GIVEN_TOTAL = 1000;
|
||||
static final int GIVEN_NOT_RETHROW = 561;
|
||||
static final int GIVEN_NOT_RETHROW = 562;
|
||||
static final int GIVEN_LOG_ONLY = 249;
|
||||
static final int GIVEN_EMPTY = 56;
|
||||
|
||||
@@ -65,8 +66,13 @@ class CatchSilenceRatchetTest {
|
||||
* 「没有证据」,判死线据此放宽)。两处都是 fail-open 且方向是「少杀一个任务」,与上面
|
||||
* BuildInfoLogger 同形状;本仓实测基线由 1001 起(原记 1002 留有 1 格无人认领的额度),
|
||||
* 加这 2 处后实测 1003。
|
||||
* 10-18 再 +1:{@code AppearancePatentTaskService.fallbackAssembleResultFile()} 的兜底自吞
|
||||
* (log.warn + return false)——{@code ResultFileJobHandler#fallbackAssembleOnFailure} 的接口
|
||||
* 契约明确要求「实现应自吞异常(兜底失败只记日志,不影响 job 已终态失败的事实)」,与
|
||||
* DeleteBrandRunService 同名兜底同形状;Worker 侧另有 catch 双保险,让判死作业的兜底每轮
|
||||
* stuck 扫描幂等重入。实测 1004。
|
||||
*/
|
||||
static final int STRICT_TOTAL = 1003;
|
||||
static final int STRICT_TOTAL = 1004;
|
||||
/**
|
||||
* 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。
|
||||
* 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。
|
||||
@@ -89,8 +95,13 @@ class CatchSilenceRatchetTest {
|
||||
* 判死线据此**放宽**(顶多让任务多活一会儿,最终仍由未启动宽限与 stale-repair 兜底)。
|
||||
* 两处的误差方向都是「少杀一个任务」,与这一格想拦的「异常到此为止、外面看不见」不同性质;
|
||||
* 抬格是为了把出处留在这份 diff 里,而不是让后来人以为基线自然涨上来的。
|
||||
* 10-18 抬 561 -> 562,逐处记名:{@code AppearancePatentTaskService.fallbackAssembleResultFile()}——
|
||||
* 失败兜底组装的自吞出口(接口契约要求,见 {@code ResultFileJobHandler#fallbackAssembleOnFailure}),
|
||||
* 与 {@code DeleteBrandRunService.fallbackAssembleResultFile} 同形状、同性质:兜底失败不得掩盖
|
||||
* 作业已终态失败的事实,也不得打断 stuck 扫描对终态失败作业的幂等重入。形状是 log+return false,
|
||||
* 因此只进本格与 total,不进 STRICT_LOG_ONLY(严格口径不数 log 后接 return)。
|
||||
*/
|
||||
static final int STRICT_NOT_RETHROW = 561;
|
||||
static final int STRICT_NOT_RETHROW = 562;
|
||||
/**
|
||||
* 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}:
|
||||
* 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出——
|
||||
|
||||
+36
@@ -181,6 +181,24 @@ class AppearancePatentDeliveryGateTest {
|
||||
}
|
||||
}
|
||||
|
||||
/** 失败兜底路径的 4 参口径:allowPartial=true 时覆盖率闸门放行(作业已判死,缺口不再有补齐窗口)。 */
|
||||
private AppearancePatentRowOutcome.Facts validatePartial(List<AppearancePatentParsedRowVo> rows,
|
||||
Map<String, AppearancePatentResultRowDto> resultMap,
|
||||
boolean allowPartial)
|
||||
throws Exception {
|
||||
Method method = AppearancePatentTaskService.class.getDeclaredMethod(
|
||||
"validateCompleteLlmCoverage", Long.class, List.class, Map.class, boolean.class);
|
||||
method.setAccessible(true);
|
||||
try {
|
||||
return (AppearancePatentRowOutcome.Facts) method.invoke(service, TASK_ID, rows, resultMap, allowPartial);
|
||||
} catch (InvocationTargetException ex) {
|
||||
if (ex.getCause() instanceof RuntimeException runtime) {
|
||||
throw runtime;
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean requestRecheck(FileTaskEntity task, TaskFileJobEntity job,
|
||||
AppearancePatentRowOutcome.Facts facts) throws Exception {
|
||||
Method method = AppearancePatentTaskService.class.getDeclaredMethod(
|
||||
@@ -254,6 +272,24 @@ class AppearancePatentDeliveryGateTest {
|
||||
assertEquals(5, facts.noDataRows());
|
||||
}
|
||||
|
||||
@Test
|
||||
void allowPartialLetsBelowFloorCoverageThroughForFailureFallback() throws Exception {
|
||||
// 同一份 50% 覆盖率:常规路径判红(belowFloorThrowsWithBothCountersInTheMessage),
|
||||
// 兜底路径放行——作业已判死,此刻拦下成品只会让用户连半张表都拿不到(线上任务 31302)
|
||||
List<AppearancePatentParsedRowVo> rows = List.of(sheetRow("1", "B0A1"), sheetRow("2", "B0A2"),
|
||||
sheetRow("3", "B0A3"));
|
||||
Map<String, AppearancePatentResultRowDto> map = new LinkedHashMap<>();
|
||||
map.putAll(submittedRows(List.of(sheetRow("1", "B0A1")), 1, true));
|
||||
map.putAll(submittedRows(List.of(sheetRow("2", "B0A2")), 0, true));
|
||||
map.putAll(submittedRows(List.of(sheetRow("3", "B0A3")), 0, false));
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task("RUNNING"));
|
||||
|
||||
AppearancePatentRowOutcome.Facts facts = validatePartial(rows, map, true);
|
||||
assertEquals(2, facts.expectedRows());
|
||||
assertEquals(1, facts.judgedRows());
|
||||
assertEquals(1, facts.missingRows());
|
||||
}
|
||||
|
||||
// ---------- 补检出口 ----------
|
||||
|
||||
@Test
|
||||
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 失败兜底组装的守卫与幂等(2026-10-18,线上任务 31302/31354 死因修复)。
|
||||
*
|
||||
* <p>组装作业重试耗尽判死后,用户要「起码失败了也可以下载」:兜底用已有分片/结论出
|
||||
* 部分结果文件。本类钉住兜底入口的分支语义;组装主体(xlsx 生成/上传)由
|
||||
* AppearancePatentSheetBuilderTest/AppearancePatentResultFormatTest/交付闸门测试覆盖。
|
||||
*/
|
||||
class AppearancePatentFallbackAssembleTest {
|
||||
|
||||
private static final long TASK_ID = 31302L;
|
||||
private static final long RESULT_ID = 9001L;
|
||||
private static final long JOB_ID = 7001L;
|
||||
|
||||
private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
private final FileResultMapper fileResultMapper = mock(FileResultMapper.class);
|
||||
private final TaskScopeStateMapper taskScopeStateMapper = mock(TaskScopeStateMapper.class);
|
||||
private final TaskChunkMapper taskChunkMapper = mock(TaskChunkMapper.class);
|
||||
private final LocalFileStorageService localFileStorageService = mock(LocalFileStorageService.class);
|
||||
private final AppearancePatentLlmClient llmClient = mock(AppearancePatentLlmClient.class);
|
||||
private final AppearancePatentTaskCacheService taskCacheService = mock(AppearancePatentTaskCacheService.class);
|
||||
private final AppearancePatentProperties properties = mock(AppearancePatentProperties.class);
|
||||
private final StorageProperties storageProperties = mock(StorageProperties.class);
|
||||
private final TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
|
||||
private final TaskProgressSnapshotService taskProgressSnapshotService = mock(TaskProgressSnapshotService.class);
|
||||
private final TransientPayloadStorageService transientPayloadStorageService = mock(TransientPayloadStorageService.class);
|
||||
private final PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
|
||||
private final DistributedJobLockService distributedJobLockService = mock(DistributedJobLockService.class);
|
||||
private final TaskDistributedLockService taskDistributedLockService = mock(TaskDistributedLockService.class);
|
||||
private final InstanceMetadata instanceMetadata = mock(InstanceMetadata.class);
|
||||
|
||||
private AppearancePatentTaskService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
when(properties.getLlmCoverageMinRatio()).thenReturn(0.95D);
|
||||
service = new AppearancePatentTaskService(
|
||||
localFileStorageService, null,
|
||||
mock(com.nanri.aiimage.common.service.ResultDownloadResolver.class),
|
||||
storageProperties, fileTaskMapper, fileResultMapper,
|
||||
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
|
||||
properties, taskFileJobService, taskProgressSnapshotService,
|
||||
mock(TaskHeartbeatPositionService.class), transientPayloadStorageService, transactionManager,
|
||||
distributedJobLockService, taskDistributedLockService, instanceMetadata,
|
||||
mock(TaskProgressLightAssembler.class), mock(UserApiSecretService.class));
|
||||
}
|
||||
|
||||
private static TaskFileJobEntity job() {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(JOB_ID);
|
||||
job.setTaskId(TASK_ID);
|
||||
job.setResultId(RESULT_ID);
|
||||
job.setModuleType("APPEARANCE_PATENT");
|
||||
job.setStatus("FAILED");
|
||||
return job;
|
||||
}
|
||||
|
||||
private static FileResultEntity result(String resultFileUrl) {
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(RESULT_ID);
|
||||
result.setTaskId(TASK_ID);
|
||||
result.setModuleType("APPEARANCE_PATENT");
|
||||
result.setResultFileUrl(resultFileUrl);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static FileTaskEntity task() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType("APPEARANCE_PATENT");
|
||||
task.setStatus("FAILED");
|
||||
task.setUserId(7L);
|
||||
return task;
|
||||
}
|
||||
|
||||
@Test
|
||||
void incompleteJobArgumentsAreRejected() {
|
||||
assertFalse(service.fallbackAssembleResultFile(null), "null 作业直接拒绝");
|
||||
assertFalse(service.fallbackAssembleResultFile(new TaskFileJobEntity()), "缺 taskId/resultId 直接拒绝");
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingResultRecordReturnsFalse() {
|
||||
when(fileResultMapper.selectById(RESULT_ID)).thenReturn(null);
|
||||
assertFalse(service.fallbackAssembleResultFile(job()), "结果记录不存在(孤儿作业)无可组装");
|
||||
verify(fileTaskMapper, never()).selectById(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void existingResultFileShortCircuitsAssemble() {
|
||||
// 幂等:stuck 扫描对终态失败作业每轮重入兜底钩子,已有结果文件就不再重做组装
|
||||
when(fileResultMapper.selectById(RESULT_ID)).thenReturn(result("appearance-patent/existing.xlsx"));
|
||||
assertTrue(service.fallbackAssembleResultFile(job()), "已有结果文件视为兜底成功");
|
||||
verify(fileTaskMapper, never()).selectById(any());
|
||||
verify(fileResultMapper, never()).updateById(any(FileResultEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingTaskRecordReturnsFalse() {
|
||||
when(fileResultMapper.selectById(RESULT_ID)).thenReturn(result(null));
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(null);
|
||||
assertFalse(service.fallbackAssembleResultFile(job()), "任务记录不存在无可组装");
|
||||
}
|
||||
|
||||
@Test
|
||||
void emptyPayloadFailsSoftlyWithoutTouchingResult() {
|
||||
// 组装抛业务异常(无可解析行)时兜底自吞:返回 false、不改写 result,作业保持终态失败
|
||||
when(fileResultMapper.selectById(RESULT_ID)).thenReturn(result(null));
|
||||
FileTaskEntity task = task();
|
||||
task.setResultJson("{}");
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
|
||||
assertFalse(service.fallbackAssembleResultFile(job()), "空载荷组装失败要被吞掉");
|
||||
verify(fileResultMapper, never()).updateById(any(FileResultEntity.class));
|
||||
}
|
||||
}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 外观专利任务心跳的组装作业续租(2026-10-18,线上任务 31354 死因修复)。
|
||||
*
|
||||
* <p>组装作业在 WAIT_LLM 停放期间执行线程已退出、Worker 执行期心跳已取消,若无人续命,
|
||||
* job 级 stuck 扫描(stuck-timeout-minutes 默认 5 分钟)会按 updated_at 判陈旧、
|
||||
* 5 次重试耗尽后误判「文件生成任务运行超时」。客户端心跳经 onHeartbeat 触发
|
||||
* touchRunningAssembleJobsIfStale 按配置间隔节流续租(与 SimilarAsinTaskHeartbeatSpi 同口径)。
|
||||
*/
|
||||
class AppearancePatentTaskHeartbeatSpiTest {
|
||||
|
||||
private final AppearancePatentTaskCacheService cacheService = mock(AppearancePatentTaskCacheService.class);
|
||||
private final TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
|
||||
private final AppearancePatentProperties properties = new AppearancePatentProperties();
|
||||
|
||||
private AppearancePatentTaskHeartbeatSpi spi;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
spi = new AppearancePatentTaskHeartbeatSpi(cacheService, taskFileJobService, properties);
|
||||
}
|
||||
|
||||
@Test
|
||||
void onHeartbeatRenewsAssembleJobLease() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(31354L);
|
||||
task.setModuleType("APPEARANCE_PATENT");
|
||||
|
||||
spi.onHeartbeat(task);
|
||||
|
||||
verify(taskFileJobService).touchRunningAssembleJobsIfStale(31354L, "APPEARANCE_PATENT",
|
||||
properties.getDbJobTouchIntervalMillis());
|
||||
}
|
||||
|
||||
@Test
|
||||
void touchIntervalBelowStuckScanFloor() {
|
||||
// 续命间隔必须小于 stuck 扫描阈值(5 分钟),否则停放作业在两跳续命之间仍会被判陈旧
|
||||
org.junit.jupiter.api.Assertions.assertTrue(properties.getDbJobTouchIntervalMillis() < 5 * 60_000L,
|
||||
"dbJobTouchIntervalMillis 应小于 stuck-timeout-minutes=5 分钟,实际 "
|
||||
+ properties.getDbJobTouchIntervalMillis());
|
||||
}
|
||||
|
||||
@Test
|
||||
void onHeartbeatWithNullTaskTouchesNothing() {
|
||||
spi.onHeartbeat(null);
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
spi.onHeartbeat(task);
|
||||
verifyNoInteractions(taskFileJobService);
|
||||
}
|
||||
|
||||
@Test
|
||||
void touchHeartbeatStillDelegatesToCache() {
|
||||
spi.touchHeartbeat(1L, null);
|
||||
verify(cacheService).touchTaskHeartbeat(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void moduleTypeMatchesRegistry() {
|
||||
org.junit.jupiter.api.Assertions.assertEquals("APPEARANCE_PATENT", spi.moduleType());
|
||||
}
|
||||
}
|
||||
+17
@@ -77,4 +77,21 @@ class AppearancePatentResultFileJobHandlerTest {
|
||||
handler.cleanup(null);
|
||||
org.mockito.Mockito.verify(appearancePatentTaskService).cleanupResultFileJob(null);
|
||||
}
|
||||
|
||||
@Test
|
||||
void fallbackAssembleDelegates() {
|
||||
// 重试耗尽(含 WAIT_LLM 停放被 stuck 棘轮判死)时委托 Service 出部分结果文件
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
when(appearancePatentTaskService.fallbackAssembleResultFile(job)).thenReturn(true);
|
||||
assertTrue(handler.fallbackAssembleOnFailure(job, "文件生成任务运行超时,已达到最大重试次数"),
|
||||
"兜底产出可下载结果时返回 true");
|
||||
}
|
||||
|
||||
@Test
|
||||
void fallbackAssembleFailureReturnsFalse() {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
when(appearancePatentTaskService.fallbackAssembleResultFile(job)).thenReturn(false);
|
||||
assertFalse(handler.fallbackAssembleOnFailure(job, "结果文件生成失败"),
|
||||
"兜底也失败时返回 false,不影响作业已终态失败的事实");
|
||||
}
|
||||
}
|
||||
|
||||
+4
-1
@@ -1,5 +1,6 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskHeartbeatSpi;
|
||||
@@ -46,7 +47,9 @@ class TaskModuleHeartbeatSpiCoverageTest {
|
||||
|
||||
private static Map<String, TaskModuleHeartbeatSpi> implementationsByModuleType() {
|
||||
Map<String, TaskModuleHeartbeatSpi> impls = new LinkedHashMap<>();
|
||||
register(impls, new AppearancePatentTaskHeartbeatSpi(mock(AppearancePatentTaskCacheService.class)));
|
||||
register(impls, new AppearancePatentTaskHeartbeatSpi(
|
||||
mock(AppearancePatentTaskCacheService.class), mock(TaskFileJobService.class),
|
||||
mock(AppearancePatentProperties.class)));
|
||||
register(impls, new CollectDataTaskHeartbeatSpi(mock(CollectDataService.class)));
|
||||
register(impls, new DeleteBrandTaskHeartbeatSpi(mock(DeleteBrandTaskCacheService.class)));
|
||||
register(impls, new PatrolDeleteTaskHeartbeatSpi(mock(PatrolDeleteTaskCacheService.class)));
|
||||
|
||||
Reference in New Issue
Block a user