diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java index 8c4b1d8d6..76f9b6690 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java @@ -72,6 +72,18 @@ public class AppearancePatentProperties { */ private int flushPendingMinutes = 1; + /** + * 组装作业在 WAIT_LLM 停放期间的租约续命间隔(毫秒)。 + * + *
组装作业等下一轮分片回传时 process 返回 false、执行线程退出、Worker 的执行期心跳 + * 随之取消,作业停在 RUNNING 只能靠客户端心跳经 {@code onHeartbeat} 触发 + * {@code touchRunningAssembleJobsIfStale} 按本间隔节流续命(与 similar-asin 同一套口径)。 + * 取值必须小于 job 级 stuck 扫描阈值(result-file-job.stuck-timeout-minutes,默认 5 分钟), + * 否则停放作业在两跳续命之间就会被判陈旧、5 次重试耗尽后被误判成 + * 「文件生成任务运行超时,已达到最大重试次数」(线上任务 31354 的死法)。 + */ + private long dbJobTouchIntervalMillis = 60000L; + /** 单个外观专利源文件最多解析的有效数据行数,防止 POI 用户模型撑爆堆。 */ private int maxParseRows = 50000; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentResultFileJobHandler.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentResultFileJobHandler.java index 4a740c669..a62ad4ba6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentResultFileJobHandler.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentResultFileJobHandler.java @@ -6,7 +6,8 @@ import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; /** * APPEARANCE_PATENT 结果文件 Job Handler(04 注册表)。 - * process 返回 Service 的 boolean(false=等待 LLM 异步结果,Worker 保持运行心跳); + * process 返回 Service 的 boolean(false=停放等待下一轮分片回传触发;此时执行线程已退出、 + * Worker 的执行期心跳已取消,停放租约由客户端心跳经 AppearancePatentTaskHeartbeatSpi#onHeartbeat 续命); * cleanup 委托 cleanupResultFileJob;支持异步 offload;owner scoped。 */ public class AppearancePatentResultFileJobHandler implements ResultFileJobHandler { @@ -32,6 +33,13 @@ public class AppearancePatentResultFileJobHandler implements ResultFileJobHandle appearancePatentTaskService.cleanupResultFileJob(job); } + @Override + public boolean fallbackAssembleOnFailure(TaskFileJobEntity job, String message) { + // 作业重试耗尽(含 WAIT_LLM 停放被 stuck 棘轮判死)时:用已有分片/结论尽力出 + // 部分结果文件供下载,别让已跑完的检测一张表都不剩(线上任务 31302/31354)。 + return appearancePatentTaskService.fallbackAssembleResultFile(job); + } + @Override public boolean supportsAsyncOffload() { return true; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskHeartbeatSpi.java index b36f8b4a5..d2eec08fd 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskHeartbeatSpi.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskHeartbeatSpi.java @@ -1,6 +1,9 @@ package com.nanri.aiimage.modules.appearancepatent.service; +import com.nanri.aiimage.config.AppearancePatentProperties; import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -10,12 +13,21 @@ import org.springframework.stereotype.Service; * *
心跳逻辑(缓存刷新 / 任务缓存回写)从 task 模块收回本模块,task 侧只依赖 SPI 接口, * 消除 task → 业务模块的编译期依赖。 + * + *
onHeartbeat 顺带给结果文件组装作业续租(与 {@code SimilarAsinTaskHeartbeatSpi} 同一套口径): + * 组装作业在 WAIT_LLM 停放期间(process 返回 false 等下一轮分片回传触发)执行线程已退出、 + * Worker 的执行期心跳已随之取消,作业停在 RUNNING 无人续命,会被 job 级 stuck 扫描按 + * updated_at 判陈旧、5 次重试耗尽后误判成「文件生成任务运行超时」(线上任务 31354)。 + * 客户端在任务执行期持续上报心跳,经这里续租即可覆盖整个停放窗口; + * 客户端也停了的场景由 stale 收尾链路与失败兜底(fallbackAssembleResultFile)接住。 */ @Service @RequiredArgsConstructor public class AppearancePatentTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { private final AppearancePatentTaskCacheService cacheService; + private final TaskFileJobService taskFileJobService; + private final AppearancePatentProperties appearancePatentProperties; @Override public String moduleType() { @@ -26,4 +38,19 @@ public class AppearancePatentTaskHeartbeatSpi implements TaskModuleHeartbeatSpi public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { cacheService.touchTaskHeartbeat(taskId); } + + @Override + public void onHeartbeat(FileTaskEntity task) { + if (task == null || task.getId() == null) { + return; + } + // touchRunningAssembleJobsIfStale 的 SQL 自带「距上次刷新超过间隔才写」的节流, + // 心跳再密也不会把库写爆,无需在这里另做节流。 + taskFileJobService.touchRunningAssembleJobsIfStale( + task.getId(), moduleType(), jobTouchIntervalMillis()); + } + + private long jobTouchIntervalMillis() { + return Math.max(1_000L, appearancePatentProperties.getDbJobTouchIntervalMillis()); + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index df9afcf56..112320f52 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -1889,6 +1889,67 @@ public class AppearancePatentTaskService { return true; } + /** + * 失败兜底:组装作业重试耗尽(WAIT_LLM 停放被 stuck 棘轮判死、或组装异常连吃 5 次重试)后调用, + * 用已回传的分片与已拿到结论的行尽力生成可下载的部分结果文件。 + * + *
用户线上点名要的保证:「起码失败了也可以下载」。判死时已跑完的检测结论仍挂在分片里, + * 不出表就一张都不剩(线上任务 31302:1196 行成品困在判死作业里零交付)。 + * + *
覆盖率闸门按「允许部分结果」放行(缺口只记行账);任务状态在这里**不收尾**——判死时 + * 任务可能仍在跑、客户端仍在心跳(stuck 棘轮只判作业、不判任务),提前置终态会把活任务误杀; + * 任务终态仍由 stale 收尾链路 / 分片回传链路按既有口径落。文件挂上 result(result_file_url) + * 后前端下载按钮即出现,与任务/作业状态无关({@code AppearancePatentHistoryAssembler#attachFileJobState})。 + * + *
幂等:结果文件已存在(result_file_url 非空)直接返回 true,不重做组装——stuck 扫描对 + * 终态失败作业每轮都会重入兜底钩子,靠这里挡住重复组装。与补传恢复(maybeRecoverTerminalFailedAssemble) + * 的重跑并发时两边都只写合法结果文件,后写者覆盖,无中间态。 + * + *
实现自吞异常(兜底失败只记日志,不影响作业已终态失败的事实),返回是否有可下载结果。
+ */
+ public boolean fallbackAssembleResultFile(TaskFileJobEntity job) {
+ if (job == null || job.getTaskId() == null || job.getResultId() == null) {
+ return false;
+ }
+ try {
+ FileResultEntity result = fileResultMapper.selectById(job.getResultId());
+ if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
+ log.warn("[appearance-patent] fallback assemble skipped because result record missing taskId={} resultId={}",
+ job.getTaskId(), job.getResultId());
+ return false;
+ }
+ if (result.getResultFileUrl() != null && !result.getResultFileUrl().isBlank()) {
+ log.info("[appearance-patent] fallback assemble skipped because result file already exists taskId={} resultId={} jobStatus={}",
+ job.getTaskId(), job.getResultId(), job.getStatus());
+ return true;
+ }
+ FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
+ if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
+ log.warn("[appearance-patent] fallback assemble skipped because task missing taskId={} resultId={}",
+ job.getTaskId(), job.getResultId());
+ return false;
+ }
+ // 与 processResultFileJobLocked 同一判法:非归属实例让给归属机(stuck 扫描两台都在跑,
+ // 不挡的话每台都会每 60 秒重入一次兜底钩子)。
+ job.setScopeKey(buildTaskOwnerScopeKey(task));
+ if (!isJobOwnedByCurrentInstance(job)) {
+ log.info("[appearance-patent] fallback assemble skipped because owner is another instance taskId={} resultId={}",
+ job.getTaskId(), job.getResultId());
+ return false;
+ }
+ AppearancePatentRowOutcome.Facts facts = assembleResultWorkbook(task, result, true);
+ fileResultMapper.updateById(result);
+ log.warn("[appearance-patent] fallback assemble produced partial result taskId={} resultId={} jobStatus={} expectedRows={} judgedRows={} missingRows={} noDataRows={}",
+ job.getTaskId(), job.getResultId(), job.getStatus(),
+ facts.expectedRows(), facts.judgedRows(), facts.missingRows(), facts.noDataRows());
+ return true;
+ } catch (Exception ex) {
+ log.warn("[appearance-patent] fallback assemble failed taskId={} resultId={} msg={}",
+ job.getTaskId(), job.getResultId(), ex.getMessage(), ex);
+ return false;
+ }
+ }
+
/** 返回真正落库的行键;没匹配到分片行的键不会出现在里面,调用方据此决定要不要撤销已检测标记。 */
private Set 组装作业重试耗尽判死后,用户要「起码失败了也可以下载」:兜底用已有分片/结论出
+ * 部分结果文件。本类钉住兜底入口的分支语义;组装主体(xlsx 生成/上传)由
+ * AppearancePatentSheetBuilderTest/AppearancePatentResultFormatTest/交付闸门测试覆盖。
+ */
+class AppearancePatentFallbackAssembleTest {
+
+ private static final long TASK_ID = 31302L;
+ private static final long RESULT_ID = 9001L;
+ private static final long JOB_ID = 7001L;
+
+ private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
+ private final FileResultMapper fileResultMapper = mock(FileResultMapper.class);
+ private final TaskScopeStateMapper taskScopeStateMapper = mock(TaskScopeStateMapper.class);
+ private final TaskChunkMapper taskChunkMapper = mock(TaskChunkMapper.class);
+ private final LocalFileStorageService localFileStorageService = mock(LocalFileStorageService.class);
+ private final AppearancePatentLlmClient llmClient = mock(AppearancePatentLlmClient.class);
+ private final AppearancePatentTaskCacheService taskCacheService = mock(AppearancePatentTaskCacheService.class);
+ private final AppearancePatentProperties properties = mock(AppearancePatentProperties.class);
+ private final StorageProperties storageProperties = mock(StorageProperties.class);
+ private final TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
+ private final TaskProgressSnapshotService taskProgressSnapshotService = mock(TaskProgressSnapshotService.class);
+ private final TransientPayloadStorageService transientPayloadStorageService = mock(TransientPayloadStorageService.class);
+ private final PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
+ private final DistributedJobLockService distributedJobLockService = mock(DistributedJobLockService.class);
+ private final TaskDistributedLockService taskDistributedLockService = mock(TaskDistributedLockService.class);
+ private final InstanceMetadata instanceMetadata = mock(InstanceMetadata.class);
+
+ private AppearancePatentTaskService service;
+
+ @BeforeEach
+ void setUp() {
+ when(properties.getLlmCoverageMinRatio()).thenReturn(0.95D);
+ service = new AppearancePatentTaskService(
+ localFileStorageService, null,
+ mock(com.nanri.aiimage.common.service.ResultDownloadResolver.class),
+ storageProperties, fileTaskMapper, fileResultMapper,
+ taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
+ properties, taskFileJobService, taskProgressSnapshotService,
+ mock(TaskHeartbeatPositionService.class), transientPayloadStorageService, transactionManager,
+ distributedJobLockService, taskDistributedLockService, instanceMetadata,
+ mock(TaskProgressLightAssembler.class), mock(UserApiSecretService.class));
+ }
+
+ private static TaskFileJobEntity job() {
+ TaskFileJobEntity job = new TaskFileJobEntity();
+ job.setId(JOB_ID);
+ job.setTaskId(TASK_ID);
+ job.setResultId(RESULT_ID);
+ job.setModuleType("APPEARANCE_PATENT");
+ job.setStatus("FAILED");
+ return job;
+ }
+
+ private static FileResultEntity result(String resultFileUrl) {
+ FileResultEntity result = new FileResultEntity();
+ result.setId(RESULT_ID);
+ result.setTaskId(TASK_ID);
+ result.setModuleType("APPEARANCE_PATENT");
+ result.setResultFileUrl(resultFileUrl);
+ return result;
+ }
+
+ private static FileTaskEntity task() {
+ FileTaskEntity task = new FileTaskEntity();
+ task.setId(TASK_ID);
+ task.setModuleType("APPEARANCE_PATENT");
+ task.setStatus("FAILED");
+ task.setUserId(7L);
+ return task;
+ }
+
+ @Test
+ void incompleteJobArgumentsAreRejected() {
+ assertFalse(service.fallbackAssembleResultFile(null), "null 作业直接拒绝");
+ assertFalse(service.fallbackAssembleResultFile(new TaskFileJobEntity()), "缺 taskId/resultId 直接拒绝");
+ }
+
+ @Test
+ void missingResultRecordReturnsFalse() {
+ when(fileResultMapper.selectById(RESULT_ID)).thenReturn(null);
+ assertFalse(service.fallbackAssembleResultFile(job()), "结果记录不存在(孤儿作业)无可组装");
+ verify(fileTaskMapper, never()).selectById(any());
+ }
+
+ @Test
+ void existingResultFileShortCircuitsAssemble() {
+ // 幂等:stuck 扫描对终态失败作业每轮重入兜底钩子,已有结果文件就不再重做组装
+ when(fileResultMapper.selectById(RESULT_ID)).thenReturn(result("appearance-patent/existing.xlsx"));
+ assertTrue(service.fallbackAssembleResultFile(job()), "已有结果文件视为兜底成功");
+ verify(fileTaskMapper, never()).selectById(any());
+ verify(fileResultMapper, never()).updateById(any(FileResultEntity.class));
+ }
+
+ @Test
+ void missingTaskRecordReturnsFalse() {
+ when(fileResultMapper.selectById(RESULT_ID)).thenReturn(result(null));
+ when(fileTaskMapper.selectById(TASK_ID)).thenReturn(null);
+ assertFalse(service.fallbackAssembleResultFile(job()), "任务记录不存在无可组装");
+ }
+
+ @Test
+ void emptyPayloadFailsSoftlyWithoutTouchingResult() {
+ // 组装抛业务异常(无可解析行)时兜底自吞:返回 false、不改写 result,作业保持终态失败
+ when(fileResultMapper.selectById(RESULT_ID)).thenReturn(result(null));
+ FileTaskEntity task = task();
+ task.setResultJson("{}");
+ when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
+
+ assertFalse(service.fallbackAssembleResultFile(job()), "空载荷组装失败要被吞掉");
+ verify(fileResultMapper, never()).updateById(any(FileResultEntity.class));
+ }
+}
diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskHeartbeatSpiTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskHeartbeatSpiTest.java
new file mode 100644
index 000000000..dc3905179
--- /dev/null
+++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskHeartbeatSpiTest.java
@@ -0,0 +1,73 @@
+package com.nanri.aiimage.modules.appearancepatent.service;
+
+import com.nanri.aiimage.config.AppearancePatentProperties;
+import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
+import com.nanri.aiimage.modules.task.service.TaskFileJobService;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+/**
+ * 外观专利任务心跳的组装作业续租(2026-10-18,线上任务 31354 死因修复)。
+ *
+ * 组装作业在 WAIT_LLM 停放期间执行线程已退出、Worker 执行期心跳已取消,若无人续命,
+ * job 级 stuck 扫描(stuck-timeout-minutes 默认 5 分钟)会按 updated_at 判陈旧、
+ * 5 次重试耗尽后误判「文件生成任务运行超时」。客户端心跳经 onHeartbeat 触发
+ * touchRunningAssembleJobsIfStale 按配置间隔节流续租(与 SimilarAsinTaskHeartbeatSpi 同口径)。
+ */
+class AppearancePatentTaskHeartbeatSpiTest {
+
+ private final AppearancePatentTaskCacheService cacheService = mock(AppearancePatentTaskCacheService.class);
+ private final TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
+ private final AppearancePatentProperties properties = new AppearancePatentProperties();
+
+ private AppearancePatentTaskHeartbeatSpi spi;
+
+ @BeforeEach
+ void setUp() {
+ spi = new AppearancePatentTaskHeartbeatSpi(cacheService, taskFileJobService, properties);
+ }
+
+ @Test
+ void onHeartbeatRenewsAssembleJobLease() {
+ FileTaskEntity task = new FileTaskEntity();
+ task.setId(31354L);
+ task.setModuleType("APPEARANCE_PATENT");
+
+ spi.onHeartbeat(task);
+
+ verify(taskFileJobService).touchRunningAssembleJobsIfStale(31354L, "APPEARANCE_PATENT",
+ properties.getDbJobTouchIntervalMillis());
+ }
+
+ @Test
+ void touchIntervalBelowStuckScanFloor() {
+ // 续命间隔必须小于 stuck 扫描阈值(5 分钟),否则停放作业在两跳续命之间仍会被判陈旧
+ org.junit.jupiter.api.Assertions.assertTrue(properties.getDbJobTouchIntervalMillis() < 5 * 60_000L,
+ "dbJobTouchIntervalMillis 应小于 stuck-timeout-minutes=5 分钟,实际 "
+ + properties.getDbJobTouchIntervalMillis());
+ }
+
+ @Test
+ void onHeartbeatWithNullTaskTouchesNothing() {
+ spi.onHeartbeat(null);
+ FileTaskEntity task = new FileTaskEntity();
+ spi.onHeartbeat(task);
+ verifyNoInteractions(taskFileJobService);
+ }
+
+ @Test
+ void touchHeartbeatStillDelegatesToCache() {
+ spi.touchHeartbeat(1L, null);
+ verify(cacheService).touchTaskHeartbeat(1L);
+ }
+
+ @Test
+ void moduleTypeMatchesRegistry() {
+ org.junit.jupiter.api.Assertions.assertEquals("APPEARANCE_PATENT", spi.moduleType());
+ }
+}
diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/AppearancePatentResultFileJobHandlerTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/AppearancePatentResultFileJobHandlerTest.java
index 51e9cf209..192de0b74 100644
--- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/AppearancePatentResultFileJobHandlerTest.java
+++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/AppearancePatentResultFileJobHandlerTest.java
@@ -77,4 +77,21 @@ class AppearancePatentResultFileJobHandlerTest {
handler.cleanup(null);
org.mockito.Mockito.verify(appearancePatentTaskService).cleanupResultFileJob(null);
}
+
+ @Test
+ void fallbackAssembleDelegates() {
+ // 重试耗尽(含 WAIT_LLM 停放被 stuck 棘轮判死)时委托 Service 出部分结果文件
+ TaskFileJobEntity job = new TaskFileJobEntity();
+ when(appearancePatentTaskService.fallbackAssembleResultFile(job)).thenReturn(true);
+ assertTrue(handler.fallbackAssembleOnFailure(job, "文件生成任务运行超时,已达到最大重试次数"),
+ "兜底产出可下载结果时返回 true");
+ }
+
+ @Test
+ void fallbackAssembleFailureReturnsFalse() {
+ TaskFileJobEntity job = new TaskFileJobEntity();
+ when(appearancePatentTaskService.fallbackAssembleResultFile(job)).thenReturn(false);
+ assertFalse(handler.fallbackAssembleOnFailure(job, "结果文件生成失败"),
+ "兜底也失败时返回 false,不影响作业已终态失败的事实");
+ }
}
diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskModuleHeartbeatSpiCoverageTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskModuleHeartbeatSpiCoverageTest.java
index 224fd5df1..cc9b1ac17 100644
--- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskModuleHeartbeatSpiCoverageTest.java
+++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskModuleHeartbeatSpiCoverageTest.java
@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.task.service;
+import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskHeartbeatSpi;
@@ -46,7 +47,9 @@ class TaskModuleHeartbeatSpiCoverageTest {
private static Map