From e08b5369d7fbe6306fe641b2f263fe53539f464a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Tue, 27 Oct 2026 15:53:10 +0800 Subject: [PATCH] =?UTF-8?q?fix(appearance-patent):=20=E4=B8=8A=E6=B8=B8?= =?UTF-8?q?=E4=BD=99=E9=A2=9D=E6=AC=A0=E8=B4=B9=E5=8D=B3=E5=88=A4=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E7=BB=88=E6=80=81=EF=BC=8C=E5=81=9C=E6=AD=A2=E4=B8=BA?= =?UTF-8?q?=E7=A9=BA=E8=BD=AC=E7=9A=84=E8=A1=8C=E7=83=A7=E5=87=BA=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B46 那一笔(f3445c21)只做到「行级一次就停、归因写对」:撞上上游 403 insufficient_user_quota 后不再跑满 3 次重试,任务本身仍停在 RUNNING/WAIT_LLM 等 Python 继续回传分片。采集端(maixiang ap_worker)只看 /task/heartbeat 的 alive,读不到终态就继续为剩下的行按次付费买出口 IP——而那些行永远进不了 LLM, 钱白花。这一笔补上 Java 侧缺的那半步:欠费即把任务落终态 FAILED,心跳随即 回 alive=false,worker 下一跳收手。 - 判据只认 TaskRetryEligibility 的 BALANCE 类(欠费 + 密钥失效),复用现成分类器 而非再抄一份字面量表;连接不可达 / 5xx / 429 / 抓取缺口一律照旧继续或按已有 结果交付,判宽了就是拿用户整单给一次网络抖动陪葬。 - 停手点放在批管线的作业执行线程(LLM 批次就在这里跑),不在心跳线程;识别到 欠费即停止后续批次,并回调宿主落终态(FileTaskMapper 与收尾口径在宿主手里)。 - 写终态前重读库里现状态:已 SUCCESS/COMPLETED/CANCELLED/FAILED 的一律不动, 因此可重入,多入口命中也只写一次。 - 终态文案 = 生产者原文(客户端欠费归因 + 上游 lastErr)+ 可执行动作,原文一个 字都不重写;不出现「请稍后重试」,不把欠费说成临时抖动。 --- .../aiimage/common/retry/LlmBalanceGuard.java | 12 + .../service/AppearancePatentTaskService.java | 13 +- .../AppearancePatentLlmBatchPipeline.java | 39 +- .../AppearancePatentResultJobSupport.java | 65 ++++ ...CandidateCollectionFaultToleranceTest.java | 6 +- ...ppearancePatentLlmArrearsTerminalTest.java | 367 ++++++++++++++++++ ...arancePatentArrearsHeartbeatAliveTest.java | 108 ++++++ 7 files changed, 605 insertions(+), 5 deletions(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentLlmArrearsTerminalTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/task/service/AppearancePatentArrearsHeartbeatAliveTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/retry/LlmBalanceGuard.java b/backend-java/src/main/java/com/nanri/aiimage/common/retry/LlmBalanceGuard.java index 4fb9af4bc..a42bed790 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/common/retry/LlmBalanceGuard.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/retry/LlmBalanceGuard.java @@ -18,6 +18,18 @@ public final class LlmBalanceGuard { /** 行级/任务级统一归因文案:说清是上游账户欠费,不是这行数据的问题。 */ public static final String REASON_LLM_BALANCE_DUE = "LLM 服务余额欠费(上游账户额度不足,非本行数据问题)"; + /** + * 任务级终态文案(2026-10-27 用户口径:「提示语要纠正一下,不能误导用户」)。 + * + *

欠费与密钥失效都是外部条件坏了:既不能写成「请稍后重试」(把欠费说成临时抖动, + * 用户会干等),也不能让人以为是自己那张表格的问题。这里只说三件事—— + * 谁的问题(上游账户/密钥)、已经做了什么(任务终止、停止继续消耗采集出口)、 + * 下一步做什么(充值或换密钥后重新发起)。 + */ + public static final String TASK_TERMINAL_GUIDANCE = + "任务已终止:上游 LLM 账户欠费或密钥失效,不是你本次上传表格的数据问题;" + + "已停止后续检测与采集,以免继续消耗代理出口额度。请充值或更换有效密钥后重新发起检测。"; + private static final String[] MARKERS = { "insufficient_user_quota", "insufficient_quota", diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index c6b914b81..af993aa6d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -367,7 +367,18 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline rows); } + /** + * 上游欠费/密钥失效的任务级处置出口(2026-10-27)。批管线只负责「识别 + 停手」, + * 把任务写成终态要 {@code FileTaskMapper} 与收尾口径,那些都在宿主 + * {@code AppearancePatentResultJobSupport} 手里,故以回调交出(与本类其余回调同一形状)。 + * + *

调用点是作业执行线程(LLM 批次就在这里跑),不是心跳线程——铁律 8 不许在心跳里判死。 + */ + @FunctionalInterface + public interface FatalLlmFailureHandler { + void onFatalLlmFailure(FileTaskEntity task, String failureMessage); + } + private final AppearancePatentProperties properties; private final AppearancePatentLlmClient llmClient; private final AppearancePatentTaskCacheService taskCacheService; @@ -71,6 +83,7 @@ public class AppearancePatentLlmBatchPipeline { private final Function apiKeyReader; private final ChunkRowMerger chunkRowMerger; private final BiFunction firstNonBlankFn; + private final FatalLlmFailureHandler fatalLlmFailureHandler; public AppearancePatentLlmBatchPipeline(AppearancePatentProperties properties, AppearancePatentLlmClient llmClient, @@ -85,7 +98,8 @@ public class AppearancePatentLlmBatchPipeline { Function promptReader, Function apiKeyReader, ChunkRowMerger chunkRowMerger, - BiFunction firstNonBlankFn) { + BiFunction firstNonBlankFn, + FatalLlmFailureHandler fatalLlmFailureHandler) { this.properties = properties; this.llmClient = llmClient; this.taskCacheService = taskCacheService; @@ -100,6 +114,7 @@ public class AppearancePatentLlmBatchPipeline { this.apiKeyReader = apiKeyReader; this.chunkRowMerger = chunkRowMerger; this.firstNonBlankFn = firstNonBlankFn; + this.fatalLlmFailureHandler = fatalLlmFailureHandler; } /** @@ -150,6 +165,7 @@ public class AppearancePatentLlmBatchPipeline { } int batchTotal = Math.max(1, (submitLimit + batchSize - 1) / batchSize); int batchIndex = 1; + String fatalLlmMessage = null; for (int i = 0; i < submitLimit; i += batchSize) { List batchRows = candidates.subList(i, Math.min(i + batchSize, submitLimit)); taskFileJobService.touchRunning(job.getId()); @@ -164,8 +180,12 @@ public class AppearancePatentLlmBatchPipeline { () -> llmClient.inspectRows(batchRows, prompt, apiKey)); } catch (Exception ex) { String message = firstNonBlank(ex.getMessage(), "LLM 检测失败"); - log.warn("[appearance-patent] llm batch failed taskId={} jobId={} rows={} batch={}/{} err={}", - task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal, message); + if (AppearancePatentResultJobSupport.isFatalLlmArrearsFailure(message)) { + fatalLlmMessage = message; + } + log.warn("[appearance-patent] llm batch failed taskId={} jobId={} rows={} batch={}/{} fatalLlmArrears={} err={}", + task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal, + fatalLlmMessage != null, message); llmRows = llmClient.markRowsFailed(batchRows, message); batchAnswered = false; } @@ -192,6 +212,19 @@ public class AppearancePatentLlmBatchPipeline { } } } + if (fatalLlmMessage != null) { + // 上游账户欠费/密钥失效:剩下每一批都拿不到结论,而 Python 采集端只看 + // /task/heartbeat 的 alive——任务不落终态,它就继续为那些永远进不了 LLM 的行 + // 按次付费买出口 IP(2026-10-27 B46 现网形状)。所以这里既停批(不再发下一批), + // 又把「落终态」交给宿主:FileTaskMapper 与收尾口径在它手里,且这一步发生在 + // 作业执行线程,不是心跳线程(铁律 8)。 + log.warn("[appearance-patent] LLM 上游欠费/密钥失效,停止后续批次并把任务落终态" + + " taskId={} jobId={} batch={}/{} pendingRows={} err={}", + task.getId(), job.getId(), batchIndex, batchTotal, + Math.max(0, submitLimit - i), fatalLlmMessage); + fatalLlmFailureHandler.onFatalLlmFailure(task, fatalLlmMessage); + return; + } log.info("[appearance-patent] llm batch done taskId={} jobId={} rows={} batch={}/{} costMs={}", task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal, System.currentTimeMillis() - startedAt); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentResultJobSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentResultJobSupport.java index a9f8175da..f1ecb8754 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentResultJobSupport.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentResultJobSupport.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.appearancepatent.service.support; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.retry.LlmBalanceGuard; import com.nanri.aiimage.common.task.TaskStatuses; import com.nanri.aiimage.common.task.GroupResultPropagator; import com.nanri.aiimage.config.AppearancePatentProperties; @@ -24,6 +25,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFinalizeSupport; +import com.nanri.aiimage.modules.task.service.TaskRetryEligibility; import lombok.extern.slf4j.Slf4j; import org.apache.poi.xssf.streaming.SXSSFWorkbook; @@ -249,6 +251,69 @@ public class AppearancePatentResultJobSupport { taskCacheService.deleteTaskCache(task.getId()); } + /** + * 只认「上游账户欠费 / LLM 密钥失效」这一类不可自愈的失败。分类口径复用 + * {@link TaskRetryEligibility}(它已经是"不可抗力不克隆重跑"的唯一判据),不再抄第二份字面量表。 + * + *

刻意只取 {@code BALANCE}:连接不可达({@code SERVICE_DOWN})、429 限流、5xx 抖动、 + * 抓取缺口都属「外部会自己恢复或补传能救」,判宽了就是拿用户的任务给一次网络抖动陪葬。 + */ + static boolean isFatalLlmArrearsFailure(String errorMessage) { + return TaskRetryEligibility.forceMajeureCategory(errorMessage) == TaskRetryEligibility.Category.BALANCE; + } + + /** + * 上游余额欠费/密钥失效 → 任务立即落终态 FAILED。 + * + *

为什么必须由 Java 侧落终态(2026-10-27 现网):{@code f3445c21} 只做到「行级不再重试三次、 + * 归因写对」,任务本身仍停在 RUNNING/WAIT_LLM 等 Python 继续回传分片。采集端 + * ({@code maixiang ap_worker})只看 {@code /task/heartbeat} 的 {@code alive},看不到终态就会 + * 继续为剩下的行按次付费买出口 IP——而那些行永远进不了 LLM,钱白花。 + * 任务一终态,{@code TaskHeartbeatService} 对其返回 {@code alive=false}(既有行为),worker 下一跳收手。 + * + *

判定发生在作业执行线程(LLM 批次就在这里跑),不在心跳线程(铁律 8)。 + * 写之前重读一次库里的现状态:已 SUCCESS/COMPLETED/CANCELLED/FAILED 的一律不动(铁律 14), + * 本方法因此可重入,停放期间同一任务被多个入口命中欠费也只写一次。 + */ + public void terminalizeOnFatalLlmFailure(FileTaskEntity task, String failureMessage) { + if (task == null || task.getId() == null) { + return; + } + if (!isFatalLlmArrearsFailure(failureMessage)) { + // 兜底闸:调用方判宽了也不许把可恢复的失败写成终态 + return; + } + FileTaskEntity current = fileTaskMapper.selectById(task.getId()); + String currentStatus = current != null ? current.getStatus() : task.getStatus(); + if (TaskStatuses.FileTask.TERMINAL.contains(currentStatus)) { + log.info("[appearance-patent] 任务已是终态,欠费归因不改写状态 taskId={} status={}", + task.getId(), currentStatus); + return; + } + String terminalMessage = fatalLlmTerminalMessage(failureMessage); + task.setStatus(STATUS_FAILED); + task.setSuccessFileCount(0); + task.setFailedFileCount(1); + task.setErrorMessage(terminalMessage); + task.setUpdatedAt(LocalDateTime.now()); + task.setFinishedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + taskCacheService.deleteTaskCache(task.getId()); + log.warn("[appearance-patent] LLM 上游欠费/密钥失效,任务落终态 FAILED 以停止为空转的行继续买出口 IP" + + " taskId={} err={}", task.getId(), terminalMessage); + } + + /** + * 终态文案 = 生产者原文(客户端的欠费归因 + 上游 lastErr)+ 可执行动作, + * 原文一个字都不在这里重写;文案里没有「稍后重试」,欠费不会被说成抖动。 + */ + static String fatalLlmTerminalMessage(String failureMessage) { + String raw = failureMessage == null ? "" : failureMessage.trim(); + return raw.isEmpty() + ? LlmBalanceGuard.TASK_TERMINAL_GUIDANCE + : raw + ";" + LlmBalanceGuard.TASK_TERMINAL_GUIDANCE; + } + public FileResultEntity findOrCreateResultRecordForAssembly(FileTaskEntity task, int rowCount) { if (task == null || task.getId() == null) { return null; diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentCandidateCollectionFaultToleranceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentCandidateCollectionFaultToleranceTest.java index 7734c5264..461ba1634 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentCandidateCollectionFaultToleranceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentCandidateCollectionFaultToleranceTest.java @@ -180,7 +180,11 @@ class AppearancePatentCandidateCollectionFaultToleranceTest { task -> "prompt", task -> "sk-test", (taskId, scopeHash, chunkIndex, rows) -> List.of(), - (preferred, fallback) -> preferred == null || preferred.isBlank() ? fallback : preferred); + (preferred, fallback) -> preferred == null || preferred.isBlank() ? fallback : preferred, + // 本夹具只驱动「候选收集」,永远不会进欠费终态分支;真进来了就说明测试形状变了 + (fatalTask, failureMessage) -> { + throw new IllegalStateException("unexpected fatal llm failure: " + failureMessage); + }); } private Set releasedKeys() { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentLlmArrearsTerminalTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentLlmArrearsTerminalTest.java new file mode 100644 index 000000000..c79d5c6ba --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentLlmArrearsTerminalTest.java @@ -0,0 +1,367 @@ +package com.nanri.aiimage.modules.appearancepatent.service; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.retry.LlmBalanceGuard; +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.common.service.ResultDownloadResolver; +import com.nanri.aiimage.common.task.TaskStatuses; +import com.nanri.aiimage.config.AppearancePatentProperties; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto; +import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentResultJobSupport; +import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentRowOutcome; +import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; +import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.transaction.PlatformTransactionManager; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 上游 LLM 账户欠费(2026-10-27 现网 B46:403 {@code insufficient_user_quota})必须把外观专利 + * 任务立刻落成终态 FAILED。 + * + *

为什么这是独立的一笔(而不是 B46 已经做完的那半):{@code f3445c21} 只让行级调用「一次就停、 + * 归因写对」,任务本身还停在 RUNNING/WAIT_LLM 等 Python 继续回传分片。采集端只看 + * {@code /task/heartbeat} 的 {@code alive},看不到终态就会继续为剩下的行按次付费买出口 IP, + * 而那些行永远进不了 LLM。任务变终态 → 心跳 {@code alive=false} → worker 下一跳收手。 + * + *

反向半边同样重要:连接不可达 / 5xx / 429 / 抓取缺口都必须照旧继续, + * 判宽了就是拿用户的整单去给一次网络抖动陪葬。 + * + *

夹具走 {@code service.submitLlmBatches} 这条生产接线(批管线 → 欠费处置回调 → + * 结果作业支撑 → FileTaskMapper),不手工 new 二十几个参数,以免测的是一套生产里不存在的 wiring。 + * 异常原文一律用外部世界的字符串或本仓常量的引用,不手抄文案。 + */ +@ExtendWith(MockitoExtension.class) +class AppearancePatentLlmArrearsTerminalTest { + + private static final Long TASK_ID = 32901L; + private static final Long JOB_ID = 24110L; + private static final String SCOPE_HASH = "scope-hash-1"; + private static final int CHUNK_INDEX = 1; + private static final String POINTER = "ptr-chunk-1.json"; + /** 两行都还没有结论 ⇒ batchSize=1 时正好两批:第二批发不发,就是「烧不烧出口」的分界。 */ + private static final String UNRESOLVED_ROWS = + "[{\"rowToken\":\"r1\",\"asin\":\"B0A0000001\",\"title\":\"RUNNING SHOE\",\"url\":\"http://img/1.jpg\",\"groupKey\":\"g1\"}," + + "{\"rowToken\":\"r2\",\"asin\":\"B0A0000002\",\"title\":\"COTTON BAG\",\"url\":\"http://img/2.jpg\",\"groupKey\":\"g2\"}]"; + /** 上游 403 原文(现网形状)。这是外部世界的字符串,不是本仓文案,抄它不算抄自家字面量。 */ + private static final String QUOTA_403_BODY = "{\"error\":{\"message\":\"预扣费额度失败, 用户[151246]剩余额度: " + + "฿0.029168, 需要预扣费额度: ฿0.032402\",\"type\":\"new_api_error\",\"code\":\"insufficient_user_quota\"}}"; + + @Mock private LocalFileStorageService localFileStorageService; + @Mock private OssStorageService ossStorageService; + @Mock private ResultDownloadResolver resultDownloadResolver; + @Mock private StorageProperties storageProperties; + @Mock private FileTaskMapper fileTaskMapper; + @Mock private FileResultMapper fileResultMapper; + @Mock private TaskScopeStateMapper taskScopeStateMapper; + @Mock private TaskChunkMapper taskChunkMapper; + @Spy private ObjectMapper objectMapper = new ObjectMapper(); + @Mock private AppearancePatentLlmClient llmClient; + @Mock private AppearancePatentTaskCacheService taskCacheService; + @Mock private AppearancePatentProperties properties; + @Mock private TaskFileJobService taskFileJobService; + @Mock private TaskProgressSnapshotService taskProgressSnapshotService; + @Mock private TaskHeartbeatPositionService taskHeartbeatPositionService; + @Mock private TransientPayloadStorageService transientPayloadStorageService; + @Mock private PlatformTransactionManager transactionManager; + @Mock private DistributedJobLockService distributedJobLockService; + @Mock private TaskDistributedLockService taskDistributedLockService; + @Mock private InstanceMetadata instanceMetadata; + @Mock private TaskProgressLightAssembler taskProgressLightAssembler; + @Mock private UserApiSecretService userApiSecretService; + + @InjectMocks private AppearancePatentTaskService service; + + @BeforeAll + static void initializeMybatisMetadata() { + MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), ""); + TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class); + TableInfoHelper.initTableInfo(assistant, FileResultEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class); + } + + @BeforeEach + void setUp() { + lenient().when(properties.getLlmBatchSize()).thenReturn(1); + lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true); + lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER), anyString())).thenReturn(UNRESOLVED_ROWS); + lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned( + anyString(), anyLong(), anyString(), any(), anyString())).thenReturn("sibling-1"); + lenient().when(taskChunkMapper.selectById(anyLong())).thenReturn(chunk()); + lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1); + lenient().when(userApiSecretService.findPlainValue(any(), anyString())).thenReturn(""); + } + + // ===== (a) 欠费 → 作业路径把任务落成终态,并带上正确文案 ===== + + /** + * 上游 403 欠费原文进作业路径:任务必须立刻 FAILED+写明动作,且第二批不再发出 + * (发一次就多扣一次用户 LLM 次数,而结论永远拿不到)。 + */ + @Test + void upstreamQuotaFailureTerminalizesTaskAndStopsRemainingBatches() throws Exception { + FileTaskEntity task = submitFailingWith(new IllegalStateException(QUOTA_403_BODY)); + + FileTaskEntity written = capturedTerminalWrite(); + assertEquals(TASK_ID, written.getId()); + assertEquals(TaskStatuses.FileTask.FAILED, written.getStatus(), "欠费必须把任务落 FAILED,不许停在 RUNNING/WAIT_LLM"); + assertNotNull(written.getFinishedAt(), "终态必须带 finished_at,否则读侧仍按在跑展示"); + assertEquals(0, written.getSuccessFileCount()); + assertEquals(1, written.getFailedFileCount()); + // 生产者原文一个字都没被重写 + assertTrue(written.getErrorMessage().contains(QUOTA_403_BODY), "实际: " + written.getErrorMessage()); + // 可执行动作(充值/换密钥后重发)由本仓文案给出 + assertTrue(written.getErrorMessage().contains(LlmBalanceGuard.TASK_TERMINAL_GUIDANCE), + "实际: " + written.getErrorMessage()); + assertEquals(TaskStatuses.FileTask.FAILED, task.getStatus(), "内存实体也要同步:宿主随后据此决定不再停放"); + verify(llmClient, times(1)).inspectRows(any(), any(), any()); + } + + /** 客户端把欠费包成 {@code REASON_LLM_BALANCE_DUE + lastErr} 那一层,同样必须落终态。 */ + @Test + void clientWrappedBalanceReasonAlsoTerminalizesTask() throws Exception { + String producerMessage = LlmBalanceGuard.REASON_LLM_BALANCE_DUE + ";lastErr=" + QUOTA_403_BODY; + + submitFailingWith(new IllegalStateException(producerMessage, new IllegalStateException(QUOTA_403_BODY))); + + FileTaskEntity written = capturedTerminalWrite(); + assertEquals(TaskStatuses.FileTask.FAILED, written.getStatus()); + assertTrue(written.getErrorMessage().contains(LlmBalanceGuard.REASON_LLM_BALANCE_DUE), + "实际: " + written.getErrorMessage()); + assertTrue(written.getErrorMessage().contains(QUOTA_403_BODY), "实际: " + written.getErrorMessage()); + verify(llmClient, times(1)).inspectRows(any(), any(), any()); + } + + /** + * 密钥失效({@code TaskRetryEligibility} 的 KEY_MARKERS 一族)与欠费同属「不修外部条件、 + * 重跑必然同结果」,也要落终态;这条钉住判据取的是 BALANCE 类而不是只认欠费字样。 + */ + @Test + void invalidLlmKeyTerminalizesTaskToo() throws Exception { + submitFailingWith(new IllegalStateException("LLM http 401: invalid api key")); + + assertEquals(TaskStatuses.FileTask.FAILED, capturedTerminalWrite().getStatus()); + } + + /** + * 提示语不许误导用户:终态文案里不能出现「请稍后重试」这类把欠费说成临时抖动的话, + * 必须写明充值/换密钥这个动作。文案取自真实落库的那一条,不是测试自己拼的。 + */ + @Test + void terminalCopyPointsAtTheRealCauseAndAnActionNotAtATransientBlip() throws Exception { + submitFailingWith(new IllegalStateException(QUOTA_403_BODY)); + + String copy = capturedTerminalWrite().getErrorMessage(); + assertFalse(copy.contains("稍后重试"), "欠费被写成抖动: " + copy); + assertFalse(copy.contains("无需修改"), "欠费被写成表格问题: " + copy); + assertTrue(copy.contains("充值"), "实际: " + copy); + assertTrue(copy.contains("密钥"), "实际: " + copy); + assertTrue(copy.contains("任务已终止"), "实际: " + copy); + } + + // ===== (b) 其他失败原因一律继续:不许误杀成终态 ===== + + /** 连上了只是这一轮 5xx:连接层与上游都会自己恢复,任务照旧停放,剩下的批次照旧跑。 */ + @Test + void gatewayFiveOhKeepsTaskRunningAndStillRunsEveryBatch() throws Exception { + FileTaskEntity task = submitFailingWith(new IllegalStateException("LLM http 502: Bad Gateway")); + + verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class)); + assertEquals(TaskStatuses.FileTask.RUNNING, task.getStatus()); + assertNull(task.getFinishedAt()); + verify(llmClient, times(2)).inspectRows(any(), any(), any()); + } + + /** 429 限流不是欠费({@code LlmBalanceGuardTest} 已钉过识别侧),这里钉任务侧同样不动。 */ + @Test + void rateLimitedBatchesKeepTaskRunning() throws Exception { + FileTaskEntity task = submitFailingWith(new IllegalStateException("LLM http 429: Too Many Requests")); + + verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class)); + assertEquals(TaskStatuses.FileTask.RUNNING, task.getStatus()); + assertNull(task.getFinishedAt()); + verify(llmClient, times(2)).inspectRows(any(), any(), any()); + } + + /** 连接根本没建立(不可达一族):归 SERVICE_DOWN,属「等服务恢复」,不是判死的理由。 */ + @Test + void connectFailureKeepsTaskRunningAndIsNotMistakenForArrears() throws Exception { + FileTaskEntity task = submitFailingWith(new IllegalStateException("Connection refused")); + + verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class)); + assertEquals(TaskStatuses.FileTask.RUNNING, task.getStatus()); + verify(llmClient, times(2)).inspectRows(any(), any(), any()); + } + + /** + * 抓取缺口(交付侧的「有 N 行未完成检测」)原文由真实生产者 {@code appendDeliveryGapMessage} + * 生成后灌进这条路径:它是可带缺口交付的部分失败,绝不能被当成欠费判死。 + */ + @Test + @SuppressWarnings("unchecked") + void deliveryGapCopyIsNotMistakenForArrears() throws Exception { + Object deliverySupport = ReflectionTestUtils.invokeMethod(service, "deliverySupport"); + String gapCopy = (String) ReflectionTestUtils.invokeMethod(deliverySupport, "appendDeliveryGapMessage", + null, new AppearancePatentRowOutcome.Facts(5, 3, 0, List.of("B0A0000001", "B0A0000002"))); + assertTrue(gapCopy.contains("未完成检测"), "夹具应当是缺口文案,实际: " + gapCopy); + + FileTaskEntity task = submitFailingWith(new IllegalStateException(gapCopy)); + + verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class)); + assertEquals(TaskStatuses.FileTask.RUNNING, task.getStatus()); + } + + // ===== (c) 已是终态的任务不被改写(铁律 14) ===== + + @Test + void successfulTaskIsNeverRewrittenToFailedByArrears() throws Exception { + assertTerminalStatusUntouchedByArrears(TaskStatuses.FileTask.SUCCESS); + } + + @Test + void cancelledTaskIsNeverRewrittenToFailedByArrears() throws Exception { + assertTerminalStatusUntouchedByArrears(TaskStatuses.FileTask.CANCELLED); + } + + /** 已经终态(含上一跳刚被欠费判死):重入只许挡回去,不许第二次写库。 */ + @Test + void alreadyFailedTaskIsNotWrittenAgain() throws Exception { + assertTerminalStatusUntouchedByArrears(TaskStatuses.FileTask.FAILED); + } + + // ===== (d) 终态之后心跳的 alive 语义(worker 收手的依据) ===== + + /** + * 欠费文案灌进既有收尾 {@code finalizeTask} 时必须落 FAILED,不能被 + * 「可带缺口交付」的抓取缺口判定救回 SUCCESS——这条是对 {@code isPartialDataGapError} + * 那段既有语义的实证,不是照抄它的注释。 + */ + @Test + void arrearsCopyIsNotSalvagedAsAPartialDataGapByFinalizeTask() throws Exception { + submitFailingWith(new IllegalStateException(QUOTA_403_BODY)); + String terminalCopy = capturedTerminalWrite().getErrorMessage(); + when(fileResultMapper.selectList(any())).thenReturn(List.of()); + + AppearancePatentResultJobSupport support = + ReflectionTestUtils.invokeMethod(service, "resultJobSupport"); + FileTaskEntity another = taskWithStatus(TaskStatuses.FileTask.RUNNING); + support.finalizeTask(another, terminalCopy, 0, true); + + assertEquals(TaskStatuses.FileTask.FAILED, another.getStatus(), + "欠费文案被当成可带缺口交付的抓取缺口了: " + another.getErrorMessage()); + assertTrue(another.getErrorMessage().contains(terminalCopy), "实际: " + another.getErrorMessage()); + } + + // ===== 辅助 ===== + + /** 用给定异常打爆 LLM 批次;返回宿主手里的那枚任务实体。 */ + private FileTaskEntity submitFailingWith(Throwable failure) throws Exception { + FileTaskEntity task = taskWithStatus(TaskStatuses.FileTask.RUNNING); + when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true); + when(llmClient.inspectRows(any(), any(), any())).thenThrow(failure); + when(llmClient.markRowsFailed(any(), anyString())).thenReturn(rows(UNRESOLVED_ROWS)); + + ReflectionTestUtils.invokeMethod(service, "submitLlmBatches", + task, new FileResultEntity(), job(), List.of(chunk())); + return task; + } + + private FileTaskEntity capturedTerminalWrite() { + ArgumentCaptor captor = ArgumentCaptor.forClass(FileTaskEntity.class); + verify(fileTaskMapper).updateById(captor.capture()); + return captor.getValue(); + } + + private void assertTerminalStatusUntouchedByArrears(String dbStatus) throws Exception { + // 现网形状:作业手里这枚实体还是 RUNNING,而库里刚被别的链路写成终态(交付成功/用户取消)。 + // 铁律 14/1:这种时刻欠费分支只许挡回去,连内存实体都不许改写。 + when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithStatus(dbStatus)); + FileTaskEntity inHand = submitFailingWith(new IllegalStateException(QUOTA_403_BODY)); + + verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class)); + assertEquals(TaskStatuses.FileTask.RUNNING, inHand.getStatus(), + "库里已是 " + dbStatus + ",内存实体却被欠费分支改写成了 " + inHand.getStatus()); + assertNull(inHand.getFinishedAt()); + } + + private List rows(String json) throws Exception { + return List.of(new ObjectMapper().readValue(json, AppearancePatentResultRowDto[].class)); + } + + private FileTaskEntity taskWithStatus(String status) { + FileTaskEntity task = new FileTaskEntity(); + task.setId(TASK_ID); + task.setModuleType(AppearancePatentTaskService.MODULE_TYPE); + task.setUserId(7L); + task.setStatus(status); + return task; + } + + private TaskFileJobEntity job() { + TaskFileJobEntity job = new TaskFileJobEntity(); + job.setId(JOB_ID); + job.setTaskId(TASK_ID); + job.setResultId(34819L); + return job; + } + + private TaskChunkEntity chunk() { + TaskChunkEntity chunk = new TaskChunkEntity(); + chunk.setId(11L); + chunk.setTaskId(TASK_ID); + chunk.setModuleType(AppearancePatentTaskService.MODULE_TYPE); + chunk.setScopeHash(SCOPE_HASH); + chunk.setChunkIndex(CHUNK_INDEX); + chunk.setPayloadJson(POINTER); + chunk.setPayloadHash("hash-old"); + return chunk; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/AppearancePatentArrearsHeartbeatAliveTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/AppearancePatentArrearsHeartbeatAliveTest.java new file mode 100644 index 000000000..d4fd615d8 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/AppearancePatentArrearsHeartbeatAliveTest.java @@ -0,0 +1,108 @@ +package com.nanri.aiimage.modules.task.service; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.nanri.aiimage.common.retry.LlmBalanceGuard; +import com.nanri.aiimage.config.AppearancePatentProperties; +import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService; +import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskHeartbeatSpi; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo; +import com.nanri.aiimage.modules.task.spi.BrandTaskHeartbeatSpi; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 欠费终态之后心跳必须回 {@code alive=false}——这是 Python 采集端({@code maixiang ap_worker}) + * 收手的唯一信号:任务还停在 RUNNING/WAIT_LLM 时它看不到终态,就会继续为剩下的行按次付费 + * 买出口 IP,而那些行永远进不了 LLM(2026-10-27 现网 B46)。 + * + *

本类不改 {@link TaskHeartbeatService},只按 {@code TaskHeartbeatServiceTest} 的夹具风格钉 + * 「外观专利任务被欠费判成 FAILED 之后」这一格读数,并留一枚 RUNNING 对照,防止整条断言 + * 因为桩没接上而变成恒 false 的假绿。 + */ +class AppearancePatentArrearsHeartbeatAliveTest { + + private static final Long TASK_ID = 32901L; + /** 上游 403 原文(外部世界的字符串)+ 本仓归因常量的引用,不手抄自家文案。 */ + private static final String QUOTA_403_BODY = "{\"error\":{\"code\":\"insufficient_user_quota\"," + + "\"message\":\"预扣费额度失败, 用户[151246]剩余额度: ฿0.029168\"}}"; + private static final String ARREARS_COPY = + LlmBalanceGuard.REASON_LLM_BALANCE_DUE + ";lastErr=" + QUOTA_403_BODY; + + private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class); + private final TaskProgressSnapshotService taskProgressSnapshotService = mock(TaskProgressSnapshotService.class); + private final BrandTaskHeartbeatSpi brandTaskHeartbeatSpi = mock(BrandTaskHeartbeatSpi.class); + private final TaskHeartbeatLivenessService livenessService = mock(TaskHeartbeatLivenessService.class); + + @BeforeAll + static void initializeMybatisMetadata() { + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new MybatisConfiguration(), ""), + FileTaskEntity.class); + } + + private TaskHeartbeatService service() { + return new TaskHeartbeatService(fileTaskMapper, taskProgressSnapshotService, brandTaskHeartbeatSpi, + livenessService, List.of(new AppearancePatentTaskHeartbeatSpi( + mock(AppearancePatentTaskCacheService.class), + mock(TaskFileJobService.class), + new AppearancePatentProperties()))); + } + + @Test + @SuppressWarnings("unchecked") + void arrearsFailedTaskHeartbeatSaysNotAlive() { + FileTaskEntity task = apTask("FAILED"); + task.setErrorMessage(ARREARS_COPY); + when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task); + + TaskHeartbeatVo result = service().heartbeat(TASK_ID, new TaskHeartbeatRequest()); + + assertFalse(result.isAlive(), "任务已终态还回 alive=true,采集端就会继续烧出口 IP"); + assertEquals("FAILED", result.getStatus()); + assertEquals("APPEARANCE_PATENT", result.getModuleType()); + // 非在跑任务不许再刷 updated_at(刷了就等于把它重新写成"还活着") + verify(fileTaskMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class)); + } + + /** 对照组:同一枚任务还是 RUNNING 时心跳照旧 alive=true,证明上一条的 false 是真判出来的。 */ + @Test + @SuppressWarnings("unchecked") + void runningTaskHeartbeatStillSaysAlive() { + FileTaskEntity task = apTask("RUNNING"); + when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task); + when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + + TaskHeartbeatVo result = service().heartbeat(TASK_ID, new TaskHeartbeatRequest()); + + assertTrue(result.isAlive()); + assertEquals("RUNNING", result.getStatus()); + } + + private FileTaskEntity apTask(String status) { + FileTaskEntity task = new FileTaskEntity(); + task.setId(TASK_ID); + task.setModuleType("APPEARANCE_PATENT"); + task.setUserId(7L); + task.setStatus(status); + return task; + } +}