fix(appearance-patent): 上游余额欠费即判任务终态,停止为空转的行烧出口

B46 那一笔(f3445c21)只做到「行级一次就停、归因写对」:撞上上游 403
insufficient_user_quota 后不再跑满 3 次重试,任务本身仍停在 RUNNING/WAIT_LLM
等 Python 继续回传分片。采集端(maixiang ap_worker)只看 /task/heartbeat 的
alive,读不到终态就继续为剩下的行按次付费买出口 IP——而那些行永远进不了 LLM,
钱白花。这一笔补上 Java 侧缺的那半步:欠费即把任务落终态 FAILED,心跳随即
回 alive=false,worker 下一跳收手。

- 判据只认 TaskRetryEligibility 的 BALANCE 类(欠费 + 密钥失效),复用现成分类器
  而非再抄一份字面量表;连接不可达 / 5xx / 429 / 抓取缺口一律照旧继续或按已有
  结果交付,判宽了就是拿用户整单给一次网络抖动陪葬。
- 停手点放在批管线的作业执行线程(LLM 批次就在这里跑),不在心跳线程;识别到
  欠费即停止后续批次,并回调宿主落终态(FileTaskMapper 与收尾口径在宿主手里)。
- 写终态前重读库里现状态:已 SUCCESS/COMPLETED/CANCELLED/FAILED 的一律不动,
  因此可重入,多入口命中也只写一次。
- 终态文案 = 生产者原文(客户端欠费归因 + 上游 lastErr)+ 可执行动作,原文一个
  字都不重写;不出现「请稍后重试」,不把欠费说成临时抖动。
This commit is contained in:
2026-10-27 15:53:10 +08:00
parent 07f973d9df
commit e08b5369d7
7 changed files with 605 additions and 5 deletions
@@ -367,7 +367,18 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
properties, llmClient, taskCacheService, taskFileJobService,
rowReader()::rowKey, this::shouldInspectRow, rowReader()::readChunkRows, this::loadSubmittedChunks,
this::isResultSubmissionComplete, this::touchJavaSideTaskActivity,
llmSubmission::readAiPrompt, llmSubmission::readApiKey, this::mergeChunkPayload, this::firstNonBlank);
llmSubmission::readAiPrompt, llmSubmission::readApiKey, this::mergeChunkPayload, this::firstNonBlank,
this::terminalizeTaskOnFatalLlmFailure);
}
/**
* 上游 LLM 账户欠费/密钥失效的任务级处置(2026-10-27):批管线识别到就停手,这里把任务落
* 终态 FAILED——只有变成终态,{@code TaskHeartbeatService} 才会对该任务回 {@code alive=false},
* Python 采集端才不再为空转的行买出口 IP。终态写法与护栏(非终态才写、文案来源)都收在
* {@link AppearancePatentResultJobSupport#terminalizeOnFatalLlmFailure}。
*/
private void terminalizeTaskOnFatalLlmFailure(FileTaskEntity task, String failureMessage) {
resultJobSupport().terminalizeOnFatalLlmFailure(task, failureMessage);
}
/**
@@ -57,6 +57,18 @@ public class AppearancePatentLlmBatchPipeline {
List<AppearancePatentResultRowDto> rows);
}
/**
* 上游欠费/密钥失效的<b>任务级处置</b>出口(2026-10-27)。批管线只负责「识别 + 停手」,
* 把任务写成终态要 {@code FileTaskMapper} 与收尾口径,那些都在宿主
* {@code AppearancePatentResultJobSupport} 手里,故以回调交出(与本类其余回调同一形状)。
*
* <p>调用点是作业执行线程(LLM 批次就在这里跑),不是心跳线程——铁律 8 不许在心跳里判死。
*/
@FunctionalInterface
public interface FatalLlmFailureHandler {
void onFatalLlmFailure(FileTaskEntity task, String failureMessage);
}
private final AppearancePatentProperties properties;
private final AppearancePatentLlmClient llmClient;
private final AppearancePatentTaskCacheService taskCacheService;
@@ -71,6 +83,7 @@ public class AppearancePatentLlmBatchPipeline {
private final Function<FileTaskEntity, String> apiKeyReader;
private final ChunkRowMerger chunkRowMerger;
private final BiFunction<String, String, String> firstNonBlankFn;
private final FatalLlmFailureHandler fatalLlmFailureHandler;
public AppearancePatentLlmBatchPipeline(AppearancePatentProperties properties,
AppearancePatentLlmClient llmClient,
@@ -85,7 +98,8 @@ public class AppearancePatentLlmBatchPipeline {
Function<FileTaskEntity, String> promptReader,
Function<FileTaskEntity, String> apiKeyReader,
ChunkRowMerger chunkRowMerger,
BiFunction<String, String, String> firstNonBlankFn) {
BiFunction<String, String, String> firstNonBlankFn,
FatalLlmFailureHandler fatalLlmFailureHandler) {
this.properties = properties;
this.llmClient = llmClient;
this.taskCacheService = taskCacheService;
@@ -100,6 +114,7 @@ public class AppearancePatentLlmBatchPipeline {
this.apiKeyReader = apiKeyReader;
this.chunkRowMerger = chunkRowMerger;
this.firstNonBlankFn = firstNonBlankFn;
this.fatalLlmFailureHandler = fatalLlmFailureHandler;
}
/**
@@ -150,6 +165,7 @@ public class AppearancePatentLlmBatchPipeline {
}
int batchTotal = Math.max(1, (submitLimit + batchSize - 1) / batchSize);
int batchIndex = 1;
String fatalLlmMessage = null;
for (int i = 0; i < submitLimit; i += batchSize) {
List<AppearancePatentResultRowDto> batchRows = candidates.subList(i, Math.min(i + batchSize, submitLimit));
taskFileJobService.touchRunning(job.getId());
@@ -164,8 +180,12 @@ public class AppearancePatentLlmBatchPipeline {
() -> llmClient.inspectRows(batchRows, prompt, apiKey));
} catch (Exception ex) {
String message = firstNonBlank(ex.getMessage(), "LLM 检测失败");
log.warn("[appearance-patent] llm batch failed taskId={} jobId={} rows={} batch={}/{} err={}",
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal, message);
if (AppearancePatentResultJobSupport.isFatalLlmArrearsFailure(message)) {
fatalLlmMessage = message;
}
log.warn("[appearance-patent] llm batch failed taskId={} jobId={} rows={} batch={}/{} fatalLlmArrears={} err={}",
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal,
fatalLlmMessage != null, message);
llmRows = llmClient.markRowsFailed(batchRows, message);
batchAnswered = false;
}
@@ -192,6 +212,19 @@ public class AppearancePatentLlmBatchPipeline {
}
}
}
if (fatalLlmMessage != null) {
// 上游账户欠费/密钥失效:剩下每一批都拿不到结论,而 Python 采集端只看
// /task/heartbeat 的 alive——任务不落终态,它就继续为那些永远进不了 LLM 的行
// 按次付费买出口 IP(2026-10-27 B46 现网形状)。所以这里既停批(不再发下一批),
// 又把「落终态」交给宿主:FileTaskMapper 与收尾口径在它手里,且这一步发生在
// 作业执行线程,不是心跳线程(铁律 8)。
log.warn("[appearance-patent] LLM 上游欠费/密钥失效,停止后续批次并把任务落终态"
+ " taskId={} jobId={} batch={}/{} pendingRows={} err={}",
task.getId(), job.getId(), batchIndex, batchTotal,
Math.max(0, submitLimit - i), fatalLlmMessage);
fatalLlmFailureHandler.onFatalLlmFailure(task, fatalLlmMessage);
return;
}
log.info("[appearance-patent] llm batch done taskId={} jobId={} rows={} batch={}/{} costMs={}",
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal,
System.currentTimeMillis() - startedAt);
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.retry.LlmBalanceGuard;
import com.nanri.aiimage.common.task.TaskStatuses;
import com.nanri.aiimage.common.task.GroupResultPropagator;
import com.nanri.aiimage.config.AppearancePatentProperties;
@@ -24,6 +25,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskFinalizeSupport;
import com.nanri.aiimage.modules.task.service.TaskRetryEligibility;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
@@ -249,6 +251,69 @@ public class AppearancePatentResultJobSupport {
taskCacheService.deleteTaskCache(task.getId());
}
/**
* 只认「上游账户欠费 / LLM 密钥失效」这一类不可自愈的失败。分类口径复用
* {@link TaskRetryEligibility}(它已经是"不可抗力不克隆重跑"的唯一判据),不再抄第二份字面量表。
*
* <p>刻意只取 {@code BALANCE}:连接不可达({@code SERVICE_DOWN})、429 限流、5xx 抖动、
* 抓取缺口都属「外部会自己恢复或补传能救」,判宽了就是拿用户的任务给一次网络抖动陪葬。
*/
static boolean isFatalLlmArrearsFailure(String errorMessage) {
return TaskRetryEligibility.forceMajeureCategory(errorMessage) == TaskRetryEligibility.Category.BALANCE;
}
/**
* 上游余额欠费/密钥失效 → 任务<b>立即</b>落终态 FAILED。
*
* <p>为什么必须由 Java 侧落终态(2026-10-27 现网):{@code f3445c21} 只做到「行级不再重试三次、
* 归因写对」,任务本身仍停在 RUNNING/WAIT_LLM 等 Python 继续回传分片。采集端
* ({@code maixiang ap_worker})只看 {@code /task/heartbeat} 的 {@code alive},看不到终态就会
* 继续为剩下的行<b>按次付费买出口 IP</b>——而那些行永远进不了 LLM,钱白花。
* 任务一终态,{@code TaskHeartbeatService} 对其返回 {@code alive=false}(既有行为),worker 下一跳收手。
*
* <p>判定发生在作业执行线程(LLM 批次就在这里跑),不在心跳线程(铁律 8)。
* 写之前重读一次库里的现状态:已 SUCCESS/COMPLETED/CANCELLED/FAILED 的一律不动(铁律 14),
* 本方法因此可重入,停放期间同一任务被多个入口命中欠费也只写一次。
*/
public void terminalizeOnFatalLlmFailure(FileTaskEntity task, String failureMessage) {
if (task == null || task.getId() == null) {
return;
}
if (!isFatalLlmArrearsFailure(failureMessage)) {
// 兜底闸:调用方判宽了也不许把可恢复的失败写成终态
return;
}
FileTaskEntity current = fileTaskMapper.selectById(task.getId());
String currentStatus = current != null ? current.getStatus() : task.getStatus();
if (TaskStatuses.FileTask.TERMINAL.contains(currentStatus)) {
log.info("[appearance-patent] 任务已是终态,欠费归因不改写状态 taskId={} status={}",
task.getId(), currentStatus);
return;
}
String terminalMessage = fatalLlmTerminalMessage(failureMessage);
task.setStatus(STATUS_FAILED);
task.setSuccessFileCount(0);
task.setFailedFileCount(1);
task.setErrorMessage(terminalMessage);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
taskCacheService.deleteTaskCache(task.getId());
log.warn("[appearance-patent] LLM 上游欠费/密钥失效,任务落终态 FAILED 以停止为空转的行继续买出口 IP"
+ " taskId={} err={}", task.getId(), terminalMessage);
}
/**
* 终态文案 = 生产者原文(客户端的欠费归因 + 上游 lastErr)+ 可执行动作,
* 原文一个字都不在这里重写;文案里没有「稍后重试」,欠费不会被说成抖动。
*/
static String fatalLlmTerminalMessage(String failureMessage) {
String raw = failureMessage == null ? "" : failureMessage.trim();
return raw.isEmpty()
? LlmBalanceGuard.TASK_TERMINAL_GUIDANCE
: raw + ";" + LlmBalanceGuard.TASK_TERMINAL_GUIDANCE;
}
public FileResultEntity findOrCreateResultRecordForAssembly(FileTaskEntity task, int rowCount) {
if (task == null || task.getId() == null) {
return null;