refactor(phase-3): 3A 拆薄第五笔——外观专利 LLM 批管线出件,服务 3135→2801

B 聚类(候选收集、批调度、轮次判死、结果行合并、批计数)原样搬进
service/support/AppearancePatentLlmBatchPipeline;读分片、分片合并、
心跳、封口判定仍留在服务侧,以回调交入。四枚同名入口保留为测试
按名反射的接缝;SourceScanTest 面数 31→32、support 8→9 并逐枚点名新件。
This commit is contained in:
2026-10-20 14:03:18 +08:00
parent 3ac859e05f
commit 4bcce9d534
4 changed files with 537 additions and 371 deletions
@@ -24,6 +24,7 @@ import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubm
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashboardVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo;
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentHistoryAssembler;
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentLlmBatchPipeline;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
@@ -88,7 +89,6 @@ import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.pipeline.AbstractFileTaskPipeline;
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
import com.nanri.aiimage.modules.usersecret.support.SecretUsageContext;
import com.nanri.aiimage.modules.usersecret.support.UserSecretModule;
@Service
@@ -783,174 +783,32 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
}
/**
* 同步检测:按批次串行调用 LLM(批内行级并发由 client 负责),
* 结果直接合并回 chunk。批间去重用 Redis 行级标记防止 submitResult 与 stale 恢复双入口重复检测。
* B 聚类(LLM 批管线)住在 {@link AppearancePatentLlmBatchPipeline}(3A 拆薄第五笔,
* 候选收集、批调度、轮次判死、结果行合并与批计数逐字搬移)。以下四枚同名入口是
* 测试按名反射点名的接缝({@code AppearancePatentRowMarkTimingTest}/
* {@code AppearancePatentChunkRowIndexTest}),不是实现本体。
*/
private void submitLlmBatches(FileTaskEntity task,
FileResultEntity result,
TaskFileJobEntity job,
List<TaskChunkEntity> chunks) {
if (chunks == null || chunks.isEmpty()) {
return;
}
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
int batchSize = Math.max(1, properties.getLlmBatchSize());
ChunkRowKeyIndex rowKeyIndex = new ChunkRowKeyIndex();
List<AppearancePatentResultRowDto> candidates = terminalizeRoundExhaustedRows(
task, collectPendingLlmCandidates(chunks, rowKeyIndex), rowKeyIndex);
boolean flushRemainder = isResultSubmissionComplete(task.getId());
if (!flushRemainder && !candidates.isEmpty()) {
LocalDateTime jobUpdatedAt = job.getUpdatedAt();
long pendingFlushMillis = flushPendingMillis();
if (jobUpdatedAt != null
&& Duration.between(jobUpdatedAt, LocalDateTime.now()).toMillis() >= pendingFlushMillis) {
flushRemainder = true;
log.warn("[appearance-patent] 零头批次等待超时,强制提交 taskId={} jobId={} pendingRows={} batchSize={} jobUpdatedAt={} flushAfterMillis={}",
task.getId(), job.getId(), candidates.size(), batchSize, jobUpdatedAt, pendingFlushMillis);
}
}
int submitLimit = (candidates.size() / batchSize) * batchSize;
if (flushRemainder && submitLimit < candidates.size()) {
submitLimit = candidates.size();
}
if (submitLimit <= 0) {
log.info("[appearance-patent] llm batch waiting for more rows taskId={} jobId={} pendingRows={} batchSize={} finalUpload={}",
task.getId(), job.getId(), candidates.size(), batchSize, flushRemainder);
return;
}
if (TransactionSynchronizationManager.isActualTransactionActive()) {
// 守住"整批 LLM 跑在事务里"这一类回归:下面循环每批开头的 touchRunning 是
// UPDATE biz_task_file_job ... WHERE id=? AND status='RUNNING',本方法一旦处在事务中,
// 这把行锁就要持到全部批次结束才释放(线上实测一批 18 分钟),另一台的卡死自愈扫描
// 于是每 110 秒撞一次 50 秒锁等待、并且整轮作废。历史上真发生过,见
// docs/ops/prod-decisions-2026-09-28.md §9。
log.error("[appearance-patent] llm 批次正跑在事务里,行锁会持到批次结束 taskId={} jobId={} pendingRows={}",
task.getId(), job.getId(), submitLimit);
}
int batchTotal = Math.max(1, (submitLimit + batchSize - 1) / batchSize);
int batchIndex = 1;
for (int i = 0; i < submitLimit; i += batchSize) {
List<AppearancePatentResultRowDto> batchRows = candidates.subList(i, Math.min(i + batchSize, submitLimit));
taskFileJobService.touchRunning(job.getId());
List<String> batchRowKeys = batchRows.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
taskCacheService.touchRowInspection(task.getId(), batchRowKeys);
taskCacheService.countRowInspectionRound(task.getId(), batchRowKeys);
long startedAt = System.currentTimeMillis();
List<AppearancePatentResultRowDto> llmRows;
boolean batchAnswered = true;
try {
llmRows = SecretUsageContext.call(task.getUserId(), UserSecretModule.APPEARANCE_PATENT.key(),
() -> llmClient.inspectRows(batchRows, prompt, apiKey));
} catch (Exception ex) {
String message = firstNonBlank(ex.getMessage(), "LLM 检测失败");
log.warn("[appearance-patent] llm batch failed taskId={} jobId={} rows={} batch={}/{} err={}",
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal, message);
llmRows = llmClient.markRowsFailed(batchRows, message);
batchAnswered = false;
}
List<String> keysToRetry = new ArrayList<>(batchRowKeys);
try {
Set<String> landedKeys = mergeLlmRowsIntoChunks(task, llmRows, batchRows, rowKeyIndex);
// 完成标记只能打在真正落库的行上——这是「发版重启会不会丢行」的分界线
if (batchAnswered) {
List<String> landed = batchRowKeys.stream().filter(landedKeys::contains).toList();
if (!landed.isEmpty()) {
taskCacheService.completeRowInspection(task.getId(), landed);
}
keysToRetry.removeAll(landed);
}
} finally {
// 本批结束时处理中名额必须让出来:没落库的行撤销后下一轮马上能重新申领,不等 TTL
if (!keysToRetry.isEmpty()) {
taskCacheService.releaseRowInspection(task.getId(), keysToRetry);
if (batchAnswered) {
// 结果拿到了却落不进分片(少回、rowToken 被改写、分片已清):
// 打成已完成就是"扣了次数永久没结论",宁可重发一轮。
log.warn("[appearance-patent] 本批结果未全部落库,撤销已检测标记交由下一轮重发 taskId={} jobId={} batch={}/{} sent={} retrying={}",
task.getId(), job.getId(), batchIndex, batchTotal, batchRowKeys.size(), keysToRetry);
}
}
}
log.info("[appearance-patent] llm batch done taskId={} jobId={} rows={} batch={}/{} costMs={}",
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal,
System.currentTimeMillis() - startedAt);
// 每批刷新任务心跳:软恢复后的检测阶段可达数小时,不刷会被通用兜底线按 updated_at 超 120 分钟误杀(2026-09-28 29473/29479)
touchJavaSideTaskActivity(task.getId());
batchIndex++;
}
llmBatchPipeline().submitLlmBatches(task, result, job, chunks);
}
/**
* 轮次已到顶的行就地判失败并打成「已完成」,从此退出候选。
*
* <p>失败行会被撤销处理中标记、下一轮重新申领——这是「不丢行」的另一半代价。没有这道闸的话,
* 上游劣化窗口内同一批行会一轮轮重发,每轮都过 {@link SecretUsageContext} 扣用户的 LLM 次数,
* 而 assemble job 走的是 deferRunning(不累加 retryCount),MAX_RETRY_COUNT 那条永不触发。
*/
private List<AppearancePatentResultRowDto> terminalizeRoundExhaustedRows(
FileTaskEntity task,
List<AppearancePatentResultRowDto> candidates,
ChunkRowKeyIndex rowKeyIndex) {
int maxRounds = properties.getLlmRowMaxRounds();
if (maxRounds <= 0 || candidates == null || candidates.isEmpty()) {
return candidates == null ? List.of() : candidates;
}
List<AppearancePatentResultRowDto> retryable = new ArrayList<>(candidates.size());
List<AppearancePatentResultRowDto> exhausted = new ArrayList<>();
for (AppearancePatentResultRowDto row : candidates) {
String key = rowKey(row);
if (!key.isBlank() && taskCacheService.rowInspectionRounds(task.getId(), key) >= maxRounds) {
exhausted.add(row);
} else {
retryable.add(row);
}
}
if (exhausted.isEmpty()) {
return retryable;
}
String reason = "重发轮次已达上限 " + maxRounds + " 轮仍未取到结论,请稍后重新发起检测";
log.warn("[appearance-patent] llm round cap reached taskId={} rows={} maxRounds={}",
task.getId(), exhausted.size(), maxRounds);
List<String> exhaustedKeys = exhausted.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
boolean rowsTerminated = false;
try {
mergeLlmRowsIntoChunks(task, llmClient.markRowsFailed(exhausted, reason),
exhausted, rowKeyIndex);
rowsTerminated = true;
} finally {
// 失败原因没落库就不能打成「已完成」,否则这批行既没结论也再不会重试
if (rowsTerminated) {
taskCacheService.completeRowInspection(task.getId(), exhaustedKeys);
} else {
taskCacheService.releaseRowInspection(task.getId(), exhaustedKeys);
}
}
return retryable;
private List<AppearancePatentResultRowDto> collectPendingLlmCandidates(
List<TaskChunkEntity> chunks, AppearancePatentLlmBatchPipeline.ChunkRowKeyIndex rowKeyIndex) {
return llmBatchPipeline().collectPendingLlmCandidates(chunks, rowKeyIndex);
}
private int countUninspectedRows(List<TaskChunkEntity> chunks) {
return llmBatchPipeline().countUninspectedRows(chunks);
}
private Set<String> mergeLlmRowsIntoChunks(FileTaskEntity task,
List<AppearancePatentResultRowDto> llmRows,
List<AppearancePatentResultRowDto> batchRows,
ChunkRowKeyIndex rowKeyIndex) {
if (llmRows == null || llmRows.isEmpty()) {
return Set.of();
}
Map<String, AppearancePatentResultRowDto> deduped = new LinkedHashMap<>();
for (int i = 0; i < llmRows.size(); i++) {
AppearancePatentResultRowDto row = llmRows.get(i);
AppearancePatentResultRowDto sourceRow = i < batchRows.size() ? batchRows.get(i) : null;
String key = rowKey(row);
if (key.isBlank()) {
if (sourceRow != null) {
key = rowKey(sourceRow);
}
}
if (key.isBlank() || !deduped.containsKey(key)) {
deduped.put(key, row);
}
}
return mergeLlmRowsIntoSubmittedChunks(task, new ArrayList<>(deduped.values()), null, null, rowKeyIndex);
AppearancePatentLlmBatchPipeline.ChunkRowKeyIndex rowKeyIndex) {
return llmBatchPipeline().mergeLlmRowsIntoChunks(task, llmRows, batchRows, rowKeyIndex);
}
/**
@@ -968,108 +826,12 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
return chunks == null ? List.of() : chunks;
}
/**
* 收集未检测的候选行:**逐行**申领——同一父号下的 {@code 2_1/2_2/2_3} 每行都各自进候选、各自送检。
*
* <p>历史上这里只挑「组代表行」,其余兄弟行指望 {@code expandRows} 在发起那一轮借用代表行的结论——
* 那是把「同一个父号」当成「同一件东西」,而子序行各有各的 sku、标题和图片链接。
* 口径(2026-10-10 用户定):这些行**都要检测**。折叠与借用一起撤掉,因为借用的后果不是省钱,
* 而是成片空行随结果文件交付(10-08 任务 30426:1211 行里只有 960 行有结论、251 行整段空白,
* 而任务报成功;10-09 任务 30455 的 281 行空行 100% 属于"同父号已有一行被判"这一形状)。
*
* <p>"这行算不算已经检过"用交付口径(标题维+外观维+结论三列齐,见
* {@link AppearancePatentRowOutcome#isJudged})。旧代码用的是"四个字段里任一可用",于是
* 只带着专利维度文本、或只带一个 error 的行会被判"已检测",从此退出候选,而表格里它那三列全是空的。
*/
private List<AppearancePatentResultRowDto> collectPendingLlmCandidates(List<TaskChunkEntity> chunks,
ChunkRowKeyIndex rowKeyIndex) {
if (chunks == null || chunks.isEmpty()) {
return List.of();
}
List<AppearancePatentResultRowDto> candidates = new ArrayList<>();
int claimBlocked = 0;
int judgedSkipped = 0;
int noDataSkipped = 0;
int chunkRows = 0;
for (TaskChunkEntity chunk : chunks) {
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
if (persistedRows.isEmpty()) {
continue;
}
// 这一遍已经把这批分片的载荷读进来了,顺手把「这片里有哪些行」记进索引:
// 下面每批落库前都要这份索引,不记的话就得每批把全部分片重读重解析一遍
rememberChunkRowKeys(rowKeyIndex, chunk, persistedRows.keySet());
chunkRows += persistedRows.size();
for (AppearancePatentResultRowDto row : persistedRows.values()) {
String key = rowKey(row);
if (key.isBlank()) {
continue;
}
if (AppearancePatentRowOutcome.isJudged(row)) {
judgedSkipped++;
continue;
}
if (!shouldInspectRow(row)) {
// 连标题带图都没采到的行:送过去也只能回"外观识别异常",白扣一次
noDataSkipped++;
continue;
}
if (!taskCacheService.beginRowInspection(chunk.getTaskId(), key)) {
// 被"处理中/已完成"标记挡住的行必须数得出来:历史上这里静默 continue,
// 于是"扣了次数却永久没结论"和"这行本来就不该检"在日志里长得一模一样
claimBlocked++;
continue;
}
candidates.add(row);
}
}
log.info("[appearance-patent] 候选收集逐行申领 taskId={} chunkRows={} judgedSkipped={} noDataSkipped={} claimBlocked={} candidates={}",
chunks.get(0).getTaskId(), chunkRows, judgedSkipped, noDataSkipped, claimBlocked, candidates.size());
return candidates;
}
/**
* 只计数、不申领。用 {@link #collectPendingLlmCandidates} 计数会把行标成处理中,
* 紧接着真正发起检测的那一步就一行也申领不到,只能等处理中标记的 TTL 过期。
*/
private int countUninspectedRows(List<TaskChunkEntity> chunks) {
if (chunks == null || chunks.isEmpty()) {
return 0;
}
int total = 0;
for (TaskChunkEntity chunk : chunks) {
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
for (AppearancePatentResultRowDto row : persistedRows.values()) {
if (!rowKey(row).isBlank() && !AppearancePatentRowOutcome.isJudged(row)) {
total++;
}
}
}
return total;
}
private int countLlmWorkUnits(List<TaskChunkEntity> chunks, int batchSize) {
if (chunks == null || chunks.isEmpty()) {
return 0;
}
int total = 0;
for (TaskChunkEntity chunk : chunks) {
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
if (persistedRows.isEmpty()) {
continue;
}
int unresolved = (int) persistedRows.values().stream()
.filter(row -> !rowKey(row).isBlank() && !AppearancePatentRowOutcome.isJudged(row)).count();
if (unresolved > 0) {
total += Math.max(1, (unresolved + batchSize - 1) / batchSize);
}
}
return total;
}
private long flushPendingMillis() {
return Math.max(1, properties.getFlushPendingMinutes()) * 60_000L;
private AppearancePatentLlmBatchPipeline llmBatchPipeline() {
return new AppearancePatentLlmBatchPipeline(
properties, llmClient, taskCacheService, taskFileJobService,
this::rowKey, this::shouldInspectRow, this::readChunkRows, this::loadSubmittedChunks,
this::isResultSubmissionComplete, this::touchJavaSideTaskActivity,
this::readAiPrompt, this::readApiKey, this::mergeChunkPayload, this::firstNonBlank);
}
private void submitLlmForSubmittedChunk(SubmitContext context) {
@@ -1680,7 +1442,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
.eq(TaskChunkEntity::getTaskId, task.getId())
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskChunkEntity::getChunkIndex));
int llmWorkUnits = countLlmWorkUnits(chunks, Math.max(1, properties.getLlmBatchSize()));
int llmWorkUnits = llmBatchPipeline().countLlmWorkUnits(chunks, Math.max(1, properties.getLlmBatchSize()));
int plannedLlmUnits = Math.max(1, llmWorkUnits);
int totalProgressUnits = Math.max(3, plannedLlmUnits + 3);
saveFileBuildProgress(task, job, totalProgressUnits, 0, "正在提交检测");
@@ -1887,102 +1649,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
}
}
/** 返回真正落库的行键;没匹配到分片行的键不会出现在里面,调用方据此决定要不要撤销已检测标记。 */
private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
List<AppearancePatentResultRowDto> llmRows,
String fallbackScopeHash,
Integer fallbackChunkIndex,
ChunkRowKeyIndex rowKeyIndex) {
if (task == null || llmRows == null || llmRows.isEmpty()) {
return Set.of();
}
List<TaskChunkEntity> chunks = loadSubmittedChunks(task.getId());
if (chunks.isEmpty()) {
return Set.of();
}
Map<String, Set<String>> rowKeysByChunk = new LinkedHashMap<>();
Map<String, TaskChunkEntity> chunkByKey = new LinkedHashMap<>();
for (TaskChunkEntity chunk : chunks) {
String chunkKey = chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex());
rowKeysByChunk.put(chunkKey, chunkRowKeySet(rowKeyIndex, chunk));
chunkByKey.put(chunkKey, chunk);
}
Map<String, Map<String, AppearancePatentResultRowDto>> mergeRowsByChunk = new LinkedHashMap<>();
for (AppearancePatentResultRowDto resultRow : llmRows) {
for (AppearancePatentResultRowDto expandedRow : List.of(resultRow)) {
String rowKey = rowKey(expandedRow);
if (rowKey.isBlank()) {
continue;
}
boolean matched = false;
for (Map.Entry<String, Set<String>> entry : rowKeysByChunk.entrySet()) {
if (entry.getValue().contains(rowKey)) {
mergeRowsByChunk.computeIfAbsent(entry.getKey(), ignored -> new LinkedHashMap<>())
.put(rowKey, expandedRow);
matched = true;
}
}
if (!matched && fallbackScopeHash != null && fallbackChunkIndex != null) {
String fallbackKey = chunkStorageKey(fallbackScopeHash, fallbackChunkIndex);
if (chunkByKey.containsKey(fallbackKey)) {
mergeRowsByChunk.computeIfAbsent(fallbackKey, ignored -> new LinkedHashMap<>())
.put(rowKey, expandedRow);
matched = true;
}
}
if (!matched) {
log.warn("[appearance-patent] llm row has no submitted chunk taskId={} rowKey={}",
task.getId(), rowKey);
}
}
}
Set<String> landedKeys = new LinkedHashSet<>();
for (Map.Entry<String, Map<String, AppearancePatentResultRowDto>> entry : mergeRowsByChunk.entrySet()) {
TaskChunkEntity chunk = chunkByKey.get(entry.getKey());
if (chunk == null || entry.getValue().isEmpty()) {
continue;
}
landedKeys.addAll(mergeChunkPayload(task.getId(), chunk.getScopeHash(), chunk.getChunkIndex(),
new ArrayList<>(entry.getValue().values())));
}
return landedKeys;
}
private String chunkStorageKey(String scopeHash, Integer chunkIndex) {
return firstNonBlank(scopeHash, "") + ":" + (chunkIndex == null ? 0 : chunkIndex);
}
/**
* 一次 {@link #submitLlmBatches} 内共享的「哪个分片里有哪些行」索引,按 payload_hash 判新鲜。
*
* <p>只存 rowKey 不存整行,索引本身不会把载荷钉在内存里。作用域必须是单次调用(局部变量,随栈丢弃):
* 本服务是单例、多个任务同时在跑批,做成字段就是多线程共享一个裸 LinkedHashMap,而且分片键永不过期、只增不减。
* 数据正确性反过来不依赖局部性——收集候选那一遍每次都会重读并重记全部分片。
*/
static final class ChunkRowKeyIndex {
private final Map<String, Snapshot> snapshots = new LinkedHashMap<>();
}
/** 把刚解析出来的分片行键记进索引(自己拷一份,不握住整行载荷),返回索引里那份。 */
private Set<String> rememberChunkRowKeys(ChunkRowKeyIndex rowKeyIndex, TaskChunkEntity chunk, Set<String> rowKeys) {
Set<String> copied = new LinkedHashSet<>(rowKeys);
rowKeyIndex.snapshots.put(chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()),
new Snapshot(chunk.getPayloadHash(), copied));
return copied;
}
/** 分片当前载荷里的全部行键;载荷哈希与索引里那次解析一致时直接复用,不重读对象存储。 */
private Set<String> chunkRowKeySet(ChunkRowKeyIndex rowKeyIndex, TaskChunkEntity chunk) {
Snapshot cached = rowKeyIndex.snapshots.get(chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()));
if (cached != null && Objects.equals(cached.payloadHash, chunk.getPayloadHash())) {
return cached.rowKeys;
}
return rememberChunkRowKeys(rowKeyIndex, chunk, readChunkRows(chunk).keySet());
}
private record Snapshot(String payloadHash, Set<String> rowKeys) {
}
private boolean isResultSubmissionComplete(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
@@ -0,0 +1,498 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.usersecret.support.SecretUsageContext;
import com.nanri.aiimage.modules.usersecret.support.UserSecretModule;
import lombok.extern.slf4j.Slf4j;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Predicate;
/**
* 外观专利的 LLM 批管线(B 聚类),从 {@code AppearancePatentTaskService} <b>原样搬移</b>
* (3A 拆薄第五笔):候选收集、批调度、轮次到顶判死、结果行合并回分片、批工作单元计数,
* 连同它们的日志文案与判序一个字没改。
*
* <p><b>搬的是批管线,不是它的宿主能力。</b>读分片载荷({@code readChunkRows})、分片合并
* ({@code mergeChunkPayload})、任务心跳({@code touchJavaSideTaskActivity})、提交是否封口
* ({@code isResultSubmissionComplete})都还留在服务侧,以回调交进来——它们分属 D/G/H 聚类,
* 跟着 B 一起拖出来就得先给它们再造一套构造器,而 3A 的账已经证明
* 「结构性成本 &gt; 可沉件收益」是要避免的事。回调是这一笔里唯一的新形状。
*
* <p><b>作用域必须是单次调用。</b>本类是单例服务里的临时协作者(每次入口新建),
* {@link ChunkRowKeyIndex} 随栈丢弃;做成服务字段就是多任务共享一个裸 LinkedHashMap,
* 而且分片键永不过期、只增不减(原注释里那条约束一并带过来)。
*
* <p>搬移后唯一的可观测差异是 logger 名:从
* {@code ...service.AppearancePatentTaskService} 变为
* {@code ...service.support.AppearancePatentLlmBatchPipeline},消息文本、级别与参数顺序未动。
*/
@Slf4j
public class AppearancePatentLlmBatchPipeline {
/** 分片载荷合并(D 聚类)留在服务侧;返回真正写进这一片载荷的行键。 */
@FunctionalInterface
public interface ChunkRowMerger {
List<String> merge(Long taskId, String scopeHash, Integer chunkIndex,
List<AppearancePatentResultRowDto> rows);
}
private final AppearancePatentProperties properties;
private final AppearancePatentLlmClient llmClient;
private final AppearancePatentTaskCacheService taskCacheService;
private final TaskFileJobService taskFileJobService;
private final Function<AppearancePatentResultRowDto, String> rowKeyer;
private final Predicate<AppearancePatentResultRowDto> shouldInspect;
private final Function<TaskChunkEntity, Map<String, AppearancePatentResultRowDto>> chunkRowReader;
private final Function<Long, List<TaskChunkEntity>> submittedChunksLoader;
private final Predicate<Long> submissionComplete;
private final Consumer<Long> taskActivityToucher;
private final Function<FileTaskEntity, String> promptReader;
private final Function<FileTaskEntity, String> apiKeyReader;
private final ChunkRowMerger chunkRowMerger;
private final BiFunction<String, String, String> firstNonBlankFn;
public AppearancePatentLlmBatchPipeline(AppearancePatentProperties properties,
AppearancePatentLlmClient llmClient,
AppearancePatentTaskCacheService taskCacheService,
TaskFileJobService taskFileJobService,
Function<AppearancePatentResultRowDto, String> rowKeyer,
Predicate<AppearancePatentResultRowDto> shouldInspect,
Function<TaskChunkEntity, Map<String, AppearancePatentResultRowDto>> chunkRowReader,
Function<Long, List<TaskChunkEntity>> submittedChunksLoader,
Predicate<Long> submissionComplete,
Consumer<Long> taskActivityToucher,
Function<FileTaskEntity, String> promptReader,
Function<FileTaskEntity, String> apiKeyReader,
ChunkRowMerger chunkRowMerger,
BiFunction<String, String, String> firstNonBlankFn) {
this.properties = properties;
this.llmClient = llmClient;
this.taskCacheService = taskCacheService;
this.taskFileJobService = taskFileJobService;
this.rowKeyer = rowKeyer;
this.shouldInspect = shouldInspect;
this.chunkRowReader = chunkRowReader;
this.submittedChunksLoader = submittedChunksLoader;
this.submissionComplete = submissionComplete;
this.taskActivityToucher = taskActivityToucher;
this.promptReader = promptReader;
this.apiKeyReader = apiKeyReader;
this.chunkRowMerger = chunkRowMerger;
this.firstNonBlankFn = firstNonBlankFn;
}
/**
* 同步检测:按批次串行调用 LLM(批内行级并发由 client 负责),
* 结果直接合并回 chunk。批间去重用 Redis 行级标记防止 submitResult 与 stale 恢复双入口重复检测。
*/
public void submitLlmBatches(FileTaskEntity task,
FileResultEntity result,
TaskFileJobEntity job,
List<TaskChunkEntity> chunks) {
if (chunks == null || chunks.isEmpty()) {
return;
}
String prompt = promptReader.apply(task);
String apiKey = apiKeyReader.apply(task);
int batchSize = Math.max(1, properties.getLlmBatchSize());
ChunkRowKeyIndex rowKeyIndex = new ChunkRowKeyIndex();
List<AppearancePatentResultRowDto> candidates = terminalizeRoundExhaustedRows(
task, collectPendingLlmCandidates(chunks, rowKeyIndex), rowKeyIndex);
boolean flushRemainder = isResultSubmissionComplete(task.getId());
if (!flushRemainder && !candidates.isEmpty()) {
LocalDateTime jobUpdatedAt = job.getUpdatedAt();
long pendingFlushMillis = flushPendingMillis();
if (jobUpdatedAt != null
&& Duration.between(jobUpdatedAt, LocalDateTime.now()).toMillis() >= pendingFlushMillis) {
flushRemainder = true;
log.warn("[appearance-patent] 零头批次等待超时,强制提交 taskId={} jobId={} pendingRows={} batchSize={} jobUpdatedAt={} flushAfterMillis={}",
task.getId(), job.getId(), candidates.size(), batchSize, jobUpdatedAt, pendingFlushMillis);
}
}
int submitLimit = (candidates.size() / batchSize) * batchSize;
if (flushRemainder && submitLimit < candidates.size()) {
submitLimit = candidates.size();
}
if (submitLimit <= 0) {
log.info("[appearance-patent] llm batch waiting for more rows taskId={} jobId={} pendingRows={} batchSize={} finalUpload={}",
task.getId(), job.getId(), candidates.size(), batchSize, flushRemainder);
return;
}
if (TransactionSynchronizationManager.isActualTransactionActive()) {
// 守住"整批 LLM 跑在事务里"这一类回归:下面循环每批开头的 touchRunning 是
// UPDATE biz_task_file_job ... WHERE id=? AND status='RUNNING',本方法一旦处在事务中,
// 这把行锁就要持到全部批次结束才释放(线上实测一批 18 分钟),另一台的卡死自愈扫描
// 于是每 110 秒撞一次 50 秒锁等待、并且整轮作废。历史上真发生过,见
// docs/ops/prod-decisions-2026-09-28.md §9。
log.error("[appearance-patent] llm 批次正跑在事务里,行锁会持到批次结束 taskId={} jobId={} pendingRows={}",
task.getId(), job.getId(), submitLimit);
}
int batchTotal = Math.max(1, (submitLimit + batchSize - 1) / batchSize);
int batchIndex = 1;
for (int i = 0; i < submitLimit; i += batchSize) {
List<AppearancePatentResultRowDto> batchRows = candidates.subList(i, Math.min(i + batchSize, submitLimit));
taskFileJobService.touchRunning(job.getId());
List<String> batchRowKeys = batchRows.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
taskCacheService.touchRowInspection(task.getId(), batchRowKeys);
taskCacheService.countRowInspectionRound(task.getId(), batchRowKeys);
long startedAt = System.currentTimeMillis();
List<AppearancePatentResultRowDto> llmRows;
boolean batchAnswered = true;
try {
llmRows = SecretUsageContext.call(task.getUserId(), UserSecretModule.APPEARANCE_PATENT.key(),
() -> llmClient.inspectRows(batchRows, prompt, apiKey));
} catch (Exception ex) {
String message = firstNonBlank(ex.getMessage(), "LLM 检测失败");
log.warn("[appearance-patent] llm batch failed taskId={} jobId={} rows={} batch={}/{} err={}",
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal, message);
llmRows = llmClient.markRowsFailed(batchRows, message);
batchAnswered = false;
}
List<String> keysToRetry = new ArrayList<>(batchRowKeys);
try {
Set<String> landedKeys = mergeLlmRowsIntoChunks(task, llmRows, batchRows, rowKeyIndex);
// 完成标记只能打在真正落库的行上——这是「发版重启会不会丢行」的分界线
if (batchAnswered) {
List<String> landed = batchRowKeys.stream().filter(landedKeys::contains).toList();
if (!landed.isEmpty()) {
taskCacheService.completeRowInspection(task.getId(), landed);
}
keysToRetry.removeAll(landed);
}
} finally {
// 本批结束时处理中名额必须让出来:没落库的行撤销后下一轮马上能重新申领,不等 TTL
if (!keysToRetry.isEmpty()) {
taskCacheService.releaseRowInspection(task.getId(), keysToRetry);
if (batchAnswered) {
// 结果拿到了却落不进分片(少回、rowToken 被改写、分片已清):
// 打成已完成就是"扣了次数永久没结论",宁可重发一轮。
log.warn("[appearance-patent] 本批结果未全部落库,撤销已检测标记交由下一轮重发 taskId={} jobId={} batch={}/{} sent={} retrying={}",
task.getId(), job.getId(), batchIndex, batchTotal, batchRowKeys.size(), keysToRetry);
}
}
}
log.info("[appearance-patent] llm batch done taskId={} jobId={} rows={} batch={}/{} costMs={}",
task.getId(), job.getId(), batchRows.size(), batchIndex, batchTotal,
System.currentTimeMillis() - startedAt);
// 每批刷新任务心跳:软恢复后的检测阶段可达数小时,不刷会被通用兜底线按 updated_at 超 120 分钟误杀(2026-09-28 29473/29479)
taskActivityToucher.accept(task.getId());
batchIndex++;
}
}
/**
* 轮次已到顶的行就地判失败并打成「已完成」,从此退出候选。
*
* <p>失败行会被撤销处理中标记、下一轮重新申领——这是「不丢行」的另一半代价。没有这道闸的话,
* 上游劣化窗口内同一批行会一轮轮重发,每轮都过 {@link SecretUsageContext} 扣用户的 LLM 次数,
* 而 assemble job 走的是 deferRunning(不累加 retryCount),MAX_RETRY_COUNT 那条永不触发。
*/
private List<AppearancePatentResultRowDto> terminalizeRoundExhaustedRows(
FileTaskEntity task,
List<AppearancePatentResultRowDto> candidates,
ChunkRowKeyIndex rowKeyIndex) {
int maxRounds = properties.getLlmRowMaxRounds();
if (maxRounds <= 0 || candidates == null || candidates.isEmpty()) {
return candidates == null ? List.of() : candidates;
}
List<AppearancePatentResultRowDto> retryable = new ArrayList<>(candidates.size());
List<AppearancePatentResultRowDto> exhausted = new ArrayList<>();
for (AppearancePatentResultRowDto row : candidates) {
String key = rowKey(row);
if (!key.isBlank() && taskCacheService.rowInspectionRounds(task.getId(), key) >= maxRounds) {
exhausted.add(row);
} else {
retryable.add(row);
}
}
if (exhausted.isEmpty()) {
return retryable;
}
String reason = "重发轮次已达上限 " + maxRounds + " 轮仍未取到结论,请稍后重新发起检测";
log.warn("[appearance-patent] llm round cap reached taskId={} rows={} maxRounds={}",
task.getId(), exhausted.size(), maxRounds);
List<String> exhaustedKeys = exhausted.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
boolean rowsTerminated = false;
try {
mergeLlmRowsIntoChunks(task, llmClient.markRowsFailed(exhausted, reason),
exhausted, rowKeyIndex);
rowsTerminated = true;
} finally {
// 失败原因没落库就不能打成「已完成」,否则这批行既没结论也再不会重试
if (rowsTerminated) {
taskCacheService.completeRowInspection(task.getId(), exhaustedKeys);
} else {
taskCacheService.releaseRowInspection(task.getId(), exhaustedKeys);
}
}
return retryable;
}
public Set<String> mergeLlmRowsIntoChunks(FileTaskEntity task,
List<AppearancePatentResultRowDto> llmRows,
List<AppearancePatentResultRowDto> batchRows,
ChunkRowKeyIndex rowKeyIndex) {
if (llmRows == null || llmRows.isEmpty()) {
return Set.of();
}
Map<String, AppearancePatentResultRowDto> deduped = new LinkedHashMap<>();
for (int i = 0; i < llmRows.size(); i++) {
AppearancePatentResultRowDto row = llmRows.get(i);
AppearancePatentResultRowDto sourceRow = i < batchRows.size() ? batchRows.get(i) : null;
String key = rowKey(row);
if (key.isBlank()) {
if (sourceRow != null) {
key = rowKey(sourceRow);
}
}
if (key.isBlank() || !deduped.containsKey(key)) {
deduped.put(key, row);
}
}
return mergeLlmRowsIntoSubmittedChunks(task, new ArrayList<>(deduped.values()), null, null, rowKeyIndex);
}
/**
* 收集未检测的候选行:**逐行**申领——同一父号下的 {@code 2_1/2_2/2_3} 每行都各自进候选、各自送检。
*
* <p>历史上这里只挑「组代表行」,其余兄弟行指望 {@code expandRows} 在发起那一轮借用代表行的结论——
* 那是把「同一个父号」当成「同一件东西」,而子序行各有各的 sku、标题和图片链接。
* 口径(2026-10-10 用户定):这些行**都要检测**。折叠与借用一起撤掉,因为借用的后果不是省钱,
* 而是成片空行随结果文件交付(10-08 任务 30426:1211 行里只有 960 行有结论、251 行整段空白,
* 而任务报成功;10-09 任务 30455 的 281 行空行 100% 属于"同父号已有一行被判"这一形状)。
*
* <p>"这行算不算已经检过"用交付口径(标题维+外观维+结论三列齐,见
* {@link AppearancePatentRowOutcome#isJudged})。旧代码用的是"四个字段里任一可用",于是
* 只带着专利维度文本、或只带一个 error 的行会被判"已检测",从此退出候选,而表格里它那三列全是空的。
*/
public List<AppearancePatentResultRowDto> collectPendingLlmCandidates(List<TaskChunkEntity> chunks,
ChunkRowKeyIndex rowKeyIndex) {
if (chunks == null || chunks.isEmpty()) {
return List.of();
}
List<AppearancePatentResultRowDto> candidates = new ArrayList<>();
int claimBlocked = 0;
int judgedSkipped = 0;
int noDataSkipped = 0;
int chunkRows = 0;
for (TaskChunkEntity chunk : chunks) {
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
if (persistedRows.isEmpty()) {
continue;
}
// 这一遍已经把这批分片的载荷读进来了,顺手把「这片里有哪些行」记进索引:
// 下面每批落库前都要这份索引,不记的话就得每批把全部分片重读重解析一遍
rememberChunkRowKeys(rowKeyIndex, chunk, persistedRows.keySet());
chunkRows += persistedRows.size();
for (AppearancePatentResultRowDto row : persistedRows.values()) {
String key = rowKey(row);
if (key.isBlank()) {
continue;
}
if (AppearancePatentRowOutcome.isJudged(row)) {
judgedSkipped++;
continue;
}
if (!shouldInspect.test(row)) {
// 连标题带图都没采到的行:送过去也只能回"外观识别异常",白扣一次
noDataSkipped++;
continue;
}
if (!taskCacheService.beginRowInspection(chunk.getTaskId(), key)) {
// 被"处理中/已完成"标记挡住的行必须数得出来:历史上这里静默 continue,
// 于是"扣了次数却永久没结论"和"这行本来就不该检"在日志里长得一模一样
claimBlocked++;
continue;
}
candidates.add(row);
}
}
log.info("[appearance-patent] 候选收集逐行申领 taskId={} chunkRows={} judgedSkipped={} noDataSkipped={} claimBlocked={} candidates={}",
chunks.get(0).getTaskId(), chunkRows, judgedSkipped, noDataSkipped, claimBlocked, candidates.size());
return candidates;
}
/**
* 只计数、不申领。用 {@link #collectPendingLlmCandidates} 计数会把行标成处理中,
* 紧接着真正发起检测的那一步就一行也申领不到,只能等处理中标记的 TTL 过期。
*/
public int countUninspectedRows(List<TaskChunkEntity> chunks) {
if (chunks == null || chunks.isEmpty()) {
return 0;
}
int total = 0;
for (TaskChunkEntity chunk : chunks) {
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
for (AppearancePatentResultRowDto row : persistedRows.values()) {
if (!rowKey(row).isBlank() && !AppearancePatentRowOutcome.isJudged(row)) {
total++;
}
}
}
return total;
}
public int countLlmWorkUnits(List<TaskChunkEntity> chunks, int batchSize) {
if (chunks == null || chunks.isEmpty()) {
return 0;
}
int total = 0;
for (TaskChunkEntity chunk : chunks) {
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
if (persistedRows.isEmpty()) {
continue;
}
int unresolved = (int) persistedRows.values().stream()
.filter(row -> !rowKey(row).isBlank() && !AppearancePatentRowOutcome.isJudged(row)).count();
if (unresolved > 0) {
total += Math.max(1, (unresolved + batchSize - 1) / batchSize);
}
}
return total;
}
private long flushPendingMillis() {
return Math.max(1, properties.getFlushPendingMinutes()) * 60_000L;
}
/** 返回真正落库的行键;没匹配到分片行的键不会出现在里面,调用方据此决定要不要撤销已检测标记。 */
private Set<String> mergeLlmRowsIntoSubmittedChunks(FileTaskEntity task,
List<AppearancePatentResultRowDto> llmRows,
String fallbackScopeHash,
Integer fallbackChunkIndex,
ChunkRowKeyIndex rowKeyIndex) {
if (task == null || llmRows == null || llmRows.isEmpty()) {
return Set.of();
}
List<TaskChunkEntity> chunks = loadSubmittedChunks(task.getId());
if (chunks.isEmpty()) {
return Set.of();
}
Map<String, Set<String>> rowKeysByChunk = new LinkedHashMap<>();
Map<String, TaskChunkEntity> chunkByKey = new LinkedHashMap<>();
for (TaskChunkEntity chunk : chunks) {
String chunkKey = chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex());
rowKeysByChunk.put(chunkKey, chunkRowKeySet(rowKeyIndex, chunk));
chunkByKey.put(chunkKey, chunk);
}
Map<String, Map<String, AppearancePatentResultRowDto>> mergeRowsByChunk = new LinkedHashMap<>();
for (AppearancePatentResultRowDto resultRow : llmRows) {
for (AppearancePatentResultRowDto expandedRow : List.of(resultRow)) {
String rowKey = rowKey(expandedRow);
if (rowKey.isBlank()) {
continue;
}
boolean matched = false;
for (Map.Entry<String, Set<String>> entry : rowKeysByChunk.entrySet()) {
if (entry.getValue().contains(rowKey)) {
mergeRowsByChunk.computeIfAbsent(entry.getKey(), ignored -> new LinkedHashMap<>())
.put(rowKey, expandedRow);
matched = true;
}
}
if (!matched && fallbackScopeHash != null && fallbackChunkIndex != null) {
String fallbackKey = chunkStorageKey(fallbackScopeHash, fallbackChunkIndex);
if (chunkByKey.containsKey(fallbackKey)) {
mergeRowsByChunk.computeIfAbsent(fallbackKey, ignored -> new LinkedHashMap<>())
.put(rowKey, expandedRow);
matched = true;
}
}
if (!matched) {
log.warn("[appearance-patent] llm row has no submitted chunk taskId={} rowKey={}",
task.getId(), rowKey);
}
}
}
Set<String> landedKeys = new LinkedHashSet<>();
for (Map.Entry<String, Map<String, AppearancePatentResultRowDto>> entry : mergeRowsByChunk.entrySet()) {
TaskChunkEntity chunk = chunkByKey.get(entry.getKey());
if (chunk == null || entry.getValue().isEmpty()) {
continue;
}
landedKeys.addAll(chunkRowMerger.merge(task.getId(), chunk.getScopeHash(), chunk.getChunkIndex(),
new ArrayList<>(entry.getValue().values())));
}
return landedKeys;
}
private String chunkStorageKey(String scopeHash, Integer chunkIndex) {
return firstNonBlank(scopeHash, "") + ":" + (chunkIndex == null ? 0 : chunkIndex);
}
/**
* 一次 {@link #submitLlmBatches} 内共享的「哪个分片里有哪些行」索引,按 payload_hash 判新鲜。
*
* <p>只存 rowKey 不存整行,索引本身不会把载荷钉在内存里。作用域必须是单次调用(局部变量,随栈丢弃):
* 本服务是单例、多个任务同时在跑批,做成字段就是多线程共享一个裸 LinkedHashMap,而且分片键永不过期、只增不减。
* 数据正确性反过来不依赖局部性——收集候选那一遍每次都会重读并重记全部分片。
*/
public static final class ChunkRowKeyIndex {
private final Map<String, Snapshot> snapshots = new LinkedHashMap<>();
}
/** 把刚解析出来的分片行键记进索引(自己拷一份,不握住整行载荷),返回索引里那份。 */
private Set<String> rememberChunkRowKeys(ChunkRowKeyIndex rowKeyIndex, TaskChunkEntity chunk, Set<String> rowKeys) {
Set<String> copied = new LinkedHashSet<>(rowKeys);
rowKeyIndex.snapshots.put(chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()),
new Snapshot(chunk.getPayloadHash(), copied));
return copied;
}
/** 分片当前载荷里的全部行键;载荷哈希与索引里那次解析一致时直接复用,不重读对象存储。 */
private Set<String> chunkRowKeySet(ChunkRowKeyIndex rowKeyIndex, TaskChunkEntity chunk) {
Snapshot cached = rowKeyIndex.snapshots.get(chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()));
if (cached != null && Objects.equals(cached.payloadHash, chunk.getPayloadHash())) {
return cached.rowKeys;
}
return rememberChunkRowKeys(rowKeyIndex, chunk, readChunkRows(chunk).keySet());
}
private record Snapshot(String payloadHash, Set<String> rowKeys) {
}
private boolean isResultSubmissionComplete(Long taskId) {
return submissionComplete.test(taskId);
}
private void touchJavaSideTaskActivity(Long taskId) {
taskActivityToucher.accept(taskId);
}
private String rowKey(AppearancePatentResultRowDto row) {
return rowKeyer.apply(row);
}
private Map<String, AppearancePatentResultRowDto> readChunkRows(TaskChunkEntity chunk) {
return chunkRowReader.apply(chunk);
}
private List<TaskChunkEntity> loadSubmittedChunks(Long taskId) {
return submittedChunksLoader.apply(taskId);
}
private String firstNonBlank(String preferred, String fallback) {
return firstNonBlankFn.apply(preferred, fallback);
}
}