fix(appearance-patent): 重试不再整单重采——派活前裁掉已采到的行+跨代切分片 scope(B96-07)

33898/33883/33885 这类单重试一次就是 4000 组从头再抓一遍(1.3-1.5 小时+按次买出口 IP),根因两条:① 重排派给客户端的仍是解析载荷的全量 groups;② ap_worker 的 chunkIndex 是相对本载荷的序号且每片都带 submissionId=str(task_id),裁剪后从 1 重新编号会与上一代同号分片撞 (task_id, module_type, scope_hash, chunk_index),落进 duplicate chunk ignored 却照旧回 ok —— 客户端以为传成功、服务端一行没收。

A:AppearancePatentResumeTrimSupport(纯裁剪)/AppearancePatentDispatchTrimSupport(下发口编排)。两条下发口(兜底拉取、页面推本机)都过 trim;已收判据不是「行存在」而是「行上有标题或图片」(失败行也会落库,按行存在裁会把 404 行整批裁掉、坏行永不重采);载荷对象已不在的分片不算已收。跨代 scope=<caller submissionId>:gen:<resumeAttempt>(首代原样,幂等),新一代开跑前清上一代输入面 completed,否则本代第一个分片就被当成「已交完」拿半批行去组装(33924 同形)。

B:裁到一组不剩时不再派活——先标完成再收口出表;已出过表(result_file_url 有值)或状态非 PENDING/RUNNING 不动,避免直链被清空再重排。SPI 对「解析期无分组」与「已收全被裁光」分两侧:后者返回空 groups 载荷而不是 null,否则认领侧会把刚收口的任务改判「任务数据不完整」FAILED。

门禁:mvn -o test 整量 5,430 例/0 失败/19 skipped BUILD SUCCESS;新增用例 14 支(裁剪 9、scope 5);TaskLifecycleDebtRatchetTest 130 遍/15 文件不变(归属谓词改走 TaskFinalizeSupport.belongsToModule);SourceScanTest 台账 57→59、support 22→24;_push_gate.py --fast overall=pass(9.5/8.8/10.0/9.6)。
This commit is contained in:
2026-11-12 13:55:15 +08:00
parent f7b206474d
commit 4d008608b1
8 changed files with 695 additions and 13 deletions
@@ -38,7 +38,14 @@ public class AppearancePatentTaskPullSpiImpl implements ClientTaskPullSpi {
AppearancePatentParsedPayloadDto payload = taskService.queuePayload(task.getId(), task.getUserId());
List<AppearancePatentParsedGroupVo> groups = payload.getGroups() == null ? List.of() : payload.getGroups();
if (groups.isEmpty()) {
// 空 groups 在 Python 侧会被静默跳过("groups/rows is empty, skip"),宁可在服务端直接判失败
// 空 groups 在 Python 侧会被静默跳过("groups/rows is empty, skip")。两种来路必须分开:
// ① 解析期就没有分组 ⇒ 认领侧判死(返回 null 会落成「任务数据不完整」);
// ② 派活前裁剪把已收行摘光了(B96-07)⇒ 服务端已经直接收口出表,这里**不能**返回 null,
// 否则认领侧把刚收口的任务改判 FAILED,用户手里那张表也跟着没了。
if (taskService.allParsedRowsCollected(task.getId())) {
log.info("[appearance-patent] 全部行已收,不再派活(服务端已直接收口出表) taskId={}", task.getId());
return emptyQueuePayload(task, payload);
}
log.warn("[appearance-patent] 兜底拉取失败:解析分组为空 taskId={}", task.getId());
return null;
}
@@ -52,6 +59,17 @@ public class AppearancePatentTaskPullSpiImpl implements ClientTaskPullSpi {
return Map.of("type", QUEUE_TYPE, "data", data);
}
/** 已收全时的空载荷:让客户端按「空 groups 跳过」,同时不触发认领侧的「数据不完整」判死。 */
private Map<String, Object> emptyQueuePayload(FileTaskEntity task, AppearancePatentParsedPayloadDto payload) {
Map<String, Object> data = new LinkedHashMap<>();
data.put("taskId", task.getId());
data.put("user_id", task.getUserId());
data.put("prompt", payload.getAiPrompt());
data.put("api_key", payload.getApiKey());
data.put("groups", List.of());
return Map.of("type", QUEUE_TYPE, "data", data);
}
@Override
public void onClaimed(FileTaskEntity task) {
// 对齐 activate:刷新模块缓存心跳,让页面立刻看到 RUNNING
@@ -14,6 +14,7 @@ import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePate
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentTaskRecordSupport;
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentDeliverySupport;
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentDeliveredRearmSupport;
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentDispatchTrimSupport;
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentExcelParser;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.AppearancePatentProperties;
@@ -838,13 +839,33 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
}
/**
* 派活前裁剪已收行(B96-07)。实现住在
* {@link com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentDispatchTrimSupport}:
* 判定本体与纯裁剪件 {@code AppearancePatentResumeTrimSupport} 放一处读,主件只留两条薄委派
* (本件已撞超长文件棘轮,棘轮口径是「要么拆」)。
*/
public AppearancePatentParsedPayloadDto queuePayload(Long taskId, Long userId) {
return payloadSupport().queuePayload(taskId, userId);
return dispatchTrimSupport().trim(taskId, payloadSupport().queuePayload(taskId, userId));
}
/** 页面/直连 API 取本机队列载荷:本单通道不是本机时拒发(P1-1 同族)。 */
public AppearancePatentParsedPayloadDto queuePayloadForLocalClient(Long taskId, Long userId) {
return payloadSupport().queuePayloadForLocalClient(taskId, userId);
return dispatchTrimSupport().trim(taskId, payloadSupport().queuePayloadForLocalClient(taskId, userId));
}
/** 给 SPI 区分「解析期就没有分组」与「已收全被裁光」:后者不许被认领侧改判 FAILED。 */
public boolean allParsedRowsCollected(Long taskId) {
return dispatchTrimSupport().allParsedRowsCollected(taskId);
}
private AppearancePatentDispatchTrimSupport dispatchTrimSupport() {
return new AppearancePatentDispatchTrimSupport(
MODULE_TYPE, fileTaskMapper, fileResultMapper, taskScopeStateMapper,
this::loadSubmittedChunks, rowReader()::readChunkRows, rowReader()::rowKey,
payloadSupport()::readParsedPayload,
this::markSubmissionCompleteInNewTransaction, this::finalizeStaleTaskInTransaction);
}
static String resolveTaskExecutionStatus(boolean waitingForAssemble, boolean executionFailed) {
@@ -147,7 +147,7 @@ public class AppearancePatentChunkSubmissionSupport {
int chunkIndex = request.getChunkIndex() == null ? 0 : request.getChunkIndex();
int chunkTotal = request.getChunkTotal() == null ? 1 : request.getChunkTotal();
boolean done = Boolean.TRUE.equals(request.getDone());
String scopeKey = firstNonBlank.pick(request.getSubmissionId(), "task:" + taskId);
String scopeKey = resolveScopeKey(request.getSubmissionId(), task);
String scopeHash = DigestUtil.sha256Hex(scopeKey);
TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
@@ -184,7 +184,7 @@ public class AppearancePatentChunkSubmissionSupport {
int chunkIndex = request.getChunkIndex() == null ? 0 : request.getChunkIndex();
int chunkTotal = request.getChunkTotal() == null ? 1 : request.getChunkTotal();
String scopeKey = firstNonBlank.pick(request.getSubmissionId(), "task:" + taskId);
String scopeKey = resolveScopeKey(request.getSubmissionId(), task);
String scopeHash = DigestUtil.sha256Hex(scopeKey);
taskCacheService.touchTaskHeartbeat(taskId);
@@ -376,6 +376,33 @@ public class AppearancePatentChunkSubmissionSupport {
return updated;
}
/**
* 上报范围键(B96-07)。
*
* <p>重排后派给客户端的是<b>裁掉已收行</b>的剩余组,客户端从 1 重新编号——若仍写同一个
* {@code task:{id}},新分片会与上一代同序号的分片撞 (task_id, module_type, scope_hash,
* chunk_index),落进下面那条「duplicate chunk ignored」却照旧回 ok:客户端以为传成功,
* 服务端一行没收,任务等到陈旧扫描判死。所以跨代必须换 scope。
*
* <p>首代({@code resumeAttempt == 0})保持原键不变:存量分片的 scope 一个字节都不动,
* 读侧(出表按行键去重、进度按 scope 聚合)不受影响。
*
* <p>⚠ 显式 submissionId <b>也要</b>带代数后缀:现网 {@code ap_worker} 每片都发
* {@code "submissionId": str(task_id)}(实测 ap_worker.py:876),只给缺省值加后缀等于
* 生产上一片没生效——裁剪后客户端从 1 重新编号,照样撞号被静默丢弃。代数只有服务端知道,
* 客户端无从标注,所以后缀统一在服务端补,且幂等(已带就不重复加)。
*/
static String resolveScopeKey(String submissionId, FileTaskEntity task) {
String explicit = submissionId == null ? "" : submissionId.trim();
String base = explicit.isEmpty() ? "task:" + task.getId() : explicit;
int attempt = task.getResumeAttempt() == null ? 0 : task.getResumeAttempt();
if (attempt <= 0) {
return base;
}
String suffix = ":gen:" + attempt;
return base.endsWith(suffix) ? base : base + suffix;
}
private void upsertScopeState(Long taskId,
String scopeKey,
String scopeHash,
@@ -0,0 +1,214 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.common.task.TaskStatuses;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskFinalizeSupport;
import lombok.extern.slf4j.Slf4j;
import java.time.LocalDateTime;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.function.LongConsumer;
/**
* 派活前的「按代续跑」编排(B96-07):裁掉已收行 → 切代完成标记 → 已收全则直接收口出表。
*
* <p>从 {@code AppearancePatentTaskService} 抽出来的原因不只是行数(那支已撞超长文件棘轮):
* 这一段是「谁该被执行方重采」的判定本体,和 {@link AppearancePatentResumeTrimSupport}(纯裁剪)
* 是一对——判定与裁剪放在一处读,才不会出现"裁剪按新口径、完成标记按旧口径"这种半修。
*
* <p>三条边界:
* <ol>
* <li><b>只认真采到的行</b>({@link AppearancePatentResumeTrimSupport#countsAsCollected}):
* 分片里失败行也会落库,按"行存在"裁会把上一代 404 的行整批裁掉,坏行永不重采。</li>
* <li><b>裁到一组不剩就直接收口出表</b>:客户端拿到空 groups 会静默跳过,任务停在 PENDING
* 等陈旧扫描,而 {@code AppearancePatentStaleTaskScanner} 只扫 RUNNING ⇒ 永远等不到出表。</li>
* <li><b>新一代开跑前清上一代的 {@code completed}</b>:不清则本代第一个分片落地就被
* {@code isResultSubmissionComplete} 当成"本代已交完",拿半批行去组装(2026-11-11 现网 33924)。</li>
* </ol>
*/
@Slf4j
public class AppearancePatentDispatchTrimSupport {
private final String moduleType;
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final Function<Long, List<TaskChunkEntity>> chunkLoader;
private final Function<TaskChunkEntity, Map<String, AppearancePatentResultRowDto>> chunkRowReader;
private final Function<AppearancePatentParsedRowVo, String> rowKeyFn;
private final Function<FileTaskEntity, AppearancePatentParsedPayloadDto> parsedPayloadReader;
private final LongConsumer submissionCompleter;
private final BiConsumer<FileTaskEntity, String> finalizer;
public AppearancePatentDispatchTrimSupport(String moduleType,
FileTaskMapper fileTaskMapper,
FileResultMapper fileResultMapper,
TaskScopeStateMapper taskScopeStateMapper,
Function<Long, List<TaskChunkEntity>> chunkLoader,
Function<TaskChunkEntity, Map<String, AppearancePatentResultRowDto>> chunkRowReader,
Function<AppearancePatentParsedRowVo, String> rowKeyFn,
Function<FileTaskEntity, AppearancePatentParsedPayloadDto> parsedPayloadReader,
LongConsumer submissionCompleter,
BiConsumer<FileTaskEntity, String> finalizer) {
this.moduleType = moduleType;
this.fileTaskMapper = fileTaskMapper;
this.fileResultMapper = fileResultMapper;
this.taskScopeStateMapper = taskScopeStateMapper;
this.chunkLoader = chunkLoader;
this.chunkRowReader = chunkRowReader;
this.rowKeyFn = rowKeyFn;
this.parsedPayloadReader = parsedPayloadReader;
this.submissionCompleter = submissionCompleter;
this.finalizer = finalizer;
}
/**
* 下发前裁剪(两条下发口共用:兜底拉取与页面推本机)。
*
* <p>为什么必须在下发值这一处:{@code retryInPlace} 只把任务复位成 PENDING,客户端照旧拿到
* 全部 4000 组,上一代已经传完的行按次再买一遍出口 IP(铁律 29(b):判定与真正被消费的值同源)。
*/
public AppearancePatentParsedPayloadDto trim(Long taskId, AppearancePatentParsedPayloadDto payload) {
if (taskId == null || payload == null || payload.getGroups() == null || payload.getGroups().isEmpty()) {
return payload;
}
Set<String> collectedKeys = collectedRowKeys(taskId);
if (collectedKeys.isEmpty()) {
return payload;
}
AppearancePatentResumeTrimSupport.TrimResult trim =
AppearancePatentResumeTrimSupport.trim(payload.getGroups(), rowKeyFn, collectedKeys);
if (trim.collectedRows() == 0) {
return payload;
}
log.info("[appearance-patent] 派活前裁剪已收行 taskId={} 原行数={} 已收={} 剩余={} 保留分组={} 丢弃分组={}",
taskId, trim.totalRows(), trim.collectedRows(), trim.remainingRows(),
trim.groups().size(), trim.droppedGroups());
payload.setGroups(trim.groups());
if (trim.allRowsCollected()) {
completeCollection(taskId, trim);
return payload;
}
resetPreviousGenerationCompletion(taskId);
return payload;
}
/** 「派活时被裁光」判别:给 SPI 区分「解析期就没有分组」与「已收全被裁光」。 */
public boolean allParsedRowsCollected(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
FileTaskEntity task = loadTask(taskId);
if (task == null) {
return false;
}
Set<String> collectedKeys = collectedRowKeys(taskId);
if (collectedKeys.isEmpty()) {
return false;
}
List<AppearancePatentParsedGroupVo> groups = parsedPayloadReader.apply(task).getGroups();
if (groups == null || groups.isEmpty()) {
return false;
}
return AppearancePatentResumeTrimSupport.trim(groups, rowKeyFn, collectedKeys).allRowsCollected();
}
/** 该任务全部已落库分片里真采到的行键(跨 scope:每一代传的行都算已收)。 */
private Set<String> collectedRowKeys(Long taskId) {
Set<String> keys = new LinkedHashSet<>();
for (TaskChunkEntity chunk : chunkLoader.apply(taskId)) {
// readChunkRows 对「载荷对象已不在」的分片返回空表并留日志:那些行不算已收,
// 宁可多采一次也不让行凭空消失(出表侧按行键去重,重采不会重复出图)。
keys.addAll(AppearancePatentResumeTrimSupport.collectableKeysOf(chunkRowReader.apply(chunk)));
}
keys.remove("");
return keys;
}
/**
* 新一代要开跑:清掉上一代留下的输入面 {@code completed=1}。
*
* <p>只清输入面({@code llm_status is null})的 completed,不动 llm 结果与分片本身——
* 已检结论就是续跑点(铁律 26 的断点续传语义)。
*/
private void resetPreviousGenerationCompletion(Long taskId) {
int reset = taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType)
.isNull(TaskScopeStateEntity::getLlmStatus)
.eq(TaskScopeStateEntity::getCompleted, 1)
.set(TaskScopeStateEntity::getCompleted, 0)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
if (reset > 0) {
log.info("[appearance-patent] 新一代开跑,清掉上一代输入面的完成标记 taskId={} scopes={}", taskId, reset);
}
}
/**
* 一条都没剩:任务直接收口并出表,不再派给任何执行方。
*
* <p>判序刻意是「先标完成再收尾」:前者让陈旧扫描不再把这条正在出表的 RUNNING 任务当
* 「客户端不回传」判死;反序则中间那一瞬它是可被扫到的活靶。
*
* <p>幂等:出过一次表({@code result_file_url} 已回填)就不再动手——重复下发时 finalize 会把
* 直链清空再重排组装,用户正下载的表会一闪。终态一律不碰。
*/
private void completeCollection(Long taskId, AppearancePatentResumeTrimSupport.TrimResult trim) {
FileTaskEntity task = loadTask(taskId);
if (task == null) {
return;
}
if (!TaskStatuses.FileTask.PENDING.equals(task.getStatus())
&& !TaskStatuses.FileTask.RUNNING.equals(task.getStatus())) {
log.info("[appearance-patent] 已收全但状态不是待执行/执行中,跳过自动出表 taskId={} status={}",
taskId, task.getStatus());
return;
}
if (hasDeliveredResult(taskId)) {
log.info("[appearance-patent] 已收全且已出过表,跳过重复收口 taskId={}", taskId);
return;
}
log.info("[appearance-patent] 全部 {}/{} 行已落库,不再派活,直接出表 taskId={}",
trim.collectedRows(), trim.totalRows(), taskId);
submissionCompleter.accept(taskId);
finalizer.accept(task, null);
}
private boolean hasDeliveredResult(Long taskId) {
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, moduleType)
.last("limit 1"));
if (rows == null || rows.isEmpty()) {
return false;
}
String url = rows.getFirst().getResultFileUrl();
return url != null && !url.isBlank();
}
/** 归属谓词走骨架出口({@code TaskFinalizeSupport.belongsToModule}),别再手写一份。 */
private FileTaskEntity loadTask(Long taskId) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !TaskFinalizeSupport.belongsToModule(moduleType, task.getModuleType())) {
return null;
}
return task;
}
}
@@ -0,0 +1,142 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
/**
* 派活前的「已收行裁剪」(B96-07)。
*
* <p>背景:外观专利的重试今天等于<b>整单重采</b>——{@code retryInPlace} 只把任务复位成
* PENDING,客户端拿到的是原始 parsed payload 的全部 groups,于是上一代已经传完的 4000 组
* 全部按次买出口 IP 再跑一遍。钱是按出口 IP 计的,这里省下来的就是客户投诉的那笔。
*
* <p>裁剪只做一件事:把<b>已经落进 {@code biz_task_chunk} 且载荷还读得回来</b>的行从分组里摘掉,
* 其余原样保留。三条边界:
* <ul>
* <li><b>读不回来的不算已收</b>——载荷对象被清理过的分片,其行键不进已收集合(宁多重采一次,
* 不能让行凭空消失;出表侧本来就按行键去重,重采不会重复出图)。</li>
* <li><b>裁完为空的组整组丢掉</b>——空 items 的组在 Python 侧会被跳过,留着只会让日志虚高。</li>
* <li><b>不动入参</b>——返回新对象,调用方拿到的解析载荷缓存不被就地改写(同一份 payload
* 还会被出表/同步侧读)。</li>
* </ul>
*
* <p>纯函数、无 Spring 依赖,判据见 {@code AppearancePatentResumeTrimTest}。
*/
public final class AppearancePatentResumeTrimSupport {
private AppearancePatentResumeTrimSupport() {
}
/** 裁剪读数:保留的分组、原行数、已收行数、丢掉的分组数。 */
public record TrimResult(List<AppearancePatentParsedGroupVo> groups,
int totalRows,
int collectedRows,
int droppedGroups) {
public boolean allRowsCollected() {
return totalRows > 0 && collectedRows >= totalRows;
}
public int remainingRows() {
return totalRows - collectedRows;
}
}
/**
* @param groups 解析分组(不会被修改)
* @param rowKeyFn 行键口径,与出表去重同一把({@code AppearancePatentResultRowReader#rowKey})
* @param collectedKeys 已落库且载荷可读的行键
*/
public static TrimResult trim(List<AppearancePatentParsedGroupVo> groups,
Function<AppearancePatentParsedRowVo, String> rowKeyFn,
Set<String> collectedKeys) {
if (groups == null || groups.isEmpty()) {
return new TrimResult(List.of(), 0, 0, 0);
}
Set<String> collected = collectedKeys == null ? Set.of() : collectedKeys;
List<AppearancePatentParsedGroupVo> kept = new ArrayList<>();
int total = 0;
int collectedRows = 0;
int droppedGroups = 0;
for (AppearancePatentParsedGroupVo group : groups) {
if (group == null) {
droppedGroups++;
continue;
}
List<AppearancePatentParsedRowVo> items = group.getItems() == null ? List.of() : group.getItems();
if (items.isEmpty()) {
// 解析期就不该有空组;真出现时空组本来就发不出去(Python 侧 skip),按原样丢掉。
droppedGroups++;
continue;
}
List<AppearancePatentParsedRowVo> remainingItems = new ArrayList<>(items.size());
for (AppearancePatentParsedRowVo row : items) {
total++;
String key = rowKeyFn == null ? "" : rowKeyFn.apply(row);
if (key != null && !key.isBlank() && collected.contains(key)) {
collectedRows++;
continue;
}
remainingItems.add(row);
}
if (remainingItems.isEmpty()) {
droppedGroups++;
continue;
}
kept.add(copyWithItems(group, remainingItems));
}
return new TrimResult(List.copyOf(kept), total, collectedRows, droppedGroups);
}
private static AppearancePatentParsedGroupVo copyWithItems(AppearancePatentParsedGroupVo group,
List<AppearancePatentParsedRowVo> items) {
AppearancePatentParsedGroupVo copy = new AppearancePatentParsedGroupVo();
copy.setSourceFileKey(group.getSourceFileKey());
copy.setSourceFilename(group.getSourceFilename());
copy.setGroupKey(group.getGroupKey());
copy.setBaseId(group.getBaseId());
copy.setDisplayId(group.getDisplayId());
copy.setItemCount(items.size());
copy.setItems(new ArrayList<>(items));
return copy;
}
/**
* 从已落库的结果行里挑出「真的采到了」的键。
*
* <p>⚠ 这一条是整个裁剪的命门:分片里<b>失败行也会落库</b>(404/墙页/采集异常都会带一行
* error 落进去,出表时写「未取得检测结果」)。若把「有这行」当成「这行采完了」,重试就会把
* 上一代<b>失败的那些行</b>整批裁掉——用户点一次「重试」,下发的只剩上代就成功的行(等于
* 再跑一遍已经好了的部分),真正坏掉的 404 行一条都不会重采,任务照样失败,钱也照旧白花。
*
* <p>判据因此不是「行存在」而是「行上有标题或图片」——与交付侧
* {@code AppearancePatentRowOutcome#hasCheckableData} 同一把尺子(唯一点径,别再抄第二份)。
* 「采到了但 LLM 没出结论」的行算已采:结论由补检链路({@code collectPendingLlmCandidates}
* +行级重发)负责,重新抓页面不会让结论自己冒出来。
*/
public static boolean countsAsCollected(AppearancePatentResultRowDto row) {
return row != null && AppearancePatentRowOutcome.hasCheckableData(null, row);
}
/** 键→结果行 表里,只保留真正采到的那些键(口径见 {@link #countsAsCollected})。 */
public static Set<String> collectableKeysOf(Map<String, AppearancePatentResultRowDto> rows) {
if (rows == null || rows.isEmpty()) {
return Set.of();
}
Set<String> keys = new LinkedHashSet<>();
for (Map.Entry<String, AppearancePatentResultRowDto> entry : rows.entrySet()) {
if (countsAsCollected(entry.getValue())) {
keys.add(entry.getKey());
}
}
return keys;
}
}
@@ -0,0 +1,198 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* B96-07 派活前裁剪的判据。
*
* <p>背景:外观专利的「重试/继续」今天是整单重采——上一代已传完的行会被再买一遍出口 IP。
* 裁剪本身只是「按行键摘掉已收行」,但三件事必须钉住,任何一条错了都比不裁剪更贵:
*
* <ol>
* <li><b>半代裁剪</b>——已收的摘、没收到的一条不许丢(丢了=用户表里缺行,且再也不会采);</li>
* <li><b>空组丢弃</b>——全收的组整组去掉(Python 侧会跳过空组,留着只会让日志虚高);</li>
* <li><b>入参不被改写</b>——解析载荷是共享对象,出表/同步侧还在读同一份。</li>
* </ol>
*/
class AppearancePatentResumeTrimTest {
private static AppearancePatentParsedRowVo row(String token) {
AppearancePatentParsedRowVo row = new AppearancePatentParsedRowVo();
row.setRowToken(token);
row.setDisplayId(token);
return row;
}
private static AppearancePatentParsedGroupVo group(String groupKey, AppearancePatentParsedRowVo... rows) {
AppearancePatentParsedGroupVo group = new AppearancePatentParsedGroupVo();
group.setGroupKey(groupKey);
group.setBaseId(groupKey);
group.setItemCount(rows.length);
group.setItems(new ArrayList<>(List.of(rows)));
return group;
}
private static Set<String> keys(String... values) {
return new LinkedHashSet<>(List.of(values));
}
@Test
void 没有已收行时原样返回且不判全收() {
List<AppearancePatentParsedGroupVo> groups = List.of(
group("g1", row("a"), row("b")),
group("g2", row("c")));
AppearancePatentResumeTrimSupport.TrimResult trim =
AppearancePatentResumeTrimSupport.trim(groups, AppearancePatentParsedRowVo::getRowToken, Set.of());
assertEquals(3, trim.totalRows());
assertEquals(0, trim.collectedRows());
assertEquals(3, trim.remainingRows());
assertEquals(2, trim.groups().size());
assertEquals(0, trim.droppedGroups());
assertFalse(trim.allRowsCollected());
}
@Test
void 部分已收只摘已收的行_其余行与分组原样保留() {
List<AppearancePatentParsedGroupVo> groups = List.of(
group("g1", row("a"), row("b"), row("c")),
group("g2", row("d"), row("e")));
AppearancePatentResumeTrimSupport.TrimResult trim =
AppearancePatentResumeTrimSupport.trim(groups, AppearancePatentParsedRowVo::getRowToken,
keys("a", "e"));
assertEquals(5, trim.totalRows());
assertEquals(2, trim.collectedRows());
assertEquals(3, trim.remainingRows());
assertEquals(2, trim.groups().size());
assertEquals(0, trim.droppedGroups());
assertEquals(List.of("b", "c"), trim.groups().get(0).getItems().stream()
.map(AppearancePatentParsedRowVo::getRowToken).toList());
assertEquals(List.of("d"), trim.groups().get(1).getItems().stream()
.map(AppearancePatentParsedRowVo::getRowToken).toList());
// itemCount 跟裁剪后的实际行数走:界面/日志读的是它,留着旧值等于谎报进度。
assertEquals(2, trim.groups().get(0).getItemCount());
assertEquals(1, trim.groups().get(1).getItemCount());
assertFalse(trim.allRowsCollected());
}
@Test
void 整组已收时整组丢掉_全收才算全收() {
List<AppearancePatentParsedGroupVo> groups = List.of(
group("g1", row("a")),
group("g2", row("b"), row("c")));
AppearancePatentResumeTrimSupport.TrimResult trim =
AppearancePatentResumeTrimSupport.trim(groups, AppearancePatentParsedRowVo::getRowToken,
keys("a", "b", "c"));
assertEquals(3, trim.totalRows());
assertEquals(3, trim.collectedRows());
assertEquals(0, trim.remainingRows());
assertTrue(trim.groups().isEmpty());
assertEquals(2, trim.droppedGroups());
assertTrue(trim.allRowsCollected());
}
@Test
void 只剩一组一行时仍然是可派活的载荷() {
List<AppearancePatentParsedGroupVo> groups = List.of(
group("g1", row("a")),
group("g2", row("b")));
AppearancePatentResumeTrimSupport.TrimResult trim =
AppearancePatentResumeTrimSupport.trim(groups, AppearancePatentParsedRowVo::getRowToken, keys("a"));
assertEquals(1, trim.groups().size());
assertEquals("g2", trim.groups().get(0).getGroupKey());
assertEquals(1, trim.droppedGroups());
assertFalse(trim.allRowsCollected());
}
@Test
void 不改写入参原对象() {
AppearancePatentParsedGroupVo original = group("g1", row("a"), row("b"));
List<AppearancePatentParsedGroupVo> groups = new ArrayList<>(List.of(original));
AppearancePatentResumeTrimSupport.TrimResult trim =
AppearancePatentResumeTrimSupport.trim(groups, AppearancePatentParsedRowVo::getRowToken, keys("a"));
assertEquals(1, groups.size());
assertSame(original, groups.get(0));
assertEquals(2, original.getItems().size());
assertEquals(2, original.getItemCount());
assertEquals(1, trim.groups().size());
assertEquals(1, trim.groups().get(0).getItems().size());
assertFalse(trim.groups().get(0) == original);
}
@Test
void 空分组与空入参不炸() {
assertEquals(0, AppearancePatentResumeTrimSupport.trim(List.of(),
AppearancePatentParsedRowVo::getRowToken, keys("a")).totalRows());
assertEquals(0, AppearancePatentResumeTrimSupport.trim(null,
AppearancePatentParsedRowVo::getRowToken, keys("a")).totalRows());
AppearancePatentParsedGroupVo empty = new AppearancePatentParsedGroupVo();
empty.setItems(new ArrayList<>());
AppearancePatentResumeTrimSupport.TrimResult trim = AppearancePatentResumeTrimSupport.trim(
List.of(empty), AppearancePatentParsedRowVo::getRowToken, keys("a"));
assertEquals(1, trim.droppedGroups());
assertFalse(trim.allRowsCollected());
}
// ---- 「已收」的判据:行存在 != 行采到了 ---------------------------------------------
// 分片里失败行也会落库(404/墙页都带一行 error)。若按「有这行」判已收,重试会把上一代
// 失败的行整批裁掉,坏行永不重采——钱花了、表还是缺行,比不裁剪更糟。
private static AppearancePatentResultRowDto resultRow(String title, String url, String error) {
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
row.setTitle(title);
row.setUrl(url);
row.setError(error);
return row;
}
@Test
void 采到标题或图片的行才算已收() {
assertTrue(AppearancePatentResumeTrimSupport.countsAsCollected(resultRow("标题", null, null)));
assertTrue(AppearancePatentResumeTrimSupport.countsAsCollected(resultRow(null, "https://x/1.jpg", null)));
}
@Test
void 失败行即使带错误原文也不算已收_必须重采() {
assertFalse(AppearancePatentResumeTrimSupport.countsAsCollected(
resultRow(null, null, "商品页 404")));
assertFalse(AppearancePatentResumeTrimSupport.countsAsCollected(resultRow(null, null, null)));
assertFalse(AppearancePatentResumeTrimSupport.countsAsCollected(null));
}
@Test
void 键集只保留真采到的行() {
String a = "file::row::2";
String b = "file::row::3";
String c = "file::row::4";
java.util.Map<String, AppearancePatentResultRowDto> rows = new java.util.LinkedHashMap<>();
rows.put(a, resultRow("标题A", null, null));
rows.put(b, resultRow(null, null, "抓取失败"));
rows.put(c, resultRow(null, "https://x/4.jpg", "部分字段缺失"));
assertEquals(keys(a, c), AppearancePatentResumeTrimSupport.collectableKeysOf(rows));
assertEquals(Set.of(), AppearancePatentResumeTrimSupport.collectableKeysOf(null));
}
}
@@ -0,0 +1,57 @@
package com.nanri.aiimage.modules.appearancepatent.service.support;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* B96-07 分片上报范围键(scope)的口径。
*
* <p>重排后派给客户端的是裁掉已收行的剩余组,客户端从 1 重新编号。若 scope 仍是
* {@code task:{id}},新分片会与上一代同序号的分片撞 (task_id, module_type, scope_hash,
* chunk_index),落进 {@code duplicate chunk ignored} 却照旧回 ok——客户端以为传成功,
* 服务端一行没收。这里钉住三件事:首代键不变(存量分片一个字节都不动)、跨代必须换键、
* 调用方显式 submissionId 仍优先。
*/
class AppearancePatentSubmissionScopeKeyTest {
private static FileTaskEntity task(long id, Integer attempt) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setResumeAttempt(attempt);
return task;
}
@Test
void 首代沿用原有的task键() {
assertEquals("task:33898", AppearancePatentChunkSubmissionSupport.resolveScopeKey(null, task(33898L, 0)));
assertEquals("task:33898", AppearancePatentChunkSubmissionSupport.resolveScopeKey(null, task(33898L, null)));
assertEquals("task:33898", AppearancePatentChunkSubmissionSupport.resolveScopeKey(" ", task(33898L, 0)));
}
@Test
void 每一代都必须换键_否则重采的分片被静默丢弃() {
assertEquals("task:33898:gen:1", AppearancePatentChunkSubmissionSupport.resolveScopeKey(null, task(33898L, 1)));
assertEquals("task:33898:gen:3", AppearancePatentChunkSubmissionSupport.resolveScopeKey(null, task(33898L, 3)));
}
@Test
void 调用方显式给的范围键在首代原样使用() {
assertEquals("client-scope-1",
AppearancePatentChunkSubmissionSupport.resolveScopeKey(" client-scope-1 ", task(33898L, 0)));
}
@Test
void 现网ap_worker每片都带submissionId_跨代也必须换键() {
// 生产实测:ap_worker.py:876 发 "submissionId": str(task_id)。只给缺省值加后缀等于没修。
assertEquals("33898:gen:1", AppearancePatentChunkSubmissionSupport.resolveScopeKey("33898", task(33898L, 1)));
assertEquals("33898:gen:3", AppearancePatentChunkSubmissionSupport.resolveScopeKey("33898", task(33898L, 3)));
}
@Test
void 已经很键幂等_不叠后缀() {
assertEquals("33898:gen:2",
AppearancePatentChunkSubmissionSupport.resolveScopeKey("33898:gen:2", task(33898L, 2)));
}
}
@@ -318,9 +318,9 @@ class SourceScanTest {
List<Path> files = SourceScan.moduleSources("modules/appearancepatent");
assertThat(files)
.as("真实集成:对真仓目录求值。57 支=2 controller+2 client+8 service 根件"
.as("真实集成:对真仓目录求值。59 支=2 controller+2 client+8 service 根件"
+ "(主件/只读查询/同步/出表作业/缓存/心跳与拉取与重试 SPI)"
+ "+22 service/support+22 model(dto7/entity1/vo14)+1 mapper,"
+ "+24 service/support+22 model(dto7/entity1/vo14)+1 mapper,"
+ "与 find 实测同号"
+ "(3A 拆薄第十三笔再落 5 支;失败任务重试再落 1 支 RetrySpi;"
+ "v1.3 Phase 1 无侵权同步底座再落 2 支:model/entity/CleanItemEntity+"
@@ -333,8 +333,11 @@ class SourceScanTest {
+ "2026-11-11 B95-02 整批失败文案分档再落 1 支 client:"
+ "AppearancePatentBrandFailureText;"
+ "2026-11-12 3A 拆薄第十四笔再落 1 支 support:"
+ "AppearancePatentDeliveredRearmSupport)")
.hasSize(57);
+ "AppearancePatentDeliveredRearmSupport;"
+ "2026-11-12 B96-07 派活前裁剪再落 2 支 support:"
+ "AppearancePatentResumeTrimSupport(纯裁剪)+"
+ "AppearancePatentDispatchTrimSupport(下发口编排))")
.hasSize(59);
assertThat(files.stream().map(p -> p.toString().replace('\\', '/')).toList())
.as("support 目录里的既有件必须在面里——聚合入口存在的理由就是拆分后的新文件要自动被看到")
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentRowNormalizer.java"));
@@ -428,16 +431,16 @@ class SourceScanTest {
}
@Test
void recursive_包含service_support下的既有22件() {
void recursive_包含service_support下的既有24件() {
List<String> support = SourceScan.moduleSources("modules/appearancepatent").stream()
.map(p -> p.toString().replace('\\', '/'))
.filter(rel -> rel.contains("/service/support/"))
.toList();
assertThat(support)
.as("真实集成:对真仓 modules/appearancepatent/service/support 求值,实测 22 支(3A 拆薄把归属谓词、陈旧扫描、结果打包、LLM 批管线、载荷读写、结果行读取、进度视图、分片回传、交付闸门、结果作业、建单解析、载荷合并、提交调度、陈旧收尾、解析分组、任务记录抽成十六枚 support 件,5→21;第十四笔再抽「继续」摘作业与清直链,21→22)。"
.as("真实集成:对真仓 modules/appearancepatent/service/support 求值,实测 24 支(3A 拆薄把归属谓词、陈旧扫描、结果打包、LLM 批管线、载荷读写、结果行读取、进度视图、分片回传、交付闸门、结果作业、建单解析、载荷合并、提交调度、陈旧收尾、解析分组、任务记录抽成十六枚 support 件,5→21;第十四笔再抽「继续」摘作业与清直链,21→22;2026-11-12 B96-07 再落派活前裁剪两件,22→24)。"
+ "这一格就是聚合入口存在的理由——拆分出去的新文件必须自动进面")
.hasSize(22);
.hasSize(24);
assertThat(support)
.as("3A 拆出去的新件必须被逐枚点名扫到:只把分母数字改大,下一回拆件又只剩「数不对」"
+ "一记红,证不了「扫得到」这件事")
@@ -457,7 +460,9 @@ class SourceScanTest {
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentStaleTaskFinalizer.java"))
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentGroupingSupport.java"))
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentTaskRecordSupport.java"))
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentDeliveredRearmSupport.java"));
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentDeliveredRearmSupport.java"))
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentResumeTrimSupport.java"))
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentDispatchTrimSupport.java"));
}
@Test