fix(appearance-patent): 重试不再整单重采——派活前裁掉已采到的行+跨代切分片 scope(B96-07)
33898/33883/33885 这类单重试一次就是 4000 组从头再抓一遍(1.3-1.5 小时+按次买出口 IP),根因两条:① 重排派给客户端的仍是解析载荷的全量 groups;② ap_worker 的 chunkIndex 是相对本载荷的序号且每片都带 submissionId=str(task_id),裁剪后从 1 重新编号会与上一代同号分片撞 (task_id, module_type, scope_hash, chunk_index),落进 duplicate chunk ignored 却照旧回 ok —— 客户端以为传成功、服务端一行没收。 A:AppearancePatentResumeTrimSupport(纯裁剪)/AppearancePatentDispatchTrimSupport(下发口编排)。两条下发口(兜底拉取、页面推本机)都过 trim;已收判据不是「行存在」而是「行上有标题或图片」(失败行也会落库,按行存在裁会把 404 行整批裁掉、坏行永不重采);载荷对象已不在的分片不算已收。跨代 scope=<caller submissionId>:gen:<resumeAttempt>(首代原样,幂等),新一代开跑前清上一代输入面 completed,否则本代第一个分片就被当成「已交完」拿半批行去组装(33924 同形)。 B:裁到一组不剩时不再派活——先标完成再收口出表;已出过表(result_file_url 有值)或状态非 PENDING/RUNNING 不动,避免直链被清空再重排。SPI 对「解析期无分组」与「已收全被裁光」分两侧:后者返回空 groups 载荷而不是 null,否则认领侧会把刚收口的任务改判「任务数据不完整」FAILED。 门禁:mvn -o test 整量 5,430 例/0 失败/19 skipped BUILD SUCCESS;新增用例 14 支(裁剪 9、scope 5);TaskLifecycleDebtRatchetTest 130 遍/15 文件不变(归属谓词改走 TaskFinalizeSupport.belongsToModule);SourceScanTest 台账 57→59、support 22→24;_push_gate.py --fast overall=pass(9.5/8.8/10.0/9.6)。
This commit is contained in:
+198
@@ -0,0 +1,198 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* B96-07 派活前裁剪的判据。
|
||||
*
|
||||
* <p>背景:外观专利的「重试/继续」今天是整单重采——上一代已传完的行会被再买一遍出口 IP。
|
||||
* 裁剪本身只是「按行键摘掉已收行」,但三件事必须钉住,任何一条错了都比不裁剪更贵:
|
||||
*
|
||||
* <ol>
|
||||
* <li><b>半代裁剪</b>——已收的摘、没收到的一条不许丢(丢了=用户表里缺行,且再也不会采);</li>
|
||||
* <li><b>空组丢弃</b>——全收的组整组去掉(Python 侧会跳过空组,留着只会让日志虚高);</li>
|
||||
* <li><b>入参不被改写</b>——解析载荷是共享对象,出表/同步侧还在读同一份。</li>
|
||||
* </ol>
|
||||
*/
|
||||
class AppearancePatentResumeTrimTest {
|
||||
|
||||
private static AppearancePatentParsedRowVo row(String token) {
|
||||
AppearancePatentParsedRowVo row = new AppearancePatentParsedRowVo();
|
||||
row.setRowToken(token);
|
||||
row.setDisplayId(token);
|
||||
return row;
|
||||
}
|
||||
|
||||
private static AppearancePatentParsedGroupVo group(String groupKey, AppearancePatentParsedRowVo... rows) {
|
||||
AppearancePatentParsedGroupVo group = new AppearancePatentParsedGroupVo();
|
||||
group.setGroupKey(groupKey);
|
||||
group.setBaseId(groupKey);
|
||||
group.setItemCount(rows.length);
|
||||
group.setItems(new ArrayList<>(List.of(rows)));
|
||||
return group;
|
||||
}
|
||||
|
||||
private static Set<String> keys(String... values) {
|
||||
return new LinkedHashSet<>(List.of(values));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 没有已收行时原样返回且不判全收() {
|
||||
List<AppearancePatentParsedGroupVo> groups = List.of(
|
||||
group("g1", row("a"), row("b")),
|
||||
group("g2", row("c")));
|
||||
|
||||
AppearancePatentResumeTrimSupport.TrimResult trim =
|
||||
AppearancePatentResumeTrimSupport.trim(groups, AppearancePatentParsedRowVo::getRowToken, Set.of());
|
||||
|
||||
assertEquals(3, trim.totalRows());
|
||||
assertEquals(0, trim.collectedRows());
|
||||
assertEquals(3, trim.remainingRows());
|
||||
assertEquals(2, trim.groups().size());
|
||||
assertEquals(0, trim.droppedGroups());
|
||||
assertFalse(trim.allRowsCollected());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 部分已收只摘已收的行_其余行与分组原样保留() {
|
||||
List<AppearancePatentParsedGroupVo> groups = List.of(
|
||||
group("g1", row("a"), row("b"), row("c")),
|
||||
group("g2", row("d"), row("e")));
|
||||
|
||||
AppearancePatentResumeTrimSupport.TrimResult trim =
|
||||
AppearancePatentResumeTrimSupport.trim(groups, AppearancePatentParsedRowVo::getRowToken,
|
||||
keys("a", "e"));
|
||||
|
||||
assertEquals(5, trim.totalRows());
|
||||
assertEquals(2, trim.collectedRows());
|
||||
assertEquals(3, trim.remainingRows());
|
||||
assertEquals(2, trim.groups().size());
|
||||
assertEquals(0, trim.droppedGroups());
|
||||
assertEquals(List.of("b", "c"), trim.groups().get(0).getItems().stream()
|
||||
.map(AppearancePatentParsedRowVo::getRowToken).toList());
|
||||
assertEquals(List.of("d"), trim.groups().get(1).getItems().stream()
|
||||
.map(AppearancePatentParsedRowVo::getRowToken).toList());
|
||||
// itemCount 跟裁剪后的实际行数走:界面/日志读的是它,留着旧值等于谎报进度。
|
||||
assertEquals(2, trim.groups().get(0).getItemCount());
|
||||
assertEquals(1, trim.groups().get(1).getItemCount());
|
||||
assertFalse(trim.allRowsCollected());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 整组已收时整组丢掉_全收才算全收() {
|
||||
List<AppearancePatentParsedGroupVo> groups = List.of(
|
||||
group("g1", row("a")),
|
||||
group("g2", row("b"), row("c")));
|
||||
|
||||
AppearancePatentResumeTrimSupport.TrimResult trim =
|
||||
AppearancePatentResumeTrimSupport.trim(groups, AppearancePatentParsedRowVo::getRowToken,
|
||||
keys("a", "b", "c"));
|
||||
|
||||
assertEquals(3, trim.totalRows());
|
||||
assertEquals(3, trim.collectedRows());
|
||||
assertEquals(0, trim.remainingRows());
|
||||
assertTrue(trim.groups().isEmpty());
|
||||
assertEquals(2, trim.droppedGroups());
|
||||
assertTrue(trim.allRowsCollected());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 只剩一组一行时仍然是可派活的载荷() {
|
||||
List<AppearancePatentParsedGroupVo> groups = List.of(
|
||||
group("g1", row("a")),
|
||||
group("g2", row("b")));
|
||||
|
||||
AppearancePatentResumeTrimSupport.TrimResult trim =
|
||||
AppearancePatentResumeTrimSupport.trim(groups, AppearancePatentParsedRowVo::getRowToken, keys("a"));
|
||||
|
||||
assertEquals(1, trim.groups().size());
|
||||
assertEquals("g2", trim.groups().get(0).getGroupKey());
|
||||
assertEquals(1, trim.droppedGroups());
|
||||
assertFalse(trim.allRowsCollected());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 不改写入参原对象() {
|
||||
AppearancePatentParsedGroupVo original = group("g1", row("a"), row("b"));
|
||||
List<AppearancePatentParsedGroupVo> groups = new ArrayList<>(List.of(original));
|
||||
|
||||
AppearancePatentResumeTrimSupport.TrimResult trim =
|
||||
AppearancePatentResumeTrimSupport.trim(groups, AppearancePatentParsedRowVo::getRowToken, keys("a"));
|
||||
|
||||
assertEquals(1, groups.size());
|
||||
assertSame(original, groups.get(0));
|
||||
assertEquals(2, original.getItems().size());
|
||||
assertEquals(2, original.getItemCount());
|
||||
assertEquals(1, trim.groups().size());
|
||||
assertEquals(1, trim.groups().get(0).getItems().size());
|
||||
assertFalse(trim.groups().get(0) == original);
|
||||
}
|
||||
|
||||
@Test
|
||||
void 空分组与空入参不炸() {
|
||||
assertEquals(0, AppearancePatentResumeTrimSupport.trim(List.of(),
|
||||
AppearancePatentParsedRowVo::getRowToken, keys("a")).totalRows());
|
||||
assertEquals(0, AppearancePatentResumeTrimSupport.trim(null,
|
||||
AppearancePatentParsedRowVo::getRowToken, keys("a")).totalRows());
|
||||
|
||||
AppearancePatentParsedGroupVo empty = new AppearancePatentParsedGroupVo();
|
||||
empty.setItems(new ArrayList<>());
|
||||
AppearancePatentResumeTrimSupport.TrimResult trim = AppearancePatentResumeTrimSupport.trim(
|
||||
List.of(empty), AppearancePatentParsedRowVo::getRowToken, keys("a"));
|
||||
assertEquals(1, trim.droppedGroups());
|
||||
assertFalse(trim.allRowsCollected());
|
||||
}
|
||||
|
||||
// ---- 「已收」的判据:行存在 != 行采到了 ---------------------------------------------
|
||||
// 分片里失败行也会落库(404/墙页都带一行 error)。若按「有这行」判已收,重试会把上一代
|
||||
// 失败的行整批裁掉,坏行永不重采——钱花了、表还是缺行,比不裁剪更糟。
|
||||
|
||||
private static AppearancePatentResultRowDto resultRow(String title, String url, String error) {
|
||||
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
|
||||
row.setTitle(title);
|
||||
row.setUrl(url);
|
||||
row.setError(error);
|
||||
return row;
|
||||
}
|
||||
|
||||
@Test
|
||||
void 采到标题或图片的行才算已收() {
|
||||
assertTrue(AppearancePatentResumeTrimSupport.countsAsCollected(resultRow("标题", null, null)));
|
||||
assertTrue(AppearancePatentResumeTrimSupport.countsAsCollected(resultRow(null, "https://x/1.jpg", null)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 失败行即使带错误原文也不算已收_必须重采() {
|
||||
assertFalse(AppearancePatentResumeTrimSupport.countsAsCollected(
|
||||
resultRow(null, null, "商品页 404")));
|
||||
assertFalse(AppearancePatentResumeTrimSupport.countsAsCollected(resultRow(null, null, null)));
|
||||
assertFalse(AppearancePatentResumeTrimSupport.countsAsCollected(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 键集只保留真采到的行() {
|
||||
String a = "file::row::2";
|
||||
String b = "file::row::3";
|
||||
String c = "file::row::4";
|
||||
java.util.Map<String, AppearancePatentResultRowDto> rows = new java.util.LinkedHashMap<>();
|
||||
rows.put(a, resultRow("标题A", null, null));
|
||||
rows.put(b, resultRow(null, null, "抓取失败"));
|
||||
rows.put(c, resultRow(null, "https://x/4.jpg", "部分字段缺失"));
|
||||
|
||||
assertEquals(keys(a, c), AppearancePatentResumeTrimSupport.collectableKeysOf(rows));
|
||||
assertEquals(Set.of(), AppearancePatentResumeTrimSupport.collectableKeysOf(null));
|
||||
}
|
||||
}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
/**
|
||||
* B96-07 分片上报范围键(scope)的口径。
|
||||
*
|
||||
* <p>重排后派给客户端的是裁掉已收行的剩余组,客户端从 1 重新编号。若 scope 仍是
|
||||
* {@code task:{id}},新分片会与上一代同序号的分片撞 (task_id, module_type, scope_hash,
|
||||
* chunk_index),落进 {@code duplicate chunk ignored} 却照旧回 ok——客户端以为传成功,
|
||||
* 服务端一行没收。这里钉住三件事:首代键不变(存量分片一个字节都不动)、跨代必须换键、
|
||||
* 调用方显式 submissionId 仍优先。
|
||||
*/
|
||||
class AppearancePatentSubmissionScopeKeyTest {
|
||||
|
||||
private static FileTaskEntity task(long id, Integer attempt) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(id);
|
||||
task.setResumeAttempt(attempt);
|
||||
return task;
|
||||
}
|
||||
|
||||
@Test
|
||||
void 首代沿用原有的task键() {
|
||||
assertEquals("task:33898", AppearancePatentChunkSubmissionSupport.resolveScopeKey(null, task(33898L, 0)));
|
||||
assertEquals("task:33898", AppearancePatentChunkSubmissionSupport.resolveScopeKey(null, task(33898L, null)));
|
||||
assertEquals("task:33898", AppearancePatentChunkSubmissionSupport.resolveScopeKey(" ", task(33898L, 0)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 每一代都必须换键_否则重采的分片被静默丢弃() {
|
||||
assertEquals("task:33898:gen:1", AppearancePatentChunkSubmissionSupport.resolveScopeKey(null, task(33898L, 1)));
|
||||
assertEquals("task:33898:gen:3", AppearancePatentChunkSubmissionSupport.resolveScopeKey(null, task(33898L, 3)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 调用方显式给的范围键在首代原样使用() {
|
||||
assertEquals("client-scope-1",
|
||||
AppearancePatentChunkSubmissionSupport.resolveScopeKey(" client-scope-1 ", task(33898L, 0)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 现网ap_worker每片都带submissionId_跨代也必须换键() {
|
||||
// 生产实测:ap_worker.py:876 发 "submissionId": str(task_id)。只给缺省值加后缀等于没修。
|
||||
assertEquals("33898:gen:1", AppearancePatentChunkSubmissionSupport.resolveScopeKey("33898", task(33898L, 1)));
|
||||
assertEquals("33898:gen:3", AppearancePatentChunkSubmissionSupport.resolveScopeKey("33898", task(33898L, 3)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 已经很键幂等_不叠后缀() {
|
||||
assertEquals("33898:gen:2",
|
||||
AppearancePatentChunkSubmissionSupport.resolveScopeKey("33898:gen:2", task(33898L, 2)));
|
||||
}
|
||||
}
|
||||
@@ -318,9 +318,9 @@ class SourceScanTest {
|
||||
List<Path> files = SourceScan.moduleSources("modules/appearancepatent");
|
||||
|
||||
assertThat(files)
|
||||
.as("真实集成:对真仓目录求值。57 支=2 controller+2 client+8 service 根件"
|
||||
.as("真实集成:对真仓目录求值。59 支=2 controller+2 client+8 service 根件"
|
||||
+ "(主件/只读查询/同步/出表作业/缓存/心跳与拉取与重试 SPI)"
|
||||
+ "+22 service/support+22 model(dto7/entity1/vo14)+1 mapper,"
|
||||
+ "+24 service/support+22 model(dto7/entity1/vo14)+1 mapper,"
|
||||
+ "与 find 实测同号"
|
||||
+ "(3A 拆薄第十三笔再落 5 支;失败任务重试再落 1 支 RetrySpi;"
|
||||
+ "v1.3 Phase 1 无侵权同步底座再落 2 支:model/entity/CleanItemEntity+"
|
||||
@@ -333,8 +333,11 @@ class SourceScanTest {
|
||||
+ "2026-11-11 B95-02 整批失败文案分档再落 1 支 client:"
|
||||
+ "AppearancePatentBrandFailureText;"
|
||||
+ "2026-11-12 3A 拆薄第十四笔再落 1 支 support:"
|
||||
+ "AppearancePatentDeliveredRearmSupport)")
|
||||
.hasSize(57);
|
||||
+ "AppearancePatentDeliveredRearmSupport;"
|
||||
+ "2026-11-12 B96-07 派活前裁剪再落 2 支 support:"
|
||||
+ "AppearancePatentResumeTrimSupport(纯裁剪)+"
|
||||
+ "AppearancePatentDispatchTrimSupport(下发口编排))")
|
||||
.hasSize(59);
|
||||
assertThat(files.stream().map(p -> p.toString().replace('\\', '/')).toList())
|
||||
.as("support 目录里的既有件必须在面里——聚合入口存在的理由就是拆分后的新文件要自动被看到")
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentRowNormalizer.java"));
|
||||
@@ -428,16 +431,16 @@ class SourceScanTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void recursive_包含service_support下的既有22件() {
|
||||
void recursive_包含service_support下的既有24件() {
|
||||
List<String> support = SourceScan.moduleSources("modules/appearancepatent").stream()
|
||||
.map(p -> p.toString().replace('\\', '/'))
|
||||
.filter(rel -> rel.contains("/service/support/"))
|
||||
.toList();
|
||||
|
||||
assertThat(support)
|
||||
.as("真实集成:对真仓 modules/appearancepatent/service/support 求值,实测 22 支(3A 拆薄把归属谓词、陈旧扫描、结果打包、LLM 批管线、载荷读写、结果行读取、进度视图、分片回传、交付闸门、结果作业、建单解析、载荷合并、提交调度、陈旧收尾、解析分组、任务记录抽成十六枚 support 件,5→21;第十四笔再抽「继续」摘作业与清直链,21→22)。"
|
||||
.as("真实集成:对真仓 modules/appearancepatent/service/support 求值,实测 24 支(3A 拆薄把归属谓词、陈旧扫描、结果打包、LLM 批管线、载荷读写、结果行读取、进度视图、分片回传、交付闸门、结果作业、建单解析、载荷合并、提交调度、陈旧收尾、解析分组、任务记录抽成十六枚 support 件,5→21;第十四笔再抽「继续」摘作业与清直链,21→22;2026-11-12 B96-07 再落派活前裁剪两件,22→24)。"
|
||||
+ "这一格就是聚合入口存在的理由——拆分出去的新文件必须自动进面")
|
||||
.hasSize(22);
|
||||
.hasSize(24);
|
||||
assertThat(support)
|
||||
.as("3A 拆出去的新件必须被逐枚点名扫到:只把分母数字改大,下一回拆件又只剩「数不对」"
|
||||
+ "一记红,证不了「扫得到」这件事")
|
||||
@@ -457,7 +460,9 @@ class SourceScanTest {
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentStaleTaskFinalizer.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentGroupingSupport.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentTaskRecordSupport.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentDeliveredRearmSupport.java"));
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentDeliveredRearmSupport.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentResumeTrimSupport.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentDispatchTrimSupport.java"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Reference in New Issue
Block a user