perf(外观专利): LLM 批循环按 payload_hash 复用分片行索引,不再每批重读对象存储

一次 submitLlmBatches 有 N 批,每批落库前都要知道「哪些行属于哪个分片」,此前这份索引
是每批现做:把该任务全部分片的载荷重读重解析一遍,读的量是 (批数+1)×分片数,全在 LLM
请求的串行路径上。改成收集候选那一遍就把行键记进一次性的索引,后续批次只对 payload_hash
变过的分片重读;载荷哈希与载荷内容在所有写入点都是同一条 UPDATE 落的,可作新鲜度锚。

落库路径一行未改:mergeChunkPayload 仍自己 selectOne + 重读最新载荷再做 CAS,索引只用于
「行属于哪一片」的匹配,不作为合并基底。

测试 AppearancePatentChunkRowIndexTest 三例,逐个做过变异验证:去掉哈希比较只咬住
「变过必须重读」,命中缓存返回空集只咬住「复用后行仍要匹配得上分片」;
全量 3413 例通过。
This commit is contained in:
2026-10-02 10:35:23 +08:00
parent 87a306458c
commit b728bb3f4d
3 changed files with 321 additions and 21 deletions
@@ -0,0 +1,265 @@
package com.nanri.aiimage.modules.appearancepatent.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.common.service.ResultDownloadResolver;
import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils;
import org.springframework.transaction.PlatformTransactionManager;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 「没变过的分片,一次检测里只解析一遍」。
*
* <p>合批后一次 {@code submitLlmBatches} 要跑 N 批,而每批落库前都得先知道「这些行属于哪个分片」。
* 这份索引此前是每批现做:把该任务**全部**分片的载荷从对象存储重读一遍、JSON 重解析一遍。
* 线上一个 211 片的任务光索引就每批 211 次读,批数一多就是批数×分片数的平方级浪费,
* 而这些读全都在 LLM 请求的串行路径上——检测阶段的墙钟时间有一半花在这里。
*
* <p>这里钉住三件事:载荷哈希没变过的分片不再解析(省下来的那部分)、
* 变过的一定要从头解析(不然索引会指向已经不存在的行,跨节点并发回传时就会把结果写错片)、
* 命中复用时行照常能匹配上自己的分片(省读省成返回空集合,就是"扣次数没结论")。
*
* <p>索引的作用域是单次调用(局部变量,随栈丢弃),这一条不做断言:收集候选那一遍本来就会
* 重读并重记每个分片,就算有人把它做成字段也不会有可见行为差别,能咬住它的只有内存占用和并发。
*/
@ExtendWith(MockitoExtension.class)
class AppearancePatentChunkRowIndexTest {
private static final Long TASK_ID = 29501L;
private static final Long JOB_ID = 20801L;
private static final String SCOPE_A = "scope-a";
private static final String SCOPE_B = "scope-b";
private static final String SCOPE_C = "scope-c";
private static final String POINTER_A = "ptr-chunk-a.json";
private static final String POINTER_B = "ptr-chunk-b.json";
private static final String POINTER_C = "ptr-chunk-c.json";
private static final String HASH_A = "hash-a-1";
private static final String HASH_B = "hash-b-1";
private static final String HASH_C = "hash-c-1";
/** 本批写入后分片 A 在库里的新哈希(模拟 mergeChunkPayload 的 CAS 生效)。 */
private static final String HASH_A_WRITTEN = "hash-a-2";
/** 6 行未检测:batchSize=2 时正好 3 批,批数大于 1 才看得见「每批重读全部分片」。 */
private static final String PENDING_ROWS = rows(6, "r", false);
/** 已检测完的分片:不产生候选、也不会被本次调用写入,是「纯粹没变过」的那两片。 */
private static final String DONE_ROWS_B = rows(3, "b", true);
private static final String DONE_ROWS_C = rows(3, "c", true);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private OssStorageService ossStorageService;
@Mock private ResultDownloadResolver resultDownloadResolver;
@Mock private StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private AppearancePatentLlmClient llmClient;
@Mock private AppearancePatentTaskCacheService taskCacheService;
@Mock private AppearancePatentProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private PlatformTransactionManager transactionManager;
@Mock private DistributedJobLockService distributedJobLockService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private InstanceMetadata instanceMetadata;
@Mock private TaskProgressLightAssembler taskProgressLightAssembler;
@Mock private UserApiSecretService userApiSecretService;
@InjectMocks private AppearancePatentTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.getLlmBatchSize()).thenReturn(2);
lenient().when(properties.getFlushPendingMinutes()).thenReturn(1);
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
lenient().when(userApiSecretService.findPlainValue(any(), anyString())).thenReturn("");
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER_A), anyString())).thenReturn(PENDING_ROWS);
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER_B), anyString())).thenReturn(DONE_ROWS_B);
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER_C), anyString())).thenReturn(DONE_ROWS_C);
// 库里每批都重新查一遍:只有 A 因为刚落库换了哈希,B/C 一个字节都没动
lenient().when(taskChunkMapper.selectList(any()))
.thenReturn(List.of(chunk(SCOPE_A, POINTER_A, HASH_A_WRITTEN),
chunk(SCOPE_B, POINTER_B, HASH_B),
chunk(SCOPE_C, POINTER_C, HASH_C)));
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(chunk(SCOPE_A, POINTER_A, HASH_A));
lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1);
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
anyString(), anyLong(), anyString(), any(), anyString())).thenReturn("sibling-1");
lenient().when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
lenient().when(llmClient.inspectRows(any(), any(), any())).thenAnswer(invocation -> {
List<AppearancePatentResultRowDto> batch = invocation.getArgument(0);
for (AppearancePatentResultRowDto row : batch) {
row.setTitleRisk("无侵权");
row.setAppearanceRisk("无侵权");
row.setConclusion("无侵权");
}
return new ArrayList<>(batch);
});
}
/**
* 一次调用 3 批,从头到尾没被改过的分片 C 只应该被解析一次(收集候选那一遍)。
*
* <p>钉住的是「省下来的那部分」:现在每批都要为 C 重读重解析一遍,3 批 + 收集 = 4 次。
*/
@Test
void chunkThatNeverChangedIsParsedOncePerRun() {
submit();
verify(transientPayloadStorageService, times(1)).resolvePayload(eq(POINTER_C), anyString());
}
/**
* 复用索引不能只省读、还得照常认得别的分片的行:第 4 批的结果照样要落进 B,而不是变成孤儿。
*
* <p>省读的实现有个致命的顺手写法——命中缓存时返回空集合或只返回「本批候选所在片」,
* 表现是行匹配不上分片、被撤销标记反复重发,用户侧就是扣了次数没结论。
*/
@Test
void reusedIndexStillMatchesRowsOfEveryChunk() {
when(transientPayloadStorageService.resolvePayload(eq(POINTER_B), anyString()))
.thenReturn(rows(2, "b", false));
when(taskChunkMapper.selectList(any())).thenReturn(List.of(
chunk(SCOPE_A, POINTER_A, HASH_A_WRITTEN),
chunk(SCOPE_B, POINTER_B, HASH_B),
chunk(SCOPE_C, POINTER_C, HASH_C)));
submit();
verify(taskCacheService).completeRowInspection(TASK_ID, List.of("r1", "r2"));
verify(taskCacheService).completeRowInspection(TASK_ID, List.of("b1", "b2"));
verify(taskCacheService, never()).releaseRowInspection(eq(TASK_ID), any());
}
/**
* 载荷哈希一变就必须重新解析——省读不能省成「拿旧行集合匹配新载荷」。
*
* <p>收集候选用的是入口那份 chunk 快照(哈希还是 hash-c-1),批次里查库拿到的是别的节点
* 回传之后的 hash-c-2。此时 C 的行集合已经不可信,必须重读。
*/
@Test
void chunkWhosePayloadHashChangedIsParsedAgain() {
ReflectionTestUtils.invokeMethod(service, "submitLlmBatches",
task(), new FileResultEntity(), job(),
List.of(chunk(SCOPE_A, POINTER_A, HASH_A), chunk(SCOPE_B, POINTER_B, HASH_B),
chunk(SCOPE_C, POINTER_C, "hash-c-stale")),
Map.of());
verify(transientPayloadStorageService, times(2)).resolvePayload(eq(POINTER_C), anyString());
}
// ===== 辅助 =====
private void submit() {
ReflectionTestUtils.invokeMethod(service, "submitLlmBatches",
task(), new FileResultEntity(), job(),
List.of(chunk(SCOPE_A, POINTER_A, HASH_A), chunk(SCOPE_B, POINTER_B, HASH_B),
chunk(SCOPE_C, POINTER_C, HASH_C)),
Map.of());
}
private FileTaskEntity task() {
FileTaskEntity task = new FileTaskEntity();
task.setId(TASK_ID);
task.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
task.setUserId(7L);
return task;
}
private com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity job() {
com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity job =
new com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity();
job.setId(JOB_ID);
job.setTaskId(TASK_ID);
return job;
}
private TaskChunkEntity chunk(String scopeHash, String pointer, String payloadHash) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(11L);
chunk.setTaskId(TASK_ID);
chunk.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(1);
chunk.setPayloadJson(pointer);
chunk.setPayloadHash(payloadHash);
return chunk;
}
private static String rows(int count, String prefix, boolean resolved) {
StringBuilder json = new StringBuilder("[");
for (int i = 1; i <= count; i++) {
if (i > 1) {
json.append(',');
}
json.append("{\"rowToken\":\"").append(prefix).append(i)
.append("\",\"asin\":\"B0A000000").append(i % 10)
.append("\",\"title\":\"PRODUCT ").append(i)
.append("\",\"url\":\"http://img/").append(i).append(".jpg\",")
.append("\"groupKey\":\"").append(prefix).append("-g").append(i).append("\"");
if (resolved) {
json.append(",\"titleRisk\":\"无侵权\",\"appearanceRisk\":\"无侵权\",\"conclusion\":\"无侵权\"");
}
json.append('}');
}
return json.append(']').toString();
}
}
@@ -135,7 +135,7 @@ class AppearancePatentRowMarkTimingTest {
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
List<AppearancePatentResultRowDto> candidates = (List<AppearancePatentResultRowDto>) ReflectionTestUtils
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()));
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()), new AppearancePatentTaskService.ChunkRowKeyIndex());
assertEquals(2, candidates.size());
verify(taskCacheService).beginRowInspection(TASK_ID, "r1");
@@ -151,7 +151,7 @@ class AppearancePatentRowMarkTimingTest {
when(taskCacheService.beginRowInspection(eq(TASK_ID), eq("r2"))).thenReturn(false);
List<AppearancePatentResultRowDto> candidates = (List<AppearancePatentResultRowDto>) ReflectionTestUtils
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()));
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()), new AppearancePatentTaskService.ChunkRowKeyIndex());
assertEquals(1, candidates.size());
assertEquals("r1", candidates.get(0).getRowToken());