perf(外观专利): LLM 批循环按 payload_hash 复用分片行索引,不再每批重读对象存储
一次 submitLlmBatches 有 N 批,每批落库前都要知道「哪些行属于哪个分片」,此前这份索引 是每批现做:把该任务全部分片的载荷重读重解析一遍,读的量是 (批数+1)×分片数,全在 LLM 请求的串行路径上。改成收集候选那一遍就把行键记进一次性的索引,后续批次只对 payload_hash 变过的分片重读;载荷哈希与载荷内容在所有写入点都是同一条 UPDATE 落的,可作新鲜度锚。 落库路径一行未改:mergeChunkPayload 仍自己 selectOne + 重读最新载荷再做 CAS,索引只用于 「行属于哪一片」的匹配,不作为合并基底。 测试 AppearancePatentChunkRowIndexTest 三例,逐个做过变异验证:去掉哈希比较只咬住 「变过必须重读」,命中缓存返回空集只咬住「复用后行仍要匹配得上分片」; 全量 3413 例通过。
This commit is contained in:
+265
@@ -0,0 +1,265 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.service.ResultDownloadResolver;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 「没变过的分片,一次检测里只解析一遍」。
|
||||
*
|
||||
* <p>合批后一次 {@code submitLlmBatches} 要跑 N 批,而每批落库前都得先知道「这些行属于哪个分片」。
|
||||
* 这份索引此前是每批现做:把该任务**全部**分片的载荷从对象存储重读一遍、JSON 重解析一遍。
|
||||
* 线上一个 211 片的任务光索引就每批 211 次读,批数一多就是批数×分片数的平方级浪费,
|
||||
* 而这些读全都在 LLM 请求的串行路径上——检测阶段的墙钟时间有一半花在这里。
|
||||
*
|
||||
* <p>这里钉住三件事:载荷哈希没变过的分片不再解析(省下来的那部分)、
|
||||
* 变过的一定要从头解析(不然索引会指向已经不存在的行,跨节点并发回传时就会把结果写错片)、
|
||||
* 命中复用时行照常能匹配上自己的分片(省读省成返回空集合,就是"扣次数没结论")。
|
||||
*
|
||||
* <p>索引的作用域是单次调用(局部变量,随栈丢弃),这一条不做断言:收集候选那一遍本来就会
|
||||
* 重读并重记每个分片,就算有人把它做成字段也不会有可见行为差别,能咬住它的只有内存占用和并发。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class AppearancePatentChunkRowIndexTest {
|
||||
|
||||
private static final Long TASK_ID = 29501L;
|
||||
private static final Long JOB_ID = 20801L;
|
||||
private static final String SCOPE_A = "scope-a";
|
||||
private static final String SCOPE_B = "scope-b";
|
||||
private static final String SCOPE_C = "scope-c";
|
||||
private static final String POINTER_A = "ptr-chunk-a.json";
|
||||
private static final String POINTER_B = "ptr-chunk-b.json";
|
||||
private static final String POINTER_C = "ptr-chunk-c.json";
|
||||
private static final String HASH_A = "hash-a-1";
|
||||
private static final String HASH_B = "hash-b-1";
|
||||
private static final String HASH_C = "hash-c-1";
|
||||
/** 本批写入后分片 A 在库里的新哈希(模拟 mergeChunkPayload 的 CAS 生效)。 */
|
||||
private static final String HASH_A_WRITTEN = "hash-a-2";
|
||||
|
||||
/** 6 行未检测:batchSize=2 时正好 3 批,批数大于 1 才看得见「每批重读全部分片」。 */
|
||||
private static final String PENDING_ROWS = rows(6, "r", false);
|
||||
/** 已检测完的分片:不产生候选、也不会被本次调用写入,是「纯粹没变过」的那两片。 */
|
||||
private static final String DONE_ROWS_B = rows(3, "b", true);
|
||||
private static final String DONE_ROWS_C = rows(3, "c", true);
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private ResultDownloadResolver resultDownloadResolver;
|
||||
@Mock private StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private AppearancePatentLlmClient llmClient;
|
||||
@Mock private AppearancePatentTaskCacheService taskCacheService;
|
||||
@Mock private AppearancePatentProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private PlatformTransactionManager transactionManager;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
@Mock private UserApiSecretService userApiSecretService;
|
||||
|
||||
@InjectMocks private AppearancePatentTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(properties.getLlmBatchSize()).thenReturn(2);
|
||||
lenient().when(properties.getFlushPendingMinutes()).thenReturn(1);
|
||||
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
|
||||
lenient().when(userApiSecretService.findPlainValue(any(), anyString())).thenReturn("");
|
||||
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER_A), anyString())).thenReturn(PENDING_ROWS);
|
||||
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER_B), anyString())).thenReturn(DONE_ROWS_B);
|
||||
lenient().when(transientPayloadStorageService.resolvePayload(eq(POINTER_C), anyString())).thenReturn(DONE_ROWS_C);
|
||||
// 库里每批都重新查一遍:只有 A 因为刚落库换了哈希,B/C 一个字节都没动
|
||||
lenient().when(taskChunkMapper.selectList(any()))
|
||||
.thenReturn(List.of(chunk(SCOPE_A, POINTER_A, HASH_A_WRITTEN),
|
||||
chunk(SCOPE_B, POINTER_B, HASH_B),
|
||||
chunk(SCOPE_C, POINTER_C, HASH_C)));
|
||||
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(chunk(SCOPE_A, POINTER_A, HASH_A));
|
||||
lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||
anyString(), anyLong(), anyString(), any(), anyString())).thenReturn("sibling-1");
|
||||
lenient().when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
lenient().when(llmClient.inspectRows(any(), any(), any())).thenAnswer(invocation -> {
|
||||
List<AppearancePatentResultRowDto> batch = invocation.getArgument(0);
|
||||
for (AppearancePatentResultRowDto row : batch) {
|
||||
row.setTitleRisk("无侵权");
|
||||
row.setAppearanceRisk("无侵权");
|
||||
row.setConclusion("无侵权");
|
||||
}
|
||||
return new ArrayList<>(batch);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 一次调用 3 批,从头到尾没被改过的分片 C 只应该被解析一次(收集候选那一遍)。
|
||||
*
|
||||
* <p>钉住的是「省下来的那部分」:现在每批都要为 C 重读重解析一遍,3 批 + 收集 = 4 次。
|
||||
*/
|
||||
@Test
|
||||
void chunkThatNeverChangedIsParsedOncePerRun() {
|
||||
submit();
|
||||
|
||||
verify(transientPayloadStorageService, times(1)).resolvePayload(eq(POINTER_C), anyString());
|
||||
}
|
||||
|
||||
/**
|
||||
* 复用索引不能只省读、还得照常认得别的分片的行:第 4 批的结果照样要落进 B,而不是变成孤儿。
|
||||
*
|
||||
* <p>省读的实现有个致命的顺手写法——命中缓存时返回空集合或只返回「本批候选所在片」,
|
||||
* 表现是行匹配不上分片、被撤销标记反复重发,用户侧就是扣了次数没结论。
|
||||
*/
|
||||
@Test
|
||||
void reusedIndexStillMatchesRowsOfEveryChunk() {
|
||||
when(transientPayloadStorageService.resolvePayload(eq(POINTER_B), anyString()))
|
||||
.thenReturn(rows(2, "b", false));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(
|
||||
chunk(SCOPE_A, POINTER_A, HASH_A_WRITTEN),
|
||||
chunk(SCOPE_B, POINTER_B, HASH_B),
|
||||
chunk(SCOPE_C, POINTER_C, HASH_C)));
|
||||
|
||||
submit();
|
||||
|
||||
verify(taskCacheService).completeRowInspection(TASK_ID, List.of("r1", "r2"));
|
||||
verify(taskCacheService).completeRowInspection(TASK_ID, List.of("b1", "b2"));
|
||||
verify(taskCacheService, never()).releaseRowInspection(eq(TASK_ID), any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 载荷哈希一变就必须重新解析——省读不能省成「拿旧行集合匹配新载荷」。
|
||||
*
|
||||
* <p>收集候选用的是入口那份 chunk 快照(哈希还是 hash-c-1),批次里查库拿到的是别的节点
|
||||
* 回传之后的 hash-c-2。此时 C 的行集合已经不可信,必须重读。
|
||||
*/
|
||||
@Test
|
||||
void chunkWhosePayloadHashChangedIsParsedAgain() {
|
||||
ReflectionTestUtils.invokeMethod(service, "submitLlmBatches",
|
||||
task(), new FileResultEntity(), job(),
|
||||
List.of(chunk(SCOPE_A, POINTER_A, HASH_A), chunk(SCOPE_B, POINTER_B, HASH_B),
|
||||
chunk(SCOPE_C, POINTER_C, "hash-c-stale")),
|
||||
Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, times(2)).resolvePayload(eq(POINTER_C), anyString());
|
||||
}
|
||||
|
||||
// ===== 辅助 =====
|
||||
|
||||
private void submit() {
|
||||
ReflectionTestUtils.invokeMethod(service, "submitLlmBatches",
|
||||
task(), new FileResultEntity(), job(),
|
||||
List.of(chunk(SCOPE_A, POINTER_A, HASH_A), chunk(SCOPE_B, POINTER_B, HASH_B),
|
||||
chunk(SCOPE_C, POINTER_C, HASH_C)),
|
||||
Map.of());
|
||||
}
|
||||
|
||||
private FileTaskEntity task() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
|
||||
task.setUserId(7L);
|
||||
return task;
|
||||
}
|
||||
|
||||
private com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity job() {
|
||||
com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity job =
|
||||
new com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity();
|
||||
job.setId(JOB_ID);
|
||||
job.setTaskId(TASK_ID);
|
||||
return job;
|
||||
}
|
||||
|
||||
private TaskChunkEntity chunk(String scopeHash, String pointer, String payloadHash) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(11L);
|
||||
chunk.setTaskId(TASK_ID);
|
||||
chunk.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(1);
|
||||
chunk.setPayloadJson(pointer);
|
||||
chunk.setPayloadHash(payloadHash);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private static String rows(int count, String prefix, boolean resolved) {
|
||||
StringBuilder json = new StringBuilder("[");
|
||||
for (int i = 1; i <= count; i++) {
|
||||
if (i > 1) {
|
||||
json.append(',');
|
||||
}
|
||||
json.append("{\"rowToken\":\"").append(prefix).append(i)
|
||||
.append("\",\"asin\":\"B0A000000").append(i % 10)
|
||||
.append("\",\"title\":\"PRODUCT ").append(i)
|
||||
.append("\",\"url\":\"http://img/").append(i).append(".jpg\",")
|
||||
.append("\"groupKey\":\"").append(prefix).append("-g").append(i).append("\"");
|
||||
if (resolved) {
|
||||
json.append(",\"titleRisk\":\"无侵权\",\"appearanceRisk\":\"无侵权\",\"conclusion\":\"无侵权\"");
|
||||
}
|
||||
json.append('}');
|
||||
}
|
||||
return json.append(']').toString();
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -135,7 +135,7 @@ class AppearancePatentRowMarkTimingTest {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
|
||||
List<AppearancePatentResultRowDto> candidates = (List<AppearancePatentResultRowDto>) ReflectionTestUtils
|
||||
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()));
|
||||
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()), new AppearancePatentTaskService.ChunkRowKeyIndex());
|
||||
|
||||
assertEquals(2, candidates.size());
|
||||
verify(taskCacheService).beginRowInspection(TASK_ID, "r1");
|
||||
@@ -151,7 +151,7 @@ class AppearancePatentRowMarkTimingTest {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), eq("r2"))).thenReturn(false);
|
||||
|
||||
List<AppearancePatentResultRowDto> candidates = (List<AppearancePatentResultRowDto>) ReflectionTestUtils
|
||||
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()));
|
||||
.invokeMethod(service, "collectPendingLlmCandidates", List.of(chunk()), new AppearancePatentTaskService.ChunkRowKeyIndex());
|
||||
|
||||
assertEquals(1, candidates.size());
|
||||
assertEquals("r1", candidates.get(0).getRowToken());
|
||||
|
||||
Reference in New Issue
Block a user