perf(货源): 分片载荷判空与建行图共用一次 GET,merge 与候选收集各省一遍对象存储读取

This commit is contained in:
2026-10-02 13:13:13 +08:00
parent 8812a3b64a
commit a83f7f51bd
3 changed files with 291 additions and 26 deletions
@@ -0,0 +1,228 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPipelineSupport;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.transaction.PlatformTransactionManager;
import java.lang.reflect.Method;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 货源链分片载荷的「一次读用两处」用例(外观 91d85ec6 的同一形态在货源侧的对应改动)。
* <p>
* 改前两处各重读一遍对象存储:
* <ul>
* <li>{@code mergeLlmRowsIntoChunk}:{@code loadSubmittedChunks} 逐片 GET 只为判空,
* 紧接着建行图又逐片 GET;</li>
* <li>{@code submitLlmForSubmittedChunk}:逐片 GET 过滤空片后,{@code collectPendingLlmCandidates}
* 再逐片 GET。</li>
* </ul>
* 用例按 GET 次数记账,并要求合并写回的行数与落库次数不变——省的是读,不是写。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceChunkReadReuseTest {
private static final String MODULE = "SIMILAR_ASIN";
private static final AtomicLong NEXT_ID = new AtomicLong(73000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private OssStorageService ossStorageService;
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(500);
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(4L * 1024L * 1024L);
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1);
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private static SimilarAsinResultRowDto row(String rowToken, String asin) {
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
r.setRowToken(rowToken);
r.setId(rowToken);
r.setAsin(asin);
r.setCountry("英国");
r.setTitle("Title " + rowToken);
r.setMainUrl("https://img.example.com/" + asin + ".jpg");
return r;
}
private static TaskChunkEntity chunk(long id, String scopeHash, int chunkIndex, String pointer, String payloadHash) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(id);
chunk.setTaskId(7304L);
chunk.setModuleType(MODULE);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setPayloadJson(pointer);
chunk.setPayloadHash(payloadHash);
return chunk;
}
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
return new ObjectMapper().writeValueAsString(rows);
}
private FileTaskEntity task() {
FileTaskEntity task = new FileTaskEntity();
task.setId(7304L);
task.setModuleType(MODULE);
task.setStatus("RUNNING");
return task;
}
/** 每个指针固定返回一份载荷,让 GET 次数可以逐片对账。 */
private void stubPayloads(Map<String, String> byPointer) {
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
.thenAnswer(invocation -> byPointer.getOrDefault(invocation.getArgument(0), "[]"));
}
@Test
void mergeChunkRows_readsEachChunkOnceInsteadOfTwice() throws Exception {
// 3 片各 1 行,回传 1 行命中第 1 片。
// 记账:建分片行图 3 次 GET + 命中片 CAS 写回前重读 1 次 = 4;改前是 3(判空探测)+3(建行图)+1=7。
List<TaskChunkEntity> chunks = List.of(
chunk(1L, "scope-1", 1, "ptr:chunk-1", "h1"),
chunk(2L, "scope-2", 2, "ptr:chunk-2", "h2"),
chunk(3L, "scope-3", 3, "ptr:chunk-3", "h3"));
stubPayloads(Map.of(
"ptr:chunk-1", rowsJson(List.of(row("r1", "B0A0000001"))),
"ptr:chunk-2", rowsJson(List.of(row("r2", "B0A0000002"))),
"ptr:chunk-3", rowsJson(List.of(row("r3", "B0A0000003")))));
when(taskChunkMapper.selectList(any())).thenReturn(chunks);
// 写回路径自己 selectOne 最新片:只命中第 1 片一次。
when(taskChunkMapper.selectOne(any())).thenReturn(chunks.get(0));
Method merge = SimilarAsinPipelineSupport.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
merge.invoke(service.pipelineSupport(), task(), null, null,
List.of(row("r1", "B0A0000001")), Map.of());
verify(transientPayloadStorageService, times(4)).resolvePayload(anyString(), anyString());
// 省的是读,写一次没少:只有命中片被改写一次。
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(
anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, times(1)).update(any(), any());
}
@Test
void emptyFilterAndCandidateCollection_shareOneReadPerChunk() {
// 3 片、其中 1 片载荷为空:判空过滤与后续候选收集必须共用同一次 GET。
List<TaskChunkEntity> chunks = List.of(
chunk(1L, "scope-1", 1, "ptr:chunk-1", "h1"),
chunk(2L, "scope-2", 2, "ptr:chunk-empty", "h2"),
chunk(3L, "scope-3", 3, "ptr:chunk-3", "h3"));
stubPayloads(Map.of(
"ptr:chunk-1", "[{\"rowToken\":\"r1\",\"id\":\"r1\",\"asin\":\"B0A0000001\",\"country\":\"英国\"}]",
"ptr:chunk-3", "[{\"rowToken\":\"r3\",\"id\":\"r3\",\"asin\":\"B0A0000003\",\"country\":\"英国\"}]"));
List<SimilarAsinPipelineSupport.SubmittedChunk> submitted =
service.pipelineSupport().readSubmittedChunks(chunks);
verify(transientPayloadStorageService, times(3)).resolvePayload(anyString(), anyString());
assertEquals(List.of("scope-1:1", "scope-3:3"), submitted.stream()
.map(entry -> entry.chunk().getScopeHash() + ":" + entry.chunk().getChunkIndex())
.toList(), "空片被跳过,非空片按入参顺序保留");
// 候选收集不再重读:GET 次数仍为 3。
var candidates = service.pipelineSupport().collectPendingLlmCandidates(submitted, Map.of());
verify(transientPayloadStorageService, times(3)).resolvePayload(anyString(), anyString());
assertEquals(2, candidates.size(), "两片各 1 行未检测 → 2 个候选,一行不因省读而丢");
}
@Test
void chunkReadResults_areNotReusedAcrossInvocations() {
// 索引只在一次调用内共享:同一批分片连着读两轮就是两轮 GET,
// 否则跨任务/跨轮次的旧载荷会悄悄决定"哪些行已经检测过"。
List<TaskChunkEntity> chunks = List.of(chunk(1L, "scope-1", 1, "ptr:chunk-1", "h1"));
stubPayloads(Map.of("ptr:chunk-1",
"[{\"rowToken\":\"r1\",\"id\":\"r1\",\"asin\":\"B0A0000001\",\"country\":\"英国\"}]"));
List<SimilarAsinPipelineSupport.SubmittedChunk> first = service.pipelineSupport().readSubmittedChunks(chunks);
List<SimilarAsinPipelineSupport.SubmittedChunk> second = service.pipelineSupport().readSubmittedChunks(chunks);
verify(transientPayloadStorageService, times(2)).resolvePayload(anyString(), anyString());
assertEquals(1, first.size());
assertEquals(1, second.size());
assertTrue(first.get(0).rows() != second.get(0).rows(), "两轮各拿自己那份载荷,不共享可变引用");
}
}
@@ -170,7 +170,8 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
@Test
void test_task_010_chunk_row_key_normal_default_path() throws Exception {
// 正常输入:2 个 chunk 各含行,llm 回传行按 rowKey 命中各自 chunk;
// 每个 chunk 的 payload 只被读取一次(索引建立),消除跨 chunk 线性扫描。
// 建索引每片一次 GET(判空与建行图共用同一次读,见 ChunkReadReuseTest),
// 外加命中片 CAS 写回前的重读 → 2+2=4 次。
List<TaskChunkEntity> chunks = List.of(
chunk(1L, "hashA", 1, "ptr:chunk-A"),
chunk(2L, "hashB", 2, "ptr:chunk-B"));
@@ -197,7 +198,7 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
merge.setAccessible(true);
merge.invoke(service.pipelineSupport(), task, null, null, llmRows, Map.of());
verify(transientPayloadStorageService, times(6)).resolvePayload(anyString(), anyString());
verify(transientPayloadStorageService, times(4)).resolvePayload(anyString(), anyString());
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, times(2)).update(any(), any());
}