perf(货源): 分片载荷判空与建行图共用一次 GET,merge 与候选收集各省一遍对象存储读取

This commit is contained in:
2026-10-02 13:13:13 +08:00
parent 8812a3b64a
commit a83f7f51bd
3 changed files with 291 additions and 26 deletions
@@ -157,8 +157,10 @@ public class SimilarAsinPipelineSupport {
if (task == null || llmRows == null || llmRows.isEmpty()) {
return;
}
List<TaskChunkEntity> chunks = loadSubmittedChunks(task.getId());
if (chunks.isEmpty()) {
// 判空与建行图共用同一次载荷读取:改前 loadSubmittedChunks 逐片 GET 只为回答"这片有没有行",
// 紧接着下面又逐片 GET 一次。
List<SubmittedChunk> submitted = readSubmittedChunks(loadSubmittedChunks(task.getId()));
if (submitted.isEmpty()) {
return;
}
// P2-11:把当前 batch 命中的图片 url 异步丢入预热队列。
@@ -180,9 +182,10 @@ public class SimilarAsinPipelineSupport {
}
Map<String, Map<String, SimilarAsinResultRowDto>> rowsByChunk = new LinkedHashMap<>();
Map<String, TaskChunkEntity> chunkByKey = new LinkedHashMap<>();
for (TaskChunkEntity chunk : chunks) {
for (SubmittedChunk entry : submitted) {
TaskChunkEntity chunk = entry.chunk();
String chunkKey = SimilarAsinChunkMergeSupport.chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex());
rowsByChunk.put(chunkKey, chunkPayloadSupport.readChunkRows(chunk));
rowsByChunk.put(chunkKey, entry.rows());
chunkByKey.put(chunkKey, chunk);
}
// Task 11:合并前按稳定 rowKey 去重,消除重复行逐行 expand/分配/写回 的 O(n²) 热点。
@@ -396,10 +399,39 @@ public class SimilarAsinPipelineSupport {
if (chunks == null || chunks.isEmpty()) {
return List.of();
}
return chunks.stream()
.filter(Objects::nonNull)
.filter(chunk -> !chunkPayloadSupport.readChunkRows(chunk).isEmpty())
.toList();
return chunks;
}
/**
* 一次作业轮次内共享的分片载荷读取结果:{@code chunk} 是库里的分片行,{@code rows} 是它当前载荷里的行。
* 只在单次调用里传递,不做字段级缓存。
*/
public record SubmittedChunk(TaskChunkEntity chunk, Map<String, SimilarAsinResultRowDto> rows) {
}
/**
* 逐片读一次载荷、跳过空片,顺序与入参一致。
* <p>
* 「这片有没有行」和「这片里有哪些行」原本是两次对象存储 GET + 整片 JSON 解析(分片数 C 就是 2C 次),
* 这里合并成一次;写回路径 {@link #mergeChunkPayload} 仍自己重读最新载荷再 CAS,不参与复用。
*/
public List<SubmittedChunk> readSubmittedChunks(List<TaskChunkEntity> chunks) {
if (chunks == null || chunks.isEmpty()) {
return List.of();
}
List<SubmittedChunk> submitted = new ArrayList<>(chunks.size());
for (TaskChunkEntity chunk : chunks) {
if (chunk == null) {
continue;
}
Map<String, SimilarAsinResultRowDto> rows = chunkPayloadSupport.readChunkRows(chunk);
if (rows.isEmpty()) {
continue;
}
submitted.add(new SubmittedChunk(chunk, rows));
}
return submitted;
}
@@ -827,11 +859,9 @@ public class SimilarAsinPipelineSupport {
.last("limit 1"));
chunks = chunk == null ? List.of() : List.of(chunk);
}
chunks = chunks == null ? List.of() : chunks.stream()
.filter(Objects::nonNull)
.filter(chunk -> !chunkPayloadSupport.readChunkRows(chunk).isEmpty())
.toList();
if (chunks.isEmpty()) {
// 分片载荷只读这一次:改前这里逐片 GET 只为过滤空片,下面 collectPendingLlmCandidates 又逐片 GET 一次。
List<SubmittedChunk> submitted = readSubmittedChunks(chunks);
if (submitted.isEmpty()) {
return;
}
FileResultEntity result = host.findOrCreateResultRecordForAssembly(task, payloadSupport.allRowCount(task));
@@ -844,7 +874,7 @@ public class SimilarAsinPipelineSupport {
return;
}
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
boolean pendingLlm = submitLlmBatches(task, result, job, chunks, allRowsByBaseId);
boolean pendingLlm = submitLlmBatchesForSubmittedChunks(task, result, job, submitted, allRowsByBaseId);
if (pendingLlm) {
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
touchJavaSideTaskActivity(task.getId());
@@ -914,19 +944,16 @@ public class SimilarAsinPipelineSupport {
}
public List<LlmCandidate> collectPendingLlmCandidates(List<TaskChunkEntity> chunks,
public List<LlmCandidate> collectPendingLlmCandidates(List<SubmittedChunk> submitted,
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
if (chunks == null || chunks.isEmpty()) {
if (submitted == null || submitted.isEmpty()) {
return List.of();
}
List<LlmCandidate> candidates = new ArrayList<>();
for (TaskChunkEntity chunk : chunks) {
Map<String, SimilarAsinResultRowDto> persistedRows = chunkPayloadSupport.readChunkRows(chunk);
if (persistedRows.isEmpty()) {
continue;
}
for (SubmittedChunk entry : submitted) {
TaskChunkEntity chunk = entry.chunk();
List<SimilarAsinResultRowDto> unresolvedRows =
enrichRowsForLlm(SimilarAsinResultTextSupport.collectPendingLlmRows(persistedRows.values()), allRowsByBaseId);
enrichRowsForLlm(SimilarAsinResultTextSupport.collectPendingLlmRows(entry.rows().values()), allRowsByBaseId);
for (SimilarAsinResultRowDto row : unresolvedRows) {
candidates.add(new LlmCandidate(chunk.getScopeHash(), chunk.getChunkIndex(), row));
}
@@ -1012,7 +1039,16 @@ public class SimilarAsinPipelineSupport {
TaskFileJobEntity job,
List<TaskChunkEntity> chunks,
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
if (chunks == null || chunks.isEmpty()) {
return submitLlmBatchesForSubmittedChunks(task, result, job, readSubmittedChunks(chunks), allRowsByBaseId);
}
public boolean submitLlmBatchesForSubmittedChunks(FileTaskEntity task,
FileResultEntity result,
TaskFileJobEntity job,
List<SubmittedChunk> submitted,
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
if (submitted == null || submitted.isEmpty()) {
return countPendingLlmStates(task.getId()) > 0;
}
String prompt = payloadSupport.readAiPrompt(task);
@@ -1027,7 +1063,7 @@ public class SimilarAsinPipelineSupport {
task.getId(), batchSize);
batchSize = 1;
}
List<LlmCandidate> candidates = collectPendingLlmCandidates(chunks, allRowsByBaseId);
List<LlmCandidate> candidates = collectPendingLlmCandidates(submitted, allRowsByBaseId);
if (candidates.isEmpty()) {
return countPendingLlmStates(task.getId()) > 0;
}
@@ -0,0 +1,228 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPipelineSupport;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.transaction.PlatformTransactionManager;
import java.lang.reflect.Method;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 货源链分片载荷的「一次读用两处」用例(外观 91d85ec6 的同一形态在货源侧的对应改动)。
* <p>
* 改前两处各重读一遍对象存储:
* <ul>
* <li>{@code mergeLlmRowsIntoChunk}:{@code loadSubmittedChunks} 逐片 GET 只为判空,
* 紧接着建行图又逐片 GET;</li>
* <li>{@code submitLlmForSubmittedChunk}:逐片 GET 过滤空片后,{@code collectPendingLlmCandidates}
* 再逐片 GET。</li>
* </ul>
* 用例按 GET 次数记账,并要求合并写回的行数与落库次数不变——省的是读,不是写。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceChunkReadReuseTest {
private static final String MODULE = "SIMILAR_ASIN";
private static final AtomicLong NEXT_ID = new AtomicLong(73000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private OssStorageService ossStorageService;
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(500);
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(4L * 1024L * 1024L);
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1);
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private static SimilarAsinResultRowDto row(String rowToken, String asin) {
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
r.setRowToken(rowToken);
r.setId(rowToken);
r.setAsin(asin);
r.setCountry("英国");
r.setTitle("Title " + rowToken);
r.setMainUrl("https://img.example.com/" + asin + ".jpg");
return r;
}
private static TaskChunkEntity chunk(long id, String scopeHash, int chunkIndex, String pointer, String payloadHash) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(id);
chunk.setTaskId(7304L);
chunk.setModuleType(MODULE);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setPayloadJson(pointer);
chunk.setPayloadHash(payloadHash);
return chunk;
}
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
return new ObjectMapper().writeValueAsString(rows);
}
private FileTaskEntity task() {
FileTaskEntity task = new FileTaskEntity();
task.setId(7304L);
task.setModuleType(MODULE);
task.setStatus("RUNNING");
return task;
}
/** 每个指针固定返回一份载荷,让 GET 次数可以逐片对账。 */
private void stubPayloads(Map<String, String> byPointer) {
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
.thenAnswer(invocation -> byPointer.getOrDefault(invocation.getArgument(0), "[]"));
}
@Test
void mergeChunkRows_readsEachChunkOnceInsteadOfTwice() throws Exception {
// 3 片各 1 行,回传 1 行命中第 1 片。
// 记账:建分片行图 3 次 GET + 命中片 CAS 写回前重读 1 次 = 4;改前是 3(判空探测)+3(建行图)+1=7。
List<TaskChunkEntity> chunks = List.of(
chunk(1L, "scope-1", 1, "ptr:chunk-1", "h1"),
chunk(2L, "scope-2", 2, "ptr:chunk-2", "h2"),
chunk(3L, "scope-3", 3, "ptr:chunk-3", "h3"));
stubPayloads(Map.of(
"ptr:chunk-1", rowsJson(List.of(row("r1", "B0A0000001"))),
"ptr:chunk-2", rowsJson(List.of(row("r2", "B0A0000002"))),
"ptr:chunk-3", rowsJson(List.of(row("r3", "B0A0000003")))));
when(taskChunkMapper.selectList(any())).thenReturn(chunks);
// 写回路径自己 selectOne 最新片:只命中第 1 片一次。
when(taskChunkMapper.selectOne(any())).thenReturn(chunks.get(0));
Method merge = SimilarAsinPipelineSupport.class.getDeclaredMethod("mergeLlmRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
merge.invoke(service.pipelineSupport(), task(), null, null,
List.of(row("r1", "B0A0000001")), Map.of());
verify(transientPayloadStorageService, times(4)).resolvePayload(anyString(), anyString());
// 省的是读,写一次没少:只有命中片被改写一次。
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(
anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, times(1)).update(any(), any());
}
@Test
void emptyFilterAndCandidateCollection_shareOneReadPerChunk() {
// 3 片、其中 1 片载荷为空:判空过滤与后续候选收集必须共用同一次 GET。
List<TaskChunkEntity> chunks = List.of(
chunk(1L, "scope-1", 1, "ptr:chunk-1", "h1"),
chunk(2L, "scope-2", 2, "ptr:chunk-empty", "h2"),
chunk(3L, "scope-3", 3, "ptr:chunk-3", "h3"));
stubPayloads(Map.of(
"ptr:chunk-1", "[{\"rowToken\":\"r1\",\"id\":\"r1\",\"asin\":\"B0A0000001\",\"country\":\"英国\"}]",
"ptr:chunk-3", "[{\"rowToken\":\"r3\",\"id\":\"r3\",\"asin\":\"B0A0000003\",\"country\":\"英国\"}]"));
List<SimilarAsinPipelineSupport.SubmittedChunk> submitted =
service.pipelineSupport().readSubmittedChunks(chunks);
verify(transientPayloadStorageService, times(3)).resolvePayload(anyString(), anyString());
assertEquals(List.of("scope-1:1", "scope-3:3"), submitted.stream()
.map(entry -> entry.chunk().getScopeHash() + ":" + entry.chunk().getChunkIndex())
.toList(), "空片被跳过,非空片按入参顺序保留");
// 候选收集不再重读:GET 次数仍为 3。
var candidates = service.pipelineSupport().collectPendingLlmCandidates(submitted, Map.of());
verify(transientPayloadStorageService, times(3)).resolvePayload(anyString(), anyString());
assertEquals(2, candidates.size(), "两片各 1 行未检测 → 2 个候选,一行不因省读而丢");
}
@Test
void chunkReadResults_areNotReusedAcrossInvocations() {
// 索引只在一次调用内共享:同一批分片连着读两轮就是两轮 GET,
// 否则跨任务/跨轮次的旧载荷会悄悄决定"哪些行已经检测过"。
List<TaskChunkEntity> chunks = List.of(chunk(1L, "scope-1", 1, "ptr:chunk-1", "h1"));
stubPayloads(Map.of("ptr:chunk-1",
"[{\"rowToken\":\"r1\",\"id\":\"r1\",\"asin\":\"B0A0000001\",\"country\":\"英国\"}]"));
List<SimilarAsinPipelineSupport.SubmittedChunk> first = service.pipelineSupport().readSubmittedChunks(chunks);
List<SimilarAsinPipelineSupport.SubmittedChunk> second = service.pipelineSupport().readSubmittedChunks(chunks);
verify(transientPayloadStorageService, times(2)).resolvePayload(anyString(), anyString());
assertEquals(1, first.size());
assertEquals(1, second.size());
assertTrue(first.get(0).rows() != second.get(0).rows(), "两轮各拿自己那份载荷,不共享可变引用");
}
}
@@ -170,7 +170,8 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
@Test
void test_task_010_chunk_row_key_normal_default_path() throws Exception {
// 正常输入:2 个 chunk 各含行,llm 回传行按 rowKey 命中各自 chunk;
// 每个 chunk 的 payload 只被读取一次(索引建立),消除跨 chunk 线性扫描。
// 建索引每片一次 GET(判空与建行图共用同一次读,见 ChunkReadReuseTest),
// 外加命中片 CAS 写回前的重读 → 2+2=4 次。
List<TaskChunkEntity> chunks = List.of(
chunk(1L, "hashA", 1, "ptr:chunk-A"),
chunk(2L, "hashB", 2, "ptr:chunk-B"));
@@ -197,7 +198,7 @@ class SimilarAsinTaskServiceRowKeyIndexTest {
merge.setAccessible(true);
merge.invoke(service.pipelineSupport(), task, null, null, llmRows, Map.of());
verify(transientPayloadStorageService, times(6)).resolvePayload(anyString(), anyString());
verify(transientPayloadStorageService, times(4)).resolvePayload(anyString(), anyString());
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, times(2)).update(any(), any());
}