From a83f7f51bdd81d44b8118c981e08043e058767da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Fri, 2 Oct 2026 13:13:13 +0800 Subject: [PATCH] =?UTF-8?q?perf(=E8=B4=A7=E6=BA=90):=20=E5=88=86=E7=89=87?= =?UTF-8?q?=E8=BD=BD=E8=8D=B7=E5=88=A4=E7=A9=BA=E4=B8=8E=E5=BB=BA=E8=A1=8C?= =?UTF-8?q?=E5=9B=BE=E5=85=B1=E7=94=A8=E4=B8=80=E6=AC=A1=20GET=EF=BC=8Cmer?= =?UTF-8?q?ge=20=E4=B8=8E=E5=80=99=E9=80=89=E6=94=B6=E9=9B=86=E5=90=84?= =?UTF-8?q?=E7=9C=81=E4=B8=80=E9=81=8D=E5=AF=B9=E8=B1=A1=E5=AD=98=E5=82=A8?= =?UTF-8?q?=E8=AF=BB=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../support/SimilarAsinPipelineSupport.java | 84 +++++-- ...ilarAsinTaskServiceChunkReadReuseTest.java | 228 ++++++++++++++++++ ...SimilarAsinTaskServiceRowKeyIndexTest.java | 5 +- 3 files changed, 291 insertions(+), 26 deletions(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkReadReuseTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineSupport.java index 82a6e8a16..a21ed57a0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineSupport.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineSupport.java @@ -157,8 +157,10 @@ public class SimilarAsinPipelineSupport { if (task == null || llmRows == null || llmRows.isEmpty()) { return; } - List chunks = loadSubmittedChunks(task.getId()); - if (chunks.isEmpty()) { + // 判空与建行图共用同一次载荷读取:改前 loadSubmittedChunks 逐片 GET 只为回答"这片有没有行", + // 紧接着下面又逐片 GET 一次。 + List submitted = readSubmittedChunks(loadSubmittedChunks(task.getId())); + if (submitted.isEmpty()) { return; } // P2-11:把当前 batch 命中的图片 url 异步丢入预热队列。 @@ -180,9 +182,10 @@ public class SimilarAsinPipelineSupport { } Map> rowsByChunk = new LinkedHashMap<>(); Map chunkByKey = new LinkedHashMap<>(); - for (TaskChunkEntity chunk : chunks) { + for (SubmittedChunk entry : submitted) { + TaskChunkEntity chunk = entry.chunk(); String chunkKey = SimilarAsinChunkMergeSupport.chunkStorageKey(chunk.getScopeHash(), chunk.getChunkIndex()); - rowsByChunk.put(chunkKey, chunkPayloadSupport.readChunkRows(chunk)); + rowsByChunk.put(chunkKey, entry.rows()); chunkByKey.put(chunkKey, chunk); } // Task 11:合并前按稳定 rowKey 去重,消除重复行逐行 expand/分配/写回 的 O(n²) 热点。 @@ -396,10 +399,39 @@ public class SimilarAsinPipelineSupport { if (chunks == null || chunks.isEmpty()) { return List.of(); } - return chunks.stream() - .filter(Objects::nonNull) - .filter(chunk -> !chunkPayloadSupport.readChunkRows(chunk).isEmpty()) - .toList(); + return chunks; + } + + + /** + * 一次作业轮次内共享的分片载荷读取结果:{@code chunk} 是库里的分片行,{@code rows} 是它当前载荷里的行。 + * 只在单次调用里传递,不做字段级缓存。 + */ + public record SubmittedChunk(TaskChunkEntity chunk, Map rows) { + } + + /** + * 逐片读一次载荷、跳过空片,顺序与入参一致。 + *

+ * 「这片有没有行」和「这片里有哪些行」原本是两次对象存储 GET + 整片 JSON 解析(分片数 C 就是 2C 次), + * 这里合并成一次;写回路径 {@link #mergeChunkPayload} 仍自己重读最新载荷再 CAS,不参与复用。 + */ + public List readSubmittedChunks(List chunks) { + if (chunks == null || chunks.isEmpty()) { + return List.of(); + } + List submitted = new ArrayList<>(chunks.size()); + for (TaskChunkEntity chunk : chunks) { + if (chunk == null) { + continue; + } + Map rows = chunkPayloadSupport.readChunkRows(chunk); + if (rows.isEmpty()) { + continue; + } + submitted.add(new SubmittedChunk(chunk, rows)); + } + return submitted; } @@ -827,11 +859,9 @@ public class SimilarAsinPipelineSupport { .last("limit 1")); chunks = chunk == null ? List.of() : List.of(chunk); } - chunks = chunks == null ? List.of() : chunks.stream() - .filter(Objects::nonNull) - .filter(chunk -> !chunkPayloadSupport.readChunkRows(chunk).isEmpty()) - .toList(); - if (chunks.isEmpty()) { + // 分片载荷只读这一次:改前这里逐片 GET 只为过滤空片,下面 collectPendingLlmCandidates 又逐片 GET 一次。 + List submitted = readSubmittedChunks(chunks); + if (submitted.isEmpty()) { return; } FileResultEntity result = host.findOrCreateResultRecordForAssembly(task, payloadSupport.allRowCount(task)); @@ -844,7 +874,7 @@ public class SimilarAsinPipelineSupport { return; } Map> allRowsByBaseId = loadAllRowsByBaseId(task); - boolean pendingLlm = submitLlmBatches(task, result, job, chunks, allRowsByBaseId); + boolean pendingLlm = submitLlmBatchesForSubmittedChunks(task, result, job, submitted, allRowsByBaseId); if (pendingLlm) { taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis()); touchJavaSideTaskActivity(task.getId()); @@ -914,19 +944,16 @@ public class SimilarAsinPipelineSupport { } - public List collectPendingLlmCandidates(List chunks, + public List collectPendingLlmCandidates(List submitted, Map> allRowsByBaseId) { - if (chunks == null || chunks.isEmpty()) { + if (submitted == null || submitted.isEmpty()) { return List.of(); } List candidates = new ArrayList<>(); - for (TaskChunkEntity chunk : chunks) { - Map persistedRows = chunkPayloadSupport.readChunkRows(chunk); - if (persistedRows.isEmpty()) { - continue; - } + for (SubmittedChunk entry : submitted) { + TaskChunkEntity chunk = entry.chunk(); List unresolvedRows = - enrichRowsForLlm(SimilarAsinResultTextSupport.collectPendingLlmRows(persistedRows.values()), allRowsByBaseId); + enrichRowsForLlm(SimilarAsinResultTextSupport.collectPendingLlmRows(entry.rows().values()), allRowsByBaseId); for (SimilarAsinResultRowDto row : unresolvedRows) { candidates.add(new LlmCandidate(chunk.getScopeHash(), chunk.getChunkIndex(), row)); } @@ -1012,7 +1039,16 @@ public class SimilarAsinPipelineSupport { TaskFileJobEntity job, List chunks, Map> allRowsByBaseId) { - if (chunks == null || chunks.isEmpty()) { + return submitLlmBatchesForSubmittedChunks(task, result, job, readSubmittedChunks(chunks), allRowsByBaseId); + } + + + public boolean submitLlmBatchesForSubmittedChunks(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + List submitted, + Map> allRowsByBaseId) { + if (submitted == null || submitted.isEmpty()) { return countPendingLlmStates(task.getId()) > 0; } String prompt = payloadSupport.readAiPrompt(task); @@ -1027,7 +1063,7 @@ public class SimilarAsinPipelineSupport { task.getId(), batchSize); batchSize = 1; } - List candidates = collectPendingLlmCandidates(chunks, allRowsByBaseId); + List candidates = collectPendingLlmCandidates(submitted, allRowsByBaseId); if (candidates.isEmpty()) { return countPendingLlmStates(task.getId()) > 0; } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkReadReuseTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkReadReuseTest.java new file mode 100644 index 000000000..75f3547bc --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkReadReuseTest.java @@ -0,0 +1,228 @@ +package com.nanri.aiimage.modules.similarasin.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.config.SimilarAsinProperties; +import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; +import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPipelineSupport; +import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.transaction.PlatformTransactionManager; + +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 货源链分片载荷的「一次读用两处」用例(外观 91d85ec6 的同一形态在货源侧的对应改动)。 + *

+ * 改前两处各重读一遍对象存储: + *

    + *
  • {@code mergeLlmRowsIntoChunk}:{@code loadSubmittedChunks} 逐片 GET 只为判空, + * 紧接着建行图又逐片 GET;
  • + *
  • {@code submitLlmForSubmittedChunk}:逐片 GET 过滤空片后,{@code collectPendingLlmCandidates} + * 再逐片 GET。
  • + *
+ * 用例按 GET 次数记账,并要求合并写回的行数与落库次数不变——省的是读,不是写。 + */ +@ExtendWith(MockitoExtension.class) +class SimilarAsinTaskServiceChunkReadReuseTest { + + private static final String MODULE = "SIMILAR_ASIN"; + private static final AtomicLong NEXT_ID = new AtomicLong(73000); + + @Mock private LocalFileStorageService localFileStorageService; + @Mock private OssStorageService ossStorageService; + @Mock private com.nanri.aiimage.config.StorageProperties storageProperties; + @Mock private FileTaskMapper fileTaskMapper; + @Mock private FileResultMapper fileResultMapper; + @Mock private TaskScopeStateMapper taskScopeStateMapper; + @Mock private TaskChunkMapper taskChunkMapper; + @Mock private SimilarAsinFilterConditionMapper filterConditionMapper; + @Spy private ObjectMapper objectMapper = new ObjectMapper(); + @Mock private SimilarAsinTaskCacheService taskCacheService; + @Mock private SimilarAsinProperties properties; + @Mock private TaskFileJobService taskFileJobService; + @Mock private TaskDistributedLockService taskDistributedLockService; + @Mock private TaskProgressSnapshotService taskProgressSnapshotService; + @Mock private TransientPayloadStorageService transientPayloadStorageService; + @Mock private PlatformTransactionManager transactionManager; + @Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService; + @Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata; + @Mock private SimilarAsinImageEmbedder imageEmbedder; + @Mock private SimilarAsinImagePrefetchService imagePrefetchService; + + @InjectMocks private SimilarAsinTaskService service; + + @BeforeAll + static void initializeMybatisMetadata() { + MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), ""); + TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class); + } + + @BeforeEach + void setUp() { + lenient().when(properties.getChunkMergeMaxRows()).thenReturn(500); + lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(4L * 1024L * 1024L); + lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned( + anyString(), any(), anyString(), any(), anyString())) + .thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3)); + lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1); + lenient().doAnswer(invocation -> { + FileTaskEntity task = invocation.getArgument(0); + task.setId(NEXT_ID.incrementAndGet()); + return 1; + }).when(fileTaskMapper).insert(any(FileTaskEntity.class)); + } + + @AfterEach + void shutdown() { + service.shutdownAssembleExecutor(); + } + + private static SimilarAsinResultRowDto row(String rowToken, String asin) { + SimilarAsinResultRowDto r = new SimilarAsinResultRowDto(); + r.setRowToken(rowToken); + r.setId(rowToken); + r.setAsin(asin); + r.setCountry("英国"); + r.setTitle("Title " + rowToken); + r.setMainUrl("https://img.example.com/" + asin + ".jpg"); + return r; + } + + private static TaskChunkEntity chunk(long id, String scopeHash, int chunkIndex, String pointer, String payloadHash) { + TaskChunkEntity chunk = new TaskChunkEntity(); + chunk.setId(id); + chunk.setTaskId(7304L); + chunk.setModuleType(MODULE); + chunk.setScopeHash(scopeHash); + chunk.setChunkIndex(chunkIndex); + chunk.setPayloadJson(pointer); + chunk.setPayloadHash(payloadHash); + return chunk; + } + + private static String rowsJson(List rows) throws Exception { + return new ObjectMapper().writeValueAsString(rows); + } + + private FileTaskEntity task() { + FileTaskEntity task = new FileTaskEntity(); + task.setId(7304L); + task.setModuleType(MODULE); + task.setStatus("RUNNING"); + return task; + } + + /** 每个指针固定返回一份载荷,让 GET 次数可以逐片对账。 */ + private void stubPayloads(Map byPointer) { + when(transientPayloadStorageService.resolvePayload(anyString(), anyString())) + .thenAnswer(invocation -> byPointer.getOrDefault(invocation.getArgument(0), "[]")); + } + + @Test + void mergeChunkRows_readsEachChunkOnceInsteadOfTwice() throws Exception { + // 3 片各 1 行,回传 1 行命中第 1 片。 + // 记账:建分片行图 3 次 GET + 命中片 CAS 写回前重读 1 次 = 4;改前是 3(判空探测)+3(建行图)+1=7。 + List chunks = List.of( + chunk(1L, "scope-1", 1, "ptr:chunk-1", "h1"), + chunk(2L, "scope-2", 2, "ptr:chunk-2", "h2"), + chunk(3L, "scope-3", 3, "ptr:chunk-3", "h3")); + stubPayloads(Map.of( + "ptr:chunk-1", rowsJson(List.of(row("r1", "B0A0000001"))), + "ptr:chunk-2", rowsJson(List.of(row("r2", "B0A0000002"))), + "ptr:chunk-3", rowsJson(List.of(row("r3", "B0A0000003"))))); + when(taskChunkMapper.selectList(any())).thenReturn(chunks); + // 写回路径自己 selectOne 最新片:只命中第 1 片一次。 + when(taskChunkMapper.selectOne(any())).thenReturn(chunks.get(0)); + + Method merge = SimilarAsinPipelineSupport.class.getDeclaredMethod("mergeLlmRowsIntoChunk", + FileTaskEntity.class, String.class, Integer.class, List.class, Map.class); + merge.setAccessible(true); + merge.invoke(service.pipelineSupport(), task(), null, null, + List.of(row("r1", "B0A0000001")), Map.of()); + + verify(transientPayloadStorageService, times(4)).resolvePayload(anyString(), anyString()); + // 省的是读,写一次没少:只有命中片被改写一次。 + verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned( + anyString(), any(), anyString(), any(), anyString()); + verify(taskChunkMapper, times(1)).update(any(), any()); + } + + @Test + void emptyFilterAndCandidateCollection_shareOneReadPerChunk() { + // 3 片、其中 1 片载荷为空:判空过滤与后续候选收集必须共用同一次 GET。 + List chunks = List.of( + chunk(1L, "scope-1", 1, "ptr:chunk-1", "h1"), + chunk(2L, "scope-2", 2, "ptr:chunk-empty", "h2"), + chunk(3L, "scope-3", 3, "ptr:chunk-3", "h3")); + stubPayloads(Map.of( + "ptr:chunk-1", "[{\"rowToken\":\"r1\",\"id\":\"r1\",\"asin\":\"B0A0000001\",\"country\":\"英国\"}]", + "ptr:chunk-3", "[{\"rowToken\":\"r3\",\"id\":\"r3\",\"asin\":\"B0A0000003\",\"country\":\"英国\"}]")); + + List submitted = + service.pipelineSupport().readSubmittedChunks(chunks); + + verify(transientPayloadStorageService, times(3)).resolvePayload(anyString(), anyString()); + assertEquals(List.of("scope-1:1", "scope-3:3"), submitted.stream() + .map(entry -> entry.chunk().getScopeHash() + ":" + entry.chunk().getChunkIndex()) + .toList(), "空片被跳过,非空片按入参顺序保留"); + + // 候选收集不再重读:GET 次数仍为 3。 + var candidates = service.pipelineSupport().collectPendingLlmCandidates(submitted, Map.of()); + verify(transientPayloadStorageService, times(3)).resolvePayload(anyString(), anyString()); + assertEquals(2, candidates.size(), "两片各 1 行未检测 → 2 个候选,一行不因省读而丢"); + } + + @Test + void chunkReadResults_areNotReusedAcrossInvocations() { + // 索引只在一次调用内共享:同一批分片连着读两轮就是两轮 GET, + // 否则跨任务/跨轮次的旧载荷会悄悄决定"哪些行已经检测过"。 + List chunks = List.of(chunk(1L, "scope-1", 1, "ptr:chunk-1", "h1")); + stubPayloads(Map.of("ptr:chunk-1", + "[{\"rowToken\":\"r1\",\"id\":\"r1\",\"asin\":\"B0A0000001\",\"country\":\"英国\"}]")); + + List first = service.pipelineSupport().readSubmittedChunks(chunks); + List second = service.pipelineSupport().readSubmittedChunks(chunks); + + verify(transientPayloadStorageService, times(2)).resolvePayload(anyString(), anyString()); + assertEquals(1, first.size()); + assertEquals(1, second.size()); + assertTrue(first.get(0).rows() != second.get(0).rows(), "两轮各拿自己那份载荷,不共享可变引用"); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyIndexTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyIndexTest.java index 555dbb5b8..57a7de26c 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyIndexTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceRowKeyIndexTest.java @@ -170,7 +170,8 @@ class SimilarAsinTaskServiceRowKeyIndexTest { @Test void test_task_010_chunk_row_key_normal_default_path() throws Exception { // 正常输入:2 个 chunk 各含行,llm 回传行按 rowKey 命中各自 chunk; - // 每个 chunk 的 payload 只被读取一次(索引建立),消除跨 chunk 线性扫描。 + // 建索引每片一次 GET(判空与建行图共用同一次读,见 ChunkReadReuseTest), + // 外加命中片 CAS 写回前的重读 → 2+2=4 次。 List chunks = List.of( chunk(1L, "hashA", 1, "ptr:chunk-A"), chunk(2L, "hashB", 2, "ptr:chunk-B")); @@ -197,7 +198,7 @@ class SimilarAsinTaskServiceRowKeyIndexTest { merge.setAccessible(true); merge.invoke(service.pipelineSupport(), task, null, null, llmRows, Map.of()); - verify(transientPayloadStorageService, times(6)).resolvePayload(anyString(), anyString()); + verify(transientPayloadStorageService, times(4)).resolvePayload(anyString(), anyString()); verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()); verify(taskChunkMapper, times(2)).update(any(), any()); }