super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-13: cap chunk merge row count and payload bytes with orphan fallback

Browse Source
This commit is contained in:
黄自达
2026-08-29 17:05:49 +08:00
parent cbe0ec8bb2
commit e201b13301
3 changed files with 523 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java
+14
View File
@@ -165,6 +165,20 @@ public class SimilarAsinProperties {
*/ */
private long maxWorkbookUncompressedBytes = 512L * 1024L * 1024L; private long maxWorkbookUncompressedBytes = 512L * 1024L * 1024L;
/**
* Task 13：单次 chunk 合并后的最大行数。mergeChunkPayload 合并后总行数超过该上限时，
* 从最旧行开始降级到 orphan 兜底（assemble 阶段 putIfAbsent 合并回结果），chunk 不无界增长。
* 默认与 maxParseRows 一致（50000）。
*/
private int chunkMergeMaxRows = 50000;
/**
* Task 13：单次 chunk 合并后 payload 的字节上限。合并后序列化字节超过该上限时，
* 从最旧行开始降级到 orphan 兜底；单行本身超过该上限时抛异常拒绝合并。
* 默认 16MB：50000 行 × 平均 300B/行 ≈ 15MB，留余量。
*/
private long chunkMergePayloadMaxBytes = 16L * 1024L * 1024L;
/** /**
* P0-4：抢 Coze 提交锁失败后下次重试间隔（毫秒）。 * P0-4：抢 Coze 提交锁失败后下次重试间隔（毫秒）。
* 原硬编码 500ms，会在指数退避算法中作为基础值（500/1000/2000/4000ms 上限 4000）。 * 原硬编码 500ms，会在指数退避算法中作为基础值（500/1000/2000/4000ms 上限 4000）。
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
+107 -1
View File
@@ -89,7 +89,9 @@ import java.time.Duration;
import java.time.Instant; import java.time.Instant;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.ZoneId; import java.time.ZoneId;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
@@ -1643,14 +1645,42 @@ public class SimilarAsinTaskService {
return; return;
} }
Map<String, SimilarAsinResultRowDto> persistedRows = readChunkRows(chunk); Map<String, SimilarAsinResultRowDto> persistedRows = readChunkRows(chunk);
int mergedRowCount = persistedRows.size() + rows.size();
// Task 13：单次合并后总行数超过上限时，从最旧行（存量优先）开始降级到 orphan 兜底，
// chunk 保持在上限内不无界增长；assemble 阶段 putIfAbsent 合并回结果不丢数据。
if (mergedRowCount > getChunkMergeMaxRows()) {
splitChunkMergeOverflow(taskId, persistedRows, rows, chunk, mergedRowCount - getChunkMergeMaxRows());
if (rows.isEmpty()) {
return;
}
}
for (SimilarAsinResultRowDto row : rows) { for (SimilarAsinResultRowDto row : rows) {
persistedRows.put(rowKey(row), row); persistedRows.put(rowKey(row), row);
} }
String payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "相似ASIN分片载荷合并失败"); String payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "相似ASIN分片载荷合并失败");
// Task 13：合并后 payload 字节超过上限时，从最旧行开始降级到 orphan 兜底；
// 降到只剩一行仍超上限时抛异常拒绝合并，防止无界 payload。
long payloadBytes = payloadJson.getBytes(StandardCharsets.UTF_8).length;
if (payloadBytes > getChunkMergePayloadMaxBytes()) {
demoteRowsToOrphan(taskId, persistedRows, payloadBytes - getChunkMergePayloadMaxBytes());
payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "相似ASIN分片载荷合并失败");
payloadBytes = payloadJson.getBytes(StandardCharsets.UTF_8).length;
if (payloadBytes > getChunkMergePayloadMaxBytes() && persistedRows.size() <= 1) {
throw new BusinessException("相似ASIN分片载荷超字节上限 taskId=" + taskId
+ " scopeHash=" + scopeHash + " chunk=" + chunkIndex
+ " bytes=" + payloadBytes + " limit=" + getChunkMergePayloadMaxBytes());
}
}
String oldPayload = chunk.getPayloadJson(); String oldPayload = chunk.getPayloadJson();
String oldPayloadHash = chunk.getPayloadHash(); String oldPayloadHash = chunk.getPayloadHash();
String newPayloadHash = DigestUtil.sha256Hex(payloadJson); String newPayloadHash = DigestUtil.sha256Hex(payloadJson);
String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); final String storedPayload;
try {
storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
} catch (Exception storeEx) {
throw new BusinessException("相似ASIN分片载荷存储失败 taskId=" + taskId + " chunk=" + chunkIndex
+ ": " + (storeEx.getMessage() == null ? "" : storeEx.getMessage()), storeEx);
}
int updated = taskChunkMapper.update(null, new LambdaUpdateWrapper<TaskChunkEntity>() int updated = taskChunkMapper.update(null, new LambdaUpdateWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getId, chunk.getId()) .eq(TaskChunkEntity::getId, chunk.getId())
.eq(TaskChunkEntity::getPayloadHash, oldPayloadHash) .eq(TaskChunkEntity::getPayloadHash, oldPayloadHash)
@@ -1672,6 +1702,82 @@ public class SimilarAsinTaskService {
throw new IllegalStateException("相似ASIN分片载荷更新失败"); throw new IllegalStateException("相似ASIN分片载荷更新失败");
} }
private int getChunkMergeMaxRows() {
int limit = properties.getChunkMergeMaxRows();
return limit > 0 ? limit : 50000;
}
private long getChunkMergePayloadMaxBytes() {
long limit = properties.getChunkMergePayloadMaxBytes();
return limit > 0 ? limit : 16L * 1024L * 1024L;
}
/**
* Task 13：按行数上限从最旧行开始降级到 orphan 兜底，保证合并后 chunk 行数不超过上限。
* 最旧优先：先降级存量行（LinkedHashMap 表头），不够再从新增行表头补齐；
* 传入的 rows 会被就地修改（保留未降级部分）。降级失败仅记日志，不阻断合并主流程。
*/
private void splitChunkMergeOverflow(Long taskId, Map<String, SimilarAsinResultRowDto> persistedRows,
List<SimilarAsinResultRowDto> rows, TaskChunkEntity chunk,
int demoteCount) {
if (demoteCount <= 0 || rows.isEmpty()) {
return;
}
List<SimilarAsinResultRowDto> demoted = new ArrayList<>();
while (demoted.size() < demoteCount && !persistedRows.isEmpty()) {
String oldestKey = persistedRows.keySet().iterator().next();
SimilarAsinResultRowDto removed = persistedRows.remove(oldestKey);
if (removed != null) {
demoted.add(removed);
}
}
for (Iterator<SimilarAsinResultRowDto> it = rows.iterator(); it.hasNext() && demoted.size() < demoteCount; ) {
SimilarAsinResultRowDto row = it.next();
if (row != null) {
demoted.add(row);
it.remove();
}
}
persistOrphanCozeRows(taskId, demoted);
log.warn("[similar-asin] chunk merge row-limit exceeded taskId={} chunk={} mergedRows={} limit={} demoted={}",
taskId, chunk.getChunkIndex(), mergedRowCountOf(persistedRows, rows), getChunkMergeMaxRows(), demoted.size());
}
/**
* Task 13：按字节上限从最旧行（LinkedHashMap 表头）开始降级到 orphan 兜底，
* 直到 payload 字节不超过上限或只剩一行；降级失败仅记日志，不阻断合并主流程。
*/
private void demoteRowsToOrphan(Long taskId, Map<String, SimilarAsinResultRowDto> persistedRows, long excessBytes) {
List<SimilarAsinResultRowDto> demoted = new ArrayList<>();
long releasedBytes = 0L;
while (persistedRows.size() > 1 && releasedBytes < excessBytes) {
String oldestKey = persistedRows.keySet().iterator().next();
SimilarAsinResultRowDto removed = persistedRows.remove(oldestKey);
if (removed != null) {
demoted.add(removed);
releasedBytes += estimateRowBytes(oldestKey, removed);
}
}
if (!demoted.isEmpty()) {
persistOrphanCozeRows(taskId, demoted);
log.warn("[similar-asin] chunk merge byte-limit exceeded taskId={} rows={} demoted={} releasedBytes={}",
taskId, demoted.size(), demoted.size(), releasedBytes);
}
}
private static int mergedRowCountOf(Map<String, SimilarAsinResultRowDto> persistedRows, List<SimilarAsinResultRowDto> rows) {
return persistedRows.size() + rows.size();
}
private long estimateRowBytes(String rowKey, SimilarAsinResultRowDto row) {
try {
String json = objectMapper.writeValueAsString(row);
return json == null ? 128L : json.getBytes(StandardCharsets.UTF_8).length;
} catch (Exception ex) {
return 128L + (rowKey == null ? 0 : rowKey.getBytes(StandardCharsets.UTF_8).length);
}
}
private List<SimilarAsinResultRowDto> expandRows(List<SimilarAsinResultRowDto> rows, private List<SimilarAsinResultRowDto> expandRows(List<SimilarAsinResultRowDto> rows,
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) { Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
if (rows == null || rows.isEmpty()) { if (rows == null || rows.isEmpty()) {
backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceChunkMergeLimitTest.java
+402
View File
@@ -0,0 +1,402 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 13：chunk 合并增加单次最大行数与 payload 字节上限。
* 校验点位于 mergeChunkPayload 单次合并入口：合并后总行数超过 chunkMergeMaxRows、
* 或 payload 字节超过 chunkMergePayloadMaxBytes 时，从最旧行开始降级到
* orphan 兜底（assemble 阶段 putIfAbsent 合并回结果，不丢数据）；
* 单行本身超过字节上限时抛可识别异常拒绝合并。低于上限的行为与旧路径完全一致。
*/
@ExtendWith(MockitoExtension.class)
class SimilarAsinTaskServiceChunkMergeLimitTest {
private static final AtomicLong NEXT_ID = new AtomicLong(90000);
@Mock private LocalFileStorageService localFileStorageService;
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@Mock private SimilarAsinCozeClient cozeClient;
@Mock private SimilarAsinTaskCacheService taskCacheService;
@Mock private SimilarAsinProperties properties;
@Mock private com.nanri.aiimage.modules.task.service.TaskFileJobService taskFileJobService;
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
@Mock private com.nanri.aiimage.modules.coze.service.CozeCredentialPoolService cozeCredentialPoolService;
@Mock private SimilarAsinImageEmbedder imageEmbedder;
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
@InjectMocks private SimilarAsinTaskService service;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
@BeforeEach
void setUp() {
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(16L * 1024L * 1024L);
lenient().when(transientPayloadStorageService.storeParsedPayloadFast(
eq(SimilarAsinTaskService.MODULE_TYPE), any(), anyString(), anyString(), eq(false)))
.thenReturn("rustfs:task-parsed/similar-asin/90000/payload.json");
lenient().doAnswer(invocation -> {
FileTaskEntity task = invocation.getArgument(0);
task.setId(NEXT_ID.incrementAndGet());
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
lenient().when(fileResultMapper.insert(any(com.nanri.aiimage.modules.task.model.entity.FileResultEntity.class))).thenReturn(1);
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenReturn(1);
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
}
@AfterEach
void shutdown() {
service.shutdownAssembleExecutor();
}
private static SimilarAsinResultRowDto row(String rowToken, String asin, String title) {
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
r.setRowToken(rowToken);
r.setId(rowToken);
r.setAsin(asin);
r.setCountry("英国");
r.setTitle(title);
return r;
}
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
return new ObjectMapper().writeValueAsString(rows);
}
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setId(id);
chunk.setTaskId(9004L);
chunk.setModuleType(SimilarAsinTaskService.MODULE_TYPE);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setPayloadJson(payloadJson);
return chunk;
}
/** 记录每次 storeChunkPayloadVersioned 收到的 payload 字符串。 */
private void stubChunkMerge(TaskChunkEntity chunk, String payloadJson, AtomicReference<String> storedPayload) throws Exception {
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq(chunk.getPayloadJson()), anyString()))
.thenReturn(payloadJson);
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
storedPayload.set(invocation.getArgument(4));
return "stored:" + invocation.getArgument(2);
});
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
}
private static void invokeMerge(SimilarAsinTaskService service, FileTaskEntity task,
String scopeHash, Integer chunkIndex,
List<SimilarAsinResultRowDto> cozeRows) throws Exception {
Method merge = SimilarAsinTaskService.class.getDeclaredMethod("mergeCozeRowsIntoChunk",
FileTaskEntity.class, String.class, Integer.class, List.class, Map.class);
merge.setAccessible(true);
merge.invoke(service, task, scopeHash, chunkIndex, cozeRows, Map.of());
}
@Test
void test_task_013_payload_row_count_chunk_normal_default_path() throws Exception {
// 正常输入：行数与 payload 字节均在上限内，合并走原路径，结果完整保留。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicReference<String> storedPayload = new AtomicReference<>("");
stubChunkMerge(chunk, rowsJson(List.of(row("r0", "B0A0000000", "存量行"))), storedPayload);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
List<SimilarAsinResultRowDto> cozeRows = List.of(
row("r1", "B0A0000001", "标题1"),
row("r2", "B0A0000002", "标题2"),
row("r3", "B0A0000003", "标题3"));
invokeMerge(service, task, "hashA", 1, cozeRows);
assertNotNull(storedPayload.get());
assertTrue(storedPayload.get().contains("\"r1\"") && storedPayload.get().contains("\"r3\""),
"上限内合并必须完整保留存量行与新增行，实际: " + storedPayload.get());
verify(transientPayloadStorageService, times(1)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, times(1)).update(any(), any());
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@Test
void test_task_013_payload_row_count_chunk_normal_multiple_items() throws Exception {
// 批量场景：多 chunk 一次 merge，全部在上限内，各 chunk 分别写回、结果不丢失。
TaskChunkEntity chunkA = chunk(1L, "hashA", 1, "ptr:chunk-A");
TaskChunkEntity chunkB = chunk(2L, "hashB", 2, "ptr:chunk-B");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunkA, chunkB));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r1", "B0A0000001", "标题1"))));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-B"), anyString()))
.thenReturn(rowsJson(List.of(row("r2", "B0A0000002", "标题2"))));
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> "stored:" + invocation.getArgument(2));
AtomicLong selectOneRound = new AtomicLong(0);
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation ->
selectOneRound.getAndIncrement() == 0 ? chunkA : chunkB);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
invokeMerge(service, task, null, null, List.of(
row("r1", "B0A0000001", "标题1-新"),
row("r2", "B0A0000002", "标题2-新")));
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskChunkMapper, times(2)).update(any(), any());
}
@Test
void test_task_013_payload_row_count_chunk_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行同一输入：每次 merge 恰好写一次，不产生重复对象、重复状态。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicLong storeCalls = new AtomicLong(0);
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
storeCalls.incrementAndGet();
return "stored:" + invocation.getArgument(2);
});
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
List<SimilarAsinResultRowDto> cozeRows = List.of(row("r1", "B0A0000001", "标题1"));
invokeMerge(service, task, "hashA", 1, cozeRows);
invokeMerge(service, task, "hashA", 1, cozeRows);
assertEquals(2, storeCalls.get(), "重复执行同一输入：每次 merge 恰好写回一次，无多余请求");
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@Test
void test_task_013_payload_row_count_chunk_boundary_empty_input() throws Exception {
// 空输入：null/空列表安全跳过，不读取 chunk、不写存储、不创建资源。
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
invokeMerge(service, task, "hashA", 1, null);
invokeMerge(service, task, "hashA", 1, List.of());
verify(taskChunkMapper, times(0)).selectList(any());
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@Test
void test_task_013_payload_row_count_chunk_boundary_single_item() throws Exception {
// 单行：不依赖批量路径，合并后结果正确。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicReference<String> storedPayload = new AtomicReference<>("");
stubChunkMerge(chunk, rowsJson(List.of(row("r0", "B0A0000000", "存量行"))), storedPayload);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
assertTrue(storedPayload.get().contains("\"r1\""), "单行合并也必须写回 chunk payload，实际: " + storedPayload.get());
verify(taskChunkMapper, times(1)).update(any(), any());
}
@Test
void test_task_013_payload_row_count_chunk_boundary_limit_and_overflow() throws Exception {
// 超限场景三连：行数超限降级、字节超限降级、单行超字节上限拒绝。
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(2);
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
AtomicLong storeCalls = new AtomicLong(0);
AtomicReference<String> lastStoredPayload = new AtomicReference<>("");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
storeCalls.incrementAndGet();
lastStoredPayload.set(invocation.getArgument(4));
return "stored:" + invocation.getArgument(2);
});
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
List<SimilarAsinResultRowDto> cozeRows = new ArrayList<>();
for (int i = 0; i < 5; i++) {
cozeRows.add(row("r" + (i + 1), "B0A00000" + (i + 1), "新行" + i));
}
// Phase A：行数超限（上限 2，存量 1 + 新增 5）→ 只保留上限内最新行，超限部分转 orphan。
invokeMerge(service, task, "hashA", 1, cozeRows);
assertTrue(lastStoredPayload.get().contains("\"r4\"") && lastStoredPayload.get().contains("\"r5\""),
"行数超限时保留上限内的最新行，实际: " + lastStoredPayload.get());
assertFalse(lastStoredPayload.get().contains("\"r0\""), "行数超限时最旧行被降级，实际: " + lastStoredPayload.get());
assertEquals(1, storeCalls.get());
verify(taskScopeStateMapper, times(1)).insert(any(TaskScopeStateEntity.class));
verify(transientPayloadStorageService, times(1))
.storeParsedPayloadEntry(anyString(), any(), anyString(), anyString(), anyString(), eq(true));
// Phase B：字节超限（行数放开）→ 从最旧行降级到字节上限内，保留最新结果。
lenient().when(properties.getChunkMergeMaxRows()).thenReturn(50000);
long oneRowBytes = rowsJson(List.of(row("r9", "B0A0000099", "样本行"))).getBytes(StandardCharsets.UTF_8).length;
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(oneRowBytes + 5L);
invokeMerge(service, task, "hashA", 1, cozeRows);
assertTrue(lastStoredPayload.get().contains("\"r5\""), "字节超限时保留最新行，实际: " + lastStoredPayload.get());
assertFalse(lastStoredPayload.get().contains("\"r0\""), "字节超限时最旧行被降级，实际: " + lastStoredPayload.get());
assertEquals(2, storeCalls.get());
verify(taskScopeStateMapper, times(2)).insert(any(TaskScopeStateEntity.class));
// Phase C：单行本身超过字节上限 → 抛可识别异常拒绝合并，不写 chunk。
lenient().when(properties.getChunkMergePayloadMaxBytes()).thenReturn(5L);
Exception ex = assertThrows(Exception.class, () -> {
try {
invokeMerge(service, task, "hashA", 1, cozeRows);
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertNotNull(ex.getMessage());
assertTrue(ex.getMessage().contains("字节上限"),
"超字节上限必须抛可识别异常，实际: " + ex.getMessage());
assertEquals(2, storeCalls.get(), "拒绝合并时不写 chunk");
}
@Test
void test_task_013_payload_row_count_chunk_invalid_input_rejected() {
// 非法输入：chunk 载荷加载失败（resolve 抛异常）时抛出可识别异常且不写 chunk。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenThrow(new IllegalStateException("rustfs down"));
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
Exception ex = assertThrows(Exception.class, () -> {
try {
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertNotNull(ex.getMessage());
assertTrue(ex.getMessage().contains("chunk"),
"chunk 读取失败消息必须可识别，实际: " + ex.getMessage());
verify(transientPayloadStorageService, times(0)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
@Test
void test_task_013_payload_row_count_chunk_dependency_failure_releases_resources() throws Exception {
// 依赖失败：payload 存储失败时抛带上下文的可识别异常、无残留状态；恢复后重试成功。
TaskChunkEntity chunk = chunk(1L, "hashA", 1, "ptr:chunk-A");
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
AtomicLong storeCalls = new AtomicLong(0);
doAnswer(invocation -> {
if (storeCalls.getAndIncrement() == 0) {
throw new IllegalStateException("rustfs down");
}
return "stored:" + invocation.getArgument(2);
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
when(taskChunkMapper.update(any(), any())).thenReturn(1);
FileTaskEntity task = new FileTaskEntity();
task.setId(9004L);
Exception ex = assertThrows(Exception.class, () -> {
try {
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
} catch (java.lang.reflect.InvocationTargetException e) {
throw e.getCause();
}
});
assertTrue(ex.getMessage() != null && ex.getMessage().contains("相似ASIN分片载荷"),
"存储失败必须抛带上下文的可识别异常，实际: " + ex.getMessage());
assertEquals(1, storeCalls.get(), "失败时只尝试一次即抛出，不静默吞错");
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
// 依赖恢复后重试成功：结果正确、无残留状态。
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
assertEquals(2, storeCalls.get(), "恢复后重试成功");
verify(taskChunkMapper, times(1)).update(any(), any());
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
}
}