task-50: 结果明细批量 upsert 写入器与唯一键幂等
CollectDataResultItemBatchWriter 按唯一键 uk_task_scope_item 批量 upsert biz_task_result_item：一次批量查询现有行（payload_hash 相等即跳过）， 分批发 INSERT ... ON DUPLICATE KEY UPDATE，替代逐行 select/insert/update； 旧格式逐行 payload 升级为引用时物理删除（引用计数兜底）。CollectDataService submitResult 接入批量写入，移除逐行 upsertResultItem。
This commit is contained in:
+239
@@ -0,0 +1,239 @@
|
||||
package com.nanri.aiimage.modules.collectdata.util;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 50：结果明细批量 upsert mapper 与幂等唯一键。
|
||||
* CollectDataResultItemBatchWriter 把整 chunk 的 accepted 行批量 upsert 到
|
||||
* biz_task_result_item：先一次批量查询现有行（payload_hash 相等即跳过，
|
||||
* 幂等），再按唯一键 uk_task_scope_item 用一条 INSERT ... ON DUPLICATE
|
||||
* KEY UPDATE 批量写入，替代逐行 select/insert/update。空输入零调用，
|
||||
* 批量失败跳过该批可恢复，无无界内存增长。
|
||||
*/
|
||||
class CollectDataResultItemBatchWriterTest {
|
||||
|
||||
private TaskResultItemMapper taskResultItemMapper;
|
||||
private TransientPayloadStorageService transientPayloadStorageService;
|
||||
private CollectDataResultDetailCodec codec;
|
||||
private CollectDataResultItemBatchWriter writer;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
taskResultItemMapper = mock(TaskResultItemMapper.class);
|
||||
transientPayloadStorageService = mock(TransientPayloadStorageService.class);
|
||||
codec = new CollectDataResultDetailCodec(new ObjectMapper());
|
||||
writer = new CollectDataResultItemBatchWriter(taskResultItemMapper, codec, transientPayloadStorageService, 10);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_050_task_normal_default_path() {
|
||||
// 正常输入：无现有行 → 一次 upsertBatch 全量写入，引用字段完整。
|
||||
when(taskResultItemMapper.selectList(any())).thenReturn(List.of());
|
||||
when(taskResultItemMapper.upsertBatch(anyList())).thenReturn(2);
|
||||
List<CollectDataResultRowVo> rows = List.of(
|
||||
row("B000000001", "Nike"), row("B000000002", "Zara"));
|
||||
|
||||
CollectDataResultItemBatchWriter.UpsertCounts counts =
|
||||
writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:detail/abc");
|
||||
|
||||
assertEquals(2, counts.insertedOrUpdated(), "两行均写入");
|
||||
assertEquals(0, counts.skipped(), "无跳过");
|
||||
verify(taskResultItemMapper).selectList(any());
|
||||
ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(taskResultItemMapper).upsertBatch(captor.capture());
|
||||
List<TaskResultItemEntity> entities = captor.getValue();
|
||||
assertEquals(2, entities.size(), "批量写入两行");
|
||||
assertEquals("asin:B000000001", entities.get(0).getItemKey(), "item_key 语义键");
|
||||
assertEquals("ACCEPTED", entities.get(0).getStatus(), "状态保持 ACCEPTED");
|
||||
assertTrue(entities.get(0).getPayloadJson().contains("rustfs:detail/abc"), "引用含对象指针");
|
||||
assertTrue(entities.get(0).getPayloadJson().contains("\"offset\":0"), "引用含 offset");
|
||||
assertEquals(2L, entities.get(0).getResultId(), "result_id 保留");
|
||||
assertEquals(1L, entities.get(0).getTaskId(), "task_id 保留");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_050_task_normal_multiple_items() {
|
||||
// 批量场景：75 行（8 批次），顺序稳定不丢失，每批数量正确。
|
||||
when(taskResultItemMapper.selectList(any())).thenReturn(List.of());
|
||||
when(taskResultItemMapper.upsertBatch(anyList())).thenAnswer(
|
||||
invocation -> ((List<?>) invocation.getArgument(0)).size());
|
||||
List<CollectDataResultRowVo> rows = new ArrayList<>();
|
||||
for (int i = 0; i < 75; i++) {
|
||||
rows.add(row("B" + String.format("%09d", i + 1), "brand-" + (i % 7)));
|
||||
}
|
||||
|
||||
CollectDataResultItemBatchWriter.UpsertCounts counts =
|
||||
writer.upsertAccepted(1L, 2L, "task:1", 3, rows, "rustfs:detail/x");
|
||||
|
||||
assertEquals(75, counts.insertedOrUpdated(), "全部写入");
|
||||
ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(taskResultItemMapper, times(8)).upsertBatch(captor.capture());
|
||||
for (int i = 0; i < 8; i++) {
|
||||
assertEquals(i < 7 ? 10 : 5, captor.getAllValues().get(i).size(), "批次 " + i + " 数量");
|
||||
}
|
||||
assertEquals("asin:B000000001", ((TaskResultItemEntity) captor.getAllValues().get(0).get(0)).getItemKey(),
|
||||
"顺序稳定");
|
||||
assertEquals("asin:B000000075", ((TaskResultItemEntity) captor.getAllValues().get(7).get(4)).getItemKey(),
|
||||
"末批末行不丢失");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_050_task_normal_repeated_operation_is_idempotent() {
|
||||
// 幂等：同一 chunk 重复提交时 payload_hash 相等 → 全部跳过不写入，
|
||||
// 无重复记录、无重复写入调用。
|
||||
String refJson = codec.encodeRef(0, 0, "rustfs:detail/x");
|
||||
TaskResultItemEntity existing = new TaskResultItemEntity();
|
||||
existing.setId(100L);
|
||||
existing.setItemKey("asin:B000000001");
|
||||
existing.setPayloadJson(refJson);
|
||||
existing.setPayloadHash(sha256(refJson));
|
||||
when(taskResultItemMapper.selectList(any())).thenReturn(List.of(existing));
|
||||
|
||||
CollectDataResultItemBatchWriter.UpsertCounts counts =
|
||||
writer.upsertAccepted(1L, 2L, "task:1", 0,
|
||||
List.of(row("B000000001", "Nike")), "rustfs:detail/x");
|
||||
|
||||
assertEquals(0, counts.insertedOrUpdated(), "hash 相等全部跳过");
|
||||
assertEquals(1, counts.skipped(), "1 行跳过");
|
||||
verify(taskResultItemMapper, never()).upsertBatch(anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_050_task_boundary_empty_input() {
|
||||
// 空输入：空列表零调用返回零计数。
|
||||
CollectDataResultItemBatchWriter.UpsertCounts counts =
|
||||
writer.upsertAccepted(1L, 2L, "task:1", 0, List.of(), "rustfs:x");
|
||||
|
||||
assertEquals(0, counts.insertedOrUpdated(), "空输入返回 0");
|
||||
assertEquals(0, counts.skipped(), "空输入无跳过");
|
||||
verify(taskResultItemMapper, never()).selectList(any());
|
||||
verify(taskResultItemMapper, never()).upsertBatch(anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_050_task_boundary_single_item() {
|
||||
// 单元素：单行正常 upsert；hash 不同时更新，唯一键幂等；
|
||||
// 旧格式逐行 payload（非引用）在升级为引用时物理删除，避免泄漏。
|
||||
TaskResultItemEntity existing = new TaskResultItemEntity();
|
||||
existing.setId(100L);
|
||||
existing.setItemKey("asin:B000000001");
|
||||
existing.setPayloadJson("{\"brand\":\"nike\",\"asin\":\"B000000001\"}");
|
||||
existing.setPayloadHash("old-hash");
|
||||
when(taskResultItemMapper.selectList(any())).thenReturn(List.of(existing));
|
||||
when(taskResultItemMapper.upsertBatch(anyList())).thenReturn(1);
|
||||
|
||||
CollectDataResultItemBatchWriter.UpsertCounts counts =
|
||||
writer.upsertAccepted(1L, 2L, "task:1", 1,
|
||||
List.of(row("B000000001", "Nike")), "rustfs:new");
|
||||
|
||||
assertEquals(1, counts.insertedOrUpdated(), "hash 不同触发 upsert 更新");
|
||||
verify(transientPayloadStorageService).deletePayloadIfPresent("{\"brand\":\"nike\",\"asin\":\"B000000001\"}");
|
||||
ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(taskResultItemMapper).upsertBatch(captor.capture());
|
||||
TaskResultItemEntity entity = (TaskResultItemEntity) captor.getValue().get(0);
|
||||
assertEquals(100L, entity.getId(), "复用现有行 id（ON DUPLICATE 命中唯一键）");
|
||||
assertTrue(entity.getPayloadJson().contains("rustfs:new"), "引用更新为新对象");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_050_task_boundary_limit_and_overflow() {
|
||||
// 上限/超限：批次大小 3，10 行 = 4 批（3+3+3+1），无无界积累。
|
||||
when(taskResultItemMapper.selectList(any())).thenReturn(List.of());
|
||||
when(taskResultItemMapper.upsertBatch(anyList())).thenAnswer(
|
||||
invocation -> ((List<?>) invocation.getArgument(0)).size());
|
||||
CollectDataResultItemBatchWriter smallWriter =
|
||||
new CollectDataResultItemBatchWriter(taskResultItemMapper, codec, transientPayloadStorageService, 3);
|
||||
List<CollectDataResultRowVo> rows = new ArrayList<>();
|
||||
for (int i = 0; i < 10; i++) {
|
||||
rows.add(row("B" + String.format("%09d", i + 1), "brand"));
|
||||
}
|
||||
|
||||
CollectDataResultItemBatchWriter.UpsertCounts counts = smallWriter.upsertAccepted(
|
||||
1L, 2L, "task:1", 0, rows, "rustfs:x");
|
||||
|
||||
assertEquals(10, counts.insertedOrUpdated(), "全部写入");
|
||||
ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(taskResultItemMapper, times(4)).upsertBatch(captor.capture());
|
||||
assertEquals(3, captor.getAllValues().get(0).size(), "首批 3 行");
|
||||
assertEquals(1, captor.getAllValues().get(3).size(), "末批 1 行");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_050_task_invalid_input_rejected() {
|
||||
// 非法参数：null/空白 ASIN 的行跳过不写入；null 行安全跳过。
|
||||
when(taskResultItemMapper.selectList(any())).thenReturn(List.of());
|
||||
when(taskResultItemMapper.upsertBatch(anyList())).thenAnswer(
|
||||
invocation -> ((List<?>) invocation.getArgument(0)).size());
|
||||
List<CollectDataResultRowVo> rows = new ArrayList<>();
|
||||
rows.add(null);
|
||||
rows.add(row(null, "nike"));
|
||||
rows.add(row("B000000001", "nike"));
|
||||
|
||||
CollectDataResultItemBatchWriter.UpsertCounts counts =
|
||||
writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:x");
|
||||
|
||||
assertEquals(1, counts.insertedOrUpdated(), "仅合法行写入");
|
||||
verify(taskResultItemMapper).upsertBatch(anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_050_task_dependency_failure_releases_resources() {
|
||||
// 依赖失败：批量 upsert 抛错时跳过该批不中断，恢复后继续，无资源泄漏。
|
||||
when(taskResultItemMapper.selectList(any())).thenReturn(List.of());
|
||||
doThrow(new RuntimeException("db down"))
|
||||
.doAnswer(invocation -> ((List<?>) invocation.getArgument(0)).size())
|
||||
.when(taskResultItemMapper).upsertBatch(anyList());
|
||||
List<CollectDataResultRowVo> rows = new ArrayList<>();
|
||||
for (int i = 0; i < 20; i++) {
|
||||
rows.add(row("B" + String.format("%09d", i + 1), "brand"));
|
||||
}
|
||||
|
||||
CollectDataResultItemBatchWriter.UpsertCounts counts =
|
||||
writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:x");
|
||||
|
||||
assertEquals(10, counts.insertedOrUpdated(), "首批失败跳过，第二批 10 行写入");
|
||||
verify(taskResultItemMapper, times(2)).upsertBatch(anyList());
|
||||
}
|
||||
|
||||
private static CollectDataResultRowVo row(String asin, String brand) {
|
||||
CollectDataResultRowVo row = new CollectDataResultRowVo();
|
||||
row.setAsin(asin);
|
||||
row.setBrand(brand);
|
||||
return row;
|
||||
}
|
||||
|
||||
private static String sha256(String value) {
|
||||
try {
|
||||
java.security.MessageDigest digest = java.security.MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user