task-44: collectdata extra JSON 批量预序列化与内容去重
persistParsedTask 逐行调用 ObjectMapper 序列化 extra 是热点；新增
CollectDataExtraJsonCodec 批量预序列化，相同 extra 只序列化一次，
输出与逐行语义完全一致，失败行降级 {}。8 个测试覆盖去重/幂等/
空输入/单元素/容量淘汰/非法输入/序列化失败，全量回归 704 通过。
This commit is contained in:
+214
@@ -0,0 +1,214 @@
|
||||
package com.nanri.aiimage.modules.collectdata.util;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Task 44：保留原始 chunk payload 的同时，减少逐行 extra JSON 的重复序列化。
|
||||
* CollectDataExtraJsonCodec 批量预序列化 extra map，内容级缓存去重相同 extra
|
||||
* 只序列化一次；输出与逐行 writeValueAsString 语义完全一致（兼容旧结构），
|
||||
* 单行序列化失败降级 "{}" 不影响其余行，缓存有界（超限淘汰最旧）。
|
||||
*/
|
||||
class CollectDataExtraJsonCodecTest {
|
||||
|
||||
@Test
|
||||
void test_task_044_payload_chunk_normal_default_path() throws Exception {
|
||||
// 正常输入：多条不同 extra 批量编码，输出与逐行序列化完全一致。
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
CollectDataExtraJsonCodec codec = new CollectDataExtraJsonCodec(mapper);
|
||||
List<Map<String, String>> extras = List.of(
|
||||
mapOf("价格", "19.99", "颜色", "黑"),
|
||||
mapOf("价格", "9.99"),
|
||||
mapOf()
|
||||
);
|
||||
|
||||
List<String> encoded = codec.encodeAll(extras);
|
||||
|
||||
assertEquals(3, encoded.size(), "输出行数一致");
|
||||
assertEquals(mapper.writeValueAsString(extras.get(0)), encoded.get(0), "与逐行序列化一致");
|
||||
assertEquals(mapper.writeValueAsString(extras.get(1)), encoded.get(1), "第二行一致");
|
||||
assertEquals("{}", encoded.get(2), "空 map 输出 {}");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_044_payload_chunk_normal_multiple_items() throws Exception {
|
||||
// 批量场景：1000 行含重复 extra，输出顺序稳定、行数不丢失、重复内容只序列化一次。
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
AtomicInteger serializations = new AtomicInteger(0);
|
||||
ObjectMapper spy = org.mockito.Mockito.spy(mapper);
|
||||
org.mockito.Mockito.doAnswer(invocation -> {
|
||||
serializations.incrementAndGet();
|
||||
return invocation.callRealMethod();
|
||||
}).when(spy).writeValueAsString(org.mockito.ArgumentMatchers.any());
|
||||
|
||||
CollectDataExtraJsonCodec codec = new CollectDataExtraJsonCodec(spy);
|
||||
List<Map<String, String>> extras = new ArrayList<>();
|
||||
Map<String, String> shared = mapOf("价格", "19.99", "卖家", "A");
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
extras.add(i % 10 == 0 ? shared : mapOf("价格", String.valueOf(i)));
|
||||
}
|
||||
|
||||
List<String> encoded = codec.encodeAll(extras);
|
||||
|
||||
assertEquals(1000, encoded.size(), "批量行数不丢失");
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
if (i % 10 == 0) {
|
||||
assertEquals(encoded.get(0), encoded.get(i), "相同 extra 输出一致");
|
||||
}
|
||||
}
|
||||
assertTrue(serializations.get() < 1000, "去重后序列化次数减少，实际=" + serializations.get());
|
||||
assertTrue(serializations.get() >= 100, "不同内容仍逐条序列化，实际=" + serializations.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_044_payload_chunk_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 幂等：同一输入两次编码输出完全一致，且第二次命中缓存不触发新序列化。
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
AtomicInteger serializations = new AtomicInteger(0);
|
||||
ObjectMapper spy = org.mockito.Mockito.spy(mapper);
|
||||
org.mockito.Mockito.doAnswer(invocation -> {
|
||||
serializations.incrementAndGet();
|
||||
return invocation.callRealMethod();
|
||||
}).when(spy).writeValueAsString(org.mockito.ArgumentMatchers.any());
|
||||
|
||||
CollectDataExtraJsonCodec codec = new CollectDataExtraJsonCodec(spy);
|
||||
List<Map<String, String>> extras = List.of(mapOf("a", "1"), mapOf("b", "2"), mapOf("a", "1"));
|
||||
|
||||
List<String> first = codec.encodeAll(extras);
|
||||
int firstCount = serializations.get();
|
||||
List<String> second = codec.encodeAll(extras);
|
||||
|
||||
assertEquals(first, second, "重复编码输出一致");
|
||||
assertTrue(firstCount <= 2, "首次编码去重后最多序列化 2 次，实际=" + firstCount);
|
||||
assertEquals(firstCount, serializations.get(), "第二次编码全部命中缓存，无新序列化");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_044_payload_chunk_boundary_empty_input() {
|
||||
// 空输入：空列表返回空输出；null 列表安全返回空列表。
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
CollectDataExtraJsonCodec codec = new CollectDataExtraJsonCodec(mapper);
|
||||
|
||||
List<String> empty = codec.encodeAll(List.of());
|
||||
assertEquals(0, empty.size(), "空列表输出空");
|
||||
|
||||
List<String> nullSafe = codec.encodeAll(null);
|
||||
assertEquals(0, nullSafe.size(), "null 列表安全返回空");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_044_payload_chunk_boundary_single_item() {
|
||||
// 单元素：单条 extra 编码正确，不依赖批量路径。
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
CollectDataExtraJsonCodec codec = new CollectDataExtraJsonCodec(mapper);
|
||||
|
||||
List<String> encoded = codec.encodeAll(List.of(mapOf("关键词", "phone case")));
|
||||
|
||||
assertEquals(1, encoded.size(), "单条输出");
|
||||
assertTrue(encoded.get(0).contains("phone case"), "单条内容正确");
|
||||
assertTrue(encoded.get(0).contains("关键词"), "单条键保留");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_044_payload_chunk_boundary_limit_and_overflow() throws Exception {
|
||||
// 上限/超限：缓存容量超限淘汰最旧条目，被淘汰内容重新序列化，
|
||||
// 仍被缓存的内容继续命中；序列化次数精确可预期。
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
AtomicInteger serializations = new AtomicInteger(0);
|
||||
ObjectMapper spy = org.mockito.Mockito.spy(mapper);
|
||||
org.mockito.Mockito.doAnswer(invocation -> {
|
||||
serializations.incrementAndGet();
|
||||
return invocation.callRealMethod();
|
||||
}).when(spy).writeValueAsString(org.mockito.ArgumentMatchers.any());
|
||||
|
||||
CollectDataExtraJsonCodec codec = new CollectDataExtraJsonCodec(spy, 32);
|
||||
List<Map<String, String>> bulk = new ArrayList<>();
|
||||
List<Map<String, String>> hot = new ArrayList<>();
|
||||
List<Map<String, String>> cold = new ArrayList<>();
|
||||
for (int i = 0; i < 100; i++) {
|
||||
bulk.add(mapOf("k" + i, "v" + i));
|
||||
}
|
||||
for (int i = 95; i < 100; i++) {
|
||||
hot.add(mapOf("k" + i, "v" + i));
|
||||
}
|
||||
for (int i = 0; i < 5; i++) {
|
||||
cold.add(mapOf("k" + i, "v" + i));
|
||||
}
|
||||
|
||||
codec.encodeAll(bulk);
|
||||
int afterBulk = serializations.get();
|
||||
assertEquals(100, afterBulk, "首轮 100 条逐条序列化");
|
||||
|
||||
codec.encodeAll(hot);
|
||||
assertEquals(afterBulk, serializations.get(), "缓存内副本全部命中，无新序列化");
|
||||
|
||||
codec.encodeAll(cold);
|
||||
assertEquals(afterBulk + 5, serializations.get(), "被淘汰的旧条目重新序列化");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_044_payload_chunk_invalid_input_rejected() {
|
||||
// 非法输入：null 行安全降级为 {}；超大 map 值仍编码不抛。
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
CollectDataExtraJsonCodec codec = new CollectDataExtraJsonCodec(mapper);
|
||||
|
||||
List<Map<String, String>> withNull = new ArrayList<>();
|
||||
withNull.add(null);
|
||||
withNull.add(mapOf("a", "b"));
|
||||
|
||||
List<String> encoded = codec.encodeAll(withNull);
|
||||
|
||||
assertEquals(2, encoded.size(), "null 行不丢弃");
|
||||
assertEquals("{}", encoded.get(0), "null 行降级为 {}");
|
||||
assertFalse(encoded.get(1).isBlank(), "正常行编码不受影响");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_044_payload_chunk_dependency_failure_releases_resources() throws Exception {
|
||||
// 依赖失败：单行序列化抛错时该行降级 {}，其余行正常；恢复后编码成功。
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
AtomicInteger calls = new AtomicInteger(0);
|
||||
ObjectMapper spy = org.mockito.Mockito.spy(mapper);
|
||||
org.mockito.Mockito.doAnswer(invocation -> {
|
||||
if (calls.getAndIncrement() == 0) {
|
||||
throw new JsonProcessingException("serializer down") {
|
||||
};
|
||||
}
|
||||
return invocation.callRealMethod();
|
||||
}).when(spy).writeValueAsString(org.mockito.ArgumentMatchers.any());
|
||||
|
||||
CollectDataExtraJsonCodec codec = new CollectDataExtraJsonCodec(spy);
|
||||
List<Map<String, String>> extras = List.of(mapOf("价格", "1"), mapOf("价格", "2"));
|
||||
|
||||
List<String> encoded = codec.encodeAll(extras);
|
||||
|
||||
assertEquals(2, encoded.size(), "失败行不丢弃");
|
||||
assertEquals("{}", encoded.get(0), "失败首行降级为 {}");
|
||||
assertTrue(encoded.get(1).contains("2"), "恢复后第二行编码正确");
|
||||
assertEquals(2, calls.get(), "首行失败一次、第二行成功一次");
|
||||
|
||||
// 失败结果已缓存：再次编码同一输入不再触发序列化（失败行不重复尝试）。
|
||||
List<String> again = codec.encodeAll(extras);
|
||||
assertEquals(encoded, again, "重复编码输出一致");
|
||||
assertEquals(2, calls.get(), "失败行降级结果已缓存，不重复尝试");
|
||||
}
|
||||
|
||||
private static Map<String, String> mapOf(String... pairs) {
|
||||
Map<String, String> map = new LinkedHashMap<>();
|
||||
for (int i = 0; i < pairs.length; i += 2) {
|
||||
map.put(pairs[i], pairs[i + 1]);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user