super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-47: 品牌检查结果任务内短期缓存，避免同品牌重复远程调用

Browse Source
This commit is contained in:
黄自达
2026-08-30 13:53:38 +08:00
parent 751f4b89fd
commit b92d3e688d
4 changed files with 305 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilterTest.java
+5 -2
View File
@@ -93,7 +93,8 @@ class CollectDataBrandBatchFilterTest {
assertEquals(first.rejected(), second.rejected(), "重复执行 rejected 一致");
assertEquals(first.accepted(), second.accepted(), "重复执行 accepted 一致");
verify(brandCheckClient, times(2)).checkAll(anyList(), any());
// 缓存生效：第二次执行全部命中缓存，不再发起远程调用。
verify(brandCheckClient, times(1)).checkAll(anyList(), any());
}
@Test
@@ -129,6 +130,7 @@ class CollectDataBrandBatchFilterTest {
row("B000000002", "")
));
assertEquals(1, blank.rejected().size(), "单行空品牌归 rejected");
// 同实例缓存生效：solo 已查过，空品牌批次无远程调用，总调用保持 1 次。
verify(brandCheckClient, times(1)).checkAll(anyList(), any());
}
@@ -148,7 +150,8 @@ class CollectDataBrandBatchFilterTest {
assertEquals(1, outcome.rejected().size(), "空品牌行归 rejected");
assertEquals(5000, outcome.accepted().size(), "非空品牌行全 accepted");
verify(brandCheckClient, times(500)).checkAll(anyList(), any());
// 每批次 10 行恰好引入 10 个新品牌，10 批次后 100 品牌全部缓存，后续批次零调用。
verify(brandCheckClient, times(10)).checkAll(anyList(), any());
}
@Test
backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandCacheTest.java
+200
View File
@@ -0,0 +1,200 @@
package com.nanri.aiimage.modules.collectdata.util;
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Task 47：为品牌检查结果增加任务内短期缓存，避免同品牌重复远程调用。
* CollectDataBrandBatchFilter 在实例内缓存品牌检查判定（FAILED /
* QUERY_FAILED / OK），后续批次命中缓存的品牌不再发起 checkAll 远程调用；
* 远程抛错不缓存（可恢复），缓存有界（超限淘汰最旧条目），
* 分类语义与无缓存时完全等价。
*/
class CollectDataBrandCacheTest {
private BrandCheckClient brandCheckClient;
private CollectDataBrandBatchFilter filter;
@BeforeEach
void setUp() {
brandCheckClient = mock(BrandCheckClient.class);
filter = new CollectDataBrandBatchFilter(brandCheckClient, 10);
}
@Test
void test_task_047_cache_brand_normal_default_path() {
// 正常输入：首查品牌发起远程检查并缓存判定，分类正确。
when(brandCheckClient.checkAll(anyList(), any())).thenReturn(
new BrandCheckClient.BrandCheckBatchResult(
List.of(), List.of("Zara"), List.of()));
CollectDataBrandBatchFilter.BrandBatchOutcome outcome = filter.filter(List.of(
row("B000000001", "Zara"),
row("B000000002", "Nike")
));
assertEquals(1, outcome.rejected().size(), "失败品牌行 rejected");
assertEquals("B000000001", outcome.rejected().get(0).getAsin());
assertEquals(1, outcome.accepted().size(), "其余行 accepted");
verify(brandCheckClient).checkAll(anyList(), any());
}
@Test
void test_task_047_cache_brand_normal_multiple_items() {
// 批量场景：多 chunk 提交时同品牌只远程检查一次，每 chunk 只查未缓存品牌。
when(brandCheckClient.checkAll(anyList(), any())).thenReturn(
new BrandCheckClient.BrandCheckBatchResult(
List.of(), List.of("Zara"), List.of()));
filter.filter(List.of(row("B000000001", "Zara"), row("B000000002", "Nike")));
CollectDataBrandBatchFilter.BrandBatchOutcome second = filter.filter(List.of(
row("B000000001", "Zara"),
row("B000000003", "Adidas")
));
assertEquals(1, second.rejected().size(), "chunk2 Zara 命中缓存仍 rejected");
assertEquals(1, second.accepted().size(), "chunk2 新品牌 Adidas accepted");
ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
verify(brandCheckClient, times(2)).checkAll(captor.capture(), any());
assertEquals(List.of("Adidas"), captor.getAllValues().get(1), "第二次只查未缓存品牌");
}
@Test
void test_task_047_cache_brand_normal_repeated_operation_is_idempotent() {
// 幂等：同一输入重复执行结果一致，第二次全部命中缓存不发远程调用。
when(brandCheckClient.checkAll(anyList(), any())).thenReturn(
new BrandCheckClient.BrandCheckBatchResult(
List.of(), List.of("Nike"), List.of()));
List<CollectDataResultRowVo> rows = List.of(
row("B000000001", "Nike"),
row("B000000002", "Adidas")
);
CollectDataBrandBatchFilter.BrandBatchOutcome first = filter.filter(rows);
CollectDataBrandBatchFilter.BrandBatchOutcome second = filter.filter(rows);
assertEquals(first.rejected(), second.rejected(), "重复执行 rejected 一致");
assertEquals(first.accepted(), second.accepted(), "重复执行 accepted 一致");
verify(brandCheckClient, times(1)).checkAll(anyList(), any());
}
@Test
void test_task_047_cache_brand_boundary_empty_input() {
// 空输入：空列表与全空品牌批次均不发起远程调用，也不写缓存。
CollectDataBrandBatchFilter.BrandBatchOutcome empty = filter.filter(List.of());
assertEquals(0, empty.rejected().size() + empty.accepted().size() + empty.queryFailed().size(),
"空列表返回空结果");
verify(brandCheckClient, never()).checkAll(anyList(), any());
CollectDataBrandBatchFilter.BrandBatchOutcome blank = filter.filter(List.of(
row("B000000001", "")
));
assertEquals(1, blank.rejected().size(), "空品牌行 rejected");
verify(brandCheckClient, never()).checkAll(anyList(), any());
}
@Test
void test_task_047_cache_brand_boundary_single_item() {
// 单元素：单品牌首查一次；同品牌再出现时命中缓存零调用。
when(brandCheckClient.checkAll(anyList(), any())).thenReturn(
new BrandCheckClient.BrandCheckBatchResult(
List.of(), List.of(), List.of()));
CollectDataBrandBatchFilter.BrandBatchOutcome single = filter.filter(List.of(
row("B000000001", "solo")
));
assertEquals(1, single.accepted().size(), "单品牌未命中失败 accepted");
CollectDataBrandBatchFilter.BrandBatchOutcome again = filter.filter(List.of(
row("B000000002", "solo")
));
assertEquals(1, again.accepted().size(), "同品牌再次出现结果一致");
verify(brandCheckClient, times(1)).checkAll(anyList(), any());
}
@Test
void test_task_047_cache_brand_boundary_limit_and_overflow() {
// 上限/超限：缓存容量 8，首批 10 个品牌淘汰最旧 2 个；第二批同品牌
// 只重查被淘汰的 2 个，其余 8 个命中缓存；无无界增长。
when(brandCheckClient.checkAll(anyList(), any())).thenReturn(
new BrandCheckClient.BrandCheckBatchResult(
List.of(), List.of(), List.of()));
CollectDataBrandBatchFilter smallCache = new CollectDataBrandBatchFilter(brandCheckClient, 10, 8);
List<CollectDataResultRowVo> first = new ArrayList<>();
List<CollectDataResultRowVo> second = new ArrayList<>();
for (int i = 0; i < 10; i++) {
first.add(row("B" + i, "brand-" + i));
second.add(row("B" + (100 + i), "brand-" + i));
}
smallCache.filter(first);
CollectDataBrandBatchFilter.BrandBatchOutcome out = smallCache.filter(second);
assertEquals(10, out.accepted().size(), "第二批 10 行全部 accepted");
ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
verify(brandCheckClient, times(2)).checkAll(captor.capture(), any());
List<String> secondCall = captor.getAllValues().get(1);
assertEquals(2, secondCall.size(), "仅重查被淘汰的最旧 2 个品牌");
assertTrue(secondCall.containsAll(List.of("brand-0", "brand-1")), "淘汰的是最旧条目");
}
@Test
void test_task_047_cache_brand_invalid_input_rejected() {
// 非法参数：checkAll 返回 null 按无失败处理并缓存；null 行安全跳过。
when(brandCheckClient.checkAll(anyList(), any())).thenReturn(null);
List<CollectDataResultRowVo> rows = new ArrayList<>();
rows.add(null);
rows.add(row("B000000001", "Nike"));
CollectDataBrandBatchFilter.BrandBatchOutcome first = filter.filter(rows);
assertEquals(1, first.accepted().size(), "null 结果按无失败处理");
assertEquals(0, first.rejected().size(), "null 行不计数");
CollectDataBrandBatchFilter.BrandBatchOutcome second = filter.filter(rows);
assertEquals(1, second.accepted().size(), "重复执行结果一致");
verify(brandCheckClient, times(1)).checkAll(anyList(), any());
}
@Test
void test_task_047_cache_brand_dependency_failure_releases_resources() {
// 依赖失败：远程抛错整批 queryFailed 且不缓存；恢复后重查成功，无残留状态。
when(brandCheckClient.checkAll(anyList(), any()))
.thenThrow(new RuntimeException("brand service down"))
.thenReturn(new BrandCheckClient.BrandCheckBatchResult(
List.of(), List.of("Zara"), List.of()));
List<CollectDataResultRowVo> rows = List.of(
row("B000000001", "Zara"), row("B000000002", "Nike"));
CollectDataBrandBatchFilter.BrandBatchOutcome failed = filter.filter(rows);
assertEquals(2, failed.queryFailed().size(), "失败批次整组降级 queryFailed");
CollectDataBrandBatchFilter.BrandBatchOutcome recovered = filter.filter(rows);
assertEquals(1, recovered.rejected().size(), "恢复后重新检查并分类");
assertEquals(1, recovered.accepted().size(), "恢复后 accepted 正确");
assertTrue(recovered.queryFailed().isEmpty(), "恢复后无残留 queryFailed");
verify(brandCheckClient, times(2)).checkAll(anyList(), any());
}
private static CollectDataResultRowVo row(String asin, String brand) {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin(asin);
row.setBrand(brand);
return row;
}
}