From 54411c6446cba006edc5e4ff73348c7643fd1e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Wed, 7 Oct 2026 13:34:18 +0800 Subject: [PATCH] =?UTF-8?q?fix(=E5=93=81=E7=89=8C=E6=A3=80=E6=B5=8B):=20?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E5=88=86=E7=89=87=E5=9B=9E=E4=BC=A0=E9=A2=86?= =?UTF-8?q?=E4=B8=80=E4=BB=BD=20chunk=20=E9=A2=84=E7=AE=97=EF=BC=8C?= =?UTF-8?q?=E7=94=A8=E5=B0=BD=E5=8D=B3=E8=BD=AC=E8=A1=A5=E6=B5=8B=E9=98=9F?= =?UTF-8?q?=E5=88=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上游 16890 卡死时每批各领一个 90 秒总超时,一个 chunk 十几批能叠到十几分钟, 桌面端是在同步等响应的路径上替它扛。现按 chunk 给 10 秒预算,预算内逐批只把 剩余额度交给远程;用尽后未查的品牌一律按查询失败入补测队列,既不按 accepted 放行也不写进无效品牌表。外观合批、品牌补测调度等后台路径仍按 90 秒走。 --- .../brand/client/BrandCheckClient.java | 28 +++- .../util/CollectDataBrandBatchFilter.java | 67 ++++++++- .../src/main/resources/application.yml | 5 + .../brand/client/BrandCheckClientTest.java | 81 +++++++++++ .../util/CollectDataBrandBatchFilterTest.java | 127 +++++++++++++++--- .../util/CollectDataBrandCacheTest.java | 33 ++--- 6 files changed, 299 insertions(+), 42 deletions(-) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/client/BrandCheckClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/client/BrandCheckClient.java index 88cc88643..b58f15016 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/client/BrandCheckClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/client/BrandCheckClient.java @@ -96,10 +96,22 @@ public class BrandCheckClient { } public BrandCheckBatchResult checkAll(List brands, String strategy) { + return checkAll(brands, strategy, properties.getTotalTimeoutMillis()); + } + + /** + * 调用方自带整批耗时预算(毫秒,{@code <= 0} 表示不限制):预算只掐「重试」, + * 每个品牌的首轮请求始终执行,所以上游只是慢而不是坏时不会因此丢结论。 + * + *

分片回传用它把一次 POST 的等待上界收到几秒:{@code total-timeout-millis} 那 90 秒 + * 是给后台批量任务(外观合批、品牌补测调度)的预算,同步等响应的桌面客户端不该替它扛 + * ——上游卡死时按批各领一个 90 秒,一个 chunk 十几批能叠到十几分钟。 + */ + public BrandCheckBatchResult checkAll(List brands, String strategy, long budgetMillis) { List distinctBrands = distinctNonBlank(brands); List failedData = new ArrayList<>(); List queryFailedData = new ArrayList<>(); - for (BrandCheckOutcome outcome : checkWords(distinctBrands, strategy).values()) { + for (BrandCheckOutcome outcome : checkWords(distinctBrands, strategy, deadlineNanos(budgetMillis)).values()) { failedData.addAll(outcome.failedData()); queryFailedData.addAll(outcome.queryFailedData()); } @@ -126,7 +138,8 @@ public class BrandCheckClient { wordsByTitle.add(words); distinctWords.addAll(words); } - Map outcomes = checkWords(new ArrayList<>(distinctWords), strategy); + Map outcomes = checkWords( + new ArrayList<>(distinctWords), strategy, deadlineNanos(properties.getTotalTimeoutMillis())); List results = new ArrayList<>(titles.size()); for (List words : wordsByTitle) { List failedData = new ArrayList<>(); @@ -144,14 +157,12 @@ public class BrandCheckClient { } /** - * 并发查一批**已去重**的词,整批共用一个耗时预算。 + * 并发查一批**已去重**的词,整批共用调用方给定的截止时刻。 * 键顺序即入参顺序,checkAll 的并集顺序因此与改动前一致。 */ - private Map checkWords(List brands, String strategy) { + private Map checkWords(List brands, String strategy, long deadlineNanos) { // 整批共用一个耗时预算:上游 16890 卡死时,单品牌 10 次重试曾把一次分片回传拖到 // 103.5 秒(taskId 28599),客户端重试预算耗尽后中止了整个采集。预算用尽即停止重试。 - long budgetMillis = properties.getTotalTimeoutMillis(); - long deadlineNanos = budgetMillis > 0L ? System.nanoTime() + budgetMillis * 1_000_000L : Long.MAX_VALUE; Map outcomes = new LinkedHashMap<>(); List> futures = new ArrayList<>(brands.size()); for (String brand : brands) { @@ -164,6 +175,11 @@ public class BrandCheckClient { return outcomes; } + /** 预算毫秒数换算成截止时刻;{@code <= 0} 表示不限制(首轮与全部重试都不掐)。 */ + private static long deadlineNanos(long budgetMillis) { + return budgetMillis > 0L ? System.nanoTime() + budgetMillis * 1_000_000L : Long.MAX_VALUE; + } + private BrandCheckOutcome checkOneBrand(String brand, String strategy, long deadlineNanos) { int attempts = Math.max(1, properties.getRetryTimes()); BrandCheckResponse response = null; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilter.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilter.java index e60c868d8..496b81a4f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilter.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilter.java @@ -26,6 +26,11 @@ import java.util.regex.Pattern; * 品牌判定结果在过滤器实例内短期缓存(同一任务多个 chunk 提交复用同一 * 实例):已判定的品牌再次出现时不再发起远程调用,避免同品牌重复请求。 * 远程抛错不写缓存(可恢复后重查);缓存有界,超限淘汰最旧条目。 + * + * 一次调用(= 一个 chunk)共用一个总耗时预算:预算用尽后不再为剩余批次发起远程 + * 检查,未判定的品牌整批归 queryFailed 转入补测队列。预算按 chunk 而不是按批, + * 是因为「每批各领一份 aiimage.brand-check.total-timeout-millis」会随批数线性叠加, + * 而这批请求正卡在客户端同步等待的分片回传里。设为 0 退回旧口径(每批各领一份)。 */ @Slf4j @Component @@ -37,9 +42,13 @@ public class CollectDataBrandBatchFilter { private static final String VERDICT_QUERY_FAILED = "QUERY_FAILED"; private static final String VERDICT_OK = "OK"; + /** 未显式配置时一次 chunk 的品牌检查预算(毫秒)。 */ + static final long DEFAULT_CHUNK_BUDGET_MILLIS = 10_000L; + private final BrandCheckClient brandCheckClient; private final int batchSize; private final int cacheCapacity; + private final long chunkBudgetMillis; /** 品牌(小写标准化)→ 判定结果;access-order LRU,超限淘汰最旧。 */ private final Map verdictCache; @@ -49,13 +58,19 @@ public class CollectDataBrandBatchFilter { this(brandCheckClient, batchSize, 512); } + public CollectDataBrandBatchFilter(BrandCheckClient brandCheckClient, int batchSize, int cacheCapacity) { + this(brandCheckClient, batchSize, cacheCapacity, DEFAULT_CHUNK_BUDGET_MILLIS); + } + @Autowired public CollectDataBrandBatchFilter(BrandCheckClient brandCheckClient, @Value("${aiimage.collect-data.brand-check-batch-size:10}") int batchSize, - @Value("${aiimage.collect-data.brand-check-cache-capacity:512}") int cacheCapacity) { + @Value("${aiimage.collect-data.brand-check-cache-capacity:512}") int cacheCapacity, + @Value("${aiimage.collect-data.brand-check-chunk-budget-millis:10000}") long chunkBudgetMillis) { this.brandCheckClient = brandCheckClient; this.batchSize = Math.max(1, batchSize); this.cacheCapacity = Math.max(1, cacheCapacity); + this.chunkBudgetMillis = Math.max(0L, chunkBudgetMillis); this.verdictCache = new LinkedHashMap<>(Math.max(16, this.cacheCapacity / 2), 0.75f, true); } @@ -69,6 +84,9 @@ public class CollectDataBrandBatchFilter { if (rows == null || rows.isEmpty()) { return new BrandBatchOutcome(rejected, queryFailed, accepted); } + long deadlineNanos = chunkBudgetMillis > 0L + ? System.nanoTime() + chunkBudgetMillis * 1_000_000L : Long.MAX_VALUE; + int budgetCutRows = 0; for (int start = 0; start < rows.size(); start += batchSize) { int end = Math.min(start + batchSize, rows.size()); List batch = rows.subList(start, end); @@ -96,9 +114,17 @@ public class CollectDataBrandBatchFilter { } continue; } + if (deadlineNanos != Long.MAX_VALUE && System.nanoTime() >= deadlineNanos) { + // 预算用尽:不再为后面的批次捶上游。已缓存的品牌仍按缓存结论归类(不花代价), + // 没结论的行归 queryFailed 入补测队列——既不按 accepted 放行(可能正是侵权品牌), + // 也不按 rejected 拉黑(没查过就不该写进无效品牌表,否则永久误杀)。 + classifyUnchecked(batch, cachedVerdicts, rejected, queryFailed, accepted); + budgetCutRows += batch.size(); + continue; + } Map batchVerdicts = new LinkedHashMap<>(cachedVerdicts); try { - batchVerdicts.putAll(checkAndCache(uncachedBrands)); + batchVerdicts.putAll(checkAndCache(uncachedBrands, remainingBudgetMillis(deadlineNanos))); } catch (RuntimeException ex) { log.warn("[collect-data] brand check batch failed, degrade batch to queryFailed err={}", ex.getMessage()); for (CollectDataResultRowVo row : batch) { @@ -110,12 +136,24 @@ public class CollectDataBrandBatchFilter { } classify(batch, batchVerdicts, rejected, queryFailed, accepted); } + if (budgetCutRows > 0) { + log.info("[collect-data] 品牌检查预算用尽,剩余批次转入补测队列 预算={}ms 延后行={} 总行={}", + chunkBudgetMillis, budgetCutRows, rows.size()); + } return new BrandBatchOutcome(rejected, queryFailed, accepted); } + /** 本批次还能用的毫秒数;0 表示不限制(交给 checkAll 按配置预算走)。 */ + private static long remainingBudgetMillis(long deadlineNanos) { + if (deadlineNanos == Long.MAX_VALUE) { + return 0L; + } + return Math.max(1L, (deadlineNanos - System.nanoTime()) / 1_000_000L); + } + /** 远程检查未缓存品牌并写入缓存;返回新查品牌(小写标准化)→ 判定映射。 */ - private Map checkAndCache(List uncachedBrands) { - BrandCheckClient.BrandCheckBatchResult check = brandCheckClient.checkAll(uncachedBrands, "Terms"); + private Map checkAndCache(List uncachedBrands, long budgetMillis) { + BrandCheckClient.BrandCheckBatchResult check = brandCheckClient.checkAll(uncachedBrands, "Terms", budgetMillis); Set failedBrands = normalizeObjectSet(check == null ? null : check.faildData()); Set queryFailedBrands = normalizeObjectSet(check == null ? null : check.queryFaildData()); Map verdicts = new LinkedHashMap<>(); @@ -170,6 +208,27 @@ public class CollectDataBrandBatchFilter { } } + /** + * 预算用尽后归一仍未检查的批次:有缓存结论的按结论分类,没结论的品牌按 + * QUERY_FAILED 处理(空品牌行仍按 rejected,与已检查批次同口径)。 + */ + private void classifyUnchecked(List batch, Map cachedVerdicts, + List rejected, + List queryFailed, + List accepted) { + Map verdicts = new LinkedHashMap<>(cachedVerdicts); + for (CollectDataResultRowVo row : batch) { + if (row == null) { + continue; + } + String brand = normalizeBrand(row.getBrand()); + if (!brand.isBlank()) { + verdicts.putIfAbsent(brand, VERDICT_QUERY_FAILED); + } + } + classify(batch, verdicts, rejected, queryFailed, accepted); + } + private void classify(List batch, Map verdicts, List rejected, List queryFailed, diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index c43d80218..6d984460d 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -401,6 +401,11 @@ aiimage: max-chunk-rows: ${AIIMAGE_COLLECT_DATA_MAX_CHUNK_ROWS:0} brand-check-batch-size: ${AIIMAGE_COLLECT_DATA_BRAND_CHECK_BATCH_SIZE:10} brand-check-cache-capacity: ${AIIMAGE_COLLECT_DATA_BRAND_CHECK_CACHE_CAPACITY:512} + # 一次分片回传(一个 chunk)里品牌检查的总耗时预算,到期后本 chunk 剩余批次不再发起 + # 远程检查、直接转入品牌补测队列。它是客户端同步等待的时延上界:上游 16890 卡死时, + # 旧的「每批各领一份 aiimage.brand-check.total-timeout-millis(90s)」会随批数线性叠加。 + # 设为 0 退回旧口径。 + brand-check-chunk-budget-millis: ${AIIMAGE_COLLECT_DATA_BRAND_CHECK_CHUNK_BUDGET_MILLIS:10000} invalid-asin-batch-size: ${AIIMAGE_COLLECT_DATA_INVALID_ASIN_BATCH_SIZE:100} result-item-batch-size: ${AIIMAGE_COLLECT_DATA_RESULT_ITEM_BATCH_SIZE:100} # ASIN 采集结果跨任务缓存(详见 AsinResultCacheService)。新鲜度口径放在服务端, diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java index abc5ea0b3..137dce93d 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/client/BrandCheckClientTest.java @@ -106,6 +106,87 @@ class BrandCheckClientTest { return new BrandCheckClient(props, null); } + /** 重试阶梯可调的构造:用于验证「耗时预算掐的是重试,不是首轮」。 */ + private BrandCheckClient newClient(HttpServer server, int retryTimes, int retryIntervalMillis, int totalTimeoutMillis) { + BrandCheckProperties props = new BrandCheckProperties(); + props.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort()); + props.setPath("/brand_check"); + props.setRetryTimes(retryTimes); + props.setRetryIntervalMillis(retryIntervalMillis); + props.setTotalTimeoutMillis(totalTimeoutMillis); + return new BrandCheckClient(props, null); + } + + @Test + void callerBudgetCutsTheRetryLadderButNeverSkipsTheFirstAttempt() throws Exception { + AtomicInteger attempts = new AtomicInteger(); + HttpServer server = startBrandCheckServer(exchange -> { + attempts.incrementAndGet(); + respond(exchange, "{\"faild_data\":[],\"query_faild_data\":[\"Apple\"]}"); + }); + try { + // 阶梯 20/40/60…:跑满 10 次要 0.9 秒,60ms 预算只够跑到第 2~3 次。 + BrandCheckClient unlimited = newClient(server, 10, 20, 0); + unlimited.checkAll(List.of("Apple"), "Terms"); + int attemptsWithoutBudget = attempts.get(); + + attempts.set(0); + BrandCheckClient client = newClient(server, 10, 20, 0); + BrandCheckClient.BrandCheckBatchResult result = client.checkAll(List.of("Apple"), "Terms", 60L); + + assertThat(attemptsWithoutBudget).as("不限制时跑满整条重试阶梯").isEqualTo(10); + assertThat(attempts.get()).as("预算用尽即停重试,但首轮始终执行") + .isBetween(1, attemptsWithoutBudget - 1); + assertThat(result.hasQueryFailedData()).as("掐在预算上仍按查询失败收尾,交调用方降级") + .isTrue(); + } finally { + server.stop(0); + } + } + + @Test + void twoArgCheckAllStillUsesTheConfiguredTotalTimeout() throws Exception { + AtomicInteger attempts = new AtomicInteger(); + HttpServer server = startBrandCheckServer(exchange -> { + attempts.incrementAndGet(); + respond(exchange, "{\"faild_data\":[],\"query_faild_data\":[\"Apple\"]}"); + }); + try { + BrandCheckClient client = newClient(server, 10, 20, 60); + + client.checkAll(List.of("Apple"), "Terms"); + + assertThat(attempts.get()).as("两参重载按 total-timeout-millis 兜底,未退化也不越界") + .isBetween(1, 9); + } finally { + server.stop(0); + } + } + + @Test + void exhaustedBudgetStillIssuesOneRequestPerBrand() throws Exception { + // 预算掐的必须是「重试」而不是首轮:检查线程池只有 3 个坑,第 4 个品牌排到时 1ms 预算 + // 早已过期。若此时连首轮都不发,品牌就在"一次都没查过"的状态下被判降级。 + AtomicInteger requests = new AtomicInteger(); + HttpServer server = startBrandCheckServer(exchange -> { + requests.incrementAndGet(); + respond(exchange, "{\"faild_data\":[],\"query_faild_data\":[\"Apple\"]}"); + }); + try { + // 重试间隔 200ms 保证第四个品牌必然在预算耗尽后才开跑,不与首轮判断抢时序。 + BrandCheckClient client = newClient(server, 3, 200, 0); + List brands = List.of("b1", "b2", "b3", "b4"); + + BrandCheckClient.BrandCheckBatchResult result = client.checkAll(brands, "Terms", 1L); + + assertThat(requests.get()).as("预算过期后每个品牌仍各发一次真实请求").isEqualTo(brands.size()); + assertThat(result.brands()).as("降级不丢品牌,入参集合原样带回").containsExactlyElementsOf(brands); + assertThat(result.hasQueryFailedData()).as("查过但没结论,按查询失败交调用方降级").isTrue(); + } finally { + server.stop(0); + } + } + private HttpServer startBrandCheckServer(com.sun.net.httpserver.HttpHandler handler) throws IOException { HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); server.createContext("/brand_check", handler); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilterTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilterTest.java index 3c2d3dd8f..b89c44109 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilterTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandBatchFilterTest.java @@ -4,14 +4,17 @@ import com.nanri.aiimage.modules.brand.client.BrandCheckClient; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -39,7 +42,7 @@ class CollectDataBrandBatchFilterTest { @Test void test_task_046_brand_normal_default_path() { // 正常输入:非空品牌批次发起检查,失败品牌行 rejected、其余 accepted。 - when(brandCheckClient.checkAll(anyList(), any())).thenReturn( + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn( new BrandCheckClient.BrandCheckBatchResult( List.of(), List.of("Zara"), List.of())); @@ -54,7 +57,7 @@ class CollectDataBrandBatchFilterTest { assertEquals(2, outcome.accepted().size(), "其余行 accepted"); assertEquals("B000000002", outcome.accepted().get(0).getAsin(), "accepted 顺序稳定"); assertEquals(0, outcome.queryFailed().size(), "无查询失败行"); - verify(brandCheckClient).checkAll(anyList(), any()); + verify(brandCheckClient).checkAll(anyList(), any(), anyLong()); } @Test @@ -64,7 +67,7 @@ class CollectDataBrandBatchFilterTest { for (int i = 0; i < 120; i++) { rows.add(row("B" + String.format("%09d", i + 1), i % 10 == 0 ? "bad-brand" : "good-" + (i / 10))); } - when(brandCheckClient.checkAll(anyList(), any())).thenReturn( + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn( new BrandCheckClient.BrandCheckBatchResult( List.of(), List.of("bad-brand"), List.of())); @@ -74,13 +77,13 @@ class CollectDataBrandBatchFilterTest { assertEquals(108, outcome.accepted().size(), "其余行 accepted"); assertEquals("B000000001", outcome.rejected().get(0).getAsin(), "rejected 顺序稳定"); assertEquals("B000000002", outcome.accepted().get(0).getAsin(), "accepted 顺序稳定"); - verify(brandCheckClient, times(12)).checkAll(anyList(), any()); + verify(brandCheckClient, times(12)).checkAll(anyList(), any(), anyLong()); } @Test void test_task_046_brand_normal_repeated_operation_is_idempotent() { // 幂等:同一输入重复执行结果一致,不产生重复对象。 - when(brandCheckClient.checkAll(anyList(), any())).thenReturn( + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn( new BrandCheckClient.BrandCheckBatchResult( List.of(), List.of("Nike"), List.of())); List rows = List.of( @@ -94,7 +97,7 @@ class CollectDataBrandBatchFilterTest { assertEquals(first.rejected(), second.rejected(), "重复执行 rejected 一致"); assertEquals(first.accepted(), second.accepted(), "重复执行 accepted 一致"); // 缓存生效:第二次执行全部命中缓存,不再发起远程调用。 - verify(brandCheckClient, times(1)).checkAll(anyList(), any()); + verify(brandCheckClient, times(1)).checkAll(anyList(), any(), anyLong()); } @Test @@ -103,20 +106,20 @@ class CollectDataBrandBatchFilterTest { CollectDataBrandBatchFilter.BrandBatchOutcome empty = filter.filter(List.of()); assertEquals(0, empty.rejected().size() + empty.accepted().size() + empty.queryFailed().size(), "空列表返回空结果"); - verify(brandCheckClient, never()).checkAll(anyList(), any()); + verify(brandCheckClient, never()).checkAll(anyList(), any(), anyLong()); CollectDataBrandBatchFilter.BrandBatchOutcome blankBatch = filter.filter(List.of( row("B000000001", ""), row("B000000002", null), row("B000000003", " ") )); assertEquals(3, blankBatch.rejected().size(), "空品牌批次行全部 rejected"); assertEquals(0, blankBatch.accepted().size(), "空品牌批次无 accepted"); - verify(brandCheckClient, never()).checkAll(anyList(), any()); + verify(brandCheckClient, never()).checkAll(anyList(), any(), anyLong()); } @Test void test_task_046_brand_boundary_single_item() { // 单元素:单行单品牌批次正常检查;单行空品牌批次不发远程调用。 - when(brandCheckClient.checkAll(anyList(), any())).thenReturn( + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn( new BrandCheckClient.BrandCheckBatchResult( List.of(), List.of(), List.of())); @@ -124,14 +127,14 @@ class CollectDataBrandBatchFilterTest { row("B000000001", "solo") )); assertEquals(1, single.accepted().size(), "单行未命中失败归 accepted"); - verify(brandCheckClient).checkAll(anyList(), any()); + verify(brandCheckClient).checkAll(anyList(), any(), anyLong()); CollectDataBrandBatchFilter.BrandBatchOutcome blank = filter.filter(List.of( row("B000000002", "") )); assertEquals(1, blank.rejected().size(), "单行空品牌归 rejected"); // 同实例缓存生效:solo 已查过,空品牌批次无远程调用,总调用保持 1 次。 - verify(brandCheckClient, times(1)).checkAll(anyList(), any()); + verify(brandCheckClient, times(1)).checkAll(anyList(), any(), anyLong()); } @Test @@ -142,7 +145,7 @@ class CollectDataBrandBatchFilterTest { rows.add(row("B" + String.format("%09d", i + 1), "brand-" + (i % 100))); } rows.add(row("B999999999", " ")); - when(brandCheckClient.checkAll(anyList(), any())).thenReturn( + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn( new BrandCheckClient.BrandCheckBatchResult( List.of(), List.of(), List.of())); @@ -151,13 +154,13 @@ class CollectDataBrandBatchFilterTest { assertEquals(1, outcome.rejected().size(), "空品牌行归 rejected"); assertEquals(5000, outcome.accepted().size(), "非空品牌行全 accepted"); // 每批次 10 行恰好引入 10 个新品牌,10 批次后 100 品牌全部缓存,后续批次零调用。 - verify(brandCheckClient, times(10)).checkAll(anyList(), any()); + verify(brandCheckClient, times(10)).checkAll(anyList(), any(), anyLong()); } @Test void test_task_046_brand_invalid_input_rejected() { // 非法参数:checkAll 返回 null 按无失败处理;null 行安全跳过不计数。 - when(brandCheckClient.checkAll(anyList(), any())).thenReturn(null); + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn(null); List rows = new ArrayList<>(); rows.add(null); @@ -168,13 +171,13 @@ class CollectDataBrandBatchFilterTest { assertEquals(2, outcome.accepted().size(), "null 结果按无失败处理,行全 accepted"); assertEquals(0, outcome.rejected().size(), "null 行不计数"); - verify(brandCheckClient).checkAll(anyList(), any()); + verify(brandCheckClient).checkAll(anyList(), any(), anyLong()); } @Test void test_task_046_brand_dependency_failure_releases_resources() { // 依赖失败:远程检查抛错时批次整组降级 queryFailed,可恢复;恢复后重新检查成功。 - when(brandCheckClient.checkAll(anyList(), any())) + when(brandCheckClient.checkAll(anyList(), any(), anyLong())) .thenThrow(new RuntimeException("brand service down")) .thenReturn(new BrandCheckClient.BrandCheckBatchResult( List.of(), List.of("Zara"), List.of())); @@ -191,6 +194,98 @@ class CollectDataBrandBatchFilterTest { assertTrue(recovered.queryFailed().isEmpty(), "恢复后无残留 queryFailed"); } + /** + * 一次调用(= 一个 chunk)共用一个总耗时预算:预算用尽后剩余批次不再捶上游。 + * 旧口径是每批各领一份 aiimage.brand-check.total-timeout-millis(90s),批数越多、 + * 客户端那次同步的分片回传 POST 等得越久(taskId 28599 实测单批就等了 103.5 秒)。 + */ + @Test + void test_chunk_budget_exhausted_stops_calling_upstream() { + AtomicInteger calls = new AtomicInteger(); + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenAnswer(invocation -> { + calls.incrementAndGet(); + Thread.sleep(30); + return new BrandCheckClient.BrandCheckBatchResult(List.of(), List.of(), List.of()); + }); + CollectDataBrandBatchFilter tight = new CollectDataBrandBatchFilter(brandCheckClient, 2, 512, 1L); + + CollectDataBrandBatchFilter.BrandBatchOutcome outcome = tight.filter(List.of( + row("B000000001", "brand-1"), row("B000000002", "brand-2"), + row("B000000003", "brand-3"), row("B000000004", "brand-4"), + row("B000000005", "brand-5"), row("B000000006", "brand-6"))); + + assertEquals(1, calls.get(), "预算用尽后不再为剩余批次发起远程检查"); + assertEquals(2, outcome.accepted().size(), "首个已检查批次照常分类"); + assertEquals(4, outcome.queryFailed().size(), "剩余批次行全部转入补测队列"); + assertEquals(0, outcome.rejected().size(), "延后行不写无效品牌表(没查过就拉黑 = 永久误杀)"); + } + + @Test + void test_cached_verdict_still_applies_and_blank_brand_rejects_after_budget_exhausted() { + // 预算用尽只做「不再花代价」,不改分类口径:缓存里有结论的仍按结论走, + // 空品牌行仍归 rejected(补测队列按品牌入队,空品牌进去就永远回不来)。 + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenAnswer(invocation -> { + Thread.sleep(30); + List brands = invocation.getArgument(0); + List hits = new ArrayList<>(brands.stream().filter("bad-brand"::equals).toList()); + return new BrandCheckClient.BrandCheckBatchResult(List.of(), hits, List.of()); + }); + CollectDataBrandBatchFilter tight = new CollectDataBrandBatchFilter(brandCheckClient, 2, 512, 1L); + + CollectDataBrandBatchFilter.BrandBatchOutcome outcome = tight.filter(List.of( + row("B000000001", "bad-brand"), row("B000000002", "seen-ok"), + row("B000000003", "bad-brand"), row("B000000004", "brand-4"), + row("B000000005", "brand-5"), row("B000000006", " "))); + + assertEquals(List.of("B000000001", "B000000003", "B000000006"), asins(outcome.rejected()), + "侵权品牌与空品牌行归 rejected"); + assertEquals(List.of("B000000002"), asins(outcome.accepted()), "已判定无侵权的行放行"); + assertEquals(List.of("B000000004", "B000000005"), asins(outcome.queryFailed()), + "只有没查过的品牌延后补测"); + } + + @Test + void test_each_batch_gets_only_the_remaining_chunk_budget() { + // 共用一个截止时刻,而不是每批各领一份预算。 + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenAnswer(invocation -> { + Thread.sleep(40); + return new BrandCheckClient.BrandCheckBatchResult(List.of(), List.of(), List.of()); + }); + CollectDataBrandBatchFilter bounded = new CollectDataBrandBatchFilter(brandCheckClient, 2, 512, 5_000L); + + bounded.filter(List.of( + row("B000000001", "brand-1"), row("B000000002", "brand-2"), + row("B000000003", "brand-3"), row("B000000004", "brand-4"), + row("B000000005", "brand-5"), row("B000000006", "brand-6"))); + + ArgumentCaptor budgets = ArgumentCaptor.forClass(Long.class); + verify(brandCheckClient, times(3)).checkAll(anyList(), any(), budgets.capture()); + List values = budgets.getAllValues(); + assertTrue(values.get(0) <= 5_000L, "首批拿到的不超过 chunk 预算"); + assertTrue(values.get(1) < values.get(0), "第二批只能用剩余的"); + assertTrue(values.get(2) < values.get(1), "预算随批次单调递减"); + } + + @Test + void test_zero_budget_passes_through_as_unlimited() { + // 逃生阀:预算配 0 表示不掐批间,把每批的预算交给 checkAll 按配置走(旧口径)。 + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn( + new BrandCheckClient.BrandCheckBatchResult(List.of(), List.of(), List.of())); + CollectDataBrandBatchFilter unbounded = new CollectDataBrandBatchFilter(brandCheckClient, 2, 512, 0L); + + CollectDataBrandBatchFilter.BrandBatchOutcome outcome = unbounded.filter(List.of( + row("B000000001", "brand-1"), row("B000000002", "brand-2"), row("B000000003", "brand-3"))); + + ArgumentCaptor budgets = ArgumentCaptor.forClass(Long.class); + verify(brandCheckClient, times(2)).checkAll(anyList(), any(), budgets.capture()); + assertEquals(List.of(0L, 0L), budgets.getAllValues(), "预算 0 逐批透传 0(不限制)"); + assertEquals(3, outcome.accepted().size(), "不限制时没有行被延后"); + } + + private static List asins(List rows) { + return rows.stream().map(CollectDataResultRowVo::getAsin).toList(); + } + private static CollectDataResultRowVo row(String asin, String brand) { CollectDataResultRowVo row = new CollectDataResultRowVo(); row.setAsin(asin); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandCacheTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandCacheTest.java index 618544ea0..befd20d5d 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandCacheTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBrandCacheTest.java @@ -13,6 +13,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -40,7 +41,7 @@ class CollectDataBrandCacheTest { @Test void test_task_047_cache_brand_normal_default_path() { // 正常输入:首查品牌发起远程检查并缓存判定,分类正确。 - when(brandCheckClient.checkAll(anyList(), any())).thenReturn( + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn( new BrandCheckClient.BrandCheckBatchResult( List.of(), List.of("Zara"), List.of())); @@ -52,13 +53,13 @@ class CollectDataBrandCacheTest { assertEquals(1, outcome.rejected().size(), "失败品牌行 rejected"); assertEquals("B000000001", outcome.rejected().get(0).getAsin()); assertEquals(1, outcome.accepted().size(), "其余行 accepted"); - verify(brandCheckClient).checkAll(anyList(), any()); + verify(brandCheckClient).checkAll(anyList(), any(), anyLong()); } @Test void test_task_047_cache_brand_normal_multiple_items() { // 批量场景:多 chunk 提交时同品牌只远程检查一次,每 chunk 只查未缓存品牌。 - when(brandCheckClient.checkAll(anyList(), any())).thenReturn( + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn( new BrandCheckClient.BrandCheckBatchResult( List.of(), List.of("Zara"), List.of())); filter.filter(List.of(row("B000000001", "Zara"), row("B000000002", "Nike"))); @@ -70,14 +71,14 @@ class CollectDataBrandCacheTest { assertEquals(1, second.rejected().size(), "chunk2 Zara 命中缓存仍 rejected"); assertEquals(1, second.accepted().size(), "chunk2 新品牌 Adidas accepted"); ArgumentCaptor captor = ArgumentCaptor.forClass(List.class); - verify(brandCheckClient, times(2)).checkAll(captor.capture(), any()); + verify(brandCheckClient, times(2)).checkAll(captor.capture(), any(), anyLong()); assertEquals(List.of("Adidas"), captor.getAllValues().get(1), "第二次只查未缓存品牌"); } @Test void test_task_047_cache_brand_normal_repeated_operation_is_idempotent() { // 幂等:同一输入重复执行结果一致,第二次全部命中缓存不发远程调用。 - when(brandCheckClient.checkAll(anyList(), any())).thenReturn( + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn( new BrandCheckClient.BrandCheckBatchResult( List.of(), List.of("Nike"), List.of())); List rows = List.of( @@ -90,7 +91,7 @@ class CollectDataBrandCacheTest { assertEquals(first.rejected(), second.rejected(), "重复执行 rejected 一致"); assertEquals(first.accepted(), second.accepted(), "重复执行 accepted 一致"); - verify(brandCheckClient, times(1)).checkAll(anyList(), any()); + verify(brandCheckClient, times(1)).checkAll(anyList(), any(), anyLong()); } @Test @@ -99,19 +100,19 @@ class CollectDataBrandCacheTest { CollectDataBrandBatchFilter.BrandBatchOutcome empty = filter.filter(List.of()); assertEquals(0, empty.rejected().size() + empty.accepted().size() + empty.queryFailed().size(), "空列表返回空结果"); - verify(brandCheckClient, never()).checkAll(anyList(), any()); + verify(brandCheckClient, never()).checkAll(anyList(), any(), anyLong()); CollectDataBrandBatchFilter.BrandBatchOutcome blank = filter.filter(List.of( row("B000000001", "") )); assertEquals(1, blank.rejected().size(), "空品牌行 rejected"); - verify(brandCheckClient, never()).checkAll(anyList(), any()); + verify(brandCheckClient, never()).checkAll(anyList(), any(), anyLong()); } @Test void test_task_047_cache_brand_boundary_single_item() { // 单元素:单品牌首查一次;同品牌再出现时命中缓存零调用。 - when(brandCheckClient.checkAll(anyList(), any())).thenReturn( + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn( new BrandCheckClient.BrandCheckBatchResult( List.of(), List.of(), List.of())); @@ -124,14 +125,14 @@ class CollectDataBrandCacheTest { row("B000000002", "solo") )); assertEquals(1, again.accepted().size(), "同品牌再次出现结果一致"); - verify(brandCheckClient, times(1)).checkAll(anyList(), any()); + verify(brandCheckClient, times(1)).checkAll(anyList(), any(), anyLong()); } @Test void test_task_047_cache_brand_boundary_limit_and_overflow() { // 上限/超限:缓存容量 8,首批 10 个品牌淘汰最旧 2 个;第二批同品牌 // 只重查被淘汰的 2 个,其余 8 个命中缓存;无无界增长。 - when(brandCheckClient.checkAll(anyList(), any())).thenReturn( + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn( new BrandCheckClient.BrandCheckBatchResult( List.of(), List.of(), List.of())); CollectDataBrandBatchFilter smallCache = new CollectDataBrandBatchFilter(brandCheckClient, 10, 8); @@ -147,7 +148,7 @@ class CollectDataBrandCacheTest { assertEquals(10, out.accepted().size(), "第二批 10 行全部 accepted"); ArgumentCaptor captor = ArgumentCaptor.forClass(List.class); - verify(brandCheckClient, times(2)).checkAll(captor.capture(), any()); + verify(brandCheckClient, times(2)).checkAll(captor.capture(), any(), anyLong()); List secondCall = captor.getAllValues().get(1); assertEquals(2, secondCall.size(), "仅重查被淘汰的最旧 2 个品牌"); assertTrue(secondCall.containsAll(List.of("brand-0", "brand-1")), "淘汰的是最旧条目"); @@ -156,7 +157,7 @@ class CollectDataBrandCacheTest { @Test void test_task_047_cache_brand_invalid_input_rejected() { // 非法参数:checkAll 返回 null 按无失败处理并缓存;null 行安全跳过。 - when(brandCheckClient.checkAll(anyList(), any())).thenReturn(null); + when(brandCheckClient.checkAll(anyList(), any(), anyLong())).thenReturn(null); List rows = new ArrayList<>(); rows.add(null); @@ -168,13 +169,13 @@ class CollectDataBrandCacheTest { CollectDataBrandBatchFilter.BrandBatchOutcome second = filter.filter(rows); assertEquals(1, second.accepted().size(), "重复执行结果一致"); - verify(brandCheckClient, times(1)).checkAll(anyList(), any()); + verify(brandCheckClient, times(1)).checkAll(anyList(), any(), anyLong()); } @Test void test_task_047_cache_brand_dependency_failure_releases_resources() { // 依赖失败:远程抛错整批 queryFailed 且不缓存;恢复后重查成功,无残留状态。 - when(brandCheckClient.checkAll(anyList(), any())) + when(brandCheckClient.checkAll(anyList(), any(), anyLong())) .thenThrow(new RuntimeException("brand service down")) .thenReturn(new BrandCheckClient.BrandCheckBatchResult( List.of(), List.of("Zara"), List.of())); @@ -188,7 +189,7 @@ class CollectDataBrandCacheTest { assertEquals(1, recovered.rejected().size(), "恢复后重新检查并分类"); assertEquals(1, recovered.accepted().size(), "恢复后 accepted 正确"); assertTrue(recovered.queryFailed().isEmpty(), "恢复后无残留 queryFailed"); - verify(brandCheckClient, times(2)).checkAll(anyList(), any()); + verify(brandCheckClient, times(2)).checkAll(anyList(), any(), anyLong()); } private static CollectDataResultRowVo row(String asin, String brand) {