From c4fc9784e8be47aab99c82c4fbc31abd6a20e06b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Wed, 28 Oct 2026 19:31:50 +0800 Subject: [PATCH] =?UTF-8?q?feat(collect-data):=20=E7=BB=93=E6=9E=9C=20Exce?= =?UTF-8?q?l=20=E8=BF=BD=E5=8A=A0=E3=80=8C=E8=A2=AB=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E6=98=8E=E7=BB=86=E3=80=8Dsheet=EF=BC=8C=E5=93=81=E7=89=8C?= =?UTF-8?q?=E6=A3=80=E6=B5=8B=E5=A4=B1=E8=B4=A5=E5=8E=9F=E5=9B=A0=E5=8F=AF?= =?UTF-8?q?=E8=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit task 32573 回传 600 行只留 1 行,被砍掉的 599 行在系统里只剩四个计数, 谁都答不出「是哪些 ASIN、死在哪一道」。四类被过滤行现在于提交时当场 逐行落库(V179 biz_collectdata_filtered_row),结果 Excel 追加第 3 个 sheet 供逐行对账。不在装配期回读 chunk 重判——去重库与无效品牌库读的 都是当前态,任务窗口期的写入会让判定漂移(错标/漏标/分类串台), 依据逐条写在 V179 头注释里。 同批:brandQueryFailedCount 与原因进历史 VO 和任务卡片;损耗通知不再对 「未配置代理」的用户承诺稍后重跑——补测队列两轮 21/21、28/28 全败、净增 0 行,那句是谎。 台账:XML_STATEMENT_TOTAL 141→143(新 mapper 两条写侧语句);静默 catch 棘轮 total 1028→1031、不重抛 579→582、只 log 181→182(三枚审计件容错, 逐处记名于 CatchSilenceRatchetTest 上方注释)。 --- .../mapper/CollectDataFilteredRowMapper.java | 21 ++ .../entity/CollectDataFilteredRowEntity.java | 51 ++++ .../model/vo/CollectDataHistoryItemVo.java | 6 + .../CollectDataBrandRecheckInjector.java | 8 + .../CollectDataExcelAssemblyService.java | 136 ++++++++++- .../CollectDataItemCleanupSpiImpl.java | 13 +- .../service/CollectDataService.java | 140 ++++++++++- .../util/CollectDataBatchQuery.java | 24 +- .../util/CollectDataFilteredRowStore.java | 225 ++++++++++++++++++ .../task/spi/CollectDataItemCleanupSpi.java | 14 +- .../src/main/resources/application.yml | 5 + .../db/V179__collectdata_filtered_row.sql | 48 ++++ .../CollectDataFilteredRowMapper.xml | 32 +++ .../aiimage/SqlLayeringConventionTest.java | 5 +- .../config/CatchSilenceRatchetTest.java | 35 ++- .../CollectDataBrandRecheckInjectorTest.java | 12 +- .../service/CollectDataFilteredSheetTest.java | 182 ++++++++++++++ .../CollectDataPartialResultNoticeTest.java | 59 ++++- ...llectDataResultFileReuploadCompatTest.java | 114 +++++++++ .../CollectDataServiceTxBoundaryTest.java | 2 +- .../CollectDataSubmitResultSnapshotTest.java | 2 +- .../util/CollectDataBatchQueryTest.java | 33 +++ .../util/CollectDataFilteredRowStoreTest.java | 183 ++++++++++++++ .../CollectDataSelfConsistencyTest.java | 3 + .../DuplicateSubmissionContractTest.java | 2 +- .../contract/SuccessTimingContractTest.java | 4 +- .../golden/submit-result-snapshot.txt | 16 +- .../brand/components/BrandCollectDataTab.vue | 10 + .../shared/api/types/modules/collect-data.ts | 4 + ...ollect-brand-query-failure-visible.test.ts | 47 ++++ 30 files changed, 1384 insertions(+), 52 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/mapper/CollectDataFilteredRowMapper.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/entity/CollectDataFilteredRowEntity.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataFilteredRowStore.java create mode 100644 backend-java/src/main/resources/db/V179__collectdata_filtered_row.sql create mode 100644 backend-java/src/main/resources/mapper/collectdata/CollectDataFilteredRowMapper.xml create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataFilteredSheetTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataResultFileReuploadCompatTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataFilteredRowStoreTest.java create mode 100644 frontend-vue/tests/collect-brand-query-failure-visible.test.ts diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/mapper/CollectDataFilteredRowMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/mapper/CollectDataFilteredRowMapper.java new file mode 100644 index 000000000..43ea4491e --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/mapper/CollectDataFilteredRowMapper.java @@ -0,0 +1,21 @@ +package com.nanri.aiimage.modules.collectdata.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.Collection; +import java.util.List; + +@Mapper +public interface CollectDataFilteredRowMapper extends BaseMapper { + + /** 一次多值 INSERT;本表无唯一键(同 ASIN 可在不同关键词/分片各被过滤一次),不做 IGNORE。 */ + int insertBatch(@Param("rows") List rows); + + /** 补测回流后给对应行盖 resolved_at(只动未回流的行),返回受影响行数。 */ + int markResolved(@Param("taskId") Long taskId, + @Param("asins") Collection asins, + @Param("detailSuffix") String detailSuffix); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/entity/CollectDataFilteredRowEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/entity/CollectDataFilteredRowEntity.java new file mode 100644 index 000000000..2bc50befc --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/entity/CollectDataFilteredRowEntity.java @@ -0,0 +1,51 @@ +package com.nanri.aiimage.modules.collectdata.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * 采集数据「被过滤行」明细(V179):四类过滤逐行带原因落库,供结果 Excel 的 + * 「被过滤明细」sheet 装配与四类计数对账。 + * + *

必须在提交时当场记,不能装配时回读 chunk 重判——去重与品牌库读的都是当前态, + * 任务窗口期的写入会让判定漂移(错标/漏标/分类串台),依据写在 V179 迁移头注释里。

+ */ +@Data +@TableName("biz_collectdata_filtered_row") +public class CollectDataFilteredRowEntity { + + /** 与历史结果表里的 ASIN 归一化口径一致(大写去空白)。 */ + public static final String REASON_DEDUPE = "DEDUPE"; + /** 命中全平台共享的无效品牌库(2026-10-06 定案不按公司区分)。 */ + public static final String REASON_INVALID_BRAND = "INVALID_BRAND"; + /** 品牌检测判定侵权,行被拒。 */ + public static final String REASON_BRAND_REJECTED = "BRAND_REJECTED"; + /** 品牌检测压根没发起查询(未配置代理 / 上游失败 / 闸已落下),行既不进结果也不判无效。 */ + public static final String REASON_BRAND_QUERY_FAILED = "BRAND_QUERY_FAILED"; + + @TableId(type = IdType.AUTO) + private Long id; + private Long taskId; + /** 产生本行的回传分片序号;0=收尾/未知。 */ + private Integer chunkIndex; + private String asin; + private String keyword; + /** 品牌按 {@code BrandLabel.strip} 的展示口径存储,与「采集数据结果」sheet 同形。 */ + private String brand; + private String reasonCode; + private String reasonDetail; + /** + * 补测回流时间;null = 当时确实没查动、至今没回来。 + * + *

这里不用 {@code status} 字符串列:TaskStatusVocabularyRatchetTest 把"模块里另抄一套 + * {@code STATUS_X = "字面量"}"当成棘轮要消灭的东西,而这列的语义本来就只有"回没回来"。 + * 换成时间戳还多得一个信息——回流发生在哪一轮,{@code RESOLVED} 把它丢了。

+ */ + private LocalDateTime resolvedAt; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataHistoryItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataHistoryItemVo.java index 9558d6542..6c0be07a0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataHistoryItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataHistoryItemVo.java @@ -46,6 +46,12 @@ public class CollectDataHistoryItemVo { @Schema(description = "品牌检测不符合的行数") private Integer brandRejectedCount; + @Schema(description = "品牌检测未返回结果的行数:这些行既不进结果、也没被判为无效品牌") + private Integer brandQueryFailedCount; + + @Schema(description = "品牌检测查不了的原因(如:用户未配置代理),供前端给出可行动指引") + private String brandQueryFailedReason; + @Schema(description = "最终结果行数") private Integer finalRowCount; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataBrandRecheckInjector.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataBrandRecheckInjector.java index a6fa382c2..43e8e9814 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataBrandRecheckInjector.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataBrandRecheckInjector.java @@ -7,6 +7,7 @@ import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo; import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery; import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter; +import com.nanri.aiimage.modules.collectdata.util.CollectDataFilteredRowStore; import com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter; import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailCodec; import com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter; @@ -70,6 +71,8 @@ public class CollectDataBrandRecheckInjector { private final ObjectMapper objectMapper; /** 只为复用 resolveCacheCountry 的单国家判定口径(缓存键缺国家会把 DE 价当 US 结论)。 */ private final CollectDataService collectDataService; + /** 补测把行补进结果后,给 V179 被过滤明细里对应行盖回流时间(保留行,不删)。 */ + private final CollectDataFilteredRowStore filteredRowStore; /** * 把补测结论落回任务:按品牌回读 chunk 全量 payload,分流 CLEAN/REJECTED。 @@ -142,6 +145,11 @@ public class CollectDataBrandRecheckInjector { } if (!acceptedKept.isEmpty()) { writeAsinCache(task, acceptedKept); + // 这些行已补进结果,给 V179 明细里对应的行盖上回流时间。 + // **不删行**:删掉就把「当时确实没查动」这条历史一起删了,而那条历史正是 + // 用户追问"我这批货当时为什么少"时唯一说得清的东西。 + filteredRowStore.markResolved(task.getId(), + acceptedKept.stream().map(CollectDataResultRowVo::getAsin).toList()); } if (acceptedRows > 0 || rejectedRows > 0) { log.info("[brand-recheck] 补测结论注入完成 taskId={} 新增结果行={} 判定无效行={} 品牌(净/败)={}/{}", diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataExcelAssemblyService.java index 399d98943..de9ff64ec 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataExcelAssemblyService.java @@ -5,6 +5,7 @@ import com.nanri.aiimage.common.text.UserMessages; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.excel.SxssfWorkbookWriter; import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto; +import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo; import lombok.extern.slf4j.Slf4j; import org.apache.poi.ss.usermodel.Row; @@ -32,6 +33,13 @@ public class CollectDataExcelAssemblyService { private static final String SHEET_SUMMARY_NAME = "结果文件"; private static final String[] SHEET_SUMMARY_HEADER = {"关键词", "FBA", "FBM", "AMZ", "无配送方式", "页数", "ASIN过滤"}; + /** + * 「被过滤明细」尾 sheet(V179):逐行列出被四道闸砍掉的 ASIN 与它死在哪一道。 + * 必须排在最后——本文件之外的读取方要么只认 sheet 0,要么按 sheet 名认列。 + */ + private static final String SHEET_FILTERED_NAME = "被过滤明细"; + private static final String[] SHEET_FILTERED_HEADER = {"品牌", "ASIN", "关键词", "过滤环节", "说明"}; + /** * 生成采集结果工作簿。 * @@ -44,7 +52,15 @@ public class CollectDataExcelAssemblyService { List items, List summaries, List rawItems) { - writeWorkbookSegmented(outputXlsx, items, summaries, () -> rawItems); + writeWorkbookSegmented(outputXlsx, items, summaries, () -> rawItems, null); + } + + /** 不出「被过滤明细」sheet 的那一档:只关心明细/汇总两表的调用方不必传 null 占位。 */ + public void writeWorkbookSegmented(File outputXlsx, + List items, + List summaries, + Supplier> rawItemsSupplier) { + writeWorkbookSegmented(outputXlsx, items, summaries, rawItemsSupplier, null); } /** @@ -55,7 +71,8 @@ public class CollectDataExcelAssemblyService { public void writeWorkbookSegmented(File outputXlsx, List items, List summaries, - Supplier> rawItemsSupplier) { + Supplier> rawItemsSupplier, + FilteredRowSource filteredSource) { // List 入参统一转成分批来源(单批即全部行),对外行为保持不变。 writeWorkbookStreaming(outputXlsx, batchConsumer -> { @@ -69,7 +86,16 @@ public class CollectDataExcelAssemblyService { if (rawItems != null && !rawItems.isEmpty()) { batchConsumer.accept(rawItems); } - }); + }, + filteredSource); + } + + /** 不出「被过滤明细」sheet 的那一档(同 writeWorkbookSegmented 的 4 参版)。 */ + public long writeWorkbookStreaming(File outputXlsx, + RowBatchSource itemsSource, + List summaries, + RowBatchSource rawItemsSource) { + return writeWorkbookStreaming(outputXlsx, itemsSource, summaries, rawItemsSource, null); } /** @@ -77,12 +103,17 @@ public class CollectDataExcelAssemblyService { * 每批写入 SXSSF 滚动窗口后即可被回收,几十万行结果集不再整体驻留堆内存, * 避免大任务生成结果文件时 OOM。 * - * @return 写入「采集数据结果」sheet 的明细行数(仅统计非 null 行) + * @param filteredSource 「被过滤明细」sheet 的数据源;null 表示不出这张表 + * @return 写入「采集数据结果」sheet 的明细行数(仅统计非 null 行)。 + * 红线:这个返回值只算明细 sheet。「被过滤明细」的行数绝不能累进来—— + * 调用方拿它写 stats.finalRowCount 与 result.rowCount,并据此跑 + * guardDetailRowsWritten 校验「下载 Excel 行数 == 保留行数」。 */ public long writeWorkbookStreaming(File outputXlsx, RowBatchSource itemsSource, List summaries, - RowBatchSource rawItemsSource) { + RowBatchSource rawItemsSource, + FilteredRowSource filteredSource) { if (outputXlsx == null) { throw new BusinessException("生成采集数据 Excel 失败: 输出文件路径为空"); } @@ -93,6 +124,7 @@ public class CollectDataExcelAssemblyService { try { long detailRowCount = writeDetailSheet(writer, itemsSource); writeSummarySheet(writer, summaries, rawItemsSource); + writeFilteredSheet(writer, filteredSource); writer.write(tmpFile); Files.move(tmpFile.toPath(), outputXlsx.toPath(), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); @@ -116,6 +148,18 @@ public class CollectDataExcelAssemblyService { void forEachBatch(Consumer> batchConsumer); } + /** + * 被过滤行(V179)的分批来源。除分批外还要自报「超出上限而未逐条记录」的缺口数—— + * 少了这个数字,sheet 只会安静地少给行,没人知道少给了多少。 + */ + public interface FilteredRowSource { + void forEachBatch(Consumer> batchConsumer); + + default long notRecordedCount() { + return 0L; + } + } + private void deleteQuietly(File file) { try { Files.deleteIfExists(file.toPath()); @@ -241,6 +285,88 @@ public class CollectDataExcelAssemblyService { } } + /** + * 「被过滤明细」尾 sheet。两条硬规矩: + *
    + *
  • 空则不建:一行被过滤都没有时多出一个只有表头的 sheet,读起来像"全被过滤了";
  • + *
  • fail-soft:它是审计件。为它打掉用户已采回的数据交付,是拿可追溯性换交付, + * 所以异常只降级这个 sheet,并在外层明细清单里写明不完整,绝不外抛。
  • + *
+ */ + private void writeFilteredSheet(SxssfWorkbookWriter writer, FilteredRowSource source) { + if (source == null) { + return; + } + Sheet[] created = new Sheet[1]; + int[] rowIndex = new int[]{1}; + boolean incomplete = false; + try { + source.forEachBatch(batch -> { + if (batch == null || batch.isEmpty()) { + return; + } + for (CollectDataFilteredRowEntity item : batch) { + if (item == null) { + continue; + } + if (created[0] == null) { + created[0] = createFilteredSheet(writer); + } + Row row = created[0].createRow(rowIndex[0]++); + SxssfWorkbookWriter.setText(row, 0, BrandLabel.strip(item.getBrand())); + SxssfWorkbookWriter.setText(row, 1, item.getAsin()); + SxssfWorkbookWriter.setText(row, 2, item.getKeyword()); + SxssfWorkbookWriter.setText(row, 3, filteredReasonLabel(item.getReasonCode())); + SxssfWorkbookWriter.setText(row, 4, item.getReasonDetail()); + } + }); + long notRecorded = source.notRecordedCount(); + if (notRecorded > 0) { + // 超出单任务追溯上限:如实说出缺口,而不是安静地少给行 + if (created[0] == null) { + created[0] = createFilteredSheet(writer); + } + Row row = created[0].createRow(rowIndex[0]); + SxssfWorkbookWriter.setText(row, 3, "未逐条记录"); + SxssfWorkbookWriter.setText(row, 4, "另有 " + notRecorded + + " 行被过滤但未逐条记录(超出单任务追溯上限),行数已计入各道闸的计数"); + } + } catch (RuntimeException ex) { + incomplete = true; + log.warn("[collect-data] filtered sheet write failed, deliver without it: {}", ex.getMessage()); + } + if (incomplete && created[0] != null) { + // 这句收尾不再套第二层 catch:它只是一次本地 POI 写,写不动说明整个 workbook + // 都写不动——那由外层 writeWorkbookStreaming 的既有出口处理(与前两张 sheet 同姿势), + // 而不是在这里再吞一层、把"交付没做成"伪装成"交付做成了但清单不完整"。 + Row row = created[0].createRow(rowIndex[0]); + SxssfWorkbookWriter.setText(row, 3, "清单不完整"); + SxssfWorkbookWriter.setText(row, 4, "本 sheet 生成过程中出错,以上仅部分被过滤行"); + } + } + + private Sheet createFilteredSheet(SxssfWorkbookWriter writer) { + Sheet sheet = writer.createSheet(SHEET_FILTERED_NAME, SHEET_FILTERED_HEADER, null); + for (int i = 0; i < SHEET_FILTERED_HEADER.length; i++) { + sheet.setColumnWidth(i, SxssfWorkbookWriter.width(i == 4 ? 46 : 18)); + } + return sheet; + } + + /** 库里存的是 reason_code(机器读),这一列给人读;未知码原样回显,不静默变空。 */ + private static String filteredReasonLabel(String reasonCode) { + if (reasonCode == null || reasonCode.isBlank()) { + return ""; + } + return switch (reasonCode) { + case CollectDataFilteredRowEntity.REASON_DEDUPE -> "已存在于去重总数据库"; + case CollectDataFilteredRowEntity.REASON_INVALID_BRAND -> "命中无效品牌库"; + case CollectDataFilteredRowEntity.REASON_BRAND_REJECTED -> "品牌检测判定侵权"; + case CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED -> "品牌检测未返回结果"; + default -> reasonCode; + }; + } + private String safe(String value) { return value == null ? "" : value; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataItemCleanupSpiImpl.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataItemCleanupSpiImpl.java index 632251391..992218408 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataItemCleanupSpiImpl.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataItemCleanupSpiImpl.java @@ -2,7 +2,9 @@ package com.nanri.aiimage.modules.collectdata.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper; +import com.nanri.aiimage.modules.collectdata.mapper.CollectDataFilteredRowMapper; import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity; +import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity; import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -10,20 +12,27 @@ import org.springframework.stereotype.Service; import java.util.List; /** - * 采集明细行的历史清理实现(2026-09 全维度审查 G5:逻辑从 task 模块收回本模块)。 + * 采集模块名下数据的清理实现(2026-09 全维度审查 G5:逻辑从 task 模块收回本模块)。 + * + *

清理范围是「本模块拥有的行」,不只 biz_collect_data_item:V179 的被过滤行明细同样 + * 归本模块,任务过期后必须一起删掉,否则它只会单向堆积。

*/ @Service @RequiredArgsConstructor public class CollectDataItemCleanupSpiImpl implements CollectDataItemCleanupSpi { private final CollectDataItemMapper collectDataItemMapper; + private final CollectDataFilteredRowMapper filteredRowMapper; @Override public int deleteItemsByTaskIds(List taskIds) { if (taskIds == null || taskIds.isEmpty()) { return 0; } - return collectDataItemMapper.delete(new LambdaQueryWrapper() + int removed = collectDataItemMapper.delete(new LambdaQueryWrapper() .in(CollectDataItemEntity::getTaskId, taskIds)); + removed += filteredRowMapper.delete(new LambdaQueryWrapper() + .in(CollectDataFilteredRowEntity::getTaskId, taskIds)); + return removed; } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java index d1932e598..a8687089f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java @@ -12,6 +12,7 @@ import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.text.BrandLabel; import com.nanri.aiimage.common.excel.ExcelStreamReader; import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService; +import com.nanri.aiimage.modules.brand.support.BrandCheckFailureNotice; import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper; import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper; import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest; @@ -23,6 +24,7 @@ import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitRowDto; import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto; import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataCountryPrefEntity; import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity; +import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryItemVo; @@ -39,6 +41,7 @@ import com.nanri.aiimage.modules.collectdata.service.support.CollectDataStaleTas import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery; import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter; import com.nanri.aiimage.modules.collectdata.util.CollectDataExtraJsonCodec; +import com.nanri.aiimage.modules.collectdata.util.CollectDataFilteredRowStore; import com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter; import com.nanri.aiimage.modules.collectdata.util.CollectDataParseLimits; import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailCodec; @@ -167,6 +170,16 @@ public class CollectDataService extends AbstractFileTaskPipeline字段注入而非塞进构造器:本类在单测里被 5 处手工 {@code new CollectDataService(...)} + * 按位置参数构造,加一个形参等于改 5 个测试的位置实参、且每条用例都要多传一个 null + * (同 {@code BrandCheckClient.taskGuard} 的理由)。生产里它是 @Component,必然存在。

+ */ + @org.springframework.beans.factory.annotation.Autowired(required = false) + private CollectDataFilteredRowStore filteredRowStore; + /** 结果明细 chunk 级编解码:accepted 行按 chunk 共享一个 RustFS 对象。 */ private final CollectDataResultDetailCodec resultDetailCodec; @@ -287,6 +300,14 @@ public class CollectDataService extends AbstractFileTaskPipeline accepted = brandOutcome.accepted(); // 结果明细改为 chunk 级存储:整个 chunk 的 accepted 行共享一个 // RustFS 对象(deterministic key,同 chunk 重提覆盖同一对象), @@ -1174,7 +1206,8 @@ public class CollectDataService extends AbstractFileTaskPipeline streamFinalRows(task.getId(), batch), stats.summaries, - batch -> streamRawRows(task.getId(), batch)); + batch -> streamRawRows(task.getId(), batch), + filteredRowSource(task.getId(), stats)); // 空表头文件不得当作成果交付:先校验再上传(见 guardDetailRowsWritten)。 guardDetailRowsWritten(task.getId(), finalRowCount, stats); String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE); @@ -1230,13 +1263,15 @@ public class CollectDataService extends AbstractFileTaskPipeline 0 && stats.brandQueryFailedCount > 0) { notificationDispatchService.pushPartialResult(task.getUserId(), task.getId(), MODULE_TYPE, task.getTaskNo(), stats.brandQueryFailedCount - + " 行商品的品牌检测未返回结果,这些行不计入结果条数," - + "也没有被判定为无效品牌,可稍后单独重跑这些 ASIN"); + + " 行商品的品牌检测未返回结果,这些行不计入结果条数,也没有被判定为无效品牌。" + + brandQueryFailedCauseForUser(stats.brandQueryFailedReason)); } } finally { FileUtil.del(xlsx); @@ -1463,6 +1498,8 @@ public class CollectDataService extends AbstractFileTaskPipeline原因缺失时**不猜上游**:那句 "classify(空串) 按上游算" 会把一个没观测到的事实 + * 说成"上游商标库暂时不可用"。这里只陈述观测得到的部分——会重试,不指责任何一方。 + */ + static String brandQueryFailedCauseForUser(String rawReason) { + if (rawReason == null || rawReason.isBlank()) { + return "系统会在后续补测轮自动重试。"; + } + return BrandCheckFailureNotice.userMessage(BrandCheckFailureNotice.classify(rawReason)); + } + + /** + * 记录本 chunk 被过滤掉的行(V179)。这份明细是审计件、不参与任何判定,所以 + * writer 缺席(单测按位置参数手工 new 本类)或预算耗尽时**静默不记**:四类计数照常 + * 累加,追溯 sheet 会写明「另有 N 行未逐条记录」,而不是为了审计件打掉用户的交付。 + */ + private int recordFilteredRows(Long taskId, int chunkIndex, CollectDataStats stats, + CollectDataBatchQuery.FilterResult filtered, + CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome) { + if (filteredRowStore == null) { + return 0; + } + int budget = maxFilteredRowsPerTask <= 0 + ? Integer.MAX_VALUE : maxFilteredRowsPerTask - stats.filteredRowsRecorded; + if (budget <= 0) { + return 0; + } + return filteredRowStore.writeBatch(taskId, chunkIndex, + filtered.filteredRows(), brandOutcome.rejected(), brandOutcome.queryFailed(), + brandOutcome.reason(), budget); + } + + /** + * 追溯 sheet 的数据源。缺口数(超出单任务上限而未逐条落库的行)在这里算出来, + * 让 sheet 能如实写「另有 N 行未逐条记录」,而不是安静地少给行。 + */ + private CollectDataExcelAssemblyService.FilteredRowSource filteredRowSource(Long taskId, + CollectDataStats stats) { + if (filteredRowStore == null) { + return null; + } + long notRecorded = filteredNotRecordedCount(stats); + return new CollectDataExcelAssemblyService.FilteredRowSource() { + @Override + public void forEachBatch(Consumer> batchConsumer) { + filteredRowStore.streamBatches(taskId, batchConsumer); + } + + @Override + public long notRecordedCount() { + return notRecorded; + } + }; + } + + /** + * 四类计数之和 − 已逐条落库的行数 = 没记下行的缺口。 + * 负数按 0:writer 跳批只会让缺口变大,出现负数说明别处算错了,不该拿它去 sheet 上写负数。 + */ + static long filteredNotRecordedCount(CollectDataStats stats) { + if (stats == null) { + return 0L; + } + long dropped = (long) stats.dedupeFilteredCount + stats.invalidFilteredCount + + stats.brandRejectedCount + stats.brandQueryFailedCount; + return Math.max(0L, dropped - stats.filteredRowsRecorded); + } + private String normalizeAsin(String value) { return normalize(value).toUpperCase(Locale.ROOT); } @@ -1585,6 +1695,19 @@ public class CollectDataService extends AbstractFileTaskPipeline summaries = new ArrayList<>(); @@ -1624,6 +1747,11 @@ public class CollectDataService extends AbstractFileTaskPipeline() .eq(CollectDataItemEntity::getTaskId, task.getId())); + // 被过滤行随任务一起删;store 抛错就让整个删除回滚,不留孤儿行 + // (它不像写入那条路可以吞——删一半比不删更糟)。 + if (filteredRowStore != null) { + filteredRowStore.deleteByTask(task.getId()); + } fileResultMapper.delete(new LambdaQueryWrapper() .eq(FileResultEntity::getTaskId, task.getId()) .eq(FileResultEntity::getModuleType, MODULE_TYPE)); @@ -1855,6 +1983,8 @@ public class CollectDataService extends AbstractFileTaskPipeline rows, Long ownerUserId) { if (rows == null || rows.isEmpty()) { - return new FilterResult(List.of(), 0, 0); + return new FilterResult(List.of(), 0, 0, List.of()); } List asins = distinctNonBlank(rows.stream() .filter(row -> row != null) @@ -89,6 +90,7 @@ public class CollectDataBatchQuery { } List out = new ArrayList<>(rows.size()); + List filtered = new ArrayList<>(); int dedupeFiltered = 0; int invalidFiltered = 0; for (CollectDataResultRowVo row : rows) { @@ -97,20 +99,34 @@ public class CollectDataBatchQuery { } if (dedupeValues.contains(row.getAsin())) { dedupeFiltered++; + filtered.add(new FilteredRow(row, CollectDataFilteredRowEntity.REASON_DEDUPE)); continue; } String brand = normalizeBrand(row.getBrand()); if (!brand.isBlank() && invalidBrands.contains(brand)) { invalidFiltered++; + filtered.add(new FilteredRow(row, CollectDataFilteredRowEntity.REASON_INVALID_BRAND)); continue; } out.add(row); } - return new FilterResult(out, dedupeFiltered, invalidFiltered); + return new FilterResult(out, dedupeFiltered, invalidFiltered, filtered); } - /** 过滤结果:保留行 + 去重命中行数 + 无效品牌命中行数。 */ - public record FilterResult(List kept, int dedupeFilteredCount, int invalidFilteredCount) { + /** + * 过滤结果:保留行 + 去重命中行数 + 无效品牌命中行数 + **被砍掉的行本身**。 + * + *

被砍行必须在这里当场交出去:本方法的两个判定读的都是当前态(去重库、无效品牌库), + * 事后回读 chunk 重判会漂移。计数与 {@code filteredRows} 由同一次循环产出,天然自洽。 + */ + public record FilterResult(List kept, + int dedupeFilteredCount, + int invalidFilteredCount, + List filteredRows) { + } + + /** 一条被过滤的行与它命中的那一道闸。 */ + public record FilteredRow(CollectDataResultRowVo row, String reasonCode) { } private static List distinctNonBlank(List values) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataFilteredRowStore.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataFilteredRowStore.java new file mode 100644 index 000000000..3bf90a2d7 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/util/CollectDataFilteredRowStore.java @@ -0,0 +1,225 @@ +package com.nanri.aiimage.modules.collectdata.util; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.nanri.aiimage.common.text.BrandLabel; +import com.nanri.aiimage.modules.collectdata.mapper.CollectDataFilteredRowMapper; +import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity; +import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.function.Consumer; +import java.util.regex.Pattern; + +/** + * 被过滤行存取器(V179):把「一个 chunk 到底丢掉了哪些行、各丢在哪一道闸」逐行落库, + * 并在装配结果 Excel 时按批读回来铺「被过滤明细」sheet。 + * + *

写侧容错口径照 {@link CollectDataInvalidAsinBatchWriter}:按批 try/catch,失败只跳批告警, + * 绝不影响回传主流程。这份明细是审计件、不是判定依据——为它打掉用户已采回的数据, + * 是拿可追溯性去换交付。也因此它不是强一致的:对不上账时以 result_json 的四个计数为准。

+ * + *

空 ASIN 的行照记不误。空品牌被判 rejected 的行在黑名单表里根本不留痕 + * ({@code CollectDataInvalidAsinBatchWriter.toEntity} 对空品牌返 null),装配期也无从反查, + * 这一格正是 V179 头注释里点名的「漏标」来源。

+ */ +@Slf4j +@Component +public class CollectDataFilteredRowStore { + + private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+"); + /** reason_detail 与库里 VARCHAR(500) 对齐,留余量给回流追加的那句。 */ + private static final int DETAIL_MAX_LENGTH = 460; + /** 装配读回的分页游标:与结果明细同量级,单批写完即释放。 */ + private static final int READ_PAGE_SIZE = 1000; + + private final CollectDataFilteredRowMapper filteredRowMapper; + private final int batchSize; + + public CollectDataFilteredRowStore(CollectDataFilteredRowMapper filteredRowMapper, + @Value("${aiimage.collect-data.filtered-row-batch-size:200}") int batchSize) { + this.filteredRowMapper = filteredRowMapper; + this.batchSize = Math.max(1, batchSize); + } + + /** + * 记录本 chunk 丢掉的行,返回**实际写入**的行数(被预算挡掉的不计)。 + * + * @param batchFiltered 去重与无效品牌两道闸砍掉的行(原因已带在行上) + * @param brandRejected 品牌检测判侵权的行 + * @param brandQueryFailed 品牌检测没发起查询/没回来的行 + * @param brandFailureReason 品牌侧失败原因(如「用户未配置代理」),来自 BrandBatchOutcome#reason + * @param budget 本任务还能记多少行;<=0 表示只留计数不再记行 + */ + public int writeBatch(Long taskId, int chunkIndex, + List batchFiltered, + List brandRejected, + List brandQueryFailed, + String brandFailureReason, + int budget) { + if (taskId == null || budget <= 0) { + return 0; + } + List entities = new ArrayList<>(); + collectInto(entities, batchFiltered, brandRejected, brandQueryFailed, brandFailureReason, taskId, chunkIndex); + if (entities.isEmpty()) { + return 0; + } + if (entities.size() > budget) { + entities = entities.subList(0, budget); + } + int written = 0; + for (int start = 0; start < entities.size(); start += batchSize) { + List batch = + entities.subList(start, Math.min(start + batchSize, entities.size())); + try { + written += filteredRowMapper.insertBatch(batch); + } catch (RuntimeException ex) { + log.warn("[collect-data] filtered row batch insert failed, skip batch taskId={} chunk={} err={}", + taskId, chunkIndex, ex.getMessage()); + } + } + return written; + } + + /** 按 id 游标分批读回本任务的被过滤行;单批交给 consumer 后即可释放。 */ + public void streamBatches(Long taskId, Consumer> consumer) { + if (taskId == null || consumer == null) { + return; + } + Long cursorId = null; + while (true) { + LambdaQueryWrapper wrapper = + new LambdaQueryWrapper() + .eq(CollectDataFilteredRowEntity::getTaskId, taskId); + if (cursorId != null) { + final Long lastId = cursorId; + wrapper.gt(CollectDataFilteredRowEntity::getId, lastId); + } + wrapper.orderByAsc(CollectDataFilteredRowEntity::getId) + .last("limit " + READ_PAGE_SIZE); + List batch = filteredRowMapper.selectList(wrapper); + if (batch == null || batch.isEmpty()) { + return; + } + consumer.accept(batch); + cursorId = batch.get(batch.size() - 1).getId(); + if (cursorId == null || batch.size() < READ_PAGE_SIZE) { + return; + } + } + } + + /** + * 补测把行补进结果后盖上回流时间,并在原因后追加一句说明。 + * **不删行**——删了就把「当时确实没查动」这条历史一起删掉了。 + */ + public int markResolved(Long taskId, Collection asins) { + if (taskId == null || asins == null || asins.isEmpty()) { + return 0; + } + List normalized = new ArrayList<>(asins.size()); + for (String asin : asins) { + String value = normalize(asin).toUpperCase(Locale.ROOT); + if (!value.isBlank()) { + normalized.add(value); + } + } + if (normalized.isEmpty()) { + return 0; + } + try { + return filteredRowMapper.markResolved(taskId, normalized, ";已由补测回流"); + } catch (RuntimeException ex) { + log.warn("[collect-data] filtered row markResolved failed taskId={} err={}", taskId, ex.getMessage()); + return 0; + } + } + + /** + * 删除本任务的被过滤行。与 {@link #writeBatch} 相反,这里**不吞异常**: + * 写失败只丢一份审计件,删失败却会让任务删一半、留下永久孤儿行。 + * 调用方在删除事务里执行它,让它随事务一起回滚。 + */ + public int deleteByTask(Long taskId) { + if (taskId == null) { + return 0; + } + return filteredRowMapper.delete(new LambdaQueryWrapper() + .eq(CollectDataFilteredRowEntity::getTaskId, taskId)); + } + + private void collectInto(List out, + List batchFiltered, + List brandRejected, + List brandQueryFailed, + String brandFailureReason, Long taskId, int chunkIndex) { + if (batchFiltered != null) { + for (CollectDataBatchQuery.FilteredRow filtered : batchFiltered) { + if (filtered == null || filtered.row() == null) { + continue; + } + out.add(toEntity(taskId, chunkIndex, filtered.row(), filtered.reasonCode(), null)); + } + } + if (brandRejected != null) { + for (CollectDataResultRowVo row : brandRejected) { + if (row == null) { + continue; + } + out.add(toEntity(taskId, chunkIndex, row, + CollectDataFilteredRowEntity.REASON_BRAND_REJECTED, null)); + } + } + if (brandQueryFailed != null) { + for (CollectDataResultRowVo row : brandQueryFailed) { + if (row == null) { + continue; + } + out.add(toEntity(taskId, chunkIndex, row, + CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED, brandFailureReason)); + } + } + } + + private CollectDataFilteredRowEntity toEntity(Long taskId, int chunkIndex, + CollectDataResultRowVo row, + String reasonCode, + String reasonDetail) { + CollectDataFilteredRowEntity entity = new CollectDataFilteredRowEntity(); + entity.setTaskId(taskId); + entity.setChunkIndex(Math.max(0, chunkIndex)); + entity.setAsin(normalize(row.getAsin()).toUpperCase(Locale.ROOT)); + entity.setKeyword(truncate(normalize(row.getKeyword()), 255)); + entity.setBrand(truncate(BrandLabel.strip(row.getBrand()), 255)); + entity.setReasonCode(reasonCode); + entity.setReasonDetail(truncate(reasonDetail, DETAIL_MAX_LENGTH)); + return entity; + } + + private static String truncate(String value, int max) { + if (value == null || value.isBlank()) { + return null; + } + return value.length() <= max ? value : value.substring(0, max); + } + + private static String normalize(String value) { + if (value == null) { + return ""; + } + String normalized = value.replace(String.valueOf((char) 0xFEFF), "") + .replace((char) 0x3000, ' ') + .replace("\r\n", " ") + .replace("\r", " ") + .replace("\n", " ") + .replace("\t", " ") + .trim(); + return WHITESPACE_PATTERN.matcher(normalized).replaceAll(" "); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/CollectDataItemCleanupSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/CollectDataItemCleanupSpi.java index 42e4bae07..2bc0a1a3a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/CollectDataItemCleanupSpi.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/CollectDataItemCleanupSpi.java @@ -3,16 +3,20 @@ package com.nanri.aiimage.modules.task.spi; import java.util.List; /** - * 采集明细行的历史清理扩展点(2026-09 全维度审查 G5)。 + * 采集模块名下数据的清理扩展点(2026-09 全维度审查 G5)。 * - *

任务历史清理时除了 task 自己的表,还要连带删除 biz_collect_data_item 的明细行; - * 该表属于采集模块,原先由 {@code ModuleHistoryCleanupService} 直接使用其 Mapper, - * 构成 task → 业务的依赖。现由采集模块实现本接口。 + *

任务历史清理时除了 task 自己的表,还要连带删除采集模块自己的行;这些表属于采集模块, + * 原先由 {@code ModuleHistoryCleanupService} 直接使用其 Mapper,构成 task → 业务的依赖。 + * 现由采集模块实现本接口。 + * + *

实现方的清理范围是「本模块拥有的全部行」,会随版本增加:目前是 + * biz_collect_data_item 与 V179 的 biz_collectdata_filtered_row。调用方不需要知道有几张表, + * 也不该在 task 模块里为新增的采集表补删除——那等于把收回来的依赖又放回去。 */ public interface CollectDataItemCleanupSpi { /** - * 删除这些任务名下的采集明细行。 + * 删除这些任务名下的采集明细行(含本模块后续的随行数据)。 * * @return 删除行数 */ diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index fe80a12f4..e34ce5c8b 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -488,6 +488,11 @@ aiimage: brand-check-chunk-budget-millis: ${AIIMAGE_COLLECT_DATA_BRAND_CHECK_CHUNK_BUDGET_MILLIS:10000} invalid-asin-batch-size: ${AIIMAGE_COLLECT_DATA_INVALID_ASIN_BATCH_SIZE:100} result-item-batch-size: ${AIIMAGE_COLLECT_DATA_RESULT_ITEM_BATCH_SIZE:100} + filtered-row-batch-size: ${AIIMAGE_COLLECT_DATA_FILTERED_ROW_BATCH_SIZE:200} + # 单任务「被过滤行」逐条落库的上限(V179,供结果 Excel 的「被过滤明细」sheet 追溯)。 + # 超限后只累加计数、不再记行,sheet 会如实写「另有 N 行未逐条记录」而不是静默少给。 + # 参照量级:task 32573 被过滤 599 行、32058 被过滤 2616 行;设为 0 表示不限。 + max-filtered-rows-per-task: ${AIIMAGE_COLLECT_DATA_MAX_FILTERED_ROWS_PER_TASK:50000} # ASIN 采集结果跨任务缓存(详见 AsinResultCacheService)。新鲜度口径放在服务端, # 采集端版本不齐时老客户端只会"问不到就照常开页",不会各按各的理解复用到过期数据。 asin-cache: diff --git a/backend-java/src/main/resources/db/V179__collectdata_filtered_row.sql b/backend-java/src/main/resources/db/V179__collectdata_filtered_row.sql new file mode 100644 index 000000000..2704a1ecf --- /dev/null +++ b/backend-java/src/main/resources/db/V179__collectdata_filtered_row.sql @@ -0,0 +1,48 @@ +-- V179: 采集数据「被过滤行」明细表 +-- +-- 背景(2026-10-28 实证,task 32573 / uid=1210):客户端回传 600 行,结果只留 1 行。 +-- 被砍掉的 599 行在系统里**只剩四个计数**(去重 48 / 黑名单命中 377 / 品牌判侵权 1 / +-- 品牌检测未出结论 173),行内容在 CollectDataBatchQuery.filter 与 +-- CollectDataBrandBatchFilter.filter 里当场丢弃。后果是任何人都答不出 +-- 「到底是哪些 ASIN、死在哪一道」:用户只看到"保留 1",运维只能逐 chunk 翻日志猜。 +-- 本表把这四类被过滤行**逐行带原因**记下来,供结果 Excel 的「被过滤明细」sheet 装配与对账。 +-- +-- 为什么必须在提交时记、不能在装配时回读 chunk 重判(三种错法,均已核到代码): +-- 1. 错标:去重与无效品牌两支查的都是**当前态**。任务窗口期(32573 跑了近 4 小时) +-- 任何后台导入,都会让当时 kept 的行在装配时"命中"。 +-- 2. 漏标:uk_data_value_brand + INSERT IGNORE 使已存在的 (asin,brand) 不落新行(零证据); +-- 空品牌的 rejected 行 toEntity 直接返 null 从不写表,但 classify 把它们算进 rejected。 +-- 3. 分类串台:本任务自己的 rejected 行会先写进 biz_invalid_asin_data,而 filter() 只认 +-- 品牌表——装配期重判会把"当时被黑名单砍"和"本次刚被判侵权"统统归成同一类。 +-- +-- 口径: +-- - 只记事实,不参与任何判定;本表任何一行写失败都**不得**影响回传主流程(writer 按批 +-- try/catch 跳批,与 CollectDataInvalidAsinBatchWriter 同口径),故它不是强一致审计件; +-- - reason_code 四类:DEDUPE / INVALID_BRAND / BRAND_REJECTED / BRAND_QUERY_FAILED; +-- - 补测回流不删行、也不另立 status 词表:resolved_at 从 NULL 变成时间戳就是唯一状态, +-- 并在 reason_detail 追加「已由补测回流」。**不删行**——删了就把"当时确实没查动"这条 +-- 历史一起删掉了; +-- - 体量闸门在应用侧(aiimage.collect-data.max-filtered-rows-per-task),不在库里做。 +-- +-- 影响表与数据量:新建空表。单任务量级参照 32573(599 行)与 32058(2616 行)。 +-- +-- 回滚(手工): +-- DROP TABLE IF EXISTS biz_collectdata_filtered_row; + +CREATE TABLE IF NOT EXISTS biz_collectdata_filtered_row ( + id BIGINT NOT NULL AUTO_INCREMENT COMMENT 'primary key', + task_id BIGINT NOT NULL COMMENT 'biz_file_task.id(COLLECT_DATA 模块)', + chunk_index INT NOT NULL DEFAULT 0 COMMENT '产生本行的回传分片序号(0=收尾/未知)', + asin VARCHAR(32) NOT NULL DEFAULT '' COMMENT '被过滤的 ASIN(归一化大写,与结果 sheet 同口径)', + keyword VARCHAR(255) NULL COMMENT '所属关键词', + brand VARCHAR(255) NULL COMMENT '品牌(BrandLabel.strip 展示口径,与明细 sheet 同形)', + reason_code VARCHAR(24) NOT NULL COMMENT 'DEDUPE / INVALID_BRAND / BRAND_REJECTED / BRAND_QUERY_FAILED', + reason_detail VARCHAR(500) NULL COMMENT '可展示的原因(如「用户未配置代理」)', + resolved_at DATETIME NULL COMMENT '补测回流时间;NULL=当时没查动且至今没回来(本表的唯一状态)', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time', + PRIMARY KEY (id), + KEY idx_filtered_task_reason (task_id, reason_code) COMMENT '装配 sheet 与四类计数对账', + KEY idx_filtered_task_asin (task_id, asin) COMMENT '补测回流按 ASIN 定位未回流行' +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = DYNAMIC + COMMENT = '采集数据被过滤行明细(四类过滤逐行带原因,供结果 Excel 追溯与对账)'; diff --git a/backend-java/src/main/resources/mapper/collectdata/CollectDataFilteredRowMapper.xml b/backend-java/src/main/resources/mapper/collectdata/CollectDataFilteredRowMapper.xml new file mode 100644 index 000000000..58f84a13c --- /dev/null +++ b/backend-java/src/main/resources/mapper/collectdata/CollectDataFilteredRowMapper.xml @@ -0,0 +1,32 @@ + + + + + + + INSERT INTO biz_collectdata_filtered_row + (task_id, chunk_index, asin, keyword, brand, reason_code, reason_detail) + VALUES + + (#{row.taskId}, #{row.chunkIndex}, #{row.asin}, + #{row.keyword,jdbcType=VARCHAR}, #{row.brand,jdbcType=VARCHAR}, + #{row.reasonCode}, #{row.reasonDetail,jdbcType=VARCHAR}) + + + + + + UPDATE biz_collectdata_filtered_row + SET resolved_at = NOW(), + reason_detail = CONCAT(IFNULL(reason_detail, ''), #{detailSuffix}) + WHERE task_id = #{taskId} + AND resolved_at IS NULL + AND asin IN + + #{asin} + + + + diff --git a/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java b/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java index 63aea7a30..5b91d9e01 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java @@ -85,7 +85,10 @@ class SqlLayeringConventionTest { // 130 → 140:关键词使用记录管理端(keywordusage 模块,1 份 XML 10 条只读 select)新写语句。 // 140 → 141:工作台客户端全量同步新增 {@code KeywordShareMapper#upsertBatchFromSync} // (词本体 UPSERT,把客户端的四级类目写进既有行),同轮登记进 NOT_MIGRATED。 - private static final int XML_STATEMENT_TOTAL = 141; + // 141 → 143:V179 采集「被过滤行」新表 {@code CollectDataFilteredRowMapper} 的两条写侧语句 + // (insertBatch 多值插入 + markResolved 盖回流时间)。该 mapper 没有注解时代, + // 不在 MigratedSqlEquivalenceTest 的基准账本里 ⇒ 不占 NOT_MIGRATED,只在这本总分母上记名。 + private static final int XML_STATEMENT_TOTAL = 143; /** * wrapper 里夹带函数级 SQL 的现状("这不是简单单表"的证据),文件名 → 条数: diff --git a/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java b/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java index 86d2678ab..7c056cba6 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java @@ -54,7 +54,9 @@ class CatchSilenceRatchetTest { // 10-27 「整表都要跑完」专项抬 573 -> 577(2 枚新增不重抛 + 2 枚既有整批抛出改逐条跳过,逐处记名见 STRICT_TOTAL 上方)。 // 10-27 同日第二批抬 577 -> 579(批量入库逐批容错+folder 去重逐条目容错,两处都是 log.warn+计数+continue, // 逐处记名见 STRICT_TOTAL 上方注释)。 - static final int GIVEN_NOT_RETHROW = 579; + // 10-28 V179 采集被过滤行追溯抬 579 -> 582(审计件三枚:逐批插入容错/回流簿记容错/尾 sheet 降级, + // 逐处记名见 STRICT_TOTAL 上方注释)。 + static final int GIVEN_NOT_RETHROW = 582; static final int GIVEN_LOG_ONLY = 249; static final int GIVEN_EMPTY = 56; @@ -181,8 +183,24 @@ class CatchSilenceRatchetTest { * 因为 {@code TaskRetryEligibility}/{@code SimilarAsinStaleReopenSupport} 会按 errorMessage * 字面量分类,写进去等于替一条正常任务编一个失败理由。 * 同批实测 total 1028、rethrow 393(未动)、notRethrow 579、logOnly 181(未动)、empty 56(未动)。 + * + *

10-28 采集「被过滤行」追溯(V179,结果 Excel 第 3 个 sheet)抬 total 1028 -> 1031,逐处记名。 + * 三枚都是审计件不得打掉交付那一形,块体各只有一句带 {@code [collect-data]} 锚点的 WARN: + *

    + *
  • {@code CollectDataFilteredRowStore.writeBatch} 一枚:被过滤行按批 INSERT,坏批点名后 + * 其余批照常落库(口径照 {@code CollectDataInvalidAsinBatchWriter})。这份明细不参与任何 + * 判定,为它打掉用户已采回的数据是拿可追溯性换交付;跳过的批会从四类计数与已落库行数 + * 的差里冒出来,sheet 上写「另有 N 行未逐条记录」,不是静默少给。
  • + *
  • {@code CollectDataFilteredRowStore.markResolved} 一枚:补测回流只盖时间戳+追加一句。 + * 必要性是实测的——{@code CollectDataBrandRecheckScheduler} 抓住 {@code apply()} 的抛出后 + * 整组留队重放,这里若外抛,一次簿记失败会让已注入的任务永远重排。
  • + *
  • {@code CollectDataExcelAssemblyService.writeFilteredSheet} 一枚:尾 sheet 读不动就交付 + * 前两表、缺这张审计表。同批我先写的第二枚("清单不完整"那行的内层 catch)已删掉: + * 那只是本地 POI 写,写不动说明整份 workbook 都写不动,该由 + * {@code writeWorkbookStreaming} 的既有出口处理,与前两张 sheet 同姿势。
  • + *
*/ - static final int STRICT_TOTAL = 1028; + static final int STRICT_TOTAL = 1031; /** * 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。 * 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。 @@ -282,8 +300,13 @@ class CatchSilenceRatchetTest { * 10-27 同日第二批再抬 577 -> 579(2 枚逐批/逐条目容错:{@code QueryAsinService.flushInsert} * 与 {@code DedupeRunService.packageFolderDedupeResultsAsZip},块体都是 log.warn+计数, * 坏项点名、其余照常交付,逐处记名见 STRICT_TOTAL 上方注释)。 + * 10-28 再抬 579 -> 582(V179 采集「被过滤行」三枚审计件容错:{@code CollectDataFilteredRowStore.writeBatch} + * 逐批插入、{@code CollectDataFilteredRowStore.markResolved} 回流簿记、 + * {@code CollectDataExcelAssemblyService.writeFilteredSheet} 尾 sheet 降级;三处都是带 + * {@code [collect-data]} 锚点的 log.warn+善后,块体除第一枚外各带一句状态置位/返回值, + * 逐处记名见 STRICT_TOTAL 上方注释)。 */ - static final int STRICT_NOT_RETHROW = 579; + static final int STRICT_NOT_RETHROW = 582; /** * 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}: * 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出—— @@ -303,8 +326,12 @@ class CatchSilenceRatchetTest { * 与 {@code getAllFileAggregates} 的逐条跳过出口——块体各只有一句带 {@code [brand-storage]} * 锚点的 WARN。两处此前是「一个文件坏 ⇒ 整批抛出」,改后其余文件照常返回,坏的那条点名可查; * 同批 {@code rebuildAggregate} 与外观专利候选收集因块内另有计数/continue 只进 NOT_RETHROW。 + * 10-28 抬 181 -> 182(V179 采集「被过滤行」的 {@code CollectDataFilteredRowStore.writeBatch}): + * 块体只有一句带 {@code [collect-data]} 锚点的 WARN(taskId+chunk+err),坏批点名后其余批照常 + * 落库。同批另两处({@code markResolved} 追加 return、{@code writeFilteredSheet} 置 incomplete 位) + * 因块内多一句只进 NOT_RETHROW。 */ - static final int STRICT_LOG_ONLY = 181; // 严格口径:块内每条顶层语句都是 log 调用才算 + static final int STRICT_LOG_ONLY = 182; // 严格口径:块内每条顶层语句都是 log 调用才算 static final int STRICT_EMPTY = 56; /** diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataBrandRecheckInjectorTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataBrandRecheckInjectorTest.java index 19c8ee677..0222c56c5 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataBrandRecheckInjectorTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataBrandRecheckInjectorTest.java @@ -60,6 +60,8 @@ class CollectDataBrandRecheckInjectorTest { private AsinResultCacheService asinResultCacheService; @Mock private CollectDataService collectDataService; + @Mock + private com.nanri.aiimage.modules.collectdata.util.CollectDataFilteredRowStore filteredRowStore; private final CollectDataResultDetailCodec codec = new CollectDataResultDetailCodec(new ObjectMapper()); private CollectDataBrandRecheckInjector injector; @@ -72,7 +74,8 @@ class CollectDataBrandRecheckInjectorTest { TaskChunkEntity.class); injector = new CollectDataBrandRecheckInjector(taskChunkMapper, fileResultMapper, collectDataBatchQuery, codec, resultItemBatchWriter, invalidAsinBatchWriter, - transientPayloadStorageService, asinResultCacheService, new ObjectMapper(), collectDataService); + transientPayloadStorageService, asinResultCacheService, new ObjectMapper(), collectDataService, + filteredRowStore); } private FileTaskEntity task(long id) { @@ -120,7 +123,7 @@ class CollectDataBrandRecheckInjectorTest { when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(payload); CollectDataResultRowVo kept = row("B01", "Nike"); when(collectDataBatchQuery.filter(anyList(), any())).thenReturn( - new CollectDataBatchQuery.FilterResult(List.of(kept), 1, 0)); + new CollectDataBatchQuery.FilterResult(List.of(kept), 1, 0, List.of())); when(transientPayloadStorageService.storeResultPayload(eq("COLLECT_DATA"), eq(100L), eq("hash1"), eq("recheck-chunk-1"), anyString())).thenReturn("ptr2"); when(resultItemBatchWriter.upsertAccepted(eq(100L), eq(9L), eq("s1"), eq(1), @@ -133,6 +136,9 @@ class CollectDataBrandRecheckInjectorTest { assertThat(outcome.acceptedRows()).isEqualTo(1); assertThat(outcome.rejectedRows()).isZero(); verify(asinResultCacheService).writeCollectDataRows(eq("US"), eq("100"), eq(List.of(kept))); + // 补回结果的行要在 V179 明细里标 RESOLVED——但只标不删:删掉就把「当时确实没查动」 + // 这条历史一起删了,而它是用户日后追问"我那批货当时为什么少"时唯一说得清的东西。 + verify(filteredRowStore).markResolved(eq(100L), eq(List.of("B01"))); } @Test @@ -180,7 +186,7 @@ class CollectDataBrandRecheckInjectorTest { "[{\"asin\":\"B01\",\"brand\":\"Nike\"}]"); // 补测窗口内该 ASIN 已被其他任务采走:复查全滤掉 when(collectDataBatchQuery.filter(anyList(), any())).thenReturn( - new CollectDataBatchQuery.FilterResult(List.of(), 1, 0)); + new CollectDataBatchQuery.FilterResult(List.of(), 1, 0, List.of())); GroupOutcome outcome = injector.apply(task(100L), Map.of("nike", RecheckVerdict.CLEAN)); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataFilteredSheetTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataFilteredSheetTest.java new file mode 100644 index 000000000..1a58b2624 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataFilteredSheetTest.java @@ -0,0 +1,182 @@ +package com.nanri.aiimage.modules.collectdata.service; + +import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto; +import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity; +import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.nio.file.Path; +import java.util.List; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 「被过滤明细」尾 sheet(V179)的形状钉子。 + * + *

背景:task 32573 回传 600 行、结果只剩 1 行,被砍的 599 行在系统里只有四个计数, + * 谁也无法回答"到底是哪些 ASIN、死在哪一道"。这张 sheet 就是为了让它可追溯。

+ * + *

三件事必须钉住:① 它排在最后(外部读取方要么只认 sheet 0、要么按 sheet 名认列); + * ② 它的行数**绝不能**累进 writeWorkbookStreaming 的返回值——调用方拿那个数写 + * stats.finalRowCount 与 result.rowCount,并据此跑 guardDetailRowsWritten 校验 + * 「下载 Excel 行数 == 保留行数」,一旦污染就等于把交付说谎重新引回来; + * ③ 一行被过滤都没有时不建这张表,只留表头会让人误以为"全被过滤了"。

+ */ +class CollectDataFilteredSheetTest { + + @TempDir + Path tempDir; + + private final CollectDataExcelAssemblyService assembly = new CollectDataExcelAssemblyService(); + + private static CollectDataResultRowVo detailRow(String asin) { + CollectDataResultRowVo row = new CollectDataResultRowVo(); + row.setBrand("nike"); + row.setAsin(asin); + row.setKeyword("phone case"); + row.setPrice("19.90"); + row.setSellerName("seller-A"); + row.setDeliveryMethod("FBA"); + return row; + } + + private static CollectDataSummaryRowDto summary() { + CollectDataSummaryRowDto item = new CollectDataSummaryRowDto(); + item.setKeyword("phone case"); + item.setFba(1); + return item; + } + + private static CollectDataFilteredRowEntity filtered(String asin, String reasonCode, String detail) { + CollectDataFilteredRowEntity entity = new CollectDataFilteredRowEntity(); + entity.setTaskId(1L); + entity.setChunkIndex(1); + entity.setAsin(asin); + entity.setKeyword("phone case"); + entity.setBrand("somebrand"); + entity.setReasonCode(reasonCode); + entity.setReasonDetail(detail); + return entity; + } + + private static CollectDataExcelAssemblyService.FilteredRowSource sourceOf( + List rows, long notRecorded) { + return new CollectDataExcelAssemblyService.FilteredRowSource() { + @Override + public void forEachBatch(Consumer> batchConsumer) { + if (!rows.isEmpty()) { + batchConsumer.accept(rows); + } + } + + @Override + public long notRecordedCount() { + return notRecorded; + } + }; + } + + @Test + void 尾sheet排在最后且明细行数不被污染() throws Exception { + File output = tempDir.resolve("filtered.xlsx").toFile(); + + long returned = assembly.writeWorkbookStreaming( + output, + consumer -> consumer.accept(List.of(detailRow("B000000001"), detailRow("B000000002"))), + List.of(summary()), + consumer -> { /* 本例不走 fallback 聚合 */ }, + sourceOf(List.of( + filtered("B000000003", CollectDataFilteredRowEntity.REASON_DEDUPE, null), + filtered("B000000004", CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED, "用户未配置代理")), + 0L)); + + // 红线:返回值只算「采集数据结果」那 2 行,被过滤的 2 行一个都不能进来 + assertThat(returned).isEqualTo(2L); + + try (Workbook workbook = WorkbookFactory.create(output)) { + assertThat(workbook.getNumberOfSheets()).isEqualTo(3); + assertThat(workbook.getSheetName(0)).isEqualTo("采集数据结果"); + assertThat(workbook.getSheetName(1)).isEqualTo("结果文件"); + assertThat(workbook.getSheetName(2)).isEqualTo("被过滤明细"); + Sheet sheet = workbook.getSheet("被过滤明细"); + // 表头 + 2 行 + assertThat(sheet.getLastRowNum()).isEqualTo(2); + assertThat(sheet.getRow(1).getCell(1).getStringCellValue()).isEqualTo("B000000003"); + assertThat(sheet.getRow(1).getCell(3).getStringCellValue()).isEqualTo("已存在于去重总数据库"); + assertThat(sheet.getRow(2).getCell(3).getStringCellValue()).isEqualTo("品牌检测未返回结果"); + assertThat(sheet.getRow(2).getCell(4).getStringCellValue()).isEqualTo("用户未配置代理"); + } + } + + @Test + void 没有被过滤行时不建这张表() throws Exception { + File output = tempDir.resolve("empty.xlsx").toFile(); + + assembly.writeWorkbookStreaming( + output, + consumer -> consumer.accept(List.of(detailRow("B000000001"))), + List.of(summary()), + consumer -> { }, + sourceOf(List.of(), 0L)); + + try (Workbook workbook = WorkbookFactory.create(output)) { + assertThat(workbook.getNumberOfSheets()).isEqualTo(2); + assertThat(workbook.getSheet("被过滤明细")).isNull(); + } + } + + @Test + void 超出上限未逐条记录时如实写出缺口() throws Exception { + File output = tempDir.resolve("gap.xlsx").toFile(); + + assembly.writeWorkbookStreaming( + output, + consumer -> consumer.accept(List.of(detailRow("B000000001"))), + List.of(summary()), + consumer -> { }, + sourceOf(List.of(filtered("B000000009", + CollectDataFilteredRowEntity.REASON_INVALID_BRAND, null)), 317L)); + + try (Workbook workbook = WorkbookFactory.create(output)) { + Sheet sheet = workbook.getSheet("被过滤明细"); + assertThat(sheet).isNotNull(); + String note = sheet.getRow(2).getCell(4).getStringCellValue(); + assertThat(note).contains("317"); + // 缺口必须说清"行数已计入各道闸的计数",否则读的人会以为又丢了 317 行 + assertThat(note).contains("已计入"); + } + } + + @Test + void 数据源抛错时降级掉这个sheet而不是打掉整份交付() throws Exception { + File output = tempDir.resolve("boom.xlsx").toFile(); + CollectDataExcelAssemblyService.FilteredRowSource broken = new CollectDataExcelAssemblyService.FilteredRowSource() { + @Override + public void forEachBatch(Consumer> batchConsumer) { + throw new IllegalStateException("明细表读不动"); + } + }; + + long returned = assembly.writeWorkbookStreaming( + output, + consumer -> consumer.accept(List.of(detailRow("B000000001"))), + List.of(summary()), + consumer -> { }, + broken); + + assertThat(returned).isEqualTo(1L); + try (Workbook workbook = WorkbookFactory.create(output)) { + // 追溯 sheet 缺了就缺了,前两表照常交付——它是审计件,不是交付物。 + // 若这里改成外抛,一次审计读失败就会把用户已采回的数据一起打掉。 + assertThat(workbook.getNumberOfSheets()).isEqualTo(2); + assertThat(workbook.getSheetName(0)).isEqualTo("采集数据结果"); + assertThat(workbook.getSheetName(1)).isEqualTo("结果文件"); + } + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataPartialResultNoticeTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataPartialResultNoticeTest.java index 7fdcd7170..f1760e4bf 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataPartialResultNoticeTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataPartialResultNoticeTest.java @@ -43,9 +43,15 @@ import static org.mockito.Mockito.when; * 上游的采集代理取不到 IP(余额不足)与 503 —— 代码在这个位置只观测到「品牌检测没返回结果」, * 观测不到原因。写死一个猜测的原因会把排查方向从代理/上游带到自身服务上。 * - *

锁定契约:文案只陈述可观测事实(缺失行数、不计入结果条数、未判无效、可重跑), - * 不指认代码没有依据的具体故障原因;真实原因留在 BrandCheckClient 的 warn 日志里 - * (brand-check 的 lastErr 字段),由运维按日志排查。 + *

锁定契约(2026-10-28 修订):文案只陈述可观测事实(缺失行数、不计入结果条数、未判无效), + * **并且只在这层真的观测得到原因时才指名原因**——品牌检测的失败原因现已由 + * {@code BrandBatchOutcome#reason} 落进 result_json(brandQueryFailedReason),所以 + * 「用户未配置代理」这类有据可查的原因必须写出来并指向「密钥设置→代理设置」。 + * + *

原契约(10-02)之所以禁止指名原因,是因为当时这一层确实观测不到;不是"永远不许说"。 + * 反过来,原因缺失时仍然不猜:不写「超时」「限流」「上游不可用」,只说系统会自动重试。 + * 旧文案那句「可稍后单独重跑这些 ASIN」是必须拆掉的误导——未配置代理时重跑一万次也不会变好 + * (现网 task 32573:补测队列两轮 21/21、28/28 全败、净增 0 行)。 */ @ExtendWith(MockitoExtension.class) class CollectDataPartialResultNoticeTest { @@ -77,13 +83,19 @@ class CollectDataPartialResultNoticeTest { } private FileTaskEntity taskWithQueryFailedRows(int queryFailedRows) { + return taskWithQueryFailedRows(queryFailedRows, null); + } + + private FileTaskEntity taskWithQueryFailedRows(int queryFailedRows, String reason) { FileTaskEntity task = new FileTaskEntity(); task.setId(TASK_ID); task.setModuleType("COLLECT_DATA"); task.setStatus("RUNNING"); task.setTaskNo("CD-29991"); task.setUserId(7L); - task.setResultJson("{\"brandQueryFailedCount\":" + queryFailedRows + "}"); + task.setResultJson("{\"brandQueryFailedCount\":" + queryFailedRows + + (reason == null ? "" : ",\"brandQueryFailedReason\":\"" + reason + "\"") + + "}"); return task; } @@ -103,9 +115,13 @@ class CollectDataPartialResultNoticeTest { } private String stubHappyPathAndCaptureDetail() { - when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7)); + return stubHappyPathAndCaptureDetail(null); + } + + private String stubHappyPathAndCaptureDetail(String reason) { + when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7, reason)); when(fileResultMapper.selectById(5001L)).thenReturn(result()); - when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L); + when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(12L); // 空明细守卫按明细表行数校验:happy path 下表内行数与写出行数一致。 when(taskResultItemMapper.selectCount(any())).thenReturn(12L); when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx"); @@ -136,14 +152,35 @@ class CollectDataPartialResultNoticeTest { assertThat(detail).contains("不计入结果条数"); assertThat(detail).contains("未返回结果"); - assertThat(detail).contains("重跑"); + // 原因缺失时不再承诺「你重跑就好」,只说系统会自动重试 + assertThat(detail).contains("自动重试"); + } + + @Test + void 原因缺失时不猜故障方() { + // classify(空串) 会算成「上游」,那是没观测到的事——这一层不能替它背书 + String detail = stubHappyPathAndCaptureDetail(); + + assertThat(detail).doesNotContain("上游"); + assertThat(detail).doesNotContain("未配置代理"); + } + + @Test + void 未配置代理必须指名原因并指向可行动入口() { + // 旧文案对配置类原因是谎:补测队列两轮 21/21、28/28 全败、净增 0 行, + // 让用户「稍后单独重跑」只会让他白等 + String detail = stubHappyPathAndCaptureDetail("用户未配置代理"); + + assertThat(detail).contains("密钥设置"); + assertThat(detail).contains("代理设置"); + assertThat(detail).doesNotContain("可稍后单独重跑这些 ASIN"); } @Test void 任务已被判失败时不推部分结果通知() { when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7)); when(fileResultMapper.selectById(5001L)).thenReturn(result()); - when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L); + when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(12L); // 空明细守卫按明细表行数校验:happy path 下表内行数与写出行数一致。 when(taskResultItemMapper.selectCount(any())).thenReturn(12L); when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx"); @@ -160,7 +197,7 @@ class CollectDataPartialResultNoticeTest { void 明细缺行时不得上传文件并把任务标成功() { when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(0)); when(fileResultMapper.selectById(5001L)).thenReturn(result()); - when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L); + when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(12L); when(taskResultItemMapper.selectCount(any())).thenReturn(20L); assertThatThrownBy(() -> service.processResultFileJob(job())) @@ -182,7 +219,7 @@ class CollectDataPartialResultNoticeTest { task.setResultJson("{\"finalRowCount\":93}"); when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task); when(fileResultMapper.selectById(5001L)).thenReturn(result()); - when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(0L); + when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(0L); when(taskResultItemMapper.selectCount(any())).thenReturn(0L); assertThatThrownBy(() -> service.processResultFileJob(job())) @@ -199,7 +236,7 @@ class CollectDataPartialResultNoticeTest { task.setResultJson("{}"); when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task); when(fileResultMapper.selectById(5001L)).thenReturn(result()); - when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(0L); + when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(0L); when(taskResultItemMapper.selectCount(any())).thenReturn(0L); when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx"); when(fileTaskMapper.update(isNull(), any())).thenReturn(1); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataResultFileReuploadCompatTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataResultFileReuploadCompatTest.java new file mode 100644 index 000000000..232b2bf7d --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataResultFileReuploadCompatTest.java @@ -0,0 +1,114 @@ +package com.nanri.aiimage.modules.collectdata.service; + +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto; +import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity; +import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo; +import com.nanri.aiimage.modules.publish.service.PublishWorkbookService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 采集结果文件被用户**回传**给别的模块时的兼容性钉子(V179 新增尾 sheet 的主要外溢风险)。 + * + *

结果 Excel 不是终点物:用户会把它再上传到上架/跟价。那两个模块用的是 + * {@code ExcelStreamReader.readAllSheets}(读全部 sheet),所以多一张 sheet 就可能多算一批行。 + * 本用例钉住「加与不加那张 sheet,回传行为逐字一致」。

+ * + *

跟价那一侧({@code PriceTrackTaskService.parseWorkbookAsinRows})的机制是 + * {@code resolveCountryCode(sheetName)} 认不出「被过滤明细」即整表跳过,且首表兜底只在 + * sheetCount==1 时生效——那条路径上的方法是私有的,本用例覆盖不到,按真机验证项处理。

+ */ +class CollectDataResultFileReuploadCompatTest { + + @TempDir + Path tempDir; + + private final CollectDataExcelAssemblyService assembly = new CollectDataExcelAssemblyService(); + private final PublishWorkbookService publishParser = new PublishWorkbookService(); + + private static CollectDataResultRowVo detailRow() { + CollectDataResultRowVo row = new CollectDataResultRowVo(); + row.setBrand("nike"); + row.setAsin("B000000001"); + row.setKeyword("phone case"); + row.setPrice("19.90"); + row.setSellerName("seller-A"); + row.setDeliveryMethod("FBA"); + return row; + } + + private static CollectDataSummaryRowDto summary() { + CollectDataSummaryRowDto item = new CollectDataSummaryRowDto(); + item.setKeyword("phone case"); + item.setFba(1); + return item; + } + + private File assemble(boolean withFilteredSheet) throws Exception { + File output = tempDir.resolve((withFilteredSheet ? "three" : "two") + ".xlsx").toFile(); + CollectDataExcelAssemblyService.FilteredRowSource source = withFilteredSheet + ? filteredSource(entity()) + : null; + assembly.writeWorkbookStreaming(output, + consumer -> consumer.accept(List.of(detailRow())), + List.of(summary()), + consumer -> { }, + source); + return output; + } + + private static CollectDataFilteredRowEntity entity() { + CollectDataFilteredRowEntity entity = new CollectDataFilteredRowEntity(); + entity.setTaskId(1L); + entity.setChunkIndex(1); + entity.setAsin("B000000002"); + entity.setKeyword("phone case"); + entity.setBrand("somebrand"); + entity.setReasonCode(CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED); + entity.setReasonDetail("用户未配置代理"); + return entity; + } + + /** {@link CollectDataExcelAssemblyService.FilteredRowSource} 有两个方法,不是函数式接口,只能匿名类。 */ + private static CollectDataExcelAssemblyService.FilteredRowSource filteredSource( + CollectDataFilteredRowEntity entity) { + return new CollectDataExcelAssemblyService.FilteredRowSource() { + @Override + public void forEachBatch(java.util.function.Consumer> consumer) { + consumer.accept(List.of(entity)); + } + + @Override + public long notRecordedCount() { + return 0L; + } + }; + } + + /** 回传解析的落点:拒绝就说清第几列不匹配,接受就说 "accepted"。 */ + private String outcomeOf(File file) { + try { + PublishWorkbookService.ParsedWorkbook parsed = publishParser.parse(file); + return "accepted rows=" + (parsed == null ? "-" : parsed.rows().size()); + } catch (BusinessException ex) { + return "rejected: " + ex.getMessage(); + } + } + + @Test + void 多出被过滤sheet后上架解析器的回传行为逐字一致() throws Exception { + String twoSheets = outcomeOf(assemble(false)); + String threeSheets = outcomeOf(assemble(true)); + + assertThat(threeSheets).isEqualTo(twoSheets); + // 并且必须是在**第一张表**就被表头校验挡下——它挡在前面,才轮不到新 sheet 出问题 + assertThat(twoSheets).startsWith("rejected: ").contains("表头不匹配").contains("第 1 列"); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataServiceTxBoundaryTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataServiceTxBoundaryTest.java index a0edafabe..d2918057f 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataServiceTxBoundaryTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataServiceTxBoundaryTest.java @@ -142,7 +142,7 @@ class CollectDataServiceTxBoundaryTest { CollectDataResultRowVo row = new CollectDataResultRowVo(); row.setAsin("B0COLLECT1"); lenient().when(collectDataBatchQuery.filter(any(), any())).thenReturn( - new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0)); + new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0, List.of())); lenient().when(brandBatchFilter.filter(any(), any())).thenReturn( new CollectDataBrandBatchFilter.BrandBatchOutcome(List.of(), List.of(), List.of(row), "")); lenient().when(resultItemBatchWriter.upsertAccepted(anyLong(), anyLong(), anyString(), anyInt(), any(), anyString())) diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/support/CollectDataSubmitResultSnapshotTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/support/CollectDataSubmitResultSnapshotTest.java index 4c815ed05..3a0009174 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/support/CollectDataSubmitResultSnapshotTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/support/CollectDataSubmitResultSnapshotTest.java @@ -335,7 +335,7 @@ class CollectDataSubmitResultSnapshotTest { List kept = new ArrayList<>(rows); f.events.add("outsideLock filter rows=" + kept.size() + " owner=" + invocation.getArgument(1) + " dedupe=0 invalid=0"); - return new CollectDataBatchQuery.FilterResult(kept, 0, 0); + return new CollectDataBatchQuery.FilterResult(kept, 0, 0, List.of()); }); when(brandFilter.filter(anyList(), any())).thenAnswer(invocation -> { @SuppressWarnings("unchecked") diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBatchQueryTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBatchQueryTest.java index 20fbcd079..be4fcde13 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBatchQueryTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataBatchQueryTest.java @@ -1,5 +1,6 @@ package com.nanri.aiimage.modules.collectdata.util; +import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity; import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo; import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService; import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper; @@ -218,6 +219,38 @@ class CollectDataBatchQueryTest { verify(dedupeService).findExistingComparableValuesForOwner(List.of("B000000001"), OWNER_USER_ID); } + /** + * 被砍的行必须连同「死在哪一道闸」一起交出去(V179)。 + * 计数与行列表出自同一次循环——两者一旦对不上,追溯 sheet 就是在说谎。 + */ + @Test + void 被砍行连同各自那道闸一起交出且与计数自洽() { + when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())) + .thenReturn(Set.of("B000000001", "B000000004")); + when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("Zara"))); + + CollectDataBatchQuery.FilterResult result = query.filter(List.of( + row("B000000001", "Nike"), + row("B000000002", "Zara"), + row("B000000003", "H&M"), + row("B000000004", "Zara") + ), OWNER_USER_ID); + + assertEquals(2, result.dedupeFilteredCount(), "ASIN 命中去重库的两行"); + assertEquals(1, result.invalidFilteredCount(), "品牌命中无效品牌库的一行"); + assertEquals(result.dedupeFilteredCount() + result.invalidFilteredCount(), + result.filteredRows().size(), "被砍行数必须等于计数之和,不得漂移"); + // B000000004 的品牌也在黑名单里,但去重先判——原因必须落在实际拦下它的那一道闸上 + assertEquals(CollectDataFilteredRowEntity.REASON_DEDUPE, + result.filteredRows().get(0).reasonCode()); + assertEquals(CollectDataFilteredRowEntity.REASON_INVALID_BRAND, + result.filteredRows().get(1).reasonCode()); + assertEquals(CollectDataFilteredRowEntity.REASON_DEDUPE, + result.filteredRows().get(2).reasonCode()); + assertEquals(List.of("B000000003"), + result.kept().stream().map(CollectDataResultRowVo::getAsin).toList(), "保留行不受新字段影响"); + } + private static CollectDataResultRowVo row(String asin, String brand) { CollectDataResultRowVo row = new CollectDataResultRowVo(); row.setAsin(asin); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataFilteredRowStoreTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataFilteredRowStoreTest.java new file mode 100644 index 000000000..073364499 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/util/CollectDataFilteredRowStoreTest.java @@ -0,0 +1,183 @@ +package com.nanri.aiimage.modules.collectdata.util; + +import com.nanri.aiimage.modules.collectdata.mapper.CollectDataFilteredRowMapper; +import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity; +import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +import java.util.Collection; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 被过滤行存取器(V179)的契约钉子。 + * + *

它守的是两件相反的事,弄反任何一件都会把系统推回"数对不上又说不清":

+ *
    + *
  • 写失败只跳批不外抛——这份明细是审计件,为它打掉用户已采回的数据交付不值;
  • + *
  • 删失败必须外抛——删一半会留下永久孤儿行,比不删更糟。
  • + *
+ */ +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class CollectDataFilteredRowStoreTest { + + @Mock + private CollectDataFilteredRowMapper mapper; + + private CollectDataFilteredRowStore store(int batchSize) { + return new CollectDataFilteredRowStore(mapper, batchSize); + } + + private static CollectDataResultRowVo row(String asin, String brand, String keyword) { + CollectDataResultRowVo row = new CollectDataResultRowVo(); + row.setAsin(asin); + row.setBrand(brand); + row.setKeyword(keyword); + return row; + } + + @SuppressWarnings("unchecked") + private List captureInserted(int expectedCalls) { + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(mapper, times(expectedCalls)).insertBatch(captor.capture()); + return captor.getAllValues().stream().flatMap(List::stream).toList(); + } + + @Test + void 三道闸的行各归各的reason_code() { + when(mapper.insertBatch(any())).thenAnswer(inv -> ((Collection) inv.getArgument(0)).size()); + List rejected = List.of(row("B000000005", "Brand: acme", "kw")); + List queryFailed = List.of(row("B000000006", "beta", "kw")); + + int written = store(200).writeBatch(1L, 3, + List.of(new CollectDataBatchQuery.FilteredRow(row("B000000001", "x", "kw"), + CollectDataFilteredRowEntity.REASON_DEDUPE), + new CollectDataBatchQuery.FilteredRow(row("B000000002", "y", "kw"), + CollectDataFilteredRowEntity.REASON_INVALID_BRAND)), + rejected, queryFailed, "用户未配置代理", 100); + + assertThat(written).isEqualTo(4); + List rows = captureInserted(1); + assertThat(rows).extracting(CollectDataFilteredRowEntity::getReasonCode).containsExactly( + CollectDataFilteredRowEntity.REASON_DEDUPE, + CollectDataFilteredRowEntity.REASON_INVALID_BRAND, + CollectDataFilteredRowEntity.REASON_BRAND_REJECTED, + CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED); + assertThat(rows).allSatisfy(entity -> + // 未回流就是 resolvedAt 为 null:本表只有这一个状态,没有第二套 STATUS_ 词表 + assertThat(entity.getResolvedAt()).isNull()); + // 品牌必须过 BrandLabel.strip:出件是用户的下一份上传物,脏值会顺着下载-再上传回流进链路 + assertThat(rows).extracting(CollectDataFilteredRowEntity::getBrand) + .containsExactly("x", "y", "acme", "beta"); + } + + @Test + void 只有品牌查询失败那类带可行动原因() { + when(mapper.insertBatch(any())).thenAnswer(inv -> ((Collection) inv.getArgument(0)).size()); + + store(200).writeBatch(1L, 1, + List.of(new CollectDataBatchQuery.FilteredRow(row("B000000001", "x", "kw"), + CollectDataFilteredRowEntity.REASON_INVALID_BRAND)), + List.of(row("B000000002", "y", "kw")), List.of(row("B000000003", "z", "kw")), + "用户未配置代理", 100); + + List rows = captureInserted(1); + // 黑名单命中/判侵权不需要额外指引,把品牌侧原因错挂到它们身上反而误导 + assertThat(rows.get(0).getReasonDetail()).isNull(); + assertThat(rows.get(1).getReasonDetail()).isNull(); + assertThat(rows.get(2).getReasonDetail()).isEqualTo("用户未配置代理"); + } + + @Test + void 空ASIN的行照记不误() { + // 空品牌被判 rejected 的行在无效品牌表里根本不留痕(toEntity 对空品牌返 null), + // 装配期也无从反查——这一格正是 V179 头注释点名的「漏标」来源,这里不能再丢。 + when(mapper.insertBatch(any())).thenAnswer(inv -> ((Collection) inv.getArgument(0)).size()); + + int written = store(200).writeBatch(1L, 1, List.of(), + List.of(row("", "", "kw")), List.of(), null, 100); + + assertThat(written).isEqualTo(1); + List rows = captureInserted(1); + assertThat(rows).hasSize(1); + assertThat(rows.get(0).getAsin()).isEmpty(); + } + + @Test + void 预算耗尽时不再记行且返回值如实反映写入量() { + when(mapper.insertBatch(any())).thenAnswer(inv -> ((Collection) inv.getArgument(0)).size()); + + int written = store(200).writeBatch(1L, 1, + List.of(new CollectDataBatchQuery.FilteredRow(row("B000000001", "x", "kw"), + CollectDataFilteredRowEntity.REASON_DEDUPE), + new CollectDataBatchQuery.FilteredRow(row("B000000002", "y", "kw"), + CollectDataFilteredRowEntity.REASON_DEDUPE)), + List.of(), List.of(), null, 1); + + assertThat(written).isEqualTo(1); + assertThat(captureInserted(1)).hasSize(1); + assertThat(store(200).writeBatch(1L, 2, + List.of(new CollectDataBatchQuery.FilteredRow(row("B000000003", "z", "kw"), + CollectDataFilteredRowEntity.REASON_DEDUPE)), + List.of(), List.of(), null, 0)).isZero(); + } + + @Test + void 写失败只跳批不外抛() { + // 审计件不值得打掉用户已采回的数据交付 + when(mapper.insertBatch(any())).thenThrow(new RuntimeException("db down")); + List rejected = List.of( + row("B000000001", "a", "kw"), row("B000000002", "b", "kw"), row("B000000003", "c", "kw")); + + assertThat(store(1).writeBatch(1L, 1, List.of(), rejected, List.of(), null, 100)).isZero(); + // 三批各失败各跳过,不中断后续批次 + verify(mapper, times(3)).insertBatch(any()); + } + + @Test + void 删失败必须外抛不留孤儿行() { + // 与写侧相反:删一半比不删更糟,必须让调用方的事务回滚 + when(mapper.delete(any(com.baomidou.mybatisplus.core.conditions.Wrapper.class))) + .thenThrow(new RuntimeException("db down")); + + assertThatThrownBy(() -> store(200).deleteByTask(7L)) + .isInstanceOf(RuntimeException.class) + .hasMessage("db down"); + } + + @Test + void 回流标记归一化ASIN并跳过空值() { + when(mapper.markResolved(anyLong(), anyCollection(), anyString())).thenReturn(2); + + assertThat(store(200).markResolved(7L, List.of(" b000000001 ", "", "B000000002"))).isEqualTo(2); + ArgumentCaptor> captor = ArgumentCaptor.forClass(Collection.class); + verify(mapper).markResolved(anyLong(), captor.capture(), anyString()); + assertThat(captor.getValue()).containsExactly("B000000001", "B000000002"); + } + + @Test + void 空输入不发起任何调用() { + assertThat(store(200).writeBatch(null, 1, List.of(), List.of(), List.of(), null, 100)).isZero(); + assertThat(store(200).markResolved(1L, List.of())).isZero(); + assertThat(store(200).deleteByTask(null)).isZero(); + verify(mapper, never()).insertBatch(any()); + verify(mapper, never()).delete(any(com.baomidou.mybatisplus.core.conditions.Wrapper.class)); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/CollectDataSelfConsistencyTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/CollectDataSelfConsistencyTest.java index 319da2acd..6d8354a4f 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/CollectDataSelfConsistencyTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/CollectDataSelfConsistencyTest.java @@ -93,6 +93,9 @@ class CollectDataSelfConsistencyTest { "resultId", "taskId", "taskNo", "sourceFilename", "resultFilename", "downloadUrl", "taskStatus", "success", "error", "rowCount", "dedupeFilteredCount", "invalidFilteredCount", "brandRejectedCount", "finalRowCount", + // 品牌检测「查不了」的行数与原因:过去这两个字段连 VO 都没出, + // 现网 task 32573 有 173 行这么丢掉,界面上却一格都没有 + "brandQueryFailedCount", "brandQueryFailedReason", "taskType", "progressPercent", "totalRows", "receivedRows", "processedRows", "collectStage", "currentKeyword", "searchCurrentPage", "searchTotalPages", "detailProcessedAsins", "detailTotalAsins", "filters", diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/DuplicateSubmissionContractTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/DuplicateSubmissionContractTest.java index 0fc8f6c39..397d39fea 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/DuplicateSubmissionContractTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/DuplicateSubmissionContractTest.java @@ -139,7 +139,7 @@ class DuplicateSubmissionContractTest { CollectDataResultRowVo row = new CollectDataResultRowVo(); row.setAsin("B0DUP1234"); lenient().when(collectDataBatchQuery.filter(any(), any())).thenReturn( - new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0)); + new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0, List.of())); lenient().when(brandBatchFilter.filter(any(), any())).thenReturn( new CollectDataBrandBatchFilter.BrandBatchOutcome(List.of(), List.of(), List.of(row), "")); lenient().when(resultItemBatchWriter.upsertAccepted(anyLong(), anyLong(), anyString(), anyInt(), any(), anyString())) diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/SuccessTimingContractTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/SuccessTimingContractTest.java index 02de57e03..e557fe51e 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/SuccessTimingContractTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/SuccessTimingContractTest.java @@ -179,8 +179,10 @@ class SuccessTimingContractTest { void noFileGeneratedNoSuccess() { // 方法名同步:实现已由 writeWorkbookSegmented 改为 writeWorkbookStreaming(流式分页读, // 避免几十万行结果集整体驻留堆),测试原先仍在 mock 旧方法名,导致"没抛异常"的假失败。 + // V179 起调用点走 5 参重载(第 5 参是「被过滤明细」sheet 的数据源),桩必须跟着换形, + // 否则严格存根会先报 PotentialStubbingProblem,把"生成失败"这条断言本身吞掉。 doThrow(new IllegalStateException("工作簿生成失败")) - .when(excelAssemblyService).writeWorkbookStreaming(any(), any(), any(), any()); + .when(excelAssemblyService).writeWorkbookStreaming(any(), any(), any(), any(), any()); assertThrows(IllegalStateException.class, () -> service.processResultFileJob(job())); diff --git a/backend-java/src/test/resources/collectdata/golden/submit-result-snapshot.txt b/backend-java/src/test/resources/collectdata/golden/submit-result-snapshot.txt index c11c5c019..0892f692a 100644 --- a/backend-java/src/test/resources/collectdata/golden/submit-result-snapshot.txt +++ b/backend-java/src/test/resources/collectdata/golden/submit-result-snapshot.txt @@ -16,7 +16,7 @@ chunk.insert scope=sub-1 idx=1/3 payload=rustfs:chunk-1 payloadHash=8640b8fb328e scope.selectOne -> none chunk.count -> 0 scope.insert chunkTotal=3 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"} -task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=f635be79fec3 +task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=9f73e6bcec11 task.load id=1001 status=RUNNING module=COLLECT_DATA thrown: none vo: taskId=1001 resultId=2001 chunk=1/3 done=false receivedRows=2 currentChunkRows=2 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=2 taskStatus=RUNNING @@ -41,7 +41,7 @@ scope.selectOne -> none chunk.count -> 0 scope.insert chunkTotal=3 received=0 completed=1 lastError=- stateJson={"phase":"RECEIVED"} result.update id=2001 success=0 rowCount=1 resultFilename=collect-data-1001-result.xlsx resultFileUrl=- fileSize=0 errorMessage=- -task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c +task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=44ebae7009e0 task.load id=1001 status=RUNNING module=COLLECT_DATA thrown: none vo: taskId=1001 resultId=2001 chunk=3/3 done=true receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING @@ -82,7 +82,7 @@ scope.selectOne -> none chunk.count -> 0 scope.insert chunkTotal=1 received=0 completed=1 lastError=- stateJson={"phase":"RECEIVED"} result.update id=2001 success=- rowCount=1 resultFilename=collect-data-1001-result.xlsx resultFileUrl=- fileSize=0 errorMessage=- -task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c +task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=44ebae7009e0 task.load id=1001 status=RUNNING module=COLLECT_DATA thrown: none vo: taskId=1001 resultId=2001 chunk=1/1 done=true receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING @@ -107,10 +107,10 @@ scope.selectOne -> none chunk.count -> 2 scope.insert chunkTotal=3 received=2 completed=0 lastError=Python 采集异常 stateJson={"phase":"RECEIVED"} result.update id=2001 success=0 rowCount=1 resultFilename=- resultFileUrl=- fileSize=- errorMessage=Python 采集异常 -task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=333 resultJson.hash=2f13867b410c +task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=388 resultJson.hash=44ebae7009e0 chunk.count -> 2 result.update id=2001 success=0 rowCount=1 resultFilename=collect-data-1001-result.xlsx resultFileUrl=- fileSize=0 errorMessage=Python 采集异常 -task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=333 resultJson.hash=2f13867b410c +task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=388 resultJson.hash=44ebae7009e0 task.load id=1001 status=FAILED module=COLLECT_DATA thrown: none vo: taskId=1001 resultId=2001 chunk=2/3 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=FAILED @@ -135,7 +135,7 @@ scope.selectOne -> none chunk.count -> 0 scope.insert chunkTotal=2 received=0 completed=0 lastError=Python 采集异常 stateJson={"phase":"RECEIVED"} result.update id=2001 success=0 rowCount=1 resultFilename=- resultFileUrl=- fileSize=- errorMessage=Python 采集异常 -task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=333 resultJson.hash=2f13867b410c +task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=388 resultJson.hash=44ebae7009e0 chunk.count -> 0 task.load id=1001 status=FAILED module=COLLECT_DATA thrown: none @@ -161,7 +161,7 @@ chunk.insert scope=sub-1 idx=1/2 payload=rustfs:chunk-1 payloadHash=a55ecd3678de scope.selectOne -> none chunk.count -> 0 scope.insert chunkTotal=2 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"} -task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c +task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=44ebae7009e0 task.load id=1001 status=RUNNING module=COLLECT_DATA thrown: none vo: taskId=1001 resultId=2001 chunk=1/2 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING @@ -182,7 +182,7 @@ chunk.insert scope=sub-1 idx=1/1 payload=rustfs:chunk-1 payloadHash=e0365b1c5568 scope.selectOne -> none chunk.count -> 0 scope.insert chunkTotal=1 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"} -task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=53ca52c40ab0 +task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=2b4337f9c7d3 task.load id=1001 status=RUNNING module=COLLECT_DATA thrown: none vo: taskId=1001 resultId=2001 chunk=1/1 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=0 taskStatus=RUNNING diff --git a/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue b/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue index 3d81981ba..1d8e3d823 100644 --- a/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue @@ -686,6 +686,14 @@ function toCollectTaskView(item: CollectDataHistoryItem): TaskItemView { const keywordProgress = taskKeywordProgress(item) const stageProgress = taskStageProgress(item) const shortfall = terminalKeywordShortfall(item) + // 品牌检测查不了的行:既不进结果、也没被判无效品牌,过去在界面上一个出口都没有 + // (task 32573 回传 600 行、其中 173 行这么丢了,用户只看到"保留 1")。 + // 这里只陈述事实与原因原文;该做什么的完整指引走通知,避免在 TS 里复刻一份 + // BrandCheckFailureNotice 的归因映射而与之漂移。 + const brandQueryFailedLine = item.brandQueryFailedCount + ? `品牌检测未出结论:${item.brandQueryFailedCount} 行` + + `${item.brandQueryFailedReason ? `(${item.brandQueryFailedReason})` : ''}` + : '' const percent = taskProgressPercent(item) const finishedAt = item.finishedAt ? formatDateTime(item.finishedAt) : isActive ? '进行中' : '-' const extraLines: string[] = isActive @@ -693,12 +701,14 @@ function toCollectTaskView(item: CollectDataHistoryItem): TaskItemView { ...(item.rowCount != null ? [`行数:${item.rowCount}`] : []), `总数据去重过滤:${item.dedupeFilteredCount ?? 0}`, `无效品牌过滤:${item.invalidFilteredCount ?? 0}`, + ...(brandQueryFailedLine ? [brandQueryFailedLine] : []), ...(keywordProgress ? [keywordProgress] : []), ...(stageProgress ? [stageProgress] : []), ] : [ `总数据去重过滤:${item.dedupeFilteredCount ?? 0}`, `无效品牌过滤:${item.invalidFilteredCount ?? 0}`, + ...(brandQueryFailedLine ? [brandQueryFailedLine] : []), ...(item.resultFilename ? [item.resultFilename] : []), ...(taskFilterSummary(item) ? [`筛选统计:${taskFilterSummary(item)}`] : []), // 终态却没跑完全部关键词:把缺口写在卡片上。此前只有那根 25% 的进度条在说话, diff --git a/frontend-vue/src/shared/api/types/modules/collect-data.ts b/frontend-vue/src/shared/api/types/modules/collect-data.ts index d2f01646a..9513bb18e 100644 --- a/frontend-vue/src/shared/api/types/modules/collect-data.ts +++ b/frontend-vue/src/shared/api/types/modules/collect-data.ts @@ -63,6 +63,10 @@ export interface CollectDataHistoryItem { dedupeFilteredCount?: number; invalidFilteredCount?: number; brandRejectedCount?: number; + /** 品牌检测未返回结果的行数:既不进结果、也没被判为无效品牌,过去在界面上完全不可见 */ + brandQueryFailedCount?: number; + /** 查不了的原因(如「用户未配置代理」),用来把用户指向可行动的入口 */ + brandQueryFailedReason?: string; finalRowCount?: number; totalRows?: number; receivedRows?: number; diff --git a/frontend-vue/tests/collect-brand-query-failure-visible.test.ts b/frontend-vue/tests/collect-brand-query-failure-visible.test.ts new file mode 100644 index 000000000..8c8cf9b9b --- /dev/null +++ b/frontend-vue/tests/collect-brand-query-failure-visible.test.ts @@ -0,0 +1,47 @@ +// 品牌检测「查不了」的行必须在任务卡上看得见。 +// +// 现网 task 32573:客户端回传 600 行,其中 173 行因为该账号未配置代理而压根没发起品牌查询, +// 这些行既不进结果、也没被判为无效品牌(CollectDataService 对 rejected/queryFailed 的区分)。 +// 而卡片只有「去重/无效品牌/品牌拒绝/保留」四格——brandQueryFailedCount 连历史 VO 都没出去, +// 前端类型里也没这个字段。于是用户看到的只有"保留 1",173 行去哪了没有任何交代。 +// +// .vue 在本套件里挂不起来(没有 @vue/test-utils,见 task-terminal-completed-behavior.test.ts +// 的说明),所以按源码钉:字段要进类型、卡片两个分支都要真的渲染出来。 +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { fileURLToPath } from 'node:url' +import { dirname, resolve } from 'node:path' + +import { readSource } from './helpers/task-terminal-ast-ruler.ts' + +const here = dirname(fileURLToPath(import.meta.url)) +const root = resolve(here, '..') +const COLLECT = 'src/pages/brand/components/BrandCollectDataTab.vue' +const TYPES = 'src/shared/api/types/modules/collect-data.ts' + +test('品牌检测失败行数与原因必须进历史类型', () => { + const src = readSource(TYPES, root) + assert.match(src, /brandQueryFailedCount\?: number/, + '后端 CollectDataHistoryItemVo 已返回这个字段,前端类型不能再缺') + assert.match(src, /brandQueryFailedReason\?: string/, + '原因也要带上来,卡片才说得出「未配置代理」这类可行动的话') +}) + +test('卡片进行中与终态两个分支都要挂上这一行', () => { + const src = readSource(COLLECT, root) + assert.match(src, /const brandQueryFailedLine = item\.brandQueryFailedCount/, + '要先按行数算出这一行,0 行时不占版面') + const usages = src.match(/\.\.\.\(brandQueryFailedLine \? \[brandQueryFailedLine\] : \[\]\)/g) ?? [] + assert.equal(usages.length, 2, + `进行中/终态两个 extraLines 分支各挂一次,实测 ${usages.length} 处`) +}) + +test('卡片不得再把无效品牌行数叫成「不符合 ASIN 过滤」', () => { + // invalidFilteredCount 是命中无效品牌库的行数,与用户自己设的 ASIN 筛选条件无关。 + // 过去同一张卡上它一会儿叫「不符合 ASIN 过滤」一会儿叫「无效品牌」,两个名字指一个数。 + const src = readSource(COLLECT, root) + assert.doesNotMatch(src, /不符合 ASIN 过滤/, + '这个名字指错了来源,统一为「无效品牌过滤」') + assert.match(src, /无效品牌过滤:\$\{item\.invalidFilteredCount/, + '必须仍读 invalidFilteredCount,只是把名字改对') +})