feat(collect-data): 结果 Excel 追加「被过滤明细」sheet,品牌检测失败原因可见

task 32573 回传 600 行只留 1 行,被砍掉的 599 行在系统里只剩四个计数,
谁都答不出「是哪些 ASIN、死在哪一道」。四类被过滤行现在于提交时当场
逐行落库(V179 biz_collectdata_filtered_row),结果 Excel 追加第 3 个
sheet 供逐行对账。不在装配期回读 chunk 重判——去重库与无效品牌库读的
都是当前态,任务窗口期的写入会让判定漂移(错标/漏标/分类串台),
依据逐条写在 V179 头注释里。

同批:brandQueryFailedCount 与原因进历史 VO 和任务卡片;损耗通知不再对
「未配置代理」的用户承诺稍后重跑——补测队列两轮 21/21、28/28 全败、净增
0 行,那句是谎。

台账:XML_STATEMENT_TOTAL 141→143(新 mapper 两条写侧语句);静默 catch
棘轮 total 1028→1031、不重抛 579→582、只 log 181→182(三枚审计件容错,
逐处记名于 CatchSilenceRatchetTest 上方注释)。
This commit is contained in:
2026-10-28 19:31:50 +08:00
parent 0e2af88184
commit c4fc9784e8
30 changed files with 1384 additions and 52 deletions
@@ -0,0 +1,21 @@
package com.nanri.aiimage.modules.collectdata.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.Collection;
import java.util.List;
@Mapper
public interface CollectDataFilteredRowMapper extends BaseMapper<CollectDataFilteredRowEntity> {
/** 一次多值 INSERT;本表无唯一键(同 ASIN 可在不同关键词/分片各被过滤一次),不做 IGNORE。 */
int insertBatch(@Param("rows") List<CollectDataFilteredRowEntity> rows);
/** 补测回流后给对应行盖 resolved_at(只动未回流的行),返回受影响行数。 */
int markResolved(@Param("taskId") Long taskId,
@Param("asins") Collection<String> asins,
@Param("detailSuffix") String detailSuffix);
}
@@ -0,0 +1,51 @@
package com.nanri.aiimage.modules.collectdata.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
/**
* 采集数据「被过滤行」明细(V179):四类过滤逐行带原因落库,供结果 Excel 的
* 「被过滤明细」sheet 装配与四类计数对账。
*
* <p>必须在提交时当场记,不能装配时回读 chunk 重判——去重与品牌库读的都是当前态,
* 任务窗口期的写入会让判定漂移(错标/漏标/分类串台),依据写在 V179 迁移头注释里。</p>
*/
@Data
@TableName("biz_collectdata_filtered_row")
public class CollectDataFilteredRowEntity {
/** 与历史结果表里的 ASIN 归一化口径一致(大写去空白)。 */
public static final String REASON_DEDUPE = "DEDUPE";
/** 命中全平台共享的无效品牌库(2026-10-06 定案不按公司区分)。 */
public static final String REASON_INVALID_BRAND = "INVALID_BRAND";
/** 品牌检测判定侵权,行被拒。 */
public static final String REASON_BRAND_REJECTED = "BRAND_REJECTED";
/** 品牌检测压根没发起查询(未配置代理 / 上游失败 / 闸已落下),行既不进结果也不判无效。 */
public static final String REASON_BRAND_QUERY_FAILED = "BRAND_QUERY_FAILED";
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
/** 产生本行的回传分片序号;0=收尾/未知。 */
private Integer chunkIndex;
private String asin;
private String keyword;
/** 品牌按 {@code BrandLabel.strip} 的展示口径存储,与「采集数据结果」sheet 同形。 */
private String brand;
private String reasonCode;
private String reasonDetail;
/**
* 补测回流时间;null = 当时确实没查动、至今没回来。
*
* <p>这里不用 {@code status} 字符串列:TaskStatusVocabularyRatchetTest 把"模块里另抄一套
* {@code STATUS_X = "字面量"}"当成棘轮要消灭的东西,而这列的语义本来就只有"回没回来"。
* 换成时间戳还多得一个信息——回流发生在哪一轮,{@code RESOLVED} 把它丢了。</p>
*/
private LocalDateTime resolvedAt;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -46,6 +46,12 @@ public class CollectDataHistoryItemVo {
@Schema(description = "品牌检测不符合的行数")
private Integer brandRejectedCount;
@Schema(description = "品牌检测未返回结果的行数:这些行既不进结果、也没被判为无效品牌")
private Integer brandQueryFailedCount;
@Schema(description = "品牌检测查不了的原因(如:用户未配置代理),供前端给出可行动指引")
private String brandQueryFailedReason;
@Schema(description = "最终结果行数")
private Integer finalRowCount;
@@ -7,6 +7,7 @@ import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
import com.nanri.aiimage.modules.collectdata.util.CollectDataFilteredRowStore;
import com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter;
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailCodec;
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter;
@@ -70,6 +71,8 @@ public class CollectDataBrandRecheckInjector {
private final ObjectMapper objectMapper;
/** 只为复用 resolveCacheCountry 的单国家判定口径(缓存键缺国家会把 DE 价当 US 结论)。 */
private final CollectDataService collectDataService;
/** 补测把行补进结果后,给 V179 被过滤明细里对应行盖回流时间(保留行,不删)。 */
private final CollectDataFilteredRowStore filteredRowStore;
/**
* 把补测结论落回任务:按品牌回读 chunk 全量 payload,分流 CLEAN/REJECTED。
@@ -142,6 +145,11 @@ public class CollectDataBrandRecheckInjector {
}
if (!acceptedKept.isEmpty()) {
writeAsinCache(task, acceptedKept);
// 这些行已补进结果,给 V179 明细里对应的行盖上回流时间。
// **不删行**:删掉就把「当时确实没查动」这条历史一起删了,而那条历史正是
// 用户追问"我这批货当时为什么少"时唯一说得清的东西。
filteredRowStore.markResolved(task.getId(),
acceptedKept.stream().map(CollectDataResultRowVo::getAsin).toList());
}
if (acceptedRows > 0 || rejectedRows > 0) {
log.info("[brand-recheck] 补测结论注入完成 taskId={} 新增结果行={} 判定无效行={} 品牌(净/败)={}/{}",
@@ -5,6 +5,7 @@ import com.nanri.aiimage.common.text.UserMessages;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.excel.SxssfWorkbookWriter;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
@@ -32,6 +33,13 @@ public class CollectDataExcelAssemblyService {
private static final String SHEET_SUMMARY_NAME = "结果文件";
private static final String[] SHEET_SUMMARY_HEADER = {"关键词", "FBA", "FBM", "AMZ", "无配送方式", "页数", "ASIN过滤"};
/**
* 「被过滤明细」尾 sheet(V179):逐行列出被四道闸砍掉的 ASIN 与它死在哪一道。
* 必须排在最后——本文件之外的读取方要么只认 sheet 0,要么按 sheet 名认列。
*/
private static final String SHEET_FILTERED_NAME = "被过滤明细";
private static final String[] SHEET_FILTERED_HEADER = {"品牌", "ASIN", "关键词", "过滤环节", "说明"};
/**
* 生成采集结果工作簿。
*
@@ -44,7 +52,15 @@ public class CollectDataExcelAssemblyService {
List<CollectDataResultRowVo> items,
List<CollectDataSummaryRowDto> summaries,
List<CollectDataResultRowVo> rawItems) {
writeWorkbookSegmented(outputXlsx, items, summaries, () -> rawItems);
writeWorkbookSegmented(outputXlsx, items, summaries, () -> rawItems, null);
}
/** 不出「被过滤明细」sheet 的那一档:只关心明细/汇总两表的调用方不必传 null 占位。 */
public void writeWorkbookSegmented(File outputXlsx,
List<CollectDataResultRowVo> items,
List<CollectDataSummaryRowDto> summaries,
Supplier<List<CollectDataResultRowVo>> rawItemsSupplier) {
writeWorkbookSegmented(outputXlsx, items, summaries, rawItemsSupplier, null);
}
/**
@@ -55,7 +71,8 @@ public class CollectDataExcelAssemblyService {
public void writeWorkbookSegmented(File outputXlsx,
List<CollectDataResultRowVo> items,
List<CollectDataSummaryRowDto> summaries,
Supplier<List<CollectDataResultRowVo>> rawItemsSupplier) {
Supplier<List<CollectDataResultRowVo>> rawItemsSupplier,
FilteredRowSource filteredSource) {
// List 入参统一转成分批来源(单批即全部行),对外行为保持不变。
writeWorkbookStreaming(outputXlsx,
batchConsumer -> {
@@ -69,7 +86,16 @@ public class CollectDataExcelAssemblyService {
if (rawItems != null && !rawItems.isEmpty()) {
batchConsumer.accept(rawItems);
}
});
},
filteredSource);
}
/** 不出「被过滤明细」sheet 的那一档(同 writeWorkbookSegmented 的 4 参版)。 */
public long writeWorkbookStreaming(File outputXlsx,
RowBatchSource itemsSource,
List<CollectDataSummaryRowDto> summaries,
RowBatchSource rawItemsSource) {
return writeWorkbookStreaming(outputXlsx, itemsSource, summaries, rawItemsSource, null);
}
/**
@@ -77,12 +103,17 @@ public class CollectDataExcelAssemblyService {
* 每批写入 SXSSF 滚动窗口后即可被回收,几十万行结果集不再整体驻留堆内存,
* 避免大任务生成结果文件时 OOM。
*
* @return 写入「采集数据结果」sheet 的明细行数(仅统计非 null 行)
* @param filteredSource 「被过滤明细」sheet 的数据源;null 表示不出这张表
* @return 写入「采集数据结果」sheet 的明细行数(仅统计非 null 行)。
* <b>红线</b>:这个返回值只算明细 sheet。「被过滤明细」的行数绝不能累进来——
* 调用方拿它写 stats.finalRowCount 与 result.rowCount,并据此跑
* guardDetailRowsWritten 校验「下载 Excel 行数 == 保留行数」。
*/
public long writeWorkbookStreaming(File outputXlsx,
RowBatchSource itemsSource,
List<CollectDataSummaryRowDto> summaries,
RowBatchSource rawItemsSource) {
RowBatchSource rawItemsSource,
FilteredRowSource filteredSource) {
if (outputXlsx == null) {
throw new BusinessException("生成采集数据 Excel 失败: 输出文件路径为空");
}
@@ -93,6 +124,7 @@ public class CollectDataExcelAssemblyService {
try {
long detailRowCount = writeDetailSheet(writer, itemsSource);
writeSummarySheet(writer, summaries, rawItemsSource);
writeFilteredSheet(writer, filteredSource);
writer.write(tmpFile);
Files.move(tmpFile.toPath(), outputXlsx.toPath(),
StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
@@ -116,6 +148,18 @@ public class CollectDataExcelAssemblyService {
void forEachBatch(Consumer<List<CollectDataResultRowVo>> batchConsumer);
}
/**
* 被过滤行(V179)的分批来源。除分批外还要自报「超出上限而未逐条记录」的缺口数——
* 少了这个数字,sheet 只会安静地少给行,没人知道少给了多少。
*/
public interface FilteredRowSource {
void forEachBatch(Consumer<List<CollectDataFilteredRowEntity>> batchConsumer);
default long notRecordedCount() {
return 0L;
}
}
private void deleteQuietly(File file) {
try {
Files.deleteIfExists(file.toPath());
@@ -241,6 +285,88 @@ public class CollectDataExcelAssemblyService {
}
}
/**
* 「被过滤明细」尾 sheet。两条硬规矩:
* <ul>
* <li><b>空则不建</b>:一行被过滤都没有时多出一个只有表头的 sheet,读起来像"全被过滤了";</li>
* <li><b>fail-soft</b>:它是审计件。为它打掉用户已采回的数据交付,是拿可追溯性换交付,
* 所以异常只降级这个 sheet,并在外层明细清单里写明不完整,绝不外抛。</li>
* </ul>
*/
private void writeFilteredSheet(SxssfWorkbookWriter writer, FilteredRowSource source) {
if (source == null) {
return;
}
Sheet[] created = new Sheet[1];
int[] rowIndex = new int[]{1};
boolean incomplete = false;
try {
source.forEachBatch(batch -> {
if (batch == null || batch.isEmpty()) {
return;
}
for (CollectDataFilteredRowEntity item : batch) {
if (item == null) {
continue;
}
if (created[0] == null) {
created[0] = createFilteredSheet(writer);
}
Row row = created[0].createRow(rowIndex[0]++);
SxssfWorkbookWriter.setText(row, 0, BrandLabel.strip(item.getBrand()));
SxssfWorkbookWriter.setText(row, 1, item.getAsin());
SxssfWorkbookWriter.setText(row, 2, item.getKeyword());
SxssfWorkbookWriter.setText(row, 3, filteredReasonLabel(item.getReasonCode()));
SxssfWorkbookWriter.setText(row, 4, item.getReasonDetail());
}
});
long notRecorded = source.notRecordedCount();
if (notRecorded > 0) {
// 超出单任务追溯上限:如实说出缺口,而不是安静地少给行
if (created[0] == null) {
created[0] = createFilteredSheet(writer);
}
Row row = created[0].createRow(rowIndex[0]);
SxssfWorkbookWriter.setText(row, 3, "未逐条记录");
SxssfWorkbookWriter.setText(row, 4, "另有 " + notRecorded
+ " 行被过滤但未逐条记录(超出单任务追溯上限),行数已计入各道闸的计数");
}
} catch (RuntimeException ex) {
incomplete = true;
log.warn("[collect-data] filtered sheet write failed, deliver without it: {}", ex.getMessage());
}
if (incomplete && created[0] != null) {
// 这句收尾不再套第二层 catch:它只是一次本地 POI 写,写不动说明整个 workbook
// 都写不动——那由外层 writeWorkbookStreaming 的既有出口处理(与前两张 sheet 同姿势),
// 而不是在这里再吞一层、把"交付没做成"伪装成"交付做成了但清单不完整"。
Row row = created[0].createRow(rowIndex[0]);
SxssfWorkbookWriter.setText(row, 3, "清单不完整");
SxssfWorkbookWriter.setText(row, 4, "本 sheet 生成过程中出错,以上仅部分被过滤行");
}
}
private Sheet createFilteredSheet(SxssfWorkbookWriter writer) {
Sheet sheet = writer.createSheet(SHEET_FILTERED_NAME, SHEET_FILTERED_HEADER, null);
for (int i = 0; i < SHEET_FILTERED_HEADER.length; i++) {
sheet.setColumnWidth(i, SxssfWorkbookWriter.width(i == 4 ? 46 : 18));
}
return sheet;
}
/** 库里存的是 reason_code(机器读),这一列给人读;未知码原样回显,不静默变空。 */
private static String filteredReasonLabel(String reasonCode) {
if (reasonCode == null || reasonCode.isBlank()) {
return "";
}
return switch (reasonCode) {
case CollectDataFilteredRowEntity.REASON_DEDUPE -> "已存在于去重总数据库";
case CollectDataFilteredRowEntity.REASON_INVALID_BRAND -> "命中无效品牌库";
case CollectDataFilteredRowEntity.REASON_BRAND_REJECTED -> "品牌检测判定侵权";
case CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED -> "品牌检测未返回结果";
default -> reasonCode;
};
}
private String safe(String value) {
return value == null ? "" : value;
}
@@ -2,7 +2,9 @@ package com.nanri.aiimage.modules.collectdata.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataFilteredRowMapper;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
@@ -10,20 +12,27 @@ import org.springframework.stereotype.Service;
import java.util.List;
/**
* 采集明细行的历史清理实现(2026-09 全维度审查 G5:逻辑从 task 模块收回本模块)。
* 采集模块名下数据的清理实现(2026-09 全维度审查 G5:逻辑从 task 模块收回本模块)。
*
* <p>清理范围是「本模块拥有的行」,不只 biz_collect_data_item:V179 的被过滤行明细同样
* 归本模块,任务过期后必须一起删掉,否则它只会单向堆积。</p>
*/
@Service
@RequiredArgsConstructor
public class CollectDataItemCleanupSpiImpl implements CollectDataItemCleanupSpi {
private final CollectDataItemMapper collectDataItemMapper;
private final CollectDataFilteredRowMapper filteredRowMapper;
@Override
public int deleteItemsByTaskIds(List<Long> taskIds) {
if (taskIds == null || taskIds.isEmpty()) {
return 0;
}
return collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
int removed = collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
.in(CollectDataItemEntity::getTaskId, taskIds));
removed += filteredRowMapper.delete(new LambdaQueryWrapper<CollectDataFilteredRowEntity>()
.in(CollectDataFilteredRowEntity::getTaskId, taskIds));
return removed;
}
}
@@ -12,6 +12,7 @@ import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.text.BrandLabel;
import com.nanri.aiimage.common.excel.ExcelStreamReader;
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
import com.nanri.aiimage.modules.brand.support.BrandCheckFailureNotice;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
@@ -23,6 +24,7 @@ import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitRowDto;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataCountryPrefEntity;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryItemVo;
@@ -39,6 +41,7 @@ import com.nanri.aiimage.modules.collectdata.service.support.CollectDataStaleTas
import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
import com.nanri.aiimage.modules.collectdata.util.CollectDataExtraJsonCodec;
import com.nanri.aiimage.modules.collectdata.util.CollectDataFilteredRowStore;
import com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter;
import com.nanri.aiimage.modules.collectdata.util.CollectDataParseLimits;
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailCodec;
@@ -167,6 +170,16 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
/** invalid ASIN 批量写入器:按批次 INSERT IGNORE,替代逐行插入。 */
private final CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
/**
* 被过滤行存取器(V179):把每个 chunk 丢掉的行逐条落库,并在装配时读回铺追溯 sheet。
*
* <p>字段注入而非塞进构造器:本类在单测里被 5 处手工 {@code new CollectDataService(...)}
* 按位置参数构造,加一个形参等于改 5 个测试的位置实参、且每条用例都要多传一个 null
* (同 {@code BrandCheckClient.taskGuard} 的理由)。生产里它是 @Component,必然存在。</p>
*/
@org.springframework.beans.factory.annotation.Autowired(required = false)
private CollectDataFilteredRowStore filteredRowStore;
/** 结果明细 chunk 级编解码:accepted 行按 chunk 共享一个 RustFS 对象。 */
private final CollectDataResultDetailCodec resultDetailCodec;
@@ -287,6 +300,14 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
@Value("${aiimage.collect-data.max-chunk-rows:0}")
private Integer maxChunkRows;
/**
* 单任务「被过滤行」逐条落库的上限;<=0 表示不限。
* 现网量级:task 32058 被过滤 2616 行、32573 被过滤 599 行。超限后只留计数,
* 追溯 sheet 会如实写「另有 N 行未逐条记录」而不是静默少给。
*/
@Value("${aiimage.collect-data.max-filtered-rows-per-task:50000}")
private int maxFilteredRowsPerTask;
/** 进度统计节流窗口(毫秒):窗口内 changed 心跳合并写,避免高频 task UPDATE。0 关闭节流。 */
@Value("${aiimage.collect-data.progress-throttle-millis:15000}")
private long progressThrottleMillis;
@@ -951,6 +972,12 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
stats.invalidFilteredCount += filtered.invalidFilteredCount();
stats.brandRejectedCount += brandOutcome.rejected().size();
stats.brandQueryFailedCount += brandOutcome.queryFailed().size();
// 只在本 chunk 真的产出 queryFailed 时才记原因;后续全好的 chunk 不该把
// 已经失败那批的原因抹掉,也不该拿一个空原因去覆盖可行动的那句。
if (!brandOutcome.queryFailed().isEmpty()
&& brandOutcome.reason() != null && !brandOutcome.reason().isBlank()) {
stats.brandQueryFailedReason = brandOutcome.reason();
}
// 只有被服务端判定为无效品牌(rejected)的行才写入无效品牌表;
// queryFailed 是品牌检测服务端到端失败(如缺 X-Token 422/超时/限流),
// 一并写入会把故障期间被误伤的品牌永久拉黑,后续同品牌商品全部被
@@ -960,6 +987,11 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
if (!brandOutcome.rejected().isEmpty()) {
invalidAsinBatchWriter.writeBatch(brandOutcome.rejected(), task.getUserId());
}
// 被过滤行逐条落库(V179):四类行此刻都在手上,过了这道就只剩四个计数了。
// 放在 rejected 写黑名单之后,是为了让「判侵权」与「命中黑名单」各归各的
// reason_code,不被事后回读混成一档(依据见 V179 头注释)。
stats.filteredRowsRecorded += recordFilteredRows(task.getId(), chunkIndex, stats,
filtered, brandOutcome);
List<CollectDataResultRowVo> accepted = brandOutcome.accepted();
// 结果明细改为 chunk 级存储:整个 chunk 的 accepted 行共享一个
// RustFS 对象(deterministic key,同 chunk 重提覆盖同一对象),
@@ -1174,7 +1206,8 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
xlsx,
batch -> streamFinalRows(task.getId(), batch),
stats.summaries,
batch -> streamRawRows(task.getId(), batch));
batch -> streamRawRows(task.getId(), batch),
filteredRowSource(task.getId(), stats));
// 空表头文件不得当作成果交付:先校验再上传(见 guardDetailRowsWritten)。
guardDetailRowsWritten(task.getId(), finalRowCount, stats);
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
@@ -1230,13 +1263,15 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
}
// 品牌查询失败的行不进结果、也不落无效品牌表(见上面 rejected/queryFailed 的区分),
// 用户端因此完全没有出口:只在任务确实翻成成功时补一条通知。
// 文案只写这一层观测得到的事实(未返回结果);真实原因(代理取不到 IP / 上游 503 / 被限流)
// 只有 BrandCheckClient 的 brand-check warn 日志知道,写进通知会把排查方向带偏。
// 这句过去固定写成「可稍后单独重跑这些 ASIN」——对配置类原因是谎:未配置代理时
// 补测队列几轮都是全败(现网 task 32573 两轮 21/21、28/28,净增 0 行),
// 重跑只会让用户白等。原因由 chunk 回传时的 BrandBatchOutcome#reason 落进 stats,
// 这里按它给出可行动的那句。
if (updated > 0 && stats.brandQueryFailedCount > 0) {
notificationDispatchService.pushPartialResult(task.getUserId(), task.getId(), MODULE_TYPE,
task.getTaskNo(), stats.brandQueryFailedCount
+ " 行商品的品牌检测未返回结果,这些行不计入结果条数,"
+ "也没有被判定为无效品牌,可稍后单独重跑这些 ASIN");
+ " 行商品的品牌检测未返回结果,这些行不计入结果条数,也没有被判定为无效品牌。"
+ brandQueryFailedCauseForUser(stats.brandQueryFailedReason));
}
} finally {
FileUtil.del(xlsx);
@@ -1463,6 +1498,8 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
stats.invalidFilteredCount = root.path("invalidFilteredCount").asInt(0);
stats.brandRejectedCount = root.path("brandRejectedCount").asInt(0);
stats.brandQueryFailedCount = root.path("brandQueryFailedCount").asInt(0);
stats.brandQueryFailedReason = root.path("brandQueryFailedReason").asText(null);
stats.filteredRowsRecorded = root.path("filteredRowsRecorded").asInt(0);
stats.finalRowCount = root.path("finalRowCount").asInt(0);
// 反序列化 Python 携带的关键词级聚合;解析失败保持空 list,仅 warn 不抛,避免影响其他统计
JsonNode summariesNode = root.get("summaries");
@@ -1499,6 +1536,8 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
payload.put("invalidFilteredCount", stats.invalidFilteredCount);
payload.put("brandRejectedCount", stats.brandRejectedCount);
payload.put("brandQueryFailedCount", stats.brandQueryFailedCount);
payload.put("brandQueryFailedReason", stats.brandQueryFailedReason);
payload.put("filteredRowsRecorded", stats.filteredRowsRecorded);
payload.put("finalRowCount", stats.finalRowCount);
// 持久化 Python 携带的关键词级聚合;后续 processResultFileJob 阶段读取用于落 Excel
payload.put("summaries", stats.summaries == null ? List.of() : stats.summaries);
@@ -1521,6 +1560,77 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
return Math.max(currentProcessed, Math.min(summaries.size(), totalRows));
}
/**
* 品牌检测失败时该让用户去做什么。归因文案与补测队列共用 {@link BrandCheckFailureNotice}
* 那一份,不在这里另写一套。
*
* <p>原因缺失时**不猜上游**:那句 "classify(空串) 按上游算" 会把一个没观测到的事实
* 说成"上游商标库暂时不可用"。这里只陈述观测得到的部分——会重试,不指责任何一方。
*/
static String brandQueryFailedCauseForUser(String rawReason) {
if (rawReason == null || rawReason.isBlank()) {
return "系统会在后续补测轮自动重试。";
}
return BrandCheckFailureNotice.userMessage(BrandCheckFailureNotice.classify(rawReason));
}
/**
* 记录本 chunk 被过滤掉的行(V179)。这份明细是审计件、不参与任何判定,所以
* writer 缺席(单测按位置参数手工 new 本类)或预算耗尽时**静默不记**:四类计数照常
* 累加,追溯 sheet 会写明「另有 N 行未逐条记录」,而不是为了审计件打掉用户的交付。
*/
private int recordFilteredRows(Long taskId, int chunkIndex, CollectDataStats stats,
CollectDataBatchQuery.FilterResult filtered,
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome) {
if (filteredRowStore == null) {
return 0;
}
int budget = maxFilteredRowsPerTask <= 0
? Integer.MAX_VALUE : maxFilteredRowsPerTask - stats.filteredRowsRecorded;
if (budget <= 0) {
return 0;
}
return filteredRowStore.writeBatch(taskId, chunkIndex,
filtered.filteredRows(), brandOutcome.rejected(), brandOutcome.queryFailed(),
brandOutcome.reason(), budget);
}
/**
* 追溯 sheet 的数据源。缺口数(超出单任务上限而未逐条落库的行)在这里算出来,
* 让 sheet 能如实写「另有 N 行未逐条记录」,而不是安静地少给行。
*/
private CollectDataExcelAssemblyService.FilteredRowSource filteredRowSource(Long taskId,
CollectDataStats stats) {
if (filteredRowStore == null) {
return null;
}
long notRecorded = filteredNotRecordedCount(stats);
return new CollectDataExcelAssemblyService.FilteredRowSource() {
@Override
public void forEachBatch(Consumer<List<CollectDataFilteredRowEntity>> batchConsumer) {
filteredRowStore.streamBatches(taskId, batchConsumer);
}
@Override
public long notRecordedCount() {
return notRecorded;
}
};
}
/**
* 四类计数之和 − 已逐条落库的行数 = 没记下行的缺口。
* 负数按 0:writer 跳批只会让缺口变大,出现负数说明别处算错了,不该拿它去 sheet 上写负数。
*/
static long filteredNotRecordedCount(CollectDataStats stats) {
if (stats == null) {
return 0L;
}
long dropped = (long) stats.dedupeFilteredCount + stats.invalidFilteredCount
+ stats.brandRejectedCount + stats.brandQueryFailedCount;
return Math.max(0L, dropped - stats.filteredRowsRecorded);
}
private String normalizeAsin(String value) {
return normalize(value).toUpperCase(Locale.ROOT);
}
@@ -1585,6 +1695,19 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
private int invalidFilteredCount;
private int brandRejectedCount;
private int brandQueryFailedCount;
/**
* 品牌检测查不了的原因(供应商拒付/用户未配置代理/出口被拒/批次异常)。
* 由 {@code CollectDataBrandBatchFilter.BrandBatchOutcome#reason} 带上来,
* 多个 chunk 按"最后一次非空为准"覆盖(与 summaries 同口径)。
* 它存在的意义是让用户看到**该做什么**,而不是"稍后重跑"——未配置代理时重跑一万次
* 也不会变好(现网 task 32573 的补测队列两轮 21/21、28/28 全败、净增 0 行)。
*/
private String brandQueryFailedReason;
/**
* 已逐条落进 biz_collectdata_filtered_row 的行数。四类计数之和减去它就是
* 「超预算没记下行」的缺口,追溯 sheet 靠这句说实话。
*/
private int filteredRowsRecorded;
private int finalRowCount;
// Python 在 done=true 时携带的关键词级聚合统计;空表示由 Java 端用 rawItems 自聚合兜底
private List<CollectDataSummaryRowDto> summaries = new ArrayList<>();
@@ -1624,6 +1747,11 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
.forEach(resultPayloadJsons::add);
collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
.eq(CollectDataItemEntity::getTaskId, task.getId()));
// 被过滤行随任务一起删;store 抛错就让整个删除回滚,不留孤儿行
// (它不像写入那条路可以吞——删一半比不删更糟)。
if (filteredRowStore != null) {
filteredRowStore.deleteByTask(task.getId());
}
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, task.getId())
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
@@ -1855,6 +1983,8 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
vo.setBrandRejectedCount(stats.brandRejectedCount);
vo.setBrandQueryFailedCount(stats.brandQueryFailedCount);
vo.setBrandQueryFailedReason(stats.brandQueryFailedReason);
vo.setFinalRowCount(stats.finalRowCount);
vo.setTotalRows(stats.totalRows);
vo.setReceivedRows(stats.receivedRows);
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.collectdata.util;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService;
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
@@ -49,7 +50,7 @@ public class CollectDataBatchQuery {
*/
public FilterResult filter(List<CollectDataResultRowVo> rows, Long ownerUserId) {
if (rows == null || rows.isEmpty()) {
return new FilterResult(List.of(), 0, 0);
return new FilterResult(List.of(), 0, 0, List.of());
}
List<String> asins = distinctNonBlank(rows.stream()
.filter(row -> row != null)
@@ -89,6 +90,7 @@ public class CollectDataBatchQuery {
}
List<CollectDataResultRowVo> out = new ArrayList<>(rows.size());
List<FilteredRow> filtered = new ArrayList<>();
int dedupeFiltered = 0;
int invalidFiltered = 0;
for (CollectDataResultRowVo row : rows) {
@@ -97,20 +99,34 @@ public class CollectDataBatchQuery {
}
if (dedupeValues.contains(row.getAsin())) {
dedupeFiltered++;
filtered.add(new FilteredRow(row, CollectDataFilteredRowEntity.REASON_DEDUPE));
continue;
}
String brand = normalizeBrand(row.getBrand());
if (!brand.isBlank() && invalidBrands.contains(brand)) {
invalidFiltered++;
filtered.add(new FilteredRow(row, CollectDataFilteredRowEntity.REASON_INVALID_BRAND));
continue;
}
out.add(row);
}
return new FilterResult(out, dedupeFiltered, invalidFiltered);
return new FilterResult(out, dedupeFiltered, invalidFiltered, filtered);
}
/** 过滤结果:保留行 + 去重命中行数 + 无效品牌命中行数。 */
public record FilterResult(List<CollectDataResultRowVo> kept, int dedupeFilteredCount, int invalidFilteredCount) {
/**
* 过滤结果:保留行 + 去重命中行数 + 无效品牌命中行数 + **被砍掉的行本身**。
*
* <p>被砍行必须在这里当场交出去:本方法的两个判定读的都是当前态(去重库、无效品牌库),
* 事后回读 chunk 重判会漂移。计数与 {@code filteredRows} 由同一次循环产出,天然自洽。
*/
public record FilterResult(List<CollectDataResultRowVo> kept,
int dedupeFilteredCount,
int invalidFilteredCount,
List<FilteredRow> filteredRows) {
}
/** 一条被过滤的行与它命中的那一道闸。 */
public record FilteredRow(CollectDataResultRowVo row, String reasonCode) {
}
private static List<String> distinctNonBlank(List<String> values) {
@@ -0,0 +1,225 @@
package com.nanri.aiimage.modules.collectdata.util;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.text.BrandLabel;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataFilteredRowMapper;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.function.Consumer;
import java.util.regex.Pattern;
/**
* 被过滤行存取器(V179):把「一个 chunk 到底丢掉了哪些行、各丢在哪一道闸」逐行落库,
* 并在装配结果 Excel 时按批读回来铺「被过滤明细」sheet。
*
* <p>写侧容错口径照 {@link CollectDataInvalidAsinBatchWriter}:按批 try/catch,失败只跳批告警,
* <b>绝不影响回传主流程</b>。这份明细是审计件、不是判定依据——为它打掉用户已采回的数据,
* 是拿可追溯性去换交付。也因此它不是强一致的:对不上账时以 result_json 的四个计数为准。</p>
*
* <p>空 ASIN 的行照记不误。空品牌被判 rejected 的行在黑名单表里根本不留痕
* ({@code CollectDataInvalidAsinBatchWriter.toEntity} 对空品牌返 null),装配期也无从反查,
* 这一格正是 V179 头注释里点名的「漏标」来源。</p>
*/
@Slf4j
@Component
public class CollectDataFilteredRowStore {
private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
/** reason_detail 与库里 VARCHAR(500) 对齐,留余量给回流追加的那句。 */
private static final int DETAIL_MAX_LENGTH = 460;
/** 装配读回的分页游标:与结果明细同量级,单批写完即释放。 */
private static final int READ_PAGE_SIZE = 1000;
private final CollectDataFilteredRowMapper filteredRowMapper;
private final int batchSize;
public CollectDataFilteredRowStore(CollectDataFilteredRowMapper filteredRowMapper,
@Value("${aiimage.collect-data.filtered-row-batch-size:200}") int batchSize) {
this.filteredRowMapper = filteredRowMapper;
this.batchSize = Math.max(1, batchSize);
}
/**
* 记录本 chunk 丢掉的行,返回**实际写入**的行数(被预算挡掉的不计)。
*
* @param batchFiltered 去重与无效品牌两道闸砍掉的行(原因已带在行上)
* @param brandRejected 品牌检测判侵权的行
* @param brandQueryFailed 品牌检测没发起查询/没回来的行
* @param brandFailureReason 品牌侧失败原因(如「用户未配置代理」),来自 BrandBatchOutcome#reason
* @param budget 本任务还能记多少行;<=0 表示只留计数不再记行
*/
public int writeBatch(Long taskId, int chunkIndex,
List<CollectDataBatchQuery.FilteredRow> batchFiltered,
List<CollectDataResultRowVo> brandRejected,
List<CollectDataResultRowVo> brandQueryFailed,
String brandFailureReason,
int budget) {
if (taskId == null || budget <= 0) {
return 0;
}
List<CollectDataFilteredRowEntity> entities = new ArrayList<>();
collectInto(entities, batchFiltered, brandRejected, brandQueryFailed, brandFailureReason, taskId, chunkIndex);
if (entities.isEmpty()) {
return 0;
}
if (entities.size() > budget) {
entities = entities.subList(0, budget);
}
int written = 0;
for (int start = 0; start < entities.size(); start += batchSize) {
List<CollectDataFilteredRowEntity> batch =
entities.subList(start, Math.min(start + batchSize, entities.size()));
try {
written += filteredRowMapper.insertBatch(batch);
} catch (RuntimeException ex) {
log.warn("[collect-data] filtered row batch insert failed, skip batch taskId={} chunk={} err={}",
taskId, chunkIndex, ex.getMessage());
}
}
return written;
}
/** 按 id 游标分批读回本任务的被过滤行;单批交给 consumer 后即可释放。 */
public void streamBatches(Long taskId, Consumer<List<CollectDataFilteredRowEntity>> consumer) {
if (taskId == null || consumer == null) {
return;
}
Long cursorId = null;
while (true) {
LambdaQueryWrapper<CollectDataFilteredRowEntity> wrapper =
new LambdaQueryWrapper<CollectDataFilteredRowEntity>()
.eq(CollectDataFilteredRowEntity::getTaskId, taskId);
if (cursorId != null) {
final Long lastId = cursorId;
wrapper.gt(CollectDataFilteredRowEntity::getId, lastId);
}
wrapper.orderByAsc(CollectDataFilteredRowEntity::getId)
.last("limit " + READ_PAGE_SIZE);
List<CollectDataFilteredRowEntity> batch = filteredRowMapper.selectList(wrapper);
if (batch == null || batch.isEmpty()) {
return;
}
consumer.accept(batch);
cursorId = batch.get(batch.size() - 1).getId();
if (cursorId == null || batch.size() < READ_PAGE_SIZE) {
return;
}
}
}
/**
* 补测把行补进结果后盖上回流时间,并在原因后追加一句说明。
* **不删行**——删了就把「当时确实没查动」这条历史一起删掉了。
*/
public int markResolved(Long taskId, Collection<String> asins) {
if (taskId == null || asins == null || asins.isEmpty()) {
return 0;
}
List<String> normalized = new ArrayList<>(asins.size());
for (String asin : asins) {
String value = normalize(asin).toUpperCase(Locale.ROOT);
if (!value.isBlank()) {
normalized.add(value);
}
}
if (normalized.isEmpty()) {
return 0;
}
try {
return filteredRowMapper.markResolved(taskId, normalized, ";已由补测回流");
} catch (RuntimeException ex) {
log.warn("[collect-data] filtered row markResolved failed taskId={} err={}", taskId, ex.getMessage());
return 0;
}
}
/**
* 删除本任务的被过滤行。与 {@link #writeBatch} 相反,这里**不吞异常**:
* 写失败只丢一份审计件,删失败却会让任务删一半、留下永久孤儿行。
* 调用方在删除事务里执行它,让它随事务一起回滚。
*/
public int deleteByTask(Long taskId) {
if (taskId == null) {
return 0;
}
return filteredRowMapper.delete(new LambdaQueryWrapper<CollectDataFilteredRowEntity>()
.eq(CollectDataFilteredRowEntity::getTaskId, taskId));
}
private void collectInto(List<CollectDataFilteredRowEntity> out,
List<CollectDataBatchQuery.FilteredRow> batchFiltered,
List<CollectDataResultRowVo> brandRejected,
List<CollectDataResultRowVo> brandQueryFailed,
String brandFailureReason, Long taskId, int chunkIndex) {
if (batchFiltered != null) {
for (CollectDataBatchQuery.FilteredRow filtered : batchFiltered) {
if (filtered == null || filtered.row() == null) {
continue;
}
out.add(toEntity(taskId, chunkIndex, filtered.row(), filtered.reasonCode(), null));
}
}
if (brandRejected != null) {
for (CollectDataResultRowVo row : brandRejected) {
if (row == null) {
continue;
}
out.add(toEntity(taskId, chunkIndex, row,
CollectDataFilteredRowEntity.REASON_BRAND_REJECTED, null));
}
}
if (brandQueryFailed != null) {
for (CollectDataResultRowVo row : brandQueryFailed) {
if (row == null) {
continue;
}
out.add(toEntity(taskId, chunkIndex, row,
CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED, brandFailureReason));
}
}
}
private CollectDataFilteredRowEntity toEntity(Long taskId, int chunkIndex,
CollectDataResultRowVo row,
String reasonCode,
String reasonDetail) {
CollectDataFilteredRowEntity entity = new CollectDataFilteredRowEntity();
entity.setTaskId(taskId);
entity.setChunkIndex(Math.max(0, chunkIndex));
entity.setAsin(normalize(row.getAsin()).toUpperCase(Locale.ROOT));
entity.setKeyword(truncate(normalize(row.getKeyword()), 255));
entity.setBrand(truncate(BrandLabel.strip(row.getBrand()), 255));
entity.setReasonCode(reasonCode);
entity.setReasonDetail(truncate(reasonDetail, DETAIL_MAX_LENGTH));
return entity;
}
private static String truncate(String value, int max) {
if (value == null || value.isBlank()) {
return null;
}
return value.length() <= max ? value : value.substring(0, max);
}
private static String normalize(String value) {
if (value == null) {
return "";
}
String normalized = value.replace(String.valueOf((char) 0xFEFF), "")
.replace((char) 0x3000, ' ')
.replace("\r\n", " ")
.replace("\r", " ")
.replace("\n", " ")
.replace("\t", " ")
.trim();
return WHITESPACE_PATTERN.matcher(normalized).replaceAll(" ");
}
}
@@ -3,16 +3,20 @@ package com.nanri.aiimage.modules.task.spi;
import java.util.List;
/**
* 采集明细行的历史清理扩展点(2026-09 全维度审查 G5)。
* 采集模块名下数据的清理扩展点(2026-09 全维度审查 G5)。
*
* <p>任务历史清理时除了 task 自己的表,还要连带删除 biz_collect_data_item 的明细行;
* 该表属于采集模块,原先由 {@code ModuleHistoryCleanupService} 直接使用其 Mapper,
* 构成 task → 业务的依赖。现由采集模块实现本接口。
* <p>任务历史清理时除了 task 自己的表,还要连带删除采集模块自己的行;这些表属于采集模块,
* 原先由 {@code ModuleHistoryCleanupService} 直接使用其 Mapper,构成 task → 业务的依赖。
* 现由采集模块实现本接口。
*
* <p>实现方的清理范围是「本模块拥有的全部行」,会随版本增加:目前是
* biz_collect_data_item 与 V179 的 biz_collectdata_filtered_row。调用方不需要知道有几张表,
* 也不该在 task 模块里为新增的采集表补删除——那等于把收回来的依赖又放回去。
*/
public interface CollectDataItemCleanupSpi {
/**
* 删除这些任务名下的采集明细行。
* 删除这些任务名下的采集明细行(含本模块后续的随行数据)。
*
* @return 删除行数
*/
@@ -488,6 +488,11 @@ aiimage:
brand-check-chunk-budget-millis: ${AIIMAGE_COLLECT_DATA_BRAND_CHECK_CHUNK_BUDGET_MILLIS:10000}
invalid-asin-batch-size: ${AIIMAGE_COLLECT_DATA_INVALID_ASIN_BATCH_SIZE:100}
result-item-batch-size: ${AIIMAGE_COLLECT_DATA_RESULT_ITEM_BATCH_SIZE:100}
filtered-row-batch-size: ${AIIMAGE_COLLECT_DATA_FILTERED_ROW_BATCH_SIZE:200}
# 单任务「被过滤行」逐条落库的上限(V179,供结果 Excel 的「被过滤明细」sheet 追溯)。
# 超限后只累加计数、不再记行,sheet 会如实写「另有 N 行未逐条记录」而不是静默少给。
# 参照量级:task 32573 被过滤 599 行、32058 被过滤 2616 行;设为 0 表示不限。
max-filtered-rows-per-task: ${AIIMAGE_COLLECT_DATA_MAX_FILTERED_ROWS_PER_TASK:50000}
# ASIN 采集结果跨任务缓存(详见 AsinResultCacheService)。新鲜度口径放在服务端,
# 采集端版本不齐时老客户端只会"问不到就照常开页",不会各按各的理解复用到过期数据。
asin-cache:
@@ -0,0 +1,48 @@
-- V179: 采集数据「被过滤行」明细表
--
-- 背景(2026-10-28 实证,task 32573 / uid=1210):客户端回传 600 行,结果只留 1 行。
-- 被砍掉的 599 行在系统里**只剩四个计数**(去重 48 / 黑名单命中 377 / 品牌判侵权 1 /
-- 品牌检测未出结论 173),行内容在 CollectDataBatchQuery.filter 与
-- CollectDataBrandBatchFilter.filter 里当场丢弃。后果是任何人都答不出
-- 「到底是哪些 ASIN、死在哪一道」:用户只看到"保留 1",运维只能逐 chunk 翻日志猜。
-- 本表把这四类被过滤行**逐行带原因**记下来,供结果 Excel 的「被过滤明细」sheet 装配与对账。
--
-- 为什么必须在提交时记、不能在装配时回读 chunk 重判(三种错法,均已核到代码):
-- 1. 错标:去重与无效品牌两支查的都是**当前态**。任务窗口期(32573 跑了近 4 小时)
-- 任何后台导入,都会让当时 kept 的行在装配时"命中"。
-- 2. 漏标:uk_data_value_brand + INSERT IGNORE 使已存在的 (asin,brand) 不落新行(零证据);
-- 空品牌的 rejected 行 toEntity 直接返 null 从不写表,但 classify 把它们算进 rejected。
-- 3. 分类串台:本任务自己的 rejected 行会先写进 biz_invalid_asin_data,而 filter() 只认
-- 品牌表——装配期重判会把"当时被黑名单砍"和"本次刚被判侵权"统统归成同一类。
--
-- 口径:
-- - 只记事实,不参与任何判定;本表任何一行写失败都**不得**影响回传主流程(writer 按批
-- try/catch 跳批,与 CollectDataInvalidAsinBatchWriter 同口径),故它不是强一致审计件;
-- - reason_code 四类:DEDUPE / INVALID_BRAND / BRAND_REJECTED / BRAND_QUERY_FAILED;
-- - 补测回流不删行、也不另立 status 词表:resolved_at 从 NULL 变成时间戳就是唯一状态,
-- 并在 reason_detail 追加「已由补测回流」。**不删行**——删了就把"当时确实没查动"这条
-- 历史一起删掉了;
-- - 体量闸门在应用侧(aiimage.collect-data.max-filtered-rows-per-task),不在库里做。
--
-- 影响表与数据量:新建空表。单任务量级参照 32573(599 行)与 32058(2616 行)。
--
-- 回滚(手工):
-- DROP TABLE IF EXISTS biz_collectdata_filtered_row;
CREATE TABLE IF NOT EXISTS biz_collectdata_filtered_row (
id BIGINT NOT NULL AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'biz_file_task.id(COLLECT_DATA 模块)',
chunk_index INT NOT NULL DEFAULT 0 COMMENT '产生本行的回传分片序号(0=收尾/未知)',
asin VARCHAR(32) NOT NULL DEFAULT '' COMMENT '被过滤的 ASIN(归一化大写,与结果 sheet 同口径)',
keyword VARCHAR(255) NULL COMMENT '所属关键词',
brand VARCHAR(255) NULL COMMENT '品牌(BrandLabel.strip 展示口径,与明细 sheet 同形)',
reason_code VARCHAR(24) NOT NULL COMMENT 'DEDUPE / INVALID_BRAND / BRAND_REJECTED / BRAND_QUERY_FAILED',
reason_detail VARCHAR(500) NULL COMMENT '可展示的原因(如「用户未配置代理」)',
resolved_at DATETIME NULL COMMENT '补测回流时间;NULL=当时没查动且至今没回来(本表的唯一状态)',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
PRIMARY KEY (id),
KEY idx_filtered_task_reason (task_id, reason_code) COMMENT '装配 sheet 与四类计数对账',
KEY idx_filtered_task_asin (task_id, asin) COMMENT '补测回流按 ASIN 定位未回流行'
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = DYNAMIC
COMMENT = '采集数据被过滤行明细(四类过滤逐行带原因,供结果 Excel 追溯与对账)';
@@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "https://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.nanri.aiimage.modules.collectdata.mapper.CollectDataFilteredRowMapper">
<!-- 被过滤行是审计件,不是判定依据:本表无唯一键,同 ASIN 可在不同关键词/分片各被记一次。
status 不在此列——库里 resolved_at 为 NULL 就是"还没回流",没有第二套词表要 Java 写。 -->
<insert id="insertBatch">
INSERT INTO biz_collectdata_filtered_row
(task_id, chunk_index, asin, keyword, brand, reason_code, reason_detail)
VALUES
<foreach collection="rows" item="row" separator=",">
(#{row.taskId}, #{row.chunkIndex}, #{row.asin},
#{row.keyword,jdbcType=VARCHAR}, #{row.brand,jdbcType=VARCHAR},
#{row.reasonCode}, #{row.reasonDetail,jdbcType=VARCHAR})
</foreach>
</insert>
<!-- 补测回流:只盖时间戳并在原因后追加一句,不删行——删了就把「当时确实没查动」这条历史一起删掉了。
resolved_at IS NULL 让重复回流天然幂等(同一 ASIN 补两次也只追加一句)。 -->
<update id="markResolved">
UPDATE biz_collectdata_filtered_row
SET resolved_at = NOW(),
reason_detail = CONCAT(IFNULL(reason_detail, ''), #{detailSuffix})
WHERE task_id = #{taskId}
AND resolved_at IS NULL
AND asin IN
<foreach collection="asins" item="asin" open="(" separator="," close=")">
#{asin}
</foreach>
</update>
</mapper>
@@ -85,7 +85,10 @@ class SqlLayeringConventionTest {
// 130 → 140:关键词使用记录管理端(keywordusage 模块,1 份 XML 10 条只读 select)新写语句。
// 140 → 141:工作台客户端全量同步新增 {@code KeywordShareMapper#upsertBatchFromSync}
// (词本体 UPSERT,把客户端的四级类目写进既有行),同轮登记进 NOT_MIGRATED。
private static final int XML_STATEMENT_TOTAL = 141;
// 141 → 143:V179 采集「被过滤行」新表 {@code CollectDataFilteredRowMapper} 的两条写侧语句
// (insertBatch 多值插入 + markResolved 盖回流时间)。该 mapper 没有注解时代,
// 不在 MigratedSqlEquivalenceTest 的基准账本里 ⇒ 不占 NOT_MIGRATED,只在这本总分母上记名。
private static final int XML_STATEMENT_TOTAL = 143;
/**
* wrapper 里夹带函数级 SQL 的现状("这不是简单单表"的证据),文件名 → 条数:
@@ -54,7 +54,9 @@ class CatchSilenceRatchetTest {
// 10-27 「整表都要跑完」专项抬 573 -> 577(2 枚新增不重抛 + 2 枚既有整批抛出改逐条跳过,逐处记名见 STRICT_TOTAL 上方)。
// 10-27 同日第二批抬 577 -> 579(批量入库逐批容错+folder 去重逐条目容错,两处都是 log.warn+计数+continue,
// 逐处记名见 STRICT_TOTAL 上方注释)。
static final int GIVEN_NOT_RETHROW = 579;
// 10-28 V179 采集被过滤行追溯抬 579 -> 582(审计件三枚:逐批插入容错/回流簿记容错/尾 sheet 降级,
// 逐处记名见 STRICT_TOTAL 上方注释)。
static final int GIVEN_NOT_RETHROW = 582;
static final int GIVEN_LOG_ONLY = 249;
static final int GIVEN_EMPTY = 56;
@@ -181,8 +183,24 @@ class CatchSilenceRatchetTest {
* 因为 {@code TaskRetryEligibility}/{@code SimilarAsinStaleReopenSupport} 会按 errorMessage
* 字面量分类,写进去等于替一条正常任务编一个失败理由。
* 同批实测 total 1028、rethrow 393(未动)、notRethrow 579、logOnly 181(未动)、empty 56(未动)。
*
* <p>10-28 采集「被过滤行」追溯(V179,结果 Excel 第 3 个 sheet)抬 total 1028 -&gt; 1031,逐处记名。
* 三枚都是<b>审计件不得打掉交付</b>那一形,块体各只有一句带 {@code [collect-data]} 锚点的 WARN:
* <ul>
* <li>{@code CollectDataFilteredRowStore.writeBatch} 一枚:被过滤行按批 INSERT,坏批点名后
* 其余批照常落库(口径照 {@code CollectDataInvalidAsinBatchWriter})。这份明细不参与任何
* 判定,为它打掉用户已采回的数据是拿可追溯性换交付;跳过的批会从四类计数与已落库行数
* 的差里冒出来,sheet 上写「另有 N 行未逐条记录」,不是静默少给。</li>
* <li>{@code CollectDataFilteredRowStore.markResolved} 一枚:补测回流只盖时间戳+追加一句。
* 必要性是实测的——{@code CollectDataBrandRecheckScheduler} 抓住 {@code apply()} 的抛出后
* <b>整组留队重放</b>,这里若外抛,一次簿记失败会让已注入的任务永远重排。</li>
* <li>{@code CollectDataExcelAssemblyService.writeFilteredSheet} 一枚:尾 sheet 读不动就交付
* 前两表、缺这张审计表。同批我先写的第二枚("清单不完整"那行的内层 catch)已删掉:
* 那只是本地 POI 写,写不动说明整份 workbook 都写不动,该由
* {@code writeWorkbookStreaming} 的既有出口处理,与前两张 sheet 同姿势。</li>
* </ul>
*/
static final int STRICT_TOTAL = 1028;
static final int STRICT_TOTAL = 1031;
/**
* 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。
* 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。
@@ -282,8 +300,13 @@ class CatchSilenceRatchetTest {
* 10-27 同日第二批再抬 577 -&gt; 579(2 枚逐批/逐条目容错:{@code QueryAsinService.flushInsert}
* 与 {@code DedupeRunService.packageFolderDedupeResultsAsZip},块体都是 log.warn+计数,
* 坏项点名、其余照常交付,逐处记名见 STRICT_TOTAL 上方注释)。
* 10-28 再抬 579 -&gt; 582(V179 采集「被过滤行」三枚审计件容错:{@code CollectDataFilteredRowStore.writeBatch}
* 逐批插入、{@code CollectDataFilteredRowStore.markResolved} 回流簿记、
* {@code CollectDataExcelAssemblyService.writeFilteredSheet} 尾 sheet 降级;三处都是带
* {@code [collect-data]} 锚点的 log.warn+善后,块体除第一枚外各带一句状态置位/返回值,
* 逐处记名见 STRICT_TOTAL 上方注释)。
*/
static final int STRICT_NOT_RETHROW = 579;
static final int STRICT_NOT_RETHROW = 582;
/**
* 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}:
* 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出——
@@ -303,8 +326,12 @@ class CatchSilenceRatchetTest {
* 与 {@code getAllFileAggregates} 的逐条跳过出口——块体各只有一句带 {@code [brand-storage]}
* 锚点的 WARN。两处此前是「一个文件坏 ⇒ 整批抛出」,改后其余文件照常返回,坏的那条点名可查;
* 同批 {@code rebuildAggregate} 与外观专利候选收集因块内另有计数/continue 只进 NOT_RETHROW。
* 10-28 抬 181 -> 182(V179 采集「被过滤行」的 {@code CollectDataFilteredRowStore.writeBatch}):
* 块体只有一句带 {@code [collect-data]} 锚点的 WARN(taskId+chunk+err),坏批点名后其余批照常
* 落库。同批另两处({@code markResolved} 追加 return、{@code writeFilteredSheet} 置 incomplete 位)
* 因块内多一句只进 NOT_RETHROW。
*/
static final int STRICT_LOG_ONLY = 181; // 严格口径:块内每条顶层语句都是 log 调用才算
static final int STRICT_LOG_ONLY = 182; // 严格口径:块内每条顶层语句都是 log 调用才算
static final int STRICT_EMPTY = 56;
/**
@@ -60,6 +60,8 @@ class CollectDataBrandRecheckInjectorTest {
private AsinResultCacheService asinResultCacheService;
@Mock
private CollectDataService collectDataService;
@Mock
private com.nanri.aiimage.modules.collectdata.util.CollectDataFilteredRowStore filteredRowStore;
private final CollectDataResultDetailCodec codec = new CollectDataResultDetailCodec(new ObjectMapper());
private CollectDataBrandRecheckInjector injector;
@@ -72,7 +74,8 @@ class CollectDataBrandRecheckInjectorTest {
TaskChunkEntity.class);
injector = new CollectDataBrandRecheckInjector(taskChunkMapper, fileResultMapper,
collectDataBatchQuery, codec, resultItemBatchWriter, invalidAsinBatchWriter,
transientPayloadStorageService, asinResultCacheService, new ObjectMapper(), collectDataService);
transientPayloadStorageService, asinResultCacheService, new ObjectMapper(), collectDataService,
filteredRowStore);
}
private FileTaskEntity task(long id) {
@@ -120,7 +123,7 @@ class CollectDataBrandRecheckInjectorTest {
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(payload);
CollectDataResultRowVo kept = row("B01", "Nike");
when(collectDataBatchQuery.filter(anyList(), any())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(kept), 1, 0));
new CollectDataBatchQuery.FilterResult(List.of(kept), 1, 0, List.of()));
when(transientPayloadStorageService.storeResultPayload(eq("COLLECT_DATA"), eq(100L), eq("hash1"),
eq("recheck-chunk-1"), anyString())).thenReturn("ptr2");
when(resultItemBatchWriter.upsertAccepted(eq(100L), eq(9L), eq("s1"), eq(1),
@@ -133,6 +136,9 @@ class CollectDataBrandRecheckInjectorTest {
assertThat(outcome.acceptedRows()).isEqualTo(1);
assertThat(outcome.rejectedRows()).isZero();
verify(asinResultCacheService).writeCollectDataRows(eq("US"), eq("100"), eq(List.of(kept)));
// 补回结果的行要在 V179 明细里标 RESOLVED——但只标不删:删掉就把「当时确实没查动」
// 这条历史一起删了,而它是用户日后追问"我那批货当时为什么少"时唯一说得清的东西。
verify(filteredRowStore).markResolved(eq(100L), eq(List.of("B01")));
}
@Test
@@ -180,7 +186,7 @@ class CollectDataBrandRecheckInjectorTest {
"[{\"asin\":\"B01\",\"brand\":\"Nike\"}]");
// 补测窗口内该 ASIN 已被其他任务采走:复查全滤掉
when(collectDataBatchQuery.filter(anyList(), any())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(), 1, 0));
new CollectDataBatchQuery.FilterResult(List.of(), 1, 0, List.of()));
GroupOutcome outcome = injector.apply(task(100L), Map.of("nike", RecheckVerdict.CLEAN));
@@ -0,0 +1,182 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.nio.file.Path;
import java.util.List;
import java.util.function.Consumer;
import static org.assertj.core.api.Assertions.assertThat;
/**
* 「被过滤明细」尾 sheet(V179)的形状钉子。
*
* <p>背景:task 32573 回传 600 行、结果只剩 1 行,被砍的 599 行在系统里只有四个计数,
* 谁也无法回答"到底是哪些 ASIN、死在哪一道"。这张 sheet 就是为了让它可追溯。</p>
*
* <p>三件事必须钉住:① 它排在最后(外部读取方要么只认 sheet 0、要么按 sheet 名认列);
* ② 它的行数**绝不能**累进 writeWorkbookStreaming 的返回值——调用方拿那个数写
* stats.finalRowCount 与 result.rowCount,并据此跑 guardDetailRowsWritten 校验
* 「下载 Excel 行数 == 保留行数」,一旦污染就等于把交付说谎重新引回来;
* ③ 一行被过滤都没有时不建这张表,只留表头会让人误以为"全被过滤了"。</p>
*/
class CollectDataFilteredSheetTest {
@TempDir
Path tempDir;
private final CollectDataExcelAssemblyService assembly = new CollectDataExcelAssemblyService();
private static CollectDataResultRowVo detailRow(String asin) {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setBrand("nike");
row.setAsin(asin);
row.setKeyword("phone case");
row.setPrice("19.90");
row.setSellerName("seller-A");
row.setDeliveryMethod("FBA");
return row;
}
private static CollectDataSummaryRowDto summary() {
CollectDataSummaryRowDto item = new CollectDataSummaryRowDto();
item.setKeyword("phone case");
item.setFba(1);
return item;
}
private static CollectDataFilteredRowEntity filtered(String asin, String reasonCode, String detail) {
CollectDataFilteredRowEntity entity = new CollectDataFilteredRowEntity();
entity.setTaskId(1L);
entity.setChunkIndex(1);
entity.setAsin(asin);
entity.setKeyword("phone case");
entity.setBrand("somebrand");
entity.setReasonCode(reasonCode);
entity.setReasonDetail(detail);
return entity;
}
private static CollectDataExcelAssemblyService.FilteredRowSource sourceOf(
List<CollectDataFilteredRowEntity> rows, long notRecorded) {
return new CollectDataExcelAssemblyService.FilteredRowSource() {
@Override
public void forEachBatch(Consumer<List<CollectDataFilteredRowEntity>> batchConsumer) {
if (!rows.isEmpty()) {
batchConsumer.accept(rows);
}
}
@Override
public long notRecordedCount() {
return notRecorded;
}
};
}
@Test
void 尾sheet排在最后且明细行数不被污染() throws Exception {
File output = tempDir.resolve("filtered.xlsx").toFile();
long returned = assembly.writeWorkbookStreaming(
output,
consumer -> consumer.accept(List.of(detailRow("B000000001"), detailRow("B000000002"))),
List.of(summary()),
consumer -> { /* 本例不走 fallback 聚合 */ },
sourceOf(List.of(
filtered("B000000003", CollectDataFilteredRowEntity.REASON_DEDUPE, null),
filtered("B000000004", CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED, "用户未配置代理")),
0L));
// 红线:返回值只算「采集数据结果」那 2 行,被过滤的 2 行一个都不能进来
assertThat(returned).isEqualTo(2L);
try (Workbook workbook = WorkbookFactory.create(output)) {
assertThat(workbook.getNumberOfSheets()).isEqualTo(3);
assertThat(workbook.getSheetName(0)).isEqualTo("采集数据结果");
assertThat(workbook.getSheetName(1)).isEqualTo("结果文件");
assertThat(workbook.getSheetName(2)).isEqualTo("被过滤明细");
Sheet sheet = workbook.getSheet("被过滤明细");
// 表头 + 2 行
assertThat(sheet.getLastRowNum()).isEqualTo(2);
assertThat(sheet.getRow(1).getCell(1).getStringCellValue()).isEqualTo("B000000003");
assertThat(sheet.getRow(1).getCell(3).getStringCellValue()).isEqualTo("已存在于去重总数据库");
assertThat(sheet.getRow(2).getCell(3).getStringCellValue()).isEqualTo("品牌检测未返回结果");
assertThat(sheet.getRow(2).getCell(4).getStringCellValue()).isEqualTo("用户未配置代理");
}
}
@Test
void 没有被过滤行时不建这张表() throws Exception {
File output = tempDir.resolve("empty.xlsx").toFile();
assembly.writeWorkbookStreaming(
output,
consumer -> consumer.accept(List.of(detailRow("B000000001"))),
List.of(summary()),
consumer -> { },
sourceOf(List.of(), 0L));
try (Workbook workbook = WorkbookFactory.create(output)) {
assertThat(workbook.getNumberOfSheets()).isEqualTo(2);
assertThat(workbook.getSheet("被过滤明细")).isNull();
}
}
@Test
void 超出上限未逐条记录时如实写出缺口() throws Exception {
File output = tempDir.resolve("gap.xlsx").toFile();
assembly.writeWorkbookStreaming(
output,
consumer -> consumer.accept(List.of(detailRow("B000000001"))),
List.of(summary()),
consumer -> { },
sourceOf(List.of(filtered("B000000009",
CollectDataFilteredRowEntity.REASON_INVALID_BRAND, null)), 317L));
try (Workbook workbook = WorkbookFactory.create(output)) {
Sheet sheet = workbook.getSheet("被过滤明细");
assertThat(sheet).isNotNull();
String note = sheet.getRow(2).getCell(4).getStringCellValue();
assertThat(note).contains("317");
// 缺口必须说清"行数已计入各道闸的计数",否则读的人会以为又丢了 317 行
assertThat(note).contains("已计入");
}
}
@Test
void 数据源抛错时降级掉这个sheet而不是打掉整份交付() throws Exception {
File output = tempDir.resolve("boom.xlsx").toFile();
CollectDataExcelAssemblyService.FilteredRowSource broken = new CollectDataExcelAssemblyService.FilteredRowSource() {
@Override
public void forEachBatch(Consumer<List<CollectDataFilteredRowEntity>> batchConsumer) {
throw new IllegalStateException("明细表读不动");
}
};
long returned = assembly.writeWorkbookStreaming(
output,
consumer -> consumer.accept(List.of(detailRow("B000000001"))),
List.of(summary()),
consumer -> { },
broken);
assertThat(returned).isEqualTo(1L);
try (Workbook workbook = WorkbookFactory.create(output)) {
// 追溯 sheet 缺了就缺了,前两表照常交付——它是审计件,不是交付物。
// 若这里改成外抛,一次审计读失败就会把用户已采回的数据一起打掉。
assertThat(workbook.getNumberOfSheets()).isEqualTo(2);
assertThat(workbook.getSheetName(0)).isEqualTo("采集数据结果");
assertThat(workbook.getSheetName(1)).isEqualTo("结果文件");
}
}
}
@@ -43,9 +43,15 @@ import static org.mockito.Mockito.when;
* 上游的采集代理取不到 IP(余额不足)与 503 —— 代码在这个位置只观测到「品牌检测没返回结果」,
* 观测不到原因。写死一个猜测的原因会把排查方向从代理/上游带到自身服务上。
*
* <p>锁定契约:文案只陈述可观测事实(缺失行数、不计入结果条数、未判无效、可重跑),
* 不指认代码没有依据的具体故障原因;真实原因留在 BrandCheckClient 的 warn 日志里
* (brand-check 的 lastErr 字段),由运维按日志排查。
* <p>锁定契约(2026-10-28 修订):文案只陈述可观测事实(缺失行数、不计入结果条数、未判无效),
* **并且只在这层真的观测得到原因时才指名原因**——品牌检测的失败原因现已由
* {@code BrandBatchOutcome#reason} 落进 result_json(brandQueryFailedReason),所以
* 「用户未配置代理」这类有据可查的原因必须写出来并指向「密钥设置→代理设置」。
*
* <p>原契约(10-02)之所以禁止指名原因,是因为当时这一层确实观测不到;不是"永远不许说"。
* 反过来,原因缺失时仍然不猜:不写「超时」「限流」「上游不可用」,只说系统会自动重试。
* 旧文案那句「可稍后单独重跑这些 ASIN」是必须拆掉的误导——未配置代理时重跑一万次也不会变好
* (现网 task 32573:补测队列两轮 21/21、28/28 全败、净增 0 行)。
*/
@ExtendWith(MockitoExtension.class)
class CollectDataPartialResultNoticeTest {
@@ -77,13 +83,19 @@ class CollectDataPartialResultNoticeTest {
}
private FileTaskEntity taskWithQueryFailedRows(int queryFailedRows) {
return taskWithQueryFailedRows(queryFailedRows, null);
}
private FileTaskEntity taskWithQueryFailedRows(int queryFailedRows, String reason) {
FileTaskEntity task = new FileTaskEntity();
task.setId(TASK_ID);
task.setModuleType("COLLECT_DATA");
task.setStatus("RUNNING");
task.setTaskNo("CD-29991");
task.setUserId(7L);
task.setResultJson("{\"brandQueryFailedCount\":" + queryFailedRows + "}");
task.setResultJson("{\"brandQueryFailedCount\":" + queryFailedRows
+ (reason == null ? "" : ",\"brandQueryFailedReason\":\"" + reason + "\"")
+ "}");
return task;
}
@@ -103,9 +115,13 @@ class CollectDataPartialResultNoticeTest {
}
private String stubHappyPathAndCaptureDetail() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7));
return stubHappyPathAndCaptureDetail(null);
}
private String stubHappyPathAndCaptureDetail(String reason) {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7, reason));
when(fileResultMapper.selectById(5001L)).thenReturn(result());
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L);
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(12L);
// 空明细守卫按明细表行数校验:happy path 下表内行数与写出行数一致。
when(taskResultItemMapper.selectCount(any())).thenReturn(12L);
when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx");
@@ -136,14 +152,35 @@ class CollectDataPartialResultNoticeTest {
assertThat(detail).contains("不计入结果条数");
assertThat(detail).contains("未返回结果");
assertThat(detail).contains("重跑");
// 原因缺失时不再承诺「你重跑就好」,只说系统会自动重试
assertThat(detail).contains("自动重试");
}
@Test
void 原因缺失时不猜故障方() {
// classify(空串) 会算成「上游」,那是没观测到的事——这一层不能替它背书
String detail = stubHappyPathAndCaptureDetail();
assertThat(detail).doesNotContain("上游");
assertThat(detail).doesNotContain("未配置代理");
}
@Test
void 未配置代理必须指名原因并指向可行动入口() {
// 旧文案对配置类原因是谎:补测队列两轮 21/21、28/28 全败、净增 0 行,
// 让用户「稍后单独重跑」只会让他白等
String detail = stubHappyPathAndCaptureDetail("用户未配置代理");
assertThat(detail).contains("密钥设置");
assertThat(detail).contains("代理设置");
assertThat(detail).doesNotContain("可稍后单独重跑这些 ASIN");
}
@Test
void 任务已被判失败时不推部分结果通知() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7));
when(fileResultMapper.selectById(5001L)).thenReturn(result());
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L);
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(12L);
// 空明细守卫按明细表行数校验:happy path 下表内行数与写出行数一致。
when(taskResultItemMapper.selectCount(any())).thenReturn(12L);
when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx");
@@ -160,7 +197,7 @@ class CollectDataPartialResultNoticeTest {
void 明细缺行时不得上传文件并把任务标成功() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(0));
when(fileResultMapper.selectById(5001L)).thenReturn(result());
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L);
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(12L);
when(taskResultItemMapper.selectCount(any())).thenReturn(20L);
assertThatThrownBy(() -> service.processResultFileJob(job()))
@@ -182,7 +219,7 @@ class CollectDataPartialResultNoticeTest {
task.setResultJson("{\"finalRowCount\":93}");
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
when(fileResultMapper.selectById(5001L)).thenReturn(result());
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(0L);
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(0L);
when(taskResultItemMapper.selectCount(any())).thenReturn(0L);
assertThatThrownBy(() -> service.processResultFileJob(job()))
@@ -199,7 +236,7 @@ class CollectDataPartialResultNoticeTest {
task.setResultJson("{}");
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
when(fileResultMapper.selectById(5001L)).thenReturn(result());
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(0L);
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(0L);
when(taskResultItemMapper.selectCount(any())).thenReturn(0L);
when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx");
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
@@ -0,0 +1,114 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.publish.service.PublishWorkbookService;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.nio.file.Path;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
/**
* 采集结果文件被用户**回传**给别的模块时的兼容性钉子(V179 新增尾 sheet 的主要外溢风险)。
*
* <p>结果 Excel 不是终点物:用户会把它再上传到上架/跟价。那两个模块用的是
* {@code ExcelStreamReader.readAllSheets}(读全部 sheet),所以多一张 sheet 就可能多算一批行。
* 本用例钉住「加与不加那张 sheet,回传行为逐字一致」。</p>
*
* <p>跟价那一侧({@code PriceTrackTaskService.parseWorkbookAsinRows})的机制是
* {@code resolveCountryCode(sheetName)} 认不出「被过滤明细」即整表跳过,且首表兜底只在
* sheetCount==1 时生效——那条路径上的方法是私有的,本用例覆盖不到,按真机验证项处理。</p>
*/
class CollectDataResultFileReuploadCompatTest {
@TempDir
Path tempDir;
private final CollectDataExcelAssemblyService assembly = new CollectDataExcelAssemblyService();
private final PublishWorkbookService publishParser = new PublishWorkbookService();
private static CollectDataResultRowVo detailRow() {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setBrand("nike");
row.setAsin("B000000001");
row.setKeyword("phone case");
row.setPrice("19.90");
row.setSellerName("seller-A");
row.setDeliveryMethod("FBA");
return row;
}
private static CollectDataSummaryRowDto summary() {
CollectDataSummaryRowDto item = new CollectDataSummaryRowDto();
item.setKeyword("phone case");
item.setFba(1);
return item;
}
private File assemble(boolean withFilteredSheet) throws Exception {
File output = tempDir.resolve((withFilteredSheet ? "three" : "two") + ".xlsx").toFile();
CollectDataExcelAssemblyService.FilteredRowSource source = withFilteredSheet
? filteredSource(entity())
: null;
assembly.writeWorkbookStreaming(output,
consumer -> consumer.accept(List.of(detailRow())),
List.of(summary()),
consumer -> { },
source);
return output;
}
private static CollectDataFilteredRowEntity entity() {
CollectDataFilteredRowEntity entity = new CollectDataFilteredRowEntity();
entity.setTaskId(1L);
entity.setChunkIndex(1);
entity.setAsin("B000000002");
entity.setKeyword("phone case");
entity.setBrand("somebrand");
entity.setReasonCode(CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED);
entity.setReasonDetail("用户未配置代理");
return entity;
}
/** {@link CollectDataExcelAssemblyService.FilteredRowSource} 有两个方法,不是函数式接口,只能匿名类。 */
private static CollectDataExcelAssemblyService.FilteredRowSource filteredSource(
CollectDataFilteredRowEntity entity) {
return new CollectDataExcelAssemblyService.FilteredRowSource() {
@Override
public void forEachBatch(java.util.function.Consumer<List<CollectDataFilteredRowEntity>> consumer) {
consumer.accept(List.of(entity));
}
@Override
public long notRecordedCount() {
return 0L;
}
};
}
/** 回传解析的落点:拒绝就说清第几列不匹配,接受就说 "accepted"。 */
private String outcomeOf(File file) {
try {
PublishWorkbookService.ParsedWorkbook parsed = publishParser.parse(file);
return "accepted rows=" + (parsed == null ? "-" : parsed.rows().size());
} catch (BusinessException ex) {
return "rejected: " + ex.getMessage();
}
}
@Test
void 多出被过滤sheet后上架解析器的回传行为逐字一致() throws Exception {
String twoSheets = outcomeOf(assemble(false));
String threeSheets = outcomeOf(assemble(true));
assertThat(threeSheets).isEqualTo(twoSheets);
// 并且必须是在**第一张表**就被表头校验挡下——它挡在前面,才轮不到新 sheet 出问题
assertThat(twoSheets).startsWith("rejected: ").contains("表头不匹配").contains("第 1 列");
}
}
@@ -142,7 +142,7 @@ class CollectDataServiceTxBoundaryTest {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin("B0COLLECT1");
lenient().when(collectDataBatchQuery.filter(any(), any())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0));
new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0, List.of()));
lenient().when(brandBatchFilter.filter(any(), any())).thenReturn(
new CollectDataBrandBatchFilter.BrandBatchOutcome(List.of(), List.of(), List.of(row), ""));
lenient().when(resultItemBatchWriter.upsertAccepted(anyLong(), anyLong(), anyString(), anyInt(), any(), anyString()))
@@ -335,7 +335,7 @@ class CollectDataSubmitResultSnapshotTest {
List<CollectDataResultRowVo> kept = new ArrayList<>(rows);
f.events.add("outsideLock filter rows=" + kept.size() + " owner=" + invocation.getArgument(1)
+ " dedupe=0 invalid=0");
return new CollectDataBatchQuery.FilterResult(kept, 0, 0);
return new CollectDataBatchQuery.FilterResult(kept, 0, 0, List.of());
});
when(brandFilter.filter(anyList(), any())).thenAnswer(invocation -> {
@SuppressWarnings("unchecked")
@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.collectdata.util;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService;
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
@@ -218,6 +219,38 @@ class CollectDataBatchQueryTest {
verify(dedupeService).findExistingComparableValuesForOwner(List.of("B000000001"), OWNER_USER_ID);
}
/**
* 被砍的行必须连同「死在哪一道闸」一起交出去(V179)。
* 计数与行列表出自同一次循环——两者一旦对不上,追溯 sheet 就是在说谎。
*/
@Test
void 被砍行连同各自那道闸一起交出且与计数自洽() {
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any()))
.thenReturn(Set.of("B000000001", "B000000004"));
when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("Zara")));
CollectDataBatchQuery.FilterResult result = query.filter(List.of(
row("B000000001", "Nike"),
row("B000000002", "Zara"),
row("B000000003", "H&M"),
row("B000000004", "Zara")
), OWNER_USER_ID);
assertEquals(2, result.dedupeFilteredCount(), "ASIN 命中去重库的两行");
assertEquals(1, result.invalidFilteredCount(), "品牌命中无效品牌库的一行");
assertEquals(result.dedupeFilteredCount() + result.invalidFilteredCount(),
result.filteredRows().size(), "被砍行数必须等于计数之和,不得漂移");
// B000000004 的品牌也在黑名单里,但去重先判——原因必须落在实际拦下它的那一道闸上
assertEquals(CollectDataFilteredRowEntity.REASON_DEDUPE,
result.filteredRows().get(0).reasonCode());
assertEquals(CollectDataFilteredRowEntity.REASON_INVALID_BRAND,
result.filteredRows().get(1).reasonCode());
assertEquals(CollectDataFilteredRowEntity.REASON_DEDUPE,
result.filteredRows().get(2).reasonCode());
assertEquals(List.of("B000000003"),
result.kept().stream().map(CollectDataResultRowVo::getAsin).toList(), "保留行不受新字段影响");
}
private static CollectDataResultRowVo row(String asin, String brand) {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin(asin);
@@ -0,0 +1,183 @@
package com.nanri.aiimage.modules.collectdata.util;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataFilteredRowMapper;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import java.util.Collection;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyCollection;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 被过滤行存取器(V179)的契约钉子。
*
* <p>它守的是两件相反的事,弄反任何一件都会把系统推回"数对不上又说不清":</p>
* <ul>
* <li><b>写</b>失败只跳批不外抛——这份明细是审计件,为它打掉用户已采回的数据交付不值;</li>
* <li><b>删</b>失败必须外抛——删一半会留下永久孤儿行,比不删更糟。</li>
* </ul>
*/
@ExtendWith(MockitoExtension.class)
@MockitoSettings(strictness = Strictness.LENIENT)
class CollectDataFilteredRowStoreTest {
@Mock
private CollectDataFilteredRowMapper mapper;
private CollectDataFilteredRowStore store(int batchSize) {
return new CollectDataFilteredRowStore(mapper, batchSize);
}
private static CollectDataResultRowVo row(String asin, String brand, String keyword) {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin(asin);
row.setBrand(brand);
row.setKeyword(keyword);
return row;
}
@SuppressWarnings("unchecked")
private List<CollectDataFilteredRowEntity> captureInserted(int expectedCalls) {
ArgumentCaptor<List<CollectDataFilteredRowEntity>> captor = ArgumentCaptor.forClass(List.class);
verify(mapper, times(expectedCalls)).insertBatch(captor.capture());
return captor.getAllValues().stream().flatMap(List::stream).toList();
}
@Test
void 三道闸的行各归各的reason_code() {
when(mapper.insertBatch(any())).thenAnswer(inv -> ((Collection<?>) inv.getArgument(0)).size());
List<CollectDataResultRowVo> rejected = List.of(row("B000000005", "Brand: acme", "kw"));
List<CollectDataResultRowVo> queryFailed = List.of(row("B000000006", "beta", "kw"));
int written = store(200).writeBatch(1L, 3,
List.of(new CollectDataBatchQuery.FilteredRow(row("B000000001", "x", "kw"),
CollectDataFilteredRowEntity.REASON_DEDUPE),
new CollectDataBatchQuery.FilteredRow(row("B000000002", "y", "kw"),
CollectDataFilteredRowEntity.REASON_INVALID_BRAND)),
rejected, queryFailed, "用户未配置代理", 100);
assertThat(written).isEqualTo(4);
List<CollectDataFilteredRowEntity> rows = captureInserted(1);
assertThat(rows).extracting(CollectDataFilteredRowEntity::getReasonCode).containsExactly(
CollectDataFilteredRowEntity.REASON_DEDUPE,
CollectDataFilteredRowEntity.REASON_INVALID_BRAND,
CollectDataFilteredRowEntity.REASON_BRAND_REJECTED,
CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED);
assertThat(rows).allSatisfy(entity ->
// 未回流就是 resolvedAt 为 null:本表只有这一个状态,没有第二套 STATUS_ 词表
assertThat(entity.getResolvedAt()).isNull());
// 品牌必须过 BrandLabel.strip:出件是用户的下一份上传物,脏值会顺着下载-再上传回流进链路
assertThat(rows).extracting(CollectDataFilteredRowEntity::getBrand)
.containsExactly("x", "y", "acme", "beta");
}
@Test
void 只有品牌查询失败那类带可行动原因() {
when(mapper.insertBatch(any())).thenAnswer(inv -> ((Collection<?>) inv.getArgument(0)).size());
store(200).writeBatch(1L, 1,
List.of(new CollectDataBatchQuery.FilteredRow(row("B000000001", "x", "kw"),
CollectDataFilteredRowEntity.REASON_INVALID_BRAND)),
List.of(row("B000000002", "y", "kw")), List.of(row("B000000003", "z", "kw")),
"用户未配置代理", 100);
List<CollectDataFilteredRowEntity> rows = captureInserted(1);
// 黑名单命中/判侵权不需要额外指引,把品牌侧原因错挂到它们身上反而误导
assertThat(rows.get(0).getReasonDetail()).isNull();
assertThat(rows.get(1).getReasonDetail()).isNull();
assertThat(rows.get(2).getReasonDetail()).isEqualTo("用户未配置代理");
}
@Test
void 空ASIN的行照记不误() {
// 空品牌被判 rejected 的行在无效品牌表里根本不留痕(toEntity 对空品牌返 null),
// 装配期也无从反查——这一格正是 V179 头注释点名的「漏标」来源,这里不能再丢。
when(mapper.insertBatch(any())).thenAnswer(inv -> ((Collection<?>) inv.getArgument(0)).size());
int written = store(200).writeBatch(1L, 1, List.of(),
List.of(row("", "", "kw")), List.of(), null, 100);
assertThat(written).isEqualTo(1);
List<CollectDataFilteredRowEntity> rows = captureInserted(1);
assertThat(rows).hasSize(1);
assertThat(rows.get(0).getAsin()).isEmpty();
}
@Test
void 预算耗尽时不再记行且返回值如实反映写入量() {
when(mapper.insertBatch(any())).thenAnswer(inv -> ((Collection<?>) inv.getArgument(0)).size());
int written = store(200).writeBatch(1L, 1,
List.of(new CollectDataBatchQuery.FilteredRow(row("B000000001", "x", "kw"),
CollectDataFilteredRowEntity.REASON_DEDUPE),
new CollectDataBatchQuery.FilteredRow(row("B000000002", "y", "kw"),
CollectDataFilteredRowEntity.REASON_DEDUPE)),
List.of(), List.of(), null, 1);
assertThat(written).isEqualTo(1);
assertThat(captureInserted(1)).hasSize(1);
assertThat(store(200).writeBatch(1L, 2,
List.of(new CollectDataBatchQuery.FilteredRow(row("B000000003", "z", "kw"),
CollectDataFilteredRowEntity.REASON_DEDUPE)),
List.of(), List.of(), null, 0)).isZero();
}
@Test
void 写失败只跳批不外抛() {
// 审计件不值得打掉用户已采回的数据交付
when(mapper.insertBatch(any())).thenThrow(new RuntimeException("db down"));
List<CollectDataResultRowVo> rejected = List.of(
row("B000000001", "a", "kw"), row("B000000002", "b", "kw"), row("B000000003", "c", "kw"));
assertThat(store(1).writeBatch(1L, 1, List.of(), rejected, List.of(), null, 100)).isZero();
// 三批各失败各跳过,不中断后续批次
verify(mapper, times(3)).insertBatch(any());
}
@Test
void 删失败必须外抛不留孤儿行() {
// 与写侧相反:删一半比不删更糟,必须让调用方的事务回滚
when(mapper.delete(any(com.baomidou.mybatisplus.core.conditions.Wrapper.class)))
.thenThrow(new RuntimeException("db down"));
assertThatThrownBy(() -> store(200).deleteByTask(7L))
.isInstanceOf(RuntimeException.class)
.hasMessage("db down");
}
@Test
void 回流标记归一化ASIN并跳过空值() {
when(mapper.markResolved(anyLong(), anyCollection(), anyString())).thenReturn(2);
assertThat(store(200).markResolved(7L, List.of(" b000000001 ", "", "B000000002"))).isEqualTo(2);
ArgumentCaptor<Collection<String>> captor = ArgumentCaptor.forClass(Collection.class);
verify(mapper).markResolved(anyLong(), captor.capture(), anyString());
assertThat(captor.getValue()).containsExactly("B000000001", "B000000002");
}
@Test
void 空输入不发起任何调用() {
assertThat(store(200).writeBatch(null, 1, List.of(), List.of(), List.of(), null, 100)).isZero();
assertThat(store(200).markResolved(1L, List.of())).isZero();
assertThat(store(200).deleteByTask(null)).isZero();
verify(mapper, never()).insertBatch(any());
verify(mapper, never()).delete(any(com.baomidou.mybatisplus.core.conditions.Wrapper.class));
}
}
@@ -93,6 +93,9 @@ class CollectDataSelfConsistencyTest {
"resultId", "taskId", "taskNo", "sourceFilename", "resultFilename", "downloadUrl",
"taskStatus", "success", "error", "rowCount", "dedupeFilteredCount",
"invalidFilteredCount", "brandRejectedCount", "finalRowCount",
// 品牌检测「查不了」的行数与原因:过去这两个字段连 VO 都没出,
// 现网 task 32573 有 173 行这么丢掉,界面上却一格都没有
"brandQueryFailedCount", "brandQueryFailedReason",
"taskType", "progressPercent", "totalRows", "receivedRows", "processedRows",
"collectStage", "currentKeyword", "searchCurrentPage", "searchTotalPages",
"detailProcessedAsins", "detailTotalAsins", "filters",
@@ -139,7 +139,7 @@ class DuplicateSubmissionContractTest {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin("B0DUP1234");
lenient().when(collectDataBatchQuery.filter(any(), any())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0));
new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0, List.of()));
lenient().when(brandBatchFilter.filter(any(), any())).thenReturn(
new CollectDataBrandBatchFilter.BrandBatchOutcome(List.of(), List.of(), List.of(row), ""));
lenient().when(resultItemBatchWriter.upsertAccepted(anyLong(), anyLong(), anyString(), anyInt(), any(), anyString()))
@@ -179,8 +179,10 @@ class SuccessTimingContractTest {
void noFileGeneratedNoSuccess() {
// 方法名同步:实现已由 writeWorkbookSegmented 改为 writeWorkbookStreaming(流式分页读,
// 避免几十万行结果集整体驻留堆),测试原先仍在 mock 旧方法名,导致"没抛异常"的假失败。
// V179 起调用点走 5 参重载(第 5 参是「被过滤明细」sheet 的数据源),桩必须跟着换形,
// 否则严格存根会先报 PotentialStubbingProblem,把"生成失败"这条断言本身吞掉。
doThrow(new IllegalStateException("工作簿生成失败"))
.when(excelAssemblyService).writeWorkbookStreaming(any(), any(), any(), any());
.when(excelAssemblyService).writeWorkbookStreaming(any(), any(), any(), any(), any());
assertThrows(IllegalStateException.class, () -> service.processResultFileJob(job()));
@@ -16,7 +16,7 @@ chunk.insert scope=sub-1 idx=1/3 payload=rustfs:chunk-1 payloadHash=8640b8fb328e
scope.selectOne -> none
chunk.count -> 0
scope.insert chunkTotal=3 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"}
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=f635be79fec3
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=9f73e6bcec11
task.load id=1001 status=RUNNING module=COLLECT_DATA
thrown: none
vo: taskId=1001 resultId=2001 chunk=1/3 done=false receivedRows=2 currentChunkRows=2 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=2 taskStatus=RUNNING
@@ -41,7 +41,7 @@ scope.selectOne -> none
chunk.count -> 0
scope.insert chunkTotal=3 received=0 completed=1 lastError=- stateJson={"phase":"RECEIVED"}
result.update id=2001 success=0 rowCount=1 resultFilename=collect-data-1001-result.xlsx resultFileUrl=- fileSize=0 errorMessage=-
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=44ebae7009e0
task.load id=1001 status=RUNNING module=COLLECT_DATA
thrown: none
vo: taskId=1001 resultId=2001 chunk=3/3 done=true receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING
@@ -82,7 +82,7 @@ scope.selectOne -> none
chunk.count -> 0
scope.insert chunkTotal=1 received=0 completed=1 lastError=- stateJson={"phase":"RECEIVED"}
result.update id=2001 success=- rowCount=1 resultFilename=collect-data-1001-result.xlsx resultFileUrl=- fileSize=0 errorMessage=-
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=44ebae7009e0
task.load id=1001 status=RUNNING module=COLLECT_DATA
thrown: none
vo: taskId=1001 resultId=2001 chunk=1/1 done=true receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING
@@ -107,10 +107,10 @@ scope.selectOne -> none
chunk.count -> 2
scope.insert chunkTotal=3 received=2 completed=0 lastError=Python 采集异常 stateJson={"phase":"RECEIVED"}
result.update id=2001 success=0 rowCount=1 resultFilename=- resultFileUrl=- fileSize=- errorMessage=Python 采集异常
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=333 resultJson.hash=2f13867b410c
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=388 resultJson.hash=44ebae7009e0
chunk.count -> 2
result.update id=2001 success=0 rowCount=1 resultFilename=collect-data-1001-result.xlsx resultFileUrl=- fileSize=0 errorMessage=Python 采集异常
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=333 resultJson.hash=2f13867b410c
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=388 resultJson.hash=44ebae7009e0
task.load id=1001 status=FAILED module=COLLECT_DATA
thrown: none
vo: taskId=1001 resultId=2001 chunk=2/3 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=FAILED
@@ -135,7 +135,7 @@ scope.selectOne -> none
chunk.count -> 0
scope.insert chunkTotal=2 received=0 completed=0 lastError=Python 采集异常 stateJson={"phase":"RECEIVED"}
result.update id=2001 success=0 rowCount=1 resultFilename=- resultFileUrl=- fileSize=- errorMessage=Python 采集异常
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=333 resultJson.hash=2f13867b410c
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=388 resultJson.hash=44ebae7009e0
chunk.count -> 0
task.load id=1001 status=FAILED module=COLLECT_DATA
thrown: none
@@ -161,7 +161,7 @@ chunk.insert scope=sub-1 idx=1/2 payload=rustfs:chunk-1 payloadHash=a55ecd3678de
scope.selectOne -> none
chunk.count -> 0
scope.insert chunkTotal=2 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"}
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=44ebae7009e0
task.load id=1001 status=RUNNING module=COLLECT_DATA
thrown: none
vo: taskId=1001 resultId=2001 chunk=1/2 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING
@@ -182,7 +182,7 @@ chunk.insert scope=sub-1 idx=1/1 payload=rustfs:chunk-1 payloadHash=e0365b1c5568
scope.selectOne -> none
chunk.count -> 0
scope.insert chunkTotal=1 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"}
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=53ca52c40ab0
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=2b4337f9c7d3
task.load id=1001 status=RUNNING module=COLLECT_DATA
thrown: none
vo: taskId=1001 resultId=2001 chunk=1/1 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=0 taskStatus=RUNNING