feat(collect-data): 结果 Excel 追加「被过滤明细」sheet,品牌检测失败原因可见
task 32573 回传 600 行只留 1 行,被砍掉的 599 行在系统里只剩四个计数, 谁都答不出「是哪些 ASIN、死在哪一道」。四类被过滤行现在于提交时当场 逐行落库(V179 biz_collectdata_filtered_row),结果 Excel 追加第 3 个 sheet 供逐行对账。不在装配期回读 chunk 重判——去重库与无效品牌库读的 都是当前态,任务窗口期的写入会让判定漂移(错标/漏标/分类串台), 依据逐条写在 V179 头注释里。 同批:brandQueryFailedCount 与原因进历史 VO 和任务卡片;损耗通知不再对 「未配置代理」的用户承诺稍后重跑——补测队列两轮 21/21、28/28 全败、净增 0 行,那句是谎。 台账:XML_STATEMENT_TOTAL 141→143(新 mapper 两条写侧语句);静默 catch 棘轮 total 1028→1031、不重抛 579→582、只 log 181→182(三枚审计件容错, 逐处记名于 CatchSilenceRatchetTest 上方注释)。
This commit is contained in:
+21
@@ -0,0 +1,21 @@
|
||||
package com.nanri.aiimage.modules.collectdata.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface CollectDataFilteredRowMapper extends BaseMapper<CollectDataFilteredRowEntity> {
|
||||
|
||||
/** 一次多值 INSERT;本表无唯一键(同 ASIN 可在不同关键词/分片各被过滤一次),不做 IGNORE。 */
|
||||
int insertBatch(@Param("rows") List<CollectDataFilteredRowEntity> rows);
|
||||
|
||||
/** 补测回流后给对应行盖 resolved_at(只动未回流的行),返回受影响行数。 */
|
||||
int markResolved(@Param("taskId") Long taskId,
|
||||
@Param("asins") Collection<String> asins,
|
||||
@Param("detailSuffix") String detailSuffix);
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 采集数据「被过滤行」明细(V179):四类过滤逐行带原因落库,供结果 Excel 的
|
||||
* 「被过滤明细」sheet 装配与四类计数对账。
|
||||
*
|
||||
* <p>必须在提交时当场记,不能装配时回读 chunk 重判——去重与品牌库读的都是当前态,
|
||||
* 任务窗口期的写入会让判定漂移(错标/漏标/分类串台),依据写在 V179 迁移头注释里。</p>
|
||||
*/
|
||||
@Data
|
||||
@TableName("biz_collectdata_filtered_row")
|
||||
public class CollectDataFilteredRowEntity {
|
||||
|
||||
/** 与历史结果表里的 ASIN 归一化口径一致(大写去空白)。 */
|
||||
public static final String REASON_DEDUPE = "DEDUPE";
|
||||
/** 命中全平台共享的无效品牌库(2026-10-06 定案不按公司区分)。 */
|
||||
public static final String REASON_INVALID_BRAND = "INVALID_BRAND";
|
||||
/** 品牌检测判定侵权,行被拒。 */
|
||||
public static final String REASON_BRAND_REJECTED = "BRAND_REJECTED";
|
||||
/** 品牌检测压根没发起查询(未配置代理 / 上游失败 / 闸已落下),行既不进结果也不判无效。 */
|
||||
public static final String REASON_BRAND_QUERY_FAILED = "BRAND_QUERY_FAILED";
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private Long taskId;
|
||||
/** 产生本行的回传分片序号;0=收尾/未知。 */
|
||||
private Integer chunkIndex;
|
||||
private String asin;
|
||||
private String keyword;
|
||||
/** 品牌按 {@code BrandLabel.strip} 的展示口径存储,与「采集数据结果」sheet 同形。 */
|
||||
private String brand;
|
||||
private String reasonCode;
|
||||
private String reasonDetail;
|
||||
/**
|
||||
* 补测回流时间;null = 当时确实没查动、至今没回来。
|
||||
*
|
||||
* <p>这里不用 {@code status} 字符串列:TaskStatusVocabularyRatchetTest 把"模块里另抄一套
|
||||
* {@code STATUS_X = "字面量"}"当成棘轮要消灭的东西,而这列的语义本来就只有"回没回来"。
|
||||
* 换成时间戳还多得一个信息——回流发生在哪一轮,{@code RESOLVED} 把它丢了。</p>
|
||||
*/
|
||||
private LocalDateTime resolvedAt;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
+6
@@ -46,6 +46,12 @@ public class CollectDataHistoryItemVo {
|
||||
@Schema(description = "品牌检测不符合的行数")
|
||||
private Integer brandRejectedCount;
|
||||
|
||||
@Schema(description = "品牌检测未返回结果的行数:这些行既不进结果、也没被判为无效品牌")
|
||||
private Integer brandQueryFailedCount;
|
||||
|
||||
@Schema(description = "品牌检测查不了的原因(如:用户未配置代理),供前端给出可行动指引")
|
||||
private String brandQueryFailedReason;
|
||||
|
||||
@Schema(description = "最终结果行数")
|
||||
private Integer finalRowCount;
|
||||
|
||||
|
||||
+8
@@ -7,6 +7,7 @@ import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataFilteredRowStore;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailCodec;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter;
|
||||
@@ -70,6 +71,8 @@ public class CollectDataBrandRecheckInjector {
|
||||
private final ObjectMapper objectMapper;
|
||||
/** 只为复用 resolveCacheCountry 的单国家判定口径(缓存键缺国家会把 DE 价当 US 结论)。 */
|
||||
private final CollectDataService collectDataService;
|
||||
/** 补测把行补进结果后,给 V179 被过滤明细里对应行盖回流时间(保留行,不删)。 */
|
||||
private final CollectDataFilteredRowStore filteredRowStore;
|
||||
|
||||
/**
|
||||
* 把补测结论落回任务:按品牌回读 chunk 全量 payload,分流 CLEAN/REJECTED。
|
||||
@@ -142,6 +145,11 @@ public class CollectDataBrandRecheckInjector {
|
||||
}
|
||||
if (!acceptedKept.isEmpty()) {
|
||||
writeAsinCache(task, acceptedKept);
|
||||
// 这些行已补进结果,给 V179 明细里对应的行盖上回流时间。
|
||||
// **不删行**:删掉就把「当时确实没查动」这条历史一起删了,而那条历史正是
|
||||
// 用户追问"我这批货当时为什么少"时唯一说得清的东西。
|
||||
filteredRowStore.markResolved(task.getId(),
|
||||
acceptedKept.stream().map(CollectDataResultRowVo::getAsin).toList());
|
||||
}
|
||||
if (acceptedRows > 0 || rejectedRows > 0) {
|
||||
log.info("[brand-recheck] 补测结论注入完成 taskId={} 新增结果行={} 判定无效行={} 品牌(净/败)={}/{}",
|
||||
|
||||
+131
-5
@@ -5,6 +5,7 @@ import com.nanri.aiimage.common.text.UserMessages;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.excel.SxssfWorkbookWriter;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
@@ -32,6 +33,13 @@ public class CollectDataExcelAssemblyService {
|
||||
private static final String SHEET_SUMMARY_NAME = "结果文件";
|
||||
private static final String[] SHEET_SUMMARY_HEADER = {"关键词", "FBA", "FBM", "AMZ", "无配送方式", "页数", "ASIN过滤"};
|
||||
|
||||
/**
|
||||
* 「被过滤明细」尾 sheet(V179):逐行列出被四道闸砍掉的 ASIN 与它死在哪一道。
|
||||
* 必须排在最后——本文件之外的读取方要么只认 sheet 0,要么按 sheet 名认列。
|
||||
*/
|
||||
private static final String SHEET_FILTERED_NAME = "被过滤明细";
|
||||
private static final String[] SHEET_FILTERED_HEADER = {"品牌", "ASIN", "关键词", "过滤环节", "说明"};
|
||||
|
||||
/**
|
||||
* 生成采集结果工作簿。
|
||||
*
|
||||
@@ -44,7 +52,15 @@ public class CollectDataExcelAssemblyService {
|
||||
List<CollectDataResultRowVo> items,
|
||||
List<CollectDataSummaryRowDto> summaries,
|
||||
List<CollectDataResultRowVo> rawItems) {
|
||||
writeWorkbookSegmented(outputXlsx, items, summaries, () -> rawItems);
|
||||
writeWorkbookSegmented(outputXlsx, items, summaries, () -> rawItems, null);
|
||||
}
|
||||
|
||||
/** 不出「被过滤明细」sheet 的那一档:只关心明细/汇总两表的调用方不必传 null 占位。 */
|
||||
public void writeWorkbookSegmented(File outputXlsx,
|
||||
List<CollectDataResultRowVo> items,
|
||||
List<CollectDataSummaryRowDto> summaries,
|
||||
Supplier<List<CollectDataResultRowVo>> rawItemsSupplier) {
|
||||
writeWorkbookSegmented(outputXlsx, items, summaries, rawItemsSupplier, null);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -55,7 +71,8 @@ public class CollectDataExcelAssemblyService {
|
||||
public void writeWorkbookSegmented(File outputXlsx,
|
||||
List<CollectDataResultRowVo> items,
|
||||
List<CollectDataSummaryRowDto> summaries,
|
||||
Supplier<List<CollectDataResultRowVo>> rawItemsSupplier) {
|
||||
Supplier<List<CollectDataResultRowVo>> rawItemsSupplier,
|
||||
FilteredRowSource filteredSource) {
|
||||
// List 入参统一转成分批来源(单批即全部行),对外行为保持不变。
|
||||
writeWorkbookStreaming(outputXlsx,
|
||||
batchConsumer -> {
|
||||
@@ -69,7 +86,16 @@ public class CollectDataExcelAssemblyService {
|
||||
if (rawItems != null && !rawItems.isEmpty()) {
|
||||
batchConsumer.accept(rawItems);
|
||||
}
|
||||
});
|
||||
},
|
||||
filteredSource);
|
||||
}
|
||||
|
||||
/** 不出「被过滤明细」sheet 的那一档(同 writeWorkbookSegmented 的 4 参版)。 */
|
||||
public long writeWorkbookStreaming(File outputXlsx,
|
||||
RowBatchSource itemsSource,
|
||||
List<CollectDataSummaryRowDto> summaries,
|
||||
RowBatchSource rawItemsSource) {
|
||||
return writeWorkbookStreaming(outputXlsx, itemsSource, summaries, rawItemsSource, null);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -77,12 +103,17 @@ public class CollectDataExcelAssemblyService {
|
||||
* 每批写入 SXSSF 滚动窗口后即可被回收,几十万行结果集不再整体驻留堆内存,
|
||||
* 避免大任务生成结果文件时 OOM。
|
||||
*
|
||||
* @return 写入「采集数据结果」sheet 的明细行数(仅统计非 null 行)
|
||||
* @param filteredSource 「被过滤明细」sheet 的数据源;null 表示不出这张表
|
||||
* @return 写入「采集数据结果」sheet 的明细行数(仅统计非 null 行)。
|
||||
* <b>红线</b>:这个返回值只算明细 sheet。「被过滤明细」的行数绝不能累进来——
|
||||
* 调用方拿它写 stats.finalRowCount 与 result.rowCount,并据此跑
|
||||
* guardDetailRowsWritten 校验「下载 Excel 行数 == 保留行数」。
|
||||
*/
|
||||
public long writeWorkbookStreaming(File outputXlsx,
|
||||
RowBatchSource itemsSource,
|
||||
List<CollectDataSummaryRowDto> summaries,
|
||||
RowBatchSource rawItemsSource) {
|
||||
RowBatchSource rawItemsSource,
|
||||
FilteredRowSource filteredSource) {
|
||||
if (outputXlsx == null) {
|
||||
throw new BusinessException("生成采集数据 Excel 失败: 输出文件路径为空");
|
||||
}
|
||||
@@ -93,6 +124,7 @@ public class CollectDataExcelAssemblyService {
|
||||
try {
|
||||
long detailRowCount = writeDetailSheet(writer, itemsSource);
|
||||
writeSummarySheet(writer, summaries, rawItemsSource);
|
||||
writeFilteredSheet(writer, filteredSource);
|
||||
writer.write(tmpFile);
|
||||
Files.move(tmpFile.toPath(), outputXlsx.toPath(),
|
||||
StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
|
||||
@@ -116,6 +148,18 @@ public class CollectDataExcelAssemblyService {
|
||||
void forEachBatch(Consumer<List<CollectDataResultRowVo>> batchConsumer);
|
||||
}
|
||||
|
||||
/**
|
||||
* 被过滤行(V179)的分批来源。除分批外还要自报「超出上限而未逐条记录」的缺口数——
|
||||
* 少了这个数字,sheet 只会安静地少给行,没人知道少给了多少。
|
||||
*/
|
||||
public interface FilteredRowSource {
|
||||
void forEachBatch(Consumer<List<CollectDataFilteredRowEntity>> batchConsumer);
|
||||
|
||||
default long notRecordedCount() {
|
||||
return 0L;
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteQuietly(File file) {
|
||||
try {
|
||||
Files.deleteIfExists(file.toPath());
|
||||
@@ -241,6 +285,88 @@ public class CollectDataExcelAssemblyService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 「被过滤明细」尾 sheet。两条硬规矩:
|
||||
* <ul>
|
||||
* <li><b>空则不建</b>:一行被过滤都没有时多出一个只有表头的 sheet,读起来像"全被过滤了";</li>
|
||||
* <li><b>fail-soft</b>:它是审计件。为它打掉用户已采回的数据交付,是拿可追溯性换交付,
|
||||
* 所以异常只降级这个 sheet,并在外层明细清单里写明不完整,绝不外抛。</li>
|
||||
* </ul>
|
||||
*/
|
||||
private void writeFilteredSheet(SxssfWorkbookWriter writer, FilteredRowSource source) {
|
||||
if (source == null) {
|
||||
return;
|
||||
}
|
||||
Sheet[] created = new Sheet[1];
|
||||
int[] rowIndex = new int[]{1};
|
||||
boolean incomplete = false;
|
||||
try {
|
||||
source.forEachBatch(batch -> {
|
||||
if (batch == null || batch.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (CollectDataFilteredRowEntity item : batch) {
|
||||
if (item == null) {
|
||||
continue;
|
||||
}
|
||||
if (created[0] == null) {
|
||||
created[0] = createFilteredSheet(writer);
|
||||
}
|
||||
Row row = created[0].createRow(rowIndex[0]++);
|
||||
SxssfWorkbookWriter.setText(row, 0, BrandLabel.strip(item.getBrand()));
|
||||
SxssfWorkbookWriter.setText(row, 1, item.getAsin());
|
||||
SxssfWorkbookWriter.setText(row, 2, item.getKeyword());
|
||||
SxssfWorkbookWriter.setText(row, 3, filteredReasonLabel(item.getReasonCode()));
|
||||
SxssfWorkbookWriter.setText(row, 4, item.getReasonDetail());
|
||||
}
|
||||
});
|
||||
long notRecorded = source.notRecordedCount();
|
||||
if (notRecorded > 0) {
|
||||
// 超出单任务追溯上限:如实说出缺口,而不是安静地少给行
|
||||
if (created[0] == null) {
|
||||
created[0] = createFilteredSheet(writer);
|
||||
}
|
||||
Row row = created[0].createRow(rowIndex[0]);
|
||||
SxssfWorkbookWriter.setText(row, 3, "未逐条记录");
|
||||
SxssfWorkbookWriter.setText(row, 4, "另有 " + notRecorded
|
||||
+ " 行被过滤但未逐条记录(超出单任务追溯上限),行数已计入各道闸的计数");
|
||||
}
|
||||
} catch (RuntimeException ex) {
|
||||
incomplete = true;
|
||||
log.warn("[collect-data] filtered sheet write failed, deliver without it: {}", ex.getMessage());
|
||||
}
|
||||
if (incomplete && created[0] != null) {
|
||||
// 这句收尾不再套第二层 catch:它只是一次本地 POI 写,写不动说明整个 workbook
|
||||
// 都写不动——那由外层 writeWorkbookStreaming 的既有出口处理(与前两张 sheet 同姿势),
|
||||
// 而不是在这里再吞一层、把"交付没做成"伪装成"交付做成了但清单不完整"。
|
||||
Row row = created[0].createRow(rowIndex[0]);
|
||||
SxssfWorkbookWriter.setText(row, 3, "清单不完整");
|
||||
SxssfWorkbookWriter.setText(row, 4, "本 sheet 生成过程中出错,以上仅部分被过滤行");
|
||||
}
|
||||
}
|
||||
|
||||
private Sheet createFilteredSheet(SxssfWorkbookWriter writer) {
|
||||
Sheet sheet = writer.createSheet(SHEET_FILTERED_NAME, SHEET_FILTERED_HEADER, null);
|
||||
for (int i = 0; i < SHEET_FILTERED_HEADER.length; i++) {
|
||||
sheet.setColumnWidth(i, SxssfWorkbookWriter.width(i == 4 ? 46 : 18));
|
||||
}
|
||||
return sheet;
|
||||
}
|
||||
|
||||
/** 库里存的是 reason_code(机器读),这一列给人读;未知码原样回显,不静默变空。 */
|
||||
private static String filteredReasonLabel(String reasonCode) {
|
||||
if (reasonCode == null || reasonCode.isBlank()) {
|
||||
return "";
|
||||
}
|
||||
return switch (reasonCode) {
|
||||
case CollectDataFilteredRowEntity.REASON_DEDUPE -> "已存在于去重总数据库";
|
||||
case CollectDataFilteredRowEntity.REASON_INVALID_BRAND -> "命中无效品牌库";
|
||||
case CollectDataFilteredRowEntity.REASON_BRAND_REJECTED -> "品牌检测判定侵权";
|
||||
case CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED -> "品牌检测未返回结果";
|
||||
default -> reasonCode;
|
||||
};
|
||||
}
|
||||
|
||||
private String safe(String value) {
|
||||
return value == null ? "" : value;
|
||||
}
|
||||
|
||||
+11
-2
@@ -2,7 +2,9 @@ package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataFilteredRowMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -10,20 +12,27 @@ import org.springframework.stereotype.Service;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 采集明细行的历史清理实现(2026-09 全维度审查 G5:逻辑从 task 模块收回本模块)。
|
||||
* 采集模块名下数据的清理实现(2026-09 全维度审查 G5:逻辑从 task 模块收回本模块)。
|
||||
*
|
||||
* <p>清理范围是「本模块拥有的行」,不只 biz_collect_data_item:V179 的被过滤行明细同样
|
||||
* 归本模块,任务过期后必须一起删掉,否则它只会单向堆积。</p>
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class CollectDataItemCleanupSpiImpl implements CollectDataItemCleanupSpi {
|
||||
|
||||
private final CollectDataItemMapper collectDataItemMapper;
|
||||
private final CollectDataFilteredRowMapper filteredRowMapper;
|
||||
|
||||
@Override
|
||||
public int deleteItemsByTaskIds(List<Long> taskIds) {
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
return collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
int removed = collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.in(CollectDataItemEntity::getTaskId, taskIds));
|
||||
removed += filteredRowMapper.delete(new LambdaQueryWrapper<CollectDataFilteredRowEntity>()
|
||||
.in(CollectDataFilteredRowEntity::getTaskId, taskIds));
|
||||
return removed;
|
||||
}
|
||||
}
|
||||
|
||||
+135
-5
@@ -12,6 +12,7 @@ import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.text.BrandLabel;
|
||||
import com.nanri.aiimage.common.excel.ExcelStreamReader;
|
||||
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
|
||||
import com.nanri.aiimage.modules.brand.support.BrandCheckFailureNotice;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
|
||||
@@ -23,6 +24,7 @@ import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitRowDto;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataCountryPrefEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryItemVo;
|
||||
@@ -39,6 +41,7 @@ import com.nanri.aiimage.modules.collectdata.service.support.CollectDataStaleTas
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataExtraJsonCodec;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataFilteredRowStore;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataParseLimits;
|
||||
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailCodec;
|
||||
@@ -167,6 +170,16 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
/** invalid ASIN 批量写入器:按批次 INSERT IGNORE,替代逐行插入。 */
|
||||
private final CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter;
|
||||
|
||||
/**
|
||||
* 被过滤行存取器(V179):把每个 chunk 丢掉的行逐条落库,并在装配时读回铺追溯 sheet。
|
||||
*
|
||||
* <p>字段注入而非塞进构造器:本类在单测里被 5 处手工 {@code new CollectDataService(...)}
|
||||
* 按位置参数构造,加一个形参等于改 5 个测试的位置实参、且每条用例都要多传一个 null
|
||||
* (同 {@code BrandCheckClient.taskGuard} 的理由)。生产里它是 @Component,必然存在。</p>
|
||||
*/
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private CollectDataFilteredRowStore filteredRowStore;
|
||||
|
||||
/** 结果明细 chunk 级编解码:accepted 行按 chunk 共享一个 RustFS 对象。 */
|
||||
private final CollectDataResultDetailCodec resultDetailCodec;
|
||||
|
||||
@@ -287,6 +300,14 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
@Value("${aiimage.collect-data.max-chunk-rows:0}")
|
||||
private Integer maxChunkRows;
|
||||
|
||||
/**
|
||||
* 单任务「被过滤行」逐条落库的上限;<=0 表示不限。
|
||||
* 现网量级:task 32058 被过滤 2616 行、32573 被过滤 599 行。超限后只留计数,
|
||||
* 追溯 sheet 会如实写「另有 N 行未逐条记录」而不是静默少给。
|
||||
*/
|
||||
@Value("${aiimage.collect-data.max-filtered-rows-per-task:50000}")
|
||||
private int maxFilteredRowsPerTask;
|
||||
|
||||
/** 进度统计节流窗口(毫秒):窗口内 changed 心跳合并写,避免高频 task UPDATE。0 关闭节流。 */
|
||||
@Value("${aiimage.collect-data.progress-throttle-millis:15000}")
|
||||
private long progressThrottleMillis;
|
||||
@@ -951,6 +972,12 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
stats.invalidFilteredCount += filtered.invalidFilteredCount();
|
||||
stats.brandRejectedCount += brandOutcome.rejected().size();
|
||||
stats.brandQueryFailedCount += brandOutcome.queryFailed().size();
|
||||
// 只在本 chunk 真的产出 queryFailed 时才记原因;后续全好的 chunk 不该把
|
||||
// 已经失败那批的原因抹掉,也不该拿一个空原因去覆盖可行动的那句。
|
||||
if (!brandOutcome.queryFailed().isEmpty()
|
||||
&& brandOutcome.reason() != null && !brandOutcome.reason().isBlank()) {
|
||||
stats.brandQueryFailedReason = brandOutcome.reason();
|
||||
}
|
||||
// 只有被服务端判定为无效品牌(rejected)的行才写入无效品牌表;
|
||||
// queryFailed 是品牌检测服务端到端失败(如缺 X-Token 422/超时/限流),
|
||||
// 一并写入会把故障期间被误伤的品牌永久拉黑,后续同品牌商品全部被
|
||||
@@ -960,6 +987,11 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
if (!brandOutcome.rejected().isEmpty()) {
|
||||
invalidAsinBatchWriter.writeBatch(brandOutcome.rejected(), task.getUserId());
|
||||
}
|
||||
// 被过滤行逐条落库(V179):四类行此刻都在手上,过了这道就只剩四个计数了。
|
||||
// 放在 rejected 写黑名单之后,是为了让「判侵权」与「命中黑名单」各归各的
|
||||
// reason_code,不被事后回读混成一档(依据见 V179 头注释)。
|
||||
stats.filteredRowsRecorded += recordFilteredRows(task.getId(), chunkIndex, stats,
|
||||
filtered, brandOutcome);
|
||||
List<CollectDataResultRowVo> accepted = brandOutcome.accepted();
|
||||
// 结果明细改为 chunk 级存储:整个 chunk 的 accepted 行共享一个
|
||||
// RustFS 对象(deterministic key,同 chunk 重提覆盖同一对象),
|
||||
@@ -1174,7 +1206,8 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
xlsx,
|
||||
batch -> streamFinalRows(task.getId(), batch),
|
||||
stats.summaries,
|
||||
batch -> streamRawRows(task.getId(), batch));
|
||||
batch -> streamRawRows(task.getId(), batch),
|
||||
filteredRowSource(task.getId(), stats));
|
||||
// 空表头文件不得当作成果交付:先校验再上传(见 guardDetailRowsWritten)。
|
||||
guardDetailRowsWritten(task.getId(), finalRowCount, stats);
|
||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||
@@ -1230,13 +1263,15 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
}
|
||||
// 品牌查询失败的行不进结果、也不落无效品牌表(见上面 rejected/queryFailed 的区分),
|
||||
// 用户端因此完全没有出口:只在任务确实翻成成功时补一条通知。
|
||||
// 文案只写这一层观测得到的事实(未返回结果);真实原因(代理取不到 IP / 上游 503 / 被限流)
|
||||
// 只有 BrandCheckClient 的 brand-check warn 日志知道,写进通知会把排查方向带偏。
|
||||
// 这句过去固定写成「可稍后单独重跑这些 ASIN」——对配置类原因是谎:未配置代理时
|
||||
// 补测队列几轮都是全败(现网 task 32573 两轮 21/21、28/28,净增 0 行),
|
||||
// 重跑只会让用户白等。原因由 chunk 回传时的 BrandBatchOutcome#reason 落进 stats,
|
||||
// 这里按它给出可行动的那句。
|
||||
if (updated > 0 && stats.brandQueryFailedCount > 0) {
|
||||
notificationDispatchService.pushPartialResult(task.getUserId(), task.getId(), MODULE_TYPE,
|
||||
task.getTaskNo(), stats.brandQueryFailedCount
|
||||
+ " 行商品的品牌检测未返回结果,这些行不计入结果条数,"
|
||||
+ "也没有被判定为无效品牌,可稍后单独重跑这些 ASIN");
|
||||
+ " 行商品的品牌检测未返回结果,这些行不计入结果条数,也没有被判定为无效品牌。"
|
||||
+ brandQueryFailedCauseForUser(stats.brandQueryFailedReason));
|
||||
}
|
||||
} finally {
|
||||
FileUtil.del(xlsx);
|
||||
@@ -1463,6 +1498,8 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
stats.invalidFilteredCount = root.path("invalidFilteredCount").asInt(0);
|
||||
stats.brandRejectedCount = root.path("brandRejectedCount").asInt(0);
|
||||
stats.brandQueryFailedCount = root.path("brandQueryFailedCount").asInt(0);
|
||||
stats.brandQueryFailedReason = root.path("brandQueryFailedReason").asText(null);
|
||||
stats.filteredRowsRecorded = root.path("filteredRowsRecorded").asInt(0);
|
||||
stats.finalRowCount = root.path("finalRowCount").asInt(0);
|
||||
// 反序列化 Python 携带的关键词级聚合;解析失败保持空 list,仅 warn 不抛,避免影响其他统计
|
||||
JsonNode summariesNode = root.get("summaries");
|
||||
@@ -1499,6 +1536,8 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
payload.put("invalidFilteredCount", stats.invalidFilteredCount);
|
||||
payload.put("brandRejectedCount", stats.brandRejectedCount);
|
||||
payload.put("brandQueryFailedCount", stats.brandQueryFailedCount);
|
||||
payload.put("brandQueryFailedReason", stats.brandQueryFailedReason);
|
||||
payload.put("filteredRowsRecorded", stats.filteredRowsRecorded);
|
||||
payload.put("finalRowCount", stats.finalRowCount);
|
||||
// 持久化 Python 携带的关键词级聚合;后续 processResultFileJob 阶段读取用于落 Excel
|
||||
payload.put("summaries", stats.summaries == null ? List.of() : stats.summaries);
|
||||
@@ -1521,6 +1560,77 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
return Math.max(currentProcessed, Math.min(summaries.size(), totalRows));
|
||||
}
|
||||
|
||||
/**
|
||||
* 品牌检测失败时该让用户去做什么。归因文案与补测队列共用 {@link BrandCheckFailureNotice}
|
||||
* 那一份,不在这里另写一套。
|
||||
*
|
||||
* <p>原因缺失时**不猜上游**:那句 "classify(空串) 按上游算" 会把一个没观测到的事实
|
||||
* 说成"上游商标库暂时不可用"。这里只陈述观测得到的部分——会重试,不指责任何一方。
|
||||
*/
|
||||
static String brandQueryFailedCauseForUser(String rawReason) {
|
||||
if (rawReason == null || rawReason.isBlank()) {
|
||||
return "系统会在后续补测轮自动重试。";
|
||||
}
|
||||
return BrandCheckFailureNotice.userMessage(BrandCheckFailureNotice.classify(rawReason));
|
||||
}
|
||||
|
||||
/**
|
||||
* 记录本 chunk 被过滤掉的行(V179)。这份明细是审计件、不参与任何判定,所以
|
||||
* writer 缺席(单测按位置参数手工 new 本类)或预算耗尽时**静默不记**:四类计数照常
|
||||
* 累加,追溯 sheet 会写明「另有 N 行未逐条记录」,而不是为了审计件打掉用户的交付。
|
||||
*/
|
||||
private int recordFilteredRows(Long taskId, int chunkIndex, CollectDataStats stats,
|
||||
CollectDataBatchQuery.FilterResult filtered,
|
||||
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome) {
|
||||
if (filteredRowStore == null) {
|
||||
return 0;
|
||||
}
|
||||
int budget = maxFilteredRowsPerTask <= 0
|
||||
? Integer.MAX_VALUE : maxFilteredRowsPerTask - stats.filteredRowsRecorded;
|
||||
if (budget <= 0) {
|
||||
return 0;
|
||||
}
|
||||
return filteredRowStore.writeBatch(taskId, chunkIndex,
|
||||
filtered.filteredRows(), brandOutcome.rejected(), brandOutcome.queryFailed(),
|
||||
brandOutcome.reason(), budget);
|
||||
}
|
||||
|
||||
/**
|
||||
* 追溯 sheet 的数据源。缺口数(超出单任务上限而未逐条落库的行)在这里算出来,
|
||||
* 让 sheet 能如实写「另有 N 行未逐条记录」,而不是安静地少给行。
|
||||
*/
|
||||
private CollectDataExcelAssemblyService.FilteredRowSource filteredRowSource(Long taskId,
|
||||
CollectDataStats stats) {
|
||||
if (filteredRowStore == null) {
|
||||
return null;
|
||||
}
|
||||
long notRecorded = filteredNotRecordedCount(stats);
|
||||
return new CollectDataExcelAssemblyService.FilteredRowSource() {
|
||||
@Override
|
||||
public void forEachBatch(Consumer<List<CollectDataFilteredRowEntity>> batchConsumer) {
|
||||
filteredRowStore.streamBatches(taskId, batchConsumer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long notRecordedCount() {
|
||||
return notRecorded;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 四类计数之和 − 已逐条落库的行数 = 没记下行的缺口。
|
||||
* 负数按 0:writer 跳批只会让缺口变大,出现负数说明别处算错了,不该拿它去 sheet 上写负数。
|
||||
*/
|
||||
static long filteredNotRecordedCount(CollectDataStats stats) {
|
||||
if (stats == null) {
|
||||
return 0L;
|
||||
}
|
||||
long dropped = (long) stats.dedupeFilteredCount + stats.invalidFilteredCount
|
||||
+ stats.brandRejectedCount + stats.brandQueryFailedCount;
|
||||
return Math.max(0L, dropped - stats.filteredRowsRecorded);
|
||||
}
|
||||
|
||||
private String normalizeAsin(String value) {
|
||||
return normalize(value).toUpperCase(Locale.ROOT);
|
||||
}
|
||||
@@ -1585,6 +1695,19 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
private int invalidFilteredCount;
|
||||
private int brandRejectedCount;
|
||||
private int brandQueryFailedCount;
|
||||
/**
|
||||
* 品牌检测查不了的原因(供应商拒付/用户未配置代理/出口被拒/批次异常)。
|
||||
* 由 {@code CollectDataBrandBatchFilter.BrandBatchOutcome#reason} 带上来,
|
||||
* 多个 chunk 按"最后一次非空为准"覆盖(与 summaries 同口径)。
|
||||
* 它存在的意义是让用户看到**该做什么**,而不是"稍后重跑"——未配置代理时重跑一万次
|
||||
* 也不会变好(现网 task 32573 的补测队列两轮 21/21、28/28 全败、净增 0 行)。
|
||||
*/
|
||||
private String brandQueryFailedReason;
|
||||
/**
|
||||
* 已逐条落进 biz_collectdata_filtered_row 的行数。四类计数之和减去它就是
|
||||
* 「超预算没记下行」的缺口,追溯 sheet 靠这句说实话。
|
||||
*/
|
||||
private int filteredRowsRecorded;
|
||||
private int finalRowCount;
|
||||
// Python 在 done=true 时携带的关键词级聚合统计;空表示由 Java 端用 rawItems 自聚合兜底
|
||||
private List<CollectDataSummaryRowDto> summaries = new ArrayList<>();
|
||||
@@ -1624,6 +1747,11 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
.forEach(resultPayloadJsons::add);
|
||||
collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.eq(CollectDataItemEntity::getTaskId, task.getId()));
|
||||
// 被过滤行随任务一起删;store 抛错就让整个删除回滚,不留孤儿行
|
||||
// (它不像写入那条路可以吞——删一半比不删更糟)。
|
||||
if (filteredRowStore != null) {
|
||||
filteredRowStore.deleteByTask(task.getId());
|
||||
}
|
||||
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, task.getId())
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
||||
@@ -1855,6 +1983,8 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
|
||||
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
|
||||
vo.setBrandRejectedCount(stats.brandRejectedCount);
|
||||
vo.setBrandQueryFailedCount(stats.brandQueryFailedCount);
|
||||
vo.setBrandQueryFailedReason(stats.brandQueryFailedReason);
|
||||
vo.setFinalRowCount(stats.finalRowCount);
|
||||
vo.setTotalRows(stats.totalRows);
|
||||
vo.setReceivedRows(stats.receivedRows);
|
||||
|
||||
+20
-4
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.collectdata.util;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService;
|
||||
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
|
||||
@@ -49,7 +50,7 @@ public class CollectDataBatchQuery {
|
||||
*/
|
||||
public FilterResult filter(List<CollectDataResultRowVo> rows, Long ownerUserId) {
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return new FilterResult(List.of(), 0, 0);
|
||||
return new FilterResult(List.of(), 0, 0, List.of());
|
||||
}
|
||||
List<String> asins = distinctNonBlank(rows.stream()
|
||||
.filter(row -> row != null)
|
||||
@@ -89,6 +90,7 @@ public class CollectDataBatchQuery {
|
||||
}
|
||||
|
||||
List<CollectDataResultRowVo> out = new ArrayList<>(rows.size());
|
||||
List<FilteredRow> filtered = new ArrayList<>();
|
||||
int dedupeFiltered = 0;
|
||||
int invalidFiltered = 0;
|
||||
for (CollectDataResultRowVo row : rows) {
|
||||
@@ -97,20 +99,34 @@ public class CollectDataBatchQuery {
|
||||
}
|
||||
if (dedupeValues.contains(row.getAsin())) {
|
||||
dedupeFiltered++;
|
||||
filtered.add(new FilteredRow(row, CollectDataFilteredRowEntity.REASON_DEDUPE));
|
||||
continue;
|
||||
}
|
||||
String brand = normalizeBrand(row.getBrand());
|
||||
if (!brand.isBlank() && invalidBrands.contains(brand)) {
|
||||
invalidFiltered++;
|
||||
filtered.add(new FilteredRow(row, CollectDataFilteredRowEntity.REASON_INVALID_BRAND));
|
||||
continue;
|
||||
}
|
||||
out.add(row);
|
||||
}
|
||||
return new FilterResult(out, dedupeFiltered, invalidFiltered);
|
||||
return new FilterResult(out, dedupeFiltered, invalidFiltered, filtered);
|
||||
}
|
||||
|
||||
/** 过滤结果:保留行 + 去重命中行数 + 无效品牌命中行数。 */
|
||||
public record FilterResult(List<CollectDataResultRowVo> kept, int dedupeFilteredCount, int invalidFilteredCount) {
|
||||
/**
|
||||
* 过滤结果:保留行 + 去重命中行数 + 无效品牌命中行数 + **被砍掉的行本身**。
|
||||
*
|
||||
* <p>被砍行必须在这里当场交出去:本方法的两个判定读的都是当前态(去重库、无效品牌库),
|
||||
* 事后回读 chunk 重判会漂移。计数与 {@code filteredRows} 由同一次循环产出,天然自洽。
|
||||
*/
|
||||
public record FilterResult(List<CollectDataResultRowVo> kept,
|
||||
int dedupeFilteredCount,
|
||||
int invalidFilteredCount,
|
||||
List<FilteredRow> filteredRows) {
|
||||
}
|
||||
|
||||
/** 一条被过滤的行与它命中的那一道闸。 */
|
||||
public record FilteredRow(CollectDataResultRowVo row, String reasonCode) {
|
||||
}
|
||||
|
||||
private static List<String> distinctNonBlank(List<String> values) {
|
||||
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
package com.nanri.aiimage.modules.collectdata.util;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.text.BrandLabel;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataFilteredRowMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 被过滤行存取器(V179):把「一个 chunk 到底丢掉了哪些行、各丢在哪一道闸」逐行落库,
|
||||
* 并在装配结果 Excel 时按批读回来铺「被过滤明细」sheet。
|
||||
*
|
||||
* <p>写侧容错口径照 {@link CollectDataInvalidAsinBatchWriter}:按批 try/catch,失败只跳批告警,
|
||||
* <b>绝不影响回传主流程</b>。这份明细是审计件、不是判定依据——为它打掉用户已采回的数据,
|
||||
* 是拿可追溯性去换交付。也因此它不是强一致的:对不上账时以 result_json 的四个计数为准。</p>
|
||||
*
|
||||
* <p>空 ASIN 的行照记不误。空品牌被判 rejected 的行在黑名单表里根本不留痕
|
||||
* ({@code CollectDataInvalidAsinBatchWriter.toEntity} 对空品牌返 null),装配期也无从反查,
|
||||
* 这一格正是 V179 头注释里点名的「漏标」来源。</p>
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class CollectDataFilteredRowStore {
|
||||
|
||||
private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
|
||||
/** reason_detail 与库里 VARCHAR(500) 对齐,留余量给回流追加的那句。 */
|
||||
private static final int DETAIL_MAX_LENGTH = 460;
|
||||
/** 装配读回的分页游标:与结果明细同量级,单批写完即释放。 */
|
||||
private static final int READ_PAGE_SIZE = 1000;
|
||||
|
||||
private final CollectDataFilteredRowMapper filteredRowMapper;
|
||||
private final int batchSize;
|
||||
|
||||
public CollectDataFilteredRowStore(CollectDataFilteredRowMapper filteredRowMapper,
|
||||
@Value("${aiimage.collect-data.filtered-row-batch-size:200}") int batchSize) {
|
||||
this.filteredRowMapper = filteredRowMapper;
|
||||
this.batchSize = Math.max(1, batchSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* 记录本 chunk 丢掉的行,返回**实际写入**的行数(被预算挡掉的不计)。
|
||||
*
|
||||
* @param batchFiltered 去重与无效品牌两道闸砍掉的行(原因已带在行上)
|
||||
* @param brandRejected 品牌检测判侵权的行
|
||||
* @param brandQueryFailed 品牌检测没发起查询/没回来的行
|
||||
* @param brandFailureReason 品牌侧失败原因(如「用户未配置代理」),来自 BrandBatchOutcome#reason
|
||||
* @param budget 本任务还能记多少行;<=0 表示只留计数不再记行
|
||||
*/
|
||||
public int writeBatch(Long taskId, int chunkIndex,
|
||||
List<CollectDataBatchQuery.FilteredRow> batchFiltered,
|
||||
List<CollectDataResultRowVo> brandRejected,
|
||||
List<CollectDataResultRowVo> brandQueryFailed,
|
||||
String brandFailureReason,
|
||||
int budget) {
|
||||
if (taskId == null || budget <= 0) {
|
||||
return 0;
|
||||
}
|
||||
List<CollectDataFilteredRowEntity> entities = new ArrayList<>();
|
||||
collectInto(entities, batchFiltered, brandRejected, brandQueryFailed, brandFailureReason, taskId, chunkIndex);
|
||||
if (entities.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
if (entities.size() > budget) {
|
||||
entities = entities.subList(0, budget);
|
||||
}
|
||||
int written = 0;
|
||||
for (int start = 0; start < entities.size(); start += batchSize) {
|
||||
List<CollectDataFilteredRowEntity> batch =
|
||||
entities.subList(start, Math.min(start + batchSize, entities.size()));
|
||||
try {
|
||||
written += filteredRowMapper.insertBatch(batch);
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[collect-data] filtered row batch insert failed, skip batch taskId={} chunk={} err={}",
|
||||
taskId, chunkIndex, ex.getMessage());
|
||||
}
|
||||
}
|
||||
return written;
|
||||
}
|
||||
|
||||
/** 按 id 游标分批读回本任务的被过滤行;单批交给 consumer 后即可释放。 */
|
||||
public void streamBatches(Long taskId, Consumer<List<CollectDataFilteredRowEntity>> consumer) {
|
||||
if (taskId == null || consumer == null) {
|
||||
return;
|
||||
}
|
||||
Long cursorId = null;
|
||||
while (true) {
|
||||
LambdaQueryWrapper<CollectDataFilteredRowEntity> wrapper =
|
||||
new LambdaQueryWrapper<CollectDataFilteredRowEntity>()
|
||||
.eq(CollectDataFilteredRowEntity::getTaskId, taskId);
|
||||
if (cursorId != null) {
|
||||
final Long lastId = cursorId;
|
||||
wrapper.gt(CollectDataFilteredRowEntity::getId, lastId);
|
||||
}
|
||||
wrapper.orderByAsc(CollectDataFilteredRowEntity::getId)
|
||||
.last("limit " + READ_PAGE_SIZE);
|
||||
List<CollectDataFilteredRowEntity> batch = filteredRowMapper.selectList(wrapper);
|
||||
if (batch == null || batch.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
consumer.accept(batch);
|
||||
cursorId = batch.get(batch.size() - 1).getId();
|
||||
if (cursorId == null || batch.size() < READ_PAGE_SIZE) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 补测把行补进结果后盖上回流时间,并在原因后追加一句说明。
|
||||
* **不删行**——删了就把「当时确实没查动」这条历史一起删掉了。
|
||||
*/
|
||||
public int markResolved(Long taskId, Collection<String> asins) {
|
||||
if (taskId == null || asins == null || asins.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
List<String> normalized = new ArrayList<>(asins.size());
|
||||
for (String asin : asins) {
|
||||
String value = normalize(asin).toUpperCase(Locale.ROOT);
|
||||
if (!value.isBlank()) {
|
||||
normalized.add(value);
|
||||
}
|
||||
}
|
||||
if (normalized.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
try {
|
||||
return filteredRowMapper.markResolved(taskId, normalized, ";已由补测回流");
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[collect-data] filtered row markResolved failed taskId={} err={}", taskId, ex.getMessage());
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除本任务的被过滤行。与 {@link #writeBatch} 相反,这里**不吞异常**:
|
||||
* 写失败只丢一份审计件,删失败却会让任务删一半、留下永久孤儿行。
|
||||
* 调用方在删除事务里执行它,让它随事务一起回滚。
|
||||
*/
|
||||
public int deleteByTask(Long taskId) {
|
||||
if (taskId == null) {
|
||||
return 0;
|
||||
}
|
||||
return filteredRowMapper.delete(new LambdaQueryWrapper<CollectDataFilteredRowEntity>()
|
||||
.eq(CollectDataFilteredRowEntity::getTaskId, taskId));
|
||||
}
|
||||
|
||||
private void collectInto(List<CollectDataFilteredRowEntity> out,
|
||||
List<CollectDataBatchQuery.FilteredRow> batchFiltered,
|
||||
List<CollectDataResultRowVo> brandRejected,
|
||||
List<CollectDataResultRowVo> brandQueryFailed,
|
||||
String brandFailureReason, Long taskId, int chunkIndex) {
|
||||
if (batchFiltered != null) {
|
||||
for (CollectDataBatchQuery.FilteredRow filtered : batchFiltered) {
|
||||
if (filtered == null || filtered.row() == null) {
|
||||
continue;
|
||||
}
|
||||
out.add(toEntity(taskId, chunkIndex, filtered.row(), filtered.reasonCode(), null));
|
||||
}
|
||||
}
|
||||
if (brandRejected != null) {
|
||||
for (CollectDataResultRowVo row : brandRejected) {
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
out.add(toEntity(taskId, chunkIndex, row,
|
||||
CollectDataFilteredRowEntity.REASON_BRAND_REJECTED, null));
|
||||
}
|
||||
}
|
||||
if (brandQueryFailed != null) {
|
||||
for (CollectDataResultRowVo row : brandQueryFailed) {
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
out.add(toEntity(taskId, chunkIndex, row,
|
||||
CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED, brandFailureReason));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private CollectDataFilteredRowEntity toEntity(Long taskId, int chunkIndex,
|
||||
CollectDataResultRowVo row,
|
||||
String reasonCode,
|
||||
String reasonDetail) {
|
||||
CollectDataFilteredRowEntity entity = new CollectDataFilteredRowEntity();
|
||||
entity.setTaskId(taskId);
|
||||
entity.setChunkIndex(Math.max(0, chunkIndex));
|
||||
entity.setAsin(normalize(row.getAsin()).toUpperCase(Locale.ROOT));
|
||||
entity.setKeyword(truncate(normalize(row.getKeyword()), 255));
|
||||
entity.setBrand(truncate(BrandLabel.strip(row.getBrand()), 255));
|
||||
entity.setReasonCode(reasonCode);
|
||||
entity.setReasonDetail(truncate(reasonDetail, DETAIL_MAX_LENGTH));
|
||||
return entity;
|
||||
}
|
||||
|
||||
private static String truncate(String value, int max) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value.length() <= max ? value : value.substring(0, max);
|
||||
}
|
||||
|
||||
private static String normalize(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
String normalized = value.replace(String.valueOf((char) 0xFEFF), "")
|
||||
.replace((char) 0x3000, ' ')
|
||||
.replace("\r\n", " ")
|
||||
.replace("\r", " ")
|
||||
.replace("\n", " ")
|
||||
.replace("\t", " ")
|
||||
.trim();
|
||||
return WHITESPACE_PATTERN.matcher(normalized).replaceAll(" ");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user