feat(collect-data): 结果 Excel 追加「被过滤明细」sheet,品牌检测失败原因可见

task 32573 回传 600 行只留 1 行,被砍掉的 599 行在系统里只剩四个计数,
谁都答不出「是哪些 ASIN、死在哪一道」。四类被过滤行现在于提交时当场
逐行落库(V179 biz_collectdata_filtered_row),结果 Excel 追加第 3 个
sheet 供逐行对账。不在装配期回读 chunk 重判——去重库与无效品牌库读的
都是当前态,任务窗口期的写入会让判定漂移(错标/漏标/分类串台),
依据逐条写在 V179 头注释里。

同批:brandQueryFailedCount 与原因进历史 VO 和任务卡片;损耗通知不再对
「未配置代理」的用户承诺稍后重跑——补测队列两轮 21/21、28/28 全败、净增
0 行,那句是谎。

台账:XML_STATEMENT_TOTAL 141→143(新 mapper 两条写侧语句);静默 catch
棘轮 total 1028→1031、不重抛 579→582、只 log 181→182(三枚审计件容错,
逐处记名于 CatchSilenceRatchetTest 上方注释)。
This commit is contained in:
2026-10-28 19:31:50 +08:00
parent 0e2af88184
commit c4fc9784e8
30 changed files with 1384 additions and 52 deletions
@@ -85,7 +85,10 @@ class SqlLayeringConventionTest {
// 130 → 140:关键词使用记录管理端(keywordusage 模块,1 份 XML 10 条只读 select)新写语句。
// 140 → 141:工作台客户端全量同步新增 {@code KeywordShareMapper#upsertBatchFromSync}
// (词本体 UPSERT,把客户端的四级类目写进既有行),同轮登记进 NOT_MIGRATED。
private static final int XML_STATEMENT_TOTAL = 141;
// 141 → 143:V179 采集「被过滤行」新表 {@code CollectDataFilteredRowMapper} 的两条写侧语句
// (insertBatch 多值插入 + markResolved 盖回流时间)。该 mapper 没有注解时代,
// 不在 MigratedSqlEquivalenceTest 的基准账本里 ⇒ 不占 NOT_MIGRATED,只在这本总分母上记名。
private static final int XML_STATEMENT_TOTAL = 143;
/**
* wrapper 里夹带函数级 SQL 的现状("这不是简单单表"的证据),文件名 → 条数:
@@ -54,7 +54,9 @@ class CatchSilenceRatchetTest {
// 10-27 「整表都要跑完」专项抬 573 -> 577(2 枚新增不重抛 + 2 枚既有整批抛出改逐条跳过,逐处记名见 STRICT_TOTAL 上方)。
// 10-27 同日第二批抬 577 -> 579(批量入库逐批容错+folder 去重逐条目容错,两处都是 log.warn+计数+continue,
// 逐处记名见 STRICT_TOTAL 上方注释)。
static final int GIVEN_NOT_RETHROW = 579;
// 10-28 V179 采集被过滤行追溯抬 579 -> 582(审计件三枚:逐批插入容错/回流簿记容错/尾 sheet 降级,
// 逐处记名见 STRICT_TOTAL 上方注释)。
static final int GIVEN_NOT_RETHROW = 582;
static final int GIVEN_LOG_ONLY = 249;
static final int GIVEN_EMPTY = 56;
@@ -181,8 +183,24 @@ class CatchSilenceRatchetTest {
* 因为 {@code TaskRetryEligibility}/{@code SimilarAsinStaleReopenSupport} 会按 errorMessage
* 字面量分类,写进去等于替一条正常任务编一个失败理由。
* 同批实测 total 1028、rethrow 393(未动)、notRethrow 579、logOnly 181(未动)、empty 56(未动)。
*
* <p>10-28 采集「被过滤行」追溯(V179,结果 Excel 第 3 个 sheet)抬 total 1028 -&gt; 1031,逐处记名。
* 三枚都是<b>审计件不得打掉交付</b>那一形,块体各只有一句带 {@code [collect-data]} 锚点的 WARN:
* <ul>
* <li>{@code CollectDataFilteredRowStore.writeBatch} 一枚:被过滤行按批 INSERT,坏批点名后
* 其余批照常落库(口径照 {@code CollectDataInvalidAsinBatchWriter})。这份明细不参与任何
* 判定,为它打掉用户已采回的数据是拿可追溯性换交付;跳过的批会从四类计数与已落库行数
* 的差里冒出来,sheet 上写「另有 N 行未逐条记录」,不是静默少给。</li>
* <li>{@code CollectDataFilteredRowStore.markResolved} 一枚:补测回流只盖时间戳+追加一句。
* 必要性是实测的——{@code CollectDataBrandRecheckScheduler} 抓住 {@code apply()} 的抛出后
* <b>整组留队重放</b>,这里若外抛,一次簿记失败会让已注入的任务永远重排。</li>
* <li>{@code CollectDataExcelAssemblyService.writeFilteredSheet} 一枚:尾 sheet 读不动就交付
* 前两表、缺这张审计表。同批我先写的第二枚("清单不完整"那行的内层 catch)已删掉:
* 那只是本地 POI 写,写不动说明整份 workbook 都写不动,该由
* {@code writeWorkbookStreaming} 的既有出口处理,与前两张 sheet 同姿势。</li>
* </ul>
*/
static final int STRICT_TOTAL = 1028;
static final int STRICT_TOTAL = 1031;
/**
* 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。
* 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。
@@ -282,8 +300,13 @@ class CatchSilenceRatchetTest {
* 10-27 同日第二批再抬 577 -&gt; 579(2 枚逐批/逐条目容错:{@code QueryAsinService.flushInsert}
* 与 {@code DedupeRunService.packageFolderDedupeResultsAsZip},块体都是 log.warn+计数,
* 坏项点名、其余照常交付,逐处记名见 STRICT_TOTAL 上方注释)。
* 10-28 再抬 579 -&gt; 582(V179 采集「被过滤行」三枚审计件容错:{@code CollectDataFilteredRowStore.writeBatch}
* 逐批插入、{@code CollectDataFilteredRowStore.markResolved} 回流簿记、
* {@code CollectDataExcelAssemblyService.writeFilteredSheet} 尾 sheet 降级;三处都是带
* {@code [collect-data]} 锚点的 log.warn+善后,块体除第一枚外各带一句状态置位/返回值,
* 逐处记名见 STRICT_TOTAL 上方注释)。
*/
static final int STRICT_NOT_RETHROW = 579;
static final int STRICT_NOT_RETHROW = 582;
/**
* 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}:
* 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出——
@@ -303,8 +326,12 @@ class CatchSilenceRatchetTest {
* 与 {@code getAllFileAggregates} 的逐条跳过出口——块体各只有一句带 {@code [brand-storage]}
* 锚点的 WARN。两处此前是「一个文件坏 ⇒ 整批抛出」,改后其余文件照常返回,坏的那条点名可查;
* 同批 {@code rebuildAggregate} 与外观专利候选收集因块内另有计数/continue 只进 NOT_RETHROW。
* 10-28 抬 181 -> 182(V179 采集「被过滤行」的 {@code CollectDataFilteredRowStore.writeBatch}):
* 块体只有一句带 {@code [collect-data]} 锚点的 WARN(taskId+chunk+err),坏批点名后其余批照常
* 落库。同批另两处({@code markResolved} 追加 return、{@code writeFilteredSheet} 置 incomplete 位)
* 因块内多一句只进 NOT_RETHROW。
*/
static final int STRICT_LOG_ONLY = 181; // 严格口径:块内每条顶层语句都是 log 调用才算
static final int STRICT_LOG_ONLY = 182; // 严格口径:块内每条顶层语句都是 log 调用才算
static final int STRICT_EMPTY = 56;
/**
@@ -60,6 +60,8 @@ class CollectDataBrandRecheckInjectorTest {
private AsinResultCacheService asinResultCacheService;
@Mock
private CollectDataService collectDataService;
@Mock
private com.nanri.aiimage.modules.collectdata.util.CollectDataFilteredRowStore filteredRowStore;
private final CollectDataResultDetailCodec codec = new CollectDataResultDetailCodec(new ObjectMapper());
private CollectDataBrandRecheckInjector injector;
@@ -72,7 +74,8 @@ class CollectDataBrandRecheckInjectorTest {
TaskChunkEntity.class);
injector = new CollectDataBrandRecheckInjector(taskChunkMapper, fileResultMapper,
collectDataBatchQuery, codec, resultItemBatchWriter, invalidAsinBatchWriter,
transientPayloadStorageService, asinResultCacheService, new ObjectMapper(), collectDataService);
transientPayloadStorageService, asinResultCacheService, new ObjectMapper(), collectDataService,
filteredRowStore);
}
private FileTaskEntity task(long id) {
@@ -120,7 +123,7 @@ class CollectDataBrandRecheckInjectorTest {
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(payload);
CollectDataResultRowVo kept = row("B01", "Nike");
when(collectDataBatchQuery.filter(anyList(), any())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(kept), 1, 0));
new CollectDataBatchQuery.FilterResult(List.of(kept), 1, 0, List.of()));
when(transientPayloadStorageService.storeResultPayload(eq("COLLECT_DATA"), eq(100L), eq("hash1"),
eq("recheck-chunk-1"), anyString())).thenReturn("ptr2");
when(resultItemBatchWriter.upsertAccepted(eq(100L), eq(9L), eq("s1"), eq(1),
@@ -133,6 +136,9 @@ class CollectDataBrandRecheckInjectorTest {
assertThat(outcome.acceptedRows()).isEqualTo(1);
assertThat(outcome.rejectedRows()).isZero();
verify(asinResultCacheService).writeCollectDataRows(eq("US"), eq("100"), eq(List.of(kept)));
// 补回结果的行要在 V179 明细里标 RESOLVED——但只标不删:删掉就把「当时确实没查动」
// 这条历史一起删了,而它是用户日后追问"我那批货当时为什么少"时唯一说得清的东西。
verify(filteredRowStore).markResolved(eq(100L), eq(List.of("B01")));
}
@Test
@@ -180,7 +186,7 @@ class CollectDataBrandRecheckInjectorTest {
"[{\"asin\":\"B01\",\"brand\":\"Nike\"}]");
// 补测窗口内该 ASIN 已被其他任务采走:复查全滤掉
when(collectDataBatchQuery.filter(anyList(), any())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(), 1, 0));
new CollectDataBatchQuery.FilterResult(List.of(), 1, 0, List.of()));
GroupOutcome outcome = injector.apply(task(100L), Map.of("nike", RecheckVerdict.CLEAN));
@@ -0,0 +1,182 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.nio.file.Path;
import java.util.List;
import java.util.function.Consumer;
import static org.assertj.core.api.Assertions.assertThat;
/**
* 「被过滤明细」尾 sheet(V179)的形状钉子。
*
* <p>背景:task 32573 回传 600 行、结果只剩 1 行,被砍的 599 行在系统里只有四个计数,
* 谁也无法回答"到底是哪些 ASIN、死在哪一道"。这张 sheet 就是为了让它可追溯。</p>
*
* <p>三件事必须钉住:① 它排在最后(外部读取方要么只认 sheet 0、要么按 sheet 名认列);
* ② 它的行数**绝不能**累进 writeWorkbookStreaming 的返回值——调用方拿那个数写
* stats.finalRowCount 与 result.rowCount,并据此跑 guardDetailRowsWritten 校验
* 「下载 Excel 行数 == 保留行数」,一旦污染就等于把交付说谎重新引回来;
* ③ 一行被过滤都没有时不建这张表,只留表头会让人误以为"全被过滤了"。</p>
*/
class CollectDataFilteredSheetTest {
@TempDir
Path tempDir;
private final CollectDataExcelAssemblyService assembly = new CollectDataExcelAssemblyService();
private static CollectDataResultRowVo detailRow(String asin) {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setBrand("nike");
row.setAsin(asin);
row.setKeyword("phone case");
row.setPrice("19.90");
row.setSellerName("seller-A");
row.setDeliveryMethod("FBA");
return row;
}
private static CollectDataSummaryRowDto summary() {
CollectDataSummaryRowDto item = new CollectDataSummaryRowDto();
item.setKeyword("phone case");
item.setFba(1);
return item;
}
private static CollectDataFilteredRowEntity filtered(String asin, String reasonCode, String detail) {
CollectDataFilteredRowEntity entity = new CollectDataFilteredRowEntity();
entity.setTaskId(1L);
entity.setChunkIndex(1);
entity.setAsin(asin);
entity.setKeyword("phone case");
entity.setBrand("somebrand");
entity.setReasonCode(reasonCode);
entity.setReasonDetail(detail);
return entity;
}
private static CollectDataExcelAssemblyService.FilteredRowSource sourceOf(
List<CollectDataFilteredRowEntity> rows, long notRecorded) {
return new CollectDataExcelAssemblyService.FilteredRowSource() {
@Override
public void forEachBatch(Consumer<List<CollectDataFilteredRowEntity>> batchConsumer) {
if (!rows.isEmpty()) {
batchConsumer.accept(rows);
}
}
@Override
public long notRecordedCount() {
return notRecorded;
}
};
}
@Test
void 尾sheet排在最后且明细行数不被污染() throws Exception {
File output = tempDir.resolve("filtered.xlsx").toFile();
long returned = assembly.writeWorkbookStreaming(
output,
consumer -> consumer.accept(List.of(detailRow("B000000001"), detailRow("B000000002"))),
List.of(summary()),
consumer -> { /* 本例不走 fallback 聚合 */ },
sourceOf(List.of(
filtered("B000000003", CollectDataFilteredRowEntity.REASON_DEDUPE, null),
filtered("B000000004", CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED, "用户未配置代理")),
0L));
// 红线:返回值只算「采集数据结果」那 2 行,被过滤的 2 行一个都不能进来
assertThat(returned).isEqualTo(2L);
try (Workbook workbook = WorkbookFactory.create(output)) {
assertThat(workbook.getNumberOfSheets()).isEqualTo(3);
assertThat(workbook.getSheetName(0)).isEqualTo("采集数据结果");
assertThat(workbook.getSheetName(1)).isEqualTo("结果文件");
assertThat(workbook.getSheetName(2)).isEqualTo("被过滤明细");
Sheet sheet = workbook.getSheet("被过滤明细");
// 表头 + 2 行
assertThat(sheet.getLastRowNum()).isEqualTo(2);
assertThat(sheet.getRow(1).getCell(1).getStringCellValue()).isEqualTo("B000000003");
assertThat(sheet.getRow(1).getCell(3).getStringCellValue()).isEqualTo("已存在于去重总数据库");
assertThat(sheet.getRow(2).getCell(3).getStringCellValue()).isEqualTo("品牌检测未返回结果");
assertThat(sheet.getRow(2).getCell(4).getStringCellValue()).isEqualTo("用户未配置代理");
}
}
@Test
void 没有被过滤行时不建这张表() throws Exception {
File output = tempDir.resolve("empty.xlsx").toFile();
assembly.writeWorkbookStreaming(
output,
consumer -> consumer.accept(List.of(detailRow("B000000001"))),
List.of(summary()),
consumer -> { },
sourceOf(List.of(), 0L));
try (Workbook workbook = WorkbookFactory.create(output)) {
assertThat(workbook.getNumberOfSheets()).isEqualTo(2);
assertThat(workbook.getSheet("被过滤明细")).isNull();
}
}
@Test
void 超出上限未逐条记录时如实写出缺口() throws Exception {
File output = tempDir.resolve("gap.xlsx").toFile();
assembly.writeWorkbookStreaming(
output,
consumer -> consumer.accept(List.of(detailRow("B000000001"))),
List.of(summary()),
consumer -> { },
sourceOf(List.of(filtered("B000000009",
CollectDataFilteredRowEntity.REASON_INVALID_BRAND, null)), 317L));
try (Workbook workbook = WorkbookFactory.create(output)) {
Sheet sheet = workbook.getSheet("被过滤明细");
assertThat(sheet).isNotNull();
String note = sheet.getRow(2).getCell(4).getStringCellValue();
assertThat(note).contains("317");
// 缺口必须说清"行数已计入各道闸的计数",否则读的人会以为又丢了 317 行
assertThat(note).contains("已计入");
}
}
@Test
void 数据源抛错时降级掉这个sheet而不是打掉整份交付() throws Exception {
File output = tempDir.resolve("boom.xlsx").toFile();
CollectDataExcelAssemblyService.FilteredRowSource broken = new CollectDataExcelAssemblyService.FilteredRowSource() {
@Override
public void forEachBatch(Consumer<List<CollectDataFilteredRowEntity>> batchConsumer) {
throw new IllegalStateException("明细表读不动");
}
};
long returned = assembly.writeWorkbookStreaming(
output,
consumer -> consumer.accept(List.of(detailRow("B000000001"))),
List.of(summary()),
consumer -> { },
broken);
assertThat(returned).isEqualTo(1L);
try (Workbook workbook = WorkbookFactory.create(output)) {
// 追溯 sheet 缺了就缺了,前两表照常交付——它是审计件,不是交付物。
// 若这里改成外抛,一次审计读失败就会把用户已采回的数据一起打掉。
assertThat(workbook.getNumberOfSheets()).isEqualTo(2);
assertThat(workbook.getSheetName(0)).isEqualTo("采集数据结果");
assertThat(workbook.getSheetName(1)).isEqualTo("结果文件");
}
}
}
@@ -43,9 +43,15 @@ import static org.mockito.Mockito.when;
* 上游的采集代理取不到 IP(余额不足)与 503 —— 代码在这个位置只观测到「品牌检测没返回结果」,
* 观测不到原因。写死一个猜测的原因会把排查方向从代理/上游带到自身服务上。
*
* <p>锁定契约:文案只陈述可观测事实(缺失行数、不计入结果条数、未判无效、可重跑),
* 不指认代码没有依据的具体故障原因;真实原因留在 BrandCheckClient 的 warn 日志里
* (brand-check 的 lastErr 字段),由运维按日志排查。
* <p>锁定契约(2026-10-28 修订):文案只陈述可观测事实(缺失行数、不计入结果条数、未判无效),
* **并且只在这层真的观测得到原因时才指名原因**——品牌检测的失败原因现已由
* {@code BrandBatchOutcome#reason} 落进 result_json(brandQueryFailedReason),所以
* 「用户未配置代理」这类有据可查的原因必须写出来并指向「密钥设置→代理设置」。
*
* <p>原契约(10-02)之所以禁止指名原因,是因为当时这一层确实观测不到;不是"永远不许说"。
* 反过来,原因缺失时仍然不猜:不写「超时」「限流」「上游不可用」,只说系统会自动重试。
* 旧文案那句「可稍后单独重跑这些 ASIN」是必须拆掉的误导——未配置代理时重跑一万次也不会变好
* (现网 task 32573:补测队列两轮 21/21、28/28 全败、净增 0 行)。
*/
@ExtendWith(MockitoExtension.class)
class CollectDataPartialResultNoticeTest {
@@ -77,13 +83,19 @@ class CollectDataPartialResultNoticeTest {
}
private FileTaskEntity taskWithQueryFailedRows(int queryFailedRows) {
return taskWithQueryFailedRows(queryFailedRows, null);
}
private FileTaskEntity taskWithQueryFailedRows(int queryFailedRows, String reason) {
FileTaskEntity task = new FileTaskEntity();
task.setId(TASK_ID);
task.setModuleType("COLLECT_DATA");
task.setStatus("RUNNING");
task.setTaskNo("CD-29991");
task.setUserId(7L);
task.setResultJson("{\"brandQueryFailedCount\":" + queryFailedRows + "}");
task.setResultJson("{\"brandQueryFailedCount\":" + queryFailedRows
+ (reason == null ? "" : ",\"brandQueryFailedReason\":\"" + reason + "\"")
+ "}");
return task;
}
@@ -103,9 +115,13 @@ class CollectDataPartialResultNoticeTest {
}
private String stubHappyPathAndCaptureDetail() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7));
return stubHappyPathAndCaptureDetail(null);
}
private String stubHappyPathAndCaptureDetail(String reason) {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7, reason));
when(fileResultMapper.selectById(5001L)).thenReturn(result());
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L);
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(12L);
// 空明细守卫按明细表行数校验:happy path 下表内行数与写出行数一致。
when(taskResultItemMapper.selectCount(any())).thenReturn(12L);
when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx");
@@ -136,14 +152,35 @@ class CollectDataPartialResultNoticeTest {
assertThat(detail).contains("不计入结果条数");
assertThat(detail).contains("未返回结果");
assertThat(detail).contains("重跑");
// 原因缺失时不再承诺「你重跑就好」,只说系统会自动重试
assertThat(detail).contains("自动重试");
}
@Test
void 原因缺失时不猜故障方() {
// classify(空串) 会算成「上游」,那是没观测到的事——这一层不能替它背书
String detail = stubHappyPathAndCaptureDetail();
assertThat(detail).doesNotContain("上游");
assertThat(detail).doesNotContain("未配置代理");
}
@Test
void 未配置代理必须指名原因并指向可行动入口() {
// 旧文案对配置类原因是谎:补测队列两轮 21/21、28/28 全败、净增 0 行,
// 让用户「稍后单独重跑」只会让他白等
String detail = stubHappyPathAndCaptureDetail("用户未配置代理");
assertThat(detail).contains("密钥设置");
assertThat(detail).contains("代理设置");
assertThat(detail).doesNotContain("可稍后单独重跑这些 ASIN");
}
@Test
void 任务已被判失败时不推部分结果通知() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(7));
when(fileResultMapper.selectById(5001L)).thenReturn(result());
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L);
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(12L);
// 空明细守卫按明细表行数校验:happy path 下表内行数与写出行数一致。
when(taskResultItemMapper.selectCount(any())).thenReturn(12L);
when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx");
@@ -160,7 +197,7 @@ class CollectDataPartialResultNoticeTest {
void 明细缺行时不得上传文件并把任务标成功() {
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(taskWithQueryFailedRows(0));
when(fileResultMapper.selectById(5001L)).thenReturn(result());
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(12L);
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(12L);
when(taskResultItemMapper.selectCount(any())).thenReturn(20L);
assertThatThrownBy(() -> service.processResultFileJob(job()))
@@ -182,7 +219,7 @@ class CollectDataPartialResultNoticeTest {
task.setResultJson("{\"finalRowCount\":93}");
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
when(fileResultMapper.selectById(5001L)).thenReturn(result());
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(0L);
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(0L);
when(taskResultItemMapper.selectCount(any())).thenReturn(0L);
assertThatThrownBy(() -> service.processResultFileJob(job()))
@@ -199,7 +236,7 @@ class CollectDataPartialResultNoticeTest {
task.setResultJson("{}");
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
when(fileResultMapper.selectById(5001L)).thenReturn(result());
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any())).thenReturn(0L);
when(excelAssemblyService.writeWorkbookStreaming(any(), any(), any(), any(), any())).thenReturn(0L);
when(taskResultItemMapper.selectCount(any())).thenReturn(0L);
when(ossStorageService.uploadResultFile(any(), anyString())).thenReturn("collect-data/29991.xlsx");
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
@@ -0,0 +1,114 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.publish.service.PublishWorkbookService;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.nio.file.Path;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
/**
* 采集结果文件被用户**回传**给别的模块时的兼容性钉子(V179 新增尾 sheet 的主要外溢风险)。
*
* <p>结果 Excel 不是终点物:用户会把它再上传到上架/跟价。那两个模块用的是
* {@code ExcelStreamReader.readAllSheets}(读全部 sheet),所以多一张 sheet 就可能多算一批行。
* 本用例钉住「加与不加那张 sheet,回传行为逐字一致」。</p>
*
* <p>跟价那一侧({@code PriceTrackTaskService.parseWorkbookAsinRows})的机制是
* {@code resolveCountryCode(sheetName)} 认不出「被过滤明细」即整表跳过,且首表兜底只在
* sheetCount==1 时生效——那条路径上的方法是私有的,本用例覆盖不到,按真机验证项处理。</p>
*/
class CollectDataResultFileReuploadCompatTest {
@TempDir
Path tempDir;
private final CollectDataExcelAssemblyService assembly = new CollectDataExcelAssemblyService();
private final PublishWorkbookService publishParser = new PublishWorkbookService();
private static CollectDataResultRowVo detailRow() {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setBrand("nike");
row.setAsin("B000000001");
row.setKeyword("phone case");
row.setPrice("19.90");
row.setSellerName("seller-A");
row.setDeliveryMethod("FBA");
return row;
}
private static CollectDataSummaryRowDto summary() {
CollectDataSummaryRowDto item = new CollectDataSummaryRowDto();
item.setKeyword("phone case");
item.setFba(1);
return item;
}
private File assemble(boolean withFilteredSheet) throws Exception {
File output = tempDir.resolve((withFilteredSheet ? "three" : "two") + ".xlsx").toFile();
CollectDataExcelAssemblyService.FilteredRowSource source = withFilteredSheet
? filteredSource(entity())
: null;
assembly.writeWorkbookStreaming(output,
consumer -> consumer.accept(List.of(detailRow())),
List.of(summary()),
consumer -> { },
source);
return output;
}
private static CollectDataFilteredRowEntity entity() {
CollectDataFilteredRowEntity entity = new CollectDataFilteredRowEntity();
entity.setTaskId(1L);
entity.setChunkIndex(1);
entity.setAsin("B000000002");
entity.setKeyword("phone case");
entity.setBrand("somebrand");
entity.setReasonCode(CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED);
entity.setReasonDetail("用户未配置代理");
return entity;
}
/** {@link CollectDataExcelAssemblyService.FilteredRowSource} 有两个方法,不是函数式接口,只能匿名类。 */
private static CollectDataExcelAssemblyService.FilteredRowSource filteredSource(
CollectDataFilteredRowEntity entity) {
return new CollectDataExcelAssemblyService.FilteredRowSource() {
@Override
public void forEachBatch(java.util.function.Consumer<List<CollectDataFilteredRowEntity>> consumer) {
consumer.accept(List.of(entity));
}
@Override
public long notRecordedCount() {
return 0L;
}
};
}
/** 回传解析的落点:拒绝就说清第几列不匹配,接受就说 "accepted"。 */
private String outcomeOf(File file) {
try {
PublishWorkbookService.ParsedWorkbook parsed = publishParser.parse(file);
return "accepted rows=" + (parsed == null ? "-" : parsed.rows().size());
} catch (BusinessException ex) {
return "rejected: " + ex.getMessage();
}
}
@Test
void 多出被过滤sheet后上架解析器的回传行为逐字一致() throws Exception {
String twoSheets = outcomeOf(assemble(false));
String threeSheets = outcomeOf(assemble(true));
assertThat(threeSheets).isEqualTo(twoSheets);
// 并且必须是在**第一张表**就被表头校验挡下——它挡在前面,才轮不到新 sheet 出问题
assertThat(twoSheets).startsWith("rejected: ").contains("表头不匹配").contains("第 1 列");
}
}
@@ -142,7 +142,7 @@ class CollectDataServiceTxBoundaryTest {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin("B0COLLECT1");
lenient().when(collectDataBatchQuery.filter(any(), any())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0));
new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0, List.of()));
lenient().when(brandBatchFilter.filter(any(), any())).thenReturn(
new CollectDataBrandBatchFilter.BrandBatchOutcome(List.of(), List.of(), List.of(row), ""));
lenient().when(resultItemBatchWriter.upsertAccepted(anyLong(), anyLong(), anyString(), anyInt(), any(), anyString()))
@@ -335,7 +335,7 @@ class CollectDataSubmitResultSnapshotTest {
List<CollectDataResultRowVo> kept = new ArrayList<>(rows);
f.events.add("outsideLock filter rows=" + kept.size() + " owner=" + invocation.getArgument(1)
+ " dedupe=0 invalid=0");
return new CollectDataBatchQuery.FilterResult(kept, 0, 0);
return new CollectDataBatchQuery.FilterResult(kept, 0, 0, List.of());
});
when(brandFilter.filter(anyList(), any())).thenAnswer(invocation -> {
@SuppressWarnings("unchecked")
@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.collectdata.util;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService;
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
@@ -218,6 +219,38 @@ class CollectDataBatchQueryTest {
verify(dedupeService).findExistingComparableValuesForOwner(List.of("B000000001"), OWNER_USER_ID);
}
/**
* 被砍的行必须连同「死在哪一道闸」一起交出去(V179)。
* 计数与行列表出自同一次循环——两者一旦对不上,追溯 sheet 就是在说谎。
*/
@Test
void 被砍行连同各自那道闸一起交出且与计数自洽() {
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any()))
.thenReturn(Set.of("B000000001", "B000000004"));
when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("Zara")));
CollectDataBatchQuery.FilterResult result = query.filter(List.of(
row("B000000001", "Nike"),
row("B000000002", "Zara"),
row("B000000003", "H&M"),
row("B000000004", "Zara")
), OWNER_USER_ID);
assertEquals(2, result.dedupeFilteredCount(), "ASIN 命中去重库的两行");
assertEquals(1, result.invalidFilteredCount(), "品牌命中无效品牌库的一行");
assertEquals(result.dedupeFilteredCount() + result.invalidFilteredCount(),
result.filteredRows().size(), "被砍行数必须等于计数之和,不得漂移");
// B000000004 的品牌也在黑名单里,但去重先判——原因必须落在实际拦下它的那一道闸上
assertEquals(CollectDataFilteredRowEntity.REASON_DEDUPE,
result.filteredRows().get(0).reasonCode());
assertEquals(CollectDataFilteredRowEntity.REASON_INVALID_BRAND,
result.filteredRows().get(1).reasonCode());
assertEquals(CollectDataFilteredRowEntity.REASON_DEDUPE,
result.filteredRows().get(2).reasonCode());
assertEquals(List.of("B000000003"),
result.kept().stream().map(CollectDataResultRowVo::getAsin).toList(), "保留行不受新字段影响");
}
private static CollectDataResultRowVo row(String asin, String brand) {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin(asin);
@@ -0,0 +1,183 @@
package com.nanri.aiimage.modules.collectdata.util;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataFilteredRowMapper;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataFilteredRowEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import java.util.Collection;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyCollection;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 被过滤行存取器(V179)的契约钉子。
*
* <p>它守的是两件相反的事,弄反任何一件都会把系统推回"数对不上又说不清":</p>
* <ul>
* <li><b>写</b>失败只跳批不外抛——这份明细是审计件,为它打掉用户已采回的数据交付不值;</li>
* <li><b>删</b>失败必须外抛——删一半会留下永久孤儿行,比不删更糟。</li>
* </ul>
*/
@ExtendWith(MockitoExtension.class)
@MockitoSettings(strictness = Strictness.LENIENT)
class CollectDataFilteredRowStoreTest {
@Mock
private CollectDataFilteredRowMapper mapper;
private CollectDataFilteredRowStore store(int batchSize) {
return new CollectDataFilteredRowStore(mapper, batchSize);
}
private static CollectDataResultRowVo row(String asin, String brand, String keyword) {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin(asin);
row.setBrand(brand);
row.setKeyword(keyword);
return row;
}
@SuppressWarnings("unchecked")
private List<CollectDataFilteredRowEntity> captureInserted(int expectedCalls) {
ArgumentCaptor<List<CollectDataFilteredRowEntity>> captor = ArgumentCaptor.forClass(List.class);
verify(mapper, times(expectedCalls)).insertBatch(captor.capture());
return captor.getAllValues().stream().flatMap(List::stream).toList();
}
@Test
void 三道闸的行各归各的reason_code() {
when(mapper.insertBatch(any())).thenAnswer(inv -> ((Collection<?>) inv.getArgument(0)).size());
List<CollectDataResultRowVo> rejected = List.of(row("B000000005", "Brand: acme", "kw"));
List<CollectDataResultRowVo> queryFailed = List.of(row("B000000006", "beta", "kw"));
int written = store(200).writeBatch(1L, 3,
List.of(new CollectDataBatchQuery.FilteredRow(row("B000000001", "x", "kw"),
CollectDataFilteredRowEntity.REASON_DEDUPE),
new CollectDataBatchQuery.FilteredRow(row("B000000002", "y", "kw"),
CollectDataFilteredRowEntity.REASON_INVALID_BRAND)),
rejected, queryFailed, "用户未配置代理", 100);
assertThat(written).isEqualTo(4);
List<CollectDataFilteredRowEntity> rows = captureInserted(1);
assertThat(rows).extracting(CollectDataFilteredRowEntity::getReasonCode).containsExactly(
CollectDataFilteredRowEntity.REASON_DEDUPE,
CollectDataFilteredRowEntity.REASON_INVALID_BRAND,
CollectDataFilteredRowEntity.REASON_BRAND_REJECTED,
CollectDataFilteredRowEntity.REASON_BRAND_QUERY_FAILED);
assertThat(rows).allSatisfy(entity ->
// 未回流就是 resolvedAt 为 null:本表只有这一个状态,没有第二套 STATUS_ 词表
assertThat(entity.getResolvedAt()).isNull());
// 品牌必须过 BrandLabel.strip:出件是用户的下一份上传物,脏值会顺着下载-再上传回流进链路
assertThat(rows).extracting(CollectDataFilteredRowEntity::getBrand)
.containsExactly("x", "y", "acme", "beta");
}
@Test
void 只有品牌查询失败那类带可行动原因() {
when(mapper.insertBatch(any())).thenAnswer(inv -> ((Collection<?>) inv.getArgument(0)).size());
store(200).writeBatch(1L, 1,
List.of(new CollectDataBatchQuery.FilteredRow(row("B000000001", "x", "kw"),
CollectDataFilteredRowEntity.REASON_INVALID_BRAND)),
List.of(row("B000000002", "y", "kw")), List.of(row("B000000003", "z", "kw")),
"用户未配置代理", 100);
List<CollectDataFilteredRowEntity> rows = captureInserted(1);
// 黑名单命中/判侵权不需要额外指引,把品牌侧原因错挂到它们身上反而误导
assertThat(rows.get(0).getReasonDetail()).isNull();
assertThat(rows.get(1).getReasonDetail()).isNull();
assertThat(rows.get(2).getReasonDetail()).isEqualTo("用户未配置代理");
}
@Test
void 空ASIN的行照记不误() {
// 空品牌被判 rejected 的行在无效品牌表里根本不留痕(toEntity 对空品牌返 null),
// 装配期也无从反查——这一格正是 V179 头注释点名的「漏标」来源,这里不能再丢。
when(mapper.insertBatch(any())).thenAnswer(inv -> ((Collection<?>) inv.getArgument(0)).size());
int written = store(200).writeBatch(1L, 1, List.of(),
List.of(row("", "", "kw")), List.of(), null, 100);
assertThat(written).isEqualTo(1);
List<CollectDataFilteredRowEntity> rows = captureInserted(1);
assertThat(rows).hasSize(1);
assertThat(rows.get(0).getAsin()).isEmpty();
}
@Test
void 预算耗尽时不再记行且返回值如实反映写入量() {
when(mapper.insertBatch(any())).thenAnswer(inv -> ((Collection<?>) inv.getArgument(0)).size());
int written = store(200).writeBatch(1L, 1,
List.of(new CollectDataBatchQuery.FilteredRow(row("B000000001", "x", "kw"),
CollectDataFilteredRowEntity.REASON_DEDUPE),
new CollectDataBatchQuery.FilteredRow(row("B000000002", "y", "kw"),
CollectDataFilteredRowEntity.REASON_DEDUPE)),
List.of(), List.of(), null, 1);
assertThat(written).isEqualTo(1);
assertThat(captureInserted(1)).hasSize(1);
assertThat(store(200).writeBatch(1L, 2,
List.of(new CollectDataBatchQuery.FilteredRow(row("B000000003", "z", "kw"),
CollectDataFilteredRowEntity.REASON_DEDUPE)),
List.of(), List.of(), null, 0)).isZero();
}
@Test
void 写失败只跳批不外抛() {
// 审计件不值得打掉用户已采回的数据交付
when(mapper.insertBatch(any())).thenThrow(new RuntimeException("db down"));
List<CollectDataResultRowVo> rejected = List.of(
row("B000000001", "a", "kw"), row("B000000002", "b", "kw"), row("B000000003", "c", "kw"));
assertThat(store(1).writeBatch(1L, 1, List.of(), rejected, List.of(), null, 100)).isZero();
// 三批各失败各跳过,不中断后续批次
verify(mapper, times(3)).insertBatch(any());
}
@Test
void 删失败必须外抛不留孤儿行() {
// 与写侧相反:删一半比不删更糟,必须让调用方的事务回滚
when(mapper.delete(any(com.baomidou.mybatisplus.core.conditions.Wrapper.class)))
.thenThrow(new RuntimeException("db down"));
assertThatThrownBy(() -> store(200).deleteByTask(7L))
.isInstanceOf(RuntimeException.class)
.hasMessage("db down");
}
@Test
void 回流标记归一化ASIN并跳过空值() {
when(mapper.markResolved(anyLong(), anyCollection(), anyString())).thenReturn(2);
assertThat(store(200).markResolved(7L, List.of(" b000000001 ", "", "B000000002"))).isEqualTo(2);
ArgumentCaptor<Collection<String>> captor = ArgumentCaptor.forClass(Collection.class);
verify(mapper).markResolved(anyLong(), captor.capture(), anyString());
assertThat(captor.getValue()).containsExactly("B000000001", "B000000002");
}
@Test
void 空输入不发起任何调用() {
assertThat(store(200).writeBatch(null, 1, List.of(), List.of(), List.of(), null, 100)).isZero();
assertThat(store(200).markResolved(1L, List.of())).isZero();
assertThat(store(200).deleteByTask(null)).isZero();
verify(mapper, never()).insertBatch(any());
verify(mapper, never()).delete(any(com.baomidou.mybatisplus.core.conditions.Wrapper.class));
}
}
@@ -93,6 +93,9 @@ class CollectDataSelfConsistencyTest {
"resultId", "taskId", "taskNo", "sourceFilename", "resultFilename", "downloadUrl",
"taskStatus", "success", "error", "rowCount", "dedupeFilteredCount",
"invalidFilteredCount", "brandRejectedCount", "finalRowCount",
// 品牌检测「查不了」的行数与原因:过去这两个字段连 VO 都没出,
// 现网 task 32573 有 173 行这么丢掉,界面上却一格都没有
"brandQueryFailedCount", "brandQueryFailedReason",
"taskType", "progressPercent", "totalRows", "receivedRows", "processedRows",
"collectStage", "currentKeyword", "searchCurrentPage", "searchTotalPages",
"detailProcessedAsins", "detailTotalAsins", "filters",
@@ -139,7 +139,7 @@ class DuplicateSubmissionContractTest {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin("B0DUP1234");
lenient().when(collectDataBatchQuery.filter(any(), any())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0));
new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0, List.of()));
lenient().when(brandBatchFilter.filter(any(), any())).thenReturn(
new CollectDataBrandBatchFilter.BrandBatchOutcome(List.of(), List.of(), List.of(row), ""));
lenient().when(resultItemBatchWriter.upsertAccepted(anyLong(), anyLong(), anyString(), anyInt(), any(), anyString()))
@@ -179,8 +179,10 @@ class SuccessTimingContractTest {
void noFileGeneratedNoSuccess() {
// 方法名同步:实现已由 writeWorkbookSegmented 改为 writeWorkbookStreaming(流式分页读,
// 避免几十万行结果集整体驻留堆),测试原先仍在 mock 旧方法名,导致"没抛异常"的假失败。
// V179 起调用点走 5 参重载(第 5 参是「被过滤明细」sheet 的数据源),桩必须跟着换形,
// 否则严格存根会先报 PotentialStubbingProblem,把"生成失败"这条断言本身吞掉。
doThrow(new IllegalStateException("工作簿生成失败"))
.when(excelAssemblyService).writeWorkbookStreaming(any(), any(), any(), any());
.when(excelAssemblyService).writeWorkbookStreaming(any(), any(), any(), any(), any());
assertThrows(IllegalStateException.class, () -> service.processResultFileJob(job()));
@@ -16,7 +16,7 @@ chunk.insert scope=sub-1 idx=1/3 payload=rustfs:chunk-1 payloadHash=8640b8fb328e
scope.selectOne -> none
chunk.count -> 0
scope.insert chunkTotal=3 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"}
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=f635be79fec3
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=9f73e6bcec11
task.load id=1001 status=RUNNING module=COLLECT_DATA
thrown: none
vo: taskId=1001 resultId=2001 chunk=1/3 done=false receivedRows=2 currentChunkRows=2 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=2 taskStatus=RUNNING
@@ -41,7 +41,7 @@ scope.selectOne -> none
chunk.count -> 0
scope.insert chunkTotal=3 received=0 completed=1 lastError=- stateJson={"phase":"RECEIVED"}
result.update id=2001 success=0 rowCount=1 resultFilename=collect-data-1001-result.xlsx resultFileUrl=- fileSize=0 errorMessage=-
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=44ebae7009e0
task.load id=1001 status=RUNNING module=COLLECT_DATA
thrown: none
vo: taskId=1001 resultId=2001 chunk=3/3 done=true receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING
@@ -82,7 +82,7 @@ scope.selectOne -> none
chunk.count -> 0
scope.insert chunkTotal=1 received=0 completed=1 lastError=- stateJson={"phase":"RECEIVED"}
result.update id=2001 success=- rowCount=1 resultFilename=collect-data-1001-result.xlsx resultFileUrl=- fileSize=0 errorMessage=-
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=44ebae7009e0
task.load id=1001 status=RUNNING module=COLLECT_DATA
thrown: none
vo: taskId=1001 resultId=2001 chunk=1/1 done=true receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING
@@ -107,10 +107,10 @@ scope.selectOne -> none
chunk.count -> 2
scope.insert chunkTotal=3 received=2 completed=0 lastError=Python 采集异常 stateJson={"phase":"RECEIVED"}
result.update id=2001 success=0 rowCount=1 resultFilename=- resultFileUrl=- fileSize=- errorMessage=Python 采集异常
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=333 resultJson.hash=2f13867b410c
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=388 resultJson.hash=44ebae7009e0
chunk.count -> 2
result.update id=2001 success=0 rowCount=1 resultFilename=collect-data-1001-result.xlsx resultFileUrl=- fileSize=0 errorMessage=Python 采集异常
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=333 resultJson.hash=2f13867b410c
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=388 resultJson.hash=44ebae7009e0
task.load id=1001 status=FAILED module=COLLECT_DATA
thrown: none
vo: taskId=1001 resultId=2001 chunk=2/3 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=FAILED
@@ -135,7 +135,7 @@ scope.selectOne -> none
chunk.count -> 0
scope.insert chunkTotal=2 received=0 completed=0 lastError=Python 采集异常 stateJson={"phase":"RECEIVED"}
result.update id=2001 success=0 rowCount=1 resultFilename=- resultFileUrl=- fileSize=- errorMessage=Python 采集异常
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=333 resultJson.hash=2f13867b410c
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=388 resultJson.hash=44ebae7009e0
chunk.count -> 0
task.load id=1001 status=FAILED module=COLLECT_DATA
thrown: none
@@ -161,7 +161,7 @@ chunk.insert scope=sub-1 idx=1/2 payload=rustfs:chunk-1 payloadHash=a55ecd3678de
scope.selectOne -> none
chunk.count -> 0
scope.insert chunkTotal=2 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"}
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=44ebae7009e0
task.load id=1001 status=RUNNING module=COLLECT_DATA
thrown: none
vo: taskId=1001 resultId=2001 chunk=1/2 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING
@@ -182,7 +182,7 @@ chunk.insert scope=sub-1 idx=1/1 payload=rustfs:chunk-1 payloadHash=e0365b1c5568
scope.selectOne -> none
chunk.count -> 0
scope.insert chunkTotal=1 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"}
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=53ca52c40ab0
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=388 resultJson.hash=2b4337f9c7d3
task.load id=1001 status=RUNNING module=COLLECT_DATA
thrown: none
vo: taskId=1001 resultId=2001 chunk=1/1 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=0 taskStatus=RUNNING