fix(tasks): 批量入库与 folder 去重逐条容错,发布文案不说谎
B49/B50 收尾三处当时被标"非 ★"跳过的同类缺陷。判据同一条:一颗坏项不许让 其余项一起不干活,也不许把"根本没拿到数据"说成"数据正常"。 1) shopkey/service/QueryAsinService.java(B49 #6,flushInsert:843-844) 复验为真:整张表共用一条读行循环,某一批 insertBatch 抛出即冒到 runImportTask 的 catch,后面的批次一个都跑不到、内存里的余行一起丢。 审计里"报数虚高"那条已被复验否掉(原状下坏批计数确实不外发),本笔不按 那个理由改;但逐批容错之后坏批计数会随后续进度外发,所以必须按"没落库" 回退:insertedCount 扣回、skippedCount 记上、坏批点名进日志与 errorMessage。 收尾新增 failIfBatchesLost():有坏批就让导入落"失败"态并报出缺口行数—— admin 侧只在 failed 分支显示 errorMessage(QueryAsinPage.vue:260-271), 留在 success 上等于把没入库的行说成导入完成。重传整表是安全的:已入库行 会被 rowSignature 认成重复跳过。 processImportFile 由 private 收到包级作测试接缝(形状同 tryAcquireFileSlot)。 该 importer 此前零夹具,新建 3 例(真 xlsx + 真 ExcelStreamReader)。 2) dedupe/service/DedupeRunService.java(B49 #4 的 folder 分支) 只在 folder 模式成立:逐文件清洗结果只落本地临时目录,唯一交付物是那个 ZIP。两颗坏项会带走整批——① 收尾的 aborted 判定排在打包之前,一个 worker 线程级异常就让打包整步跳过;② 打包循环里一个条目读不到/条目名相撞就整包 作废。现改为 folder 分支先打包再判 aborted,打包逐条目容错并点名跳过; 一个都没写进去时拒绝交空包(那才是"根本没拿到数据")。aborted 仍照旧把 终态落 FAILED,不拿部分交付冒充全量成功;缺口写在交付条目与日志上, 不写 file_task.error_message。非 folder 分支一个字节没动。 DedupeArchiveEntry 与打包方法收到包级作测试接缝。新建 3 例,含一支用 同线程执行器让第 2 个 worker 当场线程级异常的真链路用例。 3) publish/service/PublishTaskService.java(B49 #7/B50 #4,607-609) 按上一轮复验结论只改文案,不动状态语义:逐文件 status/VO 本来就交付了。 会说谎的地方是 publishResultFile(..., preserveFailure=false) 把结果行 error_message 洗成 null,"N 个文件没进这张表"在文案层等于全部成功。 缺口写在 biz_file_result.error_message(本仓既有"部分交付"载体: ShopMatch:825/PriceTrack:905/ProductRisk:827 都按它判 partialFailure; publish 侧只有 toResultVo:1216 读它,纯展示,fileReady 由 success+url 推导 不受影响)。SUCCESS 的 file_task.error_message 继续留空并就地写明理由: TaskRetryEligibility.forceMajeureCategory 与 SimilarAsinStaleReopenSupport .isStaleFailure 都按那段文字做字面量分类,写进去会喂错 stale/重开/续跑判据。 文案两个方向都不说谎:说清"只含成功处理的 X 个、另有 N 个未纳入(点名)", 并明写"不是表格格式问题、可只重新提交这些文件",不把可恢复失败推给表格。 新建 2 例(含反向钉:全成功时不得凭空造缺口文案)。 实测:mvn -o test-compile rc=0;新增 8 例改前 6 红(QueryAsin 2 红=原始 RuntimeException 直接冒出、批次被提前收走;Dedupe 3 红="Wanted but not invoked: fileResultMapper.insert"+整包作废;Publish 1 红=errorMessage 恒 null),改后 8/8 绿;相邻既有 9 支类 97 例全绿(PublishTaskServiceTest 33、 DedupeRunProgressResumeTest 14、PublishFileLevelStatusTest 8、 PublishResultFileJobHandlerTest 8、DedupeRunServiceCleanRulesTest 9、 DedupeRunServiceFileSlotTest 2、QueryAsinResolveServiceTest 13 等)。 CatchSilenceRatchetTest 按预期判红(本笔新增 2 处 catch,均在被禁改的共享台账 里,合并时由主线程统一重钉):实测 total=1028(基线 1026)、 notRethrow(incl logOnly)=579(基线 577)、logOnly=181(不变)、 rethrow=393(不变)、empty=56(不变)。
This commit is contained in:
+66
-12
@@ -367,16 +367,21 @@ public class DedupeRunService {
|
||||
}
|
||||
|
||||
try {
|
||||
if (aborted) {
|
||||
// 有文件没能正常处理完就序列化/标成功,会出现「任务成功但结果缺文件」
|
||||
throw new IllegalStateException("部分文件处理线程异常结束,结果不完整");
|
||||
}
|
||||
// folder 模式:先打包,再判 aborted。逐文件清洗结果只落在本地临时目录,唯一交付物就是这个
|
||||
// ZIP——原先一个 worker 线程级异常就让整步打包跳过,已清洗完的 N-1 个文件一起不交付
|
||||
// (B49 #4 的 folder 分支)。打包本身逐条目容错,见
|
||||
// {@link #packageFolderDedupeResultsAsZip(String, List)};aborted 仍照旧把终态落到
|
||||
// FAILED,绝不拿"部分交付"冒充全量成功。非 folder 模式一个字节不变。
|
||||
if (folderMode && !archiveEntries.isEmpty()) {
|
||||
DedupeResultItemVo zipItem = buildFolderZipResult(request, archiveEntries, task);
|
||||
synchronized (progress) {
|
||||
outcomeItems.add(0, zipItem);
|
||||
}
|
||||
}
|
||||
if (aborted) {
|
||||
// 有文件没能正常处理完就序列化/标成功,会出现「任务成功但结果缺文件」
|
||||
throw new IllegalStateException("部分文件处理线程异常结束,结果不完整");
|
||||
}
|
||||
DedupeRunVo result = buildRunVo(request, outcomeItems, progress);
|
||||
finishTaskSuccess(task, result);
|
||||
synchronized (progress) {
|
||||
@@ -620,7 +625,9 @@ public class DedupeRunService {
|
||||
private DedupeResultItemVo buildFolderZipResult(DedupeRunRequest request,
|
||||
Map<String, DedupeArchiveEntry> archiveEntries,
|
||||
FileTaskEntity task) {
|
||||
File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), new ArrayList<>(archiveEntries.values()));
|
||||
FolderPackage packaged = packageFolderDedupeResultsAsZip(
|
||||
request.getArchiveName(), new ArrayList<>(archiveEntries.values()));
|
||||
File zipFile = packaged.file();
|
||||
OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(zipFile, "DEDUPE");
|
||||
DedupeResultItemVo item = new DedupeResultItemVo();
|
||||
item.setSourceFilename(request.getArchiveName());
|
||||
@@ -628,6 +635,15 @@ public class DedupeRunService {
|
||||
item.setSuccess(true);
|
||||
item.setDownloadUrl(uploadedResult.downloadUrl());
|
||||
item.setResultFileSize(zipFile.length());
|
||||
if (!packaged.skippedEntries().isEmpty()) {
|
||||
// 缺口只写在这一条交付项上(VO + task.result_json),不写 file_task.error_message:
|
||||
// 那段文字会被按字面量分类的判据读取(TaskRetryEligibility/
|
||||
// SimilarAsinStaleReopenSupport),SUCCESS 行留空才是安全口径。
|
||||
item.setError("压缩包已交付 " + packaged.packedCount() + " 个文件,另有 "
|
||||
+ packaged.skippedEntries().size() + " 个文件的结果没能写进压缩包("
|
||||
+ String.join("、", packaged.skippedEntries())
|
||||
+ ");这些文件本次确实没交付,是临时结果文件/写入的问题,不是表格内容问题,可只重新提交这几个文件");
|
||||
}
|
||||
|
||||
FileResultEntity resultEntity = new FileResultEntity();
|
||||
resultEntity.setTaskId(task.getId());
|
||||
@@ -995,20 +1011,57 @@ public class DedupeRunService {
|
||||
return value == null ? "" : value;
|
||||
}
|
||||
|
||||
private File packageFolderDedupeResultsAsZip(String archiveName, List<DedupeArchiveEntry> archiveEntries) {
|
||||
/** 打包结果:交付的压缩包,以及逐条目容错后没能写进去的那些文件名。 */
|
||||
record FolderPackage(File file, int packedCount, List<String> skippedEntries) {
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件夹模式逐条目容错打包(B49 #4 的 folder 分支)。
|
||||
*
|
||||
* <p>原形状:一个条目读不到(结果临时文件被清掉/磁盘错)或条目名相撞,就在 {@code ZipOutputStream}
|
||||
* 里抛出并整包作废——同一批里<b>已清洗完的其余文件一起不交付</b>,用户只能整批重跑。
|
||||
* 现在坏条目自己认赔、点名跳过,其余照常入包;只有<b>一个都没写进去</b>时才整批判失败
|
||||
* (那才是"根本没拿到数据",不许交一个空包冒充成功)。压缩包本体建不起来
|
||||
* (目录/磁盘级错误)同样是整批级别,照旧抛出。
|
||||
*
|
||||
* <p>本方法不私有:它是 folder 打包唯一的可测接缝(形状同 {@link #tryAcquireFileSlot()})。
|
||||
*/
|
||||
FolderPackage packageFolderDedupeResultsAsZip(String archiveName, List<DedupeArchiveEntry> archiveEntries) {
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
|
||||
File zipFile = buildNamedOutputFile(outputDir, archiveName + ".zip");
|
||||
int packedCount = 0;
|
||||
List<String> skippedEntries = new ArrayList<>();
|
||||
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) {
|
||||
for (DedupeArchiveEntry entry : archiveEntries) {
|
||||
String zipEntryName = buildFolderZipEntry(entry.relativePath(), entry.inputName(), entry.inputName(), false);
|
||||
zos.putNextEntry(new ZipEntry(zipEntryName));
|
||||
Files.copy(entry.outputFile().toPath(), zos);
|
||||
zos.closeEntry();
|
||||
String entryName = entry.inputName();
|
||||
File source = entry.outputFile();
|
||||
// 先验可读再开条目:文件缺失/不可读与条目名相撞都发生在写入之前,
|
||||
// 这类坏项不会在包里留下半截文件(写到一半才抛的极端 IO 错仍可能留短条目)
|
||||
if (source == null || !source.exists() || !source.canRead()) {
|
||||
skippedEntries.add(entryName);
|
||||
log.error("[dedupe] 结果临时文件不可读,跳过该条目并继续其余文件 archiveName={} entry={} path={}",
|
||||
archiveName, entryName, source == null ? "null" : source.getPath());
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
zos.putNextEntry(new ZipEntry(buildFolderZipEntry(entry.relativePath(), entryName, entryName, false)));
|
||||
Files.copy(source.toPath(), zos);
|
||||
zos.closeEntry();
|
||||
packedCount++;
|
||||
} catch (Exception ex) {
|
||||
skippedEntries.add(entryName);
|
||||
log.error("[dedupe] 单个结果文件写入压缩包失败,跳过该条目并继续其余文件 archiveName={} entry={} error",
|
||||
archiveName, entryName, ex);
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException(UserMessages.with("去重结果打包失败", ex));
|
||||
}
|
||||
return zipFile;
|
||||
if (packedCount == 0) {
|
||||
throw new BusinessException("去重结果打包失败:待打包的 " + archiveEntries.size()
|
||||
+ " 个结果文件没有一个写进压缩包(临时结果文件缺失或写入异常,不是表格内容问题),本次没有交付文件夹结果");
|
||||
}
|
||||
return new FolderPackage(zipFile, packedCount, skippedEntries);
|
||||
}
|
||||
|
||||
private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) {
|
||||
@@ -1240,7 +1293,8 @@ public class DedupeRunService {
|
||||
return nanos / 1_000_000L;
|
||||
}
|
||||
|
||||
private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) {
|
||||
/** 不私有:folder 打包接缝 {@link #packageFolderDedupeResultsAsZip(String, List)} 的入参类型。 */
|
||||
record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) {
|
||||
}
|
||||
|
||||
private static final class DedupeReadResult {
|
||||
|
||||
+44
@@ -96,6 +96,8 @@ public class PublishTaskService extends AbstractFileTaskPipeline<PublishRowDto>
|
||||
private static final String STATUS_RUNNING = "RUNNING";
|
||||
private static final String STATUS_SUCCESS = "SUCCESS";
|
||||
private static final String STATUS_FAILED = "FAILED";
|
||||
/** 「部分交付」文案里最多点名几个文件,余下的用总数兜住。 */
|
||||
private static final int PARTIAL_NOTE_NAME_LIMIT = 3;
|
||||
/**
|
||||
* 保留期清理允许删除的终态集合(与 ModuleHistoryCleanupService 的 TERMINAL_STATUSES 一致)。
|
||||
* PENDING/RUNNING 绝不在此列;PUBLISH 实际只会写 SUCCESS/FAILED,取消态是无害的超集。
|
||||
@@ -603,9 +605,24 @@ public class PublishTaskService extends AbstractFileTaskPipeline<PublishRowDto>
|
||||
publishResultFile(result, packaged.file(), packaged.filename(), packaged.contentType(), rowCount, false);
|
||||
|
||||
int failedCount = (int) files.stream().filter(file -> STATUS_FAILED.equals(file.getStatus())).count();
|
||||
int unpackedCount = files.size() - successfulFiles.size();
|
||||
if (unpackedCount > 0) {
|
||||
// 只补文案、不动状态语义:文件确实交付了,success=1 与下载直链都不能改(改了就把
|
||||
// 能用的结果也扣掉)。缺口写在这一行 result 的 error_message 上——本仓把它当
|
||||
// 「部分交付」载体用(ShopMatch:825/PriceTrack:905/ProductRisk:827 都按它判
|
||||
// partialFailure),而 publish 侧只有 toResultVo 读它(纯展示),不喂任何分类判据。
|
||||
result.setErrorMessage(partialDeliveryNote(files, successfulFiles.size(), unpackedCount, failedCount));
|
||||
fileResultMapper.updateById(result);
|
||||
log.warn("[publish] 任务标成功但有文件未纳入结果 taskId={} files={} delivered={} unpacked={} failed={}",
|
||||
task.getId(), files.size(), successfulFiles.size(), unpackedCount, failedCount);
|
||||
}
|
||||
task.setStatus(STATUS_SUCCESS);
|
||||
task.setSuccessFileCount(successfulFiles.size());
|
||||
task.setFailedFileCount(failedCount);
|
||||
// SUCCESS 的 file_task 行必须留空 error_message:TaskRetryEligibility.forceMajeureCategory
|
||||
// 与 SimilarAsinStaleReopenSupport.isStaleFailure 都拿这段文字做字面量分类(stale/重开/
|
||||
// 克隆重跑),写进去会把"其实已跑完一半"的单子喂给错误的分类。所以缺口只落在上面
|
||||
// 结果行的文案与日志里,这里继续清空的语义是有意为之,不是漏判。
|
||||
task.setErrorMessage(null);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
@@ -624,6 +641,33 @@ public class PublishTaskService extends AbstractFileTaskPipeline<PublishRowDto>
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 「部分交付」提示语(B49/B50 口径):不许把「N 个文件没处理」说成整单成功,
|
||||
* 也不许反过来把可恢复的处理失败推给表格——文件级的真因本来就落在
|
||||
* {@code publish_file.error_message}({@code toFileVo} 已逐文件回传)。
|
||||
* 名单只取前几项,余下的用总数兜住,免得文案无限长。
|
||||
*/
|
||||
private String partialDeliveryNote(List<PublishFileEntity> files, int deliveredCount,
|
||||
int unpackedCount, int failedCount) {
|
||||
List<String> names = new ArrayList<>();
|
||||
for (PublishFileEntity file : files) {
|
||||
if (STATUS_SUCCESS.equals(file.getStatus())) {
|
||||
continue;
|
||||
}
|
||||
names.add(firstNonBlank(file.getSourceFilename(), "未命名文件"));
|
||||
if (names.size() >= PARTIAL_NOTE_NAME_LIMIT) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
String nameList = String.join("、", names)
|
||||
+ (unpackedCount > names.size() ? " 等 " + unpackedCount + " 个文件" : "");
|
||||
String gap = failedCount == unpackedCount
|
||||
? "另有 " + unpackedCount + " 个文件处理失败,未纳入结果(" + nameList + ")"
|
||||
: "另有 " + unpackedCount + " 个文件未纳入结果(其中 " + failedCount + " 个处理失败:" + nameList + ")";
|
||||
return "结果已生成,但只含成功处理的 " + deliveredCount + " 个文件;" + gap
|
||||
+ "。这些文件本轮确实没有结果,不是表格格式问题——各自的原因见文件列表,可只重新提交这些文件,不必重跑整单";
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除任务。
|
||||
*
|
||||
|
||||
+66
-4
@@ -451,8 +451,16 @@ public class QueryAsinService {
|
||||
}
|
||||
}
|
||||
|
||||
private void processImportFile(File tempFile, String filename, Long fallbackGroupId, Long operatorId,
|
||||
boolean superAdmin, boolean deleteMode, QueryAsinImportProgressVo progress) throws Exception {
|
||||
/**
|
||||
* 逐行读表并分批入库。本方法<b>不</b>是私有的:它是导入链唯一的可测接缝
|
||||
* (形状同 {@code DedupeRunService.tryAcquireFileSlot},本仓既有约定)。
|
||||
*
|
||||
* <p>收尾必须做一次「有没有坏批」的交代:{@link ImportContext} 现在按批容错,
|
||||
* 一颗坏批只丢它自己那几行、其余批照常入库,所以异常不会自然冒到这里。
|
||||
* 全部批次都坏=一行没落库,此时必须如实失败,不许报「导入完成」。
|
||||
*/
|
||||
void processImportFile(File tempFile, String filename, Long fallbackGroupId, Long operatorId,
|
||||
boolean superAdmin, boolean deleteMode, QueryAsinImportProgressVo progress) throws Exception {
|
||||
validateExcelFilename(filename);
|
||||
ImportContext context = new ImportContext(fallbackGroupId, operatorId, superAdmin, deleteMode, progress);
|
||||
ExcelStreamReader.readFirstSheet(tempFile, new ExcelStreamReader.SheetRowHandler() {
|
||||
@@ -471,6 +479,7 @@ public class QueryAsinService {
|
||||
if (!context.hasHeader()) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
context.failIfBatchesLost();
|
||||
}
|
||||
|
||||
private File saveMultipartToTempFile(MultipartFile file, String prefix) throws Exception {
|
||||
@@ -702,6 +711,16 @@ public class QueryAsinService {
|
||||
private int insertedCount;
|
||||
private int deletedCount;
|
||||
private int skippedCount;
|
||||
/**
|
||||
* 没能入库的批次,逐批点名(B49 #6)。原形状:一整张表共用一条读行循环,
|
||||
* 某一批 {@code insertBatch} 抛出就直接冒到 {@code runImportTask} 的 catch,
|
||||
* 于是<b>后面的批次一个都跑不到</b>、已在内存里的余行也一起丢。
|
||||
* 现在按批容错:坏批自己认赔并记在这里,其余批照常入库;最后一并交代(见
|
||||
* {@link #failIfBatchesLost()}),既不带走整表,也不把没落库的行说成导入完成。
|
||||
*/
|
||||
private final List<String> lostBatchNotes = new ArrayList<>();
|
||||
private int lostBatchRowCount;
|
||||
private int lostBatchCellCount;
|
||||
|
||||
private ImportContext(Long fallbackGroupId, Long operatorId, boolean superAdmin,
|
||||
boolean deleteMode, QueryAsinImportProgressVo progress) {
|
||||
@@ -814,6 +833,7 @@ public class QueryAsinService {
|
||||
}
|
||||
}
|
||||
List<QueryAsinEntity> insertRows = new ArrayList<>();
|
||||
int insertRowCells = 0;
|
||||
for (QueryAsinEntity target : targets) {
|
||||
int cellCount = countCountryCells(target);
|
||||
String signature = rowSignature(target);
|
||||
@@ -836,13 +856,55 @@ public class QueryAsinService {
|
||||
}
|
||||
insertRows.add(target);
|
||||
insertedCount += cellCount;
|
||||
insertRowCells += cellCount;
|
||||
existingSignatures.add(signature);
|
||||
existingByKey.computeIfAbsent(importKey(target.getGroupId(), target.getShopName()),
|
||||
ignored -> new ArrayList<>()).add(target);
|
||||
}
|
||||
if (!insertRows.isEmpty()) {
|
||||
queryAsinMapper.insertBatch(insertRows);
|
||||
if (insertRows.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
queryAsinMapper.insertBatch(insertRows);
|
||||
} catch (Exception ex) {
|
||||
// 逐批容错(B49 #6):坏批只认赔自己那几行,其余批继续由后续 flush 入库,
|
||||
// 不再让一批的数据库异常中断整张表。计数按"没落库"回退,坏批点名留痕。
|
||||
insertedCount -= insertRowCells;
|
||||
skippedCount += insertRowCells;
|
||||
lostBatchRowCount += insertRows.size();
|
||||
lostBatchCellCount += insertRowCells;
|
||||
lostBatchNotes.add("坏批 " + (lostBatchNotes.size() + 1) + "(" + insertRows.size() + " 行,"
|
||||
+ describeBatchForNote(insertRows) + ")入库失败:"
|
||||
+ UserMessages.orFallback(ex.getMessage(), "数据库写入异常"));
|
||||
log.error("[query-asin-import] 批量入库失败,跳过该批并继续其余批次 batchSize={} cells={} groups={} shops={}",
|
||||
insertRows.size(), insertRowCells, groupIds, shopNames, ex);
|
||||
}
|
||||
}
|
||||
|
||||
/** 坏批点名:取批内前几行的「分组/店铺」,让运维能在不查库的情况下定位是哪一段没进去。 */
|
||||
private String describeBatchForNote(List<QueryAsinEntity> rows) {
|
||||
List<String> names = new ArrayList<>();
|
||||
for (QueryAsinEntity row : rows) {
|
||||
names.add(row.getGroupId() + "/" + row.getShopName());
|
||||
if (names.size() >= 3) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return String.join("、", names) + (rows.size() > names.size() ? " 等" + rows.size() + " 行" : "");
|
||||
}
|
||||
|
||||
/**
|
||||
* 有批次没入库就必须让导入落「失败」态:逐批容错换来了「其余批照常入库」,
|
||||
* 代价是异常不再自然冒出去、{@code runImportTask} 会把它报成 success——那等于
|
||||
* 把没落库的行说成导入完成。这里补一次交代,并且只在真有缺口时才改口。
|
||||
*/
|
||||
private void failIfBatchesLost() {
|
||||
if (lostBatchNotes.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
throw new BusinessException("整表已跑完,但有 " + lostBatchRowCount + " 行(" + lostBatchCellCount
|
||||
+ " 个 ASIN)未入库,已入库的行不受影响;重新导入整表即可补写(已入库行会按重复跳过),"
|
||||
+ "未入库的都是数据库写入失败、不是表格内容问题:" + String.join(";", lostBatchNotes));
|
||||
}
|
||||
|
||||
private QueryAsinEntity findCompatibleInsertCandidate(QueryAsinEntity target, List<QueryAsinEntity> candidates) {
|
||||
|
||||
+262
@@ -0,0 +1,262 @@
|
||||
package com.nanri.aiimage.modules.dedupe.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeSourceFileDto;
|
||||
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunProgressVo;
|
||||
import com.nanri.aiimage.modules.dedupe.service.DedupeRunService.DedupeArchiveEntry;
|
||||
import com.nanri.aiimage.modules.dedupe.service.DedupeRunService.FolderPackage;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.AbstractExecutorService;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipFile;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 文件夹去重的打包容错(B49 #4 的 folder 分支)。
|
||||
*
|
||||
* <p>folder 模式的逐文件清洗结果只落在本地临时目录,<b>唯一交付物就是那个 ZIP</b>。原形状有两颗
|
||||
* 会把整批带走的坏项:① 打包循环里一个条目读不到(临时文件被清掉/写失败)就整包作废;
|
||||
* ② 收尾时一个 worker 线程级异常先判 {@code aborted} 再打包,于是打包整步被跳过,
|
||||
* 已清洗完的 N-1 个文件一起不交付。两条口径都是「一颗坏项不许让其余项一起不干活」,
|
||||
* 同时终态仍必须如实说「没全跑完」。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class DedupeRunServiceFolderPackageTest {
|
||||
|
||||
private static final String[] HEADERS = {"id", "ASIN", "国家", "价格", "发货类型", "品牌", "状态", "创建时间"};
|
||||
|
||||
@Mock
|
||||
private FileTaskMapper fileTaskMapper;
|
||||
@Mock
|
||||
private FileResultMapper fileResultMapper;
|
||||
@Mock
|
||||
private StorageProperties storageProperties;
|
||||
@Mock
|
||||
private OssStorageService ossStorageService;
|
||||
@Mock
|
||||
private DedupeTotalDataService dedupeTotalDataService;
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
private DedupeRunService newService() {
|
||||
return new DedupeRunService(fileTaskMapper, fileResultMapper, storageProperties, ossStorageService,
|
||||
dedupeTotalDataService, null, null, objectMapper, new InstanceMetadata("test-instance"));
|
||||
}
|
||||
|
||||
/** 逐条目容错:坏条目只带走自己,其余结果文件照常进包,并被点名报出来。 */
|
||||
@Test
|
||||
@DisplayName("一个坏条目只带走自己:其余结果文件照常进包,坏条目点名")
|
||||
void badEntryDoesNotDiscardTheWholeFolderPackage(@TempDir Path tempDir) throws Exception {
|
||||
when(storageProperties.getLocalTempDir()).thenReturn(tempDir.toString());
|
||||
File goodFirst = tempDir.resolve("a.xlsx").toFile();
|
||||
File goodLast = tempDir.resolve("b.xlsx").toFile();
|
||||
assertTrue(goodFirst.createNewFile());
|
||||
assertTrue(goodLast.createNewFile());
|
||||
// 现网真实形状:清洗结果写完后临时文件被清掉/读不到
|
||||
File vanished = tempDir.resolve("c.xlsx").toFile();
|
||||
|
||||
FolderPackage packaged = newService().packageFolderDedupeResultsAsZip("批次", List.of(
|
||||
new DedupeArchiveEntry("dir/a.xlsx", "a.xlsx", goodFirst),
|
||||
new DedupeArchiveEntry("dir/c.xlsx", "c.xlsx", vanished),
|
||||
new DedupeArchiveEntry("dir/b.xlsx", "b.xlsx", goodLast)));
|
||||
|
||||
assertEquals(2, packaged.packedCount(), "坏条目只认赔自己,其余两个必须进包");
|
||||
assertEquals(List.of("c.xlsx"), packaged.skippedEntries(), "坏条目必须点名");
|
||||
try (ZipFile zip = new ZipFile(packaged.file())) {
|
||||
List<String> names = zip.stream().map(ZipEntry::getName).toList();
|
||||
assertEquals(List.of("dir/a.xlsx", "dir/b.xlsx"), names,
|
||||
"包里就是其余可读条目,坏项不得留半截文件");
|
||||
}
|
||||
}
|
||||
|
||||
/** 反向钉:一个都没写进去时不许交空包冒充「打包成功」。 */
|
||||
@Test
|
||||
@DisplayName("一个都没写进去时不许交空包冒充成功")
|
||||
void refusesToShipPackageWhenNothingCouldBePacked(@TempDir Path tempDir) {
|
||||
when(storageProperties.getLocalTempDir()).thenReturn(tempDir.toString());
|
||||
List<DedupeArchiveEntry> allMissing = List.of(
|
||||
new DedupeArchiveEntry("dir/a.xlsx", "a.xlsx", tempDir.resolve("a.xlsx").toFile()),
|
||||
new DedupeArchiveEntry("dir/b.xlsx", "b.xlsx", tempDir.resolve("b.xlsx").toFile()));
|
||||
|
||||
BusinessException thrown = assertThrows(BusinessException.class,
|
||||
() -> newService().packageFolderDedupeResultsAsZip("批次", allMissing));
|
||||
|
||||
assertTrue(thrown.getMessage().contains(String.valueOf(allMissing.size())),
|
||||
"文案要说清有几个文件没交付,实际=" + thrown.getMessage());
|
||||
}
|
||||
|
||||
/**
|
||||
* 一颗坏 worker 不许带走整个 ZIP:先交付已清洗完的文件,终态仍如实落失败。
|
||||
* 改前形状=打包排在 {@code aborted} 判定之后,一次结果行 insert 都不会发生。
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("坏 worker 不许带走整个 ZIP:已清洗完的文件仍交付,终态仍落失败")
|
||||
void abortedWorkerStillDeliversCleansedFiles(@TempDir Path tempDir) throws Exception {
|
||||
when(storageProperties.getLocalTempDir()).thenReturn(tempDir.toString());
|
||||
writeSourceFile(tempDir.resolve("keyA.xlsx").toFile());
|
||||
writeSourceFile(tempDir.resolve("keyB.xlsx").toFile());
|
||||
when(dedupeTotalDataService.findExistingComparableValues(any(), any())).thenReturn(Set.of());
|
||||
when(dedupeTotalDataService.normalizeComparableValueOrBlank(any()))
|
||||
.thenAnswer(invocation -> {
|
||||
Object value = invocation.getArgument(0);
|
||||
return value == null ? "" : String.valueOf(value).trim().toUpperCase();
|
||||
});
|
||||
when(ossStorageService.uploadResultFileWithFreshDownloadUrl(any(), eq("DEDUPE")))
|
||||
.thenReturn(new OssStorageService.UploadedResult("oss://dedupe/批次.zip", "https://dl/批次.zip"));
|
||||
|
||||
DedupeRunService service = newService();
|
||||
ReflectionTestUtils.setField(service, "fileExecutor", secondWorkerDiesOnContact());
|
||||
|
||||
DedupeRunRequest request = new DedupeRunRequest();
|
||||
request.setUserId(1L);
|
||||
request.setArchiveName("批次");
|
||||
request.setSelectedColumns(List.of("id", "ASIN", "国家"));
|
||||
request.setFiles(List.of(sourceFile("keyA", "a.xlsx", "dir/a.xlsx"),
|
||||
sourceFile("keyB", "b.xlsx", "dir/b.xlsx")));
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(5001L);
|
||||
task.setTaskNo("DEDUPE-5001");
|
||||
task.setStatus("RUNNING");
|
||||
DedupeRunProgressVo progress = newProgress("DEDUPE-5001");
|
||||
|
||||
ReflectionTestUtils.invokeMethod(service, "runAsync", "DEDUPE-5001", task, request, progress, 1L);
|
||||
|
||||
ArgumentCaptor<FileResultEntity> rows = ArgumentCaptor.forClass(FileResultEntity.class);
|
||||
verify(fileResultMapper, atLeastOnce()).insert(rows.capture());
|
||||
assertTrue(rows.getAllValues().stream()
|
||||
.anyMatch(row -> "application/zip".equals(row.getResultContentType())
|
||||
&& row.getResultFileUrl() != null),
|
||||
"已清洗完的 A 必须打进 ZIP 交付出去,不能被坏 worker 一起带走;实际结果行="
|
||||
+ rows.getAllValues().stream().map(FileResultEntity::getResultContentType).toList());
|
||||
|
||||
// 诚实面:部分交付不等于全量成功,终态照旧落失败
|
||||
assertEquals("FAILED", task.getStatus(), "有 worker 线程级异常结束时不得标 SUCCESS");
|
||||
assertEquals("failed", progress.getStatus());
|
||||
assertNotNull(progress.getError(), "缺口必须有一句可看的原因");
|
||||
}
|
||||
|
||||
/** 第 2 个提交的 worker 当场线程级异常(不执行任务),其余同线程串行跑完。 */
|
||||
private ExecutorService secondWorkerDiesOnContact() {
|
||||
AtomicInteger submissions = new AtomicInteger();
|
||||
return new AbstractExecutorService() {
|
||||
@Override
|
||||
public void execute(Runnable command) {
|
||||
command.run();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Future<?> submit(Runnable task) {
|
||||
if (submissions.incrementAndGet() == 2) {
|
||||
CompletableFuture<?> dead = new CompletableFuture<>();
|
||||
dead.completeExceptionally(new StackOverflowError("模拟 worker 线程级异常"));
|
||||
return dead;
|
||||
}
|
||||
task.run();
|
||||
return CompletableFuture.completedFuture(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
// 测试用同线程执行器,没有常驻线程要收
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Runnable> shutdownNow() {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isShutdown() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isTerminated() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean awaitTermination(long timeout, TimeUnit unit) {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private DedupeRunProgressVo newProgress(String runId) {
|
||||
DedupeRunProgressVo progress = new DedupeRunProgressVo();
|
||||
progress.setRunId(runId);
|
||||
progress.setStatus("running");
|
||||
progress.setTotal(2);
|
||||
progress.setProcessedCount(0);
|
||||
progress.setSuccessCount(0);
|
||||
progress.setFailedCount(0);
|
||||
progress.setFinished(false);
|
||||
return progress;
|
||||
}
|
||||
|
||||
private DedupeSourceFileDto sourceFile(String fileKey, String filename, String relativePath) {
|
||||
DedupeSourceFileDto source = new DedupeSourceFileDto();
|
||||
source.setFileKey(fileKey);
|
||||
source.setOriginalFilename(filename);
|
||||
source.setRelativePath(relativePath);
|
||||
return source;
|
||||
}
|
||||
|
||||
private void writeSourceFile(File file) throws Exception {
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
|
||||
Sheet sheet = workbook.createSheet("variant_collection");
|
||||
Row headerRow = sheet.createRow(0);
|
||||
for (int i = 0; i < HEADERS.length; i++) {
|
||||
headerRow.createCell(i).setCellValue(HEADERS[i]);
|
||||
}
|
||||
String[] values = {"1", "B0AAAAAA01", "英国", "10.00", "FBM", "品牌A", "在售", "2026-09-01"};
|
||||
Row row = sheet.createRow(1);
|
||||
for (int c = 0; c < values.length; c++) {
|
||||
row.createCell(c).setCellValue(values[c]);
|
||||
}
|
||||
try (FileOutputStream fos = new FileOutputStream(file)) {
|
||||
workbook.write(fos);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
package com.nanri.aiimage.modules.publish.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.publish.mapper.PublishFileMapper;
|
||||
import com.nanri.aiimage.modules.publish.mapper.PublishItemMapper;
|
||||
import com.nanri.aiimage.modules.publish.model.entity.PublishFileEntity;
|
||||
import com.nanri.aiimage.modules.publish.model.vo.PublishResultVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.junit.jupiter.MockitoSettings;
|
||||
import org.mockito.quality.Strictness;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 上架结果组装的「部分交付」文案(B49 #7/B50 #4,上一轮复验认定"整批丢弃"表述夸大:
|
||||
* 逐文件 status 与 VO 都已交付,真问题只剩文案会说谎)。
|
||||
*
|
||||
* <p>原形状:任务里有一个文件没处理完时,结果行仍被 {@code publishResultFile(..., false)}
|
||||
* 洗成「success=1 + error_message=null」,用户端只剩 {@code failedFileCount} 一个数字,
|
||||
* 文案层面等同于"全部成功"。这里只把文案改准——
|
||||
* <b>不动状态语义</b>:文件确实交付了,success/下载直链一改就把能用的结果也扣掉;
|
||||
* 也<b>不给 SUCCESS 的 file_task 写 error_message</b>:{@code TaskRetryEligibility} 与
|
||||
* {@code SimilarAsinStaleReopenSupport} 都按那段文字做字面量分类,写进去会喂错判据。
|
||||
* 缺口落在 {@code biz_file_result.error_message}(本仓既有的「部分交付」载体,
|
||||
* publish 侧只有 {@code toResultVo} 读它,纯展示)。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
@MockitoSettings(strictness = Strictness.LENIENT)
|
||||
class PublishTaskServicePartialDeliveryNoteTest {
|
||||
|
||||
private static final Long TASK_ID = 9001L;
|
||||
private static final Long RESULT_ID = 9201L;
|
||||
private static final String FAILED_FILENAME = "店铺B-上架表.xlsx";
|
||||
|
||||
@Mock
|
||||
private PublishWorkbookService workbookService;
|
||||
@Mock
|
||||
private PublishFileMapper publishFileMapper;
|
||||
@Mock
|
||||
private PublishItemMapper publishItemMapper;
|
||||
@Mock
|
||||
private FileTaskMapper fileTaskMapper;
|
||||
@Mock
|
||||
private FileResultMapper fileResultMapper;
|
||||
@Mock
|
||||
private OssStorageService ossStorageService;
|
||||
@Mock
|
||||
private TaskFileJobService taskFileJobService;
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
private PublishTaskService newService() {
|
||||
return new PublishTaskService(
|
||||
mock(LocalFileStorageService.class), mock(ZiniaoShopSwitchService.class), workbookService,
|
||||
publishFileMapper, publishItemMapper, fileTaskMapper, fileResultMapper,
|
||||
mock(TaskChunkMapper.class), mock(TaskScopeStateMapper.class), taskFileJobService,
|
||||
mock(TaskDistributedLockService.class), mock(TransientPayloadStorageService.class),
|
||||
ossStorageService, objectMapper, mock(TransactionTemplate.class),
|
||||
mock(DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS),
|
||||
mock(InstanceMetadata.class), mock(TaskProgressLightAssembler.class),
|
||||
mock(TaskHeartbeatPositionService.class));
|
||||
}
|
||||
|
||||
private FileTaskEntity task() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType("PUBLISH");
|
||||
task.setTaskNo("PUBLISH-9001");
|
||||
task.setSourceFileCount(2);
|
||||
task.setStatus("RUNNING");
|
||||
return task;
|
||||
}
|
||||
|
||||
private FileResultEntity resultRow() {
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(RESULT_ID);
|
||||
result.setTaskId(TASK_ID);
|
||||
result.setModuleType("PUBLISH");
|
||||
result.setSuccess(0);
|
||||
result.setErrorMessage("上一轮回执:等待组装");
|
||||
return result;
|
||||
}
|
||||
|
||||
private PublishFileEntity file(long id, String filename, String status) {
|
||||
PublishFileEntity file = new PublishFileEntity();
|
||||
file.setId(id);
|
||||
file.setTaskId(TASK_ID);
|
||||
file.setSourceFilename(filename);
|
||||
file.setShopName("店铺A");
|
||||
file.setStatus(status);
|
||||
return file;
|
||||
}
|
||||
|
||||
private TaskFileJobEntity job() {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(9301L);
|
||||
job.setTaskId(TASK_ID);
|
||||
job.setResultId(RESULT_ID);
|
||||
job.setModuleType("PUBLISH");
|
||||
job.setStatus("RUNNING");
|
||||
job.setRetryCount(0);
|
||||
return job;
|
||||
}
|
||||
|
||||
private void stubAssembly(FileTaskEntity task, FileResultEntity result, File packagedFile,
|
||||
List<PublishFileEntity> files) {
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
when(fileResultMapper.selectById(RESULT_ID)).thenReturn(result);
|
||||
when(publishFileMapper.selectList(any())).thenReturn(files);
|
||||
when(workbookService.packageTaskResult(any(), any(), anyInt(), any()))
|
||||
.thenReturn(new PublishWorkbookService.PackagedResult(
|
||||
packagedFile, packagedFile.getName(), "application/zip"));
|
||||
when(ossStorageService.uploadResultFile(any(), any())).thenReturn("oss://publish/result.zip");
|
||||
when(ossStorageService.generateFreshDownloadUrl(any())).thenReturn("https://dl/publish-result.zip");
|
||||
}
|
||||
|
||||
/** 有文件没处理完:文案必须说清缺口,且不许动 SUCCESS 语义与 file_task.error_message。 */
|
||||
@Test
|
||||
@DisplayName("部分文件未纳入结果:文案点名缺口,状态语义与 file_task 文案列不动")
|
||||
void partialDeliveryIsStatedHonestlyWithoutTouchingStatus(@TempDir Path tempDir) {
|
||||
FileTaskEntity task = task();
|
||||
FileResultEntity result = resultRow();
|
||||
File packaged = tempDir.resolve("PUBLISH-9001_上架结果.zip").toFile();
|
||||
stubAssembly(task, result, packaged, List.of(
|
||||
file(1L, "店铺A-上架表.xlsx", "SUCCESS"),
|
||||
file(2L, FAILED_FILENAME, "FAILED")));
|
||||
|
||||
newService().processResultFileJob(job());
|
||||
|
||||
// 状态语义不动:任务仍 SUCCESS、计数照旧、file_task.error_message 留空
|
||||
assertEquals("SUCCESS", task.getStatus());
|
||||
assertEquals(1, task.getSuccessFileCount());
|
||||
assertEquals(1, task.getFailedFileCount());
|
||||
assertNull(task.getErrorMessage(),
|
||||
"SUCCESS 的 file_task 不许写 error_message:TaskRetryEligibility/SimilarAsinStaleReopenSupport 按它分类");
|
||||
|
||||
// 结果行仍算交付(success=1+对象 key 在),但文案不再说谎
|
||||
assertEquals(1, result.getSuccess(), "文件确实交付了,不能改成 success=0 把可用结果扣掉");
|
||||
assertEquals("oss://publish/result.zip", result.getResultFileUrl());
|
||||
String note = result.getErrorMessage();
|
||||
assertNotNull(note, "有文件未纳入结果时,交付面必须有一句缺口说明");
|
||||
assertTrue(note.contains(FAILED_FILENAME), "缺口要点名到具体文件,实际=" + note);
|
||||
assertTrue(note.contains("未纳入"), "不许把部分项没处理说成全部完成,实际=" + note);
|
||||
assertTrue(note.contains("不是表格"), "可恢复的处理失败不许说成表格问题,实际=" + note);
|
||||
|
||||
// 文案要能到达用户面:PublishResultVo 仍既有直链又有缺口说明(ready 由 success+url 推导,不受影响)
|
||||
PublishResultVo vo = (PublishResultVo) ReflectionTestUtils.invokeMethod(
|
||||
newService(), "toResultVo", result, null);
|
||||
assertNotNull(vo);
|
||||
assertTrue(Boolean.TRUE.equals(vo.getFileReady()), "写了缺口说明也不许让结果变得不可下载");
|
||||
assertEquals("https://dl/publish-result.zip", vo.getDownloadUrl());
|
||||
assertEquals(note, vo.getErrorMessage());
|
||||
}
|
||||
|
||||
/** 反向钉:全部文件都成功时不许凭空造出缺口说明。 */
|
||||
@Test
|
||||
@DisplayName("全部文件都成功:不制造缺口文案")
|
||||
void fullSuccessStaysWithoutAnyGapNote(@TempDir Path tempDir) {
|
||||
FileTaskEntity task = task();
|
||||
FileResultEntity result = resultRow();
|
||||
File packaged = tempDir.resolve("PUBLISH-9001_上架结果.zip").toFile();
|
||||
stubAssembly(task, result, packaged, List.of(
|
||||
file(1L, "店铺A-上架表.xlsx", "SUCCESS"),
|
||||
file(2L, "店铺B-上架表.xlsx", "SUCCESS")));
|
||||
|
||||
newService().processResultFileJob(job());
|
||||
|
||||
assertEquals("SUCCESS", task.getStatus());
|
||||
assertEquals(2, task.getSuccessFileCount());
|
||||
assertEquals(0, task.getFailedFileCount());
|
||||
assertNull(task.getErrorMessage());
|
||||
assertNull(result.getErrorMessage(), "一个文件都没落下时不许留任何缺口文案");
|
||||
}
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
package com.nanri.aiimage.modules.shopkey.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.shopkey.mapper.QueryAsinMapper;
|
||||
import com.nanri.aiimage.modules.shopkey.model.entity.QueryAsinEntity;
|
||||
import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinImportProgressVo;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* ASIN 导入的逐批入库容错(B49 #6)。
|
||||
*
|
||||
* <p>原形状:整张表共用一条读行循环,某一批 {@code insertBatch} 抛出就直接冒到
|
||||
* {@code runImportTask} 的 catch——<b>后面的批次一个都跑不到</b>,已在内存里的余行一起丢,
|
||||
* 用户看到的是「导入失败」而实际上一行都没进去(或只进去半张表,无从判断)。
|
||||
* 现口径:一颗坏批只认赔自己那几行,其余批照常入库;同时不许把没落库的行说成「导入完成」。
|
||||
*
|
||||
* <p>审计里「报数虚高」那条已被复验否掉(原状下坏批的计数确实不外发),本类不是按那个理由改,
|
||||
* 而是因为<b>逐批容错之后</b>坏批的计数会随后续进度外发,所以必须按「没落库」回退。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class QueryAsinServiceImportBatchToleranceTest {
|
||||
|
||||
/** 批大小取 2,5 行 ⇒ 三批:坏掉第一批仍能跑到第二、三批。 */
|
||||
private static final int BATCH_SIZE = 2;
|
||||
private static final int ROW_COUNT = 5;
|
||||
private static final Long FALLBACK_GROUP_ID = 7L;
|
||||
|
||||
@Mock
|
||||
private QueryAsinMapper queryAsinMapper;
|
||||
@Mock
|
||||
private ShopManageGroupService shopManageGroupService;
|
||||
@Mock
|
||||
private TaskPressureProperties taskPressureProperties;
|
||||
@Mock
|
||||
private StringRedisTemplate stringRedisTemplate;
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
private QueryAsinService newService() {
|
||||
when(taskPressureProperties.getDbSelectBatchSize()).thenReturn(BATCH_SIZE);
|
||||
return new QueryAsinService(queryAsinMapper, shopManageGroupService, taskPressureProperties,
|
||||
stringRedisTemplate, objectMapper);
|
||||
}
|
||||
|
||||
/** 坏批不许带走其余批:其余批照常入库,坏批点名到行,并报数不说谎。 */
|
||||
@Test
|
||||
@DisplayName("一批坏:其余批照常入库,坏批点名,缺口如实报错")
|
||||
void lostBatchDoesNotAbortRemainingBatches(@TempDir Path tempDir) throws Exception {
|
||||
when(queryAsinMapper.insertBatch(anyList()))
|
||||
.thenThrow(new RuntimeException("模拟数据库批量插入失败:Deadlock found"))
|
||||
.thenReturn(2)
|
||||
.thenReturn(1);
|
||||
QueryAsinImportProgressVo progress = new QueryAsinImportProgressVo();
|
||||
|
||||
BusinessException thrown = assertThrows(BusinessException.class,
|
||||
() -> newService().processImportFile(writeImportExcel(tempDir), "asin.xlsx",
|
||||
FALLBACK_GROUP_ID, 1L, true, false, progress));
|
||||
|
||||
ArgumentCaptor<List<QueryAsinEntity>> batches = ArgumentCaptor.forClass(List.class);
|
||||
verify(queryAsinMapper, times(3)).insertBatch(batches.capture());
|
||||
List<String> attemptedShops = batches.getAllValues().stream()
|
||||
.flatMap(batch -> batch.stream())
|
||||
.map(QueryAsinEntity::getShopName)
|
||||
.toList();
|
||||
assertEquals(ROW_COUNT, attemptedShops.size(), "三批都必须被真的试过,不能被一颗坏批提前收走");
|
||||
|
||||
List<String> persistedShops = batches.getAllValues().subList(1, batches.getAllValues().size()).stream()
|
||||
.flatMap(batch -> batch.stream())
|
||||
.map(QueryAsinEntity::getShopName)
|
||||
.toList();
|
||||
assertTrue(persistedShops.containsAll(List.of("店铺3", "店铺4", "店铺5")),
|
||||
"坏批之后的行必须真的入库,实际入库=" + persistedShops);
|
||||
|
||||
// 报数只算真落库的行:坏批按"没落库"回退
|
||||
assertEquals(ROW_COUNT - BATCH_SIZE, progress.getInsertedCount(),
|
||||
"未落库的行不许算进已导入");
|
||||
assertEquals(BATCH_SIZE, progress.getSkippedCount(), "坏批的行数要有交代");
|
||||
|
||||
// 不许把缺口说成导入完成:状态由调用方按异常落 failed,文案点名坏批与真因
|
||||
assertTrue(thrown.getMessage().contains("店铺1"), "要点名到没入库的行,实际=" + thrown.getMessage());
|
||||
assertTrue(thrown.getMessage().contains(String.valueOf(BATCH_SIZE)),
|
||||
"要说清丢了几行,实际=" + thrown.getMessage());
|
||||
}
|
||||
|
||||
/** 反向钉:全部批次都落库时不许凭空改口报失败。 */
|
||||
@Test
|
||||
@DisplayName("全部落库时照常报成功,不得凭空制造缺口")
|
||||
void fullImportStillHasNoGap(@TempDir Path tempDir) throws Exception {
|
||||
QueryAsinImportProgressVo progress = new QueryAsinImportProgressVo();
|
||||
|
||||
newService().processImportFile(writeImportExcel(tempDir), "asin.xlsx",
|
||||
FALLBACK_GROUP_ID, 1L, true, false, progress);
|
||||
|
||||
verify(queryAsinMapper, times(3)).insertBatch(anyList());
|
||||
assertEquals(ROW_COUNT, progress.getInsertedCount());
|
||||
assertEquals(0, progress.getSkippedCount());
|
||||
}
|
||||
|
||||
/** 一行都没落库时不许报「导入完成」——那才是"根本没拿到数据说成数据正常"。 */
|
||||
@Test
|
||||
@DisplayName("整表都没入库时如实失败,并说清是数据库不是表格")
|
||||
void everyBatchLostIsReportedAsNothingImported(@TempDir Path tempDir) throws Exception {
|
||||
when(queryAsinMapper.insertBatch(anyList()))
|
||||
.thenThrow(new RuntimeException("模拟数据库连接不可用"));
|
||||
QueryAsinImportProgressVo progress = new QueryAsinImportProgressVo();
|
||||
|
||||
BusinessException thrown = assertThrows(BusinessException.class,
|
||||
() -> newService().processImportFile(writeImportExcel(tempDir), "asin.xlsx",
|
||||
FALLBACK_GROUP_ID, 1L, true, false, progress));
|
||||
|
||||
verify(queryAsinMapper, times(3)).insertBatch(anyList());
|
||||
assertEquals(0, progress.getInsertedCount(), "一行都没落库就不许有已导入计数");
|
||||
assertTrue(thrown.getMessage().contains(String.valueOf(ROW_COUNT)),
|
||||
"要说清整表都没进去,实际=" + thrown.getMessage());
|
||||
}
|
||||
|
||||
/**
|
||||
* 真实形状的导入表:表头「店铺名 + DE」,逐行给不同店铺与 ASIN。
|
||||
* 分组列故意留空,走页面的 fallbackGroupId(与现网「先选分组再传表」一致)。
|
||||
*/
|
||||
private File writeImportExcel(Path tempDir) throws Exception {
|
||||
File file = tempDir.resolve("asin.xlsx").toFile();
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
|
||||
Sheet sheet = workbook.createSheet("asin");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("店铺名");
|
||||
header.createCell(1).setCellValue("DE");
|
||||
for (int r = 0; r < ROW_COUNT; r++) {
|
||||
Row row = sheet.createRow(r + 1);
|
||||
row.createCell(0).setCellValue("店铺" + (r + 1));
|
||||
row.createCell(1).setCellValue("B0TEST000" + r);
|
||||
}
|
||||
try (FileOutputStream fos = new FileOutputStream(file)) {
|
||||
workbook.write(fos);
|
||||
}
|
||||
}
|
||||
return file;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user