From 88d737a7f0d73c7125cd2842e8421bca448c2f87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Tue, 27 Oct 2026 16:02:11 +0800 Subject: [PATCH] =?UTF-8?q?fix(tasks):=20=E6=89=B9=E9=87=8F=E5=85=A5?= =?UTF-8?q?=E5=BA=93=E4=B8=8E=20folder=20=E5=8E=BB=E9=87=8D=E9=80=90?= =?UTF-8?q?=E6=9D=A1=E5=AE=B9=E9=94=99=EF=BC=8C=E5=8F=91=E5=B8=83=E6=96=87?= =?UTF-8?q?=E6=A1=88=E4=B8=8D=E8=AF=B4=E8=B0=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B49/B50 收尾三处当时被标"非 ★"跳过的同类缺陷。判据同一条:一颗坏项不许让 其余项一起不干活,也不许把"根本没拿到数据"说成"数据正常"。 1) shopkey/service/QueryAsinService.java(B49 #6,flushInsert:843-844) 复验为真:整张表共用一条读行循环,某一批 insertBatch 抛出即冒到 runImportTask 的 catch,后面的批次一个都跑不到、内存里的余行一起丢。 审计里"报数虚高"那条已被复验否掉(原状下坏批计数确实不外发),本笔不按 那个理由改;但逐批容错之后坏批计数会随后续进度外发,所以必须按"没落库" 回退:insertedCount 扣回、skippedCount 记上、坏批点名进日志与 errorMessage。 收尾新增 failIfBatchesLost():有坏批就让导入落"失败"态并报出缺口行数—— admin 侧只在 failed 分支显示 errorMessage(QueryAsinPage.vue:260-271), 留在 success 上等于把没入库的行说成导入完成。重传整表是安全的:已入库行 会被 rowSignature 认成重复跳过。 processImportFile 由 private 收到包级作测试接缝(形状同 tryAcquireFileSlot)。 该 importer 此前零夹具,新建 3 例(真 xlsx + 真 ExcelStreamReader)。 2) dedupe/service/DedupeRunService.java(B49 #4 的 folder 分支) 只在 folder 模式成立:逐文件清洗结果只落本地临时目录,唯一交付物是那个 ZIP。两颗坏项会带走整批——① 收尾的 aborted 判定排在打包之前,一个 worker 线程级异常就让打包整步跳过;② 打包循环里一个条目读不到/条目名相撞就整包 作废。现改为 folder 分支先打包再判 aborted,打包逐条目容错并点名跳过; 一个都没写进去时拒绝交空包(那才是"根本没拿到数据")。aborted 仍照旧把 终态落 FAILED,不拿部分交付冒充全量成功;缺口写在交付条目与日志上, 不写 file_task.error_message。非 folder 分支一个字节没动。 DedupeArchiveEntry 与打包方法收到包级作测试接缝。新建 3 例,含一支用 同线程执行器让第 2 个 worker 当场线程级异常的真链路用例。 3) publish/service/PublishTaskService.java(B49 #7/B50 #4,607-609) 按上一轮复验结论只改文案,不动状态语义:逐文件 status/VO 本来就交付了。 会说谎的地方是 publishResultFile(..., preserveFailure=false) 把结果行 error_message 洗成 null,"N 个文件没进这张表"在文案层等于全部成功。 缺口写在 biz_file_result.error_message(本仓既有"部分交付"载体: ShopMatch:825/PriceTrack:905/ProductRisk:827 都按它判 partialFailure; publish 侧只有 toResultVo:1216 读它,纯展示,fileReady 由 success+url 推导 不受影响)。SUCCESS 的 file_task.error_message 继续留空并就地写明理由: TaskRetryEligibility.forceMajeureCategory 与 SimilarAsinStaleReopenSupport .isStaleFailure 都按那段文字做字面量分类,写进去会喂错 stale/重开/续跑判据。 文案两个方向都不说谎:说清"只含成功处理的 X 个、另有 N 个未纳入(点名)", 并明写"不是表格格式问题、可只重新提交这些文件",不把可恢复失败推给表格。 新建 2 例(含反向钉:全成功时不得凭空造缺口文案)。 实测:mvn -o test-compile rc=0;新增 8 例改前 6 红(QueryAsin 2 红=原始 RuntimeException 直接冒出、批次被提前收走;Dedupe 3 红="Wanted but not invoked: fileResultMapper.insert"+整包作废;Publish 1 红=errorMessage 恒 null),改后 8/8 绿;相邻既有 9 支类 97 例全绿(PublishTaskServiceTest 33、 DedupeRunProgressResumeTest 14、PublishFileLevelStatusTest 8、 PublishResultFileJobHandlerTest 8、DedupeRunServiceCleanRulesTest 9、 DedupeRunServiceFileSlotTest 2、QueryAsinResolveServiceTest 13 等)。 CatchSilenceRatchetTest 按预期判红(本笔新增 2 处 catch,均在被禁改的共享台账 里,合并时由主线程统一重钉):实测 total=1028(基线 1026)、 notRethrow(incl logOnly)=579(基线 577)、logOnly=181(不变)、 rethrow=393(不变)、empty=56(不变)。 --- .../dedupe/service/DedupeRunService.java | 78 +++++- .../publish/service/PublishTaskService.java | 44 +++ .../shopkey/service/QueryAsinService.java | 70 ++++- .../DedupeRunServiceFolderPackageTest.java | 262 ++++++++++++++++++ ...ishTaskServicePartialDeliveryNoteTest.java | 209 ++++++++++++++ ...ryAsinServiceImportBatchToleranceTest.java | 164 +++++++++++ 6 files changed, 811 insertions(+), 16 deletions(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunServiceFolderPackageTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/publish/service/PublishTaskServicePartialDeliveryNoteTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/shopkey/service/QueryAsinServiceImportBatchToleranceTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java index 92d113a30..3a749e030 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java @@ -367,16 +367,21 @@ public class DedupeRunService { } try { - if (aborted) { - // 有文件没能正常处理完就序列化/标成功,会出现「任务成功但结果缺文件」 - throw new IllegalStateException("部分文件处理线程异常结束,结果不完整"); - } + // folder 模式:先打包,再判 aborted。逐文件清洗结果只落在本地临时目录,唯一交付物就是这个 + // ZIP——原先一个 worker 线程级异常就让整步打包跳过,已清洗完的 N-1 个文件一起不交付 + // (B49 #4 的 folder 分支)。打包本身逐条目容错,见 + // {@link #packageFolderDedupeResultsAsZip(String, List)};aborted 仍照旧把终态落到 + // FAILED,绝不拿"部分交付"冒充全量成功。非 folder 模式一个字节不变。 if (folderMode && !archiveEntries.isEmpty()) { DedupeResultItemVo zipItem = buildFolderZipResult(request, archiveEntries, task); synchronized (progress) { outcomeItems.add(0, zipItem); } } + if (aborted) { + // 有文件没能正常处理完就序列化/标成功,会出现「任务成功但结果缺文件」 + throw new IllegalStateException("部分文件处理线程异常结束,结果不完整"); + } DedupeRunVo result = buildRunVo(request, outcomeItems, progress); finishTaskSuccess(task, result); synchronized (progress) { @@ -620,7 +625,9 @@ public class DedupeRunService { private DedupeResultItemVo buildFolderZipResult(DedupeRunRequest request, Map archiveEntries, FileTaskEntity task) { - File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), new ArrayList<>(archiveEntries.values())); + FolderPackage packaged = packageFolderDedupeResultsAsZip( + request.getArchiveName(), new ArrayList<>(archiveEntries.values())); + File zipFile = packaged.file(); OssStorageService.UploadedResult uploadedResult = ossStorageService.uploadResultFileWithFreshDownloadUrl(zipFile, "DEDUPE"); DedupeResultItemVo item = new DedupeResultItemVo(); item.setSourceFilename(request.getArchiveName()); @@ -628,6 +635,15 @@ public class DedupeRunService { item.setSuccess(true); item.setDownloadUrl(uploadedResult.downloadUrl()); item.setResultFileSize(zipFile.length()); + if (!packaged.skippedEntries().isEmpty()) { + // 缺口只写在这一条交付项上(VO + task.result_json),不写 file_task.error_message: + // 那段文字会被按字面量分类的判据读取(TaskRetryEligibility/ + // SimilarAsinStaleReopenSupport),SUCCESS 行留空才是安全口径。 + item.setError("压缩包已交付 " + packaged.packedCount() + " 个文件,另有 " + + packaged.skippedEntries().size() + " 个文件的结果没能写进压缩包(" + + String.join("、", packaged.skippedEntries()) + + ");这些文件本次确实没交付,是临时结果文件/写入的问题,不是表格内容问题,可只重新提交这几个文件"); + } FileResultEntity resultEntity = new FileResultEntity(); resultEntity.setTaskId(task.getId()); @@ -995,20 +1011,57 @@ public class DedupeRunService { return value == null ? "" : value; } - private File packageFolderDedupeResultsAsZip(String archiveName, List archiveEntries) { + /** 打包结果:交付的压缩包,以及逐条目容错后没能写进去的那些文件名。 */ + record FolderPackage(File file, int packedCount, List skippedEntries) { + } + + /** + * 文件夹模式逐条目容错打包(B49 #4 的 folder 分支)。 + * + *

原形状:一个条目读不到(结果临时文件被清掉/磁盘错)或条目名相撞,就在 {@code ZipOutputStream} + * 里抛出并整包作废——同一批里已清洗完的其余文件一起不交付,用户只能整批重跑。 + * 现在坏条目自己认赔、点名跳过,其余照常入包;只有一个都没写进去时才整批判失败 + * (那才是"根本没拿到数据",不许交一个空包冒充成功)。压缩包本体建不起来 + * (目录/磁盘级错误)同样是整批级别,照旧抛出。 + * + *

本方法不私有:它是 folder 打包唯一的可测接缝(形状同 {@link #tryAcquireFileSlot()})。 + */ + FolderPackage packageFolderDedupeResultsAsZip(String archiveName, List archiveEntries) { File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result")); File zipFile = buildNamedOutputFile(outputDir, archiveName + ".zip"); + int packedCount = 0; + List skippedEntries = new ArrayList<>(); try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) { for (DedupeArchiveEntry entry : archiveEntries) { - String zipEntryName = buildFolderZipEntry(entry.relativePath(), entry.inputName(), entry.inputName(), false); - zos.putNextEntry(new ZipEntry(zipEntryName)); - Files.copy(entry.outputFile().toPath(), zos); - zos.closeEntry(); + String entryName = entry.inputName(); + File source = entry.outputFile(); + // 先验可读再开条目:文件缺失/不可读与条目名相撞都发生在写入之前, + // 这类坏项不会在包里留下半截文件(写到一半才抛的极端 IO 错仍可能留短条目) + if (source == null || !source.exists() || !source.canRead()) { + skippedEntries.add(entryName); + log.error("[dedupe] 结果临时文件不可读,跳过该条目并继续其余文件 archiveName={} entry={} path={}", + archiveName, entryName, source == null ? "null" : source.getPath()); + continue; + } + try { + zos.putNextEntry(new ZipEntry(buildFolderZipEntry(entry.relativePath(), entryName, entryName, false))); + Files.copy(source.toPath(), zos); + zos.closeEntry(); + packedCount++; + } catch (Exception ex) { + skippedEntries.add(entryName); + log.error("[dedupe] 单个结果文件写入压缩包失败,跳过该条目并继续其余文件 archiveName={} entry={} error", + archiveName, entryName, ex); + } } } catch (Exception ex) { throw new BusinessException(UserMessages.with("去重结果打包失败", ex)); } - return zipFile; + if (packedCount == 0) { + throw new BusinessException("去重结果打包失败:待打包的 " + archiveEntries.size() + + " 个结果文件没有一个写进压缩包(临时结果文件缺失或写入异常,不是表格内容问题),本次没有交付文件夹结果"); + } + return new FolderPackage(zipFile, packedCount, skippedEntries); } private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) { @@ -1240,7 +1293,8 @@ public class DedupeRunService { return nanos / 1_000_000L; } - private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) { + /** 不私有:folder 打包接缝 {@link #packageFolderDedupeResultsAsZip(String, List)} 的入参类型。 */ + record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) { } private static final class DedupeReadResult { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java index ea5343cde..fc1bdb5d6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java @@ -96,6 +96,8 @@ public class PublishTaskService extends AbstractFileTaskPipeline private static final String STATUS_RUNNING = "RUNNING"; private static final String STATUS_SUCCESS = "SUCCESS"; private static final String STATUS_FAILED = "FAILED"; + /** 「部分交付」文案里最多点名几个文件,余下的用总数兜住。 */ + private static final int PARTIAL_NOTE_NAME_LIMIT = 3; /** * 保留期清理允许删除的终态集合(与 ModuleHistoryCleanupService 的 TERMINAL_STATUSES 一致)。 * PENDING/RUNNING 绝不在此列;PUBLISH 实际只会写 SUCCESS/FAILED,取消态是无害的超集。 @@ -603,9 +605,24 @@ public class PublishTaskService extends AbstractFileTaskPipeline publishResultFile(result, packaged.file(), packaged.filename(), packaged.contentType(), rowCount, false); int failedCount = (int) files.stream().filter(file -> STATUS_FAILED.equals(file.getStatus())).count(); + int unpackedCount = files.size() - successfulFiles.size(); + if (unpackedCount > 0) { + // 只补文案、不动状态语义:文件确实交付了,success=1 与下载直链都不能改(改了就把 + // 能用的结果也扣掉)。缺口写在这一行 result 的 error_message 上——本仓把它当 + // 「部分交付」载体用(ShopMatch:825/PriceTrack:905/ProductRisk:827 都按它判 + // partialFailure),而 publish 侧只有 toResultVo 读它(纯展示),不喂任何分类判据。 + result.setErrorMessage(partialDeliveryNote(files, successfulFiles.size(), unpackedCount, failedCount)); + fileResultMapper.updateById(result); + log.warn("[publish] 任务标成功但有文件未纳入结果 taskId={} files={} delivered={} unpacked={} failed={}", + task.getId(), files.size(), successfulFiles.size(), unpackedCount, failedCount); + } task.setStatus(STATUS_SUCCESS); task.setSuccessFileCount(successfulFiles.size()); task.setFailedFileCount(failedCount); + // SUCCESS 的 file_task 行必须留空 error_message:TaskRetryEligibility.forceMajeureCategory + // 与 SimilarAsinStaleReopenSupport.isStaleFailure 都拿这段文字做字面量分类(stale/重开/ + // 克隆重跑),写进去会把"其实已跑完一半"的单子喂给错误的分类。所以缺口只落在上面 + // 结果行的文案与日志里,这里继续清空的语义是有意为之,不是漏判。 task.setErrorMessage(null); task.setUpdatedAt(LocalDateTime.now()); task.setFinishedAt(LocalDateTime.now()); @@ -624,6 +641,33 @@ public class PublishTaskService extends AbstractFileTaskPipeline } } + /** + * 「部分交付」提示语(B49/B50 口径):不许把「N 个文件没处理」说成整单成功, + * 也不许反过来把可恢复的处理失败推给表格——文件级的真因本来就落在 + * {@code publish_file.error_message}({@code toFileVo} 已逐文件回传)。 + * 名单只取前几项,余下的用总数兜住,免得文案无限长。 + */ + private String partialDeliveryNote(List files, int deliveredCount, + int unpackedCount, int failedCount) { + List names = new ArrayList<>(); + for (PublishFileEntity file : files) { + if (STATUS_SUCCESS.equals(file.getStatus())) { + continue; + } + names.add(firstNonBlank(file.getSourceFilename(), "未命名文件")); + if (names.size() >= PARTIAL_NOTE_NAME_LIMIT) { + break; + } + } + String nameList = String.join("、", names) + + (unpackedCount > names.size() ? " 等 " + unpackedCount + " 个文件" : ""); + String gap = failedCount == unpackedCount + ? "另有 " + unpackedCount + " 个文件处理失败,未纳入结果(" + nameList + ")" + : "另有 " + unpackedCount + " 个文件未纳入结果(其中 " + failedCount + " 个处理失败:" + nameList + ")"; + return "结果已生成,但只含成功处理的 " + deliveredCount + " 个文件;" + gap + + "。这些文件本轮确实没有结果,不是表格格式问题——各自的原因见文件列表,可只重新提交这些文件,不必重跑整单"; + } + /** * 删除任务。 * diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/QueryAsinService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/QueryAsinService.java index f9882f3ac..6bd67e190 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/QueryAsinService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/QueryAsinService.java @@ -451,8 +451,16 @@ public class QueryAsinService { } } - private void processImportFile(File tempFile, String filename, Long fallbackGroupId, Long operatorId, - boolean superAdmin, boolean deleteMode, QueryAsinImportProgressVo progress) throws Exception { + /** + * 逐行读表并分批入库。本方法不是私有的:它是导入链唯一的可测接缝 + * (形状同 {@code DedupeRunService.tryAcquireFileSlot},本仓既有约定)。 + * + *

收尾必须做一次「有没有坏批」的交代:{@link ImportContext} 现在按批容错, + * 一颗坏批只丢它自己那几行、其余批照常入库,所以异常不会自然冒到这里。 + * 全部批次都坏=一行没落库,此时必须如实失败,不许报「导入完成」。 + */ + void processImportFile(File tempFile, String filename, Long fallbackGroupId, Long operatorId, + boolean superAdmin, boolean deleteMode, QueryAsinImportProgressVo progress) throws Exception { validateExcelFilename(filename); ImportContext context = new ImportContext(fallbackGroupId, operatorId, superAdmin, deleteMode, progress); ExcelStreamReader.readFirstSheet(tempFile, new ExcelStreamReader.SheetRowHandler() { @@ -471,6 +479,7 @@ public class QueryAsinService { if (!context.hasHeader()) { throw new BusinessException("Excel 表头为空"); } + context.failIfBatchesLost(); } private File saveMultipartToTempFile(MultipartFile file, String prefix) throws Exception { @@ -702,6 +711,16 @@ public class QueryAsinService { private int insertedCount; private int deletedCount; private int skippedCount; + /** + * 没能入库的批次,逐批点名(B49 #6)。原形状:一整张表共用一条读行循环, + * 某一批 {@code insertBatch} 抛出就直接冒到 {@code runImportTask} 的 catch, + * 于是后面的批次一个都跑不到、已在内存里的余行也一起丢。 + * 现在按批容错:坏批自己认赔并记在这里,其余批照常入库;最后一并交代(见 + * {@link #failIfBatchesLost()}),既不带走整表,也不把没落库的行说成导入完成。 + */ + private final List lostBatchNotes = new ArrayList<>(); + private int lostBatchRowCount; + private int lostBatchCellCount; private ImportContext(Long fallbackGroupId, Long operatorId, boolean superAdmin, boolean deleteMode, QueryAsinImportProgressVo progress) { @@ -814,6 +833,7 @@ public class QueryAsinService { } } List insertRows = new ArrayList<>(); + int insertRowCells = 0; for (QueryAsinEntity target : targets) { int cellCount = countCountryCells(target); String signature = rowSignature(target); @@ -836,13 +856,55 @@ public class QueryAsinService { } insertRows.add(target); insertedCount += cellCount; + insertRowCells += cellCount; existingSignatures.add(signature); existingByKey.computeIfAbsent(importKey(target.getGroupId(), target.getShopName()), ignored -> new ArrayList<>()).add(target); } - if (!insertRows.isEmpty()) { - queryAsinMapper.insertBatch(insertRows); + if (insertRows.isEmpty()) { + return; } + try { + queryAsinMapper.insertBatch(insertRows); + } catch (Exception ex) { + // 逐批容错(B49 #6):坏批只认赔自己那几行,其余批继续由后续 flush 入库, + // 不再让一批的数据库异常中断整张表。计数按"没落库"回退,坏批点名留痕。 + insertedCount -= insertRowCells; + skippedCount += insertRowCells; + lostBatchRowCount += insertRows.size(); + lostBatchCellCount += insertRowCells; + lostBatchNotes.add("坏批 " + (lostBatchNotes.size() + 1) + "(" + insertRows.size() + " 行," + + describeBatchForNote(insertRows) + ")入库失败:" + + UserMessages.orFallback(ex.getMessage(), "数据库写入异常")); + log.error("[query-asin-import] 批量入库失败,跳过该批并继续其余批次 batchSize={} cells={} groups={} shops={}", + insertRows.size(), insertRowCells, groupIds, shopNames, ex); + } + } + + /** 坏批点名:取批内前几行的「分组/店铺」,让运维能在不查库的情况下定位是哪一段没进去。 */ + private String describeBatchForNote(List rows) { + List names = new ArrayList<>(); + for (QueryAsinEntity row : rows) { + names.add(row.getGroupId() + "/" + row.getShopName()); + if (names.size() >= 3) { + break; + } + } + return String.join("、", names) + (rows.size() > names.size() ? " 等" + rows.size() + " 行" : ""); + } + + /** + * 有批次没入库就必须让导入落「失败」态:逐批容错换来了「其余批照常入库」, + * 代价是异常不再自然冒出去、{@code runImportTask} 会把它报成 success——那等于 + * 把没落库的行说成导入完成。这里补一次交代,并且只在真有缺口时才改口。 + */ + private void failIfBatchesLost() { + if (lostBatchNotes.isEmpty()) { + return; + } + throw new BusinessException("整表已跑完,但有 " + lostBatchRowCount + " 行(" + lostBatchCellCount + + " 个 ASIN)未入库,已入库的行不受影响;重新导入整表即可补写(已入库行会按重复跳过)," + + "未入库的都是数据库写入失败、不是表格内容问题:" + String.join(";", lostBatchNotes)); } private QueryAsinEntity findCompatibleInsertCandidate(QueryAsinEntity target, List candidates) { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunServiceFolderPackageTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunServiceFolderPackageTest.java new file mode 100644 index 000000000..84987c89a --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunServiceFolderPackageTest.java @@ -0,0 +1,262 @@ +package com.nanri.aiimage.modules.dedupe.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.dedupe.model.dto.DedupeRunRequest; +import com.nanri.aiimage.modules.dedupe.model.dto.DedupeSourceFileDto; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeRunProgressVo; +import com.nanri.aiimage.modules.dedupe.service.DedupeRunService.DedupeArchiveEntry; +import com.nanri.aiimage.modules.dedupe.service.DedupeRunService.FolderPackage; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; + +import java.io.File; +import java.io.FileOutputStream; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 文件夹去重的打包容错(B49 #4 的 folder 分支)。 + * + *

folder 模式的逐文件清洗结果只落在本地临时目录,唯一交付物就是那个 ZIP。原形状有两颗 + * 会把整批带走的坏项:① 打包循环里一个条目读不到(临时文件被清掉/写失败)就整包作废; + * ② 收尾时一个 worker 线程级异常先判 {@code aborted} 再打包,于是打包整步被跳过, + * 已清洗完的 N-1 个文件一起不交付。两条口径都是「一颗坏项不许让其余项一起不干活」, + * 同时终态仍必须如实说「没全跑完」。 + */ +@ExtendWith(MockitoExtension.class) +class DedupeRunServiceFolderPackageTest { + + private static final String[] HEADERS = {"id", "ASIN", "国家", "价格", "发货类型", "品牌", "状态", "创建时间"}; + + @Mock + private FileTaskMapper fileTaskMapper; + @Mock + private FileResultMapper fileResultMapper; + @Mock + private StorageProperties storageProperties; + @Mock + private OssStorageService ossStorageService; + @Mock + private DedupeTotalDataService dedupeTotalDataService; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + private DedupeRunService newService() { + return new DedupeRunService(fileTaskMapper, fileResultMapper, storageProperties, ossStorageService, + dedupeTotalDataService, null, null, objectMapper, new InstanceMetadata("test-instance")); + } + + /** 逐条目容错:坏条目只带走自己,其余结果文件照常进包,并被点名报出来。 */ + @Test + @DisplayName("一个坏条目只带走自己:其余结果文件照常进包,坏条目点名") + void badEntryDoesNotDiscardTheWholeFolderPackage(@TempDir Path tempDir) throws Exception { + when(storageProperties.getLocalTempDir()).thenReturn(tempDir.toString()); + File goodFirst = tempDir.resolve("a.xlsx").toFile(); + File goodLast = tempDir.resolve("b.xlsx").toFile(); + assertTrue(goodFirst.createNewFile()); + assertTrue(goodLast.createNewFile()); + // 现网真实形状:清洗结果写完后临时文件被清掉/读不到 + File vanished = tempDir.resolve("c.xlsx").toFile(); + + FolderPackage packaged = newService().packageFolderDedupeResultsAsZip("批次", List.of( + new DedupeArchiveEntry("dir/a.xlsx", "a.xlsx", goodFirst), + new DedupeArchiveEntry("dir/c.xlsx", "c.xlsx", vanished), + new DedupeArchiveEntry("dir/b.xlsx", "b.xlsx", goodLast))); + + assertEquals(2, packaged.packedCount(), "坏条目只认赔自己,其余两个必须进包"); + assertEquals(List.of("c.xlsx"), packaged.skippedEntries(), "坏条目必须点名"); + try (ZipFile zip = new ZipFile(packaged.file())) { + List names = zip.stream().map(ZipEntry::getName).toList(); + assertEquals(List.of("dir/a.xlsx", "dir/b.xlsx"), names, + "包里就是其余可读条目,坏项不得留半截文件"); + } + } + + /** 反向钉:一个都没写进去时不许交空包冒充「打包成功」。 */ + @Test + @DisplayName("一个都没写进去时不许交空包冒充成功") + void refusesToShipPackageWhenNothingCouldBePacked(@TempDir Path tempDir) { + when(storageProperties.getLocalTempDir()).thenReturn(tempDir.toString()); + List allMissing = List.of( + new DedupeArchiveEntry("dir/a.xlsx", "a.xlsx", tempDir.resolve("a.xlsx").toFile()), + new DedupeArchiveEntry("dir/b.xlsx", "b.xlsx", tempDir.resolve("b.xlsx").toFile())); + + BusinessException thrown = assertThrows(BusinessException.class, + () -> newService().packageFolderDedupeResultsAsZip("批次", allMissing)); + + assertTrue(thrown.getMessage().contains(String.valueOf(allMissing.size())), + "文案要说清有几个文件没交付,实际=" + thrown.getMessage()); + } + + /** + * 一颗坏 worker 不许带走整个 ZIP:先交付已清洗完的文件,终态仍如实落失败。 + * 改前形状=打包排在 {@code aborted} 判定之后,一次结果行 insert 都不会发生。 + */ + @Test + @DisplayName("坏 worker 不许带走整个 ZIP:已清洗完的文件仍交付,终态仍落失败") + void abortedWorkerStillDeliversCleansedFiles(@TempDir Path tempDir) throws Exception { + when(storageProperties.getLocalTempDir()).thenReturn(tempDir.toString()); + writeSourceFile(tempDir.resolve("keyA.xlsx").toFile()); + writeSourceFile(tempDir.resolve("keyB.xlsx").toFile()); + when(dedupeTotalDataService.findExistingComparableValues(any(), any())).thenReturn(Set.of()); + when(dedupeTotalDataService.normalizeComparableValueOrBlank(any())) + .thenAnswer(invocation -> { + Object value = invocation.getArgument(0); + return value == null ? "" : String.valueOf(value).trim().toUpperCase(); + }); + when(ossStorageService.uploadResultFileWithFreshDownloadUrl(any(), eq("DEDUPE"))) + .thenReturn(new OssStorageService.UploadedResult("oss://dedupe/批次.zip", "https://dl/批次.zip")); + + DedupeRunService service = newService(); + ReflectionTestUtils.setField(service, "fileExecutor", secondWorkerDiesOnContact()); + + DedupeRunRequest request = new DedupeRunRequest(); + request.setUserId(1L); + request.setArchiveName("批次"); + request.setSelectedColumns(List.of("id", "ASIN", "国家")); + request.setFiles(List.of(sourceFile("keyA", "a.xlsx", "dir/a.xlsx"), + sourceFile("keyB", "b.xlsx", "dir/b.xlsx"))); + + FileTaskEntity task = new FileTaskEntity(); + task.setId(5001L); + task.setTaskNo("DEDUPE-5001"); + task.setStatus("RUNNING"); + DedupeRunProgressVo progress = newProgress("DEDUPE-5001"); + + ReflectionTestUtils.invokeMethod(service, "runAsync", "DEDUPE-5001", task, request, progress, 1L); + + ArgumentCaptor rows = ArgumentCaptor.forClass(FileResultEntity.class); + verify(fileResultMapper, atLeastOnce()).insert(rows.capture()); + assertTrue(rows.getAllValues().stream() + .anyMatch(row -> "application/zip".equals(row.getResultContentType()) + && row.getResultFileUrl() != null), + "已清洗完的 A 必须打进 ZIP 交付出去,不能被坏 worker 一起带走;实际结果行=" + + rows.getAllValues().stream().map(FileResultEntity::getResultContentType).toList()); + + // 诚实面:部分交付不等于全量成功,终态照旧落失败 + assertEquals("FAILED", task.getStatus(), "有 worker 线程级异常结束时不得标 SUCCESS"); + assertEquals("failed", progress.getStatus()); + assertNotNull(progress.getError(), "缺口必须有一句可看的原因"); + } + + /** 第 2 个提交的 worker 当场线程级异常(不执行任务),其余同线程串行跑完。 */ + private ExecutorService secondWorkerDiesOnContact() { + AtomicInteger submissions = new AtomicInteger(); + return new AbstractExecutorService() { + @Override + public void execute(Runnable command) { + command.run(); + } + + @Override + public Future submit(Runnable task) { + if (submissions.incrementAndGet() == 2) { + CompletableFuture dead = new CompletableFuture<>(); + dead.completeExceptionally(new StackOverflowError("模拟 worker 线程级异常")); + return dead; + } + task.run(); + return CompletableFuture.completedFuture(null); + } + + @Override + public void shutdown() { + // 测试用同线程执行器,没有常驻线程要收 + } + + @Override + public List shutdownNow() { + return List.of(); + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return true; + } + }; + } + + private DedupeRunProgressVo newProgress(String runId) { + DedupeRunProgressVo progress = new DedupeRunProgressVo(); + progress.setRunId(runId); + progress.setStatus("running"); + progress.setTotal(2); + progress.setProcessedCount(0); + progress.setSuccessCount(0); + progress.setFailedCount(0); + progress.setFinished(false); + return progress; + } + + private DedupeSourceFileDto sourceFile(String fileKey, String filename, String relativePath) { + DedupeSourceFileDto source = new DedupeSourceFileDto(); + source.setFileKey(fileKey); + source.setOriginalFilename(filename); + source.setRelativePath(relativePath); + return source; + } + + private void writeSourceFile(File file) throws Exception { + try (XSSFWorkbook workbook = new XSSFWorkbook()) { + Sheet sheet = workbook.createSheet("variant_collection"); + Row headerRow = sheet.createRow(0); + for (int i = 0; i < HEADERS.length; i++) { + headerRow.createCell(i).setCellValue(HEADERS[i]); + } + String[] values = {"1", "B0AAAAAA01", "英国", "10.00", "FBM", "品牌A", "在售", "2026-09-01"}; + Row row = sheet.createRow(1); + for (int c = 0; c < values.length; c++) { + row.createCell(c).setCellValue(values[c]); + } + try (FileOutputStream fos = new FileOutputStream(file)) { + workbook.write(fos); + } + } + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/publish/service/PublishTaskServicePartialDeliveryNoteTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/publish/service/PublishTaskServicePartialDeliveryNoteTest.java new file mode 100644 index 000000000..7caa7a7c6 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/publish/service/PublishTaskServicePartialDeliveryNoteTest.java @@ -0,0 +1,209 @@ +package com.nanri.aiimage.modules.publish.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.publish.mapper.PublishFileMapper; +import com.nanri.aiimage.modules.publish.mapper.PublishItemMapper; +import com.nanri.aiimage.modules.publish.model.entity.PublishFileEntity; +import com.nanri.aiimage.modules.publish.model.vo.PublishResultVo; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; +import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.transaction.support.TransactionTemplate; + +import java.io.File; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * 上架结果组装的「部分交付」文案(B49 #7/B50 #4,上一轮复验认定"整批丢弃"表述夸大: + * 逐文件 status 与 VO 都已交付,真问题只剩文案会说谎)。 + * + *

原形状:任务里有一个文件没处理完时,结果行仍被 {@code publishResultFile(..., false)} + * 洗成「success=1 + error_message=null」,用户端只剩 {@code failedFileCount} 一个数字, + * 文案层面等同于"全部成功"。这里只把文案改准—— + * 不动状态语义:文件确实交付了,success/下载直链一改就把能用的结果也扣掉; + * 也不给 SUCCESS 的 file_task 写 error_message:{@code TaskRetryEligibility} 与 + * {@code SimilarAsinStaleReopenSupport} 都按那段文字做字面量分类,写进去会喂错判据。 + * 缺口落在 {@code biz_file_result.error_message}(本仓既有的「部分交付」载体, + * publish 侧只有 {@code toResultVo} 读它,纯展示)。 + */ +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class PublishTaskServicePartialDeliveryNoteTest { + + private static final Long TASK_ID = 9001L; + private static final Long RESULT_ID = 9201L; + private static final String FAILED_FILENAME = "店铺B-上架表.xlsx"; + + @Mock + private PublishWorkbookService workbookService; + @Mock + private PublishFileMapper publishFileMapper; + @Mock + private PublishItemMapper publishItemMapper; + @Mock + private FileTaskMapper fileTaskMapper; + @Mock + private FileResultMapper fileResultMapper; + @Mock + private OssStorageService ossStorageService; + @Mock + private TaskFileJobService taskFileJobService; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + private PublishTaskService newService() { + return new PublishTaskService( + mock(LocalFileStorageService.class), mock(ZiniaoShopSwitchService.class), workbookService, + publishFileMapper, publishItemMapper, fileTaskMapper, fileResultMapper, + mock(TaskChunkMapper.class), mock(TaskScopeStateMapper.class), taskFileJobService, + mock(TaskDistributedLockService.class), mock(TransientPayloadStorageService.class), + ossStorageService, objectMapper, mock(TransactionTemplate.class), + mock(DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS), + mock(InstanceMetadata.class), mock(TaskProgressLightAssembler.class), + mock(TaskHeartbeatPositionService.class)); + } + + private FileTaskEntity task() { + FileTaskEntity task = new FileTaskEntity(); + task.setId(TASK_ID); + task.setModuleType("PUBLISH"); + task.setTaskNo("PUBLISH-9001"); + task.setSourceFileCount(2); + task.setStatus("RUNNING"); + return task; + } + + private FileResultEntity resultRow() { + FileResultEntity result = new FileResultEntity(); + result.setId(RESULT_ID); + result.setTaskId(TASK_ID); + result.setModuleType("PUBLISH"); + result.setSuccess(0); + result.setErrorMessage("上一轮回执:等待组装"); + return result; + } + + private PublishFileEntity file(long id, String filename, String status) { + PublishFileEntity file = new PublishFileEntity(); + file.setId(id); + file.setTaskId(TASK_ID); + file.setSourceFilename(filename); + file.setShopName("店铺A"); + file.setStatus(status); + return file; + } + + private TaskFileJobEntity job() { + TaskFileJobEntity job = new TaskFileJobEntity(); + job.setId(9301L); + job.setTaskId(TASK_ID); + job.setResultId(RESULT_ID); + job.setModuleType("PUBLISH"); + job.setStatus("RUNNING"); + job.setRetryCount(0); + return job; + } + + private void stubAssembly(FileTaskEntity task, FileResultEntity result, File packagedFile, + List files) { + when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task); + when(fileResultMapper.selectById(RESULT_ID)).thenReturn(result); + when(publishFileMapper.selectList(any())).thenReturn(files); + when(workbookService.packageTaskResult(any(), any(), anyInt(), any())) + .thenReturn(new PublishWorkbookService.PackagedResult( + packagedFile, packagedFile.getName(), "application/zip")); + when(ossStorageService.uploadResultFile(any(), any())).thenReturn("oss://publish/result.zip"); + when(ossStorageService.generateFreshDownloadUrl(any())).thenReturn("https://dl/publish-result.zip"); + } + + /** 有文件没处理完:文案必须说清缺口,且不许动 SUCCESS 语义与 file_task.error_message。 */ + @Test + @DisplayName("部分文件未纳入结果:文案点名缺口,状态语义与 file_task 文案列不动") + void partialDeliveryIsStatedHonestlyWithoutTouchingStatus(@TempDir Path tempDir) { + FileTaskEntity task = task(); + FileResultEntity result = resultRow(); + File packaged = tempDir.resolve("PUBLISH-9001_上架结果.zip").toFile(); + stubAssembly(task, result, packaged, List.of( + file(1L, "店铺A-上架表.xlsx", "SUCCESS"), + file(2L, FAILED_FILENAME, "FAILED"))); + + newService().processResultFileJob(job()); + + // 状态语义不动:任务仍 SUCCESS、计数照旧、file_task.error_message 留空 + assertEquals("SUCCESS", task.getStatus()); + assertEquals(1, task.getSuccessFileCount()); + assertEquals(1, task.getFailedFileCount()); + assertNull(task.getErrorMessage(), + "SUCCESS 的 file_task 不许写 error_message:TaskRetryEligibility/SimilarAsinStaleReopenSupport 按它分类"); + + // 结果行仍算交付(success=1+对象 key 在),但文案不再说谎 + assertEquals(1, result.getSuccess(), "文件确实交付了,不能改成 success=0 把可用结果扣掉"); + assertEquals("oss://publish/result.zip", result.getResultFileUrl()); + String note = result.getErrorMessage(); + assertNotNull(note, "有文件未纳入结果时,交付面必须有一句缺口说明"); + assertTrue(note.contains(FAILED_FILENAME), "缺口要点名到具体文件,实际=" + note); + assertTrue(note.contains("未纳入"), "不许把部分项没处理说成全部完成,实际=" + note); + assertTrue(note.contains("不是表格"), "可恢复的处理失败不许说成表格问题,实际=" + note); + + // 文案要能到达用户面:PublishResultVo 仍既有直链又有缺口说明(ready 由 success+url 推导,不受影响) + PublishResultVo vo = (PublishResultVo) ReflectionTestUtils.invokeMethod( + newService(), "toResultVo", result, null); + assertNotNull(vo); + assertTrue(Boolean.TRUE.equals(vo.getFileReady()), "写了缺口说明也不许让结果变得不可下载"); + assertEquals("https://dl/publish-result.zip", vo.getDownloadUrl()); + assertEquals(note, vo.getErrorMessage()); + } + + /** 反向钉:全部文件都成功时不许凭空造出缺口说明。 */ + @Test + @DisplayName("全部文件都成功:不制造缺口文案") + void fullSuccessStaysWithoutAnyGapNote(@TempDir Path tempDir) { + FileTaskEntity task = task(); + FileResultEntity result = resultRow(); + File packaged = tempDir.resolve("PUBLISH-9001_上架结果.zip").toFile(); + stubAssembly(task, result, packaged, List.of( + file(1L, "店铺A-上架表.xlsx", "SUCCESS"), + file(2L, "店铺B-上架表.xlsx", "SUCCESS"))); + + newService().processResultFileJob(job()); + + assertEquals("SUCCESS", task.getStatus()); + assertEquals(2, task.getSuccessFileCount()); + assertEquals(0, task.getFailedFileCount()); + assertNull(task.getErrorMessage()); + assertNull(result.getErrorMessage(), "一个文件都没落下时不许留任何缺口文案"); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopkey/service/QueryAsinServiceImportBatchToleranceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopkey/service/QueryAsinServiceImportBatchToleranceTest.java new file mode 100644 index 000000000..d87cf5bf1 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopkey/service/QueryAsinServiceImportBatchToleranceTest.java @@ -0,0 +1,164 @@ +package com.nanri.aiimage.modules.shopkey.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.TaskPressureProperties; +import com.nanri.aiimage.modules.shopkey.mapper.QueryAsinMapper; +import com.nanri.aiimage.modules.shopkey.model.entity.QueryAsinEntity; +import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinImportProgressVo; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.redis.core.StringRedisTemplate; + +import java.io.File; +import java.io.FileOutputStream; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * ASIN 导入的逐批入库容错(B49 #6)。 + * + *

原形状:整张表共用一条读行循环,某一批 {@code insertBatch} 抛出就直接冒到 + * {@code runImportTask} 的 catch——后面的批次一个都跑不到,已在内存里的余行一起丢, + * 用户看到的是「导入失败」而实际上一行都没进去(或只进去半张表,无从判断)。 + * 现口径:一颗坏批只认赔自己那几行,其余批照常入库;同时不许把没落库的行说成「导入完成」。 + * + *

审计里「报数虚高」那条已被复验否掉(原状下坏批的计数确实不外发),本类不是按那个理由改, + * 而是因为逐批容错之后坏批的计数会随后续进度外发,所以必须按「没落库」回退。 + */ +@ExtendWith(MockitoExtension.class) +class QueryAsinServiceImportBatchToleranceTest { + + /** 批大小取 2,5 行 ⇒ 三批:坏掉第一批仍能跑到第二、三批。 */ + private static final int BATCH_SIZE = 2; + private static final int ROW_COUNT = 5; + private static final Long FALLBACK_GROUP_ID = 7L; + + @Mock + private QueryAsinMapper queryAsinMapper; + @Mock + private ShopManageGroupService shopManageGroupService; + @Mock + private TaskPressureProperties taskPressureProperties; + @Mock + private StringRedisTemplate stringRedisTemplate; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + private QueryAsinService newService() { + when(taskPressureProperties.getDbSelectBatchSize()).thenReturn(BATCH_SIZE); + return new QueryAsinService(queryAsinMapper, shopManageGroupService, taskPressureProperties, + stringRedisTemplate, objectMapper); + } + + /** 坏批不许带走其余批:其余批照常入库,坏批点名到行,并报数不说谎。 */ + @Test + @DisplayName("一批坏:其余批照常入库,坏批点名,缺口如实报错") + void lostBatchDoesNotAbortRemainingBatches(@TempDir Path tempDir) throws Exception { + when(queryAsinMapper.insertBatch(anyList())) + .thenThrow(new RuntimeException("模拟数据库批量插入失败:Deadlock found")) + .thenReturn(2) + .thenReturn(1); + QueryAsinImportProgressVo progress = new QueryAsinImportProgressVo(); + + BusinessException thrown = assertThrows(BusinessException.class, + () -> newService().processImportFile(writeImportExcel(tempDir), "asin.xlsx", + FALLBACK_GROUP_ID, 1L, true, false, progress)); + + ArgumentCaptor> batches = ArgumentCaptor.forClass(List.class); + verify(queryAsinMapper, times(3)).insertBatch(batches.capture()); + List attemptedShops = batches.getAllValues().stream() + .flatMap(batch -> batch.stream()) + .map(QueryAsinEntity::getShopName) + .toList(); + assertEquals(ROW_COUNT, attemptedShops.size(), "三批都必须被真的试过,不能被一颗坏批提前收走"); + + List persistedShops = batches.getAllValues().subList(1, batches.getAllValues().size()).stream() + .flatMap(batch -> batch.stream()) + .map(QueryAsinEntity::getShopName) + .toList(); + assertTrue(persistedShops.containsAll(List.of("店铺3", "店铺4", "店铺5")), + "坏批之后的行必须真的入库,实际入库=" + persistedShops); + + // 报数只算真落库的行:坏批按"没落库"回退 + assertEquals(ROW_COUNT - BATCH_SIZE, progress.getInsertedCount(), + "未落库的行不许算进已导入"); + assertEquals(BATCH_SIZE, progress.getSkippedCount(), "坏批的行数要有交代"); + + // 不许把缺口说成导入完成:状态由调用方按异常落 failed,文案点名坏批与真因 + assertTrue(thrown.getMessage().contains("店铺1"), "要点名到没入库的行,实际=" + thrown.getMessage()); + assertTrue(thrown.getMessage().contains(String.valueOf(BATCH_SIZE)), + "要说清丢了几行,实际=" + thrown.getMessage()); + } + + /** 反向钉:全部批次都落库时不许凭空改口报失败。 */ + @Test + @DisplayName("全部落库时照常报成功,不得凭空制造缺口") + void fullImportStillHasNoGap(@TempDir Path tempDir) throws Exception { + QueryAsinImportProgressVo progress = new QueryAsinImportProgressVo(); + + newService().processImportFile(writeImportExcel(tempDir), "asin.xlsx", + FALLBACK_GROUP_ID, 1L, true, false, progress); + + verify(queryAsinMapper, times(3)).insertBatch(anyList()); + assertEquals(ROW_COUNT, progress.getInsertedCount()); + assertEquals(0, progress.getSkippedCount()); + } + + /** 一行都没落库时不许报「导入完成」——那才是"根本没拿到数据说成数据正常"。 */ + @Test + @DisplayName("整表都没入库时如实失败,并说清是数据库不是表格") + void everyBatchLostIsReportedAsNothingImported(@TempDir Path tempDir) throws Exception { + when(queryAsinMapper.insertBatch(anyList())) + .thenThrow(new RuntimeException("模拟数据库连接不可用")); + QueryAsinImportProgressVo progress = new QueryAsinImportProgressVo(); + + BusinessException thrown = assertThrows(BusinessException.class, + () -> newService().processImportFile(writeImportExcel(tempDir), "asin.xlsx", + FALLBACK_GROUP_ID, 1L, true, false, progress)); + + verify(queryAsinMapper, times(3)).insertBatch(anyList()); + assertEquals(0, progress.getInsertedCount(), "一行都没落库就不许有已导入计数"); + assertTrue(thrown.getMessage().contains(String.valueOf(ROW_COUNT)), + "要说清整表都没进去,实际=" + thrown.getMessage()); + } + + /** + * 真实形状的导入表:表头「店铺名 + DE」,逐行给不同店铺与 ASIN。 + * 分组列故意留空,走页面的 fallbackGroupId(与现网「先选分组再传表」一致)。 + */ + private File writeImportExcel(Path tempDir) throws Exception { + File file = tempDir.resolve("asin.xlsx").toFile(); + try (XSSFWorkbook workbook = new XSSFWorkbook()) { + Sheet sheet = workbook.createSheet("asin"); + Row header = sheet.createRow(0); + header.createCell(0).setCellValue("店铺名"); + header.createCell(1).setCellValue("DE"); + for (int r = 0; r < ROW_COUNT; r++) { + Row row = sheet.createRow(r + 1); + row.createCell(0).setCellValue("店铺" + (r + 1)); + row.createCell(1).setCellValue("B0TEST000" + r); + } + try (FileOutputStream fos = new FileOutputStream(file)) { + workbook.write(fos); + } + } + return file; + } +}