From 7da93112658cb673fe6a974e51c4d708465932d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Wed, 21 Oct 2026 01:17:54 +0800 Subject: [PATCH] =?UTF-8?q?refactor(phase-3):=203A=20=E6=8B=86=E8=96=84?= =?UTF-8?q?=E7=AC=AC=E5=8D=81=E4=BA=8C=E7=AC=94=E2=80=94=E2=80=94AP=20?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E7=B0=87=E5=87=BA=E4=BB=B6=EF=BC=8C=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=201409=E2=86=921127?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parse 聚类 12 段整体进 service/support/AppearancePatentParsePipeline(471 行): parseAndCreateTask 主体 + parseWorkbook/statusHeaderName/hydratePromptFields/ isAppearanceResultWorkbook/hasHeader/normalizeDisplayId/mergeHeaders/ buildAggregateScopeKey/buildAggregateSourceFilenameLabel/buildRowToken/elapsedMs, ParsedWorkbook/ParsedAppearanceRow 两记录随件;宿主能力走 5 接口 + Function/Consumer/ Supplier 回调,groupRowsByBaseId 整枚直传不抄第二份。 等值证明:12 段符号归一逐行 diff 全部 EQUAL(elapsedMs 补 static 是唯一形状增量); 本簇无按名反射点名的入口(全测试树 grep 核实),未留壳;状态字面量改吃 TaskStatuses.FileTask.PENDING。SourceScan 面数 38→39、support 15→16 并逐枚点名。 门禁:test-compile rc=0;定向(AP+十把守卫)355/0 红/2 skip;后端整量单跑 4617 例/1 红/17 skipped——红是 CatchSilenceRatchetTest 扫到另一路会话改 BrandCheckClient 的中间态(total 1008>上限 1004),其落盘后当前树实测 997、 单跑该类 9/9 复绿;本笔 catch 净增量 0(服务 4→4、新件 0)。ops ruff 全绿。 真机启动验收本格仍欠(本机无 MySQL/Redis 监听)。 --- .../service/AppearancePatentTaskService.java | 320 +----------- .../AppearancePatentParsePipeline.java | 471 ++++++++++++++++++ .../aiimage/testutil/SourceScanTest.java | 17 +- 3 files changed, 499 insertions(+), 309 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentParsePipeline.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index 2637bb75e..18a6cac4c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -29,6 +29,7 @@ import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHisto import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentHistoryAssembler; import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentLlmBatchPipeline; import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentPayloadSupport; +import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentParsePipeline; import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentProgressSupport; import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentResultRowReader; import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo; @@ -196,114 +197,23 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline sourceFiles = request.getFiles().stream() - .filter(Objects::nonNull) - .toList(); - if (sourceFiles.isEmpty()) { - throw new BusinessException("请先上传 Excel 文件"); - } - List allRows = new ArrayList<>(); - List mergedHeaders = new ArrayList<>(); - int totalRows = 0; - int droppedRows = 0; - long parseStartedAt = System.nanoTime(); - for (AppearancePatentSourceFileDto source : sourceFiles) { - if (source.getFileKey() == null || source.getFileKey().isBlank()) { - throw new BusinessException("fileKey 不能为空"); - } - File input = localFileStorageService.findLocalSourceFile(source.getFileKey()); - if (input == null || !input.exists()) { - throw new BusinessException("源文件不存在"); - } + return parsePipeline().parseAndCreateTask(request, deviceId); + } - ParsedWorkbook parsed = parseWorkbook(input, source); - int maxParseRows = Math.max(1, properties.getMaxParseRows()); - if ((long) allRows.size() + parsed.allRows().size() > maxParseRows) { - throw new BusinessException("解析总行数超过上限: " + maxParseRows); - } - totalRows += parsed.totalRows(); - droppedRows += parsed.droppedRows(); - allRows.addAll(parsed.allRows()); - mergeHeaders(mergedHeaders, parsed.headers()); - } - if (allRows.isEmpty()) { - throw new BusinessException("未解析到有效 ASIN 数据"); - } - long parsedAt = System.nanoTime(); - int groupCount = countParsedGroups(allRows); - long groupedAt = System.nanoTime(); - - FileTaskEntity task = newTaskHeader(STATUS_PENDING, "PYTHON_QUEUE", request.getUserId(), deviceId, - sourceFiles.size()); - task.setSuccessFileCount(0); - task.setFailedFileCount(0); - task.setRequestJson(writeJson(objectMapper, request, "序列化解析结果失败")); - task.setResultJson("{}"); - insertTask(task); - long taskInsertedAt = System.nanoTime(); - - String aggregateScopeKey = buildAggregateScopeKey(sourceFiles); - String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey); - String parsedPayload = payloadSupport().buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, mergedHeaders, allRows); - long payloadBuiltAt = System.nanoTime(); - String parsedPayloadPointer = payloadSupport().storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload); - long payloadStoredAt = System.nanoTime(); - task.setResultJson(payloadSupport().buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, parsedPayloadPointer)); - task.setUpdatedAt(LocalDateTime.now()); - fileTaskMapper.updateById(task); - - String sourceFilename = buildAggregateSourceFilenameLabel(sourceFiles); - // rowCount 记接受行数、success 留 NULL(本支建单不写占位值),这两列底座刻意不替它定 - FileResultEntity result = newScopeRow(task.getId(), request.getUserId(), sourceFilename, - aggregateScopeKey, allRows.size(), null); - fileResultMapper.insert(result); - - TaskScopeStateEntity scope = new TaskScopeStateEntity(); - scope.setTaskId(task.getId()); - scope.setModuleType(MODULE_TYPE); - scope.setScopeKey(aggregateScopeKey); - scope.setScopeHash(sourceScopeHash); - scope.setParsedPayloadJson(writeJson(objectMapper, parsedPayloadPointer, "保存解析载荷指针失败")); - scope.setStateJson("{\"phase\":\"PARSED\"}"); - scope.setChunkTotal(0); - scope.setReceivedChunkCount(0); - scope.setCompleted(0); - scope.setCreatedAt(LocalDateTime.now()); - scope.setUpdatedAt(LocalDateTime.now()); - taskScopeStateMapper.insert(scope); - long persistedAt = System.nanoTime(); - - AppearancePatentParseVo vo = new AppearancePatentParseVo(); - vo.setTaskId(task.getId()); - vo.setSourceFilename(sourceFilename); - vo.setSourceFileCount(sourceFiles.size()); - vo.setTotalRows(totalRows); - vo.setAcceptedRows(allRows.size()); - vo.setDroppedRows(droppedRows); - vo.setGroupCount(groupCount); - vo.setAiPrompt(normalize(request.getAiPrompt())); - vo.setItems(List.of()); - vo.setGroups(List.of()); - long finishedAt = System.nanoTime(); - log.info("[appearance-patent] parse timing taskId={} files={} rows={} groups={} totalMs={} parseMs={} groupMs={} taskInsertMs={} payloadJsonMs={} payloadStoreMs={} persistMs={} responseMs={}", - task.getId(), - sourceFiles.size(), - allRows.size(), - groupCount, - elapsedMs(startedAt, finishedAt), - elapsedMs(parseStartedAt, parsedAt), - elapsedMs(parsedAt, groupedAt), - elapsedMs(groupedAt, taskInsertedAt), - elapsedMs(taskInsertedAt, payloadBuiltAt), - elapsedMs(payloadBuiltAt, payloadStoredAt), - elapsedMs(payloadStoredAt, persistedAt), - elapsedMs(persistedAt, finishedAt)); - return vo; + /** + * 建单段(parse 聚类)住在 {@link AppearancePatentParsePipeline}(3A 拆薄第十二笔): + * 解析、建单四段落库、表头/键件与两个聚合命名件逐字搬移;底座四件、序列化出口、 + * 归一/分组/组键仍由本类以回调交出。 + */ + private AppearancePatentParsePipeline parsePipeline() { + return new AppearancePatentParsePipeline( + MODULE_TYPE, localFileStorageService, properties, + fileTaskMapper, fileResultMapper, taskScopeStateMapper, + this::requireUserId, this::newTaskHeader, this::insertTask, this::newScopeRow, + (value, message) -> writeJson(objectMapper, value, message), + this::payloadSupport, this::countParsedGroups, + this::normalize, this::firstNonBlank, this::baseId, this::buildGroupKey, + this::groupRowsByBaseId); } @Transactional @@ -1150,148 +1060,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline headers = sheet.headers(); - int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers); - boolean includeBlankStatusRows = statusCol >= 0 && isAppearanceResultWorkbook(headers); - Map headerIndex = new LinkedHashMap<>(); - for (int i = 0; i < headers.size(); i++) { - headerIndex.putIfAbsent(headers.get(i), i); - } - - List parsedRows = new ArrayList<>(); - int total = 0; - int dropped = 0; - int validRows = 0; - String currentBlockBaseId = ""; - String currentGroupKey = ""; - for (AppearancePatentExcelParser.AppearanceExcelRow parsed : sheet.rows()) { - String id = parsed.id() == null ? "" : parsed.id(); - String asin = parsed.asin() == null ? "" : parsed.asin().toUpperCase(Locale.ROOT); - String country = parsed.country() == null ? "" : parsed.country(); - if (id.isBlank() && asin.isBlank() && country.isBlank()) { - continue; - } - total++; - if (id.isBlank() || asin.isBlank() || country.isBlank()) { - dropped++; - continue; - } - validRows++; - if (validRows > maxParseRows) { - throw new BusinessException("解析行数超过上限: " + maxParseRows); - } - AppearancePatentParsedRowVo vo = new AppearancePatentParsedRowVo(); - vo.setSourceFileKey(source.getFileKey()); - vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName())); - vo.setRowIndex(parsed.rowIndex()); - vo.setSourceId(id); - vo.setDisplayId(normalizeDisplayId(id)); - String rowBaseId = baseId(vo.getDisplayId()); - if (!Objects.equals(currentBlockBaseId, rowBaseId)) { - currentBlockBaseId = rowBaseId; - currentGroupKey = buildGroupKey(source.getFileKey(), rowBaseId, vo.getRowIndex()); - } - vo.setGroupKey(currentGroupKey); - vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex())); - vo.setAsin(asin); - vo.setCountry(country); - vo.setPrice(parsed.price() == null ? "" : parsed.price()); - vo.setSku(parsed.sku() == null ? "" : parsed.sku()); - vo.setUrl(parsed.url() == null ? "" : parsed.url()); - vo.setTitle(parsed.title() == null ? "" : parsed.title()); - vo.setValues(parsed.values() == null ? new LinkedHashMap<>() : new LinkedHashMap<>(parsed.values())); - String statusValue = statusCol >= 0 && parsed.values() != null - ? parsed.values().getOrDefault(statusHeaderName(headers, statusCol), "") - : ""; - parsedRows.add(new ParsedAppearanceRow(vo, statusValue == null ? "" : statusValue)); - } - List allRows = parsedRows.stream() - .map(ParsedAppearanceRow::row) - .toList(); - hydratePromptFields(allRows); - FailedStatusRowFilter.FilterResult filteredRows = - FailedStatusRowFilter.retainRows( - parsedRows, - statusCol >= 0, - ParsedAppearanceRow::sourceStatus, - status -> FailedStatusRowFilter.matchesFailedStatus(status) - || (includeBlankStatusRows && FailedStatusRowFilter.isBlankStatus(status)) - ); - dropped += filteredRows.filteredCount(); - allRows = filteredRows.rows().stream() - .map(ParsedAppearanceRow::row) - .toList(); - if (statusCol >= 0 && validRows > 0 && allRows.isEmpty()) { - throw new BusinessException(FailedStatusRowFilter.noMatchedRowsMessage()); - } - if (allRows.isEmpty()) { - throw new BusinessException("没有解析到可检测的外观专利行,请检查表格内容"); - } - return new ParsedWorkbook(total, dropped, new ArrayList<>(headers), allRows); - } - - /** 状态列的原始表头名(流式行只带"表头→值"映射,状态值需按表头名取回)。 */ - private static String statusHeaderName(List headers, int statusCol) { - return statusCol >= 0 && statusCol < headers.size() ? headers.get(statusCol) : ""; - } - - private void hydratePromptFields(List rows) { - if (rows == null || rows.isEmpty()) { - return; - } - Map> rowsByBaseId = groupRowsByBaseId(rows); - for (List siblings : rowsByBaseId.values()) { - String title = ""; - String url = ""; - String sku = ""; - for (AppearancePatentParsedRowVo sibling : siblings) { - title = firstNonBlank(title, sibling.getTitle()); - url = firstNonBlank(url, sibling.getUrl()); - sku = firstNonBlank(sku, sibling.getSku()); - } - for (AppearancePatentParsedRowVo sibling : siblings) { - if (normalize(sibling.getTitle()).isBlank()) { - sibling.setTitle(title); - } - if (normalize(sibling.getUrl()).isBlank()) { - sibling.setUrl(url); - } - if (normalize(sibling.getSku()).isBlank()) { - sibling.setSku(sku); - } - } - } - } - - private boolean isAppearanceResultWorkbook(List headers) { - return hasHeader(headers, "状态") - && (hasHeader(headers, "结论") - || hasHeader(headers, "标题维度(商标)") - || hasHeader(headers, "外观维度(外观设计专利)") - || hasHeader(headers, "专利维度(发明/实用新型专利)")); - } - - private boolean hasHeader(List headers, String candidate) { - if (headers == null || headers.isEmpty()) { - return false; - } - // 复用 FailedStatusRowFilter.canonicalizeHeader,统一归一化规则, - // 避免专利模块自己维护一份只去半角括号的实现导致全角括号表头识别失败。 - String normalizedCandidate = FailedStatusRowFilter.canonicalizeHeader(candidate); - if (normalizedCandidate.isBlank()) { - return false; - } - for (String header : headers) { - if (normalizedCandidate.equals(FailedStatusRowFilter.canonicalizeHeader(header))) { - return true; - } - } - return false; - } private boolean shouldInspectRow(AppearancePatentResultRowDto row) { return row != null && (!normalize(row.getTitle()).isBlank() || !normalize(row.getUrl()).isBlank()); @@ -1303,58 +1072,15 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline 0 ? s.substring(0, idx) : s; } - private String normalizeDisplayId(String id) { - return id == null ? "" : id.trim(); - } - - private void mergeHeaders(List mergedHeaders, List headers) { - if (mergedHeaders == null || headers == null || headers.isEmpty()) { - return; - } - Set existing = new LinkedHashSet<>(mergedHeaders); - for (String header : headers) { - if (existing.add(header)) { - mergedHeaders.add(header); - } - } - } private String buildAggregateScopeKey(List sourceFiles) { - List fileKeys = sourceFiles == null ? List.of() : sourceFiles.stream() - .map(AppearancePatentSourceFileDto::getFileKey) - .filter(Objects::nonNull) - .map(String::trim) - .filter(value -> !value.isBlank()) - .toList(); - if (fileKeys.isEmpty()) { - return ""; - } - if (fileKeys.size() == 1) { - return fileKeys.get(0); - } - return "aggregate:" + DigestUtil.sha256Hex(String.join("|", fileKeys)); + return parsePipeline().buildAggregateScopeKey(sourceFiles); } private String buildAggregateSourceFilenameLabel(List sourceFiles) { - if (sourceFiles == null || sourceFiles.isEmpty()) { - return "appearance-patent"; - } - String firstFilename = sourceFiles.stream() - .map(AppearancePatentSourceFileDto::getOriginalFilename) - .filter(Objects::nonNull) - .map(String::trim) - .filter(value -> !value.isBlank()) - .findFirst() - .orElse("appearance-patent"); - if (sourceFiles.size() == 1) { - return firstFilename; - } - return firstFilename + " 等 " + sourceFiles.size() + " 个文件"; + return parsePipeline().buildAggregateSourceFilenameLabel(sourceFiles); } - private String buildRowToken(String sourceFileKey, Integer rowIndex) { - return normalize(sourceFileKey) + "::row::" + (rowIndex == null ? 0 : rowIndex); - } private String buildGroupKey(String sourceFileKey, String rowBaseId, Integer rowIndex) { return normalize(sourceFileKey) + "::" + normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex); @@ -1372,9 +1098,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline headers, List allRows) { - } - - private record ParsedAppearanceRow(AppearancePatentParsedRowVo row, String sourceStatus) { - } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentParsePipeline.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentParsePipeline.java new file mode 100644 index 000000000..77db29489 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentParsePipeline.java @@ -0,0 +1,471 @@ +package com.nanri.aiimage.modules.appearancepatent.service.support; + +import cn.hutool.crypto.digest.DigestUtil; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.util.FailedStatusRowFilter; +import com.nanri.aiimage.config.AppearancePatentProperties; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSourceFileDto; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParseVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo; +import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.common.task.TaskStatuses; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * 外观专利**建单段(parse)聚类**——3A 拆薄第十二笔。 + * + *

搬过来的是 {@code parseAndCreateTask} 本体(建单四段:解析 → 头行 → 载荷/索引 → 结果行/作用域行)、 + * {@code parseWorkbook}(Excel 行级组装、状态列过滤、组内字段补齐)、以及只被这两段用的表头/键件 + * ({@code statusHeaderName}/{@code hydratePromptFields}/{@code isAppearanceResultWorkbook}/ + * {@code hasHeader}/{@code normalizeDisplayId}/{@code mergeHeaders}/{@code buildRowToken}) + * 与两个聚合命名件({@code buildAggregateScopeKey}/{@code buildAggregateSourceFilenameLabel})。 + * + *

为什么这些能一次性搬、又为什么剩下的不能顺手搬: + *

    + *
  • {@code baseId}/{@code buildGroupKey}/{@code normalize}/{@code firstNonBlank} 在服务侧 + * 还被 {@code buildParsedGroups}/{@code groupRowsByBaseId}/{@code shouldInspectRow} 等 + * 非 parse 面共用 ⇒ 本件只接回调,不搬实现(不复制第二份,也不把它们从服务侧抽走)。
  • + *
  • 底座四件({@code requireUserId}/{@code newTaskHeader}/{@code insertTask}/ + * {@code newScopeRow})与序列化出口 {@code writeJson} 是 {@code protected} 的宿主能力, + * 经回调交入;本件不继承底座、也不持有 mapper 之外的底座状态。
  • + *
  • 载荷三段(组装/落库/索引 JSON)仍归 {@link AppearancePatentPayloadSupport}(I 聚类), + * 本件只经 {@link Supplier} 取用既有实例。
  • + *
+ * + *

可观测差异(如实登记):parse 分段计时那行日志的 logger 名从 + * {@code ...service.AppearancePatentTaskService} 变为 {@code ...service.support.AppearancePatentParsePipeline}, + * 消息文本、级别、字段顺序一字未动;按 logger 名做日志检索的人要跟着换。 + * + *

事务语义一字未动:本支建单全程无事务(载荷存储失败时头行已 insert), + * 与 {@code AppearancePatentCreateTaskSnapshotTest} 的 golden 一致;本件不引入任何事务包装。 + */ +public final class AppearancePatentParsePipeline { + + private static final Logger log = LoggerFactory.getLogger(AppearancePatentParsePipeline.class); + + @FunctionalInterface + public interface HeaderFactory { + FileTaskEntity create(String status, String taskMode, Long userId, String deviceId, int sourceFileCount); + } + + @FunctionalInterface + public interface ScopeRowFactory { + FileResultEntity create(Long taskId, Long userId, String sourceFilename, String sourceFileUrl, + Integer rowCount, Integer success); + } + + @FunctionalInterface + public interface JsonWriter { + String write(Object value, String failureMessage); + } + + @FunctionalInterface + public interface RowCounter { + int count(List rows); + } + + private final String moduleType; + private final LocalFileStorageService localFileStorageService; + private final AppearancePatentProperties properties; + private final FileTaskMapper fileTaskMapper; + private final FileResultMapper fileResultMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final Consumer requireUserId; + private final HeaderFactory headerFactory; + private final Function insertTask; + private final ScopeRowFactory scopeRowFactory; + private final JsonWriter jsonWriter; + private final Supplier payloadSupport; + private final RowCounter rowCounter; + private final Function normalize; + private final java.util.function.BiFunction firstNonBlank; + private final Function baseId; + private final GroupKeyBuilder groupKeyBuilder; + /** 组内归并仍住服务侧({@code buildParsedGroups}/{@code countParsedGroups} 共用同一份),本件只接回调。 */ + private final Function, Map>> groupRowsByBaseId; + + @FunctionalInterface + public interface GroupKeyBuilder { + String build(String sourceFileKey, String rowBaseId, Integer rowIndex); + } + + public AppearancePatentParsePipeline(String moduleType, + LocalFileStorageService localFileStorageService, + AppearancePatentProperties properties, + FileTaskMapper fileTaskMapper, + FileResultMapper fileResultMapper, + TaskScopeStateMapper taskScopeStateMapper, + Consumer requireUserId, + HeaderFactory headerFactory, + Function insertTask, + ScopeRowFactory scopeRowFactory, + JsonWriter jsonWriter, + Supplier payloadSupport, + RowCounter rowCounter, + Function normalize, + java.util.function.BiFunction firstNonBlank, + Function baseId, + GroupKeyBuilder groupKeyBuilder, + Function, Map>> groupRowsByBaseId) { + this.moduleType = moduleType; + this.localFileStorageService = localFileStorageService; + this.properties = properties; + this.fileTaskMapper = fileTaskMapper; + this.fileResultMapper = fileResultMapper; + this.taskScopeStateMapper = taskScopeStateMapper; + this.requireUserId = requireUserId; + this.headerFactory = headerFactory; + this.insertTask = insertTask; + this.scopeRowFactory = scopeRowFactory; + this.jsonWriter = jsonWriter; + this.payloadSupport = payloadSupport; + this.rowCounter = rowCounter; + this.normalize = normalize; + this.firstNonBlank = firstNonBlank; + this.baseId = baseId; + this.groupKeyBuilder = groupKeyBuilder; + this.groupRowsByBaseId = groupRowsByBaseId; + } + + public AppearancePatentParseVo parseAndCreateTask(AppearancePatentParseRequest request, String deviceId) { + long startedAt = System.nanoTime(); + requireUserId.accept(request.getUserId()); + if (request.getFiles() == null || request.getFiles().isEmpty()) { + throw new BusinessException("请先上传 Excel 文件"); + } + List sourceFiles = request.getFiles().stream() + .filter(Objects::nonNull) + .toList(); + if (sourceFiles.isEmpty()) { + throw new BusinessException("请先上传 Excel 文件"); + } + List allRows = new ArrayList<>(); + List mergedHeaders = new ArrayList<>(); + int totalRows = 0; + int droppedRows = 0; + long parseStartedAt = System.nanoTime(); + for (AppearancePatentSourceFileDto source : sourceFiles) { + if (source.getFileKey() == null || source.getFileKey().isBlank()) { + throw new BusinessException("fileKey 不能为空"); + } + File input = localFileStorageService.findLocalSourceFile(source.getFileKey()); + if (input == null || !input.exists()) { + throw new BusinessException("源文件不存在"); + } + + ParsedWorkbook parsed = parseWorkbook(input, source); + int maxParseRows = Math.max(1, properties.getMaxParseRows()); + if ((long) allRows.size() + parsed.allRows().size() > maxParseRows) { + throw new BusinessException("解析总行数超过上限: " + maxParseRows); + } + totalRows += parsed.totalRows(); + droppedRows += parsed.droppedRows(); + allRows.addAll(parsed.allRows()); + mergeHeaders(mergedHeaders, parsed.headers()); + } + if (allRows.isEmpty()) { + throw new BusinessException("未解析到有效 ASIN 数据"); + } + long parsedAt = System.nanoTime(); + int groupCount = rowCounter.count(allRows); + long groupedAt = System.nanoTime(); + + FileTaskEntity task = headerFactory.create(TaskStatuses.FileTask.PENDING, "PYTHON_QUEUE", request.getUserId(), deviceId, + sourceFiles.size()); + task.setSuccessFileCount(0); + task.setFailedFileCount(0); + task.setRequestJson(jsonWriter.write(request, "序列化解析结果失败")); + task.setResultJson("{}"); + insertTask.apply(task); + long taskInsertedAt = System.nanoTime(); + + String aggregateScopeKey = buildAggregateScopeKey(sourceFiles); + String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey); + String parsedPayload = payloadSupport.get().buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, mergedHeaders, allRows); + long payloadBuiltAt = System.nanoTime(); + String parsedPayloadPointer = payloadSupport.get().storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload); + long payloadStoredAt = System.nanoTime(); + task.setResultJson(payloadSupport.get().buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, parsedPayloadPointer)); + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + + String sourceFilename = buildAggregateSourceFilenameLabel(sourceFiles); + // rowCount 记接受行数、success 留 NULL(本支建单不写占位值),这两列底座刻意不替它定 + FileResultEntity result = scopeRowFactory.create(task.getId(), request.getUserId(), sourceFilename, + aggregateScopeKey, allRows.size(), null); + fileResultMapper.insert(result); + + TaskScopeStateEntity scope = new TaskScopeStateEntity(); + scope.setTaskId(task.getId()); + scope.setModuleType(moduleType); + scope.setScopeKey(aggregateScopeKey); + scope.setScopeHash(sourceScopeHash); + scope.setParsedPayloadJson(jsonWriter.write(parsedPayloadPointer, "保存解析载荷指针失败")); + scope.setStateJson("{\"phase\":\"PARSED\"}"); + scope.setChunkTotal(0); + scope.setReceivedChunkCount(0); + scope.setCompleted(0); + scope.setCreatedAt(LocalDateTime.now()); + scope.setUpdatedAt(LocalDateTime.now()); + taskScopeStateMapper.insert(scope); + long persistedAt = System.nanoTime(); + + AppearancePatentParseVo vo = new AppearancePatentParseVo(); + vo.setTaskId(task.getId()); + vo.setSourceFilename(sourceFilename); + vo.setSourceFileCount(sourceFiles.size()); + vo.setTotalRows(totalRows); + vo.setAcceptedRows(allRows.size()); + vo.setDroppedRows(droppedRows); + vo.setGroupCount(groupCount); + vo.setAiPrompt(normalize.apply(request.getAiPrompt())); + vo.setItems(List.of()); + vo.setGroups(List.of()); + long finishedAt = System.nanoTime(); + log.info("[appearance-patent] parse timing taskId={} files={} rows={} groups={} totalMs={} parseMs={} groupMs={} taskInsertMs={} payloadJsonMs={} payloadStoreMs={} persistMs={} responseMs={}", + task.getId(), + sourceFiles.size(), + allRows.size(), + groupCount, + elapsedMs(startedAt, finishedAt), + elapsedMs(parseStartedAt, parsedAt), + elapsedMs(parsedAt, groupedAt), + elapsedMs(groupedAt, taskInsertedAt), + elapsedMs(taskInsertedAt, payloadBuiltAt), + elapsedMs(payloadBuiltAt, payloadStoredAt), + elapsedMs(payloadStoredAt, persistedAt), + elapsedMs(persistedAt, finishedAt)); + return vo; + } + + private ParsedWorkbook parseWorkbook(File input, AppearancePatentSourceFileDto source) { + int maxParseRows = Math.max(1, properties.getMaxParseRows()); + AppearancePatentExcelParser.ParsedSheet sheet = new AppearancePatentExcelParser().parse(input, maxParseRows); + List headers = sheet.headers(); + int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers); + boolean includeBlankStatusRows = statusCol >= 0 && isAppearanceResultWorkbook(headers); + + Map headerIndex = new LinkedHashMap<>(); + for (int i = 0; i < headers.size(); i++) { + headerIndex.putIfAbsent(headers.get(i), i); + } + + List parsedRows = new ArrayList<>(); + int total = 0; + int dropped = 0; + int validRows = 0; + String currentBlockBaseId = ""; + String currentGroupKey = ""; + for (AppearancePatentExcelParser.AppearanceExcelRow parsed : sheet.rows()) { + String id = parsed.id() == null ? "" : parsed.id(); + String asin = parsed.asin() == null ? "" : parsed.asin().toUpperCase(Locale.ROOT); + String country = parsed.country() == null ? "" : parsed.country(); + if (id.isBlank() && asin.isBlank() && country.isBlank()) { + continue; + } + total++; + if (id.isBlank() || asin.isBlank() || country.isBlank()) { + dropped++; + continue; + } + validRows++; + if (validRows > maxParseRows) { + throw new BusinessException("解析行数超过上限: " + maxParseRows); + } + AppearancePatentParsedRowVo vo = new AppearancePatentParsedRowVo(); + vo.setSourceFileKey(source.getFileKey()); + vo.setSourceFilename(firstNonBlank.apply(source.getOriginalFilename(), input.getName())); + vo.setRowIndex(parsed.rowIndex()); + vo.setSourceId(id); + vo.setDisplayId(normalizeDisplayId(id)); + String rowBaseId = baseId.apply(vo.getDisplayId()); + if (!Objects.equals(currentBlockBaseId, rowBaseId)) { + currentBlockBaseId = rowBaseId; + currentGroupKey = groupKeyBuilder.build(source.getFileKey(), rowBaseId, vo.getRowIndex()); + } + vo.setGroupKey(currentGroupKey); + vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex())); + vo.setAsin(asin); + vo.setCountry(country); + vo.setPrice(parsed.price() == null ? "" : parsed.price()); + vo.setSku(parsed.sku() == null ? "" : parsed.sku()); + vo.setUrl(parsed.url() == null ? "" : parsed.url()); + vo.setTitle(parsed.title() == null ? "" : parsed.title()); + vo.setValues(parsed.values() == null ? new LinkedHashMap<>() : new LinkedHashMap<>(parsed.values())); + String statusValue = statusCol >= 0 && parsed.values() != null + ? parsed.values().getOrDefault(statusHeaderName(headers, statusCol), "") + : ""; + parsedRows.add(new ParsedAppearanceRow(vo, statusValue == null ? "" : statusValue)); + } + List allRows = parsedRows.stream() + .map(ParsedAppearanceRow::row) + .toList(); + hydratePromptFields(allRows); + FailedStatusRowFilter.FilterResult filteredRows = + FailedStatusRowFilter.retainRows( + parsedRows, + statusCol >= 0, + ParsedAppearanceRow::sourceStatus, + status -> FailedStatusRowFilter.matchesFailedStatus(status) + || (includeBlankStatusRows && FailedStatusRowFilter.isBlankStatus(status)) + ); + dropped += filteredRows.filteredCount(); + allRows = filteredRows.rows().stream() + .map(ParsedAppearanceRow::row) + .toList(); + if (statusCol >= 0 && validRows > 0 && allRows.isEmpty()) { + throw new BusinessException(FailedStatusRowFilter.noMatchedRowsMessage()); + } + if (allRows.isEmpty()) { + throw new BusinessException("没有解析到可检测的外观专利行,请检查表格内容"); + } + return new ParsedWorkbook(total, dropped, new ArrayList<>(headers), allRows); + } + + /** 状态列的原始表头名(流式行只带"表头→值"映射,状态值需按表头名取回)。 */ + private static String statusHeaderName(List headers, int statusCol) { + return statusCol >= 0 && statusCol < headers.size() ? headers.get(statusCol) : ""; + } + + private void hydratePromptFields(List rows) { + if (rows == null || rows.isEmpty()) { + return; + } + Map> rowsByBaseId = groupRowsByBaseId.apply(rows); + for (List siblings : rowsByBaseId.values()) { + String title = ""; + String url = ""; + String sku = ""; + for (AppearancePatentParsedRowVo sibling : siblings) { + title = firstNonBlank.apply(title, sibling.getTitle()); + url = firstNonBlank.apply(url, sibling.getUrl()); + sku = firstNonBlank.apply(sku, sibling.getSku()); + } + for (AppearancePatentParsedRowVo sibling : siblings) { + if (normalize.apply(sibling.getTitle()).isBlank()) { + sibling.setTitle(title); + } + if (normalize.apply(sibling.getUrl()).isBlank()) { + sibling.setUrl(url); + } + if (normalize.apply(sibling.getSku()).isBlank()) { + sibling.setSku(sku); + } + } + } + } + + private boolean isAppearanceResultWorkbook(List headers) { + return hasHeader(headers, "状态") + && (hasHeader(headers, "结论") + || hasHeader(headers, "标题维度(商标)") + || hasHeader(headers, "外观维度(外观设计专利)") + || hasHeader(headers, "专利维度(发明/实用新型专利)")); + } + + private boolean hasHeader(List headers, String candidate) { + if (headers == null || headers.isEmpty()) { + return false; + } + // 复用 FailedStatusRowFilter.canonicalizeHeader,统一归一化规则, + // 避免专利模块自己维护一份只去半角括号的实现导致全角括号表头识别失败。 + String normalizedCandidate = FailedStatusRowFilter.canonicalizeHeader(candidate); + if (normalizedCandidate.isBlank()) { + return false; + } + for (String header : headers) { + if (normalizedCandidate.equals(FailedStatusRowFilter.canonicalizeHeader(header))) { + return true; + } + } + return false; + } + + private String normalizeDisplayId(String id) { + return id == null ? "" : id.trim(); + } + + private void mergeHeaders(List mergedHeaders, List headers) { + if (mergedHeaders == null || headers == null || headers.isEmpty()) { + return; + } + Set existing = new LinkedHashSet<>(mergedHeaders); + for (String header : headers) { + if (existing.add(header)) { + mergedHeaders.add(header); + } + } + } + + /** 结果作业({@code AppearancePatentResultJobSupport})与建单共用,服务侧保留薄委托。 */ + public String buildAggregateScopeKey(List sourceFiles) { + List fileKeys = sourceFiles == null ? List.of() : sourceFiles.stream() + .map(AppearancePatentSourceFileDto::getFileKey) + .filter(Objects::nonNull) + .map(String::trim) + .filter(value -> !value.isBlank()) + .toList(); + if (fileKeys.isEmpty()) { + return ""; + } + if (fileKeys.size() == 1) { + return fileKeys.get(0); + } + return "aggregate:" + DigestUtil.sha256Hex(String.join("|", fileKeys)); + } + + /** 结果作业与建单共用,服务侧保留薄委托。 */ + public String buildAggregateSourceFilenameLabel(List sourceFiles) { + if (sourceFiles == null || sourceFiles.isEmpty()) { + return "appearance-patent"; + } + String firstFilename = sourceFiles.stream() + .map(AppearancePatentSourceFileDto::getOriginalFilename) + .filter(Objects::nonNull) + .map(String::trim) + .filter(value -> !value.isBlank()) + .findFirst() + .orElse("appearance-patent"); + if (sourceFiles.size() == 1) { + return firstFilename; + } + return firstFilename + " 等 " + sourceFiles.size() + " 个文件"; + } + + private String buildRowToken(String sourceFileKey, Integer rowIndex) { + return normalize.apply(sourceFileKey) + "::row::" + (rowIndex == null ? 0 : rowIndex); + } + + private static long elapsedMs(long start, long end) { + return (end - start) / 1_000_000L; + } + + private record ParsedWorkbook(int totalRows, int droppedRows, List headers, + List allRows) { + } + + private record ParsedAppearanceRow(AppearancePatentParsedRowVo row, String sourceStatus) { + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java b/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java index 74d62a9d2..2fb9bcea3 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java @@ -266,13 +266,13 @@ class SourceScanTest { // ---------------------------------------------------------------- N6 moduleSources @Test - void moduleSources_appearancepatent返回38支含拆分后的support件() { + void moduleSources_appearancepatent返回39支含拆分后的support件() { List files = SourceScan.moduleSources("modules/appearancepatent"); assertThat(files) - .as("真实集成:对真仓目录求值。38 支=1 controller+1 client+1 service 主件+3 spi/cache+" - + "15 service/support+16 model(dto7/vo9),与 find 实测同号") - .hasSize(38); + .as("真实集成:对真仓目录求值。39 支=1 controller+1 client+1 service 主件+3 spi/cache+" + + "16 service/support+16 model(dto7/vo9),与 find 实测同号(3A 拆薄第十二笔再落 1 支)") + .hasSize(39); assertThat(files.stream().map(p -> p.toString().replace('\\', '/')).toList()) .as("support 目录里的既有件必须在面里——聚合入口存在的理由就是拆分后的新文件要自动被看到") .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentRowNormalizer.java")); @@ -373,11 +373,11 @@ class SourceScanTest { .toList(); assertThat(support) - .as("真实集成:对真仓 modules/appearancepatent/service/support 求值,实测 15 支(3A 拆薄把归属谓词、陈旧扫描、结果打包、LLM 批管线、载荷读写、结果行读取、进度视图、分片回传、交付闸门、结果作业抽成十枚 support 件,5→15)。" + .as("真实集成:对真仓 modules/appearancepatent/service/support 求值,实测 16 支(3A 拆薄把归属谓词、陈旧扫描、结果打包、LLM 批管线、载荷读写、结果行读取、进度视图、分片回传、交付闸门、结果作业、建单解析抽成十一枚 support 件,5→16)。" + "这一格就是聚合入口存在的理由——拆分出去的新文件必须自动进面") - .hasSize(15); + .hasSize(16); assertThat(support) - .as("3A 拆出去的十枚新件必须被逐枚点名扫到:只把分母数字改大,下一回拆件又只剩「数不对」" + .as("3A 拆出去的十一枚新件必须被逐枚点名扫到:只把分母数字改大,下一回拆件又只剩「数不对」" + "一记红,证不了「扫得到」这件事") .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentTaskOwnership.java")) .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentStaleTaskScanner.java")) @@ -388,7 +388,8 @@ class SourceScanTest { .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentProgressSupport.java")) .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentChunkSubmissionSupport.java")) .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentDeliverySupport.java")) - .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentResultJobSupport.java")); + .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentResultJobSupport.java")) + .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentParsePipeline.java")); } @Test