diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index 3a62681d9..29be2e1dd 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -25,6 +25,8 @@ import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashb import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo; import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentHistoryAssembler; import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentLlmBatchPipeline; +import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentPayloadSupport; +import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentResultRowReader; import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo; import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo; import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo; @@ -107,11 +109,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline rawRows = flattenSubmittedRows(request); + List rawRows = rowReader().flattenSubmittedRows(request); String payloadJson = writeJson(objectMapper, rawRows, "结果序列化失败"); TaskChunkEntity chunk = new TaskChunkEntity(); @@ -547,7 +544,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline payloads = collectTransientTaskPayloads(taskId); + List payloads = payloadSupport().collectTransientTaskPayloads(taskId); fileResultMapper.delete(new LambdaQueryWrapper().eq(FileResultEntity::getTaskId, taskId).eq(FileResultEntity::getModuleType, MODULE_TYPE)); taskScopeStateMapper.delete(new LambdaQueryWrapper().eq(TaskScopeStateEntity::getTaskId, taskId).eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); taskChunkMapper.delete(new LambdaQueryWrapper().eq(TaskChunkEntity::getTaskId, taskId).eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); @@ -621,7 +618,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline taskCacheService.deleteTaskCache(taskId)); } @@ -706,9 +703,9 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline rawRows = flattenSubmittedRows(request); + List rawRows = rowReader().flattenSubmittedRows(request); String payloadJson = writeJson(objectMapper, rawRows, "结果序列化失败"); - String storedPayload = storeSharedChunkPayload(taskId, scopeHash, chunkIndex, payloadJson); + String storedPayload = payloadSupport().storeSharedChunkPayload(taskId, scopeHash, chunkIndex, payloadJson); return new PreparedSubmittedChunk(task, scopeKey, scopeHash, chunkIndex, chunkTotal, done, request.getError(), payloadJson, storedPayload, DigestUtil.sha256Hex(payloadJson)); } @@ -775,7 +772,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline writeJson(objectMapper, value, message)); + } + + /** + * H 聚类(结果行读取与行键)住在 {@link AppearancePatentResultRowReader}(3A 拆薄第七笔): + * 回传分组展开、解析行计数、分片载荷读取、按键分页读取逐字搬移;解析载荷读取与归一再由本类回调交出。 + */ + private AppearancePatentResultRowReader rowReader() { + return new AppearancePatentResultRowReader( + MODULE_TYPE, objectMapper, transientPayloadStorageService, taskChunkMapper, + this::normalize, payloadSupport()::readParsedPayload); + } + private void submitLlmForSubmittedChunk(SubmitContext context) { if (context == null || context.task() == null || context.task().getId() == null) { return; @@ -848,7 +869,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline { - finalizeTask(task, error, allRowCount(task), true); + finalizeTask(task, error, rowReader().allRowCount(task), true); return null; }); } @@ -933,7 +954,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline persistedRows = readChunkRows(chunk); + Map persistedRows = rowReader().readChunkRows(chunk); for (AppearancePatentResultRowDto row : rows) { - String key = rowKey(row); + String key = rowReader().rowKey(row); AppearancePatentResultRowDto submittedRow = persistedRows.get(key); if (submittedRow != null) { row.setPrice(submittedRow.getPrice()); @@ -1121,7 +1142,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline updateWrapper = new LambdaUpdateWrapper() .eq(TaskChunkEntity::getId, chunk.getId()) .set(TaskChunkEntity::getPayloadJson, storedPayload) @@ -1137,7 +1158,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline !key.isBlank()).toList(); + return rows.stream().map(rowReader()::rowKey).filter(key -> !key.isBlank()).toList(); } // CAS 冲突:读回行上的当前哈希,否则只有一句 conflict 无从判断是谁改的(线上 28459 至今未能定位) String currentHash = currentPayloadHash(chunk.getId()); @@ -1206,64 +1227,10 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline flattenSubmittedRows(AppearancePatentSubmitResultRequest request) { - if (request == null) { - return List.of(); - } - List groups = request.getGroups(); - if (groups != null && !groups.isEmpty()) { - List rows = new ArrayList<>(); - for (AppearancePatentResultGroupDto group : groups) { - if (group == null || group.getItems() == null || group.getItems().isEmpty()) { - continue; - } - for (AppearancePatentResultRowDto row : group.getItems()) { - if (row == null) { - continue; - } - if (normalize(row.getGroupKey()).isBlank()) { - row.setGroupKey(group.getGroupKey()); - } - if (normalize(row.getSourceFileKey()).isBlank()) { - row.setSourceFileKey(group.getSourceFileKey()); - } - if (normalize(row.getSourceFilename()).isBlank()) { - row.setSourceFilename(group.getSourceFilename()); - } - rows.add(row); - } - } - return rows; - } - return request.getItems() == null ? List.of() : request.getItems(); - } - - private int allRowCount(FileTaskEntity task) { - try { - AppearancePatentParsedPayloadDto payload = readParsedPayload(task); - if (payload.getAllItems() != null && !payload.getAllItems().isEmpty()) { - return payload.getAllItems().size(); - } - if (payload.getItems() != null && !payload.getItems().isEmpty()) { - return payload.getItems().size(); - } - if (payload.getGroups() != null && !payload.getGroups().isEmpty()) { - return payload.getGroups().stream() - .map(AppearancePatentParsedGroupVo::getItems) - .filter(Objects::nonNull) - .mapToInt(List::size) - .sum(); - } - return 0; - } catch (Exception ex) { - log.warn("[appearance-patent] read all row count failed taskId={} err={}", task.getId(), ex.getMessage()); - return 0; - } - } private String readAiPrompt(FileTaskEntity task) { try { - return readParsedPayload(task).getAiPrompt(); + return payloadSupport().readParsedPayload(task).getAiPrompt(); } catch (Exception ignored) { return ""; } @@ -1272,7 +1239,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline rows = fileResultMapper.selectList(new LambdaQueryWrapper() .eq(FileResultEntity::getTaskId, task.getId()) .eq(FileResultEntity::getModuleType, MODULE_TYPE) .last("limit 1")); if (!rows.isEmpty()) { result = rows.getFirst(); - } else if (assembleWorkbook && hasPersistedResultRows) { + } else if (assembleWorkbook && persistedRowsPresent) { result = createResultRecordForAssembly(task, rowCount); } if (result != null) { - boolean shouldAssembleResult = assembleWorkbook && (finalError == null || finalError.isBlank() || hasPersistedResultRows); + boolean shouldAssembleResult = assembleWorkbook && (finalError == null || finalError.isBlank() || persistedRowsPresent); if (shouldAssembleResult && shouldAssembleSynchronously()) { try { deliveryFacts = assembleResultWorkbook(task, result); @@ -1328,7 +1295,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline 0 ? rowCount : result.getRowCount()); - boolean shouldAssembleResult = assembleWorkbook && (!failed || hasPersistedResultRows); + boolean shouldAssembleResult = assembleWorkbook && (!failed || persistedRowsPresent); if (shouldAssembleResult) { result.setResultFilename(safeFileStem(result.getSourceFilename()) + "-result.xlsx"); result.setResultFileUrl(null); @@ -1374,7 +1341,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline sourceFiles = payload.getSourceFiles() == null ? List.of() : payload.getSourceFiles(); FileResultEntity result = new FileResultEntity(); result.setTaskId(task.getId()); @@ -1498,16 +1465,16 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline parsedRows = parsed.getAllItems() == null ? List.of() : parsed.getAllItems(); AppearancePatentRowOutcome.Facts facts = AppearancePatentRowOutcome.Facts.EMPTY; for (AppearancePatentResultPackager.SourceRows item : splitRowsBySourceFile(parsed, parsedRows, result.getSourceFilename())) { Map resultMap = - loadPersistedResultRowsForRowsWithRetry(task.getId(), item.rows()); + rowReader().loadPersistedResultRowsForRowsWithRetry(task.getId(), item.rows()); facts = facts.plus(AppearancePatentRowOutcome.summarize( - item.rows(), row -> findResultRow(row, resultMap))); + item.rows(), row -> rowReader().findResultRow(row, resultMap))); } return facts; } @@ -1796,7 +1763,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline() .eq(TaskChunkEntity::getTaskId, job.getTaskId()) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); @@ -1817,7 +1784,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline parsedRows = parsed.getAllItems() == null ? List.of() : parsed.getAllItems(); List sourceRows = splitRowsBySourceFile(parsed, parsedRows, result.getSourceFilename()); @@ -1838,19 +1805,19 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline resultMap = - loadPersistedResultRowsForRowsWithRetry(task.getId(), item.rows()); + rowReader().loadPersistedResultRowsForRowsWithRetry(task.getId(), item.rows()); // 覆盖率先于「本文件无结果就跳过」判定:整源文件的 LLM 结果全丢(分片被跳过、 // 结果清空)时覆盖率是 0%,必须在这里判掉,不能让 zip 少一个文件还报任务成功。 deliveryFacts = deliveryFacts.plus( validateCompleteLlmCoverage(task.getId(), item.rows(), resultMap, allowPartial)); - List receivedRows = filterReceivedParsedRows(item.rows(), resultMap); + List receivedRows = rowReader().filterReceivedParsedRows(item.rows(), resultMap); if (receivedRows.isEmpty()) { resultMap.clear(); continue; } receivedRowCount += receivedRows.size(); resolvedRows += receivedRows.stream() - .filter(row -> findResultRow(row, resultMap) != null) + .filter(row -> rowReader().findResultRow(row, resultMap) != null) .count(); reasonRows += resultMap.values().stream() .filter(this::hasReasonFields) @@ -1858,7 +1825,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline findResultRow(row, resultMap), + row -> rowReader().findResultRow(row, resultMap), AppearancePatentResultRowDto::getConclusion, AppearancePatentResultRowDto::setConclusion, List.of("已侵权", "侵权"), @@ -1952,20 +1919,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline filterReceivedParsedRows(List parsedRows, - Map resultMap) { - if (parsedRows == null || parsedRows.isEmpty() || resultMap == null || resultMap.isEmpty()) { - return List.of(); - } - List receivedRows = new ArrayList<>(); - for (AppearancePatentParsedRowVo parsedRow : parsedRows) { - if (findResultRow(parsedRow, resultMap) != null) { - receivedRows.add(parsedRow); - } - } - return receivedRows; - } - /** * 覆盖率判定:分母必须是**真正会写进结果文件的那批行**,且"该行有没有数据"要同时看 * 用户上传表的解析行与客户端回传的结果行——只看解析行,用户表不带标题列时分母恒为 0, @@ -1986,7 +1939,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline findResultRow(row, resultMap)); + AppearancePatentRowOutcome.summarize(parsedRows, row -> rowReader().findResultRow(row, resultMap)); FileTaskEntity task = fileTaskMapper.selectById(taskId); boolean taskAlreadyFailed = task != null && STATUS_FAILED.equals(task.getStatus()); if (facts.missingRows() > 0 || facts.noDataRows() > 0) { @@ -2022,7 +1975,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline() - .eq(TaskChunkEntity::getTaskId, taskId) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); - return count != null && count > 0; - } - - private Map loadPersistedResultRowsForRowsWithRetry( - Long taskId, List requestedRows) { - Set requestedKeys = resultLookupKeysForParsedRows(requestedRows); - Map result = - loadPersistedResultRowsForKeys(taskId, requestedKeys); - if (requestedKeys.isEmpty() || !result.isEmpty()) { - return result; - } - for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) { - sleepBeforeResultRowsRetry(attempt); - result = loadPersistedResultRowsForKeys(taskId, requestedKeys); - if (!result.isEmpty()) { - log.info("[appearance-patent] result rows recovered after retry taskId={} attempt={} rows={}", - taskId, attempt, result.size()); - } - } - return result; - } - - - private Map loadPersistedResultRowsForKeys( - Long taskId, Set requestedKeys) { - Map result = new LinkedHashMap<>(); - if (taskId == null || requestedKeys == null || requestedKeys.isEmpty()) { - return result; - } - Long lastChunkId = null; - while (true) { - LambdaQueryWrapper query = new LambdaQueryWrapper() - .eq(TaskChunkEntity::getTaskId, taskId) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) - .orderByAsc(TaskChunkEntity::getId) - .last("LIMIT " + RESULT_ROWS_CHUNK_PAGE_SIZE); - if (lastChunkId != null) { - query.gt(TaskChunkEntity::getId, lastChunkId); - } - List page = taskChunkMapper.selectList(query); - if (page == null || page.isEmpty()) { - break; - } - // 整页取 chunk 后逐片解载荷:原先每轮 LIMIT 1 只取一行,一个结果文件要把该任务的 - // chunk 表逐行翻完,每行一次 SQL + 一次对象读 + 一次整片 JSON 反序列化 - for (TaskChunkEntity chunk : page) { - appendRequestedResultRows(result, readChunkRows(chunk).values(), requestedKeys); - } - TaskChunkEntity tail = page.getLast(); - lastChunkId = tail == null ? null : tail.getId(); - if (lastChunkId == null || page.size() < RESULT_ROWS_CHUNK_PAGE_SIZE) { - break; - } - } - return result; - } - - private Set resultLookupKeysForParsedRows(List rows) { - Set keys = new LinkedHashSet<>(); - for (AppearancePatentParsedRowVo row : rows == null ? List.of() : rows) { - String primary = rowKey(row); - if (!primary.isBlank()) { - keys.add(primary); - } - String legacy = legacyRowKey(row); - if (!legacy.isBlank()) { - keys.add(legacy); - } - } - return keys; - } - - private void appendRequestedResultRows(Map sink, - Iterable rows, - Set requestedKeys) { - if (rows == null || requestedKeys == null || requestedKeys.isEmpty()) { - return; - } - for (AppearancePatentResultRowDto row : rows) { - if (row == null) { - continue; - } - String primary = rowKey(row); - String legacy = legacyRowKey(row); - String matchedKey = requestedKeys.contains(primary) - ? primary : requestedKeys.contains(legacy) ? legacy : ""; - if (!matchedKey.isBlank()) { - sink.put(matchedKey, row); - } - } - } - - - private void sleepBeforeResultRowsRetry(int attempt) { - try { - Thread.sleep(RESULT_ROWS_READ_RETRY_DELAY_MS * attempt); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } - } private void writeResultWorkbook(File xlsx, AppearancePatentParsedPayloadDto parsed, @@ -2175,7 +2021,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline resultMap) { try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) { AppearancePatentSheetBuilder.buildResultSheet(workbook, receivedRows, resultMap, - parsedRow -> findResultRow(parsedRow, resultMap)); + parsedRow -> rowReader().findResultRow(parsedRow, resultMap)); workbook.write(fos); workbook.dispose(); } catch (Exception ex) { @@ -2470,226 +2316,16 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline sourceFiles, List headers, List allRows) { - AppearancePatentParsedPayloadDto payload = new AppearancePatentParsedPayloadDto(); - payload.setAiPrompt(normalize(aiPrompt)); - payload.setApiKey(normalize(apiKey)); - payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles); - payload.setHeaders(headers == null ? List.of() : headers); - payload.setItems(List.of()); - payload.setGroups(List.of()); - payload.setAllItems(allRows == null ? List.of() : allRows); - return writeJson(objectMapper, payload, "保存解析结果失败"); - } - - private String buildTaskResultJson(String aiPrompt, String apiKey, List sourceFiles, String parsedPayloadPointer) { - Map payload = new LinkedHashMap<>(); - payload.put("aiPrompt", normalize(aiPrompt)); - payload.put("apiKey", normalize(apiKey)); - payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream() - .map(AppearancePatentSourceFileDto::getFileKey) - .filter(Objects::nonNull) - .toList()); - payload.put("parsedPayloadRef", parsedPayloadPointer); - payload.put("ownerInstanceId", currentInstanceId()); - return writeJson(objectMapper, payload, "保存任务结果索引失败"); - } - - private String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) { - requireSharedTransientPayloadStorage("parsed payload"); - return transientPayloadStorageService.storeParsedPayloadFast(MODULE_TYPE, taskId, scopeHash, parsedPayloadJson, false); - } - private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId, long waitMillis) { return taskDistributedLockService.acquire(MODULE_TYPE, taskId, TASK_LOCK_TTL, waitMillis); } - private String storeSharedChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, String payloadJson) { - requireSharedTransientPayloadStorage("chunk payload"); - return transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); - } - - private String storeSharedChunkPayloadVersioned(Long taskId, String scopeHash, Integer chunkIndex, String payloadJson) { - requireSharedTransientPayloadStorage("merged chunk payload"); - return transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); - } - - - private void requireSharedTransientPayloadStorage(String payloadType) { - if (!transientPayloadStorageService.isSharedWriteEnabled()) { - throw new BusinessException("外观专利" + payloadType + "必须写入共享临时存储 RustFS,请检查 aiimage.transient-storage 配置"); - } - } - private long elapsedMs(long start, long end) { return (end - start) / 1_000_000L; } - private AppearancePatentParsedPayloadDto readParsedPayload(FileTaskEntity task) { - try { - JsonNode root = objectMapper.readTree(task.getResultJson()); - String pointer = root.path("parsedPayloadRef").asText(""); - if (!pointer.isBlank()) { - String json = transientPayloadStorageService.resolvePayload(pointer, "read appearance patent parsed payload failed"); - return hydrateParsedPayloadRows(objectMapper.readValue(json, AppearancePatentParsedPayloadDto.class)); - } - if (root.path("allItems").isArray()) { - return hydrateParsedPayloadRows(objectMapper.treeToValue(root, AppearancePatentParsedPayloadDto.class)); - } - } catch (Exception ex) { - throw new BusinessException("读取解析载荷失败"); - } - return hydrateParsedPayloadRows(new AppearancePatentParsedPayloadDto()); - } - public AppearancePatentParsedPayloadDto queuePayload(Long taskId, Long userId) { - FileTaskEntity task = fileTaskMapper.selectById(taskId); - if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) { - throw new BusinessException("任务不存在"); - } - ensureTaskOwnedByCurrentInstance(task, "queue payload"); - AppearancePatentParsedPayloadDto payload = readParsedPayload(task); - AppearancePatentParsedPayloadDto queuePayload = new AppearancePatentParsedPayloadDto(); - queuePayload.setAiPrompt(payload.getAiPrompt()); - queuePayload.setApiKey(payload.getApiKey()); - queuePayload.setGroups(payload.getGroups() == null ? List.of() : payload.getGroups()); - queuePayload.setItems(List.of()); - queuePayload.setAllItems(List.of()); - queuePayload.setHeaders(List.of()); - queuePayload.setSourceFiles(List.of()); - return queuePayload; - } - - private AppearancePatentParsedPayloadDto hydrateParsedPayloadRows(AppearancePatentParsedPayloadDto payload) { - if (payload == null) { - return new AppearancePatentParsedPayloadDto(); - } - List rows = payload.getAllItems(); - if (rows == null || rows.isEmpty()) { - rows = payload.getItems(); - } - if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) { - rows = payload.getGroups().stream() - .filter(Objects::nonNull) - .flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream()) - .toList(); - } - if (rows == null) { - rows = List.of(); - } - if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) { - payload.setAllItems(rows); - } - if (payload.getItems() == null || payload.getItems().isEmpty()) { - payload.setItems(rows); - } - if ((payload.getGroups() == null || payload.getGroups().isEmpty()) && !rows.isEmpty()) { - payload.setGroups(buildParsedGroups(rows)); - } - return payload; - } - - private Map readChunkRows(TaskChunkEntity chunk) { - Map rows = new LinkedHashMap<>(); - if (chunk == null || chunk.getPayloadJson() == null || chunk.getPayloadJson().isBlank()) { - return rows; - } - try { - String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read appearance patent chunk failed"); - JsonNode array = objectMapper.readTree(payloadJson); - if (!array.isArray()) { - return rows; - } - for (JsonNode node : array) { - AppearancePatentResultRowDto row = objectMapper.treeToValue(node, AppearancePatentResultRowDto.class); - rows.put(rowKey(row), row); - } - } catch (Exception ex) { - log.warn("[appearance-patent] read chunk payload failed taskId={} chunk={} err={}", - chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage()); - if (isPayloadMissing(ex)) { - // payload 对象已不在(被清理或从未写入):重试多少次都读不回来。继续抛会让 - // ASSEMBLE_RESULT job 的终态回调每轮重跑兜底组装 → 再读同一个缺失对象 → 无限循环 - // (线上任务 28459 每 10~30 秒重试一次)。跳过该分片,让任务按已有分片出部分结果, - // 与品牌/相似ASIN「失败也产出可下载的部分结果」同一口径。 - log.warn("[appearance-patent] chunk payload 已不存在,跳过该分片(任务按已有分片出结果)" - + " taskId={} chunk={}", chunk.getTaskId(), chunk.getChunkIndex()); - return rows; - } - throw new BusinessException("读取外观专利结果分片失败,分片=" - + chunk.getChunkIndex() + ": " + ex.getMessage(), ex); - } - return rows; - } - - /** payload 对象已不存在(RustFS 返回 NoSuchKey:message 为 "The specified key does not exist.")。 - * 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。 */ - private static boolean isPayloadMissing(Throwable error) { - Throwable cursor = error; - while (cursor != null) { - String message = cursor.getMessage(); - if (message != null && message.contains("does not exist")) { - return true; - } - cursor = cursor.getCause(); - } - return false; - } - - private String rowKey(AppearancePatentParsedRowVo row) { - if (row == null) { - return ""; - } - String rowToken = normalize(row.getRowToken()); - if (!rowToken.isBlank()) { - return rowToken; - } - return legacyRowKey(row); - } - - private String rowKey(AppearancePatentResultRowDto row) { - if (row == null) { - return ""; - } - String rowToken = normalize(row.getRowToken()); - if (!rowToken.isBlank()) { - return rowToken; - } - return legacyRowKey(row); - } - - private AppearancePatentResultRowDto findResultRow(AppearancePatentParsedRowVo parsedRow, - Map resultMap) { - if (parsedRow == null || resultMap == null || resultMap.isEmpty()) { - return null; - } - AppearancePatentResultRowDto resultRow = resultMap.get(rowKey(parsedRow)); - if (resultRow != null) { - return resultRow; - } - String legacyKey = legacyRowKey(parsedRow); - if (legacyKey.isBlank()) { - return null; - } - return resultMap.get(legacyKey); - } - - private String legacyRowKey(AppearancePatentParsedRowVo row) { - if (row == null) { - return ""; - } - return rowKey(row.getDisplayId(), row.getAsin(), row.getCountry()); - } - - private String legacyRowKey(AppearancePatentResultRowDto row) { - if (row == null) { - return ""; - } - return rowKey(row.getId(), row.getAsin(), row.getCountry()); - } - - private String rowKey(String id, String asin, String country) { - return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country); + return payloadSupport().queuePayload(taskId, userId); } static String resolveTaskExecutionStatus(boolean waitingForAssemble, boolean executionFailed) { @@ -2699,66 +2335,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline collectTransientTaskPayloads(Long taskId) { - if (taskId == null || taskId <= 0) { - return List.of(); - } - List payloads = new ArrayList<>(); - List scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper() - .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) - .eq(TaskScopeStateEntity::getTaskId, taskId) - .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); - if (scopes != null) { - for (TaskScopeStateEntity scope : scopes) { - if (scope == null) { - continue; - } - if (scope.getParsedPayloadJson() != null && !scope.getParsedPayloadJson().isBlank()) { - payloads.add(scope.getParsedPayloadJson()); - } - if (scope.getStateJson() != null && !scope.getStateJson().isBlank()) { - payloads.add(scope.getStateJson()); - } - } - } - List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() - .select(TaskChunkEntity::getPayloadJson) - .eq(TaskChunkEntity::getTaskId, taskId) - .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); - if (chunks != null) { - for (TaskChunkEntity chunk : chunks) { - if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) { - payloads.add(chunk.getPayloadJson()); - } - } - } - return payloads; - } - - /** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */ - private void deletePayloadsAfterCommit(List payloads, Long taskId) { - if (payloads == null || payloads.isEmpty()) { - return; - } - runAfterCommit(() -> { - for (String payload : payloads) { - try { - transientPayloadStorageService.deletePayloadIfPresent(payload); - } catch (Exception ex) { - log.warn("[appearance-patent] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage()); - } - } - }); - } - - /** - * 删除范围/分片的远端载荷(保留给无事务的清理链路调用)。 - */ - private void deleteTransientTaskPayloads(Long taskId) { - deletePayloadsAfterCommit(collectTransientTaskPayloads(taskId), taskId); - } - /** 有活动事务则注册 afterCommit,否则立即执行。 */ private void runAfterCommit(Runnable action) { if (TransactionSynchronizationManager.isSynchronizationActive()) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentPayloadSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentPayloadSupport.java new file mode 100644 index 000000000..a4a60d2fd --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentPayloadSupport.java @@ -0,0 +1,261 @@ +package com.nanri.aiimage.modules.appearancepatent.service.support; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSourceFileDto; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * 外观专利的载荷/JSON 读写(I 聚类),从 {@code AppearancePatentTaskService} 原样搬移 + * (3A 拆薄第六笔):解析载荷的组装/落库/读取、分片载荷写入、任务载荷指针收集与事务后删除, + * 连同异常文案与删除顺序一个字没改。 + * + *

搬的是载荷读写,不是宿主能力。字符串归一({@code normalize})、解析分组重建 + * ({@code buildParsedGroups})、实例 id、归属校验与 afterCommit 调度都还留在服务侧, + * 以回调交进来——它们横跨解析/归属/事务三个面,跟着载荷一起拖出来只会把依赖面放大。 + */ +@Slf4j +public class AppearancePatentPayloadSupport { + + /** 序列化失败时的统一出口(服务侧 {@code writeJson} 的薄适配)。 */ + @FunctionalInterface + public interface JsonWriter { + String write(Object value, String failureMessage); + } + + private final String moduleType; + private final ObjectMapper objectMapper; + private final TransientPayloadStorageService transientPayloadStorageService; + private final FileTaskMapper fileTaskMapper; + private final TaskChunkMapper taskChunkMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final Function normalizer; + private final Function, List> groupBuilder; + private final Supplier currentInstanceId; + private final BiConsumer ownershipChecker; + private final Consumer afterCommitRunner; + private final JsonWriter jsonWriter; + + public AppearancePatentPayloadSupport(String moduleType, + ObjectMapper objectMapper, + TransientPayloadStorageService transientPayloadStorageService, + FileTaskMapper fileTaskMapper, + TaskChunkMapper taskChunkMapper, + TaskScopeStateMapper taskScopeStateMapper, + Function normalizer, + Function, List> groupBuilder, + Supplier currentInstanceId, + BiConsumer ownershipChecker, + Consumer afterCommitRunner, + JsonWriter jsonWriter) { + this.moduleType = moduleType; + this.objectMapper = objectMapper; + this.transientPayloadStorageService = transientPayloadStorageService; + this.fileTaskMapper = fileTaskMapper; + this.taskChunkMapper = taskChunkMapper; + this.taskScopeStateMapper = taskScopeStateMapper; + this.normalizer = normalizer; + this.groupBuilder = groupBuilder; + this.currentInstanceId = currentInstanceId; + this.ownershipChecker = ownershipChecker; + this.afterCommitRunner = afterCommitRunner; + this.jsonWriter = jsonWriter; + } + + public String buildParsedPayloadJson(String aiPrompt, String apiKey, + List sourceFiles, + List headers, + List allRows) { + AppearancePatentParsedPayloadDto payload = new AppearancePatentParsedPayloadDto(); + payload.setAiPrompt(normalizer.apply(aiPrompt)); + payload.setApiKey(normalizer.apply(apiKey)); + payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles); + payload.setHeaders(headers == null ? List.of() : headers); + payload.setItems(List.of()); + payload.setGroups(List.of()); + payload.setAllItems(allRows == null ? List.of() : allRows); + return jsonWriter.write(payload, "保存解析结果失败"); + } + + public String buildTaskResultJson(String aiPrompt, String apiKey, + List sourceFiles, + String parsedPayloadPointer) { + Map payload = new LinkedHashMap<>(); + payload.put("aiPrompt", normalizer.apply(aiPrompt)); + payload.put("apiKey", normalizer.apply(apiKey)); + payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream() + .map(AppearancePatentSourceFileDto::getFileKey) + .filter(Objects::nonNull) + .toList()); + payload.put("parsedPayloadRef", parsedPayloadPointer); + payload.put("ownerInstanceId", currentInstanceId.get()); + return jsonWriter.write(payload, "保存任务结果索引失败"); + } + + public String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) { + requireSharedTransientPayloadStorage("parsed payload"); + return transientPayloadStorageService.storeParsedPayloadFast(moduleType, taskId, scopeHash, parsedPayloadJson, false); + } + + public String storeSharedChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, String payloadJson) { + requireSharedTransientPayloadStorage("chunk payload"); + return transientPayloadStorageService.storeChunkPayload(moduleType, taskId, scopeHash, chunkIndex, payloadJson); + } + + public String storeSharedChunkPayloadVersioned(Long taskId, String scopeHash, Integer chunkIndex, String payloadJson) { + requireSharedTransientPayloadStorage("merged chunk payload"); + return transientPayloadStorageService.storeChunkPayloadVersioned(moduleType, taskId, scopeHash, chunkIndex, payloadJson); + } + + private void requireSharedTransientPayloadStorage(String payloadType) { + if (!transientPayloadStorageService.isSharedWriteEnabled()) { + throw new BusinessException("外观专利" + payloadType + "必须写入共享临时存储 RustFS,请检查 aiimage.transient-storage 配置"); + } + } + + public AppearancePatentParsedPayloadDto readParsedPayload(FileTaskEntity task) { + try { + JsonNode root = objectMapper.readTree(task.getResultJson()); + String pointer = root.path("parsedPayloadRef").asText(""); + if (!pointer.isBlank()) { + String json = transientPayloadStorageService.resolvePayload(pointer, "read appearance patent parsed payload failed"); + return hydrateParsedPayloadRows(objectMapper.readValue(json, AppearancePatentParsedPayloadDto.class)); + } + if (root.path("allItems").isArray()) { + return hydrateParsedPayloadRows(objectMapper.treeToValue(root, AppearancePatentParsedPayloadDto.class)); + } + } catch (Exception ex) { + throw new BusinessException("读取解析载荷失败"); + } + return hydrateParsedPayloadRows(new AppearancePatentParsedPayloadDto()); + } + + public AppearancePatentParsedPayloadDto queuePayload(Long taskId, Long userId) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !moduleType.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) { + throw new BusinessException("任务不存在"); + } + ownershipChecker.accept(task, "queue payload"); + AppearancePatentParsedPayloadDto payload = readParsedPayload(task); + AppearancePatentParsedPayloadDto queuePayload = new AppearancePatentParsedPayloadDto(); + queuePayload.setAiPrompt(payload.getAiPrompt()); + queuePayload.setApiKey(payload.getApiKey()); + queuePayload.setGroups(payload.getGroups() == null ? List.of() : payload.getGroups()); + queuePayload.setItems(List.of()); + queuePayload.setAllItems(List.of()); + queuePayload.setHeaders(List.of()); + queuePayload.setSourceFiles(List.of()); + return queuePayload; + } + + private AppearancePatentParsedPayloadDto hydrateParsedPayloadRows(AppearancePatentParsedPayloadDto payload) { + if (payload == null) { + return new AppearancePatentParsedPayloadDto(); + } + List rows = payload.getAllItems(); + if (rows == null || rows.isEmpty()) { + rows = payload.getItems(); + } + if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) { + rows = payload.getGroups().stream() + .filter(Objects::nonNull) + .flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream()) + .toList(); + } + if (rows == null) { + rows = List.of(); + } + if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) { + payload.setAllItems(rows); + } + if (payload.getItems() == null || payload.getItems().isEmpty()) { + payload.setItems(rows); + } + if ((payload.getGroups() == null || payload.getGroups().isEmpty()) && !rows.isEmpty()) { + payload.setGroups(groupBuilder.apply(rows)); + } + return payload; + } + + /** 只读收集任务范围/分片载荷指针,供事务提交后做远端删除。 */ + public List collectTransientTaskPayloads(Long taskId) { + if (taskId == null || taskId <= 0) { + return List.of(); + } + List payloads = new ArrayList<>(); + List scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, moduleType)); + if (scopes != null) { + for (TaskScopeStateEntity scope : scopes) { + if (scope == null) { + continue; + } + if (scope.getParsedPayloadJson() != null && !scope.getParsedPayloadJson().isBlank()) { + payloads.add(scope.getParsedPayloadJson()); + } + if (scope.getStateJson() != null && !scope.getStateJson().isBlank()) { + payloads.add(scope.getStateJson()); + } + } + } + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .select(TaskChunkEntity::getPayloadJson) + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, moduleType)); + if (chunks != null) { + for (TaskChunkEntity chunk : chunks) { + if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) { + payloads.add(chunk.getPayloadJson()); + } + } + } + return payloads; + } + + /** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */ + public void deletePayloadsAfterCommit(List payloads, Long taskId) { + if (payloads == null || payloads.isEmpty()) { + return; + } + afterCommitRunner.accept(() -> { + for (String payload : payloads) { + try { + transientPayloadStorageService.deletePayloadIfPresent(payload); + } catch (Exception ex) { + log.warn("[appearance-patent] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage()); + } + } + }); + } + + /** + * 删除范围/分片的远端载荷(保留给无事务的清理链路调用)。 + */ + public void deleteTransientTaskPayloads(Long taskId) { + deletePayloadsAfterCommit(collectTransientTaskPayloads(taskId), taskId); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentResultRowReader.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentResultRowReader.java new file mode 100644 index 000000000..963dce1cc --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentResultRowReader.java @@ -0,0 +1,344 @@ +package com.nanri.aiimage.modules.appearancepatent.service.support; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultGroupDto; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; + +/** + * 外观专利的结果行读取(H 聚类),从 {@code AppearancePatentTaskService} 原样搬移 + * (3A 拆薄第七笔):回传分组展开、解析行计数、分片载荷读取、结果行按键分页读取与行键口径, + * 连同分页大小、重试次数、缺失载荷的跳过判据一个字没改。 + * + *

搬的是读取与行键,不是宿主。解析载荷读取(I 聚类)仍由服务侧传入; + * 字符串归一也以回调交入,避免把解析面的工具再抄一份。分片读取的异常语义保持原样: + * 只有 RustFS 的 "does not exist" 才允许跳过该分片,网络类失败仍抛出以便重试。 + */ +@Slf4j +public class AppearancePatentResultRowReader { + + /** 结果行读取时 chunk 的 keyset 页大小,与 similarasin 侧 loadChunksKeyset 的 500 同口径。 */ + private static final int RESULT_ROWS_CHUNK_PAGE_SIZE = 500; + private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3; + private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L; + + private final String moduleType; + private final ObjectMapper objectMapper; + private final TransientPayloadStorageService transientPayloadStorageService; + private final TaskChunkMapper taskChunkMapper; + private final Function normalizer; + private final Function parsedPayloadReader; + + public AppearancePatentResultRowReader(String moduleType, + ObjectMapper objectMapper, + TransientPayloadStorageService transientPayloadStorageService, + TaskChunkMapper taskChunkMapper, + Function normalizer, + Function parsedPayloadReader) { + this.moduleType = moduleType; + this.objectMapper = objectMapper; + this.transientPayloadStorageService = transientPayloadStorageService; + this.taskChunkMapper = taskChunkMapper; + this.normalizer = normalizer; + this.parsedPayloadReader = parsedPayloadReader; + } + + public List flattenSubmittedRows(AppearancePatentSubmitResultRequest request) { + if (request == null) { + return List.of(); + } + List groups = request.getGroups(); + if (groups != null && !groups.isEmpty()) { + List rows = new ArrayList<>(); + for (AppearancePatentResultGroupDto group : groups) { + if (group == null || group.getItems() == null || group.getItems().isEmpty()) { + continue; + } + for (AppearancePatentResultRowDto row : group.getItems()) { + if (row == null) { + continue; + } + if (normalizer.apply(row.getGroupKey()).isBlank()) { + row.setGroupKey(group.getGroupKey()); + } + if (normalizer.apply(row.getSourceFileKey()).isBlank()) { + row.setSourceFileKey(group.getSourceFileKey()); + } + if (normalizer.apply(row.getSourceFilename()).isBlank()) { + row.setSourceFilename(group.getSourceFilename()); + } + rows.add(row); + } + } + return rows; + } + return request.getItems() == null ? List.of() : request.getItems(); + } + + public int allRowCount(FileTaskEntity task) { + try { + AppearancePatentParsedPayloadDto payload = parsedPayloadReader.apply(task); + if (payload.getAllItems() != null && !payload.getAllItems().isEmpty()) { + return payload.getAllItems().size(); + } + if (payload.getItems() != null && !payload.getItems().isEmpty()) { + return payload.getItems().size(); + } + if (payload.getGroups() != null && !payload.getGroups().isEmpty()) { + return payload.getGroups().stream() + .map(AppearancePatentParsedGroupVo::getItems) + .filter(Objects::nonNull) + .mapToInt(List::size) + .sum(); + } + return 0; + } catch (Exception ex) { + log.warn("[appearance-patent] read all row count failed taskId={} err={}", task.getId(), ex.getMessage()); + return 0; + } + } + + public List filterReceivedParsedRows(List parsedRows, + Map resultMap) { + if (parsedRows == null || parsedRows.isEmpty() || resultMap == null || resultMap.isEmpty()) { + return List.of(); + } + List receivedRows = new ArrayList<>(); + for (AppearancePatentParsedRowVo parsedRow : parsedRows) { + if (findResultRow(parsedRow, resultMap) != null) { + receivedRows.add(parsedRow); + } + } + return receivedRows; + } + + public boolean hasPersistedResultRows(Long taskId) { + if (taskId == null) { + return false; + } + Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, moduleType)); + return count != null && count > 0; + } + + public Map loadPersistedResultRowsForRowsWithRetry( + Long taskId, List requestedRows) { + Set requestedKeys = resultLookupKeysForParsedRows(requestedRows); + Map result = + loadPersistedResultRowsForKeys(taskId, requestedKeys); + if (requestedKeys.isEmpty() || !result.isEmpty()) { + return result; + } + for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) { + sleepBeforeResultRowsRetry(attempt); + result = loadPersistedResultRowsForKeys(taskId, requestedKeys); + if (!result.isEmpty()) { + log.info("[appearance-patent] result rows recovered after retry taskId={} attempt={} rows={}", + taskId, attempt, result.size()); + } + } + return result; + } + + private Map loadPersistedResultRowsForKeys( + Long taskId, Set requestedKeys) { + Map result = new LinkedHashMap<>(); + if (taskId == null || requestedKeys == null || requestedKeys.isEmpty()) { + return result; + } + Long lastChunkId = null; + while (true) { + LambdaQueryWrapper query = new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, moduleType) + .orderByAsc(TaskChunkEntity::getId) + .last("LIMIT " + RESULT_ROWS_CHUNK_PAGE_SIZE); + if (lastChunkId != null) { + query.gt(TaskChunkEntity::getId, lastChunkId); + } + List page = taskChunkMapper.selectList(query); + if (page == null || page.isEmpty()) { + break; + } + // 整页取 chunk 后逐片解载荷:原先每轮 LIMIT 1 只取一行,一个结果文件要把该任务的 + // chunk 表逐行翻完,每行一次 SQL + 一次对象读 + 一次整片 JSON 反序列化 + for (TaskChunkEntity chunk : page) { + appendRequestedResultRows(result, readChunkRows(chunk).values(), requestedKeys); + } + TaskChunkEntity tail = page.getLast(); + lastChunkId = tail == null ? null : tail.getId(); + if (lastChunkId == null || page.size() < RESULT_ROWS_CHUNK_PAGE_SIZE) { + break; + } + } + return result; + } + + private Set resultLookupKeysForParsedRows(List rows) { + Set keys = new LinkedHashSet<>(); + for (AppearancePatentParsedRowVo row : rows == null ? List.of() : rows) { + String primary = rowKey(row); + if (!primary.isBlank()) { + keys.add(primary); + } + String legacy = legacyRowKey(row); + if (!legacy.isBlank()) { + keys.add(legacy); + } + } + return keys; + } + + private void appendRequestedResultRows(Map sink, + Iterable rows, + Set requestedKeys) { + if (rows == null || requestedKeys == null || requestedKeys.isEmpty()) { + return; + } + for (AppearancePatentResultRowDto row : rows) { + if (row == null) { + continue; + } + String primary = rowKey(row); + String legacy = legacyRowKey(row); + String matchedKey = requestedKeys.contains(primary) + ? primary : requestedKeys.contains(legacy) ? legacy : ""; + if (!matchedKey.isBlank()) { + sink.put(matchedKey, row); + } + } + } + + private void sleepBeforeResultRowsRetry(int attempt) { + try { + Thread.sleep(RESULT_ROWS_READ_RETRY_DELAY_MS * attempt); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + + public Map readChunkRows(TaskChunkEntity chunk) { + Map rows = new LinkedHashMap<>(); + if (chunk == null || chunk.getPayloadJson() == null || chunk.getPayloadJson().isBlank()) { + return rows; + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read appearance patent chunk failed"); + JsonNode array = objectMapper.readTree(payloadJson); + if (!array.isArray()) { + return rows; + } + for (JsonNode node : array) { + AppearancePatentResultRowDto row = objectMapper.treeToValue(node, AppearancePatentResultRowDto.class); + rows.put(rowKey(row), row); + } + } catch (Exception ex) { + log.warn("[appearance-patent] read chunk payload failed taskId={} chunk={} err={}", + chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage()); + if (isPayloadMissing(ex)) { + // payload 对象已不在(被清理或从未写入):重试多少次都读不回来。继续抛会让 + // ASSEMBLE_RESULT job 的终态回调每轮重跑兜底组装 → 再读同一个缺失对象 → 无限循环 + // (线上任务 28459 每 10~30 秒重试一次)。跳过该分片,让任务按已有分片出部分结果, + // 与品牌/相似ASIN「失败也产出可下载的部分结果」同一口径。 + log.warn("[appearance-patent] chunk payload 已不存在,跳过该分片(任务按已有分片出结果)" + + " taskId={} chunk={}", chunk.getTaskId(), chunk.getChunkIndex()); + return rows; + } + throw new BusinessException("读取外观专利结果分片失败,分片=" + + chunk.getChunkIndex() + ": " + ex.getMessage(), ex); + } + return rows; + } + + /** payload 对象已不存在(RustFS 返回 NoSuchKey:message 为 "The specified key does not exist.")。 + * 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。 */ + private static boolean isPayloadMissing(Throwable error) { + Throwable cursor = error; + while (cursor != null) { + String message = cursor.getMessage(); + if (message != null && message.contains("does not exist")) { + return true; + } + cursor = cursor.getCause(); + } + return false; + } + + public String rowKey(AppearancePatentParsedRowVo row) { + if (row == null) { + return ""; + } + String rowToken = normalizer.apply(row.getRowToken()); + if (!rowToken.isBlank()) { + return rowToken; + } + return legacyRowKey(row); + } + + public String rowKey(AppearancePatentResultRowDto row) { + if (row == null) { + return ""; + } + String rowToken = normalizer.apply(row.getRowToken()); + if (!rowToken.isBlank()) { + return rowToken; + } + return legacyRowKey(row); + } + + public AppearancePatentResultRowDto findResultRow(AppearancePatentParsedRowVo parsedRow, + Map resultMap) { + if (parsedRow == null || resultMap == null || resultMap.isEmpty()) { + return null; + } + AppearancePatentResultRowDto resultRow = resultMap.get(rowKey(parsedRow)); + if (resultRow != null) { + return resultRow; + } + String legacyKey = legacyRowKey(parsedRow); + if (legacyKey.isBlank()) { + return null; + } + return resultMap.get(legacyKey); + } + + private String legacyRowKey(AppearancePatentParsedRowVo row) { + if (row == null) { + return ""; + } + return rowKey(row.getDisplayId(), row.getAsin(), row.getCountry()); + } + + private String legacyRowKey(AppearancePatentResultRowDto row) { + if (row == null) { + return ""; + } + return rowKey(row.getId(), row.getAsin(), row.getCountry()); + } + + private String rowKey(String id, String asin, String country) { + return normalizer.apply(id) + "::" + normalizer.apply(asin).toUpperCase(Locale.ROOT) + "::" + normalizer.apply(country); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java b/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java index 1c8d08f69..101a8465b 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java @@ -266,13 +266,13 @@ class SourceScanTest { // ---------------------------------------------------------------- N6 moduleSources @Test - void moduleSources_appearancepatent返回32支含拆分后的support件() { + void moduleSources_appearancepatent返回34支含拆分后的support件() { List files = SourceScan.moduleSources("modules/appearancepatent"); assertThat(files) - .as("真实集成:对真仓目录求值。32 支=1 controller+1 client+1 service 主件+3 spi/cache+" - + "9 service/support+16 model(dto7/vo9),与 find 实测同号") - .hasSize(32); + .as("真实集成:对真仓目录求值。34 支=1 controller+1 client+1 service 主件+3 spi/cache+" + + "11 service/support+16 model(dto7/vo9),与 find 实测同号") + .hasSize(34); assertThat(files.stream().map(p -> p.toString().replace('\\', '/')).toList()) .as("support 目录里的既有件必须在面里——聚合入口存在的理由就是拆分后的新文件要自动被看到") .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentRowNormalizer.java")); @@ -366,23 +366,25 @@ class SourceScanTest { } @Test - void recursive_包含service_support下的既有9件() { + void recursive_包含service_support下的既有11件() { List support = SourceScan.moduleSources("modules/appearancepatent").stream() .map(p -> p.toString().replace('\\', '/')) .filter(rel -> rel.contains("/service/support/")) .toList(); assertThat(support) - .as("真实集成:对真仓 modules/appearancepatent/service/support 求值,实测 9 支(3A 拆薄把归属谓词、陈旧扫描、结果打包、LLM 批管线抽成四枚 support 件,5→9)。" + .as("真实集成:对真仓 modules/appearancepatent/service/support 求值,实测 11 支(3A 拆薄把归属谓词、陈旧扫描、结果打包、LLM 批管线、载荷读写、结果行读取抽成六枚 support 件,5→11)。" + "这一格就是聚合入口存在的理由——拆分出去的新文件必须自动进面") - .hasSize(9); + .hasSize(11); assertThat(support) - .as("3A 拆出去的四枚新件必须被逐枚点名扫到:只把分母数字改大,下一回拆件又只剩「数不对」" + .as("3A 拆出去的六枚新件必须被逐枚点名扫到:只把分母数字改大,下一回拆件又只剩「数不对」" + "一记红,证不了「扫得到」这件事") .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentTaskOwnership.java")) .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentStaleTaskScanner.java")) .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentResultPackager.java")) - .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentLlmBatchPipeline.java")); + .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentLlmBatchPipeline.java")) + .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentPayloadSupport.java")) + .anyMatch(rel -> rel.endsWith("service/support/AppearancePatentResultRowReader.java")); } @Test