refactor(phase-3): 3A 拆薄第六七笔——AP 载荷与结果行读取出件,服务 2801→2377
I 聚类(解析/分片载荷组装、读取、事务后删除)进 AppearancePatentPayloadSupport; H 聚类(回传分组展开、解析行计数、分片载荷读取、分页结果行与行键口径)进 AppearancePatentResultRowReader。归一、归属、分组、afterCommit 与解析载荷读取 仍由服务回调交出。SourceScan 面数 32→34、support 9→11 并逐枚点名新件。
This commit is contained in:
+68
-492
@@ -25,6 +25,8 @@ import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashb
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentHistoryAssembler;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentLlmBatchPipeline;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentPayloadSupport;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentResultRowReader;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
|
||||
@@ -107,11 +109,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
||||
private static final int CHUNK_PAYLOAD_MERGE_RETRY_LIMIT = 3;
|
||||
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
|
||||
|
||||
/** 结果行读取时 chunk 的 keyset 页大小,与 similarasin 侧 loadChunksKeyset 的 500 同口径。 */
|
||||
private static final int RESULT_ROWS_CHUNK_PAGE_SIZE = 500;
|
||||
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
|
||||
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
|
||||
private static final long TASK_LOCK_WAIT_MILLIS = 10000L;
|
||||
private static final long TASK_LOCK_RETRY_DELAY_MILLIS = 200L;
|
||||
@@ -253,11 +250,11 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
|
||||
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
|
||||
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
|
||||
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, mergedHeaders, allRows);
|
||||
String parsedPayload = payloadSupport().buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, mergedHeaders, allRows);
|
||||
long payloadBuiltAt = System.nanoTime();
|
||||
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
|
||||
String parsedPayloadPointer = payloadSupport().storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
|
||||
long payloadStoredAt = System.nanoTime();
|
||||
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, parsedPayloadPointer));
|
||||
task.setResultJson(payloadSupport().buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, parsedPayloadPointer));
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
|
||||
@@ -537,7 +534,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
|
||||
.last("limit 1"));
|
||||
if (existing == null) {
|
||||
List<AppearancePatentResultRowDto> rawRows = flattenSubmittedRows(request);
|
||||
List<AppearancePatentResultRowDto> rawRows = rowReader().flattenSubmittedRows(request);
|
||||
String payloadJson = writeJson(objectMapper, rawRows, "结果序列化失败");
|
||||
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
@@ -547,7 +544,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
chunk.setChunkTotal(chunkTotal);
|
||||
String storedPayload = storeSharedChunkPayload(taskId, scopeHash, chunkIndex, payloadJson);
|
||||
String storedPayload = payloadSupport().storeSharedChunkPayload(taskId, scopeHash, chunkIndex, payloadJson);
|
||||
chunk.setPayloadJson(storedPayload);
|
||||
chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson));
|
||||
chunk.setCreatedAt(LocalDateTime.now());
|
||||
@@ -593,7 +590,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
SubmitContext context = new SubmitContext(task, scopeKey, scopeHash, chunkIndex,
|
||||
Boolean.TRUE.equals(request.getDone()), request.getError());
|
||||
if (Boolean.TRUE.equals(request.getDone()) || request.getError() != null && !request.getError().isBlank()) {
|
||||
finalizeTask(task, request.getError(), allRowCount(task), true);
|
||||
finalizeTask(task, request.getError(), rowReader().allRowCount(task), true);
|
||||
} else {
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
@@ -613,7 +610,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
List<String> payloads = collectTransientTaskPayloads(taskId);
|
||||
List<String> payloads = payloadSupport().collectTransientTaskPayloads(taskId);
|
||||
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>().eq(FileResultEntity::getTaskId, taskId).eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
||||
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>().eq(TaskScopeStateEntity::getTaskId, taskId).eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>().eq(TaskChunkEntity::getTaskId, taskId).eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
@@ -621,7 +618,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
|
||||
fileTaskMapper.deleteById(taskId);
|
||||
// 事务提交后再做远端删除与缓存清理
|
||||
deletePayloadsAfterCommit(payloads, taskId);
|
||||
payloadSupport().deletePayloadsAfterCommit(payloads, taskId);
|
||||
runAfterCommit(() -> taskCacheService.deleteTaskCache(taskId));
|
||||
}
|
||||
|
||||
@@ -706,9 +703,9 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
return new PreparedSubmittedChunk(task, scopeKey, scopeHash, chunkIndex, chunkTotal, done,
|
||||
request.getError(), null, null, null);
|
||||
}
|
||||
List<AppearancePatentResultRowDto> rawRows = flattenSubmittedRows(request);
|
||||
List<AppearancePatentResultRowDto> rawRows = rowReader().flattenSubmittedRows(request);
|
||||
String payloadJson = writeJson(objectMapper, rawRows, "结果序列化失败");
|
||||
String storedPayload = storeSharedChunkPayload(taskId, scopeHash, chunkIndex, payloadJson);
|
||||
String storedPayload = payloadSupport().storeSharedChunkPayload(taskId, scopeHash, chunkIndex, payloadJson);
|
||||
return new PreparedSubmittedChunk(task, scopeKey, scopeHash, chunkIndex, chunkTotal, done,
|
||||
request.getError(), payloadJson, storedPayload, DigestUtil.sha256Hex(payloadJson));
|
||||
}
|
||||
@@ -775,7 +772,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
}
|
||||
upsertScopeState(task.getId(), context.scopeKey(), context.scopeHash(), null, context.error(), context.forceFlush(), false);
|
||||
if (context.forceFlush() || context.error() != null && !context.error().isBlank()) {
|
||||
finalizeTask(task, context.error(), allRowCount(task), true);
|
||||
finalizeTask(task, context.error(), rowReader().allRowCount(task), true);
|
||||
return;
|
||||
}
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
@@ -829,11 +826,35 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
private AppearancePatentLlmBatchPipeline llmBatchPipeline() {
|
||||
return new AppearancePatentLlmBatchPipeline(
|
||||
properties, llmClient, taskCacheService, taskFileJobService,
|
||||
this::rowKey, this::shouldInspectRow, this::readChunkRows, this::loadSubmittedChunks,
|
||||
rowReader()::rowKey, this::shouldInspectRow, rowReader()::readChunkRows, this::loadSubmittedChunks,
|
||||
this::isResultSubmissionComplete, this::touchJavaSideTaskActivity,
|
||||
this::readAiPrompt, this::readApiKey, this::mergeChunkPayload, this::firstNonBlank);
|
||||
}
|
||||
|
||||
/**
|
||||
* I 聚类(载荷/JSON 读写)住在 {@link AppearancePatentPayloadSupport}(3A 拆薄第六笔,
|
||||
* 解析载荷组装/读取、分片载荷写入、事务后删除逐字搬移);归一、分组、归属、afterCommit
|
||||
* 仍由本类以回调交出。
|
||||
*/
|
||||
private AppearancePatentPayloadSupport payloadSupport() {
|
||||
return new AppearancePatentPayloadSupport(
|
||||
MODULE_TYPE, objectMapper, transientPayloadStorageService,
|
||||
fileTaskMapper, taskChunkMapper, taskScopeStateMapper,
|
||||
this::normalize, this::buildParsedGroups, this::currentInstanceId,
|
||||
this::ensureTaskOwnedByCurrentInstance, this::runAfterCommit,
|
||||
(value, message) -> writeJson(objectMapper, value, message));
|
||||
}
|
||||
|
||||
/**
|
||||
* H 聚类(结果行读取与行键)住在 {@link AppearancePatentResultRowReader}(3A 拆薄第七笔):
|
||||
* 回传分组展开、解析行计数、分片载荷读取、按键分页读取逐字搬移;解析载荷读取与归一再由本类回调交出。
|
||||
*/
|
||||
private AppearancePatentResultRowReader rowReader() {
|
||||
return new AppearancePatentResultRowReader(
|
||||
MODULE_TYPE, objectMapper, transientPayloadStorageService, taskChunkMapper,
|
||||
this::normalize, payloadSupport()::readParsedPayload);
|
||||
}
|
||||
|
||||
private void submitLlmForSubmittedChunk(SubmitContext context) {
|
||||
if (context == null || context.task() == null || context.task().getId() == null) {
|
||||
return;
|
||||
@@ -848,7 +869,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
task.getId());
|
||||
return;
|
||||
}
|
||||
FileResultEntity result = findOrCreateResultRecordForAssembly(task, allRowCount(task));
|
||||
FileResultEntity result = findOrCreateResultRecordForAssembly(task, rowReader().allRowCount(task));
|
||||
if (result == null) {
|
||||
log.warn("[appearance-patent] stale recovery could not create result record taskId={}", task.getId());
|
||||
return;
|
||||
@@ -883,7 +904,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
if (chunks.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
FileResultEntity result = findOrCreateResultRecordForAssembly(task, allRowCount(task));
|
||||
FileResultEntity result = findOrCreateResultRecordForAssembly(task, rowReader().allRowCount(task));
|
||||
if (result == null) {
|
||||
return;
|
||||
}
|
||||
@@ -916,11 +937,11 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
return;
|
||||
}
|
||||
if (transactionManager == null) {
|
||||
finalizeTask(task, error, allRowCount(task), true);
|
||||
finalizeTask(task, error, rowReader().allRowCount(task), true);
|
||||
return;
|
||||
}
|
||||
inNewTransaction(() -> {
|
||||
finalizeTask(task, error, allRowCount(task), true);
|
||||
finalizeTask(task, error, rowReader().allRowCount(task), true);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
@@ -933,7 +954,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
boolean uploadComplete = isResultSubmissionComplete(taskId);
|
||||
long activeAssembleJobs = taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE);
|
||||
log.info("[appearance-patent] stale recovery probe taskId={} uploadComplete={} activeAssembleJobs={} persistedRows={}",
|
||||
taskId, uploadComplete, activeAssembleJobs, hasPersistedResultRows(taskId));
|
||||
taskId, uploadComplete, activeAssembleJobs, rowReader().hasPersistedResultRows(taskId));
|
||||
// 已有「重试耗尽且已终态收尾」的 assemble job:说明恢复已经试过、缺失是永久的。
|
||||
// 再重建只会每 30 秒空转一轮,而且恢复过程刷新任务心跳会让任务永远 RUNNING
|
||||
// (线上任务 28459 实测:48 分钟里每隔 30 秒重建一次 job)。返回 false 交给
|
||||
@@ -942,7 +963,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
log.warn("[appearance-patent] stale recovery 放弃:已有重试耗尽的 assemble job,按失败收尾 taskId={}", taskId);
|
||||
return false;
|
||||
}
|
||||
if (!hasPersistedResultRows(taskId)) {
|
||||
if (!rowReader().hasPersistedResultRows(taskId)) {
|
||||
log.info("[appearance-patent] stale recovery aborted because no persisted rows taskId={}", taskId);
|
||||
return false;
|
||||
}
|
||||
@@ -1108,9 +1129,9 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
if (chunk == null) {
|
||||
return List.of();
|
||||
}
|
||||
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
|
||||
Map<String, AppearancePatentResultRowDto> persistedRows = rowReader().readChunkRows(chunk);
|
||||
for (AppearancePatentResultRowDto row : rows) {
|
||||
String key = rowKey(row);
|
||||
String key = rowReader().rowKey(row);
|
||||
AppearancePatentResultRowDto submittedRow = persistedRows.get(key);
|
||||
if (submittedRow != null) {
|
||||
row.setPrice(submittedRow.getPrice());
|
||||
@@ -1121,7 +1142,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
String oldPayload = chunk.getPayloadJson();
|
||||
String oldPayloadHash = chunk.getPayloadHash();
|
||||
String newPayloadHash = DigestUtil.sha256Hex(payloadJson);
|
||||
String storedPayload = storeSharedChunkPayloadVersioned(taskId, scopeHash, chunkIndex, payloadJson);
|
||||
String storedPayload = payloadSupport().storeSharedChunkPayloadVersioned(taskId, scopeHash, chunkIndex, payloadJson);
|
||||
LambdaUpdateWrapper<TaskChunkEntity> updateWrapper = new LambdaUpdateWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getId, chunk.getId())
|
||||
.set(TaskChunkEntity::getPayloadJson, storedPayload)
|
||||
@@ -1137,7 +1158,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
log.debug("[appearance-patent] chunk payload replaced taskId={} scopeHash={} chunk={} oldPayload={} newPayload={} attempt={}",
|
||||
taskId, scopeHash, chunkIndex, oldPayload, storedPayload, attempt);
|
||||
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
||||
return rows.stream().map(this::rowKey).filter(key -> !key.isBlank()).toList();
|
||||
return rows.stream().map(rowReader()::rowKey).filter(key -> !key.isBlank()).toList();
|
||||
}
|
||||
// CAS 冲突:读回行上的当前哈希,否则只有一句 conflict 无从判断是谁改的(线上 28459 至今未能定位)
|
||||
String currentHash = currentPayloadHash(chunk.getId());
|
||||
@@ -1206,64 +1227,10 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<AppearancePatentResultRowDto> flattenSubmittedRows(AppearancePatentSubmitResultRequest request) {
|
||||
if (request == null) {
|
||||
return List.of();
|
||||
}
|
||||
List<AppearancePatentResultGroupDto> groups = request.getGroups();
|
||||
if (groups != null && !groups.isEmpty()) {
|
||||
List<AppearancePatentResultRowDto> rows = new ArrayList<>();
|
||||
for (AppearancePatentResultGroupDto group : groups) {
|
||||
if (group == null || group.getItems() == null || group.getItems().isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
for (AppearancePatentResultRowDto row : group.getItems()) {
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
if (normalize(row.getGroupKey()).isBlank()) {
|
||||
row.setGroupKey(group.getGroupKey());
|
||||
}
|
||||
if (normalize(row.getSourceFileKey()).isBlank()) {
|
||||
row.setSourceFileKey(group.getSourceFileKey());
|
||||
}
|
||||
if (normalize(row.getSourceFilename()).isBlank()) {
|
||||
row.setSourceFilename(group.getSourceFilename());
|
||||
}
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
return request.getItems() == null ? List.of() : request.getItems();
|
||||
}
|
||||
|
||||
private int allRowCount(FileTaskEntity task) {
|
||||
try {
|
||||
AppearancePatentParsedPayloadDto payload = readParsedPayload(task);
|
||||
if (payload.getAllItems() != null && !payload.getAllItems().isEmpty()) {
|
||||
return payload.getAllItems().size();
|
||||
}
|
||||
if (payload.getItems() != null && !payload.getItems().isEmpty()) {
|
||||
return payload.getItems().size();
|
||||
}
|
||||
if (payload.getGroups() != null && !payload.getGroups().isEmpty()) {
|
||||
return payload.getGroups().stream()
|
||||
.map(AppearancePatentParsedGroupVo::getItems)
|
||||
.filter(Objects::nonNull)
|
||||
.mapToInt(List::size)
|
||||
.sum();
|
||||
}
|
||||
return 0;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent] read all row count failed taskId={} err={}", task.getId(), ex.getMessage());
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private String readAiPrompt(FileTaskEntity task) {
|
||||
try {
|
||||
return readParsedPayload(task).getAiPrompt();
|
||||
return payloadSupport().readParsedPayload(task).getAiPrompt();
|
||||
} catch (Exception ignored) {
|
||||
return "";
|
||||
}
|
||||
@@ -1272,7 +1239,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
/** 密钥读取:任务 payload 优先(兼容老任务与显式覆盖),为空时按任务归属用户从服务端密钥表兜底。 */
|
||||
private String readApiKey(FileTaskEntity task) {
|
||||
try {
|
||||
String fromPayload = normalize(readParsedPayload(task).getApiKey());
|
||||
String fromPayload = normalize(payloadSupport().readParsedPayload(task).getApiKey());
|
||||
if (!fromPayload.isEmpty()) {
|
||||
return fromPayload;
|
||||
}
|
||||
@@ -1293,18 +1260,18 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
String finalError = error;
|
||||
AppearancePatentRowOutcome.Facts deliveryFacts = AppearancePatentRowOutcome.Facts.EMPTY;
|
||||
FileResultEntity result = null;
|
||||
boolean hasPersistedResultRows = hasPersistedResultRows(task.getId());
|
||||
boolean persistedRowsPresent = rowReader().hasPersistedResultRows(task.getId());
|
||||
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, task.getId())
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||
.last("limit 1"));
|
||||
if (!rows.isEmpty()) {
|
||||
result = rows.getFirst();
|
||||
} else if (assembleWorkbook && hasPersistedResultRows) {
|
||||
} else if (assembleWorkbook && persistedRowsPresent) {
|
||||
result = createResultRecordForAssembly(task, rowCount);
|
||||
}
|
||||
if (result != null) {
|
||||
boolean shouldAssembleResult = assembleWorkbook && (finalError == null || finalError.isBlank() || hasPersistedResultRows);
|
||||
boolean shouldAssembleResult = assembleWorkbook && (finalError == null || finalError.isBlank() || persistedRowsPresent);
|
||||
if (shouldAssembleResult && shouldAssembleSynchronously()) {
|
||||
try {
|
||||
deliveryFacts = assembleResultWorkbook(task, result);
|
||||
@@ -1328,7 +1295,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
result.setSuccess(failed ? 0 : 1);
|
||||
result.setErrorMessage(finalError);
|
||||
result.setRowCount(rowCount > 0 ? rowCount : result.getRowCount());
|
||||
boolean shouldAssembleResult = assembleWorkbook && (!failed || hasPersistedResultRows);
|
||||
boolean shouldAssembleResult = assembleWorkbook && (!failed || persistedRowsPresent);
|
||||
if (shouldAssembleResult) {
|
||||
result.setResultFilename(safeFileStem(result.getSourceFilename()) + "-result.xlsx");
|
||||
result.setResultFileUrl(null);
|
||||
@@ -1374,7 +1341,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
}
|
||||
|
||||
private FileResultEntity createResultRecordForAssembly(FileTaskEntity task, int rowCount) {
|
||||
AppearancePatentParsedPayloadDto payload = readParsedPayload(task);
|
||||
AppearancePatentParsedPayloadDto payload = payloadSupport().readParsedPayload(task);
|
||||
List<AppearancePatentSourceFileDto> sourceFiles = payload.getSourceFiles() == null ? List.of() : payload.getSourceFiles();
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setTaskId(task.getId());
|
||||
@@ -1498,16 +1465,16 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
* {@link AppearancePatentRowOutcome#summarize},两边不许各判一套。
|
||||
*/
|
||||
private AppearancePatentRowOutcome.Facts measureDeliveryFacts(FileTaskEntity task, FileResultEntity result) {
|
||||
AppearancePatentParsedPayloadDto parsed = readParsedPayload(task);
|
||||
AppearancePatentParsedPayloadDto parsed = payloadSupport().readParsedPayload(task);
|
||||
List<AppearancePatentParsedRowVo> parsedRows = parsed.getAllItems() == null
|
||||
? List.of() : parsed.getAllItems();
|
||||
AppearancePatentRowOutcome.Facts facts = AppearancePatentRowOutcome.Facts.EMPTY;
|
||||
for (AppearancePatentResultPackager.SourceRows item
|
||||
: splitRowsBySourceFile(parsed, parsedRows, result.getSourceFilename())) {
|
||||
Map<String, AppearancePatentResultRowDto> resultMap =
|
||||
loadPersistedResultRowsForRowsWithRetry(task.getId(), item.rows());
|
||||
rowReader().loadPersistedResultRowsForRowsWithRetry(task.getId(), item.rows());
|
||||
facts = facts.plus(AppearancePatentRowOutcome.summarize(
|
||||
item.rows(), row -> findResultRow(row, resultMap)));
|
||||
item.rows(), row -> rowReader().findResultRow(row, resultMap)));
|
||||
}
|
||||
return facts;
|
||||
}
|
||||
@@ -1796,7 +1763,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
if (job == null || job.getTaskId() == null) {
|
||||
return;
|
||||
}
|
||||
deleteTransientTaskPayloads(job.getTaskId());
|
||||
payloadSupport().deleteTransientTaskPayloads(job.getTaskId());
|
||||
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, job.getTaskId())
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
@@ -1817,7 +1784,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
*/
|
||||
private AppearancePatentRowOutcome.Facts assembleResultWorkbook(FileTaskEntity task, FileResultEntity result,
|
||||
boolean allowPartial) {
|
||||
AppearancePatentParsedPayloadDto parsed = readParsedPayload(task);
|
||||
AppearancePatentParsedPayloadDto parsed = payloadSupport().readParsedPayload(task);
|
||||
List<AppearancePatentParsedRowVo> parsedRows = parsed.getAllItems() == null
|
||||
? List.of() : parsed.getAllItems();
|
||||
List<AppearancePatentResultPackager.SourceRows> sourceRows = splitRowsBySourceFile(parsed, parsedRows, result.getSourceFilename());
|
||||
@@ -1838,19 +1805,19 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
try {
|
||||
for (AppearancePatentResultPackager.SourceRows item : sourceRows) {
|
||||
Map<String, AppearancePatentResultRowDto> resultMap =
|
||||
loadPersistedResultRowsForRowsWithRetry(task.getId(), item.rows());
|
||||
rowReader().loadPersistedResultRowsForRowsWithRetry(task.getId(), item.rows());
|
||||
// 覆盖率先于「本文件无结果就跳过」判定:整源文件的 LLM 结果全丢(分片被跳过、
|
||||
// 结果清空)时覆盖率是 0%,必须在这里判掉,不能让 zip 少一个文件还报任务成功。
|
||||
deliveryFacts = deliveryFacts.plus(
|
||||
validateCompleteLlmCoverage(task.getId(), item.rows(), resultMap, allowPartial));
|
||||
List<AppearancePatentParsedRowVo> receivedRows = filterReceivedParsedRows(item.rows(), resultMap);
|
||||
List<AppearancePatentParsedRowVo> receivedRows = rowReader().filterReceivedParsedRows(item.rows(), resultMap);
|
||||
if (receivedRows.isEmpty()) {
|
||||
resultMap.clear();
|
||||
continue;
|
||||
}
|
||||
receivedRowCount += receivedRows.size();
|
||||
resolvedRows += receivedRows.stream()
|
||||
.filter(row -> findResultRow(row, resultMap) != null)
|
||||
.filter(row -> rowReader().findResultRow(row, resultMap) != null)
|
||||
.count();
|
||||
reasonRows += resultMap.values().stream()
|
||||
.filter(this::hasReasonFields)
|
||||
@@ -1858,7 +1825,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
conclusionPropagated += GroupResultPropagator.propagateByGroup(
|
||||
receivedRows,
|
||||
AppearancePatentParsedRowVo::getDisplayId,
|
||||
row -> findResultRow(row, resultMap),
|
||||
row -> rowReader().findResultRow(row, resultMap),
|
||||
AppearancePatentResultRowDto::getConclusion,
|
||||
AppearancePatentResultRowDto::setConclusion,
|
||||
List.of("已侵权", "侵权"),
|
||||
@@ -1952,20 +1919,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
return resultPackager().safeFileStem(filename);
|
||||
}
|
||||
|
||||
private List<AppearancePatentParsedRowVo> filterReceivedParsedRows(List<AppearancePatentParsedRowVo> parsedRows,
|
||||
Map<String, AppearancePatentResultRowDto> resultMap) {
|
||||
if (parsedRows == null || parsedRows.isEmpty() || resultMap == null || resultMap.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<AppearancePatentParsedRowVo> receivedRows = new ArrayList<>();
|
||||
for (AppearancePatentParsedRowVo parsedRow : parsedRows) {
|
||||
if (findResultRow(parsedRow, resultMap) != null) {
|
||||
receivedRows.add(parsedRow);
|
||||
}
|
||||
}
|
||||
return receivedRows;
|
||||
}
|
||||
|
||||
/**
|
||||
* 覆盖率判定:分母必须是**真正会写进结果文件的那批行**,且"该行有没有数据"要同时看
|
||||
* 用户上传表的解析行与客户端回传的结果行——只看解析行,用户表不带标题列时分母恒为 0,
|
||||
@@ -1986,7 +1939,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
return AppearancePatentRowOutcome.Facts.EMPTY;
|
||||
}
|
||||
AppearancePatentRowOutcome.Facts facts =
|
||||
AppearancePatentRowOutcome.summarize(parsedRows, row -> findResultRow(row, resultMap));
|
||||
AppearancePatentRowOutcome.summarize(parsedRows, row -> rowReader().findResultRow(row, resultMap));
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
boolean taskAlreadyFailed = task != null && STATUS_FAILED.equals(task.getStatus());
|
||||
if (facts.missingRows() > 0 || facts.noDataRows() > 0) {
|
||||
@@ -2022,7 +1975,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
if (parsedRow == null) {
|
||||
continue;
|
||||
}
|
||||
if (AppearancePatentRowOutcome.classify(parsedRow, findResultRow(parsedRow, resultMap))
|
||||
if (AppearancePatentRowOutcome.classify(parsedRow, rowReader().findResultRow(parsedRow, resultMap))
|
||||
!= AppearancePatentRowOutcome.Kind.NOT_INSPECTED) {
|
||||
continue;
|
||||
}
|
||||
@@ -2061,113 +2014,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
return existing.isBlank() ? gap.toString() : existing + ";" + gap;
|
||||
}
|
||||
|
||||
private boolean hasPersistedResultRows(Long taskId) {
|
||||
if (taskId == null) {
|
||||
return false;
|
||||
}
|
||||
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
return count != null && count > 0;
|
||||
}
|
||||
|
||||
private Map<String, AppearancePatentResultRowDto> loadPersistedResultRowsForRowsWithRetry(
|
||||
Long taskId, List<AppearancePatentParsedRowVo> requestedRows) {
|
||||
Set<String> requestedKeys = resultLookupKeysForParsedRows(requestedRows);
|
||||
Map<String, AppearancePatentResultRowDto> result =
|
||||
loadPersistedResultRowsForKeys(taskId, requestedKeys);
|
||||
if (requestedKeys.isEmpty() || !result.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) {
|
||||
sleepBeforeResultRowsRetry(attempt);
|
||||
result = loadPersistedResultRowsForKeys(taskId, requestedKeys);
|
||||
if (!result.isEmpty()) {
|
||||
log.info("[appearance-patent] result rows recovered after retry taskId={} attempt={} rows={}",
|
||||
taskId, attempt, result.size());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
private Map<String, AppearancePatentResultRowDto> loadPersistedResultRowsForKeys(
|
||||
Long taskId, Set<String> requestedKeys) {
|
||||
Map<String, AppearancePatentResultRowDto> result = new LinkedHashMap<>();
|
||||
if (taskId == null || requestedKeys == null || requestedKeys.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
Long lastChunkId = null;
|
||||
while (true) {
|
||||
LambdaQueryWrapper<TaskChunkEntity> query = new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.orderByAsc(TaskChunkEntity::getId)
|
||||
.last("LIMIT " + RESULT_ROWS_CHUNK_PAGE_SIZE);
|
||||
if (lastChunkId != null) {
|
||||
query.gt(TaskChunkEntity::getId, lastChunkId);
|
||||
}
|
||||
List<TaskChunkEntity> page = taskChunkMapper.selectList(query);
|
||||
if (page == null || page.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
// 整页取 chunk 后逐片解载荷:原先每轮 LIMIT 1 只取一行,一个结果文件要把该任务的
|
||||
// chunk 表逐行翻完,每行一次 SQL + 一次对象读 + 一次整片 JSON 反序列化
|
||||
for (TaskChunkEntity chunk : page) {
|
||||
appendRequestedResultRows(result, readChunkRows(chunk).values(), requestedKeys);
|
||||
}
|
||||
TaskChunkEntity tail = page.getLast();
|
||||
lastChunkId = tail == null ? null : tail.getId();
|
||||
if (lastChunkId == null || page.size() < RESULT_ROWS_CHUNK_PAGE_SIZE) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private Set<String> resultLookupKeysForParsedRows(List<AppearancePatentParsedRowVo> rows) {
|
||||
Set<String> keys = new LinkedHashSet<>();
|
||||
for (AppearancePatentParsedRowVo row : rows == null ? List.<AppearancePatentParsedRowVo>of() : rows) {
|
||||
String primary = rowKey(row);
|
||||
if (!primary.isBlank()) {
|
||||
keys.add(primary);
|
||||
}
|
||||
String legacy = legacyRowKey(row);
|
||||
if (!legacy.isBlank()) {
|
||||
keys.add(legacy);
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
private void appendRequestedResultRows(Map<String, AppearancePatentResultRowDto> sink,
|
||||
Iterable<AppearancePatentResultRowDto> rows,
|
||||
Set<String> requestedKeys) {
|
||||
if (rows == null || requestedKeys == null || requestedKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (AppearancePatentResultRowDto row : rows) {
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
String primary = rowKey(row);
|
||||
String legacy = legacyRowKey(row);
|
||||
String matchedKey = requestedKeys.contains(primary)
|
||||
? primary : requestedKeys.contains(legacy) ? legacy : "";
|
||||
if (!matchedKey.isBlank()) {
|
||||
sink.put(matchedKey, row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void sleepBeforeResultRowsRetry(int attempt) {
|
||||
try {
|
||||
Thread.sleep(RESULT_ROWS_READ_RETRY_DELAY_MS * attempt);
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
private void writeResultWorkbook(File xlsx,
|
||||
AppearancePatentParsedPayloadDto parsed,
|
||||
@@ -2175,7 +2021,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
Map<String, AppearancePatentResultRowDto> resultMap) {
|
||||
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
|
||||
AppearancePatentSheetBuilder.buildResultSheet(workbook, receivedRows, resultMap,
|
||||
parsedRow -> findResultRow(parsedRow, resultMap));
|
||||
parsedRow -> rowReader().findResultRow(parsedRow, resultMap));
|
||||
workbook.write(fos);
|
||||
workbook.dispose();
|
||||
} catch (Exception ex) {
|
||||
@@ -2470,226 +2316,16 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
private String buildParsedPayloadJson(String aiPrompt, String apiKey, List<AppearancePatentSourceFileDto> sourceFiles, List<String> headers, List<AppearancePatentParsedRowVo> allRows) {
|
||||
AppearancePatentParsedPayloadDto payload = new AppearancePatentParsedPayloadDto();
|
||||
payload.setAiPrompt(normalize(aiPrompt));
|
||||
payload.setApiKey(normalize(apiKey));
|
||||
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
|
||||
payload.setHeaders(headers == null ? List.of() : headers);
|
||||
payload.setItems(List.of());
|
||||
payload.setGroups(List.of());
|
||||
payload.setAllItems(allRows == null ? List.of() : allRows);
|
||||
return writeJson(objectMapper, payload, "保存解析结果失败");
|
||||
}
|
||||
|
||||
private String buildTaskResultJson(String aiPrompt, String apiKey, List<AppearancePatentSourceFileDto> sourceFiles, String parsedPayloadPointer) {
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put("aiPrompt", normalize(aiPrompt));
|
||||
payload.put("apiKey", normalize(apiKey));
|
||||
payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream()
|
||||
.map(AppearancePatentSourceFileDto::getFileKey)
|
||||
.filter(Objects::nonNull)
|
||||
.toList());
|
||||
payload.put("parsedPayloadRef", parsedPayloadPointer);
|
||||
payload.put("ownerInstanceId", currentInstanceId());
|
||||
return writeJson(objectMapper, payload, "保存任务结果索引失败");
|
||||
}
|
||||
|
||||
private String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) {
|
||||
requireSharedTransientPayloadStorage("parsed payload");
|
||||
return transientPayloadStorageService.storeParsedPayloadFast(MODULE_TYPE, taskId, scopeHash, parsedPayloadJson, false);
|
||||
}
|
||||
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId, long waitMillis) {
|
||||
return taskDistributedLockService.acquire(MODULE_TYPE, taskId, TASK_LOCK_TTL, waitMillis);
|
||||
}
|
||||
|
||||
private String storeSharedChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
requireSharedTransientPayloadStorage("chunk payload");
|
||||
return transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
|
||||
}
|
||||
|
||||
private String storeSharedChunkPayloadVersioned(Long taskId, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
requireSharedTransientPayloadStorage("merged chunk payload");
|
||||
return transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
|
||||
}
|
||||
|
||||
|
||||
private void requireSharedTransientPayloadStorage(String payloadType) {
|
||||
if (!transientPayloadStorageService.isSharedWriteEnabled()) {
|
||||
throw new BusinessException("外观专利" + payloadType + "必须写入共享临时存储 RustFS,请检查 aiimage.transient-storage 配置");
|
||||
}
|
||||
}
|
||||
|
||||
private long elapsedMs(long start, long end) {
|
||||
return (end - start) / 1_000_000L;
|
||||
}
|
||||
|
||||
private AppearancePatentParsedPayloadDto readParsedPayload(FileTaskEntity task) {
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(task.getResultJson());
|
||||
String pointer = root.path("parsedPayloadRef").asText("");
|
||||
if (!pointer.isBlank()) {
|
||||
String json = transientPayloadStorageService.resolvePayload(pointer, "read appearance patent parsed payload failed");
|
||||
return hydrateParsedPayloadRows(objectMapper.readValue(json, AppearancePatentParsedPayloadDto.class));
|
||||
}
|
||||
if (root.path("allItems").isArray()) {
|
||||
return hydrateParsedPayloadRows(objectMapper.treeToValue(root, AppearancePatentParsedPayloadDto.class));
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取解析载荷失败");
|
||||
}
|
||||
return hydrateParsedPayloadRows(new AppearancePatentParsedPayloadDto());
|
||||
}
|
||||
|
||||
public AppearancePatentParsedPayloadDto queuePayload(Long taskId, Long userId) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
ensureTaskOwnedByCurrentInstance(task, "queue payload");
|
||||
AppearancePatentParsedPayloadDto payload = readParsedPayload(task);
|
||||
AppearancePatentParsedPayloadDto queuePayload = new AppearancePatentParsedPayloadDto();
|
||||
queuePayload.setAiPrompt(payload.getAiPrompt());
|
||||
queuePayload.setApiKey(payload.getApiKey());
|
||||
queuePayload.setGroups(payload.getGroups() == null ? List.of() : payload.getGroups());
|
||||
queuePayload.setItems(List.of());
|
||||
queuePayload.setAllItems(List.of());
|
||||
queuePayload.setHeaders(List.of());
|
||||
queuePayload.setSourceFiles(List.of());
|
||||
return queuePayload;
|
||||
}
|
||||
|
||||
private AppearancePatentParsedPayloadDto hydrateParsedPayloadRows(AppearancePatentParsedPayloadDto payload) {
|
||||
if (payload == null) {
|
||||
return new AppearancePatentParsedPayloadDto();
|
||||
}
|
||||
List<AppearancePatentParsedRowVo> rows = payload.getAllItems();
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
rows = payload.getItems();
|
||||
}
|
||||
if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) {
|
||||
rows = payload.getGroups().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream())
|
||||
.toList();
|
||||
}
|
||||
if (rows == null) {
|
||||
rows = List.of();
|
||||
}
|
||||
if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) {
|
||||
payload.setAllItems(rows);
|
||||
}
|
||||
if (payload.getItems() == null || payload.getItems().isEmpty()) {
|
||||
payload.setItems(rows);
|
||||
}
|
||||
if ((payload.getGroups() == null || payload.getGroups().isEmpty()) && !rows.isEmpty()) {
|
||||
payload.setGroups(buildParsedGroups(rows));
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
private Map<String, AppearancePatentResultRowDto> readChunkRows(TaskChunkEntity chunk) {
|
||||
Map<String, AppearancePatentResultRowDto> rows = new LinkedHashMap<>();
|
||||
if (chunk == null || chunk.getPayloadJson() == null || chunk.getPayloadJson().isBlank()) {
|
||||
return rows;
|
||||
}
|
||||
try {
|
||||
String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read appearance patent chunk failed");
|
||||
JsonNode array = objectMapper.readTree(payloadJson);
|
||||
if (!array.isArray()) {
|
||||
return rows;
|
||||
}
|
||||
for (JsonNode node : array) {
|
||||
AppearancePatentResultRowDto row = objectMapper.treeToValue(node, AppearancePatentResultRowDto.class);
|
||||
rows.put(rowKey(row), row);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent] read chunk payload failed taskId={} chunk={} err={}",
|
||||
chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage());
|
||||
if (isPayloadMissing(ex)) {
|
||||
// payload 对象已不在(被清理或从未写入):重试多少次都读不回来。继续抛会让
|
||||
// ASSEMBLE_RESULT job 的终态回调每轮重跑兜底组装 → 再读同一个缺失对象 → 无限循环
|
||||
// (线上任务 28459 每 10~30 秒重试一次)。跳过该分片,让任务按已有分片出部分结果,
|
||||
// 与品牌/相似ASIN「失败也产出可下载的部分结果」同一口径。
|
||||
log.warn("[appearance-patent] chunk payload 已不存在,跳过该分片(任务按已有分片出结果)"
|
||||
+ " taskId={} chunk={}", chunk.getTaskId(), chunk.getChunkIndex());
|
||||
return rows;
|
||||
}
|
||||
throw new BusinessException("读取外观专利结果分片失败,分片="
|
||||
+ chunk.getChunkIndex() + ": " + ex.getMessage(), ex);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** payload 对象已不存在(RustFS 返回 NoSuchKey:message 为 "The specified key does not exist.")。
|
||||
* 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。 */
|
||||
private static boolean isPayloadMissing(Throwable error) {
|
||||
Throwable cursor = error;
|
||||
while (cursor != null) {
|
||||
String message = cursor.getMessage();
|
||||
if (message != null && message.contains("does not exist")) {
|
||||
return true;
|
||||
}
|
||||
cursor = cursor.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private String rowKey(AppearancePatentParsedRowVo row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
}
|
||||
String rowToken = normalize(row.getRowToken());
|
||||
if (!rowToken.isBlank()) {
|
||||
return rowToken;
|
||||
}
|
||||
return legacyRowKey(row);
|
||||
}
|
||||
|
||||
private String rowKey(AppearancePatentResultRowDto row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
}
|
||||
String rowToken = normalize(row.getRowToken());
|
||||
if (!rowToken.isBlank()) {
|
||||
return rowToken;
|
||||
}
|
||||
return legacyRowKey(row);
|
||||
}
|
||||
|
||||
private AppearancePatentResultRowDto findResultRow(AppearancePatentParsedRowVo parsedRow,
|
||||
Map<String, AppearancePatentResultRowDto> resultMap) {
|
||||
if (parsedRow == null || resultMap == null || resultMap.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
AppearancePatentResultRowDto resultRow = resultMap.get(rowKey(parsedRow));
|
||||
if (resultRow != null) {
|
||||
return resultRow;
|
||||
}
|
||||
String legacyKey = legacyRowKey(parsedRow);
|
||||
if (legacyKey.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return resultMap.get(legacyKey);
|
||||
}
|
||||
|
||||
private String legacyRowKey(AppearancePatentParsedRowVo row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
}
|
||||
return rowKey(row.getDisplayId(), row.getAsin(), row.getCountry());
|
||||
}
|
||||
|
||||
private String legacyRowKey(AppearancePatentResultRowDto row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
}
|
||||
return rowKey(row.getId(), row.getAsin(), row.getCountry());
|
||||
}
|
||||
|
||||
private String rowKey(String id, String asin, String country) {
|
||||
return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
|
||||
return payloadSupport().queuePayload(taskId, userId);
|
||||
}
|
||||
|
||||
static String resolveTaskExecutionStatus(boolean waitingForAssemble, boolean executionFailed) {
|
||||
@@ -2699,66 +2335,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
return executionFailed ? STATUS_FAILED : STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
/** 只读收集任务范围/分片载荷指针,供事务提交后做远端删除。 */
|
||||
private List<String> collectTransientTaskPayloads(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> payloads = new ArrayList<>();
|
||||
List<TaskScopeStateEntity> scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||
if (scopes != null) {
|
||||
for (TaskScopeStateEntity scope : scopes) {
|
||||
if (scope == null) {
|
||||
continue;
|
||||
}
|
||||
if (scope.getParsedPayloadJson() != null && !scope.getParsedPayloadJson().isBlank()) {
|
||||
payloads.add(scope.getParsedPayloadJson());
|
||||
}
|
||||
if (scope.getStateJson() != null && !scope.getStateJson().isBlank()) {
|
||||
payloads.add(scope.getStateJson());
|
||||
}
|
||||
}
|
||||
}
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.select(TaskChunkEntity::getPayloadJson)
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
if (chunks != null) {
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
|
||||
payloads.add(chunk.getPayloadJson());
|
||||
}
|
||||
}
|
||||
}
|
||||
return payloads;
|
||||
}
|
||||
|
||||
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
|
||||
private void deletePayloadsAfterCommit(List<String> payloads, Long taskId) {
|
||||
if (payloads == null || payloads.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
runAfterCommit(() -> {
|
||||
for (String payload : payloads) {
|
||||
try {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(payload);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage());
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除范围/分片的远端载荷(保留给无事务的清理链路调用)。
|
||||
*/
|
||||
private void deleteTransientTaskPayloads(Long taskId) {
|
||||
deletePayloadsAfterCommit(collectTransientTaskPayloads(taskId), taskId);
|
||||
}
|
||||
|
||||
/** 有活动事务则注册 afterCommit,否则立即执行。 */
|
||||
private void runAfterCommit(Runnable action) {
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSourceFileDto;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* 外观专利的载荷/JSON 读写(I 聚类),从 {@code AppearancePatentTaskService} <b>原样搬移</b>
|
||||
* (3A 拆薄第六笔):解析载荷的组装/落库/读取、分片载荷写入、任务载荷指针收集与事务后删除,
|
||||
* 连同异常文案与删除顺序一个字没改。
|
||||
*
|
||||
* <p><b>搬的是载荷读写,不是宿主能力。</b>字符串归一({@code normalize})、解析分组重建
|
||||
* ({@code buildParsedGroups})、实例 id、归属校验与 afterCommit 调度都还留在服务侧,
|
||||
* 以回调交进来——它们横跨解析/归属/事务三个面,跟着载荷一起拖出来只会把依赖面放大。
|
||||
*/
|
||||
@Slf4j
|
||||
public class AppearancePatentPayloadSupport {
|
||||
|
||||
/** 序列化失败时的统一出口(服务侧 {@code writeJson} 的薄适配)。 */
|
||||
@FunctionalInterface
|
||||
public interface JsonWriter {
|
||||
String write(Object value, String failureMessage);
|
||||
}
|
||||
|
||||
private final String moduleType;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final TaskChunkMapper taskChunkMapper;
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final Function<String, String> normalizer;
|
||||
private final Function<List<AppearancePatentParsedRowVo>, List<AppearancePatentParsedGroupVo>> groupBuilder;
|
||||
private final Supplier<String> currentInstanceId;
|
||||
private final BiConsumer<FileTaskEntity, String> ownershipChecker;
|
||||
private final Consumer<Runnable> afterCommitRunner;
|
||||
private final JsonWriter jsonWriter;
|
||||
|
||||
public AppearancePatentPayloadSupport(String moduleType,
|
||||
ObjectMapper objectMapper,
|
||||
TransientPayloadStorageService transientPayloadStorageService,
|
||||
FileTaskMapper fileTaskMapper,
|
||||
TaskChunkMapper taskChunkMapper,
|
||||
TaskScopeStateMapper taskScopeStateMapper,
|
||||
Function<String, String> normalizer,
|
||||
Function<List<AppearancePatentParsedRowVo>, List<AppearancePatentParsedGroupVo>> groupBuilder,
|
||||
Supplier<String> currentInstanceId,
|
||||
BiConsumer<FileTaskEntity, String> ownershipChecker,
|
||||
Consumer<Runnable> afterCommitRunner,
|
||||
JsonWriter jsonWriter) {
|
||||
this.moduleType = moduleType;
|
||||
this.objectMapper = objectMapper;
|
||||
this.transientPayloadStorageService = transientPayloadStorageService;
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
this.taskChunkMapper = taskChunkMapper;
|
||||
this.taskScopeStateMapper = taskScopeStateMapper;
|
||||
this.normalizer = normalizer;
|
||||
this.groupBuilder = groupBuilder;
|
||||
this.currentInstanceId = currentInstanceId;
|
||||
this.ownershipChecker = ownershipChecker;
|
||||
this.afterCommitRunner = afterCommitRunner;
|
||||
this.jsonWriter = jsonWriter;
|
||||
}
|
||||
|
||||
public String buildParsedPayloadJson(String aiPrompt, String apiKey,
|
||||
List<AppearancePatentSourceFileDto> sourceFiles,
|
||||
List<String> headers,
|
||||
List<AppearancePatentParsedRowVo> allRows) {
|
||||
AppearancePatentParsedPayloadDto payload = new AppearancePatentParsedPayloadDto();
|
||||
payload.setAiPrompt(normalizer.apply(aiPrompt));
|
||||
payload.setApiKey(normalizer.apply(apiKey));
|
||||
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
|
||||
payload.setHeaders(headers == null ? List.of() : headers);
|
||||
payload.setItems(List.of());
|
||||
payload.setGroups(List.of());
|
||||
payload.setAllItems(allRows == null ? List.of() : allRows);
|
||||
return jsonWriter.write(payload, "保存解析结果失败");
|
||||
}
|
||||
|
||||
public String buildTaskResultJson(String aiPrompt, String apiKey,
|
||||
List<AppearancePatentSourceFileDto> sourceFiles,
|
||||
String parsedPayloadPointer) {
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put("aiPrompt", normalizer.apply(aiPrompt));
|
||||
payload.put("apiKey", normalizer.apply(apiKey));
|
||||
payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream()
|
||||
.map(AppearancePatentSourceFileDto::getFileKey)
|
||||
.filter(Objects::nonNull)
|
||||
.toList());
|
||||
payload.put("parsedPayloadRef", parsedPayloadPointer);
|
||||
payload.put("ownerInstanceId", currentInstanceId.get());
|
||||
return jsonWriter.write(payload, "保存任务结果索引失败");
|
||||
}
|
||||
|
||||
public String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) {
|
||||
requireSharedTransientPayloadStorage("parsed payload");
|
||||
return transientPayloadStorageService.storeParsedPayloadFast(moduleType, taskId, scopeHash, parsedPayloadJson, false);
|
||||
}
|
||||
|
||||
public String storeSharedChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
requireSharedTransientPayloadStorage("chunk payload");
|
||||
return transientPayloadStorageService.storeChunkPayload(moduleType, taskId, scopeHash, chunkIndex, payloadJson);
|
||||
}
|
||||
|
||||
public String storeSharedChunkPayloadVersioned(Long taskId, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
requireSharedTransientPayloadStorage("merged chunk payload");
|
||||
return transientPayloadStorageService.storeChunkPayloadVersioned(moduleType, taskId, scopeHash, chunkIndex, payloadJson);
|
||||
}
|
||||
|
||||
private void requireSharedTransientPayloadStorage(String payloadType) {
|
||||
if (!transientPayloadStorageService.isSharedWriteEnabled()) {
|
||||
throw new BusinessException("外观专利" + payloadType + "必须写入共享临时存储 RustFS,请检查 aiimage.transient-storage 配置");
|
||||
}
|
||||
}
|
||||
|
||||
public AppearancePatentParsedPayloadDto readParsedPayload(FileTaskEntity task) {
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(task.getResultJson());
|
||||
String pointer = root.path("parsedPayloadRef").asText("");
|
||||
if (!pointer.isBlank()) {
|
||||
String json = transientPayloadStorageService.resolvePayload(pointer, "read appearance patent parsed payload failed");
|
||||
return hydrateParsedPayloadRows(objectMapper.readValue(json, AppearancePatentParsedPayloadDto.class));
|
||||
}
|
||||
if (root.path("allItems").isArray()) {
|
||||
return hydrateParsedPayloadRows(objectMapper.treeToValue(root, AppearancePatentParsedPayloadDto.class));
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取解析载荷失败");
|
||||
}
|
||||
return hydrateParsedPayloadRows(new AppearancePatentParsedPayloadDto());
|
||||
}
|
||||
|
||||
public AppearancePatentParsedPayloadDto queuePayload(Long taskId, Long userId) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !moduleType.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
ownershipChecker.accept(task, "queue payload");
|
||||
AppearancePatentParsedPayloadDto payload = readParsedPayload(task);
|
||||
AppearancePatentParsedPayloadDto queuePayload = new AppearancePatentParsedPayloadDto();
|
||||
queuePayload.setAiPrompt(payload.getAiPrompt());
|
||||
queuePayload.setApiKey(payload.getApiKey());
|
||||
queuePayload.setGroups(payload.getGroups() == null ? List.of() : payload.getGroups());
|
||||
queuePayload.setItems(List.of());
|
||||
queuePayload.setAllItems(List.of());
|
||||
queuePayload.setHeaders(List.of());
|
||||
queuePayload.setSourceFiles(List.of());
|
||||
return queuePayload;
|
||||
}
|
||||
|
||||
private AppearancePatentParsedPayloadDto hydrateParsedPayloadRows(AppearancePatentParsedPayloadDto payload) {
|
||||
if (payload == null) {
|
||||
return new AppearancePatentParsedPayloadDto();
|
||||
}
|
||||
List<AppearancePatentParsedRowVo> rows = payload.getAllItems();
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
rows = payload.getItems();
|
||||
}
|
||||
if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) {
|
||||
rows = payload.getGroups().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream())
|
||||
.toList();
|
||||
}
|
||||
if (rows == null) {
|
||||
rows = List.of();
|
||||
}
|
||||
if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) {
|
||||
payload.setAllItems(rows);
|
||||
}
|
||||
if (payload.getItems() == null || payload.getItems().isEmpty()) {
|
||||
payload.setItems(rows);
|
||||
}
|
||||
if ((payload.getGroups() == null || payload.getGroups().isEmpty()) && !rows.isEmpty()) {
|
||||
payload.setGroups(groupBuilder.apply(rows));
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
/** 只读收集任务范围/分片载荷指针,供事务提交后做远端删除。 */
|
||||
public List<String> collectTransientTaskPayloads(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> payloads = new ArrayList<>();
|
||||
List<TaskScopeStateEntity> scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, moduleType));
|
||||
if (scopes != null) {
|
||||
for (TaskScopeStateEntity scope : scopes) {
|
||||
if (scope == null) {
|
||||
continue;
|
||||
}
|
||||
if (scope.getParsedPayloadJson() != null && !scope.getParsedPayloadJson().isBlank()) {
|
||||
payloads.add(scope.getParsedPayloadJson());
|
||||
}
|
||||
if (scope.getStateJson() != null && !scope.getStateJson().isBlank()) {
|
||||
payloads.add(scope.getStateJson());
|
||||
}
|
||||
}
|
||||
}
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.select(TaskChunkEntity::getPayloadJson)
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, moduleType));
|
||||
if (chunks != null) {
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
|
||||
payloads.add(chunk.getPayloadJson());
|
||||
}
|
||||
}
|
||||
}
|
||||
return payloads;
|
||||
}
|
||||
|
||||
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
|
||||
public void deletePayloadsAfterCommit(List<String> payloads, Long taskId) {
|
||||
if (payloads == null || payloads.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
afterCommitRunner.accept(() -> {
|
||||
for (String payload : payloads) {
|
||||
try {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(payload);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage());
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除范围/分片的远端载荷(保留给无事务的清理链路调用)。
|
||||
*/
|
||||
public void deleteTransientTaskPayloads(Long taskId) {
|
||||
deletePayloadsAfterCommit(collectTransientTaskPayloads(taskId), taskId);
|
||||
}
|
||||
}
|
||||
+344
@@ -0,0 +1,344 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultGroupDto;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
|
||||
/**
|
||||
* 外观专利的结果行读取(H 聚类),从 {@code AppearancePatentTaskService} <b>原样搬移</b>
|
||||
* (3A 拆薄第七笔):回传分组展开、解析行计数、分片载荷读取、结果行按键分页读取与行键口径,
|
||||
* 连同分页大小、重试次数、缺失载荷的跳过判据一个字没改。
|
||||
*
|
||||
* <p><b>搬的是读取与行键,不是宿主。</b>解析载荷读取(I 聚类)仍由服务侧传入;
|
||||
* 字符串归一也以回调交入,避免把解析面的工具再抄一份。分片读取的异常语义保持原样:
|
||||
* 只有 RustFS 的 "does not exist" 才允许跳过该分片,网络类失败仍抛出以便重试。
|
||||
*/
|
||||
@Slf4j
|
||||
public class AppearancePatentResultRowReader {
|
||||
|
||||
/** 结果行读取时 chunk 的 keyset 页大小,与 similarasin 侧 loadChunksKeyset 的 500 同口径。 */
|
||||
private static final int RESULT_ROWS_CHUNK_PAGE_SIZE = 500;
|
||||
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
|
||||
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
|
||||
|
||||
private final String moduleType;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||
private final TaskChunkMapper taskChunkMapper;
|
||||
private final Function<String, String> normalizer;
|
||||
private final Function<FileTaskEntity, AppearancePatentParsedPayloadDto> parsedPayloadReader;
|
||||
|
||||
public AppearancePatentResultRowReader(String moduleType,
|
||||
ObjectMapper objectMapper,
|
||||
TransientPayloadStorageService transientPayloadStorageService,
|
||||
TaskChunkMapper taskChunkMapper,
|
||||
Function<String, String> normalizer,
|
||||
Function<FileTaskEntity, AppearancePatentParsedPayloadDto> parsedPayloadReader) {
|
||||
this.moduleType = moduleType;
|
||||
this.objectMapper = objectMapper;
|
||||
this.transientPayloadStorageService = transientPayloadStorageService;
|
||||
this.taskChunkMapper = taskChunkMapper;
|
||||
this.normalizer = normalizer;
|
||||
this.parsedPayloadReader = parsedPayloadReader;
|
||||
}
|
||||
|
||||
public List<AppearancePatentResultRowDto> flattenSubmittedRows(AppearancePatentSubmitResultRequest request) {
|
||||
if (request == null) {
|
||||
return List.of();
|
||||
}
|
||||
List<AppearancePatentResultGroupDto> groups = request.getGroups();
|
||||
if (groups != null && !groups.isEmpty()) {
|
||||
List<AppearancePatentResultRowDto> rows = new ArrayList<>();
|
||||
for (AppearancePatentResultGroupDto group : groups) {
|
||||
if (group == null || group.getItems() == null || group.getItems().isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
for (AppearancePatentResultRowDto row : group.getItems()) {
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
if (normalizer.apply(row.getGroupKey()).isBlank()) {
|
||||
row.setGroupKey(group.getGroupKey());
|
||||
}
|
||||
if (normalizer.apply(row.getSourceFileKey()).isBlank()) {
|
||||
row.setSourceFileKey(group.getSourceFileKey());
|
||||
}
|
||||
if (normalizer.apply(row.getSourceFilename()).isBlank()) {
|
||||
row.setSourceFilename(group.getSourceFilename());
|
||||
}
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
return request.getItems() == null ? List.of() : request.getItems();
|
||||
}
|
||||
|
||||
public int allRowCount(FileTaskEntity task) {
|
||||
try {
|
||||
AppearancePatentParsedPayloadDto payload = parsedPayloadReader.apply(task);
|
||||
if (payload.getAllItems() != null && !payload.getAllItems().isEmpty()) {
|
||||
return payload.getAllItems().size();
|
||||
}
|
||||
if (payload.getItems() != null && !payload.getItems().isEmpty()) {
|
||||
return payload.getItems().size();
|
||||
}
|
||||
if (payload.getGroups() != null && !payload.getGroups().isEmpty()) {
|
||||
return payload.getGroups().stream()
|
||||
.map(AppearancePatentParsedGroupVo::getItems)
|
||||
.filter(Objects::nonNull)
|
||||
.mapToInt(List::size)
|
||||
.sum();
|
||||
}
|
||||
return 0;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent] read all row count failed taskId={} err={}", task.getId(), ex.getMessage());
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
public List<AppearancePatentParsedRowVo> filterReceivedParsedRows(List<AppearancePatentParsedRowVo> parsedRows,
|
||||
Map<String, AppearancePatentResultRowDto> resultMap) {
|
||||
if (parsedRows == null || parsedRows.isEmpty() || resultMap == null || resultMap.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<AppearancePatentParsedRowVo> receivedRows = new ArrayList<>();
|
||||
for (AppearancePatentParsedRowVo parsedRow : parsedRows) {
|
||||
if (findResultRow(parsedRow, resultMap) != null) {
|
||||
receivedRows.add(parsedRow);
|
||||
}
|
||||
}
|
||||
return receivedRows;
|
||||
}
|
||||
|
||||
public boolean hasPersistedResultRows(Long taskId) {
|
||||
if (taskId == null) {
|
||||
return false;
|
||||
}
|
||||
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, moduleType));
|
||||
return count != null && count > 0;
|
||||
}
|
||||
|
||||
public Map<String, AppearancePatentResultRowDto> loadPersistedResultRowsForRowsWithRetry(
|
||||
Long taskId, List<AppearancePatentParsedRowVo> requestedRows) {
|
||||
Set<String> requestedKeys = resultLookupKeysForParsedRows(requestedRows);
|
||||
Map<String, AppearancePatentResultRowDto> result =
|
||||
loadPersistedResultRowsForKeys(taskId, requestedKeys);
|
||||
if (requestedKeys.isEmpty() || !result.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) {
|
||||
sleepBeforeResultRowsRetry(attempt);
|
||||
result = loadPersistedResultRowsForKeys(taskId, requestedKeys);
|
||||
if (!result.isEmpty()) {
|
||||
log.info("[appearance-patent] result rows recovered after retry taskId={} attempt={} rows={}",
|
||||
taskId, attempt, result.size());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private Map<String, AppearancePatentResultRowDto> loadPersistedResultRowsForKeys(
|
||||
Long taskId, Set<String> requestedKeys) {
|
||||
Map<String, AppearancePatentResultRowDto> result = new LinkedHashMap<>();
|
||||
if (taskId == null || requestedKeys == null || requestedKeys.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
Long lastChunkId = null;
|
||||
while (true) {
|
||||
LambdaQueryWrapper<TaskChunkEntity> query = new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, moduleType)
|
||||
.orderByAsc(TaskChunkEntity::getId)
|
||||
.last("LIMIT " + RESULT_ROWS_CHUNK_PAGE_SIZE);
|
||||
if (lastChunkId != null) {
|
||||
query.gt(TaskChunkEntity::getId, lastChunkId);
|
||||
}
|
||||
List<TaskChunkEntity> page = taskChunkMapper.selectList(query);
|
||||
if (page == null || page.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
// 整页取 chunk 后逐片解载荷:原先每轮 LIMIT 1 只取一行,一个结果文件要把该任务的
|
||||
// chunk 表逐行翻完,每行一次 SQL + 一次对象读 + 一次整片 JSON 反序列化
|
||||
for (TaskChunkEntity chunk : page) {
|
||||
appendRequestedResultRows(result, readChunkRows(chunk).values(), requestedKeys);
|
||||
}
|
||||
TaskChunkEntity tail = page.getLast();
|
||||
lastChunkId = tail == null ? null : tail.getId();
|
||||
if (lastChunkId == null || page.size() < RESULT_ROWS_CHUNK_PAGE_SIZE) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private Set<String> resultLookupKeysForParsedRows(List<AppearancePatentParsedRowVo> rows) {
|
||||
Set<String> keys = new LinkedHashSet<>();
|
||||
for (AppearancePatentParsedRowVo row : rows == null ? List.<AppearancePatentParsedRowVo>of() : rows) {
|
||||
String primary = rowKey(row);
|
||||
if (!primary.isBlank()) {
|
||||
keys.add(primary);
|
||||
}
|
||||
String legacy = legacyRowKey(row);
|
||||
if (!legacy.isBlank()) {
|
||||
keys.add(legacy);
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
private void appendRequestedResultRows(Map<String, AppearancePatentResultRowDto> sink,
|
||||
Iterable<AppearancePatentResultRowDto> rows,
|
||||
Set<String> requestedKeys) {
|
||||
if (rows == null || requestedKeys == null || requestedKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (AppearancePatentResultRowDto row : rows) {
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
String primary = rowKey(row);
|
||||
String legacy = legacyRowKey(row);
|
||||
String matchedKey = requestedKeys.contains(primary)
|
||||
? primary : requestedKeys.contains(legacy) ? legacy : "";
|
||||
if (!matchedKey.isBlank()) {
|
||||
sink.put(matchedKey, row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void sleepBeforeResultRowsRetry(int attempt) {
|
||||
try {
|
||||
Thread.sleep(RESULT_ROWS_READ_RETRY_DELAY_MS * attempt);
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
public Map<String, AppearancePatentResultRowDto> readChunkRows(TaskChunkEntity chunk) {
|
||||
Map<String, AppearancePatentResultRowDto> rows = new LinkedHashMap<>();
|
||||
if (chunk == null || chunk.getPayloadJson() == null || chunk.getPayloadJson().isBlank()) {
|
||||
return rows;
|
||||
}
|
||||
try {
|
||||
String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read appearance patent chunk failed");
|
||||
JsonNode array = objectMapper.readTree(payloadJson);
|
||||
if (!array.isArray()) {
|
||||
return rows;
|
||||
}
|
||||
for (JsonNode node : array) {
|
||||
AppearancePatentResultRowDto row = objectMapper.treeToValue(node, AppearancePatentResultRowDto.class);
|
||||
rows.put(rowKey(row), row);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent] read chunk payload failed taskId={} chunk={} err={}",
|
||||
chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage());
|
||||
if (isPayloadMissing(ex)) {
|
||||
// payload 对象已不在(被清理或从未写入):重试多少次都读不回来。继续抛会让
|
||||
// ASSEMBLE_RESULT job 的终态回调每轮重跑兜底组装 → 再读同一个缺失对象 → 无限循环
|
||||
// (线上任务 28459 每 10~30 秒重试一次)。跳过该分片,让任务按已有分片出部分结果,
|
||||
// 与品牌/相似ASIN「失败也产出可下载的部分结果」同一口径。
|
||||
log.warn("[appearance-patent] chunk payload 已不存在,跳过该分片(任务按已有分片出结果)"
|
||||
+ " taskId={} chunk={}", chunk.getTaskId(), chunk.getChunkIndex());
|
||||
return rows;
|
||||
}
|
||||
throw new BusinessException("读取外观专利结果分片失败,分片="
|
||||
+ chunk.getChunkIndex() + ": " + ex.getMessage(), ex);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** payload 对象已不存在(RustFS 返回 NoSuchKey:message 为 "The specified key does not exist.")。
|
||||
* 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。 */
|
||||
private static boolean isPayloadMissing(Throwable error) {
|
||||
Throwable cursor = error;
|
||||
while (cursor != null) {
|
||||
String message = cursor.getMessage();
|
||||
if (message != null && message.contains("does not exist")) {
|
||||
return true;
|
||||
}
|
||||
cursor = cursor.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public String rowKey(AppearancePatentParsedRowVo row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
}
|
||||
String rowToken = normalizer.apply(row.getRowToken());
|
||||
if (!rowToken.isBlank()) {
|
||||
return rowToken;
|
||||
}
|
||||
return legacyRowKey(row);
|
||||
}
|
||||
|
||||
public String rowKey(AppearancePatentResultRowDto row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
}
|
||||
String rowToken = normalizer.apply(row.getRowToken());
|
||||
if (!rowToken.isBlank()) {
|
||||
return rowToken;
|
||||
}
|
||||
return legacyRowKey(row);
|
||||
}
|
||||
|
||||
public AppearancePatentResultRowDto findResultRow(AppearancePatentParsedRowVo parsedRow,
|
||||
Map<String, AppearancePatentResultRowDto> resultMap) {
|
||||
if (parsedRow == null || resultMap == null || resultMap.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
AppearancePatentResultRowDto resultRow = resultMap.get(rowKey(parsedRow));
|
||||
if (resultRow != null) {
|
||||
return resultRow;
|
||||
}
|
||||
String legacyKey = legacyRowKey(parsedRow);
|
||||
if (legacyKey.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return resultMap.get(legacyKey);
|
||||
}
|
||||
|
||||
private String legacyRowKey(AppearancePatentParsedRowVo row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
}
|
||||
return rowKey(row.getDisplayId(), row.getAsin(), row.getCountry());
|
||||
}
|
||||
|
||||
private String legacyRowKey(AppearancePatentResultRowDto row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
}
|
||||
return rowKey(row.getId(), row.getAsin(), row.getCountry());
|
||||
}
|
||||
|
||||
private String rowKey(String id, String asin, String country) {
|
||||
return normalizer.apply(id) + "::" + normalizer.apply(asin).toUpperCase(Locale.ROOT) + "::" + normalizer.apply(country);
|
||||
}
|
||||
}
|
||||
@@ -266,13 +266,13 @@ class SourceScanTest {
|
||||
// ---------------------------------------------------------------- N6 moduleSources
|
||||
|
||||
@Test
|
||||
void moduleSources_appearancepatent返回32支含拆分后的support件() {
|
||||
void moduleSources_appearancepatent返回34支含拆分后的support件() {
|
||||
List<Path> files = SourceScan.moduleSources("modules/appearancepatent");
|
||||
|
||||
assertThat(files)
|
||||
.as("真实集成:对真仓目录求值。32 支=1 controller+1 client+1 service 主件+3 spi/cache+"
|
||||
+ "9 service/support+16 model(dto7/vo9),与 find 实测同号")
|
||||
.hasSize(32);
|
||||
.as("真实集成:对真仓目录求值。34 支=1 controller+1 client+1 service 主件+3 spi/cache+"
|
||||
+ "11 service/support+16 model(dto7/vo9),与 find 实测同号")
|
||||
.hasSize(34);
|
||||
assertThat(files.stream().map(p -> p.toString().replace('\\', '/')).toList())
|
||||
.as("support 目录里的既有件必须在面里——聚合入口存在的理由就是拆分后的新文件要自动被看到")
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentRowNormalizer.java"));
|
||||
@@ -366,23 +366,25 @@ class SourceScanTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void recursive_包含service_support下的既有9件() {
|
||||
void recursive_包含service_support下的既有11件() {
|
||||
List<String> support = SourceScan.moduleSources("modules/appearancepatent").stream()
|
||||
.map(p -> p.toString().replace('\\', '/'))
|
||||
.filter(rel -> rel.contains("/service/support/"))
|
||||
.toList();
|
||||
|
||||
assertThat(support)
|
||||
.as("真实集成:对真仓 modules/appearancepatent/service/support 求值,实测 9 支(3A 拆薄把归属谓词、陈旧扫描、结果打包、LLM 批管线抽成四枚 support 件,5→9)。"
|
||||
.as("真实集成:对真仓 modules/appearancepatent/service/support 求值,实测 11 支(3A 拆薄把归属谓词、陈旧扫描、结果打包、LLM 批管线、载荷读写、结果行读取抽成六枚 support 件,5→11)。"
|
||||
+ "这一格就是聚合入口存在的理由——拆分出去的新文件必须自动进面")
|
||||
.hasSize(9);
|
||||
.hasSize(11);
|
||||
assertThat(support)
|
||||
.as("3A 拆出去的四枚新件必须被逐枚点名扫到:只把分母数字改大,下一回拆件又只剩「数不对」"
|
||||
.as("3A 拆出去的六枚新件必须被逐枚点名扫到:只把分母数字改大,下一回拆件又只剩「数不对」"
|
||||
+ "一记红,证不了「扫得到」这件事")
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentTaskOwnership.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentStaleTaskScanner.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentResultPackager.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentLlmBatchPipeline.java"));
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentLlmBatchPipeline.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentPayloadSupport.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentResultRowReader.java"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Reference in New Issue
Block a user