refactor(phase-3): 3A 拆薄第九笔——AP 回传分片 prepare/persist/scope 出件,服务 2161→1942
This commit is contained in:
+32
-251
@@ -7,6 +7,9 @@ import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.task.TaskResultAdmission;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentChunkSubmissionSupport;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentChunkSubmissionSupport.PreparedSubmittedChunk;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentChunkSubmissionSupport.SubmitContext;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentExcelParser;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.util.GroupResultPropagator;
|
||||
@@ -189,7 +192,8 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
|
||||
/**
|
||||
* 本支回传准入走 DB(底座 javadoc 点的「AppearancePatent/Publish/CollectData 一派」)。
|
||||
* 现由 {@code prepareSubmittedChunk} 经 {@link #requireSubmissionTask} 真实调用——
|
||||
* 现由 {@link AppearancePatentChunkSubmissionSupport#prepareSubmittedChunk} 经
|
||||
* {@link #requireSubmissionTask} 真实调用——
|
||||
* 那一格此前是四行内联 {@code selectById + 模块校验 +「任务不存在」},逐字同形。
|
||||
*/
|
||||
@Override
|
||||
@@ -396,9 +400,9 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
if (transactionManager != null) {
|
||||
// 纯计算(flatten 分组展开/序列化/payload 存储/哈希)在事务开始前完成
|
||||
PreparedSubmittedChunk prepared = prepareSubmittedChunk(taskId, request);
|
||||
SubmitContext context = inNewTransaction(() -> persistSubmittedChunk(prepared));
|
||||
SubmitContext context = inNewTransaction(() -> submissionSupport().persistSubmittedChunk(prepared));
|
||||
inNewTransaction(() -> {
|
||||
completeSubmittedChunk(context);
|
||||
submissionSupport().completeSubmittedChunk(context);
|
||||
return null;
|
||||
});
|
||||
scheduleLlmPipelineForSubmittedChunk(context);
|
||||
@@ -467,7 +471,7 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
scope.setCreatedAt(LocalDateTime.now());
|
||||
}
|
||||
scope.setChunkTotal(chunkTotal);
|
||||
scope.setReceivedChunkCount(countChunks(taskId, scopeHash));
|
||||
scope.setReceivedChunkCount(submissionSupport().countChunks(taskId, scopeHash));
|
||||
scope.setLastChunkAt(LocalDateTime.now());
|
||||
scope.setLastError(request.getError());
|
||||
scope.setCompleted(Boolean.TRUE.equals(request.getDone()) ? 1 : 0);
|
||||
@@ -571,106 +575,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
staleTaskScanner().sweepNoUpload(this::finalizeStaleTask);
|
||||
}
|
||||
|
||||
/**
|
||||
* /result 提交的纯计算阶段(无事务注解):读任务 + 校验归属 + 分组展开 +
|
||||
* 序列化 + payload 存储 + 哈希预计算,全部在事务开始前完成。
|
||||
* 重复 chunk(查重命中)时不做存储与哈希,落库由 persist 按同一查重短路。
|
||||
*/
|
||||
private PreparedSubmittedChunk prepareSubmittedChunk(Long taskId, AppearancePatentSubmitResultRequest request) {
|
||||
FileTaskEntity task = requireSubmissionTask(taskId);
|
||||
ensureTaskOwnedByCurrentInstance(task, "submit result");
|
||||
int chunkIndex = request.getChunkIndex() == null ? 0 : request.getChunkIndex();
|
||||
int chunkTotal = request.getChunkTotal() == null ? 1 : request.getChunkTotal();
|
||||
boolean done = Boolean.TRUE.equals(request.getDone());
|
||||
String scopeKey = firstNonBlank(request.getSubmissionId(), "task:" + taskId);
|
||||
String scopeHash = DigestUtil.sha256Hex(scopeKey);
|
||||
|
||||
TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
|
||||
.last("limit 1"));
|
||||
if (existing != null) {
|
||||
return new PreparedSubmittedChunk(task, scopeKey, scopeHash, chunkIndex, chunkTotal, done,
|
||||
request.getError(), null, null, null);
|
||||
}
|
||||
List<AppearancePatentResultRowDto> rawRows = rowReader().flattenSubmittedRows(request);
|
||||
String payloadJson = writeJson(objectMapper, rawRows, "结果序列化失败");
|
||||
String storedPayload = payloadSupport().storeSharedChunkPayload(taskId, scopeHash, chunkIndex, payloadJson);
|
||||
return new PreparedSubmittedChunk(task, scopeKey, scopeHash, chunkIndex, chunkTotal, done,
|
||||
request.getError(), payloadJson, storedPayload, DigestUtil.sha256Hex(payloadJson));
|
||||
}
|
||||
|
||||
private SubmitContext persistSubmittedChunk(PreparedSubmittedChunk prepared) {
|
||||
Long taskId = prepared.task().getId();
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||
TaskResultAdmission.rejectNotRunning(task.getStatus());
|
||||
}
|
||||
|
||||
ensureTaskOwnedByCurrentInstance(task, "submit result");
|
||||
taskCacheService.touchTaskHeartbeat(taskId);
|
||||
|
||||
TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskChunkEntity::getScopeHash, prepared.scopeHash())
|
||||
.eq(TaskChunkEntity::getChunkIndex, prepared.chunkIndex())
|
||||
.last("limit 1"));
|
||||
if (existing == null && prepared.storedPayload() != null) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setTaskId(taskId);
|
||||
chunk.setModuleType(MODULE_TYPE);
|
||||
chunk.setScopeKey(prepared.scopeKey());
|
||||
chunk.setScopeHash(prepared.scopeHash());
|
||||
chunk.setChunkIndex(prepared.chunkIndex());
|
||||
chunk.setChunkTotal(prepared.chunkTotal());
|
||||
chunk.setPayloadJson(prepared.storedPayload());
|
||||
chunk.setPayloadHash(prepared.payloadHash());
|
||||
chunk.setCreatedAt(LocalDateTime.now());
|
||||
chunk.setUpdatedAt(LocalDateTime.now());
|
||||
try {
|
||||
taskChunkMapper.insert(chunk);
|
||||
} catch (DuplicateKeyException ex) {
|
||||
log.info("[appearance-patent] duplicate chunk inserted concurrently taskId={} scope={} chunk={}",
|
||||
taskId, prepared.scopeKey(), prepared.chunkIndex());
|
||||
}
|
||||
} else if (existing != null) {
|
||||
log.info("[appearance-patent] duplicate chunk ignored taskId={} scope={} chunk={}",
|
||||
taskId, prepared.scopeKey(), prepared.chunkIndex());
|
||||
}
|
||||
|
||||
upsertScopeState(taskId, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkTotal(),
|
||||
prepared.error(), prepared.done(), false);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
return new SubmitContext(task, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkIndex(),
|
||||
prepared.done(), prepared.error());
|
||||
}
|
||||
|
||||
private void completeSubmittedChunk(SubmitContext context) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(context.task().getId());
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||
log.info("[appearance-patent] skip completion because task already finalized taskId={} status={}",
|
||||
task.getId(), task.getStatus());
|
||||
return;
|
||||
}
|
||||
upsertScopeState(task.getId(), context.scopeKey(), context.scopeHash(), null, context.error(), context.forceFlush(), false);
|
||||
if (context.forceFlush() || context.error() != null && !context.error().isBlank()) {
|
||||
finalizeTask(task, context.error(), rowReader().allRowCount(task), true);
|
||||
return;
|
||||
}
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
|
||||
/**
|
||||
* B 聚类(LLM 批管线)住在 {@link AppearancePatentLlmBatchPipeline}(3A 拆薄第五笔,
|
||||
* 候选收集、批调度、轮次判死、结果行合并与批计数逐字搬移)。以下四枚同名入口是
|
||||
@@ -757,6 +661,28 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
taskFileJobService, taskProgressSnapshotService, objectMapper);
|
||||
}
|
||||
|
||||
/**
|
||||
* D 聚类(分片回传与范围状态)住在 {@link AppearancePatentChunkSubmissionSupport}
|
||||
* (3A 拆薄第九笔)。任务准入、归属、分组展开、行计数、载荷写入、序列化与收尾由本类回调交出。
|
||||
*/
|
||||
/**
|
||||
* 按名反射接缝:{@code AppearancePatentTaskServiceTxBoundaryTest} 点名本方法,
|
||||
* 用来证明载荷哈希在事务开始前就算好。实现本体在 {@link AppearancePatentChunkSubmissionSupport}。
|
||||
*/
|
||||
private PreparedSubmittedChunk prepareSubmittedChunk(Long taskId, AppearancePatentSubmitResultRequest request) {
|
||||
return submissionSupport().prepareSubmittedChunk(taskId, request);
|
||||
}
|
||||
|
||||
private AppearancePatentChunkSubmissionSupport submissionSupport() {
|
||||
return new AppearancePatentChunkSubmissionSupport(
|
||||
MODULE_TYPE, fileTaskMapper, taskChunkMapper, taskScopeStateMapper, taskCacheService,
|
||||
this::requireSubmissionTask, this::ensureTaskOwnedByCurrentInstance,
|
||||
rowReader()::flattenSubmittedRows, rowReader()::allRowCount,
|
||||
payloadSupport()::storeSharedChunkPayload,
|
||||
(value, message) -> writeJson(objectMapper, value, message),
|
||||
this::finalizeTask, this::firstNonBlank);
|
||||
}
|
||||
|
||||
private void submitLlmForSubmittedChunk(SubmitContext context) {
|
||||
if (context == null || context.task() == null || context.task().getId() == null) {
|
||||
return;
|
||||
@@ -872,8 +798,8 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
if (!uploadComplete) {
|
||||
// 封口是多行 scope 状态更新,单独占一个短事务;紧随其后的 LLM 批次不在事务里
|
||||
int forcedScopes = transactionManager == null
|
||||
? markSubmissionCompleteOnPythonTimeout(taskId)
|
||||
: inNewTransaction(() -> markSubmissionCompleteOnPythonTimeout(taskId));
|
||||
? submissionSupport().markSubmissionCompleteOnPythonTimeout(taskId)
|
||||
: inNewTransaction(() -> submissionSupport().markSubmissionCompleteOnPythonTimeout(taskId));
|
||||
if (forcedScopes <= 0) {
|
||||
log.warn("[appearance-patent] stale recovery aborted because submission could not be marked complete taskId={}",
|
||||
taskId);
|
||||
@@ -891,122 +817,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
return true;
|
||||
}
|
||||
|
||||
private int markSubmissionCompleteOnPythonTimeout(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return 0;
|
||||
}
|
||||
List<TaskScopeStateEntity> inputStates = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.isNull(TaskScopeStateEntity::getLlmStatus)
|
||||
.orderByDesc(TaskScopeStateEntity::getUpdatedAt));
|
||||
if (inputStates == null || inputStates.isEmpty()) {
|
||||
TaskChunkEntity latestChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.orderByDesc(TaskChunkEntity::getUpdatedAt)
|
||||
.last("limit 1"));
|
||||
if (latestChunk == null || latestChunk.getScopeHash() == null || latestChunk.getScopeHash().isBlank()) {
|
||||
log.warn("[appearance-patent] stale recovery found no latest chunk to complete taskId={}", taskId);
|
||||
return 0;
|
||||
}
|
||||
log.info("[appearance-patent] stale recovery recreating input scope from chunk taskId={} scopeKey={} scopeHash={} chunkTotal={}",
|
||||
taskId, latestChunk.getScopeKey(), latestChunk.getScopeHash(), latestChunk.getChunkTotal());
|
||||
upsertScopeState(taskId,
|
||||
firstNonBlank(latestChunk.getScopeKey(), "task:" + taskId),
|
||||
latestChunk.getScopeHash(),
|
||||
latestChunk.getChunkTotal(),
|
||||
null,
|
||||
true,
|
||||
false);
|
||||
return 1;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = 0;
|
||||
for (TaskScopeStateEntity state : inputStates) {
|
||||
if (state == null || state.getId() == null) {
|
||||
continue;
|
||||
}
|
||||
state.setCompleted(1);
|
||||
if (state.getLastChunkAt() == null) {
|
||||
state.setLastChunkAt(now);
|
||||
}
|
||||
state.setUpdatedAt(now);
|
||||
state.setStateJson("{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}");
|
||||
taskScopeStateMapper.updateById(state);
|
||||
updated++;
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
|
||||
private void upsertScopeState(Long taskId,
|
||||
String scopeKey,
|
||||
String scopeHash,
|
||||
Integer chunkTotal,
|
||||
String error,
|
||||
boolean completed,
|
||||
boolean llmDone) {
|
||||
TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
|
||||
.last("limit 1"));
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
if (scope == null) {
|
||||
scope = new TaskScopeStateEntity();
|
||||
scope.setTaskId(taskId);
|
||||
scope.setModuleType(MODULE_TYPE);
|
||||
scope.setScopeKey(scopeKey);
|
||||
scope.setScopeHash(scopeHash);
|
||||
scope.setCreatedAt(now);
|
||||
}
|
||||
if (chunkTotal != null) {
|
||||
scope.setChunkTotal(chunkTotal);
|
||||
}
|
||||
scope.setReceivedChunkCount(countChunks(taskId, scopeHash));
|
||||
scope.setLastChunkAt(now);
|
||||
scope.setLastError(error);
|
||||
scope.setCompleted(completed ? 1 : 0);
|
||||
scope.setUpdatedAt(now);
|
||||
scope.setStateJson(llmDone
|
||||
? "{\"phase\":\"RECEIVED\",\"llm\":\"DONE\"}"
|
||||
: "{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}");
|
||||
if (scope.getId() == null) {
|
||||
try {
|
||||
taskScopeStateMapper.insert(scope);
|
||||
log.info("[appearance-patent] scope state inserted taskId={} scope={} scopeHash={} completed={} llmDone={}",
|
||||
taskId, scopeKey, scopeHash, completed, llmDone);
|
||||
return;
|
||||
} catch (DuplicateKeyException ex) {
|
||||
log.info("[appearance-patent] duplicate scope state inserted concurrently taskId={} scope={}", taskId, scopeKey);
|
||||
scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
|
||||
.last("limit 1"));
|
||||
if (scope == null) {
|
||||
log.info("[appearance-patent] scope state winner not committed yet, skip duplicate updater taskId={} scope={}",
|
||||
taskId, scopeKey);
|
||||
return;
|
||||
}
|
||||
if (chunkTotal != null) {
|
||||
scope.setChunkTotal(chunkTotal);
|
||||
}
|
||||
scope.setReceivedChunkCount(countChunks(taskId, scopeHash));
|
||||
scope.setLastChunkAt(now);
|
||||
scope.setLastError(error);
|
||||
scope.setCompleted(completed ? 1 : 0);
|
||||
scope.setUpdatedAt(now);
|
||||
scope.setStateJson(llmDone
|
||||
? "{\"phase\":\"RECEIVED\",\"llm\":\"DONE\"}"
|
||||
: "{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}");
|
||||
}
|
||||
}
|
||||
taskScopeStateMapper.updateById(scope);
|
||||
log.info("[appearance-patent] scope state updated taskId={} scope={} scopeHash={} completed={} llmDone={}",
|
||||
taskId, scopeKey, scopeHash, completed, llmDone);
|
||||
}
|
||||
|
||||
private <T> T inNewTransaction(Supplier<T> action) {
|
||||
TransactionTemplate template = new TransactionTemplate(transactionManager);
|
||||
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
|
||||
@@ -2083,15 +1893,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
return normalize(sourceFileKey) + "::" + normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex);
|
||||
}
|
||||
|
||||
private int countChunks(Long taskId, String scopeHash) {
|
||||
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskChunkEntity::getScopeHash, scopeHash));
|
||||
return count == null ? 0 : count.intValue();
|
||||
}
|
||||
|
||||
|
||||
private String firstNonBlank(String preferred, String fallback) {
|
||||
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||
}
|
||||
@@ -2133,26 +1934,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline<Appear
|
||||
action.run();
|
||||
}
|
||||
|
||||
private record SubmitContext(FileTaskEntity task,
|
||||
String scopeKey,
|
||||
String scopeHash,
|
||||
Integer chunkIndex,
|
||||
boolean forceFlush,
|
||||
String error) {
|
||||
}
|
||||
|
||||
private record PreparedSubmittedChunk(FileTaskEntity task,
|
||||
String scopeKey,
|
||||
String scopeHash,
|
||||
Integer chunkIndex,
|
||||
Integer chunkTotal,
|
||||
boolean done,
|
||||
String error,
|
||||
String payloadJson,
|
||||
String storedPayload,
|
||||
String payloadHash) {
|
||||
}
|
||||
|
||||
private record ParsedWorkbook(int totalRows, int droppedRows, List<String> headers, List<AppearancePatentParsedRowVo> allRows) {
|
||||
}
|
||||
|
||||
|
||||
+362
@@ -0,0 +1,362 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||
|
||||
import cn.hutool.crypto.digest.DigestUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.task.TaskResultAdmission;
|
||||
import com.nanri.aiimage.common.task.TaskStatuses;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 外观专利的分片回传与范围状态(D 聚类),从 {@code AppearancePatentTaskService}
|
||||
* <b>原样搬移</b>(3A 拆薄第九笔):prepare/persist/complete 三段、Python 超时封口、
|
||||
* 范围状态 upsert 与分片计数,连同事务边界之外的纯计算形状、重复分片的短路与日志一字没改。
|
||||
*
|
||||
* <p><b>搬的是分片落库,不是宿主能力。</b>任务加载(底座准入)、归属校验、回传分组展开、
|
||||
* 解析行计数、载荷写入、序列化出口、任务收尾都还留在服务侧,以回调交进来——
|
||||
* 它们分属底座/B/H/I/E 聚类,跟着 D 一起拖出来会把依赖面放大。
|
||||
*/
|
||||
@Slf4j
|
||||
public class AppearancePatentChunkSubmissionSupport {
|
||||
|
||||
@FunctionalInterface
|
||||
public interface SubmissionTaskLoader {
|
||||
FileTaskEntity load(Long taskId);
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface OwnershipChecker {
|
||||
void check(FileTaskEntity task, String operation);
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface RowFlattener {
|
||||
List<AppearancePatentResultRowDto> flatten(AppearancePatentSubmitResultRequest request);
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface RowCounter {
|
||||
int count(FileTaskEntity task);
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface ChunkPayloadWriter {
|
||||
String store(Long taskId, String scopeHash, Integer chunkIndex, String payloadJson);
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface JsonWriter {
|
||||
String write(Object value, String failureMessage);
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface TaskFinalizer {
|
||||
void finalizeTask(FileTaskEntity task, String error, int rowCount, boolean assembleWorkbook);
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface FirstNonBlank {
|
||||
String pick(String preferred, String fallback);
|
||||
}
|
||||
|
||||
public record SubmitContext(FileTaskEntity task,
|
||||
String scopeKey,
|
||||
String scopeHash,
|
||||
Integer chunkIndex,
|
||||
boolean forceFlush,
|
||||
String error) {
|
||||
}
|
||||
|
||||
public record PreparedSubmittedChunk(FileTaskEntity task,
|
||||
String scopeKey,
|
||||
String scopeHash,
|
||||
Integer chunkIndex,
|
||||
Integer chunkTotal,
|
||||
boolean done,
|
||||
String error,
|
||||
String payloadJson,
|
||||
String storedPayload,
|
||||
String payloadHash) {
|
||||
}
|
||||
|
||||
private static final String STATUS_RUNNING = TaskStatuses.FileTask.RUNNING;
|
||||
|
||||
private final String moduleType;
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final TaskChunkMapper taskChunkMapper;
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final AppearancePatentTaskCacheService taskCacheService;
|
||||
private final SubmissionTaskLoader taskLoader;
|
||||
private final OwnershipChecker ownershipChecker;
|
||||
private final RowFlattener rowFlattener;
|
||||
private final RowCounter rowCounter;
|
||||
private final ChunkPayloadWriter chunkPayloadWriter;
|
||||
private final JsonWriter jsonWriter;
|
||||
private final TaskFinalizer finalizer;
|
||||
private final FirstNonBlank firstNonBlank;
|
||||
|
||||
public AppearancePatentChunkSubmissionSupport(String moduleType,
|
||||
FileTaskMapper fileTaskMapper,
|
||||
TaskChunkMapper taskChunkMapper,
|
||||
TaskScopeStateMapper taskScopeStateMapper,
|
||||
AppearancePatentTaskCacheService taskCacheService,
|
||||
SubmissionTaskLoader taskLoader,
|
||||
OwnershipChecker ownershipChecker,
|
||||
RowFlattener rowFlattener,
|
||||
RowCounter rowCounter,
|
||||
ChunkPayloadWriter chunkPayloadWriter,
|
||||
JsonWriter jsonWriter,
|
||||
TaskFinalizer finalizer,
|
||||
FirstNonBlank firstNonBlank) {
|
||||
this.moduleType = moduleType;
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
this.taskChunkMapper = taskChunkMapper;
|
||||
this.taskScopeStateMapper = taskScopeStateMapper;
|
||||
this.taskCacheService = taskCacheService;
|
||||
this.taskLoader = taskLoader;
|
||||
this.ownershipChecker = ownershipChecker;
|
||||
this.rowFlattener = rowFlattener;
|
||||
this.rowCounter = rowCounter;
|
||||
this.chunkPayloadWriter = chunkPayloadWriter;
|
||||
this.jsonWriter = jsonWriter;
|
||||
this.finalizer = finalizer;
|
||||
this.firstNonBlank = firstNonBlank;
|
||||
}
|
||||
|
||||
/**
|
||||
* /result 提交的纯计算阶段(无事务注解):读任务 + 校验归属 + 分组展开 +
|
||||
* 序列化 + payload 存储 + 哈希预计算,全部在事务开始前完成。
|
||||
* 重复 chunk(查重命中)时不做存储与哈希,落库由 persist 按同一查重短路。
|
||||
*/
|
||||
public PreparedSubmittedChunk prepareSubmittedChunk(Long taskId, AppearancePatentSubmitResultRequest request) {
|
||||
FileTaskEntity task = taskLoader.load(taskId);
|
||||
ownershipChecker.check(task, "submit result");
|
||||
int chunkIndex = request.getChunkIndex() == null ? 0 : request.getChunkIndex();
|
||||
int chunkTotal = request.getChunkTotal() == null ? 1 : request.getChunkTotal();
|
||||
boolean done = Boolean.TRUE.equals(request.getDone());
|
||||
String scopeKey = firstNonBlank.pick(request.getSubmissionId(), "task:" + taskId);
|
||||
String scopeHash = DigestUtil.sha256Hex(scopeKey);
|
||||
|
||||
TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, moduleType)
|
||||
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
|
||||
.last("limit 1"));
|
||||
if (existing != null) {
|
||||
return new PreparedSubmittedChunk(task, scopeKey, scopeHash, chunkIndex, chunkTotal, done,
|
||||
request.getError(), null, null, null);
|
||||
}
|
||||
List<AppearancePatentResultRowDto> rawRows = rowFlattener.flatten(request);
|
||||
String payloadJson = jsonWriter.write(rawRows, "结果序列化失败");
|
||||
String storedPayload = chunkPayloadWriter.store(taskId, scopeHash, chunkIndex, payloadJson);
|
||||
return new PreparedSubmittedChunk(task, scopeKey, scopeHash, chunkIndex, chunkTotal, done,
|
||||
request.getError(), payloadJson, storedPayload, DigestUtil.sha256Hex(payloadJson));
|
||||
}
|
||||
|
||||
public SubmitContext persistSubmittedChunk(PreparedSubmittedChunk prepared) {
|
||||
Long taskId = prepared.task().getId();
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !moduleType.equals(task.getModuleType())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||
TaskResultAdmission.rejectNotRunning(task.getStatus());
|
||||
}
|
||||
|
||||
ownershipChecker.check(task, "submit result");
|
||||
taskCacheService.touchTaskHeartbeat(taskId);
|
||||
|
||||
TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, moduleType)
|
||||
.eq(TaskChunkEntity::getScopeHash, prepared.scopeHash())
|
||||
.eq(TaskChunkEntity::getChunkIndex, prepared.chunkIndex())
|
||||
.last("limit 1"));
|
||||
if (existing == null && prepared.storedPayload() != null) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setTaskId(taskId);
|
||||
chunk.setModuleType(moduleType);
|
||||
chunk.setScopeKey(prepared.scopeKey());
|
||||
chunk.setScopeHash(prepared.scopeHash());
|
||||
chunk.setChunkIndex(prepared.chunkIndex());
|
||||
chunk.setChunkTotal(prepared.chunkTotal());
|
||||
chunk.setPayloadJson(prepared.storedPayload());
|
||||
chunk.setPayloadHash(prepared.payloadHash());
|
||||
chunk.setCreatedAt(LocalDateTime.now());
|
||||
chunk.setUpdatedAt(LocalDateTime.now());
|
||||
try {
|
||||
taskChunkMapper.insert(chunk);
|
||||
} catch (DuplicateKeyException ex) {
|
||||
log.info("[appearance-patent] duplicate chunk inserted concurrently taskId={} scope={} chunk={}",
|
||||
taskId, prepared.scopeKey(), prepared.chunkIndex());
|
||||
}
|
||||
} else if (existing != null) {
|
||||
log.info("[appearance-patent] duplicate chunk ignored taskId={} scope={} chunk={}",
|
||||
taskId, prepared.scopeKey(), prepared.chunkIndex());
|
||||
}
|
||||
|
||||
upsertScopeState(taskId, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkTotal(),
|
||||
prepared.error(), prepared.done(), false);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
return new SubmitContext(task, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkIndex(),
|
||||
prepared.done(), prepared.error());
|
||||
}
|
||||
|
||||
public void completeSubmittedChunk(SubmitContext context) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(context.task().getId());
|
||||
if (task == null || !moduleType.equals(task.getModuleType())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||
log.info("[appearance-patent] skip completion because task already finalized taskId={} status={}",
|
||||
task.getId(), task.getStatus());
|
||||
return;
|
||||
}
|
||||
upsertScopeState(task.getId(), context.scopeKey(), context.scopeHash(), null, context.error(), context.forceFlush(), false);
|
||||
if (context.forceFlush() || context.error() != null && !context.error().isBlank()) {
|
||||
finalizer.finalizeTask(task, context.error(), rowCounter.count(task), true);
|
||||
return;
|
||||
}
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
|
||||
public int markSubmissionCompleteOnPythonTimeout(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return 0;
|
||||
}
|
||||
List<TaskScopeStateEntity> inputStates = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||
.isNull(TaskScopeStateEntity::getLlmStatus)
|
||||
.orderByDesc(TaskScopeStateEntity::getUpdatedAt));
|
||||
if (inputStates == null || inputStates.isEmpty()) {
|
||||
TaskChunkEntity latestChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, moduleType)
|
||||
.orderByDesc(TaskChunkEntity::getUpdatedAt)
|
||||
.last("limit 1"));
|
||||
if (latestChunk == null || latestChunk.getScopeHash() == null || latestChunk.getScopeHash().isBlank()) {
|
||||
log.warn("[appearance-patent] stale recovery found no latest chunk to complete taskId={}", taskId);
|
||||
return 0;
|
||||
}
|
||||
log.info("[appearance-patent] stale recovery recreating input scope from chunk taskId={} scopeKey={} scopeHash={} chunkTotal={}",
|
||||
taskId, latestChunk.getScopeKey(), latestChunk.getScopeHash(), latestChunk.getChunkTotal());
|
||||
upsertScopeState(taskId,
|
||||
firstNonBlank.pick(latestChunk.getScopeKey(), "task:" + taskId),
|
||||
latestChunk.getScopeHash(),
|
||||
latestChunk.getChunkTotal(),
|
||||
null,
|
||||
true,
|
||||
false);
|
||||
return 1;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = 0;
|
||||
for (TaskScopeStateEntity state : inputStates) {
|
||||
if (state == null || state.getId() == null) {
|
||||
continue;
|
||||
}
|
||||
state.setCompleted(1);
|
||||
if (state.getLastChunkAt() == null) {
|
||||
state.setLastChunkAt(now);
|
||||
}
|
||||
state.setUpdatedAt(now);
|
||||
state.setStateJson("{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}");
|
||||
taskScopeStateMapper.updateById(state);
|
||||
updated++;
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
|
||||
private void upsertScopeState(Long taskId,
|
||||
String scopeKey,
|
||||
String scopeHash,
|
||||
Integer chunkTotal,
|
||||
String error,
|
||||
boolean completed,
|
||||
boolean llmDone) {
|
||||
TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
|
||||
.last("limit 1"));
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
if (scope == null) {
|
||||
scope = new TaskScopeStateEntity();
|
||||
scope.setTaskId(taskId);
|
||||
scope.setModuleType(moduleType);
|
||||
scope.setScopeKey(scopeKey);
|
||||
scope.setScopeHash(scopeHash);
|
||||
scope.setCreatedAt(now);
|
||||
}
|
||||
if (chunkTotal != null) {
|
||||
scope.setChunkTotal(chunkTotal);
|
||||
}
|
||||
scope.setReceivedChunkCount(countChunks(taskId, scopeHash));
|
||||
scope.setLastChunkAt(now);
|
||||
scope.setLastError(error);
|
||||
scope.setCompleted(completed ? 1 : 0);
|
||||
scope.setUpdatedAt(now);
|
||||
scope.setStateJson(llmDone
|
||||
? "{\"phase\":\"RECEIVED\",\"llm\":\"DONE\"}"
|
||||
: "{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}");
|
||||
if (scope.getId() == null) {
|
||||
try {
|
||||
taskScopeStateMapper.insert(scope);
|
||||
log.info("[appearance-patent] scope state inserted taskId={} scope={} scopeHash={} completed={} llmDone={}",
|
||||
taskId, scopeKey, scopeHash, completed, llmDone);
|
||||
return;
|
||||
} catch (DuplicateKeyException ex) {
|
||||
log.info("[appearance-patent] duplicate scope state inserted concurrently taskId={} scope={}", taskId, scopeKey);
|
||||
scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
|
||||
.last("limit 1"));
|
||||
if (scope == null) {
|
||||
log.info("[appearance-patent] scope state winner not committed yet, skip duplicate updater taskId={} scope={}",
|
||||
taskId, scopeKey);
|
||||
return;
|
||||
}
|
||||
if (chunkTotal != null) {
|
||||
scope.setChunkTotal(chunkTotal);
|
||||
}
|
||||
scope.setReceivedChunkCount(countChunks(taskId, scopeHash));
|
||||
scope.setLastChunkAt(now);
|
||||
scope.setLastError(error);
|
||||
scope.setCompleted(completed ? 1 : 0);
|
||||
scope.setUpdatedAt(now);
|
||||
scope.setStateJson(llmDone
|
||||
? "{\"phase\":\"RECEIVED\",\"llm\":\"DONE\"}"
|
||||
: "{\"phase\":\"RECEIVED\",\"llm\":\"PENDING\"}");
|
||||
}
|
||||
}
|
||||
taskScopeStateMapper.updateById(scope);
|
||||
log.info("[appearance-patent] scope state updated taskId={} scope={} scopeHash={} completed={} llmDone={}",
|
||||
taskId, scopeKey, scopeHash, completed, llmDone);
|
||||
}
|
||||
|
||||
public int countChunks(Long taskId, String scopeHash) {
|
||||
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, moduleType)
|
||||
.eq(TaskChunkEntity::getScopeHash, scopeHash));
|
||||
return count == null ? 0 : count.intValue();
|
||||
}
|
||||
}
|
||||
@@ -266,13 +266,13 @@ class SourceScanTest {
|
||||
// ---------------------------------------------------------------- N6 moduleSources
|
||||
|
||||
@Test
|
||||
void moduleSources_appearancepatent返回35支含拆分后的support件() {
|
||||
void moduleSources_appearancepatent返回36支含拆分后的support件() {
|
||||
List<Path> files = SourceScan.moduleSources("modules/appearancepatent");
|
||||
|
||||
assertThat(files)
|
||||
.as("真实集成:对真仓目录求值。35 支=1 controller+1 client+1 service 主件+3 spi/cache+"
|
||||
+ "12 service/support+16 model(dto7/vo9),与 find 实测同号")
|
||||
.hasSize(35);
|
||||
.as("真实集成:对真仓目录求值。36 支=1 controller+1 client+1 service 主件+3 spi/cache+"
|
||||
+ "13 service/support+16 model(dto7/vo9),与 find 实测同号")
|
||||
.hasSize(36);
|
||||
assertThat(files.stream().map(p -> p.toString().replace('\\', '/')).toList())
|
||||
.as("support 目录里的既有件必须在面里——聚合入口存在的理由就是拆分后的新文件要自动被看到")
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentRowNormalizer.java"));
|
||||
@@ -366,18 +366,18 @@ class SourceScanTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void recursive_包含service_support下的既有12件() {
|
||||
void recursive_包含service_support下的既有13件() {
|
||||
List<String> support = SourceScan.moduleSources("modules/appearancepatent").stream()
|
||||
.map(p -> p.toString().replace('\\', '/'))
|
||||
.filter(rel -> rel.contains("/service/support/"))
|
||||
.toList();
|
||||
|
||||
assertThat(support)
|
||||
.as("真实集成:对真仓 modules/appearancepatent/service/support 求值,实测 12 支(3A 拆薄把归属谓词、陈旧扫描、结果打包、LLM 批管线、载荷读写、结果行读取、进度视图抽成七枚 support 件,5→12)。"
|
||||
.as("真实集成:对真仓 modules/appearancepatent/service/support 求值,实测 13 支(3A 拆薄把归属谓词、陈旧扫描、结果打包、LLM 批管线、载荷读写、结果行读取、进度视图、分片回传抽成八枚 support 件,5→13)。"
|
||||
+ "这一格就是聚合入口存在的理由——拆分出去的新文件必须自动进面")
|
||||
.hasSize(12);
|
||||
.hasSize(13);
|
||||
assertThat(support)
|
||||
.as("3A 拆出去的七枚新件必须被逐枚点名扫到:只把分母数字改大,下一回拆件又只剩「数不对」"
|
||||
.as("3A 拆出去的八枚新件必须被逐枚点名扫到:只把分母数字改大,下一回拆件又只剩「数不对」"
|
||||
+ "一记红,证不了「扫得到」这件事")
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentTaskOwnership.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentStaleTaskScanner.java"))
|
||||
@@ -385,7 +385,8 @@ class SourceScanTest {
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentLlmBatchPipeline.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentPayloadSupport.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentResultRowReader.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentProgressSupport.java"));
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentProgressSupport.java"))
|
||||
.anyMatch(rel -> rel.endsWith("service/support/AppearancePatentChunkSubmissionSupport.java"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Reference in New Issue
Block a user