完善任务存储、权限及货源查询流程

This commit is contained in:
supernijia
2026-08-14 22:49:15 +08:00
parent 91cd02eeb4
commit cf59e23f15
21 changed files with 2584 additions and 566 deletions
@@ -655,138 +655,44 @@ public class SimilarAsinTaskService {
}
public void submitResult(Long taskId, SimilarAsinSubmitResultRequest request) {
submitResultLocked(taskId, request);
try (TaskDistributedLockService.LockHandle ignored = requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) {
submitResultLocked(taskId, request);
}
}
private void submitResultLocked(Long taskId, SimilarAsinSubmitResultRequest request) {
if (transactionManager != null) {
SubmitContext context = inNewTransaction(() -> persistSubmittedChunk(taskId, request));
inNewTransaction(() -> {
completeSubmittedChunk(context);
return null;
});
scheduleCozePipelineForSubmittedChunk(context);
return;
if (request == null) {
throw new BusinessException("结果请求不能为空");
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
PreparedSubmittedChunk prepared = prepareSubmittedChunk(taskId, request);
PersistSubmittedChunkResult persisted = transactionManager == null
? persistSubmittedChunk(prepared)
: inNewTransaction(() -> persistSubmittedChunk(prepared));
SubmitContext context = persisted.context();
// A thrown transaction may already have committed but lost its ACK. Keep
// that candidate; cleanup is only safe after a successful transaction
// has proved that another chunk won the unique key.
if (!persisted.payloadPersisted()) {
cleanupPreparedSubmittedChunkIfUnreferenced(prepared);
}
if (!STATUS_RUNNING.equals(task.getStatus())) {
throw new BusinessException("任务不是运行中状态");
}
ensureTaskOwnedByCurrentInstance(task, "submit result");
int chunkIndex = request.getChunkIndex() == null ? 0 : request.getChunkIndex();
int chunkTotal = request.getChunkTotal() == null ? 1 : request.getChunkTotal();
String scopeKey = firstNonBlank(request.getSubmissionId(), "task:" + taskId);
String scopeHash = DigestUtil.sha256Hex(scopeKey);
taskCacheService.touchTaskHeartbeat(taskId);
TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
.last("limit 1"));
if (existing == null) {
List<SimilarAsinResultRowDto> rawRows = flattenSubmittedRows(request);
String payloadJson = writeJson(rawRows, "结果序列化失败");
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setTaskId(taskId);
chunk.setModuleType(MODULE_TYPE);
chunk.setScopeKey(scopeKey);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setChunkTotal(chunkTotal);
// P0-2:改用 versioned key(chunk-{index}-{uuid}),避免并发回调下 deterministic key
// 覆盖造成的 read chunk payload failed / 数据丢失。loser 拿到的是自己独有的 object,
// DuplicateKeyException 后可以安全删除,不会影响 winner 的 chunk。
String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
// P0-3:判断本次 store 是否走了 RustFS → local 兜底。
boolean localFallback = transientPayloadStorageService.wasLastStoreLocalFallback();
chunk.setPayloadJson(storedPayload);
chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson));
chunk.setCreatedAt(LocalDateTime.now());
chunk.setUpdatedAt(LocalDateTime.now());
try {
taskChunkMapper.insert(chunk);
} catch (DuplicateKeyException ex) {
// P0-2:versioned key 每次都是独立 object,loser 的 storedPayload 不会被 winner 引用,
// 这里直接清理掉以免 RustFS / local 上残留孤儿对象。
log.info("[similar-asin] duplicate chunk inserted concurrently taskId={} scope={} chunk={} cleanupLoser={}",
taskId, scopeKey, chunkIndex, storedPayload != null);
try {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
} catch (Exception cleanupEx) {
log.warn("[similar-asin] cleanup loser chunk payload failed taskId={} chunk={} err={}",
taskId, chunkIndex, cleanupEx.getMessage());
}
}
// P0-3:本次 chunk 落到本地时把 task 锁到当前实例,避免其他实例 assemble 时读不到。
if (localFallback) {
bindTaskToCurrentOwnerForLocalFallback(task, scopeHash, chunkIndex);
}
if (persisted.finalizeResult() == null) {
taskCacheService.touchTaskHeartbeat(taskId);
} else {
log.info("[similar-asin] duplicate chunk ignored taskId={} scope={} chunk={}", taskId, scopeKey, chunkIndex);
}
TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.last("limit 1"));
if (scope == null) {
scope = new TaskScopeStateEntity();
scope.setTaskId(taskId);
scope.setModuleType(MODULE_TYPE);
scope.setScopeKey(scopeKey);
scope.setScopeHash(scopeHash);
scope.setCreatedAt(LocalDateTime.now());
}
scope.setChunkTotal(chunkTotal);
scope.setReceivedChunkCount(resolveReceivedChunkProgress(taskId, scopeHash, scope.getChunkTotal()));
scope.setLastChunkAt(LocalDateTime.now());
scope.setLastError(request.getError());
scope.setCompleted(Boolean.TRUE.equals(request.getDone()) ? 1 : 0);
scope.setUpdatedAt(LocalDateTime.now());
scope.setStateJson("{\"phase\":\"RECEIVED\",\"coze\":\"PENDING\"}");
if (scope.getId() == null) {
taskScopeStateMapper.insert(scope);
} else {
taskScopeStateMapper.updateById(scope);
}
SubmitContext context = new SubmitContext(task, scopeKey, scopeHash, chunkIndex,
Boolean.TRUE.equals(request.getDone()), request.getError());
if (Boolean.TRUE.equals(request.getDone()) || request.getError() != null && !request.getError().isBlank()) {
finalizeTask(task, request.getError(), allRowCount(task), true);
} else {
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
applyFinalizeSideEffects(persisted.finalizeResult());
}
scheduleCozePipelineForSubmittedChunk(context);
}
@Transactional
public void deleteTask(Long taskId, Long userId) {
List<String> payloads;
try (TaskDistributedLockService.LockHandle ignored = requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) {
throw new BusinessException("任务不存在");
payloads = transactionManager == null
? deleteTaskRecords(taskId, userId)
: inNewTransaction(() -> deleteTaskRecords(taskId, userId));
}
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>().eq(FileResultEntity::getTaskId, taskId).eq(FileResultEntity::getModuleType, MODULE_TYPE));
deleteTransientTaskPayloads(taskId);
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>().eq(TaskScopeStateEntity::getTaskId, taskId).eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>().eq(TaskChunkEntity::getTaskId, taskId).eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
deleteTransientPayloads(payloads, taskId);
taskCacheService.deleteTaskCache(taskId);
// 同步清理 task_file_job,避免被删除任务遗留的 PENDING/FAILED 行被 TaskResultFileJobWorker 反复扫描出“任务不存在”。
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
fileTaskMapper.deleteById(taskId);
// P1-1:任务被删除时一并清理滑窗记录,防止内存泄漏。
clearPoisonWindow(taskId);
}
}
public void deleteHistory(Long resultId, Long userId) {
@@ -799,6 +705,69 @@ public class SimilarAsinTaskService {
fileResultMapper.deleteById(resultId);
}
private List<String> deleteTaskRecords(Long taskId, Long userId) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) {
throw new BusinessException("任务不存在");
}
List<String> payloads = collectTransientTaskPayloads(taskId);
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
fileTaskMapper.deleteById(taskId);
return payloads;
}
private void deleteTransientPayloads(List<String> payloads, Long taskId) {
if (payloads == null || payloads.isEmpty()) {
return;
}
for (String payload : new LinkedHashSet<>(payloads)) {
try {
if (isTransientPayloadStillReferenced(payload)) {
log.info("[similar-asin] skip task payload delete because it is still referenced taskId={} pointer={}",
taskId, transientPayloadStorageService.extractPointer(payload));
continue;
}
transientPayloadStorageService.deletePayloadIfPresent(payload);
} catch (Exception ex) {
log.warn("[similar-asin] delete task payload deferred taskId={} pointer={} err={}",
taskId, transientPayloadStorageService.extractPointer(payload), ex.getMessage());
}
}
}
private boolean isTransientPayloadStillReferenced(String payload) {
String pointer = transientPayloadStorageService.extractPointer(payload);
if (pointer == null) {
return false;
}
LinkedHashSet<String> values = new LinkedHashSet<>();
values.add(payload);
values.add(pointer);
try {
values.add(objectMapper.writeValueAsString(pointer));
} catch (Exception ignored) {
}
Long chunkCount = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.in(TaskChunkEntity::getPayloadJson, values));
if (chunkCount != null && chunkCount > 0L) {
return true;
}
Long scopeCount = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.and(wrapper -> wrapper.in(TaskScopeStateEntity::getParsedPayloadJson, values)
.or()
.in(TaskScopeStateEntity::getStateJson, values)));
return scopeCount != null && scopeCount > 0L;
}
public String resolveResultDownloadUrl(Long resultId, Long userId) {
FileResultEntity row = fileResultMapper.selectById(resultId);
if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) {
@@ -873,14 +842,7 @@ public class SimilarAsinTaskService {
continue;
}
try (taskLockHandle) {
if (transactionManager != null) {
inNewTransaction(() -> {
finalizeStaleTask(task.getId(), "Python interrupted before uploading final similar ASIN result");
return null;
});
} else {
finalizeStaleTask(task.getId(), "Python interrupted before uploading final similar ASIN result");
}
finalizeStaleTask(task.getId(), "Python interrupted before uploading final similar ASIN result");
}
}
}
@@ -889,19 +851,6 @@ public class SimilarAsinTaskService {
if (taskId == null || taskId <= 0) {
return;
}
if (transactionManager != null) {
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(taskId, 0L);
if (taskLockHandle == null) {
return;
}
try (taskLockHandle) {
inNewTransaction(() -> {
finalizeStaleTask(taskId, "Python interrupted before uploading final similar ASIN result");
return null;
});
}
return;
}
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(taskId, 0L);
if (taskLockHandle == null) {
return;
@@ -943,7 +892,7 @@ public class SimilarAsinTaskService {
return updatedMillis <= thresholdMillis;
}
private SubmitContext persistSubmittedChunk(Long taskId, SimilarAsinSubmitResultRequest request) {
private PreparedSubmittedChunk prepareSubmittedChunk(Long taskId, SimilarAsinSubmitResultRequest request) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
@@ -958,62 +907,132 @@ public class SimilarAsinTaskService {
boolean done = Boolean.TRUE.equals(request.getDone());
String scopeKey = firstNonBlank(request.getSubmissionId(), "task:" + taskId);
String scopeHash = DigestUtil.sha256Hex(scopeKey);
taskCacheService.touchTaskHeartbeat(taskId);
boolean terminalCallback = done || request.getError() != null && !request.getError().isBlank();
SubmittedTaskMetadata taskMetadata = terminalCallback
? readSubmittedTaskMetadata(task)
: null;
TaskChunkEntity existing = findSubmittedChunk(taskId, scopeHash, chunkIndex);
if (existing != null) {
return new PreparedSubmittedChunk(
taskId, scopeKey, scopeHash, chunkIndex, chunkTotal, done, request.getError(),
null, null, false, taskMetadata);
}
String payloadJson = writeJson(flattenSubmittedRows(request), "结果序列化失败");
String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(
MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
boolean localFallback = transientPayloadStorageService.wasLastStoreLocalFallback();
return new PreparedSubmittedChunk(
taskId, scopeKey, scopeHash, chunkIndex, chunkTotal, done, request.getError(),
payloadJson, storedPayload, localFallback, taskMetadata);
}
TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
private SubmittedTaskMetadata readSubmittedTaskMetadata(FileTaskEntity task) {
SimilarAsinParsedPayloadDto payload = readParsedPayload(task);
List<SimilarAsinSourceFileDto> sourceFiles = payload.getSourceFiles() == null
? List.of()
: List.copyOf(payload.getSourceFiles());
return new SubmittedTaskMetadata(rowCount(payload), sourceFiles);
}
private int rowCount(SimilarAsinParsedPayloadDto payload) {
if (payload == null) {
return 0;
}
if (payload.getAllItems() != null && !payload.getAllItems().isEmpty()) {
return payload.getAllItems().size();
}
if (payload.getItems() != null && !payload.getItems().isEmpty()) {
return payload.getItems().size();
}
if (payload.getGroups() != null && !payload.getGroups().isEmpty()) {
return payload.getGroups().stream()
.map(SimilarAsinParsedGroupVo::getItems)
.filter(Objects::nonNull)
.mapToInt(List::size)
.sum();
}
return 0;
}
private PersistSubmittedChunkResult persistSubmittedChunk(PreparedSubmittedChunk prepared) {
Long taskId = prepared.taskId();
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
}
if (!STATUS_RUNNING.equals(task.getStatus())) {
throw new BusinessException("任务不是运行中状态");
}
ensureTaskOwnedByCurrentInstance(task, "submit result");
boolean payloadPersisted = prepared.storedPayload() == null;
TaskChunkEntity existing = findSubmittedChunk(taskId, prepared.scopeHash(), prepared.chunkIndex());
if (existing == null && prepared.storedPayload() != null) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setTaskId(taskId);
chunk.setModuleType(MODULE_TYPE);
chunk.setScopeKey(prepared.scopeKey());
chunk.setScopeHash(prepared.scopeHash());
chunk.setChunkIndex(prepared.chunkIndex());
chunk.setChunkTotal(prepared.chunkTotal());
chunk.setPayloadJson(prepared.storedPayload());
chunk.setPayloadHash(DigestUtil.sha256Hex(prepared.payloadJson()));
chunk.setCreatedAt(LocalDateTime.now());
chunk.setUpdatedAt(LocalDateTime.now());
try {
if (taskChunkMapper.insert(chunk) != 1) {
throw new IllegalStateException("Chunk insert affected no row taskId=" + taskId
+ " chunk=" + prepared.chunkIndex());
}
payloadPersisted = true;
} catch (DuplicateKeyException ex) {
log.info("[similar-asin] duplicate chunk inserted concurrently taskId={} scope={} chunk={} cleanupLoser={}",
taskId, prepared.scopeKey(), prepared.chunkIndex(), prepared.storedPayload() != null);
}
} else {
log.info("[similar-asin] duplicate chunk ignored taskId={} scope={} chunk={}",
taskId, prepared.scopeKey(), prepared.chunkIndex());
}
upsertScopeState(taskId, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkTotal(),
prepared.error(), prepared.done(), false);
if (prepared.localFallback() && payloadPersisted) {
bindTaskToCurrentOwnerForLocalFallback(
task, prepared.scopeHash(), prepared.chunkIndex());
}
SubmitContext context = new SubmitContext(task, prepared.scopeKey(), prepared.scopeHash(), prepared.chunkIndex(),
prepared.done(), prepared.error(), prepared.taskMetadata());
FinalizeTaskResult finalizeResult = completeSubmittedChunk(context);
return new PersistSubmittedChunkResult(context, payloadPersisted, finalizeResult);
}
private TaskChunkEntity findSubmittedChunk(Long taskId, String scopeHash, Integer chunkIndex) {
return taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
.last("limit 1"));
if (existing == null) {
List<SimilarAsinResultRowDto> rawRows = flattenSubmittedRows(request);
String payloadJson = writeJson(rawRows, "结果序列化失败");
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setTaskId(taskId);
chunk.setModuleType(MODULE_TYPE);
chunk.setScopeKey(scopeKey);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setChunkTotal(chunkTotal);
// P0-2:与 submitResultLocked 同步切到 versioned key(chunk-{index}-{uuid}),
// 避免并发回调互相覆盖造成 read chunk payload failed。
String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
// P0-3:拿到 pointer 后立刻读 ThreadLocal 标记,决定是否需要把 task 锁到当前实例。
boolean localFallback = transientPayloadStorageService.wasLastStoreLocalFallback();
chunk.setPayloadJson(storedPayload);
chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson));
chunk.setCreatedAt(LocalDateTime.now());
chunk.setUpdatedAt(LocalDateTime.now());
try {
taskChunkMapper.insert(chunk);
} catch (DuplicateKeyException ex) {
// P0-2:versioned key 后 loser 的 storedPayload 不会被 winner 引用,可以安全清理。
log.info("[similar-asin] duplicate chunk inserted concurrently taskId={} scope={} chunk={} cleanupLoser={}",
taskId, scopeKey, chunkIndex, storedPayload != null);
try {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
} catch (Exception cleanupEx) {
log.warn("[similar-asin] cleanup loser chunk payload failed taskId={} chunk={} err={}",
taskId, chunkIndex, cleanupEx.getMessage());
}
}
// P0-3:fallback 到 local 时把 task 与当前实例绑定,确保后续 assemble 走对实例。
if (localFallback) {
bindTaskToCurrentOwnerForLocalFallback(task, scopeHash, chunkIndex);
}
} else {
log.info("[similar-asin] duplicate chunk ignored taskId={} scope={} chunk={}", taskId, scopeKey, chunkIndex);
}
upsertScopeState(taskId, scopeKey, scopeHash, chunkTotal, request.getError(), done, false);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
return new SubmitContext(task, scopeKey, scopeHash, chunkIndex, done, request.getError());
}
private void completeSubmittedChunk(SubmitContext context) {
private void cleanupPreparedSubmittedChunkIfUnreferenced(PreparedSubmittedChunk prepared) {
if (prepared == null || prepared.storedPayload() == null) {
return;
}
try {
if (isTransientPayloadStillReferenced(prepared.storedPayload())) {
log.info("[similar-asin] keep prepared chunk payload because it is referenced taskId={} chunk={}",
prepared.taskId(), prepared.chunkIndex());
return;
}
transientPayloadStorageService.deletePayloadIfPresent(prepared.storedPayload());
} catch (Exception ex) {
log.warn("[similar-asin] cleanup uncommitted chunk payload failed taskId={} chunk={} err={}",
prepared.taskId(), prepared.chunkIndex(), ex.getMessage());
}
}
private FinalizeTaskResult completeSubmittedChunk(SubmitContext context) {
FileTaskEntity task = fileTaskMapper.selectById(context.task().getId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
@@ -1021,15 +1040,13 @@ public class SimilarAsinTaskService {
if (!STATUS_RUNNING.equals(task.getStatus())) {
log.info("[similar-asin] skip completion because task already finalized taskId={} status={}",
task.getId(), task.getStatus());
return;
return null;
}
upsertScopeState(task.getId(), context.scopeKey(), context.scopeHash(), null, context.error(), context.forceFlush(), false);
if (context.forceFlush() || context.error() != null && !context.error().isBlank()) {
finalizeTask(task, context.error(), allRowCount(task), true);
return;
return finalizeTask(task, context.error(), context.taskMetadata(), true);
}
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
touchJavaSideTaskActivity(task.getId());
return null;
}
private void submitCozeForSubmittedChunk(SubmitContext context) {
@@ -1118,7 +1135,11 @@ public class SimilarAsinTaskService {
if (tryRecoverTimedOutPythonTask(task)) {
return;
}
finalizeTask(task, error, allRowCount(task), true);
SubmittedTaskMetadata metadata = readSubmittedTaskMetadata(task);
FinalizeTaskResult result = transactionManager == null
? finalizeTask(task, error, metadata, true)
: inNewTransaction(() -> finalizeTask(task, error, metadata, true));
applyFinalizeSideEffects(result);
}
private boolean tryRecoverTimedOutPythonTask(FileTaskEntity task) {
@@ -1143,7 +1164,7 @@ public class SimilarAsinTaskService {
log.warn("[similar-asin] Python 超时恢复继续推进 Coze/文件收尾 taskId={} pendingCozeStates={} activeAssembleJobs={}",
taskId, pendingCozeStates, activeAssembleJobs);
}
submitCozeForSubmittedChunk(new SubmitContext(task, null, null, null, true, null));
submitCozeForSubmittedChunk(new SubmitContext(task, null, null, null, true, null, null));
touchJavaSideTaskActivity(taskId);
return true;
}
@@ -1220,7 +1241,7 @@ public class SimilarAsinTaskService {
scope.setReceivedChunkCount(countChunks(taskId, scopeHash));
scope.setLastChunkAt(now);
scope.setLastError(error);
scope.setCompleted(completed ? 1 : 0);
scope.setCompleted(completed || Integer.valueOf(1).equals(scope.getCompleted()) ? 1 : 0);
scope.setUpdatedAt(now);
scope.setStateJson(cozeDone
? "{\"phase\":\"RECEIVED\",\"coze\":\"DONE\"}"
@@ -1247,7 +1268,7 @@ public class SimilarAsinTaskService {
scope.setReceivedChunkCount(resolveReceivedChunkProgress(taskId, scopeHash, scope.getChunkTotal()));
scope.setLastChunkAt(now);
scope.setLastError(error);
scope.setCompleted(completed ? 1 : 0);
scope.setCompleted(completed || Integer.valueOf(1).equals(scope.getCompleted()) ? 1 : 0);
scope.setUpdatedAt(now);
scope.setStateJson(cozeDone
? "{\"phase\":\"RECEIVED\",\"coze\":\"DONE\"}"
@@ -1573,21 +1594,7 @@ public class SimilarAsinTaskService {
private int allRowCount(FileTaskEntity task) {
try {
SimilarAsinParsedPayloadDto payload = readParsedPayload(task);
if (payload.getAllItems() != null && !payload.getAllItems().isEmpty()) {
return payload.getAllItems().size();
}
if (payload.getItems() != null && !payload.getItems().isEmpty()) {
return payload.getItems().size();
}
if (payload.getGroups() != null && !payload.getGroups().isEmpty()) {
return payload.getGroups().stream()
.map(SimilarAsinParsedGroupVo::getItems)
.filter(Objects::nonNull)
.mapToInt(List::size)
.sum();
}
return 0;
return rowCount(readParsedPayload(task));
} catch (Exception ex) {
log.warn("[similar-asin] read all row count failed taskId={} err={}", task.getId(), ex.getMessage());
return 0;
@@ -1623,8 +1630,12 @@ public class SimilarAsinTaskService {
return Math.max(1, configured);
}
private void finalizeTask(FileTaskEntity task, String error, int rowCount, boolean assembleWorkbook) {
private FinalizeTaskResult finalizeTask(FileTaskEntity task,
String error,
SubmittedTaskMetadata taskMetadata,
boolean assembleWorkbook) {
String finalError = error;
int rowCount = taskMetadata == null ? 0 : taskMetadata.rowCount();
FileResultEntity result = null;
boolean hasPersistedResultRows = hasPersistedResultRows(task.getId());
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
@@ -1634,20 +1645,13 @@ public class SimilarAsinTaskService {
if (!rows.isEmpty()) {
result = rows.getFirst();
} else if (assembleWorkbook && hasPersistedResultRows) {
result = createResultRecordForAssembly(task, rowCount);
result = createResultRecordForAssembly(task, rowCount,
taskMetadata == null ? null : taskMetadata.sourceFiles());
}
if (result != null) {
boolean shouldAssembleResult = assembleWorkbook && (finalError == null || finalError.isBlank() || hasPersistedResultRows);
if (shouldAssembleResult && shouldAssembleSynchronously()) {
try {
// P0-3:stale-recovery 同步 finalize 路径走这里,确保
// assemble 之前缓冲的 cozeRows 已合并到 chunk。
flushBufferedCozeResults(task.getId());
assembleResultWorkbook(task, result);
} catch (Exception ex) {
log.warn("[相似ASIN] 组装结果工作簿失败 任务ID={} 错误={}", task.getId(), ex.getMessage());
finalError = firstNonBlank(finalError, "生成相似ASIN结果失败");
}
throw new IllegalStateException("Synchronous result assembly is not allowed in a database transaction");
}
}
boolean failed = finalError != null && !finalError.isBlank();
@@ -1677,11 +1681,16 @@ public class SimilarAsinTaskService {
log.warn("[相似ASIN] 跳过失败任务的工作簿组装,因为没有已持久化结果行 任务ID={}", task.getId());
}
}
taskCacheService.deleteTaskCache(task.getId());
// P1-1:任务进入终态(SUCCESS/FAILED)后清理滑窗记录,避免长任务残留内存。
// 处于 waitingForAssemble(仍为 RUNNING,等待异步 assemble)时不清理,待 assemble 完真正进入终态再由后续路径触发。
if (!waitingForAssemble) {
clearPoisonWindow(task.getId());
return new FinalizeTaskResult(task.getId(), !waitingForAssemble);
}
private void applyFinalizeSideEffects(FinalizeTaskResult result) {
if (result == null || result.taskId() == null) {
return;
}
taskCacheService.deleteTaskCache(result.taskId());
if (result.terminal()) {
clearPoisonWindow(result.taskId());
}
}
@@ -1716,12 +1725,19 @@ public class SimilarAsinTaskService {
private FileResultEntity createResultRecordForAssembly(FileTaskEntity task, int rowCount) {
SimilarAsinParsedPayloadDto payload = readParsedPayload(task);
List<SimilarAsinSourceFileDto> sourceFiles = payload.getSourceFiles() == null ? List.of() : payload.getSourceFiles();
return createResultRecordForAssembly(task, rowCount,
payload.getSourceFiles() == null ? List.of() : payload.getSourceFiles());
}
private FileResultEntity createResultRecordForAssembly(FileTaskEntity task,
int rowCount,
List<SimilarAsinSourceFileDto> sourceFiles) {
List<SimilarAsinSourceFileDto> resolvedSourceFiles = sourceFiles == null ? List.of() : sourceFiles;
FileResultEntity result = new FileResultEntity();
result.setTaskId(task.getId());
result.setModuleType(MODULE_TYPE);
result.setSourceFilename(buildAggregateSourceFilenameLabel(sourceFiles));
result.setSourceFileUrl(buildAggregateScopeKey(sourceFiles));
result.setSourceFilename(buildAggregateSourceFilenameLabel(resolvedSourceFiles));
result.setSourceFileUrl(buildAggregateScopeKey(resolvedSourceFiles));
result.setRowCount(rowCount);
result.setUserId(task.getUserId());
result.setCreatedAt(LocalDateTime.now());
@@ -4032,7 +4048,7 @@ public class SimilarAsinTaskService {
*/
private void bindTaskToCurrentOwnerForLocalFallback(FileTaskEntity task, String scopeHash, Integer chunkIndex) {
if (task == null || task.getId() == null) {
return;
throw new IllegalStateException("Cannot bind local fallback without a task ID");
}
String currentInstance = currentInstanceId();
try {
@@ -4054,28 +4070,30 @@ public class SimilarAsinTaskService {
update.setId(task.getId());
update.setResultJson(updatedJson);
update.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(update);
if (fileTaskMapper.updateById(update) != 1) {
throw new IllegalStateException("Task owner update affected no row");
}
log.warn("[similar-asin] task bound to current instance due to rustfs fallback taskId={} instanceId={} chunk={}",
task.getId(), currentInstance, chunkIndex);
} else if (!Objects.equals(existingOwner, currentInstance)) {
log.error("[similar-asin] rustfs fallback on non-owner instance, chunk will be unreachable taskId={} chunk={} owner={} current={}",
task.getId(), chunkIndex, existingOwner, currentInstance);
throw new IllegalStateException("Local fallback task is owned by instance=" + existingOwner
+ " current=" + currentInstance);
}
} catch (Exception ex) {
log.warn("[similar-asin] bind owner for local fallback failed taskId={} err={}", task.getId(), ex.getMessage());
throw new IllegalStateException("Failed to bind local fallback task owner taskId=" + task.getId(), ex);
}
// 把 fallback 信号写到 task_scope_state.state_json,方便排查 / 后续告警钩子。
if (scopeHash == null || scopeHash.isBlank()) {
return;
}
try {
if (scopeHash == null || scopeHash.isBlank()) {
return;
}
TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, task.getId())
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.last("limit 1"));
if (scope == null) {
return;
throw new IllegalStateException("Local fallback scope state does not exist");
}
ObjectNode stateNode;
String stateJson = scope.getStateJson();
@@ -4106,11 +4124,13 @@ public class SimilarAsinTaskService {
fallbackArr.add(tag);
scope.setStateJson(objectMapper.writeValueAsString(stateNode));
scope.setUpdatedAt(LocalDateTime.now());
taskScopeStateMapper.updateById(scope);
if (taskScopeStateMapper.updateById(scope) != 1) {
throw new IllegalStateException("Local fallback scope update affected no row");
}
}
} catch (Exception ex) {
log.warn("[similar-asin] write local-fallback state failed taskId={} chunk={} err={}",
task.getId(), chunkIndex, ex.getMessage());
throw new IllegalStateException("Failed to record local fallback state taskId="
+ task.getId() + " chunk=" + chunkIndex, ex);
}
}
@@ -6385,17 +6405,22 @@ public class SimilarAsinTaskService {
}
private void deleteTransientTaskPayloads(Long taskId) {
deleteTransientPayloads(collectTransientTaskPayloads(taskId), taskId);
}
private List<String> collectTransientTaskPayloads(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
return List.of();
}
List<String> payloads = new ArrayList<>();
List<TaskScopeStateEntity> scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (scopes != null) {
for (TaskScopeStateEntity scope : scopes) {
transientPayloadStorageService.deletePayloadIfPresent(scope.getParsedPayloadJson());
transientPayloadStorageService.deletePayloadIfPresent(scope.getStateJson());
payloads.add(scope.getParsedPayloadJson());
payloads.add(scope.getStateJson());
}
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
@@ -6404,9 +6429,29 @@ public class SimilarAsinTaskService {
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
payloads.add(chunk.getPayloadJson());
}
}
payloads.removeIf(value -> value == null || value.isBlank());
return payloads;
}
private record PreparedSubmittedChunk(Long taskId,
String scopeKey,
String scopeHash,
Integer chunkIndex,
Integer chunkTotal,
boolean done,
String error,
String payloadJson,
String storedPayload,
boolean localFallback,
SubmittedTaskMetadata taskMetadata) {
}
private record PersistSubmittedChunkResult(SubmitContext context,
boolean payloadPersisted,
FinalizeTaskResult finalizeResult) {
}
private record SubmitContext(FileTaskEntity task,
@@ -6414,7 +6459,16 @@ public class SimilarAsinTaskService {
String scopeHash,
Integer chunkIndex,
boolean forceFlush,
String error) {
String error,
SubmittedTaskMetadata taskMetadata) {
}
private record SubmittedTaskMetadata(int rowCount,
List<SimilarAsinSourceFileDto> sourceFiles) {
}
private record FinalizeTaskResult(Long taskId,
boolean terminal) {
}
private record CozeBatchContext(Long jobId,