task-12: extend coze result buffering beyond poll path
- bufferCozeRowsOrMerge: unified DONE result persist entry point; buffers to transient storage when toggle on, falls back to immediate chunk merge on failure (result never lost), no-op on empty input - submit/retry synchronous immediate DONE results now buffer too (previously always merged immediately); submit path persists a DONE scope state row to carry the buffer pointer (persistImmediateCozeDoneState) - poll path drops the batchTotal>1 gate: single-batch tasks also buffer - CozeCandidate/CozeBatchContext records package-private for test access
This commit is contained in:
+121
-36
@@ -2616,6 +2616,17 @@ public class SimilarAsinTaskService {
|
||||
if (!emptyResultMessage.isBlank()) {
|
||||
throw new IllegalStateException(emptyResultMessage);
|
||||
}
|
||||
// Task 12：同步 immediate DONE 结果也走缓冲（原立即 merge），由 finalize/assemble 前
|
||||
// 一次性 flush 合并到 chunk，减少 chunk payload 频繁读写。先落一条 DONE state 承载
|
||||
// 缓冲 pointer；缓冲关闭/失败/重复时回退立即 merge，结果不丢失。
|
||||
if (isCozeResultBufferEnabled()) {
|
||||
TaskScopeStateEntity doneState = persistImmediateCozeDoneState(task, result, job, batchRows,
|
||||
batchScopeKey, batchScopeHash, batchIndex, batchTotal, submit.credentialName());
|
||||
if (doneState != null) {
|
||||
bufferCozeRowsOrMerge(doneState, readCozeBatchContext(doneState), cozeRows, task, allRowsByBaseId);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
mergeCozeRowsIntoChunk(task, null, null, cozeRows, allRowsByBaseId);
|
||||
return false;
|
||||
}
|
||||
@@ -2851,31 +2862,23 @@ public class SimilarAsinTaskService {
|
||||
if (!failureMessage.isBlank()) {
|
||||
cozeRows = cozeClient.markRowsFailed(batchRows, failureMessage);
|
||||
}
|
||||
// P0-3：仅"DONE 且 batchTotal>1 且 feature toggle 开启"时缓冲 cozeRows 到 transient storage，
|
||||
// 由 finalize 阶段一次性合并到 chunk。失败 batch / 单 batch 任务保留原立即 merge 路径。
|
||||
boolean buffered = false;
|
||||
if (failureMessage.isBlank()
|
||||
&& isCozeResultBufferEnabled()
|
||||
&& context.batchTotal() != null && context.batchTotal() > 1
|
||||
&& cozeRows != null && !cozeRows.isEmpty()) {
|
||||
CozeBatchContext bufferedContext = bufferCozeResultForFlush(state, context, cozeRows);
|
||||
if (bufferedContext != null
|
||||
&& bufferedContext.resultPayloadPointer() != null
|
||||
&& !bufferedContext.resultPayloadPointer().isBlank()) {
|
||||
context = bufferedContext;
|
||||
buffered = true;
|
||||
// Task 12：DONE 结果统一走 bufferCozeRowsOrMerge（原 P0-3 仅 poll 且 batchTotal>1 缓冲；
|
||||
// 现单 batch、submit/retry 同步 immediate 结果也缓冲），缓冲失败回退立即 merge，
|
||||
// flush 在 finalize/assemble 前一次性合并到 chunk，结果不丢失。
|
||||
// 失败行（markRowsFailed）保持立即 merge 语义不变。
|
||||
if (failureMessage.isBlank()) {
|
||||
if (cozeRows != null && !cozeRows.isEmpty()) {
|
||||
FileTaskEntity pollTask = taskForPoll(state.getTaskId());
|
||||
bufferCozeRowsOrMerge(state, context, cozeRows, pollTask,
|
||||
pollTask == null ? Map.of() : allRowsByBaseIdForPoll(pollTask));
|
||||
}
|
||||
}
|
||||
if (!buffered) {
|
||||
FileTaskEntity task = taskForPoll(state.getTaskId());
|
||||
if (task != null) {
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
||||
} else {
|
||||
FileTaskEntity pollTask = taskForPoll(state.getTaskId());
|
||||
if (pollTask != null) {
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(pollTask);
|
||||
try {
|
||||
mergeCozeRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), cozeRows, allRowsByBaseId);
|
||||
mergeCozeRowsIntoChunk(pollTask, context.chunkScopeHash(), context.chunkIndex(), cozeRows, allRowsByBaseId);
|
||||
} catch (Exception mergeEx) {
|
||||
if (failureMessage.isBlank()) {
|
||||
throw mergeEx;
|
||||
}
|
||||
log.warn("[similar-asin] coze failed result merge failed, mark state terminal anyway taskId={} stateId={} executeId={} err={}",
|
||||
state.getTaskId(), state.getId(), state.getCozeExecuteId(),
|
||||
firstNonBlank(mergeEx.getMessage(), mergeEx.getClass().getSimpleName()));
|
||||
@@ -3406,7 +3409,9 @@ public class SimilarAsinTaskService {
|
||||
if (!emptyResultMessage.isBlank()) {
|
||||
throw new IllegalStateException(emptyResultMessage);
|
||||
}
|
||||
mergeCozeRowsIntoChunk(task, null, null, cozeRows, allRowsByBaseId);
|
||||
// Task 12：retry 同步 immediate DONE 结果也走缓冲（原立即 merge），
|
||||
// 缓冲失败回退立即 merge；flush 在 finalize 时一次性完成。
|
||||
bufferCozeRowsOrMerge(state, context, cozeRows, task, allRowsByBaseId);
|
||||
markCozeStateTerminal(state, COZE_STATUS_DONE, null);
|
||||
maybeFinalizeCozeJobLocked(state.getTaskId(), context);
|
||||
return;
|
||||
@@ -4048,6 +4053,86 @@ public class SimilarAsinTaskService {
|
||||
return properties.isCozeResultBufferEnabled();
|
||||
}
|
||||
|
||||
/**
|
||||
* Task 12：统一 Coze DONE 结果落库入口。
|
||||
* 缓冲开关开启时把 cozeRows 写入 transient storage（pointer 存进 state.stateJson），
|
||||
* 由 flushBufferedCozeResults 在 finalize/assemble 前一次性合并到 chunk；
|
||||
* 缓冲失败（存储异常 / state 更新失败 / 开关关闭）回退立即 merge，结果不丢失。
|
||||
* 空 rows / 空 state / 空 context 直接返回，不产生任何写入。
|
||||
*/
|
||||
private void bufferCozeRowsOrMerge(TaskScopeStateEntity state,
|
||||
CozeBatchContext context,
|
||||
List<SimilarAsinResultRowDto> cozeRows,
|
||||
FileTaskEntity task,
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
|
||||
if (state == null || context == null || cozeRows == null || cozeRows.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
if (!isCozeResultBufferEnabled()) {
|
||||
mergeCozeRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), cozeRows, allRowsByBaseId);
|
||||
return;
|
||||
}
|
||||
CozeBatchContext bufferedContext = bufferCozeResultForFlush(state, context, cozeRows);
|
||||
if (bufferedContext == null
|
||||
|| bufferedContext.resultPayloadPointer() == null
|
||||
|| bufferedContext.resultPayloadPointer().isBlank()) {
|
||||
// 缓冲失败：回退立即 merge，避免结果悬挂在 transient storage 之外。
|
||||
mergeCozeRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), cozeRows, allRowsByBaseId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Task 12：为 submit 同步 immediate DONE 结果落一条 DONE state 承载缓冲 pointer。
|
||||
* 与 saveCozeBatchState（SUBMITTED 异步）不同，该 state 直接以 DONE 终态插入，
|
||||
* 不会被 countPendingCozeStates 扫描；缓冲失败/重复插入时返回 null，调用方回退立即 merge。
|
||||
*/
|
||||
private TaskScopeStateEntity persistImmediateCozeDoneState(FileTaskEntity task,
|
||||
FileResultEntity result,
|
||||
TaskFileJobEntity job,
|
||||
List<SimilarAsinResultRowDto> batchRows,
|
||||
String batchScopeKey,
|
||||
String batchScopeHash,
|
||||
int batchIndex,
|
||||
int batchTotal,
|
||||
String credentialName) {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
CozeBatchContext context = new CozeBatchContext(
|
||||
job.getId(),
|
||||
result.getId(),
|
||||
null,
|
||||
null,
|
||||
batchIndex,
|
||||
batchTotal,
|
||||
currentInstanceId(),
|
||||
0,
|
||||
credentialName,
|
||||
null
|
||||
);
|
||||
TaskScopeStateEntity state = new TaskScopeStateEntity();
|
||||
state.setTaskId(task.getId());
|
||||
state.setModuleType(MODULE_TYPE);
|
||||
state.setScopeKey(batchScopeKey);
|
||||
state.setScopeHash(batchScopeHash);
|
||||
state.setStateJson(writeJson(context, "serialize immediate coze done state context failed"));
|
||||
state.setCozeStatus(COZE_STATUS_DONE);
|
||||
state.setCozeSubmittedAt(now);
|
||||
state.setCozeCompletedAt(now);
|
||||
state.setCozeAttemptCount(0);
|
||||
state.setChunkTotal(batchTotal);
|
||||
state.setReceivedChunkCount(batchIndex);
|
||||
state.setCompleted(1);
|
||||
state.setCreatedAt(now);
|
||||
state.setUpdatedAt(now);
|
||||
try {
|
||||
taskScopeStateMapper.insert(state);
|
||||
return state;
|
||||
} catch (DuplicateKeyException ex) {
|
||||
log.info("[similar-asin] duplicate immediate done coze state ignored taskId={} scope={}",
|
||||
task.getId(), batchScopeKey);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* P0-3：在 finalize 之前一次性把所有 DONE/FAILED state 上缓冲的 cozeRows
|
||||
* 按 chunkScopeHash 分组合并到 chunk。把每个 batch 的"loadSubmittedChunks +
|
||||
@@ -4291,9 +4376,9 @@ public class SimilarAsinTaskService {
|
||||
return "coze:task:" + taskId + ":rows:" + DigestUtil.sha256Hex(rowKeys.toString());
|
||||
}
|
||||
|
||||
private record CozeCandidate(String chunkScopeHash,
|
||||
Integer chunkIndex,
|
||||
SimilarAsinResultRowDto row) {
|
||||
record CozeCandidate(String chunkScopeHash,
|
||||
Integer chunkIndex,
|
||||
SimilarAsinResultRowDto row) {
|
||||
}
|
||||
|
||||
private record PythonUploadProgress(int current, int total, String unit) {
|
||||
@@ -6789,16 +6874,16 @@ public class SimilarAsinTaskService {
|
||||
boolean terminal) {
|
||||
}
|
||||
|
||||
private record CozeBatchContext(Long jobId,
|
||||
Long resultId,
|
||||
String chunkScopeHash,
|
||||
Integer chunkIndex,
|
||||
Integer batchIndex,
|
||||
Integer batchTotal,
|
||||
String ownerInstanceId,
|
||||
Integer submitRetryCount,
|
||||
String credentialName,
|
||||
String resultPayloadPointer) {
|
||||
record CozeBatchContext(Long jobId,
|
||||
Long resultId,
|
||||
String chunkScopeHash,
|
||||
Integer chunkIndex,
|
||||
Integer batchIndex,
|
||||
Integer batchTotal,
|
||||
String ownerInstanceId,
|
||||
Integer submitRetryCount,
|
||||
String credentialName,
|
||||
String resultPayloadPointer) {
|
||||
}
|
||||
|
||||
private static class SourceRowsBuilder {
|
||||
|
||||
Reference in New Issue
Block a user