处理数据量binlog问题

This commit is contained in:
supernijia
2026-07-28 15:49:41 +08:00
parent e453e822c0
commit 28423e015b
8 changed files with 187 additions and 15 deletions
@@ -1026,7 +1026,7 @@ public class SimilarAsinTaskService {
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
boolean pendingCoze = submitCozeBatches(task, result, job, chunks, allRowsByBaseId);
if (pendingCoze) {
taskFileJobService.touchRunning(job.getId());
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
touchJavaSideTaskActivity(task.getId());
} else if (isResultSubmissionComplete(task.getId())) {
maybeFinalizeCozeJobLocked(task.getId(), new CozeBatchContext(
@@ -1715,7 +1715,7 @@ public class SimilarAsinTaskService {
finalizeTimedOutCozeStatesForTask(task.getId());
}
if (countPendingCozeStates(task.getId()) > 0) {
taskFileJobService.touchRunning(job.getId());
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
touchJavaSideTaskActivity(task.getId());
saveFileBuildProgress(task, job, totalProgressUnits, 1, "Coze 已提交,等待结果");
return false;
@@ -1737,13 +1737,13 @@ public class SimilarAsinTaskService {
saveFileBuildProgress(task, job, totalProgressUnits, 0, "正在提交 Coze");
boolean pendingCoze = submitCozeBatches(task, result, job, chunks, allRowsByBaseId);
if (pendingCoze) {
taskFileJobService.touchRunning(job.getId());
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
touchJavaSideTaskActivity(task.getId());
saveFileBuildProgress(task, job, totalProgressUnits, 1, "Coze 已提交,等待结果");
return false;
}
if (STATUS_RUNNING.equals(task.getStatus()) && !isResultSubmissionComplete(task.getId())) {
taskFileJobService.touchRunning(job.getId());
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
touchJavaSideTaskActivity(task.getId());
saveFileBuildProgress(task, job, totalProgressUnits, Math.max(1, cozeWorkUnits),
"等待 Python 上传,每 " + batchSize + " 行提交一次 Coze");
@@ -2140,7 +2140,7 @@ public class SimilarAsinTaskService {
if (batchRows.isEmpty()) {
return false;
}
taskFileJobService.touchRunning(job.getId());
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
String batchScopeKey = buildCozeBatchScopeKey(task.getId(), batchRows);
String batchScopeHash = DigestUtil.sha256Hex(batchScopeKey);
TaskScopeStateEntity existing = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
@@ -2188,7 +2188,7 @@ public class SimilarAsinTaskService {
if (CozeFailureClassifier.isThrottleLockTimeout(message)) {
savePendingCozeBatchState(task, result, job, batchRows, batchScopeKey, batchScopeHash,
batchIndex, batchTotal, message, credential.name());
taskFileJobService.touchRunning(job.getId());
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
touchJavaSideTaskActivity(task.getId());
return true;
}
@@ -2359,12 +2359,11 @@ public class SimilarAsinTaskService {
state.getTaskId(), state.getId(), context.ownerInstanceId(), currentInstanceId());
return;
}
taskFileJobService.touchRunning(context.jobId());
taskFileJobService.touchRunningIfStale(context.jobId(), properties.getDbJobTouchIntervalMillis());
try {
// P0-2 最小风险变体:优先复用预取阶段的 HTTP 响应;未命中(单 state 或失败兜底)走原同步 HTTP。
SimilarAsinCozeClient.CozePollResponse poll = pollWorkflowCached(state, context.credentialName());
if (!poll.hasPayload() && !poll.isFinished() && !isCozeStateTimedOut(state)) {
updateCozeStateRunning(state, null);
return;
}
String failureMessage = poll.isFailed()
@@ -2552,7 +2551,7 @@ public class SimilarAsinTaskService {
.set(TaskScopeStateEntity::getCompleted, 0)
.set(TaskScopeStateEntity::getUpdatedAt, now));
if (updated > 0) {
taskFileJobService.touchRunning(context.jobId());
taskFileJobService.touchRunningIfStale(context.jobId(), properties.getDbJobTouchIntervalMillis());
touchJavaSideTaskActivity(state.getTaskId());
log.info("[similar-asin] coze retry submitted taskId={} stateId={} oldExecuteId={} newExecuteId={} chunk={} batch={}/{} retry={}/{} failure={}",
state.getTaskId(), state.getId(), state.getCozeExecuteId(), submit.executeId(),
@@ -2568,7 +2567,7 @@ public class SimilarAsinTaskService {
log.warn("[similar-asin] coze retry deferred by throttle lock taskId={} stateId={} executeId={}",
state.getTaskId(), state.getId(), state.getCozeExecuteId());
deferStateForResubmit(state, msg);
taskFileJobService.touchRunning(context.jobId());
taskFileJobService.touchRunningIfStale(context.jobId(), properties.getDbJobTouchIntervalMillis());
touchJavaSideTaskActivity(state.getTaskId());
return true;
}
@@ -2824,7 +2823,7 @@ public class SimilarAsinTaskService {
}
if (submittedAny) {
markCozeStateTerminal(state, COZE_STATUS_DONE, "split retry submitted after failure: " + firstNonBlank(failureMessage, "unknown"));
taskFileJobService.touchRunning(context.jobId());
taskFileJobService.touchRunningIfStale(context.jobId(), properties.getDbJobTouchIntervalMillis());
touchJavaSideTaskActivity(state.getTaskId());
maybeFinalizeCozeJob(state.getTaskId(), context);
log.info("[similar-asin] coze split retry submitted taskId={} stateId={} chunk={} batch={}/{} parts={} retry={}/{} poisonRow={} failure={}",
@@ -2840,7 +2839,7 @@ public class SimilarAsinTaskService {
log.warn("[similar-asin] coze split retry deferred by throttle lock taskId={} stateId={} parts={}",
state.getTaskId(), state.getId(), partitions.size());
deferStateForResubmit(state, msg);
taskFileJobService.touchRunning(context.jobId());
taskFileJobService.touchRunningIfStale(context.jobId(), properties.getDbJobTouchIntervalMillis());
touchJavaSideTaskActivity(state.getTaskId());
return true;
}
@@ -2940,7 +2939,7 @@ public class SimilarAsinTaskService {
return;
}
try {
taskFileJobService.touchRunning(context.jobId());
taskFileJobService.touchRunningIfStale(context.jobId(), properties.getDbJobTouchIntervalMillis());
SimilarAsinCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
cozeClient.credentialByName(context.credentialName()), false);
@@ -3246,6 +3245,8 @@ public class SimilarAsinTaskService {
.le(TaskScopeStateEntity::getCozeLastPolledAt, now.minus(Duration.ofMillis(intervalMillis))))
.set(TaskScopeStateEntity::getCozeStatus, COZE_STATUS_RUNNING)
.set(TaskScopeStateEntity::getCozeLastPolledAt, now)
.set(TaskScopeStateEntity::getCozeAttemptCount, cozeAttemptCount(state) + 1)
.set(TaskScopeStateEntity::getCozeError, null)
.set(TaskScopeStateEntity::getUpdatedAt, now)) > 0;
}
@@ -3304,7 +3305,7 @@ public class SimilarAsinTaskService {
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task != null && STATUS_RUNNING.equals(task.getStatus()) && !isResultSubmissionComplete(taskId)) {
taskFileJobService.touchRunning(job.getId());
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
touchJavaSideTaskActivity(taskId);
return;
}
@@ -3812,11 +3813,16 @@ public class SimilarAsinTaskService {
if (taskId == null || taskId <= 0) {
return;
}
LocalDateTime now = LocalDateTime.now();
LocalDateTime cutoff = now.minus(Duration.ofMillis(Math.max(1_000L, properties.getDbTaskTouchIntervalMillis())));
fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, taskId)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
.and(wrapper -> wrapper.isNull(FileTaskEntity::getUpdatedAt)
.or()
.le(FileTaskEntity::getUpdatedAt, cutoff))
.set(FileTaskEntity::getUpdatedAt, now));
}
private String buildCozeBatchScopeKey(Long taskId, List<SimilarAsinResultRowDto> batchRows) {