处理数据量binlog问题

This commit is contained in:
supernijia
2026-07-28 15:49:41 +08:00
parent e453e822c0
commit 28423e015b
8 changed files with 187 additions and 15 deletions
@@ -11,6 +11,7 @@ import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
@@ -212,6 +213,21 @@ public class TaskFileJobService {
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
}
public void touchRunningIfStale(Long jobId, long intervalMillis) {
if (jobId == null || jobId <= 0) {
return;
}
LocalDateTime now = LocalDateTime.now();
LocalDateTime cutoff = now.minus(Duration.ofMillis(Math.max(1_000L, intervalMillis)));
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, jobId)
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.and(wrapper -> wrapper.isNull(TaskFileJobEntity::getUpdatedAt)
.or()
.le(TaskFileJobEntity::getUpdatedAt, cutoff))
.set(TaskFileJobEntity::getUpdatedAt, now));
}
public boolean deferRunning(Long jobId, String message) {
if (jobId == null || jobId <= 0) {
return false;
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
@@ -25,6 +26,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -62,6 +64,7 @@ public class TaskHeartbeatService {
private final WithdrawTaskCacheService withdrawTaskCacheService;
private final AppearancePatentTaskCacheService appearancePatentTaskCacheService;
private final SimilarAsinTaskCacheService similarAsinTaskCacheService;
private final SimilarAsinProperties similarAsinProperties;
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
@@ -120,6 +123,9 @@ public class TaskHeartbeatService {
task.getId(), task.getUserId(), moduleType, status);
return TaskHeartbeatVo.notAlive(moduleType, status, "task is not running");
}
if (MODULE_SIMILAR_ASIN.equals(moduleType)) {
return touchSimilarAsinHeartbeat(task);
}
LocalDateTime now = LocalDateTime.now();
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
@@ -138,6 +144,39 @@ public class TaskHeartbeatService {
return TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING);
}
private TaskHeartbeatVo touchSimilarAsinHeartbeat(FileTaskEntity task) {
LocalDateTime now = LocalDateTime.now();
long intervalMillis = Math.max(1_000L, similarAsinProperties.getDbTaskTouchIntervalMillis());
LocalDateTime cutoff = now.minus(Duration.ofMillis(intervalMillis));
boolean checkpointDue = task.getUpdatedAt() == null || !task.getUpdatedAt().isAfter(cutoff);
similarAsinTaskCacheService.touchTaskHeartbeat(task.getId());
if (!checkpointDue) {
saveFileTaskCache(MODULE_SIMILAR_ASIN, task);
return TaskHeartbeatVo.alive(MODULE_SIMILAR_ASIN, STATUS_RUNNING);
}
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.and(wrapper -> wrapper.isNull(FileTaskEntity::getUpdatedAt)
.or()
.le(FileTaskEntity::getUpdatedAt, cutoff))
.set(FileTaskEntity::getUpdatedAt, now));
if (updated <= 0) {
FileTaskEntity latest = fileTaskMapper.selectById(task.getId());
if (latest == null || !STATUS_RUNNING.equals(latest.getStatus())) {
return TaskHeartbeatVo.notAlive(MODULE_SIMILAR_ASIN,
latest == null ? task.getStatus() : latest.getStatus(), "task is not running");
}
task = latest;
} else {
task.setUpdatedAt(now);
}
saveFileTaskCache(MODULE_SIMILAR_ASIN, task);
return TaskHeartbeatVo.alive(MODULE_SIMILAR_ASIN, STATUS_RUNNING);
}
private TaskHeartbeatVo touchBrandTaskIfRunning(BrandCrawlTaskEntity task, TaskHeartbeatRequest request) {
if (task == null) {
return null;
@@ -12,6 +12,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.util.Objects;
@Service
@RequiredArgsConstructor
@@ -61,6 +62,10 @@ public class TaskProgressSnapshotService {
if (existing == null) {
throw new BusinessException("保存任务进度快照失败");
}
if (isUnchanged(existing, status, totalCount, successCount, failedCount, pendingCount,
currentScopeKey, message, snapshotJson)) {
return;
}
taskProgressSnapshotMapper.update(null, new LambdaUpdateWrapper<TaskProgressSnapshotEntity>()
.eq(TaskProgressSnapshotEntity::getId, existing.getId())
.set(TaskProgressSnapshotEntity::getStatus, status)
@@ -74,6 +79,25 @@ public class TaskProgressSnapshotService {
.set(TaskProgressSnapshotEntity::getUpdatedAt, now));
}
private boolean isUnchanged(TaskProgressSnapshotEntity existing,
String status,
int totalCount,
int successCount,
int failedCount,
int pendingCount,
String currentScopeKey,
String message,
String snapshotJson) {
return Objects.equals(existing.getStatus(), status)
&& Objects.equals(existing.getTotalCount(), totalCount)
&& Objects.equals(existing.getSuccessCount(), successCount)
&& Objects.equals(existing.getFailedCount(), failedCount)
&& Objects.equals(existing.getPendingCount(), pendingCount)
&& Objects.equals(existing.getCurrentScopeKey(), currentScopeKey)
&& Objects.equals(existing.getMessage(), message)
&& Objects.equals(existing.getSnapshotJson(), snapshotJson);
}
public TaskProgressSnapshotEntity find(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return null;