处理数据量binlog问题
This commit is contained in:
+16
@@ -11,6 +11,7 @@ import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
@@ -212,6 +213,21 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
}
|
||||
|
||||
public void touchRunningIfStale(Long jobId, long intervalMillis) {
|
||||
if (jobId == null || jobId <= 0) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LocalDateTime cutoff = now.minus(Duration.ofMillis(Math.max(1_000L, intervalMillis)));
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, jobId)
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.and(wrapper -> wrapper.isNull(TaskFileJobEntity::getUpdatedAt)
|
||||
.or()
|
||||
.le(TaskFileJobEntity::getUpdatedAt, cutoff))
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now));
|
||||
}
|
||||
|
||||
public boolean deferRunning(Long jobId, String message) {
|
||||
if (jobId == null || jobId <= 0) {
|
||||
return false;
|
||||
|
||||
+39
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||
@@ -25,6 +26,7 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
@@ -62,6 +64,7 @@ public class TaskHeartbeatService {
|
||||
private final WithdrawTaskCacheService withdrawTaskCacheService;
|
||||
private final AppearancePatentTaskCacheService appearancePatentTaskCacheService;
|
||||
private final SimilarAsinTaskCacheService similarAsinTaskCacheService;
|
||||
private final SimilarAsinProperties similarAsinProperties;
|
||||
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
||||
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||
|
||||
@@ -120,6 +123,9 @@ public class TaskHeartbeatService {
|
||||
task.getId(), task.getUserId(), moduleType, status);
|
||||
return TaskHeartbeatVo.notAlive(moduleType, status, "task is not running");
|
||||
}
|
||||
if (MODULE_SIMILAR_ASIN.equals(moduleType)) {
|
||||
return touchSimilarAsinHeartbeat(task);
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
@@ -138,6 +144,39 @@ public class TaskHeartbeatService {
|
||||
return TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING);
|
||||
}
|
||||
|
||||
private TaskHeartbeatVo touchSimilarAsinHeartbeat(FileTaskEntity task) {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
long intervalMillis = Math.max(1_000L, similarAsinProperties.getDbTaskTouchIntervalMillis());
|
||||
LocalDateTime cutoff = now.minus(Duration.ofMillis(intervalMillis));
|
||||
boolean checkpointDue = task.getUpdatedAt() == null || !task.getUpdatedAt().isAfter(cutoff);
|
||||
|
||||
similarAsinTaskCacheService.touchTaskHeartbeat(task.getId());
|
||||
if (!checkpointDue) {
|
||||
saveFileTaskCache(MODULE_SIMILAR_ASIN, task);
|
||||
return TaskHeartbeatVo.alive(MODULE_SIMILAR_ASIN, STATUS_RUNNING);
|
||||
}
|
||||
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.and(wrapper -> wrapper.isNull(FileTaskEntity::getUpdatedAt)
|
||||
.or()
|
||||
.le(FileTaskEntity::getUpdatedAt, cutoff))
|
||||
.set(FileTaskEntity::getUpdatedAt, now));
|
||||
if (updated <= 0) {
|
||||
FileTaskEntity latest = fileTaskMapper.selectById(task.getId());
|
||||
if (latest == null || !STATUS_RUNNING.equals(latest.getStatus())) {
|
||||
return TaskHeartbeatVo.notAlive(MODULE_SIMILAR_ASIN,
|
||||
latest == null ? task.getStatus() : latest.getStatus(), "task is not running");
|
||||
}
|
||||
task = latest;
|
||||
} else {
|
||||
task.setUpdatedAt(now);
|
||||
}
|
||||
saveFileTaskCache(MODULE_SIMILAR_ASIN, task);
|
||||
return TaskHeartbeatVo.alive(MODULE_SIMILAR_ASIN, STATUS_RUNNING);
|
||||
}
|
||||
|
||||
private TaskHeartbeatVo touchBrandTaskIfRunning(BrandCrawlTaskEntity task, TaskHeartbeatRequest request) {
|
||||
if (task == null) {
|
||||
return null;
|
||||
|
||||
+24
@@ -12,6 +12,7 @@ import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Objects;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -61,6 +62,10 @@ public class TaskProgressSnapshotService {
|
||||
if (existing == null) {
|
||||
throw new BusinessException("保存任务进度快照失败");
|
||||
}
|
||||
if (isUnchanged(existing, status, totalCount, successCount, failedCount, pendingCount,
|
||||
currentScopeKey, message, snapshotJson)) {
|
||||
return;
|
||||
}
|
||||
taskProgressSnapshotMapper.update(null, new LambdaUpdateWrapper<TaskProgressSnapshotEntity>()
|
||||
.eq(TaskProgressSnapshotEntity::getId, existing.getId())
|
||||
.set(TaskProgressSnapshotEntity::getStatus, status)
|
||||
@@ -74,6 +79,25 @@ public class TaskProgressSnapshotService {
|
||||
.set(TaskProgressSnapshotEntity::getUpdatedAt, now));
|
||||
}
|
||||
|
||||
private boolean isUnchanged(TaskProgressSnapshotEntity existing,
|
||||
String status,
|
||||
int totalCount,
|
||||
int successCount,
|
||||
int failedCount,
|
||||
int pendingCount,
|
||||
String currentScopeKey,
|
||||
String message,
|
||||
String snapshotJson) {
|
||||
return Objects.equals(existing.getStatus(), status)
|
||||
&& Objects.equals(existing.getTotalCount(), totalCount)
|
||||
&& Objects.equals(existing.getSuccessCount(), successCount)
|
||||
&& Objects.equals(existing.getFailedCount(), failedCount)
|
||||
&& Objects.equals(existing.getPendingCount(), pendingCount)
|
||||
&& Objects.equals(existing.getCurrentScopeKey(), currentScopeKey)
|
||||
&& Objects.equals(existing.getMessage(), message)
|
||||
&& Objects.equals(existing.getSnapshotJson(), snapshotJson);
|
||||
}
|
||||
|
||||
public TaskProgressSnapshotEntity find(Long taskId, String moduleType) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||
return null;
|
||||
|
||||
Reference in New Issue
Block a user