super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-122: N+1 批量化修复（claim/stuck-reset/stale-check 逐行 selectById 改 selectBatchIds 批量回读 + Map 装配，行为等价）

Browse Source
This commit is contained in:
黄自达
2026-09-02 03:13:05 +08:00
parent 29625f703c
commit 445b7780c5
7 changed files with 536 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java
+39 -9
View File
@@ -36,8 +36,10 @@ import java.nio.file.attribute.FileTime;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
@@ -178,6 +180,7 @@ public class DeleteBrandStaleTaskService {
.lt(FileTaskEntity::getUpdatedAt, threshold)
.last("limit 200"));
List<StaleFinalizeEntry> entries = new ArrayList<>();
for (FileTaskEntity task : runningTasks) {
Map<Object, Object> progress = deleteBrandTaskCacheService.getProgress(task.getId());
long lastHeartbeatAt = 0L;
@@ -203,28 +206,52 @@ public class DeleteBrandStaleTaskService {
if (taskLockHandle == null) {
continue;
}
boolean finalizeThrew;
try (taskLockHandle) {
try {
deleteBrandRunService.tryFinalizeTask(task.getId(), true);
FileTaskEntity refreshed = fileTaskMapper.selectById(task.getId());
try {
deleteBrandRunService.tryFinalizeTask(task.getId(), true);
finalizeThrew = false;
} catch (Exception ex) {
finalizeThrew = true;
log.warn("[stale-check] delete-brand finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
}
}
entries.add(new StaleFinalizeEntry(task, lastHeartbeatAt, completedScopeCount, hasStartedProgress, finalizeThrew));
}
if (entries.isEmpty()) {
return;
}
// 两阶段：全部 finalize 后一次 IN 批量回读，替代逐任务 selectById。
List<Long> refreshTaskIds = entries.stream()
.filter(entry -> !entry.finalizeThrew)
.map(entry -> entry.task.getId())
.toList();
Map<Long, FileTaskEntity> refreshedById = refreshTaskIds.isEmpty()
? Map.of()
: fileTaskMapper.selectBatchIds(refreshTaskIds).stream()
.collect(Collectors.toMap(FileTaskEntity::getId, task -> task, (a, b) -> a));
for (StaleFinalizeEntry entry : entries) {
FileTaskEntity task = entry.task;
if (!entry.finalizeThrew) {
FileTaskEntity refreshed = refreshedById.get(task.getId());
if (refreshed != null && !"RUNNING".equals(refreshed.getStatus())) {
deleteBrandTaskCacheService.saveTaskCache(refreshed);
log.info("[stale-check] delete-brand finalized before timeout-fail taskId={} status={} updatedAt={} finishedAt={}",
refreshed.getId(), refreshed.getStatus(), refreshed.getUpdatedAt(), refreshed.getFinishedAt());
continue;
}
} catch (Exception ex) {
log.warn("[stale-check] delete-brand finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
}
log.warn("[stale-check] delete-brand failing stale task -> taskId={} updatedAt={} createdAt={} lastHeartbeatAt={} timeoutMinutes={} completedScopes={} hasStartedProgress={}",
task.getId(),
task.getUpdatedAt(),
task.getCreatedAt(),
lastHeartbeatAt,
entry.lastHeartbeatAt,
minutes,
completedScopeCount,
hasStartedProgress);
entry.completedScopeCount,
entry.hasStartedProgress);
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
@@ -238,10 +265,13 @@ public class DeleteBrandStaleTaskService {
deleteBrandTaskCacheService.delete(task.getId());
log.warn("[stale-check] delete-brand failed taskId={} reason=timeout", task.getId());
}
}
}
}
private record StaleFinalizeEntry(FileTaskEntity task, long lastHeartbeatAt, int completedScopeCount,
boolean hasStartedProgress, boolean finalizeThrew) {
}
private ProductRiskStaleCheckStats failStaleProductRiskResolveTasks() {
ProductRiskStaleCheckStats stats = new ProductRiskStaleCheckStats();
long minutes = Math.max(1L, deleteBrandProgressProperties.getProductRiskStaleTimeoutMinutes());
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java
+50 -9
View File
@@ -17,6 +17,7 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
@@ -142,15 +143,22 @@ public class TaskFileJobService {
if (candidates == null || candidates.isEmpty()) {
return List.of();
}
List<TaskFileJobEntity> claimed = new ArrayList<>();
List<Long> claimedIds = new ArrayList<>();
for (TaskFileJobEntity candidate : candidates) {
if (candidate == null || candidate.getId() == null) {
continue;
}
if (!markRunning(candidate.getId())) {
continue;
if (markRunning(candidate.getId())) {
claimedIds.add(candidate.getId());
}
TaskFileJobEntity claim = taskFileJobMapper.selectById(candidate.getId());
}
if (claimedIds.isEmpty()) {
return List.of();
}
Map<Long, TaskFileJobEntity> claimedById = refreshJobsByIds(claimedIds);
List<TaskFileJobEntity> claimed = new ArrayList<>();
for (Long id : claimedIds) {
TaskFileJobEntity claim = claimedById.get(id);
if (claim != null && "RUNNING".equals(claim.getStatus())) {
claimed.add(claim);
}
@@ -259,10 +267,16 @@ public class TaskFileJobService {
.last("limit " + Math.max(1, Math.min(limit, 200))));
int reset = 0;
List<TaskFileJobEntity> exhaustedJobs = new ArrayList<>();
List<Long> eventJobIds = new ArrayList<>();
List<Long> exhaustedJobIds = new ArrayList<>();
Map<Long, TaskFileJobEntity> originalExhaustedById = new LinkedHashMap<>();
Map<Long, TaskFileJobEntity> originalJobsById = jobs.stream()
.collect(Collectors.toMap(TaskFileJobEntity::getId, job -> job, (a, b) -> a));
for (TaskFileJobEntity job : jobs) {
if ("FAILED".equals(job.getStatus())
&& job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT) {
exhaustedJobs.add(job);
exhaustedJobIds.add(job.getId());
originalExhaustedById.put(job.getId(), job);
continue;
}
if ("PENDING".equals(job.getStatus())) {
@@ -279,7 +293,7 @@ public class TaskFileJobService {
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
if (updated > 0) {
publishDispatchEvent(taskFileJobMapper.selectById(job.getId()));
eventJobIds.add(job.getId());
}
continue;
}
@@ -300,8 +314,7 @@ public class TaskFileJobService {
.set(TaskFileJobEntity::getFinishedAt, now)
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
if (updated > 0) {
TaskFileJobEntity exhausted = taskFileJobMapper.selectById(job.getId());
exhaustedJobs.add(exhausted == null ? job : exhausted);
exhaustedJobIds.add(job.getId());
}
continue;
}
@@ -312,12 +325,40 @@ public class TaskFileJobService {
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
if (updated > 0) {
reset++;
publishDispatchEvent(taskFileJobMapper.selectById(job.getId()));
eventJobIds.add(job.getId());
}
}
if (!eventJobIds.isEmpty()) {
Map<Long, TaskFileJobEntity> refreshedById = refreshJobsByIds(eventJobIds);
for (Long jobId : eventJobIds) {
publishDispatchEvent(refreshedById.get(jobId));
}
}
if (!exhaustedJobIds.isEmpty()) {
List<Long> refreshIds = exhaustedJobIds.stream()
.filter(id -> !originalExhaustedById.containsKey(id))
.toList();
Map<Long, TaskFileJobEntity> refreshedById = refreshIds.isEmpty()
? Map.of()
: refreshJobsByIds(refreshIds);
for (Long jobId : exhaustedJobIds) {
TaskFileJobEntity original = originalExhaustedById.get(jobId);
if (original != null) {
exhaustedJobs.add(original);
continue;
}
TaskFileJobEntity refreshed = refreshedById.get(jobId);
exhaustedJobs.add(refreshed == null ? originalJobsById.get(jobId) : refreshed);
}
}
return new StuckJobResetResult(reset, List.copyOf(exhaustedJobs));
}
private Map<Long, TaskFileJobEntity> refreshJobsByIds(List<Long> jobIds) {
return taskFileJobMapper.selectBatchIds(jobIds).stream()
.collect(Collectors.toMap(TaskFileJobEntity::getId, job -> job, (a, b) -> a));
}
public record StuckJobResetResult(int resetCount, List<TaskFileJobEntity> exhaustedJobs) {
}