新需求更新 同步更新
This commit is contained in:
+18
@@ -17,4 +17,22 @@ public class TaskHeartbeatRequest {
|
||||
|
||||
@Schema(description = "可选的总进度", example = "1100")
|
||||
private Integer total;
|
||||
|
||||
@Schema(description = "采集数据任务当前阶段", example = "detail")
|
||||
private String collectStage;
|
||||
|
||||
@Schema(description = "采集数据任务当前关键词", example = "women bodysuit")
|
||||
private String currentKeyword;
|
||||
|
||||
@Schema(description = "搜索页当前页码", example = "2")
|
||||
private Integer searchCurrentPage;
|
||||
|
||||
@Schema(description = "搜索页总页数", example = "7")
|
||||
private Integer searchTotalPages;
|
||||
|
||||
@Schema(description = "详情页已处理 ASIN 数", example = "34")
|
||||
private Integer detailProcessedAsins;
|
||||
|
||||
@Schema(description = "详情页 ASIN 总数", example = "120")
|
||||
private Integer detailTotalAsins;
|
||||
}
|
||||
|
||||
+2
@@ -25,4 +25,6 @@ public class TaskFileJobEntity {
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
private LocalDateTime finishedAt;
|
||||
/** Parent task/result terminal-failure callback completion marker. */
|
||||
private LocalDateTime terminalCallbackAt;
|
||||
}
|
||||
|
||||
+150
-24
@@ -71,7 +71,8 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, null)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, null));
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(existing.getId());
|
||||
publishDispatchEvent(refreshed);
|
||||
return refreshed;
|
||||
@@ -148,7 +149,8 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getRetryCount, 0)
|
||||
.set(TaskFileJobEntity::getErrorMessage, null)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, null));
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (updated > 0) {
|
||||
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(jobId);
|
||||
publishDispatchEvent(refreshed);
|
||||
@@ -159,37 +161,70 @@ public class TaskFileJobService {
|
||||
|
||||
@Transactional
|
||||
public int resetStuckRunningJobs(int stuckMinutes, int limit) {
|
||||
return resetStuckRunningJobsDetailed(stuckMinutes, limit).resetCount();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public StuckJobResetResult resetStuckRunningJobsDetailed(int stuckMinutes, int limit) {
|
||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes));
|
||||
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.lt(TaskFileJobEntity::getUpdatedAt, threshold)
|
||||
.and(wrapper -> wrapper
|
||||
.and(running -> running
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.and(age -> age
|
||||
.lt(TaskFileJobEntity::getUpdatedAt, threshold)
|
||||
.or()
|
||||
.isNull(TaskFileJobEntity::getUpdatedAt)))
|
||||
.or(exhausted -> exhausted
|
||||
.eq(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.ge(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.isNull(TaskFileJobEntity::getTerminalCallbackAt)))
|
||||
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
|
||||
.last("limit " + Math.max(1, Math.min(limit, 200))));
|
||||
int reset = 0;
|
||||
List<TaskFileJobEntity> exhaustedJobs = new ArrayList<>();
|
||||
for (TaskFileJobEntity job : jobs) {
|
||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
||||
if (retryCount >= MAX_RETRY_COUNT) {
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if ("FAILED".equals(job.getStatus())
|
||||
&& job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT) {
|
||||
exhaustedJobs.add(job);
|
||||
continue;
|
||||
}
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
||||
int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1);
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LambdaUpdateWrapper<TaskFileJobEntity> update = new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now);
|
||||
if (nextRetryCount >= MAX_RETRY_COUNT) {
|
||||
int updated = taskFileJobMapper.update(null, update
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已达到最大重试次数")
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (updated > 0) {
|
||||
TaskFileJobEntity exhausted = taskFileJobMapper.selectById(job.getId());
|
||||
exhaustedJobs.add(exhausted == null ? job : exhausted);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
int updated = taskFileJobMapper.update(null, update
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (updated > 0) {
|
||||
reset++;
|
||||
publishDispatchEvent(taskFileJobMapper.selectById(job.getId()));
|
||||
}
|
||||
}
|
||||
return reset;
|
||||
return new StuckJobResetResult(reset, List.copyOf(exhaustedJobs));
|
||||
}
|
||||
|
||||
public record StuckJobResetResult(int resetCount, List<TaskFileJobEntity> exhaustedJobs) {
|
||||
}
|
||||
|
||||
public boolean markRunning(Long jobId) {
|
||||
@@ -199,7 +234,37 @@ public class TaskFileJobService {
|
||||
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, jobId)
|
||||
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
|
||||
.and(wrapper -> wrapper.isNull(TaskFileJobEntity::getRetryCount)
|
||||
.or()
|
||||
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT))
|
||||
.set(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null)) > 0;
|
||||
}
|
||||
|
||||
/** Returns the persisted RUNNING row so queued execution can later fence itself with updatedAt. */
|
||||
public TaskFileJobEntity claimRunning(Long jobId) {
|
||||
if (!markRunning(jobId)) {
|
||||
return null;
|
||||
}
|
||||
TaskFileJobEntity claim = taskFileJobMapper.selectById(jobId);
|
||||
return claim != null && "RUNNING".equals(claim.getStatus()) ? claim : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Activates a previously queued claim. The updatedAt CAS prevents a timed-out old queue entry from
|
||||
* starting after the same job has already been reset or claimed again.
|
||||
*/
|
||||
public boolean activateRunningClaim(TaskFileJobEntity claim) {
|
||||
if (claim == null || claim.getId() == null || claim.getUpdatedAt() == null) {
|
||||
return false;
|
||||
}
|
||||
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, claim.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(TaskFileJobEntity::getUpdatedAt, claim.getUpdatedAt())
|
||||
.eq(claim.getRetryCount() != null, TaskFileJobEntity::getRetryCount, claim.getRetryCount())
|
||||
.isNull(claim.getRetryCount() == null, TaskFileJobEntity::getRetryCount)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())) > 0;
|
||||
}
|
||||
|
||||
@@ -228,6 +293,23 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now));
|
||||
}
|
||||
|
||||
public void touchRunningAssembleJobsIfStale(Long taskId, String moduleType, long intervalMillis) {
|
||||
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LocalDateTime cutoff = now.minus(Duration.ofMillis(Math.max(1_000L, intervalMillis)));
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getTaskId, taskId)
|
||||
.eq(TaskFileJobEntity::getModuleType, moduleType)
|
||||
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.and(wrapper -> wrapper.isNull(TaskFileJobEntity::getUpdatedAt)
|
||||
.or()
|
||||
.le(TaskFileJobEntity::getUpdatedAt, cutoff))
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now));
|
||||
}
|
||||
|
||||
public boolean deferRunning(Long jobId, String message) {
|
||||
if (jobId == null || jobId <= 0) {
|
||||
return false;
|
||||
@@ -252,10 +334,14 @@ public class TaskFileJobService {
|
||||
.eq(TaskFileJobEntity::getId, jobId)
|
||||
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
|
||||
.ne(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.and(wrapper -> wrapper.isNull(TaskFileJobEntity::getRetryCount)
|
||||
.or()
|
||||
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT))
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, null));
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (updated <= 0) {
|
||||
return false;
|
||||
}
|
||||
@@ -265,13 +351,19 @@ public class TaskFileJobService {
|
||||
}
|
||||
|
||||
public void markSuccess(TaskFileJobEntity job, String resultFileUrl) {
|
||||
if (job == null || job.getId() == null) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.set(TaskFileJobEntity::getStatus, "SUCCESS")
|
||||
.set(TaskFileJobEntity::getErrorMessage, null)
|
||||
.set(TaskFileJobEntity::getResultFileUrl, resultFileUrl)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, now));
|
||||
}
|
||||
|
||||
public TaskFileJobEntity findById(Long jobId) {
|
||||
@@ -283,19 +375,52 @@ public class TaskFileJobService {
|
||||
|
||||
public boolean isRetryExhausted(Long jobId) {
|
||||
TaskFileJobEntity job = findById(jobId);
|
||||
return job != null && job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT;
|
||||
return job != null
|
||||
&& "FAILED".equals(job.getStatus())
|
||||
&& job.getRetryCount() != null
|
||||
&& job.getRetryCount() >= MAX_RETRY_COUNT;
|
||||
}
|
||||
|
||||
public void markFailed(TaskFileJobEntity job, String message) {
|
||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount() + 1;
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
TaskFileJobEntity current = job == null || job.getId() == null
|
||||
? null
|
||||
: taskFileJobMapper.selectById(job.getId());
|
||||
if (current == null || "SUCCESS".equals(current.getStatus())) {
|
||||
return;
|
||||
}
|
||||
int currentRetryCount = current.getRetryCount() == null ? 0 : current.getRetryCount();
|
||||
int retryCount = Math.min(MAX_RETRY_COUNT, currentRetryCount + 1);
|
||||
LambdaUpdateWrapper<TaskFileJobEntity> update = new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
|
||||
.eq(current.getRetryCount() != null, TaskFileJobEntity::getRetryCount, current.getRetryCount())
|
||||
.isNull(current.getRetryCount() == null, TaskFileJobEntity::getRetryCount)
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getRetryCount, retryCount)
|
||||
.set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, retryCount >= MAX_RETRY_COUNT ? LocalDateTime.now() : null));
|
||||
.set(TaskFileJobEntity::getFinishedAt,
|
||||
retryCount >= MAX_RETRY_COUNT ? LocalDateTime.now() : null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null);
|
||||
taskFileJobMapper.update(null, update);
|
||||
}
|
||||
|
||||
public boolean markFailureFinalized(Long jobId, String message) {
|
||||
if (jobId == null || jobId <= 0) {
|
||||
return false;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, jobId)
|
||||
.eq(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.ge(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.isNull(TaskFileJobEntity::getTerminalCallbackAt)
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getErrorMessage,
|
||||
message == null ? "结果文件生成失败" : message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, now)) > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -310,7 +435,8 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
|
||||
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, LocalDateTime.now()));
|
||||
}
|
||||
|
||||
public TaskFileJobEntity findAssembleJob(Long taskId, String moduleType, Long resultId) {
|
||||
|
||||
+6
@@ -7,6 +7,7 @@ import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCa
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||
import com.nanri.aiimage.modules.brand.service.BrandTaskProgressCacheService;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
|
||||
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService;
|
||||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
|
||||
@@ -49,6 +50,7 @@ public class TaskHeartbeatService {
|
||||
private static final String MODULE_WITHDRAW = "WITHDRAW";
|
||||
private static final String MODULE_APPEARANCE_PATENT = "APPEARANCE_PATENT";
|
||||
private static final String MODULE_SIMILAR_ASIN = "SIMILAR_ASIN";
|
||||
private static final String MODULE_COLLECT_DATA = "COLLECT_DATA";
|
||||
private static final String MODULE_BRAND = "BRAND";
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
@@ -65,8 +67,10 @@ public class TaskHeartbeatService {
|
||||
private final AppearancePatentTaskCacheService appearancePatentTaskCacheService;
|
||||
private final SimilarAsinTaskCacheService similarAsinTaskCacheService;
|
||||
private final SimilarAsinProperties similarAsinProperties;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
||||
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||
private final CollectDataService collectDataService;
|
||||
|
||||
public TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
@@ -151,6 +155,7 @@ public class TaskHeartbeatService {
|
||||
boolean checkpointDue = task.getUpdatedAt() == null || !task.getUpdatedAt().isAfter(cutoff);
|
||||
|
||||
similarAsinTaskCacheService.touchTaskHeartbeat(task.getId());
|
||||
taskFileJobService.touchRunningAssembleJobsIfStale(task.getId(), MODULE_SIMILAR_ASIN, intervalMillis);
|
||||
if (!checkpointDue) {
|
||||
saveFileTaskCache(MODULE_SIMILAR_ASIN, task);
|
||||
return TaskHeartbeatVo.alive(MODULE_SIMILAR_ASIN, STATUS_RUNNING);
|
||||
@@ -239,6 +244,7 @@ public class TaskHeartbeatService {
|
||||
case MODULE_DELETE_BRAND -> {
|
||||
deleteBrandTaskCacheService.saveProgress(taskId, buildDeleteBrandHeartbeatProgress(request), true);
|
||||
}
|
||||
case MODULE_COLLECT_DATA -> collectDataService.updateProgress(taskId, request);
|
||||
default -> {
|
||||
}
|
||||
}
|
||||
|
||||
+87
-15
@@ -18,6 +18,7 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -28,6 +29,10 @@ import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -66,6 +71,15 @@ public class TaskResultFileJobWorker {
|
||||
@Value("${aiimage.result-file-job.stuck-timeout-minutes:30}")
|
||||
private int stuckTimeoutMinutes;
|
||||
|
||||
@Value("${aiimage.result-file-job.heartbeat-interval-ms:60000}")
|
||||
private long heartbeatIntervalMillis = 60000L;
|
||||
|
||||
private final ScheduledExecutorService jobHeartbeatExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> {
|
||||
Thread thread = new Thread(runnable, "task-file-job-heartbeat");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.result-file-job.local-worker-delay-ms:15000}")
|
||||
public void runPendingJobs() {
|
||||
if (!localWorkerEnabled) {
|
||||
@@ -90,9 +104,17 @@ public class TaskResultFileJobWorker {
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.result-file-job.stuck-scan-delay-ms:60000}")
|
||||
public void resetStuckJobs() {
|
||||
int reset = taskFileJobService.resetStuckRunningJobs(stuckTimeoutMinutes, batchSize);
|
||||
if (reset > 0) {
|
||||
log.warn("[task-file-job] reset stuck running jobs count={} timeoutMinutes={}", reset, stuckTimeoutMinutes);
|
||||
TaskFileJobService.StuckJobResetResult result =
|
||||
taskFileJobService.resetStuckRunningJobsDetailed(stuckTimeoutMinutes, batchSize);
|
||||
if (result.resetCount() > 0 || !result.exhaustedJobs().isEmpty()) {
|
||||
log.warn("[task-file-job] handled stuck running jobs resetCount={} exhaustedCount={} timeoutMinutes={}",
|
||||
result.resetCount(), result.exhaustedJobs().size(), stuckTimeoutMinutes);
|
||||
}
|
||||
for (TaskFileJobEntity job : result.exhaustedJobs()) {
|
||||
String message = job.getErrorMessage() == null || job.getErrorMessage().isBlank()
|
||||
? "文件生成任务运行超时,已达到最大重试次数"
|
||||
: job.getErrorMessage();
|
||||
finalizeRetryExhausted(job, message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,19 +124,49 @@ public class TaskResultFileJobWorker {
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), ownerFromScopeKey(job.getScopeKey()), currentInstanceId());
|
||||
return;
|
||||
}
|
||||
if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) {
|
||||
if (job == null || job.getId() == null) {
|
||||
return;
|
||||
}
|
||||
TaskFileJobEntity claim = taskFileJobService.claimRunning(job.getId());
|
||||
if (claim == null) {
|
||||
return;
|
||||
}
|
||||
if ("APPEARANCE_PATENT".equals(job.getModuleType()) || "SIMILAR_ASIN".equals(job.getModuleType())) {
|
||||
try {
|
||||
cozeTaskExecutor.execute(() -> processInternal(job));
|
||||
cozeTaskExecutor.execute(() -> processClaimedWithHeartbeat(job, claim));
|
||||
return;
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[task-file-job] coze module offload failed, fallback inline jobId={} taskId={} moduleType={} msg={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
processInternal(job);
|
||||
processClaimedWithHeartbeat(job, claim);
|
||||
}
|
||||
|
||||
private ScheduledFuture<?> startJobHeartbeat(TaskFileJobEntity job) {
|
||||
long interval = Math.max(1000L, heartbeatIntervalMillis);
|
||||
return jobHeartbeatExecutor.scheduleWithFixedDelay(() -> {
|
||||
try {
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-file-job] heartbeat failed jobId={} taskId={} moduleType={} msg={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage());
|
||||
}
|
||||
}, interval, interval, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
private void processClaimedWithHeartbeat(TaskFileJobEntity job, TaskFileJobEntity claim) {
|
||||
if (!taskFileJobService.activateRunningClaim(claim)) {
|
||||
log.info("[task-file-job] queued claim expired before execution jobId={} taskId={} moduleType={} claimUpdatedAt={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), claim.getUpdatedAt());
|
||||
return;
|
||||
}
|
||||
ScheduledFuture<?> heartbeat = startJobHeartbeat(job);
|
||||
try {
|
||||
processInternal(job);
|
||||
} finally {
|
||||
heartbeat.cancel(false);
|
||||
}
|
||||
}
|
||||
|
||||
private void processInternal(TaskFileJobEntity job) {
|
||||
@@ -130,13 +182,6 @@ public class TaskResultFileJobWorker {
|
||||
TaskDistributedLockService.LockHandle lockHandle =
|
||||
taskDistributedLockService.acquire(job.getModuleType(), job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS);
|
||||
if (lockHandle == null) {
|
||||
TaskFileJobEntity latestOnBusy = taskFileJobService.findById(job.getId());
|
||||
if (latestOnBusy != null && "RUNNING".equals(latestOnBusy.getStatus())) {
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
log.info("[task-file-job] process skipped because task lock is busy and job is already running jobId={} taskId={} moduleType={} resultId={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
|
||||
return;
|
||||
}
|
||||
taskFileJobService.requeue(job.getId(), "Task is busy, waiting for previous task operation");
|
||||
log.info("[task-file-job] process requeued because task lock is busy jobId={} taskId={} moduleType={} resultId={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
|
||||
@@ -194,12 +239,39 @@ public class TaskResultFileJobWorker {
|
||||
log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), message);
|
||||
taskFileJobService.markFailed(job, message);
|
||||
if ("SHOP_DATA_CRAWL".equals(job.getModuleType()) && taskFileJobService.isRetryExhausted(job.getId())) {
|
||||
shopDataCrawlTaskService.handleResultFileJobFailure(job, message);
|
||||
if (taskFileJobService.isRetryExhausted(job.getId())) {
|
||||
finalizeRetryExhausted(job, message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void finalizeRetryExhausted(TaskFileJobEntity job, String message) {
|
||||
try {
|
||||
notifyRetryExhausted(job, message);
|
||||
if (!taskFileJobService.markFailureFinalized(job.getId(), message)) {
|
||||
log.warn("[task-file-job] exhausted job terminal callback was already finalized or claim was lost jobId={} taskId={} moduleType={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType());
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
// Keep the terminal-callback marker empty so the scheduled scan retries this idempotent callback.
|
||||
log.error("[task-file-job] failed to finalize exhausted job jobId={} taskId={} moduleType={} msg={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void notifyRetryExhausted(TaskFileJobEntity job, String message) {
|
||||
if ("SHOP_DATA_CRAWL".equals(job.getModuleType())) {
|
||||
shopDataCrawlTaskService.handleResultFileJobFailure(job, message);
|
||||
} else if ("SIMILAR_ASIN".equals(job.getModuleType())) {
|
||||
similarAsinTaskService.handleResultFileJobFailure(job, message);
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
void shutdownJobHeartbeatExecutor() {
|
||||
jobHeartbeatExecutor.shutdownNow();
|
||||
}
|
||||
|
||||
private boolean isOrphanJobFailure(Exception ex, String message) {
|
||||
if (!(ex instanceof BusinessException)) {
|
||||
return false;
|
||||
|
||||
Reference in New Issue
Block a user