修改完善这个专利部分
This commit is contained in:
+14
@@ -4,6 +4,8 @@ import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.core.task.TaskExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.event.TransactionPhase;
|
||||
import org.springframework.transaction.event.TransactionalEventListener;
|
||||
@@ -15,6 +17,8 @@ public class TaskFileJobDispatchCoordinator {
|
||||
|
||||
private final TaskFileJobPublisher taskFileJobPublisher;
|
||||
private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher;
|
||||
@Qualifier("taskFileJobDispatchExecutor")
|
||||
private final TaskExecutor taskFileJobDispatchExecutor;
|
||||
|
||||
@TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT, fallbackExecution = true)
|
||||
public void onTaskFileJobDispatch(TaskFileJobDispatchEvent event) {
|
||||
@@ -29,6 +33,16 @@ public class TaskFileJobDispatchCoordinator {
|
||||
message.setScopeKey(event.scopeKey());
|
||||
message.setJobType(event.jobType());
|
||||
message.setCreatedAt(event.createdAt());
|
||||
try {
|
||||
taskFileJobDispatchExecutor.execute(() -> dispatchAfterCommit(event, message));
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[task-file-job] dispatch executor rejected after commit jobId={} taskId={} moduleType={} msg={}",
|
||||
event.jobId(), event.taskId(), event.moduleType(), ex.getMessage(), ex);
|
||||
dispatchAfterCommit(event, message);
|
||||
}
|
||||
}
|
||||
|
||||
private void dispatchAfterCommit(TaskFileJobDispatchEvent event, TaskFileJobMessage message) {
|
||||
boolean published = taskFileJobPublisher.publish(message);
|
||||
if (published) {
|
||||
return;
|
||||
|
||||
+36
-3
@@ -8,12 +8,17 @@ import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class TaskFileJobPublisher {
|
||||
|
||||
private final ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider;
|
||||
private final AtomicInteger consecutiveFailures = new AtomicInteger();
|
||||
private final AtomicLong circuitOpenUntilMillis = new AtomicLong();
|
||||
|
||||
@Value("${aiimage.result-file-job.mq-enabled:false}")
|
||||
private boolean mqEnabled;
|
||||
@@ -21,6 +26,15 @@ public class TaskFileJobPublisher {
|
||||
@Value("${aiimage.result-file-job.topic:aiimage-result-file-job}")
|
||||
private String topic;
|
||||
|
||||
@Value("${aiimage.result-file-job.mq-send-timeout-ms:1000}")
|
||||
private long mqSendTimeoutMillis;
|
||||
|
||||
@Value("${aiimage.result-file-job.mq-failure-threshold:3}")
|
||||
private int mqFailureThreshold;
|
||||
|
||||
@Value("${aiimage.result-file-job.mq-circuit-open-ms:60000}")
|
||||
private long mqCircuitOpenMillis;
|
||||
|
||||
public boolean publish(TaskFileJobMessage message) {
|
||||
if (message == null || message.getJobId() == null) {
|
||||
return false;
|
||||
@@ -30,6 +44,13 @@ public class TaskFileJobPublisher {
|
||||
message.getJobId(), message.getTaskId(), message.getModuleType());
|
||||
return false;
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
long openUntil = circuitOpenUntilMillis.get();
|
||||
if (openUntil > now) {
|
||||
log.info("[task-file-job] RocketMQ publish circuit open, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} remainingMs={}",
|
||||
message.getJobId(), message.getTaskId(), message.getModuleType(), openUntil - now);
|
||||
return false;
|
||||
}
|
||||
RocketMQTemplate template = rocketMQTemplateProvider.getIfAvailable();
|
||||
if (template == null) {
|
||||
log.warn("[task-file-job] RocketMQTemplate unavailable, jobId={} taskId={} moduleType={}",
|
||||
@@ -37,13 +58,25 @@ public class TaskFileJobPublisher {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
template.convertAndSend(topic, message);
|
||||
template.syncSend(topic, message, Math.max(100L, mqSendTimeoutMillis));
|
||||
consecutiveFailures.set(0);
|
||||
circuitOpenUntilMillis.set(0L);
|
||||
log.info("[task-file-job] mq published jobId={} taskId={} moduleType={} topic={}",
|
||||
message.getJobId(), message.getTaskId(), message.getModuleType(), topic);
|
||||
return true;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-file-job] publish RocketMQ failed, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} msg={}",
|
||||
message.getJobId(), message.getTaskId(), message.getModuleType(), ex.getMessage());
|
||||
int failures = consecutiveFailures.incrementAndGet();
|
||||
int threshold = Math.max(1, mqFailureThreshold);
|
||||
if (failures >= threshold) {
|
||||
long nextOpenUntil = System.currentTimeMillis() + Math.max(1000L, mqCircuitOpenMillis);
|
||||
circuitOpenUntilMillis.set(nextOpenUntil);
|
||||
log.warn("[task-file-job] publish RocketMQ failed, circuit opened and fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} failures={} openMs={} msg={}",
|
||||
message.getJobId(), message.getTaskId(), message.getModuleType(), failures,
|
||||
Math.max(1000L, mqCircuitOpenMillis), ex.getMessage());
|
||||
} else {
|
||||
log.warn("[task-file-job] publish RocketMQ failed, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} failures={}/{} msg={}",
|
||||
message.getJobId(), message.getTaskId(), message.getModuleType(), failures, threshold, ex.getMessage());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
+61
-2
@@ -12,6 +12,7 @@ import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -76,6 +77,41 @@ public class TaskFileJobService {
|
||||
.last("limit " + Math.max(1, Math.min(limit, 100))));
|
||||
}
|
||||
|
||||
public List<TaskFileJobEntity> listRunnableJobsForOwner(int limit, String owner) {
|
||||
String normalizedOwner = owner == null ? "" : owner.trim();
|
||||
if (normalizedOwner.isBlank()) {
|
||||
return listRunnableJobs(limit);
|
||||
}
|
||||
String ownerMarker = ":owner:" + normalizedOwner;
|
||||
int safeLimit = Math.max(1, Math.min(limit, 100));
|
||||
|
||||
List<TaskFileJobEntity> ownerJobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
|
||||
.lt(TaskFileJobEntity::getRetryCount, 5)
|
||||
.in(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN"))
|
||||
.like(TaskFileJobEntity::getScopeKey, ownerMarker)
|
||||
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
|
||||
.last("limit " + safeLimit));
|
||||
if (ownerJobs.size() >= safeLimit) {
|
||||
return ownerJobs;
|
||||
}
|
||||
|
||||
List<TaskFileJobEntity> jobs = new ArrayList<>(ownerJobs);
|
||||
LambdaQueryWrapper<TaskFileJobEntity> genericWrapper = new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
|
||||
.lt(TaskFileJobEntity::getRetryCount, 5)
|
||||
.and(wrapper -> wrapper
|
||||
.notIn(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN"))
|
||||
.or()
|
||||
.isNull(TaskFileJobEntity::getScopeKey)
|
||||
.or()
|
||||
.notLike(TaskFileJobEntity::getScopeKey, ":owner:"))
|
||||
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
|
||||
.last("limit " + (safeLimit - jobs.size()));
|
||||
jobs.addAll(taskFileJobMapper.selectList(genericWrapper));
|
||||
return jobs;
|
||||
}
|
||||
|
||||
public List<TaskFileJobEntity> listJobs(String status, String moduleType, Long taskId, int limit) {
|
||||
LambdaQueryWrapper<TaskFileJobEntity> wrapper = new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.orderByDesc(TaskFileJobEntity::getUpdatedAt)
|
||||
@@ -130,7 +166,8 @@ public class TaskFileJobService {
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
|
||||
continue;
|
||||
}
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
@@ -168,6 +205,19 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
}
|
||||
|
||||
public boolean deferRunning(Long jobId, String message) {
|
||||
if (jobId == null || jobId <= 0) {
|
||||
return false;
|
||||
}
|
||||
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, jobId)
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)) > 0;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public boolean requeue(Long jobId, String message) {
|
||||
if (jobId == null || jobId <= 0) {
|
||||
@@ -198,14 +248,23 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
|
||||
}
|
||||
|
||||
public TaskFileJobEntity findById(Long jobId) {
|
||||
if (jobId == null || jobId <= 0) {
|
||||
return null;
|
||||
}
|
||||
return taskFileJobMapper.selectById(jobId);
|
||||
}
|
||||
|
||||
public void markFailed(TaskFileJobEntity job, String message) {
|
||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount() + 1;
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getRetryCount, retryCount)
|
||||
.set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, retryCount >= 5 ? LocalDateTime.now() : null));
|
||||
}
|
||||
|
||||
public TaskFileJobEntity findAssembleJob(Long taskId, String moduleType, Long resultId) {
|
||||
|
||||
+27
-2
@@ -62,7 +62,7 @@ public class TaskResultFileJobWorker {
|
||||
if (!localWorkerEnabled) {
|
||||
return;
|
||||
}
|
||||
List<TaskFileJobEntity> jobs = taskFileJobService.listRunnableJobs(batchSize);
|
||||
List<TaskFileJobEntity> jobs = taskFileJobService.listRunnableJobsForOwner(batchSize, currentInstanceId());
|
||||
if (jobs == null || jobs.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
@@ -111,6 +111,12 @@ public class TaskResultFileJobWorker {
|
||||
private void processInternal(TaskFileJobEntity job) {
|
||||
long startedAt = System.currentTimeMillis();
|
||||
try {
|
||||
TaskFileJobEntity latest = taskFileJobService.findById(job.getId());
|
||||
if (latest != null && "SUCCESS".equals(latest.getStatus())) {
|
||||
log.info("[task-file-job] process skipped because job already succeeded jobId={} taskId={} moduleType={} resultId={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
|
||||
return;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle lockHandle =
|
||||
taskDistributedLockService.acquire(job.getModuleType(), job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS);
|
||||
if (lockHandle == null) {
|
||||
@@ -120,9 +126,22 @@ public class TaskResultFileJobWorker {
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
latest = taskFileJobService.findById(job.getId());
|
||||
if (latest != null && "SUCCESS".equals(latest.getStatus())) {
|
||||
log.info("[task-file-job] process skipped after lock because job already succeeded jobId={} taskId={} moduleType={} resultId={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
|
||||
return;
|
||||
}
|
||||
boolean completed = dispatch(job);
|
||||
if (!completed) {
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
if (isOwnerScopedCozeJob(job)) {
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
log.info("[task-file-job] process waiting for async coze result jobId={} taskId={} moduleType={} resultId={} elapsedMs={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
|
||||
System.currentTimeMillis() - startedAt);
|
||||
return;
|
||||
}
|
||||
taskFileJobService.deferRunning(job.getId(), "Waiting for Coze/file assembly to continue");
|
||||
log.info("[task-file-job] process deferred jobId={} taskId={} moduleType={} resultId={} elapsedMs={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
|
||||
System.currentTimeMillis() - startedAt);
|
||||
@@ -136,6 +155,12 @@ public class TaskResultFileJobWorker {
|
||||
System.currentTimeMillis() - startedAt, resultFileUrl);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
TaskFileJobEntity latest = taskFileJobService.findById(job.getId());
|
||||
if (latest != null && "SUCCESS".equals(latest.getStatus())) {
|
||||
log.info("[task-file-job] process failure ignored because job already succeeded jobId={} taskId={} moduleType={} resultId={} msg={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), ex.getMessage());
|
||||
return;
|
||||
}
|
||||
String message = ex.getMessage() == null ? "结果文件生成失败" : ex.getMessage();
|
||||
log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), message);
|
||||
|
||||
Reference in New Issue
Block a user