feat(phase-3): AC-9~12 后端任务引擎稳定性(队列隔离/心跳降级/清理瘦投影/终态与超时)
- AC-9(S1-B2/B3):视频/图像任务改走独立 videoQueueExecutor(aiimage.task-queue.video-max-concurrent, 默认 6),不再与结果文件组装共用 12 格 taskQueueExecutor;TaskQueueGate 增 queue 标签区分指标; scheduled worker 本地派发被拒改回队(requeue),不再压回 16 线程调度池内联组装。 - AC-10(S1-B4):CollectDataService.updateProgress 去掉任务锁,改无条件短事务 + updated_at 乐观 CAS, 只写 result_json/updated_at 两列;TaskHeartbeatService 对 SPI 进度写异常降级, TaskHeartbeatVo 新增 progressDegraded/warning,alive 仍为 true。 - AC-11(S1-B5/B6):ModuleHistoryCleanupService 批查与四路指针查询全部瘦投影; application.yml 增 server.shutdown=graceful 与 spring.lifecycle.timeout-per-shutdown-phase=30s; 虚拟线程池 destroy 改为 await 在途任务(上限 aiimage.task-queue.shutdown-await-seconds)。 - AC-12(S1-B1/B7/B8/B9):PENDING 僵尸重试计数可到 MAX 并就地翻 FAILED 进 exhaustedJobs; 修正 selectAdminTaskDetail 的双转义(>= -> >=)并按真实 SQL 修正迁移基准账本; MyBatis default-statement-timeout=60s(AIIMAGE_DB_DEFAULT_STATEMENT_TIMEOUT_SECONDS); 店铺数据两处删除收进 @Transactional,对象回收沿用 afterCommit 注册。 - 契约激活:PendingPhase3*Test 14 例全部去 @Disabled;同步改写被新契约反转的既有判据 (CollectDataProgressThrottleTest 锁/updateById 口径、OwnerRouting 的内联分支改显式关闭派发形态、 FaultInjection 指标按新 queue 标签查找)与三处棘轮(catch total 1031->1034、 notRethrow 582->585、超管调用点 17->18,均逐条记名)。
This commit is contained in:
@@ -75,11 +75,58 @@ public class TaskFileJobConfig {
|
||||
return executor;
|
||||
}
|
||||
|
||||
/** 与 {@code spring.lifecycle.timeout-per-shutdown-phase} 同窗口:等在途任务收尾的上限(秒)。 */
|
||||
@Value("${aiimage.task-queue.shutdown-await-seconds:30}")
|
||||
private int shutdownAwaitSeconds = 30;
|
||||
|
||||
/**
|
||||
* S1-B6:发版硬切在途作业的根因之一——这个池的 destroy 只 shutdown 不等待。
|
||||
* 现在 shutdown() 会等到在途任务收尾(上限 {@code aiimage.task-queue.shutdown-await-seconds})。
|
||||
*/
|
||||
@Bean(destroyMethod = "shutdown")
|
||||
public ExecutorService taskQueueVirtualThreadExecutor() {
|
||||
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
|
||||
ExecutorService delegate = Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
|
||||
.name("task-queue-", 0)
|
||||
.factory());
|
||||
long awaitSeconds = Math.max(1, shutdownAwaitSeconds);
|
||||
return new java.util.concurrent.AbstractExecutorService() {
|
||||
@Override
|
||||
public void execute(Runnable command) {
|
||||
delegate.execute(command);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
delegate.shutdown();
|
||||
try {
|
||||
if (!delegate.awaitTermination(awaitSeconds, TimeUnit.SECONDS)) {
|
||||
delegate.shutdownNow();
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Runnable> shutdownNow() {
|
||||
return delegate.shutdownNow();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isShutdown() {
|
||||
return delegate.isShutdown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isTerminated() {
|
||||
return delegate.isTerminated();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
|
||||
return delegate.awaitTermination(timeout, unit);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -132,6 +179,41 @@ public class TaskFileJobConfig {
|
||||
return new TaskQueueGate(semaphoreLimited, maxWaiting, meterRegistryProvider);
|
||||
}
|
||||
|
||||
/**
|
||||
* S1-B2:图像/视频任务(Coze 读超时 1 小时)与 43 个模块的结果文件组装共用
|
||||
* {@code taskQueueExecutor} 的 12 格许可时,12 个慢 Coze 请求就能把全池占满一小时。
|
||||
* 这里给视频派发/轮询一支独立许可 + 独立等待闸门的执行器;
|
||||
* 线程仍跑在同一只虚拟线程池上(虚拟线程廉价,隔离的是"正在执行"的许可数)。
|
||||
*/
|
||||
@Bean("videoQueueExecutor")
|
||||
public TaskExecutor videoQueueExecutor(
|
||||
@Qualifier("taskQueueVirtualThreadExecutor") ExecutorService taskQueueVirtualThreadExecutor,
|
||||
@Value("${aiimage.task-queue.video-max-concurrent:6}") int videoMaxConcurrent,
|
||||
@Value("${aiimage.task-queue.video-max-waiting:1000}") int videoMaxWaiting,
|
||||
ObjectProvider<MeterRegistry> meterRegistryProvider) {
|
||||
Semaphore semaphore = new Semaphore(Math.max(1, videoMaxConcurrent));
|
||||
TaskExecutor semaphoreLimited = new ConcurrentTaskExecutor(command -> {
|
||||
if (command == null) {
|
||||
throw new IllegalArgumentException("task 不能为 null");
|
||||
}
|
||||
taskQueueVirtualThreadExecutor.execute(() -> {
|
||||
boolean acquired = false;
|
||||
try {
|
||||
semaphore.acquire();
|
||||
acquired = true;
|
||||
command.run();
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
} finally {
|
||||
if (acquired) {
|
||||
semaphore.release();
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
return new TaskQueueGate(semaphoreLimited, videoMaxWaiting, meterRegistryProvider, "video");
|
||||
}
|
||||
|
||||
@Bean
|
||||
public ResultFileJobHandlerRegistry resultFileJobHandlerRegistry(List<ResultFileJobHandler> handlers) {
|
||||
ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers);
|
||||
|
||||
@@ -26,13 +26,20 @@ public class TaskQueueGate implements TaskExecutor {
|
||||
private final TaskExecutor delegate;
|
||||
private final int maxWaiting;
|
||||
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
|
||||
private final String queueName;
|
||||
private final AtomicInteger waiting = new AtomicInteger();
|
||||
|
||||
public TaskQueueGate(TaskExecutor delegate, int maxWaiting,
|
||||
ObjectProvider<MeterRegistry> meterRegistryProvider) {
|
||||
this(delegate, maxWaiting, meterRegistryProvider, "task-queue");
|
||||
}
|
||||
|
||||
public TaskQueueGate(TaskExecutor delegate, int maxWaiting,
|
||||
ObjectProvider<MeterRegistry> meterRegistryProvider, String queueName) {
|
||||
this.delegate = delegate;
|
||||
this.maxWaiting = Math.max(1, maxWaiting);
|
||||
this.meterRegistryProvider = meterRegistryProvider;
|
||||
this.queueName = queueName == null || queueName.isBlank() ? "task-queue" : queueName;
|
||||
}
|
||||
|
||||
public int waiting() {
|
||||
@@ -47,8 +54,8 @@ public class TaskQueueGate implements TaskExecutor {
|
||||
}
|
||||
if (!tryReserveWaitingSlot()) {
|
||||
recordRejected("queue-full");
|
||||
log.warn("[task-queue][gate] waiting queue full, reject submit waiting={} limit={}",
|
||||
waiting.get(), maxWaiting);
|
||||
log.warn("[{}][gate] waiting queue full, reject submit waiting={} limit={}",
|
||||
queueName, waiting.get(), maxWaiting);
|
||||
throw new TaskRejectedException("task 等待队列已满,limit=" + maxWaiting
|
||||
+ ", waiting=" + waiting.get());
|
||||
}
|
||||
@@ -67,8 +74,8 @@ public class TaskQueueGate implements TaskExecutor {
|
||||
waiting.decrementAndGet();
|
||||
recordQueueWait(System.nanoTime() - queuedAt);
|
||||
recordRejected("delegate-rejected");
|
||||
log.warn("[task-queue][gate] delegate rejected submit waiting={} limit={} msg={}",
|
||||
waiting.get(), maxWaiting, ex.getMessage(), ex);
|
||||
log.warn("[{}][gate] delegate rejected submit waiting={} limit={} msg={}",
|
||||
queueName, waiting.get(), maxWaiting, ex.getMessage(), ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
@@ -101,7 +108,8 @@ public class TaskQueueGate implements TaskExecutor {
|
||||
private void recordRejected(String reason) {
|
||||
MeterRegistry registry = meterRegistry();
|
||||
if (registry != null) {
|
||||
registry.counter("aiimage.task-queue.submit.rejected.total", "reason", reason).increment();
|
||||
registry.counter("aiimage.task-queue.submit.rejected.total",
|
||||
"queue", queueName, "reason", reason).increment();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,6 +117,7 @@ public class TaskQueueGate implements TaskExecutor {
|
||||
MeterRegistry registry = meterRegistry();
|
||||
if (registry != null && waitNanos >= 0L) {
|
||||
Timer.builder("aiimage.task-queue.queue.wait.duration")
|
||||
.tag("queue", queueName)
|
||||
.register(registry)
|
||||
.record(waitNanos, TimeUnit.NANOSECONDS);
|
||||
}
|
||||
@@ -118,6 +127,7 @@ public class TaskQueueGate implements TaskExecutor {
|
||||
MeterRegistry registry = meterRegistry();
|
||||
if (registry != null && durationNanos >= 0L) {
|
||||
Timer.builder("aiimage.task-queue.execution.duration")
|
||||
.tag("queue", queueName)
|
||||
.register(registry)
|
||||
.record(durationNanos, TimeUnit.NANOSECONDS);
|
||||
}
|
||||
|
||||
+26
-14
@@ -549,22 +549,21 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
/**
|
||||
* 进度心跳。
|
||||
*
|
||||
* <p>事务边界:Redis 任务锁在事务外获取(自旋等待最长 TASK_LOCK_WAIT_MILLIS,
|
||||
* 放在 @Transactional 里会白占一个 Hikari 连接),DB 段(统计持久化 + 任务行更新)
|
||||
* 仍在一个事务内。
|
||||
* <p>S1-B4:这里**不再取任务锁**。锁是给组装/提交的写路径用的,心跳在持锁窗口里去抢锁
|
||||
* (最长等 5s,拿不到就 40902)会让客户端把还在跑的任务当"任务不在了"处理。
|
||||
* 改法是无锁短事务 + 乐观 CAS:读到的 {@code updated_at} 作为写条件,
|
||||
* 与组装/分片提交并发时心跳让路(跳过本次进度写,下一条心跳自然补上),绝不覆盖对方刚写的 result_json。
|
||||
*/
|
||||
public void updateProgress(Long taskId, TaskHeartbeatRequest request) {
|
||||
if (taskId == null || taskId <= 0 || request == null) {
|
||||
return;
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||
if (transactionTemplate == null) {
|
||||
// 单测场景(@InjectMocks 未注入事务模板):退化为直接执行 DB 段
|
||||
updateProgressLocked(taskId, request);
|
||||
return;
|
||||
}
|
||||
transactionTemplate.executeWithoutResult(status -> updateProgressLocked(taskId, request));
|
||||
if (transactionTemplate == null) {
|
||||
// 单测场景(@InjectMocks 未注入事务模板):退化为直接执行 DB 段
|
||||
updateProgressLocked(taskId, request);
|
||||
return;
|
||||
}
|
||||
transactionTemplate.executeWithoutResult(status -> updateProgressLocked(taskId, request));
|
||||
}
|
||||
|
||||
private void updateProgressLocked(Long taskId, TaskHeartbeatRequest request) {
|
||||
@@ -622,10 +621,23 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
return;
|
||||
}
|
||||
persistStats(task, stats);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
// 节流时钟只在写成功之后推进:抛异常时不执行到这里,库里那一版也没前进,
|
||||
// 下一条心跳就会立刻补刷,而不是被一次失败的写留下的窗口挡掉。
|
||||
LocalDateTime progressUpdatedAt = LocalDateTime.now();
|
||||
// 只写进度两列(result_json/updated_at),不再整实体 updateById:整实体回写会把
|
||||
// 读到的旧 status/request_json 一并覆盖回去,在无锁心跳下足以让并发终态被"复活"。
|
||||
// 条件 updated_at 匹配=乐观 CAS,与组装/分片提交撞车时让路,绝不清掉对方刚写的统计。
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.eq(task.getUpdatedAt() != null, FileTaskEntity::getUpdatedAt, task.getUpdatedAt())
|
||||
.isNull(task.getUpdatedAt() == null, FileTaskEntity::getUpdatedAt)
|
||||
.set(FileTaskEntity::getResultJson, task.getResultJson())
|
||||
.set(FileTaskEntity::getUpdatedAt, progressUpdatedAt));
|
||||
if (updated <= 0) {
|
||||
log.info("[collect-data] 进度心跳让路:任务行已被并发写入或已离 RUNNING taskId={}", taskId);
|
||||
return;
|
||||
}
|
||||
// 节流时钟只在写成功之后推进:被 CAS 挡下的那轮不算数,下一条心跳立刻补刷,
|
||||
// 而不是被一次失败的写留下的窗口挡掉。
|
||||
lastProgressFlushMillis.put(taskId, System.currentTimeMillis());
|
||||
}
|
||||
|
||||
|
||||
+6
-6
@@ -69,7 +69,7 @@ public class ImageVideoAsyncTaskService {
|
||||
private final ImageVideoWorkflowConfigService workflowConfigService;
|
||||
private final ImageVideoArchiveService archiveService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TaskExecutor taskQueueExecutor;
|
||||
private final TaskExecutor videoQueueExecutor;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
private final com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
|
||||
@@ -79,7 +79,7 @@ public class ImageVideoAsyncTaskService {
|
||||
ImageVideoWorkflowConfigService workflowConfigService,
|
||||
ImageVideoArchiveService archiveService,
|
||||
ObjectMapper objectMapper,
|
||||
@Qualifier("taskQueueExecutor") TaskExecutor taskQueueExecutor,
|
||||
@Qualifier("videoQueueExecutor") TaskExecutor videoQueueExecutor,
|
||||
InstanceMetadata instanceMetadata,
|
||||
com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService) {
|
||||
this.taskMapper = taskMapper;
|
||||
@@ -87,7 +87,7 @@ public class ImageVideoAsyncTaskService {
|
||||
this.workflowConfigService = workflowConfigService;
|
||||
this.archiveService = archiveService;
|
||||
this.objectMapper = objectMapper;
|
||||
this.taskQueueExecutor = taskQueueExecutor;
|
||||
this.videoQueueExecutor = videoQueueExecutor;
|
||||
this.instanceMetadata = instanceMetadata;
|
||||
this.distributedJobLockService = distributedJobLockService;
|
||||
}
|
||||
@@ -156,7 +156,7 @@ public class ImageVideoAsyncTaskService {
|
||||
.or().eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, ""))
|
||||
.orderByAsc(ImageVideoAsyncTaskEntity::getId)
|
||||
.last("LIMIT " + DISPATCH_BATCH_SIZE));
|
||||
tasks.forEach(task -> taskQueueExecutor.execute(() -> executeTask(task.getId())));
|
||||
tasks.forEach(task -> videoQueueExecutor.execute(() -> executeTask(task.getId())));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,7 +178,7 @@ public class ImageVideoAsyncTaskService {
|
||||
.or().eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, ""))
|
||||
.orderByAsc(ImageVideoAsyncTaskEntity::getUpdatedAt)
|
||||
.last("LIMIT " + POLL_BATCH_SIZE));
|
||||
tasks.forEach(task -> taskQueueExecutor.execute(() -> pollTask(task.getId())));
|
||||
tasks.forEach(task -> videoQueueExecutor.execute(() -> pollTask(task.getId())));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -281,7 +281,7 @@ public class ImageVideoAsyncTaskService {
|
||||
taskMapper.insert(task);
|
||||
log.info("[image-video] async task submitted taskId={} type={} owner={}",
|
||||
task.getId(), task.getTaskType(), task.getOwnerInstanceId());
|
||||
taskQueueExecutor.execute(() -> executeTask(task.getId()));
|
||||
videoQueueExecutor.execute(() -> executeTask(task.getId()));
|
||||
return toVo(task);
|
||||
}
|
||||
|
||||
|
||||
+6
@@ -21,6 +21,7 @@ import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
@@ -100,7 +101,12 @@ public class ShopDataCrawlAdminTasksService {
|
||||
* 管理端真删该店这条店铺数据记录(累计档 + 文件 + 明细),仅允许删除已生成文件的终态记录。
|
||||
*
|
||||
* <p>S0-6/AC-7:真删比下载更危险,同一把归属尺子必须在动手之前落。
|
||||
*
|
||||
* <p>S1-B9:四次写(删明细 → 解绑结果文件 → 删成员/档 → 回收对象)必须在同一个 DB 事务里,
|
||||
* 中途失败不能留半删状态。对象回收交给 {@code deleteResultObjectIfUnreferenced}
|
||||
* 注册的 afterCommit,事务回滚时不会误删远端文件。
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void deleteAdminDailyFile(Long dailyFileId, List<String> visibleShopKeys) {
|
||||
if (dailyFileId == null || dailyFileId <= 0) {
|
||||
throw new BusinessException("参数不合法");
|
||||
|
||||
+3
@@ -10,6 +10,7 @@ import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
@@ -221,6 +222,8 @@ public class ShopDataCrawlDailyFileService {
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
/** S1-B9:删成员 + 删档两次写必须同事务,中途失败不能留"成员没了档还在"的半删状态。 */
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void deleteDailyFile(Long dailyFileId) {
|
||||
if (dailyFileId == null || dailyFileId <= 0) {
|
||||
return;
|
||||
|
||||
+17
@@ -19,6 +19,16 @@ public class TaskHeartbeatVo {
|
||||
@Schema(description = "提示信息", example = "ok")
|
||||
private String message;
|
||||
|
||||
/**
|
||||
* 进度更新失败但任务仍存活时置 true(S1-B4:组装/提交持锁期间心跳不许降级成"任务不在了")。
|
||||
* alive 与 progressDegraded 是两件事:前者是活性,后者只是这次进度没写进去。
|
||||
*/
|
||||
@Schema(description = "进度更新是否已降级(任务仍在运行,本次进度未落库)", example = "false")
|
||||
private boolean progressDegraded;
|
||||
|
||||
@Schema(description = "降级原因(仅 progressDegraded=true 时非空)")
|
||||
private String warning;
|
||||
|
||||
public static TaskHeartbeatVo alive(String moduleType, String status) {
|
||||
TaskHeartbeatVo vo = new TaskHeartbeatVo();
|
||||
vo.setAlive(true);
|
||||
@@ -28,6 +38,13 @@ public class TaskHeartbeatVo {
|
||||
return vo;
|
||||
}
|
||||
|
||||
public static TaskHeartbeatVo aliveDegraded(String moduleType, String status, String warning) {
|
||||
TaskHeartbeatVo vo = alive(moduleType, status);
|
||||
vo.setProgressDegraded(true);
|
||||
vo.setWarning(warning);
|
||||
return vo;
|
||||
}
|
||||
|
||||
public static TaskHeartbeatVo notAlive(String moduleType, String status, String message) {
|
||||
TaskHeartbeatVo vo = new TaskHeartbeatVo();
|
||||
vo.setAlive(false);
|
||||
|
||||
+9
@@ -137,6 +137,11 @@ public class ModuleHistoryCleanupService {
|
||||
List<Long> skippedRetainedTaskSamples = new ArrayList<>(LOG_SAMPLE_IDS);
|
||||
while (true) {
|
||||
List<FileTaskEntity> page = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
// S1-B5:批查只取指针列。整实体一批 500 行会把 request_json/result_json/
|
||||
// error_message 三只大列全拉进堆,夜间清理足以把堆打满。
|
||||
.select(FileTaskEntity::getId, FileTaskEntity::getModuleType,
|
||||
FileTaskEntity::getStatus, FileTaskEntity::getUpdatedAt,
|
||||
FileTaskEntity::getFinishedAt)
|
||||
.in(FileTaskEntity::getModuleType, moduleTypes)
|
||||
.gt(FileTaskEntity::getId, cursor)
|
||||
.orderByAsc(FileTaskEntity::getId)
|
||||
@@ -364,12 +369,14 @@ public class ModuleHistoryCleanupService {
|
||||
private List<String> collectPayloadPointers(List<String> moduleTypes, List<Long> cleanupTaskIds, int max) {
|
||||
java.util.Set<String> pointers = new java.util.LinkedHashSet<>();
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.select(TaskChunkEntity::getPayloadJson)
|
||||
.in(TaskChunkEntity::getModuleType, moduleTypes)
|
||||
.in(TaskChunkEntity::getTaskId, cleanupTaskIds));
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
collectPointer(pointers, chunk.getPayloadJson(), max);
|
||||
}
|
||||
List<TaskScopeStateEntity> scopeStates = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
|
||||
.in(TaskScopeStateEntity::getModuleType, moduleTypes)
|
||||
.in(TaskScopeStateEntity::getTaskId, cleanupTaskIds));
|
||||
for (TaskScopeStateEntity scopeState : scopeStates) {
|
||||
@@ -377,12 +384,14 @@ public class ModuleHistoryCleanupService {
|
||||
collectPointer(pointers, scopeState.getStateJson(), max);
|
||||
}
|
||||
List<TaskResultItemEntity> resultItems = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.select(TaskResultItemEntity::getPayloadJson)
|
||||
.in(TaskResultItemEntity::getModuleType, moduleTypes)
|
||||
.in(TaskResultItemEntity::getTaskId, cleanupTaskIds));
|
||||
for (TaskResultItemEntity resultItem : resultItems) {
|
||||
collectPointer(pointers, resultItem.getPayloadJson(), max);
|
||||
}
|
||||
List<TaskResultPayloadEntity> resultPayloads = taskResultPayloadMapper.selectList(new LambdaQueryWrapper<TaskResultPayloadEntity>()
|
||||
.select(TaskResultPayloadEntity::getPayloadJson)
|
||||
.in(TaskResultPayloadEntity::getModuleType, moduleTypes)
|
||||
.in(TaskResultPayloadEntity::getTaskId, cleanupTaskIds));
|
||||
for (TaskResultPayloadEntity resultPayload : resultPayloads) {
|
||||
|
||||
+23
-7
@@ -410,20 +410,36 @@ public class TaskFileJobService {
|
||||
continue;
|
||||
}
|
||||
if ("PENDING".equals(job.getStatus())) {
|
||||
// PENDING 僵尸兜底:补发 dispatch 事件(worker claim 幂等),
|
||||
// 并累加 retryCount 防无限重发;注意不翻转状态,
|
||||
// 避免与真实 PENDING 竞争 worker 的 claim 条件更新冲突。
|
||||
// PENDING 僵尸兜底:补发 dispatch 事件(worker claim 幂等),并累加 retryCount
|
||||
// 防无限重发。S1-B7:计数必须能走到 MAX_RETRY_COUNT 并就地翻 FAILED——
|
||||
// 之前 Math.min(MAX_RETRY_COUNT - 1, ...) 把僵尸钉死在 4,而判终态的条件是
|
||||
// retry_count >= MAX_RETRY_COUNT,于是每轮重派、永不到终态、终态回调永不触发。
|
||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
||||
int nextRetryCount = Math.min(MAX_RETRY_COUNT - 1, retryCount + 1);
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1);
|
||||
boolean exhaustedNow = nextRetryCount >= MAX_RETRY_COUNT;
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LambdaUpdateWrapper<TaskFileJobEntity> pendingUpdate = new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now);
|
||||
if (exhaustedNow) {
|
||||
pendingUpdate
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getErrorMessage,
|
||||
"文件生成任务排队未被执行,已达到最大重试次数")
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null);
|
||||
}
|
||||
int updated = taskFileJobMapper.update(null, pendingUpdate);
|
||||
if (updated > 0) {
|
||||
eventJobIds.add(job.getId());
|
||||
if (exhaustedNow) {
|
||||
exhaustedJobIds.add(job.getId());
|
||||
} else {
|
||||
eventJobIds.add(job.getId());
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
+18
-2
@@ -90,7 +90,11 @@ public class TaskHeartbeatService {
|
||||
}
|
||||
|
||||
if (fileResult != null && fileResult.isAlive() && brandResult != null && brandResult.isAlive()) {
|
||||
return TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING);
|
||||
TaskHeartbeatVo merged = fileResult.isProgressDegraded()
|
||||
? TaskHeartbeatVo.aliveDegraded(fileResult.getModuleType() + "," + MODULE_BRAND,
|
||||
STATUS_RUNNING, fileResult.getWarning())
|
||||
: TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING);
|
||||
return merged;
|
||||
}
|
||||
if (fileResult != null && fileResult.isAlive()) {
|
||||
return fileResult;
|
||||
@@ -206,9 +210,21 @@ public class TaskHeartbeatService {
|
||||
}
|
||||
|
||||
if (handler != null) {
|
||||
handler.touchHeartbeat(task.getId(), request);
|
||||
String degradeWarning = null;
|
||||
try {
|
||||
handler.touchHeartbeat(task.getId(), request);
|
||||
} catch (RuntimeException ex) {
|
||||
// S1-B4:模块的进度写可能因并发写库/锁超时抛任何运行时异常;
|
||||
// 心跳表达的是活性,进度写不进去只能降级,绝不能让客户端把 40902 当"任务不在了"。
|
||||
degradeWarning = "进度更新失败(已降级,任务仍在运行): " + ex.getMessage();
|
||||
log.warn("[task-heartbeat] module progress degraded taskId={} moduleType={} err={}",
|
||||
task.getId(), moduleType, ex.getMessage());
|
||||
}
|
||||
handler.onHeartbeat(task);
|
||||
handler.saveTaskCache(task);
|
||||
return degradeWarning == null
|
||||
? TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING)
|
||||
: TaskHeartbeatVo.aliveDegraded(moduleType, STATUS_RUNNING, degradeWarning);
|
||||
}
|
||||
return TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING);
|
||||
}
|
||||
|
||||
+18
-1
@@ -44,6 +44,13 @@ public class TaskResultFileJobWorker {
|
||||
@Value("${aiimage.result-file-job.local-worker-enabled:true}")
|
||||
private boolean localWorkerEnabled;
|
||||
|
||||
/**
|
||||
* 与 {@code TaskFileJobLocalDispatcher} 读同一键:只有部署方显式关掉本地派发时才允许
|
||||
* 退回"调度线程内联执行"这一历史语义;默认状态下派发被拒一律回队。
|
||||
*/
|
||||
@Value("${aiimage.result-file-job.local-dispatch-enabled:true}")
|
||||
private boolean localDispatchEnabled = true;
|
||||
|
||||
@Value("${aiimage.result-file-job.batch-size:20}")
|
||||
private int batchSize;
|
||||
|
||||
@@ -88,7 +95,17 @@ public class TaskResultFileJobWorker {
|
||||
// 失败,job 被静默丢弃并挂到 stuck 扫描才回收,定时兜底等于失效。
|
||||
boolean dispatched = taskFileJobLocalDispatcher.dispatchClaimed(job);
|
||||
if (!dispatched) {
|
||||
processClaimed(job);
|
||||
if (!localDispatchEnabled) {
|
||||
// 部署方显式关闭本地派发:保留"调度线程内联执行"的历史语义。
|
||||
processClaimed(job);
|
||||
continue;
|
||||
}
|
||||
// S1-B3:派发被拒(在途上限或执行器拒绝)时绝不能内联——那是把分钟级的
|
||||
// Excel 组装压回 16 线程的调度池,拖停 stale 修复/视频派发这些兜底任务。
|
||||
// 回队后由归属实例下一轮扫描补发,与 process() 通道的处理口径一致。
|
||||
log.warn("[task-file-job] local dispatch rejected, requeue for next round jobId={} taskId={} moduleType={} stage=REQUEUE",
|
||||
job.getId(), job.getTaskId(), job.getModuleType());
|
||||
taskFileJobService.requeue(job.getId(), "本地派发被拒,稍后重投");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user