feat(phase-3): AC-9~12 后端任务引擎稳定性(队列隔离/心跳降级/清理瘦投影/终态与超时)
- AC-9(S1-B2/B3):视频/图像任务改走独立 videoQueueExecutor(aiimage.task-queue.video-max-concurrent, 默认 6),不再与结果文件组装共用 12 格 taskQueueExecutor;TaskQueueGate 增 queue 标签区分指标; scheduled worker 本地派发被拒改回队(requeue),不再压回 16 线程调度池内联组装。 - AC-10(S1-B4):CollectDataService.updateProgress 去掉任务锁,改无条件短事务 + updated_at 乐观 CAS, 只写 result_json/updated_at 两列;TaskHeartbeatService 对 SPI 进度写异常降级, TaskHeartbeatVo 新增 progressDegraded/warning,alive 仍为 true。 - AC-11(S1-B5/B6):ModuleHistoryCleanupService 批查与四路指针查询全部瘦投影; application.yml 增 server.shutdown=graceful 与 spring.lifecycle.timeout-per-shutdown-phase=30s; 虚拟线程池 destroy 改为 await 在途任务(上限 aiimage.task-queue.shutdown-await-seconds)。 - AC-12(S1-B1/B7/B8/B9):PENDING 僵尸重试计数可到 MAX 并就地翻 FAILED 进 exhaustedJobs; 修正 selectAdminTaskDetail 的双转义(>= -> >=)并按真实 SQL 修正迁移基准账本; MyBatis default-statement-timeout=60s(AIIMAGE_DB_DEFAULT_STATEMENT_TIMEOUT_SECONDS); 店铺数据两处删除收进 @Transactional,对象回收沿用 afterCommit 注册。 - 契约激活:PendingPhase3*Test 14 例全部去 @Disabled;同步改写被新契约反转的既有判据 (CollectDataProgressThrottleTest 锁/updateById 口径、OwnerRouting 的内联分支改显式关闭派发形态、 FaultInjection 指标按新 queue 标签查找)与三处棘轮(catch total 1031->1034、 notRethrow 582->585、超管调用点 17->18,均逐条记名)。
This commit is contained in:
@@ -75,11 +75,58 @@ public class TaskFileJobConfig {
|
||||
return executor;
|
||||
}
|
||||
|
||||
/** 与 {@code spring.lifecycle.timeout-per-shutdown-phase} 同窗口:等在途任务收尾的上限(秒)。 */
|
||||
@Value("${aiimage.task-queue.shutdown-await-seconds:30}")
|
||||
private int shutdownAwaitSeconds = 30;
|
||||
|
||||
/**
|
||||
* S1-B6:发版硬切在途作业的根因之一——这个池的 destroy 只 shutdown 不等待。
|
||||
* 现在 shutdown() 会等到在途任务收尾(上限 {@code aiimage.task-queue.shutdown-await-seconds})。
|
||||
*/
|
||||
@Bean(destroyMethod = "shutdown")
|
||||
public ExecutorService taskQueueVirtualThreadExecutor() {
|
||||
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
|
||||
ExecutorService delegate = Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
|
||||
.name("task-queue-", 0)
|
||||
.factory());
|
||||
long awaitSeconds = Math.max(1, shutdownAwaitSeconds);
|
||||
return new java.util.concurrent.AbstractExecutorService() {
|
||||
@Override
|
||||
public void execute(Runnable command) {
|
||||
delegate.execute(command);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
delegate.shutdown();
|
||||
try {
|
||||
if (!delegate.awaitTermination(awaitSeconds, TimeUnit.SECONDS)) {
|
||||
delegate.shutdownNow();
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Runnable> shutdownNow() {
|
||||
return delegate.shutdownNow();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isShutdown() {
|
||||
return delegate.isShutdown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isTerminated() {
|
||||
return delegate.isTerminated();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
|
||||
return delegate.awaitTermination(timeout, unit);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -132,6 +179,41 @@ public class TaskFileJobConfig {
|
||||
return new TaskQueueGate(semaphoreLimited, maxWaiting, meterRegistryProvider);
|
||||
}
|
||||
|
||||
/**
|
||||
* S1-B2:图像/视频任务(Coze 读超时 1 小时)与 43 个模块的结果文件组装共用
|
||||
* {@code taskQueueExecutor} 的 12 格许可时,12 个慢 Coze 请求就能把全池占满一小时。
|
||||
* 这里给视频派发/轮询一支独立许可 + 独立等待闸门的执行器;
|
||||
* 线程仍跑在同一只虚拟线程池上(虚拟线程廉价,隔离的是"正在执行"的许可数)。
|
||||
*/
|
||||
@Bean("videoQueueExecutor")
|
||||
public TaskExecutor videoQueueExecutor(
|
||||
@Qualifier("taskQueueVirtualThreadExecutor") ExecutorService taskQueueVirtualThreadExecutor,
|
||||
@Value("${aiimage.task-queue.video-max-concurrent:6}") int videoMaxConcurrent,
|
||||
@Value("${aiimage.task-queue.video-max-waiting:1000}") int videoMaxWaiting,
|
||||
ObjectProvider<MeterRegistry> meterRegistryProvider) {
|
||||
Semaphore semaphore = new Semaphore(Math.max(1, videoMaxConcurrent));
|
||||
TaskExecutor semaphoreLimited = new ConcurrentTaskExecutor(command -> {
|
||||
if (command == null) {
|
||||
throw new IllegalArgumentException("task 不能为 null");
|
||||
}
|
||||
taskQueueVirtualThreadExecutor.execute(() -> {
|
||||
boolean acquired = false;
|
||||
try {
|
||||
semaphore.acquire();
|
||||
acquired = true;
|
||||
command.run();
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
} finally {
|
||||
if (acquired) {
|
||||
semaphore.release();
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
return new TaskQueueGate(semaphoreLimited, videoMaxWaiting, meterRegistryProvider, "video");
|
||||
}
|
||||
|
||||
@Bean
|
||||
public ResultFileJobHandlerRegistry resultFileJobHandlerRegistry(List<ResultFileJobHandler> handlers) {
|
||||
ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers);
|
||||
|
||||
@@ -26,13 +26,20 @@ public class TaskQueueGate implements TaskExecutor {
|
||||
private final TaskExecutor delegate;
|
||||
private final int maxWaiting;
|
||||
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
|
||||
private final String queueName;
|
||||
private final AtomicInteger waiting = new AtomicInteger();
|
||||
|
||||
public TaskQueueGate(TaskExecutor delegate, int maxWaiting,
|
||||
ObjectProvider<MeterRegistry> meterRegistryProvider) {
|
||||
this(delegate, maxWaiting, meterRegistryProvider, "task-queue");
|
||||
}
|
||||
|
||||
public TaskQueueGate(TaskExecutor delegate, int maxWaiting,
|
||||
ObjectProvider<MeterRegistry> meterRegistryProvider, String queueName) {
|
||||
this.delegate = delegate;
|
||||
this.maxWaiting = Math.max(1, maxWaiting);
|
||||
this.meterRegistryProvider = meterRegistryProvider;
|
||||
this.queueName = queueName == null || queueName.isBlank() ? "task-queue" : queueName;
|
||||
}
|
||||
|
||||
public int waiting() {
|
||||
@@ -47,8 +54,8 @@ public class TaskQueueGate implements TaskExecutor {
|
||||
}
|
||||
if (!tryReserveWaitingSlot()) {
|
||||
recordRejected("queue-full");
|
||||
log.warn("[task-queue][gate] waiting queue full, reject submit waiting={} limit={}",
|
||||
waiting.get(), maxWaiting);
|
||||
log.warn("[{}][gate] waiting queue full, reject submit waiting={} limit={}",
|
||||
queueName, waiting.get(), maxWaiting);
|
||||
throw new TaskRejectedException("task 等待队列已满,limit=" + maxWaiting
|
||||
+ ", waiting=" + waiting.get());
|
||||
}
|
||||
@@ -67,8 +74,8 @@ public class TaskQueueGate implements TaskExecutor {
|
||||
waiting.decrementAndGet();
|
||||
recordQueueWait(System.nanoTime() - queuedAt);
|
||||
recordRejected("delegate-rejected");
|
||||
log.warn("[task-queue][gate] delegate rejected submit waiting={} limit={} msg={}",
|
||||
waiting.get(), maxWaiting, ex.getMessage(), ex);
|
||||
log.warn("[{}][gate] delegate rejected submit waiting={} limit={} msg={}",
|
||||
queueName, waiting.get(), maxWaiting, ex.getMessage(), ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
@@ -101,7 +108,8 @@ public class TaskQueueGate implements TaskExecutor {
|
||||
private void recordRejected(String reason) {
|
||||
MeterRegistry registry = meterRegistry();
|
||||
if (registry != null) {
|
||||
registry.counter("aiimage.task-queue.submit.rejected.total", "reason", reason).increment();
|
||||
registry.counter("aiimage.task-queue.submit.rejected.total",
|
||||
"queue", queueName, "reason", reason).increment();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,6 +117,7 @@ public class TaskQueueGate implements TaskExecutor {
|
||||
MeterRegistry registry = meterRegistry();
|
||||
if (registry != null && waitNanos >= 0L) {
|
||||
Timer.builder("aiimage.task-queue.queue.wait.duration")
|
||||
.tag("queue", queueName)
|
||||
.register(registry)
|
||||
.record(waitNanos, TimeUnit.NANOSECONDS);
|
||||
}
|
||||
@@ -118,6 +127,7 @@ public class TaskQueueGate implements TaskExecutor {
|
||||
MeterRegistry registry = meterRegistry();
|
||||
if (registry != null && durationNanos >= 0L) {
|
||||
Timer.builder("aiimage.task-queue.execution.duration")
|
||||
.tag("queue", queueName)
|
||||
.register(registry)
|
||||
.record(durationNanos, TimeUnit.NANOSECONDS);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user