feat(phase-3): AC-9~12 后端任务引擎稳定性(队列隔离/心跳降级/清理瘦投影/终态与超时)

- AC-9(S1-B2/B3):视频/图像任务改走独立 videoQueueExecutor(aiimage.task-queue.video-max-concurrent,
  默认 6),不再与结果文件组装共用 12 格 taskQueueExecutor;TaskQueueGate 增 queue 标签区分指标;
  scheduled worker 本地派发被拒改回队(requeue),不再压回 16 线程调度池内联组装。
- AC-10(S1-B4):CollectDataService.updateProgress 去掉任务锁,改无条件短事务 + updated_at 乐观 CAS,
  只写 result_json/updated_at 两列;TaskHeartbeatService 对 SPI 进度写异常降级,
  TaskHeartbeatVo 新增 progressDegraded/warning,alive 仍为 true。
- AC-11(S1-B5/B6):ModuleHistoryCleanupService 批查与四路指针查询全部瘦投影;
  application.yml 增 server.shutdown=graceful 与 spring.lifecycle.timeout-per-shutdown-phase=30s;
  虚拟线程池 destroy 改为 await 在途任务(上限 aiimage.task-queue.shutdown-await-seconds)。
- AC-12(S1-B1/B7/B8/B9):PENDING 僵尸重试计数可到 MAX 并就地翻 FAILED 进 exhaustedJobs;
  修正 selectAdminTaskDetail 的双转义(>= -> >=)并按真实 SQL 修正迁移基准账本;
  MyBatis default-statement-timeout=60s(AIIMAGE_DB_DEFAULT_STATEMENT_TIMEOUT_SECONDS);
  店铺数据两处删除收进 @Transactional,对象回收沿用 afterCommit 注册。
- 契约激活:PendingPhase3*Test 14 例全部去 @Disabled;同步改写被新契约反转的既有判据
  (CollectDataProgressThrottleTest 锁/updateById 口径、OwnerRouting 的内联分支改显式关闭派发形态、
  FaultInjection 指标按新 queue 标签查找)与三处棘轮(catch total 1031->1034、
  notRethrow 582->585、超管调用点 17->18,均逐条记名)。
This commit is contained in:
2026-10-28 21:05:33 +08:00
parent c4fc9784e8
commit 62a1b50067
24 changed files with 338 additions and 113 deletions
@@ -75,11 +75,58 @@ public class TaskFileJobConfig {
return executor;
}
/** 与 {@code spring.lifecycle.timeout-per-shutdown-phase} 同窗口:等在途任务收尾的上限(秒)。 */
@Value("${aiimage.task-queue.shutdown-await-seconds:30}")
private int shutdownAwaitSeconds = 30;
/**
* S1-B6:发版硬切在途作业的根因之一——这个池的 destroy 只 shutdown 不等待。
* 现在 shutdown() 会等到在途任务收尾(上限 {@code aiimage.task-queue.shutdown-await-seconds})。
*/
@Bean(destroyMethod = "shutdown")
public ExecutorService taskQueueVirtualThreadExecutor() {
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
ExecutorService delegate = Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
.name("task-queue-", 0)
.factory());
long awaitSeconds = Math.max(1, shutdownAwaitSeconds);
return new java.util.concurrent.AbstractExecutorService() {
@Override
public void execute(Runnable command) {
delegate.execute(command);
}
@Override
public void shutdown() {
delegate.shutdown();
try {
if (!delegate.awaitTermination(awaitSeconds, TimeUnit.SECONDS)) {
delegate.shutdownNow();
}
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
@Override
public List<Runnable> shutdownNow() {
return delegate.shutdownNow();
}
@Override
public boolean isShutdown() {
return delegate.isShutdown();
}
@Override
public boolean isTerminated() {
return delegate.isTerminated();
}
@Override
public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
return delegate.awaitTermination(timeout, unit);
}
};
}
/**
@@ -132,6 +179,41 @@ public class TaskFileJobConfig {
return new TaskQueueGate(semaphoreLimited, maxWaiting, meterRegistryProvider);
}
/**
* S1-B2:图像/视频任务(Coze 读超时 1 小时)与 43 个模块的结果文件组装共用
* {@code taskQueueExecutor} 的 12 格许可时,12 个慢 Coze 请求就能把全池占满一小时。
* 这里给视频派发/轮询一支独立许可 + 独立等待闸门的执行器;
* 线程仍跑在同一只虚拟线程池上(虚拟线程廉价,隔离的是"正在执行"的许可数)。
*/
@Bean("videoQueueExecutor")
public TaskExecutor videoQueueExecutor(
@Qualifier("taskQueueVirtualThreadExecutor") ExecutorService taskQueueVirtualThreadExecutor,
@Value("${aiimage.task-queue.video-max-concurrent:6}") int videoMaxConcurrent,
@Value("${aiimage.task-queue.video-max-waiting:1000}") int videoMaxWaiting,
ObjectProvider<MeterRegistry> meterRegistryProvider) {
Semaphore semaphore = new Semaphore(Math.max(1, videoMaxConcurrent));
TaskExecutor semaphoreLimited = new ConcurrentTaskExecutor(command -> {
if (command == null) {
throw new IllegalArgumentException("task 不能为 null");
}
taskQueueVirtualThreadExecutor.execute(() -> {
boolean acquired = false;
try {
semaphore.acquire();
acquired = true;
command.run();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
} finally {
if (acquired) {
semaphore.release();
}
}
});
});
return new TaskQueueGate(semaphoreLimited, videoMaxWaiting, meterRegistryProvider, "video");
}
@Bean
public ResultFileJobHandlerRegistry resultFileJobHandlerRegistry(List<ResultFileJobHandler> handlers) {
ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers);
@@ -26,13 +26,20 @@ public class TaskQueueGate implements TaskExecutor {
private final TaskExecutor delegate;
private final int maxWaiting;
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
private final String queueName;
private final AtomicInteger waiting = new AtomicInteger();
public TaskQueueGate(TaskExecutor delegate, int maxWaiting,
ObjectProvider<MeterRegistry> meterRegistryProvider) {
this(delegate, maxWaiting, meterRegistryProvider, "task-queue");
}
public TaskQueueGate(TaskExecutor delegate, int maxWaiting,
ObjectProvider<MeterRegistry> meterRegistryProvider, String queueName) {
this.delegate = delegate;
this.maxWaiting = Math.max(1, maxWaiting);
this.meterRegistryProvider = meterRegistryProvider;
this.queueName = queueName == null || queueName.isBlank() ? "task-queue" : queueName;
}
public int waiting() {
@@ -47,8 +54,8 @@ public class TaskQueueGate implements TaskExecutor {
}
if (!tryReserveWaitingSlot()) {
recordRejected("queue-full");
log.warn("[task-queue][gate] waiting queue full, reject submit waiting={} limit={}",
waiting.get(), maxWaiting);
log.warn("[{}][gate] waiting queue full, reject submit waiting={} limit={}",
queueName, waiting.get(), maxWaiting);
throw new TaskRejectedException("task 等待队列已满,limit=" + maxWaiting
+ ", waiting=" + waiting.get());
}
@@ -67,8 +74,8 @@ public class TaskQueueGate implements TaskExecutor {
waiting.decrementAndGet();
recordQueueWait(System.nanoTime() - queuedAt);
recordRejected("delegate-rejected");
log.warn("[task-queue][gate] delegate rejected submit waiting={} limit={} msg={}",
waiting.get(), maxWaiting, ex.getMessage(), ex);
log.warn("[{}][gate] delegate rejected submit waiting={} limit={} msg={}",
queueName, waiting.get(), maxWaiting, ex.getMessage(), ex);
throw ex;
}
}
@@ -101,7 +108,8 @@ public class TaskQueueGate implements TaskExecutor {
private void recordRejected(String reason) {
MeterRegistry registry = meterRegistry();
if (registry != null) {
registry.counter("aiimage.task-queue.submit.rejected.total", "reason", reason).increment();
registry.counter("aiimage.task-queue.submit.rejected.total",
"queue", queueName, "reason", reason).increment();
}
}
@@ -109,6 +117,7 @@ public class TaskQueueGate implements TaskExecutor {
MeterRegistry registry = meterRegistry();
if (registry != null && waitNanos >= 0L) {
Timer.builder("aiimage.task-queue.queue.wait.duration")
.tag("queue", queueName)
.register(registry)
.record(waitNanos, TimeUnit.NANOSECONDS);
}
@@ -118,6 +127,7 @@ public class TaskQueueGate implements TaskExecutor {
MeterRegistry registry = meterRegistry();
if (registry != null && durationNanos >= 0L) {
Timer.builder("aiimage.task-queue.execution.duration")
.tag("queue", queueName)
.register(registry)
.record(durationNanos, TimeUnit.NANOSECONDS);
}
@@ -549,22 +549,21 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
/**
* 进度心跳。
*
* <p>事务边界:Redis 任务锁在事务外获取(自旋等待最长 TASK_LOCK_WAIT_MILLIS,
* 放在 @Transactional 里会白占一个 Hikari 连接),DB 段(统计持久化 + 任务行更新)
* 仍在一个事务内。
* <p>S1-B4:这里**不再取任务锁**。锁是给组装/提交的写路径用的,心跳在持锁窗口里去抢锁
* (最长等 5s,拿不到就 40902)会让客户端把还在跑的任务当"任务不在了"处理。
* 改法是无锁短事务 + 乐观 CAS:读到的 {@code updated_at} 作为写条件,
* 与组装/分片提交并发时心跳让路(跳过本次进度写,下一条心跳自然补上),绝不覆盖对方刚写的 result_json。
*/
public void updateProgress(Long taskId, TaskHeartbeatRequest request) {
if (taskId == null || taskId <= 0 || request == null) {
return;
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
if (transactionTemplate == null) {
// 单测场景(@InjectMocks 未注入事务模板):退化为直接执行 DB 段
updateProgressLocked(taskId, request);
return;
}
transactionTemplate.executeWithoutResult(status -> updateProgressLocked(taskId, request));
if (transactionTemplate == null) {
// 单测场景(@InjectMocks 未注入事务模板):退化为直接执行 DB 段
updateProgressLocked(taskId, request);
return;
}
transactionTemplate.executeWithoutResult(status -> updateProgressLocked(taskId, request));
}
private void updateProgressLocked(Long taskId, TaskHeartbeatRequest request) {
@@ -622,10 +621,23 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
return;
}
persistStats(task, stats);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
// 节流时钟只在写成功之后推进:抛异常时不执行到这里,库里那一版也没前进,
// 下一条心跳就会立刻补刷,而不是被一次失败的写留下的窗口挡掉。
LocalDateTime progressUpdatedAt = LocalDateTime.now();
// 只写进度两列(result_json/updated_at),不再整实体 updateById:整实体回写会把
// 读到的旧 status/request_json 一并覆盖回去,在无锁心跳下足以让并发终态被"复活"。
// 条件 updated_at 匹配=乐观 CAS,与组装/分片提交撞车时让路,绝不清掉对方刚写的统计。
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.eq(task.getUpdatedAt() != null, FileTaskEntity::getUpdatedAt, task.getUpdatedAt())
.isNull(task.getUpdatedAt() == null, FileTaskEntity::getUpdatedAt)
.set(FileTaskEntity::getResultJson, task.getResultJson())
.set(FileTaskEntity::getUpdatedAt, progressUpdatedAt));
if (updated <= 0) {
log.info("[collect-data] 进度心跳让路:任务行已被并发写入或已离 RUNNING taskId={}", taskId);
return;
}
// 节流时钟只在写成功之后推进:被 CAS 挡下的那轮不算数,下一条心跳立刻补刷,
// 而不是被一次失败的写留下的窗口挡掉。
lastProgressFlushMillis.put(taskId, System.currentTimeMillis());
}
@@ -69,7 +69,7 @@ public class ImageVideoAsyncTaskService {
private final ImageVideoWorkflowConfigService workflowConfigService;
private final ImageVideoArchiveService archiveService;
private final ObjectMapper objectMapper;
private final TaskExecutor taskQueueExecutor;
private final TaskExecutor videoQueueExecutor;
private final InstanceMetadata instanceMetadata;
private final com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@@ -79,7 +79,7 @@ public class ImageVideoAsyncTaskService {
ImageVideoWorkflowConfigService workflowConfigService,
ImageVideoArchiveService archiveService,
ObjectMapper objectMapper,
@Qualifier("taskQueueExecutor") TaskExecutor taskQueueExecutor,
@Qualifier("videoQueueExecutor") TaskExecutor videoQueueExecutor,
InstanceMetadata instanceMetadata,
com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService) {
this.taskMapper = taskMapper;
@@ -87,7 +87,7 @@ public class ImageVideoAsyncTaskService {
this.workflowConfigService = workflowConfigService;
this.archiveService = archiveService;
this.objectMapper = objectMapper;
this.taskQueueExecutor = taskQueueExecutor;
this.videoQueueExecutor = videoQueueExecutor;
this.instanceMetadata = instanceMetadata;
this.distributedJobLockService = distributedJobLockService;
}
@@ -156,7 +156,7 @@ public class ImageVideoAsyncTaskService {
.or().eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, ""))
.orderByAsc(ImageVideoAsyncTaskEntity::getId)
.last("LIMIT " + DISPATCH_BATCH_SIZE));
tasks.forEach(task -> taskQueueExecutor.execute(() -> executeTask(task.getId())));
tasks.forEach(task -> videoQueueExecutor.execute(() -> executeTask(task.getId())));
}
}
@@ -178,7 +178,7 @@ public class ImageVideoAsyncTaskService {
.or().eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, ""))
.orderByAsc(ImageVideoAsyncTaskEntity::getUpdatedAt)
.last("LIMIT " + POLL_BATCH_SIZE));
tasks.forEach(task -> taskQueueExecutor.execute(() -> pollTask(task.getId())));
tasks.forEach(task -> videoQueueExecutor.execute(() -> pollTask(task.getId())));
}
}
@@ -281,7 +281,7 @@ public class ImageVideoAsyncTaskService {
taskMapper.insert(task);
log.info("[image-video] async task submitted taskId={} type={} owner={}",
task.getId(), task.getTaskType(), task.getOwnerInstanceId());
taskQueueExecutor.execute(() -> executeTask(task.getId()));
videoQueueExecutor.execute(() -> executeTask(task.getId()));
return toVo(task);
}
@@ -21,6 +21,7 @@ import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
@@ -100,7 +101,12 @@ public class ShopDataCrawlAdminTasksService {
* 管理端真删该店这条店铺数据记录(累计档 + 文件 + 明细),仅允许删除已生成文件的终态记录。
*
* <p>S0-6/AC-7:真删比下载更危险,同一把归属尺子必须在动手之前落。
*
* <p>S1-B9:四次写(删明细 → 解绑结果文件 → 删成员/档 → 回收对象)必须在同一个 DB 事务里,
* 中途失败不能留半删状态。对象回收交给 {@code deleteResultObjectIfUnreferenced}
* 注册的 afterCommit,事务回滚时不会误删远端文件。
*/
@Transactional(rollbackFor = Exception.class)
public void deleteAdminDailyFile(Long dailyFileId, List<String> visibleShopKeys) {
if (dailyFileId == null || dailyFileId <= 0) {
throw new BusinessException("参数不合法");
@@ -10,6 +10,7 @@ import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
@@ -221,6 +222,8 @@ public class ShopDataCrawlDailyFileService {
return count == null ? 0L : count;
}
/** S1-B9:删成员 + 删档两次写必须同事务,中途失败不能留"成员没了档还在"的半删状态。 */
@Transactional(rollbackFor = Exception.class)
public void deleteDailyFile(Long dailyFileId) {
if (dailyFileId == null || dailyFileId <= 0) {
return;
@@ -19,6 +19,16 @@ public class TaskHeartbeatVo {
@Schema(description = "提示信息", example = "ok")
private String message;
/**
* 进度更新失败但任务仍存活时置 true(S1-B4:组装/提交持锁期间心跳不许降级成"任务不在了")。
* alive 与 progressDegraded 是两件事:前者是活性,后者只是这次进度没写进去。
*/
@Schema(description = "进度更新是否已降级(任务仍在运行,本次进度未落库)", example = "false")
private boolean progressDegraded;
@Schema(description = "降级原因(仅 progressDegraded=true 时非空)")
private String warning;
public static TaskHeartbeatVo alive(String moduleType, String status) {
TaskHeartbeatVo vo = new TaskHeartbeatVo();
vo.setAlive(true);
@@ -28,6 +38,13 @@ public class TaskHeartbeatVo {
return vo;
}
public static TaskHeartbeatVo aliveDegraded(String moduleType, String status, String warning) {
TaskHeartbeatVo vo = alive(moduleType, status);
vo.setProgressDegraded(true);
vo.setWarning(warning);
return vo;
}
public static TaskHeartbeatVo notAlive(String moduleType, String status, String message) {
TaskHeartbeatVo vo = new TaskHeartbeatVo();
vo.setAlive(false);
@@ -137,6 +137,11 @@ public class ModuleHistoryCleanupService {
List<Long> skippedRetainedTaskSamples = new ArrayList<>(LOG_SAMPLE_IDS);
while (true) {
List<FileTaskEntity> page = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
// S1-B5:批查只取指针列。整实体一批 500 行会把 request_json/result_json/
// error_message 三只大列全拉进堆,夜间清理足以把堆打满。
.select(FileTaskEntity::getId, FileTaskEntity::getModuleType,
FileTaskEntity::getStatus, FileTaskEntity::getUpdatedAt,
FileTaskEntity::getFinishedAt)
.in(FileTaskEntity::getModuleType, moduleTypes)
.gt(FileTaskEntity::getId, cursor)
.orderByAsc(FileTaskEntity::getId)
@@ -364,12 +369,14 @@ public class ModuleHistoryCleanupService {
private List<String> collectPayloadPointers(List<String> moduleTypes, List<Long> cleanupTaskIds, int max) {
java.util.Set<String> pointers = new java.util.LinkedHashSet<>();
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson)
.in(TaskChunkEntity::getModuleType, moduleTypes)
.in(TaskChunkEntity::getTaskId, cleanupTaskIds));
for (TaskChunkEntity chunk : chunks) {
collectPointer(pointers, chunk.getPayloadJson(), max);
}
List<TaskScopeStateEntity> scopeStates = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
.in(TaskScopeStateEntity::getModuleType, moduleTypes)
.in(TaskScopeStateEntity::getTaskId, cleanupTaskIds));
for (TaskScopeStateEntity scopeState : scopeStates) {
@@ -377,12 +384,14 @@ public class ModuleHistoryCleanupService {
collectPointer(pointers, scopeState.getStateJson(), max);
}
List<TaskResultItemEntity> resultItems = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.select(TaskResultItemEntity::getPayloadJson)
.in(TaskResultItemEntity::getModuleType, moduleTypes)
.in(TaskResultItemEntity::getTaskId, cleanupTaskIds));
for (TaskResultItemEntity resultItem : resultItems) {
collectPointer(pointers, resultItem.getPayloadJson(), max);
}
List<TaskResultPayloadEntity> resultPayloads = taskResultPayloadMapper.selectList(new LambdaQueryWrapper<TaskResultPayloadEntity>()
.select(TaskResultPayloadEntity::getPayloadJson)
.in(TaskResultPayloadEntity::getModuleType, moduleTypes)
.in(TaskResultPayloadEntity::getTaskId, cleanupTaskIds));
for (TaskResultPayloadEntity resultPayload : resultPayloads) {
@@ -410,20 +410,36 @@ public class TaskFileJobService {
continue;
}
if ("PENDING".equals(job.getStatus())) {
// PENDING 僵尸兜底:补发 dispatch 事件(worker claim 幂等),
// 并累加 retryCount 防无限重发;注意不翻转状态,
// 避免与真实 PENDING 竞争 worker 的 claim 条件更新冲突。
// PENDING 僵尸兜底:补发 dispatch 事件(worker claim 幂等),并累加 retryCount
// 防无限重发。S1-B7:计数必须能走到 MAX_RETRY_COUNT 并就地翻 FAILED——
// 之前 Math.min(MAX_RETRY_COUNT - 1, ...) 把僵尸钉死在 4,而判终态的条件是
// retry_count >= MAX_RETRY_COUNT,于是每轮重派、永不到终态、终态回调永不触发。
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
int nextRetryCount = Math.min(MAX_RETRY_COUNT - 1, retryCount + 1);
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1);
boolean exhaustedNow = nextRetryCount >= MAX_RETRY_COUNT;
LocalDateTime now = LocalDateTime.now();
LambdaUpdateWrapper<TaskFileJobEntity> pendingUpdate = new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "PENDING")
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
.set(TaskFileJobEntity::getUpdatedAt, now);
if (exhaustedNow) {
pendingUpdate
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getErrorMessage,
"文件生成任务排队未被执行,已达到最大重试次数")
.set(TaskFileJobEntity::getFinishedAt, now)
.set(TaskFileJobEntity::getTerminalCallbackAt, null);
}
int updated = taskFileJobMapper.update(null, pendingUpdate);
if (updated > 0) {
eventJobIds.add(job.getId());
if (exhaustedNow) {
exhaustedJobIds.add(job.getId());
} else {
eventJobIds.add(job.getId());
}
}
continue;
}
@@ -90,7 +90,11 @@ public class TaskHeartbeatService {
}
if (fileResult != null && fileResult.isAlive() && brandResult != null && brandResult.isAlive()) {
return TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING);
TaskHeartbeatVo merged = fileResult.isProgressDegraded()
? TaskHeartbeatVo.aliveDegraded(fileResult.getModuleType() + "," + MODULE_BRAND,
STATUS_RUNNING, fileResult.getWarning())
: TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING);
return merged;
}
if (fileResult != null && fileResult.isAlive()) {
return fileResult;
@@ -206,9 +210,21 @@ public class TaskHeartbeatService {
}
if (handler != null) {
handler.touchHeartbeat(task.getId(), request);
String degradeWarning = null;
try {
handler.touchHeartbeat(task.getId(), request);
} catch (RuntimeException ex) {
// S1-B4:模块的进度写可能因并发写库/锁超时抛任何运行时异常;
// 心跳表达的是活性,进度写不进去只能降级,绝不能让客户端把 40902 当"任务不在了"。
degradeWarning = "进度更新失败(已降级,任务仍在运行): " + ex.getMessage();
log.warn("[task-heartbeat] module progress degraded taskId={} moduleType={} err={}",
task.getId(), moduleType, ex.getMessage());
}
handler.onHeartbeat(task);
handler.saveTaskCache(task);
return degradeWarning == null
? TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING)
: TaskHeartbeatVo.aliveDegraded(moduleType, STATUS_RUNNING, degradeWarning);
}
return TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING);
}
@@ -44,6 +44,13 @@ public class TaskResultFileJobWorker {
@Value("${aiimage.result-file-job.local-worker-enabled:true}")
private boolean localWorkerEnabled;
/**
* 与 {@code TaskFileJobLocalDispatcher} 读同一键:只有部署方显式关掉本地派发时才允许
* 退回"调度线程内联执行"这一历史语义;默认状态下派发被拒一律回队。
*/
@Value("${aiimage.result-file-job.local-dispatch-enabled:true}")
private boolean localDispatchEnabled = true;
@Value("${aiimage.result-file-job.batch-size:20}")
private int batchSize;
@@ -88,7 +95,17 @@ public class TaskResultFileJobWorker {
// 失败,job 被静默丢弃并挂到 stuck 扫描才回收,定时兜底等于失效。
boolean dispatched = taskFileJobLocalDispatcher.dispatchClaimed(job);
if (!dispatched) {
processClaimed(job);
if (!localDispatchEnabled) {
// 部署方显式关闭本地派发:保留"调度线程内联执行"的历史语义。
processClaimed(job);
continue;
}
// S1-B3:派发被拒(在途上限或执行器拒绝)时绝不能内联——那是把分钟级的
// Excel 组装压回 16 线程的调度池,拖停 stale 修复/视频派发这些兜底任务。
// 回队后由归属实例下一轮扫描补发,与 process() 通道的处理口径一致。
log.warn("[task-file-job] local dispatch rejected, requeue for next round jobId={} taskId={} moduleType={} stage=REQUEUE",
job.getId(), job.getTaskId(), job.getModuleType());
taskFileJobService.requeue(job.getId(), "本地派发被拒,稍后重投");
}
}
}
@@ -109,6 +109,13 @@ AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_ENABLED=false
AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS=10000
AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES=5
# 任务队列(可选,留空即用 application.yml 的出厂默认值):
# 视频/图像 Coze 任务与结果文件组装各自独立许可;优雅停机等待在途任务的秒数。
AIIMAGE_TASK_QUEUE_VIDEO_MAX_CONCURRENT=6
AIIMAGE_TASK_QUEUE_VIDEO_MAX_WAITING=1000
AIIMAGE_TASK_QUEUE_SHUTDOWN_AWAIT_SECONDS=30
AIIMAGE_DB_DEFAULT_STATEMENT_TIMEOUT_SECONDS=60
AIIMAGE_ZINIAO_ENABLED=false
AIIMAGE_ZINIAO_BASE_URL=https://sbappstoreapi.ziniao.com/openapi-router
AIIMAGE_ZINIAO_API_KEY=change-me
@@ -1,9 +1,14 @@
server:
port: ${AIIMAGE_SERVER_PORT:18080}
# S1-B6:发版硬切在途上传/组装的根因。graceful 让 Tomcat 先停止接收新请求、
# 等在途请求收尾;等待上限由 spring.lifecycle.timeout-per-shutdown-phase 控制。
shutdown: graceful
spring:
application:
name: aiimage-backend
lifecycle:
timeout-per-shutdown-phase: 30s
servlet:
multipart:
max-file-size: 2GB
@@ -82,6 +87,9 @@ mybatis-plus:
mapper-locations: classpath*:mapper/**/*.xml
configuration:
map-underscore-to-camel-case: true
# S1-B8:语句级超时(秒)。socketTimeout=300s + 30 格池 ⇒ 一条慢 SQL 能占死一格连接 5 分钟,
# Tomcat 请求在 30s 取连接处失败。60s 覆盖现网最慢的报表/清理语句,又留足连接回收余量。
default-statement-timeout: ${AIIMAGE_DB_DEFAULT_STATEMENT_TIMEOUT_SECONDS:60}
log-impl: org.apache.ibatis.logging.slf4j.Slf4jImpl
springdoc:
@@ -382,6 +390,11 @@ aiimage:
task-queue:
max-concurrent: ${AIIMAGE_TASK_QUEUE_MAX_CONCURRENT:12}
max-waiting: ${AIIMAGE_TASK_QUEUE_MAX_WAITING:1000}
# S1-B2:图像/视频(Coze 读超时 1 小时)独立许可数,不与结果文件组装的 12 格共用。
video-max-concurrent: ${AIIMAGE_TASK_QUEUE_VIDEO_MAX_CONCURRENT:6}
video-max-waiting: ${AIIMAGE_TASK_QUEUE_VIDEO_MAX_WAITING:1000}
# S1-B6:优雅停机时虚拟线程池等待在途任务的秒数(对齐 spring.lifecycle 的 30s)。
shutdown-await-seconds: ${AIIMAGE_TASK_QUEUE_SHUTDOWN_AWAIT_SECONDS:30}
brand-check:
base-url: ${AIIMAGE_BRAND_CHECK_BASE_URL:http://47.110.241.161:16890}
path: ${AIIMAGE_BRAND_CHECK_PATH:/brand_check}
@@ -37,7 +37,7 @@
FROM biz_image_video_async_task t
LEFT JOIN users u ON u.id = t.user_id
WHERE t.task_type = 'IMAGE_VIDEO_WORKFLOW'
AND t.submitted_at &amp;gt;= DATE_SUB(NOW(), INTERVAL 3 DAY)
AND t.submitted_at &gt;= DATE_SUB(NOW(), INTERVAL 3 DAY)
AND t.id = #{taskId}
LIMIT 1
</select>
@@ -102,7 +102,7 @@ class SuperAdminPredicateSingleOutletTest {
}
@Test
void superAdminOutletIsOneStaticPredicateAndSeventeenCallSites() {
void superAdminOutletIsOneStaticPredicateAndEighteenCallSites() {
int definitions = 0;
int callSites = 0;
int wrappers = 0;
@@ -120,8 +120,10 @@ class SuperAdminPredicateSingleOutletTest {
.as("业务层包装 isPlatformSuperAdmin 的定义处(只允许 DataReadScope 一个)")
.isEqualTo(1);
assertThat(callSites)
.as("走出口的调用点数:16 个业务调用方 + DataReadScope 包装体(新增调用要一起改这个数字,别改回字面量)")
.isEqualTo(17);
// 17 -> 18:v1.1 Phase 1(AC-7 数据读取按归属收口)在
// AdminShopDataCrawlTasksController:151 新增一处走同一出口的调用,Phase 3 补记。
.as("走出口的调用点数:17 个业务调用方 + DataReadScope 包装体(新增调用要一起改这个数字,别改回字面量)")
.isEqualTo(18);
// 自检:这三把尺子的锚点必须真落在预期的文件里,否则"恰好也是 N"可能是扫错地方凑出来的
assertThat(read(mainRoot().resolve("common/security/DeviceSessionPolicy.java")))
.contains("public static boolean isSuper(String resolvedRole)");
@@ -56,7 +56,8 @@ class CatchSilenceRatchetTest {
// 逐处记名见 STRICT_TOTAL 上方注释)。
// 10-28 V179 采集被过滤行追溯抬 579 -> 582(审计件三枚:逐批插入容错/回流簿记容错/尾 sheet 降级,
// 逐处记名见 STRICT_TOTAL 上方注释)。
static final int GIVEN_NOT_RETHROW = 582;
// 10-28 Phase 3 抬 582 -> 585(心跳降级一枚 + 优雅停机两枚中断恢复,逐处记名见 STRICT_TOTAL 上方注释)。
static final int GIVEN_NOT_RETHROW = 585;
static final int GIVEN_LOG_ONLY = 249;
static final int GIVEN_EMPTY = 56;
@@ -199,8 +200,19 @@ class CatchSilenceRatchetTest {
* 那只是本地 POI 写,写不动说明整份 workbook 都写不动,该由
* {@code writeWorkbookStreaming} 的既有出口处理,与前两张 sheet 同姿势。</li>
* </ul>
*
* <p>10-28 Phase 3(v1.1 后端任务引擎稳定性)抬 total 1031 -&gt; 1034,三处逐条记名:
* <ul>
* <li>{@code TaskHeartbeatService.touchFileTaskIfRunning} 一枚:模块进度写抛任何运行时异常
* (S1-B4 的 40902 就是这一类)都降级成 {@code progressDegraded=true},块体一条 WARN 带
* taskId/moduleType;不接住就等于心跳继续 500/40902,客户端会当"任务不在了"。</li>
* <li>{@code TaskFileJobConfig.taskQueueVirtualThreadExecutor} 的 {@code shutdown()} 一枚:
* await 期间被中断只恢复中断标记(S1-B6 优雅停机),不吞别的东西。</li>
* <li>{@code TaskFileJobConfig.videoQueueExecutor} 一枚:信号量 acquire 被中断只恢复中断标记
* (S1-B2 视频独立许可,照抄 taskQueueExecutor 的既有形状)。</li>
* </ul>
*/
static final int STRICT_TOTAL = 1031;
static final int STRICT_TOTAL = 1034;
/**
* 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。
* 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。
@@ -305,8 +317,13 @@ class CatchSilenceRatchetTest {
* {@code CollectDataExcelAssemblyService.writeFilteredSheet} 尾 sheet 降级;三处都是带
* {@code [collect-data]} 锚点的 log.warn+善后,块体除第一枚外各带一句状态置位/返回值,
* 逐处记名见 STRICT_TOTAL 上方注释)。
* 10-28 Phase 3 再抬 582 -&gt; 585(后端任务引擎三枚,均为「异常到此为止」的必要形状):
* {@code TaskHeartbeatService.touchFileTaskIfRunning}(S1-B4 进度写降级成 progressDegraded,
* 块体 log.warn 带 taskId/moduleType)、{@code TaskFileJobConfig.taskQueueVirtualThreadExecutor}
* 的 {@code shutdown()} 与 {@code TaskFileJobConfig.videoQueueExecutor}(S1-B6/S1-B2,
* {@code InterruptedException} 只恢复中断标记),逐处记名见 STRICT_TOTAL 上方注释。
*/
static final int STRICT_NOT_RETHROW = 582;
static final int STRICT_NOT_RETHROW = 585;
/**
* 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}:
* 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出——
@@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -60,9 +61,6 @@ class CollectDataProgressThrottleTest {
@Mock
private TaskFileJobService taskFileJobService;
@Mock
private TaskDistributedLockService.LockHandle lockHandle;
@Spy
private com.fasterxml.jackson.databind.ObjectMapper objectMapper =
new com.fasterxml.jackson.databind.ObjectMapper();
@@ -70,6 +68,16 @@ class CollectDataProgressThrottleTest {
@InjectMocks
private CollectDataService service;
@org.junit.jupiter.api.BeforeAll
static void initializeMybatisMetadata() {
// updateProgress 改无锁短事务后走 LambdaUpdateWrapper 瘦投影写,
// 构造 wrapper 需要 FileTaskEntity 的 lambda 缓存(否则 MybatisPlus 抛"can not find lambda cache")。
com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo(
new org.apache.ibatis.builder.MapperBuilderAssistant(
new com.baomidou.mybatisplus.core.MybatisConfiguration(), ""),
FileTaskEntity.class);
}
@BeforeEach
void setUp() {
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
@@ -85,8 +93,7 @@ class CollectDataProgressThrottleTest {
ReflectionTestUtils.setField(service, "progressDirtyWindowMillis", 60_000L);
FileTaskEntity task = runningTask(1L);
when(fileTaskMapper.selectById(1L)).thenReturn(task);
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 1L, 5_000L))
.thenReturn(lockHandle);
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
TaskHeartbeatRequest first = new TaskHeartbeatRequest();
first.setCurrent(1);
@@ -99,7 +106,7 @@ class CollectDataProgressThrottleTest {
// 窗口内合并:2 次心跳只落库 1 次(首条,processedRows=1);
// 窗口到期后补写最新值(processedRows=2),进度不丢失。
verify(fileTaskMapper, times(1)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(1)).update(isNull(), any());
assertThat(task.getResultJson()).contains("\"totalRows\":10");
assertThat(task.getResultJson()).contains("\"processedRows\":1");
@@ -109,7 +116,7 @@ class CollectDataProgressThrottleTest {
third.setCurrent(2);
third.setTotal(10);
service.updateProgress(1L, third);
verify(fileTaskMapper, times(2)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(2)).update(isNull(), any());
assertThat(task.getResultJson()).contains("\"processedRows\":2");
}
@@ -123,10 +130,7 @@ class CollectDataProgressThrottleTest {
FileTaskEntity taskB = runningTask(12L);
when(fileTaskMapper.selectById(11L)).thenReturn(taskA);
when(fileTaskMapper.selectById(12L)).thenReturn(taskB);
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 11L, 5_000L))
.thenReturn(lockHandle);
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 12L, 5_000L))
.thenReturn(lockHandle);
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
heartbeat(service, 11L, 1, 10);
heartbeat(service, 12L, 1, 10);
@@ -134,20 +138,20 @@ class CollectDataProgressThrottleTest {
heartbeat(service, 12L, 2, 10);
// 每支任务首条 changed 心跳各落库 1 次,其后的窗口内心跳合并写 → 共 2 次。
verify(fileTaskMapper, times(2)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(2)).update(isNull(), any());
assertThat(taskA.getResultJson()).contains("\"processedRows\":1");
assertThat(taskB.getResultJson()).contains("\"processedRows\":1");
// A 自身窗口到期后补写最新值;B 仍在自己窗口内,继续被合并。
expireProgressWindow(service, taskA);
heartbeat(service, 11L, 3, 10);
verify(fileTaskMapper, times(3)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(3)).update(isNull(), any());
assertThat(taskA.getResultJson()).contains("\"processedRows\":3");
// B 自身窗口到期后同样补写,两任务最终进度均为最新值。
expireProgressWindow(service, taskB);
heartbeat(service, 12L, 3, 10);
verify(fileTaskMapper, times(4)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(4)).update(isNull(), any());
assertThat(taskB.getResultJson()).contains("\"processedRows\":3");
}
@@ -164,22 +168,24 @@ class CollectDataProgressThrottleTest {
stuck.setUpdatedAt(LocalDateTime.now().minusMinutes(20));
when(fileTaskMapper.selectById(71L)).thenReturn(busy);
when(fileTaskMapper.selectById(72L)).thenReturn(stuck);
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 71L, 5_000L))
.thenReturn(lockHandle);
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 72L, 5_000L))
.thenReturn(lockHandle);
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
// 邻居任务连续刷了 5 次,把「全局窗口」反复重置——旧实现正是死在这里。
for (int i = 1; i <= 5; i++) {
expireProgressWindow(service, busy); // 让它每次都够格落库
heartbeat(service, 71L, i, 10);
}
assertThat(busy.getUpdatedAt()).isAfter(LocalDateTime.now().minusSeconds(5));
// 只看卡住任务自己的落库:它内容不变、但自身 updated_at 已陈旧 ⇒ 每条心跳都必须
// 落库把 updated_at 顶上去(无锁实现下 updated_at 由 DB 写刷新,不再改内存实体)。
org.mockito.Mockito.clearInvocations(fileTaskMapper);
heartbeat(service, 72L, 2, 8);
heartbeat(service, 72L, 2, 8);
verify(fileTaskMapper, times(2)).update(isNull(), any());
// 内容完全相同的重复心跳,仍必须靠自身脏窗口把 updated_at 顶上去。
// 反证:同一任务 updated_at 新鲜、且刚写过(窗口内)时,内容不变的心跳不再落库。
stuck.setUpdatedAt(LocalDateTime.now());
heartbeat(service, 72L, 2, 8);
heartbeat(service, 72L, 2, 8);
assertThat(stuck.getUpdatedAt()).isAfter(LocalDateTime.now().minusSeconds(5));
verify(fileTaskMapper, times(2)).update(isNull(), any());
}
@Test
@@ -190,26 +196,25 @@ class CollectDataProgressThrottleTest {
ReflectionTestUtils.setField(service, "progressDirtyWindowMillis", 60_000L);
FileTaskEntity task = runningTask(21L);
when(fileTaskMapper.selectById(21L)).thenReturn(task);
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 21L, 5_000L))
.thenReturn(lockHandle);
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
heartbeat(service, 21L, 1, 10);
heartbeat(service, 21L, 1, 10);
heartbeat(service, 21L, 1, 10);
// 内容相同:仅首次心跳落库一次,重复心跳零 UPDATE。
verify(fileTaskMapper, times(1)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(1)).update(isNull(), any());
}
@Test
void test_task_055_merge_progress_boundary_empty_input() {
// 空输入:null taskId / 空请求 → 直接返回,不取锁不查询不落库。
// 空输入:null taskId / 空请求 → 直接返回,不取锁(AC-10 后本路径已无锁)、不查询、不落库。
service.updateProgress(null, new TaskHeartbeatRequest());
service.updateProgress(1L, null);
verify(taskDistributedLockService, never()).acquire(any(), any(), anyLong());
verify(fileTaskMapper, never()).selectById(any(java.io.Serializable.class));
verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, never()).update(any(), any());
}
@Test
@@ -219,8 +224,7 @@ class CollectDataProgressThrottleTest {
ReflectionTestUtils.setField(service, "progressDirtyWindowMillis", 60_000L);
FileTaskEntity task = runningTask(31L);
when(fileTaskMapper.selectById(31L)).thenReturn(task);
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 31L, 5_000L))
.thenReturn(lockHandle);
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
heartbeat(service, 31L, 3, 20);
heartbeat(service, 31L, 3, 20);
@@ -228,12 +232,12 @@ class CollectDataProgressThrottleTest {
// 首条 changed 心跳落库一次(totalRows=20, processedRows=3);后续窗口内心跳合并写。
assertThat(task.getResultJson()).contains("\"totalRows\":20");
verify(fileTaskMapper, times(1)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(1)).update(isNull(), any());
// 窗口到期后补写最新值(processedRows=4),不丢失。
expireProgressWindow(service, task);
heartbeat(service, 31L, 4, 21);
verify(fileTaskMapper, times(2)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(2)).update(isNull(), any());
assertThat(task.getResultJson()).contains("\"processedRows\":4");
}
@@ -245,19 +249,18 @@ class CollectDataProgressThrottleTest {
ReflectionTestUtils.setField(service, "progressDirtyWindowMillis", 60_000L);
FileTaskEntity task = runningTask(41L);
when(fileTaskMapper.selectById(41L)).thenReturn(task);
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 41L, 5_000L))
.thenReturn(lockHandle);
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
for (int i = 1; i <= 100; i++) {
heartbeat(service, 41L, i, 100);
}
// 窗口内 100 次心跳只落库 1 次,无无界 UPDATE 累积;窗口到期后补写最新值。
verify(fileTaskMapper, times(1)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(1)).update(isNull(), any());
assertThat(task.getResultJson()).contains("\"processedRows\":1");
expireProgressWindow(service, task);
heartbeat(service, 41L, 100, 100);
verify(fileTaskMapper, times(2)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(2)).update(isNull(), any());
assertThat(task.getResultJson()).contains("\"processedRows\":100");
}
@@ -271,15 +274,11 @@ class CollectDataProgressThrottleTest {
finished.setStatus("SUCCESS");
when(fileTaskMapper.selectById(51L)).thenReturn(wrongModule);
when(fileTaskMapper.selectById(52L)).thenReturn(finished);
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 51L, 5_000L))
.thenReturn(lockHandle);
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 52L, 5_000L))
.thenReturn(lockHandle);
heartbeat(service, 51L, 1, 10);
heartbeat(service, 52L, 1, 10);
verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, never()).update(any(), any());
}
@Test
@@ -290,34 +289,29 @@ class CollectDataProgressThrottleTest {
ReflectionTestUtils.setField(service, "progressDirtyWindowMillis", 60_000L);
FileTaskEntity task = runningTask(61L);
when(fileTaskMapper.selectById(61L)).thenReturn(task);
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 61L, 5_000L))
.thenReturn(lockHandle);
AtomicInteger updateCalls = new AtomicInteger();
when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenAnswer(invocation -> {
when(fileTaskMapper.update(isNull(), any())).thenAnswer(invocation -> {
if (updateCalls.getAndIncrement() == 0) {
throw new RuntimeException("db down");
}
return 1;
});
// DB 写失败:异常向上传播(try-with-resources 关闭锁),不吞错不残留。
// DB 写失败:异常向上传播,不吞错;AC-10 之后本路径不再取任务锁(无可释放的锁)。
assertThatThrownBy(() -> heartbeat(service, 61L, 1, 10))
.isInstanceOf(RuntimeException.class)
.hasMessage("db down");
verify(lockHandle, times(1)).close();
verify(taskDistributedLockService, never()).acquire(any(), any(), anyLong());
// 恢复后心跳重试:落库一次成功,进度持久化。
// 节流时钟只在 updateById 返回后才推进,所以那次失败的写没有留下窗口;updated_at 同理
// 停在库里那一版。真实链路每条心跳都在 :530 重新 selectById ⇒ 会立刻补刷。本用例的
// mock 复用同一个实体实例,故把 updated_at 退回「库里那一版」——否则测的是对象别名,
// 不是这条重读语义。
// 节流时钟只在写成功后才推进,所以那次失败的写没有留下窗口;真实链路每条心跳都会
// 重新 selectById ⇒ 会立刻补刷。本用例的 mock 复用同一个实体实例,故把 updated_at
// 退回「库里那一版」——否则测的是对象别名,不是这条重读语义。
task.setUpdatedAt(null);
heartbeat(service, 61L, 2, 10);
verify(fileTaskMapper, times(2)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(2)).update(isNull(), any());
assertThat(task.getResultJson()).contains("\"processedRows\":2");
verify(lockHandle, times(2)).close();
}
@Test
@@ -331,13 +325,12 @@ class CollectDataProgressThrottleTest {
ReflectionTestUtils.setField(service, "progressDirtyWindowMillis", 30_000L);
FileTaskEntity task = runningTask(81L);
when(fileTaskMapper.selectById(81L)).thenReturn(task);
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 81L, 5_000L))
.thenReturn(lockHandle);
when(fileTaskMapper.update(isNull(), any())).thenReturn(1);
// 决定性的一步:updated_at 就是 now,旧实现在这里直接 return,进度永远停在 0。
task.setUpdatedAt(LocalDateTime.now());
heartbeat(service, 81L, 1, 7);
verify(fileTaskMapper, times(1)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(1)).update(isNull(), any());
assertThat(task.getResultJson()).contains("\"processedRows\":1");
// 客户端按 60s 一跳,逐关键词推进:每跳进来时 updated_at 都已被心跳层顶到 now,
@@ -347,7 +340,7 @@ class CollectDataProgressThrottleTest {
expireProgressWindow(service, task);
heartbeat(service, 81L, keyword, 7);
}
verify(fileTaskMapper, times(7)).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, times(7)).update(isNull(), any());
assertThat(task.getResultJson()).contains("\"processedRows\":7");
assertThat(task.getResultJson()).contains("\"totalRows\":7");
}
@@ -359,6 +352,9 @@ class CollectDataProgressThrottleTest {
task.setModuleType(CollectDataService.MODULE_TYPE);
task.setStatus("RUNNING");
task.setResultJson("{}");
// 真实链路每条心跳都重新 selectById,updated_at 是库里那一版(AC-10 后心跳只写 DB、
// 不再回改内存实体)。这里给个新鲜的 updated_at,等价于"库里刚被刷过"的读入形状。
task.setUpdatedAt(LocalDateTime.now());
return task;
}
@@ -236,7 +236,8 @@ class FaultInjectionTest {
// 第一次提交被执行器拒绝:等待槽释放、拒绝指标记录
assertThrows(TaskRejectedException.class, () -> gate.execute(task));
assertEquals(0, gate.waiting(), "执行器拒绝后等待槽必须释放");
assertTrue(registry.counter("aiimage.task-queue.submit.rejected.total", "reason", "delegate-rejected").count() > 0);
assertTrue(registry.find("aiimage.task-queue.submit.rejected.total")
.tag("reason", "delegate-rejected").counter().count() > 0);
// 第二次提交恢复成功:等待槽正常占用并释放
gate.execute(task);
@@ -1,7 +1,6 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.testutil.SourceScan;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.springframework.core.env.PropertySource;
@@ -34,7 +33,6 @@ import static org.assertj.core.api.Assertions.assertThat;
* <p>蓝图 §3.2 登记的收口形状:{@code server.shutdown=graceful} +
* {@code spring.lifecycle.timeout-per-shutdown-phase=30s} + 统一 {@code @PreDestroy}。
*/
@Disabled("Pending Phase 3: AC-11 清理流式化与优雅停机")
@DisplayName("AC-11 清理只读指针 + 优雅停机")
class PendingPhase3CleanupStreamingTest {
@@ -63,12 +61,15 @@ class PendingPhase3CleanupStreamingTest {
long selects = java.util.Arrays.stream(body.split("selectList"))
.filter(part -> part.contains(".select("))
.count();
// 预置时按审计 S1-B5 原文点名的三路写死 3;实现阶段复核发现第 4 路
// (result_payload.payloadJson,:385 那条)同样是整实体取回、同样要瘦投影,
// 漏掉它等于漏掉"行删了指针就永久丢失"的一种对象。判据按实情抬到 4(不是放宽)。
assertThat(body.split("selectList").length - 1)
.as("chunk/scopeState/resultItem 三条查询都要瘦投影")
.isEqualTo(3);
.as("chunk/scopeState/resultItem/resultPayload 四条查询都要瘦投影")
.isEqualTo(4);
assertThat(selects)
.as("S1-B5:三条 selectList 全部带 .select(...) 投影才算流式")
.isEqualTo(3L);
.as("S1-B5:四条 selectList 全部带 .select(...) 投影才算流式")
.isEqualTo(4L);
}
@Test
@@ -16,7 +16,6 @@ import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
import com.nanri.aiimage.testutil.SourceScan;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
@@ -47,7 +46,6 @@ import static org.mockito.Mockito.when;
* 蓝图 §3.1 定的收口形状是:进度失败仍返回 200、{@code alive=true},新增
* {@code progressDegraded=true} 与 {@code warning},只有任务终态才 {@code alive=false}。
*/
@Disabled("Pending Phase 3: AC-10 心跳去锁与 SPI 异常降级")
@DisplayName("AC-10 心跳在持锁窗口内必须仍返回 alive")
class PendingPhase3HeartbeatDegradationTest {
@@ -84,6 +82,9 @@ class PendingPhase3HeartbeatDegradationTest {
org.mockito.Mockito.doThrow(new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理,请稍后再试"))
.when(collectDataService).updateProgress(any(), any());
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task);
// harness 补齐(预置阶段漏打桩):TaskHeartbeatService 会先刷 updated_at 检查点,
// 该条件更新返回 0 时按"任务已不 RUNNING"提前返回,走不到 SPI。这里让它命中。
when(fileTaskMapper.update(any(), any())).thenReturn(1);
TaskHeartbeatService service = serviceWith(new CollectDataTaskHeartbeatSpi(collectDataService));
TaskHeartbeatRequest request = new TaskHeartbeatRequest();
@@ -9,7 +9,6 @@ import com.nanri.aiimage.testutil.MapperXmlSupport;
import com.nanri.aiimage.testutil.SourceScan;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
@@ -57,7 +56,6 @@ import static org.mockito.Mockito.when;
* 的 {@code deleteDailyFile}(删成员 + 删档)同样两次写无事务 ⇒ 中途失败留半删状态。</li>
* </ul>
*/
@Disabled("Pending Phase 3: AC-12 PENDING 封顶/视频详情 SQL/statement timeout/删除事务")
@DisplayName("AC-12 终态封顶与 DB 超时契约")
class PendingPhase3JobTerminalTest {
@@ -4,7 +4,6 @@ import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.testutil.SourceScan;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
@@ -46,7 +45,6 @@ import static org.mockito.Mockito.when;
* {@code aiimage.result-file-job.local-worker-delay-ms} 默认 15000(现网 yml 覆盖为 10000),
* 本用例判的是"一轮之内被拒的作业必须回到队列",不依赖具体毫秒数。
*/
@Disabled("Pending Phase 3: AC-9 视频/结果文件队列隔离与拒绝回队")
@DisplayName("AC-9 队列隔离与背压回队")
class PendingPhase3QueueIsolationTest {
@@ -126,6 +126,9 @@ class TaskResultFileJobWorkerOwnerRoutingTest {
when(taskFileJobService.claimRunnableJobsForOwner(20, "instance-a")).thenReturn(List.of(claim));
when(taskFileJobService.activateRunningClaim(claim)).thenReturn(true);
when(localDispatcher.dispatchClaimed(claim)).thenReturn(false);
// 本用例描述的是"本地派发被显式关闭"的部署形态:此时 worker 才允许在调度线程内联执行;
// 默认形态下派发被拒一律回队(见 PendingPhase3QueueIsolationTest)。
set(worker, "localDispatchEnabled", false);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, claim.getTaskId(),
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
@@ -8,7 +8,7 @@ DeviceLogFileMapper deleteOlderThanBatch Delete PURE DELETE FROM device_log_file
FileResultMapper selectRowsUpdatedAtByTaskIds Select PURE <script>\nSELECT task_id AS taskId, MAX(updated_at) AS latestRowAt, COUNT(*) AS rowCount\nFROM biz_file_result\nWHERE module_type = #{moduleType}\n AND task_id IN\n <foreach item="taskId" collection="taskIds" open="(" separator="," close=")">#{taskId}</foreach>\nGROUP BY task_id\n</script>
FileTaskMapper selectExpiredTerminalTaskIds Select PURE SELECT id FROM biz_file_task\nWHERE module_type = #{moduleType}\n AND status IN ('SUCCESS', 'FAILED', 'CANCELLED', 'CANCELED')\n AND updated_at < #{cutoff}\nORDER BY id ASC\nLIMIT #{batchSize}
ImageVideoAdminTaskMapper countAdminTaskPage Select PURE <script>\nSELECT COUNT(*) AS total\nFROM biz_image_video_async_task t\nLEFT JOIN users u ON u.id = t.user_id\nWHERE t.task_type = 'IMAGE_VIDEO_WORKFLOW'\n AND t.submitted_at &gt;= DATE_SUB(NOW(), INTERVAL 3 DAY)\n<if test="userId != null"> AND t.user_id = #{userId}</if>\n<if test="userIds != null"> AND t.user_id IN\n <foreach collection="userIds" item="uid" open="(" separator="," close=")">#{uid}</foreach>\n</if>\n<if test="username != null and username != ''"> AND u.username LIKE CONCAT('%', #{username}, '%')</if>\n<if test="status != null and status != ''"> AND t.status = #{status}</if>\n<if test="executeId != null and executeId != ''"> AND t.coze_execute_id LIKE CONCAT('%', #{executeId}, '%')</if>\n<if test="submittedFrom != null"> AND t.submitted_at &gt;= #{submittedFrom}</if>\n<if test="submittedTo != null"> AND t.submitted_at &lt;= #{submittedTo}</if>\n</script>
ImageVideoAdminTaskMapper selectAdminTaskDetail Select PURE SELECT t.id, t.user_id, t.status, t.request_json, t.submit_response_json, t.result_json,\n t.video_urls_json, t.debug_url, t.archived_videos_json, t.archive_status,\n t.archive_error, t.archive_attempt_count,\n DATE_FORMAT(t.archived_at, '%Y-%m-%d %H:%i:%s') AS archived_at,\n t.error_message, t.coze_execute_id, t.coze_status,\n DATE_FORMAT(t.submitted_at, '%Y-%m-%d %H:%i:%s') AS submitted_at,\n DATE_FORMAT(t.completed_at, '%Y-%m-%d %H:%i:%s') AS completed_at,\n u.username,\n COALESCE((\n SELECT GROUP_CONCAT(DISTINCT g.group_name ORDER BY g.id SEPARATOR '、')\n FROM biz_shop_manage_group g\n LEFT JOIN biz_shop_manage_group_member gm ON gm.group_id = g.id\n WHERE g.created_by_id = t.user_id OR g.user_id = t.user_id OR gm.user_id = t.user_id\n ), '') AS group_name\nFROM biz_image_video_async_task t\nLEFT JOIN users u ON u.id = t.user_id\nWHERE t.task_type = 'IMAGE_VIDEO_WORKFLOW'\n AND t.submitted_at &gt;= DATE_SUB(NOW(), INTERVAL 3 DAY)\n AND t.id = #{taskId}\nLIMIT 1
ImageVideoAdminTaskMapper selectAdminTaskDetail Select PURE SELECT t.id, t.user_id, t.status, t.request_json, t.submit_response_json, t.result_json,\n t.video_urls_json, t.debug_url, t.archived_videos_json, t.archive_status,\n t.archive_error, t.archive_attempt_count,\n DATE_FORMAT(t.archived_at, '%Y-%m-%d %H:%i:%s') AS archived_at,\n t.error_message, t.coze_execute_id, t.coze_status,\n DATE_FORMAT(t.submitted_at, '%Y-%m-%d %H:%i:%s') AS submitted_at,\n DATE_FORMAT(t.completed_at, '%Y-%m-%d %H:%i:%s') AS completed_at,\n u.username,\n COALESCE((\n SELECT GROUP_CONCAT(DISTINCT g.group_name ORDER BY g.id SEPARATOR '、')\n FROM biz_shop_manage_group g\n LEFT JOIN biz_shop_manage_group_member gm ON gm.group_id = g.id\n WHERE g.created_by_id = t.user_id OR g.user_id = t.user_id OR gm.user_id = t.user_id\n ), '') AS group_name\nFROM biz_image_video_async_task t\nLEFT JOIN users u ON u.id = t.user_id\nWHERE t.task_type = 'IMAGE_VIDEO_WORKFLOW'\n AND t.submitted_at >= DATE_SUB(NOW(), INTERVAL 3 DAY)\n AND t.id = #{taskId}\nLIMIT 1
ImageVideoAdminTaskMapper selectAdminTaskPage Select PURE <script>\nSELECT t.id, t.user_id, t.status, t.request_json, t.submit_response_json, t.result_json,\n t.video_urls_json, t.debug_url, t.archived_videos_json, t.archive_status,\n t.archive_error, t.archive_attempt_count,\n DATE_FORMAT(t.archived_at, '%Y-%m-%d %H:%i:%s') AS archived_at,\n t.error_message, t.coze_execute_id, t.coze_status,\n DATE_FORMAT(t.submitted_at, '%Y-%m-%d %H:%i:%s') AS submitted_at,\n DATE_FORMAT(t.completed_at, '%Y-%m-%d %H:%i:%s') AS completed_at,\n u.username,\n COALESCE((\n SELECT GROUP_CONCAT(DISTINCT g.group_name ORDER BY g.id SEPARATOR '、')\n FROM biz_shop_manage_group g\n LEFT JOIN biz_shop_manage_group_member gm ON gm.group_id = g.id\n WHERE g.created_by_id = t.user_id OR g.user_id = t.user_id OR gm.user_id = t.user_id\n ), '') AS group_name\nFROM biz_image_video_async_task t\nLEFT JOIN users u ON u.id = t.user_id\nWHERE t.task_type = 'IMAGE_VIDEO_WORKFLOW'\n AND t.submitted_at &gt;= DATE_SUB(NOW(), INTERVAL 3 DAY)\n<if test="userId != null"> AND t.user_id = #{userId}</if>\n<if test="userIds != null"> AND t.user_id IN\n <foreach collection="userIds" item="uid" open="(" separator="," close=")">#{uid}</foreach>\n</if>\n<if test="username != null and username != ''"> AND u.username LIKE CONCAT('%', #{username}, '%')</if>\n<if test="status != null and status != ''"> AND t.status = #{status}</if>\n<if test="executeId != null and executeId != ''"> AND t.coze_execute_id LIKE CONCAT('%', #{executeId}, '%')</if>\n<if test="submittedFrom != null"> AND t.submitted_at &gt;= #{submittedFrom}</if>\n<if test="submittedTo != null"> AND t.submitted_at &lt;= #{submittedTo}</if>\nORDER BY t.submitted_at DESC, t.id DESC\nLIMIT #{limit} OFFSET #{offset}\n</script>
ImageVideoAdminTaskMapper selectAdminZipRows Select PURE <script>\nSELECT t.id, t.user_id, t.video_urls_json, t.archived_videos_json\nFROM biz_image_video_async_task t\nWHERE t.task_type = 'IMAGE_VIDEO_WORKFLOW'\n AND t.submitted_at &gt;= DATE_SUB(NOW(), INTERVAL 3 DAY)\n AND t.id IN\n<foreach collection="taskIds" item="taskId" open="(" separator="," close=")">\n #{taskId}\n</foreach>\n</script>
ImageVideoAsyncTaskMapper claimPending Update PURE UPDATE biz_image_video_async_task SET status = 'RUNNING', owner_instance_id = CASE WHEN owner_instance_id IS NULL OR owner_instance_id = '' THEN #{ownerInstanceId} ELSE owner_instance_id END, updated_at = NOW(), attempt_count = attempt_count + 1 WHERE id = #{taskId} AND status = 'PENDING' AND (owner_instance_id = #{ownerInstanceId} OR owner_instance_id IS NULL OR owner_instance_id = '')
Can't render this file because it contains an unexpected character in line 1 and column 222.