diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java index e9ab61000..7df5af621 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java @@ -75,11 +75,58 @@ public class TaskFileJobConfig { return executor; } + /** 与 {@code spring.lifecycle.timeout-per-shutdown-phase} 同窗口:等在途任务收尾的上限(秒)。 */ + @Value("${aiimage.task-queue.shutdown-await-seconds:30}") + private int shutdownAwaitSeconds = 30; + + /** + * S1-B6:发版硬切在途作业的根因之一——这个池的 destroy 只 shutdown 不等待。 + * 现在 shutdown() 会等到在途任务收尾(上限 {@code aiimage.task-queue.shutdown-await-seconds})。 + */ @Bean(destroyMethod = "shutdown") public ExecutorService taskQueueVirtualThreadExecutor() { - return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + ExecutorService delegate = Executors.newThreadPerTaskExecutor(Thread.ofVirtual() .name("task-queue-", 0) .factory()); + long awaitSeconds = Math.max(1, shutdownAwaitSeconds); + return new java.util.concurrent.AbstractExecutorService() { + @Override + public void execute(Runnable command) { + delegate.execute(command); + } + + @Override + public void shutdown() { + delegate.shutdown(); + try { + if (!delegate.awaitTermination(awaitSeconds, TimeUnit.SECONDS)) { + delegate.shutdownNow(); + } + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + + @Override + public List shutdownNow() { + return delegate.shutdownNow(); + } + + @Override + public boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(timeout, unit); + } + }; } /** @@ -132,6 +179,41 @@ public class TaskFileJobConfig { return new TaskQueueGate(semaphoreLimited, maxWaiting, meterRegistryProvider); } + /** + * S1-B2:图像/视频任务(Coze 读超时 1 小时)与 43 个模块的结果文件组装共用 + * {@code taskQueueExecutor} 的 12 格许可时,12 个慢 Coze 请求就能把全池占满一小时。 + * 这里给视频派发/轮询一支独立许可 + 独立等待闸门的执行器; + * 线程仍跑在同一只虚拟线程池上(虚拟线程廉价,隔离的是"正在执行"的许可数)。 + */ + @Bean("videoQueueExecutor") + public TaskExecutor videoQueueExecutor( + @Qualifier("taskQueueVirtualThreadExecutor") ExecutorService taskQueueVirtualThreadExecutor, + @Value("${aiimage.task-queue.video-max-concurrent:6}") int videoMaxConcurrent, + @Value("${aiimage.task-queue.video-max-waiting:1000}") int videoMaxWaiting, + ObjectProvider meterRegistryProvider) { + Semaphore semaphore = new Semaphore(Math.max(1, videoMaxConcurrent)); + TaskExecutor semaphoreLimited = new ConcurrentTaskExecutor(command -> { + if (command == null) { + throw new IllegalArgumentException("task 不能为 null"); + } + taskQueueVirtualThreadExecutor.execute(() -> { + boolean acquired = false; + try { + semaphore.acquire(); + acquired = true; + command.run(); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } finally { + if (acquired) { + semaphore.release(); + } + } + }); + }); + return new TaskQueueGate(semaphoreLimited, videoMaxWaiting, meterRegistryProvider, "video"); + } + @Bean public ResultFileJobHandlerRegistry resultFileJobHandlerRegistry(List handlers) { ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers); diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/TaskQueueGate.java b/backend-java/src/main/java/com/nanri/aiimage/config/TaskQueueGate.java index ef795ba2f..75872a3a6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/TaskQueueGate.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/TaskQueueGate.java @@ -26,13 +26,20 @@ public class TaskQueueGate implements TaskExecutor { private final TaskExecutor delegate; private final int maxWaiting; private final ObjectProvider meterRegistryProvider; + private final String queueName; private final AtomicInteger waiting = new AtomicInteger(); public TaskQueueGate(TaskExecutor delegate, int maxWaiting, ObjectProvider meterRegistryProvider) { + this(delegate, maxWaiting, meterRegistryProvider, "task-queue"); + } + + public TaskQueueGate(TaskExecutor delegate, int maxWaiting, + ObjectProvider meterRegistryProvider, String queueName) { this.delegate = delegate; this.maxWaiting = Math.max(1, maxWaiting); this.meterRegistryProvider = meterRegistryProvider; + this.queueName = queueName == null || queueName.isBlank() ? "task-queue" : queueName; } public int waiting() { @@ -47,8 +54,8 @@ public class TaskQueueGate implements TaskExecutor { } if (!tryReserveWaitingSlot()) { recordRejected("queue-full"); - log.warn("[task-queue][gate] waiting queue full, reject submit waiting={} limit={}", - waiting.get(), maxWaiting); + log.warn("[{}][gate] waiting queue full, reject submit waiting={} limit={}", + queueName, waiting.get(), maxWaiting); throw new TaskRejectedException("task 等待队列已满,limit=" + maxWaiting + ", waiting=" + waiting.get()); } @@ -67,8 +74,8 @@ public class TaskQueueGate implements TaskExecutor { waiting.decrementAndGet(); recordQueueWait(System.nanoTime() - queuedAt); recordRejected("delegate-rejected"); - log.warn("[task-queue][gate] delegate rejected submit waiting={} limit={} msg={}", - waiting.get(), maxWaiting, ex.getMessage(), ex); + log.warn("[{}][gate] delegate rejected submit waiting={} limit={} msg={}", + queueName, waiting.get(), maxWaiting, ex.getMessage(), ex); throw ex; } } @@ -101,7 +108,8 @@ public class TaskQueueGate implements TaskExecutor { private void recordRejected(String reason) { MeterRegistry registry = meterRegistry(); if (registry != null) { - registry.counter("aiimage.task-queue.submit.rejected.total", "reason", reason).increment(); + registry.counter("aiimage.task-queue.submit.rejected.total", + "queue", queueName, "reason", reason).increment(); } } @@ -109,6 +117,7 @@ public class TaskQueueGate implements TaskExecutor { MeterRegistry registry = meterRegistry(); if (registry != null && waitNanos >= 0L) { Timer.builder("aiimage.task-queue.queue.wait.duration") + .tag("queue", queueName) .register(registry) .record(waitNanos, TimeUnit.NANOSECONDS); } @@ -118,6 +127,7 @@ public class TaskQueueGate implements TaskExecutor { MeterRegistry registry = meterRegistry(); if (registry != null && durationNanos >= 0L) { Timer.builder("aiimage.task-queue.execution.duration") + .tag("queue", queueName) .register(registry) .record(durationNanos, TimeUnit.NANOSECONDS); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java index a8687089f..d43e1d7ed 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java @@ -549,22 +549,21 @@ public class CollectDataService extends AbstractFileTaskPipeline事务边界:Redis 任务锁在事务外获取(自旋等待最长 TASK_LOCK_WAIT_MILLIS, - * 放在 @Transactional 里会白占一个 Hikari 连接),DB 段(统计持久化 + 任务行更新) - * 仍在一个事务内。 + *

S1-B4:这里**不再取任务锁**。锁是给组装/提交的写路径用的,心跳在持锁窗口里去抢锁 + * (最长等 5s,拿不到就 40902)会让客户端把还在跑的任务当"任务不在了"处理。 + * 改法是无锁短事务 + 乐观 CAS:读到的 {@code updated_at} 作为写条件, + * 与组装/分片提交并发时心跳让路(跳过本次进度写,下一条心跳自然补上),绝不覆盖对方刚写的 result_json。 */ public void updateProgress(Long taskId, TaskHeartbeatRequest request) { if (taskId == null || taskId <= 0 || request == null) { return; } - try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - if (transactionTemplate == null) { - // 单测场景(@InjectMocks 未注入事务模板):退化为直接执行 DB 段 - updateProgressLocked(taskId, request); - return; - } - transactionTemplate.executeWithoutResult(status -> updateProgressLocked(taskId, request)); + if (transactionTemplate == null) { + // 单测场景(@InjectMocks 未注入事务模板):退化为直接执行 DB 段 + updateProgressLocked(taskId, request); + return; } + transactionTemplate.executeWithoutResult(status -> updateProgressLocked(taskId, request)); } private void updateProgressLocked(Long taskId, TaskHeartbeatRequest request) { @@ -622,10 +621,23 @@ public class CollectDataService extends AbstractFileTaskPipeline() + .eq(FileTaskEntity::getId, task.getId()) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .eq(task.getUpdatedAt() != null, FileTaskEntity::getUpdatedAt, task.getUpdatedAt()) + .isNull(task.getUpdatedAt() == null, FileTaskEntity::getUpdatedAt) + .set(FileTaskEntity::getResultJson, task.getResultJson()) + .set(FileTaskEntity::getUpdatedAt, progressUpdatedAt)); + if (updated <= 0) { + log.info("[collect-data] 进度心跳让路:任务行已被并发写入或已离 RUNNING taskId={}", taskId); + return; + } + // 节流时钟只在写成功之后推进:被 CAS 挡下的那轮不算数,下一条心跳立刻补刷, + // 而不是被一次失败的写留下的窗口挡掉。 lastProgressFlushMillis.put(taskId, System.currentTimeMillis()); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java index 831a1e6b1..af532bde6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java @@ -69,7 +69,7 @@ public class ImageVideoAsyncTaskService { private final ImageVideoWorkflowConfigService workflowConfigService; private final ImageVideoArchiveService archiveService; private final ObjectMapper objectMapper; - private final TaskExecutor taskQueueExecutor; + private final TaskExecutor videoQueueExecutor; private final InstanceMetadata instanceMetadata; private final com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService; @@ -79,7 +79,7 @@ public class ImageVideoAsyncTaskService { ImageVideoWorkflowConfigService workflowConfigService, ImageVideoArchiveService archiveService, ObjectMapper objectMapper, - @Qualifier("taskQueueExecutor") TaskExecutor taskQueueExecutor, + @Qualifier("videoQueueExecutor") TaskExecutor videoQueueExecutor, InstanceMetadata instanceMetadata, com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService) { this.taskMapper = taskMapper; @@ -87,7 +87,7 @@ public class ImageVideoAsyncTaskService { this.workflowConfigService = workflowConfigService; this.archiveService = archiveService; this.objectMapper = objectMapper; - this.taskQueueExecutor = taskQueueExecutor; + this.videoQueueExecutor = videoQueueExecutor; this.instanceMetadata = instanceMetadata; this.distributedJobLockService = distributedJobLockService; } @@ -156,7 +156,7 @@ public class ImageVideoAsyncTaskService { .or().eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, "")) .orderByAsc(ImageVideoAsyncTaskEntity::getId) .last("LIMIT " + DISPATCH_BATCH_SIZE)); - tasks.forEach(task -> taskQueueExecutor.execute(() -> executeTask(task.getId()))); + tasks.forEach(task -> videoQueueExecutor.execute(() -> executeTask(task.getId()))); } } @@ -178,7 +178,7 @@ public class ImageVideoAsyncTaskService { .or().eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, "")) .orderByAsc(ImageVideoAsyncTaskEntity::getUpdatedAt) .last("LIMIT " + POLL_BATCH_SIZE)); - tasks.forEach(task -> taskQueueExecutor.execute(() -> pollTask(task.getId()))); + tasks.forEach(task -> videoQueueExecutor.execute(() -> pollTask(task.getId()))); } } @@ -281,7 +281,7 @@ public class ImageVideoAsyncTaskService { taskMapper.insert(task); log.info("[image-video] async task submitted taskId={} type={} owner={}", task.getId(), task.getTaskType(), task.getOwnerInstanceId()); - taskQueueExecutor.execute(() -> executeTask(task.getId())); + videoQueueExecutor.execute(() -> executeTask(task.getId())); return toVo(task); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java index fc3064b7e..a556df344 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java @@ -21,6 +21,7 @@ import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -100,7 +101,12 @@ public class ShopDataCrawlAdminTasksService { * 管理端真删该店这条店铺数据记录(累计档 + 文件 + 明细),仅允许删除已生成文件的终态记录。 * *

S0-6/AC-7:真删比下载更危险,同一把归属尺子必须在动手之前落。 + * + *

S1-B9:四次写(删明细 → 解绑结果文件 → 删成员/档 → 回收对象)必须在同一个 DB 事务里, + * 中途失败不能留半删状态。对象回收交给 {@code deleteResultObjectIfUnreferenced} + * 注册的 afterCommit,事务回滚时不会误删远端文件。 */ + @Transactional(rollbackFor = Exception.class) public void deleteAdminDailyFile(Long dailyFileId, List visibleShopKeys) { if (dailyFileId == null || dailyFileId <= 0) { throw new BusinessException("参数不合法"); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlDailyFileService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlDailyFileService.java index 579492d8e..9da1c16fe 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlDailyFileService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlDailyFileService.java @@ -10,6 +10,7 @@ import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import lombok.RequiredArgsConstructor; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -221,6 +222,8 @@ public class ShopDataCrawlDailyFileService { return count == null ? 0L : count; } + /** S1-B9:删成员 + 删档两次写必须同事务,中途失败不能留"成员没了档还在"的半删状态。 */ + @Transactional(rollbackFor = Exception.class) public void deleteDailyFile(Long dailyFileId) { if (dailyFileId == null || dailyFileId <= 0) { return; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/vo/TaskHeartbeatVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/vo/TaskHeartbeatVo.java index 28eb07ea3..74f08d440 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/vo/TaskHeartbeatVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/vo/TaskHeartbeatVo.java @@ -19,6 +19,16 @@ public class TaskHeartbeatVo { @Schema(description = "提示信息", example = "ok") private String message; + /** + * 进度更新失败但任务仍存活时置 true(S1-B4:组装/提交持锁期间心跳不许降级成"任务不在了")。 + * alive 与 progressDegraded 是两件事:前者是活性,后者只是这次进度没写进去。 + */ + @Schema(description = "进度更新是否已降级(任务仍在运行,本次进度未落库)", example = "false") + private boolean progressDegraded; + + @Schema(description = "降级原因(仅 progressDegraded=true 时非空)") + private String warning; + public static TaskHeartbeatVo alive(String moduleType, String status) { TaskHeartbeatVo vo = new TaskHeartbeatVo(); vo.setAlive(true); @@ -28,6 +38,13 @@ public class TaskHeartbeatVo { return vo; } + public static TaskHeartbeatVo aliveDegraded(String moduleType, String status, String warning) { + TaskHeartbeatVo vo = alive(moduleType, status); + vo.setProgressDegraded(true); + vo.setWarning(warning); + return vo; + } + public static TaskHeartbeatVo notAlive(String moduleType, String status, String message) { TaskHeartbeatVo vo = new TaskHeartbeatVo(); vo.setAlive(false); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java index c71a68589..ac04695fb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java @@ -137,6 +137,11 @@ public class ModuleHistoryCleanupService { List skippedRetainedTaskSamples = new ArrayList<>(LOG_SAMPLE_IDS); while (true) { List page = fileTaskMapper.selectList(new LambdaQueryWrapper() + // S1-B5:批查只取指针列。整实体一批 500 行会把 request_json/result_json/ + // error_message 三只大列全拉进堆,夜间清理足以把堆打满。 + .select(FileTaskEntity::getId, FileTaskEntity::getModuleType, + FileTaskEntity::getStatus, FileTaskEntity::getUpdatedAt, + FileTaskEntity::getFinishedAt) .in(FileTaskEntity::getModuleType, moduleTypes) .gt(FileTaskEntity::getId, cursor) .orderByAsc(FileTaskEntity::getId) @@ -364,12 +369,14 @@ public class ModuleHistoryCleanupService { private List collectPayloadPointers(List moduleTypes, List cleanupTaskIds, int max) { java.util.Set pointers = new java.util.LinkedHashSet<>(); List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .select(TaskChunkEntity::getPayloadJson) .in(TaskChunkEntity::getModuleType, moduleTypes) .in(TaskChunkEntity::getTaskId, cleanupTaskIds)); for (TaskChunkEntity chunk : chunks) { collectPointer(pointers, chunk.getPayloadJson(), max); } List scopeStates = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) .in(TaskScopeStateEntity::getModuleType, moduleTypes) .in(TaskScopeStateEntity::getTaskId, cleanupTaskIds)); for (TaskScopeStateEntity scopeState : scopeStates) { @@ -377,12 +384,14 @@ public class ModuleHistoryCleanupService { collectPointer(pointers, scopeState.getStateJson(), max); } List resultItems = taskResultItemMapper.selectList(new LambdaQueryWrapper() + .select(TaskResultItemEntity::getPayloadJson) .in(TaskResultItemEntity::getModuleType, moduleTypes) .in(TaskResultItemEntity::getTaskId, cleanupTaskIds)); for (TaskResultItemEntity resultItem : resultItems) { collectPointer(pointers, resultItem.getPayloadJson(), max); } List resultPayloads = taskResultPayloadMapper.selectList(new LambdaQueryWrapper() + .select(TaskResultPayloadEntity::getPayloadJson) .in(TaskResultPayloadEntity::getModuleType, moduleTypes) .in(TaskResultPayloadEntity::getTaskId, cleanupTaskIds)); for (TaskResultPayloadEntity resultPayload : resultPayloads) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java index b7865034f..0bc352a3c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java @@ -410,20 +410,36 @@ public class TaskFileJobService { continue; } if ("PENDING".equals(job.getStatus())) { - // PENDING 僵尸兜底:补发 dispatch 事件(worker claim 幂等), - // 并累加 retryCount 防无限重发;注意不翻转状态, - // 避免与真实 PENDING 竞争 worker 的 claim 条件更新冲突。 + // PENDING 僵尸兜底:补发 dispatch 事件(worker claim 幂等),并累加 retryCount + // 防无限重发。S1-B7:计数必须能走到 MAX_RETRY_COUNT 并就地翻 FAILED—— + // 之前 Math.min(MAX_RETRY_COUNT - 1, ...) 把僵尸钉死在 4,而判终态的条件是 + // retry_count >= MAX_RETRY_COUNT,于是每轮重派、永不到终态、终态回调永不触发。 int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount(); - int nextRetryCount = Math.min(MAX_RETRY_COUNT - 1, retryCount + 1); - int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper() + int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1); + boolean exhaustedNow = nextRetryCount >= MAX_RETRY_COUNT; + LocalDateTime now = LocalDateTime.now(); + LambdaUpdateWrapper pendingUpdate = new LambdaUpdateWrapper() .eq(TaskFileJobEntity::getId, job.getId()) .eq(TaskFileJobEntity::getStatus, "PENDING") .eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt()) .isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt) .set(TaskFileJobEntity::getRetryCount, nextRetryCount) - .set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())); + .set(TaskFileJobEntity::getUpdatedAt, now); + if (exhaustedNow) { + pendingUpdate + .set(TaskFileJobEntity::getStatus, "FAILED") + .set(TaskFileJobEntity::getErrorMessage, + "文件生成任务排队未被执行,已达到最大重试次数") + .set(TaskFileJobEntity::getFinishedAt, now) + .set(TaskFileJobEntity::getTerminalCallbackAt, null); + } + int updated = taskFileJobMapper.update(null, pendingUpdate); if (updated > 0) { - eventJobIds.add(job.getId()); + if (exhaustedNow) { + exhaustedJobIds.add(job.getId()); + } else { + eventJobIds.add(job.getId()); + } } continue; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java index ae9489add..7341a1dd7 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java @@ -90,7 +90,11 @@ public class TaskHeartbeatService { } if (fileResult != null && fileResult.isAlive() && brandResult != null && brandResult.isAlive()) { - return TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING); + TaskHeartbeatVo merged = fileResult.isProgressDegraded() + ? TaskHeartbeatVo.aliveDegraded(fileResult.getModuleType() + "," + MODULE_BRAND, + STATUS_RUNNING, fileResult.getWarning()) + : TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING); + return merged; } if (fileResult != null && fileResult.isAlive()) { return fileResult; @@ -206,9 +210,21 @@ public class TaskHeartbeatService { } if (handler != null) { - handler.touchHeartbeat(task.getId(), request); + String degradeWarning = null; + try { + handler.touchHeartbeat(task.getId(), request); + } catch (RuntimeException ex) { + // S1-B4:模块的进度写可能因并发写库/锁超时抛任何运行时异常; + // 心跳表达的是活性,进度写不进去只能降级,绝不能让客户端把 40902 当"任务不在了"。 + degradeWarning = "进度更新失败(已降级,任务仍在运行): " + ex.getMessage(); + log.warn("[task-heartbeat] module progress degraded taskId={} moduleType={} err={}", + task.getId(), moduleType, ex.getMessage()); + } handler.onHeartbeat(task); handler.saveTaskCache(task); + return degradeWarning == null + ? TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING) + : TaskHeartbeatVo.aliveDegraded(moduleType, STATUS_RUNNING, degradeWarning); } return TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java index c3a43042f..012a428f6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java @@ -44,6 +44,13 @@ public class TaskResultFileJobWorker { @Value("${aiimage.result-file-job.local-worker-enabled:true}") private boolean localWorkerEnabled; + /** + * 与 {@code TaskFileJobLocalDispatcher} 读同一键:只有部署方显式关掉本地派发时才允许 + * 退回"调度线程内联执行"这一历史语义;默认状态下派发被拒一律回队。 + */ + @Value("${aiimage.result-file-job.local-dispatch-enabled:true}") + private boolean localDispatchEnabled = true; + @Value("${aiimage.result-file-job.batch-size:20}") private int batchSize; @@ -88,7 +95,17 @@ public class TaskResultFileJobWorker { // 失败,job 被静默丢弃并挂到 stuck 扫描才回收,定时兜底等于失效。 boolean dispatched = taskFileJobLocalDispatcher.dispatchClaimed(job); if (!dispatched) { - processClaimed(job); + if (!localDispatchEnabled) { + // 部署方显式关闭本地派发:保留"调度线程内联执行"的历史语义。 + processClaimed(job); + continue; + } + // S1-B3:派发被拒(在途上限或执行器拒绝)时绝不能内联——那是把分钟级的 + // Excel 组装压回 16 线程的调度池,拖停 stale 修复/视频派发这些兜底任务。 + // 回队后由归属实例下一轮扫描补发,与 process() 通道的处理口径一致。 + log.warn("[task-file-job] local dispatch rejected, requeue for next round jobId={} taskId={} moduleType={} stage=REQUEUE", + job.getId(), job.getTaskId(), job.getModuleType()); + taskFileJobService.requeue(job.getId(), "本地派发被拒,稍后重投"); } } } diff --git a/backend-java/src/main/resources/application-local.example.yml b/backend-java/src/main/resources/application-local.example.yml index 5d2e9cc36..9192007f2 100644 --- a/backend-java/src/main/resources/application-local.example.yml +++ b/backend-java/src/main/resources/application-local.example.yml @@ -109,6 +109,13 @@ AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_ENABLED=false AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS=10000 AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES=5 +# 任务队列(可选,留空即用 application.yml 的出厂默认值): +# 视频/图像 Coze 任务与结果文件组装各自独立许可;优雅停机等待在途任务的秒数。 +AIIMAGE_TASK_QUEUE_VIDEO_MAX_CONCURRENT=6 +AIIMAGE_TASK_QUEUE_VIDEO_MAX_WAITING=1000 +AIIMAGE_TASK_QUEUE_SHUTDOWN_AWAIT_SECONDS=30 +AIIMAGE_DB_DEFAULT_STATEMENT_TIMEOUT_SECONDS=60 + AIIMAGE_ZINIAO_ENABLED=false AIIMAGE_ZINIAO_BASE_URL=https://sbappstoreapi.ziniao.com/openapi-router AIIMAGE_ZINIAO_API_KEY=change-me diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index e34ce5c8b..3e120b84a 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -1,9 +1,14 @@ server: port: ${AIIMAGE_SERVER_PORT:18080} + # S1-B6:发版硬切在途上传/组装的根因。graceful 让 Tomcat 先停止接收新请求、 + # 等在途请求收尾;等待上限由 spring.lifecycle.timeout-per-shutdown-phase 控制。 + shutdown: graceful spring: application: name: aiimage-backend + lifecycle: + timeout-per-shutdown-phase: 30s servlet: multipart: max-file-size: 2GB @@ -82,6 +87,9 @@ mybatis-plus: mapper-locations: classpath*:mapper/**/*.xml configuration: map-underscore-to-camel-case: true + # S1-B8:语句级超时(秒)。socketTimeout=300s + 30 格池 ⇒ 一条慢 SQL 能占死一格连接 5 分钟, + # Tomcat 请求在 30s 取连接处失败。60s 覆盖现网最慢的报表/清理语句,又留足连接回收余量。 + default-statement-timeout: ${AIIMAGE_DB_DEFAULT_STATEMENT_TIMEOUT_SECONDS:60} log-impl: org.apache.ibatis.logging.slf4j.Slf4jImpl springdoc: @@ -382,6 +390,11 @@ aiimage: task-queue: max-concurrent: ${AIIMAGE_TASK_QUEUE_MAX_CONCURRENT:12} max-waiting: ${AIIMAGE_TASK_QUEUE_MAX_WAITING:1000} + # S1-B2:图像/视频(Coze 读超时 1 小时)独立许可数,不与结果文件组装的 12 格共用。 + video-max-concurrent: ${AIIMAGE_TASK_QUEUE_VIDEO_MAX_CONCURRENT:6} + video-max-waiting: ${AIIMAGE_TASK_QUEUE_VIDEO_MAX_WAITING:1000} + # S1-B6:优雅停机时虚拟线程池等待在途任务的秒数(对齐 spring.lifecycle 的 30s)。 + shutdown-await-seconds: ${AIIMAGE_TASK_QUEUE_SHUTDOWN_AWAIT_SECONDS:30} brand-check: base-url: ${AIIMAGE_BRAND_CHECK_BASE_URL:http://47.110.241.161:16890} path: ${AIIMAGE_BRAND_CHECK_PATH:/brand_check} diff --git a/backend-java/src/main/resources/mapper/imagevideo/ImageVideoAdminTaskMapper.xml b/backend-java/src/main/resources/mapper/imagevideo/ImageVideoAdminTaskMapper.xml index 7d2a55c28..a8fecda03 100644 --- a/backend-java/src/main/resources/mapper/imagevideo/ImageVideoAdminTaskMapper.xml +++ b/backend-java/src/main/resources/mapper/imagevideo/ImageVideoAdminTaskMapper.xml @@ -37,7 +37,7 @@ FROM biz_image_video_async_task t LEFT JOIN users u ON u.id = t.user_id WHERE t.task_type = 'IMAGE_VIDEO_WORKFLOW' - AND t.submitted_at &gt;= DATE_SUB(NOW(), INTERVAL 3 DAY) + AND t.submitted_at >= DATE_SUB(NOW(), INTERVAL 3 DAY) AND t.id = #{taskId} LIMIT 1 diff --git a/backend-java/src/test/java/com/nanri/aiimage/common/security/SuperAdminPredicateSingleOutletTest.java b/backend-java/src/test/java/com/nanri/aiimage/common/security/SuperAdminPredicateSingleOutletTest.java index 6b37df943..d1d8d4d90 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/common/security/SuperAdminPredicateSingleOutletTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/common/security/SuperAdminPredicateSingleOutletTest.java @@ -102,7 +102,7 @@ class SuperAdminPredicateSingleOutletTest { } @Test - void superAdminOutletIsOneStaticPredicateAndSeventeenCallSites() { + void superAdminOutletIsOneStaticPredicateAndEighteenCallSites() { int definitions = 0; int callSites = 0; int wrappers = 0; @@ -120,8 +120,10 @@ class SuperAdminPredicateSingleOutletTest { .as("业务层包装 isPlatformSuperAdmin 的定义处(只允许 DataReadScope 一个)") .isEqualTo(1); assertThat(callSites) - .as("走出口的调用点数:16 个业务调用方 + DataReadScope 包装体(新增调用要一起改这个数字,别改回字面量)") - .isEqualTo(17); + // 17 -> 18:v1.1 Phase 1(AC-7 数据读取按归属收口)在 + // AdminShopDataCrawlTasksController:151 新增一处走同一出口的调用,Phase 3 补记。 + .as("走出口的调用点数:17 个业务调用方 + DataReadScope 包装体(新增调用要一起改这个数字,别改回字面量)") + .isEqualTo(18); // 自检:这三把尺子的锚点必须真落在预期的文件里,否则"恰好也是 N"可能是扫错地方凑出来的 assertThat(read(mainRoot().resolve("common/security/DeviceSessionPolicy.java"))) .contains("public static boolean isSuper(String resolvedRole)"); diff --git a/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java b/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java index 7c056cba6..80d1fb4ed 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java @@ -56,7 +56,8 @@ class CatchSilenceRatchetTest { // 逐处记名见 STRICT_TOTAL 上方注释)。 // 10-28 V179 采集被过滤行追溯抬 579 -> 582(审计件三枚:逐批插入容错/回流簿记容错/尾 sheet 降级, // 逐处记名见 STRICT_TOTAL 上方注释)。 - static final int GIVEN_NOT_RETHROW = 582; + // 10-28 Phase 3 抬 582 -> 585(心跳降级一枚 + 优雅停机两枚中断恢复,逐处记名见 STRICT_TOTAL 上方注释)。 + static final int GIVEN_NOT_RETHROW = 585; static final int GIVEN_LOG_ONLY = 249; static final int GIVEN_EMPTY = 56; @@ -199,8 +200,19 @@ class CatchSilenceRatchetTest { * 那只是本地 POI 写,写不动说明整份 workbook 都写不动,该由 * {@code writeWorkbookStreaming} 的既有出口处理,与前两张 sheet 同姿势。 * + * + *

10-28 Phase 3(v1.1 后端任务引擎稳定性)抬 total 1031 -> 1034,三处逐条记名: + *

    + *
  • {@code TaskHeartbeatService.touchFileTaskIfRunning} 一枚:模块进度写抛任何运行时异常 + * (S1-B4 的 40902 就是这一类)都降级成 {@code progressDegraded=true},块体一条 WARN 带 + * taskId/moduleType;不接住就等于心跳继续 500/40902,客户端会当"任务不在了"。
  • + *
  • {@code TaskFileJobConfig.taskQueueVirtualThreadExecutor} 的 {@code shutdown()} 一枚: + * await 期间被中断只恢复中断标记(S1-B6 优雅停机),不吞别的东西。
  • + *
  • {@code TaskFileJobConfig.videoQueueExecutor} 一枚:信号量 acquire 被中断只恢复中断标记 + * (S1-B2 视频独立许可,照抄 taskQueueExecutor 的既有形状)。
  • + *
*/ - static final int STRICT_TOTAL = 1031; + static final int STRICT_TOTAL = 1034; /** * 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。 * 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。 @@ -305,8 +317,13 @@ class CatchSilenceRatchetTest { * {@code CollectDataExcelAssemblyService.writeFilteredSheet} 尾 sheet 降级;三处都是带 * {@code [collect-data]} 锚点的 log.warn+善后,块体除第一枚外各带一句状态置位/返回值, * 逐处记名见 STRICT_TOTAL 上方注释)。 + * 10-28 Phase 3 再抬 582 -> 585(后端任务引擎三枚,均为「异常到此为止」的必要形状): + * {@code TaskHeartbeatService.touchFileTaskIfRunning}(S1-B4 进度写降级成 progressDegraded, + * 块体 log.warn 带 taskId/moduleType)、{@code TaskFileJobConfig.taskQueueVirtualThreadExecutor} + * 的 {@code shutdown()} 与 {@code TaskFileJobConfig.videoQueueExecutor}(S1-B6/S1-B2, + * {@code InterruptedException} 只恢复中断标记),逐处记名见 STRICT_TOTAL 上方注释。 */ - static final int STRICT_NOT_RETHROW = 582; + static final int STRICT_NOT_RETHROW = 585; /** * 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}: * 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出—— diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataProgressThrottleTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataProgressThrottleTest.java index eef502f8b..ed7f27d40 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataProgressThrottleTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataProgressThrottleTest.java @@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -60,9 +61,6 @@ class CollectDataProgressThrottleTest { @Mock private TaskFileJobService taskFileJobService; - @Mock - private TaskDistributedLockService.LockHandle lockHandle; - @Spy private com.fasterxml.jackson.databind.ObjectMapper objectMapper = new com.fasterxml.jackson.databind.ObjectMapper(); @@ -70,6 +68,16 @@ class CollectDataProgressThrottleTest { @InjectMocks private CollectDataService service; + @org.junit.jupiter.api.BeforeAll + static void initializeMybatisMetadata() { + // updateProgress 改无锁短事务后走 LambdaUpdateWrapper 瘦投影写, + // 构造 wrapper 需要 FileTaskEntity 的 lambda 缓存(否则 MybatisPlus 抛"can not find lambda cache")。 + com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo( + new org.apache.ibatis.builder.MapperBuilderAssistant( + new com.baomidou.mybatisplus.core.MybatisConfiguration(), ""), + FileTaskEntity.class); + } + @BeforeEach void setUp() { ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L); @@ -85,8 +93,7 @@ class CollectDataProgressThrottleTest { ReflectionTestUtils.setField(service, "progressDirtyWindowMillis", 60_000L); FileTaskEntity task = runningTask(1L); when(fileTaskMapper.selectById(1L)).thenReturn(task); - when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 1L, 5_000L)) - .thenReturn(lockHandle); + when(fileTaskMapper.update(isNull(), any())).thenReturn(1); TaskHeartbeatRequest first = new TaskHeartbeatRequest(); first.setCurrent(1); @@ -99,7 +106,7 @@ class CollectDataProgressThrottleTest { // 窗口内合并:2 次心跳只落库 1 次(首条,processedRows=1); // 窗口到期后补写最新值(processedRows=2),进度不丢失。 - verify(fileTaskMapper, times(1)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(1)).update(isNull(), any()); assertThat(task.getResultJson()).contains("\"totalRows\":10"); assertThat(task.getResultJson()).contains("\"processedRows\":1"); @@ -109,7 +116,7 @@ class CollectDataProgressThrottleTest { third.setCurrent(2); third.setTotal(10); service.updateProgress(1L, third); - verify(fileTaskMapper, times(2)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(2)).update(isNull(), any()); assertThat(task.getResultJson()).contains("\"processedRows\":2"); } @@ -123,10 +130,7 @@ class CollectDataProgressThrottleTest { FileTaskEntity taskB = runningTask(12L); when(fileTaskMapper.selectById(11L)).thenReturn(taskA); when(fileTaskMapper.selectById(12L)).thenReturn(taskB); - when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 11L, 5_000L)) - .thenReturn(lockHandle); - when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 12L, 5_000L)) - .thenReturn(lockHandle); + when(fileTaskMapper.update(isNull(), any())).thenReturn(1); heartbeat(service, 11L, 1, 10); heartbeat(service, 12L, 1, 10); @@ -134,20 +138,20 @@ class CollectDataProgressThrottleTest { heartbeat(service, 12L, 2, 10); // 每支任务首条 changed 心跳各落库 1 次,其后的窗口内心跳合并写 → 共 2 次。 - verify(fileTaskMapper, times(2)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(2)).update(isNull(), any()); assertThat(taskA.getResultJson()).contains("\"processedRows\":1"); assertThat(taskB.getResultJson()).contains("\"processedRows\":1"); // A 自身窗口到期后补写最新值;B 仍在自己窗口内,继续被合并。 expireProgressWindow(service, taskA); heartbeat(service, 11L, 3, 10); - verify(fileTaskMapper, times(3)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(3)).update(isNull(), any()); assertThat(taskA.getResultJson()).contains("\"processedRows\":3"); // B 自身窗口到期后同样补写,两任务最终进度均为最新值。 expireProgressWindow(service, taskB); heartbeat(service, 12L, 3, 10); - verify(fileTaskMapper, times(4)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(4)).update(isNull(), any()); assertThat(taskB.getResultJson()).contains("\"processedRows\":3"); } @@ -164,22 +168,24 @@ class CollectDataProgressThrottleTest { stuck.setUpdatedAt(LocalDateTime.now().minusMinutes(20)); when(fileTaskMapper.selectById(71L)).thenReturn(busy); when(fileTaskMapper.selectById(72L)).thenReturn(stuck); - when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 71L, 5_000L)) - .thenReturn(lockHandle); - when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 72L, 5_000L)) - .thenReturn(lockHandle); + when(fileTaskMapper.update(isNull(), any())).thenReturn(1); // 邻居任务连续刷了 5 次,把「全局窗口」反复重置——旧实现正是死在这里。 for (int i = 1; i <= 5; i++) { expireProgressWindow(service, busy); // 让它每次都够格落库 heartbeat(service, 71L, i, 10); } - assertThat(busy.getUpdatedAt()).isAfter(LocalDateTime.now().minusSeconds(5)); + // 只看卡住任务自己的落库:它内容不变、但自身 updated_at 已陈旧 ⇒ 每条心跳都必须 + // 落库把 updated_at 顶上去(无锁实现下 updated_at 由 DB 写刷新,不再改内存实体)。 + org.mockito.Mockito.clearInvocations(fileTaskMapper); + heartbeat(service, 72L, 2, 8); + heartbeat(service, 72L, 2, 8); + verify(fileTaskMapper, times(2)).update(isNull(), any()); - // 内容完全相同的重复心跳,仍必须靠自身脏窗口把 updated_at 顶上去。 + // 反证:同一任务 updated_at 新鲜、且刚写过(窗口内)时,内容不变的心跳不再落库。 + stuck.setUpdatedAt(LocalDateTime.now()); heartbeat(service, 72L, 2, 8); - heartbeat(service, 72L, 2, 8); - assertThat(stuck.getUpdatedAt()).isAfter(LocalDateTime.now().minusSeconds(5)); + verify(fileTaskMapper, times(2)).update(isNull(), any()); } @Test @@ -190,26 +196,25 @@ class CollectDataProgressThrottleTest { ReflectionTestUtils.setField(service, "progressDirtyWindowMillis", 60_000L); FileTaskEntity task = runningTask(21L); when(fileTaskMapper.selectById(21L)).thenReturn(task); - when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 21L, 5_000L)) - .thenReturn(lockHandle); + when(fileTaskMapper.update(isNull(), any())).thenReturn(1); heartbeat(service, 21L, 1, 10); heartbeat(service, 21L, 1, 10); heartbeat(service, 21L, 1, 10); // 内容相同:仅首次心跳落库一次,重复心跳零 UPDATE。 - verify(fileTaskMapper, times(1)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(1)).update(isNull(), any()); } @Test void test_task_055_merge_progress_boundary_empty_input() { - // 空输入:null taskId / 空请求 → 直接返回,不取锁不查询不落库。 + // 空输入:null taskId / 空请求 → 直接返回,不取锁(AC-10 后本路径已无锁)、不查询、不落库。 service.updateProgress(null, new TaskHeartbeatRequest()); service.updateProgress(1L, null); verify(taskDistributedLockService, never()).acquire(any(), any(), anyLong()); verify(fileTaskMapper, never()).selectById(any(java.io.Serializable.class)); - verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, never()).update(any(), any()); } @Test @@ -219,8 +224,7 @@ class CollectDataProgressThrottleTest { ReflectionTestUtils.setField(service, "progressDirtyWindowMillis", 60_000L); FileTaskEntity task = runningTask(31L); when(fileTaskMapper.selectById(31L)).thenReturn(task); - when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 31L, 5_000L)) - .thenReturn(lockHandle); + when(fileTaskMapper.update(isNull(), any())).thenReturn(1); heartbeat(service, 31L, 3, 20); heartbeat(service, 31L, 3, 20); @@ -228,12 +232,12 @@ class CollectDataProgressThrottleTest { // 首条 changed 心跳落库一次(totalRows=20, processedRows=3);后续窗口内心跳合并写。 assertThat(task.getResultJson()).contains("\"totalRows\":20"); - verify(fileTaskMapper, times(1)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(1)).update(isNull(), any()); // 窗口到期后补写最新值(processedRows=4),不丢失。 expireProgressWindow(service, task); heartbeat(service, 31L, 4, 21); - verify(fileTaskMapper, times(2)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(2)).update(isNull(), any()); assertThat(task.getResultJson()).contains("\"processedRows\":4"); } @@ -245,19 +249,18 @@ class CollectDataProgressThrottleTest { ReflectionTestUtils.setField(service, "progressDirtyWindowMillis", 60_000L); FileTaskEntity task = runningTask(41L); when(fileTaskMapper.selectById(41L)).thenReturn(task); - when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 41L, 5_000L)) - .thenReturn(lockHandle); + when(fileTaskMapper.update(isNull(), any())).thenReturn(1); for (int i = 1; i <= 100; i++) { heartbeat(service, 41L, i, 100); } // 窗口内 100 次心跳只落库 1 次,无无界 UPDATE 累积;窗口到期后补写最新值。 - verify(fileTaskMapper, times(1)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(1)).update(isNull(), any()); assertThat(task.getResultJson()).contains("\"processedRows\":1"); expireProgressWindow(service, task); heartbeat(service, 41L, 100, 100); - verify(fileTaskMapper, times(2)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(2)).update(isNull(), any()); assertThat(task.getResultJson()).contains("\"processedRows\":100"); } @@ -271,15 +274,11 @@ class CollectDataProgressThrottleTest { finished.setStatus("SUCCESS"); when(fileTaskMapper.selectById(51L)).thenReturn(wrongModule); when(fileTaskMapper.selectById(52L)).thenReturn(finished); - when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 51L, 5_000L)) - .thenReturn(lockHandle); - when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 52L, 5_000L)) - .thenReturn(lockHandle); heartbeat(service, 51L, 1, 10); heartbeat(service, 52L, 1, 10); - verify(fileTaskMapper, never()).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, never()).update(any(), any()); } @Test @@ -290,34 +289,29 @@ class CollectDataProgressThrottleTest { ReflectionTestUtils.setField(service, "progressDirtyWindowMillis", 60_000L); FileTaskEntity task = runningTask(61L); when(fileTaskMapper.selectById(61L)).thenReturn(task); - when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 61L, 5_000L)) - .thenReturn(lockHandle); AtomicInteger updateCalls = new AtomicInteger(); - when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenAnswer(invocation -> { + when(fileTaskMapper.update(isNull(), any())).thenAnswer(invocation -> { if (updateCalls.getAndIncrement() == 0) { throw new RuntimeException("db down"); } return 1; }); - // DB 写失败:异常向上传播(try-with-resources 关闭锁),不吞错不残留。 + // DB 写失败:异常向上传播,不吞错;AC-10 之后本路径不再取任务锁(无可释放的锁)。 assertThatThrownBy(() -> heartbeat(service, 61L, 1, 10)) .isInstanceOf(RuntimeException.class) .hasMessage("db down"); - verify(lockHandle, times(1)).close(); + verify(taskDistributedLockService, never()).acquire(any(), any(), anyLong()); // 恢复后心跳重试:落库一次成功,进度持久化。 - // 节流时钟只在 updateById 返回后才推进,所以那次失败的写没有留下窗口;updated_at 同理 - // 停在库里那一版。真实链路每条心跳都在 :530 重新 selectById ⇒ 会立刻补刷。本用例的 - // mock 复用同一个实体实例,故把 updated_at 退回「库里那一版」——否则测的是对象别名, - // 不是这条重读语义。 + // 节流时钟只在写成功后才推进,所以那次失败的写没有留下窗口;真实链路每条心跳都会 + // 重新 selectById ⇒ 会立刻补刷。本用例的 mock 复用同一个实体实例,故把 updated_at + // 退回「库里那一版」——否则测的是对象别名,不是这条重读语义。 task.setUpdatedAt(null); heartbeat(service, 61L, 2, 10); - verify(fileTaskMapper, times(2)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(2)).update(isNull(), any()); assertThat(task.getResultJson()).contains("\"processedRows\":2"); - - verify(lockHandle, times(2)).close(); } @Test @@ -331,13 +325,12 @@ class CollectDataProgressThrottleTest { ReflectionTestUtils.setField(service, "progressDirtyWindowMillis", 30_000L); FileTaskEntity task = runningTask(81L); when(fileTaskMapper.selectById(81L)).thenReturn(task); - when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, 81L, 5_000L)) - .thenReturn(lockHandle); + when(fileTaskMapper.update(isNull(), any())).thenReturn(1); // 决定性的一步:updated_at 就是 now,旧实现在这里直接 return,进度永远停在 0。 task.setUpdatedAt(LocalDateTime.now()); heartbeat(service, 81L, 1, 7); - verify(fileTaskMapper, times(1)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(1)).update(isNull(), any()); assertThat(task.getResultJson()).contains("\"processedRows\":1"); // 客户端按 60s 一跳,逐关键词推进:每跳进来时 updated_at 都已被心跳层顶到 now, @@ -347,7 +340,7 @@ class CollectDataProgressThrottleTest { expireProgressWindow(service, task); heartbeat(service, 81L, keyword, 7); } - verify(fileTaskMapper, times(7)).updateById(any(FileTaskEntity.class)); + verify(fileTaskMapper, times(7)).update(isNull(), any()); assertThat(task.getResultJson()).contains("\"processedRows\":7"); assertThat(task.getResultJson()).contains("\"totalRows\":7"); } @@ -359,6 +352,9 @@ class CollectDataProgressThrottleTest { task.setModuleType(CollectDataService.MODULE_TYPE); task.setStatus("RUNNING"); task.setResultJson("{}"); + // 真实链路每条心跳都重新 selectById,updated_at 是库里那一版(AC-10 后心跳只写 DB、 + // 不再回改内存实体)。这里给个新鲜的 updated_at,等价于"库里刚被刷过"的读入形状。 + task.setUpdatedAt(LocalDateTime.now()); return task; } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/object/FaultInjectionTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/object/FaultInjectionTest.java index 376a8d346..3e9e445df 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/object/FaultInjectionTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/file/service/object/FaultInjectionTest.java @@ -236,7 +236,8 @@ class FaultInjectionTest { // 第一次提交被执行器拒绝:等待槽释放、拒绝指标记录 assertThrows(TaskRejectedException.class, () -> gate.execute(task)); assertEquals(0, gate.waiting(), "执行器拒绝后等待槽必须释放"); - assertTrue(registry.counter("aiimage.task-queue.submit.rejected.total", "reason", "delegate-rejected").count() > 0); + assertTrue(registry.find("aiimage.task-queue.submit.rejected.total") + .tag("reason", "delegate-rejected").counter().count() > 0); // 第二次提交恢复成功:等待槽正常占用并释放 gate.execute(task); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3CleanupStreamingTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3CleanupStreamingTest.java index 1d5ca2d23..729241320 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3CleanupStreamingTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3CleanupStreamingTest.java @@ -1,7 +1,6 @@ package com.nanri.aiimage.modules.task.service; import com.nanri.aiimage.testutil.SourceScan; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.core.env.PropertySource; @@ -34,7 +33,6 @@ import static org.assertj.core.api.Assertions.assertThat; *

蓝图 §3.2 登记的收口形状:{@code server.shutdown=graceful} + * {@code spring.lifecycle.timeout-per-shutdown-phase=30s} + 统一 {@code @PreDestroy}。 */ -@Disabled("Pending Phase 3: AC-11 清理流式化与优雅停机") @DisplayName("AC-11 清理只读指针 + 优雅停机") class PendingPhase3CleanupStreamingTest { @@ -63,12 +61,15 @@ class PendingPhase3CleanupStreamingTest { long selects = java.util.Arrays.stream(body.split("selectList")) .filter(part -> part.contains(".select(")) .count(); + // 预置时按审计 S1-B5 原文点名的三路写死 3;实现阶段复核发现第 4 路 + // (result_payload.payloadJson,:385 那条)同样是整实体取回、同样要瘦投影, + // 漏掉它等于漏掉"行删了指针就永久丢失"的一种对象。判据按实情抬到 4(不是放宽)。 assertThat(body.split("selectList").length - 1) - .as("chunk/scopeState/resultItem 三条查询都要瘦投影") - .isEqualTo(3); + .as("chunk/scopeState/resultItem/resultPayload 四条查询都要瘦投影") + .isEqualTo(4); assertThat(selects) - .as("S1-B5:三条 selectList 全部带 .select(...) 投影才算流式") - .isEqualTo(3L); + .as("S1-B5:四条 selectList 全部带 .select(...) 投影才算流式") + .isEqualTo(4L); } @Test diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3HeartbeatDegradationTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3HeartbeatDegradationTest.java index 91d13287b..6fa5e5f0e 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3HeartbeatDegradationTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3HeartbeatDegradationTest.java @@ -16,7 +16,6 @@ import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; import com.nanri.aiimage.testutil.SourceScan; import org.apache.ibatis.builder.MapperBuilderAssistant; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -47,7 +46,6 @@ import static org.mockito.Mockito.when; * 蓝图 §3.1 定的收口形状是:进度失败仍返回 200、{@code alive=true},新增 * {@code progressDegraded=true} 与 {@code warning},只有任务终态才 {@code alive=false}。 */ -@Disabled("Pending Phase 3: AC-10 心跳去锁与 SPI 异常降级") @DisplayName("AC-10 心跳在持锁窗口内必须仍返回 alive") class PendingPhase3HeartbeatDegradationTest { @@ -84,6 +82,9 @@ class PendingPhase3HeartbeatDegradationTest { org.mockito.Mockito.doThrow(new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理,请稍后再试")) .when(collectDataService).updateProgress(any(), any()); when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task); + // harness 补齐(预置阶段漏打桩):TaskHeartbeatService 会先刷 updated_at 检查点, + // 该条件更新返回 0 时按"任务已不 RUNNING"提前返回,走不到 SPI。这里让它命中。 + when(fileTaskMapper.update(any(), any())).thenReturn(1); TaskHeartbeatService service = serviceWith(new CollectDataTaskHeartbeatSpi(collectDataService)); TaskHeartbeatRequest request = new TaskHeartbeatRequest(); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3JobTerminalTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3JobTerminalTest.java index 1643c2779..79b929dc4 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3JobTerminalTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3JobTerminalTest.java @@ -9,7 +9,6 @@ import com.nanri.aiimage.testutil.MapperXmlSupport; import com.nanri.aiimage.testutil.SourceScan; import org.apache.ibatis.builder.MapperBuilderAssistant; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -57,7 +56,6 @@ import static org.mockito.Mockito.when; * 的 {@code deleteDailyFile}(删成员 + 删档)同样两次写无事务 ⇒ 中途失败留半删状态。 * */ -@Disabled("Pending Phase 3: AC-12 PENDING 封顶/视频详情 SQL/statement timeout/删除事务") @DisplayName("AC-12 终态封顶与 DB 超时契约") class PendingPhase3JobTerminalTest { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3QueueIsolationTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3QueueIsolationTest.java index e87d7b9ae..43a30540f 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3QueueIsolationTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/PendingPhase3QueueIsolationTest.java @@ -4,7 +4,6 @@ import com.nanri.aiimage.config.InstanceMetadata; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; import com.nanri.aiimage.testutil.SourceScan; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.test.util.ReflectionTestUtils; @@ -46,7 +45,6 @@ import static org.mockito.Mockito.when; * {@code aiimage.result-file-job.local-worker-delay-ms} 默认 15000(现网 yml 覆盖为 10000), * 本用例判的是"一轮之内被拒的作业必须回到队列",不依赖具体毫秒数。 */ -@Disabled("Pending Phase 3: AC-9 视频/结果文件队列隔离与拒绝回队") @DisplayName("AC-9 队列隔离与背压回队") class PendingPhase3QueueIsolationTest { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOwnerRoutingTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOwnerRoutingTest.java index d0dcbe3b9..fbfb3607b 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOwnerRoutingTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOwnerRoutingTest.java @@ -126,6 +126,9 @@ class TaskResultFileJobWorkerOwnerRoutingTest { when(taskFileJobService.claimRunnableJobsForOwner(20, "instance-a")).thenReturn(List.of(claim)); when(taskFileJobService.activateRunningClaim(claim)).thenReturn(true); when(localDispatcher.dispatchClaimed(claim)).thenReturn(false); + // 本用例描述的是"本地派发被显式关闭"的部署形态:此时 worker 才允许在调度线程内联执行; + // 默认形态下派发被拒一律回队(见 PendingPhase3QueueIsolationTest)。 + set(worker, "localDispatchEnabled", false); when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, claim.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock); diff --git a/backend-java/src/test/resources/sql-layering/migrated-statements.tsv b/backend-java/src/test/resources/sql-layering/migrated-statements.tsv index 3cf0ce3a7..4615c32b8 100644 --- a/backend-java/src/test/resources/sql-layering/migrated-statements.tsv +++ b/backend-java/src/test/resources/sql-layering/migrated-statements.tsv @@ -8,7 +8,7 @@ DeviceLogFileMapper deleteOlderThanBatch Delete PURE DELETE FROM device_log_file FileResultMapper selectRowsUpdatedAtByTaskIds Select PURE FileTaskMapper selectExpiredTerminalTaskIds Select PURE SELECT id FROM biz_file_task\nWHERE module_type = #{moduleType}\n AND status IN ('SUCCESS', 'FAILED', 'CANCELLED', 'CANCELED')\n AND updated_at < #{cutoff}\nORDER BY id ASC\nLIMIT #{batchSize} ImageVideoAdminTaskMapper countAdminTaskPage Select PURE -ImageVideoAdminTaskMapper selectAdminTaskDetail Select PURE SELECT t.id, t.user_id, t.status, t.request_json, t.submit_response_json, t.result_json,\n t.video_urls_json, t.debug_url, t.archived_videos_json, t.archive_status,\n t.archive_error, t.archive_attempt_count,\n DATE_FORMAT(t.archived_at, '%Y-%m-%d %H:%i:%s') AS archived_at,\n t.error_message, t.coze_execute_id, t.coze_status,\n DATE_FORMAT(t.submitted_at, '%Y-%m-%d %H:%i:%s') AS submitted_at,\n DATE_FORMAT(t.completed_at, '%Y-%m-%d %H:%i:%s') AS completed_at,\n u.username,\n COALESCE((\n SELECT GROUP_CONCAT(DISTINCT g.group_name ORDER BY g.id SEPARATOR '、')\n FROM biz_shop_manage_group g\n LEFT JOIN biz_shop_manage_group_member gm ON gm.group_id = g.id\n WHERE g.created_by_id = t.user_id OR g.user_id = t.user_id OR gm.user_id = t.user_id\n ), '') AS group_name\nFROM biz_image_video_async_task t\nLEFT JOIN users u ON u.id = t.user_id\nWHERE t.task_type = 'IMAGE_VIDEO_WORKFLOW'\n AND t.submitted_at >= DATE_SUB(NOW(), INTERVAL 3 DAY)\n AND t.id = #{taskId}\nLIMIT 1 +ImageVideoAdminTaskMapper selectAdminTaskDetail Select PURE SELECT t.id, t.user_id, t.status, t.request_json, t.submit_response_json, t.result_json,\n t.video_urls_json, t.debug_url, t.archived_videos_json, t.archive_status,\n t.archive_error, t.archive_attempt_count,\n DATE_FORMAT(t.archived_at, '%Y-%m-%d %H:%i:%s') AS archived_at,\n t.error_message, t.coze_execute_id, t.coze_status,\n DATE_FORMAT(t.submitted_at, '%Y-%m-%d %H:%i:%s') AS submitted_at,\n DATE_FORMAT(t.completed_at, '%Y-%m-%d %H:%i:%s') AS completed_at,\n u.username,\n COALESCE((\n SELECT GROUP_CONCAT(DISTINCT g.group_name ORDER BY g.id SEPARATOR '、')\n FROM biz_shop_manage_group g\n LEFT JOIN biz_shop_manage_group_member gm ON gm.group_id = g.id\n WHERE g.created_by_id = t.user_id OR g.user_id = t.user_id OR gm.user_id = t.user_id\n ), '') AS group_name\nFROM biz_image_video_async_task t\nLEFT JOIN users u ON u.id = t.user_id\nWHERE t.task_type = 'IMAGE_VIDEO_WORKFLOW'\n AND t.submitted_at >= DATE_SUB(NOW(), INTERVAL 3 DAY)\n AND t.id = #{taskId}\nLIMIT 1 ImageVideoAdminTaskMapper selectAdminTaskPage Select PURE ImageVideoAdminTaskMapper selectAdminZipRows Select PURE ImageVideoAsyncTaskMapper claimPending Update PURE UPDATE biz_image_video_async_task SET status = 'RUNNING', owner_instance_id = CASE WHEN owner_instance_id IS NULL OR owner_instance_id = '' THEN #{ownerInstanceId} ELSE owner_instance_id END, updated_at = NOW(), attempt_count = attempt_count + 1 WHERE id = #{taskId} AND status = 'PENDING' AND (owner_instance_id = #{ownerInstanceId} OR owner_instance_id IS NULL OR owner_instance_id = '')