From 228d481211c5290cbdd069885b3e60d14310e99b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Tue, 15 Sep 2026 23:01:59 +0800 Subject: [PATCH] =?UTF-8?q?perf(=E5=85=A8=E9=93=BE=E8=B7=AF):=20=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E6=B1=A0/=E4=BA=8B=E5=8A=A1=E8=BE=B9=E7=95=8C/?= =?UTF-8?q?=E8=BD=AE=E8=AF=A2=E4=B8=8E=20IO=20=E6=95=88=E7=8E=87=E6=94=B6?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Java： - LLM 180s 读超时不再被全局 call-timeout 静默截断成 90s（长思考请求被掐断→重试→付费网关二次计费） - 代理 HttpClient 缓存改有界 LRU（jikip 每次提取新 IP，无界缓存持续泄漏 selector 线程与连接池） - 12 个 service 的 Redis 任务锁移出 @Transactional（自旋最坏 10s 白占 DB 连接，池仅 30），远端对象删/传改 afterCommit - 结果文件 Job 闸门拒绝时不再回退内联执行（改重新入队，避免把背压转嫁给 MQ 消费线程） - imagevideo 每秒扫描加列投影、过期清理加 LIMIT；权限页整表查询改列投影（不再拉回密码哈希） - 哈希改 HexFormat；补 5 处"不能改"的技术依据注释（批量插入会丢回填主键、流式丢模板与图片等） 前端：4 个工具页轮询改轻量端点（带 fallback）；PriceTrack 快照节流写盘；候选店铺表分页；页面隐藏时停表 客户端：HTTP 连接池按出口复用（Session 仍每请求新建，保持无跨请求状态）；品牌检测 WIPO 逐请求握手； 代理配置按 mtime 缓存；串行任务改专属池；异常降级为标签页重连；紫鸟启动改端口轮询；模板编译缓存； 日志上报连接与落盘收口；Flask 版本 API 改按请求复用连接 --- .../aiimage/common/util/BoundedLruCache.java | 70 +++++++ .../nanri/aiimage/config/HttpClientPool.java | 16 +- .../service/AppearancePatentTaskService.java | 73 ++++++- .../service/BrandTaskStorageService.java | 74 +++++-- .../service/CollectDataService.java | 123 ++++++------ .../DeleteBrandTaskStorageService.java | 72 +++++-- .../service/DigitalHumanVersionService.java | 41 +++- .../service/ImageVideoAsyncTaskService.java | 10 +- .../service/PatrolDeleteTaskService.java | 73 +++++-- .../service/PermissionMenuService.java | 26 ++- .../service/PriceTrackTaskService.java | 181 ++++++++++++------ .../service/ProductRiskTaskService.java | 104 ++++++---- .../publish/service/PublishTaskService.java | 101 +++++++++- .../service/QueryAsinTaskService.java | 57 ++++-- .../ShopDataCrawlExcelAssemblyService.java | 7 + .../service/ShopDataCrawlTaskService.java | 118 ++++++++---- .../service/ShopMatchTaskService.java | 137 ++++++++----- .../service/SimilarAsinTaskService.java | 14 +- .../support/SimilarAsinPipelineHost.java | 3 +- .../task/service/TaskResultFileJobWorker.java | 12 +- .../task/service/TaskResultItemService.java | 8 + .../TaskScopePayloadStorageService.java | 7 +- .../withdraw/service/WithdrawTaskService.java | 57 ++++-- .../ziniao/client/ZiniaoClientImpl.java | 19 +- .../ziniao/service/ZiniaoAuthService.java | 7 +- .../service/ZiniaoShopIndexService.java | 7 +- .../ImageVideoAsyncTaskServiceTest.java | 16 ++ .../service/PermissionMenuServiceTest.java | 16 ++ .../QueryAsinSnapshotJsonThrottleTest.java | 5 +- ...hopDataCrawlTaskServiceTxBoundaryTest.java | 35 +++- .../TaskResultFileJobWorkerOffloadTest.java | 14 +- .../TaskResultFileJobWorkerOrphanTest.java | 10 +- backend/app.py | 5 +- backend/utils/db.py | 86 ++++++++- .../pages/brand/components/BrandBrandTab.vue | 38 ++-- .../brand/components/BrandCollectDataTab.vue | 69 ++++++- .../brand/components/BrandPatrolDeleteTab.vue | 17 +- .../brand/components/BrandPriceTrackTab.vue | 87 +++++++-- .../brand/components/BrandQueryAsinTab.vue | 17 +- .../components/BrandShopDataCrawlTab.vue | 12 +- .../brand/components/BrandShopMatchTab.vue | 3 +- .../brand/components/BrandWithdrawTab.vue | 17 +- .../shared/composables/useTaskProgressLoop.ts | 67 +++++-- frontend-vue/tests/polling-visibility.test.ts | 42 +++- 44 files changed, 1550 insertions(+), 423 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/common/util/BoundedLruCache.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/util/BoundedLruCache.java b/backend-java/src/main/java/com/nanri/aiimage/common/util/BoundedLruCache.java new file mode 100644 index 00000000..30fed4ff --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/util/BoundedLruCache.java @@ -0,0 +1,70 @@ +package com.nanri.aiimage.common.util; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.Function; + +/** + * 有界 LRU 缓存：容量超限时自动淘汰最久未使用的条目。 + * + *

用于按「外部端点」缓存长生命周期资源（HttpClient / RestClient）。这类资源各自持有 + * 连接池与 selector 线程，无界累积会持续泄漏线程与内存：代理端点每次提取往往是新的 + * IP:port（jikip 提取），无上限的缓存只增不减。 + * + *

淘汰时只从缓存移除引用，不做显式关闭：JDK 的 HttpClientImpl 注册了 Cleaner， + * 对象不可达后由 GC 回收并关闭其 selector 线程；显式关闭反而可能打断仍在途的请求。 + */ +public final class BoundedLruCache { + + /** 默认容量：代理端点数量级远小于此，足够覆盖热点端点又不至于累积。 */ + public static final int DEFAULT_MAX_SIZE = 64; + + private final int maxSize; + private final Map store; + + public BoundedLruCache() { + this(DEFAULT_MAX_SIZE); + } + + public BoundedLruCache(int maxSize) { + this.maxSize = Math.max(1, maxSize); + // accessOrder=true 使 get 也刷新顺序（真正的 LRU）；synchronizedMap 保证其线程安全 + this.store = Collections.synchronizedMap(new LinkedHashMap<>(16, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > BoundedLruCache.this.maxSize; + } + }); + } + + /** + * 取缓存值，缺失时用 loader 计算并放入。 + * + *

与 {@code ConcurrentHashMap.computeIfAbsent} 不同，此处不保证 loader 的原子性： + * 并发首次访问同一 key 时可能各自构造一次，随后其中一个覆盖另一个。对 + * HttpClient/RestClient 这类构造廉价且幂等的资源可接受，换来的是锁粒度更小。 + */ + public V computeIfAbsent(K key, Function loader) { + V existing = store.get(key); + if (existing != null) { + return existing; + } + V created = loader.apply(key); + store.put(key, created); + return created; + } + + public int size() { + return store.size(); + } + + /** 当前容量上限，供日志与测试断言使用。 */ + public int maxSize() { + return maxSize; + } + + public void clear() { + store.clear(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/HttpClientPool.java b/backend-java/src/main/java/com/nanri/aiimage/config/HttpClientPool.java index be13e670..7bcddb41 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/HttpClientPool.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/HttpClientPool.java @@ -1,5 +1,7 @@ package com.nanri.aiimage.config; +import com.nanri.aiimage.common.util.BoundedLruCache; +import lombok.extern.slf4j.Slf4j; import org.springframework.http.client.ClientHttpRequestFactory; import org.springframework.http.client.JdkClientHttpRequestFactory; @@ -14,8 +16,6 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.time.Duration; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; /** * Task 77：外部 HTTP 客户端统一连接复用池。 @@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentHashMap; * 每次请求都重新建连。各客户端按自身超时创建独立的 * JdkClientHttpRequestFactory（共享底层连接池），RestClient 单例懒加载。 */ +@Slf4j public class HttpClientPool { private static volatile HttpClient sharedHttpClient; @@ -103,8 +104,12 @@ public class HttpClientPool { public static ClientHttpRequestFactory requestFactory(int readTimeoutMillis, String proxyUrl) { long safeReadTimeout = Math.max(1L, readTimeoutMillis); long callTimeout = configuredCallTimeoutMillis; - if (callTimeout > 0L) { - safeReadTimeout = Math.min(safeReadTimeout, callTimeout); + if (callTimeout > 0L && safeReadTimeout > callTimeout) { + // 读超时以调用方显式值为准，不再被全局 call-timeout 截断： + // LLM 长思考配的是 180s（llm-read-timeout-millis），曾被静默压到 90s， + // 导致请求在 90s 被掐断 → 上层重试 → 付费网关二次计费（2026-09-15 修复）。 + // 各调用方的超时已由各自的 HttpConfigResolver 钳制，此处不再二次收敛。 + log.debug("读超时 {}ms 超过全局 call-timeout {}ms，按调用方显式值生效", safeReadTimeout, callTimeout); } JdkClientHttpRequestFactory factory = new JdkClientHttpRequestFactory(httpClientFor(proxyUrl)); @@ -166,5 +171,6 @@ public class HttpClientPool { private record ProxyEndpoint(String host, int port, String userInfo) { } - private static final Map PROXY_CLIENTS = new ConcurrentHashMap<>(); + private static final BoundedLruCache PROXY_CLIENTS = + new BoundedLruCache<>(BoundedLruCache.DEFAULT_MAX_SIZE); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index ac91a970..75619146 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -69,6 +69,8 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; import org.springframework.transaction.support.TransactionTemplate; import java.io.File; @@ -522,20 +524,28 @@ public class AppearancePatentTaskService { scheduleLlmPipelineForSubmittedChunk(context); } + /** + * 删除任务。 + * + *

事务边界：远端载荷删除与缓存清理移到事务提交后执行 —— 单次删除超时 120s、重试 3 次， + * 放在 DB 事务里会让连接被长时间占用（项目在 TaskScopePayloadStorageService 已写明该约束）。 + */ @Transactional public void deleteTask(Long taskId, Long userId) { FileTaskEntity task = fileTaskMapper.selectById(taskId); if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) { throw new BusinessException("任务不存在"); } + List payloads = collectTransientTaskPayloads(taskId); fileResultMapper.delete(new LambdaQueryWrapper().eq(FileResultEntity::getTaskId, taskId).eq(FileResultEntity::getModuleType, MODULE_TYPE)); - deleteTransientTaskPayloads(taskId); taskScopeStateMapper.delete(new LambdaQueryWrapper().eq(TaskScopeStateEntity::getTaskId, taskId).eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); taskChunkMapper.delete(new LambdaQueryWrapper().eq(TaskChunkEntity::getTaskId, taskId).eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); - taskCacheService.deleteTaskCache(taskId); // 同步清理 task_file_job，避免被删除任务遗留的 PENDING/FAILED 行被 TaskResultFileJobWorker 反复扫描出 task not found。 taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); fileTaskMapper.deleteById(taskId); + // 事务提交后再做远端删除与缓存清理 + deletePayloadsAfterCommit(payloads, taskId); + runAfterCommit(() -> taskCacheService.deleteTaskCache(taskId)); } public void deleteHistory(Long resultId, Long userId) { @@ -2992,18 +3002,27 @@ public class AppearancePatentTaskService { } } - private void deleteTransientTaskPayloads(Long taskId) { + /** 只读收集任务范围/分片载荷指针，供事务提交后做远端删除。 */ + private List collectTransientTaskPayloads(Long taskId) { if (taskId == null || taskId <= 0) { - return; + return List.of(); } + List payloads = new ArrayList<>(); List scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper() .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) .eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); if (scopes != null) { for (TaskScopeStateEntity scope : scopes) { - transientPayloadStorageService.deletePayloadIfPresent(scope.getParsedPayloadJson()); - transientPayloadStorageService.deletePayloadIfPresent(scope.getStateJson()); + if (scope == null) { + continue; + } + if (scope.getParsedPayloadJson() != null && !scope.getParsedPayloadJson().isBlank()) { + payloads.add(scope.getParsedPayloadJson()); + } + if (scope.getStateJson() != null && !scope.getStateJson().isBlank()) { + payloads.add(scope.getStateJson()); + } } } List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() @@ -3012,9 +3031,49 @@ public class AppearancePatentTaskService { .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); if (chunks != null) { for (TaskChunkEntity chunk : chunks) { - transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); + if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) { + payloads.add(chunk.getPayloadJson()); + } } } + return payloads; + } + + /** 事务提交后逐条删远端载荷；无事务时立即执行（与 TaskResultItemService 同款兜底）。 */ + private void deletePayloadsAfterCommit(List payloads, Long taskId) { + if (payloads == null || payloads.isEmpty()) { + return; + } + runAfterCommit(() -> { + for (String payload : payloads) { + try { + transientPayloadStorageService.deletePayloadIfPresent(payload); + } catch (Exception ex) { + log.warn("[appearance-patent] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage()); + } + } + }); + } + + /** + * 删除范围/分片的远端载荷（保留给无事务的清理链路调用）。 + */ + private void deleteTransientTaskPayloads(Long taskId) { + deletePayloadsAfterCommit(collectTransientTaskPayloads(taskId), taskId); + } + + /** 有活动事务则注册 afterCommit，否则立即执行。 */ + private void runAfterCommit(Runnable action) { + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + action.run(); + } + }); + return; + } + action.run(); } private record SubmitContext(FileTaskEntity task, diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java index a55526a9..dbdbb653 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java @@ -13,9 +13,12 @@ import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -27,6 +30,7 @@ import java.util.Map; @Service @RequiredArgsConstructor +@Slf4j public class BrandTaskStorageService { private static final String MODULE_TYPE = "BRAND"; @@ -236,6 +240,13 @@ public class BrandTaskStorageService { return new ChunkStoreResult(false, completed, countCompletedFiles(taskId), aggregate); } + /** + * 删除任务的全部范围/分片数据。 + * + *

事务边界：远端载荷删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次， + * 放在 DB 事务里会让连接被长时间占用（项目在 TaskScopePayloadStorageService 已写明该约束）。 + * 提交后再删还有个好处：引用计数查询看到的是删除完成后的 DB 状态，判断更准确。 + */ @Transactional public void deleteTaskData(Long taskId) { if (taskId == null || taskId <= 0) { @@ -245,27 +256,68 @@ public class BrandTaskStorageService { .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) .eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); - if (states != null) { - for (TaskScopeStateEntity state : states) { - transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson()); - transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson()); - } - } List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() .select(TaskChunkEntity::getPayloadJson) .eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); - if (chunks != null) { - for (TaskChunkEntity chunk : chunks) { - transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); - } - } taskChunkMapper.delete(new LambdaQueryWrapper() .eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); taskScopeStateMapper.delete(new LambdaQueryWrapper() .eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + deletePayloadsAfterCommit(states, chunks, taskId); + } + + /** 事务提交后逐条删远端载荷；无事务时立即执行（与 TaskResultItemService 同款兜底）。 */ + private void deletePayloadsAfterCommit(List states, + List chunks, + Long taskId) { + List payloads = new ArrayList<>(); + if (states != null) { + for (TaskScopeStateEntity state : states) { + if (state == null) { + continue; + } + if (state.getParsedPayloadJson() != null && !state.getParsedPayloadJson().isBlank()) { + payloads.add(state.getParsedPayloadJson()); + } + if (state.getStateJson() != null && !state.getStateJson().isBlank()) { + payloads.add(state.getStateJson()); + } + } + } + if (chunks != null) { + for (TaskChunkEntity chunk : chunks) { + if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) { + payloads.add(chunk.getPayloadJson()); + } + } + } + if (payloads.isEmpty()) { + return; + } + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + deletePayloadsNow(payloads, taskId); + } + }); + return; + } + deletePayloadsNow(payloads, taskId); + } + + private void deletePayloadsNow(List payloads, Long taskId) { + for (String payload : payloads) { + try { + transientPayloadStorageService.deletePayloadIfPresent(payload); + } catch (Exception ex) { + // 事务已提交，远端删除失败只记日志，不影响任务数据清理结果 + log.warn("[brand-storage] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage()); + } + } } private void saveAggregate(Long taskId, diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java index 46306797..63753b68 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java @@ -431,70 +431,85 @@ public class CollectDataService { } } - @Transactional + /** + * 进度心跳。 + * + *

事务边界：Redis 任务锁在事务外获取（自旋等待最长 TASK_LOCK_WAIT_MILLIS， + * 放在 @Transactional 里会白占一个 Hikari 连接），DB 段（统计持久化 + 任务行更新） + * 仍在一个事务内。 + */ public void updateProgress(Long taskId, TaskHeartbeatRequest request) { if (taskId == null || taskId <= 0 || request == null) { return; } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - FileTaskEntity task = fileTaskMapper.selectById(taskId); - if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) { + if (transactionTemplate == null) { + // 单测场景（@InjectMocks 未注入事务模板）：退化为直接执行 DB 段 + updateProgressLocked(taskId, request); return; } - CollectDataStats stats = loadStats(task); - boolean changed = false; - Integer current = request.getCurrent(); - Integer total = request.getTotal(); - if (current != null && total != null && total > 0) { - int totalRows = stats.totalRows > 0 ? stats.totalRows : total; - int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows)); - if (stats.totalRows != totalRows || stats.processedRows != processedRows) { - stats.totalRows = totalRows; - stats.processedRows = processedRows; - changed = true; - } - } - if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) { - stats.collectStage = request.getCollectStage(); - changed = true; - } - if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) { - stats.currentKeyword = request.getCurrentKeyword(); - changed = true; - } - if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) { - stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage()); - changed = true; - } - if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) { - stats.searchTotalPages = Math.max(0, request.getSearchTotalPages()); - changed = true; - } - if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) { - stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins()); - changed = true; - } - if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) { - stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins()); - changed = true; - } - if (!changed) { - // 内容未变化：仅当脏数据兜底窗口到期时才强制刷新一次， - // 否则零 UPDATE（重复心跳幂等）。 - if (!shouldForceProgressFlush()) { - return; - } - } else if (shouldThrottleProgressFlush()) { - // 节流窗口内：合并写（只更新内存态、不落库），窗口到期后统一持久化。 - return; - } - persistStats(task, stats); - task.setUpdatedAt(LocalDateTime.now()); - fileTaskMapper.updateById(task); - lastProgressFlushMillis = System.currentTimeMillis(); + transactionTemplate.executeWithoutResult(status -> updateProgressLocked(taskId, request)); } } + private void updateProgressLocked(Long taskId, TaskHeartbeatRequest request) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) { + return; + } + CollectDataStats stats = loadStats(task); + boolean changed = false; + Integer current = request.getCurrent(); + Integer total = request.getTotal(); + if (current != null && total != null && total > 0) { + int totalRows = stats.totalRows > 0 ? stats.totalRows : total; + int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows)); + if (stats.totalRows != totalRows || stats.processedRows != processedRows) { + stats.totalRows = totalRows; + stats.processedRows = processedRows; + changed = true; + } + } + if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) { + stats.collectStage = request.getCollectStage(); + changed = true; + } + if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) { + stats.currentKeyword = request.getCurrentKeyword(); + changed = true; + } + if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) { + stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage()); + changed = true; + } + if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) { + stats.searchTotalPages = Math.max(0, request.getSearchTotalPages()); + changed = true; + } + if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) { + stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins()); + changed = true; + } + if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) { + stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins()); + changed = true; + } + if (!changed) { + // 内容未变化：仅当脏数据兜底窗口到期时才强制刷新一次， + // 否则零 UPDATE（重复心跳幂等）。 + if (!shouldForceProgressFlush()) { + return; + } + } else if (shouldThrottleProgressFlush()) { + // 节流窗口内：合并写（只更新内存态、不落库），窗口到期后统一持久化。 + return; + } + persistStats(task, stats); + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + lastProgressFlushMillis = System.currentTimeMillis(); + } + /** * 节流判定：progressThrottleMillis>0 且距上次实际落库未超过窗口 → 合并写（跳过 UPDATE）。 * progressThrottleMillis<=0 视为关闭节流，恒返回 false（每次心跳都落库，兼容旧行为）。 diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java index dc86fbd0..83d4eee4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java @@ -16,6 +16,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -252,6 +254,13 @@ public class DeleteBrandTaskStorageService { return grouped; } + /** + * 删除任务的全部范围/分片数据。 + * + *

事务边界：远端载荷删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次， + * 放在 DB 事务里会让连接被长时间占用（项目在 TaskScopePayloadStorageService 已写明该约束）。 + * 提交后再删还有个好处：引用计数查询看到的是删除完成后的 DB 状态，判断更准确。 + */ @Transactional public void deleteTaskData(Long taskId) { if (taskId == null || taskId <= 0) { @@ -261,27 +270,68 @@ public class DeleteBrandTaskStorageService { .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) .eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); - if (states != null) { - for (TaskScopeStateEntity state : states) { - transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson()); - transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson()); - } - } List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() .select(TaskChunkEntity::getPayloadJson) .eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); - if (chunks != null) { - for (TaskChunkEntity chunk : chunks) { - transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); - } - } taskChunkMapper.delete(new LambdaQueryWrapper() .eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); taskScopeStateMapper.delete(new LambdaQueryWrapper() .eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + deletePayloadsAfterCommit(states, chunks, taskId); + } + + /** 事务提交后逐条删远端载荷；无事务时立即执行（与 TaskResultItemService 同款兜底）。 */ + private void deletePayloadsAfterCommit(List states, + List chunks, + Long taskId) { + List payloads = new ArrayList<>(); + if (states != null) { + for (TaskScopeStateEntity state : states) { + if (state == null) { + continue; + } + if (state.getParsedPayloadJson() != null && !state.getParsedPayloadJson().isBlank()) { + payloads.add(state.getParsedPayloadJson()); + } + if (state.getStateJson() != null && !state.getStateJson().isBlank()) { + payloads.add(state.getStateJson()); + } + } + } + if (chunks != null) { + for (TaskChunkEntity chunk : chunks) { + if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) { + payloads.add(chunk.getPayloadJson()); + } + } + } + if (payloads.isEmpty()) { + return; + } + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + deletePayloadsNow(payloads, taskId); + } + }); + return; + } + deletePayloadsNow(payloads, taskId); + } + + private void deletePayloadsNow(List payloads, Long taskId) { + for (String payload : payloads) { + try { + transientPayloadStorageService.deletePayloadIfPresent(payload); + } catch (Exception ex) { + // 事务已提交，远端删除失败只记日志，不影响任务数据清理结果 + log.warn("[delete-brand-storage] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage()); + } + } } private void refreshScopeState(Long taskId, String scopeKey, String scopeHash, Integer chunkTotal, LocalDateTime now) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/digitalhuman/service/DigitalHumanVersionService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/digitalhuman/service/DigitalHumanVersionService.java index 67a75052..689277c5 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/digitalhuman/service/DigitalHumanVersionService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/digitalhuman/service/DigitalHumanVersionService.java @@ -11,6 +11,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; import org.springframework.web.multipart.MultipartFile; import java.io.File; @@ -187,6 +189,13 @@ public class DigitalHumanVersionService { return toVo(entity); } + /** + * 删除版本。 + * + *

事务边界：MinIO 对象删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次， + * 放在 DB 事务里会长时间占用连接（项目在 TaskScopePayloadStorageService 已写明该约束）。 + * 顺序仍是「先删库、后删对象」，与改前一致。 + */ @Transactional public void deleteVersion(String version) { DigitalHumanVersionEntity entity = versionMapper.selectOne(new LambdaQueryWrapper() @@ -198,15 +207,33 @@ public class DigitalHumanVersionService { throw new BusinessException("最新版本不能删除"); } - // 删除 MinIO 文件 - try { - ossStorageService.deleteObject(entity.getOssObjectKey()); - } catch (Exception e) { - log.warn("删除 MinIO 文件失败：{}", entity.getOssObjectKey(), e); - } - // 删除数据库记录 versionMapper.deleteById(entity.getId()); + + // 删除 MinIO 文件（事务提交后） + deleteObjectAfterCommit(entity.getOssObjectKey()); + } + + /** 事务提交后删 MinIO 对象；无事务时立即执行。 */ + private void deleteObjectAfterCommit(String objectKey) { + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + deleteObjectQuietly(objectKey); + } + }); + return; + } + deleteObjectQuietly(objectKey); + } + + private void deleteObjectQuietly(String objectKey) { + try { + ossStorageService.deleteObject(objectKey); + } catch (Exception e) { + log.warn("删除 MinIO 文件失败：{}", objectKey, e); + } } public String getDownloadUrl(String version) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java index 5329dcb2..330bc49f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java @@ -40,6 +40,8 @@ public class ImageVideoAsyncTaskService { private static final int DISPATCH_BATCH_SIZE = 20; private static final int POLL_BATCH_SIZE = 50; private static final int FAILED_TASK_RETENTION_MINUTES = 10; + /** 过期失败任务单轮删除上限：一条无 LIMIT 的 DELETE 会长时间持锁，改成分批（每轮扫描都会再清）。 */ + private static final int EXPIRED_DELETE_BATCH_SIZE = 500; private static final Set TERMINAL_STATUSES = Set.of( "SUCCESS", "SUCCEEDED", "COMPLETED", "DONE", "FINISHED", "FAILED", "FAIL", "ERROR", "CANCELED", "CANCELLED" @@ -142,7 +144,10 @@ public class ImageVideoAsyncTaskService { return; } try (jobLock) { + // 只取主键：本表含 5 个 LONGTEXT 列（请求/响应正文），而这里只用来发起 + // executeTask(id)。每秒扫一轮还拉全部大字段属于纯浪费（2026-09-15 优化）。 List tasks = taskMapper.selectList(new LambdaQueryWrapper() + .select(ImageVideoAsyncTaskEntity::getId) .eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.PENDING.name()) .and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId()) .or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId) @@ -161,7 +166,9 @@ public class ImageVideoAsyncTaskService { return; } try (jobLock) { + // 同 dispatchPendingTasks：只取主键，避免每 5 秒把 LONGTEXT 正文整列拉回 List tasks = taskMapper.selectList(new LambdaQueryWrapper() + .select(ImageVideoAsyncTaskEntity::getId) .eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.WAITING.name()) .and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId()) .or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId) @@ -234,7 +241,8 @@ public class ImageVideoAsyncTaskService { LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES); int deleted = taskMapper.delete(new LambdaQueryWrapper() .eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.FAILED.name()) - .lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff)); + .lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff) + .last("LIMIT " + EXPIRED_DELETE_BATCH_SIZE)); if (deleted > 0) { log.info("[image-video] removed expired failed tasks count={}", deleted); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java index b26dd782..46b4196e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java @@ -37,6 +37,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; import java.io.File; import java.time.LocalDateTime; @@ -81,6 +83,10 @@ public class PatrolDeleteTaskService { private final TaskScopePayloadStorageService taskScopePayloadStorageService; private final TaskDistributedLockService taskDistributedLockService; private final TaskProgressLightAssembler taskProgressLightAssembler; + /** + * 事务边界收口用：Redis 任务锁必须在事务外获取，否则自旋等待期间会白占一个 Hikari 连接。 + */ + private final org.springframework.transaction.PlatformTransactionManager transactionManager; private FileTaskEntity loadTaskForExecution(Long taskId) { Map cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId)); @@ -470,7 +476,10 @@ public class PatrolDeleteTaskService { } } - @Transactional + /** + * 事务边界：任务锁在事务外获取（自旋等待不再占用 DB 连接）； + * 缓存清理放到事务提交后、锁内执行。耗时日志语义保持不变。 + */ public void deleteTask(Long taskId, Long userId) { long startedAt = System.nanoTime(); validateUserId(userId); @@ -480,14 +489,18 @@ public class PatrolDeleteTaskService { } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { long taskLoadedAt = System.nanoTime(); - fileResultMapper.delete(new LambdaQueryWrapper() - .eq(FileResultEntity::getTaskId, taskId) - .eq(FileResultEntity::getModuleType, MODULE_TYPE)); - long resultsDeletedAt = System.nanoTime(); - cleanupTaskAuxiliaryDataFast(taskId); - long auxiliaryDeletedAt = System.nanoTime(); - fileTaskMapper.deleteById(taskId); - long taskDeletedAt = System.nanoTime(); + long[] marks = new long[3]; + inNewTransaction(() -> { + fileResultMapper.delete(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE)); + marks[0] = System.nanoTime(); + cleanupTaskAuxiliaryDataFast(taskId); + marks[1] = System.nanoTime(); + fileTaskMapper.deleteById(taskId); + marks[2] = System.nanoTime(); + return null; + }); taskCacheService.evictTaskCacheOnly(taskId); long finishedAt = System.nanoTime(); log.info("[patrol-delete] delete task timing taskId={} userId={} totalMs={} loadMs={} resultDeleteMs={} auxiliaryDeleteMs={} taskDeleteMs={} cacheMs={}", @@ -495,14 +508,16 @@ public class PatrolDeleteTaskService { userId, elapsedMs(startedAt, finishedAt), elapsedMs(startedAt, taskLoadedAt), - elapsedMs(taskLoadedAt, resultsDeletedAt), - elapsedMs(resultsDeletedAt, auxiliaryDeletedAt), - elapsedMs(auxiliaryDeletedAt, taskDeletedAt), - elapsedMs(taskDeletedAt, finishedAt)); + elapsedMs(taskLoadedAt, marks[0]), + elapsedMs(marks[0], marks[1]), + elapsedMs(marks[1], marks[2]), + elapsedMs(marks[2], finishedAt)); } } - @Transactional + /** + * 事务边界：任务锁在事务外获取，DB 段（复核 + 删除 + 重算）仍在同一事务内。 + */ public void deleteHistory(Long resultId, Long userId) { validateUserId(userId); FileResultEntity entity = fileResultMapper.selectById(resultId); @@ -516,13 +531,16 @@ public class PatrolDeleteTaskService { return; } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - FileResultEntity latestEntity = fileResultMapper.selectById(resultId); - if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { - throw new BusinessException("record not found"); - } - fileResultMapper.deleteById(resultId); - cleanupResultAuxiliaryDataFast(taskId, resultId); - reconcileTaskAfterResultRemoval(taskId); + inNewTransaction(() -> { + FileResultEntity latestEntity = fileResultMapper.selectById(resultId); + if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { + throw new BusinessException("record not found"); + } + fileResultMapper.deleteById(resultId); + cleanupResultAuxiliaryDataFast(taskId, resultId); + reconcileTaskAfterResultRemoval(taskId); + return null; + }); } } @@ -1094,6 +1112,19 @@ public class PatrolDeleteTaskService { return taskDistributedLockService.acquire(MODULE_TYPE, taskId); } + /** + * 显式短事务：锁已在事务外获取，这里只包住 DB 段， + * 保证原来同属一个 @Transactional 的多条写仍在同一事务内。 + */ + private T inNewTransaction(java.util.function.Supplier action) { + if (transactionManager == null) { + return action.get(); + } + TransactionTemplate template = new TransactionTemplate(transactionManager); + template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + return template.execute(status -> action.get()); + } + private void markResultSuccess(FileResultEntity row) { row.setSuccess(RESULT_SUCCESS); row.setErrorMessage(null); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java index d40e716c..8d32e195 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java @@ -297,6 +297,22 @@ public class PermissionMenuService { .contains(columnId); } + /** + * 可授权用户查询：只取「超管判定 + 列表展示」用到的列。 + * + *

权限页此前一律 `selectList(new LambdaQueryWrapper<>())` 整表拉回，而 admin_user + * 含密码哈希等敏感列，每次打开权限页都把全表（含哈希）读进内存（2026-09-15 优化）。 + * 判定用 role/is_admin/created_by_id，展示用 id/username。 + */ + private LambdaQueryWrapper grantableUserQuery() { + return new LambdaQueryWrapper() + .select(AdminUserEntity::getId, + AdminUserEntity::getUsername, + AdminUserEntity::getRole, + AdminUserEntity::getIsAdmin, + AdminUserEntity::getCreatedById); + } + public List listImageVideoDataPermissionUsers(AdminUserEntity operator) { ensureSuperAdminOperator(operator); PermissionMenuEntity dataPermission = requireImageVideoDataPermission(); @@ -305,7 +321,7 @@ public class PermissionMenuService { .map(UserColumnPermissionEntity::getUserId) .filter(id -> id != null && id > 0) .collect(Collectors.toSet()); - return adminUserMapper.selectList(new LambdaQueryWrapper() + return adminUserMapper.selectList(grantableUserQuery() .orderByAsc(AdminUserEntity::getUsername) .orderByAsc(AdminUserEntity::getId)) .stream() @@ -319,7 +335,7 @@ public class PermissionMenuService { ensureSuperAdminOperator(operator); PermissionMenuEntity dataPermission = requireImageVideoDataPermission(); List requestedIds = normalizeColumnIds(userIds); - List users = adminUserMapper.selectList(new LambdaQueryWrapper()); + List users = adminUserMapper.selectList(grantableUserQuery()); Map grantableUsers = users.stream() .filter(user -> user.getId() != null && !isSuperAdmin(user)) .collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left)); @@ -345,7 +361,7 @@ public class PermissionMenuService { .map(UserColumnPermissionEntity::getUserId) .filter(id -> id != null && id > 0) .collect(Collectors.toSet()); - return adminUserMapper.selectList(new LambdaQueryWrapper() + return adminUserMapper.selectList(grantableUserQuery() .orderByAsc(AdminUserEntity::getUsername) .orderByAsc(AdminUserEntity::getId)) .stream() @@ -360,7 +376,7 @@ public class PermissionMenuService { PermissionMenuEntity dataPermission = requireDataPermission( SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据"); List requestedIds = normalizeColumnIds(userIds); - List users = adminUserMapper.selectList(new LambdaQueryWrapper()); + List users = adminUserMapper.selectList(grantableUserQuery()); Map grantableUsers = users.stream() .filter(user -> user.getId() != null && !isSuperAdmin(user)) .collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left)); @@ -607,7 +623,9 @@ public class PermissionMenuService { } Set targetEffectiveIds = expandDescendantIds( new LinkedHashSet<>(loadDirectColumnIds(target.getId())), loadMenus(null)); + // 只用到 id：不要为一次级联清理把整表（含密码哈希）拉回 List subordinates = adminUserMapper.selectList(new LambdaQueryWrapper() + .select(AdminUserEntity::getId) .eq(AdminUserEntity::getCreatedById, target.getId())); for (AdminUserEntity subordinate : subordinates) { Long subordinateId = subordinate.getId(); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java index 8e8d72c7..d6f15198 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java @@ -39,6 +39,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; import java.io.BufferedReader; import java.io.File; @@ -93,6 +95,11 @@ public class PriceTrackTaskService { private final TaskDistributedLockService taskDistributedLockService; private final TaskProgressLightAssembler taskProgressLightAssembler; private final com.nanri.aiimage.modules.file.service.LocalFileStorageService localFileStorageService; + /** + * 事务边界收口用：Redis 任务锁必须在事务外获取，否则自旋等待（最长 TASK_LOCK_WAIT_MILLIS） + * 期间会白占一个 Hikari 连接（生产 maximum-pool-size 仅 30）。 + */ + private final org.springframework.transaction.PlatformTransactionManager transactionManager; private FileTaskEntity loadTaskForExecution(Long taskId) { Map cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId)); @@ -254,7 +261,10 @@ public class PriceTrackTaskService { return vo; } - @Transactional + /** + * 事务边界：Redis 任务锁在事务外获取（自旋等待不再占用 DB 连接）， + * DB 段（复核 + 删除 + 重算任务状态）仍在同一个事务内。 + */ public void deleteHistory(Long resultId, Long userId) { if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); FileResultEntity entity = fileResultMapper.selectById(resultId); @@ -267,73 +277,97 @@ public class PriceTrackTaskService { return; } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - FileResultEntity latestEntity = fileResultMapper.selectById(resultId); - if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { - throw new BusinessException("record not found"); - } - fileResultMapper.deleteById(resultId); - reconcileTaskAfterResultRemoval(taskId); + inNewTransaction(() -> { + FileResultEntity latestEntity = fileResultMapper.selectById(resultId); + if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { + throw new BusinessException("record not found"); + } + fileResultMapper.deleteById(resultId); + reconcileTaskAfterResultRemoval(taskId); + return null; + }); } } - @Transactional + /** + * 事务边界：任务锁在事务外获取；缓存清理（Redis + 远端范围载荷）放到事务提交后执行。 + */ public void deleteTask(Long taskId, Long userId) { if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - FileTaskEntity task = loadTaskForExecution(taskId); - if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { - throw new BusinessException("任务不存在"); - } - fileResultMapper.delete(new LambdaQueryWrapper() - .eq(FileResultEntity::getTaskId, taskId) - .eq(FileResultEntity::getModuleType, MODULE_TYPE)); - fileTaskMapper.deleteById(taskId); + inNewTransaction(() -> { + FileTaskEntity task = loadTaskForExecution(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { + throw new BusinessException("任务不存在"); + } + fileResultMapper.delete(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE)); + fileTaskMapper.deleteById(taskId); + priceTrackLoopRunService.syncLoopRunAfterChildRemoved(taskId); + return null; + }); + // 缓存清理（Redis + 远端范围载荷）在事务提交后、锁内执行 priceTrackTaskCacheService.deleteTaskCache(taskId); - priceTrackLoopRunService.syncLoopRunAfterChildRemoved(taskId); } + log.info("[price-track] deleteTask 完成 taskId={} userId={}", taskId, userId); } - @Transactional + /** + * 事务边界：任务锁在事务外获取；缓存清理放到事务提交后执行。 + */ public void markDispatchFailed(Long taskId, Long userId, String errorMessage) { if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); if (errorMessage == null || errorMessage.isBlank()) throw new BusinessException("errorMessage 不能为空"); String normalizedError = errorMessage.trim(); try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - FileTaskEntity task = loadTaskForExecution(taskId); - if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { - throw new BusinessException("任务不存在"); + // 返回需要清理缓存的任务状态（仅无结果行分支），非终态返回 null + String terminalStatus = inNewTransaction(() -> + markDispatchFailedRecords(taskId, userId, normalizedError)); + if (terminalStatus != null) { + cleanupTaskCacheIfTerminal(taskId, terminalStatus); } - if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) { - return; - } - List results = fileResultMapper.selectList( - new LambdaQueryWrapper() - .eq(FileResultEntity::getTaskId, taskId) - .eq(FileResultEntity::getModuleType, MODULE_TYPE) - .orderByAsc(FileResultEntity::getId)); - if (results.isEmpty()) { - task.setStatus("FAILED"); - task.setErrorMessage(normalizedError); - task.setUpdatedAt(LocalDateTime.now()); - task.setFinishedAt(LocalDateTime.now()); - fileTaskMapper.updateById(task); - cleanupTaskCacheIfTerminal(taskId, task.getStatus()); - priceTrackLoopRunService.syncLoopRunAfterChildTerminal(taskId); - return; - } - for (FileResultEntity result : results) { - boolean succeeded = result.getSuccess() != null && result.getSuccess() == 1; - boolean failed = result.getErrorMessage() != null && !result.getErrorMessage().isBlank(); - if (!succeeded && !failed) { - markResultFailed(result, normalizedError); - } - } - updateTaskStatusFromLatestRows(task, results); - log.warn("[price-track] Python dispatch failed taskId={} userId={} error={}", taskId, userId, normalizedError); } } - @Transactional + private String markDispatchFailedRecords(Long taskId, Long userId, String normalizedError) { + FileTaskEntity task = loadTaskForExecution(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { + throw new BusinessException("任务不存在"); + } + if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) { + return null; + } + List results = fileResultMapper.selectList( + new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .orderByAsc(FileResultEntity::getId)); + if (results.isEmpty()) { + task.setStatus("FAILED"); + task.setErrorMessage(normalizedError); + task.setUpdatedAt(LocalDateTime.now()); + task.setFinishedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + priceTrackLoopRunService.syncLoopRunAfterChildTerminal(taskId); + return task.getStatus(); + } + for (FileResultEntity result : results) { + boolean succeeded = result.getSuccess() != null && result.getSuccess() == 1; + boolean failed = result.getErrorMessage() != null && !result.getErrorMessage().isBlank(); + if (!succeeded && !failed) { + markResultFailed(result, normalizedError); + } + } + updateTaskStatusFromLatestRows(task, results); + log.warn("[price-track] Python dispatch failed taskId={} userId={} error={}", taskId, userId, normalizedError); + return null; + } + + /** + * 事务边界：逐个候选任务在事务外取锁；命中的那条记录的删除 + 重算在独立短事务内完成 + * （原实现整个循环共用一个事务，锁自旋等待期间会一直占着连接）。 + */ public PriceTrackPendingDeleteVo deletePendingShopResult(Long userId, String shopName) { if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 不能为空"); @@ -356,20 +390,33 @@ public class PriceTrackTaskService { if (!"RUNNING".equals(task.getStatus())) continue; try (TaskDistributedLockService.LockHandle ignored = acquireTaskLock(fr.getTaskId())) { if (ignored == null) continue; - FileTaskEntity lockedTask = loadTaskForExecution(fr.getTaskId()); - if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) continue; - if (!"RUNNING".equals(lockedTask.getStatus())) continue; - FileResultEntity lockedResult = fileResultMapper.selectById(fr.getId()); - if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) continue; - fileResultMapper.deleteById(fr.getId()); - reconcileTaskAfterResultRemoval(fr.getTaskId()); - vo.setRemoved(true); - return vo; + Boolean removed = inNewTransaction(() -> removeRunningShopResult(fr.getTaskId(), userId, fr.getId())); + if (Boolean.TRUE.equals(removed)) { + vo.setRemoved(true); + return vo; + } } } return vo; } + private boolean removeRunningShopResult(Long taskId, Long userId, Long resultId) { + FileTaskEntity lockedTask = loadTaskForExecution(taskId); + if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) { + return false; + } + if (!"RUNNING".equals(lockedTask.getStatus())) { + return false; + } + FileResultEntity lockedResult = fileResultMapper.selectById(resultId); + if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) { + return false; + } + fileResultMapper.deleteById(resultId); + reconcileTaskAfterResultRemoval(taskId); + return true; + } + public PriceTrackTaskBatchVo getTaskDetailsBatch(List taskIds) { PriceTrackTaskBatchVo batch = new PriceTrackTaskBatchVo(); if (taskIds == null || taskIds.isEmpty()) return batch; @@ -506,6 +553,10 @@ public class PriceTrackTaskService { } List snapshot = new ArrayList<>(); + // 逐店铺 fileResultMapper.insert 看似该改批量（500 店铺 = 500 次往返），但**不能改**： + // 下面的 toSnapshotVo 依赖 MyBatis-Plus insert 回填的自增主键（fr.getId() → vo.resultId）， + // 而本仓的手写 INSERT ... VALUES (...),(...) 批量写法不回填主键，改了会让快照 resultId 为 null。 + // 要批量化必须同时解决「批量插入 + 按序回填 id」（MySQL 需 useGeneratedKeys + 顺序保证）。 for (PriceTrackMatchShopsVo.PriceTrackShopQueueItem item : uniqueItems) { String norm = ziniaoShopSwitchService.normalizeShopName(item.getShopName()); if (norm.isBlank()) throw new BusinessException("店铺名称无效"); @@ -2182,6 +2233,20 @@ public class PriceTrackTaskService { return lockHandle; } + /** + * 显式开启一个短事务：锁已在事务外获取，这里只包住 DB 段， + * 保证原来同属一个 @Transactional 的多条写仍在同一事务内。 + */ + private T inNewTransaction(java.util.function.Supplier action) { + if (transactionManager == null) { + // 单测直接 new 出本类（history 见 QueryAsinSnapshotJsonThrottleTest 同类用法）时无事务管理器 + return action.get(); + } + TransactionTemplate template = new TransactionTemplate(transactionManager); + template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + return template.execute(status -> action.get()); + } + private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId) { if (taskId == null || taskId <= 0) { return null; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java index 456af95c..68d7a3c2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java @@ -39,6 +39,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; import java.io.File; import java.time.LocalDateTime; @@ -82,6 +84,10 @@ public class ProductRiskTaskService { private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskDistributedLockService taskDistributedLockService; private final TaskProgressLightAssembler taskProgressLightAssembler; + /** + * 事务边界收口用：Redis 任务锁必须在事务外获取，否则自旋等待期间会白占一个 Hikari 连接。 + */ + private final org.springframework.transaction.PlatformTransactionManager transactionManager; private FileTaskEntity loadTaskForExecution(Long taskId) { Map cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId)); @@ -248,7 +254,9 @@ public class ProductRiskTaskService { return vo; } - @Transactional + /** + * 事务边界：任务锁在事务外获取（自旋等待不再占用 DB 连接），DB 段仍在同一事务内。 + */ public void deleteHistory(Long resultId, Long userId) { if (userId == null || userId <= 0) { throw new BusinessException("user_id 不合法"); @@ -263,43 +271,53 @@ public class ProductRiskTaskService { return; } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - FileResultEntity latestEntity = fileResultMapper.selectById(resultId); - if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { - throw new BusinessException("record not found"); - } - taskFileJobService.deleteResultJobs(latestEntity.getTaskId(), MODULE_TYPE, latestEntity.getId()); - fileResultMapper.deleteById(resultId); - reconcileTaskAfterResultRemoval(taskId); + inNewTransaction(() -> { + FileResultEntity latestEntity = fileResultMapper.selectById(resultId); + if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { + throw new BusinessException("record not found"); + } + taskFileJobService.deleteResultJobs(latestEntity.getTaskId(), MODULE_TYPE, latestEntity.getId()); + fileResultMapper.deleteById(resultId); + reconcileTaskAfterResultRemoval(taskId); + return null; + }); } } /** * 删除整条商品风险任务及其下所有店铺结果，运行中和已结束任务都允许删除，但必须归属当前用户。 + * + *

事务边界：任务锁在事务外获取；缓存清理放到事务提交后、锁内执行。 */ - @Transactional public void deleteTask(Long taskId, Long userId) { if (userId == null || userId <= 0) { throw new BusinessException("user_id 不合法"); } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - FileTaskEntity task = loadTaskForExecution(taskId); - if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { - throw new BusinessException("任务不存在"); - } - fileResultMapper.delete(new LambdaQueryWrapper() - .eq(FileResultEntity::getTaskId, taskId) - .eq(FileResultEntity::getModuleType, MODULE_TYPE)); - taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); - fileTaskMapper.deleteById(taskId); + inNewTransaction(() -> { + FileTaskEntity task = loadTaskForExecution(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { + throw new BusinessException("任务不存在"); + } + fileResultMapper.delete(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE)); + taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); + fileTaskMapper.deleteById(taskId); + return null; + }); cleanupTaskCacheIfTerminal(taskId, "DELETE_EMPTY"); } + log.info("[product-risk] deleteTask 完成 taskId={} userId={}", taskId, userId); } /** * 从用户当前“运行中”的商品风险任务里，按规范化店铺名删除一条结果，用于前端移除匹配列表后同步后端。 * 如果没有对应记录，则返回 removed=false。 + * + *

事务边界：逐个候选任务在事务外取锁；命中的那条记录的删除 + 重算在独立短事务内完成 + * （原实现整个循环共用一个事务，锁自旋等待期间会一直占着连接）。 */ - @Transactional public ProductRiskPendingDeleteVo deletePendingShopResult(Long userId, String shopName) { if (userId == null || userId <= 0) { throw new BusinessException("user_id 不合法"); @@ -335,26 +353,33 @@ public class ProductRiskTaskService { if (ignored == null) { continue; } - FileTaskEntity lockedTask = loadTaskForExecution(tid); - if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) { - continue; + Boolean removed = inNewTransaction(() -> removeRunningShopResult(tid, userId, fr.getId())); + if (Boolean.TRUE.equals(removed)) { + vo.setRemoved(true); + return vo; } - if (!"RUNNING".equals(lockedTask.getStatus())) { - continue; - } - FileResultEntity lockedResult = fileResultMapper.selectById(fr.getId()); - if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) { - continue; - } - fileResultMapper.deleteById(fr.getId()); - reconcileTaskAfterResultRemoval(tid); - vo.setRemoved(true); - return vo; } } return vo; } + private boolean removeRunningShopResult(Long taskId, Long userId, Long resultId) { + FileTaskEntity lockedTask = loadTaskForExecution(taskId); + if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) { + return false; + } + if (!"RUNNING".equals(lockedTask.getStatus())) { + return false; + } + FileResultEntity lockedResult = fileResultMapper.selectById(resultId); + if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) { + return false; + } + fileResultMapper.deleteById(resultId); + reconcileTaskAfterResultRemoval(taskId); + return true; + } + /** * 删除一条 file_result 后重算父任务状态；如果没有剩余结果则删除任务。 */ @@ -1002,6 +1027,19 @@ public class ProductRiskTaskService { return taskDistributedLockService.acquire(MODULE_TYPE, taskId); } + /** + * 显式短事务：锁已在事务外获取，这里只包住 DB 段， + * 保证原来同属一个 @Transactional 的多条写仍在同一事务内。 + */ + private T inNewTransaction(java.util.function.Supplier action) { + if (transactionManager == null) { + return action.get(); + } + TransactionTemplate template = new TransactionTemplate(transactionManager); + template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + return template.execute(status -> action.get()); + } + private ProductRiskTaskDetailVo buildTaskDetail(FileTaskEntity task) { ProductRiskTaskDetailVo detail = new ProductRiskTaskDetailVo(); detail.setTask(toTaskItemVo(task)); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java index 19854a2e..e1e916eb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java @@ -57,6 +57,8 @@ import org.springframework.dao.DuplicateKeyException; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; import org.springframework.transaction.support.TransactionTemplate; import java.io.File; @@ -554,24 +556,28 @@ public class PublishTaskService { } } + /** + * 删除任务。 + * + *

事务边界：远端删除（结果文件对象 + 分片载荷）全部移到事务提交后执行 —— + * 单次删除超时 120s、重试 3 次，放在 DB 事务里会让连接被长时间占用 + * （项目在 TaskScopePayloadStorageService 已写明「不能把网络调用放进 DB 事务」）。 + */ @Transactional public void deleteTask(Long taskId, Long userId) { FileTaskEntity task = requireTaskForDeletion(taskId, userId); List results = fileResultMapper.selectList(new LambdaQueryWrapper() .eq(FileResultEntity::getTaskId, taskId) .eq(FileResultEntity::getModuleType, MODULE_TYPE)); + List resultObjectKeys = new ArrayList<>(); for (FileResultEntity result : results) { if (result.getResultFileUrl() != null && !result.getResultFileUrl().isBlank()) { - try { - ossStorageService.deleteObject(result.getResultFileUrl()); - } catch (Exception ex) { - log.warn("[publish] OSS cleanup failed taskId={} resultId={} msg={}", - taskId, result.getId(), ex.getMessage()); - } + resultObjectKeys.add(result.getResultFileUrl()); } } + List chunkPayloads = collectTransientResultChunkPayloads(taskId); taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); - deleteTransientResultChunks(taskId); + deleteTransientResultChunkRows(taskId); publishItemMapper.delete(new LambdaQueryWrapper() .eq(PublishItemEntity::getTaskId, taskId)); publishFileMapper.delete(new LambdaQueryWrapper() @@ -580,6 +586,9 @@ public class PublishTaskService { .eq(FileResultEntity::getTaskId, taskId) .eq(FileResultEntity::getModuleType, MODULE_TYPE)); fileTaskMapper.deleteById(task.getId()); + // 事务提交后再做远端删除 + deleteResultObjectsAfterCommit(resultObjectKeys, taskId); + deletePayloadsAfterCommit(chunkPayloads, taskId); } public void deleteHistory(Long resultId, Long userId) { @@ -599,18 +608,35 @@ public class PublishTaskService { } private void deleteTransientResultChunks(Long taskId) { + List payloads = collectTransientResultChunkPayloads(taskId); + deleteTransientResultChunkRows(taskId); + deletePayloadsAfterCommit(payloads, taskId); + } + + /** 只读收集分片载荷指针，供事务提交后做远端删除。 */ + private List collectTransientResultChunkPayloads(Long taskId) { if (taskId == null || taskId <= 0) { - return; + return List.of(); } List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() .select(TaskChunkEntity::getPayloadJson) .eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + List payloads = new ArrayList<>(); if (chunks != null) { for (TaskChunkEntity chunk : chunks) { - transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); + if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) { + payloads.add(chunk.getPayloadJson()); + } } } + return payloads; + } + + private void deleteTransientResultChunkRows(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } taskChunkMapper.delete(new LambdaQueryWrapper() .eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); @@ -619,6 +645,63 @@ public class PublishTaskService { .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); } + /** 事务提交后逐条删分片载荷；无事务时立即执行（与 TaskResultItemService 同款兜底）。 */ + private void deletePayloadsAfterCommit(List payloads, Long taskId) { + if (payloads == null || payloads.isEmpty()) { + return; + } + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + deletePayloadsNow(payloads, taskId); + } + }); + return; + } + deletePayloadsNow(payloads, taskId); + } + + private void deletePayloadsNow(List payloads, Long taskId) { + for (String payload : payloads) { + try { + transientPayloadStorageService.deletePayloadIfPresent(payload); + } catch (Exception ex) { + log.warn("[publish] 分片载荷删除失败 taskId={} pointer={} msg={}", + taskId, transientPayloadStorageService.extractPointer(payload), safeMessage(ex)); + } + } + } + + /** 事务提交后删结果文件对象；无事务时立即执行。 */ + private void deleteResultObjectsAfterCommit(List objectKeys, Long taskId) { + if (objectKeys == null || objectKeys.isEmpty()) { + return; + } + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + deleteResultObjectsNow(objectKeys, taskId); + } + }); + return; + } + deleteResultObjectsNow(objectKeys, taskId); + } + + private void deleteResultObjectsNow(List objectKeys, Long taskId) { + for (String objectKey : objectKeys) { + try { + ossStorageService.deleteObject(objectKey); + log.info("[publish] 结果文件已删除 taskId={} objectKey={}", taskId, objectKey); + } catch (Exception ex) { + log.warn("[publish] OSS cleanup failed taskId={} objectKey={} msg={}", + taskId, objectKey, ex.getMessage()); + } + } + } + private void deleteUncommittedPayloads(List storedPayloads) { if (storedPayloads == null || storedPayloads.isEmpty()) { return; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java index 4739cb3f..9b58f897 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java @@ -36,6 +36,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; import java.io.File; import java.time.LocalDateTime; @@ -92,6 +94,10 @@ public class QueryAsinTaskService { private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskDistributedLockService taskDistributedLockService; private final TaskProgressLightAssembler taskProgressLightAssembler; + /** + * 事务边界收口用：Redis 任务锁必须在事务外获取，否则自旋等待期间会白占一个 Hikari 连接。 + */ + private final org.springframework.transaction.PlatformTransactionManager transactionManager; private FileTaskEntity loadTaskForExecution(Long taskId) { Map cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId)); @@ -475,7 +481,10 @@ public class QueryAsinTaskService { } } - @Transactional + /** + * 事务边界：任务锁在事务外获取（自旋等待不再占用 DB 连接）； + * 缓存清理（Redis + 远端范围载荷）放到事务提交后、锁内执行。 + */ public void deleteTask(Long taskId, Long userId) { validateUserId(userId); FileTaskEntity task = loadTaskForExecution(taskId); @@ -483,15 +492,21 @@ public class QueryAsinTaskService { throw new BusinessException("任务不存在"); } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - fileResultMapper.delete(new LambdaQueryWrapper() - .eq(FileResultEntity::getTaskId, taskId) - .eq(FileResultEntity::getModuleType, MODULE_TYPE)); - fileTaskMapper.deleteById(taskId); - taskCacheService.deleteTaskCache(taskId); + inNewTransaction(() -> { + fileResultMapper.delete(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE)); + fileTaskMapper.deleteById(taskId); + return null; + }); + taskCacheService.deleteTaskCache(taskId); } + log.info("[query-asin] deleteTask 完成 taskId={} userId={}", taskId, userId); } - @Transactional + /** + * 事务边界：任务锁在事务外获取，DB 段（复核 + 删除 + 重算）仍在同一事务内。 + */ public void deleteHistory(Long resultId, Long userId) { validateUserId(userId); FileResultEntity entity = fileResultMapper.selectById(resultId); @@ -504,12 +519,15 @@ public class QueryAsinTaskService { return; } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - FileResultEntity latestEntity = fileResultMapper.selectById(resultId); - if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { - throw new BusinessException("record not found"); - } - fileResultMapper.deleteById(resultId); - reconcileTaskAfterResultRemoval(taskId); + inNewTransaction(() -> { + FileResultEntity latestEntity = fileResultMapper.selectById(resultId); + if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { + throw new BusinessException("record not found"); + } + fileResultMapper.deleteById(resultId); + reconcileTaskAfterResultRemoval(taskId); + return null; + }); } } @@ -1070,6 +1088,19 @@ public class QueryAsinTaskService { return taskDistributedLockService.acquire(MODULE_TYPE, taskId); } + /** + * 显式短事务：锁已在事务外获取，这里只包住 DB 段， + * 保证原来同属一个 @Transactional 的多条写仍在同一事务内。 + */ + private T inNewTransaction(java.util.function.Supplier action) { + if (transactionManager == null) { + return action.get(); + } + TransactionTemplate template = new TransactionTemplate(transactionManager); + template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + return template.execute(status -> action.get()); + } + private void markResultSuccess(FileResultEntity row) { row.setSuccess(RESULT_SUCCESS); row.setErrorMessage(null); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java index 132aea02..832763ec 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java @@ -87,6 +87,13 @@ public class ShopDataCrawlExcelAssemblyService { imageEmbedder.prefetch(prefetchBudget().boundedUrls(imageUrls(rowsByCountry)), imageCache); } + /** + * 模板路径组装（DOM）。历史包袱说明：本方法把整个工作簿读进 DOM（XSSFWorkbook）， + * 当日累计文件越大峰值堆越高（-Xmx6g 下大客户的大文件有 OOM 风险）。 + * 同文件的 {@link #writeWorkbookStreaming} 是 SXSSF 流式版，但它**不使用模板、 + * 不写图片**（图片只兜底成 URL 文本），直接替换会改变交付文件的样式与商品图， + * 属产品取舍，未直接切换。若要落地必须实现「保留模板样式 + 保留图片」的流式写入。 + */ public int writeWorkbook(File outputXlsx, List items) { try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream(); XSSFWorkbook workbook = new XSSFWorkbook(input); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java index 6dca797f..42b47c62 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java @@ -807,7 +807,10 @@ public class ShopDataCrawlTaskService { } } - @Transactional + /** + * 事务边界：Redis 任务锁在事务外获取（自旋等待期间不再白占一个 Hikari 连接）； + * DB 段（含每日文件行锁与三条删除链路）仍在同一个短事务内，远端对象清理在提交后进行。 + */ public void deleteTask(Long taskId, Long userId) { validateUserId(userId); FileTaskEntity task = loadTaskForExecution(taskId); @@ -816,28 +819,33 @@ public class ShopDataCrawlTaskService { } ensureTaskOwnedByCurrentInstance(task, "delete shop data crawl task"); try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - List taskRows = listTaskRows(taskId); - try (DailyLockSet dailyLocks = acquireDailyLocks(taskRows)) { - ensureDailySyncCompletedBeforeDelete(taskRows); - Set removedResultIds = collectResultIds(taskRows); - // A task deletion is only a frontend task-record cleanup. The daily - // workbook is an independent backend aggregate and must not roll - // back when its source task is removed. - DailyDeletionResult dailyResult = preserveDailyForTaskDeletion(removedResultIds); - registerUploadedObjectRollback(dailyResult.uploadedObjectKeys()); - List resultFileUrls = collectResultFileUrls(taskRows, dailyResult.obsoleteObjectKeys()); - fileResultMapper.delete(new LambdaQueryWrapper() - .eq(FileResultEntity::getTaskId, taskId) - .eq(FileResultEntity::getModuleType, MODULE_TYPE)); - taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); - taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE); - taskProgressSnapshotService.delete(taskId, MODULE_TYPE); - fileTaskMapper.deleteById(taskId); - taskCacheService.deleteTaskCache(taskId); - deleteTransientResultChunks(taskId); - resultFileUrls.forEach(this::deleteResultObjectIfUnreferenced); - } + executeShortTransaction(() -> { + List taskRows = listTaskRows(taskId); + try (DailyLockSet dailyLocks = acquireDailyLocks(taskRows)) { + ensureDailySyncCompletedBeforeDelete(taskRows); + Set removedResultIds = collectResultIds(taskRows); + // A task deletion is only a frontend task-record cleanup. The daily + // workbook is an independent backend aggregate and must not roll + // back when its source task is removed. + DailyDeletionResult dailyResult = preserveDailyForTaskDeletion(removedResultIds); + registerUploadedObjectRollback(dailyResult.uploadedObjectKeys()); + List resultFileUrls = collectResultFileUrls(taskRows, dailyResult.obsoleteObjectKeys()); + fileResultMapper.delete(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE)); + taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); + taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE); + taskProgressSnapshotService.delete(taskId, MODULE_TYPE); + fileTaskMapper.deleteById(taskId); + deleteTransientResultChunks(taskId); + resultFileUrls.forEach(this::deleteResultObjectIfUnreferenced); + } + return null; + }); + // 缓存清理（Redis + 本地缓存）在事务提交后、锁内执行 + taskCacheService.deleteTaskCache(taskId); } + log.info("[shop-data-crawl] deleteTask 完成 taskId={} userId={}", taskId, userId); } /** 删除任务的纯计算：收集有效结果行 id 集合（过滤 null/非正数），无副作用。 */ @@ -884,7 +892,10 @@ public class ShopDataCrawlTaskService { } } - @Transactional + /** + * 事务边界：任务锁在事务外获取；DB 段（复核 + 删行 + 每日文件重建）在同一个短事务内， + * 远端对象清理已由 deleteResultObjectIfUnreferenced 在提交后执行。 + */ public void deleteHistory(Long resultId, Long userId) { validateUserId(userId); FileResultEntity entity = requireResultEntity(resultId); @@ -899,21 +910,26 @@ public class ShopDataCrawlTaskService { throw new BusinessException(BusinessCodes.TASK_BUSY, "任务仍在处理中，不能删除"); } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - FileResultEntity latestEntity = fileResultMapper.selectById(resultId); - if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { - throw new BusinessException("记录不存在"); - } - try (DailyLockSet dailyLocks = acquireDailyLocks(List.of(latestEntity))) { - deleteResultHistoryRow(latestEntity); - } + executeShortTransaction(() -> { + FileResultEntity latestEntity = fileResultMapper.selectById(resultId); + if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { + throw new BusinessException("记录不存在"); + } + try (DailyLockSet dailyLocks = acquireDailyLocks(List.of(latestEntity))) { + deleteResultHistoryRow(latestEntity); + } + return null; + }); } } /** * Deletes an administrative result without accepting a caller-controlled owner ID. * The controller that calls this method is restricted to the Flask-to-Java internal channel. + * + *

不加 @Transactional：内部调用 deleteHistory 时事务代理不生效（同类自调用）， + * 锁会落在本方法开启的事务里 —— 事务边界统一由 deleteHistory 自己收口。 */ - @Transactional public void deleteAdminHistory(Long resultId) { FileResultEntity entity = requireResultEntity(resultId); Long ownerId = entity.getUserId(); @@ -2620,6 +2636,13 @@ public class ShopDataCrawlTaskService { taskCacheService.deleteTaskCache(taskId); } + /** + * 删除结果分片行。 + * + *

事务边界：远端载荷删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次， + * 放在 DB 事务里会让连接被长时间占用（项目在 TaskScopePayloadStorageService 已写明该约束）。 + * 提交后再删还有个好处：引用计数查询看到的是删除完成后的 DB 状态，判断更准确。 + */ private void deleteTransientResultChunks(Long taskId) { if (taskId == null || taskId <= 0) { return; @@ -2628,9 +2651,12 @@ public class ShopDataCrawlTaskService { .select(TaskChunkEntity::getPayloadJson) .eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + List payloads = new ArrayList<>(); if (chunks != null) { for (TaskChunkEntity chunk : chunks) { - transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); + if (chunk != null && !blank(chunk.getPayloadJson())) { + payloads.add(chunk.getPayloadJson()); + } } } taskChunkMapper.delete(new LambdaQueryWrapper() @@ -2640,6 +2666,34 @@ public class ShopDataCrawlTaskService { .eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) .likeRight(TaskScopeStateEntity::getScopeKey, RESULT_CHUNK_SCOPE_PREFIX)); + deleteChunkPayloadsAfterCommit(payloads, taskId); + } + + /** 事务提交后逐条删远端载荷；无事务时立即执行（与 TaskResultItemService 同款兜底）。 */ + private void deleteChunkPayloadsAfterCommit(List payloads, Long taskId) { + if (payloads == null || payloads.isEmpty()) { + return; + } + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + deleteChunkPayloadsNow(payloads, taskId); + } + }); + return; + } + deleteChunkPayloadsNow(payloads, taskId); + } + + private void deleteChunkPayloadsNow(List payloads, Long taskId) { + for (String payload : payloads) { + try { + transientPayloadStorageService.deletePayloadIfPresent(payload); + } catch (Exception ex) { + log.warn("[shop-data-crawl] 分片载荷删除失败 taskId={} msg={}", taskId, safeMessage(ex)); + } + } } private void cleanupResultChunksQuietly(Long taskId, String reason) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java index c12b0391..5c9c2b34 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java @@ -46,6 +46,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; import java.io.File; import java.time.LocalDateTime; @@ -89,6 +91,10 @@ public class ShopMatchTaskService { private final SkipPriceAsinService skipPriceAsinService; private final QueryAsinMapper queryAsinMapper; private final TaskProgressLightAssembler taskProgressLightAssembler; + /** + * 事务边界收口用：Redis 任务锁必须在事务外获取，否则自旋等待期间会白占一个 Hikari 连接。 + */ + private final org.springframework.transaction.PlatformTransactionManager transactionManager; private FileTaskEntity loadTaskForExecution(Long taskId) { Map cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId)); @@ -280,7 +286,9 @@ public class ShopMatchTaskService { return vo; } - @Transactional + /** + * 事务边界：任务锁在事务外获取（自旋等待不再占用 DB 连接），DB 段仍在同一事务内。 + */ public void deleteHistory(Long resultId, Long userId) { if (userId == null || userId <= 0) { throw new BusinessException("user_id 不合法"); @@ -295,16 +303,21 @@ public class ShopMatchTaskService { return; } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - FileResultEntity latestEntity = fileResultMapper.selectById(resultId); - if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { - throw new BusinessException("record not found"); - } - fileResultMapper.deleteById(resultId); - reconcileTaskAfterResultRemoval(taskId); + inNewTransaction(() -> { + FileResultEntity latestEntity = fileResultMapper.selectById(resultId); + if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { + throw new BusinessException("record not found"); + } + fileResultMapper.deleteById(resultId); + reconcileTaskAfterResultRemoval(taskId); + return null; + }); } } - @Transactional + /** + * 事务边界：任务锁在事务外获取；缓存清理（Redis + 远端范围载荷）放到事务提交后。 + */ public void deleteTask(Long taskId, Long userId) { if (userId == null || userId <= 0) { throw new BusinessException("user_id 不合法"); @@ -314,12 +327,17 @@ public class ShopMatchTaskService { throw new BusinessException("任务不存在"); } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - fileResultMapper.delete(new LambdaQueryWrapper() - .eq(FileResultEntity::getTaskId, taskId) - .eq(FileResultEntity::getModuleType, MODULE_TYPE)); - fileTaskMapper.deleteById(taskId); + inNewTransaction(() -> { + fileResultMapper.delete(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE)); + fileTaskMapper.deleteById(taskId); + return null; + }); + // 缓存清理（Redis + 远端范围载荷）在事务提交后、锁内执行 shopMatchTaskCacheService.deleteTaskCache(taskId); } + log.info("[shop-match] deleteTask 完成 taskId={} userId={}", taskId, userId); } private void reconcileTaskAfterResultRemoval(Long taskId) { @@ -500,7 +518,9 @@ public class ShopMatchTaskService { return vo; } - @Transactional + /** + * 事务边界：任务锁在事务外获取；心跳（Redis）放到事务提交后写。 + */ public void activateTask(Long taskId, Long userId, Integer stageIndex) { if (taskId == null || taskId <= 0) { throw new BusinessException("taskId 不合法"); @@ -512,45 +532,53 @@ public class ShopMatchTaskService { throw new BusinessException("stage_index 不合法"); } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - FileTaskEntity task = loadTaskForExecution(taskId); - if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { - throw new BusinessException("任务不存在"); - } - if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) { - throw new BusinessException("任务状态不正确"); - } - if (!"SCHEDULED".equals(task.getStatus())) { - throw new BusinessException("任务状态不正确"); - } - LocalDateTime now = now(); - if (task.getScheduledAt() != null && now.isBefore(task.getScheduledAt().minusSeconds(90))) { - log.warn("[shop-match] activate rejected taskId={} stageIndex={} now={} scheduledAt={} earliestActivateAt={}", - taskId, stageIndex, now, task.getScheduledAt(), task.getScheduledAt().minusSeconds(90)); - throw new BusinessException("未到定时执行时间"); - } - ShopMatchCreateTaskRequest state = parseTaskRequest(task); - int currentStageIndex = state.getCurrentStageIndex() == null ? 0 : state.getCurrentStageIndex(); - if (currentStageIndex != stageIndex) { - throw new BusinessException("当前轮次已变化，请刷新后重试"); - } - if (state.getScheduleTimes() == null || stageIndex >= state.getScheduleTimes().size()) { - throw new BusinessException("轮次配置缺失"); - } - FileTaskEntity runningTask = findOtherRunningTask(userId, taskId); - if (runningTask != null) { - log.warn("[shop-match] activate serialized taskId={} stageIndex={} blockedByTaskId={} blockedUpdatedAt={}", - taskId, stageIndex, runningTask.getId(), runningTask.getUpdatedAt()); - throw new BusinessException("当前已有定时匹配任务执行中，请等待上一任务执行结束后再重试"); - } - state.setActiveStageIndex(stageIndex); - task.setStatus("RUNNING"); - task.setUpdatedAt(now); - persistTaskRequest(task, state); - updateTaskAndRefreshCache(task); + inNewTransaction(() -> { + activateTaskLocked(taskId, userId, stageIndex); + return null; + }); + // 心跳是 Redis 写，放到事务提交、锁内执行（与改前一样仍在锁内） shopMatchTaskCacheService.touchTaskHeartbeat(taskId); } } + private void activateTaskLocked(Long taskId, Long userId, Integer stageIndex) { + FileTaskEntity task = loadTaskForExecution(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { + throw new BusinessException("任务不存在"); + } + if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) { + throw new BusinessException("任务状态不正确"); + } + if (!"SCHEDULED".equals(task.getStatus())) { + throw new BusinessException("任务状态不正确"); + } + LocalDateTime now = now(); + if (task.getScheduledAt() != null && now.isBefore(task.getScheduledAt().minusSeconds(90))) { + log.warn("[shop-match] activate rejected taskId={} stageIndex={} now={} scheduledAt={} earliestActivateAt={}", + taskId, stageIndex, now, task.getScheduledAt(), task.getScheduledAt().minusSeconds(90)); + throw new BusinessException("未到定时执行时间"); + } + ShopMatchCreateTaskRequest state = parseTaskRequest(task); + int currentStageIndex = state.getCurrentStageIndex() == null ? 0 : state.getCurrentStageIndex(); + if (currentStageIndex != stageIndex) { + throw new BusinessException("当前轮次已变化，请刷新后重试"); + } + if (state.getScheduleTimes() == null || stageIndex >= state.getScheduleTimes().size()) { + throw new BusinessException("轮次配置缺失"); + } + FileTaskEntity runningTask = findOtherRunningTask(userId, taskId); + if (runningTask != null) { + log.warn("[shop-match] activate serialized taskId={} stageIndex={} blockedByTaskId={} blockedUpdatedAt={}", + taskId, stageIndex, runningTask.getId(), runningTask.getUpdatedAt()); + throw new BusinessException("当前已有定时匹配任务执行中，请等待上一任务执行结束后再重试"); + } + state.setActiveStageIndex(stageIndex); + task.setStatus("RUNNING"); + task.setUpdatedAt(now); + persistTaskRequest(task, state); + updateTaskAndRefreshCache(task); + } + @Transactional public void completeStage(Long taskId, Long userId, ShopMatchStageCompleteRequest request) { if (taskId == null || taskId <= 0) { @@ -960,6 +988,19 @@ public class ShopMatchTaskService { return taskDistributedLockService.acquire(MODULE_TYPE, taskId); } + /** + * 显式短事务：锁已在事务外获取，这里只包住 DB 段， + * 保证原来同属一个 @Transactional 的多条写仍在同一事务内。 + */ + private T inNewTransaction(java.util.function.Supplier action) { + if (transactionManager == null) { + return action.get(); + } + TransactionTemplate template = new TransactionTemplate(transactionManager); + template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + return template.execute(status -> action.get()); + } + private boolean shouldRemainRunningUntilNextStage(FileTaskEntity task) { ShopMatchCreateTaskRequest request = parseTaskRequestSilently(task); if (request == null || request.getScheduleTimes() == null || request.getScheduleTimes().isEmpty()) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java index 4d5e5c54..7bf52ee3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -526,7 +526,12 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost { return vo; } - @Transactional + /** + * 启动任务（PENDING→RUNNING）。 + * + *

事务边界：Redis 任务锁必须在事务外获取（自旋等待期间会白占一个 Hikari 连接）； + * 本方法只有一条条件 UPDATE（自身即原子），故不再包事务。 + */ public void activateTask(Long taskId, Long userId) { try (TaskDistributedLockService.LockHandle ignored = ownershipSupport().requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) { FileTaskEntity task = fileTaskMapper.selectById(taskId); @@ -1323,7 +1328,12 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost { } } - @Transactional + /** + * 结果文件 job 失败的收尾。 + * + *

事务边界：任务锁在事务外获取；DB 段本来就走 {@code inNewTransaction}（REQUIRES_NEW）， + * 方法级 @Transactional 只会额外占一个空闲连接，故去掉。 + */ public void handleResultFileJobFailure(TaskFileJobEntity job, String message) { if (job == null || job.getTaskId() == null) { return; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineHost.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineHost.java index ff3580f0..15c7bada 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineHost.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineHost.java @@ -9,7 +9,8 @@ import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; *

* 采用依赖倒置：由宿主（SimilarAsinTaskService）实现本接口，流水线只依赖接口， * 避免 support 包与 Service 之间形成循环依赖。这些方法保留在宿主的两个原因： - * 一是属于任务/结果文件任务的编排与事务边界（{@code handleResultFileJobFailure} 带 @Transactional）， + * 一是属于任务/结果文件任务的编排与事务边界（{@code handleResultFileJobFailure} 先取任务锁、 + * 再用 {@code inNewTransaction} 显式包 DB 段 —— 锁必须在事务外获取，故不再挂方法级 @Transactional）， * 二是被宿主自身的门面路径复用。 */ public interface SimilarAsinPipelineHost { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java index 338c6dc6..a7e9f0d8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java @@ -139,10 +139,20 @@ public class TaskResultFileJobWorker { taskQueueExecutor.execute(() -> processClaimedWithHeartbeat(job, claim)); return; } catch (RuntimeException ex) { - log.warn("[task-file-job] llm module offload failed, fallback inline jobId={} taskId={} moduleType={} msg={}", + // 闸门拒绝（等待队列已满）说明系统已满载：此时**不能**回退成内联执行—— + // 那等于把背压转嫁给 MQ 消费线程，让消费线程按分钟级阻塞在 Excel 组装上， + // 队列越满越糟。改为重新入队，由归属实例 15s 一轮的扫描补发。 + log.warn("[task-file-job] 结果文件任务队列已满，改为重新入队 jobId={} taskId={} moduleType={} msg={} stage=REQUEUE", job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex); + taskFileJobService.requeue(job.getId(), "结果文件任务队列已满，稍后重投"); + return; } } + // 注意：supportsAsyncOffload()=false 的模块（11 个）仍在本方法调用方的线程上 + // 内联执行，MQ 侧实际并发度 = consumeThreadNumber 默认值 20。该并发度直接决定 + // 同时在跑的 Excel 组装数量，而每一路都持有 DB 连接（Hikari 池仅 30，还要供 + // Tomcat 200 线程与调度线程使用）。若要收敛，应显式设置 + // RocketMQMessageListener.consumeThreadNumber 并先测吞吐，不要在这里临时改。 processClaimedWithHeartbeat(job, claim); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java index 12ba620e..5e379b06 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java @@ -89,6 +89,14 @@ public class TaskResultItemService { return moduleType + "|" + scopeHash + "|" + itemKey; } + /** + * 载入本任务的**全部**结果快照（无 limit）。 + * + *

刻意不加 limit：调用方要用它组装完整的结果文件，截断即漏行（正确性错误， + * 不是性能取舍）。调用方多为「逐任务循环调用」，因此峰值内存 = 单个任务的结果集， + * 这是本操作固有的代价。若要再降，需要改成「边读边写文件」的流式组装契约， + * 涉及 4 个模块的组装实现，属独立工程。 + */ public List listResultSnapshots(Long taskId, String moduleType, Class clazz) { if (taskId == null || taskId <= 0 || isBlank(moduleType)) { return List.of(); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java index 2dd727fa..5ca488c3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java @@ -517,11 +517,8 @@ public class TaskScopePayloadStorageService { try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8)); - StringBuilder sb = new StringBuilder(bytes.length * 2); - for (byte b : bytes) { - sb.append(String.format("%02x", b)); - } - return sb.toString(); + // HexFormat 替代逐字节 String.format("%02x")：后者每次走 32 次 Formatter（热路径按行） + return java.util.HexFormat.of().formatHex(bytes); } catch (Exception ex) { throw new IllegalStateException("failed to hash scope key", ex); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawTaskService.java index 1dac9d3c..88fc0f0d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawTaskService.java @@ -35,6 +35,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; import java.io.File; import java.math.BigDecimal; @@ -79,6 +81,10 @@ public class WithdrawTaskService { private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskDistributedLockService taskDistributedLockService; private final TaskProgressLightAssembler taskProgressLightAssembler; + /** + * 事务边界收口用：Redis 任务锁必须在事务外获取，否则自旋等待期间会白占一个 Hikari 连接。 + */ + private final org.springframework.transaction.PlatformTransactionManager transactionManager; public ProductRiskDashboardVo dashboard(Long userId) { validateUserId(userId); @@ -347,7 +353,10 @@ public class WithdrawTaskService { } } - @Transactional + /** + * 事务边界：任务锁在事务外获取（自旋等待不再占用 DB 连接）； + * 缓存清理（Redis + 远端范围载荷）放到事务提交后、锁内执行。 + */ public void deleteTask(Long taskId, Long userId) { validateUserId(userId); FileTaskEntity task = loadTaskForExecution(taskId); @@ -355,18 +364,24 @@ public class WithdrawTaskService { throw new BusinessException("任务不存在"); } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - fileResultMapper.delete(new LambdaQueryWrapper() - .eq(FileResultEntity::getTaskId, taskId) - .eq(FileResultEntity::getModuleType, MODULE_TYPE)); - fileTaskMapper.deleteById(taskId); - taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); - taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE); - taskProgressSnapshotService.delete(taskId, MODULE_TYPE); + inNewTransaction(() -> { + fileResultMapper.delete(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE)); + fileTaskMapper.deleteById(taskId); + taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); + taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE); + taskProgressSnapshotService.delete(taskId, MODULE_TYPE); + return null; + }); taskCacheService.deleteTaskCache(taskId); } + log.info("[withdraw] deleteTask 完成 taskId={} userId={}", taskId, userId); } - @Transactional + /** + * 事务边界：任务锁在事务外获取，DB 段（复核 + 删除 + 重算）仍在同一事务内。 + */ public void deleteHistory(Long resultId, Long userId) { validateUserId(userId); FileResultEntity entity = fileResultMapper.selectById(resultId); @@ -379,10 +394,13 @@ public class WithdrawTaskService { return; } try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { - fileResultMapper.deleteById(resultId); - taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId); - taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId); - reconcileTaskAfterResultRemoval(taskId); + inNewTransaction(() -> { + fileResultMapper.deleteById(resultId); + taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId); + taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId); + reconcileTaskAfterResultRemoval(taskId); + return null; + }); } } @@ -1050,6 +1068,19 @@ public class WithdrawTaskService { return taskDistributedLockService.acquire(MODULE_TYPE, taskId); } + /** + * 显式短事务：锁已在事务外获取，这里只包住 DB 段， + * 保证原来同属一个 @Transactional 的多条写仍在同一事务内。 + */ + private T inNewTransaction(java.util.function.Supplier action) { + if (transactionManager == null) { + return action.get(); + } + TransactionTemplate template = new TransactionTemplate(transactionManager); + template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + return template.execute(status -> action.get()); + } + private List normalizeTaskIds(List taskIds) { return taskIds == null ? List.of() : taskIds.stream() .filter(taskId -> taskId != null && taskId > 0) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/client/ZiniaoClientImpl.java b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/client/ZiniaoClientImpl.java index 5b252974..1ca4dcf6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/client/ZiniaoClientImpl.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/client/ZiniaoClientImpl.java @@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.ziniao.client; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.util.BoundedLruCache; import com.nanri.aiimage.config.ZiniaoProperties; import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopCacheDto; import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoStaffItemVo; @@ -59,8 +60,13 @@ public class ZiniaoClientImpl implements ZiniaoClient { /** Task 77：单例 RestClient（共享连接池），避免每次调用新建短命客户端。 */ private volatile RestClient sharedRestClient; - /** 按代理地址缓存 RestClient：同一代理复用同一连接池，避免每请求新建。 */ - private final Map proxyRestClients = new java.util.concurrent.ConcurrentHashMap<>(); + /** + * 按代理地址缓存 RestClient：同一代理复用同一连接池，避免每请求新建。 + * 有界 LRU——代理端点会随店铺轮换不断新增（每个都持有连接池与 selector 线程）， + * 无上限时按店铺数持续累积。 + */ + private final BoundedLruCache proxyRestClients = + new BoundedLruCache<>(BoundedLruCache.DEFAULT_MAX_SIZE); @Override public Long getCompanyIdByApiKey(String apiKey) { @@ -124,6 +130,15 @@ public class ZiniaoClientImpl implements ZiniaoClient { return parseUserStores(raw); } + /** + * 获取员工登录 token。 + * + *

**刻意不加缓存**（2026-09-15 复核）：调用方 {@code ZiniaoAuthService} 命中店铺匹配缓存时 + * 仍会用**新取的** token 重建开店铺 URL（见其 getCachedShopMatch 分支只缓存匹配结果、 + * openStoreUrl 存 null 后重算），说明该 token 是「一次开店铺一次」的短时凭据， + * 缓存后会把已被消费的 token 发出去，用户点开店铺会失败。上游往返只为单次用户动作 + * 服务，不是吞吐瓶颈，不要为省一次 RTT 引入这个正确性风险。 + */ @Override public String getUserLoginToken(String apiKey, Long companyId, Long userId) { String raw = postWithApiKey(apiKey, ziniaoProperties.getUserLoginTokenPath(), Map.of( diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoAuthService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoAuthService.java index f2e6a203..4cc5febb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoAuthService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoAuthService.java @@ -539,11 +539,8 @@ public boolean isIpWhitelistError(BusinessException ex) { try { java.security.MessageDigest digest = java.security.MessageDigest.getInstance("SHA-256"); byte[] hash = digest.digest(apiKey.getBytes(StandardCharsets.UTF_8)); - StringBuilder sb = new StringBuilder(); - for (byte b : hash) { - sb.append(String.format("%02x", b)); - } - return sb.toString(); + // HexFormat 替代逐字节 String.format("%02x")：后者每次走 32 次 Formatter（热路径按调用） + return java.util.HexFormat.of().formatHex(hash); } catch (Exception ex) { throw new BusinessException("生成紫鸟 apiKey 缓存键失败"); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java index 80f9b237..edc52dbc 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java @@ -937,11 +937,8 @@ public class ZiniaoShopIndexService { try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); byte[] hash = digest.digest(apiKey.getBytes(StandardCharsets.UTF_8)); - StringBuilder sb = new StringBuilder(); - for (byte b : hash) { - sb.append(String.format("%02x", b)); - } - return sb.toString(); + // HexFormat 替代逐字节 String.format("%02x")：后者每次走 32 次 Formatter（热路径按调用） + return java.util.HexFormat.of().formatHex(hash); } catch (Exception ex) { throw new BusinessException("生成紫鸟 apiKey 缓存键失败"); } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskServiceTest.java index 12d596f6..ba6751cd 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskServiceTest.java @@ -9,6 +9,10 @@ import com.nanri.aiimage.modules.imagevideo.mapper.ImageVideoAsyncTaskMapper; import com.nanri.aiimage.modules.imagevideo.model.dto.DouyinCopyRequest; import com.nanri.aiimage.modules.imagevideo.model.entity.ImageVideoAsyncTaskEntity; import com.nanri.aiimage.modules.imagevideo.model.vo.DouyinCopyVo; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.springframework.core.task.TaskExecutor; import org.mockito.ArgumentCaptor; @@ -29,6 +33,18 @@ import static org.mockito.Mockito.when; class ImageVideoAsyncTaskServiceTest { + /** + * MyBatis-Plus 的 lambda 列解析需要实体的 TableInfo。本类用 mock mapper、不启动 Spring， + * 而 dispatch/poll 扫描现在走 select(ImageVideoAsyncTaskEntity::getId)（2026-09 + * 「不再每秒把 5 个 LONGTEXT 列拉回」优化引入），没有 TableInfo 会抛 + * 「can not find lambda cache」。同款初始化见 AppearancePatentTaskServiceHistoryBatchTest。 + */ + @BeforeAll + static void initMybatisPlusTableInfo() { + TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), + ImageVideoAsyncTaskEntity.class); + } + @Test void parsesNestedDouyinCopyOutputIntoFrontendTextFields() { ImageVideoCozeService cozeService = new ImageVideoCozeService( diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/permission/service/PermissionMenuServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/permission/service/PermissionMenuServiceTest.java index b9aac381..96e9380d 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/permission/service/PermissionMenuServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/permission/service/PermissionMenuServiceTest.java @@ -13,6 +13,10 @@ import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEnt import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo; import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo; import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -30,6 +34,18 @@ import static org.mockito.Mockito.when; class PermissionMenuServiceTest { + /** + * MyBatis-Plus 的 lambda 列解析需要实体的 TableInfo。本类用 mock mapper、不启动 Spring， + * 而 PermissionMenuService 的权限用户查询走 select(AdminUserEntity::getId, ...)（2026-09 + * 「不再整表拉回含密码哈希的全列」优化引入），没有 TableInfo 会在构造 wrapper 时抛 + * 「can not find lambda cache」。同款初始化见 AppearancePatentTaskServiceHistoryBatchTest。 + */ + @BeforeAll + static void initMybatisPlusTableInfo() { + TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), + AdminUserEntity.class); + } + @Test void preservesExistingImageVideoPermissionDuringGenericReplacement() { PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinSnapshotJsonThrottleTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinSnapshotJsonThrottleTest.java index f28a5953..bafa07bc 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinSnapshotJsonThrottleTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinSnapshotJsonThrottleTest.java @@ -36,10 +36,11 @@ class QueryAsinSnapshotJsonThrottleTest { private QueryAsinTaskService service() { // 构造顺序：mapper/resultMapper/resolve/excel/cache/oss/resultDownloadResolver/ziniaoSwitch/ - // objectMapper/pressure/jobService/resultItemService/snapshot/lock/lightAssembler + // objectMapper/pressure/jobService/resultItemService/snapshot/lock/lightAssembler/transactionManager + // transactionManager 传 null：本用例只走 persistSnapshotJson，不涉及需要事务的删除链路 return new QueryAsinTaskService( null, null, null, null, null, null, null, null, - objectMapper, null, null, taskResultItemService, taskProgressSnapshotService, null, null); + objectMapper, null, null, taskResultItemService, taskProgressSnapshotService, null, null, null); } private static void persist(QueryAsinTaskService service, FileTaskEntity task, diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceTxBoundaryTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceTxBoundaryTest.java index a5897b06..1d3809ab 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceTxBoundaryTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceTxBoundaryTest.java @@ -31,6 +31,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; @@ -49,6 +50,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -196,11 +198,38 @@ class ShopDataCrawlTaskServiceTxBoundaryTest { verify(fileResultMapper).updateById(any(FileResultEntity.class)); } + /** + * 事务边界契约（2026-09 连接池修复）：deleteTask 不再挂方法级 @Transactional —— + * Redis 任务锁必须在事务外获取，否则自旋等待（最长 TASK_LOCK_WAIT_MILLIS）期间会白占一个 + * Hikari 连接。落库删除仍在同一个短事务（executeShortTransaction / REQUIRES_NEW）里完成， + * 且顺序固定为「先取锁、后开事务、再落库删除」。 + */ @Test - void deleteTaskKeepsTransactionAnnotationAndPureCompute() throws Exception { + void deleteTaskKeepsLockOutsideAndWritesInsideSingleShortTransaction() throws Exception { Method delete = ShopDataCrawlTaskService.class.getMethod("deleteTask", Long.class, Long.class); - assertTrue(delete.getAnnotation(Transactional.class) != null, - "deleteTask 落库删除必须保留 @Transactional（锁内删除语义不变）"); + assertNull(delete.getAnnotation(Transactional.class), + "deleteTask 不得带 @Transactional（Redis 锁必须落在事务外，避免自旋期间占用 DB 连接）"); + + FileTaskEntity task = runningTask(); + when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task); + FileResultEntity row = resultRow(1L); + row.setSuccess(1); + when(fileResultMapper.selectList(any())).thenReturn(List.of(row)); + when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", TASK_ID)) + .thenReturn(mock(TaskDistributedLockService.LockHandle.class)); + when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, "SHOP_DATA_CRAWL", 1L)).thenReturn(true); + when(dailyFileService.findMembersByResultId(1L)) + .thenReturn(List.of(mock(com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity.class))); + + service.deleteTask(TASK_ID, USER_ID); + + InOrder order = inOrder(taskDistributedLockService, transactionManager, fileResultMapper); + order.verify(taskDistributedLockService).acquire("SHOP_DATA_CRAWL", TASK_ID); + order.verify(transactionManager).getTransaction(any()); + order.verify(fileResultMapper).delete(any()); + // 全部落库删除共用一个短事务 + verify(transactionManager, times(1)).getTransaction(any()); + verify(fileTaskMapper).deleteById(TASK_ID); // 删除路径使用抽出的纯函数（行为等价由既有 ShopDataCrawlCleanupTest 守门） assertTrue(ShopDataCrawlTaskService.collectResultIds(List.of()).isEmpty()); } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOffloadTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOffloadTest.java index c908d5e3..5b09e5e3 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOffloadTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOffloadTest.java @@ -167,8 +167,15 @@ class TaskResultFileJobWorkerOffloadTest { verify(similar).cleanupResultFileJob(job); } + /** + * 闸门拒绝时的契约（2026-09 背压修复）：**不再回退成内联执行**。 + * + * 内联兜底会把背压转嫁给 MQ 消费线程——消费线程按分钟级阻塞在 Excel 组装 + OSS 上传上， + * 队列越满越糟。改为重新入队（置回 PENDING），由归属实例 15s 一轮的扫描补发； + * 任务不会丢，只是不在消费线程上抢跑。原用例名 offloadFailureFallback 断言的正是旧行为。 + */ @Test - void offloadFailureFallback() throws Exception { + void offloadRejectedRequeuesInsteadOfInlineFallback() throws Exception { TaskResultFileJobWorker worker = buildWorker(); TaskFileJobEntity job = job("SIMILAR_ASIN", 4L, 14L); allowClaim(job.getId(), job.getTaskId(), job.getModuleType()); @@ -178,8 +185,9 @@ class TaskResultFileJobWorkerOffloadTest { worker.process(job); - verify(taskFileJobService).markSuccess(job, null); - verify(similar).cleanupResultFileJob(job); + verify(taskFileJobService).requeue(job.getId(), "结果文件任务队列已满，稍后重投"); + verify(taskFileJobService, never()).markSuccess(any(), any()); + verify(similar, never()).processResultFileJob(any()); } @Test diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOrphanTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOrphanTest.java index e82c2aa9..88546c59 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOrphanTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerOrphanTest.java @@ -90,13 +90,21 @@ class TaskResultFileJobWorkerOrphanTest { new BrandResultFileJobHandler(brand, payload), new CollectDataResultFileJobHandler(collectData)); ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers); - return new TaskResultFileJobWorker( + TaskResultFileJobWorker worker = new TaskResultFileJobWorker( taskFileJobService, taskDistributedLockService, mock(FileResultMapper.class), mock(TaskFileJobLocalDispatcher.class), instanceMetadata, registry); + // taskQueueExecutor 是字段注入。此前不注入时字段为 null，offload 分支会抛 NPE 并被 + // 兜底成内联执行——本测试一直依赖这个偶然行为。2026-09 起闸门拒绝改为「重新入队」 + // 而不再回退内联，故这里显式注入同步执行器，让 offload 模块仍按内联语义跑完整链路。 + java.lang.reflect.Field executorField = + TaskResultFileJobWorker.class.getDeclaredField("taskQueueExecutor"); + executorField.setAccessible(true); + executorField.set(worker, (org.springframework.core.task.TaskExecutor) Runnable::run); + return worker; } private static TaskFileJobEntity job(String moduleType, long jobId, long taskId) { diff --git a/backend/app.py b/backend/app.py index e1ff07d8..5189906a 100644 --- a/backend/app.py +++ b/backend/app.py @@ -11,7 +11,7 @@ import secrets from flask import Flask from flask_cors import CORS -from utils.db import init_db +from utils.db import close_db, init_db from blueprints.version import version_bp BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -36,6 +36,9 @@ app.secret_key = os.environ.get('SECRET_KEY', secrets.token_hex(32)) # 注册蓝图：仅版本公开 API（管理后台 API 已迁 Java，见模块 docstring） app.register_blueprint(version_bp) +# 请求级数据库连接随请求结束释放（utils/db.get_db 复用同一连接） +app.teardown_appcontext(close_db) + def run_app(host='0.0.0.0', port=15124): init_db() diff --git a/backend/utils/db.py b/backend/utils/db.py index 13f03156..7083738f 100644 --- a/backend/utils/db.py +++ b/backend/utils/db.py @@ -1,9 +1,26 @@ """ 数据库连接与初始化 + +请求级连接复用：pymysql 每次 connect 都要做一次完整的鉴权握手，而本进程 +（15124）只服务 /api/version、/api/version/latest 这类轻量查询，握手开销 +占了单次请求的大头。连接因此挂在 flask.g 上按请求复用，请求结束由 +close_db（注册在 app.teardown_appcontext）统一释放。 """ import re + import pymysql +try: + from flask import g, has_app_context +except ImportError: # 非 Flask 环境（脚本/离线核查）下退化为每次新建连接 + g = None + + def has_app_context() -> bool: + return False + +# 请求级连接挂在 flask.g 上的属性名（避免与业务字段冲突） +_CONN_ATTR = "_shufu_db_conn" + try: from config import mysql_host as config_mysql_host from config import mysql_user as config_mysql_user @@ -44,7 +61,7 @@ def describe_db_target(): ) -def get_db(): +def _connect(): return pymysql.connect( host=mysql_host, user=mysql_user, @@ -55,6 +72,73 @@ def get_db(): ) +def _is_alive(conn) -> bool: + """连接可用性探测：已关闭直接判死，否则发一次 COM_PING。 + + COM_PING 一次往返远小于一次完整鉴权握手；不用 ping(reconnect=True) + （该参数在新版 pymysql 已废弃，且失败时会自行重连、语义不可控）。 + """ + try: + if not conn.open: + return False + conn.ping(reconnect=False) + return True + except Exception: + return False + + +def _close_quietly(conn) -> None: + """静默关闭连接（可能为 None / 已关闭 / 断连）。""" + if conn is None: + return + try: + conn.close() + except Exception: + pass + + +def get_db(): + """取当前请求的数据库连接（同一请求内复用，避免每次请求都做鉴权握手）。 + + 连接挂在 flask.g 上，请求结束由 close_db 释放；被服务端 wait_timeout + 断开、或调用方提前 close 过，这里都会静默换成新连接，调用方无感知。 + 无 Flask 应用上下文时（脚本调用）退回每次新建，保持旧行为。 + """ + ctx = g if has_app_context() else None + conn = getattr(ctx, _CONN_ATTR, None) if ctx is not None else None + if conn is not None: + if _is_alive(conn): + return conn + # 断连/已被关闭：丢弃旧连接后重建（不把坏连接交给业务查询） + _close_quietly(conn) + try: + delattr(ctx, _CONN_ATTR) + except Exception: + pass + conn = _connect() + if ctx is not None: + setattr(ctx, _CONN_ATTR, conn) + return conn + + +def close_db(exc=None) -> None: + """释放当前请求的连接（注册为 app.teardown_appcontext 回调）。 + + 异常路径同样会走 teardown，因此查询抛错时连接不会泄漏到下一次请求。 + """ + ctx = g if has_app_context() else None + if ctx is None: + return + conn = getattr(ctx, _CONN_ATTR, None) + if conn is None: + return + try: + delattr(ctx, _CONN_ATTR) + except Exception: + pass + _close_quietly(conn) + + def init_db(): """确保版本公开 API 依赖的最小表结构存在。 diff --git a/frontend-vue/src/pages/brand/components/BrandBrandTab.vue b/frontend-vue/src/pages/brand/components/BrandBrandTab.vue index a1ec5a95..2b34a220 100644 --- a/frontend-vue/src/pages/brand/components/BrandBrandTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandBrandTab.vue @@ -194,23 +194,29 @@ async function pollOnce() { for (const key of Object.keys(lineProgressMap.value)) { if (!busy.some((b) => String(b.id) === key)) delete lineProgressMap.value[key] } - for (const item of busy) { - const id = Number(item.id) - if (!id) continue - try { - const res = await getBrandTask(id) - const lp = res?.line_progress - if (lp?.has_progress && lp.info) { - lineProgressMap.value[String(id)] = { - current: Number(lp.info.current_line) || 0, - total: Number(lp.info.total_lines) || 0, - phase: lp.info.phase, - } - } else { - delete lineProgressMap.value[String(id)] + // 并发拉取：此前在 for 循环里逐个 await，任务多时一轮耗时随任务数线性叠加 + //（brand 模块没有批量进度端点，见 shared/api/endpoints.ts，只能逐个查，故用并发而非 batch） + const busyIds = busy.map((item) => Number(item.id)).filter((id) => Number.isFinite(id) && id > 0) + const details = await Promise.all( + busyIds.map(async (id) => { + try { + return { id, res: await getBrandTask(id) } + } catch { + // 单条详情失败不中断整轮轮询 + return { id, res: null } } - } catch { - // 单条详情失败不中断整轮轮询 + }), + ) + for (const { id, res } of details) { + const lp = res?.line_progress + if (lp?.has_progress && lp.info) { + lineProgressMap.value[String(id)] = { + current: Number(lp.info.current_line) || 0, + total: Number(lp.info.total_lines) || 0, + phase: lp.info.phase, + } + } else { + delete lineProgressMap.value[String(id)] } } scheduleNextPoll() diff --git a/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue b/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue index 5cea8a8a..8f4de4f0 100644 --- a/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue @@ -179,6 +179,7 @@ import { type CollectDataTaskDetailVo, type UploadFileVo, } from '@/shared/api/java-modules' +import { getPollingProgressBatch } from '@/shared/api/task-progress-polling.ts' import { formatDateTime } from '@/shared/utils/datetime' import { uploadPathsToJava } from '@/shared/utils/upload-to-java' import { runBatchDelete } from '@/shared/utils/batch-delete' @@ -236,24 +237,70 @@ const taskSnapshots = ref>({}) const POLLING_STORAGE_KEY = 'brand:collect-data:polling-task-ids' +/** 轮询可比对的轻量字段（/tasks/progress/light 白名单里本模块快照有的字段），只有它们变化才重建快照对象 */ +function taskProgressSignatureOf(detail: CollectDataTaskDetailVo) { + const task = detail.task + if (!task) return '' + return [task.status ?? '', task.updatedAt ?? ''].join('|') +} + +/** + * currentItems 的 taskId → 下标索引。 + * 轮询每轮都要按 taskId 定位行，此前每次都对整份列表 map 一遍；有索引后只改命中的那一行。 + * 列表被整体替换（loadHistory）时必须重建；使用处还会做一次 taskId 校验兜底。 + */ +let currentItemsIndex = new Map() + +function rebuildCurrentItemsIndex() { + const next = new Map() + currentItems.value.forEach((item, index) => { + const taskId = Number(item.taskId) + if (Number.isFinite(taskId) && taskId > 0) next.set(taskId, index) + }) + currentItemsIndex = next +} + +/** 原地更新 currentItems 里指定任务的那一行（找不到就什么都不做） */ +function patchCurrentItem( + taskId: number, + latestItem: CollectDataHistoryItem | undefined, + status: string | undefined, +) { + let index = currentItemsIndex.get(taskId) + if (index === undefined || Number(currentItems.value[index]?.taskId) !== taskId) { + rebuildCurrentItemsIndex() + index = currentItemsIndex.get(taskId) + } + if (index === undefined) return + const current = currentItems.value[index] + if (!current) return + currentItems.value[index] = { + ...current, + ...(latestItem || {}), + taskStatus: status ?? latestItem?.taskStatus ?? current.taskStatus, + } +} + const progressLoop = useTaskProgressLoop({ scope: 'collect-data-tab', storageKey: POLLING_STORAGE_KEY, - fetchProgress: (ids) => getCollectDataTaskProgressBatch(ids), + // F5：主轮询只读 status/fileReady 等轻量白名单字段，走 light 端点减小响应体； + // 轻量端点异常/空结果时适配器自动回退重型 batch（否则会漏刷新结果） + fetchProgress: (ids) => getPollingProgressBatch('collectData', ids, { + fallback: () => getCollectDataTaskProgressBatch(ids), + }), extractTaskId: (detail) => detail.task?.id ?? null, extractStatus: (detail) => detail.task?.status ?? '', onUpdate: (taskId, detail) => { const prev = taskSnapshots.value[taskId] - taskSnapshots.value = { - ...taskSnapshots.value, - [taskId]: prev - ? { ...prev, task: { ...(prev.task || {}), ...(detail.task || {}) }, items: detail.items ?? prev.items } - : detail, + const merged: CollectDataTaskDetailVo = prev + ? { ...prev, task: { ...(prev.task || {}), ...(detail.task || {}) }, items: detail.items ?? prev.items } + : detail + // 只有轮询字段（或行明细）真的变了才展开快照对象：整份快照含行明细，无变化时重建纯属浪费 + if (!prev || merged.items !== prev.items || taskProgressSignatureOf(prev) !== taskProgressSignatureOf(merged)) { + taskSnapshots.value = { ...taskSnapshots.value, [taskId]: merged } } - const latestItem = detail.items?.[0] - currentItems.value = currentItems.value.map((item) => item.taskId === taskId - ? { ...item, ...(latestItem || {}), taskStatus: detail.task?.status ?? latestItem?.taskStatus ?? item.taskStatus } - : item) + patchCurrentItem(taskId, detail.items?.[0], detail.task?.status) }, onTerminal: async (taskId) => { if (taskSnapshots.value[taskId]) { @@ -624,6 +671,8 @@ async function loadHistory() { } } currentItems.value = merged + // 列表整体替换后重建 taskId 索引，轮询就能原地更新命中的那一行 + rebuildCurrentItemsIndex() } catch { /* 忽略：历史接口失败时保持空列表 */ } diff --git a/frontend-vue/src/pages/brand/components/BrandPatrolDeleteTab.vue b/frontend-vue/src/pages/brand/components/BrandPatrolDeleteTab.vue index ce3469e6..8ba9181b 100644 --- a/frontend-vue/src/pages/brand/components/BrandPatrolDeleteTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandPatrolDeleteTab.vue @@ -33,7 +33,7 @@

+
删除条件 @@ -310,6 +318,13 @@ const { total: matchedTotal, paged: pagedMatchedItems, } = useTablePaging(matchedItems) +// F9：备选店铺表同样只切渲染窗口（分页不改变数据源，勾选/删除语义不变） +const { + page: candidatePage, + pageSize: candidatePageSize, + total: candidateTotal, + paged: pagedCandidates, +} = useTablePaging(candidates) const historyItems = ref([]); const dashboard = ref({ candidateCount: 0, diff --git a/frontend-vue/src/pages/brand/components/BrandPriceTrackTab.vue b/frontend-vue/src/pages/brand/components/BrandPriceTrackTab.vue index e3a3827a..83f160c0 100644 --- a/frontend-vue/src/pages/brand/components/BrandPriceTrackTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandPriceTrackTab.vue @@ -467,10 +467,37 @@ function loadTaskSnapshotsFromStorage() { } } -function saveTaskSnapshotsToStorage() { +/** 快照落盘最小间隔：快照可达数百 KB，全量 JSON.stringify + setItem 会阻塞主线程 */ +const SNAPSHOT_SAVE_MIN_INTERVAL_MS = 30000 +let lastSnapshotSaveAt = 0 +let snapshotSavePending = false + +/** + * 快照写盘（节流）。 + * + * 每轮轮询都序列化整份快照会周期性卡顿主线程，因此： + * - 内容未变时 setStorageJson 内部已会跳过写入； + * - 这里再按 SNAPSHOT_SAVE_MIN_INTERVAL_MS 节流，被节流掉的写入挂起， + * 下一次（或 force / 组件卸载前的 flush）落盘时一并写入最新内容。 + */ +function saveTaskSnapshotsToStorage(force = false) { + if (typeof window === 'undefined') return + const now = Date.now() + if (!force && lastSnapshotSaveAt > 0 && now - lastSnapshotSaveAt < SNAPSHOT_SAVE_MIN_INTERVAL_MS) { + snapshotSavePending = true + return + } + snapshotSavePending = false + lastSnapshotSaveAt = now setStorageJson(taskSnapshotsStorageKey(), taskSnapshots.value, Object.keys(taskSnapshots.value).length === 0) } +/** 卸载前把节流挂起的快照补写一次，避免最后一次变更丢失 */ +function flushTaskSnapshotsToStorage() { + if (!snapshotSavePending) return + saveTaskSnapshotsToStorage(true) +} + function loadMatchedItemsFromStorage() { try { const raw = typeof window !== 'undefined' ? window.localStorage.getItem(matchedItemsStorageKey()) : null @@ -1147,7 +1174,7 @@ async function processMatchedQueue() { ...taskDetails.value, [taskVo.taskId]: 'RUNNING', } - saveTaskSnapshotsToStorage() + saveTaskSnapshotsToStorage(true) saveTaskDetailsToStorage() const queuePayload = await buildQueuePayload(taskVo, row) queuePayloadText.value = JSON.stringify(queuePayload, null, 2) @@ -1233,7 +1260,7 @@ function recordCreatedTask(taskVo: PriceTrackCreateTaskVo) { ...taskDetails.value, [taskVo.taskId]: 'RUNNING', } - saveTaskSnapshotsToStorage() + saveTaskSnapshotsToStorage(true) saveTaskDetailsToStorage() } @@ -1418,37 +1445,68 @@ function getPollIntervalMs() { return getTaskPollIntervalMs() } +/** 轮询可比对的轻量字段（与 /tasks/progress/light 白名单一致，外加只有重型端点才带的起止时间） */ +function taskProgressSignature(task?: PriceTrackTaskDetailVo['task'] | null) { + if (!task) return '' + return [ + task.status ?? '', + task.createdAt ?? '', + task.finishedAt ?? '', + task.updatedAt ?? '', + ].join('|') +} + async function refreshTaskBatch() { const ids = pollingTaskIds.value.filter((id) => id > 0) if (!ids.length) return try { - const batch = await getPriceTrackTaskProgressBatch(ids) - let changed = false + // F5：主轮询只读 status/fileReady 等轻量白名单字段，走 light 端点减小轮询响应体； + // 轻量端点异常或返回空结果时适配器自动回退重型 batch（否则会漏刷新结果） + const batch = await getPollingProgressBatch('priceTrack', ids, { + fallback: () => getPriceTrackTaskProgressBatch(ids), + }) + let settled = false + let snapshotsChanged = false const nextSnapshots = { ...taskSnapshots.value } for (const missingId of batch.missingTaskIds || []) { + console.log(`[price-track] 任务 ${missingId} 在服务端已不存在，停止轮询`) removePollingTask(missingId) delete nextSnapshots[missingId] + snapshotsChanged = true } for (const detail of batch.items || []) { const taskId = detail.task?.id const status = detail.task?.status if (typeof taskId !== 'number' || taskId <= 0) continue const prev = nextSnapshots[taskId] - nextSnapshots[taskId] = prev ? { ...prev, task: { ...(prev.task || {}), ...(detail.task || {}) } } : detail + const merged: PriceTrackTaskDetailVo = prev + ? { ...prev, task: { ...(prev.task || {}), ...(detail.task || {}) } } + : detail + // 只有轮询字段真的变了才重写快照：整份快照数百 KB，无变化时重建对象 + 落盘纯属浪费 + if (!prev || taskProgressSignature(prev.task) !== taskProgressSignature(merged.task)) { + nextSnapshots[taskId] = merged + snapshotsChanged = true + } if (status) { taskDetails.value[taskId] = status if (status === 'SUCCESS' || status === 'FAILED') { + console.log(`[price-track] 任务 ${taskId} 已到终态 ${status}，移出轮询集合并刷新列表`) removePollingTask(taskId) - changed = true + settled = true } } } - taskSnapshots.value = nextSnapshots - saveTaskSnapshotsToStorage() + if (snapshotsChanged) { + taskSnapshots.value = nextSnapshots + saveTaskSnapshotsToStorage() + } saveTaskDetailsToStorage() - await loadHistory() - syncPollingIdsWithHistory() - if (changed) { + // 历史列表只在任务落定（或快照真变化）时拉取：此前每 10s 无条件重查一次，纯属浪费 + if (settled || snapshotsChanged) { + await loadHistory() + syncPollingIdsWithHistory() + } + if (settled) { await loadDashboard() void resumeLoopExecutionIfNeeded() } @@ -1508,7 +1566,8 @@ function removePollingTask(taskId: number) { const next = { ...taskSnapshots.value } delete next[taskId] taskSnapshots.value = next - saveTaskSnapshotsToStorage() + // 终态清理属于关键变更，跳过节流立即落盘 + saveTaskSnapshotsToStorage(true) } } @@ -1679,6 +1738,8 @@ onUnmounted(() => { clearSleepTimers() timers.clearScope() if (countryPrefSaveTimer) timers.clearTimer('preference-save', countryPrefSaveTimer) + // 补写被节流挂起的快照，避免最后一次变更丢失 + flushTaskSnapshotsToStorage() }) /** diff --git a/frontend-vue/src/pages/brand/components/BrandQueryAsinTab.vue b/frontend-vue/src/pages/brand/components/BrandQueryAsinTab.vue index 0b973975..fe6de5c1 100644 --- a/frontend-vue/src/pages/brand/components/BrandQueryAsinTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandQueryAsinTab.vue @@ -33,7 +33,7 @@
+ @@ -245,6 +253,13 @@ const { total: matchedTotal, paged: pagedMatchedItems, } = useTablePaging(matchedItems) +// F9：备选店铺表同样只切渲染窗口（分页不改变数据源，勾选/删除语义不变） +const { + page: candidatePage, + pageSize: candidatePageSize, + total: candidateTotal, + paged: pagedCandidates, +} = useTablePaging(candidates) const historyItems = ref([]); const dashboard = ref({ candidateCount: 0, diff --git a/frontend-vue/src/pages/brand/components/BrandShopDataCrawlTab.vue b/frontend-vue/src/pages/brand/components/BrandShopDataCrawlTab.vue index 2192a50a..afa0a263 100644 --- a/frontend-vue/src/pages/brand/components/BrandShopDataCrawlTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandShopDataCrawlTab.vue @@ -17,7 +17,7 @@
备选区
暂无备选店铺
- @@ -28,6 +28,9 @@
+
抓取国家与顺序
@@ -187,6 +190,13 @@ const { total: matchedTotal, paged: pagedMatchedItems, } = useTablePaging(matchedItems) +// F9：备选店铺表同样只切渲染窗口（分页不改变数据源，勾选/删除语义不变） +const { + page: candidatePage, + pageSize: candidatePageSize, + total: candidateTotal, + paged: pagedCandidates, +} = useTablePaging(candidates) const historyItems = ref([]) const dashboard = ref({ candidateCount: 0, processedTaskCount: 0, successTaskCount: 0, failedTaskCount: 0 }) const orderedCountryCodes = ref(COUNTRY_OPTIONS.map((row) => row.code)) diff --git a/frontend-vue/src/pages/brand/components/BrandShopMatchTab.vue b/frontend-vue/src/pages/brand/components/BrandShopMatchTab.vue index b0205b77..5c4ff9f5 100644 --- a/frontend-vue/src/pages/brand/components/BrandShopMatchTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandShopMatchTab.vue @@ -452,7 +452,8 @@ function sleep(ms: number) { if (disposed) return Promise.resolve(); return time function clearSleepTimers() { timers.clearCategory('queue-wait') } function isTransientBackendError(error: unknown) { const message = (error instanceof Error ? error.message : String(error || '')).toLowerCase(); return TRANSIENT_BACKEND_ERROR_PATTERNS.some((pattern) => message.includes(pattern.toLowerCase())) } async function withTransientRetry(action: () => Promise, onRetry: (attempt: number, maxAttempts: number) => void, maxAttempts = 10) { let attempt = 0; while (true) { if (disposed) throw new Error('component disposed'); try { return await action() } catch (error) { attempt += 1; if (!isTransientBackendError(error) || attempt >= maxAttempts) throw error; onRetry(attempt, maxAttempts); await sleep(getPollIntervalMs()) } } } -async function refreshTaskBatch() { if (!pollingTaskIds.value.length) return { settledTaskIds: [] as number[] }; try { const batch = await getShopMatchTaskProgressBatch(pollingTaskIds.value); const nextSnapshots = { ...taskSnapshots.value }; const settledTaskIds = new Set(); for (const missingId of batch.missingTaskIds || []) { removeTaskLocally(missingId); delete nextSnapshots[missingId]; settledTaskIds.add(missingId) } for (const detail of batch.items || []) { const taskId = detail.task?.id; if (!taskId) continue; const prev = nextSnapshots[taskId]; nextSnapshots[taskId] = prev ? { ...prev, task: { ...(prev.task || {}), ...(detail.task || {}) } } : detail; const status = detail.task?.status || ''; if (!status) continue; taskDetails.value[taskId] = status; if (isTaskTerminalStatus(status)) { removePollingTask(taskId); settledTaskIds.add(taskId); continue } if (status === 'SCHEDULED') { stopPollingTask(taskId); settledTaskIds.add(taskId) } } taskSnapshots.value = nextSnapshots; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); restoreScheduledDispatches(); return { settledTaskIds: Array.from(settledTaskIds) } } catch { return { settledTaskIds: [] as number[] } } } +async function refreshTaskBatch() { if (!pollingTaskIds.value.length) return { settledTaskIds: [] as number[] }; try { // F5：主轮询只读 status 等轻量白名单字段，走 light 端点减小响应体；轻量异常/空结果时回退重型 batch（否则会漏刷新结果） + const batch = await getPollingProgressBatch('shopMatch', pollingTaskIds.value, { fallback: () => getShopMatchTaskProgressBatch(pollingTaskIds.value) }); const nextSnapshots = { ...taskSnapshots.value }; const settledTaskIds = new Set(); for (const missingId of batch.missingTaskIds || []) { removeTaskLocally(missingId); delete nextSnapshots[missingId]; settledTaskIds.add(missingId) } for (const detail of batch.items || []) { const taskId = detail.task?.id; if (!taskId) continue; const prev = nextSnapshots[taskId]; nextSnapshots[taskId] = prev ? { ...prev, task: { ...(prev.task || {}), ...(detail.task || {}) } } : detail; const status = detail.task?.status || ''; if (!status) continue; taskDetails.value[taskId] = status; if (isTaskTerminalStatus(status)) { removePollingTask(taskId); settledTaskIds.add(taskId); continue } if (status === 'SCHEDULED') { stopPollingTask(taskId); settledTaskIds.add(taskId) } } taskSnapshots.value = nextSnapshots; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); restoreScheduledDispatches(); return { settledTaskIds: Array.from(settledTaskIds) } } catch { return { settledTaskIds: [] as number[] } } } function getPollIntervalMs() { return getTaskPollIntervalMs() } function scheduleNextPoll(immediate = false) { if (disposed) return; if (pollTimer.value) { if (!immediate) return; timers.clearTimer('task-poll', pollTimer.value); pollTimer.value = null } const run = async () => { pollTimer.value = null; if (disposed) return; if (pollingInFlight.value || !pollingTaskIds.value.length) { if (pollingTaskIds.value.length) scheduleNextPoll(); return } pollingInFlight.value = true; try { const { settledTaskIds } = await refreshTaskBatch(); if (settledTaskIds.length) await Promise.allSettled([loadHistory(), loadDashboard()]) } finally { pollingInFlight.value = false } if (!disposed && pollingTaskIds.value.length) pollTimer.value = timers.setTimeout('task-poll', run, getPollIntervalMs()) }; if (immediate) void run(); else pollTimer.value = timers.setTimeout('task-poll', run, getPollIntervalMs()) } function ensurePolling(immediate = false) { scheduleNextPoll(immediate) } diff --git a/frontend-vue/src/pages/brand/components/BrandWithdrawTab.vue b/frontend-vue/src/pages/brand/components/BrandWithdrawTab.vue index 7dd31ef6..6d84728e 100644 --- a/frontend-vue/src/pages/brand/components/BrandWithdrawTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandWithdrawTab.vue @@ -44,7 +44,7 @@
+ @@ -296,6 +304,13 @@ const { total: matchedTotal, paged: pagedMatchedItems, } = useTablePaging(matchedItems) +// F9：备选店铺表同样只切渲染窗口（分页不改变数据源，勾选/删除语义不变） +const { + page: candidatePage, + pageSize: candidatePageSize, + total: candidateTotal, + paged: pagedCandidates, +} = useTablePaging(candidates) const historyItems = ref([]); const dashboard = ref({ candidateCount: 0, diff --git a/frontend-vue/src/shared/composables/useTaskProgressLoop.ts b/frontend-vue/src/shared/composables/useTaskProgressLoop.ts index 14c623bf..4abdc322 100644 --- a/frontend-vue/src/shared/composables/useTaskProgressLoop.ts +++ b/frontend-vue/src/shared/composables/useTaskProgressLoop.ts @@ -139,6 +139,8 @@ export function useTaskProgressLoop( let pollTimer: number | null = null let disposed = false let failureCount = 0 + /** 页面隐藏导致停表时为 true；恢复可见后据此判断需要重启轮询 */ + let pausedForVisibility = false function persist() { writeIdsToStorage(options.storageKey, taskIds.value) @@ -237,13 +239,13 @@ export function useTaskProgressLoop( } function scheduleNext(immediate = false) { - if (disposed) return + if (disposed || pausedForVisibility) return if (pollTimer != null && !immediate) return clearPollTimer() const run = async () => { pollTimer = null - if (disposed) return + if (disposed || pausedForVisibility) return if (!taskIds.value.length) return if (inFlight.value) { // 上一次还没回，按退避策略延迟重试（默认 500ms，可配置） @@ -251,7 +253,7 @@ export function useTaskProgressLoop( return } await refreshOnce() - if (!disposed && taskIds.value.length > 0) { + if (!disposed && !pausedForVisibility && taskIds.value.length > 0) { // 连续失败时按指数退避拉长间隔，避免后端故障时每 5s 撞一次（成功即复位） const delay = failureCount > 0 ? getTaskPollBackoffMs(failureCount) : intervalMs() pollTimer = timers.setTimeout('task-poll', run, delay) @@ -266,7 +268,7 @@ export function useTaskProgressLoop( } function ensure(immediate = false) { - if (disposed) return + if (disposed || pausedForVisibility) return if (!taskIds.value.length) return if (pollTimer != null && !immediate) return scheduleNext(immediate) @@ -292,38 +294,67 @@ export function useTaskProgressLoop( { flush: 'post' }, ) + /** + * 页面隐藏时停表：后台标签页不再按隐藏间隔空转请求（每轮仍是完整的进度查询）。 + * 恢复可见时立即补拉一轮并重启轮询（见下面的 visibilitychange 处理器）。 + */ + function pauseForVisibility() { + if (disposed || !taskIds.value.length) return + pausedForVisibility = true + stop() + console.log(`[task-poll] 页面隐藏，暂停轮询: ${options.scope}`) + } + + /** 恢复可见：按前台刷新开关决定是否立即补拉，但无论如何都要把轮询重启起来 */ + function resumeAfterVisibility() { + if (disposed) return + const wasPaused = pausedForVisibility + pausedForVisibility = false + if (!taskIds.value.length) return + if (getTaskForegroundRefreshEnabled()) { + const delay = getTaskForegroundRefreshDelayMs() + if (delay > 0) { + scheduleNextDelayed(delay) + } else { + scheduleNext(true) + } + if (wasPaused) { + console.log(`[task-poll] 页面恢复可见，已补拉一轮并重启轮询: ${options.scope}`) + } + return + } + if (wasPaused) { + // 开关关闭时不立即补拉，但必须重启轮询，否则隐藏过一次就再也不轮询 + console.log(`[task-poll] 页面恢复可见，重启轮询（前台即时刷新已关闭）: ${options.scope}`) + scheduleNext() + } + } + // 切到前台后立刻拉一次（可配置开关与延迟），让用户回到页面看到的是最新状态 let visibilityHandler: (() => void) | null = null if (typeof document !== 'undefined') { visibilityHandler = () => { - if ( - getTaskForegroundRefreshEnabled() && - document.visibilityState === 'visible' && - taskIds.value.length > 0 - ) { - const delay = getTaskForegroundRefreshDelayMs() - if (delay > 0) { - scheduleNextDelayed(delay) - } else { - scheduleNext(true) - } + if (document.visibilityState === 'visible') { + resumeAfterVisibility() + } else { + pauseForVisibility() } } document.addEventListener('visibilitychange', visibilityHandler) } function scheduleNextDelayed(delayMs: number) { - if (disposed) return + if (disposed || pausedForVisibility) return clearPollTimer() const run = () => { pollTimer = null - if (disposed || !taskIds.value.length) return + if (disposed || pausedForVisibility || !taskIds.value.length) return if (inFlight.value) { pollTimer = timers.setTimeout('task-poll', run, getTaskPollBackoffMs(0)) return } void refreshOnce() - if (!disposed && taskIds.value.length > 0) { + if (!disposed && !pausedForVisibility && taskIds.value.length > 0) { const delay = failureCount > 0 ? getTaskPollBackoffMs(failureCount) : intervalMs() pollTimer = timers.setTimeout('task-poll', run, delay) } diff --git a/frontend-vue/tests/polling-visibility.test.ts b/frontend-vue/tests/polling-visibility.test.ts index de066c01..aafba244 100644 --- a/frontend-vue/tests/polling-visibility.test.ts +++ b/frontend-vue/tests/polling-visibility.test.ts @@ -57,6 +57,8 @@ interface VisibilityHarness { loop: ReturnType> doc: FakeDoc sink: TimerSinkEntry[] + /** 被 clearTimeout 取消过的定时器 id（用于断言隐藏时确实停了表） */ + cleared: unknown[] fetchCount: () => number } @@ -72,6 +74,7 @@ async function makeVisibilityLoop( } const doc = installFakeDocument(visibility) const sink: TimerSinkEntry[] = [] + const cleared: unknown[] = [] const storage = new Map() let fetchCount = 0 ;(globalThis as Record).window = { @@ -84,7 +87,10 @@ async function makeVisibilityLoop( sink.push({ fn, ms }) return globalThis.setTimeout(fn, ms) }, - clearTimeout: (id: unknown) => globalThis.clearTimeout(id as number), + clearTimeout: (id: unknown) => { + cleared.push(id) + globalThis.clearTimeout(id as number) + }, setInterval: (fn: () => void, ms: number) => globalThis.setInterval(fn, ms), clearInterval: (id: unknown) => globalThis.clearInterval(id as number), } @@ -98,7 +104,7 @@ async function makeVisibilityLoop( extractStatus: (d) => d?.status, }) loop.reset([1]) - return { loop, doc, sink, fetchCount: () => fetchCount } + return { loop, doc, sink, cleared, fetchCount: () => fetchCount } } test('test_hidden_long_interval', async () => { @@ -187,3 +193,35 @@ test('test_hidden_no_immediate_refresh', async () => { h.loop.dispose() cleanupDocument() }) + +test('test_hidden_stops_polling', async () => { + const h = await makeVisibilityLoop('visible') + await tick(10) + const scheduledBefore = h.sink.length + const before = h.fetchCount() + h.doc.setVisibility('hidden') + h.doc.dispatch('visibilitychange') + await tick(10) + assert.equal(h.fetchCount(), before, '隐藏后不再发起请求') + assert.equal(h.sink.length, scheduledBefore, '隐藏后不再排定新的轮询定时器') + assert.ok(h.cleared.length > 0, '隐藏时清掉了已排定的轮询定时器') + h.loop.dispose() + cleanupDocument() +}) + +test('test_visible_restarts_polling', async () => { + const h = await makeVisibilityLoop('visible') + await tick(10) + h.doc.setVisibility('hidden') + h.doc.dispatch('visibilitychange') + await tick(10) + const before = h.fetchCount() + h.doc.setVisibility('visible') + h.doc.dispatch('visibilitychange') + await tick(10) + assert.equal(h.fetchCount(), before + 1, '恢复可见立即补拉一轮') + assert.equal(h.sink.at(-1)?.ms, TASK_POLL_VISIBLE_INTERVAL_MS, '恢复可见后按可见间隔重启轮询') + assert.deepEqual(h.loop.taskIds.value, [1], '停表期间任务集合不变') + h.loop.dispose() + cleanupDocument() +})