perf(全链路): 连接池/事务边界/轮询与 IO 效率收口

Java:
- LLM 180s 读超时不再被全局 call-timeout 静默截断成 90s(长思考请求被掐断→重试→付费网关二次计费)
- 代理 HttpClient 缓存改有界 LRU(jikip 每次提取新 IP,无界缓存持续泄漏 selector 线程与连接池)
- 12 个 service 的 Redis 任务锁移出 @Transactional(自旋最坏 10s 白占 DB 连接,池仅 30),远端对象删/传改 afterCommit
- 结果文件 Job 闸门拒绝时不再回退内联执行(改重新入队,避免把背压转嫁给 MQ 消费线程)
- imagevideo 每秒扫描加列投影、过期清理加 LIMIT;权限页整表查询改列投影(不再拉回密码哈希)
- 哈希改 HexFormat;补 5 处"不能改"的技术依据注释(批量插入会丢回填主键、流式丢模板与图片等)

前端:4 个工具页轮询改轻量端点(带 fallback);PriceTrack 快照节流写盘;候选店铺表分页;页面隐藏时停表

客户端:HTTP 连接池按出口复用(Session 仍每请求新建,保持无跨请求状态);品牌检测 WIPO 逐请求握手;
代理配置按 mtime 缓存;串行任务改专属池;异常降级为标签页重连;紫鸟启动改端口轮询;模板编译缓存;
日志上报连接与落盘收口;Flask 版本 API 改按请求复用连接
This commit is contained in:
2026-09-15 23:01:59 +08:00
parent d6f8368493
commit 228d481211
44 changed files with 1550 additions and 423 deletions
@@ -0,0 +1,70 @@
package com.nanri.aiimage.common.util;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.function.Function;
/**
* 有界 LRU 缓存:容量超限时自动淘汰最久未使用的条目。
*
* <p>用于按「外部端点」缓存长生命周期资源(HttpClient / RestClient)。这类资源各自持有
* 连接池与 selector 线程,无界累积会持续泄漏线程与内存:代理端点每次提取往往是新的
* IP:port(jikip 提取),无上限的缓存只增不减。
*
* <p>淘汰时只从缓存移除引用,不做显式关闭:JDK 的 HttpClientImpl 注册了 Cleaner
* 对象不可达后由 GC 回收并关闭其 selector 线程;显式关闭反而可能打断仍在途的请求。
*/
public final class BoundedLruCache<K, V> {
/** 默认容量:代理端点数量级远小于此,足够覆盖热点端点又不至于累积。 */
public static final int DEFAULT_MAX_SIZE = 64;
private final int maxSize;
private final Map<K, V> store;
public BoundedLruCache() {
this(DEFAULT_MAX_SIZE);
}
public BoundedLruCache(int maxSize) {
this.maxSize = Math.max(1, maxSize);
// accessOrder=true 使 get 也刷新顺序(真正的 LRU);synchronizedMap 保证其线程安全
this.store = Collections.synchronizedMap(new LinkedHashMap<>(16, 0.75f, true) {
@Override
protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
return size() > BoundedLruCache.this.maxSize;
}
});
}
/**
* 取缓存值,缺失时用 loader 计算并放入。
*
* <p>与 {@code ConcurrentHashMap.computeIfAbsent} 不同,此处不保证 loader 的原子性:
* 并发首次访问同一 key 时可能各自构造一次,随后其中一个覆盖另一个。对
* HttpClient/RestClient 这类构造廉价且幂等的资源可接受,换来的是锁粒度更小。
*/
public V computeIfAbsent(K key, Function<K, V> loader) {
V existing = store.get(key);
if (existing != null) {
return existing;
}
V created = loader.apply(key);
store.put(key, created);
return created;
}
public int size() {
return store.size();
}
/** 当前容量上限,供日志与测试断言使用。 */
public int maxSize() {
return maxSize;
}
public void clear() {
store.clear();
}
}
@@ -1,5 +1,7 @@
package com.nanri.aiimage.config; package com.nanri.aiimage.config;
import com.nanri.aiimage.common.util.BoundedLruCache;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.client.ClientHttpRequestFactory; import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.JdkClientHttpRequestFactory; import org.springframework.http.client.JdkClientHttpRequestFactory;
@@ -14,8 +16,6 @@ import java.net.http.HttpClient;
import java.net.http.HttpRequest; import java.net.http.HttpRequest;
import java.net.http.HttpResponse; import java.net.http.HttpResponse;
import java.time.Duration; import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/** /**
* Task 77:外部 HTTP 客户端统一连接复用池。 * Task 77:外部 HTTP 客户端统一连接复用池。
@@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentHashMap;
* 每次请求都重新建连。各客户端按自身超时创建独立的 * 每次请求都重新建连。各客户端按自身超时创建独立的
* JdkClientHttpRequestFactory(共享底层连接池),RestClient 单例懒加载。 * JdkClientHttpRequestFactory(共享底层连接池),RestClient 单例懒加载。
*/ */
@Slf4j
public class HttpClientPool { public class HttpClientPool {
private static volatile HttpClient sharedHttpClient; private static volatile HttpClient sharedHttpClient;
@@ -103,8 +104,12 @@ public class HttpClientPool {
public static ClientHttpRequestFactory requestFactory(int readTimeoutMillis, String proxyUrl) { public static ClientHttpRequestFactory requestFactory(int readTimeoutMillis, String proxyUrl) {
long safeReadTimeout = Math.max(1L, readTimeoutMillis); long safeReadTimeout = Math.max(1L, readTimeoutMillis);
long callTimeout = configuredCallTimeoutMillis; long callTimeout = configuredCallTimeoutMillis;
if (callTimeout > 0L) { if (callTimeout > 0L && safeReadTimeout > callTimeout) {
safeReadTimeout = Math.min(safeReadTimeout, callTimeout); // 读超时以调用方显式值为准,不再被全局 call-timeout 截断:
// LLM 长思考配的是 180sllm-read-timeout-millis),曾被静默压到 90s
// 导致请求在 90s 被掐断 → 上层重试 → 付费网关二次计费(2026-09-15 修复)。
// 各调用方的超时已由各自的 HttpConfigResolver 钳制,此处不再二次收敛。
log.debug("读超时 {}ms 超过全局 call-timeout {}ms,按调用方显式值生效", safeReadTimeout, callTimeout);
} }
JdkClientHttpRequestFactory factory = JdkClientHttpRequestFactory factory =
new JdkClientHttpRequestFactory(httpClientFor(proxyUrl)); new JdkClientHttpRequestFactory(httpClientFor(proxyUrl));
@@ -166,5 +171,6 @@ public class HttpClientPool {
private record ProxyEndpoint(String host, int port, String userInfo) { private record ProxyEndpoint(String host, int port, String userInfo) {
} }
private static final Map<ProxyEndpoint, HttpClient> PROXY_CLIENTS = new ConcurrentHashMap<>(); private static final BoundedLruCache<ProxyEndpoint, HttpClient> PROXY_CLIENTS =
new BoundedLruCache<>(BoundedLruCache.DEFAULT_MAX_SIZE);
} }
@@ -69,6 +69,8 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.transaction.support.TransactionTemplate; import org.springframework.transaction.support.TransactionTemplate;
import java.io.File; import java.io.File;
@@ -522,20 +524,28 @@ public class AppearancePatentTaskService {
scheduleLlmPipelineForSubmittedChunk(context); scheduleLlmPipelineForSubmittedChunk(context);
} }
/**
* 删除任务。
*
* <p>事务边界:远端载荷删除与缓存清理移到事务提交后执行 —— 单次删除超时 120s、重试 3 次,
* 放在 DB 事务里会让连接被长时间占用(项目在 TaskScopePayloadStorageService 已写明该约束)。
*/
@Transactional @Transactional
public void deleteTask(Long taskId, Long userId) { public void deleteTask(Long taskId, Long userId) {
FileTaskEntity task = fileTaskMapper.selectById(taskId); FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) { if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) {
throw new BusinessException("任务不存在"); throw new BusinessException("任务不存在");
} }
List<String> payloads = collectTransientTaskPayloads(taskId);
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>().eq(FileResultEntity::getTaskId, taskId).eq(FileResultEntity::getModuleType, MODULE_TYPE)); fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>().eq(FileResultEntity::getTaskId, taskId).eq(FileResultEntity::getModuleType, MODULE_TYPE));
deleteTransientTaskPayloads(taskId);
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>().eq(TaskScopeStateEntity::getTaskId, taskId).eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>().eq(TaskScopeStateEntity::getTaskId, taskId).eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>().eq(TaskChunkEntity::getTaskId, taskId).eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>().eq(TaskChunkEntity::getTaskId, taskId).eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskCacheService.deleteTaskCache(taskId);
// 同步清理 task_file_job,避免被删除任务遗留的 PENDING/FAILED 行被 TaskResultFileJobWorker 反复扫描出 task not found。 // 同步清理 task_file_job,避免被删除任务遗留的 PENDING/FAILED 行被 TaskResultFileJobWorker 反复扫描出 task not found。
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
fileTaskMapper.deleteById(taskId); fileTaskMapper.deleteById(taskId);
// 事务提交后再做远端删除与缓存清理
deletePayloadsAfterCommit(payloads, taskId);
runAfterCommit(() -> taskCacheService.deleteTaskCache(taskId));
} }
public void deleteHistory(Long resultId, Long userId) { public void deleteHistory(Long resultId, Long userId) {
@@ -2992,18 +3002,27 @@ public class AppearancePatentTaskService {
} }
} }
private void deleteTransientTaskPayloads(Long taskId) { /** 只读收集任务范围/分片载荷指针,供事务提交后做远端删除。 */
private List<String> collectTransientTaskPayloads(Long taskId) {
if (taskId == null || taskId <= 0) { if (taskId == null || taskId <= 0) {
return; return List.of();
} }
List<String> payloads = new ArrayList<>();
List<TaskScopeStateEntity> scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>() List<TaskScopeStateEntity> scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (scopes != null) { if (scopes != null) {
for (TaskScopeStateEntity scope : scopes) { for (TaskScopeStateEntity scope : scopes) {
transientPayloadStorageService.deletePayloadIfPresent(scope.getParsedPayloadJson()); if (scope == null) {
transientPayloadStorageService.deletePayloadIfPresent(scope.getStateJson()); continue;
}
if (scope.getParsedPayloadJson() != null && !scope.getParsedPayloadJson().isBlank()) {
payloads.add(scope.getParsedPayloadJson());
}
if (scope.getStateJson() != null && !scope.getStateJson().isBlank()) {
payloads.add(scope.getStateJson());
}
} }
} }
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>() List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
@@ -3012,9 +3031,49 @@ public class AppearancePatentTaskService {
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); .eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) { if (chunks != null) {
for (TaskChunkEntity chunk : chunks) { for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
payloads.add(chunk.getPayloadJson());
}
} }
} }
return payloads;
}
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
private void deletePayloadsAfterCommit(List<String> payloads, Long taskId) {
if (payloads == null || payloads.isEmpty()) {
return;
}
runAfterCommit(() -> {
for (String payload : payloads) {
try {
transientPayloadStorageService.deletePayloadIfPresent(payload);
} catch (Exception ex) {
log.warn("[appearance-patent] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage());
}
}
});
}
/**
* 删除范围/分片的远端载荷(保留给无事务的清理链路调用)。
*/
private void deleteTransientTaskPayloads(Long taskId) {
deletePayloadsAfterCommit(collectTransientTaskPayloads(taskId), taskId);
}
/** 有活动事务则注册 afterCommit,否则立即执行。 */
private void runAfterCommit(Runnable action) {
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
action.run();
}
});
return;
}
action.run();
} }
private record SubmitContext(FileTaskEntity task, private record SubmitContext(FileTaskEntity task,
@@ -13,9 +13,12 @@ import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DuplicateKeyException; import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.security.MessageDigest; import java.security.MessageDigest;
@@ -27,6 +30,7 @@ import java.util.Map;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@Slf4j
public class BrandTaskStorageService { public class BrandTaskStorageService {
private static final String MODULE_TYPE = "BRAND"; private static final String MODULE_TYPE = "BRAND";
@@ -236,6 +240,13 @@ public class BrandTaskStorageService {
return new ChunkStoreResult(false, completed, countCompletedFiles(taskId), aggregate); return new ChunkStoreResult(false, completed, countCompletedFiles(taskId), aggregate);
} }
/**
* 删除任务的全部范围/分片数据。
*
* <p>事务边界:远端载荷删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次,
* 放在 DB 事务里会让连接被长时间占用(项目在 TaskScopePayloadStorageService 已写明该约束)。
* 提交后再删还有个好处:引用计数查询看到的是删除完成后的 DB 状态,判断更准确。
*/
@Transactional @Transactional
public void deleteTaskData(Long taskId) { public void deleteTaskData(Long taskId) {
if (taskId == null || taskId <= 0) { if (taskId == null || taskId <= 0) {
@@ -245,27 +256,68 @@ public class BrandTaskStorageService {
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (states != null) {
for (TaskScopeStateEntity state : states) {
transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson());
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
}
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>() List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson) .select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); .eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>() taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); .eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>() taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
deletePayloadsAfterCommit(states, chunks, taskId);
}
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
private void deletePayloadsAfterCommit(List<TaskScopeStateEntity> states,
List<TaskChunkEntity> chunks,
Long taskId) {
List<String> payloads = new ArrayList<>();
if (states != null) {
for (TaskScopeStateEntity state : states) {
if (state == null) {
continue;
}
if (state.getParsedPayloadJson() != null && !state.getParsedPayloadJson().isBlank()) {
payloads.add(state.getParsedPayloadJson());
}
if (state.getStateJson() != null && !state.getStateJson().isBlank()) {
payloads.add(state.getStateJson());
}
}
}
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
payloads.add(chunk.getPayloadJson());
}
}
}
if (payloads.isEmpty()) {
return;
}
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deletePayloadsNow(payloads, taskId);
}
});
return;
}
deletePayloadsNow(payloads, taskId);
}
private void deletePayloadsNow(List<String> payloads, Long taskId) {
for (String payload : payloads) {
try {
transientPayloadStorageService.deletePayloadIfPresent(payload);
} catch (Exception ex) {
// 事务已提交,远端删除失败只记日志,不影响任务数据清理结果
log.warn("[brand-storage] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage());
}
}
} }
private void saveAggregate(Long taskId, private void saveAggregate(Long taskId,
@@ -431,70 +431,85 @@ public class CollectDataService {
} }
} }
@Transactional /**
* 进度心跳。
*
* <p>事务边界:Redis 任务锁在事务外获取(自旋等待最长 TASK_LOCK_WAIT_MILLIS
* 放在 @Transactional 里会白占一个 Hikari 连接),DB 段(统计持久化 + 任务行更新)
* 仍在一个事务内。
*/
public void updateProgress(Long taskId, TaskHeartbeatRequest request) { public void updateProgress(Long taskId, TaskHeartbeatRequest request) {
if (taskId == null || taskId <= 0 || request == null) { if (taskId == null || taskId <= 0 || request == null) {
return; return;
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId); if (transactionTemplate == null) {
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) { // 单测场景(@InjectMocks 未注入事务模板):退化为直接执行 DB 段
updateProgressLocked(taskId, request);
return; return;
} }
CollectDataStats stats = loadStats(task); transactionTemplate.executeWithoutResult(status -> updateProgressLocked(taskId, request));
boolean changed = false;
Integer current = request.getCurrent();
Integer total = request.getTotal();
if (current != null && total != null && total > 0) {
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
stats.totalRows = totalRows;
stats.processedRows = processedRows;
changed = true;
}
}
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
stats.collectStage = request.getCollectStage();
changed = true;
}
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
stats.currentKeyword = request.getCurrentKeyword();
changed = true;
}
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
changed = true;
}
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
changed = true;
}
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
changed = true;
}
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
changed = true;
}
if (!changed) {
// 内容未变化:仅当脏数据兜底窗口到期时才强制刷新一次,
// 否则零 UPDATE(重复心跳幂等)。
if (!shouldForceProgressFlush()) {
return;
}
} else if (shouldThrottleProgressFlush()) {
// 节流窗口内:合并写(只更新内存态、不落库),窗口到期后统一持久化。
return;
}
persistStats(task, stats);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
lastProgressFlushMillis = System.currentTimeMillis();
} }
} }
private void updateProgressLocked(Long taskId, TaskHeartbeatRequest request) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
return;
}
CollectDataStats stats = loadStats(task);
boolean changed = false;
Integer current = request.getCurrent();
Integer total = request.getTotal();
if (current != null && total != null && total > 0) {
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
stats.totalRows = totalRows;
stats.processedRows = processedRows;
changed = true;
}
}
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
stats.collectStage = request.getCollectStage();
changed = true;
}
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
stats.currentKeyword = request.getCurrentKeyword();
changed = true;
}
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
changed = true;
}
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
changed = true;
}
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
changed = true;
}
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
changed = true;
}
if (!changed) {
// 内容未变化:仅当脏数据兜底窗口到期时才强制刷新一次,
// 否则零 UPDATE(重复心跳幂等)。
if (!shouldForceProgressFlush()) {
return;
}
} else if (shouldThrottleProgressFlush()) {
// 节流窗口内:合并写(只更新内存态、不落库),窗口到期后统一持久化。
return;
}
persistStats(task, stats);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
lastProgressFlushMillis = System.currentTimeMillis();
}
/** /**
* 节流判定:progressThrottleMillis>0 且距上次实际落库未超过窗口 → 合并写(跳过 UPDATE)。 * 节流判定:progressThrottleMillis>0 且距上次实际落库未超过窗口 → 合并写(跳过 UPDATE)。
* progressThrottleMillis<=0 视为关闭节流,恒返回 false(每次心跳都落库,兼容旧行为)。 * progressThrottleMillis<=0 视为关闭节流,恒返回 false(每次心跳都落库,兼容旧行为)。
@@ -16,6 +16,8 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DuplicateKeyException; import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.security.MessageDigest; import java.security.MessageDigest;
@@ -252,6 +254,13 @@ public class DeleteBrandTaskStorageService {
return grouped; return grouped;
} }
/**
* 删除任务的全部范围/分片数据。
*
* <p>事务边界:远端载荷删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次,
* 放在 DB 事务里会让连接被长时间占用(项目在 TaskScopePayloadStorageService 已写明该约束)。
* 提交后再删还有个好处:引用计数查询看到的是删除完成后的 DB 状态,判断更准确。
*/
@Transactional @Transactional
public void deleteTaskData(Long taskId) { public void deleteTaskData(Long taskId) {
if (taskId == null || taskId <= 0) { if (taskId == null || taskId <= 0) {
@@ -261,27 +270,68 @@ public class DeleteBrandTaskStorageService {
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (states != null) {
for (TaskScopeStateEntity state : states) {
transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson());
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
}
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>() List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson) .select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); .eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>() taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); .eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>() taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
deletePayloadsAfterCommit(states, chunks, taskId);
}
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
private void deletePayloadsAfterCommit(List<TaskScopeStateEntity> states,
List<TaskChunkEntity> chunks,
Long taskId) {
List<String> payloads = new ArrayList<>();
if (states != null) {
for (TaskScopeStateEntity state : states) {
if (state == null) {
continue;
}
if (state.getParsedPayloadJson() != null && !state.getParsedPayloadJson().isBlank()) {
payloads.add(state.getParsedPayloadJson());
}
if (state.getStateJson() != null && !state.getStateJson().isBlank()) {
payloads.add(state.getStateJson());
}
}
}
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
payloads.add(chunk.getPayloadJson());
}
}
}
if (payloads.isEmpty()) {
return;
}
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deletePayloadsNow(payloads, taskId);
}
});
return;
}
deletePayloadsNow(payloads, taskId);
}
private void deletePayloadsNow(List<String> payloads, Long taskId) {
for (String payload : payloads) {
try {
transientPayloadStorageService.deletePayloadIfPresent(payload);
} catch (Exception ex) {
// 事务已提交,远端删除失败只记日志,不影响任务数据清理结果
log.warn("[delete-brand-storage] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage());
}
}
} }
private void refreshScopeState(Long taskId, String scopeKey, String scopeHash, Integer chunkTotal, LocalDateTime now) { private void refreshScopeState(Long taskId, String scopeKey, String scopeHash, Integer chunkTotal, LocalDateTime now) {
@@ -11,6 +11,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import java.io.File; import java.io.File;
@@ -187,6 +189,13 @@ public class DigitalHumanVersionService {
return toVo(entity); return toVo(entity);
} }
/**
* 删除版本。
*
* <p>事务边界:MinIO 对象删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次,
* 放在 DB 事务里会长时间占用连接(项目在 TaskScopePayloadStorageService 已写明该约束)。
* 顺序仍是「先删库、后删对象」,与改前一致。
*/
@Transactional @Transactional
public void deleteVersion(String version) { public void deleteVersion(String version) {
DigitalHumanVersionEntity entity = versionMapper.selectOne(new LambdaQueryWrapper<DigitalHumanVersionEntity>() DigitalHumanVersionEntity entity = versionMapper.selectOne(new LambdaQueryWrapper<DigitalHumanVersionEntity>()
@@ -198,15 +207,33 @@ public class DigitalHumanVersionService {
throw new BusinessException("最新版本不能删除"); throw new BusinessException("最新版本不能删除");
} }
// 删除 MinIO 文件
try {
ossStorageService.deleteObject(entity.getOssObjectKey());
} catch (Exception e) {
log.warn("删除 MinIO 文件失败:{}", entity.getOssObjectKey(), e);
}
// 删除数据库记录 // 删除数据库记录
versionMapper.deleteById(entity.getId()); versionMapper.deleteById(entity.getId());
// 删除 MinIO 文件(事务提交后)
deleteObjectAfterCommit(entity.getOssObjectKey());
}
/** 事务提交后删 MinIO 对象;无事务时立即执行。 */
private void deleteObjectAfterCommit(String objectKey) {
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deleteObjectQuietly(objectKey);
}
});
return;
}
deleteObjectQuietly(objectKey);
}
private void deleteObjectQuietly(String objectKey) {
try {
ossStorageService.deleteObject(objectKey);
} catch (Exception e) {
log.warn("删除 MinIO 文件失败:{}", objectKey, e);
}
} }
public String getDownloadUrl(String version) { public String getDownloadUrl(String version) {
@@ -40,6 +40,8 @@ public class ImageVideoAsyncTaskService {
private static final int DISPATCH_BATCH_SIZE = 20; private static final int DISPATCH_BATCH_SIZE = 20;
private static final int POLL_BATCH_SIZE = 50; private static final int POLL_BATCH_SIZE = 50;
private static final int FAILED_TASK_RETENTION_MINUTES = 10; private static final int FAILED_TASK_RETENTION_MINUTES = 10;
/** 过期失败任务单轮删除上限:一条无 LIMIT 的 DELETE 会长时间持锁,改成分批(每轮扫描都会再清)。 */
private static final int EXPIRED_DELETE_BATCH_SIZE = 500;
private static final Set<String> TERMINAL_STATUSES = Set.of( private static final Set<String> TERMINAL_STATUSES = Set.of(
"SUCCESS", "SUCCEEDED", "COMPLETED", "DONE", "FINISHED", "SUCCESS", "SUCCEEDED", "COMPLETED", "DONE", "FINISHED",
"FAILED", "FAIL", "ERROR", "CANCELED", "CANCELLED" "FAILED", "FAIL", "ERROR", "CANCELED", "CANCELLED"
@@ -142,7 +144,10 @@ public class ImageVideoAsyncTaskService {
return; return;
} }
try (jobLock) { try (jobLock) {
// 只取主键:本表含 5 个 LONGTEXT 列(请求/响应正文),而这里只用来发起
// executeTask(id)。每秒扫一轮还拉全部大字段属于纯浪费(2026-09-15 优化)。
List<ImageVideoAsyncTaskEntity> tasks = taskMapper.selectList(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>() List<ImageVideoAsyncTaskEntity> tasks = taskMapper.selectList(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
.select(ImageVideoAsyncTaskEntity::getId)
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.PENDING.name()) .eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.PENDING.name())
.and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId()) .and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
.or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId) .or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
@@ -161,7 +166,9 @@ public class ImageVideoAsyncTaskService {
return; return;
} }
try (jobLock) { try (jobLock) {
// 同 dispatchPendingTasks:只取主键,避免每 5 秒把 LONGTEXT 正文整列拉回
List<ImageVideoAsyncTaskEntity> tasks = taskMapper.selectList(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>() List<ImageVideoAsyncTaskEntity> tasks = taskMapper.selectList(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
.select(ImageVideoAsyncTaskEntity::getId)
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.WAITING.name()) .eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.WAITING.name())
.and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId()) .and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
.or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId) .or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
@@ -234,7 +241,8 @@ public class ImageVideoAsyncTaskService {
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES); LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES);
int deleted = taskMapper.delete(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>() int deleted = taskMapper.delete(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.FAILED.name()) .eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.FAILED.name())
.lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff)); .lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff)
.last("LIMIT " + EXPIRED_DELETE_BATCH_SIZE));
if (deleted > 0) { if (deleted > 0) {
log.info("[image-video] removed expired failed tasks count={}", deleted); log.info("[image-video] removed expired failed tasks count={}", deleted);
} }
@@ -37,6 +37,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.File; import java.io.File;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@@ -81,6 +83,10 @@ public class PatrolDeleteTaskService {
private final TaskScopePayloadStorageService taskScopePayloadStorageService; private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler; private final TaskProgressLightAssembler taskProgressLightAssembler;
/**
* 事务边界收口用:Redis 任务锁必须在事务外获取,否则自旋等待期间会白占一个 Hikari 连接。
*/
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
private FileTaskEntity loadTaskForExecution(Long taskId) { private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId)); Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -470,7 +476,10 @@ public class PatrolDeleteTaskService {
} }
} }
@Transactional /**
* 事务边界:任务锁在事务外获取(自旋等待不再占用 DB 连接);
* 缓存清理放到事务提交后、锁内执行。耗时日志语义保持不变。
*/
public void deleteTask(Long taskId, Long userId) { public void deleteTask(Long taskId, Long userId) {
long startedAt = System.nanoTime(); long startedAt = System.nanoTime();
validateUserId(userId); validateUserId(userId);
@@ -480,14 +489,18 @@ public class PatrolDeleteTaskService {
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
long taskLoadedAt = System.nanoTime(); long taskLoadedAt = System.nanoTime();
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>() long[] marks = new long[3];
.eq(FileResultEntity::getTaskId, taskId) inNewTransaction(() -> {
.eq(FileResultEntity::getModuleType, MODULE_TYPE)); fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
long resultsDeletedAt = System.nanoTime(); .eq(FileResultEntity::getTaskId, taskId)
cleanupTaskAuxiliaryDataFast(taskId); .eq(FileResultEntity::getModuleType, MODULE_TYPE));
long auxiliaryDeletedAt = System.nanoTime(); marks[0] = System.nanoTime();
fileTaskMapper.deleteById(taskId); cleanupTaskAuxiliaryDataFast(taskId);
long taskDeletedAt = System.nanoTime(); marks[1] = System.nanoTime();
fileTaskMapper.deleteById(taskId);
marks[2] = System.nanoTime();
return null;
});
taskCacheService.evictTaskCacheOnly(taskId); taskCacheService.evictTaskCacheOnly(taskId);
long finishedAt = System.nanoTime(); long finishedAt = System.nanoTime();
log.info("[patrol-delete] delete task timing taskId={} userId={} totalMs={} loadMs={} resultDeleteMs={} auxiliaryDeleteMs={} taskDeleteMs={} cacheMs={}", log.info("[patrol-delete] delete task timing taskId={} userId={} totalMs={} loadMs={} resultDeleteMs={} auxiliaryDeleteMs={} taskDeleteMs={} cacheMs={}",
@@ -495,14 +508,16 @@ public class PatrolDeleteTaskService {
userId, userId,
elapsedMs(startedAt, finishedAt), elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, taskLoadedAt), elapsedMs(startedAt, taskLoadedAt),
elapsedMs(taskLoadedAt, resultsDeletedAt), elapsedMs(taskLoadedAt, marks[0]),
elapsedMs(resultsDeletedAt, auxiliaryDeletedAt), elapsedMs(marks[0], marks[1]),
elapsedMs(auxiliaryDeletedAt, taskDeletedAt), elapsedMs(marks[1], marks[2]),
elapsedMs(taskDeletedAt, finishedAt)); elapsedMs(marks[2], finishedAt));
} }
} }
@Transactional /**
* 事务边界:任务锁在事务外获取,DB 段(复核 + 删除 + 重算)仍在同一事务内。
*/
public void deleteHistory(Long resultId, Long userId) { public void deleteHistory(Long resultId, Long userId) {
validateUserId(userId); validateUserId(userId);
FileResultEntity entity = fileResultMapper.selectById(resultId); FileResultEntity entity = fileResultMapper.selectById(resultId);
@@ -516,13 +531,16 @@ public class PatrolDeleteTaskService {
return; return;
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId); inNewTransaction(() -> {
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
throw new BusinessException("record not found"); if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
} throw new BusinessException("record not found");
fileResultMapper.deleteById(resultId); }
cleanupResultAuxiliaryDataFast(taskId, resultId); fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId); cleanupResultAuxiliaryDataFast(taskId, resultId);
reconcileTaskAfterResultRemoval(taskId);
return null;
});
} }
} }
@@ -1094,6 +1112,19 @@ public class PatrolDeleteTaskService {
return taskDistributedLockService.acquire(MODULE_TYPE, taskId); return taskDistributedLockService.acquire(MODULE_TYPE, taskId);
} }
/**
* 显式短事务:锁已在事务外获取,这里只包住 DB 段,
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内。
*/
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
if (transactionManager == null) {
return action.get();
}
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return template.execute(status -> action.get());
}
private void markResultSuccess(FileResultEntity row) { private void markResultSuccess(FileResultEntity row) {
row.setSuccess(RESULT_SUCCESS); row.setSuccess(RESULT_SUCCESS);
row.setErrorMessage(null); row.setErrorMessage(null);
@@ -297,6 +297,22 @@ public class PermissionMenuService {
.contains(columnId); .contains(columnId);
} }
/**
* 可授权用户查询:只取「超管判定 + 列表展示」用到的列。
*
* <p>权限页此前一律 `selectList(new LambdaQueryWrapper<>())` 整表拉回,而 admin_user
* 含密码哈希等敏感列,每次打开权限页都把全表(含哈希)读进内存(2026-09-15 优化)。
* 判定用 role/is_admin/created_by_id,展示用 id/username。
*/
private LambdaQueryWrapper<AdminUserEntity> grantableUserQuery() {
return new LambdaQueryWrapper<AdminUserEntity>()
.select(AdminUserEntity::getId,
AdminUserEntity::getUsername,
AdminUserEntity::getRole,
AdminUserEntity::getIsAdmin,
AdminUserEntity::getCreatedById);
}
public List<ImageVideoDataPermissionUserVo> listImageVideoDataPermissionUsers(AdminUserEntity operator) { public List<ImageVideoDataPermissionUserVo> listImageVideoDataPermissionUsers(AdminUserEntity operator) {
ensureSuperAdminOperator(operator); ensureSuperAdminOperator(operator);
PermissionMenuEntity dataPermission = requireImageVideoDataPermission(); PermissionMenuEntity dataPermission = requireImageVideoDataPermission();
@@ -305,7 +321,7 @@ public class PermissionMenuService {
.map(UserColumnPermissionEntity::getUserId) .map(UserColumnPermissionEntity::getUserId)
.filter(id -> id != null && id > 0) .filter(id -> id != null && id > 0)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
return adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>() return adminUserMapper.selectList(grantableUserQuery()
.orderByAsc(AdminUserEntity::getUsername) .orderByAsc(AdminUserEntity::getUsername)
.orderByAsc(AdminUserEntity::getId)) .orderByAsc(AdminUserEntity::getId))
.stream() .stream()
@@ -319,7 +335,7 @@ public class PermissionMenuService {
ensureSuperAdminOperator(operator); ensureSuperAdminOperator(operator);
PermissionMenuEntity dataPermission = requireImageVideoDataPermission(); PermissionMenuEntity dataPermission = requireImageVideoDataPermission();
List<Long> requestedIds = normalizeColumnIds(userIds); List<Long> requestedIds = normalizeColumnIds(userIds);
List<AdminUserEntity> users = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>()); List<AdminUserEntity> users = adminUserMapper.selectList(grantableUserQuery());
Map<Long, AdminUserEntity> grantableUsers = users.stream() Map<Long, AdminUserEntity> grantableUsers = users.stream()
.filter(user -> user.getId() != null && !isSuperAdmin(user)) .filter(user -> user.getId() != null && !isSuperAdmin(user))
.collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left)); .collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left));
@@ -345,7 +361,7 @@ public class PermissionMenuService {
.map(UserColumnPermissionEntity::getUserId) .map(UserColumnPermissionEntity::getUserId)
.filter(id -> id != null && id > 0) .filter(id -> id != null && id > 0)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
return adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>() return adminUserMapper.selectList(grantableUserQuery()
.orderByAsc(AdminUserEntity::getUsername) .orderByAsc(AdminUserEntity::getUsername)
.orderByAsc(AdminUserEntity::getId)) .orderByAsc(AdminUserEntity::getId))
.stream() .stream()
@@ -360,7 +376,7 @@ public class PermissionMenuService {
PermissionMenuEntity dataPermission = requireDataPermission( PermissionMenuEntity dataPermission = requireDataPermission(
SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据"); SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据");
List<Long> requestedIds = normalizeColumnIds(userIds); List<Long> requestedIds = normalizeColumnIds(userIds);
List<AdminUserEntity> users = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>()); List<AdminUserEntity> users = adminUserMapper.selectList(grantableUserQuery());
Map<Long, AdminUserEntity> grantableUsers = users.stream() Map<Long, AdminUserEntity> grantableUsers = users.stream()
.filter(user -> user.getId() != null && !isSuperAdmin(user)) .filter(user -> user.getId() != null && !isSuperAdmin(user))
.collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left)); .collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left));
@@ -607,7 +623,9 @@ public class PermissionMenuService {
} }
Set<Long> targetEffectiveIds = expandDescendantIds( Set<Long> targetEffectiveIds = expandDescendantIds(
new LinkedHashSet<>(loadDirectColumnIds(target.getId())), loadMenus(null)); new LinkedHashSet<>(loadDirectColumnIds(target.getId())), loadMenus(null));
// 只用到 id:不要为一次级联清理把整表(含密码哈希)拉回
List<AdminUserEntity> subordinates = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>() List<AdminUserEntity> subordinates = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>()
.select(AdminUserEntity::getId)
.eq(AdminUserEntity::getCreatedById, target.getId())); .eq(AdminUserEntity::getCreatedById, target.getId()));
for (AdminUserEntity subordinate : subordinates) { for (AdminUserEntity subordinate : subordinates) {
Long subordinateId = subordinate.getId(); Long subordinateId = subordinate.getId();
@@ -39,6 +39,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.File; import java.io.File;
@@ -93,6 +95,11 @@ public class PriceTrackTaskService {
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler; private final TaskProgressLightAssembler taskProgressLightAssembler;
private final com.nanri.aiimage.modules.file.service.LocalFileStorageService localFileStorageService; private final com.nanri.aiimage.modules.file.service.LocalFileStorageService localFileStorageService;
/**
* 事务边界收口用:Redis 任务锁必须在事务外获取,否则自旋等待(最长 TASK_LOCK_WAIT_MILLIS
* 期间会白占一个 Hikari 连接(生产 maximum-pool-size 仅 30)。
*/
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
private FileTaskEntity loadTaskForExecution(Long taskId) { private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId)); Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -254,7 +261,10 @@ public class PriceTrackTaskService {
return vo; return vo;
} }
@Transactional /**
* 事务边界:Redis 任务锁在事务外获取(自旋等待不再占用 DB 连接),
* DB 段(复核 + 删除 + 重算任务状态)仍在同一个事务内。
*/
public void deleteHistory(Long resultId, Long userId) { public void deleteHistory(Long resultId, Long userId) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
FileResultEntity entity = fileResultMapper.selectById(resultId); FileResultEntity entity = fileResultMapper.selectById(resultId);
@@ -267,73 +277,97 @@ public class PriceTrackTaskService {
return; return;
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId); inNewTransaction(() -> {
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
throw new BusinessException("record not found"); if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
} throw new BusinessException("record not found");
fileResultMapper.deleteById(resultId); }
reconcileTaskAfterResultRemoval(taskId); fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
return null;
});
} }
} }
@Transactional /**
* 事务边界:任务锁在事务外获取;缓存清理(Redis + 远端范围载荷)放到事务提交后执行。
*/
public void deleteTask(Long taskId, Long userId) { public void deleteTask(Long taskId, Long userId) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = loadTaskForExecution(taskId); inNewTransaction(() -> {
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { FileTaskEntity task = loadTaskForExecution(taskId);
throw new BusinessException("任务不存在"); if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
} throw new BusinessException("任务不存在");
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>() }
.eq(FileResultEntity::getTaskId, taskId) fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)); .eq(FileResultEntity::getTaskId, taskId)
fileTaskMapper.deleteById(taskId); .eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(taskId);
priceTrackLoopRunService.syncLoopRunAfterChildRemoved(taskId);
return null;
});
// 缓存清理(Redis + 远端范围载荷)在事务提交后、锁内执行
priceTrackTaskCacheService.deleteTaskCache(taskId); priceTrackTaskCacheService.deleteTaskCache(taskId);
priceTrackLoopRunService.syncLoopRunAfterChildRemoved(taskId);
} }
log.info("[price-track] deleteTask 完成 taskId={} userId={}", taskId, userId);
} }
@Transactional /**
* 事务边界:任务锁在事务外获取;缓存清理放到事务提交后执行。
*/
public void markDispatchFailed(Long taskId, Long userId, String errorMessage) { public void markDispatchFailed(Long taskId, Long userId, String errorMessage) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
if (errorMessage == null || errorMessage.isBlank()) throw new BusinessException("errorMessage 不能为空"); if (errorMessage == null || errorMessage.isBlank()) throw new BusinessException("errorMessage 不能为空");
String normalizedError = errorMessage.trim(); String normalizedError = errorMessage.trim();
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = loadTaskForExecution(taskId); // 返回需要清理缓存的任务状态(仅无结果行分支),非终态返回 null
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { String terminalStatus = inNewTransaction(() ->
throw new BusinessException("任务不存在"); markDispatchFailedRecords(taskId, userId, normalizedError));
if (terminalStatus != null) {
cleanupTaskCacheIfTerminal(taskId, terminalStatus);
} }
if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) {
return;
}
List<FileResultEntity> results = fileResultMapper.selectList(
new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.orderByAsc(FileResultEntity::getId));
if (results.isEmpty()) {
task.setStatus("FAILED");
task.setErrorMessage(normalizedError);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
cleanupTaskCacheIfTerminal(taskId, task.getStatus());
priceTrackLoopRunService.syncLoopRunAfterChildTerminal(taskId);
return;
}
for (FileResultEntity result : results) {
boolean succeeded = result.getSuccess() != null && result.getSuccess() == 1;
boolean failed = result.getErrorMessage() != null && !result.getErrorMessage().isBlank();
if (!succeeded && !failed) {
markResultFailed(result, normalizedError);
}
}
updateTaskStatusFromLatestRows(task, results);
log.warn("[price-track] Python dispatch failed taskId={} userId={} error={}", taskId, userId, normalizedError);
} }
} }
@Transactional private String markDispatchFailedRecords(Long taskId, Long userId, String normalizedError) {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在");
}
if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) {
return null;
}
List<FileResultEntity> results = fileResultMapper.selectList(
new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.orderByAsc(FileResultEntity::getId));
if (results.isEmpty()) {
task.setStatus("FAILED");
task.setErrorMessage(normalizedError);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
priceTrackLoopRunService.syncLoopRunAfterChildTerminal(taskId);
return task.getStatus();
}
for (FileResultEntity result : results) {
boolean succeeded = result.getSuccess() != null && result.getSuccess() == 1;
boolean failed = result.getErrorMessage() != null && !result.getErrorMessage().isBlank();
if (!succeeded && !failed) {
markResultFailed(result, normalizedError);
}
}
updateTaskStatusFromLatestRows(task, results);
log.warn("[price-track] Python dispatch failed taskId={} userId={} error={}", taskId, userId, normalizedError);
return null;
}
/**
* 事务边界:逐个候选任务在事务外取锁;命中的那条记录的删除 + 重算在独立短事务内完成
* (原实现整个循环共用一个事务,锁自旋等待期间会一直占着连接)。
*/
public PriceTrackPendingDeleteVo deletePendingShopResult(Long userId, String shopName) { public PriceTrackPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 不能为空"); if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 不能为空");
@@ -356,20 +390,33 @@ public class PriceTrackTaskService {
if (!"RUNNING".equals(task.getStatus())) continue; if (!"RUNNING".equals(task.getStatus())) continue;
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLock(fr.getTaskId())) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLock(fr.getTaskId())) {
if (ignored == null) continue; if (ignored == null) continue;
FileTaskEntity lockedTask = loadTaskForExecution(fr.getTaskId()); Boolean removed = inNewTransaction(() -> removeRunningShopResult(fr.getTaskId(), userId, fr.getId()));
if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) continue; if (Boolean.TRUE.equals(removed)) {
if (!"RUNNING".equals(lockedTask.getStatus())) continue; vo.setRemoved(true);
FileResultEntity lockedResult = fileResultMapper.selectById(fr.getId()); return vo;
if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) continue; }
fileResultMapper.deleteById(fr.getId());
reconcileTaskAfterResultRemoval(fr.getTaskId());
vo.setRemoved(true);
return vo;
} }
} }
return vo; return vo;
} }
private boolean removeRunningShopResult(Long taskId, Long userId, Long resultId) {
FileTaskEntity lockedTask = loadTaskForExecution(taskId);
if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) {
return false;
}
if (!"RUNNING".equals(lockedTask.getStatus())) {
return false;
}
FileResultEntity lockedResult = fileResultMapper.selectById(resultId);
if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) {
return false;
}
fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
return true;
}
public PriceTrackTaskBatchVo getTaskDetailsBatch(List<Long> taskIds) { public PriceTrackTaskBatchVo getTaskDetailsBatch(List<Long> taskIds) {
PriceTrackTaskBatchVo batch = new PriceTrackTaskBatchVo(); PriceTrackTaskBatchVo batch = new PriceTrackTaskBatchVo();
if (taskIds == null || taskIds.isEmpty()) return batch; if (taskIds == null || taskIds.isEmpty()) return batch;
@@ -506,6 +553,10 @@ public class PriceTrackTaskService {
} }
List<PriceTrackResultItemVo> snapshot = new ArrayList<>(); List<PriceTrackResultItemVo> snapshot = new ArrayList<>();
// 逐店铺 fileResultMapper.insert 看似该改批量(500 店铺 = 500 次往返),但**不能改**:
// 下面的 toSnapshotVo 依赖 MyBatis-Plus insert 回填的自增主键(fr.getId() → vo.resultId),
// 而本仓的手写 INSERT ... VALUES (...),(...) 批量写法不回填主键,改了会让快照 resultId 为 null。
// 要批量化必须同时解决「批量插入 + 按序回填 id」(MySQL 需 useGeneratedKeys + 顺序保证)。
for (PriceTrackMatchShopsVo.PriceTrackShopQueueItem item : uniqueItems) { for (PriceTrackMatchShopsVo.PriceTrackShopQueueItem item : uniqueItems) {
String norm = ziniaoShopSwitchService.normalizeShopName(item.getShopName()); String norm = ziniaoShopSwitchService.normalizeShopName(item.getShopName());
if (norm.isBlank()) throw new BusinessException("店铺名称无效"); if (norm.isBlank()) throw new BusinessException("店铺名称无效");
@@ -2182,6 +2233,20 @@ public class PriceTrackTaskService {
return lockHandle; return lockHandle;
} }
/**
* 显式开启一个短事务:锁已在事务外获取,这里只包住 DB 段,
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内。
*/
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
if (transactionManager == null) {
// 单测直接 new 出本类(history 见 QueryAsinSnapshotJsonThrottleTest 同类用法)时无事务管理器
return action.get();
}
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return template.execute(status -> action.get());
}
private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId) { private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId) {
if (taskId == null || taskId <= 0) { if (taskId == null || taskId <= 0) {
return null; return null;
@@ -39,6 +39,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.File; import java.io.File;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@@ -82,6 +84,10 @@ public class ProductRiskTaskService {
private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler; private final TaskProgressLightAssembler taskProgressLightAssembler;
/**
* 事务边界收口用:Redis 任务锁必须在事务外获取,否则自旋等待期间会白占一个 Hikari 连接。
*/
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
private FileTaskEntity loadTaskForExecution(Long taskId) { private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId)); Map<Long, FileTaskEntity> cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -248,7 +254,9 @@ public class ProductRiskTaskService {
return vo; return vo;
} }
@Transactional /**
* 事务边界:任务锁在事务外获取(自旋等待不再占用 DB 连接),DB 段仍在同一事务内。
*/
public void deleteHistory(Long resultId, Long userId) { public void deleteHistory(Long resultId, Long userId) {
if (userId == null || userId <= 0) { if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
@@ -263,43 +271,53 @@ public class ProductRiskTaskService {
return; return;
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId); inNewTransaction(() -> {
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
throw new BusinessException("record not found"); if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
} throw new BusinessException("record not found");
taskFileJobService.deleteResultJobs(latestEntity.getTaskId(), MODULE_TYPE, latestEntity.getId()); }
fileResultMapper.deleteById(resultId); taskFileJobService.deleteResultJobs(latestEntity.getTaskId(), MODULE_TYPE, latestEntity.getId());
reconcileTaskAfterResultRemoval(taskId); fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
return null;
});
} }
} }
/** /**
* 删除整条商品风险任务及其下所有店铺结果,运行中和已结束任务都允许删除,但必须归属当前用户。 * 删除整条商品风险任务及其下所有店铺结果,运行中和已结束任务都允许删除,但必须归属当前用户。
*
* <p>事务边界:任务锁在事务外获取;缓存清理放到事务提交后、锁内执行。
*/ */
@Transactional
public void deleteTask(Long taskId, Long userId) { public void deleteTask(Long taskId, Long userId) {
if (userId == null || userId <= 0) { if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = loadTaskForExecution(taskId); inNewTransaction(() -> {
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { FileTaskEntity task = loadTaskForExecution(taskId);
throw new BusinessException("任务不存在"); if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
} throw new BusinessException("任务不存在");
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>() }
.eq(FileResultEntity::getTaskId, taskId) fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)); .eq(FileResultEntity::getTaskId, taskId)
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); .eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(taskId); taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
fileTaskMapper.deleteById(taskId);
return null;
});
cleanupTaskCacheIfTerminal(taskId, "DELETE_EMPTY"); cleanupTaskCacheIfTerminal(taskId, "DELETE_EMPTY");
} }
log.info("[product-risk] deleteTask 完成 taskId={} userId={}", taskId, userId);
} }
/** /**
* 从用户当前“运行中”的商品风险任务里,按规范化店铺名删除一条结果,用于前端移除匹配列表后同步后端。 * 从用户当前“运行中”的商品风险任务里,按规范化店铺名删除一条结果,用于前端移除匹配列表后同步后端。
* 如果没有对应记录,则返回 removed=false。 * 如果没有对应记录,则返回 removed=false。
*
* <p>事务边界:逐个候选任务在事务外取锁;命中的那条记录的删除 + 重算在独立短事务内完成
* (原实现整个循环共用一个事务,锁自旋等待期间会一直占着连接)。
*/ */
@Transactional
public ProductRiskPendingDeleteVo deletePendingShopResult(Long userId, String shopName) { public ProductRiskPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
if (userId == null || userId <= 0) { if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
@@ -335,26 +353,33 @@ public class ProductRiskTaskService {
if (ignored == null) { if (ignored == null) {
continue; continue;
} }
FileTaskEntity lockedTask = loadTaskForExecution(tid); Boolean removed = inNewTransaction(() -> removeRunningShopResult(tid, userId, fr.getId()));
if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) { if (Boolean.TRUE.equals(removed)) {
continue; vo.setRemoved(true);
return vo;
} }
if (!"RUNNING".equals(lockedTask.getStatus())) {
continue;
}
FileResultEntity lockedResult = fileResultMapper.selectById(fr.getId());
if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) {
continue;
}
fileResultMapper.deleteById(fr.getId());
reconcileTaskAfterResultRemoval(tid);
vo.setRemoved(true);
return vo;
} }
} }
return vo; return vo;
} }
private boolean removeRunningShopResult(Long taskId, Long userId, Long resultId) {
FileTaskEntity lockedTask = loadTaskForExecution(taskId);
if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) {
return false;
}
if (!"RUNNING".equals(lockedTask.getStatus())) {
return false;
}
FileResultEntity lockedResult = fileResultMapper.selectById(resultId);
if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) {
return false;
}
fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
return true;
}
/** /**
* 删除一条 file_result 后重算父任务状态;如果没有剩余结果则删除任务。 * 删除一条 file_result 后重算父任务状态;如果没有剩余结果则删除任务。
*/ */
@@ -1002,6 +1027,19 @@ public class ProductRiskTaskService {
return taskDistributedLockService.acquire(MODULE_TYPE, taskId); return taskDistributedLockService.acquire(MODULE_TYPE, taskId);
} }
/**
* 显式短事务:锁已在事务外获取,这里只包住 DB 段,
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内。
*/
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
if (transactionManager == null) {
return action.get();
}
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return template.execute(status -> action.get());
}
private ProductRiskTaskDetailVo buildTaskDetail(FileTaskEntity task) { private ProductRiskTaskDetailVo buildTaskDetail(FileTaskEntity task) {
ProductRiskTaskDetailVo detail = new ProductRiskTaskDetailVo(); ProductRiskTaskDetailVo detail = new ProductRiskTaskDetailVo();
detail.setTask(toTaskItemVo(task)); detail.setTask(toTaskItemVo(task));
@@ -57,6 +57,8 @@ import org.springframework.dao.DuplicateKeyException;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.transaction.support.TransactionTemplate; import org.springframework.transaction.support.TransactionTemplate;
import java.io.File; import java.io.File;
@@ -554,24 +556,28 @@ public class PublishTaskService {
} }
} }
/**
* 删除任务。
*
* <p>事务边界:远端删除(结果文件对象 + 分片载荷)全部移到事务提交后执行 ——
* 单次删除超时 120s、重试 3 次,放在 DB 事务里会让连接被长时间占用
* (项目在 TaskScopePayloadStorageService 已写明「不能把网络调用放进 DB 事务」)。
*/
@Transactional @Transactional
public void deleteTask(Long taskId, Long userId) { public void deleteTask(Long taskId, Long userId) {
FileTaskEntity task = requireTaskForDeletion(taskId, userId); FileTaskEntity task = requireTaskForDeletion(taskId, userId);
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId) .eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)); .eq(FileResultEntity::getModuleType, MODULE_TYPE));
List<String> resultObjectKeys = new ArrayList<>();
for (FileResultEntity result : results) { for (FileResultEntity result : results) {
if (result.getResultFileUrl() != null && !result.getResultFileUrl().isBlank()) { if (result.getResultFileUrl() != null && !result.getResultFileUrl().isBlank()) {
try { resultObjectKeys.add(result.getResultFileUrl());
ossStorageService.deleteObject(result.getResultFileUrl());
} catch (Exception ex) {
log.warn("[publish] OSS cleanup failed taskId={} resultId={} msg={}",
taskId, result.getId(), ex.getMessage());
}
} }
} }
List<String> chunkPayloads = collectTransientResultChunkPayloads(taskId);
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
deleteTransientResultChunks(taskId); deleteTransientResultChunkRows(taskId);
publishItemMapper.delete(new LambdaQueryWrapper<PublishItemEntity>() publishItemMapper.delete(new LambdaQueryWrapper<PublishItemEntity>()
.eq(PublishItemEntity::getTaskId, taskId)); .eq(PublishItemEntity::getTaskId, taskId));
publishFileMapper.delete(new LambdaQueryWrapper<PublishFileEntity>() publishFileMapper.delete(new LambdaQueryWrapper<PublishFileEntity>()
@@ -580,6 +586,9 @@ public class PublishTaskService {
.eq(FileResultEntity::getTaskId, taskId) .eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)); .eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(task.getId()); fileTaskMapper.deleteById(task.getId());
// 事务提交后再做远端删除
deleteResultObjectsAfterCommit(resultObjectKeys, taskId);
deletePayloadsAfterCommit(chunkPayloads, taskId);
} }
public void deleteHistory(Long resultId, Long userId) { public void deleteHistory(Long resultId, Long userId) {
@@ -599,18 +608,35 @@ public class PublishTaskService {
} }
private void deleteTransientResultChunks(Long taskId) { private void deleteTransientResultChunks(Long taskId) {
List<String> payloads = collectTransientResultChunkPayloads(taskId);
deleteTransientResultChunkRows(taskId);
deletePayloadsAfterCommit(payloads, taskId);
}
/** 只读收集分片载荷指针,供事务提交后做远端删除。 */
private List<String> collectTransientResultChunkPayloads(Long taskId) {
if (taskId == null || taskId <= 0) { if (taskId == null || taskId <= 0) {
return; return List.of();
} }
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>() List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson) .select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); .eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
List<String> payloads = new ArrayList<>();
if (chunks != null) { if (chunks != null) {
for (TaskChunkEntity chunk : chunks) { for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
payloads.add(chunk.getPayloadJson());
}
} }
} }
return payloads;
}
private void deleteTransientResultChunkRows(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>() taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); .eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
@@ -619,6 +645,63 @@ public class PublishTaskService {
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
} }
/** 事务提交后逐条删分片载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
private void deletePayloadsAfterCommit(List<String> payloads, Long taskId) {
if (payloads == null || payloads.isEmpty()) {
return;
}
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deletePayloadsNow(payloads, taskId);
}
});
return;
}
deletePayloadsNow(payloads, taskId);
}
private void deletePayloadsNow(List<String> payloads, Long taskId) {
for (String payload : payloads) {
try {
transientPayloadStorageService.deletePayloadIfPresent(payload);
} catch (Exception ex) {
log.warn("[publish] 分片载荷删除失败 taskId={} pointer={} msg={}",
taskId, transientPayloadStorageService.extractPointer(payload), safeMessage(ex));
}
}
}
/** 事务提交后删结果文件对象;无事务时立即执行。 */
private void deleteResultObjectsAfterCommit(List<String> objectKeys, Long taskId) {
if (objectKeys == null || objectKeys.isEmpty()) {
return;
}
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deleteResultObjectsNow(objectKeys, taskId);
}
});
return;
}
deleteResultObjectsNow(objectKeys, taskId);
}
private void deleteResultObjectsNow(List<String> objectKeys, Long taskId) {
for (String objectKey : objectKeys) {
try {
ossStorageService.deleteObject(objectKey);
log.info("[publish] 结果文件已删除 taskId={} objectKey={}", taskId, objectKey);
} catch (Exception ex) {
log.warn("[publish] OSS cleanup failed taskId={} objectKey={} msg={}",
taskId, objectKey, ex.getMessage());
}
}
}
private void deleteUncommittedPayloads(List<String> storedPayloads) { private void deleteUncommittedPayloads(List<String> storedPayloads) {
if (storedPayloads == null || storedPayloads.isEmpty()) { if (storedPayloads == null || storedPayloads.isEmpty()) {
return; return;
@@ -36,6 +36,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.File; import java.io.File;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@@ -92,6 +94,10 @@ public class QueryAsinTaskService {
private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler; private final TaskProgressLightAssembler taskProgressLightAssembler;
/**
* 事务边界收口用:Redis 任务锁必须在事务外获取,否则自旋等待期间会白占一个 Hikari 连接。
*/
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
private FileTaskEntity loadTaskForExecution(Long taskId) { private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId)); Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -475,7 +481,10 @@ public class QueryAsinTaskService {
} }
} }
@Transactional /**
* 事务边界:任务锁在事务外获取(自旋等待不再占用 DB 连接);
* 缓存清理(Redis + 远端范围载荷)放到事务提交后、锁内执行。
*/
public void deleteTask(Long taskId, Long userId) { public void deleteTask(Long taskId, Long userId) {
validateUserId(userId); validateUserId(userId);
FileTaskEntity task = loadTaskForExecution(taskId); FileTaskEntity task = loadTaskForExecution(taskId);
@@ -483,15 +492,21 @@ public class QueryAsinTaskService {
throw new BusinessException("任务不存在"); throw new BusinessException("任务不存在");
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>() inNewTransaction(() -> {
.eq(FileResultEntity::getTaskId, taskId) fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)); .eq(FileResultEntity::getTaskId, taskId)
fileTaskMapper.deleteById(taskId); .eq(FileResultEntity::getModuleType, MODULE_TYPE));
taskCacheService.deleteTaskCache(taskId); fileTaskMapper.deleteById(taskId);
return null;
});
taskCacheService.deleteTaskCache(taskId);
} }
log.info("[query-asin] deleteTask 完成 taskId={} userId={}", taskId, userId);
} }
@Transactional /**
* 事务边界:任务锁在事务外获取,DB 段(复核 + 删除 + 重算)仍在同一事务内。
*/
public void deleteHistory(Long resultId, Long userId) { public void deleteHistory(Long resultId, Long userId) {
validateUserId(userId); validateUserId(userId);
FileResultEntity entity = fileResultMapper.selectById(resultId); FileResultEntity entity = fileResultMapper.selectById(resultId);
@@ -504,12 +519,15 @@ public class QueryAsinTaskService {
return; return;
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId); inNewTransaction(() -> {
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
throw new BusinessException("record not found"); if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
} throw new BusinessException("record not found");
fileResultMapper.deleteById(resultId); }
reconcileTaskAfterResultRemoval(taskId); fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
return null;
});
} }
} }
@@ -1070,6 +1088,19 @@ public class QueryAsinTaskService {
return taskDistributedLockService.acquire(MODULE_TYPE, taskId); return taskDistributedLockService.acquire(MODULE_TYPE, taskId);
} }
/**
* 显式短事务:锁已在事务外获取,这里只包住 DB 段,
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内。
*/
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
if (transactionManager == null) {
return action.get();
}
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return template.execute(status -> action.get());
}
private void markResultSuccess(FileResultEntity row) { private void markResultSuccess(FileResultEntity row) {
row.setSuccess(RESULT_SUCCESS); row.setSuccess(RESULT_SUCCESS);
row.setErrorMessage(null); row.setErrorMessage(null);
@@ -87,6 +87,13 @@ public class ShopDataCrawlExcelAssemblyService {
imageEmbedder.prefetch(prefetchBudget().boundedUrls(imageUrls(rowsByCountry)), imageCache); imageEmbedder.prefetch(prefetchBudget().boundedUrls(imageUrls(rowsByCountry)), imageCache);
} }
/**
* 模板路径组装DOM历史包袱说明本方法把整个工作簿读进 DOMXSSFWorkbook
* 当日累计文件越大峰值堆越高-Xmx6g 下大客户的大文件有 OOM 风险
* 同文件的 {@link #writeWorkbookStreaming} SXSSF 流式版但它**不使用模板
* 不写图片**图片只兜底成 URL 文本直接替换会改变交付文件的样式与商品图
* 属产品取舍未直接切换若要落地必须实现保留模板样式 + 保留图片的流式写入
*/
public int writeWorkbook(File outputXlsx, List<ShopDataCrawlResultItemVo> items) { public int writeWorkbook(File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream(); try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream();
XSSFWorkbook workbook = new XSSFWorkbook(input); XSSFWorkbook workbook = new XSSFWorkbook(input);
@@ -807,7 +807,10 @@ public class ShopDataCrawlTaskService {
} }
} }
@Transactional /**
* 事务边界Redis 任务锁在事务外获取自旋等待期间不再白占一个 Hikari 连接
* DB 含每日文件行锁与三条删除链路仍在同一个短事务内远端对象清理在提交后进行
*/
public void deleteTask(Long taskId, Long userId) { public void deleteTask(Long taskId, Long userId) {
validateUserId(userId); validateUserId(userId);
FileTaskEntity task = loadTaskForExecution(taskId); FileTaskEntity task = loadTaskForExecution(taskId);
@@ -816,28 +819,33 @@ public class ShopDataCrawlTaskService {
} }
ensureTaskOwnedByCurrentInstance(task, "delete shop data crawl task"); ensureTaskOwnedByCurrentInstance(task, "delete shop data crawl task");
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
List<FileResultEntity> taskRows = listTaskRows(taskId); executeShortTransaction(() -> {
try (DailyLockSet dailyLocks = acquireDailyLocks(taskRows)) { List<FileResultEntity> taskRows = listTaskRows(taskId);
ensureDailySyncCompletedBeforeDelete(taskRows); try (DailyLockSet dailyLocks = acquireDailyLocks(taskRows)) {
Set<Long> removedResultIds = collectResultIds(taskRows); ensureDailySyncCompletedBeforeDelete(taskRows);
// A task deletion is only a frontend task-record cleanup. The daily Set<Long> removedResultIds = collectResultIds(taskRows);
// workbook is an independent backend aggregate and must not roll // A task deletion is only a frontend task-record cleanup. The daily
// back when its source task is removed. // workbook is an independent backend aggregate and must not roll
DailyDeletionResult dailyResult = preserveDailyForTaskDeletion(removedResultIds); // back when its source task is removed.
registerUploadedObjectRollback(dailyResult.uploadedObjectKeys()); DailyDeletionResult dailyResult = preserveDailyForTaskDeletion(removedResultIds);
List<String> resultFileUrls = collectResultFileUrls(taskRows, dailyResult.obsoleteObjectKeys()); registerUploadedObjectRollback(dailyResult.uploadedObjectKeys());
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>() List<String> resultFileUrls = collectResultFileUrls(taskRows, dailyResult.obsoleteObjectKeys());
.eq(FileResultEntity::getTaskId, taskId) fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)); .eq(FileResultEntity::getTaskId, taskId)
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); .eq(FileResultEntity::getModuleType, MODULE_TYPE));
taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE); taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
taskProgressSnapshotService.delete(taskId, MODULE_TYPE); taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE);
fileTaskMapper.deleteById(taskId); taskProgressSnapshotService.delete(taskId, MODULE_TYPE);
taskCacheService.deleteTaskCache(taskId); fileTaskMapper.deleteById(taskId);
deleteTransientResultChunks(taskId); deleteTransientResultChunks(taskId);
resultFileUrls.forEach(this::deleteResultObjectIfUnreferenced); resultFileUrls.forEach(this::deleteResultObjectIfUnreferenced);
} }
return null;
});
// 缓存清理Redis + 本地缓存在事务提交后锁内执行
taskCacheService.deleteTaskCache(taskId);
} }
log.info("[shop-data-crawl] deleteTask 完成 taskId={} userId={}", taskId, userId);
} }
/** 删除任务的纯计算:收集有效结果行 id 集合(过滤 null/非正数),无副作用。 */ /** 删除任务的纯计算:收集有效结果行 id 集合(过滤 null/非正数),无副作用。 */
@@ -884,7 +892,10 @@ public class ShopDataCrawlTaskService {
} }
} }
@Transactional /**
* 事务边界任务锁在事务外获取DB 复核 + 删行 + 每日文件重建在同一个短事务内
* 远端对象清理已由 deleteResultObjectIfUnreferenced 在提交后执行
*/
public void deleteHistory(Long resultId, Long userId) { public void deleteHistory(Long resultId, Long userId) {
validateUserId(userId); validateUserId(userId);
FileResultEntity entity = requireResultEntity(resultId); FileResultEntity entity = requireResultEntity(resultId);
@@ -899,21 +910,26 @@ public class ShopDataCrawlTaskService {
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务仍在处理中,不能删除"); throw new BusinessException(BusinessCodes.TASK_BUSY, "任务仍在处理中,不能删除");
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId); executeShortTransaction(() -> {
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
throw new BusinessException("记录不存在"); if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
} throw new BusinessException("记录不存在");
try (DailyLockSet dailyLocks = acquireDailyLocks(List.of(latestEntity))) { }
deleteResultHistoryRow(latestEntity); try (DailyLockSet dailyLocks = acquireDailyLocks(List.of(latestEntity))) {
} deleteResultHistoryRow(latestEntity);
}
return null;
});
} }
} }
/** /**
* Deletes an administrative result without accepting a caller-controlled owner ID. * Deletes an administrative result without accepting a caller-controlled owner ID.
* The controller that calls this method is restricted to the Flask-to-Java internal channel. * The controller that calls this method is restricted to the Flask-to-Java internal channel.
*
* <p>不加 @Transactional内部调用 deleteHistory 时事务代理不生效同类自调用
* 锁会落在本方法开启的事务里 事务边界统一由 deleteHistory 自己收口
*/ */
@Transactional
public void deleteAdminHistory(Long resultId) { public void deleteAdminHistory(Long resultId) {
FileResultEntity entity = requireResultEntity(resultId); FileResultEntity entity = requireResultEntity(resultId);
Long ownerId = entity.getUserId(); Long ownerId = entity.getUserId();
@@ -2620,6 +2636,13 @@ public class ShopDataCrawlTaskService {
taskCacheService.deleteTaskCache(taskId); taskCacheService.deleteTaskCache(taskId);
} }
/**
* 删除结果分片行
*
* <p>事务边界远端载荷删除移到事务提交后执行 单次删除超时 120s重试 3
* 放在 DB 事务里会让连接被长时间占用项目在 TaskScopePayloadStorageService 已写明该约束
* 提交后再删还有个好处引用计数查询看到的是删除完成后的 DB 状态判断更准确
*/
private void deleteTransientResultChunks(Long taskId) { private void deleteTransientResultChunks(Long taskId) {
if (taskId == null || taskId <= 0) { if (taskId == null || taskId <= 0) {
return; return;
@@ -2628,9 +2651,12 @@ public class ShopDataCrawlTaskService {
.select(TaskChunkEntity::getPayloadJson) .select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); .eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
List<String> payloads = new ArrayList<>();
if (chunks != null) { if (chunks != null) {
for (TaskChunkEntity chunk : chunks) { for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); if (chunk != null && !blank(chunk.getPayloadJson())) {
payloads.add(chunk.getPayloadJson());
}
} }
} }
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>() taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
@@ -2640,6 +2666,34 @@ public class ShopDataCrawlTaskService {
.eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.likeRight(TaskScopeStateEntity::getScopeKey, RESULT_CHUNK_SCOPE_PREFIX)); .likeRight(TaskScopeStateEntity::getScopeKey, RESULT_CHUNK_SCOPE_PREFIX));
deleteChunkPayloadsAfterCommit(payloads, taskId);
}
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
private void deleteChunkPayloadsAfterCommit(List<String> payloads, Long taskId) {
if (payloads == null || payloads.isEmpty()) {
return;
}
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deleteChunkPayloadsNow(payloads, taskId);
}
});
return;
}
deleteChunkPayloadsNow(payloads, taskId);
}
private void deleteChunkPayloadsNow(List<String> payloads, Long taskId) {
for (String payload : payloads) {
try {
transientPayloadStorageService.deletePayloadIfPresent(payload);
} catch (Exception ex) {
log.warn("[shop-data-crawl] 分片载荷删除失败 taskId={} msg={}", taskId, safeMessage(ex));
}
}
} }
private void cleanupResultChunksQuietly(Long taskId, String reason) { private void cleanupResultChunksQuietly(Long taskId, String reason) {
@@ -46,6 +46,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.File; import java.io.File;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@@ -89,6 +91,10 @@ public class ShopMatchTaskService {
private final SkipPriceAsinService skipPriceAsinService; private final SkipPriceAsinService skipPriceAsinService;
private final QueryAsinMapper queryAsinMapper; private final QueryAsinMapper queryAsinMapper;
private final TaskProgressLightAssembler taskProgressLightAssembler; private final TaskProgressLightAssembler taskProgressLightAssembler;
/**
* 事务边界收口用Redis 任务锁必须在事务外获取否则自旋等待期间会白占一个 Hikari 连接
*/
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
private FileTaskEntity loadTaskForExecution(Long taskId) { private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId)); Map<Long, FileTaskEntity> cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -280,7 +286,9 @@ public class ShopMatchTaskService {
return vo; return vo;
} }
@Transactional /**
* 事务边界任务锁在事务外获取自旋等待不再占用 DB 连接DB 段仍在同一事务内
*/
public void deleteHistory(Long resultId, Long userId) { public void deleteHistory(Long resultId, Long userId) {
if (userId == null || userId <= 0) { if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
@@ -295,16 +303,21 @@ public class ShopMatchTaskService {
return; return;
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId); inNewTransaction(() -> {
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) { FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
throw new BusinessException("record not found"); if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
} throw new BusinessException("record not found");
fileResultMapper.deleteById(resultId); }
reconcileTaskAfterResultRemoval(taskId); fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
return null;
});
} }
} }
@Transactional /**
* 事务边界任务锁在事务外获取缓存清理Redis + 远端范围载荷放到事务提交后
*/
public void deleteTask(Long taskId, Long userId) { public void deleteTask(Long taskId, Long userId) {
if (userId == null || userId <= 0) { if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
@@ -314,12 +327,17 @@ public class ShopMatchTaskService {
throw new BusinessException("任务不存在"); throw new BusinessException("任务不存在");
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>() inNewTransaction(() -> {
.eq(FileResultEntity::getTaskId, taskId) fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)); .eq(FileResultEntity::getTaskId, taskId)
fileTaskMapper.deleteById(taskId); .eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(taskId);
return null;
});
// 缓存清理Redis + 远端范围载荷在事务提交后锁内执行
shopMatchTaskCacheService.deleteTaskCache(taskId); shopMatchTaskCacheService.deleteTaskCache(taskId);
} }
log.info("[shop-match] deleteTask 完成 taskId={} userId={}", taskId, userId);
} }
private void reconcileTaskAfterResultRemoval(Long taskId) { private void reconcileTaskAfterResultRemoval(Long taskId) {
@@ -500,7 +518,9 @@ public class ShopMatchTaskService {
return vo; return vo;
} }
@Transactional /**
* 事务边界任务锁在事务外获取心跳Redis放到事务提交后写
*/
public void activateTask(Long taskId, Long userId, Integer stageIndex) { public void activateTask(Long taskId, Long userId, Integer stageIndex) {
if (taskId == null || taskId <= 0) { if (taskId == null || taskId <= 0) {
throw new BusinessException("taskId 不合法"); throw new BusinessException("taskId 不合法");
@@ -512,45 +532,53 @@ public class ShopMatchTaskService {
throw new BusinessException("stage_index 不合法"); throw new BusinessException("stage_index 不合法");
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = loadTaskForExecution(taskId); inNewTransaction(() -> {
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { activateTaskLocked(taskId, userId, stageIndex);
throw new BusinessException("任务不存在"); return null;
} });
if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) { // 心跳是 Redis 放到事务提交锁内执行与改前一样仍在锁内
throw new BusinessException("任务状态不正确");
}
if (!"SCHEDULED".equals(task.getStatus())) {
throw new BusinessException("任务状态不正确");
}
LocalDateTime now = now();
if (task.getScheduledAt() != null && now.isBefore(task.getScheduledAt().minusSeconds(90))) {
log.warn("[shop-match] activate rejected taskId={} stageIndex={} now={} scheduledAt={} earliestActivateAt={}",
taskId, stageIndex, now, task.getScheduledAt(), task.getScheduledAt().minusSeconds(90));
throw new BusinessException("未到定时执行时间");
}
ShopMatchCreateTaskRequest state = parseTaskRequest(task);
int currentStageIndex = state.getCurrentStageIndex() == null ? 0 : state.getCurrentStageIndex();
if (currentStageIndex != stageIndex) {
throw new BusinessException("当前轮次已变化,请刷新后重试");
}
if (state.getScheduleTimes() == null || stageIndex >= state.getScheduleTimes().size()) {
throw new BusinessException("轮次配置缺失");
}
FileTaskEntity runningTask = findOtherRunningTask(userId, taskId);
if (runningTask != null) {
log.warn("[shop-match] activate serialized taskId={} stageIndex={} blockedByTaskId={} blockedUpdatedAt={}",
taskId, stageIndex, runningTask.getId(), runningTask.getUpdatedAt());
throw new BusinessException("当前已有定时匹配任务执行中,请等待上一任务执行结束后再重试");
}
state.setActiveStageIndex(stageIndex);
task.setStatus("RUNNING");
task.setUpdatedAt(now);
persistTaskRequest(task, state);
updateTaskAndRefreshCache(task);
shopMatchTaskCacheService.touchTaskHeartbeat(taskId); shopMatchTaskCacheService.touchTaskHeartbeat(taskId);
} }
} }
private void activateTaskLocked(Long taskId, Long userId, Integer stageIndex) {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在");
}
if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) {
throw new BusinessException("任务状态不正确");
}
if (!"SCHEDULED".equals(task.getStatus())) {
throw new BusinessException("任务状态不正确");
}
LocalDateTime now = now();
if (task.getScheduledAt() != null && now.isBefore(task.getScheduledAt().minusSeconds(90))) {
log.warn("[shop-match] activate rejected taskId={} stageIndex={} now={} scheduledAt={} earliestActivateAt={}",
taskId, stageIndex, now, task.getScheduledAt(), task.getScheduledAt().minusSeconds(90));
throw new BusinessException("未到定时执行时间");
}
ShopMatchCreateTaskRequest state = parseTaskRequest(task);
int currentStageIndex = state.getCurrentStageIndex() == null ? 0 : state.getCurrentStageIndex();
if (currentStageIndex != stageIndex) {
throw new BusinessException("当前轮次已变化,请刷新后重试");
}
if (state.getScheduleTimes() == null || stageIndex >= state.getScheduleTimes().size()) {
throw new BusinessException("轮次配置缺失");
}
FileTaskEntity runningTask = findOtherRunningTask(userId, taskId);
if (runningTask != null) {
log.warn("[shop-match] activate serialized taskId={} stageIndex={} blockedByTaskId={} blockedUpdatedAt={}",
taskId, stageIndex, runningTask.getId(), runningTask.getUpdatedAt());
throw new BusinessException("当前已有定时匹配任务执行中,请等待上一任务执行结束后再重试");
}
state.setActiveStageIndex(stageIndex);
task.setStatus("RUNNING");
task.setUpdatedAt(now);
persistTaskRequest(task, state);
updateTaskAndRefreshCache(task);
}
@Transactional @Transactional
public void completeStage(Long taskId, Long userId, ShopMatchStageCompleteRequest request) { public void completeStage(Long taskId, Long userId, ShopMatchStageCompleteRequest request) {
if (taskId == null || taskId <= 0) { if (taskId == null || taskId <= 0) {
@@ -960,6 +988,19 @@ public class ShopMatchTaskService {
return taskDistributedLockService.acquire(MODULE_TYPE, taskId); return taskDistributedLockService.acquire(MODULE_TYPE, taskId);
} }
/**
* 显式短事务锁已在事务外获取这里只包住 DB
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内
*/
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
if (transactionManager == null) {
return action.get();
}
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return template.execute(status -> action.get());
}
private boolean shouldRemainRunningUntilNextStage(FileTaskEntity task) { private boolean shouldRemainRunningUntilNextStage(FileTaskEntity task) {
ShopMatchCreateTaskRequest request = parseTaskRequestSilently(task); ShopMatchCreateTaskRequest request = parseTaskRequestSilently(task);
if (request == null || request.getScheduleTimes() == null || request.getScheduleTimes().isEmpty()) { if (request == null || request.getScheduleTimes() == null || request.getScheduleTimes().isEmpty()) {
@@ -526,7 +526,12 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
return vo; return vo;
} }
@Transactional /**
* 启动任务PENDINGRUNNING
*
* <p>事务边界Redis 任务锁必须在事务外获取自旋等待期间会白占一个 Hikari 连接
* 本方法只有一条条件 UPDATE自身即原子故不再包事务
*/
public void activateTask(Long taskId, Long userId) { public void activateTask(Long taskId, Long userId) {
try (TaskDistributedLockService.LockHandle ignored = ownershipSupport().requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) { try (TaskDistributedLockService.LockHandle ignored = ownershipSupport().requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId); FileTaskEntity task = fileTaskMapper.selectById(taskId);
@@ -1323,7 +1328,12 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
} }
} }
@Transactional /**
* 结果文件 job 失败的收尾
*
* <p>事务边界任务锁在事务外获取DB 段本来就走 {@code inNewTransaction}REQUIRES_NEW
* 方法级 @Transactional 只会额外占一个空闲连接故去掉
*/
public void handleResultFileJobFailure(TaskFileJobEntity job, String message) { public void handleResultFileJobFailure(TaskFileJobEntity job, String message) {
if (job == null || job.getTaskId() == null) { if (job == null || job.getTaskId() == null) {
return; return;
@@ -9,7 +9,8 @@ import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
* <p> * <p>
* 采用依赖倒置由宿主SimilarAsinTaskService实现本接口流水线只依赖接口 * 采用依赖倒置由宿主SimilarAsinTaskService实现本接口流水线只依赖接口
* 避免 support 包与 Service 之间形成循环依赖这些方法保留在宿主的两个原因 * 避免 support 包与 Service 之间形成循环依赖这些方法保留在宿主的两个原因
* 一是属于任务/结果文件任务的编排与事务边界{@code handleResultFileJobFailure} @Transactional * 一是属于任务/结果文件任务的编排与事务边界{@code handleResultFileJobFailure} 先取任务锁
* 再用 {@code inNewTransaction} 显式包 DB 锁必须在事务外获取故不再挂方法级 @Transactional
* 二是被宿主自身的门面路径复用 * 二是被宿主自身的门面路径复用
*/ */
public interface SimilarAsinPipelineHost { public interface SimilarAsinPipelineHost {
@@ -139,10 +139,20 @@ public class TaskResultFileJobWorker {
taskQueueExecutor.execute(() -> processClaimedWithHeartbeat(job, claim)); taskQueueExecutor.execute(() -> processClaimedWithHeartbeat(job, claim));
return; return;
} catch (RuntimeException ex) { } catch (RuntimeException ex) {
log.warn("[task-file-job] llm module offload failed, fallback inline jobId={} taskId={} moduleType={} msg={}", // 闸门拒绝等待队列已满说明系统已满载此时**不能**回退成内联执行
// 那等于把背压转嫁给 MQ 消费线程让消费线程按分钟级阻塞在 Excel 组装上
// 队列越满越糟改为重新入队由归属实例 15s 一轮的扫描补发
log.warn("[task-file-job] 结果文件任务队列已满,改为重新入队 jobId={} taskId={} moduleType={} msg={} stage=REQUEUE",
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex); job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex);
taskFileJobService.requeue(job.getId(), "结果文件任务队列已满,稍后重投");
return;
} }
} }
// 注意supportsAsyncOffload()=false 的模块11 仍在本方法调用方的线程上
// 内联执行MQ 侧实际并发度 = consumeThreadNumber 默认值 20该并发度直接决定
// 同时在跑的 Excel 组装数量而每一路都持有 DB 连接Hikari 池仅 30还要供
// Tomcat 200 线程与调度线程使用若要收敛应显式设置
// RocketMQMessageListener.consumeThreadNumber 并先测吞吐不要在这里临时改
processClaimedWithHeartbeat(job, claim); processClaimedWithHeartbeat(job, claim);
} }
@@ -89,6 +89,14 @@ public class TaskResultItemService {
return moduleType + "|" + scopeHash + "|" + itemKey; return moduleType + "|" + scopeHash + "|" + itemKey;
} }
/**
* 载入本任务的**全部**结果快照 limit
*
* <p>刻意不加 limit调用方要用它组装完整的结果文件截断即漏行正确性错误
* 不是性能取舍调用方多为逐任务循环调用因此峰值内存 = 单个任务的结果集
* 这是本操作固有的代价若要再降需要改成边读边写文件的流式组装契约
* 涉及 4 个模块的组装实现属独立工程
*/
public <T> List<T> listResultSnapshots(Long taskId, String moduleType, Class<T> clazz) { public <T> List<T> listResultSnapshots(Long taskId, String moduleType, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) { if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return List.of(); return List.of();
@@ -517,11 +517,8 @@ public class TaskScopePayloadStorageService {
try { try {
MessageDigest digest = MessageDigest.getInstance("SHA-256"); MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8)); byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2); // HexFormat 替代逐字节 String.format("%02x")后者每次走 32 Formatter热路径按行
for (byte b : bytes) { return java.util.HexFormat.of().formatHex(bytes);
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) { } catch (Exception ex) {
throw new IllegalStateException("failed to hash scope key", ex); throw new IllegalStateException("failed to hash scope key", ex);
} }
@@ -35,6 +35,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.File; import java.io.File;
import java.math.BigDecimal; import java.math.BigDecimal;
@@ -79,6 +81,10 @@ public class WithdrawTaskService {
private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskDistributedLockService taskDistributedLockService; private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler; private final TaskProgressLightAssembler taskProgressLightAssembler;
/**
* 事务边界收口用Redis 任务锁必须在事务外获取否则自旋等待期间会白占一个 Hikari 连接
*/
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
public ProductRiskDashboardVo dashboard(Long userId) { public ProductRiskDashboardVo dashboard(Long userId) {
validateUserId(userId); validateUserId(userId);
@@ -347,7 +353,10 @@ public class WithdrawTaskService {
} }
} }
@Transactional /**
* 事务边界任务锁在事务外获取自旋等待不再占用 DB 连接
* 缓存清理Redis + 远端范围载荷放到事务提交后锁内执行
*/
public void deleteTask(Long taskId, Long userId) { public void deleteTask(Long taskId, Long userId) {
validateUserId(userId); validateUserId(userId);
FileTaskEntity task = loadTaskForExecution(taskId); FileTaskEntity task = loadTaskForExecution(taskId);
@@ -355,18 +364,24 @@ public class WithdrawTaskService {
throw new BusinessException("任务不存在"); throw new BusinessException("任务不存在");
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>() inNewTransaction(() -> {
.eq(FileResultEntity::getTaskId, taskId) fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)); .eq(FileResultEntity::getTaskId, taskId)
fileTaskMapper.deleteById(taskId); .eq(FileResultEntity::getModuleType, MODULE_TYPE));
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); fileTaskMapper.deleteById(taskId);
taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE); taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
taskProgressSnapshotService.delete(taskId, MODULE_TYPE); taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE);
taskProgressSnapshotService.delete(taskId, MODULE_TYPE);
return null;
});
taskCacheService.deleteTaskCache(taskId); taskCacheService.deleteTaskCache(taskId);
} }
log.info("[withdraw] deleteTask 完成 taskId={} userId={}", taskId, userId);
} }
@Transactional /**
* 事务边界任务锁在事务外获取DB 复核 + 删除 + 重算仍在同一事务内
*/
public void deleteHistory(Long resultId, Long userId) { public void deleteHistory(Long resultId, Long userId) {
validateUserId(userId); validateUserId(userId);
FileResultEntity entity = fileResultMapper.selectById(resultId); FileResultEntity entity = fileResultMapper.selectById(resultId);
@@ -379,10 +394,13 @@ public class WithdrawTaskService {
return; return;
} }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) { try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
fileResultMapper.deleteById(resultId); inNewTransaction(() -> {
taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId); fileResultMapper.deleteById(resultId);
taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId); taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId);
reconcileTaskAfterResultRemoval(taskId); taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId);
reconcileTaskAfterResultRemoval(taskId);
return null;
});
} }
} }
@@ -1050,6 +1068,19 @@ public class WithdrawTaskService {
return taskDistributedLockService.acquire(MODULE_TYPE, taskId); return taskDistributedLockService.acquire(MODULE_TYPE, taskId);
} }
/**
* 显式短事务锁已在事务外获取这里只包住 DB
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内
*/
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
if (transactionManager == null) {
return action.get();
}
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return template.execute(status -> action.get());
}
private List<Long> normalizeTaskIds(List<Long> taskIds) { private List<Long> normalizeTaskIds(List<Long> taskIds) {
return taskIds == null ? List.of() : taskIds.stream() return taskIds == null ? List.of() : taskIds.stream()
.filter(taskId -> taskId != null && taskId > 0) .filter(taskId -> taskId != null && taskId > 0)
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.ziniao.client;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.BoundedLruCache;
import com.nanri.aiimage.config.ZiniaoProperties; import com.nanri.aiimage.config.ZiniaoProperties;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopCacheDto; import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopCacheDto;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoStaffItemVo; import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoStaffItemVo;
@@ -59,8 +60,13 @@ public class ZiniaoClientImpl implements ZiniaoClient {
/** Task 77:单例 RestClient(共享连接池),避免每次调用新建短命客户端。 */ /** Task 77:单例 RestClient(共享连接池),避免每次调用新建短命客户端。 */
private volatile RestClient sharedRestClient; private volatile RestClient sharedRestClient;
/** 按代理地址缓存 RestClient:同一代理复用同一连接池,避免每请求新建。 */ /**
private final Map<String, RestClient> proxyRestClients = new java.util.concurrent.ConcurrentHashMap<>(); * 按代理地址缓存 RestClient同一代理复用同一连接池避免每请求新建
* 有界 LRU代理端点会随店铺轮换不断新增每个都持有连接池与 selector 线程
* 无上限时按店铺数持续累积
*/
private final BoundedLruCache<String, RestClient> proxyRestClients =
new BoundedLruCache<>(BoundedLruCache.DEFAULT_MAX_SIZE);
@Override @Override
public Long getCompanyIdByApiKey(String apiKey) { public Long getCompanyIdByApiKey(String apiKey) {
@@ -124,6 +130,15 @@ public class ZiniaoClientImpl implements ZiniaoClient {
return parseUserStores(raw); return parseUserStores(raw);
} }
/**
* 获取员工登录 token
*
* <p>**刻意不加缓存**2026-09-15 复核调用方 {@code ZiniaoAuthService} 命中店铺匹配缓存时
* 仍会用**新取的** token 重建开店铺 URL见其 getCachedShopMatch 分支只缓存匹配结果
* openStoreUrl null 后重算说明该 token 一次开店铺一次的短时凭据
* 缓存后会把已被消费的 token 发出去用户点开店铺会失败上游往返只为单次用户动作
* 服务不是吞吐瓶颈不要为省一次 RTT 引入这个正确性风险
*/
@Override @Override
public String getUserLoginToken(String apiKey, Long companyId, Long userId) { public String getUserLoginToken(String apiKey, Long companyId, Long userId) {
String raw = postWithApiKey(apiKey, ziniaoProperties.getUserLoginTokenPath(), Map.of( String raw = postWithApiKey(apiKey, ziniaoProperties.getUserLoginTokenPath(), Map.of(
@@ -539,11 +539,8 @@ public boolean isIpWhitelistError(BusinessException ex) {
try { try {
java.security.MessageDigest digest = java.security.MessageDigest.getInstance("SHA-256"); java.security.MessageDigest digest = java.security.MessageDigest.getInstance("SHA-256");
byte[] hash = digest.digest(apiKey.getBytes(StandardCharsets.UTF_8)); byte[] hash = digest.digest(apiKey.getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(); // HexFormat 替代逐字节 String.format("%02x")后者每次走 32 Formatter热路径按调用
for (byte b : hash) { return java.util.HexFormat.of().formatHex(hash);
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) { } catch (Exception ex) {
throw new BusinessException("生成紫鸟 apiKey 缓存键失败"); throw new BusinessException("生成紫鸟 apiKey 缓存键失败");
} }
@@ -937,11 +937,8 @@ public class ZiniaoShopIndexService {
try { try {
MessageDigest digest = MessageDigest.getInstance("SHA-256"); MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] hash = digest.digest(apiKey.getBytes(StandardCharsets.UTF_8)); byte[] hash = digest.digest(apiKey.getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(); // HexFormat 替代逐字节 String.format("%02x")后者每次走 32 Formatter热路径按调用
for (byte b : hash) { return java.util.HexFormat.of().formatHex(hash);
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) { } catch (Exception ex) {
throw new BusinessException("生成紫鸟 apiKey 缓存键失败"); throw new BusinessException("生成紫鸟 apiKey 缓存键失败");
} }
@@ -9,6 +9,10 @@ import com.nanri.aiimage.modules.imagevideo.mapper.ImageVideoAsyncTaskMapper;
import com.nanri.aiimage.modules.imagevideo.model.dto.DouyinCopyRequest; import com.nanri.aiimage.modules.imagevideo.model.dto.DouyinCopyRequest;
import com.nanri.aiimage.modules.imagevideo.model.entity.ImageVideoAsyncTaskEntity; import com.nanri.aiimage.modules.imagevideo.model.entity.ImageVideoAsyncTaskEntity;
import com.nanri.aiimage.modules.imagevideo.model.vo.DouyinCopyVo; import com.nanri.aiimage.modules.imagevideo.model.vo.DouyinCopyVo;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.core.task.TaskExecutor; import org.springframework.core.task.TaskExecutor;
import org.mockito.ArgumentCaptor; import org.mockito.ArgumentCaptor;
@@ -29,6 +33,18 @@ import static org.mockito.Mockito.when;
class ImageVideoAsyncTaskServiceTest { class ImageVideoAsyncTaskServiceTest {
/**
* MyBatis-Plus lambda 列解析需要实体的 TableInfo本类用 mock mapper不启动 Spring
* dispatch/poll 扫描现在走 select(ImageVideoAsyncTaskEntity::getId)2026-09
* 不再每秒把 5 LONGTEXT 列拉回优化引入没有 TableInfo 会抛
* can not find lambda cache同款初始化见 AppearancePatentTaskServiceHistoryBatchTest
*/
@BeforeAll
static void initMybatisPlusTableInfo() {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
ImageVideoAsyncTaskEntity.class);
}
@Test @Test
void parsesNestedDouyinCopyOutputIntoFrontendTextFields() { void parsesNestedDouyinCopyOutputIntoFrontendTextFields() {
ImageVideoCozeService cozeService = new ImageVideoCozeService( ImageVideoCozeService cozeService = new ImageVideoCozeService(
@@ -13,6 +13,10 @@ import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEnt
import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo; import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo; import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo; import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor; import org.mockito.ArgumentCaptor;
@@ -30,6 +34,18 @@ import static org.mockito.Mockito.when;
class PermissionMenuServiceTest { class PermissionMenuServiceTest {
/**
* MyBatis-Plus lambda 列解析需要实体的 TableInfo本类用 mock mapper不启动 Spring
* PermissionMenuService 的权限用户查询走 select(AdminUserEntity::getId, ...)2026-09
* 不再整表拉回含密码哈希的全列优化引入没有 TableInfo 会在构造 wrapper 时抛
* can not find lambda cache同款初始化见 AppearancePatentTaskServiceHistoryBatchTest
*/
@BeforeAll
static void initMybatisPlusTableInfo() {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
AdminUserEntity.class);
}
@Test @Test
void preservesExistingImageVideoPermissionDuringGenericReplacement() { void preservesExistingImageVideoPermissionDuringGenericReplacement() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class); PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
@@ -36,10 +36,11 @@ class QueryAsinSnapshotJsonThrottleTest {
private QueryAsinTaskService service() { private QueryAsinTaskService service() {
// 构造顺序mapper/resultMapper/resolve/excel/cache/oss/resultDownloadResolver/ziniaoSwitch/ // 构造顺序mapper/resultMapper/resolve/excel/cache/oss/resultDownloadResolver/ziniaoSwitch/
// objectMapper/pressure/jobService/resultItemService/snapshot/lock/lightAssembler // objectMapper/pressure/jobService/resultItemService/snapshot/lock/lightAssembler/transactionManager
// transactionManager null本用例只走 persistSnapshotJson不涉及需要事务的删除链路
return new QueryAsinTaskService( return new QueryAsinTaskService(
null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
objectMapper, null, null, taskResultItemService, taskProgressSnapshotService, null, null); objectMapper, null, null, taskResultItemService, taskProgressSnapshotService, null, null, null);
} }
private static void persist(QueryAsinTaskService service, FileTaskEntity task, private static void persist(QueryAsinTaskService service, FileTaskEntity task,
@@ -31,6 +31,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;
import org.mockito.InOrder;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.Spy; import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
@@ -49,6 +50,7 @@ import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never; import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times; import static org.mockito.Mockito.times;
@@ -196,11 +198,38 @@ class ShopDataCrawlTaskServiceTxBoundaryTest {
verify(fileResultMapper).updateById(any(FileResultEntity.class)); verify(fileResultMapper).updateById(any(FileResultEntity.class));
} }
/**
* 事务边界契约2026-09 连接池修复deleteTask 不再挂方法级 @Transactional
* Redis 任务锁必须在事务外获取否则自旋等待最长 TASK_LOCK_WAIT_MILLIS期间会白占一个
* Hikari 连接落库删除仍在同一个短事务executeShortTransaction / REQUIRES_NEW里完成
* 且顺序固定为先取锁后开事务再落库删除
*/
@Test @Test
void deleteTaskKeepsTransactionAnnotationAndPureCompute() throws Exception { void deleteTaskKeepsLockOutsideAndWritesInsideSingleShortTransaction() throws Exception {
Method delete = ShopDataCrawlTaskService.class.getMethod("deleteTask", Long.class, Long.class); Method delete = ShopDataCrawlTaskService.class.getMethod("deleteTask", Long.class, Long.class);
assertTrue(delete.getAnnotation(Transactional.class) != null, assertNull(delete.getAnnotation(Transactional.class),
"deleteTask 落库删除必须保留 @Transactional锁内删除语义不变"); "deleteTask 不得带 @TransactionalRedis 锁必须落在事务外,避免自旋期间占用 DB 连接");
FileTaskEntity task = runningTask();
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
FileResultEntity row = resultRow(1L);
row.setSuccess(1);
when(fileResultMapper.selectList(any())).thenReturn(List.of(row));
when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", TASK_ID))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, "SHOP_DATA_CRAWL", 1L)).thenReturn(true);
when(dailyFileService.findMembersByResultId(1L))
.thenReturn(List.of(mock(com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity.class)));
service.deleteTask(TASK_ID, USER_ID);
InOrder order = inOrder(taskDistributedLockService, transactionManager, fileResultMapper);
order.verify(taskDistributedLockService).acquire("SHOP_DATA_CRAWL", TASK_ID);
order.verify(transactionManager).getTransaction(any());
order.verify(fileResultMapper).delete(any());
// 全部落库删除共用一个短事务
verify(transactionManager, times(1)).getTransaction(any());
verify(fileTaskMapper).deleteById(TASK_ID);
// 删除路径使用抽出的纯函数行为等价由既有 ShopDataCrawlCleanupTest 守门 // 删除路径使用抽出的纯函数行为等价由既有 ShopDataCrawlCleanupTest 守门
assertTrue(ShopDataCrawlTaskService.collectResultIds(List.of()).isEmpty()); assertTrue(ShopDataCrawlTaskService.collectResultIds(List.of()).isEmpty());
} }
@@ -167,8 +167,15 @@ class TaskResultFileJobWorkerOffloadTest {
verify(similar).cleanupResultFileJob(job); verify(similar).cleanupResultFileJob(job);
} }
/**
* 闸门拒绝时的契约2026-09 背压修复**不再回退成内联执行**
*
* 内联兜底会把背压转嫁给 MQ 消费线程消费线程按分钟级阻塞在 Excel 组装 + OSS 上传上
* 队列越满越糟改为重新入队置回 PENDING由归属实例 15s 一轮的扫描补发
* 任务不会丢只是不在消费线程上抢跑原用例名 offloadFailureFallback 断言的正是旧行为
*/
@Test @Test
void offloadFailureFallback() throws Exception { void offloadRejectedRequeuesInsteadOfInlineFallback() throws Exception {
TaskResultFileJobWorker worker = buildWorker(); TaskResultFileJobWorker worker = buildWorker();
TaskFileJobEntity job = job("SIMILAR_ASIN", 4L, 14L); TaskFileJobEntity job = job("SIMILAR_ASIN", 4L, 14L);
allowClaim(job.getId(), job.getTaskId(), job.getModuleType()); allowClaim(job.getId(), job.getTaskId(), job.getModuleType());
@@ -178,8 +185,9 @@ class TaskResultFileJobWorkerOffloadTest {
worker.process(job); worker.process(job);
verify(taskFileJobService).markSuccess(job, null); verify(taskFileJobService).requeue(job.getId(), "结果文件任务队列已满,稍后重投");
verify(similar).cleanupResultFileJob(job); verify(taskFileJobService, never()).markSuccess(any(), any());
verify(similar, never()).processResultFileJob(any());
} }
@Test @Test
@@ -90,13 +90,21 @@ class TaskResultFileJobWorkerOrphanTest {
new BrandResultFileJobHandler(brand, payload), new BrandResultFileJobHandler(brand, payload),
new CollectDataResultFileJobHandler(collectData)); new CollectDataResultFileJobHandler(collectData));
ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers); ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers);
return new TaskResultFileJobWorker( TaskResultFileJobWorker worker = new TaskResultFileJobWorker(
taskFileJobService, taskFileJobService,
taskDistributedLockService, taskDistributedLockService,
mock(FileResultMapper.class), mock(FileResultMapper.class),
mock(TaskFileJobLocalDispatcher.class), mock(TaskFileJobLocalDispatcher.class),
instanceMetadata, instanceMetadata,
registry); registry);
// taskQueueExecutor 是字段注入此前不注入时字段为 nulloffload 分支会抛 NPE 并被
// 兜底成内联执行本测试一直依赖这个偶然行为2026-09 起闸门拒绝改为重新入队
// 而不再回退内联故这里显式注入同步执行器 offload 模块仍按内联语义跑完整链路
java.lang.reflect.Field executorField =
TaskResultFileJobWorker.class.getDeclaredField("taskQueueExecutor");
executorField.setAccessible(true);
executorField.set(worker, (org.springframework.core.task.TaskExecutor) Runnable::run);
return worker;
} }
private static TaskFileJobEntity job(String moduleType, long jobId, long taskId) { private static TaskFileJobEntity job(String moduleType, long jobId, long taskId) {
+4 -1
View File
@@ -11,7 +11,7 @@ import secrets
from flask import Flask from flask import Flask
from flask_cors import CORS from flask_cors import CORS
from utils.db import init_db from utils.db import close_db, init_db
from blueprints.version import version_bp from blueprints.version import version_bp
BASE_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -36,6 +36,9 @@ app.secret_key = os.environ.get('SECRET_KEY', secrets.token_hex(32))
# 注册蓝图:仅版本公开 API(管理后台 API 已迁 Java,见模块 docstring # 注册蓝图:仅版本公开 API(管理后台 API 已迁 Java,见模块 docstring
app.register_blueprint(version_bp) app.register_blueprint(version_bp)
# 请求级数据库连接随请求结束释放(utils/db.get_db 复用同一连接)
app.teardown_appcontext(close_db)
def run_app(host='0.0.0.0', port=15124): def run_app(host='0.0.0.0', port=15124):
init_db() init_db()
+85 -1
View File
@@ -1,9 +1,26 @@
""" """
数据库连接与初始化 数据库连接与初始化
请求级连接复用pymysql 每次 connect 都要做一次完整的鉴权握手而本进程
15124只服务 /api/version/api/version/latest 这类轻量查询握手开销
占了单次请求的大头连接因此挂在 flask.g 上按请求复用请求结束由
close_db注册在 app.teardown_appcontext统一释放
""" """
import re import re
import pymysql import pymysql
try:
from flask import g, has_app_context
except ImportError: # 非 Flask 环境(脚本/离线核查)下退化为每次新建连接
g = None
def has_app_context() -> bool:
return False
# 请求级连接挂在 flask.g 上的属性名(避免与业务字段冲突)
_CONN_ATTR = "_shufu_db_conn"
try: try:
from config import mysql_host as config_mysql_host from config import mysql_host as config_mysql_host
from config import mysql_user as config_mysql_user from config import mysql_user as config_mysql_user
@@ -44,7 +61,7 @@ def describe_db_target():
) )
def get_db(): def _connect():
return pymysql.connect( return pymysql.connect(
host=mysql_host, host=mysql_host,
user=mysql_user, user=mysql_user,
@@ -55,6 +72,73 @@ def get_db():
) )
def _is_alive(conn) -> bool:
"""连接可用性探测:已关闭直接判死,否则发一次 COM_PING。
COM_PING 一次往返远小于一次完整鉴权握手不用 ping(reconnect=True)
该参数在新版 pymysql 已废弃且失败时会自行重连语义不可控
"""
try:
if not conn.open:
return False
conn.ping(reconnect=False)
return True
except Exception:
return False
def _close_quietly(conn) -> None:
"""静默关闭连接(可能为 None / 已关闭 / 断连)。"""
if conn is None:
return
try:
conn.close()
except Exception:
pass
def get_db():
"""取当前请求的数据库连接(同一请求内复用,避免每次请求都做鉴权握手)。
连接挂在 flask.g 请求结束由 close_db 释放被服务端 wait_timeout
断开或调用方提前 close 这里都会静默换成新连接调用方无感知
Flask 应用上下文时脚本调用退回每次新建保持旧行为
"""
ctx = g if has_app_context() else None
conn = getattr(ctx, _CONN_ATTR, None) if ctx is not None else None
if conn is not None:
if _is_alive(conn):
return conn
# 断连/已被关闭:丢弃旧连接后重建(不把坏连接交给业务查询)
_close_quietly(conn)
try:
delattr(ctx, _CONN_ATTR)
except Exception:
pass
conn = _connect()
if ctx is not None:
setattr(ctx, _CONN_ATTR, conn)
return conn
def close_db(exc=None) -> None:
"""释放当前请求的连接(注册为 app.teardown_appcontext 回调)。
异常路径同样会走 teardown因此查询抛错时连接不会泄漏到下一次请求
"""
ctx = g if has_app_context() else None
if ctx is None:
return
conn = getattr(ctx, _CONN_ATTR, None)
if conn is None:
return
try:
delattr(ctx, _CONN_ATTR)
except Exception:
pass
_close_quietly(conn)
def init_db(): def init_db():
"""确保版本公开 API 依赖的最小表结构存在。 """确保版本公开 API 依赖的最小表结构存在。
@@ -194,23 +194,29 @@ async function pollOnce() {
for (const key of Object.keys(lineProgressMap.value)) { for (const key of Object.keys(lineProgressMap.value)) {
if (!busy.some((b) => String(b.id) === key)) delete lineProgressMap.value[key] if (!busy.some((b) => String(b.id) === key)) delete lineProgressMap.value[key]
} }
for (const item of busy) { // for await线
const id = Number(item.id) //brand shared/api/endpoints.ts batch
if (!id) continue const busyIds = busy.map((item) => Number(item.id)).filter((id) => Number.isFinite(id) && id > 0)
try { const details = await Promise.all(
const res = await getBrandTask(id) busyIds.map(async (id) => {
const lp = res?.line_progress try {
if (lp?.has_progress && lp.info) { return { id, res: await getBrandTask(id) }
lineProgressMap.value[String(id)] = { } catch {
current: Number(lp.info.current_line) || 0, //
total: Number(lp.info.total_lines) || 0, return { id, res: null }
phase: lp.info.phase,
}
} else {
delete lineProgressMap.value[String(id)]
} }
} catch { }),
// )
for (const { id, res } of details) {
const lp = res?.line_progress
if (lp?.has_progress && lp.info) {
lineProgressMap.value[String(id)] = {
current: Number(lp.info.current_line) || 0,
total: Number(lp.info.total_lines) || 0,
phase: lp.info.phase,
}
} else {
delete lineProgressMap.value[String(id)]
} }
} }
scheduleNextPoll() scheduleNextPoll()
@@ -179,6 +179,7 @@ import {
type CollectDataTaskDetailVo, type CollectDataTaskDetailVo,
type UploadFileVo, type UploadFileVo,
} from '@/shared/api/java-modules' } from '@/shared/api/java-modules'
import { getPollingProgressBatch } from '@/shared/api/task-progress-polling.ts'
import { formatDateTime } from '@/shared/utils/datetime' import { formatDateTime } from '@/shared/utils/datetime'
import { uploadPathsToJava } from '@/shared/utils/upload-to-java' import { uploadPathsToJava } from '@/shared/utils/upload-to-java'
import { runBatchDelete } from '@/shared/utils/batch-delete' import { runBatchDelete } from '@/shared/utils/batch-delete'
@@ -236,24 +237,70 @@ const taskSnapshots = ref<Record<number, CollectDataTaskDetailVo>>({})
const POLLING_STORAGE_KEY = 'brand:collect-data:polling-task-ids' const POLLING_STORAGE_KEY = 'brand:collect-data:polling-task-ids'
/** 轮询可比对的轻量字段(/tasks/progress/light 白名单里本模块快照有的字段),只有它们变化才重建快照对象 */
function taskProgressSignatureOf(detail: CollectDataTaskDetailVo) {
const task = detail.task
if (!task) return ''
return [task.status ?? '', task.updatedAt ?? ''].join('|')
}
/**
* currentItems taskId 下标索引
* 轮询每轮都要按 taskId 定位行此前每次都对整份列表 map 一遍有索引后只改命中的那一行
* 列表被整体替换loadHistory时必须重建使用处还会做一次 taskId 校验兜底
*/
let currentItemsIndex = new Map<number, number>()
function rebuildCurrentItemsIndex() {
const next = new Map<number, number>()
currentItems.value.forEach((item, index) => {
const taskId = Number(item.taskId)
if (Number.isFinite(taskId) && taskId > 0) next.set(taskId, index)
})
currentItemsIndex = next
}
/** 原地更新 currentItems 里指定任务的那一行(找不到就什么都不做) */
function patchCurrentItem(
taskId: number,
latestItem: CollectDataHistoryItem | undefined,
status: string | undefined,
) {
let index = currentItemsIndex.get(taskId)
if (index === undefined || Number(currentItems.value[index]?.taskId) !== taskId) {
rebuildCurrentItemsIndex()
index = currentItemsIndex.get(taskId)
}
if (index === undefined) return
const current = currentItems.value[index]
if (!current) return
currentItems.value[index] = {
...current,
...(latestItem || {}),
taskStatus: status ?? latestItem?.taskStatus ?? current.taskStatus,
}
}
const progressLoop = useTaskProgressLoop<CollectDataTaskDetailVo>({ const progressLoop = useTaskProgressLoop<CollectDataTaskDetailVo>({
scope: 'collect-data-tab', scope: 'collect-data-tab',
storageKey: POLLING_STORAGE_KEY, storageKey: POLLING_STORAGE_KEY,
fetchProgress: (ids) => getCollectDataTaskProgressBatch(ids), // F5 status/fileReady light
// /退 batch
fetchProgress: (ids) => getPollingProgressBatch('collectData', ids, {
fallback: () => getCollectDataTaskProgressBatch(ids),
}),
extractTaskId: (detail) => detail.task?.id ?? null, extractTaskId: (detail) => detail.task?.id ?? null,
extractStatus: (detail) => detail.task?.status ?? '', extractStatus: (detail) => detail.task?.status ?? '',
onUpdate: (taskId, detail) => { onUpdate: (taskId, detail) => {
const prev = taskSnapshots.value[taskId] const prev = taskSnapshots.value[taskId]
taskSnapshots.value = { const merged: CollectDataTaskDetailVo = prev
...taskSnapshots.value, ? { ...prev, task: { ...(prev.task || {}), ...(detail.task || {}) }, items: detail.items ?? prev.items }
[taskId]: prev : detail
? { ...prev, task: { ...(prev.task || {}), ...(detail.task || {}) }, items: detail.items ?? prev.items } //
: detail, if (!prev || merged.items !== prev.items || taskProgressSignatureOf(prev) !== taskProgressSignatureOf(merged)) {
taskSnapshots.value = { ...taskSnapshots.value, [taskId]: merged }
} }
const latestItem = detail.items?.[0] patchCurrentItem(taskId, detail.items?.[0], detail.task?.status)
currentItems.value = currentItems.value.map((item) => item.taskId === taskId
? { ...item, ...(latestItem || {}), taskStatus: detail.task?.status ?? latestItem?.taskStatus ?? item.taskStatus }
: item)
}, },
onTerminal: async (taskId) => { onTerminal: async (taskId) => {
if (taskSnapshots.value[taskId]) { if (taskSnapshots.value[taskId]) {
@@ -624,6 +671,8 @@ async function loadHistory() {
} }
} }
currentItems.value = merged currentItems.value = merged
// taskId
rebuildCurrentItemsIndex()
} catch { } catch {
/* 忽略:历史接口失败时保持空列表 */ /* 忽略:历史接口失败时保持空列表 */
} }
@@ -33,7 +33,7 @@
</div> </div>
<div v-else class="candidate-table-scroll"> <div v-else class="candidate-table-scroll">
<el-table <el-table
:data="candidates" :data="pagedCandidates"
row-key="id" row-key="id"
height="260" height="260"
class="candidate-table" class="candidate-table"
@@ -59,6 +59,14 @@
</el-table-column> </el-table-column>
</el-table> </el-table>
</div> </div>
<el-pagination
v-if="candidateTotal > candidatePageSize"
class="candidate-pagination"
layout="total, prev, pager, next"
:total="candidateTotal"
:page-size="candidatePageSize"
v-model:current-page="candidatePage"
/>
<div class="section-title condition-title"> <div class="section-title condition-title">
<span>删除条件</span> <span>删除条件</span>
@@ -310,6 +318,13 @@ const {
total: matchedTotal, total: matchedTotal,
paged: pagedMatchedItems, paged: pagedMatchedItems,
} = useTablePaging(matchedItems) } = useTablePaging(matchedItems)
// F9/
const {
page: candidatePage,
pageSize: candidatePageSize,
total: candidateTotal,
paged: pagedCandidates,
} = useTablePaging(candidates)
const historyItems = ref<PatrolDeleteHistoryItem[]>([]); const historyItems = ref<PatrolDeleteHistoryItem[]>([]);
const dashboard = ref<PatrolDeleteDashboardVo>({ const dashboard = ref<PatrolDeleteDashboardVo>({
candidateCount: 0, candidateCount: 0,
@@ -467,10 +467,37 @@ function loadTaskSnapshotsFromStorage() {
} }
} }
function saveTaskSnapshotsToStorage() { /** 快照落盘最小间隔:快照可达数百 KB,全量 JSON.stringify + setItem 会阻塞主线程 */
const SNAPSHOT_SAVE_MIN_INTERVAL_MS = 30000
let lastSnapshotSaveAt = 0
let snapshotSavePending = false
/**
* 快照写盘节流
*
* 每轮轮询都序列化整份快照会周期性卡顿主线程因此
* - 内容未变时 setStorageJson 内部已会跳过写入
* - 这里再按 SNAPSHOT_SAVE_MIN_INTERVAL_MS 节流被节流掉的写入挂起
* 下一次 force / 组件卸载前的 flush落盘时一并写入最新内容
*/
function saveTaskSnapshotsToStorage(force = false) {
if (typeof window === 'undefined') return
const now = Date.now()
if (!force && lastSnapshotSaveAt > 0 && now - lastSnapshotSaveAt < SNAPSHOT_SAVE_MIN_INTERVAL_MS) {
snapshotSavePending = true
return
}
snapshotSavePending = false
lastSnapshotSaveAt = now
setStorageJson(taskSnapshotsStorageKey(), taskSnapshots.value, Object.keys(taskSnapshots.value).length === 0) setStorageJson(taskSnapshotsStorageKey(), taskSnapshots.value, Object.keys(taskSnapshots.value).length === 0)
} }
/** 卸载前把节流挂起的快照补写一次,避免最后一次变更丢失 */
function flushTaskSnapshotsToStorage() {
if (!snapshotSavePending) return
saveTaskSnapshotsToStorage(true)
}
function loadMatchedItemsFromStorage() { function loadMatchedItemsFromStorage() {
try { try {
const raw = typeof window !== 'undefined' ? window.localStorage.getItem(matchedItemsStorageKey()) : null const raw = typeof window !== 'undefined' ? window.localStorage.getItem(matchedItemsStorageKey()) : null
@@ -1147,7 +1174,7 @@ async function processMatchedQueue() {
...taskDetails.value, ...taskDetails.value,
[taskVo.taskId]: 'RUNNING', [taskVo.taskId]: 'RUNNING',
} }
saveTaskSnapshotsToStorage() saveTaskSnapshotsToStorage(true)
saveTaskDetailsToStorage() saveTaskDetailsToStorage()
const queuePayload = await buildQueuePayload(taskVo, row) const queuePayload = await buildQueuePayload(taskVo, row)
queuePayloadText.value = JSON.stringify(queuePayload, null, 2) queuePayloadText.value = JSON.stringify(queuePayload, null, 2)
@@ -1233,7 +1260,7 @@ function recordCreatedTask(taskVo: PriceTrackCreateTaskVo) {
...taskDetails.value, ...taskDetails.value,
[taskVo.taskId]: 'RUNNING', [taskVo.taskId]: 'RUNNING',
} }
saveTaskSnapshotsToStorage() saveTaskSnapshotsToStorage(true)
saveTaskDetailsToStorage() saveTaskDetailsToStorage()
} }
@@ -1418,37 +1445,68 @@ function getPollIntervalMs() {
return getTaskPollIntervalMs() return getTaskPollIntervalMs()
} }
/** 轮询可比对的轻量字段(与 /tasks/progress/light 白名单一致,外加只有重型端点才带的起止时间) */
function taskProgressSignature(task?: PriceTrackTaskDetailVo['task'] | null) {
if (!task) return ''
return [
task.status ?? '',
task.createdAt ?? '',
task.finishedAt ?? '',
task.updatedAt ?? '',
].join('|')
}
async function refreshTaskBatch() { async function refreshTaskBatch() {
const ids = pollingTaskIds.value.filter((id) => id > 0) const ids = pollingTaskIds.value.filter((id) => id > 0)
if (!ids.length) return if (!ids.length) return
try { try {
const batch = await getPriceTrackTaskProgressBatch(ids) // F5 status/fileReady light
let changed = false // 退 batch
const batch = await getPollingProgressBatch('priceTrack', ids, {
fallback: () => getPriceTrackTaskProgressBatch(ids),
})
let settled = false
let snapshotsChanged = false
const nextSnapshots = { ...taskSnapshots.value } const nextSnapshots = { ...taskSnapshots.value }
for (const missingId of batch.missingTaskIds || []) { for (const missingId of batch.missingTaskIds || []) {
console.log(`[price-track] 任务 ${missingId} 在服务端已不存在,停止轮询`)
removePollingTask(missingId) removePollingTask(missingId)
delete nextSnapshots[missingId] delete nextSnapshots[missingId]
snapshotsChanged = true
} }
for (const detail of batch.items || []) { for (const detail of batch.items || []) {
const taskId = detail.task?.id const taskId = detail.task?.id
const status = detail.task?.status const status = detail.task?.status
if (typeof taskId !== 'number' || taskId <= 0) continue if (typeof taskId !== 'number' || taskId <= 0) continue
const prev = nextSnapshots[taskId] const prev = nextSnapshots[taskId]
nextSnapshots[taskId] = prev ? { ...prev, task: { ...(prev.task || {}), ...(detail.task || {}) } } : detail const merged: PriceTrackTaskDetailVo = prev
? { ...prev, task: { ...(prev.task || {}), ...(detail.task || {}) } }
: detail
// KB +
if (!prev || taskProgressSignature(prev.task) !== taskProgressSignature(merged.task)) {
nextSnapshots[taskId] = merged
snapshotsChanged = true
}
if (status) { if (status) {
taskDetails.value[taskId] = status taskDetails.value[taskId] = status
if (status === 'SUCCESS' || status === 'FAILED') { if (status === 'SUCCESS' || status === 'FAILED') {
console.log(`[price-track] 任务 ${taskId} 已到终态 ${status},移出轮询集合并刷新列表`)
removePollingTask(taskId) removePollingTask(taskId)
changed = true settled = true
} }
} }
} }
taskSnapshots.value = nextSnapshots if (snapshotsChanged) {
saveTaskSnapshotsToStorage() taskSnapshots.value = nextSnapshots
saveTaskSnapshotsToStorage()
}
saveTaskDetailsToStorage() saveTaskDetailsToStorage()
await loadHistory() // 10s
syncPollingIdsWithHistory() if (settled || snapshotsChanged) {
if (changed) { await loadHistory()
syncPollingIdsWithHistory()
}
if (settled) {
await loadDashboard() await loadDashboard()
void resumeLoopExecutionIfNeeded() void resumeLoopExecutionIfNeeded()
} }
@@ -1508,7 +1566,8 @@ function removePollingTask(taskId: number) {
const next = { ...taskSnapshots.value } const next = { ...taskSnapshots.value }
delete next[taskId] delete next[taskId]
taskSnapshots.value = next taskSnapshots.value = next
saveTaskSnapshotsToStorage() //
saveTaskSnapshotsToStorage(true)
} }
} }
@@ -1679,6 +1738,8 @@ onUnmounted(() => {
clearSleepTimers() clearSleepTimers()
timers.clearScope() timers.clearScope()
if (countryPrefSaveTimer) timers.clearTimer('preference-save', countryPrefSaveTimer) if (countryPrefSaveTimer) timers.clearTimer('preference-save', countryPrefSaveTimer)
//
flushTaskSnapshotsToStorage()
}) })
/** /**
@@ -33,7 +33,7 @@
</div> </div>
<div v-else class="candidate-table-scroll"> <div v-else class="candidate-table-scroll">
<el-table <el-table
:data="candidates" :data="pagedCandidates"
row-key="id" row-key="id"
height="260" height="260"
class="candidate-table" class="candidate-table"
@@ -59,6 +59,14 @@
</el-table-column> </el-table-column>
</el-table> </el-table>
</div> </div>
<el-pagination
v-if="candidateTotal > candidatePageSize"
class="candidate-pagination"
layout="total, prev, pager, next"
:total="candidateTotal"
:page-size="candidatePageSize"
v-model:current-page="candidatePage"
/>
<ZiniaoVersionSetting v-model="ziniaoVersion" /> <ZiniaoVersionSetting v-model="ziniaoVersion" />
@@ -245,6 +253,13 @@ const {
total: matchedTotal, total: matchedTotal,
paged: pagedMatchedItems, paged: pagedMatchedItems,
} = useTablePaging(matchedItems) } = useTablePaging(matchedItems)
// F9/
const {
page: candidatePage,
pageSize: candidatePageSize,
total: candidateTotal,
paged: pagedCandidates,
} = useTablePaging(candidates)
const historyItems = ref<QueryAsinHistoryItem[]>([]); const historyItems = ref<QueryAsinHistoryItem[]>([]);
const dashboard = ref<QueryAsinDashboardVo>({ const dashboard = ref<QueryAsinDashboardVo>({
candidateCount: 0, candidateCount: 0,
@@ -17,7 +17,7 @@
<div class="section-title">备选区</div> <div class="section-title">备选区</div>
<div v-if="!candidates.length" class="empty-candidates">暂无备选店铺</div> <div v-if="!candidates.length" class="empty-candidates">暂无备选店铺</div>
<div v-else class="candidate-table-scroll"> <div v-else class="candidate-table-scroll">
<el-table :data="candidates" row-key="id" height="250" class="candidate-table" <el-table :data="pagedCandidates" row-key="id" height="250" class="candidate-table"
@selection-change="onSelectionChange"> @selection-change="onSelectionChange">
<el-table-column type="selection" width="42" /> <el-table-column type="selection" width="42" />
<el-table-column prop="shop_name" label="店铺名" min-width="140" show-overflow-tooltip /> <el-table-column prop="shop_name" label="店铺名" min-width="140" show-overflow-tooltip />
@@ -28,6 +28,9 @@
</el-table-column> </el-table-column>
</el-table> </el-table>
</div> </div>
<el-pagination v-if="candidateTotal > candidatePageSize" class="candidate-pagination"
layout="total, prev, pager, next" :total="candidateTotal" :page-size="candidatePageSize"
v-model:current-page="candidatePage" />
<div class="section-title">抓取国家与顺序</div> <div class="section-title">抓取国家与顺序</div>
<div class="country-pref-checks"> <div class="country-pref-checks">
@@ -187,6 +190,13 @@ const {
total: matchedTotal, total: matchedTotal,
paged: pagedMatchedItems, paged: pagedMatchedItems,
} = useTablePaging(matchedItems) } = useTablePaging(matchedItems)
// F9/
const {
page: candidatePage,
pageSize: candidatePageSize,
total: candidateTotal,
paged: pagedCandidates,
} = useTablePaging(candidates)
const historyItems = ref<ShopDataCrawlHistoryItem[]>([]) const historyItems = ref<ShopDataCrawlHistoryItem[]>([])
const dashboard = ref<ShopDataCrawlDashboardVo>({ candidateCount: 0, processedTaskCount: 0, successTaskCount: 0, failedTaskCount: 0 }) const dashboard = ref<ShopDataCrawlDashboardVo>({ candidateCount: 0, processedTaskCount: 0, successTaskCount: 0, failedTaskCount: 0 })
const orderedCountryCodes = ref<string[]>(COUNTRY_OPTIONS.map((row) => row.code)) const orderedCountryCodes = ref<string[]>(COUNTRY_OPTIONS.map((row) => row.code))
@@ -452,7 +452,8 @@ function sleep(ms: number) { if (disposed) return Promise.resolve(); return time
function clearSleepTimers() { timers.clearCategory('queue-wait') } function clearSleepTimers() { timers.clearCategory('queue-wait') }
function isTransientBackendError(error: unknown) { const message = (error instanceof Error ? error.message : String(error || '')).toLowerCase(); return TRANSIENT_BACKEND_ERROR_PATTERNS.some((pattern) => message.includes(pattern.toLowerCase())) } function isTransientBackendError(error: unknown) { const message = (error instanceof Error ? error.message : String(error || '')).toLowerCase(); return TRANSIENT_BACKEND_ERROR_PATTERNS.some((pattern) => message.includes(pattern.toLowerCase())) }
async function withTransientRetry<T>(action: () => Promise<T>, onRetry: (attempt: number, maxAttempts: number) => void, maxAttempts = 10) { let attempt = 0; while (true) { if (disposed) throw new Error('component disposed'); try { return await action() } catch (error) { attempt += 1; if (!isTransientBackendError(error) || attempt >= maxAttempts) throw error; onRetry(attempt, maxAttempts); await sleep(getPollIntervalMs()) } } } async function withTransientRetry<T>(action: () => Promise<T>, onRetry: (attempt: number, maxAttempts: number) => void, maxAttempts = 10) { let attempt = 0; while (true) { if (disposed) throw new Error('component disposed'); try { return await action() } catch (error) { attempt += 1; if (!isTransientBackendError(error) || attempt >= maxAttempts) throw error; onRetry(attempt, maxAttempts); await sleep(getPollIntervalMs()) } } }
async function refreshTaskBatch() { if (!pollingTaskIds.value.length) return { settledTaskIds: [] as number[] }; try { const batch = await getShopMatchTaskProgressBatch(pollingTaskIds.value); const nextSnapshots = { ...taskSnapshots.value }; const settledTaskIds = new Set<number>(); for (const missingId of batch.missingTaskIds || []) { removeTaskLocally(missingId); delete nextSnapshots[missingId]; settledTaskIds.add(missingId) } for (const detail of batch.items || []) { const taskId = detail.task?.id; if (!taskId) continue; const prev = nextSnapshots[taskId]; nextSnapshots[taskId] = prev ? { ...prev, task: { ...(prev.task || {}), ...(detail.task || {}) } } : detail; const status = detail.task?.status || ''; if (!status) continue; taskDetails.value[taskId] = status; if (isTaskTerminalStatus(status)) { removePollingTask(taskId); settledTaskIds.add(taskId); continue } if (status === 'SCHEDULED') { stopPollingTask(taskId); settledTaskIds.add(taskId) } } taskSnapshots.value = nextSnapshots; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); restoreScheduledDispatches(); return { settledTaskIds: Array.from(settledTaskIds) } } catch { return { settledTaskIds: [] as number[] } } } async function refreshTaskBatch() { if (!pollingTaskIds.value.length) return { settledTaskIds: [] as number[] }; try { // F5 status light /退 batch
const batch = await getPollingProgressBatch('shopMatch', pollingTaskIds.value, { fallback: () => getShopMatchTaskProgressBatch(pollingTaskIds.value) }); const nextSnapshots = { ...taskSnapshots.value }; const settledTaskIds = new Set<number>(); for (const missingId of batch.missingTaskIds || []) { removeTaskLocally(missingId); delete nextSnapshots[missingId]; settledTaskIds.add(missingId) } for (const detail of batch.items || []) { const taskId = detail.task?.id; if (!taskId) continue; const prev = nextSnapshots[taskId]; nextSnapshots[taskId] = prev ? { ...prev, task: { ...(prev.task || {}), ...(detail.task || {}) } } : detail; const status = detail.task?.status || ''; if (!status) continue; taskDetails.value[taskId] = status; if (isTaskTerminalStatus(status)) { removePollingTask(taskId); settledTaskIds.add(taskId); continue } if (status === 'SCHEDULED') { stopPollingTask(taskId); settledTaskIds.add(taskId) } } taskSnapshots.value = nextSnapshots; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); restoreScheduledDispatches(); return { settledTaskIds: Array.from(settledTaskIds) } } catch { return { settledTaskIds: [] as number[] } } }
function getPollIntervalMs() { return getTaskPollIntervalMs() } function getPollIntervalMs() { return getTaskPollIntervalMs() }
function scheduleNextPoll(immediate = false) { if (disposed) return; if (pollTimer.value) { if (!immediate) return; timers.clearTimer('task-poll', pollTimer.value); pollTimer.value = null } const run = async () => { pollTimer.value = null; if (disposed) return; if (pollingInFlight.value || !pollingTaskIds.value.length) { if (pollingTaskIds.value.length) scheduleNextPoll(); return } pollingInFlight.value = true; try { const { settledTaskIds } = await refreshTaskBatch(); if (settledTaskIds.length) await Promise.allSettled([loadHistory(), loadDashboard()]) } finally { pollingInFlight.value = false } if (!disposed && pollingTaskIds.value.length) pollTimer.value = timers.setTimeout('task-poll', run, getPollIntervalMs()) }; if (immediate) void run(); else pollTimer.value = timers.setTimeout('task-poll', run, getPollIntervalMs()) } function scheduleNextPoll(immediate = false) { if (disposed) return; if (pollTimer.value) { if (!immediate) return; timers.clearTimer('task-poll', pollTimer.value); pollTimer.value = null } const run = async () => { pollTimer.value = null; if (disposed) return; if (pollingInFlight.value || !pollingTaskIds.value.length) { if (pollingTaskIds.value.length) scheduleNextPoll(); return } pollingInFlight.value = true; try { const { settledTaskIds } = await refreshTaskBatch(); if (settledTaskIds.length) await Promise.allSettled([loadHistory(), loadDashboard()]) } finally { pollingInFlight.value = false } if (!disposed && pollingTaskIds.value.length) pollTimer.value = timers.setTimeout('task-poll', run, getPollIntervalMs()) }; if (immediate) void run(); else pollTimer.value = timers.setTimeout('task-poll', run, getPollIntervalMs()) }
function ensurePolling(immediate = false) { scheduleNextPoll(immediate) } function ensurePolling(immediate = false) { scheduleNextPoll(immediate) }
@@ -44,7 +44,7 @@
</div> </div>
<div v-else class="candidate-table-scroll"> <div v-else class="candidate-table-scroll">
<el-table <el-table
:data="candidates" :data="pagedCandidates"
row-key="id" row-key="id"
height="260" height="260"
class="candidate-table" class="candidate-table"
@@ -70,6 +70,14 @@
</el-table-column> </el-table-column>
</el-table> </el-table>
</div> </div>
<el-pagination
v-if="candidateTotal > candidatePageSize"
class="candidate-pagination"
layout="total, prev, pager, next"
:total="candidateTotal"
:page-size="candidatePageSize"
v-model:current-page="candidatePage"
/>
<ZiniaoVersionSetting v-model="ziniaoVersion" /> <ZiniaoVersionSetting v-model="ziniaoVersion" />
@@ -296,6 +304,13 @@ const {
total: matchedTotal, total: matchedTotal,
paged: pagedMatchedItems, paged: pagedMatchedItems,
} = useTablePaging(matchedItems) } = useTablePaging(matchedItems)
// F9/
const {
page: candidatePage,
pageSize: candidatePageSize,
total: candidateTotal,
paged: pagedCandidates,
} = useTablePaging(candidates)
const historyItems = ref<WithdrawHistoryItem[]>([]); const historyItems = ref<WithdrawHistoryItem[]>([]);
const dashboard = ref<WithdrawDashboardVo>({ const dashboard = ref<WithdrawDashboardVo>({
candidateCount: 0, candidateCount: 0,
@@ -139,6 +139,8 @@ export function useTaskProgressLoop<TDetail>(
let pollTimer: number | null = null let pollTimer: number | null = null
let disposed = false let disposed = false
let failureCount = 0 let failureCount = 0
/** 页面隐藏导致停表时为 true;恢复可见后据此判断需要重启轮询 */
let pausedForVisibility = false
function persist() { function persist() {
writeIdsToStorage(options.storageKey, taskIds.value) writeIdsToStorage(options.storageKey, taskIds.value)
@@ -237,13 +239,13 @@ export function useTaskProgressLoop<TDetail>(
} }
function scheduleNext(immediate = false) { function scheduleNext(immediate = false) {
if (disposed) return if (disposed || pausedForVisibility) return
if (pollTimer != null && !immediate) return if (pollTimer != null && !immediate) return
clearPollTimer() clearPollTimer()
const run = async () => { const run = async () => {
pollTimer = null pollTimer = null
if (disposed) return if (disposed || pausedForVisibility) return
if (!taskIds.value.length) return if (!taskIds.value.length) return
if (inFlight.value) { if (inFlight.value) {
// 上一次还没回,按退避策略延迟重试(默认 500ms,可配置) // 上一次还没回,按退避策略延迟重试(默认 500ms,可配置)
@@ -251,7 +253,7 @@ export function useTaskProgressLoop<TDetail>(
return return
} }
await refreshOnce() await refreshOnce()
if (!disposed && taskIds.value.length > 0) { if (!disposed && !pausedForVisibility && taskIds.value.length > 0) {
// 连续失败时按指数退避拉长间隔,避免后端故障时每 5s 撞一次(成功即复位) // 连续失败时按指数退避拉长间隔,避免后端故障时每 5s 撞一次(成功即复位)
const delay = failureCount > 0 ? getTaskPollBackoffMs(failureCount) : intervalMs() const delay = failureCount > 0 ? getTaskPollBackoffMs(failureCount) : intervalMs()
pollTimer = timers.setTimeout('task-poll', run, delay) pollTimer = timers.setTimeout('task-poll', run, delay)
@@ -266,7 +268,7 @@ export function useTaskProgressLoop<TDetail>(
} }
function ensure(immediate = false) { function ensure(immediate = false) {
if (disposed) return if (disposed || pausedForVisibility) return
if (!taskIds.value.length) return if (!taskIds.value.length) return
if (pollTimer != null && !immediate) return if (pollTimer != null && !immediate) return
scheduleNext(immediate) scheduleNext(immediate)
@@ -292,38 +294,67 @@ export function useTaskProgressLoop<TDetail>(
{ flush: 'post' }, { flush: 'post' },
) )
/**
*
* visibilitychange
*/
function pauseForVisibility() {
if (disposed || !taskIds.value.length) return
pausedForVisibility = true
stop()
console.log(`[task-poll] 页面隐藏,暂停轮询: ${options.scope}`)
}
/** 恢复可见:按前台刷新开关决定是否立即补拉,但无论如何都要把轮询重启起来 */
function resumeAfterVisibility() {
if (disposed) return
const wasPaused = pausedForVisibility
pausedForVisibility = false
if (!taskIds.value.length) return
if (getTaskForegroundRefreshEnabled()) {
const delay = getTaskForegroundRefreshDelayMs()
if (delay > 0) {
scheduleNextDelayed(delay)
} else {
scheduleNext(true)
}
if (wasPaused) {
console.log(`[task-poll] 页面恢复可见,已补拉一轮并重启轮询: ${options.scope}`)
}
return
}
if (wasPaused) {
// 开关关闭时不立即补拉,但必须重启轮询,否则隐藏过一次就再也不轮询
console.log(`[task-poll] 页面恢复可见,重启轮询(前台即时刷新已关闭): ${options.scope}`)
scheduleNext()
}
}
// 切到前台后立刻拉一次(可配置开关与延迟),让用户回到页面看到的是最新状态 // 切到前台后立刻拉一次(可配置开关与延迟),让用户回到页面看到的是最新状态
let visibilityHandler: (() => void) | null = null let visibilityHandler: (() => void) | null = null
if (typeof document !== 'undefined') { if (typeof document !== 'undefined') {
visibilityHandler = () => { visibilityHandler = () => {
if ( if (document.visibilityState === 'visible') {
getTaskForegroundRefreshEnabled() && resumeAfterVisibility()
document.visibilityState === 'visible' && } else {
taskIds.value.length > 0 pauseForVisibility()
) {
const delay = getTaskForegroundRefreshDelayMs()
if (delay > 0) {
scheduleNextDelayed(delay)
} else {
scheduleNext(true)
}
} }
} }
document.addEventListener('visibilitychange', visibilityHandler) document.addEventListener('visibilitychange', visibilityHandler)
} }
function scheduleNextDelayed(delayMs: number) { function scheduleNextDelayed(delayMs: number) {
if (disposed) return if (disposed || pausedForVisibility) return
clearPollTimer() clearPollTimer()
const run = () => { const run = () => {
pollTimer = null pollTimer = null
if (disposed || !taskIds.value.length) return if (disposed || pausedForVisibility || !taskIds.value.length) return
if (inFlight.value) { if (inFlight.value) {
pollTimer = timers.setTimeout('task-poll', run, getTaskPollBackoffMs(0)) pollTimer = timers.setTimeout('task-poll', run, getTaskPollBackoffMs(0))
return return
} }
void refreshOnce() void refreshOnce()
if (!disposed && taskIds.value.length > 0) { if (!disposed && !pausedForVisibility && taskIds.value.length > 0) {
const delay = failureCount > 0 ? getTaskPollBackoffMs(failureCount) : intervalMs() const delay = failureCount > 0 ? getTaskPollBackoffMs(failureCount) : intervalMs()
pollTimer = timers.setTimeout('task-poll', run, delay) pollTimer = timers.setTimeout('task-poll', run, delay)
} }
+40 -2
View File
@@ -57,6 +57,8 @@ interface VisibilityHarness {
loop: ReturnType<typeof useTaskProgressLoop<{ taskId?: number; status?: string }>> loop: ReturnType<typeof useTaskProgressLoop<{ taskId?: number; status?: string }>>
doc: FakeDoc doc: FakeDoc
sink: TimerSinkEntry[] sink: TimerSinkEntry[]
/** 被 clearTimeout 取消过的定时器 id(用于断言隐藏时确实停了表) */
cleared: unknown[]
fetchCount: () => number fetchCount: () => number
} }
@@ -72,6 +74,7 @@ async function makeVisibilityLoop(
} }
const doc = installFakeDocument(visibility) const doc = installFakeDocument(visibility)
const sink: TimerSinkEntry[] = [] const sink: TimerSinkEntry[] = []
const cleared: unknown[] = []
const storage = new Map<string, string>() const storage = new Map<string, string>()
let fetchCount = 0 let fetchCount = 0
;(globalThis as Record<string, unknown>).window = { ;(globalThis as Record<string, unknown>).window = {
@@ -84,7 +87,10 @@ async function makeVisibilityLoop(
sink.push({ fn, ms }) sink.push({ fn, ms })
return globalThis.setTimeout(fn, ms) return globalThis.setTimeout(fn, ms)
}, },
clearTimeout: (id: unknown) => globalThis.clearTimeout(id as number), clearTimeout: (id: unknown) => {
cleared.push(id)
globalThis.clearTimeout(id as number)
},
setInterval: (fn: () => void, ms: number) => globalThis.setInterval(fn, ms), setInterval: (fn: () => void, ms: number) => globalThis.setInterval(fn, ms),
clearInterval: (id: unknown) => globalThis.clearInterval(id as number), clearInterval: (id: unknown) => globalThis.clearInterval(id as number),
} }
@@ -98,7 +104,7 @@ async function makeVisibilityLoop(
extractStatus: (d) => d?.status, extractStatus: (d) => d?.status,
}) })
loop.reset([1]) loop.reset([1])
return { loop, doc, sink, fetchCount: () => fetchCount } return { loop, doc, sink, cleared, fetchCount: () => fetchCount }
} }
test('test_hidden_long_interval', async () => { test('test_hidden_long_interval', async () => {
@@ -187,3 +193,35 @@ test('test_hidden_no_immediate_refresh', async () => {
h.loop.dispose() h.loop.dispose()
cleanupDocument() cleanupDocument()
}) })
test('test_hidden_stops_polling', async () => {
const h = await makeVisibilityLoop('visible')
await tick(10)
const scheduledBefore = h.sink.length
const before = h.fetchCount()
h.doc.setVisibility('hidden')
h.doc.dispatch('visibilitychange')
await tick(10)
assert.equal(h.fetchCount(), before, '隐藏后不再发起请求')
assert.equal(h.sink.length, scheduledBefore, '隐藏后不再排定新的轮询定时器')
assert.ok(h.cleared.length > 0, '隐藏时清掉了已排定的轮询定时器')
h.loop.dispose()
cleanupDocument()
})
test('test_visible_restarts_polling', async () => {
const h = await makeVisibilityLoop('visible')
await tick(10)
h.doc.setVisibility('hidden')
h.doc.dispatch('visibilitychange')
await tick(10)
const before = h.fetchCount()
h.doc.setVisibility('visible')
h.doc.dispatch('visibilitychange')
await tick(10)
assert.equal(h.fetchCount(), before + 1, '恢复可见立即补拉一轮')
assert.equal(h.sink.at(-1)?.ms, TASK_POLL_VISIBLE_INTERVAL_MS, '恢复可见后按可见间隔重启轮询')
assert.deepEqual(h.loop.taskIds.value, [1], '停表期间任务集合不变')
h.loop.dispose()
cleanupDocument()
})