perf(全链路): 连接池/事务边界/轮询与 IO 效率收口

Java:
- LLM 180s 读超时不再被全局 call-timeout 静默截断成 90s(长思考请求被掐断→重试→付费网关二次计费)
- 代理 HttpClient 缓存改有界 LRU(jikip 每次提取新 IP,无界缓存持续泄漏 selector 线程与连接池)
- 12 个 service 的 Redis 任务锁移出 @Transactional(自旋最坏 10s 白占 DB 连接,池仅 30),远端对象删/传改 afterCommit
- 结果文件 Job 闸门拒绝时不再回退内联执行(改重新入队,避免把背压转嫁给 MQ 消费线程)
- imagevideo 每秒扫描加列投影、过期清理加 LIMIT;权限页整表查询改列投影(不再拉回密码哈希)
- 哈希改 HexFormat;补 5 处"不能改"的技术依据注释(批量插入会丢回填主键、流式丢模板与图片等)

前端:4 个工具页轮询改轻量端点(带 fallback);PriceTrack 快照节流写盘;候选店铺表分页;页面隐藏时停表

客户端:HTTP 连接池按出口复用(Session 仍每请求新建,保持无跨请求状态);品牌检测 WIPO 逐请求握手;
代理配置按 mtime 缓存;串行任务改专属池;异常降级为标签页重连;紫鸟启动改端口轮询;模板编译缓存;
日志上报连接与落盘收口;Flask 版本 API 改按请求复用连接
This commit is contained in:
2026-09-15 23:01:59 +08:00
parent d6f8368493
commit 228d481211
44 changed files with 1550 additions and 423 deletions
@@ -0,0 +1,70 @@
package com.nanri.aiimage.common.util;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.function.Function;
/**
* 有界 LRU 缓存:容量超限时自动淘汰最久未使用的条目。
*
* <p>用于按「外部端点」缓存长生命周期资源(HttpClient / RestClient)。这类资源各自持有
* 连接池与 selector 线程,无界累积会持续泄漏线程与内存:代理端点每次提取往往是新的
* IP:port(jikip 提取),无上限的缓存只增不减。
*
* <p>淘汰时只从缓存移除引用,不做显式关闭:JDK 的 HttpClientImpl 注册了 Cleaner
* 对象不可达后由 GC 回收并关闭其 selector 线程;显式关闭反而可能打断仍在途的请求。
*/
public final class BoundedLruCache<K, V> {
/** 默认容量:代理端点数量级远小于此,足够覆盖热点端点又不至于累积。 */
public static final int DEFAULT_MAX_SIZE = 64;
private final int maxSize;
private final Map<K, V> store;
public BoundedLruCache() {
this(DEFAULT_MAX_SIZE);
}
public BoundedLruCache(int maxSize) {
this.maxSize = Math.max(1, maxSize);
// accessOrder=true 使 get 也刷新顺序(真正的 LRU);synchronizedMap 保证其线程安全
this.store = Collections.synchronizedMap(new LinkedHashMap<>(16, 0.75f, true) {
@Override
protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
return size() > BoundedLruCache.this.maxSize;
}
});
}
/**
* 取缓存值,缺失时用 loader 计算并放入。
*
* <p>与 {@code ConcurrentHashMap.computeIfAbsent} 不同,此处不保证 loader 的原子性:
* 并发首次访问同一 key 时可能各自构造一次,随后其中一个覆盖另一个。对
* HttpClient/RestClient 这类构造廉价且幂等的资源可接受,换来的是锁粒度更小。
*/
public V computeIfAbsent(K key, Function<K, V> loader) {
V existing = store.get(key);
if (existing != null) {
return existing;
}
V created = loader.apply(key);
store.put(key, created);
return created;
}
public int size() {
return store.size();
}
/** 当前容量上限,供日志与测试断言使用。 */
public int maxSize() {
return maxSize;
}
public void clear() {
store.clear();
}
}
@@ -1,5 +1,7 @@
package com.nanri.aiimage.config;
import com.nanri.aiimage.common.util.BoundedLruCache;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.JdkClientHttpRequestFactory;
@@ -14,8 +16,6 @@ import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* Task 77:外部 HTTP 客户端统一连接复用池。
@@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentHashMap;
* 每次请求都重新建连。各客户端按自身超时创建独立的
* JdkClientHttpRequestFactory(共享底层连接池),RestClient 单例懒加载。
*/
@Slf4j
public class HttpClientPool {
private static volatile HttpClient sharedHttpClient;
@@ -103,8 +104,12 @@ public class HttpClientPool {
public static ClientHttpRequestFactory requestFactory(int readTimeoutMillis, String proxyUrl) {
long safeReadTimeout = Math.max(1L, readTimeoutMillis);
long callTimeout = configuredCallTimeoutMillis;
if (callTimeout > 0L) {
safeReadTimeout = Math.min(safeReadTimeout, callTimeout);
if (callTimeout > 0L && safeReadTimeout > callTimeout) {
// 读超时以调用方显式值为准,不再被全局 call-timeout 截断:
// LLM 长思考配的是 180sllm-read-timeout-millis),曾被静默压到 90s
// 导致请求在 90s 被掐断 → 上层重试 → 付费网关二次计费(2026-09-15 修复)。
// 各调用方的超时已由各自的 HttpConfigResolver 钳制,此处不再二次收敛。
log.debug("读超时 {}ms 超过全局 call-timeout {}ms,按调用方显式值生效", safeReadTimeout, callTimeout);
}
JdkClientHttpRequestFactory factory =
new JdkClientHttpRequestFactory(httpClientFor(proxyUrl));
@@ -166,5 +171,6 @@ public class HttpClientPool {
private record ProxyEndpoint(String host, int port, String userInfo) {
}
private static final Map<ProxyEndpoint, HttpClient> PROXY_CLIENTS = new ConcurrentHashMap<>();
private static final BoundedLruCache<ProxyEndpoint, HttpClient> PROXY_CLIENTS =
new BoundedLruCache<>(BoundedLruCache.DEFAULT_MAX_SIZE);
}
@@ -69,6 +69,8 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.File;
@@ -522,20 +524,28 @@ public class AppearancePatentTaskService {
scheduleLlmPipelineForSubmittedChunk(context);
}
/**
* 删除任务。
*
* <p>事务边界:远端载荷删除与缓存清理移到事务提交后执行 —— 单次删除超时 120s、重试 3 次,
* 放在 DB 事务里会让连接被长时间占用(项目在 TaskScopePayloadStorageService 已写明该约束)。
*/
@Transactional
public void deleteTask(Long taskId, Long userId) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) {
throw new BusinessException("任务不存在");
}
List<String> payloads = collectTransientTaskPayloads(taskId);
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>().eq(FileResultEntity::getTaskId, taskId).eq(FileResultEntity::getModuleType, MODULE_TYPE));
deleteTransientTaskPayloads(taskId);
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>().eq(TaskScopeStateEntity::getTaskId, taskId).eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>().eq(TaskChunkEntity::getTaskId, taskId).eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskCacheService.deleteTaskCache(taskId);
// 同步清理 task_file_job,避免被删除任务遗留的 PENDING/FAILED 行被 TaskResultFileJobWorker 反复扫描出 task not found。
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
fileTaskMapper.deleteById(taskId);
// 事务提交后再做远端删除与缓存清理
deletePayloadsAfterCommit(payloads, taskId);
runAfterCommit(() -> taskCacheService.deleteTaskCache(taskId));
}
public void deleteHistory(Long resultId, Long userId) {
@@ -2992,18 +3002,27 @@ public class AppearancePatentTaskService {
}
}
private void deleteTransientTaskPayloads(Long taskId) {
/** 只读收集任务范围/分片载荷指针,供事务提交后做远端删除。 */
private List<String> collectTransientTaskPayloads(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
return List.of();
}
List<String> payloads = new ArrayList<>();
List<TaskScopeStateEntity> scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (scopes != null) {
for (TaskScopeStateEntity scope : scopes) {
transientPayloadStorageService.deletePayloadIfPresent(scope.getParsedPayloadJson());
transientPayloadStorageService.deletePayloadIfPresent(scope.getStateJson());
if (scope == null) {
continue;
}
if (scope.getParsedPayloadJson() != null && !scope.getParsedPayloadJson().isBlank()) {
payloads.add(scope.getParsedPayloadJson());
}
if (scope.getStateJson() != null && !scope.getStateJson().isBlank()) {
payloads.add(scope.getStateJson());
}
}
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
@@ -3012,9 +3031,49 @@ public class AppearancePatentTaskService {
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
payloads.add(chunk.getPayloadJson());
}
}
}
return payloads;
}
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
private void deletePayloadsAfterCommit(List<String> payloads, Long taskId) {
if (payloads == null || payloads.isEmpty()) {
return;
}
runAfterCommit(() -> {
for (String payload : payloads) {
try {
transientPayloadStorageService.deletePayloadIfPresent(payload);
} catch (Exception ex) {
log.warn("[appearance-patent] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage());
}
}
});
}
/**
* 删除范围/分片的远端载荷(保留给无事务的清理链路调用)。
*/
private void deleteTransientTaskPayloads(Long taskId) {
deletePayloadsAfterCommit(collectTransientTaskPayloads(taskId), taskId);
}
/** 有活动事务则注册 afterCommit,否则立即执行。 */
private void runAfterCommit(Runnable action) {
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
action.run();
}
});
return;
}
action.run();
}
private record SubmitContext(FileTaskEntity task,
@@ -13,9 +13,12 @@ import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
@@ -27,6 +30,7 @@ import java.util.Map;
@Service
@RequiredArgsConstructor
@Slf4j
public class BrandTaskStorageService {
private static final String MODULE_TYPE = "BRAND";
@@ -236,6 +240,13 @@ public class BrandTaskStorageService {
return new ChunkStoreResult(false, completed, countCompletedFiles(taskId), aggregate);
}
/**
* 删除任务的全部范围/分片数据。
*
* <p>事务边界:远端载荷删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次,
* 放在 DB 事务里会让连接被长时间占用(项目在 TaskScopePayloadStorageService 已写明该约束)。
* 提交后再删还有个好处:引用计数查询看到的是删除完成后的 DB 状态,判断更准确。
*/
@Transactional
public void deleteTaskData(Long taskId) {
if (taskId == null || taskId <= 0) {
@@ -245,27 +256,68 @@ public class BrandTaskStorageService {
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (states != null) {
for (TaskScopeStateEntity state : states) {
transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson());
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
}
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
deletePayloadsAfterCommit(states, chunks, taskId);
}
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
private void deletePayloadsAfterCommit(List<TaskScopeStateEntity> states,
List<TaskChunkEntity> chunks,
Long taskId) {
List<String> payloads = new ArrayList<>();
if (states != null) {
for (TaskScopeStateEntity state : states) {
if (state == null) {
continue;
}
if (state.getParsedPayloadJson() != null && !state.getParsedPayloadJson().isBlank()) {
payloads.add(state.getParsedPayloadJson());
}
if (state.getStateJson() != null && !state.getStateJson().isBlank()) {
payloads.add(state.getStateJson());
}
}
}
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
payloads.add(chunk.getPayloadJson());
}
}
}
if (payloads.isEmpty()) {
return;
}
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deletePayloadsNow(payloads, taskId);
}
});
return;
}
deletePayloadsNow(payloads, taskId);
}
private void deletePayloadsNow(List<String> payloads, Long taskId) {
for (String payload : payloads) {
try {
transientPayloadStorageService.deletePayloadIfPresent(payload);
} catch (Exception ex) {
// 事务已提交,远端删除失败只记日志,不影响任务数据清理结果
log.warn("[brand-storage] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage());
}
}
}
private void saveAggregate(Long taskId,
@@ -431,70 +431,85 @@ public class CollectDataService {
}
}
@Transactional
/**
* 进度心跳。
*
* <p>事务边界:Redis 任务锁在事务外获取(自旋等待最长 TASK_LOCK_WAIT_MILLIS
* 放在 @Transactional 里会白占一个 Hikari 连接),DB 段(统计持久化 + 任务行更新)
* 仍在一个事务内。
*/
public void updateProgress(Long taskId, TaskHeartbeatRequest request) {
if (taskId == null || taskId <= 0 || request == null) {
return;
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
if (transactionTemplate == null) {
// 单测场景(@InjectMocks 未注入事务模板):退化为直接执行 DB 段
updateProgressLocked(taskId, request);
return;
}
CollectDataStats stats = loadStats(task);
boolean changed = false;
Integer current = request.getCurrent();
Integer total = request.getTotal();
if (current != null && total != null && total > 0) {
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
stats.totalRows = totalRows;
stats.processedRows = processedRows;
changed = true;
}
}
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
stats.collectStage = request.getCollectStage();
changed = true;
}
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
stats.currentKeyword = request.getCurrentKeyword();
changed = true;
}
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
changed = true;
}
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
changed = true;
}
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
changed = true;
}
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
changed = true;
}
if (!changed) {
// 内容未变化:仅当脏数据兜底窗口到期时才强制刷新一次,
// 否则零 UPDATE(重复心跳幂等)。
if (!shouldForceProgressFlush()) {
return;
}
} else if (shouldThrottleProgressFlush()) {
// 节流窗口内:合并写(只更新内存态、不落库),窗口到期后统一持久化。
return;
}
persistStats(task, stats);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
lastProgressFlushMillis = System.currentTimeMillis();
transactionTemplate.executeWithoutResult(status -> updateProgressLocked(taskId, request));
}
}
private void updateProgressLocked(Long taskId, TaskHeartbeatRequest request) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
return;
}
CollectDataStats stats = loadStats(task);
boolean changed = false;
Integer current = request.getCurrent();
Integer total = request.getTotal();
if (current != null && total != null && total > 0) {
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
stats.totalRows = totalRows;
stats.processedRows = processedRows;
changed = true;
}
}
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
stats.collectStage = request.getCollectStage();
changed = true;
}
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
stats.currentKeyword = request.getCurrentKeyword();
changed = true;
}
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
changed = true;
}
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
changed = true;
}
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
changed = true;
}
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
changed = true;
}
if (!changed) {
// 内容未变化:仅当脏数据兜底窗口到期时才强制刷新一次,
// 否则零 UPDATE(重复心跳幂等)。
if (!shouldForceProgressFlush()) {
return;
}
} else if (shouldThrottleProgressFlush()) {
// 节流窗口内:合并写(只更新内存态、不落库),窗口到期后统一持久化。
return;
}
persistStats(task, stats);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
lastProgressFlushMillis = System.currentTimeMillis();
}
/**
* 节流判定:progressThrottleMillis>0 且距上次实际落库未超过窗口 → 合并写(跳过 UPDATE)。
* progressThrottleMillis<=0 视为关闭节流,恒返回 false(每次心跳都落库,兼容旧行为)。
@@ -16,6 +16,8 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
@@ -252,6 +254,13 @@ public class DeleteBrandTaskStorageService {
return grouped;
}
/**
* 删除任务的全部范围/分片数据。
*
* <p>事务边界:远端载荷删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次,
* 放在 DB 事务里会让连接被长时间占用(项目在 TaskScopePayloadStorageService 已写明该约束)。
* 提交后再删还有个好处:引用计数查询看到的是删除完成后的 DB 状态,判断更准确。
*/
@Transactional
public void deleteTaskData(Long taskId) {
if (taskId == null || taskId <= 0) {
@@ -261,27 +270,68 @@ public class DeleteBrandTaskStorageService {
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (states != null) {
for (TaskScopeStateEntity state : states) {
transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson());
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
}
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
deletePayloadsAfterCommit(states, chunks, taskId);
}
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
private void deletePayloadsAfterCommit(List<TaskScopeStateEntity> states,
List<TaskChunkEntity> chunks,
Long taskId) {
List<String> payloads = new ArrayList<>();
if (states != null) {
for (TaskScopeStateEntity state : states) {
if (state == null) {
continue;
}
if (state.getParsedPayloadJson() != null && !state.getParsedPayloadJson().isBlank()) {
payloads.add(state.getParsedPayloadJson());
}
if (state.getStateJson() != null && !state.getStateJson().isBlank()) {
payloads.add(state.getStateJson());
}
}
}
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
payloads.add(chunk.getPayloadJson());
}
}
}
if (payloads.isEmpty()) {
return;
}
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deletePayloadsNow(payloads, taskId);
}
});
return;
}
deletePayloadsNow(payloads, taskId);
}
private void deletePayloadsNow(List<String> payloads, Long taskId) {
for (String payload : payloads) {
try {
transientPayloadStorageService.deletePayloadIfPresent(payload);
} catch (Exception ex) {
// 事务已提交,远端删除失败只记日志,不影响任务数据清理结果
log.warn("[delete-brand-storage] 载荷删除失败 taskId={} msg={}", taskId, ex.getMessage());
}
}
}
private void refreshScopeState(Long taskId, String scopeKey, String scopeHash, Integer chunkTotal, LocalDateTime now) {
@@ -11,6 +11,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
@@ -187,6 +189,13 @@ public class DigitalHumanVersionService {
return toVo(entity);
}
/**
* 删除版本。
*
* <p>事务边界:MinIO 对象删除移到事务提交后执行 —— 单次删除超时 120s、重试 3 次,
* 放在 DB 事务里会长时间占用连接(项目在 TaskScopePayloadStorageService 已写明该约束)。
* 顺序仍是「先删库、后删对象」,与改前一致。
*/
@Transactional
public void deleteVersion(String version) {
DigitalHumanVersionEntity entity = versionMapper.selectOne(new LambdaQueryWrapper<DigitalHumanVersionEntity>()
@@ -198,15 +207,33 @@ public class DigitalHumanVersionService {
throw new BusinessException("最新版本不能删除");
}
// 删除 MinIO 文件
try {
ossStorageService.deleteObject(entity.getOssObjectKey());
} catch (Exception e) {
log.warn("删除 MinIO 文件失败:{}", entity.getOssObjectKey(), e);
}
// 删除数据库记录
versionMapper.deleteById(entity.getId());
// 删除 MinIO 文件(事务提交后)
deleteObjectAfterCommit(entity.getOssObjectKey());
}
/** 事务提交后删 MinIO 对象;无事务时立即执行。 */
private void deleteObjectAfterCommit(String objectKey) {
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deleteObjectQuietly(objectKey);
}
});
return;
}
deleteObjectQuietly(objectKey);
}
private void deleteObjectQuietly(String objectKey) {
try {
ossStorageService.deleteObject(objectKey);
} catch (Exception e) {
log.warn("删除 MinIO 文件失败:{}", objectKey, e);
}
}
public String getDownloadUrl(String version) {
@@ -40,6 +40,8 @@ public class ImageVideoAsyncTaskService {
private static final int DISPATCH_BATCH_SIZE = 20;
private static final int POLL_BATCH_SIZE = 50;
private static final int FAILED_TASK_RETENTION_MINUTES = 10;
/** 过期失败任务单轮删除上限:一条无 LIMIT 的 DELETE 会长时间持锁,改成分批(每轮扫描都会再清)。 */
private static final int EXPIRED_DELETE_BATCH_SIZE = 500;
private static final Set<String> TERMINAL_STATUSES = Set.of(
"SUCCESS", "SUCCEEDED", "COMPLETED", "DONE", "FINISHED",
"FAILED", "FAIL", "ERROR", "CANCELED", "CANCELLED"
@@ -142,7 +144,10 @@ public class ImageVideoAsyncTaskService {
return;
}
try (jobLock) {
// 只取主键本表含 5 LONGTEXT 请求/响应正文而这里只用来发起
// executeTask(id)每秒扫一轮还拉全部大字段属于纯浪费2026-09-15 优化
List<ImageVideoAsyncTaskEntity> tasks = taskMapper.selectList(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
.select(ImageVideoAsyncTaskEntity::getId)
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.PENDING.name())
.and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
.or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
@@ -161,7 +166,9 @@ public class ImageVideoAsyncTaskService {
return;
}
try (jobLock) {
// dispatchPendingTasks只取主键避免每 5 秒把 LONGTEXT 正文整列拉回
List<ImageVideoAsyncTaskEntity> tasks = taskMapper.selectList(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
.select(ImageVideoAsyncTaskEntity::getId)
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.WAITING.name())
.and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
.or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
@@ -234,7 +241,8 @@ public class ImageVideoAsyncTaskService {
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES);
int deleted = taskMapper.delete(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.FAILED.name())
.lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff));
.lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff)
.last("LIMIT " + EXPIRED_DELETE_BATCH_SIZE));
if (deleted > 0) {
log.info("[image-video] removed expired failed tasks count={}", deleted);
}
@@ -37,6 +37,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.File;
import java.time.LocalDateTime;
@@ -81,6 +83,10 @@ public class PatrolDeleteTaskService {
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler;
/**
* 事务边界收口用Redis 任务锁必须在事务外获取否则自旋等待期间会白占一个 Hikari 连接
*/
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -470,7 +476,10 @@ public class PatrolDeleteTaskService {
}
}
@Transactional
/**
* 事务边界任务锁在事务外获取自旋等待不再占用 DB 连接
* 缓存清理放到事务提交后锁内执行耗时日志语义保持不变
*/
public void deleteTask(Long taskId, Long userId) {
long startedAt = System.nanoTime();
validateUserId(userId);
@@ -480,14 +489,18 @@ public class PatrolDeleteTaskService {
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
long taskLoadedAt = System.nanoTime();
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
long resultsDeletedAt = System.nanoTime();
cleanupTaskAuxiliaryDataFast(taskId);
long auxiliaryDeletedAt = System.nanoTime();
fileTaskMapper.deleteById(taskId);
long taskDeletedAt = System.nanoTime();
long[] marks = new long[3];
inNewTransaction(() -> {
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
marks[0] = System.nanoTime();
cleanupTaskAuxiliaryDataFast(taskId);
marks[1] = System.nanoTime();
fileTaskMapper.deleteById(taskId);
marks[2] = System.nanoTime();
return null;
});
taskCacheService.evictTaskCacheOnly(taskId);
long finishedAt = System.nanoTime();
log.info("[patrol-delete] delete task timing taskId={} userId={} totalMs={} loadMs={} resultDeleteMs={} auxiliaryDeleteMs={} taskDeleteMs={} cacheMs={}",
@@ -495,14 +508,16 @@ public class PatrolDeleteTaskService {
userId,
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, taskLoadedAt),
elapsedMs(taskLoadedAt, resultsDeletedAt),
elapsedMs(resultsDeletedAt, auxiliaryDeletedAt),
elapsedMs(auxiliaryDeletedAt, taskDeletedAt),
elapsedMs(taskDeletedAt, finishedAt));
elapsedMs(taskLoadedAt, marks[0]),
elapsedMs(marks[0], marks[1]),
elapsedMs(marks[1], marks[2]),
elapsedMs(marks[2], finishedAt));
}
}
@Transactional
/**
* 事务边界任务锁在事务外获取DB 复核 + 删除 + 重算仍在同一事务内
*/
public void deleteHistory(Long resultId, Long userId) {
validateUserId(userId);
FileResultEntity entity = fileResultMapper.selectById(resultId);
@@ -516,13 +531,16 @@ public class PatrolDeleteTaskService {
return;
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("record not found");
}
fileResultMapper.deleteById(resultId);
cleanupResultAuxiliaryDataFast(taskId, resultId);
reconcileTaskAfterResultRemoval(taskId);
inNewTransaction(() -> {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("record not found");
}
fileResultMapper.deleteById(resultId);
cleanupResultAuxiliaryDataFast(taskId, resultId);
reconcileTaskAfterResultRemoval(taskId);
return null;
});
}
}
@@ -1094,6 +1112,19 @@ public class PatrolDeleteTaskService {
return taskDistributedLockService.acquire(MODULE_TYPE, taskId);
}
/**
* 显式短事务锁已在事务外获取这里只包住 DB
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内
*/
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
if (transactionManager == null) {
return action.get();
}
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return template.execute(status -> action.get());
}
private void markResultSuccess(FileResultEntity row) {
row.setSuccess(RESULT_SUCCESS);
row.setErrorMessage(null);
@@ -297,6 +297,22 @@ public class PermissionMenuService {
.contains(columnId);
}
/**
* 可授权用户查询只取超管判定 + 列表展示用到的列
*
* <p>权限页此前一律 `selectList(new LambdaQueryWrapper<>())` 整表拉回 admin_user
* 含密码哈希等敏感列每次打开权限页都把全表含哈希读进内存2026-09-15 优化
* 判定用 role/is_admin/created_by_id展示用 id/username
*/
private LambdaQueryWrapper<AdminUserEntity> grantableUserQuery() {
return new LambdaQueryWrapper<AdminUserEntity>()
.select(AdminUserEntity::getId,
AdminUserEntity::getUsername,
AdminUserEntity::getRole,
AdminUserEntity::getIsAdmin,
AdminUserEntity::getCreatedById);
}
public List<ImageVideoDataPermissionUserVo> listImageVideoDataPermissionUsers(AdminUserEntity operator) {
ensureSuperAdminOperator(operator);
PermissionMenuEntity dataPermission = requireImageVideoDataPermission();
@@ -305,7 +321,7 @@ public class PermissionMenuService {
.map(UserColumnPermissionEntity::getUserId)
.filter(id -> id != null && id > 0)
.collect(Collectors.toSet());
return adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>()
return adminUserMapper.selectList(grantableUserQuery()
.orderByAsc(AdminUserEntity::getUsername)
.orderByAsc(AdminUserEntity::getId))
.stream()
@@ -319,7 +335,7 @@ public class PermissionMenuService {
ensureSuperAdminOperator(operator);
PermissionMenuEntity dataPermission = requireImageVideoDataPermission();
List<Long> requestedIds = normalizeColumnIds(userIds);
List<AdminUserEntity> users = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>());
List<AdminUserEntity> users = adminUserMapper.selectList(grantableUserQuery());
Map<Long, AdminUserEntity> grantableUsers = users.stream()
.filter(user -> user.getId() != null && !isSuperAdmin(user))
.collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left));
@@ -345,7 +361,7 @@ public class PermissionMenuService {
.map(UserColumnPermissionEntity::getUserId)
.filter(id -> id != null && id > 0)
.collect(Collectors.toSet());
return adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>()
return adminUserMapper.selectList(grantableUserQuery()
.orderByAsc(AdminUserEntity::getUsername)
.orderByAsc(AdminUserEntity::getId))
.stream()
@@ -360,7 +376,7 @@ public class PermissionMenuService {
PermissionMenuEntity dataPermission = requireDataPermission(
SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据");
List<Long> requestedIds = normalizeColumnIds(userIds);
List<AdminUserEntity> users = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>());
List<AdminUserEntity> users = adminUserMapper.selectList(grantableUserQuery());
Map<Long, AdminUserEntity> grantableUsers = users.stream()
.filter(user -> user.getId() != null && !isSuperAdmin(user))
.collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left));
@@ -607,7 +623,9 @@ public class PermissionMenuService {
}
Set<Long> targetEffectiveIds = expandDescendantIds(
new LinkedHashSet<>(loadDirectColumnIds(target.getId())), loadMenus(null));
// 只用到 id不要为一次级联清理把整表含密码哈希拉回
List<AdminUserEntity> subordinates = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>()
.select(AdminUserEntity::getId)
.eq(AdminUserEntity::getCreatedById, target.getId()));
for (AdminUserEntity subordinate : subordinates) {
Long subordinateId = subordinate.getId();
@@ -39,6 +39,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.BufferedReader;
import java.io.File;
@@ -93,6 +95,11 @@ public class PriceTrackTaskService {
private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler;
private final com.nanri.aiimage.modules.file.service.LocalFileStorageService localFileStorageService;
/**
* 事务边界收口用Redis 任务锁必须在事务外获取否则自旋等待最长 TASK_LOCK_WAIT_MILLIS
* 期间会白占一个 Hikari 连接生产 maximum-pool-size 30
*/
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -254,7 +261,10 @@ public class PriceTrackTaskService {
return vo;
}
@Transactional
/**
* 事务边界Redis 任务锁在事务外获取自旋等待不再占用 DB 连接
* DB 复核 + 删除 + 重算任务状态仍在同一个事务内
*/
public void deleteHistory(Long resultId, Long userId) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
FileResultEntity entity = fileResultMapper.selectById(resultId);
@@ -267,73 +277,97 @@ public class PriceTrackTaskService {
return;
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("record not found");
}
fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
inNewTransaction(() -> {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("record not found");
}
fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
return null;
});
}
}
@Transactional
/**
* 事务边界任务锁在事务外获取缓存清理Redis + 远端范围载荷放到事务提交后执行
*/
public void deleteTask(Long taskId, Long userId) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在");
}
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(taskId);
inNewTransaction(() -> {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在");
}
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(taskId);
priceTrackLoopRunService.syncLoopRunAfterChildRemoved(taskId);
return null;
});
// 缓存清理Redis + 远端范围载荷在事务提交后锁内执行
priceTrackTaskCacheService.deleteTaskCache(taskId);
priceTrackLoopRunService.syncLoopRunAfterChildRemoved(taskId);
}
log.info("[price-track] deleteTask 完成 taskId={} userId={}", taskId, userId);
}
@Transactional
/**
* 事务边界任务锁在事务外获取缓存清理放到事务提交后执行
*/
public void markDispatchFailed(Long taskId, Long userId, String errorMessage) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
if (errorMessage == null || errorMessage.isBlank()) throw new BusinessException("errorMessage 不能为空");
String normalizedError = errorMessage.trim();
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在");
// 返回需要清理缓存的任务状态仅无结果行分支非终态返回 null
String terminalStatus = inNewTransaction(() ->
markDispatchFailedRecords(taskId, userId, normalizedError));
if (terminalStatus != null) {
cleanupTaskCacheIfTerminal(taskId, terminalStatus);
}
if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) {
return;
}
List<FileResultEntity> results = fileResultMapper.selectList(
new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.orderByAsc(FileResultEntity::getId));
if (results.isEmpty()) {
task.setStatus("FAILED");
task.setErrorMessage(normalizedError);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
cleanupTaskCacheIfTerminal(taskId, task.getStatus());
priceTrackLoopRunService.syncLoopRunAfterChildTerminal(taskId);
return;
}
for (FileResultEntity result : results) {
boolean succeeded = result.getSuccess() != null && result.getSuccess() == 1;
boolean failed = result.getErrorMessage() != null && !result.getErrorMessage().isBlank();
if (!succeeded && !failed) {
markResultFailed(result, normalizedError);
}
}
updateTaskStatusFromLatestRows(task, results);
log.warn("[price-track] Python dispatch failed taskId={} userId={} error={}", taskId, userId, normalizedError);
}
}
@Transactional
private String markDispatchFailedRecords(Long taskId, Long userId, String normalizedError) {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在");
}
if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) {
return null;
}
List<FileResultEntity> results = fileResultMapper.selectList(
new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.orderByAsc(FileResultEntity::getId));
if (results.isEmpty()) {
task.setStatus("FAILED");
task.setErrorMessage(normalizedError);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
priceTrackLoopRunService.syncLoopRunAfterChildTerminal(taskId);
return task.getStatus();
}
for (FileResultEntity result : results) {
boolean succeeded = result.getSuccess() != null && result.getSuccess() == 1;
boolean failed = result.getErrorMessage() != null && !result.getErrorMessage().isBlank();
if (!succeeded && !failed) {
markResultFailed(result, normalizedError);
}
}
updateTaskStatusFromLatestRows(task, results);
log.warn("[price-track] Python dispatch failed taskId={} userId={} error={}", taskId, userId, normalizedError);
return null;
}
/**
* 事务边界逐个候选任务在事务外取锁命中的那条记录的删除 + 重算在独立短事务内完成
* 原实现整个循环共用一个事务锁自旋等待期间会一直占着连接
*/
public PriceTrackPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 不能为空");
@@ -356,20 +390,33 @@ public class PriceTrackTaskService {
if (!"RUNNING".equals(task.getStatus())) continue;
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLock(fr.getTaskId())) {
if (ignored == null) continue;
FileTaskEntity lockedTask = loadTaskForExecution(fr.getTaskId());
if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) continue;
if (!"RUNNING".equals(lockedTask.getStatus())) continue;
FileResultEntity lockedResult = fileResultMapper.selectById(fr.getId());
if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) continue;
fileResultMapper.deleteById(fr.getId());
reconcileTaskAfterResultRemoval(fr.getTaskId());
vo.setRemoved(true);
return vo;
Boolean removed = inNewTransaction(() -> removeRunningShopResult(fr.getTaskId(), userId, fr.getId()));
if (Boolean.TRUE.equals(removed)) {
vo.setRemoved(true);
return vo;
}
}
}
return vo;
}
private boolean removeRunningShopResult(Long taskId, Long userId, Long resultId) {
FileTaskEntity lockedTask = loadTaskForExecution(taskId);
if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) {
return false;
}
if (!"RUNNING".equals(lockedTask.getStatus())) {
return false;
}
FileResultEntity lockedResult = fileResultMapper.selectById(resultId);
if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) {
return false;
}
fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
return true;
}
public PriceTrackTaskBatchVo getTaskDetailsBatch(List<Long> taskIds) {
PriceTrackTaskBatchVo batch = new PriceTrackTaskBatchVo();
if (taskIds == null || taskIds.isEmpty()) return batch;
@@ -506,6 +553,10 @@ public class PriceTrackTaskService {
}
List<PriceTrackResultItemVo> snapshot = new ArrayList<>();
// 逐店铺 fileResultMapper.insert 看似该改批量500 店铺 = 500 次往返**不能改**
// 下面的 toSnapshotVo 依赖 MyBatis-Plus insert 回填的自增主键fr.getId() vo.resultId
// 而本仓的手写 INSERT ... VALUES (...),(...) 批量写法不回填主键改了会让快照 resultId null
// 要批量化必须同时解决批量插入 + 按序回填 idMySQL useGeneratedKeys + 顺序保证
for (PriceTrackMatchShopsVo.PriceTrackShopQueueItem item : uniqueItems) {
String norm = ziniaoShopSwitchService.normalizeShopName(item.getShopName());
if (norm.isBlank()) throw new BusinessException("店铺名称无效");
@@ -2182,6 +2233,20 @@ public class PriceTrackTaskService {
return lockHandle;
}
/**
* 显式开启一个短事务锁已在事务外获取这里只包住 DB
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内
*/
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
if (transactionManager == null) {
// 单测直接 new 出本类history QueryAsinSnapshotJsonThrottleTest 同类用法时无事务管理器
return action.get();
}
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return template.execute(status -> action.get());
}
private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId) {
if (taskId == null || taskId <= 0) {
return null;
@@ -39,6 +39,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.File;
import java.time.LocalDateTime;
@@ -82,6 +84,10 @@ public class ProductRiskTaskService {
private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler;
/**
* 事务边界收口用Redis 任务锁必须在事务外获取否则自旋等待期间会白占一个 Hikari 连接
*/
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -248,7 +254,9 @@ public class ProductRiskTaskService {
return vo;
}
@Transactional
/**
* 事务边界任务锁在事务外获取自旋等待不再占用 DB 连接DB 段仍在同一事务内
*/
public void deleteHistory(Long resultId, Long userId) {
if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法");
@@ -263,43 +271,53 @@ public class ProductRiskTaskService {
return;
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("record not found");
}
taskFileJobService.deleteResultJobs(latestEntity.getTaskId(), MODULE_TYPE, latestEntity.getId());
fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
inNewTransaction(() -> {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("record not found");
}
taskFileJobService.deleteResultJobs(latestEntity.getTaskId(), MODULE_TYPE, latestEntity.getId());
fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
return null;
});
}
}
/**
* 删除整条商品风险任务及其下所有店铺结果运行中和已结束任务都允许删除但必须归属当前用户
*
* <p>事务边界任务锁在事务外获取缓存清理放到事务提交后锁内执行
*/
@Transactional
public void deleteTask(Long taskId, Long userId) {
if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法");
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在");
}
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
fileTaskMapper.deleteById(taskId);
inNewTransaction(() -> {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在");
}
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
fileTaskMapper.deleteById(taskId);
return null;
});
cleanupTaskCacheIfTerminal(taskId, "DELETE_EMPTY");
}
log.info("[product-risk] deleteTask 完成 taskId={} userId={}", taskId, userId);
}
/**
* 从用户当前运行中的商品风险任务里按规范化店铺名删除一条结果用于前端移除匹配列表后同步后端
* 如果没有对应记录则返回 removed=false
*
* <p>事务边界逐个候选任务在事务外取锁命中的那条记录的删除 + 重算在独立短事务内完成
* 原实现整个循环共用一个事务锁自旋等待期间会一直占着连接
*/
@Transactional
public ProductRiskPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法");
@@ -335,26 +353,33 @@ public class ProductRiskTaskService {
if (ignored == null) {
continue;
}
FileTaskEntity lockedTask = loadTaskForExecution(tid);
if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) {
continue;
Boolean removed = inNewTransaction(() -> removeRunningShopResult(tid, userId, fr.getId()));
if (Boolean.TRUE.equals(removed)) {
vo.setRemoved(true);
return vo;
}
if (!"RUNNING".equals(lockedTask.getStatus())) {
continue;
}
FileResultEntity lockedResult = fileResultMapper.selectById(fr.getId());
if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) {
continue;
}
fileResultMapper.deleteById(fr.getId());
reconcileTaskAfterResultRemoval(tid);
vo.setRemoved(true);
return vo;
}
}
return vo;
}
private boolean removeRunningShopResult(Long taskId, Long userId, Long resultId) {
FileTaskEntity lockedTask = loadTaskForExecution(taskId);
if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) {
return false;
}
if (!"RUNNING".equals(lockedTask.getStatus())) {
return false;
}
FileResultEntity lockedResult = fileResultMapper.selectById(resultId);
if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) {
return false;
}
fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
return true;
}
/**
* 删除一条 file_result 后重算父任务状态如果没有剩余结果则删除任务
*/
@@ -1002,6 +1027,19 @@ public class ProductRiskTaskService {
return taskDistributedLockService.acquire(MODULE_TYPE, taskId);
}
/**
* 显式短事务锁已在事务外获取这里只包住 DB
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内
*/
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
if (transactionManager == null) {
return action.get();
}
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return template.execute(status -> action.get());
}
private ProductRiskTaskDetailVo buildTaskDetail(FileTaskEntity task) {
ProductRiskTaskDetailVo detail = new ProductRiskTaskDetailVo();
detail.setTask(toTaskItemVo(task));
@@ -57,6 +57,8 @@ import org.springframework.dao.DuplicateKeyException;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.File;
@@ -554,24 +556,28 @@ public class PublishTaskService {
}
}
/**
* 删除任务
*
* <p>事务边界远端删除结果文件对象 + 分片载荷全部移到事务提交后执行
* 单次删除超时 120s重试 3 放在 DB 事务里会让连接被长时间占用
* 项目在 TaskScopePayloadStorageService 已写明不能把网络调用放进 DB 事务
*/
@Transactional
public void deleteTask(Long taskId, Long userId) {
FileTaskEntity task = requireTaskForDeletion(taskId, userId);
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
List<String> resultObjectKeys = new ArrayList<>();
for (FileResultEntity result : results) {
if (result.getResultFileUrl() != null && !result.getResultFileUrl().isBlank()) {
try {
ossStorageService.deleteObject(result.getResultFileUrl());
} catch (Exception ex) {
log.warn("[publish] OSS cleanup failed taskId={} resultId={} msg={}",
taskId, result.getId(), ex.getMessage());
}
resultObjectKeys.add(result.getResultFileUrl());
}
}
List<String> chunkPayloads = collectTransientResultChunkPayloads(taskId);
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
deleteTransientResultChunks(taskId);
deleteTransientResultChunkRows(taskId);
publishItemMapper.delete(new LambdaQueryWrapper<PublishItemEntity>()
.eq(PublishItemEntity::getTaskId, taskId));
publishFileMapper.delete(new LambdaQueryWrapper<PublishFileEntity>()
@@ -580,6 +586,9 @@ public class PublishTaskService {
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(task.getId());
// 事务提交后再做远端删除
deleteResultObjectsAfterCommit(resultObjectKeys, taskId);
deletePayloadsAfterCommit(chunkPayloads, taskId);
}
public void deleteHistory(Long resultId, Long userId) {
@@ -599,18 +608,35 @@ public class PublishTaskService {
}
private void deleteTransientResultChunks(Long taskId) {
List<String> payloads = collectTransientResultChunkPayloads(taskId);
deleteTransientResultChunkRows(taskId);
deletePayloadsAfterCommit(payloads, taskId);
}
/** 只读收集分片载荷指针,供事务提交后做远端删除。 */
private List<String> collectTransientResultChunkPayloads(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
return List.of();
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
List<String> payloads = new ArrayList<>();
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
if (chunk != null && chunk.getPayloadJson() != null && !chunk.getPayloadJson().isBlank()) {
payloads.add(chunk.getPayloadJson());
}
}
}
return payloads;
}
private void deleteTransientResultChunkRows(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
@@ -619,6 +645,63 @@ public class PublishTaskService {
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
}
/** 事务提交后逐条删分片载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
private void deletePayloadsAfterCommit(List<String> payloads, Long taskId) {
if (payloads == null || payloads.isEmpty()) {
return;
}
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deletePayloadsNow(payloads, taskId);
}
});
return;
}
deletePayloadsNow(payloads, taskId);
}
private void deletePayloadsNow(List<String> payloads, Long taskId) {
for (String payload : payloads) {
try {
transientPayloadStorageService.deletePayloadIfPresent(payload);
} catch (Exception ex) {
log.warn("[publish] 分片载荷删除失败 taskId={} pointer={} msg={}",
taskId, transientPayloadStorageService.extractPointer(payload), safeMessage(ex));
}
}
}
/** 事务提交后删结果文件对象;无事务时立即执行。 */
private void deleteResultObjectsAfterCommit(List<String> objectKeys, Long taskId) {
if (objectKeys == null || objectKeys.isEmpty()) {
return;
}
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deleteResultObjectsNow(objectKeys, taskId);
}
});
return;
}
deleteResultObjectsNow(objectKeys, taskId);
}
private void deleteResultObjectsNow(List<String> objectKeys, Long taskId) {
for (String objectKey : objectKeys) {
try {
ossStorageService.deleteObject(objectKey);
log.info("[publish] 结果文件已删除 taskId={} objectKey={}", taskId, objectKey);
} catch (Exception ex) {
log.warn("[publish] OSS cleanup failed taskId={} objectKey={} msg={}",
taskId, objectKey, ex.getMessage());
}
}
}
private void deleteUncommittedPayloads(List<String> storedPayloads) {
if (storedPayloads == null || storedPayloads.isEmpty()) {
return;
@@ -36,6 +36,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.File;
import java.time.LocalDateTime;
@@ -92,6 +94,10 @@ public class QueryAsinTaskService {
private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler;
/**
* 事务边界收口用Redis 任务锁必须在事务外获取否则自旋等待期间会白占一个 Hikari 连接
*/
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -475,7 +481,10 @@ public class QueryAsinTaskService {
}
}
@Transactional
/**
* 事务边界任务锁在事务外获取自旋等待不再占用 DB 连接
* 缓存清理Redis + 远端范围载荷放到事务提交后锁内执行
*/
public void deleteTask(Long taskId, Long userId) {
validateUserId(userId);
FileTaskEntity task = loadTaskForExecution(taskId);
@@ -483,15 +492,21 @@ public class QueryAsinTaskService {
throw new BusinessException("任务不存在");
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(taskId);
taskCacheService.deleteTaskCache(taskId);
inNewTransaction(() -> {
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(taskId);
return null;
});
taskCacheService.deleteTaskCache(taskId);
}
log.info("[query-asin] deleteTask 完成 taskId={} userId={}", taskId, userId);
}
@Transactional
/**
* 事务边界任务锁在事务外获取DB 复核 + 删除 + 重算仍在同一事务内
*/
public void deleteHistory(Long resultId, Long userId) {
validateUserId(userId);
FileResultEntity entity = fileResultMapper.selectById(resultId);
@@ -504,12 +519,15 @@ public class QueryAsinTaskService {
return;
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("record not found");
}
fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
inNewTransaction(() -> {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("record not found");
}
fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
return null;
});
}
}
@@ -1070,6 +1088,19 @@ public class QueryAsinTaskService {
return taskDistributedLockService.acquire(MODULE_TYPE, taskId);
}
/**
* 显式短事务锁已在事务外获取这里只包住 DB
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内
*/
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
if (transactionManager == null) {
return action.get();
}
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return template.execute(status -> action.get());
}
private void markResultSuccess(FileResultEntity row) {
row.setSuccess(RESULT_SUCCESS);
row.setErrorMessage(null);
@@ -87,6 +87,13 @@ public class ShopDataCrawlExcelAssemblyService {
imageEmbedder.prefetch(prefetchBudget().boundedUrls(imageUrls(rowsByCountry)), imageCache);
}
/**
* 模板路径组装DOM历史包袱说明本方法把整个工作簿读进 DOMXSSFWorkbook
* 当日累计文件越大峰值堆越高-Xmx6g 下大客户的大文件有 OOM 风险
* 同文件的 {@link #writeWorkbookStreaming} SXSSF 流式版但它**不使用模板
* 不写图片**图片只兜底成 URL 文本直接替换会改变交付文件的样式与商品图
* 属产品取舍未直接切换若要落地必须实现保留模板样式 + 保留图片的流式写入
*/
public int writeWorkbook(File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream();
XSSFWorkbook workbook = new XSSFWorkbook(input);
@@ -807,7 +807,10 @@ public class ShopDataCrawlTaskService {
}
}
@Transactional
/**
* 事务边界Redis 任务锁在事务外获取自旋等待期间不再白占一个 Hikari 连接
* DB 含每日文件行锁与三条删除链路仍在同一个短事务内远端对象清理在提交后进行
*/
public void deleteTask(Long taskId, Long userId) {
validateUserId(userId);
FileTaskEntity task = loadTaskForExecution(taskId);
@@ -816,28 +819,33 @@ public class ShopDataCrawlTaskService {
}
ensureTaskOwnedByCurrentInstance(task, "delete shop data crawl task");
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
List<FileResultEntity> taskRows = listTaskRows(taskId);
try (DailyLockSet dailyLocks = acquireDailyLocks(taskRows)) {
ensureDailySyncCompletedBeforeDelete(taskRows);
Set<Long> removedResultIds = collectResultIds(taskRows);
// A task deletion is only a frontend task-record cleanup. The daily
// workbook is an independent backend aggregate and must not roll
// back when its source task is removed.
DailyDeletionResult dailyResult = preserveDailyForTaskDeletion(removedResultIds);
registerUploadedObjectRollback(dailyResult.uploadedObjectKeys());
List<String> resultFileUrls = collectResultFileUrls(taskRows, dailyResult.obsoleteObjectKeys());
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE);
taskProgressSnapshotService.delete(taskId, MODULE_TYPE);
fileTaskMapper.deleteById(taskId);
taskCacheService.deleteTaskCache(taskId);
deleteTransientResultChunks(taskId);
resultFileUrls.forEach(this::deleteResultObjectIfUnreferenced);
}
executeShortTransaction(() -> {
List<FileResultEntity> taskRows = listTaskRows(taskId);
try (DailyLockSet dailyLocks = acquireDailyLocks(taskRows)) {
ensureDailySyncCompletedBeforeDelete(taskRows);
Set<Long> removedResultIds = collectResultIds(taskRows);
// A task deletion is only a frontend task-record cleanup. The daily
// workbook is an independent backend aggregate and must not roll
// back when its source task is removed.
DailyDeletionResult dailyResult = preserveDailyForTaskDeletion(removedResultIds);
registerUploadedObjectRollback(dailyResult.uploadedObjectKeys());
List<String> resultFileUrls = collectResultFileUrls(taskRows, dailyResult.obsoleteObjectKeys());
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE);
taskProgressSnapshotService.delete(taskId, MODULE_TYPE);
fileTaskMapper.deleteById(taskId);
deleteTransientResultChunks(taskId);
resultFileUrls.forEach(this::deleteResultObjectIfUnreferenced);
}
return null;
});
// 缓存清理Redis + 本地缓存在事务提交后锁内执行
taskCacheService.deleteTaskCache(taskId);
}
log.info("[shop-data-crawl] deleteTask 完成 taskId={} userId={}", taskId, userId);
}
/** 删除任务的纯计算:收集有效结果行 id 集合(过滤 null/非正数),无副作用。 */
@@ -884,7 +892,10 @@ public class ShopDataCrawlTaskService {
}
}
@Transactional
/**
* 事务边界任务锁在事务外获取DB 复核 + 删行 + 每日文件重建在同一个短事务内
* 远端对象清理已由 deleteResultObjectIfUnreferenced 在提交后执行
*/
public void deleteHistory(Long resultId, Long userId) {
validateUserId(userId);
FileResultEntity entity = requireResultEntity(resultId);
@@ -899,21 +910,26 @@ public class ShopDataCrawlTaskService {
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务仍在处理中,不能删除");
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("记录不存在");
}
try (DailyLockSet dailyLocks = acquireDailyLocks(List.of(latestEntity))) {
deleteResultHistoryRow(latestEntity);
}
executeShortTransaction(() -> {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("记录不存在");
}
try (DailyLockSet dailyLocks = acquireDailyLocks(List.of(latestEntity))) {
deleteResultHistoryRow(latestEntity);
}
return null;
});
}
}
/**
* Deletes an administrative result without accepting a caller-controlled owner ID.
* The controller that calls this method is restricted to the Flask-to-Java internal channel.
*
* <p>不加 @Transactional内部调用 deleteHistory 时事务代理不生效同类自调用
* 锁会落在本方法开启的事务里 事务边界统一由 deleteHistory 自己收口
*/
@Transactional
public void deleteAdminHistory(Long resultId) {
FileResultEntity entity = requireResultEntity(resultId);
Long ownerId = entity.getUserId();
@@ -2620,6 +2636,13 @@ public class ShopDataCrawlTaskService {
taskCacheService.deleteTaskCache(taskId);
}
/**
* 删除结果分片行
*
* <p>事务边界远端载荷删除移到事务提交后执行 单次删除超时 120s重试 3
* 放在 DB 事务里会让连接被长时间占用项目在 TaskScopePayloadStorageService 已写明该约束
* 提交后再删还有个好处引用计数查询看到的是删除完成后的 DB 状态判断更准确
*/
private void deleteTransientResultChunks(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
@@ -2628,9 +2651,12 @@ public class ShopDataCrawlTaskService {
.select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
List<String> payloads = new ArrayList<>();
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
if (chunk != null && !blank(chunk.getPayloadJson())) {
payloads.add(chunk.getPayloadJson());
}
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
@@ -2640,6 +2666,34 @@ public class ShopDataCrawlTaskService {
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.likeRight(TaskScopeStateEntity::getScopeKey, RESULT_CHUNK_SCOPE_PREFIX));
deleteChunkPayloadsAfterCommit(payloads, taskId);
}
/** 事务提交后逐条删远端载荷;无事务时立即执行(与 TaskResultItemService 同款兜底)。 */
private void deleteChunkPayloadsAfterCommit(List<String> payloads, Long taskId) {
if (payloads == null || payloads.isEmpty()) {
return;
}
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
deleteChunkPayloadsNow(payloads, taskId);
}
});
return;
}
deleteChunkPayloadsNow(payloads, taskId);
}
private void deleteChunkPayloadsNow(List<String> payloads, Long taskId) {
for (String payload : payloads) {
try {
transientPayloadStorageService.deletePayloadIfPresent(payload);
} catch (Exception ex) {
log.warn("[shop-data-crawl] 分片载荷删除失败 taskId={} msg={}", taskId, safeMessage(ex));
}
}
}
private void cleanupResultChunksQuietly(Long taskId, String reason) {
@@ -46,6 +46,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.File;
import java.time.LocalDateTime;
@@ -89,6 +91,10 @@ public class ShopMatchTaskService {
private final SkipPriceAsinService skipPriceAsinService;
private final QueryAsinMapper queryAsinMapper;
private final TaskProgressLightAssembler taskProgressLightAssembler;
/**
* 事务边界收口用Redis 任务锁必须在事务外获取否则自旋等待期间会白占一个 Hikari 连接
*/
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -280,7 +286,9 @@ public class ShopMatchTaskService {
return vo;
}
@Transactional
/**
* 事务边界任务锁在事务外获取自旋等待不再占用 DB 连接DB 段仍在同一事务内
*/
public void deleteHistory(Long resultId, Long userId) {
if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法");
@@ -295,16 +303,21 @@ public class ShopMatchTaskService {
return;
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("record not found");
}
fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
inNewTransaction(() -> {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("record not found");
}
fileResultMapper.deleteById(resultId);
reconcileTaskAfterResultRemoval(taskId);
return null;
});
}
}
@Transactional
/**
* 事务边界任务锁在事务外获取缓存清理Redis + 远端范围载荷放到事务提交后
*/
public void deleteTask(Long taskId, Long userId) {
if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法");
@@ -314,12 +327,17 @@ public class ShopMatchTaskService {
throw new BusinessException("任务不存在");
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(taskId);
inNewTransaction(() -> {
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(taskId);
return null;
});
// 缓存清理Redis + 远端范围载荷在事务提交后锁内执行
shopMatchTaskCacheService.deleteTaskCache(taskId);
}
log.info("[shop-match] deleteTask 完成 taskId={} userId={}", taskId, userId);
}
private void reconcileTaskAfterResultRemoval(Long taskId) {
@@ -500,7 +518,9 @@ public class ShopMatchTaskService {
return vo;
}
@Transactional
/**
* 事务边界任务锁在事务外获取心跳Redis放到事务提交后写
*/
public void activateTask(Long taskId, Long userId, Integer stageIndex) {
if (taskId == null || taskId <= 0) {
throw new BusinessException("taskId 不合法");
@@ -512,45 +532,53 @@ public class ShopMatchTaskService {
throw new BusinessException("stage_index 不合法");
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在");
}
if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) {
throw new BusinessException("任务状态不正确");
}
if (!"SCHEDULED".equals(task.getStatus())) {
throw new BusinessException("任务状态不正确");
}
LocalDateTime now = now();
if (task.getScheduledAt() != null && now.isBefore(task.getScheduledAt().minusSeconds(90))) {
log.warn("[shop-match] activate rejected taskId={} stageIndex={} now={} scheduledAt={} earliestActivateAt={}",
taskId, stageIndex, now, task.getScheduledAt(), task.getScheduledAt().minusSeconds(90));
throw new BusinessException("未到定时执行时间");
}
ShopMatchCreateTaskRequest state = parseTaskRequest(task);
int currentStageIndex = state.getCurrentStageIndex() == null ? 0 : state.getCurrentStageIndex();
if (currentStageIndex != stageIndex) {
throw new BusinessException("当前轮次已变化,请刷新后重试");
}
if (state.getScheduleTimes() == null || stageIndex >= state.getScheduleTimes().size()) {
throw new BusinessException("轮次配置缺失");
}
FileTaskEntity runningTask = findOtherRunningTask(userId, taskId);
if (runningTask != null) {
log.warn("[shop-match] activate serialized taskId={} stageIndex={} blockedByTaskId={} blockedUpdatedAt={}",
taskId, stageIndex, runningTask.getId(), runningTask.getUpdatedAt());
throw new BusinessException("当前已有定时匹配任务执行中,请等待上一任务执行结束后再重试");
}
state.setActiveStageIndex(stageIndex);
task.setStatus("RUNNING");
task.setUpdatedAt(now);
persistTaskRequest(task, state);
updateTaskAndRefreshCache(task);
inNewTransaction(() -> {
activateTaskLocked(taskId, userId, stageIndex);
return null;
});
// 心跳是 Redis 放到事务提交锁内执行与改前一样仍在锁内
shopMatchTaskCacheService.touchTaskHeartbeat(taskId);
}
}
private void activateTaskLocked(Long taskId, Long userId, Integer stageIndex) {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在");
}
if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) {
throw new BusinessException("任务状态不正确");
}
if (!"SCHEDULED".equals(task.getStatus())) {
throw new BusinessException("任务状态不正确");
}
LocalDateTime now = now();
if (task.getScheduledAt() != null && now.isBefore(task.getScheduledAt().minusSeconds(90))) {
log.warn("[shop-match] activate rejected taskId={} stageIndex={} now={} scheduledAt={} earliestActivateAt={}",
taskId, stageIndex, now, task.getScheduledAt(), task.getScheduledAt().minusSeconds(90));
throw new BusinessException("未到定时执行时间");
}
ShopMatchCreateTaskRequest state = parseTaskRequest(task);
int currentStageIndex = state.getCurrentStageIndex() == null ? 0 : state.getCurrentStageIndex();
if (currentStageIndex != stageIndex) {
throw new BusinessException("当前轮次已变化,请刷新后重试");
}
if (state.getScheduleTimes() == null || stageIndex >= state.getScheduleTimes().size()) {
throw new BusinessException("轮次配置缺失");
}
FileTaskEntity runningTask = findOtherRunningTask(userId, taskId);
if (runningTask != null) {
log.warn("[shop-match] activate serialized taskId={} stageIndex={} blockedByTaskId={} blockedUpdatedAt={}",
taskId, stageIndex, runningTask.getId(), runningTask.getUpdatedAt());
throw new BusinessException("当前已有定时匹配任务执行中,请等待上一任务执行结束后再重试");
}
state.setActiveStageIndex(stageIndex);
task.setStatus("RUNNING");
task.setUpdatedAt(now);
persistTaskRequest(task, state);
updateTaskAndRefreshCache(task);
}
@Transactional
public void completeStage(Long taskId, Long userId, ShopMatchStageCompleteRequest request) {
if (taskId == null || taskId <= 0) {
@@ -960,6 +988,19 @@ public class ShopMatchTaskService {
return taskDistributedLockService.acquire(MODULE_TYPE, taskId);
}
/**
* 显式短事务锁已在事务外获取这里只包住 DB
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内
*/
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
if (transactionManager == null) {
return action.get();
}
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return template.execute(status -> action.get());
}
private boolean shouldRemainRunningUntilNextStage(FileTaskEntity task) {
ShopMatchCreateTaskRequest request = parseTaskRequestSilently(task);
if (request == null || request.getScheduleTimes() == null || request.getScheduleTimes().isEmpty()) {
@@ -526,7 +526,12 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
return vo;
}
@Transactional
/**
* 启动任务PENDINGRUNNING
*
* <p>事务边界Redis 任务锁必须在事务外获取自旋等待期间会白占一个 Hikari 连接
* 本方法只有一条条件 UPDATE自身即原子故不再包事务
*/
public void activateTask(Long taskId, Long userId) {
try (TaskDistributedLockService.LockHandle ignored = ownershipSupport().requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
@@ -1323,7 +1328,12 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
}
}
@Transactional
/**
* 结果文件 job 失败的收尾
*
* <p>事务边界任务锁在事务外获取DB 段本来就走 {@code inNewTransaction}REQUIRES_NEW
* 方法级 @Transactional 只会额外占一个空闲连接故去掉
*/
public void handleResultFileJobFailure(TaskFileJobEntity job, String message) {
if (job == null || job.getTaskId() == null) {
return;
@@ -9,7 +9,8 @@ import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
* <p>
* 采用依赖倒置由宿主SimilarAsinTaskService实现本接口流水线只依赖接口
* 避免 support 包与 Service 之间形成循环依赖这些方法保留在宿主的两个原因
* 一是属于任务/结果文件任务的编排与事务边界{@code handleResultFileJobFailure} @Transactional
* 一是属于任务/结果文件任务的编排与事务边界{@code handleResultFileJobFailure} 先取任务锁
* 再用 {@code inNewTransaction} 显式包 DB 锁必须在事务外获取故不再挂方法级 @Transactional
* 二是被宿主自身的门面路径复用
*/
public interface SimilarAsinPipelineHost {
@@ -139,10 +139,20 @@ public class TaskResultFileJobWorker {
taskQueueExecutor.execute(() -> processClaimedWithHeartbeat(job, claim));
return;
} catch (RuntimeException ex) {
log.warn("[task-file-job] llm module offload failed, fallback inline jobId={} taskId={} moduleType={} msg={}",
// 闸门拒绝等待队列已满说明系统已满载此时**不能**回退成内联执行
// 那等于把背压转嫁给 MQ 消费线程让消费线程按分钟级阻塞在 Excel 组装上
// 队列越满越糟改为重新入队由归属实例 15s 一轮的扫描补发
log.warn("[task-file-job] 结果文件任务队列已满,改为重新入队 jobId={} taskId={} moduleType={} msg={} stage=REQUEUE",
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex);
taskFileJobService.requeue(job.getId(), "结果文件任务队列已满,稍后重投");
return;
}
}
// 注意supportsAsyncOffload()=false 的模块11 仍在本方法调用方的线程上
// 内联执行MQ 侧实际并发度 = consumeThreadNumber 默认值 20该并发度直接决定
// 同时在跑的 Excel 组装数量而每一路都持有 DB 连接Hikari 池仅 30还要供
// Tomcat 200 线程与调度线程使用若要收敛应显式设置
// RocketMQMessageListener.consumeThreadNumber 并先测吞吐不要在这里临时改
processClaimedWithHeartbeat(job, claim);
}
@@ -89,6 +89,14 @@ public class TaskResultItemService {
return moduleType + "|" + scopeHash + "|" + itemKey;
}
/**
* 载入本任务的**全部**结果快照 limit
*
* <p>刻意不加 limit调用方要用它组装完整的结果文件截断即漏行正确性错误
* 不是性能取舍调用方多为逐任务循环调用因此峰值内存 = 单个任务的结果集
* 这是本操作固有的代价若要再降需要改成边读边写文件的流式组装契约
* 涉及 4 个模块的组装实现属独立工程
*/
public <T> List<T> listResultSnapshots(Long taskId, String moduleType, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return List.of();
@@ -517,11 +517,8 @@ public class TaskScopePayloadStorageService {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
// HexFormat 替代逐字节 String.format("%02x")后者每次走 32 Formatter热路径按行
return java.util.HexFormat.of().formatHex(bytes);
} catch (Exception ex) {
throw new IllegalStateException("failed to hash scope key", ex);
}
@@ -35,6 +35,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import java.io.File;
import java.math.BigDecimal;
@@ -79,6 +81,10 @@ public class WithdrawTaskService {
private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressLightAssembler taskProgressLightAssembler;
/**
* 事务边界收口用Redis 任务锁必须在事务外获取否则自旋等待期间会白占一个 Hikari 连接
*/
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
public ProductRiskDashboardVo dashboard(Long userId) {
validateUserId(userId);
@@ -347,7 +353,10 @@ public class WithdrawTaskService {
}
}
@Transactional
/**
* 事务边界任务锁在事务外获取自旋等待不再占用 DB 连接
* 缓存清理Redis + 远端范围载荷放到事务提交后锁内执行
*/
public void deleteTask(Long taskId, Long userId) {
validateUserId(userId);
FileTaskEntity task = loadTaskForExecution(taskId);
@@ -355,18 +364,24 @@ public class WithdrawTaskService {
throw new BusinessException("任务不存在");
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(taskId);
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE);
taskProgressSnapshotService.delete(taskId, MODULE_TYPE);
inNewTransaction(() -> {
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
fileTaskMapper.deleteById(taskId);
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
taskResultItemService.deleteTaskItems(taskId, MODULE_TYPE);
taskProgressSnapshotService.delete(taskId, MODULE_TYPE);
return null;
});
taskCacheService.deleteTaskCache(taskId);
}
log.info("[withdraw] deleteTask 完成 taskId={} userId={}", taskId, userId);
}
@Transactional
/**
* 事务边界任务锁在事务外获取DB 复核 + 删除 + 重算仍在同一事务内
*/
public void deleteHistory(Long resultId, Long userId) {
validateUserId(userId);
FileResultEntity entity = fileResultMapper.selectById(resultId);
@@ -379,10 +394,13 @@ public class WithdrawTaskService {
return;
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
fileResultMapper.deleteById(resultId);
taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId);
taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId);
reconcileTaskAfterResultRemoval(taskId);
inNewTransaction(() -> {
fileResultMapper.deleteById(resultId);
taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId);
taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId);
reconcileTaskAfterResultRemoval(taskId);
return null;
});
}
}
@@ -1050,6 +1068,19 @@ public class WithdrawTaskService {
return taskDistributedLockService.acquire(MODULE_TYPE, taskId);
}
/**
* 显式短事务锁已在事务外获取这里只包住 DB
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内
*/
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
if (transactionManager == null) {
return action.get();
}
TransactionTemplate template = new TransactionTemplate(transactionManager);
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return template.execute(status -> action.get());
}
private List<Long> normalizeTaskIds(List<Long> taskIds) {
return taskIds == null ? List.of() : taskIds.stream()
.filter(taskId -> taskId != null && taskId > 0)
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.ziniao.client;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.BoundedLruCache;
import com.nanri.aiimage.config.ZiniaoProperties;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopCacheDto;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoStaffItemVo;
@@ -59,8 +60,13 @@ public class ZiniaoClientImpl implements ZiniaoClient {
/** Task 77:单例 RestClient(共享连接池),避免每次调用新建短命客户端。 */
private volatile RestClient sharedRestClient;
/** 按代理地址缓存 RestClient:同一代理复用同一连接池,避免每请求新建。 */
private final Map<String, RestClient> proxyRestClients = new java.util.concurrent.ConcurrentHashMap<>();
/**
* 按代理地址缓存 RestClient同一代理复用同一连接池避免每请求新建
* 有界 LRU代理端点会随店铺轮换不断新增每个都持有连接池与 selector 线程
* 无上限时按店铺数持续累积
*/
private final BoundedLruCache<String, RestClient> proxyRestClients =
new BoundedLruCache<>(BoundedLruCache.DEFAULT_MAX_SIZE);
@Override
public Long getCompanyIdByApiKey(String apiKey) {
@@ -124,6 +130,15 @@ public class ZiniaoClientImpl implements ZiniaoClient {
return parseUserStores(raw);
}
/**
* 获取员工登录 token
*
* <p>**刻意不加缓存**2026-09-15 复核调用方 {@code ZiniaoAuthService} 命中店铺匹配缓存时
* 仍会用**新取的** token 重建开店铺 URL见其 getCachedShopMatch 分支只缓存匹配结果
* openStoreUrl null 后重算说明该 token 一次开店铺一次的短时凭据
* 缓存后会把已被消费的 token 发出去用户点开店铺会失败上游往返只为单次用户动作
* 服务不是吞吐瓶颈不要为省一次 RTT 引入这个正确性风险
*/
@Override
public String getUserLoginToken(String apiKey, Long companyId, Long userId) {
String raw = postWithApiKey(apiKey, ziniaoProperties.getUserLoginTokenPath(), Map.of(
@@ -539,11 +539,8 @@ public boolean isIpWhitelistError(BusinessException ex) {
try {
java.security.MessageDigest digest = java.security.MessageDigest.getInstance("SHA-256");
byte[] hash = digest.digest(apiKey.getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder();
for (byte b : hash) {
sb.append(String.format("%02x", b));
}
return sb.toString();
// HexFormat 替代逐字节 String.format("%02x")后者每次走 32 Formatter热路径按调用
return java.util.HexFormat.of().formatHex(hash);
} catch (Exception ex) {
throw new BusinessException("生成紫鸟 apiKey 缓存键失败");
}
@@ -937,11 +937,8 @@ public class ZiniaoShopIndexService {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] hash = digest.digest(apiKey.getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder();
for (byte b : hash) {
sb.append(String.format("%02x", b));
}
return sb.toString();
// HexFormat 替代逐字节 String.format("%02x")后者每次走 32 Formatter热路径按调用
return java.util.HexFormat.of().formatHex(hash);
} catch (Exception ex) {
throw new BusinessException("生成紫鸟 apiKey 缓存键失败");
}
@@ -9,6 +9,10 @@ import com.nanri.aiimage.modules.imagevideo.mapper.ImageVideoAsyncTaskMapper;
import com.nanri.aiimage.modules.imagevideo.model.dto.DouyinCopyRequest;
import com.nanri.aiimage.modules.imagevideo.model.entity.ImageVideoAsyncTaskEntity;
import com.nanri.aiimage.modules.imagevideo.model.vo.DouyinCopyVo;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.springframework.core.task.TaskExecutor;
import org.mockito.ArgumentCaptor;
@@ -29,6 +33,18 @@ import static org.mockito.Mockito.when;
class ImageVideoAsyncTaskServiceTest {
/**
* MyBatis-Plus lambda 列解析需要实体的 TableInfo本类用 mock mapper不启动 Spring
* dispatch/poll 扫描现在走 select(ImageVideoAsyncTaskEntity::getId)2026-09
* 不再每秒把 5 LONGTEXT 列拉回优化引入没有 TableInfo 会抛
* can not find lambda cache同款初始化见 AppearancePatentTaskServiceHistoryBatchTest
*/
@BeforeAll
static void initMybatisPlusTableInfo() {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
ImageVideoAsyncTaskEntity.class);
}
@Test
void parsesNestedDouyinCopyOutputIntoFrontendTextFields() {
ImageVideoCozeService cozeService = new ImageVideoCozeService(
@@ -13,6 +13,10 @@ import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEnt
import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
@@ -30,6 +34,18 @@ import static org.mockito.Mockito.when;
class PermissionMenuServiceTest {
/**
* MyBatis-Plus lambda 列解析需要实体的 TableInfo本类用 mock mapper不启动 Spring
* PermissionMenuService 的权限用户查询走 select(AdminUserEntity::getId, ...)2026-09
* 不再整表拉回含密码哈希的全列优化引入没有 TableInfo 会在构造 wrapper 时抛
* can not find lambda cache同款初始化见 AppearancePatentTaskServiceHistoryBatchTest
*/
@BeforeAll
static void initMybatisPlusTableInfo() {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
AdminUserEntity.class);
}
@Test
void preservesExistingImageVideoPermissionDuringGenericReplacement() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
@@ -36,10 +36,11 @@ class QueryAsinSnapshotJsonThrottleTest {
private QueryAsinTaskService service() {
// 构造顺序mapper/resultMapper/resolve/excel/cache/oss/resultDownloadResolver/ziniaoSwitch/
// objectMapper/pressure/jobService/resultItemService/snapshot/lock/lightAssembler
// objectMapper/pressure/jobService/resultItemService/snapshot/lock/lightAssembler/transactionManager
// transactionManager null本用例只走 persistSnapshotJson不涉及需要事务的删除链路
return new QueryAsinTaskService(
null, null, null, null, null, null, null, null,
objectMapper, null, null, taskResultItemService, taskProgressSnapshotService, null, null);
objectMapper, null, null, taskResultItemService, taskProgressSnapshotService, null, null, null);
}
private static void persist(QueryAsinTaskService service, FileTaskEntity task,
@@ -31,6 +31,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
@@ -49,6 +50,7 @@ import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@@ -196,11 +198,38 @@ class ShopDataCrawlTaskServiceTxBoundaryTest {
verify(fileResultMapper).updateById(any(FileResultEntity.class));
}
/**
* 事务边界契约2026-09 连接池修复deleteTask 不再挂方法级 @Transactional
* Redis 任务锁必须在事务外获取否则自旋等待最长 TASK_LOCK_WAIT_MILLIS期间会白占一个
* Hikari 连接落库删除仍在同一个短事务executeShortTransaction / REQUIRES_NEW里完成
* 且顺序固定为先取锁后开事务再落库删除
*/
@Test
void deleteTaskKeepsTransactionAnnotationAndPureCompute() throws Exception {
void deleteTaskKeepsLockOutsideAndWritesInsideSingleShortTransaction() throws Exception {
Method delete = ShopDataCrawlTaskService.class.getMethod("deleteTask", Long.class, Long.class);
assertTrue(delete.getAnnotation(Transactional.class) != null,
"deleteTask 落库删除必须保留 @Transactional锁内删除语义不变");
assertNull(delete.getAnnotation(Transactional.class),
"deleteTask 不得带 @TransactionalRedis 锁必须落在事务外,避免自旋期间占用 DB 连接");
FileTaskEntity task = runningTask();
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
FileResultEntity row = resultRow(1L);
row.setSuccess(1);
when(fileResultMapper.selectList(any())).thenReturn(List.of(row));
when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", TASK_ID))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, "SHOP_DATA_CRAWL", 1L)).thenReturn(true);
when(dailyFileService.findMembersByResultId(1L))
.thenReturn(List.of(mock(com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity.class)));
service.deleteTask(TASK_ID, USER_ID);
InOrder order = inOrder(taskDistributedLockService, transactionManager, fileResultMapper);
order.verify(taskDistributedLockService).acquire("SHOP_DATA_CRAWL", TASK_ID);
order.verify(transactionManager).getTransaction(any());
order.verify(fileResultMapper).delete(any());
// 全部落库删除共用一个短事务
verify(transactionManager, times(1)).getTransaction(any());
verify(fileTaskMapper).deleteById(TASK_ID);
// 删除路径使用抽出的纯函数行为等价由既有 ShopDataCrawlCleanupTest 守门
assertTrue(ShopDataCrawlTaskService.collectResultIds(List.of()).isEmpty());
}
@@ -167,8 +167,15 @@ class TaskResultFileJobWorkerOffloadTest {
verify(similar).cleanupResultFileJob(job);
}
/**
* 闸门拒绝时的契约2026-09 背压修复**不再回退成内联执行**
*
* 内联兜底会把背压转嫁给 MQ 消费线程消费线程按分钟级阻塞在 Excel 组装 + OSS 上传上
* 队列越满越糟改为重新入队置回 PENDING由归属实例 15s 一轮的扫描补发
* 任务不会丢只是不在消费线程上抢跑原用例名 offloadFailureFallback 断言的正是旧行为
*/
@Test
void offloadFailureFallback() throws Exception {
void offloadRejectedRequeuesInsteadOfInlineFallback() throws Exception {
TaskResultFileJobWorker worker = buildWorker();
TaskFileJobEntity job = job("SIMILAR_ASIN", 4L, 14L);
allowClaim(job.getId(), job.getTaskId(), job.getModuleType());
@@ -178,8 +185,9 @@ class TaskResultFileJobWorkerOffloadTest {
worker.process(job);
verify(taskFileJobService).markSuccess(job, null);
verify(similar).cleanupResultFileJob(job);
verify(taskFileJobService).requeue(job.getId(), "结果文件任务队列已满,稍后重投");
verify(taskFileJobService, never()).markSuccess(any(), any());
verify(similar, never()).processResultFileJob(any());
}
@Test
@@ -90,13 +90,21 @@ class TaskResultFileJobWorkerOrphanTest {
new BrandResultFileJobHandler(brand, payload),
new CollectDataResultFileJobHandler(collectData));
ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers);
return new TaskResultFileJobWorker(
TaskResultFileJobWorker worker = new TaskResultFileJobWorker(
taskFileJobService,
taskDistributedLockService,
mock(FileResultMapper.class),
mock(TaskFileJobLocalDispatcher.class),
instanceMetadata,
registry);
// taskQueueExecutor 是字段注入此前不注入时字段为 nulloffload 分支会抛 NPE 并被
// 兜底成内联执行本测试一直依赖这个偶然行为2026-09 起闸门拒绝改为重新入队
// 而不再回退内联故这里显式注入同步执行器 offload 模块仍按内联语义跑完整链路
java.lang.reflect.Field executorField =
TaskResultFileJobWorker.class.getDeclaredField("taskQueueExecutor");
executorField.setAccessible(true);
executorField.set(worker, (org.springframework.core.task.TaskExecutor) Runnable::run);
return worker;
}
private static TaskFileJobEntity job(String moduleType, long jobId, long taskId) {