perf(全链路): 连接池/事务边界/轮询与 IO 效率收口
Java: - LLM 180s 读超时不再被全局 call-timeout 静默截断成 90s(长思考请求被掐断→重试→付费网关二次计费) - 代理 HttpClient 缓存改有界 LRU(jikip 每次提取新 IP,无界缓存持续泄漏 selector 线程与连接池) - 12 个 service 的 Redis 任务锁移出 @Transactional(自旋最坏 10s 白占 DB 连接,池仅 30),远端对象删/传改 afterCommit - 结果文件 Job 闸门拒绝时不再回退内联执行(改重新入队,避免把背压转嫁给 MQ 消费线程) - imagevideo 每秒扫描加列投影、过期清理加 LIMIT;权限页整表查询改列投影(不再拉回密码哈希) - 哈希改 HexFormat;补 5 处"不能改"的技术依据注释(批量插入会丢回填主键、流式丢模板与图片等) 前端:4 个工具页轮询改轻量端点(带 fallback);PriceTrack 快照节流写盘;候选店铺表分页;页面隐藏时停表 客户端:HTTP 连接池按出口复用(Session 仍每请求新建,保持无跨请求状态);品牌检测 WIPO 逐请求握手; 代理配置按 mtime 缓存;串行任务改专属池;异常降级为标签页重连;紫鸟启动改端口轮询;模板编译缓存; 日志上报连接与落盘收口;Flask 版本 API 改按请求复用连接
This commit is contained in:
+123
-58
@@ -39,6 +39,8 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.TransactionDefinition;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
@@ -93,6 +95,11 @@ public class PriceTrackTaskService {
|
||||
private final TaskDistributedLockService taskDistributedLockService;
|
||||
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
private final com.nanri.aiimage.modules.file.service.LocalFileStorageService localFileStorageService;
|
||||
/**
|
||||
* 事务边界收口用:Redis 任务锁必须在事务外获取,否则自旋等待(最长 TASK_LOCK_WAIT_MILLIS)
|
||||
* 期间会白占一个 Hikari 连接(生产 maximum-pool-size 仅 30)。
|
||||
*/
|
||||
private final org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
|
||||
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
||||
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
||||
@@ -254,7 +261,10 @@ public class PriceTrackTaskService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
/**
|
||||
* 事务边界:Redis 任务锁在事务外获取(自旋等待不再占用 DB 连接),
|
||||
* DB 段(复核 + 删除 + 重算任务状态)仍在同一个事务内。
|
||||
*/
|
||||
public void deleteHistory(Long resultId, Long userId) {
|
||||
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
|
||||
FileResultEntity entity = fileResultMapper.selectById(resultId);
|
||||
@@ -267,73 +277,97 @@ public class PriceTrackTaskService {
|
||||
return;
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
|
||||
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
|
||||
throw new BusinessException("record not found");
|
||||
}
|
||||
fileResultMapper.deleteById(resultId);
|
||||
reconcileTaskAfterResultRemoval(taskId);
|
||||
inNewTransaction(() -> {
|
||||
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
|
||||
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
|
||||
throw new BusinessException("record not found");
|
||||
}
|
||||
fileResultMapper.deleteById(resultId);
|
||||
reconcileTaskAfterResultRemoval(taskId);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
/**
|
||||
* 事务边界:任务锁在事务外获取;缓存清理(Redis + 远端范围载荷)放到事务提交后执行。
|
||||
*/
|
||||
public void deleteTask(Long taskId, Long userId) {
|
||||
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||
FileTaskEntity task = loadTaskForExecution(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, taskId)
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
||||
fileTaskMapper.deleteById(taskId);
|
||||
inNewTransaction(() -> {
|
||||
FileTaskEntity task = loadTaskForExecution(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, taskId)
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
||||
fileTaskMapper.deleteById(taskId);
|
||||
priceTrackLoopRunService.syncLoopRunAfterChildRemoved(taskId);
|
||||
return null;
|
||||
});
|
||||
// 缓存清理(Redis + 远端范围载荷)在事务提交后、锁内执行
|
||||
priceTrackTaskCacheService.deleteTaskCache(taskId);
|
||||
priceTrackLoopRunService.syncLoopRunAfterChildRemoved(taskId);
|
||||
}
|
||||
log.info("[price-track] deleteTask 完成 taskId={} userId={}", taskId, userId);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
/**
|
||||
* 事务边界:任务锁在事务外获取;缓存清理放到事务提交后执行。
|
||||
*/
|
||||
public void markDispatchFailed(Long taskId, Long userId, String errorMessage) {
|
||||
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
|
||||
if (errorMessage == null || errorMessage.isBlank()) throw new BusinessException("errorMessage 不能为空");
|
||||
String normalizedError = errorMessage.trim();
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||
FileTaskEntity task = loadTaskForExecution(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
// 返回需要清理缓存的任务状态(仅无结果行分支),非终态返回 null
|
||||
String terminalStatus = inNewTransaction(() ->
|
||||
markDispatchFailedRecords(taskId, userId, normalizedError));
|
||||
if (terminalStatus != null) {
|
||||
cleanupTaskCacheIfTerminal(taskId, terminalStatus);
|
||||
}
|
||||
if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) {
|
||||
return;
|
||||
}
|
||||
List<FileResultEntity> results = fileResultMapper.selectList(
|
||||
new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, taskId)
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||
.orderByAsc(FileResultEntity::getId));
|
||||
if (results.isEmpty()) {
|
||||
task.setStatus("FAILED");
|
||||
task.setErrorMessage(normalizedError);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
cleanupTaskCacheIfTerminal(taskId, task.getStatus());
|
||||
priceTrackLoopRunService.syncLoopRunAfterChildTerminal(taskId);
|
||||
return;
|
||||
}
|
||||
for (FileResultEntity result : results) {
|
||||
boolean succeeded = result.getSuccess() != null && result.getSuccess() == 1;
|
||||
boolean failed = result.getErrorMessage() != null && !result.getErrorMessage().isBlank();
|
||||
if (!succeeded && !failed) {
|
||||
markResultFailed(result, normalizedError);
|
||||
}
|
||||
}
|
||||
updateTaskStatusFromLatestRows(task, results);
|
||||
log.warn("[price-track] Python dispatch failed taskId={} userId={} error={}", taskId, userId, normalizedError);
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
private String markDispatchFailedRecords(Long taskId, Long userId, String normalizedError) {
|
||||
FileTaskEntity task = loadTaskForExecution(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) {
|
||||
return null;
|
||||
}
|
||||
List<FileResultEntity> results = fileResultMapper.selectList(
|
||||
new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, taskId)
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||
.orderByAsc(FileResultEntity::getId));
|
||||
if (results.isEmpty()) {
|
||||
task.setStatus("FAILED");
|
||||
task.setErrorMessage(normalizedError);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
priceTrackLoopRunService.syncLoopRunAfterChildTerminal(taskId);
|
||||
return task.getStatus();
|
||||
}
|
||||
for (FileResultEntity result : results) {
|
||||
boolean succeeded = result.getSuccess() != null && result.getSuccess() == 1;
|
||||
boolean failed = result.getErrorMessage() != null && !result.getErrorMessage().isBlank();
|
||||
if (!succeeded && !failed) {
|
||||
markResultFailed(result, normalizedError);
|
||||
}
|
||||
}
|
||||
updateTaskStatusFromLatestRows(task, results);
|
||||
log.warn("[price-track] Python dispatch failed taskId={} userId={} error={}", taskId, userId, normalizedError);
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 事务边界:逐个候选任务在事务外取锁;命中的那条记录的删除 + 重算在独立短事务内完成
|
||||
* (原实现整个循环共用一个事务,锁自旋等待期间会一直占着连接)。
|
||||
*/
|
||||
public PriceTrackPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
|
||||
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
|
||||
if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 不能为空");
|
||||
@@ -356,20 +390,33 @@ public class PriceTrackTaskService {
|
||||
if (!"RUNNING".equals(task.getStatus())) continue;
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLock(fr.getTaskId())) {
|
||||
if (ignored == null) continue;
|
||||
FileTaskEntity lockedTask = loadTaskForExecution(fr.getTaskId());
|
||||
if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) continue;
|
||||
if (!"RUNNING".equals(lockedTask.getStatus())) continue;
|
||||
FileResultEntity lockedResult = fileResultMapper.selectById(fr.getId());
|
||||
if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) continue;
|
||||
fileResultMapper.deleteById(fr.getId());
|
||||
reconcileTaskAfterResultRemoval(fr.getTaskId());
|
||||
vo.setRemoved(true);
|
||||
return vo;
|
||||
Boolean removed = inNewTransaction(() -> removeRunningShopResult(fr.getTaskId(), userId, fr.getId()));
|
||||
if (Boolean.TRUE.equals(removed)) {
|
||||
vo.setRemoved(true);
|
||||
return vo;
|
||||
}
|
||||
}
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
private boolean removeRunningShopResult(Long taskId, Long userId, Long resultId) {
|
||||
FileTaskEntity lockedTask = loadTaskForExecution(taskId);
|
||||
if (lockedTask == null || !MODULE_TYPE.equals(lockedTask.getModuleType()) || !userId.equals(lockedTask.getUserId())) {
|
||||
return false;
|
||||
}
|
||||
if (!"RUNNING".equals(lockedTask.getStatus())) {
|
||||
return false;
|
||||
}
|
||||
FileResultEntity lockedResult = fileResultMapper.selectById(resultId);
|
||||
if (lockedResult == null || !MODULE_TYPE.equals(lockedResult.getModuleType())) {
|
||||
return false;
|
||||
}
|
||||
fileResultMapper.deleteById(resultId);
|
||||
reconcileTaskAfterResultRemoval(taskId);
|
||||
return true;
|
||||
}
|
||||
|
||||
public PriceTrackTaskBatchVo getTaskDetailsBatch(List<Long> taskIds) {
|
||||
PriceTrackTaskBatchVo batch = new PriceTrackTaskBatchVo();
|
||||
if (taskIds == null || taskIds.isEmpty()) return batch;
|
||||
@@ -506,6 +553,10 @@ public class PriceTrackTaskService {
|
||||
}
|
||||
|
||||
List<PriceTrackResultItemVo> snapshot = new ArrayList<>();
|
||||
// 逐店铺 fileResultMapper.insert 看似该改批量(500 店铺 = 500 次往返),但**不能改**:
|
||||
// 下面的 toSnapshotVo 依赖 MyBatis-Plus insert 回填的自增主键(fr.getId() → vo.resultId),
|
||||
// 而本仓的手写 INSERT ... VALUES (...),(...) 批量写法不回填主键,改了会让快照 resultId 为 null。
|
||||
// 要批量化必须同时解决「批量插入 + 按序回填 id」(MySQL 需 useGeneratedKeys + 顺序保证)。
|
||||
for (PriceTrackMatchShopsVo.PriceTrackShopQueueItem item : uniqueItems) {
|
||||
String norm = ziniaoShopSwitchService.normalizeShopName(item.getShopName());
|
||||
if (norm.isBlank()) throw new BusinessException("店铺名称无效");
|
||||
@@ -2182,6 +2233,20 @@ public class PriceTrackTaskService {
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
/**
|
||||
* 显式开启一个短事务:锁已在事务外获取,这里只包住 DB 段,
|
||||
* 保证原来同属一个 @Transactional 的多条写仍在同一事务内。
|
||||
*/
|
||||
private <T> T inNewTransaction(java.util.function.Supplier<T> action) {
|
||||
if (transactionManager == null) {
|
||||
// 单测直接 new 出本类(history 见 QueryAsinSnapshotJsonThrottleTest 同类用法)时无事务管理器
|
||||
return action.get();
|
||||
}
|
||||
TransactionTemplate template = new TransactionTemplate(transactionManager);
|
||||
template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
|
||||
return template.execute(status -> action.get());
|
||||
}
|
||||
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return null;
|
||||
|
||||
Reference in New Issue
Block a user