perf(全链路): 连接池/事务边界/轮询与 IO 效率收口
Java: - LLM 180s 读超时不再被全局 call-timeout 静默截断成 90s(长思考请求被掐断→重试→付费网关二次计费) - 代理 HttpClient 缓存改有界 LRU(jikip 每次提取新 IP,无界缓存持续泄漏 selector 线程与连接池) - 12 个 service 的 Redis 任务锁移出 @Transactional(自旋最坏 10s 白占 DB 连接,池仅 30),远端对象删/传改 afterCommit - 结果文件 Job 闸门拒绝时不再回退内联执行(改重新入队,避免把背压转嫁给 MQ 消费线程) - imagevideo 每秒扫描加列投影、过期清理加 LIMIT;权限页整表查询改列投影(不再拉回密码哈希) - 哈希改 HexFormat;补 5 处"不能改"的技术依据注释(批量插入会丢回填主键、流式丢模板与图片等) 前端:4 个工具页轮询改轻量端点(带 fallback);PriceTrack 快照节流写盘;候选店铺表分页;页面隐藏时停表 客户端:HTTP 连接池按出口复用(Session 仍每请求新建,保持无跨请求状态);品牌检测 WIPO 逐请求握手; 代理配置按 mtime 缓存;串行任务改专属池;异常降级为标签页重连;紫鸟启动改端口轮询;模板编译缓存; 日志上报连接与落盘收口;Flask 版本 API 改按请求复用连接
This commit is contained in:
+69
-54
@@ -431,70 +431,85 @@ public class CollectDataService {
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
/**
|
||||
* 进度心跳。
|
||||
*
|
||||
* <p>事务边界:Redis 任务锁在事务外获取(自旋等待最长 TASK_LOCK_WAIT_MILLIS,
|
||||
* 放在 @Transactional 里会白占一个 Hikari 连接),DB 段(统计持久化 + 任务行更新)
|
||||
* 仍在一个事务内。
|
||||
*/
|
||||
public void updateProgress(Long taskId, TaskHeartbeatRequest request) {
|
||||
if (taskId == null || taskId <= 0 || request == null) {
|
||||
return;
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
|
||||
if (transactionTemplate == null) {
|
||||
// 单测场景(@InjectMocks 未注入事务模板):退化为直接执行 DB 段
|
||||
updateProgressLocked(taskId, request);
|
||||
return;
|
||||
}
|
||||
CollectDataStats stats = loadStats(task);
|
||||
boolean changed = false;
|
||||
Integer current = request.getCurrent();
|
||||
Integer total = request.getTotal();
|
||||
if (current != null && total != null && total > 0) {
|
||||
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
|
||||
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
|
||||
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
|
||||
stats.totalRows = totalRows;
|
||||
stats.processedRows = processedRows;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
|
||||
stats.collectStage = request.getCollectStage();
|
||||
changed = true;
|
||||
}
|
||||
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
|
||||
stats.currentKeyword = request.getCurrentKeyword();
|
||||
changed = true;
|
||||
}
|
||||
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
|
||||
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
|
||||
changed = true;
|
||||
}
|
||||
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
|
||||
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
|
||||
changed = true;
|
||||
}
|
||||
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
|
||||
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
|
||||
changed = true;
|
||||
}
|
||||
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
|
||||
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
|
||||
changed = true;
|
||||
}
|
||||
if (!changed) {
|
||||
// 内容未变化:仅当脏数据兜底窗口到期时才强制刷新一次,
|
||||
// 否则零 UPDATE(重复心跳幂等)。
|
||||
if (!shouldForceProgressFlush()) {
|
||||
return;
|
||||
}
|
||||
} else if (shouldThrottleProgressFlush()) {
|
||||
// 节流窗口内:合并写(只更新内存态、不落库),窗口到期后统一持久化。
|
||||
return;
|
||||
}
|
||||
persistStats(task, stats);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
lastProgressFlushMillis = System.currentTimeMillis();
|
||||
transactionTemplate.executeWithoutResult(status -> updateProgressLocked(taskId, request));
|
||||
}
|
||||
}
|
||||
|
||||
private void updateProgressLocked(Long taskId, TaskHeartbeatRequest request) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
|
||||
return;
|
||||
}
|
||||
CollectDataStats stats = loadStats(task);
|
||||
boolean changed = false;
|
||||
Integer current = request.getCurrent();
|
||||
Integer total = request.getTotal();
|
||||
if (current != null && total != null && total > 0) {
|
||||
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
|
||||
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
|
||||
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
|
||||
stats.totalRows = totalRows;
|
||||
stats.processedRows = processedRows;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
|
||||
stats.collectStage = request.getCollectStage();
|
||||
changed = true;
|
||||
}
|
||||
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
|
||||
stats.currentKeyword = request.getCurrentKeyword();
|
||||
changed = true;
|
||||
}
|
||||
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
|
||||
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
|
||||
changed = true;
|
||||
}
|
||||
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
|
||||
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
|
||||
changed = true;
|
||||
}
|
||||
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
|
||||
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
|
||||
changed = true;
|
||||
}
|
||||
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
|
||||
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
|
||||
changed = true;
|
||||
}
|
||||
if (!changed) {
|
||||
// 内容未变化:仅当脏数据兜底窗口到期时才强制刷新一次,
|
||||
// 否则零 UPDATE(重复心跳幂等)。
|
||||
if (!shouldForceProgressFlush()) {
|
||||
return;
|
||||
}
|
||||
} else if (shouldThrottleProgressFlush()) {
|
||||
// 节流窗口内:合并写(只更新内存态、不落库),窗口到期后统一持久化。
|
||||
return;
|
||||
}
|
||||
persistStats(task, stats);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
lastProgressFlushMillis = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
/**
|
||||
* 节流判定:progressThrottleMillis>0 且距上次实际落库未超过窗口 → 合并写(跳过 UPDATE)。
|
||||
* progressThrottleMillis<=0 视为关闭节流,恒返回 false(每次心跳都落库,兼容旧行为)。
|
||||
|
||||
Reference in New Issue
Block a user