perf(全链路): 连接池/事务边界/轮询与 IO 效率收口

Java:
- LLM 180s 读超时不再被全局 call-timeout 静默截断成 90s(长思考请求被掐断→重试→付费网关二次计费)
- 代理 HttpClient 缓存改有界 LRU(jikip 每次提取新 IP,无界缓存持续泄漏 selector 线程与连接池)
- 12 个 service 的 Redis 任务锁移出 @Transactional(自旋最坏 10s 白占 DB 连接,池仅 30),远端对象删/传改 afterCommit
- 结果文件 Job 闸门拒绝时不再回退内联执行(改重新入队,避免把背压转嫁给 MQ 消费线程)
- imagevideo 每秒扫描加列投影、过期清理加 LIMIT;权限页整表查询改列投影(不再拉回密码哈希)
- 哈希改 HexFormat;补 5 处"不能改"的技术依据注释(批量插入会丢回填主键、流式丢模板与图片等)

前端:4 个工具页轮询改轻量端点(带 fallback);PriceTrack 快照节流写盘;候选店铺表分页;页面隐藏时停表

客户端:HTTP 连接池按出口复用(Session 仍每请求新建,保持无跨请求状态);品牌检测 WIPO 逐请求握手;
代理配置按 mtime 缓存;串行任务改专属池;异常降级为标签页重连;紫鸟启动改端口轮询;模板编译缓存;
日志上报连接与落盘收口;Flask 版本 API 改按请求复用连接
This commit is contained in:
2026-09-15 23:01:59 +08:00
parent 0dd0b1237f
commit 06b3fc3502
44 changed files with 1550 additions and 423 deletions
@@ -431,70 +431,85 @@ public class CollectDataService {
}
}
@Transactional
/**
* 进度心跳。
*
* <p>事务边界:Redis 任务锁在事务外获取(自旋等待最长 TASK_LOCK_WAIT_MILLIS,
* 放在 @Transactional 里会白占一个 Hikari 连接),DB 段(统计持久化 + 任务行更新)
* 仍在一个事务内。
*/
public void updateProgress(Long taskId, TaskHeartbeatRequest request) {
if (taskId == null || taskId <= 0 || request == null) {
return;
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
if (transactionTemplate == null) {
// 单测场景(@InjectMocks 未注入事务模板):退化为直接执行 DB 段
updateProgressLocked(taskId, request);
return;
}
CollectDataStats stats = loadStats(task);
boolean changed = false;
Integer current = request.getCurrent();
Integer total = request.getTotal();
if (current != null && total != null && total > 0) {
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
stats.totalRows = totalRows;
stats.processedRows = processedRows;
changed = true;
}
}
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
stats.collectStage = request.getCollectStage();
changed = true;
}
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
stats.currentKeyword = request.getCurrentKeyword();
changed = true;
}
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
changed = true;
}
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
changed = true;
}
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
changed = true;
}
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
changed = true;
}
if (!changed) {
// 内容未变化:仅当脏数据兜底窗口到期时才强制刷新一次,
// 否则零 UPDATE(重复心跳幂等)。
if (!shouldForceProgressFlush()) {
return;
}
} else if (shouldThrottleProgressFlush()) {
// 节流窗口内:合并写(只更新内存态、不落库),窗口到期后统一持久化。
return;
}
persistStats(task, stats);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
lastProgressFlushMillis = System.currentTimeMillis();
transactionTemplate.executeWithoutResult(status -> updateProgressLocked(taskId, request));
}
}
private void updateProgressLocked(Long taskId, TaskHeartbeatRequest request) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
return;
}
CollectDataStats stats = loadStats(task);
boolean changed = false;
Integer current = request.getCurrent();
Integer total = request.getTotal();
if (current != null && total != null && total > 0) {
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
stats.totalRows = totalRows;
stats.processedRows = processedRows;
changed = true;
}
}
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
stats.collectStage = request.getCollectStage();
changed = true;
}
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
stats.currentKeyword = request.getCurrentKeyword();
changed = true;
}
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
changed = true;
}
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
changed = true;
}
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
changed = true;
}
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
changed = true;
}
if (!changed) {
// 内容未变化:仅当脏数据兜底窗口到期时才强制刷新一次,
// 否则零 UPDATE(重复心跳幂等)。
if (!shouldForceProgressFlush()) {
return;
}
} else if (shouldThrottleProgressFlush()) {
// 节流窗口内:合并写(只更新内存态、不落库),窗口到期后统一持久化。
return;
}
persistStats(task, stats);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
lastProgressFlushMillis = System.currentTimeMillis();
}
/**
* 节流判定:progressThrottleMillis>0 且距上次实际落库未超过窗口 → 合并写(跳过 UPDATE)。
* progressThrottleMillis<=0 视为关闭节流,恒返回 false(每次心跳都落库,兼容旧行为)。