perf(全链路): 连接池/事务边界/轮询与 IO 效率收口

Java:
- LLM 180s 读超时不再被全局 call-timeout 静默截断成 90s(长思考请求被掐断→重试→付费网关二次计费)
- 代理 HttpClient 缓存改有界 LRU(jikip 每次提取新 IP,无界缓存持续泄漏 selector 线程与连接池)
- 12 个 service 的 Redis 任务锁移出 @Transactional(自旋最坏 10s 白占 DB 连接,池仅 30),远端对象删/传改 afterCommit
- 结果文件 Job 闸门拒绝时不再回退内联执行(改重新入队,避免把背压转嫁给 MQ 消费线程)
- imagevideo 每秒扫描加列投影、过期清理加 LIMIT;权限页整表查询改列投影(不再拉回密码哈希)
- 哈希改 HexFormat;补 5 处"不能改"的技术依据注释(批量插入会丢回填主键、流式丢模板与图片等)

前端:4 个工具页轮询改轻量端点(带 fallback);PriceTrack 快照节流写盘;候选店铺表分页;页面隐藏时停表

客户端:HTTP 连接池按出口复用(Session 仍每请求新建,保持无跨请求状态);品牌检测 WIPO 逐请求握手;
代理配置按 mtime 缓存;串行任务改专属池;异常降级为标签页重连;紫鸟启动改端口轮询;模板编译缓存;
日志上报连接与落盘收口;Flask 版本 API 改按请求复用连接
This commit is contained in:
2026-09-15 23:01:59 +08:00
parent 0dd0b1237f
commit 06b3fc3502
44 changed files with 1550 additions and 423 deletions
@@ -9,6 +9,10 @@ import com.nanri.aiimage.modules.imagevideo.mapper.ImageVideoAsyncTaskMapper;
import com.nanri.aiimage.modules.imagevideo.model.dto.DouyinCopyRequest;
import com.nanri.aiimage.modules.imagevideo.model.entity.ImageVideoAsyncTaskEntity;
import com.nanri.aiimage.modules.imagevideo.model.vo.DouyinCopyVo;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.springframework.core.task.TaskExecutor;
import org.mockito.ArgumentCaptor;
@@ -29,6 +33,18 @@ import static org.mockito.Mockito.when;
class ImageVideoAsyncTaskServiceTest {
/**
* MyBatis-Plus 的 lambda 列解析需要实体的 TableInfo。本类用 mock mapper、不启动 Spring,
* 而 dispatch/poll 扫描现在走 select(ImageVideoAsyncTaskEntity::getId)(2026-09
* 「不再每秒把 5 个 LONGTEXT 列拉回」优化引入),没有 TableInfo 会抛
* 「can not find lambda cache」。同款初始化见 AppearancePatentTaskServiceHistoryBatchTest。
*/
@BeforeAll
static void initMybatisPlusTableInfo() {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
ImageVideoAsyncTaskEntity.class);
}
@Test
void parsesNestedDouyinCopyOutputIntoFrontendTextFields() {
ImageVideoCozeService cozeService = new ImageVideoCozeService(
@@ -13,6 +13,10 @@ import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEnt
import com.nanri.aiimage.modules.permission.model.vo.ImageVideoDataPermissionUserVo;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
@@ -30,6 +34,18 @@ import static org.mockito.Mockito.when;
class PermissionMenuServiceTest {
/**
* MyBatis-Plus 的 lambda 列解析需要实体的 TableInfo。本类用 mock mapper、不启动 Spring,
* 而 PermissionMenuService 的权限用户查询走 select(AdminUserEntity::getId, ...)(2026-09
* 「不再整表拉回含密码哈希的全列」优化引入),没有 TableInfo 会在构造 wrapper 时抛
* 「can not find lambda cache」。同款初始化见 AppearancePatentTaskServiceHistoryBatchTest。
*/
@BeforeAll
static void initMybatisPlusTableInfo() {
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
AdminUserEntity.class);
}
@Test
void preservesExistingImageVideoPermissionDuringGenericReplacement() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
@@ -36,10 +36,11 @@ class QueryAsinSnapshotJsonThrottleTest {
private QueryAsinTaskService service() {
// 构造顺序:mapper/resultMapper/resolve/excel/cache/oss/resultDownloadResolver/ziniaoSwitch/
// objectMapper/pressure/jobService/resultItemService/snapshot/lock/lightAssembler
// objectMapper/pressure/jobService/resultItemService/snapshot/lock/lightAssembler/transactionManager
// transactionManager 传 null:本用例只走 persistSnapshotJson,不涉及需要事务的删除链路
return new QueryAsinTaskService(
null, null, null, null, null, null, null, null,
objectMapper, null, null, taskResultItemService, taskProgressSnapshotService, null, null);
objectMapper, null, null, taskResultItemService, taskProgressSnapshotService, null, null, null);
}
private static void persist(QueryAsinTaskService service, FileTaskEntity task,
@@ -31,6 +31,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
@@ -49,6 +50,7 @@ import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@@ -196,11 +198,38 @@ class ShopDataCrawlTaskServiceTxBoundaryTest {
verify(fileResultMapper).updateById(any(FileResultEntity.class));
}
/**
* 事务边界契约(2026-09 连接池修复):deleteTask 不再挂方法级 @Transactional ——
* Redis 任务锁必须在事务外获取,否则自旋等待(最长 TASK_LOCK_WAIT_MILLIS)期间会白占一个
* Hikari 连接。落库删除仍在同一个短事务(executeShortTransaction / REQUIRES_NEW)里完成,
* 且顺序固定为「先取锁、后开事务、再落库删除」。
*/
@Test
void deleteTaskKeepsTransactionAnnotationAndPureCompute() throws Exception {
void deleteTaskKeepsLockOutsideAndWritesInsideSingleShortTransaction() throws Exception {
Method delete = ShopDataCrawlTaskService.class.getMethod("deleteTask", Long.class, Long.class);
assertTrue(delete.getAnnotation(Transactional.class) != null,
"deleteTask 落库删除必须保留 @Transactional(锁内删除语义不变)");
assertNull(delete.getAnnotation(Transactional.class),
"deleteTask 不得带 @Transactional(Redis 锁必须落在事务外,避免自旋期间占用 DB 连接)");
FileTaskEntity task = runningTask();
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
FileResultEntity row = resultRow(1L);
row.setSuccess(1);
when(fileResultMapper.selectList(any())).thenReturn(List.of(row));
when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", TASK_ID))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, "SHOP_DATA_CRAWL", 1L)).thenReturn(true);
when(dailyFileService.findMembersByResultId(1L))
.thenReturn(List.of(mock(com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity.class)));
service.deleteTask(TASK_ID, USER_ID);
InOrder order = inOrder(taskDistributedLockService, transactionManager, fileResultMapper);
order.verify(taskDistributedLockService).acquire("SHOP_DATA_CRAWL", TASK_ID);
order.verify(transactionManager).getTransaction(any());
order.verify(fileResultMapper).delete(any());
// 全部落库删除共用一个短事务
verify(transactionManager, times(1)).getTransaction(any());
verify(fileTaskMapper).deleteById(TASK_ID);
// 删除路径使用抽出的纯函数(行为等价由既有 ShopDataCrawlCleanupTest 守门)
assertTrue(ShopDataCrawlTaskService.collectResultIds(List.of()).isEmpty());
}
@@ -167,8 +167,15 @@ class TaskResultFileJobWorkerOffloadTest {
verify(similar).cleanupResultFileJob(job);
}
/**
* 闸门拒绝时的契约(2026-09 背压修复):**不再回退成内联执行**。
*
* 内联兜底会把背压转嫁给 MQ 消费线程——消费线程按分钟级阻塞在 Excel 组装 + OSS 上传上,
* 队列越满越糟。改为重新入队(置回 PENDING),由归属实例 15s 一轮的扫描补发;
* 任务不会丢,只是不在消费线程上抢跑。原用例名 offloadFailureFallback 断言的正是旧行为。
*/
@Test
void offloadFailureFallback() throws Exception {
void offloadRejectedRequeuesInsteadOfInlineFallback() throws Exception {
TaskResultFileJobWorker worker = buildWorker();
TaskFileJobEntity job = job("SIMILAR_ASIN", 4L, 14L);
allowClaim(job.getId(), job.getTaskId(), job.getModuleType());
@@ -178,8 +185,9 @@ class TaskResultFileJobWorkerOffloadTest {
worker.process(job);
verify(taskFileJobService).markSuccess(job, null);
verify(similar).cleanupResultFileJob(job);
verify(taskFileJobService).requeue(job.getId(), "结果文件任务队列已满,稍后重投");
verify(taskFileJobService, never()).markSuccess(any(), any());
verify(similar, never()).processResultFileJob(any());
}
@Test
@@ -90,13 +90,21 @@ class TaskResultFileJobWorkerOrphanTest {
new BrandResultFileJobHandler(brand, payload),
new CollectDataResultFileJobHandler(collectData));
ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(handlers);
return new TaskResultFileJobWorker(
TaskResultFileJobWorker worker = new TaskResultFileJobWorker(
taskFileJobService,
taskDistributedLockService,
mock(FileResultMapper.class),
mock(TaskFileJobLocalDispatcher.class),
instanceMetadata,
registry);
// taskQueueExecutor 是字段注入。此前不注入时字段为 null,offload 分支会抛 NPE 并被
// 兜底成内联执行——本测试一直依赖这个偶然行为。2026-09 起闸门拒绝改为「重新入队」
// 而不再回退内联,故这里显式注入同步执行器,让 offload 模块仍按内联语义跑完整链路。
java.lang.reflect.Field executorField =
TaskResultFileJobWorker.class.getDeclaredField("taskQueueExecutor");
executorField.setAccessible(true);
executorField.set(worker, (org.springframework.core.task.TaskExecutor) Runnable::run);
return worker;
}
private static TaskFileJobEntity job(String moduleType, long jobId, long taskId) {