feat(phase-5): B60 console 分页与矩阵预聚合+P1-B5/6/7/8 性能热点收口
接管另一路会话的 Phase 5 在飞件收口(5A/5C 主体由那一路落码,本笔补齐 P1-B7 与两处台账并整量验证)。 - 5A/B60:撞款 console 的行窗口在服务端真分页(响应 dup 行数 ≤ pageSize,total_dup 仍是未分页 总数)、dupGroup 判据收归服务端单一出处(前端 dupWithinGroup 删掉,两套同名规则不再分叉)、 排序先于切窗;后管矩阵改预聚合+列窗 40。读数(200 列 × 50 行):每行 3 店 8.4 → 5.3 → 0.7ms (12.4×),每行 40 店 83.7 → 18.4 → 1.8ms(45.3×)。服务端不设列窗是判断不是漏做:shops 每行 只有 5 个标量,而矩阵的列来自已分页的行——给 shops 再加分页会把"全局指标读全集"和"列窗读本页" 两套语义搅在一起。浏览器首屏/交互耗时本机拿不到,登记为 Phase 6 现网复量。 - P1-B6:claim 循环的父任务终态判定折成一次批量 IN(TaskFileJobBatchQueries+Mapper XML, 铁律 1 收口在 Mapper),逐行 CAS 认领一字不动。 - P1-B5:V187 只建 idx_biz_file_task_resume_of;审计给的候选集索引**有意不建**(V184 前缀已覆盖, OR/IS-NULL 与 COALESCE 不可索引),判据把这条负判定钉住。真库 EXPLAIN 本机无 MySQL,顺延现网。 - P1-B8:结果载荷的对象存储删除移到 afterCommit,事务里不再做 120s 超时的网络调用。 - P1-B7:结果集读取按窗提交+收割(窗口=线程池队列容量的一半)。审计把它记成"无 limit",但截断 是正确性错误不是性能取舍,真能收的是"整份结果集被读两遍"——一次性提交时每支 Future 把解好的 载荷扣在自己身上直到全部收割完,峰值=结果集+另一份整副本;分窗后副本只驻留一窗。流式组装 契约要改 4 个模块的调用方,本 Phase 未做,别读成 P1-B7 已闭环。 - SourceScanTest 台账随之登记:裸名 25 → 26、调用方 18 → 19(P1-B7 的窗口判据只能打在源码上—— 线程池只有 8 个线程,"同时开始读几行"改前改后读数相同,差异只体现在堆上,mock 数不出来)。
This commit is contained in:
+11
-3
@@ -141,7 +141,7 @@ public class AdminShopDataDuplicateCheckController {
|
||||
}
|
||||
|
||||
@GetMapping("/duplicate-check-console")
|
||||
@Operation(summary = "撞款控制台快照:指标/店铺/命中筛选的撞款全集/分组统计")
|
||||
@Operation(summary = "撞款控制台快照:指标/店铺/撞款行窗口(分页)/分组统计")
|
||||
public ApiResponse<Object> console(
|
||||
HttpServletRequest request,
|
||||
@RequestParam(name = "asin", required = false) String asin,
|
||||
@@ -151,13 +151,20 @@ public class AdminShopDataDuplicateCheckController {
|
||||
@RequestParam(name = "site", required = false) String site,
|
||||
@RequestParam(name = "date_from", required = false) String dateFrom,
|
||||
@RequestParam(name = "date_to", required = false) String dateTo,
|
||||
@Parameter(description = "行窗口页码(从 1 起)") @RequestParam(name = "page", required = false) String page,
|
||||
@Parameter(description = "行窗口每页行数(响应体 dup 行数 ≤ 本值)")
|
||||
@RequestParam(name = "page_size", required = false) String pageSize,
|
||||
@Parameter(description = "撞款行范围=指定分组的成员店铺(只裁行,不动指标/店铺/分组统计)")
|
||||
@RequestParam(name = "dup_group", required = false) String dupGroup,
|
||||
@Parameter(description = "公司 ID") @RequestParam(name = "company_id", required = false) Long companyId) {
|
||||
RequestOperator operator = requireDuplicateCheckAccess(request);
|
||||
int safePage = parsePage(page);
|
||||
int safeSize = clamp(parseSize(pageSize, 100), 1, 200);
|
||||
ShopDataDuplicateCheckQueryService.Filters filters =
|
||||
ShopDataDuplicateCheckQueryService.Filters.clean(asin, shopName, country, site, dateFrom, dateTo);
|
||||
DuplicateScanView scan = scanService.loadLatest();
|
||||
if (scan == null) {
|
||||
return ApiResponse.success(queryService.pendingConsole());
|
||||
return ApiResponse.success(queryService.pendingConsole(safePage, safeSize));
|
||||
}
|
||||
List<Long> companyMemberIds = operator.superAdmin()
|
||||
? dataReadScope.readScope(operator.user(), companyId).userIdsOr(null)
|
||||
@@ -165,7 +172,8 @@ public class AdminShopDataDuplicateCheckController {
|
||||
Set<String> visibleKeys = queryService.resolveVisibleShopKeys(
|
||||
operator.id(), operator.superAdmin(), companyMemberIds);
|
||||
Set<String> effectiveKeys = queryService.withGroupFilter(visibleKeys, scan, group);
|
||||
return ApiResponse.success(queryService.consoleData(scan, effectiveKeys, filters));
|
||||
return ApiResponse.success(queryService.consoleData(scan, effectiveKeys, filters,
|
||||
safePage, safeSize, dupGroup));
|
||||
}
|
||||
|
||||
@GetMapping("/duplicate-check-ledger")
|
||||
|
||||
+57
-6
@@ -280,22 +280,73 @@ public class ShopDataDuplicateCheckQueryService {
|
||||
"page", page, "page_size", pageSize, "scanned_at", "");
|
||||
}
|
||||
|
||||
/** 撞款控制台一次快照:summary/shops(全局可见口径) + 命中筛选的撞款全集 + 分组统计。 */
|
||||
public Map<String, Object> consoleData(DuplicateScanView view, Set<String> visibleKeys, Filters filters) {
|
||||
/**
|
||||
* 撞款控制台一次快照:summary/shops(全局可见口径) + 命中筛选的撞款**行窗口** + 分组统计。
|
||||
*
|
||||
* <p>B60:{@code dup} 原先返回全集(现网可累到数万行的 JSON),前端翻页/切分组都在这份
|
||||
* 整数组上做客户端切片——分页参数在这里真生效(响应体行数 ≤ pageSize),{@code total_dup}
|
||||
* 仍是<b>该范围内未分页的总数</b>(KPI 与分页控件读它,不是窗口长度)。
|
||||
*
|
||||
* <p>{@code dupGroup} 非空时行集先按该分组裁剪(组内 ≥2 店才算撞款)。判据与
|
||||
* {@link #groupStats} 同源(成员键=可见店铺的 groupName 拆分),而不是前端再算一份:
|
||||
* 原先页面用 {@code dupWithinGroup} 自己裁了一遍,两套规则同名不同实现,分页一上就会分叉。
|
||||
*
|
||||
* <p>排序在切窗<b>之前</b>:行窗口必须稳定,否则同一页每次刷新拿到的是不同行。
|
||||
* ASIN 是 {@code [A-Z0-9]},这里的字符串序与页面原先的 localeCompare 同序。
|
||||
*
|
||||
* <p>蓝图 5A 那句「限制列数」落在<b>渲染侧</b>(后管 {@code duplicate-matrix-window.ts} 的
|
||||
* 列窗 40),服务端<b>不设</b>列窗,别把它当成没做完:矩阵的列本来就是从这份<b>已分页的行</b>
|
||||
* 里数出来的({@code storeColumnsOf(rows)}),而 {@code shops} 是筛选项字典+KPI 的原料、
|
||||
* 每行只有 5 个标量字段(见 {@code DuplicateShop})。给 {@code shops} 再加一层分页,只会把
|
||||
* 「全局指标读全集」和「列窗读本页」两套语义搅成一处——那才是 B60 要防的分叉。
|
||||
*/
|
||||
public Map<String, Object> consoleData(DuplicateScanView view, Set<String> visibleKeys, Filters filters,
|
||||
int page, int pageSize, String dupGroup) {
|
||||
List<DuplicateShop> shops = cropShops(view.payload().shops(), visibleKeys);
|
||||
List<DuplicateItem> cropped = cropItems(view.payload().items(), visibleKeys);
|
||||
String source = view.summary() == null ? "" : safe(view.summary().source());
|
||||
DuplicateScanSummary summary = recomputeSummary(cropped, shops, source);
|
||||
List<DuplicateItem> dup = filterItems(cropped, true, filters);
|
||||
List<DuplicateItem> rows = dupRowsInScope(shops, dupGroup, filterItems(cropped, true, filters));
|
||||
rows.sort(Comparator.comparing((DuplicateItem item) -> safe(item.asin())));
|
||||
List<Map<String, Object>> groups = groupStats(shops, filterItems(cropped, false, filters));
|
||||
int total = rows.size();
|
||||
int from = Math.min(Math.max(0, page - 1) * pageSize, total);
|
||||
int to = Math.min(from + pageSize, total);
|
||||
List<DuplicateItem> window = rows.isEmpty() ? List.of() : new ArrayList<>(rows.subList(from, to));
|
||||
return mapOf("pending", false, "scanned_at", safe(view.scannedAt()), "summary", summary,
|
||||
"shops", shops, "dup", dup, "groups", groups, "total_dup", dup.size());
|
||||
"shops", shops, "dup", window, "groups", groups, "total_dup", total,
|
||||
"page", page, "page_size", pageSize);
|
||||
}
|
||||
|
||||
public Map<String, Object> pendingConsole() {
|
||||
/** 分组范围内的撞款行集:按成员店铺裁剪后仍是 ≥2 店的才算撞款。 */
|
||||
private List<DuplicateItem> dupRowsInScope(List<DuplicateShop> visibleShops, String dupGroup,
|
||||
List<DuplicateItem> matched) {
|
||||
String group = safe(dupGroup).trim();
|
||||
if (group.isEmpty()) {
|
||||
return matched;
|
||||
}
|
||||
Set<String> memberKeys = new LinkedHashSet<>();
|
||||
for (DuplicateShop shop : visibleShops) {
|
||||
for (String g : splitGroups(safe(shop.groupName()))) {
|
||||
if (g.equals(group)) {
|
||||
memberKeys.add(shopKey(shop.shopName()));
|
||||
}
|
||||
}
|
||||
}
|
||||
List<DuplicateItem> rows = new ArrayList<>();
|
||||
for (DuplicateItem item : cropItems(matched, memberKeys)) {
|
||||
if (item.shopCount() >= 2) {
|
||||
rows.add(item);
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
public Map<String, Object> pendingConsole(int page, int pageSize) {
|
||||
return mapOf("pending", true, "scanned_at", "", "summary", Collections.emptyMap(),
|
||||
"shops", Collections.emptyList(), "dup", Collections.emptyList(),
|
||||
"groups", Collections.emptyList(), "total_dup", 0);
|
||||
"groups", Collections.emptyList(), "total_dup", 0,
|
||||
"page", page, "page_size", pageSize);
|
||||
}
|
||||
|
||||
/** 单条台账行:品牌取首个非空;店铺/分组/站点保持出现顺序去重;最早/最近上架取归一化时间。 */
|
||||
|
||||
+13
@@ -5,6 +5,9 @@ import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface TaskFileJobMapper extends BaseMapper<TaskFileJobEntity> {
|
||||
|
||||
@@ -14,4 +17,14 @@ public interface TaskFileJobMapper extends BaseMapper<TaskFileJobEntity> {
|
||||
* 语句在 {@code mapper/TaskFileJobMapper.xml}(本仓约定:自定义 SQL 一律进 XML)。
|
||||
*/
|
||||
String selectFileTaskStatus(@Param("taskId") Long taskId);
|
||||
|
||||
/**
|
||||
* {@link #selectFileTaskStatus} 的批量档(P1-B6):一次问回「状态落在给定终态值域里」的任务 id。
|
||||
*
|
||||
* <p>终态值域由调用方传(正本仍是 {@code TaskStatuses.FileTask.TERMINAL})——不在 XML 里
|
||||
* 再抄一份字符串,否则改值域时两边会悄悄分叉。查不到行的 id 不出现在结果里=非终态,
|
||||
* 与单条档「品牌任务的 job 返回 null」的语义逐字对齐。
|
||||
*/
|
||||
List<Long> selectTerminalFileTaskIds(@Param("taskIds") Collection<Long> taskIds,
|
||||
@Param("terminalStatuses") Collection<String> terminalStatuses);
|
||||
}
|
||||
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.task.TaskStatuses;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* {@code biz_task_file_job} / {@code biz_file_task} 的<b>批量读</b>(P1-B5/P1-B6):
|
||||
* 一批候选进来,一次往返出 id 集。
|
||||
*
|
||||
* <p>为什么从 {@link TaskFileJobService} 里单立一支:那支服务已经挂在可读性基线的超长文件
|
||||
* 名单上(只许减),而这几条是纯查询——不写状态、不发派发事件、不碰本机在飞表,与服务侧的
|
||||
* 写路径没有任何共享可变状态,所以它是<b>静态工具</b>而不是又一支注入进来的 Bean:
|
||||
* {@code TaskFileJobService} 的构造器已经是别处十几支用例的直接依赖,多加一个参数只会把
|
||||
* 无关用例一起改红。
|
||||
*
|
||||
* <p>把「批量预取」与「逐行 CAS」分成两件事也是有意的:认领的并发正确性只来自逐行 CAS
|
||||
* ({@code markRunning}),批量预取只用来跳过已终态/已在补检的行;两者混进一个方法,
|
||||
* 那个前提就再也读不出来了。
|
||||
*/
|
||||
public final class TaskFileJobBatchQueries {
|
||||
|
||||
private TaskFileJobBatchQueries() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 这批任务里「还在补检(名下有未终态的组装作业)」的 id(P1-B5)。
|
||||
*
|
||||
* <p>交付缺口触发的补检轮会把作业排回 PENDING/RUNNING({@link TaskFileJobService#requeue}),
|
||||
* 所以"有活着的组装作业"="这单还在补检"。而任务行可能早就落成 FAILED(心跳超时或交付判死先落),
|
||||
* 此时按任务状态去克隆重跑=同一份活开两份:现网 2026-10-24 一枚 760 行的外观专利因此被克隆成
|
||||
* 4 枚、每枚都全量重跑一遍,四枚卡片的开始时间还都是同一秒(克隆件继承原件 created_at)。
|
||||
* 调用方 {@code TaskResumeService} 就是拿这个集合跳过那一类的。
|
||||
*
|
||||
* <p>原先这是逐条 {@code selectCount}(N 个候选=N 次往返,P1-B5 点名的那处 N+1);巡检按几百个
|
||||
* 候选跑,所以折成一次 {@code selectList} 取 id 列——判据一字未动(同一 jobType、同一两态)。
|
||||
*/
|
||||
public static Set<Long> taskIdsWithUnfinishedAssembleJob(TaskFileJobMapper mapper, Collection<Long> taskIds) {
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
List<TaskFileJobEntity> rows = mapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.select(TaskFileJobEntity::getTaskId)
|
||||
.in(TaskFileJobEntity::getTaskId, taskIds)
|
||||
.eq(TaskFileJobEntity::getJobType, TaskFileJobService.JOB_TYPE_ASSEMBLE_RESULT)
|
||||
.in(TaskFileJobEntity::getStatus, TaskStatuses.Job.PENDING, TaskStatuses.Job.RUNNING));
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
Set<Long> ids = new LinkedHashSet<>();
|
||||
for (TaskFileJobEntity row : rows) {
|
||||
if (row != null && row.getTaskId() != null) {
|
||||
ids.add(row.getTaskId());
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
/**
|
||||
* 这批候选作业里「父任务已终态」的任务 id(P1-B6):先折出候选的 taskId,再一次批量读。
|
||||
*
|
||||
* <p>终态值域仍只有 {@code TaskStatuses.FileTask.TERMINAL} 一处正本——它作为参数进 SQL,
|
||||
* 不在 XML 里另抄一份字符串(抄了就又是第二份词汇表)。单条档判据见
|
||||
* {@link TaskFileJobService#isOwningTaskTerminal(Long)},两边同源。
|
||||
*/
|
||||
public static Set<Long> terminalOwningTaskIds(TaskFileJobMapper mapper, List<TaskFileJobEntity> candidates) {
|
||||
Set<Long> taskIds = new LinkedHashSet<>();
|
||||
if (candidates != null) {
|
||||
for (TaskFileJobEntity candidate : candidates) {
|
||||
if (candidate != null && candidate.getTaskId() != null && candidate.getTaskId() > 0) {
|
||||
taskIds.add(candidate.getTaskId());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (taskIds.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
List<Long> terminal = mapper.selectTerminalFileTaskIds(taskIds, TaskStatuses.FileTask.TERMINAL);
|
||||
return terminal == null || terminal.isEmpty() ? Set.of() : new LinkedHashSet<>(terminal);
|
||||
}
|
||||
}
|
||||
+10
-16
@@ -16,6 +16,7 @@ import org.springframework.transaction.annotation.Transactional;
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -143,22 +144,12 @@ public class TaskFileJobService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 该任务名下还有没有**未终态**的组装作业(PENDING/RUNNING)。
|
||||
*
|
||||
* <p>交付缺口触发的补检轮就是把作业排回这两态({@link #requeue}),所以"有活着的组装作业"
|
||||
* ="这单还在补检"。而任务行可能早就落成 FAILED(心跳超时或交付判死先落),此时按任务状态
|
||||
* 去克隆重跑,等于同一份活开两份:现网 2026-10-24 一枚 760 行的外观专利因此被克隆成 4 枚、
|
||||
* 每枚都全量重跑一遍,四枚卡片的开始时间还都是同一秒(克隆件继承原件 created_at)。
|
||||
* 这批任务里「还在补检」的 id(P1-B5 的批量档):判据与来由都在
|
||||
* {@link TaskFileJobBatchQueries#taskIdsWithUnfinishedAssembleJob},这里只是服务侧的门面——
|
||||
* 调用方({@code TaskResumeService})认的是这支服务,不该为了读侧再注入一件东西。
|
||||
*/
|
||||
public boolean hasUnfinishedAssembleJob(Long taskId) {
|
||||
if (taskId == null) {
|
||||
return false;
|
||||
}
|
||||
Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getTaskId, taskId)
|
||||
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
|
||||
.in(TaskFileJobEntity::getStatus, TaskStatuses.Job.PENDING, TaskStatuses.Job.RUNNING));
|
||||
return count != null && count > 0;
|
||||
public Set<Long> taskIdsWithUnfinishedAssembleJob(Collection<Long> taskIds) {
|
||||
return TaskFileJobBatchQueries.taskIdsWithUnfinishedAssembleJob(taskFileJobMapper, taskIds);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -296,12 +287,15 @@ public class TaskFileJobService {
|
||||
if (candidates == null || candidates.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
// P1-B6:父任务状态原本逐条点查(一批 100 个候选=100 次 SELECT),折成一次批量读;
|
||||
// 逐行 CAS 认领不动——那才是并发不重复派发的根据。
|
||||
Set<Long> terminalTaskIds = TaskFileJobBatchQueries.terminalOwningTaskIds(taskFileJobMapper, candidates);
|
||||
List<Long> claimedIds = new ArrayList<>();
|
||||
for (TaskFileJobEntity candidate : candidates) {
|
||||
if (candidate == null || candidate.getId() == null) {
|
||||
continue;
|
||||
}
|
||||
if (isOwningTaskTerminal(candidate.getTaskId())) {
|
||||
if (terminalTaskIds.contains(candidate.getTaskId())) {
|
||||
noteTerminalTaskStillDeliverable(candidate);
|
||||
}
|
||||
if (markRunning(candidate.getId())) {
|
||||
|
||||
+34
-19
@@ -35,6 +35,8 @@ public class TaskResultItemService {
|
||||
*/
|
||||
private final ExecutorService snapshotReadPool =
|
||||
com.nanri.aiimage.common.support.ThreadPools.boundedFixed("task-result-read", 8);
|
||||
/** 指针化载荷并发读的窗口:见 {@link #listResultSnapshots} 的 P1-B7 说明。 */
|
||||
static final int SNAPSHOT_READ_WINDOW = com.nanri.aiimage.common.support.ThreadPools.DEFAULT_QUEUE_CAPACITY / 2;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||
|
||||
@@ -95,7 +97,11 @@ public class TaskResultItemService {
|
||||
* <p>刻意不加 limit:调用方要用它组装完整的结果文件,截断即漏行(正确性错误,
|
||||
* 不是性能取舍)。调用方多为「逐任务循环调用」,因此峰值内存 = 单个任务的结果集,
|
||||
* 这是本操作固有的代价。若要再降,需要改成「边读边写文件」的流式组装契约,
|
||||
* 涉及 4 个模块的组装实现,属独立工程。
|
||||
* 涉及 4 个模块的组装实现(patroldelete/queryasin/shopdatacrawl/withdraw),属独立工程。
|
||||
*
|
||||
* <p>P1-B7 在本方法里能收的是**另一件事**:整份结果集不需要被读两遍。指针化载荷那条分支
|
||||
* 按窗口提交+收割(见下方 P1-B7 注释),副本只驻留一窗;返回仍是整份 {@code List},
|
||||
* 「流式组装契约」那半没做,别把本段读成 P1-B7 已闭环。
|
||||
*/
|
||||
public <T> List<T> listResultSnapshots(Long taskId, String moduleType, Class<T> clazz) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||
@@ -129,26 +135,35 @@ public class TaskResultItemService {
|
||||
T value = readPayload(rows.get(0), clazz);
|
||||
return value == null ? List.of() : List.of(value);
|
||||
}
|
||||
List<Future<T>> futures = new ArrayList<>(rows.size());
|
||||
for (TaskResultItemEntity row : rows) {
|
||||
futures.add(snapshotReadPool.submit(() -> readPayload(row, clazz)));
|
||||
}
|
||||
// P1-B7(2026-11-03):**分窗提交并收割**,不再把整份结果集一次性交进线程池。
|
||||
// 一次性提交时,每支 Future 完成后仍把解好的载荷扣在自己身上(FutureTask.outcome 不清),
|
||||
// 而这一批 Future 又要等最后一个才释放 ⇒ 峰值=「结果集」+「同样多的另一份副本」。
|
||||
// 一窗一收之后,窗内的副本收完就可回收,峰值退化成「结果集」+「一窗」。
|
||||
// 窗口取队列容量的一半:8 线程+512 队列(ThreadPools.DEFAULT_QUEUE_CAPACITY)在窗内永不排满,
|
||||
// 也就不会落到 CallerRunsPolicy 上把请求线程变成同步读对象存储的那个人。
|
||||
List<T> out = new ArrayList<>(rows.size());
|
||||
for (Future<T> future : futures) {
|
||||
try {
|
||||
T value = future.get();
|
||||
if (value != null) {
|
||||
out.add(value);
|
||||
for (int from = 0; from < rows.size(); from += SNAPSHOT_READ_WINDOW) {
|
||||
int to = Math.min(from + SNAPSHOT_READ_WINDOW, rows.size());
|
||||
List<Future<T>> window = new ArrayList<>(to - from);
|
||||
for (TaskResultItemEntity row : rows.subList(from, to)) {
|
||||
window.add(snapshotReadPool.submit(() -> readPayload(row, clazz)));
|
||||
}
|
||||
for (Future<T> future : window) {
|
||||
try {
|
||||
T value = future.get();
|
||||
if (value != null) {
|
||||
out.add(value);
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new BusinessException("读取任务结果明细被中断");
|
||||
} catch (ExecutionException ex) {
|
||||
Throwable cause = ex.getCause();
|
||||
if (cause instanceof BusinessException businessException) {
|
||||
throw businessException;
|
||||
}
|
||||
throw new BusinessException("读取任务结果明细失败", cause);
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new BusinessException("读取任务结果明细被中断");
|
||||
} catch (ExecutionException ex) {
|
||||
Throwable cause = ex.getCause();
|
||||
if (cause instanceof BusinessException businessException) {
|
||||
throw businessException;
|
||||
}
|
||||
throw new BusinessException("读取任务结果明细失败", cause);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
|
||||
+28
-3
@@ -10,6 +10,8 @@ import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
@@ -95,10 +97,33 @@ public class TaskResultPayloadService {
|
||||
return;
|
||||
}
|
||||
TaskResultPayloadEntity existing = findLatest(taskId, moduleType, hash(scopeKey.trim()));
|
||||
if (existing != null) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(existing.getPayloadJson());
|
||||
taskResultPayloadMapper.deleteById(existing.getId());
|
||||
if (existing == null) {
|
||||
return;
|
||||
}
|
||||
// P1-B8:先落库、提交之后再删对象存储。原先的顺序(存储删完再 deleteById)把一次
|
||||
// 网络调用(超时 120s、重试 3 次)连同它的引用计数反查一起塞进 DB 事务:连接被长时间
|
||||
// 占住,且回滚时对象已经没了、行还在=读得到一个不存在的载荷。同一批结果明细行
|
||||
// (TaskResultItemService#deletePayloadsAfterCommit,2026-09)已经改过这个形状,
|
||||
// 这里补齐漏掉的这一处;提交后删还有一层好处——引用反查看到的是行已删除的 DB 状态。
|
||||
taskResultPayloadMapper.deleteById(existing.getId());
|
||||
deletePayloadAfterCommit(existing.getPayloadJson());
|
||||
}
|
||||
|
||||
/** 事务提交后再删对象存储上的载荷;无事务上下文时直接删(与 TaskResultItemService 同口径)。 */
|
||||
private void deletePayloadAfterCommit(String payloadValue) {
|
||||
if (payloadValue == null || payloadValue.isBlank()) {
|
||||
return;
|
||||
}
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||
@Override
|
||||
public void afterCommit() {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(payloadValue);
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
transientPayloadStorageService.deletePayloadIfPresent(payloadValue);
|
||||
}
|
||||
|
||||
private TaskResultPayloadEntity findLatest(Long taskId, String moduleType, String scopeHash) {
|
||||
|
||||
+40
-9
@@ -13,6 +13,7 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
@@ -67,8 +68,8 @@ public class TaskResumeService {
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
/**
|
||||
* 查"这单名下还有没有活着的组装作业"——补检轮还活着的意思,见
|
||||
* {@link TaskFileJobService#hasUnfinishedAssembleJob(Long)}。
|
||||
* 查"这批单名下还有没有活着的组装作业"——补检轮还活着的意思,见
|
||||
* {@link TaskFileJobService#taskIdsWithUnfinishedAssembleJob(Collection)}。
|
||||
*/
|
||||
private final TaskFileJobService taskFileJobs;
|
||||
/** moduleType → 克隆重试实现;只有注册了本 SPI 的模块才会自动重试(SPI 现在只作「支持重试」白名单用)。 */
|
||||
@@ -201,6 +202,18 @@ public class TaskResumeService {
|
||||
return stats;
|
||||
}
|
||||
stats.scannedTaskCount = candidates.size();
|
||||
// P1-B5:下面两条判定原先逐行各问一次库(500 个候选=1,000 次往返,且
|
||||
// `resume_of_task_id` 在 V187 之前无索引=每次还是一场全表扫)。折成两次批量查,
|
||||
// 循环里只做集合判定。原地复位不新增子任务行(2026-11-01 起),所以预取的集合
|
||||
// 不会在本轮内被自己写脏。
|
||||
Set<Long> candidateIds = new LinkedHashSet<>();
|
||||
for (FileTaskEntity candidate : candidates) {
|
||||
if (candidate.getId() != null) {
|
||||
candidateIds.add(candidate.getId());
|
||||
}
|
||||
}
|
||||
Set<Long> parentsWithChild = parentsWithRetryChild(candidateIds);
|
||||
Set<Long> recheckingTaskIds = taskFileJobs.taskIdsWithUnfinishedAssembleJob(candidateIds);
|
||||
for (FileTaskEntity original : candidates) {
|
||||
if (original.getUserId() == null || original.getUserId() <= 0) {
|
||||
stats.skippedTaskCount++;
|
||||
@@ -226,7 +239,7 @@ public class TaskResumeService {
|
||||
abbreviate(original.getErrorMessage()));
|
||||
continue;
|
||||
}
|
||||
if (hasRetryChild(original.getId())) {
|
||||
if (parentsWithChild.contains(original.getId())) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
@@ -234,7 +247,7 @@ public class TaskResumeService {
|
||||
// (见 AppearancePatentDeliverySupport.requestRecheckForMissingRows),此时克隆=同一份活
|
||||
// 开两份、还各按全量重跑一遍。作业真卡死另有 TaskResultFileJobWorker 的 stuck 扫描兜底,
|
||||
// 不会因为这条例外就永远没人管。
|
||||
if (taskFileJobs.hasUnfinishedAssembleJob(original.getId())) {
|
||||
if (recheckingTaskIds.contains(original.getId())) {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[task-resume] 不重试 taskId={} moduleType={} 原因=组装作业仍在补检(PENDING/RUNNING)",
|
||||
original.getId(), original.getModuleType());
|
||||
@@ -305,11 +318,29 @@ public class TaskResumeService {
|
||||
}
|
||||
}
|
||||
|
||||
/** 该原任务是否已经有重试任务(反查 resume_of_task_id)。 */
|
||||
private boolean hasRetryChild(Long originalTaskId) {
|
||||
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getResumeOfTaskId, originalTaskId));
|
||||
return count != null && count > 0;
|
||||
/**
|
||||
* 这批原任务里哪些已经有重试子任务(反查 {@code resume_of_task_id})。
|
||||
*
|
||||
* <p>P1-B5:判据与原来的单条档逐字相同(同一列、同一等值),只是把 N 次点查折成一次
|
||||
* {@code IN} 批查——那列在 V187 之前没有索引,巡检一轮 500 个候选就是 500 场全表扫。
|
||||
*/
|
||||
private Set<Long> parentsWithRetryChild(Collection<Long> originalTaskIds) {
|
||||
if (originalTaskIds == null || originalTaskIds.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
List<FileTaskEntity> children = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getResumeOfTaskId)
|
||||
.in(FileTaskEntity::getResumeOfTaskId, originalTaskIds));
|
||||
if (children == null || children.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
Set<Long> parents = new LinkedHashSet<>();
|
||||
for (FileTaskEntity child : children) {
|
||||
if (child != null && child.getResumeOfTaskId() != null) {
|
||||
parents.add(child.getResumeOfTaskId());
|
||||
}
|
||||
}
|
||||
return parents;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user