perf(代码审查落地): 后端 A1-A8 与前端 C1-C8 加固,retention 清理补索引(V141)
A 组(backend-java): - A1 载荷引用计数不再无条件回落全表 COUNT - A2 结果行读取由 LIMIT 1 改整页 - A3 ziniao 索引缓存过期判断下推 SQL - A4 循环子任务认领改 CAS,失败一律抛 - A5 建单事务里不碰外部 IO - A6 连接池补网络层超时(connectTimeout/socketTimeout,env 可覆盖) - A7 日志留现场 + 异步落盘 - A8 其余:货源图片预取、结果工作簿装配等 - V141:三条 retention DELETE 的可用索引,其中 biz_price_track_loop_run 用 STORED 生成列承载 COALESCE 时间线(该表实测 881 行/17MB,COPY 秒级) C 组(frontend-vue):URL 取值走 safe-url、行号取值走 row-id、 交付视频任务状态抽成 delivery-video-task-state.ts,各补用例。
This commit is contained in:
+4
-2
@@ -15,12 +15,14 @@ public interface PriceTrackLoopRunMapper extends BaseMapper<PriceTrackLoopRunEnt
|
||||
* 分批删除已结束且超过保留期的循环批次行。
|
||||
*
|
||||
* <p>只删终态行:RUNNING 的批次删了会让后续 dispatch / childFinished 找不到记录。
|
||||
* 时间线用 COALESCE 兜底——终态行本应写 finished_at,历史行可能只有 updated_at。
|
||||
* 时间线用 V141 的 STORED 生成列 {@code cleanup_at}(= COALESCE(finished_at, updated_at,
|
||||
* created_at),口径与改造前完全一致):原来这个表达式直接写在 WHERE 里不可走索引,
|
||||
* 每条 DELETE 都要全表扫描并对扫过的行加锁,而 (status, cleanup_at) 索引已就位。
|
||||
*/
|
||||
@Delete("""
|
||||
DELETE FROM biz_price_track_loop_run
|
||||
WHERE status IN ('SUCCESS', 'FAILED', 'STOPPED')
|
||||
AND COALESCE(finished_at, updated_at, created_at) < #{cutoff}
|
||||
AND cleanup_at < #{cutoff}
|
||||
LIMIT #{batchSize}
|
||||
""")
|
||||
int deleteFinishedBefore(@Param("cutoff") LocalDateTime cutoff, @Param("batchSize") int batchSize);
|
||||
|
||||
+25
-8
@@ -169,6 +169,14 @@ public class PriceTrackLoopRunService {
|
||||
return toVo(entity);
|
||||
}
|
||||
|
||||
/**
|
||||
* 把刚创建的子任务认领为循环任务的当前子任务。
|
||||
*
|
||||
* <p>认领失败一律抛异常而不是静默返回:调用方 {@code PriceTrackTaskService.createTask}
|
||||
* 与本方法同处一个事务(默认传播 REQUIRED),抛出会让刚 insert 的子任务连同扣次一起回滚。
|
||||
* 原先各处判不到就 {@code return},子任务照样建成并真实执行——同一个店铺槽位跑出两个
|
||||
* RUNNING 子任务(重复采集、重复扣用户次数),且第一个失去跟踪。
|
||||
*/
|
||||
@Transactional
|
||||
public void bindChildTask(Long loopRunId, Long childTaskId, Integer roundIndex, Integer shopIndex) {
|
||||
if (loopRunId == null || childTaskId == null) {
|
||||
@@ -176,26 +184,35 @@ public class PriceTrackLoopRunService {
|
||||
}
|
||||
PriceTrackLoopRunEntity entity = loopRunMapper.selectById(loopRunId);
|
||||
if (entity == null || isTerminal(entity.getStatus())) {
|
||||
return;
|
||||
throw new BusinessException("循环任务已结束或不存在,子任务未绑定");
|
||||
}
|
||||
if (entity.getCurrentRound() == null || entity.getCurrentShopIndex() == null) {
|
||||
return;
|
||||
throw new BusinessException("循环任务轮次状态缺失,子任务未绑定");
|
||||
}
|
||||
if (!entity.getCurrentRound().equals(roundIndex) || !entity.getCurrentShopIndex().equals(shopIndex)) {
|
||||
return;
|
||||
throw new BusinessException("循环任务轮次或店铺指针已推进,本次派发已作废");
|
||||
}
|
||||
if (entity.getActiveTaskId() != null && !isChildTaskTerminal(entity.getActiveTaskId())) {
|
||||
return;
|
||||
throw new BusinessException("当前仍有执行中的子任务,本次派发已作废");
|
||||
}
|
||||
entity.setActiveTaskId(childTaskId);
|
||||
entity.setUpdatedAt(LocalDateTime.now());
|
||||
// 字段级更新 + status CAS:整行 updateById 会覆盖并发写入的 stop_requested,
|
||||
// 导致「停止循环」请求被静默丢弃
|
||||
loopRunMapper.update(null, new LambdaUpdateWrapper<PriceTrackLoopRunEntity>()
|
||||
// 认领槽位必须连带 CAS 掉读取时的 active_task_id:只比 status 时两个并发派发都能通过
|
||||
// (status 没变),后写的那个把先绑的子任务顶掉,两个 RUNNING 子任务同时真实执行。
|
||||
Long expectedActiveTaskId = entity.getActiveTaskId();
|
||||
LambdaUpdateWrapper<PriceTrackLoopRunEntity> claim = new LambdaUpdateWrapper<PriceTrackLoopRunEntity>()
|
||||
.eq(PriceTrackLoopRunEntity::getId, entity.getId())
|
||||
.eq(PriceTrackLoopRunEntity::getStatus, entity.getStatus())
|
||||
.set(PriceTrackLoopRunEntity::getActiveTaskId, childTaskId)
|
||||
.set(PriceTrackLoopRunEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
.set(PriceTrackLoopRunEntity::getUpdatedAt, LocalDateTime.now());
|
||||
if (expectedActiveTaskId == null) {
|
||||
claim.isNull(PriceTrackLoopRunEntity::getActiveTaskId);
|
||||
} else {
|
||||
claim.eq(PriceTrackLoopRunEntity::getActiveTaskId, expectedActiveTaskId);
|
||||
}
|
||||
if (loopRunMapper.update(null, claim) == 0) {
|
||||
throw new BusinessException("循环任务槽位已被并发派发认领,本次派发已作废");
|
||||
}
|
||||
log.info("[price-track-loop] child bound loopRunId={} childTaskId={} roundIndex={} shopIndex={}",
|
||||
loopRunId, childTaskId, roundIndex, shopIndex);
|
||||
}
|
||||
|
||||
+99
-82
@@ -460,7 +460,6 @@ public class PriceTrackTaskService {
|
||||
return result;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public PriceTrackCreateTaskVo createTask(PriceTrackCreateTaskRequest request, String deviceId) {
|
||||
if (request.isStatusMode() == request.isAsinMode()) throw new BusinessException("跟价模式必须二选一");
|
||||
if (request.isAsinMode() && (request.getAsinFiles() == null || request.getAsinFiles().isEmpty())) {
|
||||
@@ -490,91 +489,98 @@ public class PriceTrackTaskService {
|
||||
log.info("[price-track] createTask skipAsins countries={} asinMode={}",
|
||||
skipAsinsByCountry.keySet(), request.isAsinMode());
|
||||
|
||||
// 创建任务记录
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType(MODULE_TYPE);
|
||||
task.setTaskMode("PYTHON_QUEUE");
|
||||
task.setStatus("RUNNING");
|
||||
task.setSourceFileCount(uniqueItems.size());
|
||||
task.setSuccessFileCount(0);
|
||||
task.setFailedFileCount(0);
|
||||
task.setCreatedBy("user:" + request.getUserId());
|
||||
task.setUserId(request.getUserId());
|
||||
task.setDeviceId(AdminAuthSupport.normalizeDeviceId(deviceId));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.insert(task);
|
||||
priceTrackTaskCacheService.touchTaskHeartbeat(task.getId());
|
||||
if (request.isAsinMode()) {
|
||||
saveTaskAsinRowsPayload(task.getId(), asinRowsForPayload);
|
||||
}
|
||||
if (request.getLoopRunId() != null) {
|
||||
priceTrackLoopRunService.bindChildTask(request.getLoopRunId(), task.getId(), request.getRoundIndex(), request.getShopIndex());
|
||||
}
|
||||
|
||||
List<PriceTrackResultItemVo> snapshot = new ArrayList<>();
|
||||
// 逐店铺 fileResultMapper.insert 看似该改批量(500 店铺 = 500 次往返),但**不能改**:
|
||||
// 下面的 toSnapshotVo 依赖 MyBatis-Plus insert 回填的自增主键(fr.getId() → vo.resultId),
|
||||
// 而本仓的手写 INSERT ... VALUES (...),(...) 批量写法不回填主键,改了会让快照 resultId 为 null。
|
||||
// 要批量化必须同时解决「批量插入 + 按序回填 id」(MySQL 需 useGeneratedKeys + 顺序保证)。
|
||||
for (PriceTrackMatchShopsVo.PriceTrackShopQueueItem item : uniqueItems) {
|
||||
String norm = ziniaoShopSwitchService.normalizeShopName(item.getShopName());
|
||||
if (norm.isBlank()) throw new BusinessException("店铺名称无效");
|
||||
FileResultEntity fr = new FileResultEntity();
|
||||
fr.setTaskId(task.getId());
|
||||
fr.setModuleType(MODULE_TYPE);
|
||||
fr.setSourceFilename(norm);
|
||||
fr.setSourceFileUrl(Objects.toString(item.getShopId(), null));
|
||||
fr.setUserId(request.getUserId());
|
||||
fr.setSuccess(0);
|
||||
fr.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(fr);
|
||||
PriceTrackResultItemVo snapshotItem = toSnapshotVo(fr, item, task.getStatus());
|
||||
snapshotItem.setLoopRunId(request.getLoopRunId());
|
||||
snapshotItem.setRoundIndex(request.getRoundIndex());
|
||||
snapshot.add(snapshotItem);
|
||||
}
|
||||
|
||||
// 保存请求上下文快照,供 Python 消费和前端轮询展示使用
|
||||
try {
|
||||
Map<String, Object> ctx = new LinkedHashMap<>();
|
||||
ctx.put("userId", request.getUserId());
|
||||
ctx.put("statusMode", request.isStatusMode());
|
||||
ctx.put("asinMode", request.isAsinMode());
|
||||
ctx.put("countryCodes", request.getCountryCodes());
|
||||
// 不再保存全量 ASIN 数据到 requestJson
|
||||
// ctx.put("skipAsinsByCountry", skipAsinsByCountry);
|
||||
// ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry);
|
||||
// ctx.put("asinRowsByCountry", asinRowsByCountry);
|
||||
// ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin);
|
||||
// 文件模式:保存文件路径,供后续分页读取
|
||||
// Excel/CSV 解析(POI 全表读,可达秒级)刻意留在事务之外:事务里只做 DB 写与载荷落库,
|
||||
// 避免创建跟价任务时长时间占住连接池连接(本仓历史上 biz_task_file_job 一类锁等
|
||||
// 就是被「事务内含外部 IO」放大的)。载荷 PUT 仍留在事务内:它失败必须整体回滚,
|
||||
// 不能让「任务已建、ASIN 载荷为空」的半成品进入 Python 队列。
|
||||
final Map<String, List<Map<String, String>>> parsedAsinRows = asinRowsForPayload;
|
||||
return inRequiredTransaction(() -> {
|
||||
// 创建任务记录
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType(MODULE_TYPE);
|
||||
task.setTaskMode("PYTHON_QUEUE");
|
||||
task.setStatus("RUNNING");
|
||||
task.setSourceFileCount(uniqueItems.size());
|
||||
task.setSuccessFileCount(0);
|
||||
task.setFailedFileCount(0);
|
||||
task.setCreatedBy("user:" + request.getUserId());
|
||||
task.setUserId(request.getUserId());
|
||||
task.setDeviceId(AdminAuthSupport.normalizeDeviceId(deviceId));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.insert(task);
|
||||
priceTrackTaskCacheService.touchTaskHeartbeat(task.getId());
|
||||
if (request.isAsinMode()) {
|
||||
ctx.put("asinFiles", request.getAsinFiles());
|
||||
saveTaskAsinRowsPayload(task.getId(), parsedAsinRows);
|
||||
}
|
||||
if (request.getLoopRunId() != null) {
|
||||
priceTrackLoopRunService.bindChildTask(request.getLoopRunId(), task.getId(), request.getRoundIndex(), request.getShopIndex());
|
||||
}
|
||||
ctx.put("items", uniqueItems);
|
||||
ctx.put("loopRunId", request.getLoopRunId());
|
||||
ctx.put("roundIndex", request.getRoundIndex());
|
||||
ctx.put("shopIndex", request.getShopIndex());
|
||||
task.setRequestJson(objectMapper.writeValueAsString(ctx));
|
||||
task.setResultJson(objectMapper.writeValueAsString(snapshot));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("序列化任务数据失败");
|
||||
}
|
||||
fileTaskMapper.updateById(task);
|
||||
priceTrackTaskCacheService.saveTaskCache(task);
|
||||
|
||||
// TODO: Python 侧轮询 RUNNING 任务,pickup 后开始处理
|
||||
// Java 侧通过 submitResult 接口接收 Python 回传结果
|
||||
List<PriceTrackResultItemVo> snapshot = new ArrayList<>();
|
||||
// 逐店铺 fileResultMapper.insert 看似该改批量(500 店铺 = 500 次往返),但**不能改**:
|
||||
// 下面的 toSnapshotVo 依赖 MyBatis-Plus insert 回填的自增主键(fr.getId() → vo.resultId),
|
||||
// 而本仓的手写 INSERT ... VALUES (...),(...) 批量写法不回填主键,改了会让快照 resultId 为 null。
|
||||
// 要批量化必须同时解决「批量插入 + 按序回填 id」(MySQL 需 useGeneratedKeys + 顺序保证)。
|
||||
for (PriceTrackMatchShopsVo.PriceTrackShopQueueItem item : uniqueItems) {
|
||||
String norm = ziniaoShopSwitchService.normalizeShopName(item.getShopName());
|
||||
if (norm.isBlank()) throw new BusinessException("店铺名称无效");
|
||||
FileResultEntity fr = new FileResultEntity();
|
||||
fr.setTaskId(task.getId());
|
||||
fr.setModuleType(MODULE_TYPE);
|
||||
fr.setSourceFilename(norm);
|
||||
fr.setSourceFileUrl(Objects.toString(item.getShopId(), null));
|
||||
fr.setUserId(request.getUserId());
|
||||
fr.setSuccess(0);
|
||||
fr.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(fr);
|
||||
PriceTrackResultItemVo snapshotItem = toSnapshotVo(fr, item, task.getStatus());
|
||||
snapshotItem.setLoopRunId(request.getLoopRunId());
|
||||
snapshotItem.setRoundIndex(request.getRoundIndex());
|
||||
snapshot.add(snapshotItem);
|
||||
}
|
||||
|
||||
PriceTrackCreateTaskVo vo = new PriceTrackCreateTaskVo();
|
||||
vo.setTaskId(task.getId());
|
||||
vo.setItems(snapshot);
|
||||
vo.setSkipAsinsByCountry(skipAsinsByCountry);
|
||||
vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry);
|
||||
vo.setAsinRowsByCountry(asinRowsByCountry);
|
||||
vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin);
|
||||
return vo;
|
||||
// 保存请求上下文快照,供 Python 消费和前端轮询展示使用
|
||||
try {
|
||||
Map<String, Object> ctx = new LinkedHashMap<>();
|
||||
ctx.put("userId", request.getUserId());
|
||||
ctx.put("statusMode", request.isStatusMode());
|
||||
ctx.put("asinMode", request.isAsinMode());
|
||||
ctx.put("countryCodes", request.getCountryCodes());
|
||||
// 不再保存全量 ASIN 数据到 requestJson
|
||||
// ctx.put("skipAsinsByCountry", skipAsinsByCountry);
|
||||
// ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry);
|
||||
// ctx.put("asinRowsByCountry", asinRowsByCountry);
|
||||
// ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin);
|
||||
// 文件模式:保存文件路径,供后续分页读取
|
||||
if (request.isAsinMode()) {
|
||||
ctx.put("asinFiles", request.getAsinFiles());
|
||||
}
|
||||
ctx.put("items", uniqueItems);
|
||||
ctx.put("loopRunId", request.getLoopRunId());
|
||||
ctx.put("roundIndex", request.getRoundIndex());
|
||||
ctx.put("shopIndex", request.getShopIndex());
|
||||
task.setRequestJson(objectMapper.writeValueAsString(ctx));
|
||||
task.setResultJson(objectMapper.writeValueAsString(snapshot));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("序列化任务数据失败");
|
||||
}
|
||||
fileTaskMapper.updateById(task);
|
||||
priceTrackTaskCacheService.saveTaskCache(task);
|
||||
|
||||
// TODO: Python 侧轮询 RUNNING 任务,pickup 后开始处理
|
||||
// Java 侧通过 submitResult 接口接收 Python 回传结果
|
||||
|
||||
PriceTrackCreateTaskVo vo = new PriceTrackCreateTaskVo();
|
||||
vo.setTaskId(task.getId());
|
||||
vo.setItems(snapshot);
|
||||
vo.setSkipAsinsByCountry(skipAsinsByCountry);
|
||||
vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry);
|
||||
vo.setAsinRowsByCountry(asinRowsByCountry);
|
||||
vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin);
|
||||
return vo;
|
||||
});
|
||||
}
|
||||
|
||||
public void submitResult(Long taskId, PriceTrackSubmitResultRequest request) {
|
||||
@@ -2263,6 +2269,17 @@ public class PriceTrackTaskService {
|
||||
return template.execute(status -> action.get());
|
||||
}
|
||||
|
||||
/**
|
||||
* 默认传播(REQUIRED)事务段:供「外部 IO 已移出、只剩 DB 写」的方法体使用。
|
||||
* 与 {@link #inNewTransaction} 的差别是不挂起外层事务;单测直接 new 本类时退化为直调。
|
||||
*/
|
||||
private <T> T inRequiredTransaction(java.util.function.Supplier<T> action) {
|
||||
if (transactionManager == null) {
|
||||
return action.get();
|
||||
}
|
||||
return new TransactionTemplate(transactionManager).execute(status -> action.get());
|
||||
}
|
||||
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId) {
|
||||
return TaskFinalizeSupport.acquireTaskLock(taskDistributedLockService, MODULE_TYPE, taskId);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user