perf(代码审查落地): 后端 A1-A8 与前端 C1-C8 加固,retention 清理补索引(V141)
A 组(backend-java): - A1 载荷引用计数不再无条件回落全表 COUNT - A2 结果行读取由 LIMIT 1 改整页 - A3 ziniao 索引缓存过期判断下推 SQL - A4 循环子任务认领改 CAS,失败一律抛 - A5 建单事务里不碰外部 IO - A6 连接池补网络层超时(connectTimeout/socketTimeout,env 可覆盖) - A7 日志留现场 + 异步落盘 - A8 其余:货源图片预取、结果工作簿装配等 - V141:三条 retention DELETE 的可用索引,其中 biz_price_track_loop_run 用 STORED 生成列承载 COALESCE 时间线(该表实测 881 行/17MB,COPY 秒级) C 组(frontend-vue):URL 取值走 safe-url、行号取值走 row-id、 交付视频任务状态抽成 delivery-video-task-state.ts,各补用例。
This commit is contained in:
+12
-5
@@ -121,6 +121,9 @@ public class AppearancePatentTaskService {
|
||||
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
||||
private static final int CHUNK_PAYLOAD_MERGE_RETRY_LIMIT = 3;
|
||||
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
|
||||
|
||||
/** 结果行读取时 chunk 的 keyset 页大小,与 similarasin 侧 loadChunksKeyset 的 500 同口径。 */
|
||||
private static final int RESULT_ROWS_CHUNK_PAGE_SIZE = 500;
|
||||
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
|
||||
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
|
||||
private static final long TASK_LOCK_WAIT_MILLIS = 10000L;
|
||||
@@ -2538,7 +2541,7 @@ public class AppearancePatentTaskService {
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.orderByAsc(TaskChunkEntity::getId)
|
||||
.last("LIMIT 1");
|
||||
.last("LIMIT " + RESULT_ROWS_CHUNK_PAGE_SIZE);
|
||||
if (lastChunkId != null) {
|
||||
query.gt(TaskChunkEntity::getId, lastChunkId);
|
||||
}
|
||||
@@ -2546,10 +2549,14 @@ public class AppearancePatentTaskService {
|
||||
if (page == null || page.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
TaskChunkEntity chunk = page.getFirst();
|
||||
appendRequestedResultRows(result, readChunkRows(chunk).values(), requestedKeys);
|
||||
lastChunkId = chunk.getId();
|
||||
if (lastChunkId == null) {
|
||||
// 整页取 chunk 后逐片解载荷:原先每轮 LIMIT 1 只取一行,一个结果文件要把该任务的
|
||||
// chunk 表逐行翻完,每行一次 SQL + 一次对象读 + 一次整片 JSON 反序列化
|
||||
for (TaskChunkEntity chunk : page) {
|
||||
appendRequestedResultRows(result, readChunkRows(chunk).values(), requestedKeys);
|
||||
}
|
||||
TaskChunkEntity tail = page.getLast();
|
||||
lastChunkId = tail == null ? null : tail.getId();
|
||||
if (lastChunkId == null || page.size() < RESULT_ROWS_CHUNK_PAGE_SIZE) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
+27
-15
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.ImageVideoProperties;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.imagevideo.mapper.ImageVideoAsyncTaskMapper;
|
||||
import com.nanri.aiimage.modules.imagevideo.model.entity.ImageVideoAsyncTaskEntity;
|
||||
@@ -46,6 +47,7 @@ public class ImageVideoArchiveService {
|
||||
private final OssStorageService ossStorageService;
|
||||
private final ImageVideoProperties properties;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
|
||||
void captureSubmitResponse(ImageVideoAsyncTaskEntity task, Object response) {
|
||||
if (!isWorkflowTask(task) || response == null) {
|
||||
@@ -107,22 +109,32 @@ public class ImageVideoArchiveService {
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.image-video.retention-cleanup-delay-ms:600000}")
|
||||
public void cleanupExpiredWorkflowTasks() {
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusDays(RETENTION_DAYS);
|
||||
List<ImageVideoAsyncTaskEntity> expiredTasks = taskMapper.selectList(
|
||||
new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
|
||||
.eq(ImageVideoAsyncTaskEntity::getTaskType, TASK_TYPE)
|
||||
.lt(ImageVideoAsyncTaskEntity::getSubmittedAt, cutoff)
|
||||
.orderByAsc(ImageVideoAsyncTaskEntity::getId)
|
||||
.last("LIMIT " + RETENTION_CLEANUP_BATCH_SIZE));
|
||||
int deleted = 0;
|
||||
for (ImageVideoAsyncTaskEntity task : expiredTasks) {
|
||||
if (!deleteArchivedObjects(task)) {
|
||||
continue;
|
||||
}
|
||||
deleted += taskMapper.deleteById(task.getId());
|
||||
// 归档认领有 CAS 兜底,但过期清理没有:两个节点各自 selectList 拿到同一批 200 行后
|
||||
// 会重复删同一批 OSS 对象、重复 deleteById,且任一侧中途失败都会给对方留下
|
||||
// 「对象已删、行还在」的半截状态。锁 TTL 给到 20 分钟,覆盖最慢的一批远端删除。
|
||||
DistributedJobLockService.LockHandle jobLock = distributedJobLockService.tryLock(
|
||||
"image-video:retention-cleanup", java.time.Duration.ofMinutes(20));
|
||||
if (jobLock == null) {
|
||||
return;
|
||||
}
|
||||
if (deleted > 0) {
|
||||
log.info("[image-video] expired workflow task cleanup deleted={} cutoff={}", deleted, cutoff);
|
||||
try (jobLock) {
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusDays(RETENTION_DAYS);
|
||||
List<ImageVideoAsyncTaskEntity> expiredTasks = taskMapper.selectList(
|
||||
new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
|
||||
.eq(ImageVideoAsyncTaskEntity::getTaskType, TASK_TYPE)
|
||||
.lt(ImageVideoAsyncTaskEntity::getSubmittedAt, cutoff)
|
||||
.orderByAsc(ImageVideoAsyncTaskEntity::getId)
|
||||
.last("LIMIT " + RETENTION_CLEANUP_BATCH_SIZE));
|
||||
int deleted = 0;
|
||||
for (ImageVideoAsyncTaskEntity task : expiredTasks) {
|
||||
if (!deleteArchivedObjects(task)) {
|
||||
continue;
|
||||
}
|
||||
deleted += taskMapper.deleteById(task.getId());
|
||||
}
|
||||
if (deleted > 0) {
|
||||
log.info("[image-video] expired workflow task cleanup deleted={} cutoff={}", deleted, cutoff);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+17
-7
@@ -238,13 +238,23 @@ public class ImageVideoAsyncTaskService {
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.image-video.failed-task-cleanup-delay-ms:60000}")
|
||||
public void cleanupFailedTasks() {
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES);
|
||||
int deleted = taskMapper.delete(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
|
||||
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.FAILED.name())
|
||||
.lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff)
|
||||
.last("LIMIT " + EXPIRED_DELETE_BATCH_SIZE));
|
||||
if (deleted > 0) {
|
||||
log.info("[image-video] removed expired failed tasks count={}", deleted);
|
||||
// 本方法是全类唯一直接对共享表发 DELETE ... LIMIT 的定时任务:两节点各跑一份时会
|
||||
// 命中同一批 (status, updated_at) 区间并互相等行锁。扫描/轮询/接管三条已用任务锁,
|
||||
// 这里补齐;锁超时后另一节点接管,删除本身幂等,不会漏清也不会重复扣。
|
||||
DistributedJobLockService.LockHandle jobLock =
|
||||
distributedJobLockService.tryLock("image-video:failed-task-cleanup", java.time.Duration.ofSeconds(120));
|
||||
if (jobLock == null) {
|
||||
return;
|
||||
}
|
||||
try (jobLock) {
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES);
|
||||
int deleted = taskMapper.delete(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
|
||||
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.FAILED.name())
|
||||
.lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff)
|
||||
.last("LIMIT " + EXPIRED_DELETE_BATCH_SIZE));
|
||||
if (deleted > 0) {
|
||||
log.info("[image-video] removed expired failed tasks count={}", deleted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+4
-2
@@ -15,12 +15,14 @@ public interface PriceTrackLoopRunMapper extends BaseMapper<PriceTrackLoopRunEnt
|
||||
* 分批删除已结束且超过保留期的循环批次行。
|
||||
*
|
||||
* <p>只删终态行:RUNNING 的批次删了会让后续 dispatch / childFinished 找不到记录。
|
||||
* 时间线用 COALESCE 兜底——终态行本应写 finished_at,历史行可能只有 updated_at。
|
||||
* 时间线用 V141 的 STORED 生成列 {@code cleanup_at}(= COALESCE(finished_at, updated_at,
|
||||
* created_at),口径与改造前完全一致):原来这个表达式直接写在 WHERE 里不可走索引,
|
||||
* 每条 DELETE 都要全表扫描并对扫过的行加锁,而 (status, cleanup_at) 索引已就位。
|
||||
*/
|
||||
@Delete("""
|
||||
DELETE FROM biz_price_track_loop_run
|
||||
WHERE status IN ('SUCCESS', 'FAILED', 'STOPPED')
|
||||
AND COALESCE(finished_at, updated_at, created_at) < #{cutoff}
|
||||
AND cleanup_at < #{cutoff}
|
||||
LIMIT #{batchSize}
|
||||
""")
|
||||
int deleteFinishedBefore(@Param("cutoff") LocalDateTime cutoff, @Param("batchSize") int batchSize);
|
||||
|
||||
+25
-8
@@ -169,6 +169,14 @@ public class PriceTrackLoopRunService {
|
||||
return toVo(entity);
|
||||
}
|
||||
|
||||
/**
|
||||
* 把刚创建的子任务认领为循环任务的当前子任务。
|
||||
*
|
||||
* <p>认领失败一律抛异常而不是静默返回:调用方 {@code PriceTrackTaskService.createTask}
|
||||
* 与本方法同处一个事务(默认传播 REQUIRED),抛出会让刚 insert 的子任务连同扣次一起回滚。
|
||||
* 原先各处判不到就 {@code return},子任务照样建成并真实执行——同一个店铺槽位跑出两个
|
||||
* RUNNING 子任务(重复采集、重复扣用户次数),且第一个失去跟踪。
|
||||
*/
|
||||
@Transactional
|
||||
public void bindChildTask(Long loopRunId, Long childTaskId, Integer roundIndex, Integer shopIndex) {
|
||||
if (loopRunId == null || childTaskId == null) {
|
||||
@@ -176,26 +184,35 @@ public class PriceTrackLoopRunService {
|
||||
}
|
||||
PriceTrackLoopRunEntity entity = loopRunMapper.selectById(loopRunId);
|
||||
if (entity == null || isTerminal(entity.getStatus())) {
|
||||
return;
|
||||
throw new BusinessException("循环任务已结束或不存在,子任务未绑定");
|
||||
}
|
||||
if (entity.getCurrentRound() == null || entity.getCurrentShopIndex() == null) {
|
||||
return;
|
||||
throw new BusinessException("循环任务轮次状态缺失,子任务未绑定");
|
||||
}
|
||||
if (!entity.getCurrentRound().equals(roundIndex) || !entity.getCurrentShopIndex().equals(shopIndex)) {
|
||||
return;
|
||||
throw new BusinessException("循环任务轮次或店铺指针已推进,本次派发已作废");
|
||||
}
|
||||
if (entity.getActiveTaskId() != null && !isChildTaskTerminal(entity.getActiveTaskId())) {
|
||||
return;
|
||||
throw new BusinessException("当前仍有执行中的子任务,本次派发已作废");
|
||||
}
|
||||
entity.setActiveTaskId(childTaskId);
|
||||
entity.setUpdatedAt(LocalDateTime.now());
|
||||
// 字段级更新 + status CAS:整行 updateById 会覆盖并发写入的 stop_requested,
|
||||
// 导致「停止循环」请求被静默丢弃
|
||||
loopRunMapper.update(null, new LambdaUpdateWrapper<PriceTrackLoopRunEntity>()
|
||||
// 认领槽位必须连带 CAS 掉读取时的 active_task_id:只比 status 时两个并发派发都能通过
|
||||
// (status 没变),后写的那个把先绑的子任务顶掉,两个 RUNNING 子任务同时真实执行。
|
||||
Long expectedActiveTaskId = entity.getActiveTaskId();
|
||||
LambdaUpdateWrapper<PriceTrackLoopRunEntity> claim = new LambdaUpdateWrapper<PriceTrackLoopRunEntity>()
|
||||
.eq(PriceTrackLoopRunEntity::getId, entity.getId())
|
||||
.eq(PriceTrackLoopRunEntity::getStatus, entity.getStatus())
|
||||
.set(PriceTrackLoopRunEntity::getActiveTaskId, childTaskId)
|
||||
.set(PriceTrackLoopRunEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
.set(PriceTrackLoopRunEntity::getUpdatedAt, LocalDateTime.now());
|
||||
if (expectedActiveTaskId == null) {
|
||||
claim.isNull(PriceTrackLoopRunEntity::getActiveTaskId);
|
||||
} else {
|
||||
claim.eq(PriceTrackLoopRunEntity::getActiveTaskId, expectedActiveTaskId);
|
||||
}
|
||||
if (loopRunMapper.update(null, claim) == 0) {
|
||||
throw new BusinessException("循环任务槽位已被并发派发认领,本次派发已作废");
|
||||
}
|
||||
log.info("[price-track-loop] child bound loopRunId={} childTaskId={} roundIndex={} shopIndex={}",
|
||||
loopRunId, childTaskId, roundIndex, shopIndex);
|
||||
}
|
||||
|
||||
+99
-82
@@ -460,7 +460,6 @@ public class PriceTrackTaskService {
|
||||
return result;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public PriceTrackCreateTaskVo createTask(PriceTrackCreateTaskRequest request, String deviceId) {
|
||||
if (request.isStatusMode() == request.isAsinMode()) throw new BusinessException("跟价模式必须二选一");
|
||||
if (request.isAsinMode() && (request.getAsinFiles() == null || request.getAsinFiles().isEmpty())) {
|
||||
@@ -490,91 +489,98 @@ public class PriceTrackTaskService {
|
||||
log.info("[price-track] createTask skipAsins countries={} asinMode={}",
|
||||
skipAsinsByCountry.keySet(), request.isAsinMode());
|
||||
|
||||
// 创建任务记录
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType(MODULE_TYPE);
|
||||
task.setTaskMode("PYTHON_QUEUE");
|
||||
task.setStatus("RUNNING");
|
||||
task.setSourceFileCount(uniqueItems.size());
|
||||
task.setSuccessFileCount(0);
|
||||
task.setFailedFileCount(0);
|
||||
task.setCreatedBy("user:" + request.getUserId());
|
||||
task.setUserId(request.getUserId());
|
||||
task.setDeviceId(AdminAuthSupport.normalizeDeviceId(deviceId));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.insert(task);
|
||||
priceTrackTaskCacheService.touchTaskHeartbeat(task.getId());
|
||||
if (request.isAsinMode()) {
|
||||
saveTaskAsinRowsPayload(task.getId(), asinRowsForPayload);
|
||||
}
|
||||
if (request.getLoopRunId() != null) {
|
||||
priceTrackLoopRunService.bindChildTask(request.getLoopRunId(), task.getId(), request.getRoundIndex(), request.getShopIndex());
|
||||
}
|
||||
|
||||
List<PriceTrackResultItemVo> snapshot = new ArrayList<>();
|
||||
// 逐店铺 fileResultMapper.insert 看似该改批量(500 店铺 = 500 次往返),但**不能改**:
|
||||
// 下面的 toSnapshotVo 依赖 MyBatis-Plus insert 回填的自增主键(fr.getId() → vo.resultId),
|
||||
// 而本仓的手写 INSERT ... VALUES (...),(...) 批量写法不回填主键,改了会让快照 resultId 为 null。
|
||||
// 要批量化必须同时解决「批量插入 + 按序回填 id」(MySQL 需 useGeneratedKeys + 顺序保证)。
|
||||
for (PriceTrackMatchShopsVo.PriceTrackShopQueueItem item : uniqueItems) {
|
||||
String norm = ziniaoShopSwitchService.normalizeShopName(item.getShopName());
|
||||
if (norm.isBlank()) throw new BusinessException("店铺名称无效");
|
||||
FileResultEntity fr = new FileResultEntity();
|
||||
fr.setTaskId(task.getId());
|
||||
fr.setModuleType(MODULE_TYPE);
|
||||
fr.setSourceFilename(norm);
|
||||
fr.setSourceFileUrl(Objects.toString(item.getShopId(), null));
|
||||
fr.setUserId(request.getUserId());
|
||||
fr.setSuccess(0);
|
||||
fr.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(fr);
|
||||
PriceTrackResultItemVo snapshotItem = toSnapshotVo(fr, item, task.getStatus());
|
||||
snapshotItem.setLoopRunId(request.getLoopRunId());
|
||||
snapshotItem.setRoundIndex(request.getRoundIndex());
|
||||
snapshot.add(snapshotItem);
|
||||
}
|
||||
|
||||
// 保存请求上下文快照,供 Python 消费和前端轮询展示使用
|
||||
try {
|
||||
Map<String, Object> ctx = new LinkedHashMap<>();
|
||||
ctx.put("userId", request.getUserId());
|
||||
ctx.put("statusMode", request.isStatusMode());
|
||||
ctx.put("asinMode", request.isAsinMode());
|
||||
ctx.put("countryCodes", request.getCountryCodes());
|
||||
// 不再保存全量 ASIN 数据到 requestJson
|
||||
// ctx.put("skipAsinsByCountry", skipAsinsByCountry);
|
||||
// ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry);
|
||||
// ctx.put("asinRowsByCountry", asinRowsByCountry);
|
||||
// ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin);
|
||||
// 文件模式:保存文件路径,供后续分页读取
|
||||
// Excel/CSV 解析(POI 全表读,可达秒级)刻意留在事务之外:事务里只做 DB 写与载荷落库,
|
||||
// 避免创建跟价任务时长时间占住连接池连接(本仓历史上 biz_task_file_job 一类锁等
|
||||
// 就是被「事务内含外部 IO」放大的)。载荷 PUT 仍留在事务内:它失败必须整体回滚,
|
||||
// 不能让「任务已建、ASIN 载荷为空」的半成品进入 Python 队列。
|
||||
final Map<String, List<Map<String, String>>> parsedAsinRows = asinRowsForPayload;
|
||||
return inRequiredTransaction(() -> {
|
||||
// 创建任务记录
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType(MODULE_TYPE);
|
||||
task.setTaskMode("PYTHON_QUEUE");
|
||||
task.setStatus("RUNNING");
|
||||
task.setSourceFileCount(uniqueItems.size());
|
||||
task.setSuccessFileCount(0);
|
||||
task.setFailedFileCount(0);
|
||||
task.setCreatedBy("user:" + request.getUserId());
|
||||
task.setUserId(request.getUserId());
|
||||
task.setDeviceId(AdminAuthSupport.normalizeDeviceId(deviceId));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.insert(task);
|
||||
priceTrackTaskCacheService.touchTaskHeartbeat(task.getId());
|
||||
if (request.isAsinMode()) {
|
||||
ctx.put("asinFiles", request.getAsinFiles());
|
||||
saveTaskAsinRowsPayload(task.getId(), parsedAsinRows);
|
||||
}
|
||||
if (request.getLoopRunId() != null) {
|
||||
priceTrackLoopRunService.bindChildTask(request.getLoopRunId(), task.getId(), request.getRoundIndex(), request.getShopIndex());
|
||||
}
|
||||
ctx.put("items", uniqueItems);
|
||||
ctx.put("loopRunId", request.getLoopRunId());
|
||||
ctx.put("roundIndex", request.getRoundIndex());
|
||||
ctx.put("shopIndex", request.getShopIndex());
|
||||
task.setRequestJson(objectMapper.writeValueAsString(ctx));
|
||||
task.setResultJson(objectMapper.writeValueAsString(snapshot));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("序列化任务数据失败");
|
||||
}
|
||||
fileTaskMapper.updateById(task);
|
||||
priceTrackTaskCacheService.saveTaskCache(task);
|
||||
|
||||
// TODO: Python 侧轮询 RUNNING 任务,pickup 后开始处理
|
||||
// Java 侧通过 submitResult 接口接收 Python 回传结果
|
||||
List<PriceTrackResultItemVo> snapshot = new ArrayList<>();
|
||||
// 逐店铺 fileResultMapper.insert 看似该改批量(500 店铺 = 500 次往返),但**不能改**:
|
||||
// 下面的 toSnapshotVo 依赖 MyBatis-Plus insert 回填的自增主键(fr.getId() → vo.resultId),
|
||||
// 而本仓的手写 INSERT ... VALUES (...),(...) 批量写法不回填主键,改了会让快照 resultId 为 null。
|
||||
// 要批量化必须同时解决「批量插入 + 按序回填 id」(MySQL 需 useGeneratedKeys + 顺序保证)。
|
||||
for (PriceTrackMatchShopsVo.PriceTrackShopQueueItem item : uniqueItems) {
|
||||
String norm = ziniaoShopSwitchService.normalizeShopName(item.getShopName());
|
||||
if (norm.isBlank()) throw new BusinessException("店铺名称无效");
|
||||
FileResultEntity fr = new FileResultEntity();
|
||||
fr.setTaskId(task.getId());
|
||||
fr.setModuleType(MODULE_TYPE);
|
||||
fr.setSourceFilename(norm);
|
||||
fr.setSourceFileUrl(Objects.toString(item.getShopId(), null));
|
||||
fr.setUserId(request.getUserId());
|
||||
fr.setSuccess(0);
|
||||
fr.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(fr);
|
||||
PriceTrackResultItemVo snapshotItem = toSnapshotVo(fr, item, task.getStatus());
|
||||
snapshotItem.setLoopRunId(request.getLoopRunId());
|
||||
snapshotItem.setRoundIndex(request.getRoundIndex());
|
||||
snapshot.add(snapshotItem);
|
||||
}
|
||||
|
||||
PriceTrackCreateTaskVo vo = new PriceTrackCreateTaskVo();
|
||||
vo.setTaskId(task.getId());
|
||||
vo.setItems(snapshot);
|
||||
vo.setSkipAsinsByCountry(skipAsinsByCountry);
|
||||
vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry);
|
||||
vo.setAsinRowsByCountry(asinRowsByCountry);
|
||||
vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin);
|
||||
return vo;
|
||||
// 保存请求上下文快照,供 Python 消费和前端轮询展示使用
|
||||
try {
|
||||
Map<String, Object> ctx = new LinkedHashMap<>();
|
||||
ctx.put("userId", request.getUserId());
|
||||
ctx.put("statusMode", request.isStatusMode());
|
||||
ctx.put("asinMode", request.isAsinMode());
|
||||
ctx.put("countryCodes", request.getCountryCodes());
|
||||
// 不再保存全量 ASIN 数据到 requestJson
|
||||
// ctx.put("skipAsinsByCountry", skipAsinsByCountry);
|
||||
// ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry);
|
||||
// ctx.put("asinRowsByCountry", asinRowsByCountry);
|
||||
// ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin);
|
||||
// 文件模式:保存文件路径,供后续分页读取
|
||||
if (request.isAsinMode()) {
|
||||
ctx.put("asinFiles", request.getAsinFiles());
|
||||
}
|
||||
ctx.put("items", uniqueItems);
|
||||
ctx.put("loopRunId", request.getLoopRunId());
|
||||
ctx.put("roundIndex", request.getRoundIndex());
|
||||
ctx.put("shopIndex", request.getShopIndex());
|
||||
task.setRequestJson(objectMapper.writeValueAsString(ctx));
|
||||
task.setResultJson(objectMapper.writeValueAsString(snapshot));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("序列化任务数据失败");
|
||||
}
|
||||
fileTaskMapper.updateById(task);
|
||||
priceTrackTaskCacheService.saveTaskCache(task);
|
||||
|
||||
// TODO: Python 侧轮询 RUNNING 任务,pickup 后开始处理
|
||||
// Java 侧通过 submitResult 接口接收 Python 回传结果
|
||||
|
||||
PriceTrackCreateTaskVo vo = new PriceTrackCreateTaskVo();
|
||||
vo.setTaskId(task.getId());
|
||||
vo.setItems(snapshot);
|
||||
vo.setSkipAsinsByCountry(skipAsinsByCountry);
|
||||
vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry);
|
||||
vo.setAsinRowsByCountry(asinRowsByCountry);
|
||||
vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin);
|
||||
return vo;
|
||||
});
|
||||
}
|
||||
|
||||
public void submitResult(Long taskId, PriceTrackSubmitResultRequest request) {
|
||||
@@ -2263,6 +2269,17 @@ public class PriceTrackTaskService {
|
||||
return template.execute(status -> action.get());
|
||||
}
|
||||
|
||||
/**
|
||||
* 默认传播(REQUIRED)事务段:供「外部 IO 已移出、只剩 DB 写」的方法体使用。
|
||||
* 与 {@link #inNewTransaction} 的差别是不挂起外层事务;单测直接 new 本类时退化为直调。
|
||||
*/
|
||||
private <T> T inRequiredTransaction(java.util.function.Supplier<T> action) {
|
||||
if (transactionManager == null) {
|
||||
return action.get();
|
||||
}
|
||||
return new TransactionTemplate(transactionManager).execute(status -> action.get());
|
||||
}
|
||||
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId) {
|
||||
return TaskFinalizeSupport.acquireTaskLock(taskDistributedLockService, MODULE_TYPE, taskId);
|
||||
}
|
||||
|
||||
+27
-1
@@ -17,6 +17,8 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
@@ -117,8 +119,12 @@ public class ShopKeyService {
|
||||
/**
|
||||
* 后台手动检测白名单:绕过缓存真实打一次紫鸟接口,并重置自动重试次数。
|
||||
* 白名单放行需紫鸟公司侧授权,自动重试达阈值后会停试,此按钮供授权后重新验证。
|
||||
*
|
||||
* <p>不加 {@code @Transactional}:本方法只有一次单语句 UPDATE,没有需要原子提交的多元写,
|
||||
* 而 {@code probeApiKeyUpstream} 是真实出网的紫鸟调用(上游代理挂起时按超时计可达数十秒)。
|
||||
* 挂在事务上等于把连接池的连接按上游 RTT 扣住,多次点击即可耗尽池——这正是本仓
|
||||
* 「事务内含外部 IO」要避免的形态。
|
||||
*/
|
||||
@Transactional
|
||||
public ShopKeyItemVo checkWhitelist(Long id) {
|
||||
ShopKeyEntity entity = getById(id);
|
||||
ziniaoApiKeyProvider.invalidateProxyUrlCache();
|
||||
@@ -214,7 +220,27 @@ public class ShopKeyService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
/**
|
||||
* 密钥变更后重刷紫鸟店铺索引。
|
||||
*
|
||||
* invalidateIndex() 是「删 1 张持久表全部索引行 + 清 7 类瞬时缓存」的重操作,
|
||||
* 放在 create/update/delete 的事务里等于把一次单行写放大成几十条 DELETE 的持锁时间,
|
||||
* 而缓存本来就不参与回滚。因此:有事务时挂到提交之后执行,没有事务时照旧立即执行。
|
||||
*/
|
||||
private void triggerShopIndexRefresh() {
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||
@Override
|
||||
public void afterCommit() {
|
||||
invalidateIndexSafely();
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
invalidateIndexSafely();
|
||||
}
|
||||
|
||||
private void invalidateIndexSafely() {
|
||||
try {
|
||||
ziniaoShopIndexService.invalidateIndex();
|
||||
} catch (Exception ex) {
|
||||
|
||||
+5
-7
@@ -58,6 +58,9 @@ public class SimilarAsinImagePrefetchService {
|
||||
/** 批量命中检查/批量 touch 单次 IN 上限,避免超长 SQL 参数列表。 */
|
||||
private static final int CACHE_LOOKUP_BATCH_SIZE = 500;
|
||||
|
||||
/** url_hash 十六进制编码用:逐字节 String.format 在按 URL 调用的路径上会放大成几万次格式化。 */
|
||||
private static final java.util.HexFormat HEX_FORMAT = java.util.HexFormat.of();
|
||||
|
||||
/** 单个 task 最多缓存的待预热 URL 数,避免持续回传时内存无界增长。 */
|
||||
private static final int MAX_PENDING_URLS_PER_TASK = 5_000;
|
||||
/** 单机最多同时维护的预热 task 数。 */
|
||||
@@ -316,7 +319,7 @@ public class SimilarAsinImagePrefetchService {
|
||||
* 未命中 url 不产生任何 touch。输入按传入顺序返回,未命中为 null。
|
||||
* 失败/开关关闭返回空列表,由调用方走回退路径。
|
||||
*/
|
||||
List<byte[]> lookupBatch(List<String> urls) {
|
||||
public List<byte[]> lookupBatch(List<String> urls) {
|
||||
List<byte[]> result = new ArrayList<>();
|
||||
if (!properties.isImageDbCacheEnabled()) {
|
||||
return result;
|
||||
@@ -415,12 +418,7 @@ public class SimilarAsinImagePrefetchService {
|
||||
private static String sha256Hex(String value) {
|
||||
try {
|
||||
MessageDigest md = MessageDigest.getInstance("SHA-256");
|
||||
byte[] digest = md.digest(value.getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(digest.length * 2);
|
||||
for (byte b : digest) {
|
||||
sb.append(String.format("%02x", b & 0xFF));
|
||||
}
|
||||
return sb.toString();
|
||||
return HEX_FORMAT.formatHex(md.digest(value.getBytes(StandardCharsets.UTF_8)));
|
||||
} catch (NoSuchAlgorithmException ex) {
|
||||
// SHA-256 在 JDK 中是必备算法,正常环境下不会落到这里。
|
||||
return null;
|
||||
|
||||
+12
-1
@@ -132,6 +132,8 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
||||
private static final int MAX_LIGHT_TASK_IDS = 200;
|
||||
/** progressBatch 的 IN 子句上限:看板单次轮询的在册任务量级远小于它,取值留了 5 倍余量。 */
|
||||
private static final int MAX_BATCH_TASK_IDS = 1000;
|
||||
/** Task 19:Python 回传逐行日志采样频率(每 N 行记一行)。 */
|
||||
private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L;
|
||||
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
|
||||
@@ -657,10 +659,19 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
|
||||
public SimilarAsinTaskBatchVo progressBatch(List<Long> taskIds) {
|
||||
SimilarAsinTaskBatchVo vo = new SimilarAsinTaskBatchVo();
|
||||
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
|
||||
List<Long> requestedIds = taskIds == null ? List.of() : taskIds.stream()
|
||||
.filter(id -> id != null && id > 0)
|
||||
.distinct()
|
||||
.toList();
|
||||
// 上限只为挡住超长 IN 子句。被截的尾部不进 missingTaskIds(与 progressLight 同口径:
|
||||
// 看板会把 missing 当成「这张单不存在」,而它们只是本轮没查),改为记 WARN 留现场。
|
||||
if (requestedIds.size() > MAX_BATCH_TASK_IDS) {
|
||||
log.warn("[similar-asin] progressBatch 收到 {} 个 id,超过上限 {},本轮只刷新前 {} 个",
|
||||
requestedIds.size(), MAX_BATCH_TASK_IDS, MAX_BATCH_TASK_IDS);
|
||||
}
|
||||
List<Long> normalizedIds = requestedIds.size() > MAX_BATCH_TASK_IDS
|
||||
? requestedIds.subList(0, MAX_BATCH_TASK_IDS)
|
||||
: requestedIds;
|
||||
if (normalizedIds.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
|
||||
+72
-30
@@ -91,6 +91,9 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
private static final int IMG_COL_PUZZLE1 = 13;
|
||||
private static final int IMG_COL_PUZZLE2 = 14;
|
||||
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
|
||||
|
||||
/** 结果行读取时 chunk / orphan scope_state 的 keyset 页大小,与 loadChunksKeyset 的 500 同口径。 */
|
||||
private static final int RESULT_ROWS_PAGE_SIZE = 500;
|
||||
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
|
||||
|
||||
private final SimilarAsinProperties properties;
|
||||
@@ -562,6 +565,11 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
/**
|
||||
* Select and decode one chunk payload at a time. Only rows needed by the
|
||||
* current source workbook are retained in the returned map.
|
||||
*
|
||||
* <p>按 keyset 整页取 chunk(与 {@link SimilarAsinChunkMergeSupport#loadChunksKeyset} 同口径):
|
||||
* 原先每轮 {@code LIMIT 1} 只取一行,一个源文件要把该任务的 chunk 表逐行翻完,
|
||||
* 每行一次 SQL + 一次对象读 + 一次整片 JSON 反序列化;请求行全部命中后立即收尾,
|
||||
* 不再把剩余 chunk 也解一遍。
|
||||
*/
|
||||
private Map<String, SimilarAsinResultRowDto> loadPersistedResultRowsForRows(
|
||||
Long taskId,
|
||||
@@ -578,7 +586,7 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.orderByAsc(TaskChunkEntity::getId)
|
||||
.last("LIMIT 1");
|
||||
.last("LIMIT " + RESULT_ROWS_PAGE_SIZE);
|
||||
if (lastChunkId != null) {
|
||||
query.gt(TaskChunkEntity::getId, lastChunkId);
|
||||
}
|
||||
@@ -586,10 +594,16 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
if (page == null || page.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
TaskChunkEntity chunk = page.getFirst();
|
||||
appendRequestedResultRows(result, chunkPayloadSupport.readChunkRows(chunk).values(), requestedKeys, false);
|
||||
lastChunkId = chunk.getId();
|
||||
if (lastChunkId == null) {
|
||||
for (TaskChunkEntity chunk : page) {
|
||||
appendRequestedResultRows(result, chunkPayloadSupport.readChunkRows(chunk).values(),
|
||||
requestedKeys, false);
|
||||
}
|
||||
TaskChunkEntity tail = page.getLast();
|
||||
lastChunkId = tail == null ? null : tail.getId();
|
||||
if (lastChunkId == null || page.size() < RESULT_ROWS_PAGE_SIZE) {
|
||||
break;
|
||||
}
|
||||
if (allRequestedRowsResolved(requestedRows, result)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -597,6 +611,21 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 请求行是否都已取到结果。判据必须与 {@link SimilarAsinChunkMergeSupport#findResultRow}
|
||||
* 的匹配口径一致(主键或 legacy 键任一命中即算已取到)——直接用 result.size() 与
|
||||
* requestedKeys.size() 比会永远不成立,因为 requestedKeys 每行都塞了主键 + legacy 两个键。
|
||||
*/
|
||||
private boolean allRequestedRowsResolved(List<SimilarAsinParsedRowVo> requestedRows,
|
||||
Map<String, SimilarAsinResultRowDto> result) {
|
||||
for (SimilarAsinParsedRowVo row : requestedRows) {
|
||||
if (SimilarAsinChunkMergeSupport.findResultRow(row, result) == null) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private Set<String> resultLookupKeys(List<SimilarAsinParsedRowVo> rows) {
|
||||
Set<String> keys = new LinkedHashSet<>();
|
||||
for (SimilarAsinParsedRowVo row : rows == null ? List.<SimilarAsinParsedRowVo>of() : rows) {
|
||||
@@ -651,7 +680,7 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.likeRight(TaskScopeStateEntity::getScopeKey, SimilarAsinChunkPayloadSupport.ORPHAN_SCOPE_KEY_PREFIX)
|
||||
.orderByAsc(TaskScopeStateEntity::getId)
|
||||
.last("LIMIT 1");
|
||||
.last("LIMIT " + RESULT_ROWS_PAGE_SIZE);
|
||||
if (lastStateId != null) {
|
||||
query.gt(TaskScopeStateEntity::getId, lastStateId);
|
||||
}
|
||||
@@ -659,29 +688,33 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
if (page == null || page.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
TaskScopeStateEntity state = page.getFirst();
|
||||
lastStateId = state.getId();
|
||||
if (lastStateId == null) {
|
||||
return;
|
||||
}
|
||||
if (state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
String payloadJson = transientPayloadStorageService.resolvePayload(
|
||||
state.getParsedPayloadJson(), "read similar ASIN orphan rows failed");
|
||||
JsonNode array = objectMapper.readTree(payloadJson);
|
||||
if (!array.isArray()) {
|
||||
TaskScopeStateEntity tail = page.getLast();
|
||||
lastStateId = tail == null ? null : tail.getId();
|
||||
boolean lastPage = page.size() < RESULT_ROWS_PAGE_SIZE || lastStateId == null;
|
||||
for (TaskScopeStateEntity state : page) {
|
||||
if (state == null || state.getParsedPayloadJson() == null
|
||||
|| state.getParsedPayloadJson().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
List<SimilarAsinResultRowDto> matchingRows = new ArrayList<>();
|
||||
for (JsonNode node : array) {
|
||||
matchingRows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class));
|
||||
try {
|
||||
String payloadJson = transientPayloadStorageService.resolvePayload(
|
||||
state.getParsedPayloadJson(), "read similar ASIN orphan rows failed");
|
||||
JsonNode array = objectMapper.readTree(payloadJson);
|
||||
if (!array.isArray()) {
|
||||
continue;
|
||||
}
|
||||
List<SimilarAsinResultRowDto> matchingRows = new ArrayList<>();
|
||||
for (JsonNode node : array) {
|
||||
matchingRows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class));
|
||||
}
|
||||
appendRequestedResultRows(sink, matchingRows, requestedKeys, true);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] read requested orphan rows failed taskId={} stateId={} err={}",
|
||||
taskId, state.getId(), ex.getMessage());
|
||||
}
|
||||
appendRequestedResultRows(sink, matchingRows, requestedKeys, true);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] read requested orphan rows failed taskId={} stateId={} err={}",
|
||||
taskId, state.getId(), ex.getMessage());
|
||||
}
|
||||
if (lastPage) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -788,14 +821,23 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
if (!prefetchUrls.isEmpty()) {
|
||||
// DB cache 命中的缩略图也立即落到任务临时目录,避免整批字节停留在堆中。
|
||||
long dbCacheStart = System.currentTimeMillis();
|
||||
List<String> remainingUrls = new ArrayList<>(prefetchUrls.size());
|
||||
int dbHit = 0;
|
||||
// 一次 IN 批量查(lookupBatch 内部只对命中项 touch last_used_at):
|
||||
// 原先逐 URL 调 lookup(),按注释里的量级(4 源 × 4000 URL)每次装配要发几千条单行 SQL。
|
||||
Set<String> pendingUrlSet = new LinkedHashSet<>();
|
||||
for (String url : prefetchUrls) {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
if (url == null || imageSpool.get(url) != null || taskImageCache.containsKey(url)) {
|
||||
continue;
|
||||
}
|
||||
byte[] cachedBytes = imagePrefetchService.lookup(url);
|
||||
pendingUrlSet.add(url);
|
||||
}
|
||||
List<String> pendingUrls = new ArrayList<>(pendingUrlSet);
|
||||
List<byte[]> cachedBatch = imagePrefetchService.lookupBatch(pendingUrls);
|
||||
List<String> remainingUrls = new ArrayList<>();
|
||||
int dbHit = 0;
|
||||
for (int i = 0; i < pendingUrls.size(); i++) {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
String url = pendingUrls.get(i);
|
||||
byte[] cachedBytes = i < cachedBatch.size() ? cachedBatch.get(i) : null;
|
||||
if (cachedBytes == null) {
|
||||
remainingUrls.add(url);
|
||||
continue;
|
||||
|
||||
+10
-11
@@ -323,33 +323,32 @@ public class TransientPayloadStorageService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 带 task_id 收敛的 chunk 引用计数;带 task 查不到时回退全局查询,
|
||||
* 保证即使指针格式与预期不符也不会漏判引用(漏判会导致对象被误删)。
|
||||
* 带 task_id 收敛的 chunk 引用计数。
|
||||
*
|
||||
* <p>只有在指针解析不出 taskId 时才退回全局查询。指针可解析时不再回落全局:
|
||||
* 对象 key 里带 taskId、引用不跨任务(见 {@link #isStillReferenced} 的口径说明),
|
||||
* 而物理删除又约定在 DB 行删除之后执行,所以带 task 查到 0 才是正常状态——
|
||||
* 原先「查到 0 就无条件再全局查一次」让每次删除都必然退化成
|
||||
* {@code payload_json}(JSON 列,无索引)上的全表扫描,而本方法是删除路径的公共链路且被逐行调用。
|
||||
*/
|
||||
private Long referencedChunkCount(Long taskId, List<String> values) {
|
||||
if (taskId != null) {
|
||||
Long scoped = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
return taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.in(TaskChunkEntity::getPayloadJson, values));
|
||||
if (scoped != null && scoped > 0L) {
|
||||
return scoped;
|
||||
}
|
||||
}
|
||||
return taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getPayloadJson, values));
|
||||
}
|
||||
|
||||
/** 带 task_id 收敛的 scope_state 引用计数;回退逻辑同 {@link #referencedChunkCount}。 */
|
||||
/** 带 task_id 收敛的 scope_state 引用计数;回落口径同 {@link #referencedChunkCount}。 */
|
||||
private Long referencedScopeStateCount(Long taskId, List<String> values) {
|
||||
if (taskId != null) {
|
||||
Long scoped = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
return taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
|
||||
.or()
|
||||
.in(TaskScopeStateEntity::getStateJson, values)));
|
||||
if (scoped != null && scoped > 0L) {
|
||||
return scoped;
|
||||
}
|
||||
}
|
||||
return taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
|
||||
|
||||
+14
-6
@@ -76,16 +76,21 @@ public class ZiniaoMemoryStoreService {
|
||||
public <T> List<T> listByType(String cacheType, Class<T> valueType, int limit) {
|
||||
String normalizedType = normalizeRequired(cacheType, "cacheType 不能为空");
|
||||
int safeLimit = Math.max(limit, 1);
|
||||
// 过期判断必须在 SQL 里做:原来先按 cache_key 取 LIMIT N 再在 Java 里滤掉过期行,
|
||||
// 于是过期行会白占 N 这个窗口——同一 cache_type 存量超过 N 时,排在窗口之后的
|
||||
// 有效行根本取不到,调用方(店铺名索引兜底扫描)把「其实存在」判成未命中。
|
||||
// 同时过期行不再把 payload_json 整列拖回应用侧反序列化。
|
||||
// 走 uk_type_key(cache_type, cache_key):等值 + 排序都命中索引,expires_at 作回表过滤。
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
List<ZiniaoMemoryStoreEntity> entities = ziniaoMemoryStoreMapper.selectList(new LambdaQueryWrapper<ZiniaoMemoryStoreEntity>()
|
||||
.eq(ZiniaoMemoryStoreEntity::getCacheType, normalizedType)
|
||||
.gt(ZiniaoMemoryStoreEntity::getExpiresAt, now)
|
||||
.orderByAsc(ZiniaoMemoryStoreEntity::getCacheKey)
|
||||
.last("LIMIT " + safeLimit));
|
||||
if (entities.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
return entities.stream()
|
||||
.filter(entity -> entity.getExpiresAt() != null && entity.getExpiresAt().isAfter(now))
|
||||
.map(entity -> deserialize(entity.getPayloadJson(), valueType))
|
||||
.filter(Objects::nonNull)
|
||||
.toList();
|
||||
@@ -133,8 +138,11 @@ public class ZiniaoMemoryStoreService {
|
||||
entity.setCreatedAt(now);
|
||||
entity.setUpdatedAt(now);
|
||||
ziniaoMemoryStoreMapper.insert(entity);
|
||||
// 店铺索引条目按「每店两把键(shopId + 名称别名)× 每轮刷新」成批写入,
|
||||
// 逐行 INFO 等于把刷新规模原样刷进日志文件;汇总行数已在
|
||||
// ZiniaoShopIndexService 的 `[ziniao-index] refresh persist` 里给出,这里降到 DEBUG。
|
||||
if (CACHE_TYPE_SHOP_INDEX_ENTRY.equals(normalizedType)) {
|
||||
log.info("[ziniao-shop-index-db] insert cacheKey={} companyNameSample={} expiresAt={} bytes={}",
|
||||
log.debug("[ziniao-shop-index-db] insert cacheKey={} companyNameSample={} expiresAt={} bytes={}",
|
||||
normalizedKey, sampleCompanyNameForLog(companyName), expiresAt, payloadJson.length());
|
||||
}
|
||||
return;
|
||||
@@ -146,7 +154,7 @@ public class ZiniaoMemoryStoreService {
|
||||
entity.setUpdatedAt(now);
|
||||
ziniaoMemoryStoreMapper.updateById(entity);
|
||||
if (CACHE_TYPE_SHOP_INDEX_ENTRY.equals(normalizedType)) {
|
||||
log.info("[ziniao-shop-index-db] update cacheKey={} id={} companyNameSample={} expiresAt={} bytes={}",
|
||||
log.debug("[ziniao-shop-index-db] update cacheKey={} id={} companyNameSample={} expiresAt={} bytes={}",
|
||||
normalizedKey, entity.getId(), sampleCompanyNameForLog(companyName), expiresAt, payloadJson.length());
|
||||
}
|
||||
}
|
||||
@@ -159,7 +167,7 @@ public class ZiniaoMemoryStoreService {
|
||||
if (entity != null) {
|
||||
ziniaoMemoryStoreMapper.deleteById(entity.getId());
|
||||
if (CACHE_TYPE_SHOP_INDEX_ENTRY.equals(normalizedType)) {
|
||||
log.info("[ziniao-shop-index-db] delete cacheKey={} id={} companyNameSample={}",
|
||||
log.debug("[ziniao-shop-index-db] delete cacheKey={} id={} companyNameSample={}",
|
||||
entity.getCacheKey(), entity.getId(), sampleCompanyNameForLog(entity.getCompanyName()));
|
||||
}
|
||||
}
|
||||
@@ -200,7 +208,7 @@ public class ZiniaoMemoryStoreService {
|
||||
}
|
||||
updated += ziniaoMemoryStoreMapper.updateById(row);
|
||||
if (CACHE_TYPE_SHOP_INDEX_ENTRY.equals(row.getCacheType())) {
|
||||
log.info("[ziniao-shop-index-db] mark-stale cacheKey={} id={} companyNameSample={} expiresAt={} bytes={}",
|
||||
log.debug("[ziniao-shop-index-db] mark-stale cacheKey={} id={} companyNameSample={} expiresAt={} bytes={}",
|
||||
row.getCacheKey(), row.getId(), sampleCompanyNameForLog(row.getCompanyName()),
|
||||
row.getExpiresAt(), row.getPayloadJson() == null ? 0 : row.getPayloadJson().length());
|
||||
}
|
||||
|
||||
@@ -27,6 +27,13 @@ spring:
|
||||
max-lifetime: ${AIIMAGE_DB_POOL_MAX_LIFETIME_MS:1500000}
|
||||
keepalive-time: ${AIIMAGE_DB_POOL_KEEPALIVE_TIME_MS:120000}
|
||||
leak-detection-threshold: ${AIIMAGE_DB_POOL_LEAK_DETECT_MS:15000}
|
||||
# 网络层超时兜底:此前只有 Hikari 的取连接/校验超时,socket 读没有上限——
|
||||
# 一条卡在 DB 侧的语句会把 Tomcat 工作线程挂到服务端结束为止,只能人工 kill 容器。
|
||||
# 刻意走 data-source-properties 而不是改上面 URL 的兜底默认值:生产用 AIIMAGE_DB_URL
|
||||
# 整体覆盖 URL,只改兜底值对线上无效。单位为毫秒(Connector/J 口径)。
|
||||
data-source-properties:
|
||||
connectTimeout: ${AIIMAGE_DB_CONNECT_TIMEOUT_MS:5000}
|
||||
socketTimeout: ${AIIMAGE_DB_SOCKET_TIMEOUT_MS:300000}
|
||||
flyway:
|
||||
enabled: ${AIIMAGE_FLYWAY_ENABLED:true}
|
||||
locations: classpath:db
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
-- 保留清理(retention)三条分批 DELETE 的可用索引。
|
||||
--
|
||||
-- 三条 DELETE 的 WHERE 都落不到任何现有索引上,只能全表扫描;InnoDB 的 DELETE 会对扫描过的行
|
||||
-- 加锁,03:50/04:20/04:40 三条又挤在同一窗口,于是清理任务与同期写入互相等待(生产的锁等待
|
||||
-- 与慢 SQL 有一条来源就在这里)。这里只追加索引/生成列,不改任何历史迁移文件。
|
||||
|
||||
-- 1) device_log_file:DELETE ... WHERE log_date < #{cutoff} LIMIT n
|
||||
-- uk_device_log_file(source, device_id, file_name, log_date) 里 log_date 在第 4 列,
|
||||
-- 不带前三列时左前缀用不上;idx_device_log_last_upload 是另一条时间线(V127:24-25)。
|
||||
ALTER TABLE device_log_file
|
||||
ADD KEY idx_device_log_log_date (log_date),
|
||||
ALGORITHM = INPLACE, LOCK = NONE;
|
||||
|
||||
-- 2) shop_data_duplicate_scan:DELETE ... WHERE created_at < #{cutoff} AND id < #{protectFromId} LIMIT n
|
||||
-- 现有 idx_status_created(status, created_at) 首列是 status,而本 DELETE 不带 status 条件(V108)。
|
||||
ALTER TABLE shop_data_duplicate_scan
|
||||
ADD KEY idx_sds_created_at (created_at),
|
||||
ALGORITHM = INPLACE, LOCK = NONE;
|
||||
|
||||
-- 3) biz_price_track_loop_run:DELETE ... WHERE status IN ('SUCCESS','FAILED','STOPPED')
|
||||
-- AND COALESCE(finished_at, updated_at, created_at) < #{cutoff} LIMIT n
|
||||
-- 被 COALESCE 包住的表达式不可走索引(非 sargable),所以落一个 STORED 生成列承载同一时间线,
|
||||
-- 再建 (status, cleanup_at) 复合索引;现有 idx_user_status(user_id, status) 首列是 user_id,
|
||||
-- 同样用不上(V26:20-21)。
|
||||
--
|
||||
-- 应用侧不写这一列:PriceTrackLoopRunEntity 没有 cleanup_at 字段,MyBatis-Plus 按实体列生成
|
||||
-- INSERT/UPDATE 的列清单,不会带上它。加 STORED 生成列需要重建表(ALGORITHM=COPY),
|
||||
-- 该表行数=循环批次数量级,且每行带 3 个 MEDIUMTEXT(V26:12-14),
|
||||
-- 上线前需在从库实测耗时再定执行窗口。
|
||||
ALTER TABLE biz_price_track_loop_run
|
||||
ADD COLUMN cleanup_at DATETIME GENERATED ALWAYS AS (COALESCE(finished_at, updated_at, created_at)) STORED
|
||||
COMMENT '保留清理时间线(生成列,由 finished_at/updated_at/created_at 兜底,应用勿写)',
|
||||
ALGORITHM = COPY;
|
||||
|
||||
ALTER TABLE biz_price_track_loop_run
|
||||
ADD KEY idx_loop_run_cleanup (status, cleanup_at),
|
||||
ALGORITHM = INPLACE, LOCK = NONE;
|
||||
@@ -26,16 +26,30 @@
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
|
||||
<fileNamePattern>${APP_LOG_ARCHIVE}</fileNamePattern>
|
||||
<maxFileSize>50MB</maxFileSize>
|
||||
<maxHistory>1</maxHistory>
|
||||
<!-- 原先 maxHistory=1 且 cleanHistoryOnStart=true:每次发版重启就把一天以上的
|
||||
历史日志抹掉,跨天问题(跟价/外观任务 4~24 小时)事后无现场可查。
|
||||
磁盘占用由 totalSizeCap 兜住,不靠砍天数。 -->
|
||||
<maxHistory>14</maxHistory>
|
||||
<totalSizeCap>500MB</totalSizeCap>
|
||||
<cleanHistoryOnStart>true</cleanHistoryOnStart>
|
||||
<cleanHistoryOnStart>false</cleanHistoryOnStart>
|
||||
</rollingPolicy>
|
||||
</appender>
|
||||
|
||||
<!-- 文件落盘改异步:高频调度日志(1s/2s 级)此前同步占住业务线程,
|
||||
而巡检的日志速率判据正是靠这些行。discardingThreshold=0 表示队列满时
|
||||
宁可阻塞调用线程也不丢日志——丢日志会让"最近时刻"判据假死。 -->
|
||||
<appender name="FILE_ASYNC" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<queueSize>8192</queueSize>
|
||||
<discardingThreshold>0</discardingThreshold>
|
||||
<neverBlock>false</neverBlock>
|
||||
<includeCallerData>false</includeCallerData>
|
||||
<appender-ref ref="FILE"/>
|
||||
</appender>
|
||||
|
||||
<logger name="com.nanri.aiimage" level="${APP_LOG_LEVEL}"/>
|
||||
|
||||
<root level="${ROOT_LOG_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
<appender-ref ref="FILE"/>
|
||||
<appender-ref ref="FILE_ASYNC"/>
|
||||
</root>
|
||||
</configuration>
|
||||
|
||||
+6
-4
@@ -27,7 +27,7 @@ class ImageVideoArchiveServiceTest {
|
||||
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
|
||||
OssStorageService ossStorageService = mock(OssStorageService.class);
|
||||
ImageVideoArchiveService archiveService = new ImageVideoArchiveService(
|
||||
taskMapper, ossStorageService, new ImageVideoProperties(), new ObjectMapper());
|
||||
taskMapper, ossStorageService, new ImageVideoProperties(), new ObjectMapper(), mock(com.nanri.aiimage.common.service.DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS));
|
||||
ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity();
|
||||
task.setId(10L);
|
||||
task.setTaskType(ImageVideoArchiveService.TASK_TYPE);
|
||||
@@ -46,7 +46,7 @@ class ImageVideoArchiveServiceTest {
|
||||
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
|
||||
OssStorageService ossStorageService = mock(OssStorageService.class);
|
||||
ImageVideoArchiveService archiveService = new ImageVideoArchiveService(
|
||||
taskMapper, ossStorageService, new ImageVideoProperties(), new ObjectMapper());
|
||||
taskMapper, ossStorageService, new ImageVideoProperties(), new ObjectMapper(), mock(com.nanri.aiimage.common.service.DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS));
|
||||
ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity();
|
||||
task.setId(11L);
|
||||
task.setTaskType(ImageVideoArchiveService.TASK_TYPE);
|
||||
@@ -82,7 +82,8 @@ class ImageVideoArchiveServiceTest {
|
||||
mock(ImageVideoAsyncTaskMapper.class),
|
||||
mock(OssStorageService.class),
|
||||
new ImageVideoProperties(),
|
||||
objectMapper);
|
||||
objectMapper,
|
||||
mock(com.nanri.aiimage.common.service.DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS));
|
||||
ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity();
|
||||
task.setTaskType(ImageVideoArchiveService.TASK_TYPE);
|
||||
|
||||
@@ -106,7 +107,8 @@ class ImageVideoArchiveServiceTest {
|
||||
mock(ImageVideoAsyncTaskMapper.class),
|
||||
mock(OssStorageService.class),
|
||||
new ImageVideoProperties(),
|
||||
new ObjectMapper());
|
||||
new ObjectMapper(),
|
||||
mock(com.nanri.aiimage.common.service.DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS));
|
||||
ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity();
|
||||
task.setTaskType(ImageVideoArchiveService.TASK_TYPE);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user