diff --git a/backend-java/pom.xml b/backend-java/pom.xml index f897b13ea..859fc62b1 100644 --- a/backend-java/pom.xml +++ b/backend-java/pom.xml @@ -185,6 +185,19 @@ + + org.apache.maven.plugins + maven-jar-plugin + + + + db/aiimage.sql + + + org.apache.maven.plugins maven-surefire-plugin diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index 72a4e26e3..2e09a45b2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -121,6 +121,9 @@ public class AppearancePatentTaskService { private static final String CONTENT_TYPE_ZIP = "application/zip"; private static final int CHUNK_PAYLOAD_MERGE_RETRY_LIMIT = 3; private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3; + + /** 结果行读取时 chunk 的 keyset 页大小,与 similarasin 侧 loadChunksKeyset 的 500 同口径。 */ + private static final int RESULT_ROWS_CHUNK_PAGE_SIZE = 500; private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L; private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5); private static final long TASK_LOCK_WAIT_MILLIS = 10000L; @@ -2538,7 +2541,7 @@ public class AppearancePatentTaskService { .eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) .orderByAsc(TaskChunkEntity::getId) - .last("LIMIT 1"); + .last("LIMIT " + RESULT_ROWS_CHUNK_PAGE_SIZE); if (lastChunkId != null) { query.gt(TaskChunkEntity::getId, lastChunkId); } @@ -2546,10 +2549,14 @@ public class AppearancePatentTaskService { if (page == null || page.isEmpty()) { break; } - TaskChunkEntity chunk = page.getFirst(); - appendRequestedResultRows(result, readChunkRows(chunk).values(), requestedKeys); - lastChunkId = chunk.getId(); - if (lastChunkId == null) { + // 整页取 chunk 后逐片解载荷:原先每轮 LIMIT 1 只取一行,一个结果文件要把该任务的 + // chunk 表逐行翻完,每行一次 SQL + 一次对象读 + 一次整片 JSON 反序列化 + for (TaskChunkEntity chunk : page) { + appendRequestedResultRows(result, readChunkRows(chunk).values(), requestedKeys); + } + TaskChunkEntity tail = page.getLast(); + lastChunkId = tail == null ? null : tail.getId(); + if (lastChunkId == null || page.size() < RESULT_ROWS_CHUNK_PAGE_SIZE) { break; } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoArchiveService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoArchiveService.java index a762dbd72..3b3fa3c14 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoArchiveService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoArchiveService.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.config.ImageVideoProperties; +import com.nanri.aiimage.common.service.DistributedJobLockService; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.imagevideo.mapper.ImageVideoAsyncTaskMapper; import com.nanri.aiimage.modules.imagevideo.model.entity.ImageVideoAsyncTaskEntity; @@ -46,6 +47,7 @@ public class ImageVideoArchiveService { private final OssStorageService ossStorageService; private final ImageVideoProperties properties; private final ObjectMapper objectMapper; + private final com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService; void captureSubmitResponse(ImageVideoAsyncTaskEntity task, Object response) { if (!isWorkflowTask(task) || response == null) { @@ -107,22 +109,32 @@ public class ImageVideoArchiveService { @Scheduled(fixedDelayString = "${aiimage.image-video.retention-cleanup-delay-ms:600000}") public void cleanupExpiredWorkflowTasks() { - LocalDateTime cutoff = LocalDateTime.now().minusDays(RETENTION_DAYS); - List expiredTasks = taskMapper.selectList( - new LambdaQueryWrapper() - .eq(ImageVideoAsyncTaskEntity::getTaskType, TASK_TYPE) - .lt(ImageVideoAsyncTaskEntity::getSubmittedAt, cutoff) - .orderByAsc(ImageVideoAsyncTaskEntity::getId) - .last("LIMIT " + RETENTION_CLEANUP_BATCH_SIZE)); - int deleted = 0; - for (ImageVideoAsyncTaskEntity task : expiredTasks) { - if (!deleteArchivedObjects(task)) { - continue; - } - deleted += taskMapper.deleteById(task.getId()); + // 归档认领有 CAS 兜底,但过期清理没有:两个节点各自 selectList 拿到同一批 200 行后 + // 会重复删同一批 OSS 对象、重复 deleteById,且任一侧中途失败都会给对方留下 + // 「对象已删、行还在」的半截状态。锁 TTL 给到 20 分钟,覆盖最慢的一批远端删除。 + DistributedJobLockService.LockHandle jobLock = distributedJobLockService.tryLock( + "image-video:retention-cleanup", java.time.Duration.ofMinutes(20)); + if (jobLock == null) { + return; } - if (deleted > 0) { - log.info("[image-video] expired workflow task cleanup deleted={} cutoff={}", deleted, cutoff); + try (jobLock) { + LocalDateTime cutoff = LocalDateTime.now().minusDays(RETENTION_DAYS); + List expiredTasks = taskMapper.selectList( + new LambdaQueryWrapper() + .eq(ImageVideoAsyncTaskEntity::getTaskType, TASK_TYPE) + .lt(ImageVideoAsyncTaskEntity::getSubmittedAt, cutoff) + .orderByAsc(ImageVideoAsyncTaskEntity::getId) + .last("LIMIT " + RETENTION_CLEANUP_BATCH_SIZE)); + int deleted = 0; + for (ImageVideoAsyncTaskEntity task : expiredTasks) { + if (!deleteArchivedObjects(task)) { + continue; + } + deleted += taskMapper.deleteById(task.getId()); + } + if (deleted > 0) { + log.info("[image-video] expired workflow task cleanup deleted={} cutoff={}", deleted, cutoff); + } } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java index 330bc49fe..f4a30684d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoAsyncTaskService.java @@ -238,13 +238,23 @@ public class ImageVideoAsyncTaskService { @Scheduled(fixedDelayString = "${aiimage.image-video.failed-task-cleanup-delay-ms:60000}") public void cleanupFailedTasks() { - LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES); - int deleted = taskMapper.delete(new LambdaQueryWrapper() - .eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.FAILED.name()) - .lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff) - .last("LIMIT " + EXPIRED_DELETE_BATCH_SIZE)); - if (deleted > 0) { - log.info("[image-video] removed expired failed tasks count={}", deleted); + // 本方法是全类唯一直接对共享表发 DELETE ... LIMIT 的定时任务:两节点各跑一份时会 + // 命中同一批 (status, updated_at) 区间并互相等行锁。扫描/轮询/接管三条已用任务锁, + // 这里补齐;锁超时后另一节点接管,删除本身幂等,不会漏清也不会重复扣。 + DistributedJobLockService.LockHandle jobLock = + distributedJobLockService.tryLock("image-video:failed-task-cleanup", java.time.Duration.ofSeconds(120)); + if (jobLock == null) { + return; + } + try (jobLock) { + LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES); + int deleted = taskMapper.delete(new LambdaQueryWrapper() + .eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.FAILED.name()) + .lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff) + .last("LIMIT " + EXPIRED_DELETE_BATCH_SIZE)); + if (deleted > 0) { + log.info("[image-video] removed expired failed tasks count={}", deleted); + } } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/mapper/PriceTrackLoopRunMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/mapper/PriceTrackLoopRunMapper.java index 09b9a10c7..65bc58188 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/mapper/PriceTrackLoopRunMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/mapper/PriceTrackLoopRunMapper.java @@ -15,12 +15,14 @@ public interface PriceTrackLoopRunMapper extends BaseMapper只删终态行:RUNNING 的批次删了会让后续 dispatch / childFinished 找不到记录。 - * 时间线用 COALESCE 兜底——终态行本应写 finished_at,历史行可能只有 updated_at。 + * 时间线用 V141 的 STORED 生成列 {@code cleanup_at}(= COALESCE(finished_at, updated_at, + * created_at),口径与改造前完全一致):原来这个表达式直接写在 WHERE 里不可走索引, + * 每条 DELETE 都要全表扫描并对扫过的行加锁,而 (status, cleanup_at) 索引已就位。 */ @Delete(""" DELETE FROM biz_price_track_loop_run WHERE status IN ('SUCCESS', 'FAILED', 'STOPPED') - AND COALESCE(finished_at, updated_at, created_at) < #{cutoff} + AND cleanup_at < #{cutoff} LIMIT #{batchSize} """) int deleteFinishedBefore(@Param("cutoff") LocalDateTime cutoff, @Param("batchSize") int batchSize); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackLoopRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackLoopRunService.java index 57ea03d89..9d7201fed 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackLoopRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackLoopRunService.java @@ -169,6 +169,14 @@ public class PriceTrackLoopRunService { return toVo(entity); } + /** + * 把刚创建的子任务认领为循环任务的当前子任务。 + * + *

认领失败一律抛异常而不是静默返回:调用方 {@code PriceTrackTaskService.createTask} + * 与本方法同处一个事务(默认传播 REQUIRED),抛出会让刚 insert 的子任务连同扣次一起回滚。 + * 原先各处判不到就 {@code return},子任务照样建成并真实执行——同一个店铺槽位跑出两个 + * RUNNING 子任务(重复采集、重复扣用户次数),且第一个失去跟踪。 + */ @Transactional public void bindChildTask(Long loopRunId, Long childTaskId, Integer roundIndex, Integer shopIndex) { if (loopRunId == null || childTaskId == null) { @@ -176,26 +184,35 @@ public class PriceTrackLoopRunService { } PriceTrackLoopRunEntity entity = loopRunMapper.selectById(loopRunId); if (entity == null || isTerminal(entity.getStatus())) { - return; + throw new BusinessException("循环任务已结束或不存在,子任务未绑定"); } if (entity.getCurrentRound() == null || entity.getCurrentShopIndex() == null) { - return; + throw new BusinessException("循环任务轮次状态缺失,子任务未绑定"); } if (!entity.getCurrentRound().equals(roundIndex) || !entity.getCurrentShopIndex().equals(shopIndex)) { - return; + throw new BusinessException("循环任务轮次或店铺指针已推进,本次派发已作废"); } if (entity.getActiveTaskId() != null && !isChildTaskTerminal(entity.getActiveTaskId())) { - return; + throw new BusinessException("当前仍有执行中的子任务,本次派发已作废"); } - entity.setActiveTaskId(childTaskId); - entity.setUpdatedAt(LocalDateTime.now()); // 字段级更新 + status CAS:整行 updateById 会覆盖并发写入的 stop_requested, // 导致「停止循环」请求被静默丢弃 - loopRunMapper.update(null, new LambdaUpdateWrapper() + // 认领槽位必须连带 CAS 掉读取时的 active_task_id:只比 status 时两个并发派发都能通过 + // (status 没变),后写的那个把先绑的子任务顶掉,两个 RUNNING 子任务同时真实执行。 + Long expectedActiveTaskId = entity.getActiveTaskId(); + LambdaUpdateWrapper claim = new LambdaUpdateWrapper() .eq(PriceTrackLoopRunEntity::getId, entity.getId()) .eq(PriceTrackLoopRunEntity::getStatus, entity.getStatus()) .set(PriceTrackLoopRunEntity::getActiveTaskId, childTaskId) - .set(PriceTrackLoopRunEntity::getUpdatedAt, LocalDateTime.now())); + .set(PriceTrackLoopRunEntity::getUpdatedAt, LocalDateTime.now()); + if (expectedActiveTaskId == null) { + claim.isNull(PriceTrackLoopRunEntity::getActiveTaskId); + } else { + claim.eq(PriceTrackLoopRunEntity::getActiveTaskId, expectedActiveTaskId); + } + if (loopRunMapper.update(null, claim) == 0) { + throw new BusinessException("循环任务槽位已被并发派发认领,本次派发已作废"); + } log.info("[price-track-loop] child bound loopRunId={} childTaskId={} roundIndex={} shopIndex={}", loopRunId, childTaskId, roundIndex, shopIndex); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java index 67e7162ef..89e5e904c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java @@ -460,7 +460,6 @@ public class PriceTrackTaskService { return result; } - @Transactional public PriceTrackCreateTaskVo createTask(PriceTrackCreateTaskRequest request, String deviceId) { if (request.isStatusMode() == request.isAsinMode()) throw new BusinessException("跟价模式必须二选一"); if (request.isAsinMode() && (request.getAsinFiles() == null || request.getAsinFiles().isEmpty())) { @@ -490,91 +489,98 @@ public class PriceTrackTaskService { log.info("[price-track] createTask skipAsins countries={} asinMode={}", skipAsinsByCountry.keySet(), request.isAsinMode()); - // 创建任务记录 - FileTaskEntity task = new FileTaskEntity(); - task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr()); - task.setModuleType(MODULE_TYPE); - task.setTaskMode("PYTHON_QUEUE"); - task.setStatus("RUNNING"); - task.setSourceFileCount(uniqueItems.size()); - task.setSuccessFileCount(0); - task.setFailedFileCount(0); - task.setCreatedBy("user:" + request.getUserId()); - task.setUserId(request.getUserId()); - task.setDeviceId(AdminAuthSupport.normalizeDeviceId(deviceId)); - task.setCreatedAt(LocalDateTime.now()); - task.setUpdatedAt(LocalDateTime.now()); - fileTaskMapper.insert(task); - priceTrackTaskCacheService.touchTaskHeartbeat(task.getId()); - if (request.isAsinMode()) { - saveTaskAsinRowsPayload(task.getId(), asinRowsForPayload); - } - if (request.getLoopRunId() != null) { - priceTrackLoopRunService.bindChildTask(request.getLoopRunId(), task.getId(), request.getRoundIndex(), request.getShopIndex()); - } - - List snapshot = new ArrayList<>(); - // 逐店铺 fileResultMapper.insert 看似该改批量(500 店铺 = 500 次往返),但**不能改**: - // 下面的 toSnapshotVo 依赖 MyBatis-Plus insert 回填的自增主键(fr.getId() → vo.resultId), - // 而本仓的手写 INSERT ... VALUES (...),(...) 批量写法不回填主键,改了会让快照 resultId 为 null。 - // 要批量化必须同时解决「批量插入 + 按序回填 id」(MySQL 需 useGeneratedKeys + 顺序保证)。 - for (PriceTrackMatchShopsVo.PriceTrackShopQueueItem item : uniqueItems) { - String norm = ziniaoShopSwitchService.normalizeShopName(item.getShopName()); - if (norm.isBlank()) throw new BusinessException("店铺名称无效"); - FileResultEntity fr = new FileResultEntity(); - fr.setTaskId(task.getId()); - fr.setModuleType(MODULE_TYPE); - fr.setSourceFilename(norm); - fr.setSourceFileUrl(Objects.toString(item.getShopId(), null)); - fr.setUserId(request.getUserId()); - fr.setSuccess(0); - fr.setCreatedAt(LocalDateTime.now()); - fileResultMapper.insert(fr); - PriceTrackResultItemVo snapshotItem = toSnapshotVo(fr, item, task.getStatus()); - snapshotItem.setLoopRunId(request.getLoopRunId()); - snapshotItem.setRoundIndex(request.getRoundIndex()); - snapshot.add(snapshotItem); - } - - // 保存请求上下文快照,供 Python 消费和前端轮询展示使用 - try { - Map ctx = new LinkedHashMap<>(); - ctx.put("userId", request.getUserId()); - ctx.put("statusMode", request.isStatusMode()); - ctx.put("asinMode", request.isAsinMode()); - ctx.put("countryCodes", request.getCountryCodes()); - // 不再保存全量 ASIN 数据到 requestJson - // ctx.put("skipAsinsByCountry", skipAsinsByCountry); - // ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry); - // ctx.put("asinRowsByCountry", asinRowsByCountry); - // ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin); - // 文件模式:保存文件路径,供后续分页读取 + // Excel/CSV 解析(POI 全表读,可达秒级)刻意留在事务之外:事务里只做 DB 写与载荷落库, + // 避免创建跟价任务时长时间占住连接池连接(本仓历史上 biz_task_file_job 一类锁等 + // 就是被「事务内含外部 IO」放大的)。载荷 PUT 仍留在事务内:它失败必须整体回滚, + // 不能让「任务已建、ASIN 载荷为空」的半成品进入 Python 队列。 + final Map>> parsedAsinRows = asinRowsForPayload; + return inRequiredTransaction(() -> { + // 创建任务记录 + FileTaskEntity task = new FileTaskEntity(); + task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr()); + task.setModuleType(MODULE_TYPE); + task.setTaskMode("PYTHON_QUEUE"); + task.setStatus("RUNNING"); + task.setSourceFileCount(uniqueItems.size()); + task.setSuccessFileCount(0); + task.setFailedFileCount(0); + task.setCreatedBy("user:" + request.getUserId()); + task.setUserId(request.getUserId()); + task.setDeviceId(AdminAuthSupport.normalizeDeviceId(deviceId)); + task.setCreatedAt(LocalDateTime.now()); + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.insert(task); + priceTrackTaskCacheService.touchTaskHeartbeat(task.getId()); if (request.isAsinMode()) { - ctx.put("asinFiles", request.getAsinFiles()); + saveTaskAsinRowsPayload(task.getId(), parsedAsinRows); + } + if (request.getLoopRunId() != null) { + priceTrackLoopRunService.bindChildTask(request.getLoopRunId(), task.getId(), request.getRoundIndex(), request.getShopIndex()); } - ctx.put("items", uniqueItems); - ctx.put("loopRunId", request.getLoopRunId()); - ctx.put("roundIndex", request.getRoundIndex()); - ctx.put("shopIndex", request.getShopIndex()); - task.setRequestJson(objectMapper.writeValueAsString(ctx)); - task.setResultJson(objectMapper.writeValueAsString(snapshot)); - } catch (Exception ex) { - throw new BusinessException("序列化任务数据失败"); - } - fileTaskMapper.updateById(task); - priceTrackTaskCacheService.saveTaskCache(task); - // TODO: Python 侧轮询 RUNNING 任务,pickup 后开始处理 - // Java 侧通过 submitResult 接口接收 Python 回传结果 + List snapshot = new ArrayList<>(); + // 逐店铺 fileResultMapper.insert 看似该改批量(500 店铺 = 500 次往返),但**不能改**: + // 下面的 toSnapshotVo 依赖 MyBatis-Plus insert 回填的自增主键(fr.getId() → vo.resultId), + // 而本仓的手写 INSERT ... VALUES (...),(...) 批量写法不回填主键,改了会让快照 resultId 为 null。 + // 要批量化必须同时解决「批量插入 + 按序回填 id」(MySQL 需 useGeneratedKeys + 顺序保证)。 + for (PriceTrackMatchShopsVo.PriceTrackShopQueueItem item : uniqueItems) { + String norm = ziniaoShopSwitchService.normalizeShopName(item.getShopName()); + if (norm.isBlank()) throw new BusinessException("店铺名称无效"); + FileResultEntity fr = new FileResultEntity(); + fr.setTaskId(task.getId()); + fr.setModuleType(MODULE_TYPE); + fr.setSourceFilename(norm); + fr.setSourceFileUrl(Objects.toString(item.getShopId(), null)); + fr.setUserId(request.getUserId()); + fr.setSuccess(0); + fr.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(fr); + PriceTrackResultItemVo snapshotItem = toSnapshotVo(fr, item, task.getStatus()); + snapshotItem.setLoopRunId(request.getLoopRunId()); + snapshotItem.setRoundIndex(request.getRoundIndex()); + snapshot.add(snapshotItem); + } - PriceTrackCreateTaskVo vo = new PriceTrackCreateTaskVo(); - vo.setTaskId(task.getId()); - vo.setItems(snapshot); - vo.setSkipAsinsByCountry(skipAsinsByCountry); - vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry); - vo.setAsinRowsByCountry(asinRowsByCountry); - vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin); - return vo; + // 保存请求上下文快照,供 Python 消费和前端轮询展示使用 + try { + Map ctx = new LinkedHashMap<>(); + ctx.put("userId", request.getUserId()); + ctx.put("statusMode", request.isStatusMode()); + ctx.put("asinMode", request.isAsinMode()); + ctx.put("countryCodes", request.getCountryCodes()); + // 不再保存全量 ASIN 数据到 requestJson + // ctx.put("skipAsinsByCountry", skipAsinsByCountry); + // ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry); + // ctx.put("asinRowsByCountry", asinRowsByCountry); + // ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin); + // 文件模式:保存文件路径,供后续分页读取 + if (request.isAsinMode()) { + ctx.put("asinFiles", request.getAsinFiles()); + } + ctx.put("items", uniqueItems); + ctx.put("loopRunId", request.getLoopRunId()); + ctx.put("roundIndex", request.getRoundIndex()); + ctx.put("shopIndex", request.getShopIndex()); + task.setRequestJson(objectMapper.writeValueAsString(ctx)); + task.setResultJson(objectMapper.writeValueAsString(snapshot)); + } catch (Exception ex) { + throw new BusinessException("序列化任务数据失败"); + } + fileTaskMapper.updateById(task); + priceTrackTaskCacheService.saveTaskCache(task); + + // TODO: Python 侧轮询 RUNNING 任务,pickup 后开始处理 + // Java 侧通过 submitResult 接口接收 Python 回传结果 + + PriceTrackCreateTaskVo vo = new PriceTrackCreateTaskVo(); + vo.setTaskId(task.getId()); + vo.setItems(snapshot); + vo.setSkipAsinsByCountry(skipAsinsByCountry); + vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry); + vo.setAsinRowsByCountry(asinRowsByCountry); + vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin); + return vo; + }); } public void submitResult(Long taskId, PriceTrackSubmitResultRequest request) { @@ -2263,6 +2269,17 @@ public class PriceTrackTaskService { return template.execute(status -> action.get()); } + /** + * 默认传播(REQUIRED)事务段:供「外部 IO 已移出、只剩 DB 写」的方法体使用。 + * 与 {@link #inNewTransaction} 的差别是不挂起外层事务;单测直接 new 本类时退化为直调。 + */ + private T inRequiredTransaction(java.util.function.Supplier action) { + if (transactionManager == null) { + return action.get(); + } + return new TransactionTemplate(transactionManager).execute(status -> action.get()); + } + private TaskDistributedLockService.LockHandle acquireTaskLock(Long taskId) { return TaskFinalizeSupport.acquireTaskLock(taskDistributedLockService, MODULE_TYPE, taskId); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopKeyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopKeyService.java index e63d3e9b2..a7f972f81 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopKeyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopKeyService.java @@ -17,6 +17,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -117,8 +119,12 @@ public class ShopKeyService { /** * 后台手动检测白名单:绕过缓存真实打一次紫鸟接口,并重置自动重试次数。 * 白名单放行需紫鸟公司侧授权,自动重试达阈值后会停试,此按钮供授权后重新验证。 + * + *

不加 {@code @Transactional}:本方法只有一次单语句 UPDATE,没有需要原子提交的多元写, + * 而 {@code probeApiKeyUpstream} 是真实出网的紫鸟调用(上游代理挂起时按超时计可达数十秒)。 + * 挂在事务上等于把连接池的连接按上游 RTT 扣住,多次点击即可耗尽池——这正是本仓 + * 「事务内含外部 IO」要避免的形态。 */ - @Transactional public ShopKeyItemVo checkWhitelist(Long id) { ShopKeyEntity entity = getById(id); ziniaoApiKeyProvider.invalidateProxyUrlCache(); @@ -214,7 +220,27 @@ public class ShopKeyService { return vo; } + /** + * 密钥变更后重刷紫鸟店铺索引。 + * + * invalidateIndex() 是「删 1 张持久表全部索引行 + 清 7 类瞬时缓存」的重操作, + * 放在 create/update/delete 的事务里等于把一次单行写放大成几十条 DELETE 的持锁时间, + * 而缓存本来就不参与回滚。因此:有事务时挂到提交之后执行,没有事务时照旧立即执行。 + */ private void triggerShopIndexRefresh() { + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + invalidateIndexSafely(); + } + }); + return; + } + invalidateIndexSafely(); + } + + private void invalidateIndexSafely() { try { ziniaoShopIndexService.invalidateIndex(); } catch (Exception ex) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinImagePrefetchService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinImagePrefetchService.java index 0abe69537..4bdd4cc35 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinImagePrefetchService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinImagePrefetchService.java @@ -58,6 +58,9 @@ public class SimilarAsinImagePrefetchService { /** 批量命中检查/批量 touch 单次 IN 上限,避免超长 SQL 参数列表。 */ private static final int CACHE_LOOKUP_BATCH_SIZE = 500; + /** url_hash 十六进制编码用:逐字节 String.format 在按 URL 调用的路径上会放大成几万次格式化。 */ + private static final java.util.HexFormat HEX_FORMAT = java.util.HexFormat.of(); + /** 单个 task 最多缓存的待预热 URL 数,避免持续回传时内存无界增长。 */ private static final int MAX_PENDING_URLS_PER_TASK = 5_000; /** 单机最多同时维护的预热 task 数。 */ @@ -316,7 +319,7 @@ public class SimilarAsinImagePrefetchService { * 未命中 url 不产生任何 touch。输入按传入顺序返回,未命中为 null。 * 失败/开关关闭返回空列表,由调用方走回退路径。 */ - List lookupBatch(List urls) { + public List lookupBatch(List urls) { List result = new ArrayList<>(); if (!properties.isImageDbCacheEnabled()) { return result; @@ -415,12 +418,7 @@ public class SimilarAsinImagePrefetchService { private static String sha256Hex(String value) { try { MessageDigest md = MessageDigest.getInstance("SHA-256"); - byte[] digest = md.digest(value.getBytes(StandardCharsets.UTF_8)); - StringBuilder sb = new StringBuilder(digest.length * 2); - for (byte b : digest) { - sb.append(String.format("%02x", b & 0xFF)); - } - return sb.toString(); + return HEX_FORMAT.formatHex(md.digest(value.getBytes(StandardCharsets.UTF_8))); } catch (NoSuchAlgorithmException ex) { // SHA-256 在 JDK 中是必备算法,正常环境下不会落到这里。 return null; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java index 9f1abdd93..03a504d1b 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -132,6 +132,8 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost { private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_ZIP = "application/zip"; private static final int MAX_LIGHT_TASK_IDS = 200; + /** progressBatch 的 IN 子句上限:看板单次轮询的在册任务量级远小于它,取值留了 5 倍余量。 */ + private static final int MAX_BATCH_TASK_IDS = 1000; /** Task 19:Python 回传逐行日志采样频率(每 N 行记一行)。 */ private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L; private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5); @@ -657,10 +659,19 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost { public SimilarAsinTaskBatchVo progressBatch(List taskIds) { SimilarAsinTaskBatchVo vo = new SimilarAsinTaskBatchVo(); - List normalizedIds = taskIds == null ? List.of() : taskIds.stream() + List requestedIds = taskIds == null ? List.of() : taskIds.stream() .filter(id -> id != null && id > 0) .distinct() .toList(); + // 上限只为挡住超长 IN 子句。被截的尾部不进 missingTaskIds(与 progressLight 同口径: + // 看板会把 missing 当成「这张单不存在」,而它们只是本轮没查),改为记 WARN 留现场。 + if (requestedIds.size() > MAX_BATCH_TASK_IDS) { + log.warn("[similar-asin] progressBatch 收到 {} 个 id,超过上限 {},本轮只刷新前 {} 个", + requestedIds.size(), MAX_BATCH_TASK_IDS, MAX_BATCH_TASK_IDS); + } + List normalizedIds = requestedIds.size() > MAX_BATCH_TASK_IDS + ? requestedIds.subList(0, MAX_BATCH_TASK_IDS) + : requestedIds; if (normalizedIds.isEmpty()) { return vo; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinResultWorkbookAssembler.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinResultWorkbookAssembler.java index 7818ffa4e..b1599de34 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinResultWorkbookAssembler.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinResultWorkbookAssembler.java @@ -91,6 +91,9 @@ public class SimilarAsinResultWorkbookAssembler { private static final int IMG_COL_PUZZLE1 = 13; private static final int IMG_COL_PUZZLE2 = 14; private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3; + + /** 结果行读取时 chunk / orphan scope_state 的 keyset 页大小,与 loadChunksKeyset 的 500 同口径。 */ + private static final int RESULT_ROWS_PAGE_SIZE = 500; private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L; private final SimilarAsinProperties properties; @@ -562,6 +565,11 @@ public class SimilarAsinResultWorkbookAssembler { /** * Select and decode one chunk payload at a time. Only rows needed by the * current source workbook are retained in the returned map. + * + *

按 keyset 整页取 chunk(与 {@link SimilarAsinChunkMergeSupport#loadChunksKeyset} 同口径): + * 原先每轮 {@code LIMIT 1} 只取一行,一个源文件要把该任务的 chunk 表逐行翻完, + * 每行一次 SQL + 一次对象读 + 一次整片 JSON 反序列化;请求行全部命中后立即收尾, + * 不再把剩余 chunk 也解一遍。 */ private Map loadPersistedResultRowsForRows( Long taskId, @@ -578,7 +586,7 @@ public class SimilarAsinResultWorkbookAssembler { .eq(TaskChunkEntity::getTaskId, taskId) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) .orderByAsc(TaskChunkEntity::getId) - .last("LIMIT 1"); + .last("LIMIT " + RESULT_ROWS_PAGE_SIZE); if (lastChunkId != null) { query.gt(TaskChunkEntity::getId, lastChunkId); } @@ -586,10 +594,16 @@ public class SimilarAsinResultWorkbookAssembler { if (page == null || page.isEmpty()) { break; } - TaskChunkEntity chunk = page.getFirst(); - appendRequestedResultRows(result, chunkPayloadSupport.readChunkRows(chunk).values(), requestedKeys, false); - lastChunkId = chunk.getId(); - if (lastChunkId == null) { + for (TaskChunkEntity chunk : page) { + appendRequestedResultRows(result, chunkPayloadSupport.readChunkRows(chunk).values(), + requestedKeys, false); + } + TaskChunkEntity tail = page.getLast(); + lastChunkId = tail == null ? null : tail.getId(); + if (lastChunkId == null || page.size() < RESULT_ROWS_PAGE_SIZE) { + break; + } + if (allRequestedRowsResolved(requestedRows, result)) { break; } } @@ -597,6 +611,21 @@ public class SimilarAsinResultWorkbookAssembler { return result; } + /** + * 请求行是否都已取到结果。判据必须与 {@link SimilarAsinChunkMergeSupport#findResultRow} + * 的匹配口径一致(主键或 legacy 键任一命中即算已取到)——直接用 result.size() 与 + * requestedKeys.size() 比会永远不成立,因为 requestedKeys 每行都塞了主键 + legacy 两个键。 + */ + private boolean allRequestedRowsResolved(List requestedRows, + Map result) { + for (SimilarAsinParsedRowVo row : requestedRows) { + if (SimilarAsinChunkMergeSupport.findResultRow(row, result) == null) { + return false; + } + } + return true; + } + private Set resultLookupKeys(List rows) { Set keys = new LinkedHashSet<>(); for (SimilarAsinParsedRowVo row : rows == null ? List.of() : rows) { @@ -651,7 +680,7 @@ public class SimilarAsinResultWorkbookAssembler { .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) .likeRight(TaskScopeStateEntity::getScopeKey, SimilarAsinChunkPayloadSupport.ORPHAN_SCOPE_KEY_PREFIX) .orderByAsc(TaskScopeStateEntity::getId) - .last("LIMIT 1"); + .last("LIMIT " + RESULT_ROWS_PAGE_SIZE); if (lastStateId != null) { query.gt(TaskScopeStateEntity::getId, lastStateId); } @@ -659,29 +688,33 @@ public class SimilarAsinResultWorkbookAssembler { if (page == null || page.isEmpty()) { return; } - TaskScopeStateEntity state = page.getFirst(); - lastStateId = state.getId(); - if (lastStateId == null) { - return; - } - if (state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) { - continue; - } - try { - String payloadJson = transientPayloadStorageService.resolvePayload( - state.getParsedPayloadJson(), "read similar ASIN orphan rows failed"); - JsonNode array = objectMapper.readTree(payloadJson); - if (!array.isArray()) { + TaskScopeStateEntity tail = page.getLast(); + lastStateId = tail == null ? null : tail.getId(); + boolean lastPage = page.size() < RESULT_ROWS_PAGE_SIZE || lastStateId == null; + for (TaskScopeStateEntity state : page) { + if (state == null || state.getParsedPayloadJson() == null + || state.getParsedPayloadJson().isBlank()) { continue; } - List matchingRows = new ArrayList<>(); - for (JsonNode node : array) { - matchingRows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class)); + try { + String payloadJson = transientPayloadStorageService.resolvePayload( + state.getParsedPayloadJson(), "read similar ASIN orphan rows failed"); + JsonNode array = objectMapper.readTree(payloadJson); + if (!array.isArray()) { + continue; + } + List matchingRows = new ArrayList<>(); + for (JsonNode node : array) { + matchingRows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class)); + } + appendRequestedResultRows(sink, matchingRows, requestedKeys, true); + } catch (Exception ex) { + log.warn("[similar-asin] read requested orphan rows failed taskId={} stateId={} err={}", + taskId, state.getId(), ex.getMessage()); } - appendRequestedResultRows(sink, matchingRows, requestedKeys, true); - } catch (Exception ex) { - log.warn("[similar-asin] read requested orphan rows failed taskId={} stateId={} err={}", - taskId, state.getId(), ex.getMessage()); + } + if (lastPage) { + return; } } } @@ -788,14 +821,23 @@ public class SimilarAsinResultWorkbookAssembler { if (!prefetchUrls.isEmpty()) { // DB cache 命中的缩略图也立即落到任务临时目录,避免整批字节停留在堆中。 long dbCacheStart = System.currentTimeMillis(); - List remainingUrls = new ArrayList<>(prefetchUrls.size()); - int dbHit = 0; + // 一次 IN 批量查(lookupBatch 内部只对命中项 touch last_used_at): + // 原先逐 URL 调 lookup(),按注释里的量级(4 源 × 4000 URL)每次装配要发几千条单行 SQL。 + Set pendingUrlSet = new LinkedHashSet<>(); for (String url : prefetchUrls) { - ensureResultAssemblyNotInterrupted(); if (url == null || imageSpool.get(url) != null || taskImageCache.containsKey(url)) { continue; } - byte[] cachedBytes = imagePrefetchService.lookup(url); + pendingUrlSet.add(url); + } + List pendingUrls = new ArrayList<>(pendingUrlSet); + List cachedBatch = imagePrefetchService.lookupBatch(pendingUrls); + List remainingUrls = new ArrayList<>(); + int dbHit = 0; + for (int i = 0; i < pendingUrls.size(); i++) { + ensureResultAssemblyNotInterrupted(); + String url = pendingUrls.get(i); + byte[] cachedBytes = i < cachedBatch.size() ? cachedBatch.get(i) : null; if (cachedBytes == null) { remainingUrls.add(url); continue; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java index b4fa9c6a5..fbb639006 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java @@ -323,33 +323,32 @@ public class TransientPayloadStorageService { } /** - * 带 task_id 收敛的 chunk 引用计数;带 task 查不到时回退全局查询, - * 保证即使指针格式与预期不符也不会漏判引用(漏判会导致对象被误删)。 + * 带 task_id 收敛的 chunk 引用计数。 + * + *

只有在指针解析不出 taskId 时才退回全局查询。指针可解析时不再回落全局: + * 对象 key 里带 taskId、引用不跨任务(见 {@link #isStillReferenced} 的口径说明), + * 而物理删除又约定在 DB 行删除之后执行,所以带 task 查到 0 才是正常状态—— + * 原先「查到 0 就无条件再全局查一次」让每次删除都必然退化成 + * {@code payload_json}(JSON 列,无索引)上的全表扫描,而本方法是删除路径的公共链路且被逐行调用。 */ private Long referencedChunkCount(Long taskId, List values) { if (taskId != null) { - Long scoped = taskChunkMapper.selectCount(new LambdaQueryWrapper() + return taskChunkMapper.selectCount(new LambdaQueryWrapper() .eq(TaskChunkEntity::getTaskId, taskId) .in(TaskChunkEntity::getPayloadJson, values)); - if (scoped != null && scoped > 0L) { - return scoped; - } } return taskChunkMapper.selectCount(new LambdaQueryWrapper() .in(TaskChunkEntity::getPayloadJson, values)); } - /** 带 task_id 收敛的 scope_state 引用计数;回退逻辑同 {@link #referencedChunkCount}。 */ + /** 带 task_id 收敛的 scope_state 引用计数;回落口径同 {@link #referencedChunkCount}。 */ private Long referencedScopeStateCount(Long taskId, List values) { if (taskId != null) { - Long scoped = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() + return taskScopeStateMapper.selectCount(new LambdaQueryWrapper() .eq(TaskScopeStateEntity::getTaskId, taskId) .and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values) .or() .in(TaskScopeStateEntity::getStateJson, values))); - if (scoped != null && scoped > 0L) { - return scoped; - } } return taskScopeStateMapper.selectCount(new LambdaQueryWrapper() .and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/memory/service/ZiniaoMemoryStoreService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/memory/service/ZiniaoMemoryStoreService.java index 7ee5066c3..a0938bfd0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/memory/service/ZiniaoMemoryStoreService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/memory/service/ZiniaoMemoryStoreService.java @@ -76,16 +76,21 @@ public class ZiniaoMemoryStoreService { public List listByType(String cacheType, Class valueType, int limit) { String normalizedType = normalizeRequired(cacheType, "cacheType 不能为空"); int safeLimit = Math.max(limit, 1); + // 过期判断必须在 SQL 里做:原来先按 cache_key 取 LIMIT N 再在 Java 里滤掉过期行, + // 于是过期行会白占 N 这个窗口——同一 cache_type 存量超过 N 时,排在窗口之后的 + // 有效行根本取不到,调用方(店铺名索引兜底扫描)把「其实存在」判成未命中。 + // 同时过期行不再把 payload_json 整列拖回应用侧反序列化。 + // 走 uk_type_key(cache_type, cache_key):等值 + 排序都命中索引,expires_at 作回表过滤。 + LocalDateTime now = LocalDateTime.now(); List entities = ziniaoMemoryStoreMapper.selectList(new LambdaQueryWrapper() .eq(ZiniaoMemoryStoreEntity::getCacheType, normalizedType) + .gt(ZiniaoMemoryStoreEntity::getExpiresAt, now) .orderByAsc(ZiniaoMemoryStoreEntity::getCacheKey) .last("LIMIT " + safeLimit)); if (entities.isEmpty()) { return List.of(); } - LocalDateTime now = LocalDateTime.now(); return entities.stream() - .filter(entity -> entity.getExpiresAt() != null && entity.getExpiresAt().isAfter(now)) .map(entity -> deserialize(entity.getPayloadJson(), valueType)) .filter(Objects::nonNull) .toList(); @@ -133,8 +138,11 @@ public class ZiniaoMemoryStoreService { entity.setCreatedAt(now); entity.setUpdatedAt(now); ziniaoMemoryStoreMapper.insert(entity); + // 店铺索引条目按「每店两把键(shopId + 名称别名)× 每轮刷新」成批写入, + // 逐行 INFO 等于把刷新规模原样刷进日志文件;汇总行数已在 + // ZiniaoShopIndexService 的 `[ziniao-index] refresh persist` 里给出,这里降到 DEBUG。 if (CACHE_TYPE_SHOP_INDEX_ENTRY.equals(normalizedType)) { - log.info("[ziniao-shop-index-db] insert cacheKey={} companyNameSample={} expiresAt={} bytes={}", + log.debug("[ziniao-shop-index-db] insert cacheKey={} companyNameSample={} expiresAt={} bytes={}", normalizedKey, sampleCompanyNameForLog(companyName), expiresAt, payloadJson.length()); } return; @@ -146,7 +154,7 @@ public class ZiniaoMemoryStoreService { entity.setUpdatedAt(now); ziniaoMemoryStoreMapper.updateById(entity); if (CACHE_TYPE_SHOP_INDEX_ENTRY.equals(normalizedType)) { - log.info("[ziniao-shop-index-db] update cacheKey={} id={} companyNameSample={} expiresAt={} bytes={}", + log.debug("[ziniao-shop-index-db] update cacheKey={} id={} companyNameSample={} expiresAt={} bytes={}", normalizedKey, entity.getId(), sampleCompanyNameForLog(companyName), expiresAt, payloadJson.length()); } } @@ -159,7 +167,7 @@ public class ZiniaoMemoryStoreService { if (entity != null) { ziniaoMemoryStoreMapper.deleteById(entity.getId()); if (CACHE_TYPE_SHOP_INDEX_ENTRY.equals(normalizedType)) { - log.info("[ziniao-shop-index-db] delete cacheKey={} id={} companyNameSample={}", + log.debug("[ziniao-shop-index-db] delete cacheKey={} id={} companyNameSample={}", entity.getCacheKey(), entity.getId(), sampleCompanyNameForLog(entity.getCompanyName())); } } @@ -200,7 +208,7 @@ public class ZiniaoMemoryStoreService { } updated += ziniaoMemoryStoreMapper.updateById(row); if (CACHE_TYPE_SHOP_INDEX_ENTRY.equals(row.getCacheType())) { - log.info("[ziniao-shop-index-db] mark-stale cacheKey={} id={} companyNameSample={} expiresAt={} bytes={}", + log.debug("[ziniao-shop-index-db] mark-stale cacheKey={} id={} companyNameSample={} expiresAt={} bytes={}", row.getCacheKey(), row.getId(), sampleCompanyNameForLog(row.getCompanyName()), row.getExpiresAt(), row.getPayloadJson() == null ? 0 : row.getPayloadJson().length()); } diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index 832fed2b8..e0bf3f82d 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -27,6 +27,13 @@ spring: max-lifetime: ${AIIMAGE_DB_POOL_MAX_LIFETIME_MS:1500000} keepalive-time: ${AIIMAGE_DB_POOL_KEEPALIVE_TIME_MS:120000} leak-detection-threshold: ${AIIMAGE_DB_POOL_LEAK_DETECT_MS:15000} + # 网络层超时兜底:此前只有 Hikari 的取连接/校验超时,socket 读没有上限—— + # 一条卡在 DB 侧的语句会把 Tomcat 工作线程挂到服务端结束为止,只能人工 kill 容器。 + # 刻意走 data-source-properties 而不是改上面 URL 的兜底默认值:生产用 AIIMAGE_DB_URL + # 整体覆盖 URL,只改兜底值对线上无效。单位为毫秒(Connector/J 口径)。 + data-source-properties: + connectTimeout: ${AIIMAGE_DB_CONNECT_TIMEOUT_MS:5000} + socketTimeout: ${AIIMAGE_DB_SOCKET_TIMEOUT_MS:300000} flyway: enabled: ${AIIMAGE_FLYWAY_ENABLED:true} locations: classpath:db diff --git a/backend-java/src/main/resources/db/V141__retention_cleanup_indexes.sql b/backend-java/src/main/resources/db/V141__retention_cleanup_indexes.sql new file mode 100644 index 000000000..65e7616d5 --- /dev/null +++ b/backend-java/src/main/resources/db/V141__retention_cleanup_indexes.sql @@ -0,0 +1,37 @@ +-- 保留清理(retention)三条分批 DELETE 的可用索引。 +-- +-- 三条 DELETE 的 WHERE 都落不到任何现有索引上,只能全表扫描;InnoDB 的 DELETE 会对扫描过的行 +-- 加锁,03:50/04:20/04:40 三条又挤在同一窗口,于是清理任务与同期写入互相等待(生产的锁等待 +-- 与慢 SQL 有一条来源就在这里)。这里只追加索引/生成列,不改任何历史迁移文件。 + +-- 1) device_log_file:DELETE ... WHERE log_date < #{cutoff} LIMIT n +-- uk_device_log_file(source, device_id, file_name, log_date) 里 log_date 在第 4 列, +-- 不带前三列时左前缀用不上;idx_device_log_last_upload 是另一条时间线(V127:24-25)。 +ALTER TABLE device_log_file + ADD KEY idx_device_log_log_date (log_date), + ALGORITHM = INPLACE, LOCK = NONE; + +-- 2) shop_data_duplicate_scan:DELETE ... WHERE created_at < #{cutoff} AND id < #{protectFromId} LIMIT n +-- 现有 idx_status_created(status, created_at) 首列是 status,而本 DELETE 不带 status 条件(V108)。 +ALTER TABLE shop_data_duplicate_scan + ADD KEY idx_sds_created_at (created_at), + ALGORITHM = INPLACE, LOCK = NONE; + +-- 3) biz_price_track_loop_run:DELETE ... WHERE status IN ('SUCCESS','FAILED','STOPPED') +-- AND COALESCE(finished_at, updated_at, created_at) < #{cutoff} LIMIT n +-- 被 COALESCE 包住的表达式不可走索引(非 sargable),所以落一个 STORED 生成列承载同一时间线, +-- 再建 (status, cleanup_at) 复合索引;现有 idx_user_status(user_id, status) 首列是 user_id, +-- 同样用不上(V26:20-21)。 +-- +-- 应用侧不写这一列:PriceTrackLoopRunEntity 没有 cleanup_at 字段,MyBatis-Plus 按实体列生成 +-- INSERT/UPDATE 的列清单,不会带上它。加 STORED 生成列需要重建表(ALGORITHM=COPY), +-- 该表行数=循环批次数量级,且每行带 3 个 MEDIUMTEXT(V26:12-14), +-- 上线前需在从库实测耗时再定执行窗口。 +ALTER TABLE biz_price_track_loop_run + ADD COLUMN cleanup_at DATETIME GENERATED ALWAYS AS (COALESCE(finished_at, updated_at, created_at)) STORED + COMMENT '保留清理时间线(生成列,由 finished_at/updated_at/created_at 兜底,应用勿写)', + ALGORITHM = COPY; + +ALTER TABLE biz_price_track_loop_run + ADD KEY idx_loop_run_cleanup (status, cleanup_at), + ALGORITHM = INPLACE, LOCK = NONE; diff --git a/backend-java/src/main/resources/logback-spring.xml b/backend-java/src/main/resources/logback-spring.xml index 6ea853a44..973add4f8 100644 --- a/backend-java/src/main/resources/logback-spring.xml +++ b/backend-java/src/main/resources/logback-spring.xml @@ -26,16 +26,30 @@ ${APP_LOG_ARCHIVE} 50MB - 1 + + 14 500MB - true + false + + + 8192 + 0 + false + false + + + - + diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoArchiveServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoArchiveServiceTest.java index 23c593924..0010d363a 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoArchiveServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/imagevideo/service/ImageVideoArchiveServiceTest.java @@ -27,7 +27,7 @@ class ImageVideoArchiveServiceTest { ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class); OssStorageService ossStorageService = mock(OssStorageService.class); ImageVideoArchiveService archiveService = new ImageVideoArchiveService( - taskMapper, ossStorageService, new ImageVideoProperties(), new ObjectMapper()); + taskMapper, ossStorageService, new ImageVideoProperties(), new ObjectMapper(), mock(com.nanri.aiimage.common.service.DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS)); ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity(); task.setId(10L); task.setTaskType(ImageVideoArchiveService.TASK_TYPE); @@ -46,7 +46,7 @@ class ImageVideoArchiveServiceTest { ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class); OssStorageService ossStorageService = mock(OssStorageService.class); ImageVideoArchiveService archiveService = new ImageVideoArchiveService( - taskMapper, ossStorageService, new ImageVideoProperties(), new ObjectMapper()); + taskMapper, ossStorageService, new ImageVideoProperties(), new ObjectMapper(), mock(com.nanri.aiimage.common.service.DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS)); ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity(); task.setId(11L); task.setTaskType(ImageVideoArchiveService.TASK_TYPE); @@ -82,7 +82,8 @@ class ImageVideoArchiveServiceTest { mock(ImageVideoAsyncTaskMapper.class), mock(OssStorageService.class), new ImageVideoProperties(), - objectMapper); + objectMapper, + mock(com.nanri.aiimage.common.service.DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS)); ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity(); task.setTaskType(ImageVideoArchiveService.TASK_TYPE); @@ -106,7 +107,8 @@ class ImageVideoArchiveServiceTest { mock(ImageVideoAsyncTaskMapper.class), mock(OssStorageService.class), new ImageVideoProperties(), - new ObjectMapper()); + new ObjectMapper(), + mock(com.nanri.aiimage.common.service.DistributedJobLockService.class, org.mockito.Answers.RETURNS_MOCKS)); ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity(); task.setTaskType(ImageVideoArchiveService.TASK_TYPE); diff --git a/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue b/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue index 4ec1acc60..d08cd9153 100644 --- a/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandCollectDataTab.vue @@ -135,7 +135,7 @@ ', + 'vbscript:msgbox(1)', + 'file:///C:/Windows/win.ini', + 'http://[bad', + ]) { + assert.equal(safeHttpUrl(bad), '#', `${bad} 必须被拦成 #`) + } +}) + +test('test_safe_http_url_handles_empty_input', () => { + for (const empty of [undefined, null, '', ' ']) { + assert.equal(safeHttpUrl(empty as string | null | undefined), '#') + } +}) + +test('test_safe_http_url_uses_page_origin_when_window_exists', () => { + const previous = (globalThis as Record).window + ;(globalThis as Record).window = { location: { href: 'https://app.example.com/#group-front' } } + try { + assert.equal(safeHttpUrl('/api/result.zip'), '/api/result.zip') + // 带 origin 的伪协议不会因为「相对路径」被放行 + assert.equal(safeHttpUrl('javascript:alert(1)'), '#') + } finally { + if (previous === undefined) delete (globalThis as Record).window + else (globalThis as Record).window = previous + } +}) + +test('test_create_row_id_is_unique_per_call', () => { + const ids = new Set() + for (let i = 0; i < 2000; i += 1) ids.add(createRowId('asset')) + assert.equal(ids.size, 2000, '同一页会话内行 id 不能重复') + for (const id of ids) assert.match(id, /^asset:/) +}) + +test('test_create_row_id_falls_back_without_random_uuid', () => { + const cryptoGlobal = (globalThis as { crypto?: Crypto }).crypto + if (!cryptoGlobal) return // 没有 crypto 全局时走的就是降级路径 + const restore = Object.getOwnPropertyDescriptor(cryptoGlobal, 'randomUUID') + try { + // 1) randomUUID 缺失(http 非安全上下文下 WebView 不提供) + Object.defineProperty(cryptoGlobal, 'randomUUID', { value: undefined, configurable: true, writable: true }) + const missing = createRowId('row') + assert.match(missing, /^row:[0-9a-z]+-[0-9a-z]+-[0-9a-z]+$/, '降级实现要带前缀且分三段') + // 2) 存在但会抛(部分内核在非安全上下文直接抛错) + Object.defineProperty(cryptoGlobal, 'randomUUID', { + value: () => { throw new Error('not a secure context') }, + configurable: true, writable: true, + }) + const thrown = createRowId('row') + assert.match(thrown, /^row:/, '抛错也必须产出可用 id') + assert.notEqual(thrown, missing) + } finally { + if (restore) Object.defineProperty(cryptoGlobal, 'randomUUID', restore) + else delete (cryptoGlobal as unknown as Record).randomUUID + } +}) + +test('test_every_dynamic_href_outlet_goes_through_safe_http_url', () => { + const files = listVueFiles(srcRoot) + const sources: Record = {} + for (const file of files) { + sources[relative(srcRoot, file).replace(/\\/g, '/')] = readFileSync(file, 'utf-8') + } + const bindings = collectHrefBindings(sources) + assert.ok(bindings.length >= 4, `扫描到的 :href 出口只有 ${bindings.length} 处,检测面失效`) + const violations = unvettedBindings(bindings, vettedWrapperNames(Object.values(sources))) + assert.deepEqual(violations, [], '存在未经协议过滤的动态 :href') +}) + +test('test_href_outlet_detector_actually_flags_raw_binding', () => { + const wrappers = vettedWrapperNames(['function sourceFileHref(item) {\n return safeHttpUrl(item?.url)\n}']) + assert.deepEqual([...wrappers], ['sourceFileHref'], '包装函数识别失效') + const synthetic: HrefBinding[] = [ + { file: 'a.vue', line: '' }, + { file: 'b.vue', line: '' }, + { file: 'c.vue', line: '' }, + { file: 'd.vue', line: '' }, + ] + const violations = unvettedBindings(synthetic, wrappers) + assert.deepEqual(violations.map((v) => v.file), ['d.vue'], '检测必须只漏放原始绑定') +})