diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/pipeline/AbstractFileTaskPipeline.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/pipeline/AbstractFileTaskPipeline.java new file mode 100644 index 000000000..5372b503d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/pipeline/AbstractFileTaskPipeline.java @@ -0,0 +1,394 @@ +package com.nanri.aiimage.modules.task.service.pipeline; + +import cn.hutool.core.util.IdUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.nanri.aiimage.common.exception.BusinessCodes; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.security.AdminAuthSupport; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.AbstractModuleTaskCacheService; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskFinalizeSupport; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; + +/** + * 后端任务管线四段(文件任务→解析→跑批→出结果)的公共底座。 + * + *

本类只沉「七支巨石实测逐字相同的那一截」,不承诺统一四段。签名逐条来自 + * 2026-10-18 对七支现网代码的定点实贴(AppearancePatent 3,377/ShopDataCrawl 3,195/ + * DeleteBrand 2,304/PriceTrack 2,302/Publish 2,249/CollectData 2,162/ShopMatch 2,027), + * 每段方法的注释里都写了它的出处行号,Phase 3 迁移时被迁移支必须能逐字替换, + * 换不上就是本类缺件,不是迁移方自由发挥。 + * + *

四段实测结论(决定了下面有什么、没有什么): + *

+ * + *

缓存与心跳复用 {@link AbstractModuleTaskCacheService},不新造第二套:4/7 支有模块缓存服务 + * (AppearancePatent/ShopDataCrawl/ShopMatch/PriceTrack),其余三支没有—— + * 故 {@link #taskCacheService()} 默认 {@code null},心跳只在有缓存服务的支上落, + * 与现网一致(DeleteBrand 走自家 {@code saveProgress},其缓存服务不继承本抽象, + * 见 {@code DeleteBrandRunService:1265} 注释)。 + * + *

依赖严格限于 task 模块内部+{@code common}+{@code modules.file},不触碰任何业务模块, + * 满足 ArchUnit 冻结线 {@code TASK_TO_BUSINESS_BASELINE = 0} + * ({@code ArchitectureBoundaryTest:39})。泛型 {@code T} 即该支店铺维度合并载荷类型, + * 与 {@link AbstractModuleTaskCacheService} 的 {@code T} 同义,因此两者可直接对接。 + */ +@Slf4j +public abstract class AbstractFileTaskPipeline { + + /** 终态任务被再次回传时的拒绝形状。两支差异不是文案,是线协议。 */ + public enum TerminalRejection { + /** + * 带 {@link BusinessCodes#TASK_ALREADY_FINISHED}(40901): + * {@code GlobalExceptionHandler} 转 HTTP 200 + success=true,Python 侧按幂等忽略、停止重试。 + * 现网出处:ShopMatch {@code :652}/PriceTrack {@code :603}/CollectData {@code :864/:904}。 + */ + IDEMPOTENT, + /** + * 不带 code({@code BusinessException(String)}):响应 success=false,调用方会当错误处理。 + * 现网出处:ShopDataCrawl {@code :633}/DeleteBrand {@code :1362-1374}/Publish {@code :1344}。 + */ + ERROR + } + + protected final FileTaskMapper fileTaskMapper; + protected final FileResultMapper fileResultMapper; + protected final TaskDistributedLockService taskDistributedLockService; + protected final TaskFileJobService taskFileJobService; + protected final OssStorageService ossStorageService; + + protected AbstractFileTaskPipeline(FileTaskMapper fileTaskMapper, + FileResultMapper fileResultMapper, + TaskDistributedLockService taskDistributedLockService, + TaskFileJobService taskFileJobService, + OssStorageService ossStorageService) { + this.fileTaskMapper = fileTaskMapper; + this.fileResultMapper = fileResultMapper; + this.taskDistributedLockService = taskDistributedLockService; + this.taskFileJobService = taskFileJobService; + this.ossStorageService = ossStorageService; + } + + // ------------------------------------------------------------------ + // 钩子:实测真差异处,只有这四条 + // ------------------------------------------------------------------ + + /** 该支模块标识,全库唯一(同时是结果文件 Job 的注册键与锁键)。 */ + protected abstract String moduleType(); + + /** + * 回传准入用任务加载:实测两派——AppearancePatent/Publish/CollectData 直接 + * {@code fileTaskMapper.selectById}(DB),ShopDataCrawl/ShopMatch/PriceTrack 走 + * 缓存优先({@link TaskFinalizeSupport#loadTaskForExecution}),DeleteBrand 用本地缓存件。 + * 派别由子类定,底座不替它选。 + */ + protected abstract FileTaskEntity loadTaskForSubmission(Long taskId); + + /** + * 该支的模块缓存服务;无缓存服务的三支返回 {@code null}。 + * 返回非空时 {@link #admitSubmission} 会按现网形状打一次心跳。 + */ + protected AbstractModuleTaskCacheService taskCacheService() { + return null; + } + + /** + * 归属实例校验钩子,默认放过。现网只有 AppearancePatent{@code :2026}/ + * Publish{@code :1375}/ShopDataCrawl{@code :3114} 三支实现(多实例转发用), + * 其余四支不校验——所以默认必须是空实现而不是抛错。 + */ + protected void ensureTaskOwnedByCurrentInstance(FileTaskEntity task, String operation) { + } + + // ------------------------------------------------------------------ + // 段 1:解析建单——task 头行与结果行的落库形状 + // ------------------------------------------------------------------ + + /** + * 建任务头行(不落库)。七支逐字相同的字段集合在此钉死: + * {@code taskNo = -<雪花id>}、{@code createdBy = "user:" + userId}、 + * {@code deviceId = AdminAuthSupport.normalizeDeviceId(deviceId)}、{@code createdAt/updatedAt} + * 取同一时刻。后者对 AppearancePatent{@code :207-208} 与 DeleteBrand{@code :156-157} + * 是一处微差——它们各调了两次 {@code LocalDateTime.now()}(可能差几微秒),底座只取一次; + * 两列都只用于「建单时刻」,无人拿它们互相比较,故按 Majority 的单次写法沉。 + * 出处:ShopDataCrawl{@code :575-587}/ShopMatch{@code :473-486}/Publish{@code :1054-1070} + * /CollectData{@code :329-341}/AppearancePatent{@code :196-214}/PriceTrack{@code :501-513} + * /DeleteBrand{@code :146-157}。 + * + *

{@code taskMode} 六支 {@code "PYTHON_QUEUE"}、DeleteBrand 一支 {@code "IMMEDIATE"}; + * {@code status} 建单时各不相同——{@code RUNNING}(ShopDataCrawl/PriceTrack/DeleteBrand)、 + * {@code PENDING}(AppearancePatent/CollectData)、{@code SCHEDULED}(ShopMatch 带排期时)、 + * 乃至 Publish 在全部文件解析失败时直接 {@code FAILED}({@code :1058})。 + * 两者一律由调用方显式传入,不在底座设默认值——设了就会有人不再想它。 + * + *

{@code successFileCount}/{@code failedFileCount} 刻意不由底座初始化:建单时五支写 0 + * (AppearancePatent{@code :202-203}/ShopDataCrawl{@code :581-582}/ShopMatch{@code :479-480}/ + * PriceTrack{@code :507-508}/CollectData{@code :333-334}),Publish{@code :1060-1061} 写的是 + * 真实失败数,而 DeleteBrand{@code :146-157} 首次 insert 两列都留 NULL、到 {@code :268-278} + * 循环后才回填。底座一旦默认写 0,就把 DeleteBrand 的「还没统计」偷换成了「零成功零失败」。 + * 其余模块专有列({@code requestJson}/{@code resultJson}/{@code scheduledAt} + * /{@code errorMessage}/{@code finishedAt})同样请在返回的实体上继续 set,再交 {@link #insertTask}。 + */ + protected final FileTaskEntity newTaskHeader(String status, + String taskMode, + Long userId, + String deviceId, + int sourceFileCount) { + requireUserId(userId); + LocalDateTime now = LocalDateTime.now(); + FileTaskEntity task = new FileTaskEntity(); + task.setTaskNo(moduleType() + "-" + IdUtil.getSnowflakeNextIdStr()); + task.setModuleType(moduleType()); + task.setTaskMode(taskMode); + task.setStatus(status); + task.setSourceFileCount(sourceFileCount); + task.setCreatedBy("user:" + userId); + task.setUserId(userId); + task.setDeviceId(AdminAuthSupport.normalizeDeviceId(deviceId)); + task.setCreatedAt(now); + task.setUpdatedAt(now); + return task; + } + + /** 落库任务头行。主键由 {@code IdType.AUTO} 回填,返回同一实体以便调用方继续用 id。 */ + protected final FileTaskEntity insertTask(FileTaskEntity task) { + fileTaskMapper.insert(task); + return task; + } + + /** + * 建一个「作用域结果行」(一支一店/一支一文件)。七支逐字相同的字段集合: + * {@code taskId/moduleType/sourceFilename/sourceFileUrl/userId/createdAt}。 + * {@code rowCount} 与 {@code success} 刻意可空——现网 ShopDataCrawl{@code :598-606} + * 与 ShopMatch{@code :496-504} 不写 rowCount,{@code success} 则有三值: + * {@code null}(AppearancePatent{@code :229-237} 不写)、{@code 0}(多数)、 + * {@code -1}(ShopDataCrawl 的 {@code RESULT_PENDING})。底座不替它们归一。 + * + *

{@code sourceFilename} 在 shop 族是店铺名、在 file 族是源文件名, + * 且回传侧正是用它当作用域键({@code ShopMatchTaskService:674} {@code row.getSourceFilename()}), + * 所以这里的入参就是后续 {@link #listScopeRows} 的匹配键,别传展示用的名字。 + */ + protected final FileResultEntity newScopeRow(Long taskId, + Long userId, + String sourceFilename, + String sourceFileUrl, + Integer rowCount, + Integer success) { + FileResultEntity row = new FileResultEntity(); + row.setTaskId(taskId); + row.setModuleType(moduleType()); + row.setSourceFilename(sourceFilename); + row.setSourceFileUrl(sourceFileUrl); + row.setRowCount(rowCount); + row.setSuccess(success); + row.setUserId(userId); + row.setCreatedAt(LocalDateTime.now()); + return row; + } + + /** + * 逐条插入结果行。刻意不批插:PriceTrack{@code :524-527} 的注释写明了原因—— + * 后续要用 {@code useGeneratedKeys} 回填的 {@code resultId} 去挂载荷,批插拿不到逐行主键。 + */ + protected final List insertScopeRows(List rows) { + List saved = new ArrayList<>(rows.size()); + for (FileResultEntity row : rows) { + fileResultMapper.insert(row); + saved.add(row); + } + return saved; + } + + // ------------------------------------------------------------------ + // 段 3:回传落库——准入序列 + // ------------------------------------------------------------------ + + /** {@code taskId} 非法即拒。出处:ShopDataCrawl{@code :619}/ShopMatch{@code :631}/PriceTrack{@code :592}。 */ + protected final void requireTaskId(Long taskId) { + if (taskId == null || taskId <= 0) { + throw new BusinessException("taskId 不合法"); + } + } + + /** {@code user_id} 非法即拒;七支文案逐字相同({@code user_id 不合法})。 */ + protected final void requireUserId(Long userId) { + if (userId == null || userId <= 0) { + throw new BusinessException("user_id 不合法"); + } + } + + /** + * 取任务分布式锁,竞争即 {@link BusinessCodes#TASK_BUSY} 拒绝(不静默降级)。 + * 委托既有 {@link TaskFinalizeSupport#acquireTaskLockOrThrow},不另立一套锁语义。 + */ + protected final TaskDistributedLockService.LockHandle acquireSubmissionLockOrThrow(Long taskId, String logTag) { + return TaskFinalizeSupport.acquireTaskLockOrThrow(taskDistributedLockService, moduleType(), taskId, logTag); + } + + /** 带等待时长的变体(CollectData/ShopDataCrawl 等自有 {@code TASK_LOCK_WAIT_MILLIS})。 */ + protected final TaskDistributedLockService.LockHandle acquireSubmissionLockOrThrow(Long taskId, + String logTag, + long waitMillis) { + TaskDistributedLockService.LockHandle lockHandle = + TaskFinalizeSupport.acquireTaskLock(taskDistributedLockService, moduleType(), taskId, waitMillis); + if (lockHandle == null) { + log.warn("[{}] 任务锁竞争,拒绝本次提交 taskId={}", logTag, taskId); + throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试"); + } + return lockHandle; + } + + /** + * 回传准入:校验 taskId → 加载任务 → 模块校验 → 终态拒绝 → 归属校验 → 心跳。 + * 出处:ShopMatch{@code :647-655} 对 PriceTrack{@code :597-605} 除终态那格的日志外逐字相同; + * ShopDataCrawl{@code :627-636} 同形但步序相反(见下)。 + * + *

终态怎么拒由 {@code terminalRejection} 决定(见 {@link TerminalRejection} 的线协议后果), + * 归属校验是否发生由 {@code ownershipOperation} 决定(传 {@code null} = 不校验, + * 对应现网 ShopMatch/PriceTrack/DeleteBrand/CollectData 四支不查归属)。 + * + *

步序按 ShopMatch/PriceTrack 定:终态判定在归属校验之前。ShopDataCrawl 现网是反的 + * ({@code :631} 先查归属、{@code :632} 再判终态),一支既被别人持有又已终态的任务, + * 走它会得 {@code TaskOwnerMismatchException}、走本方法是终态拒绝。所以 ShopDataCrawl 迁移时 + * 要么保留自己的步序(传 {@code null} 后自查归属),要么把这条差异单独立账, + * 不许悄悄套用本默认序就算迁完。 + * + *

本方法不打日志:七支里只有 PriceTrack{@code :602} 在终态拒绝前打一行 warn, + * 其余六支沉默。日志不是公共形状,所以那一行留在它自己的调用点,别当成已沉掉后把它删了。 + */ + protected final FileTaskEntity admitSubmission(Long taskId, + TerminalRejection terminalRejection, + String ownershipOperation) { + requireTaskId(taskId); + FileTaskEntity task = loadTaskForSubmission(taskId); + if (task == null || !moduleType().equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + String status = task.getStatus(); + if ("SUCCESS".equals(status) || "FAILED".equals(status)) { + if (terminalRejection == TerminalRejection.IDEMPOTENT) { + throw new BusinessException(BusinessCodes.TASK_ALREADY_FINISHED, "任务已结束,拒绝重复提交"); + } + throw new BusinessException("任务已结束,拒绝重复提交"); + } + if (ownershipOperation != null) { + ensureTaskOwnedByCurrentInstance(task, ownershipOperation); + } + AbstractModuleTaskCacheService cacheService = taskCacheService(); + if (cacheService != null) { + cacheService.touchTaskHeartbeat(taskId); + } + return task; + } + + /** + * 该任务的全部结果行,按主键升序——回传侧的作用域匹配与收尾判定都依赖这个顺序。 + * 出处:ShopMatch{@code :656-659}/PriceTrack{@code :606-610}/DeleteBrand 同款 selectList。 + */ + protected final List listScopeRows(Long taskId) { + return fileResultMapper.selectList(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, moduleType()) + .orderByAsc(FileResultEntity::getId)); + } + + /** + * 挂一个结果文件组装 Job。{@code scopeKey} 实测是四形的,不是笔误: + * 归属实例域 {@code task:{id}:owner:{instance}}(Publish{@code :1046/:1216}、 + * ShopDataCrawl{@code :1920}、AppearancePatent{@code :1753})/固定 {@code task:{id}} + * (CollectData{@code :1122})/店铺名(ShopMatch{@code :883}、PriceTrack{@code :905}) + * /文件身份(DeleteBrand{@code :1505/:1770})。底座只转发,不解释形状。 + */ + protected final void enqueueResultAssembly(Long taskId, Long resultId, String scopeKey) { + taskFileJobService.enqueueAssembleResult(taskId, moduleType(), resultId, scopeKey); + } + + // ------------------------------------------------------------------ + // 段 4:出结果——上传与结果行 6 列落库 + // ------------------------------------------------------------------ + + /** + * 把已组装好的结果文件上传对象存储,并写回结果行的 6 列后落库。 + * 出处(六支同形):ShopMatch{@code :828-840}/PriceTrack{@code :845-857}/ + * Publish{@code :590-598}/CollectData{@code :1166-1190}/AppearancePatent{@code :1876} + * /DeleteBrand{@code :1921-1929}。 + * + *

ShopDataCrawl 不在此件适用范围内:它写 5 列走的是另一条路—— + * {@code updateResultFileColumns:1880-1892} 用 {@code LambdaUpdateWrapper} 按 id 集合批量 + * {@code .in(ids).eq(success, RESULT_SUCCESS).set(...)},既不 {@code updateById} 载入的实体、 + * 不写 {@code success}/{@code errorMessage},还会在 {@code :1919} 传 {@code objectKey=null} + * (终态无工作簿那格)。硬套本方法会把它的「只刷成功行、不动状态位」语义改掉, + * 故 3B 迁移那一支必须保留本地写法,别把这句读成「七支都能换」。 + * + *

{@code preserveFailure} 是 ShopMatch/PriceTrack 的「失败店铺部分结果」语义: + * 结果行已带 {@code errorMessage} 时只挂文件、{@code success} 保持 0, + * 否则组装一落地就把行“洗成成功”,用户再也看不到「这个店铺其实没跑完」 + * (原注释见 {@code ShopMatchTaskService:823-825})。其余五支恒传 {@code false} + * (它们无条件 {@code success=1} 并清 {@code errorMessage})。 + * + *

临时文件不由本方法删:现网七支的清理点不同(有的在本层 {@code finally}、 + * 有的在外层),谁上传谁负责,与本类 javadoc 说的「只沉逐字相同那截」一致。 + */ + protected final void publishResultFile(FileResultEntity row, + File resultFile, + String filename, + String contentType, + int rowCount, + boolean preserveFailure) { + String objectKey = ossStorageService.uploadResultFile(resultFile, moduleType()); + if (preserveFailure) { + row.setSuccess(0); + } else { + row.setSuccess(1); + row.setErrorMessage(null); + } + row.setResultFilename(filename); + row.setResultFileUrl(objectKey); + row.setResultFileSize(resultFile.length()); + row.setResultContentType(contentType); + row.setRowCount(rowCount); + fileResultMapper.updateById(row); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/pipeline/AbstractShopPayloadTaskPipeline.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/pipeline/AbstractShopPayloadTaskPipeline.java new file mode 100644 index 000000000..45a1f4fd2 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/pipeline/AbstractShopPayloadTaskPipeline.java @@ -0,0 +1,149 @@ +package com.nanri.aiimage.modules.task.service.pipeline; + +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** + * 「店铺维度合并载荷」家族的回传落库(段 3)骨架——{@link AbstractFileTaskPipeline} 的家族层。 + * + *

为什么只有这一族值得再下一层。2026-10-18 实贴七支的 {@code submitResult}, + * 合并模型实测分三族:chunk+scope(AppearancePatent/Publish/CollectData)、 + * shop-payload(ShopDataCrawl/ShopMatch/PriceTrack)、DeleteBrand 自有分片存储。 + * 其中 ShopMatch 与 PriceTrack 两支的 {@code submitResult} 是行级孪生件: + * 校验→取锁→准入→按店铺名建索引→逐行合并→{@code changed} 闸门→重读结果行→重算任务状态 + * 整段结构一致({@code ShopMatchTaskService:630-710} 对 {@code PriceTrackTaskService:591-663}), + * 差异全在「单行怎么处理」那一步里。ShopDataCrawl 只共用「准入」与「用 + * {@code row.sourceFilename} 当作用域匹配键」这两件事({@code :644-648}):它的索引是自建的 + * {@code normalizePayloadByShop(shops, 首行店铺名)}({@code :641}),尾部没有 {@code changed} + * 闸门而是无条件 {@code persistProgressOrSnapshot}+{@code tryFinalizeTask}({@code :679-681}), + * 单行体整段走快照机——所以本模板对 ShopDataCrawl 不适用,那一支保留自己的写法。 + * + *

刻意不沉的两处(沉了就是替现网做决定,Phase 3 会拿不到行为对撞): + *

+ * + *

店铺名归一化留成钩子:两支现网都调 {@code ZiniaoShopSwitchService.normalizeShopName} + * ({@code ShopMatchTaskService:665}/{@code PriceTrackTaskService:614}),但该 bean 在 + * {@code modules.ziniao} 下,底座直接依赖它会把 task→业务 的冻结线(0)顶开。 + */ +public abstract class AbstractShopPayloadTaskPipeline extends AbstractFileTaskPipeline { + + protected AbstractShopPayloadTaskPipeline(FileTaskMapper fileTaskMapper, + FileResultMapper fileResultMapper, + TaskDistributedLockService taskDistributedLockService, + TaskFileJobService taskFileJobService, + OssStorageService ossStorageService) { + super(fileTaskMapper, fileResultMapper, taskDistributedLockService, + taskFileJobService, ossStorageService); + } + + /** + * 店铺名归一化(同时是结果行 {@code sourceFilename} 与载荷索引键的构造规则)。 + * 两支必须用同一规则建与查,否则回传永远匹配不上行。 + */ + protected abstract String normalizeShopKey(String rawShopName); + + /** + * 单行处理:合并该店铺载荷,并按各自现网语义决定「失败置位/等待更多分片/挂组装 Job」。 + * 调用即代表本次回传命中了该行({@code changed} 由模板在调用前置位,与两支现网一致)。 + */ + protected abstract void applyShopResultRow(Long taskId, + FileTaskEntity task, + FileResultEntity row, + String shopKey, + T incoming); + + /** 本轮有行被命中后的收尾:两支现网都是拿最新结果行重算任务状态。 */ + protected abstract void onShopRowsRecomputed(FileTaskEntity task, List latestRows); + + /** + * 锁竞争日志标签。历史值是各支小写短名({@code shop-match}/{@code price-track}/ + * {@code shop-data-crawl}),与 {@link #moduleType()} 的大写枚举形不同, + * 默认返回 moduleType 仅供未覆写时不缺标签,两支迁移时必须按历史值覆写。 + */ + protected String submissionLockLogTag() { + return moduleType(); + } + + /** + * 段 3「回传落库」整段骨架。入参 {@code incoming} 即各支请求里的 {@code shops} 列表。 + * + *

闸门与终态拒绝形状由 {@code terminalRejection} 决定;{@code ownershipOperation} + * 传 {@code null} 表示不校验归属(两支现网都不校验)。 + * + *

结果行匹配键取 {@code row.getSourceFilename()},因此建单时写进该列的必须是 + * {@link #normalizeShopKey(String)} 归一化后的店铺名({@code ShopMatchTaskService:499} + * 与 {@code PriceTrackTaskService:534} 正是这么写的)。 + * + *

校验在取锁之前(按 PriceTrack{@code :592-596} 的形状):ShopMatch 现网把 + * {@code shops} 非空检查放在锁内({@code :634} 取锁 → {@code :643} 才检查)。两者对调用方 + * 的可见结果相同,本模板选前者是为了「明显不合法的请求不去抢锁」。迁移这一支时把这句 + * 微差登记在案,别让它变成「顺手改的」没人知道。 + */ + protected final void submitShopScopedResults(Long taskId, + List incoming, + Function rawShopNameOf, + TerminalRejection terminalRejection, + String ownershipOperation) { + requireTaskId(taskId); + if (incoming == null || incoming.isEmpty()) { + throw new BusinessException("shops 不能为空"); + } + try (TaskDistributedLockService.LockHandle ignored = + acquireSubmissionLockOrThrow(taskId, submissionLockLogTag())) { + FileTaskEntity task = admitSubmission(taskId, terminalRejection, ownershipOperation); + Map payloadByShopKey = indexByShopKey(incoming, rawShopNameOf); + boolean changed = false; + for (FileResultEntity row : listScopeRows(taskId)) { + String shopKey = row.getSourceFilename(); + T item = shopKey == null ? null : payloadByShopKey.get(shopKey); + if (item == null) { + continue; + } + changed = true; + applyShopResultRow(taskId, task, row, shopKey, item); + } + if (!changed) { + return; + } + onShopRowsRecomputed(task, listScopeRows(taskId)); + } + } + + /** + * 回传载荷按店铺名建索引:跳过 null 项与归一化后为空的项,保持请求顺序。 + * 出处:{@code ShopMatchTaskService:660-669}/{@code PriceTrackTaskService:611-616}(逐字同形)。 + */ + protected final Map indexByShopKey(List incoming, Function rawShopNameOf) { + Map indexed = new LinkedHashMap<>(); + for (T item : incoming) { + if (item == null) { + continue; + } + String shopKey = normalizeShopKey(rawShopNameOf.apply(item)); + if (shopKey != null && !shopKey.isBlank()) { + indexed.put(shopKey, item); + } + } + return indexed; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/pipeline/AbstractFileTaskPipelineTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/pipeline/AbstractFileTaskPipelineTest.java new file mode 100644 index 000000000..3c0125be2 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/pipeline/AbstractFileTaskPipelineTest.java @@ -0,0 +1,404 @@ +package com.nanri.aiimage.modules.task.service.pipeline; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.AbstractModuleTaskCacheService; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +/** + * 任务管线底座的行为契约(Phase 2 新增公共件,AGENTS.md §3 阶段 3 第 2 条要求覆盖正常/边界/异常)。 + * + *

这些用例锁的不是「基类能跑」,而是Phase 3 迁移时七支必须换得上的那截形状: + * 每条断言旁边注着它抄自哪支现网代码的哪一行。任何一条改了,就是在替现网做决定。 + */ +class AbstractFileTaskPipelineTest { + + private static final String MODULE_TYPE = "PIPE_TEST"; + + private FileTaskMapper fileTaskMapper; + private FileResultMapper fileResultMapper; + private TaskDistributedLockService lockService; + private TaskFileJobService jobService; + private OssStorageService ossStorageService; + private TestPipeline pipeline; + + @BeforeEach + void setUp() { + // LambdaQueryWrapper 的列名解析吃 lambda 缓存,未预置会在 getTargetSql() 处炸(仓内 136 支同此套路) + TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), + FileResultEntity.class); + fileTaskMapper = mock(FileTaskMapper.class); + fileResultMapper = mock(FileResultMapper.class); + lockService = mock(TaskDistributedLockService.class); + jobService = mock(TaskFileJobService.class); + ossStorageService = mock(OssStorageService.class); + pipeline = new TestPipeline(fileTaskMapper, fileResultMapper, lockService, jobService, ossStorageService); + } + + // ------------------------------------------------------------------ + // 段 1:解析建单 + // ------------------------------------------------------------------ + + @Test + void taskHeaderCarriesTheSevenModuleCommonFieldsVerbatim() { + FileTaskEntity task = pipeline.newTaskHeader("RUNNING", "PYTHON_QUEUE", 42L, " dev-1 ", 3); + + assertTrue(task.getTaskNo().startsWith(MODULE_TYPE + "-"), "taskNo 必须是 -<雪花>"); + assertEquals(MODULE_TYPE, task.getModuleType()); + assertEquals("PYTHON_QUEUE", task.getTaskMode()); + assertEquals("RUNNING", task.getStatus()); + assertEquals(3, task.getSourceFileCount()); + assertEquals("user:42", task.getCreatedBy()); + assertEquals(42L, task.getUserId()); + assertEquals("dev-1", task.getDeviceId(), "deviceId 走 trim(AdminAuthSupport.normalizeDeviceId)"); + assertNotNull(task.getCreatedAt()); + assertSame(task.getCreatedAt(), task.getUpdatedAt()); + } + + @Test + void blankDeviceIdStaysUnrecordedRatherThanEmptyString() { + assertNull(pipeline.newTaskHeader("PENDING", "IMMEDIATE", 1L, " ", 0).getDeviceId()); + } + + @Test + void taskHeaderDoesNotInventDefaultsForModuleDecidedColumns() { + FileTaskEntity task = pipeline.newTaskHeader("SCHEDULED", "PYTHON_QUEUE", 7L, "dev", 1); + // ShopMatch 的 scheduledAt、Publish 的 finishedAt/errorMessage、各支 requestJson/resultJson + // 都由调用方自己 set:底座一设默认值,迁移时就会把「没写」变成「写了个错的」。 + assertNull(task.getScheduledAt()); + assertNull(task.getFinishedAt()); + assertNull(task.getErrorMessage()); + assertNull(task.getRequestJson()); + assertNull(task.getResultJson()); + // 两个计数列同理:DeleteBrand 建单时留 NULL、循环后才回填(:146-157 → :268-278), + // Publish 写的是真实失败数(:1061)——底座一-default 0 就把「还没统计」偷换成「零成功零失败」。 + assertNull(task.getSuccessFileCount()); + assertNull(task.getFailedFileCount()); + } + + @Test + void illegalUserIdIsRejectedBeforeAnyRowIsBuilt() { + for (Long bad : new Long[]{null, 0L, -1L}) { + BusinessException ex = assertThrows(BusinessException.class, + () -> pipeline.newTaskHeader("RUNNING", "PYTHON_QUEUE", bad, "dev", 1)); + assertEquals("user_id 不合法", ex.getMessage(), "七支文案逐字相同,不许改"); + } + verifyNoInteractions(fileTaskMapper, fileResultMapper); + } + + @Test + void insertTaskWritesOnceAndReturnsTheSameEntityForIdBackfill() { + FileTaskEntity task = pipeline.newTaskHeader("RUNNING", "PYTHON_QUEUE", 9L, "dev", 1); + assertSame(task, pipeline.insertTask(task)); + verify(fileTaskMapper, times(1)).insert(task); + } + + @Test + void scopeRowKeepsNullableColumnsUntouched() { + // ShopDataCrawl/ShopMatch 建单时不写 rowCount,AppearancePatent 不写 success: + // 底座一归一,结果行的「未开始」就被读成「0 行且失败」。 + FileResultEntity row = pipeline.newScopeRow(100L, 9L, "shop-a", "shop-id-1", null, null); + assertNull(row.getRowCount()); + assertNull(row.getSuccess()); + assertEquals(100L, row.getTaskId()); + assertEquals(MODULE_TYPE, row.getModuleType()); + assertEquals("shop-a", row.getSourceFilename(), "sourceFilename 是回传侧的作用域匹配键"); + assertEquals("shop-id-1", row.getSourceFileUrl()); + assertEquals(9L, row.getUserId()); + assertNotNull(row.getCreatedAt()); + + assertEquals(-1, pipeline.newScopeRow(1L, 1L, "s", "u", null, -1).getSuccess(), + "ShopDataCrawl 的 RESULT_PENDING=-1 必须原样传下去"); + } + + @Test + void scopeRowsAreInsertedOneByOneNotBatched() { + List rows = List.of( + pipeline.newScopeRow(1L, 1L, "a", "ua", 10, 0), + pipeline.newScopeRow(1L, 1L, "b", "ub", 20, 0)); + + List saved = pipeline.insertScopeRows(rows); + + assertEquals(2, saved.size()); + assertSame(rows.get(0), saved.get(0)); + assertSame(rows.get(1), saved.get(1)); + verify(fileResultMapper, times(2)).insert(any(FileResultEntity.class)); + } + + @Test + void emptyScopeRowListInsertsNothing() { + assertTrue(pipeline.insertScopeRows(List.of()).isEmpty()); + verify(fileResultMapper, never()).insert(any(FileResultEntity.class)); + } + + // ------------------------------------------------------------------ + // 段 3:回传准入 + // ------------------------------------------------------------------ + + @Test + void illegalTaskIdRejectedWithoutTouchingAnyCollaborator() { + for (Long bad : new Long[]{null, 0L, -5L}) { + assertEquals("taskId 不合法", + assertThrows(BusinessException.class, () -> pipeline.requireTaskId(bad)).getMessage()); + } + verifyNoInteractions(fileTaskMapper, fileResultMapper, lockService, jobService); + } + + @Test + void missingTaskOrForeignModuleTypeIsBothReportedAsAbsent() { + pipeline.loadedTask = null; + assertAdmissionFails("任务不存在"); + + FileTaskEntity foreign = new FileTaskEntity(); + foreign.setModuleType("SOMETHING_ELSE"); + foreign.setStatus("RUNNING"); + pipeline.loadedTask = foreign; + assertAdmissionFails("任务不存在"); + } + + @Test + void terminalTaskRejectionKeepsTheTwoWireProtocolShapes() { + for (String terminal : List.of("SUCCESS", "FAILED")) { + pipeline.loadedTask = taskWithStatus(terminal); + + BusinessException idempotent = assertThrows(BusinessException.class, + () -> pipeline.admitSubmission(1L, AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null)); + assertEquals(40901, idempotent.getCode(), + "40901 → HTTP 200 + success=true,Python 停止重试(BusinessCodes 已拆码,不可含糊)"); + assertEquals("任务已结束,拒绝重复提交", idempotent.getMessage()); + + BusinessException error = assertThrows(BusinessException.class, + () -> pipeline.admitSubmission(1L, AbstractFileTaskPipeline.TerminalRejection.ERROR, null)); + assertNull(error.getCode(), "无 code 的拒绝会让 success=false,与 IDEMPOTENT 是两件事"); + assertEquals("任务已结束,拒绝重复提交", error.getMessage()); + assertEquals(terminal, pipeline.loadedTask.getStatus()); + } + } + + @Test + @SuppressWarnings("unchecked") + void runningTaskIsAdmittedAndHeartbeatOnlyWhenTheModuleHasACacheService() { + pipeline.loadedTask = taskWithStatus("RUNNING"); + + pipeline.admitSubmission(1L, AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null); + assertNull(pipeline.ownershipOperation, "ownershipOperation 传 null 即不校验归属(四支现网如此)"); + assertTrue(pipeline.cacheRequested, "准入必须真的问一次缓存服务,否则这个钩子是死代码"); + verifyNoInteractions(ossStorageService, jobService); + + pipeline.cacheServiceHolder = mock(AbstractModuleTaskCacheService.class); + pipeline.admitSubmission(1L, AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null); + verify(pipeline.cacheServiceHolder, times(1)).touchTaskHeartbeat(1L); + + pipeline.cacheServiceHolder = mock(AbstractModuleTaskCacheService.class); + pipeline.admitSubmission(2L, AbstractFileTaskPipeline.TerminalRejection.ERROR, "submit shop data crawl result"); + assertEquals("submit shop data crawl result", pipeline.ownershipOperation, + "AP/Publish/SDC 三支要归属校验,钩子必须真被走到"); + } + + @Test + void lockContentionRejectsSubmissionWithTaskBusyCode() { + when(lockService.acquire(anyString(), anyLong())).thenReturn(null); + BusinessException ex = assertThrows(BusinessException.class, + () -> pipeline.acquireSubmissionLockOrThrow(1L, "shop-match")); + assertEquals(40902, ex.getCode(), "锁竞争必须 success=false 让调用方重试,不能静默降级"); + assertEquals("任务正在处理中,请稍后再试", ex.getMessage()); + + when(lockService.acquire(anyString(), anyLong(), anyLong())).thenReturn(null); + assertEquals(40902, assertThrows(BusinessException.class, + () -> pipeline.acquireSubmissionLockOrThrow(1L, "shop-match", 5_000L)).getCode()); + } + + @Test + void lockHandleIsHandedBackWhenAcquired() { + TaskDistributedLockService.LockHandle handle = mock(TaskDistributedLockService.LockHandle.class); + when(lockService.acquire(MODULE_TYPE, 1L)).thenReturn(handle); + assertSame(handle, pipeline.acquireSubmissionLockOrThrow(1L, "shop-match")); + } + + @Test + void scopeRowsAreListedForThisTaskAndModuleOrderedById() { + List rows = List.of(new FileResultEntity(), new FileResultEntity()); + when(fileResultMapper.selectList(any())).thenReturn(rows); + + assertSame(rows, pipeline.listScopeRows(77L)); + + // 光看返回值抓不到「读了别的任务/别的模块的行」——mock 的 selectList 是无条件回桩的, + // 所以必须把 wrapper 抓出来钉住条件列与值(仓内既有惯例)。 + ArgumentCaptor> captor = + ArgumentCaptor.forClass( + com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper.class); + verify(fileResultMapper).selectList(captor.capture()); + captor.getValue().getSqlSegment(); // MP 惯例:先求值一次才铺出参数 + String sql = captor.getValue().getSqlSegment(); + assertTrue(sql.contains("task_id") && sql.contains("module_type"), + "条件必须双卡 taskId+moduleType,少了后者就会把别的模块的结果行读进来:" + sql); + assertTrue(captor.getValue().getParamNameValuePairs().containsValue(77L)); + assertTrue(captor.getValue().getParamNameValuePairs().containsValue(MODULE_TYPE)); + assertTrue(sql.toLowerCase().contains("order by"), + "回传收尾判定依赖行序,丢了顺序就是行为变更:" + sql); + } + + @Test + void resultAssemblyJobIsEnqueuedUnderTheModuleType() { + pipeline.enqueueResultAssembly(5L, 6L, "task:5:owner:i-1"); + verify(jobService).enqueueAssembleResult(5L, MODULE_TYPE, 6L, "task:5:owner:i-1"); + } + + // ------------------------------------------------------------------ + // 段 4:出结果 + // ------------------------------------------------------------------ + + @Test + void publishResultFileUploadsThenWritesTheSixColumns(@TempDir java.nio.file.Path dir) throws IOException { + File xlsx = write(dir, "a.xlsx", "hello"); + FileResultEntity row = resultRowWithFailure("上一轮有部分行没跑完"); + when(ossStorageService.uploadResultFile(xlsx, MODULE_TYPE)).thenReturn("object/key/a"); + + pipeline.publishResultFile(row, xlsx, "店铺A.xlsx", "application/vnd.ms-excel", 12, false); + + verify(ossStorageService).uploadResultFile(xlsx, MODULE_TYPE); + assertEquals("店铺A.xlsx", row.getResultFilename()); + assertEquals("object/key/a", row.getResultFileUrl()); + assertEquals(xlsx.length(), row.getResultFileSize(), "文件大小取真实上传件字节数"); + assertEquals("application/vnd.ms-excel", row.getResultContentType()); + assertEquals(12, row.getRowCount()); + assertEquals(1, row.getSuccess()); + assertNull(row.getErrorMessage(), "非 preserveFailure 必须清掉旧失败原因(六支现网无条件 success=1)"); + verify(fileResultMapper).updateById(row); + } + + @Test + void preserveFailureKeepsTheRowVisiblyFailedWhileStillAttachingTheFile(@TempDir java.nio.file.Path dir) + throws IOException { + File xlsx = write(dir, "b.xlsx", "data"); + FileResultEntity row = resultRowWithFailure("店铺处理失败"); + row.setSuccess(0); + + pipeline.publishResultFile(row, xlsx, "店铺B.xlsx", "application/vnd.ms-excel", 3, true); + + assertEquals(0, row.getSuccess(), "失败店铺的部分结果被组装后仍是失败态,否则用户看不到「其实没跑完」"); + assertEquals("店铺处理失败", row.getErrorMessage()); + assertEquals("店铺B.xlsx", row.getResultFilename(), "文件照挂,用户仍能下载已跑出来的行"); + verify(fileResultMapper).updateById(row); + } + + @Test + void uploadFailureLeavesTheRowUntouched(@TempDir java.nio.file.Path dir) throws IOException { + File xlsx = write(dir, "c.xlsx", "data"); + FileResultEntity row = resultRowWithFailure(null); + when(ossStorageService.uploadResultFile(any(), anyString())).thenThrow(new IllegalStateException("对象存储不可达")); + + assertThrows(IllegalStateException.class, + () -> pipeline.publishResultFile(row, xlsx, "店铺C.xlsx", "x", 1, false)); + + verify(fileResultMapper, never()).updateById(any(FileResultEntity.class)); + assertNull(row.getResultFileUrl(), "上传没成就不能把 URL 写进去,否则下载即 404"); + } + + // ------------------------------------------------------------------ + // 夹具 + // ------------------------------------------------------------------ + + private void assertAdmissionFails(String expectedMessage) { + BusinessException ex = assertThrows(BusinessException.class, + () -> pipeline.admitSubmission(1L, AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null)); + assertEquals(expectedMessage, ex.getMessage()); + } + + private static FileTaskEntity taskWithStatus(String status) { + FileTaskEntity task = new FileTaskEntity(); + task.setId(1L); + task.setModuleType(MODULE_TYPE); + task.setStatus(status); + return task; + } + + private static FileResultEntity resultRowWithFailure(String errorMessage) { + FileResultEntity row = new FileResultEntity(); + row.setId(500L); + row.setTaskId(1L); + row.setModuleType(MODULE_TYPE); + row.setSuccess(0); + row.setErrorMessage(errorMessage); + return row; + } + + private static File write(java.nio.file.Path dir, String name, String content) throws IOException { + return Files.write(dir.resolve(name), content.getBytes(StandardCharsets.UTF_8)).toFile(); + } + + /** 被测底座的具体子类:只补三个钩子,公共件一律不重写(重写就等于没沉)。 */ + private static class TestPipeline extends AbstractFileTaskPipeline> { + + FileTaskEntity loadedTask; + String ownershipOperation; + boolean cacheRequested; + AbstractModuleTaskCacheService> cacheServiceHolder; + + TestPipeline(FileTaskMapper fileTaskMapper, + FileResultMapper fileResultMapper, + TaskDistributedLockService lockService, + TaskFileJobService jobService, + OssStorageService ossStorageService) { + super(fileTaskMapper, fileResultMapper, lockService, jobService, ossStorageService); + } + + @Override + protected String moduleType() { + return MODULE_TYPE; + } + + @Override + protected FileTaskEntity loadTaskForSubmission(Long taskId) { + return loadedTask; + } + + @Override + protected AbstractModuleTaskCacheService> taskCacheService() { + cacheRequested = true; + return cacheServiceHolder; + } + + @Override + protected void ensureTaskOwnedByCurrentInstance(FileTaskEntity task, String operation) { + this.ownershipOperation = operation; + } + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/pipeline/AbstractShopPayloadTaskPipelineTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/pipeline/AbstractShopPayloadTaskPipelineTest.java new file mode 100644 index 000000000..5d7806ff8 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/pipeline/AbstractShopPayloadTaskPipelineTest.java @@ -0,0 +1,276 @@ +package com.nanri.aiimage.modules.task.service.pipeline; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +/** + * 「店铺维度合并载荷」家族骨架的行为契约。 + * + *

逐条对撞的是 ShopMatch{@code submitResult:630-710} 与 PriceTrack{@code submitResult:591-663} + * 这两支行级孪生件的共同形状:命中与否由 {@code row.sourceFilename} 对上归一化店铺名决定, + * 一个都没命中就不重读也不重算状态,任何退出路径都必须把任务锁还回去。 + */ +class AbstractShopPayloadTaskPipelineTest { + + private static final String MODULE_TYPE = "SHOP_PAYLOAD_TEST"; + + private FileResultMapper fileResultMapper; + private TaskDistributedLockService lockService; + private TaskDistributedLockService.LockHandle lockHandle; + private ShopPipeline pipeline; + + @BeforeEach + void setUp() { + TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), + FileResultEntity.class); + fileResultMapper = mock(FileResultMapper.class); + lockService = mock(TaskDistributedLockService.class); + lockHandle = mock(TaskDistributedLockService.LockHandle.class); + when(lockService.acquire(anyString(), any())).thenReturn(lockHandle); + pipeline = new ShopPipeline(mock(FileTaskMapper.class), fileResultMapper, lockService, + mock(TaskFileJobService.class), mock(OssStorageService.class)); + pipeline.loadedTask = runningTask(); + } + + @Test + void matchedShopRowsAreHandledOnceAndStatusIsRecomputedFromFreshRows() { + FileResultEntity shopA = row("shop-a"); + FileResultEntity shopB = row("shop-b"); + List firstRead = List.of(shopA, shopB); + List secondRead = List.of(row("shop-a")); + // 第一次读=本轮待处理行,第二次读=changed 闸门后重算状态用的最新行(两支现网都是重读,不复用旧行) + when(fileResultMapper.selectList(any())).thenReturn(firstRead, secondRead); + + pipeline.submitShopScopedResults(1L, List.of("Shop-A"), String::valueOf, + AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null); + + assertEquals(1, pipeline.handled.size(), "只处理对上的那一行,对不上的行不许被顺手改写"); + assertEquals("shop-a", pipeline.handled.get(0).shopKey); + assertSame(shopA, pipeline.handled.get(0).row); + assertEquals("Shop-A", pipeline.handled.get(0).incoming, "钩子收的是原始入参,合并时机留给各支自己定"); + assertSame(pipeline.loadedTask, pipeline.handled.get(0).task); + assertEquals(1, pipeline.recomputed.size()); + assertSame(secondRead, pipeline.recomputed.get(0), "收尾必须拿重读后的最新行,不能吃本轮的旧快照"); + verify(lockHandle, times(1)).close(); + } + + @Test + void noMatchedRowLeavesStatusUntouchedButStillReleasesTheLock() { + when(fileResultMapper.selectList(any())).thenReturn(List.of(row("shop-x"))); + + pipeline.submitShopScopedResults(1L, List.of("Shop-Z"), String::valueOf, + AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null); + + assertTrue(pipeline.handled.isEmpty()); + assertTrue(pipeline.recomputed.isEmpty(), "changed 闸门是两支共有的:没改动就不许重算任务状态"); + verify(lockHandle, times(1)).close(); + } + + @Test + void rowWithoutScopeKeyIsSkippedInsteadOfCrashingTheLoop() { + when(fileResultMapper.selectList(any())).thenReturn(List.of(row(null))); + + pipeline.submitShopScopedResults(1L, List.of("Shop-A"), String::valueOf, + AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null); + + assertTrue(pipeline.handled.isEmpty()); + assertTrue(pipeline.recomputed.isEmpty()); + verify(lockHandle, times(1)).close(); + } + + @Test + void emptyShopListIsRejectedBeforeTakingTheLock() { + BusinessException empty = assertThrows(BusinessException.class, + () -> pipeline.submitShopScopedResults(1L, List.of(), String::valueOf, + AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null)); + assertEquals("shops 不能为空", empty.getMessage()); + + BusinessException nullList = assertThrows(BusinessException.class, + () -> pipeline.submitShopScopedResults(1L, null, String::valueOf, + AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null)); + assertEquals("shops 不能为空", nullList.getMessage()); + + verifyNoInteractions(lockService, fileResultMapper); + } + + @Test + void illegalTaskIdKeepsItsOwnMessageAndDoesNotReachTheShopValidation() { + BusinessException ex = assertThrows(BusinessException.class, + () -> pipeline.submitShopScopedResults(0L, List.of("Shop-A"), String::valueOf, + AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null)); + assertEquals("taskId 不合法", ex.getMessage()); + verifyNoInteractions(lockService, fileResultMapper); + } + + @Test + void terminalTaskRejectionStillReleasesTheLock() { + pipeline.loadedTask.setStatus("SUCCESS"); + + BusinessException ex = assertThrows(BusinessException.class, + () -> pipeline.submitShopScopedResults(1L, List.of("Shop-A"), String::valueOf, + AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null)); + + assertEquals(40901, ex.getCode(), "终态重复提交必须回 40901,让 Python 侧幂等忽略"); + verify(lockHandle, times(1)).close(); + verify(fileResultMapper, never()).selectList(any()); + } + + @Test + void hookFailurePropagatesAndTheLockIsNotLeaked() { + when(fileResultMapper.selectList(any())).thenReturn(List.of(row("shop-a"))); + pipeline.failOnHandle = new IllegalStateException("合并载荷时对象存储不可达"); + + assertThrows(IllegalStateException.class, + () -> pipeline.submitShopScopedResults(1L, List.of("Shop-A"), String::valueOf, + AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null)); + + verify(lockHandle, times(1)).close(); + assertTrue(pipeline.recomputed.isEmpty(), "单行炸了就不许带着半截状态去重算任务"); + } + + @Test + void incomingIndexSkipsNullAndUnnormalizableShopNamesKeepingRequestOrder() { + // 每项尾缀一个序号,好让「同名后者覆盖」这件事看得见(List.of 不收 null,这里用 Arrays.asList) + List incoming = java.util.Arrays.asList("Shop-B#1", null, " ", "Shop-A#1", "Shop-B#2"); + Function shopNameOf = item -> item.split("#")[0]; + + var indexed = pipeline.indexByShopKey(incoming, shopNameOf); + + assertEquals(List.of("shop-b", "shop-a"), List.copyOf(indexed.keySet()), + "空名与归一化后为空的项不进索引;请求顺序保留(两支都用 LinkedHashMap)"); + assertEquals("Shop-B#2", indexed.get("shop-b"), "同名后到的覆盖先到的——两支现网 put 同形,不许改成合并"); + } + + @Test + void shopKeyNormalizerIsUsedForBothIndexingAndMatching() { + // 行上的 sourceFilename 是建单时按同一规则写进去的归一化名(ShopMatch:601/PriceTrack:528-543) + when(fileResultMapper.selectList(any())).thenReturn(List.of(row("shop-a"))); + + pipeline.submitShopScopedResults(1L, List.of("Shop-A"), String::valueOf, + AbstractFileTaskPipeline.TerminalRejection.IDEMPOTENT, null); + + assertEquals(List.of("Shop-A"), pipeline.normalized, "索引侧必须调归一化钩子,两侧同规则才对得上"); + assertEquals(1, pipeline.handled.size()); + assertEquals("shop-a", pipeline.handled.get(0).shopKey); + } + + @Test + void lockLogTagDefaultsToModuleTypeSoSubclassesMustOverrideIt() { + assertEquals(MODULE_TYPE, pipeline.submissionLockLogTag(), + "默认给个不缺的标签;现网各支是小写短名(shop-match/price-track),迁移时必须覆写回历史值"); + + ShopPipeline tagged = new ShopPipeline(mock(FileTaskMapper.class), mock(FileResultMapper.class), + mock(TaskDistributedLockService.class), mock(TaskFileJobService.class), + mock(OssStorageService.class)) { + @Override + protected String submissionLockLogTag() { + return "shop-match"; + } + }; + assertEquals("shop-match", tagged.submissionLockLogTag()); + } + + // ------------------------------------------------------------------ + // 夹具 + // ------------------------------------------------------------------ + + private static FileTaskEntity runningTask() { + FileTaskEntity task = new FileTaskEntity(); + task.setId(1L); + task.setModuleType(MODULE_TYPE); + task.setStatus("RUNNING"); + return task; + } + + private static FileResultEntity row(String shopKey) { + FileResultEntity row = new FileResultEntity(); + row.setTaskId(1L); + row.setModuleType(MODULE_TYPE); + row.setSourceFilename(shopKey); + row.setSuccess(0); + return row; + } + + /** 家族层的具体子类:三个钩子只做记录,不引入任何模块语义。 */ + private static class ShopPipeline extends AbstractShopPayloadTaskPipeline { + + FileTaskEntity loadedTask; + final List handled = new ArrayList<>(); + final List> recomputed = new ArrayList<>(); + final List normalized = new ArrayList<>(); + RuntimeException failOnHandle; + + ShopPipeline(FileTaskMapper fileTaskMapper, + FileResultMapper fileResultMapper, + TaskDistributedLockService lockService, + TaskFileJobService jobService, + OssStorageService ossStorageService) { + super(fileTaskMapper, fileResultMapper, lockService, jobService, ossStorageService); + } + + @Override + protected String moduleType() { + return MODULE_TYPE; + } + + @Override + protected FileTaskEntity loadTaskForSubmission(Long taskId) { + return loadedTask; + } + + @Override + protected String normalizeShopKey(String rawShopName) { + normalized.add(rawShopName); + return rawShopName == null ? null : rawShopName.trim().toLowerCase(); + } + + @Override + protected void applyShopResultRow(Long taskId, + FileTaskEntity task, + FileResultEntity row, + String shopKey, + String incoming) { + if (failOnHandle != null) { + throw failOnHandle; + } + handled.add(new Handled(taskId, task, row, shopKey, incoming)); + } + + @Override + protected void onShopRowsRecomputed(FileTaskEntity task, List latestRows) { + recomputed.add(latestRows); + } + } + + private record Handled(Long taskId, FileTaskEntity task, FileResultEntity row, String shopKey, String incoming) { + } +}