diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCandidateVo.java b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskCandidateVo.java similarity index 92% rename from backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCandidateVo.java rename to backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskCandidateVo.java index 72606c3f..69bc9098 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCandidateVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskCandidateVo.java @@ -1,4 +1,4 @@ -package com.nanri.aiimage.modules.productrisk.model.vo; +package com.nanri.aiimage.common.model.vo; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCountryPreferenceVo.java b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskCountryPreferenceVo.java similarity index 91% rename from backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCountryPreferenceVo.java rename to backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskCountryPreferenceVo.java index c1c741db..6839c193 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCountryPreferenceVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskCountryPreferenceVo.java @@ -1,4 +1,4 @@ -package com.nanri.aiimage.modules.productrisk.model.vo; +package com.nanri.aiimage.common.model.vo; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskDashboardVo.java b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskDashboardVo.java similarity index 94% rename from backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskDashboardVo.java rename to backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskDashboardVo.java index 11268553..9e9498ac 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskDashboardVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskDashboardVo.java @@ -1,4 +1,4 @@ -package com.nanri.aiimage.modules.productrisk.model.vo; +package com.nanri.aiimage.common.model.vo; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskHistoryVo.java b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskHistoryVo.java similarity index 89% rename from backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskHistoryVo.java rename to backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskHistoryVo.java index 2611ca06..0b38e03f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskHistoryVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskHistoryVo.java @@ -1,4 +1,4 @@ -package com.nanri.aiimage.modules.productrisk.model.vo; +package com.nanri.aiimage.common.model.vo; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskMatchShopsVo.java b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskMatchShopsVo.java similarity index 89% rename from backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskMatchShopsVo.java rename to backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskMatchShopsVo.java index baecf492..b4ad71f2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskMatchShopsVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskMatchShopsVo.java @@ -1,4 +1,4 @@ -package com.nanri.aiimage.modules.productrisk.model.vo; +package com.nanri.aiimage.common.model.vo; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskResultItemVo.java similarity index 98% rename from backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskResultItemVo.java rename to backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskResultItemVo.java index 7407b0c0..c9c0599a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskResultItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskResultItemVo.java @@ -1,4 +1,4 @@ -package com.nanri.aiimage.modules.productrisk.model.vo; +package com.nanri.aiimage.common.model.vo; import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskShopQueueItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskShopQueueItemVo.java similarity index 98% rename from backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskShopQueueItemVo.java rename to backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskShopQueueItemVo.java index 089ad379..de22a14b 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskShopQueueItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskShopQueueItemVo.java @@ -1,4 +1,4 @@ -package com.nanri.aiimage.modules.productrisk.model.vo; +package com.nanri.aiimage.common.model.vo; import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskTaskBatchVo.java b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskTaskBatchVo.java similarity index 91% rename from backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskTaskBatchVo.java rename to backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskTaskBatchVo.java index 774cfd18..f5d1ef74 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskTaskBatchVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskTaskBatchVo.java @@ -1,4 +1,4 @@ -package com.nanri.aiimage.modules.productrisk.model.vo; +package com.nanri.aiimage.common.model.vo; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskTaskDetailVo.java b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskTaskDetailVo.java similarity index 91% rename from backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskTaskDetailVo.java rename to backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskTaskDetailVo.java index 61c59da5..b75162e7 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskTaskDetailVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskTaskDetailVo.java @@ -1,4 +1,4 @@ -package com.nanri.aiimage.modules.productrisk.model.vo; +package com.nanri.aiimage.common.model.vo; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskTaskItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskTaskItemVo.java similarity index 97% rename from backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskTaskItemVo.java rename to backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskTaskItemVo.java index 3e5d5dee..a834a6a4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskTaskItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/model/vo/ProductRiskTaskItemVo.java @@ -1,4 +1,4 @@ -package com.nanri.aiimage.modules.productrisk.model.vo; +package com.nanri.aiimage.common.model.vo; import com.fasterxml.jackson.annotation.JsonProperty; import com.nanri.aiimage.modules.shopmatch.model.vo.ShopMatchTaskStageVo; diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/module/TaskModuleRegistry.java b/backend-java/src/main/java/com/nanri/aiimage/common/module/TaskModuleRegistry.java new file mode 100644 index 00000000..e8ec10f8 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/module/TaskModuleRegistry.java @@ -0,0 +1,111 @@ +package com.nanri.aiimage.common.module; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 任务模块清单的**单一来源**（2026-09 全维度审查 G6）。 + * + * 背景：新增一个工具模块此前要同时改 5 处枚举（结果文件 Job 白名单、按天清理清单、 + * 站内通知模块名、任务心跳分支、陈旧判死巡检线），其中 4 处没有任何自检， + * 漏改表现为"功能静默不生效"。现在把**数据驱动的三处**（结果文件 Job、按天清理、通知中文名） + * 统一从这里派生，并由 {@code TaskModuleCoverageTest} 守住覆盖面。 + * + * 心跳与陈旧判死两处是代码分支驱动（switch / 逐模块委派），无法纯数据派生， + * 新增模块时仍需实现对应 Handler，测试会提示缺失。 + */ +public final class TaskModuleRegistry { + + /** + * 单个任务模块的声明。 + * + * @param type moduleType（与 biz_file_task.module_type 一致） + * @param label 站内通知/界面用中文名 + * @param resultFileJob 是否产出结果文件（需要注册 ResultFileJobHandler） + * @param ageCleanup 是否参与按天清理（ModuleCleanupProperties） + * @param delegatedStaleCheck 是否以"委派"方式并入 stale-check 巡检线 + * （DeleteBrandStaleTaskService.delegatedStaleChecks，必须逐模块登记动作） + * @param selfScheduledStaleCheck 该模块的陈旧判死自带更快的调度（publish 60s / collect-data 30s）， + * 刻意不并入 2 分钟一轮的巡检线——并入会显著拉长判死时延。 + * 这类模块的"不在委托名单"是设计差异，不是漏接；由覆盖面测试固定。 + */ + public record Module(String type, String label, boolean resultFileJob, boolean ageCleanup, + boolean delegatedStaleCheck, boolean selfScheduledStaleCheck) { + } + + private static final List MODULES = List.of( + new Module("PUBLISH", "上架", true, false, false, true), + new Module("DEDUPE", "数据去重", false, true, false, false), + new Module("SPLIT", "数据拆分", false, true, false, false), + new Module("CONVERT", "格式转换", false, true, false, false), + new Module("DELETE_BRAND", "删除ASIN", true, true, false, false), + new Module("PRODUCT_RISK_RESOLVE", "商品风险解决", true, true, false, false), + new Module("PRICE_TRACK", "跟价", true, true, false, false), + new Module("SHOP_MATCH", "定时匹配", true, true, false, false), + new Module("PATROL_DELETE", "巡店删除", true, true, false, false), + new Module("QUERY_ASIN", "查询ASIN", true, true, false, false), + new Module("WITHDRAW", "取款", true, true, false, false), + new Module("APPEARANCE_PATENT", "外观专利检测", true, true, true, false), + new Module("SIMILAR_ASIN", "货源查询", true, true, true, false), + new Module("COLLECT_DATA", "采集数据", true, true, false, true), + new Module("SHOP_DATA_CRAWL", "店铺数据抓取", true, false, true, false), + new Module("BRAND", "品牌检测", true, false, true, false) + ); + + private TaskModuleRegistry() { + } + + public static List modules() { + return MODULES; + } + + /** 全部 moduleType。 */ + public static Set moduleTypes() { + return MODULES.stream().map(Module::type).collect(Collectors.toUnmodifiableSet()); + } + + /** moduleType → 中文名（站内通知等展示用）。 */ + public static Map labels() { + Map labels = new LinkedHashMap<>(); + for (Module module : MODULES) { + labels.put(module.type(), module.label()); + } + return Map.copyOf(labels); + } + + /** 产出结果文件的模块（需要注册 Handler 的模块）。 */ + public static Set resultFileJobModuleTypes() { + return MODULES.stream().filter(Module::resultFileJob).map(Module::type) + .collect(Collectors.toUnmodifiableSet()); + } + + /** 参与按天清理的模块。 */ + public static Set ageCleanupModuleTypes() { + return MODULES.stream().filter(Module::ageCleanup).map(Module::type) + .collect(Collectors.toUnmodifiableSet()); + } + + /** 自带更快巡检节奏、刻意不并入集中巡检线的模块。 */ + public static Set selfScheduledStaleCheckModuleTypes() { + return MODULES.stream().filter(Module::selfScheduledStaleCheck).map(Module::type) + .collect(Collectors.toUnmodifiableSet()); + } + + /** 以"委派"方式并入 stale-check 巡检线的模块（DeleteBrandStaleTaskService 必须逐模块登记动作）。 */ + public static Set delegatedStaleCheckModuleTypes() { + return MODULES.stream().filter(Module::delegatedStaleCheck).map(Module::type) + .collect(Collectors.toUnmodifiableSet()); + } + + public static String labelOf(String moduleType) { + for (Module module : MODULES) { + if (module.type().equals(moduleType)) { + return module.label(); + } + } + return moduleType; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/service/NodeSharedStore.java b/backend-java/src/main/java/com/nanri/aiimage/common/service/NodeSharedStore.java new file mode 100644 index 00000000..5a00d370 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/service/NodeSharedStore.java @@ -0,0 +1,169 @@ +package com.nanri.aiimage.common.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; + +import java.time.Duration; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 跨节点的小容量状态存储（2026-09 全维度审查 D2）。 + * + *

用途：导入/导出任务的进度、归属用户、分组等状态。原先只存节点本地内存， + * 客户端轮询落到另一节点就报"任务不存在"（nginx 的 user_id 亲和只覆盖常态）。 + * 现在本地 Map 作快路径、Redis 作跨节点真源，任何节点都能读到。 + * + *

写节流：调用方可能**逐行**刷新进度（数十万行），逐次写 Redis 不可接受； + * 默认 500ms 内只写一次（终态用 {@link #putNow} 立即落库）。 + * + *

语义取舍：读-改-写不保证原子（进度类状态可接受）；{@code redis}/{@code objectMapper} + * 为空时退化为纯本地（单测与未注入场景）。 + */ +@Slf4j +public final class NodeSharedStore { + + private static final long DEFAULT_WRITE_THROTTLE_MILLIS = 500L; + + private final String keyPrefix; + private final Duration ttl; + private final Class valueType; + private final StringRedisTemplate redis; + private final ObjectMapper objectMapper; + private final long writeThrottleMillis; + + private final ConcurrentHashMap local = new ConcurrentHashMap<>(); + private final ConcurrentHashMap lastWriteAt = new ConcurrentHashMap<>(); + + public NodeSharedStore(String keyPrefix, + Duration ttl, + Class valueType, + StringRedisTemplate redis, + ObjectMapper objectMapper) { + this(keyPrefix, ttl, valueType, redis, objectMapper, DEFAULT_WRITE_THROTTLE_MILLIS); + } + + public NodeSharedStore(String keyPrefix, + Duration ttl, + Class valueType, + StringRedisTemplate redis, + ObjectMapper objectMapper, + long writeThrottleMillis) { + this.keyPrefix = keyPrefix; + this.ttl = ttl; + this.valueType = valueType; + this.redis = redis; + this.objectMapper = objectMapper; + this.writeThrottleMillis = Math.max(0L, writeThrottleMillis); + } + + /** 本地优先；本地没有则读 Redis 并回填本地（跨节点可见）。 */ + public V get(K key) { + if (key == null) { + return null; + } + V cached = local.get(key); + if (cached != null) { + return cached; + } + V remote = readRemote(key); + if (remote != null) { + local.put(key, remote); + } + return remote; + } + + public boolean containsKey(K key) { + return get(key) != null; + } + + /** 写入并（按节流）同步到 Redis。 */ + public void put(K key, V value) { + if (key == null || value == null) { + return; + } + local.put(key, value); + if (!throttleAllowsWrite(key)) { + return; + } + writeRemote(key, value); + } + + /** 立即写入 Redis（终态、归属等一次性状态用）。 */ + public void putNow(K key, V value) { + if (key == null || value == null) { + return; + } + local.put(key, value); + writeRemote(key, value); + } + + /** 删除（本地 + Redis），返回删除前的值（可能为 null）。 */ + public V remove(K key) { + if (key == null) { + return null; + } + V previous = local.remove(key); + lastWriteAt.remove(key); + if (redis != null) { + try { + redis.delete(fullKey(key)); + } catch (Exception ex) { + log.warn("[node-shared-store] 删除远端状态失败 key={} msg={}", fullKey(key), ex.getMessage()); + } + } + return previous; + } + + /** 本地快照（仅用于日志/统计；不含其它节点的写入）。 */ + public int localSize() { + return local.size(); + } + + private boolean throttleAllowsWrite(K key) { + if (writeThrottleMillis <= 0L) { + return true; + } + long now = System.currentTimeMillis(); + AtomicLong last = lastWriteAt.computeIfAbsent(key, ignored -> new AtomicLong(0L)); + long previous = last.get(); + if (now - previous < writeThrottleMillis) { + return false; + } + return last.compareAndSet(previous, now); + } + + private V readRemote(K key) { + if (redis == null || objectMapper == null) { + return null; + } + try { + String json = redis.opsForValue().get(fullKey(key)); + if (json == null || json.isBlank()) { + return null; + } + return objectMapper.readValue(json, valueType); + } catch (Exception ex) { + log.warn("[node-shared-store] 读取远端状态失败 key={} msg={}", fullKey(key), ex.getMessage()); + return null; + } + } + + private void writeRemote(K key, V value) { + if (redis == null || objectMapper == null) { + return; + } + try { + String json = objectMapper.writeValueAsString(value); + redis.opsForValue().set(fullKey(key), json, ttl); + } catch (Exception ex) { + // 写失败不影响本地进度（下次 put 会重试），只留线索 + log.warn("[node-shared-store] 写入远端状态失败 key={} msg={}", fullKey(key), ex.getMessage()); + } + } + + private String fullKey(K key) { + return keyPrefix + ":" + key; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/util/ExcelStreamReader.java b/backend-java/src/main/java/com/nanri/aiimage/common/util/ExcelStreamReader.java index 77725f91..af1c8bda 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/common/util/ExcelStreamReader.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/util/ExcelStreamReader.java @@ -56,6 +56,13 @@ public final class ExcelStreamReader { default void onHeader(String sheetName, Integer sheetNo, Map headerMap) throws Exception { } + /** + * 表头之后回调本次 sheet 的近似总行数（EasyExcel 基于 sheet 尺寸，可能为 null）。 + * 流式解析无法在读完前得到精确行数，需要展示进度/做前置上限校验的调用方可用它近似。 + */ + default void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) throws Exception { + } + void onRow(String sheetName, Integer sheetNo, int rowIndex, Map headerMap, Map rowMap) throws Exception; } @@ -77,6 +84,7 @@ public final class ExcelStreamReader { currentHeaderMap = normalizedHeadMap; try { handler.onHeader(sheetName(context), sheetNo(context), currentHeaderMap); + handler.onSheetTotal(sheetName(context), sheetNo(context), approximateTotalRows(context)); } catch (RuntimeException ex) { throw ex; } catch (Exception ex) { @@ -109,6 +117,15 @@ public final class ExcelStreamReader { public void doAfterAllAnalysed(AnalysisContext context) { } + private Integer approximateTotalRows(AnalysisContext context) { + try { + return context.readSheetHolder() == null ? null + : context.readSheetHolder().getApproximateTotalRowNumber(); + } catch (Exception ex) { + return null; + } + } + private String sheetName(AnalysisContext context) { return context.readSheetHolder() == null ? "" : context.readSheetHolder().getSheetName(); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/util/PendingDeleteJournal.java b/backend-java/src/main/java/com/nanri/aiimage/common/util/PendingDeleteJournal.java new file mode 100644 index 00000000..f1eb9dab --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/util/PendingDeleteJournal.java @@ -0,0 +1,138 @@ +package com.nanri.aiimage.common.util; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * 待删对象队列的本地落盘日志：进程重启后仍能恢复「待删除对象」清单。 + * + *

使用场景：对象存储删除失败后的补偿队列原本只在节点内存里，重启即丢， + * 对应对象会一直残留在桶里直到生命周期规则过期。这里用一行一个 objectKey 的 + * 追加日志做最小持久化——入队追加、队列收敛后整体重写、启动时回放。 + * + *

并发：所有方法内部同步，调用方无需额外加锁；文件损坏/读写异常只记日志不抛出， + * 保证补偿链路本身不会因为落盘失败而中断业务。 + * + *

上限：{@code maxEntries} 用于防止日志在异常堆积时无界增长（超出后丢弃最早的记录， + * 与内存队列的容量准入语义一致——丢的是「待删对象」，最坏结果是对象残留）。 + */ +@Slf4j +public final class PendingDeleteJournal { + + private final Path path; + private final int maxEntries; + private final Object lock = new Object(); + + public PendingDeleteJournal(Path path, int maxEntries) { + this.path = path; + this.maxEntries = Math.max(1, maxEntries); + } + + public Path getPath() { + return path; + } + + /** 追加一条待删对象（重复追加由回放时的 Set 语义去重）。 */ + public void record(String objectKey) { + if (objectKey == null || objectKey.isBlank()) { + return; + } + synchronized (lock) { + try { + Path parent = path.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + Files.writeString(path, sanitize(objectKey) + System.lineSeparator(), + StandardCharsets.UTF_8, StandardOpenOption.CREATE, StandardOpenOption.APPEND); + } catch (Exception ex) { + log.warn("[delete-journal] 追加待删对象失败 path={} msg={}", path, ex.getMessage()); + } + } + } + + /** 启动回放：返回日志中的待删对象（按首次出现顺序去重）。 */ + public List readAll() { + synchronized (lock) { + if (!Files.isRegularFile(path)) { + return List.of(); + } + List lines; + try { + lines = Files.readAllLines(path, StandardCharsets.UTF_8); + } catch (Exception ex) { + log.warn("[delete-journal] 读取待删对象日志失败 path={} msg={}", path, ex.getMessage()); + return List.of(); + } + Set unique = new LinkedHashSet<>(); + for (String line : lines) { + if (line == null) { + continue; + } + String key = line.trim(); + if (!key.isEmpty()) { + unique.add(key); + } + } + return new ArrayList<>(unique); + } + } + + /** 队列收敛（成功删除/引用仍在）后整体重写为剩余的待删对象；剩余为空则删除日志。 */ + public void rewrite(Collection remaining) { + synchronized (lock) { + Set keep = new LinkedHashSet<>(); + if (remaining != null) { + for (String key : remaining) { + if (key != null && !key.isBlank()) { + keep.add(sanitize(key)); + if (keep.size() >= maxEntries) { + break; + } + } + } + } + try { + if (keep.isEmpty()) { + Files.deleteIfExists(path); + return; + } + Path parent = path.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + Path tmp = path.resolveSibling(path.getFileName() + ".tmp"); + StringBuilder content = new StringBuilder(); + for (String key : keep) { + content.append(key).append(System.lineSeparator()); + } + Files.writeString(tmp, content.toString(), StandardCharsets.UTF_8, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + try { + Files.move(tmp, path, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException atomicUnsupported) { + // 少数文件系统不支持 ATOMIC_MOVE，退化为普通替换 + Files.move(tmp, path, StandardCopyOption.REPLACE_EXISTING); + } + } catch (Exception ex) { + log.warn("[delete-journal] 重写待删对象日志失败 path={} msg={}", path, ex.getMessage()); + } + } + } + + /** 单行一条记录：去掉换行避免破坏行结构。 */ + private static String sanitize(String objectKey) { + return objectKey.replace('\n', ' ').replace('\r', ' ').trim(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/AdminApiGuardFilter.java b/backend-java/src/main/java/com/nanri/aiimage/config/AdminApiGuardFilter.java index b130cac5..96237322 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/AdminApiGuardFilter.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/AdminApiGuardFilter.java @@ -79,6 +79,9 @@ public class AdminApiGuardFilter extends OncePerRequestFilter { "/api/shop-data-crawl", "/api/withdraw", "/api/task-file-jobs", + // 2026-09 复核补：该前缀全部接口已在 controller 内 requireAdmin（含可换取员工店铺登录 + // 令牌的 /shops/open），此处纳入名单是"开关打开后的第二层"；开关关闭时行为不变 + "/api/ziniao", // /api/tasks/{taskId}/interrupted 仅凭 taskId 即可把 RUNNING 任务置为 FAILED， // 匿名遍历 taskId 就能批量打断线上任务 "/api/tasks", diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java index e71e83ec..0903523f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java @@ -1,5 +1,6 @@ package com.nanri.aiimage.config; +import com.nanri.aiimage.common.module.TaskModuleRegistry; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -19,5 +20,6 @@ public class ModuleCleanupProperties { private int batchSize = 500; // SHOP_DATA_CRAWL keeps one per-shop daily workbook in its task service // and must not be removed by the age-based sweep. - private List moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA")); + /** 参与按天清理的模块：取自模块注册表（G6）。 */ + private List moduleTypes = new ArrayList<>(TaskModuleRegistry.ageCleanupModuleTypes()); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java index cf638f38..47e4f02b 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java @@ -33,6 +33,7 @@ import io.micrometer.core.instrument.MeterRegistry; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import com.nanri.aiimage.common.module.TaskModuleRegistry; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.task.TaskExecutor; @@ -52,11 +53,11 @@ import java.util.concurrent.Semaphore; public class TaskFileJobConfig { /** 结果文件 Job 支持的全部 moduleType（启动校验枚举源，见 ResultFileJobHandlerRegistry.validateCoverage） */ - public static final Set RESULT_FILE_JOB_MODULE_TYPES = Set.of( - "SHOP_MATCH", "PRICE_TRACK", "PRODUCT_RISK_RESOLVE", - "PUBLISH", "QUERY_ASIN", "SHOP_DATA_CRAWL", "WITHDRAW", - "PATROL_DELETE", "APPEARANCE_PATENT", "SIMILAR_ASIN", - "DELETE_BRAND", "BRAND", "COLLECT_DATA"); + /** + * 结果文件 Job 支持的 moduleType：取自模块注册表（G6）， + * 与 {@code ResultFileJobHandlerRegistry.validateCoverage} 的启动自检配合使用。 + */ + public static final Set RESULT_FILE_JOB_MODULE_TYPES = TaskModuleRegistry.resultFileJobModuleTypes(); @Bean("taskFileJobDispatchExecutor") public TaskExecutor taskFileJobDispatchExecutor( diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskHeartbeatSpi.java new file mode 100644 index 00000000..c52be579 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskHeartbeatSpi.java @@ -0,0 +1,30 @@ +package com.nanri.aiimage.modules.appearancepatent.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +/** + * 外观专利检测 的任务心跳实现（2026-09 全维度审查 G5）。 + * + *

心跳逻辑（缓存刷新 / 任务缓存回写）从 task 模块收回本模块，task 侧只依赖 SPI 接口， + * 消除 task → 业务模块的编译期依赖。 + */ +@Service +@RequiredArgsConstructor +public class AppearancePatentTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { + + private final AppearancePatentTaskCacheService cacheService; + + @Override + public String moduleType() { + return "APPEARANCE_PATENT"; + } + + @Override + public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { + cacheService.touchTaskHeartbeat(taskId); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentExcelParser.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentExcelParser.java index 7f12b4ed..1ab7e4fb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentExcelParser.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentExcelParser.java @@ -1,17 +1,17 @@ package com.nanri.aiimage.modules.appearancepatent.service.support; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.util.ExcelStreamReader; import lombok.extern.slf4j.Slf4j; -import org.apache.poi.ss.usermodel.DataFormatter; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; -import org.apache.poi.ss.usermodel.WorkbookFactory; +import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; +import java.io.PushbackInputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -37,9 +37,8 @@ public class AppearancePatentExcelParser { if (input == null) { throw new IllegalArgumentException("input must not be null"); } - try (FileInputStream fis = new FileInputStream(input); - Workbook workbook = WorkbookFactory.create(fis)) { - return parseWorkbook(workbook, maxRows); + try (InputStream inputStream = new FileInputStream(input)) { + return readStreaming(inputStream, maxRows); } catch (BusinessException ex) { throw ex; } catch (Exception ex) { @@ -52,8 +51,8 @@ public class AppearancePatentExcelParser { if (input == null) { throw new IllegalArgumentException("input must not be null"); } - try (Workbook workbook = WorkbookFactory.create(input)) { - return parseWorkbook(workbook, DEFAULT_MAX_ROWS); + try { + return readStreaming(input, DEFAULT_MAX_ROWS); } catch (BusinessException ex) { throw ex; } catch (Exception ex) { @@ -62,83 +61,154 @@ public class AppearancePatentExcelParser { } } - private ParsedSheet parseWorkbook(Workbook workbook, int maxRows) { - int safeMaxRows = Math.max(1, maxRows); - DataFormatter formatter = new DataFormatter(); - Sheet sheet = workbook.getSheetAt(0); - Row header = sheet.getRow(0); - if (header == null) { + /** + * 流式解析（EasyExcel SAX），替代 POI WorkbookFactory 全量 DOM 加载： + * 大表不再整表驻留堆内存，行数上限在迭代过程中即时生效（超限抛错）。 + * 语义与原 POI 路径一致：cell 归一化、表头别名匹配、空行跳过、必填表头缺失抛错、无字段截断。 + * 注意不关闭传入的 InputStream（由调用方负责）。 + */ + private ParsedSheet readStreaming(InputStream inputStream, int maxRows) throws Exception { + // EasyExcel 会把非 zip 文本当 CSV 解析成功；原 WorkbookFactory 只认 xlsx/xls， + // 这里先做文件魔数校验，保持「垃圾文件→解析 Excel 失败」的语义并拒绝 CSV 误解析。 + PushbackInputStream pb = new PushbackInputStream(new BufferedInputStream(inputStream, 8192), 8); + requireExcelMagic(pb); + SheetContext ctx = new SheetContext(Math.max(1, maxRows)); + ExcelStreamReader.readFirstSheet(pb, new ExcelStreamReader.SheetRowHandler() { + @Override + public void onHeader(String sheetName, Integer sheetNo, Map headerMap) { + ctx.initHeader(headerMap); + } + + @Override + public void onRow(String sheetName, Integer sheetNo, int rowIndex, + Map headerMap, Map rowMap) { + ctx.consumeRow(rowIndex, rowMap); + } + }); + if (ctx.headers == null) { + // 空表 / 表头行整行为空（EasyExcel 不上报表头回调）：与旧实现 header == null 一致 throw new BusinessException("Excel 表头为空"); } - Map headerMap = buildHeaderMap(header, formatter); - List headers = readHeaders(header, formatter); - int idCol = findRequiredHeader(headerMap, "id"); - int asinCol = findRequiredHeader(headerMap, "asin"); - int countryCol = findRequiredHeader(headerMap, "国家", "country"); - int priceCol = findOptionalHeaderExact(headerMap, "价格", "price"); - int skuCol = findOptionalHeaderExact(headerMap, - "sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku"); - int urlCol = findOptionalHeaderExact(headerMap, - "url", "rul", "link", "image", "img", "pic", "picture", - "链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接"); - int titleCol = findOptionalHeaderExact(headerMap, - "标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称"); + return new ParsedSheet(ctx.headers, ctx.rows); + } - List rows = new ArrayList<>(); - for (int i = 1; i <= sheet.getLastRowNum(); i++) { - Row row = sheet.getRow(i); - if (row == null) { - continue; + /** 校验文件头魔数：xlsx=PK(zip)、xls=OLE2(CFD)。判非抛「解析 Excel 失败」；用 unread 回退已读字节。 */ + private void requireExcelMagic(PushbackInputStream in) throws IOException { + byte[] head = new byte[8]; + int n = 0; + while (n < head.length) { + int r = in.read(head, n, head.length - n); + if (r < 0) { + break; } - String id = cell(row, idCol, formatter); - String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT); - String country = cell(row, countryCol, formatter); + n += r; + } + if (n > 0) { + in.unread(head, 0, n); + } + boolean isZip = n >= 4 && head[0] == 'P' && head[1] == 'K' && (head[2] == 3 || head[2] == 5 || head[2] == 7); + boolean isOle2 = n >= 8 + && (head[0] & 0xFF) == 0xD0 && (head[1] & 0xFF) == 0xCF + && (head[2] & 0xFF) == 0x11 && (head[3] & 0xFF) == 0xE0; + if (!isZip && !isOle2) { + log.warn("[appearance-patent] parse rejected non-excel magic head={}", Arrays.copyOf(head, Math.max(n, 0))); + throw new BusinessException("解析 Excel 失败"); + } + } + + /** 单表解析上下文：表头就绪后逐行累积结果行。 */ + private final class SheetContext { + + private final int maxRows; + private final List rows = new ArrayList<>(); + private List headers; + private int idCol; + private int asinCol; + private int countryCol; + private int priceCol; + private int skuCol; + private int urlCol; + private int titleCol; + + SheetContext(int maxRows) { + this.maxRows = maxRows; + } + + void initHeader(Map rawHeaderMap) { + // EasyExcel 回调给出 列号 → 表头文本，与旧 Row 遍历等价（缺列一般为 null/空串） + int lastColumnCount = lastColumnCount(rawHeaderMap); + Map map = new LinkedHashMap<>(); + List headerNames = new ArrayList<>(); + for (int i = 0; i < lastColumnCount; i++) { + String val = normalize(rawHeaderMap.getOrDefault(i, "")); + headerNames.add(val.isBlank() ? "列" + (i + 1) : val); + if (!val.isBlank()) { + map.putIfAbsent(val.toLowerCase(Locale.ROOT), i); + } + } + this.headers = headerNames; + this.idCol = findRequiredHeader(map, "id"); + this.asinCol = findRequiredHeader(map, "asin"); + this.countryCol = findRequiredHeader(map, "国家", "country"); + this.priceCol = findOptionalHeaderExact(map, "价格", "price"); + this.skuCol = findOptionalHeaderExact(map, + "sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku"); + this.urlCol = findOptionalHeaderExact(map, + "url", "rul", "link", "image", "img", "pic", "picture", + "链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接"); + this.titleCol = findOptionalHeaderExact(map, + "标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称"); + } + + void consumeRow(int rowIndex, Map rowMap) { + if (headers == null) { + return; + } + // EasyExcel rowIndex 从 0 起（0 为表头），POI 原实现行号同样 0 起并 +1 展示 + String id = streamCell(rowMap, idCol); + String asin = streamCell(rowMap, asinCol).toUpperCase(Locale.ROOT); + String country = streamCell(rowMap, countryCol); if (id.isBlank() && asin.isBlank() && country.isBlank()) { - continue; + return; } - if (rows.size() >= safeMaxRows) { - throw new BusinessException("解析行数超过上限: " + safeMaxRows); + if (rows.size() >= maxRows) { + throw new BusinessException("解析行数超过上限: " + maxRows); + } + Map values = new LinkedHashMap<>(); + for (int i = 0; i < headers.size(); i++) { + values.put(headers.get(i), streamCell(rowMap, i)); } rows.add(new AppearanceExcelRow( - i + 1, + rowIndex + 1, id, asin, country, - priceCol >= 0 ? cell(row, priceCol, formatter) : "", - skuCol >= 0 ? cell(row, skuCol, formatter) : "", - urlCol >= 0 ? cell(row, urlCol, formatter) : "", - titleCol >= 0 ? cell(row, titleCol, formatter) : "", - readRowValues(row, headers, formatter))); + priceCol >= 0 ? streamCell(rowMap, priceCol) : "", + skuCol >= 0 ? streamCell(rowMap, skuCol) : "", + urlCol >= 0 ? streamCell(rowMap, urlCol) : "", + titleCol >= 0 ? streamCell(rowMap, titleCol) : "", + values)); } - return new ParsedSheet(headers, rows); - } - private Map buildHeaderMap(Row header, DataFormatter formatter) { - Map map = new LinkedHashMap<>(); - for (int i = 0; i < header.getLastCellNum(); i++) { - String val = normalize(formatter.formatCellValue(header.getCell(i))); - if (!val.isBlank()) { - map.putIfAbsent(val.toLowerCase(Locale.ROOT), i); + private int lastColumnCount(Map rowMap) { + if (rowMap == null || rowMap.isEmpty()) { + return 0; } + int lastColumnCount = 0; + for (Integer columnIndex : rowMap.keySet()) { + if (columnIndex != null && columnIndex >= lastColumnCount) { + lastColumnCount = columnIndex + 1; + } + } + return lastColumnCount; } - return map; - } - private List readHeaders(Row header, DataFormatter formatter) { - List headers = new ArrayList<>(); - for (int i = 0; i < header.getLastCellNum(); i++) { - String val = normalize(formatter.formatCellValue(header.getCell(i))); - headers.add(val.isBlank() ? "列" + (i + 1) : val); + private String streamCell(Map rowMap, int col) { + if (col < 0 || rowMap == null) { + return ""; + } + return normalize(rowMap.get(col)); } - return headers; - } - - private Map readRowValues(Row row, List headers, DataFormatter formatter) { - Map values = new LinkedHashMap<>(); - for (int i = 0; i < headers.size(); i++) { - values.put(headers.get(i), cell(row, i, formatter)); - } - return values; } private int findRequiredHeader(Map map, String... names) { @@ -177,10 +247,6 @@ public class AppearancePatentExcelParser { return normalized.replaceAll("[\\s_\\-()（）\\[\\]{}:：/\\\\]+", ""); } - private String cell(Row row, int col, DataFormatter formatter) { - return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col))); - } - private String normalize(String val) { return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " "); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandCrawlTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandCrawlTaskHeartbeatSpi.java new file mode 100644 index 00000000..e940fa66 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandCrawlTaskHeartbeatSpi.java @@ -0,0 +1,97 @@ +package com.nanri.aiimage.modules.brand.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper; +import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity; +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo; +import com.nanri.aiimage.modules.task.spi.BrandTaskHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; + +/** + * 品牌检测任务（brand_crawl_tasks）的心跳/中断实现（2026-09 全维度审查 G5）。 + * + *

品牌任务有独立的表与状态字面量（running/pending/cancelled），原先这段逻辑散在 + * {@code TaskHeartbeatService} 里并直接依赖本模块的 Mapper 与进度缓存；现整体收回本模块。 + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class BrandCrawlTaskHeartbeatSpi implements BrandTaskHeartbeatSpi { + + private static final String MODULE_BRAND = "BRAND"; + private static final String BRAND_STATUS_RUNNING = "running"; + + private final BrandCrawlTaskMapper brandCrawlTaskMapper; + private final BrandTaskProgressCacheService brandTaskProgressCacheService; + + @Override + public TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request) { + BrandCrawlTaskEntity task = selectTask(taskId); + if (task == null) { + return null; + } + String status = task.getStatus(); + if (!BRAND_STATUS_RUNNING.equals(status)) { + log.warn("[task-heartbeat] brand task is not running taskId={} actualUserId={} status={}", + task.getId(), task.getUserId(), status); + return TaskHeartbeatVo.notAlive(MODULE_BRAND, status, "task is not running"); + } + int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(BrandCrawlTaskEntity::getId, task.getId()) + .eq(BrandCrawlTaskEntity::getStatus, BRAND_STATUS_RUNNING) + .set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now())); + if (updated <= 0) { + BrandCrawlTaskEntity latest = brandCrawlTaskMapper.selectById(task.getId()); + log.warn("[task-heartbeat] brand task heartbeat update missed taskId={} actualUserId={} status={} latestStatus={}", + task.getId(), task.getUserId(), status, + latest == null ? null : latest.getStatus()); + return TaskHeartbeatVo.notAlive(MODULE_BRAND, latest == null ? status : latest.getStatus(), + "task is not running"); + } + brandTaskProgressCacheService.touchHeartbeat( + task.getId(), + request == null ? null : request.getPhase(), + request == null ? null : request.getCurrent(), + request == null ? null : request.getTotal()); + return TaskHeartbeatVo.alive(MODULE_BRAND, BRAND_STATUS_RUNNING); + } + + @Override + public TaskHeartbeatVo markInterrupted(Long taskId, String reason) { + BrandCrawlTaskEntity task = selectTask(taskId); + if (task == null) { + return null; + } + String status = task.getStatus(); + if ("running".equalsIgnoreCase(status) || "pending".equalsIgnoreCase(status)) { + // 与 file 分支统一改为条件更新：整行 updateById 会拿读取快照覆盖并发写入的字段 + // （客户端重启上报与品牌任务自身状态流转同时发生时） + int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(BrandCrawlTaskEntity::getId, task.getId()) + .in(BrandCrawlTaskEntity::getStatus, "running", "pending", "RUNNING", "PENDING") + .set(BrandCrawlTaskEntity::getStatus, "cancelled") + .set(BrandCrawlTaskEntity::getErrorMessage, reason) + .set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now())); + if (updated > 0) { + log.warn("[task-interrupted] brand task marked cancelled by client restart taskId={} reason={}", + taskId, reason); + return TaskHeartbeatVo.notAlive(MODULE_BRAND, "cancelled", "marked cancelled"); + } + } + log.info("[task-interrupted] brand task not in running/pending, skipped taskId={} status={}", taskId, status); + return TaskHeartbeatVo.notAlive(MODULE_BRAND, status, "task is not running"); + } + + private BrandCrawlTaskEntity selectTask(Long taskId) { + LambdaQueryWrapper brandQuery = new LambdaQueryWrapper() + .eq(BrandCrawlTaskEntity::getId, taskId) + .last("limit 1"); + return brandCrawlTaskMapper.selectOne(brandQuery); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java index 10012d17..1115bab5 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.common.util.ExcelStreamReader; import com.nanri.aiimage.config.BrandProgressProperties; import com.nanri.aiimage.config.HttpClientPool; import com.nanri.aiimage.config.StorageProperties; @@ -39,12 +40,7 @@ import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.DataFormatter; -import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; -import org.apache.poi.ss.usermodel.WorkbookFactory; import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.stereotype.Service; @@ -882,81 +878,133 @@ public class BrandTaskService { } private ParsedBrandFile parseBrandFile(File inputFile) { - DataFormatter formatter = new DataFormatter(); - try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = WorkbookFactory.create(fis)) { - Sheet sheet = workbook.getSheetAt(0); - Row headerRow = sheet.getRow(0); - if (headerRow == null) { - throw new BusinessException("Excel 表头为空"); - } - List columns = extractHeaders(headerRow, formatter); - if (columns.isEmpty()) { - throw new BusinessException("未读取到有效表头"); - } - Map headerIndexes = buildHeaderIndexes(headerRow, formatter, columns); - List> rows = new ArrayList<>(); - Set uniqueBrands = new LinkedHashSet<>(); - for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { - Row row = sheet.getRow(rowNum); - if (row == null) { - continue; - } - Map rowData = new LinkedHashMap<>(); - for (String column : columns) { - Integer index = headerIndexes.get(column); - String value = index == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(index))); - rowData.put(column, value); - } - String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), "")); - if (brand.isBlank()) { - rowData.put("__rowIndex", rowNum + 1); - rows.add(rowData); - continue; - } - if (uniqueBrands.add(brand)) { - rowData.put("__rowIndex", rowNum + 1); - rows.add(rowData); - } - } - return new ParsedBrandFile(sheet.getSheetName(), columns, rows, new ArrayList<>(uniqueBrands)); + // 2026-09 全维度审查 C5：改逐行流式解析（ExcelStreamReader → EasyExcel SAX）。 + // 此前 WorkbookFactory.create 把整表读成 DOM，品牌源文件（几十万行）会整表驻留堆内存。 + // 解析结果（columns/rows/uniqueBrands/行号）与错误文案不变，行号沿用 sheet 绝对 0 基行号 +1。 + if (!hasExcelMagic(inputFile)) { + log.warn("[brand] 源文件不是 Excel（疑似 CSV/文本），拒绝解析 file={}", inputFile.getName()); + throw new BusinessException("读取 Excel 失败"); + } + BrandFileSheetContext context = new BrandFileSheetContext(); + try { + ExcelStreamReader.readFirstSheet(inputFile, context); + } catch (BusinessException ex) { + throw ex; } catch (IOException ex) { throw new BusinessException("读取 Excel 失败"); } + return context.finish(); } - private List extractHeaders(Row headerRow, DataFormatter formatter) { - List headers = new ArrayList<>(); - Set seen = new LinkedHashSet<>(); - for (int i = 0; i < headerRow.getLastCellNum(); i++) { - Cell cell = headerRow.getCell(i); - String value = normalizeHeaderValue(cell == null ? null : formatter.formatCellValue(cell)); - if (value.isBlank() || seen.contains(value)) { - continue; - } - seen.add(value); - headers.add(value); - if ("缩略图地址8".equals(value)) { - break; - } + /** + * 文件魔数校验：xlsx=PK(zip)、xls=OLE2(CFD)。 + * EasyExcel 会把非 zip 文本当 CSV 静默解析成功，而旧 POI WorkbookFactory 只认 xlsx/xls（垃圾文件直接失败）， + * 这里保持「非 Excel 源文件 → 读取失败」的语义（与 SimilarAsinExcelParser 同口径）。 + */ + private boolean hasExcelMagic(File file) { + byte[] head = new byte[8]; + try (InputStream inputStream = new FileInputStream(file)) { + int n = inputStream.readNBytes(head, 0, head.length); + boolean isZip = n >= 4 && head[0] == 'P' && head[1] == 'K' && (head[2] == 3 || head[2] == 5 || head[2] == 7); + boolean isOle2 = n >= 8 + && (head[0] & 0xFF) == 0xD0 && (head[1] & 0xFF) == 0xCF + && (head[2] & 0xFF) == 0x11 && (head[3] & 0xFF) == 0xE0; + return isZip || isOle2; + } catch (IOException ex) { + return false; } - return headers; } - private Map buildHeaderIndexes(Row headerRow, DataFormatter formatter, List columns) { - Map headerIndexes = new LinkedHashMap<>(); - Set allowed = new LinkedHashSet<>(columns); - for (int i = 0; i < headerRow.getLastCellNum(); i++) { - Cell cell = headerRow.getCell(i); - String value = normalizeHeaderValue(cell == null ? null : formatter.formatCellValue(cell)); - if (value.isBlank() || headerIndexes.containsKey(value) || !allowed.contains(value)) { - continue; + /** 品牌源文件流式解析上下文：表头解析 + 按品牌去重累积结果行（内存不随整表 DOM 放大）。 */ + private final class BrandFileSheetContext implements ExcelStreamReader.SheetRowHandler { + + private final List> rows = new ArrayList<>(); + private final Set uniqueBrands = new LinkedHashSet<>(); + private List columns; + private Map headerIndexes; + private String sheetName = ""; + + @Override + public void onHeader(String sheetName, Integer sheetNo, Map headerMap) { + this.sheetName = sheetName == null ? "" : sheetName; + int lastColumnCount = lastColumnCount(headerMap); + List headers = new ArrayList<>(); + Set seen = new LinkedHashSet<>(); + for (int i = 0; i < lastColumnCount; i++) { + String value = normalizeHeaderValue(headerMap.get(i)); + if (value.isBlank() || seen.contains(value)) { + continue; + } + seen.add(value); + headers.add(value); + if ("缩略图地址8".equals(value)) { + break; + } } - headerIndexes.put(value, i); - if ("缩略图地址8".equals(value)) { - break; + if (headers.isEmpty()) { + throw new BusinessException("未读取到有效表头"); + } + Map indexes = new LinkedHashMap<>(); + Set allowed = new LinkedHashSet<>(headers); + for (int i = 0; i < lastColumnCount; i++) { + String value = normalizeHeaderValue(headerMap.get(i)); + if (value.isBlank() || indexes.containsKey(value) || !allowed.contains(value)) { + continue; + } + indexes.put(value, i); + if ("缩略图地址8".equals(value)) { + break; + } + } + this.columns = headers; + this.headerIndexes = indexes; + } + + @Override + public void onRow(String sheetName, Integer sheetNo, int rowIndex, + Map headerMap, Map rowMap) { + if (columns == null) { + // 表头行整行为空时 EasyExcel 不上报表头回调：与旧实现 headerRow == null 一致 + throw new BusinessException("Excel 表头为空"); + } + Map rowData = new LinkedHashMap<>(); + for (String column : columns) { + Integer index = headerIndexes.get(column); + String value = index == null ? "" : normalizeCellText(rowMap.get(index)); + rowData.put(column, value); + } + String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), "")); + if (brand.isBlank()) { + rowData.put("__rowIndex", rowIndex + 1); + rows.add(rowData); + return; + } + if (uniqueBrands.add(brand)) { + rowData.put("__rowIndex", rowIndex + 1); + rows.add(rowData); } } - return headerIndexes; + + private ParsedBrandFile finish() { + if (columns == null) { + // 空表（无任何行）或表头行缺失：与旧实现 headerRow == null 一致 + throw new BusinessException("Excel 表头为空"); + } + return new ParsedBrandFile(sheetName, columns, rows, new ArrayList<>(uniqueBrands)); + } + + private int lastColumnCount(Map headerMap) { + if (headerMap == null || headerMap.isEmpty()) { + return 0; + } + int lastColumnCount = 0; + for (Integer columnIndex : headerMap.keySet()) { + if (columnIndex != null && columnIndex >= lastColumnCount) { + lastColumnCount = columnIndex + 1; + } + } + return lastColumnCount; + } } private String normalizeHeaderValue(String value) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStaleRepairSpiImpl.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStaleRepairSpiImpl.java new file mode 100644 index 00000000..49ce6942 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStaleRepairSpiImpl.java @@ -0,0 +1,47 @@ +package com.nanri.aiimage.modules.brand.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper; +import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity; +import com.nanri.aiimage.modules.task.spi.BrandTaskStaleRepairSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.util.List; + +/** + * brand_crawl_tasks 的陈旧修复实现（2026-09 全维度审查 G5：逻辑从 task 模块收回本模块）。 + */ +@Service +@RequiredArgsConstructor +public class BrandTaskStaleRepairSpiImpl implements BrandTaskStaleRepairSpi { + + private static final String STATUS_PENDING = "pending"; + private static final String STATUS_RUNNING = "running"; + private static final String STATUS_FAILED = "failed"; + + private final BrandCrawlTaskMapper brandCrawlTaskMapper; + + @Override + public List failStaleBrandTasks(LocalDateTime cutoff, int limit) { + List stale = brandCrawlTaskMapper.selectList( + new LambdaQueryWrapper() + .select(BrandCrawlTaskEntity::getId) + .in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING) + .lt(BrandCrawlTaskEntity::getUpdatedAt, cutoff) + .last("limit " + Math.max(1, limit))); + if (stale.isEmpty()) { + return List.of(); + } + List ids = stale.stream().map(BrandCrawlTaskEntity::getId).toList(); + brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() + .in(BrandCrawlTaskEntity::getId, ids) + .in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING) + .lt(BrandCrawlTaskEntity::getUpdatedAt, cutoff) + .set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED) + .set(BrandCrawlTaskEntity::getErrorMessage, "任务长期无心跳，已自动失败")); + return ids; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataItemCleanupSpiImpl.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataItemCleanupSpiImpl.java new file mode 100644 index 00000000..63225139 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataItemCleanupSpiImpl.java @@ -0,0 +1,29 @@ +package com.nanri.aiimage.modules.collectdata.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper; +import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity; +import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * 采集明细行的历史清理实现（2026-09 全维度审查 G5：逻辑从 task 模块收回本模块）。 + */ +@Service +@RequiredArgsConstructor +public class CollectDataItemCleanupSpiImpl implements CollectDataItemCleanupSpi { + + private final CollectDataItemMapper collectDataItemMapper; + + @Override + public int deleteItemsByTaskIds(List taskIds) { + if (taskIds == null || taskIds.isEmpty()) { + return 0; + } + return collectDataItemMapper.delete(new LambdaQueryWrapper() + .in(CollectDataItemEntity::getTaskId, taskIds)); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataTaskHeartbeatSpi.java new file mode 100644 index 00000000..03021d0d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataTaskHeartbeatSpi.java @@ -0,0 +1,28 @@ +package com.nanri.aiimage.modules.collectdata.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +/** + * 采集数据的任务心跳实现（2026-09 全维度审查 G5）。 + * + *

进度由采集模块自己的导入进度表维护，心跳直接把客户端上报的进度写进去。 + */ +@Service +@RequiredArgsConstructor +public class CollectDataTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { + + private final CollectDataService collectDataService; + + @Override + public String moduleType() { + return "COLLECT_DATA"; + } + + @Override + public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { + collectDataService.updateProgress(taskId, request); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java index 5edda316..27cddfed 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java @@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.dedupe.service; import cn.hutool.core.util.IdUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.util.ExcelStreamReader; import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper; import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest; import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest; @@ -122,7 +123,10 @@ public class DedupeTotalDataService { String safeCountry = country == null ? "" : country.trim(); AccessScope scope = resolveAccessScope(operatorId); LambdaQueryWrapper query = new LambdaQueryWrapper() - .like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword) + // 2026-09 全维度审查 C6：关键字由"包含"改为"前缀"匹配——后台该输入框标注的是 + // 「输入 ASIN」（相邻的用户名框才是"模糊搜索"），前缀匹配才能命中 data_value 索引； + // 此前的 %kw% 前置通配使 idx 永不生效，93 万行表每翻页两次全表扫 + .likeRight(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword) .like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername) .apply(!safeCountry.isEmpty(), "FIND_IN_SET({0}, IFNULL(country, '')) > 0", @@ -667,101 +671,162 @@ public class DedupeTotalDataService { throw new BusinessException("仅支持 .xlsx 或 .xls 文件"); } - DataFormatter formatter = new DataFormatter(); - try (Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) { - Sheet sheet = workbook.getSheetAt(0); - Row headerRow = sheet.getRow(0); - if (headerRow == null) { - throw new BusinessException("Excel 表头为空"); - } + // 2026-09 全维度审查 C5：改逐行流式解析（ExcelStreamReader → EasyExcel SAX）。 + // 此前 WorkbookFactory.create 把整表读成 DOM（50 万行约 1~2GB 堆），且行数上限在 + // 加载完之后才校验（等于没有防线）。现在内存只保留单批待插入值与文件内去重集合， + // 行数上限在迭代过程中即时生效。 + ImportSession session = new ImportSession(progress, groupId, uploaderUserId, uploaderUsername); + try { + ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() { + private Integer countryColumnIndex; + private boolean headerReady; - Map headerMap = new HashMap<>(); - for (int i = 0; i < headerRow.getLastCellNum(); i++) { - Cell cell = headerRow.getCell(i); - String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell)); - if (!value.isBlank() && !headerMap.containsKey(value)) { - headerMap.put(value, i); + @Override + public void onHeader(String sheetName, Integer sheetNo, Map headerMap) { + session.headerIndex = buildHeaderIndex(headerMap); + if (session.headerIndex.get("ASIN") == null) { + throw new BusinessException("缺少 ASIN 列"); + } + countryColumnIndex = session.headerIndex.get("国家"); + headerReady = true; } - } - Integer asinIndex = headerMap.get("ASIN"); - if (asinIndex == null) { - throw new BusinessException("缺少 ASIN 列"); - } - Integer countryIndex = headerMap.get("国家"); + @Override + public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) { + if (progress != null && approximateTotalRows != null && approximateTotalRows > 0) { + progress.setTotalRows(approximateTotalRows); + } + } - Set seenInFile = new HashSet<>(); - List pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE); - Map pendingCountries = new HashMap<>(); - int totalRows = Math.max(sheet.getLastRowNum(), 0); - if (maxImportRows > 0 && totalRows > maxImportRows) { + @Override + public void onRow(String sheetName, Integer sheetNo, int rowIndex, + Map headerMap, Map rowMap) { + if (!headerReady) { + // 无表头（空表）：与旧 DOM 实现一致，不处理任何数据行 + return; + } + session.acceptRow(rowMap, countryColumnIndex); + } + }); + } catch (BusinessException e) { + throw e; + } catch (Exception e) { + log.error("[dedupe-total-data] 导入 Excel 失败 filename={} processedRows={}", + filename, session.processedRows(), e); + throw new BusinessException("导入 Excel 失败", e); + } + return session.finish(); + } + + /** 表头名 → 列索引（首次出现优先、忽略空表头），与旧 DOM 实现口径一致。 */ + private Map buildHeaderIndex(Map headerMap) { + Map index = new HashMap<>(); + if (headerMap == null) { + return index; + } + for (Map.Entry entry : headerMap.entrySet()) { + if (entry.getKey() == null) { + continue; + } + String value = normalizeExcelText(entry.getValue()); + if (!value.isBlank() && !index.containsKey(value)) { + index.put(value, entry.getKey()); + } + } + return index; + } + + /** 流式解析下按列索引取行内文本。 */ + private String cellText(Map rowMap, Integer columnIndex) { + if (rowMap == null || columnIndex == null) { + return null; + } + return rowMap.get(columnIndex); + } + + /** 流式导入的累加器：单批待插入值 + 计数与进度（内存只随批大小增长，不随文件行数增长）。 */ + private final class ImportSession { + + private final DedupeTotalDataImportProgressVo progress; + private final Long groupId; + private final Long uploaderUserId; + private final String uploaderUsername; + private final Set seenInFile = new HashSet<>(); + private final List pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE); + private final Map pendingCountries = new HashMap<>(); + private Map headerIndex = Map.of(); + private int processedRows; + private int asinCount; + private int insertedCount; + private int skippedCount; + + private ImportSession(DedupeTotalDataImportProgressVo progress, Long groupId, + Long uploaderUserId, String uploaderUsername) { + this.progress = progress; + this.groupId = groupId; + this.uploaderUserId = uploaderUserId; + this.uploaderUsername = uploaderUsername; + } + + private void acceptRow(Map rowMap, Integer countryColumnIndex) { + processedRows++; + if (maxImportRows > 0 && processedRows > maxImportRows) { + // 流式下无法前置知道总行数：超限即中断，已插入批次保留（与旧实现的"截断"语义不同， + // 但旧实现在加载后才校验，实际是 OOM 之后才失败） throw new BusinessException("导入行数超过上限: " + maxImportRows); } - int asinCount = 0; - int insertedCount = 0; - int skippedCount = 0; - if (progress != null) { - progress.setTotalRows(totalRows); - progress.setProcessedRows(0); - progress.setAsinCount(0); - progress.setInsertedCount(0); - progress.setSkippedCount(0); - progress.setErrorMessage(null); + Integer asinIndex = headerIndex.get("ASIN"); + String asin = asinIndex == null ? "" : normalizeComparableValueOrBlank(cellText(rowMap, asinIndex)); + if (asin.isBlank()) { + skippedCount++; + updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount); + return; } + asinCount++; + if (!seenInFile.add(asin)) { + skippedCount++; + updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount); + return; + } + pendingValues.add(asin); + if (countryColumnIndex != null) { + pendingCountries.put(asin, parseCountries(cellText(rowMap, countryColumnIndex))); + } + if (pendingValues.size() >= IMPORT_BATCH_SIZE) { + flushBatch(); + } + updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount); + } - for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { - Row row = sheet.getRow(rowNum); - if (row == null) { - skippedCount++; - updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount); - continue; - } - String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex))); - if (asin.isBlank()) { - skippedCount++; - updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount); - continue; - } - asinCount++; - if (!seenInFile.add(asin)) { - skippedCount++; - updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount); - continue; - } - pendingValues.add(asin); - if (countryIndex != null) { - pendingCountries.put(asin, parseCountries(formatter.formatCellValue(row.getCell(countryIndex)))); - } - if (pendingValues.size() >= IMPORT_BATCH_SIZE) { - ImportBatchResult batch = insertImportBatch( - pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername); - insertedCount += batch.insertedCount(); - skippedCount += batch.skippedCount(); - pendingValues.clear(); - pendingCountries.clear(); - } - updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount); + private void flushBatch() { + if (pendingValues.isEmpty()) { + return; } - if (!pendingValues.isEmpty()) { - ImportBatchResult batch = insertImportBatch( - pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername); - insertedCount += batch.insertedCount(); - skippedCount += batch.skippedCount(); - pendingValues.clear(); - pendingCountries.clear(); - } - updateImportProgress(progress, totalRows, asinCount, insertedCount, skippedCount); + ImportBatchResult batch = insertImportBatch( + pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername); + insertedCount += batch.insertedCount(); + skippedCount += batch.skippedCount(); + pendingValues.clear(); + pendingCountries.clear(); + } + private int processedRows() { + return processedRows; + } + + private DedupeTotalDataImportVo finish() { + flushBatch(); + updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount); + if (progress != null && processedRows > progress.getTotalRows()) { + // 近似总行数缺失（或与实际物理行数不一致）时用实际处理行数兜底，避免展示成 x/0 + progress.setTotalRows(processedRows); + } DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo(); - vo.setTotalRows(totalRows); + vo.setTotalRows(processedRows); vo.setAsinCount(asinCount); vo.setInsertedCount(insertedCount); vo.setSkippedCount(skippedCount); return vo; - } catch (BusinessException e) { - throw e; - } catch (Exception e) { - throw new BusinessException("导入 Excel 失败"); } } @@ -850,112 +915,120 @@ public class DedupeTotalDataService { throw new BusinessException("仅支持 .xlsx 或 .xls 文件"); } - DataFormatter formatter = new DataFormatter(); - try (Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) { - Sheet sheet = workbook.getSheetAt(0); - Row headerRow = sheet.getRow(0); - if (headerRow == null) { - throw new BusinessException("Excel 表头为空"); - } + // 2026-09 全维度审查 C5：与导入同样改流式解析，避免 50 万行时 DOM 占 1~2GB 堆 + DeleteSession session = new DeleteSession(progress, scope, groupId); + try { + ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() { + private boolean headerReady; - Map headerMap = new HashMap<>(); - for (int i = 0; i < headerRow.getLastCellNum(); i++) { - Cell cell = headerRow.getCell(i); - String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell)); - if (!value.isBlank() && !headerMap.containsKey(value)) { - headerMap.put(value, i); - } - } - - Integer asinIndex = headerMap.get("ASIN"); - if (asinIndex == null) { - throw new BusinessException("缺少 ASIN 列"); - } - - Set seenInFile = new HashSet<>(); - List pendingDeletes = new ArrayList<>(); - int totalRows = Math.max(sheet.getLastRowNum(), 0); - if (maxImportRows > 0 && totalRows > maxImportRows) { - throw new BusinessException("导入行数超过上限: " + maxImportRows); - } - int asinCount = 0; - int deletedCount = 0; - int skippedCount = 0; - if (progress != null) { - progress.setTotalRows(totalRows); - progress.setProcessedRows(0); - progress.setAsinCount(0); - progress.setInsertedCount(0); - progress.setSkippedCount(0); - progress.setErrorMessage(null); - } - - for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { - Row row = sheet.getRow(rowNum); - if (row == null) { - skippedCount++; - if (progress != null) { - progress.setProcessedRows(rowNum); - progress.setAsinCount(asinCount); - progress.setInsertedCount(deletedCount); - progress.setSkippedCount(skippedCount); + @Override + public void onHeader(String sheetName, Integer sheetNo, Map headerMap) { + session.headerIndex = buildHeaderIndex(headerMap); + if (session.headerIndex.get("ASIN") == null) { + throw new BusinessException("缺少 ASIN 列"); } - continue; - } - String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex))); - if (asin.isBlank()) { - skippedCount++; - if (progress != null) { - progress.setProcessedRows(rowNum); - progress.setAsinCount(asinCount); - progress.setInsertedCount(deletedCount); - progress.setSkippedCount(skippedCount); - } - continue; - } - asinCount++; - String dataValue = asin; - if (!seenInFile.add(dataValue)) { - skippedCount++; - if (progress != null) { - progress.setProcessedRows(rowNum); - progress.setAsinCount(asinCount); - progress.setInsertedCount(deletedCount); - progress.setSkippedCount(skippedCount); - } - continue; + headerReady = true; } - // 批量删除：攒批 + IN 一次删，避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1 - pendingDeletes.add(dataValue); - if (pendingDeletes.size() >= DELETE_BATCH_SIZE) { - deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId); - pendingDeletes = new ArrayList<>(); + @Override + public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) { + if (progress != null && approximateTotalRows != null && approximateTotalRows > 0) { + progress.setTotalRows(approximateTotalRows); + } } - if (progress != null) { - progress.setProcessedRows(rowNum); - progress.setAsinCount(asinCount); - progress.setInsertedCount(deletedCount); - progress.setSkippedCount(skippedCount); - } - } - if (!pendingDeletes.isEmpty()) { - deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId); - } - DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo(); - vo.setTotalRows(totalRows); - vo.setAsinCount(asinCount); - vo.setInsertedCount(deletedCount); - vo.setSkippedCount(skippedCount); - return vo; + @Override + public void onRow(String sheetName, Integer sheetNo, int rowIndex, + Map headerMap, Map rowMap) { + if (!headerReady) { + return; + } + session.acceptRow(rowMap); + } + }); } catch (BusinessException e) { throw e; } catch (Exception e) { // 此前丢弃原始异常：排查时只能看到一句「删除 Excel 匹配数据失败」 - log.error("[dedupe-total-data] 删除 Excel 匹配数据失败", e); + log.error("[dedupe-total-data] 删除 Excel 匹配数据失败 filename={} processedRows={}", + filename, session.processedRows(), e); throw new BusinessException("删除 Excel 匹配数据失败", e); } + return session.finish(); + } + + /** 流式删除匹配的累加器：单批待删值 + 计数与进度。 */ + private final class DeleteSession { + + private final DedupeTotalDataImportProgressVo progress; + private final AccessScope scope; + private final Long groupId; + private final Set seenInFile = new HashSet<>(); + private final List pendingDeletes = new ArrayList<>(); + private Map headerIndex = Map.of(); + private int processedRows; + private int asinCount; + private int deletedCount; + private int skippedCount; + + private DeleteSession(DedupeTotalDataImportProgressVo progress, AccessScope scope, Long groupId) { + this.progress = progress; + this.scope = scope; + this.groupId = groupId; + } + + private void acceptRow(Map rowMap) { + processedRows++; + if (maxImportRows > 0 && processedRows > maxImportRows) { + throw new BusinessException("导入行数超过上限: " + maxImportRows); + } + Integer asinIndex = headerIndex.get("ASIN"); + String asin = asinIndex == null ? "" : normalizeComparableValueOrBlank(cellText(rowMap, asinIndex)); + if (asin.isBlank() || !seenInFile.add(asin)) { + skippedCount++; + updateProgress(); + return; + } + asinCount++; + // 批量删除：攒批 + IN 一次删，避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1 + pendingDeletes.add(asin); + if (pendingDeletes.size() >= DELETE_BATCH_SIZE) { + deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId); + pendingDeletes.clear(); + } + updateProgress(); + } + + private void updateProgress() { + if (progress == null) { + return; + } + progress.setProcessedRows(processedRows); + progress.setAsinCount(asinCount); + progress.setInsertedCount(deletedCount); + progress.setSkippedCount(skippedCount); + } + + private int processedRows() { + return processedRows; + } + + private DedupeTotalDataImportVo finish() { + if (!pendingDeletes.isEmpty()) { + deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId); + pendingDeletes.clear(); + } + updateProgress(); + if (progress != null && processedRows > progress.getTotalRows()) { + progress.setTotalRows(processedRows); + } + DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo(); + vo.setTotalRows(processedRows); + vo.setAsinCount(asinCount); + vo.setInsertedCount(deletedCount); + vo.setSkippedCount(skippedCount); + return vo; + } } @Transactional diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java index 9ed70ad2..d0a7fa81 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java @@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.util.ExcelStreamReader; import com.nanri.aiimage.config.DeleteBrandProgressProperties; import com.nanri.aiimage.config.TaskPressureProperties; import com.nanri.aiimage.modules.deletebrand.model.cache.DeleteBrandParsedFileCacheDto; @@ -44,10 +45,8 @@ import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.DataFormatter; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.stereotype.Service; import org.springframework.transaction.PlatformTransactionManager; @@ -59,6 +58,7 @@ import org.springframework.transaction.support.TransactionTemplate; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.IOException; import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; @@ -515,62 +515,130 @@ public class DeleteBrandRunService { } private ParsedDeleteBrandFile parseDeleteBrandFile(File inputFile) { - DataFormatter formatter = new DataFormatter(); - try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) { - Sheet sheet = workbook.getSheetAt(0); - Row titleRow = sheet.getRow(0); - Row headerRow = sheet.getRow(1); - if (titleRow == null || headerRow == null) { - throw new BusinessException("Excel 表头不完整"); + // 2026-09 全维度审查 C5：改逐行流式解析（ExcelStreamReader → EasyExcel SAX）。 + // 此前 WorkbookFactory.create 把整表读成 DOM，删除品牌源文件（几十万行）会整表驻留堆内存。 + // 解析结果（国家分组/预览行/行号）与错误文案不变，行号沿用 sheet 绝对 0 基行号 +1。 + if (!hasExcelMagic(inputFile)) { + log.warn("[delete-brand] 源文件不是 Excel（疑似 CSV/文本），拒绝解析 file={}", inputFile.getName()); + throw new BusinessException("读取删除品牌 Excel 失败"); + } + DeleteBrandSheetContext context = new DeleteBrandSheetContext(); + try { + ExcelStreamReader.readFirstSheet(inputFile, context); + } catch (BusinessException ex) { + throw ex; + } catch (Exception ex) { + // 此前该 catch 既不记日志也不带 cause，线上只有一句「读取删除品牌 Excel 失败」 + log.warn("[delete-brand] 读取删除品牌 Excel 失败 file={} err={}", inputFile.getName(), ex.getMessage(), ex); + throw new BusinessException("读取删除品牌 Excel 失败"); + } + return context.finish(); + } + + /** + * 文件魔数校验：xlsx=PK(zip)、xls=OLE2(CFD)。 + * EasyExcel 会把非 zip 文本当 CSV 静默解析成功，而旧 POI WorkbookFactory 只认 xlsx/xls（垃圾文件直接失败）， + * 这里保持「非 Excel 源文件 → 读取失败」的语义（与 SimilarAsinExcelParser 同口径）。 + */ + private boolean hasExcelMagic(File file) { + byte[] head = new byte[8]; + try (FileInputStream inputStream = new FileInputStream(file)) { + int n = inputStream.readNBytes(head, 0, head.length); + boolean isZip = n >= 4 && head[0] == 'P' && head[1] == 'K' && (head[2] == 3 || head[2] == 5 || head[2] == 7); + boolean isOle2 = n >= 8 + && (head[0] & 0xFF) == 0xD0 && (head[1] & 0xFF) == 0xCF + && (head[2] & 0xFF) == 0x11 && (head[3] & 0xFF) == 0xE0; + return isZip || isOle2; + } catch (IOException ex) { + return false; + } + } + + /** + * 删除品牌源文件流式解析上下文：两行表头（国家标题行 + 字段行）解析列组 + 逐行按国家收集 ASIN。 + * dataStartRowIndex 语义与旧实现一致（删除ASIN 表头记为 2，其余"状态"表头记为 1）。 + */ + private final class DeleteBrandSheetContext implements ExcelStreamReader.SheetRowHandler { + + private final Map> grouped = new LinkedHashMap<>(); + private final Map displayCountryNames = new LinkedHashMap<>(); + private Map titleRowMap; + private boolean titleRowSeen; + private boolean headerRowSeen; + private List pairs; + private int firstDataRow = 2; + + @Override + public void onHeader(String sheetName, Integer sheetNo, Map headerMap) { + // 第 0 行 = 国家标题行 + titleRowSeen = true; + titleRowMap = headerMap == null ? Map.of() : headerMap; + } + + @Override + public void onRow(String sheetName, Integer sheetNo, int rowIndex, + Map headerMap, Map rowMap) { + if (!headerRowSeen) { + if (rowIndex != 1) { + // 第 1 行整行为空（EasyExcel 不上报）：与旧实现 headerRow == null 一致 + throw new BusinessException("Excel 表头不完整"); + } + if (!titleRowSeen) { + // 第 0 行整行为空（EasyExcel 不上报表头回调）：与旧实现 titleRow == null 一致 + throw new BusinessException("Excel 表头不完整"); + } + headerRowSeen = true; + pairs = resolveCountryPairs(titleRowMap, rowMap); + if (pairs.isEmpty()) { + throw new BusinessException("未识别到删除品牌表头"); + } + firstDataRow = pairs.stream() + .mapToInt(CountryColumnPair::dataStartRowIndex) + .min() + .orElse(2); } - - List pairs = resolveCountryPairs(titleRow, headerRow, formatter); - if (pairs.isEmpty()) { - throw new BusinessException("未识别到删除品牌表头"); + if (rowIndex < firstDataRow) { + return; } + consumeDataRow(rowIndex, rowMap); + } - Map> grouped = new LinkedHashMap<>(); - Map displayCountryNames = new LinkedHashMap<>(); - - int firstDataRow = pairs.stream() - .mapToInt(CountryColumnPair::dataStartRowIndex) - .min() - .orElse(2); - for (int rowNum = firstDataRow; rowNum <= sheet.getLastRowNum(); rowNum++) { - Row row = sheet.getRow(rowNum); - if (row == null) { + private void consumeDataRow(int rowNum, Map rowMap) { + for (CountryColumnPair pair : pairs) { + if (rowNum < pair.dataStartRowIndex()) { continue; } - for (CountryColumnPair pair : pairs) { - if (rowNum < pair.dataStartRowIndex()) { - continue; - } - String country = pair.country(); - String asin = normalizeCellText(formatter.formatCellValue(row.getCell(pair.asinColumnIndex()))); - String status = normalizeCellText(formatter.formatCellValue(row.getCell(pair.statusColumnIndex()))); - if (asin.isBlank() && status.isBlank()) { - continue; - } - - String asinKey = normalizeAsinKey(asin); - if (asinKey.isBlank()) { - continue; - } - - displayCountryNames.putIfAbsent(country, country); - Map countryMap = grouped.computeIfAbsent(country, ignored -> new LinkedHashMap<>()); - if (countryMap.containsKey(asinKey)) { - continue; - } - - DeleteBrandCountryAsinVo item = new DeleteBrandCountryAsinVo(); - item.setRowIndex(rowNum + 1); - item.setAsin(asin); - item.setStatus(status); - countryMap.put(asinKey, item); + String country = pair.country(); + String asin = normalizeCellText(cellText(rowMap, pair.asinColumnIndex())); + String status = normalizeCellText(cellText(rowMap, pair.statusColumnIndex())); + if (asin.isBlank() && status.isBlank()) { + continue; } - } + String asinKey = normalizeAsinKey(asin); + if (asinKey.isBlank()) { + continue; + } + + displayCountryNames.putIfAbsent(country, country); + Map countryMap = grouped.computeIfAbsent(country, ignored -> new LinkedHashMap<>()); + if (countryMap.containsKey(asinKey)) { + continue; + } + + DeleteBrandCountryAsinVo item = new DeleteBrandCountryAsinVo(); + item.setRowIndex(rowNum + 1); + item.setAsin(asin); + item.setStatus(status); + countryMap.put(asinKey, item); + } + } + + private ParsedDeleteBrandFile finish() { + if (!headerRowSeen) { + // 空表 / 表头行缺失：与旧实现 titleRow == null || headerRow == null 一致 + throw new BusinessException("Excel 表头不完整"); + } List countries = new ArrayList<>(); List previewRows = new ArrayList<>(); int totalRows = 0; @@ -598,10 +666,6 @@ public class DeleteBrandRunService { } return new ParsedDeleteBrandFile(totalRows, countries, previewRows); - } catch (BusinessException ex) { - throw ex; - } catch (Exception ex) { - throw new BusinessException("读取删除品牌 Excel 失败"); } } @@ -612,13 +676,13 @@ public class DeleteBrandRunService { return value.replace(" ", "").trim().toUpperCase(Locale.ROOT); } - private List resolveCountryPairs(Row titleRow, Row headerRow, DataFormatter formatter) { + private List resolveCountryPairs(Map titleRowMap, Map headerRowMap) { List pairs = new ArrayList<>(); - int lastCellNum = Math.max(titleRow.getLastCellNum(), headerRow.getLastCellNum()); + int lastCellNum = Math.max(lastColumnCount(titleRowMap), lastColumnCount(headerRowMap)); for (int i = 0; i < lastCellNum; i++) { - String title = normalizeCellText(formatter.formatCellValue(titleRow.getCell(i))); - String firstHeader = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i))); - String secondHeader = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i + 1))); + String title = normalizeCellText(cellText(titleRowMap, i)); + String firstHeader = normalizeCellText(cellText(headerRowMap, i)); + String secondHeader = normalizeCellText(cellText(headerRowMap, i + 1)); if (title.isBlank()) { continue; } @@ -633,6 +697,27 @@ public class DeleteBrandRunService { return pairs; } + /** 流式解析下按列索引取行内文本（与旧 formatter.formatCellValue(row.getCell(index)) 口径一致）。 */ + private String cellText(Map rowMap, Integer columnIndex) { + if (rowMap == null || columnIndex == null || columnIndex < 0) { + return ""; + } + return rowMap.get(columnIndex); + } + + private int lastColumnCount(Map rowMap) { + if (rowMap == null || rowMap.isEmpty()) { + return 0; + } + int lastColumnCount = 0; + for (Integer columnIndex : rowMap.keySet()) { + if (columnIndex != null && columnIndex >= lastColumnCount) { + lastColumnCount = columnIndex + 1; + } + } + return lastColumnCount; + } + private String resolveShopName(DeleteBrandSourceFileDto sourceFile) { String name = sourceFile.getOriginalFilename(); if (name == null || name.isBlank()) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java index 97320bc5..d38eca1a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java @@ -38,6 +38,7 @@ import java.time.Duration; import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -103,13 +104,9 @@ public class DeleteBrandStaleTaskService { ShopMatchStaleCheckStats patrolDeleteStats = failStalePatrolDeleteTasks(); ShopMatchStaleCheckStats queryAsinStats = failStaleQueryAsinTasks(); ShopMatchStaleCheckStats withdrawStats = failStaleWithdrawTasks(); - runModuleStaleCheck("brand", brandTaskService::failStaleRunningTasks); - runModuleStaleCheck("appearance-patent", appearancePatentTaskService::finalizeStaleTasks); - runModuleStaleCheck("similar-asin", similarAsinTaskService::finalizeStaleTasks); - // 商品管理采集并入本巡检线（2026-09）：此前它自带 @Scheduled 且按 owner 过滤， - // 双实例下 owner 宕机即无人判死（P1-8）；并入后由本方法的 job 锁保证单实例扫描， - // 它自己改为全局判死 + 任务锁 + status CAS - runModuleStaleCheck("shop-data-crawl", shopDataCrawlTaskService::finalizeStaleTasks); + for (Map.Entry delegated : delegatedStaleChecks().entrySet()) { + runModuleStaleCheck(delegated.getKey(), delegated.getValue()); + } // 周期每 2 分钟一轮，各模块 summary 合并为单行，避免定期刷屏 // 注意：每段占位符数量必须与实参一致——此前每段 5 个占位符只传 4 个参数， // 导致 withdraw 之后的取值整体错位、末尾 elapsedMs/thread 打成字面量 @@ -125,6 +122,23 @@ public class DeleteBrandStaleTaskService { } } + /** + * 委派式陈旧判死：moduleType → 处理动作。 + * {@code TaskModuleRegistry} 中 delegatedStaleCheck=true 的模块都必须在这里登记， + * 由 TaskModuleCoverageTest 守住一致性（新增模块漏登记会红测试）。 + */ + Map delegatedStaleChecks() { + Map handlers = new LinkedHashMap<>(); + handlers.put("BRAND", brandTaskService::failStaleRunningTasks); + handlers.put("APPEARANCE_PATENT", appearancePatentTaskService::finalizeStaleTasks); + handlers.put("SIMILAR_ASIN", similarAsinTaskService::finalizeStaleTasks); + // 商品管理采集并入本巡检线（2026-09）：此前它自带 @Scheduled 且按 owner 过滤， + // 双实例下 owner 宕机即无人判死（P1-8）；并入后由本方法的 job 锁保证单实例扫描， + // 它自己改为全局判死 + 任务锁 + status CAS + handlers.put("SHOP_DATA_CRAWL", shopDataCrawlTaskService::finalizeStaleTasks); + return handlers; + } + private void runModuleStaleCheck(String moduleName, Runnable action) { long startedAt = System.currentTimeMillis(); try { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskHeartbeatSpi.java new file mode 100644 index 00000000..8c99ba69 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskHeartbeatSpi.java @@ -0,0 +1,57 @@ +package com.nanri.aiimage.modules.deletebrand.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * 删除 ASIN 的任务心跳实现（2026-09 全维度审查 G5）。 + * + *

本模块的心跳不写独立的"心跳键"，而是整段进度写进任务缓存（saveProgress，forceWrite=true）， + * 并携带客户端上报的 phase/current/total。 + */ +@Service +@RequiredArgsConstructor +public class DeleteBrandTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { + + private final DeleteBrandTaskCacheService cacheService; + + @Override + public String moduleType() { + return "DELETE_BRAND"; + } + + @Override + public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { + cacheService.saveProgress(taskId, buildHeartbeatProgress(request), true); + } + + @Override + public void saveTaskCache(FileTaskEntity task) { + cacheService.saveTaskCache(task); + } + + private Map buildHeartbeatProgress(TaskHeartbeatRequest request) { + String now = String.valueOf(System.currentTimeMillis()); + Map values = new LinkedHashMap<>(); + values.put("last_heartbeat_at", now); + values.put("updated_at", now); + if (request != null) { + putIfPresent(values, "phase", request.getPhase()); + putIfPresent(values, "current", request.getCurrent()); + putIfPresent(values, "total", request.getTotal()); + } + return values; + } + + private void putIfPresent(Map values, String key, Object value) { + if (value != null) { + values.put(key, String.valueOf(value)); + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/digitalhuman/controller/DigitalHumanVersionController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/digitalhuman/controller/DigitalHumanVersionController.java index 98030f0b..e60bc09a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/digitalhuman/controller/DigitalHumanVersionController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/digitalhuman/controller/DigitalHumanVersionController.java @@ -1,12 +1,14 @@ package com.nanri.aiimage.modules.digitalhuman.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.modules.admin.support.AdminAuthSupport; import com.nanri.aiimage.modules.digitalhuman.model.dto.UploadVersionRequest; import com.nanri.aiimage.modules.digitalhuman.model.vo.DigitalHumanVersionVo; import com.nanri.aiimage.modules.digitalhuman.service.DigitalHumanVersionService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import org.springframework.http.MediaType; @@ -27,10 +29,22 @@ import java.util.List; public class DigitalHumanVersionController { private final DigitalHumanVersionService versionService; + private final AdminAuthSupport adminAuthSupport; + + /** + * 写操作（上传/发布/设为最新/删除）必须管理员身份：2026-09 全维度审查 A4—— + * 此前整套接口匿名可达，任何人可发布/删除客户端版本。 + * 读接口（列表/latest/详情/download-url）保持匿名：桌面客户端自身要查版本与下载地址。 + */ + private void requireAdmin(HttpServletRequest request) { + adminAuthSupport.requireAdminOrInternal(request); + } @PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) @Operation(summary = "上传新版本", description = "上传数字人程序新版本到 OSS，并记录版本信息") - public ApiResponse uploadVersion(@Valid @ModelAttribute UploadVersionRequest request) { + public ApiResponse uploadVersion(@Valid @ModelAttribute UploadVersionRequest request, + HttpServletRequest httpRequest) { + requireAdmin(httpRequest); DigitalHumanVersionVo vo = versionService.uploadVersion( request.getVersion(), request.getFile(), @@ -63,21 +77,27 @@ public class DigitalHumanVersionController { @PostMapping("/{version}/release") @Operation(summary = "发布版本", description = "将草稿状态的版本发布，状态变更为 RELEASED") public ApiResponse releaseVersion( - @Parameter(description = "版本号", example = "1.0.0") @PathVariable String version) { + @Parameter(description = "版本号", example = "1.0.0") @PathVariable String version, + HttpServletRequest request) { + requireAdmin(request); return ApiResponse.success(versionService.releaseVersion(version)); } @PostMapping("/{version}/set-latest") @Operation(summary = "设为最新版本", description = "将指定已发布版本设置为最新版本，清除其他版本的最新标记") public ApiResponse setLatest( - @Parameter(description = "版本号", example = "1.0.0") @PathVariable String version) { + @Parameter(description = "版本号", example = "1.0.0") @PathVariable String version, + HttpServletRequest request) { + requireAdmin(request); return ApiResponse.success(versionService.setLatest(version)); } @DeleteMapping("/{version}") @Operation(summary = "删除版本", description = "删除指定版本，同时删除 OSS 上的文件。最新版本不允许删除") public ApiResponse deleteVersion( - @Parameter(description = "版本号", example = "1.0.0") @PathVariable String version) { + @Parameter(description = "版本号", example = "1.0.0") @PathVariable String version, + HttpServletRequest request) { + requireAdmin(request); versionService.deleteVersion(version); return ApiResponse.success(null); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java index 83f87a82..3e0a3748 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java @@ -3,34 +3,46 @@ package com.nanri.aiimage.modules.file.service; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.IdUtil; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.util.ExcelStreamReader; import com.nanri.aiimage.config.StorageProperties; import com.nanri.aiimage.modules.file.model.vo.ExcelInfoVo; import com.nanri.aiimage.modules.file.model.vo.UploadFileVo; import lombok.RequiredArgsConstructor; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.DataFormatter; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; @Service @RequiredArgsConstructor +@Slf4j public class LocalFileStorageService { /** fileKey → 文件名 索引容量：超限淘汰最旧条目，避免索引自身无界增长。 */ static final int SOURCE_FILE_INDEX_CAPACITY = 1024; + /** + * 禁止上传的可执行/脚本类扩展名（2026-09 全维度审查 A3）。 + * 上传接口是桌面端文件中转口（临时目录落盘，2GB/次），此前对类型无任何限制； + * 这里挡掉"落盘后一旦被其它组件误执行/误引用即可造成危害"的扩展名。 + * 业务文件（xlsx/xls/csv/txt/json/zip/图片/视频）不受影响； + * 如需调整由运维在 application.yml 覆盖 aiimage.files.upload-blocked-extensions。 + */ + private static final String DEFAULT_BLOCKED_EXTENSIONS = + "exe,bat,cmd,com,scr,msi,dll,sh,ps1,vbs,vbe,wsf,hta,jsp,jspx,php,jar,so,dylib"; + + @Value("${aiimage.files.upload-blocked-extensions:" + DEFAULT_BLOCKED_EXTENSIONS + "}") + private String blockedExtensions = DEFAULT_BLOCKED_EXTENSIONS; + private final StorageProperties storageProperties; /** 临时目录标准路径：供调用方做"文件必须落在上传临时目录内"的穿越校验。 */ @@ -51,6 +63,7 @@ public class LocalFileStorageService { FileUtil.mkdir(tempDir); String fileKey = IdUtil.fastSimpleUUID(); String extName = FileUtil.extName(file.getOriginalFilename()); + rejectBlockedExtension(file.getOriginalFilename(), extName); String filename = fileKey + (extName.isEmpty() ? "" : "." + extName); File target = FileUtil.file(tempDir, filename); file.transferTo(target); @@ -65,49 +78,125 @@ public class LocalFileStorageService { return vo; } + /** 扩展名黑名单校验：命中即拒绝落盘（大小写不敏感）。 */ + private void rejectBlockedExtension(String originalFilename, String extName) { + if (extName == null || extName.isBlank()) { + return; + } + String normalized = extName.trim().toLowerCase(java.util.Locale.ROOT); + for (String blocked : blockedExtensions.split(",")) { + if (!blocked.isBlank() && blocked.trim().toLowerCase(java.util.Locale.ROOT).equals(normalized)) { + log.warn("[file-upload] 拒绝上传被禁扩展名 originalFilename={} ext={}", originalFilename, normalized); + throw new BusinessException("不支持上传该类型文件：." + normalized); + } + } + } + public ExcelInfoVo getExcelInfo(String fileKey) throws IOException { File inputFile = findLocalSourceFile(fileKey); if (inputFile == null || !inputFile.exists()) { throw new BusinessException("源文件不存在"); } - DataFormatter formatter = new DataFormatter(); - try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) { - Sheet sheet = workbook.getSheetAt(0); - Row headerRow = sheet.getRow(0); - if (headerRow == null) { - throw new BusinessException("Excel 表头为空"); - } + // 2026-09 全维度审查 C5：改流式解析（ExcelStreamReader → EasyExcel SAX）。 + // 此前 WorkbookFactory.create 把整表读成 DOM，只为取表头与行数却让整表驻留堆内存。 + // 返回值结构与字段含义不变：headers 同旧口径（去重、空表头跳过、命中「缩略图地址8」截断）， + // totalRows 仍是"数据行数"（旧实现取 lastRowNum，等于物理末行行号-1）。 + try { + ExcelInfoStreamingReader reader = new ExcelInfoStreamingReader(); + ExcelStreamReader.readFirstSheet(inputFile, reader); + reader.assertHeaderSeen(); + ExcelInfoVo vo = new ExcelInfoVo(); + vo.setHeaders(reader.headers()); + vo.setTotalRows(reader.totalRows()); + return vo; + } catch (BusinessException e) { + throw e; + } catch (Exception e) { + log.warn("[file-excel-info] 读取 Excel 信息失败 fileKey={} err={}", fileKey, e.getMessage(), e); + throw new BusinessException("读取 Excel 信息失败"); + } + } - Map headerMap = new HashMap<>(); - List headers = new ArrayList<>(); - for (int i = 0; i < headerRow.getLastCellNum(); i++) { - Cell cell = headerRow.getCell(i); - String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell)); + /** + * 表头与行数的流式读取：内存不随文件行数增长。 + * 表头行整行为空（EasyExcel 不上报表头回调）时视为「Excel 表头为空」——旧实现仅在 getRow(0)==null + * 时报同一错误，此处口径略严（不存在的表头与空表头同样处理）。 + */ + private final class ExcelInfoStreamingReader implements ExcelStreamReader.SheetRowHandler { + + private final List headers = new ArrayList<>(); + private final Set seenHeaders = new LinkedHashSet<>(); + private Integer approximateTotalRows; + private boolean headerSeen; + private boolean headerTruncated; + private int countedRows; + + @Override + public void onHeader(String sheetName, Integer sheetNo, Map headerMap) { + headerSeen = true; + int lastColumnCount = lastHeaderColumnCount(headerMap); + for (int i = 0; i < lastColumnCount && !headerTruncated; i++) { + String value = normalizeCellText(headerMap.get(i)); if (value.isBlank()) { continue; } value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty() ? value : value.split("idASIN国家状态价格变体数量", 2)[0].trim(); - if (headerMap.containsKey(value)) { + if (!seenHeaders.add(value)) { continue; } - headerMap.put(value, i); headers.add(value); if ("缩略图地址8".equals(value)) { - break; + headerTruncated = true; } } + } - ExcelInfoVo vo = new ExcelInfoVo(); - vo.setHeaders(headers); - vo.setTotalRows(Math.max(sheet.getLastRowNum(), 0)); - return vo; - } catch (BusinessException e) { - throw e; - } catch (Exception e) { - throw new BusinessException("读取 Excel 信息失败"); + @Override + public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) { + this.approximateTotalRows = approximateTotalRows; + } + + @Override + public void onRow(String sheetName, Integer sheetNo, int rowIndex, + Map headerMap, Map rowMap) { + countedRows++; + } + + private void assertHeaderSeen() { + if (!headerSeen) { + throw new BusinessException("Excel 表头为空"); + } + } + + private List headers() { + return headers; + } + + /** + * onSheetTotal 的近似值 = 含表头的物理末行行号（POI lastRowNum+1 口径），换算回旧的 + * "数据行数"口径；近似值缺失（dimension 标签缺失）或偏小时用遍历计数兜底。 + */ + private int totalRows() { + int approximateDataRows = approximateTotalRows == null || approximateTotalRows <= 0 + ? 0 + : approximateTotalRows - 1; + return Math.max(approximateDataRows, countedRows); + } + + private int lastHeaderColumnCount(Map headerMap) { + if (headerMap == null || headerMap.isEmpty()) { + return 0; + } + int lastColumnCount = 0; + for (Integer columnIndex : headerMap.keySet()) { + if (columnIndex != null && columnIndex >= lastColumnCount) { + lastColumnCount = columnIndex + 1; + } + } + return lastColumnCount; } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsDeleteRetryService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsDeleteRetryService.java index 92539b7f..22cca3f1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsDeleteRetryService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsDeleteRetryService.java @@ -1,11 +1,15 @@ package com.nanri.aiimage.modules.file.service.object; +import com.nanri.aiimage.common.util.PendingDeleteJournal; import com.nanri.aiimage.config.TransientStorageProperties; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import java.nio.file.Path; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashSet; @@ -26,6 +30,48 @@ public class RustfsDeleteRetryService { private final ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); private final ConcurrentHashMap pending = new ConcurrentHashMap<>(); private final Object admissionLock = new Object(); + /** + * 待删对象的本地落盘日志：内存队列重启即丢，对应对象会一直残留在桶里直到生命周期过期。 + * 入队时追加、每轮重试后按 pending 收敛重写、启动时回放。 + */ + private volatile PendingDeleteJournal journal; + + /** + * 启动回放：把上次进程残留的待删对象重新入队（容量准入、去重语义与在线入队一致）， + * 随后由 {@link #retryPendingDeletes()} 的周期任务继续重试。 + */ + @EventListener(ApplicationReadyEvent.class) + public void replayJournalOnStartup() { + if (!properties.isDeleteRetryEnabled()) { + return; + } + List objectKeys = journal().readAll(); + if (objectKeys.isEmpty()) { + return; + } + int replayed = 0; + for (String objectKey : objectKeys) { + enqueue(objectKey, null); + replayed++; + } + log.info("[rustfs] 启动回放待删对象 journalSize={} replayed={} pending={}", + objectKeys.size(), replayed, pending.size()); + } + + private PendingDeleteJournal journal() { + PendingDeleteJournal local = journal; + if (local != null) { + return local; + } + synchronized (admissionLock) { + if (journal == null) { + Path path = Path.of(System.getProperty("java.io.tmpdir"), + "aiimage-delete-journal", "rustfs-delete-retry.log"); + journal = new PendingDeleteJournal(path, Math.max(1, properties.getDeleteRetryQueueCapacity())); + } + return journal; + } + } public void enqueue(String objectKey, Throwable cause) { if (!properties.isDeleteRetryEnabled() || objectKey == null || objectKey.isBlank()) { @@ -97,6 +143,8 @@ public class RustfsDeleteRetryService { } retryNextInvocation.forEach(this::requeue); if (processed > 0) { + // 收敛落盘：成功删除的对象从日志移除，仍失败/待重试的保留 + journal().rewrite(pending.keySet()); log.info("[rustfs] delete retry batch completed processed={} success={} failed={} pending={}", processed, success, failed, pending.size()); } @@ -124,6 +172,8 @@ public class RustfsDeleteRetryService { } item = new RetryItem(objectKey); pending.put(objectKey, item); + // 只在"新准入"时记账：重写以 pending 为准，重复失败无需重复追加 + journal().record(objectKey); return item; } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/controller/ImageVideoController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/controller/ImageVideoController.java index 26b3bf36..f43d5050 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/controller/ImageVideoController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/imagevideo/controller/ImageVideoController.java @@ -1,6 +1,9 @@ package com.nanri.aiimage.modules.imagevideo.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity; +import com.nanri.aiimage.modules.admin.support.AdminAuthSupport; import com.nanri.aiimage.modules.imagevideo.model.dto.DouyinCopyRequest; import com.nanri.aiimage.modules.imagevideo.model.dto.ImageVideoSecretSaveRequest; import com.nanri.aiimage.modules.imagevideo.model.dto.ImageVideoVoiceCloneRequest; @@ -17,6 +20,7 @@ import com.nanri.aiimage.modules.imagevideo.service.ImageVideoAsyncTaskService; import com.nanri.aiimage.modules.imagevideo.service.ImageVideoSecretService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.servlet.http.HttpServletRequest; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import org.springframework.web.bind.annotation.GetMapping; @@ -36,21 +40,42 @@ import org.springframework.web.multipart.MultipartFile; public class ImageVideoController { private final ImageVideoCozeService imageVideoCozeService; + private final AdminAuthSupport adminAuthSupport; private final ImageVideoSecretService imageVideoSecretService; private final ImageVideoAsyncTaskService imageVideoAsyncTaskService; @GetMapping("/secrets") @Operation(summary = "查询视频密钥配置状态") - public ApiResponse secretStatus(@RequestParam("user_id") Long userId) { - return ApiResponse.success(imageVideoSecretService.status(userId)); + public ApiResponse secretStatus(@RequestParam("user_id") Long userId, + HttpServletRequest request) { + return ApiResponse.success(imageVideoSecretService.status(resolveSelfUserId(request, userId))); } @PutMapping("/secrets") @Operation(summary = "保存视频密钥配置") - public ApiResponse saveSecrets(@Valid @RequestBody ImageVideoSecretSaveRequest request) { + public ApiResponse saveSecrets(@Valid @RequestBody ImageVideoSecretSaveRequest request, + HttpServletRequest httpRequest) { + resolveSelfUserId(httpRequest, request.getUserId()); return ApiResponse.success(imageVideoSecretService.save(request)); } + /** + * 2026-09 全维度审查 A7：视频密钥接口此前匿名可达，且 userId 取自请求体/查询串 + * ——任何人可读写他人密钥配置。现要求登录身份（或可信内部代理）， + * 且只允许操作自己（超管例外，保留后台代查能力）。 + */ + private Long resolveSelfUserId(HttpServletRequest request, Long requestedUserId) { + AdminUserEntity me = adminAuthSupport.requireUserOrInternal(request); + if (requestedUserId == null || requestedUserId <= 0) { + throw new BusinessException("用户 id 不合法"); + } + boolean superAdmin = "super_admin".equals(adminAuthSupport.currentRole(me)); + if (!superAdmin && !requestedUserId.equals(me.getId())) { + throw new BusinessException(403, "只能操作自己的视频密钥配置"); + } + return requestedUserId; + } + @PostMapping("/douyin-copy") @Operation(summary = "抖音文案识别和仿写") public ApiResponse douyinCopy(@Valid @RequestBody DouyinCopyRequest request) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/notification/service/NotificationScanScheduler.java b/backend-java/src/main/java/com/nanri/aiimage/modules/notification/service/NotificationScanScheduler.java index 5babe247..bcaa40ed 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/notification/service/NotificationScanScheduler.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/notification/service/NotificationScanScheduler.java @@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.notification.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.nanri.aiimage.config.HttpClientPool; import com.nanri.aiimage.config.NotificationProperties; +import com.nanri.aiimage.common.module.TaskModuleRegistry; import com.nanri.aiimage.common.service.DistributedJobLockService; import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper; import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity; @@ -54,24 +55,11 @@ public class NotificationScanScheduler { private static final DateTimeFormatter HOUR_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHH"); - /** 任务模块类型 → 中文名（与 TaskHeartbeatService 的模块常量对齐）。 */ - private static final Map MODULE_LABELS = Map.ofEntries( - Map.entry("APPEARANCE_PATENT", "外观专利检测"), - Map.entry("SIMILAR_ASIN", "货源查询"), - Map.entry("PATROL_DELETE", "巡店删除"), - Map.entry("PRICE_TRACK", "跟价"), - Map.entry("PRODUCT_RISK_RESOLVE", "商品风险解决"), - Map.entry("QUERY_ASIN", "查询ASIN"), - Map.entry("WITHDRAW", "取款"), - Map.entry("SHOP_DATA_CRAWL", "店铺数据抓取"), - Map.entry("SHOP_MATCH", "定时匹配"), - Map.entry("BRAND", "品牌检测"), - Map.entry("COLLECT_DATA", "采集数据"), - Map.entry("DELETE_BRAND", "删除ASIN"), - Map.entry("SPLIT", "数据拆分"), - Map.entry("CONVERT", "格式转换"), - Map.entry("PUBLISH", "上架"), - Map.entry("DEDUPE", "数据去重")); + /** + * 任务模块类型 → 中文名：直接取模块注册表（G6），新增模块只需在 + * {@code TaskModuleRegistry} 追加一行，不再需要同步这份表。 + */ + private static final Map MODULE_LABELS = TaskModuleRegistry.labels(); private final FileTaskMapper fileTaskMapper; private final BrandCrawlTaskMapper brandCrawlTaskMapper; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/controller/PatrolDeleteController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/controller/PatrolDeleteController.java index 5f778b7b..5a70605e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/controller/PatrolDeleteController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/controller/PatrolDeleteController.java @@ -13,9 +13,9 @@ import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteResolveService import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteResolveService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteResolveService.java index 1a6eb107..8ed184fe 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteResolveService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteResolveService.java @@ -10,9 +10,9 @@ import com.nanri.aiimage.modules.patroldelete.model.entity.PatrolDeleteShopCandi import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteConditionVo; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo; import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskHeartbeatSpi.java new file mode 100644 index 00000000..c9c7e432 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskHeartbeatSpi.java @@ -0,0 +1,35 @@ +package com.nanri.aiimage.modules.patroldelete.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +/** + * 巡店删除 的任务心跳实现（2026-09 全维度审查 G5）。 + * + *

心跳逻辑（缓存刷新 / 任务缓存回写）从 task 模块收回本模块，task 侧只依赖 SPI 接口， + * 消除 task → 业务模块的编译期依赖。 + */ +@Service +@RequiredArgsConstructor +public class PatrolDeleteTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { + + private final PatrolDeleteTaskCacheService cacheService; + + @Override + public String moduleType() { + return "PATROL_DELETE"; + } + + @Override + public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { + cacheService.touchTaskHeartbeat(taskId); + } + + @Override + public void saveTaskCache(FileTaskEntity task) { + cacheService.saveTaskCache(task); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java index 9ac68a6b..b26dd782 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java @@ -21,7 +21,7 @@ import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteTaskBatchVo; import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteCreateTaskVo; import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteHistoryVo; import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteResultItemVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskHeartbeatSpi.java new file mode 100644 index 00000000..f7e78e15 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskHeartbeatSpi.java @@ -0,0 +1,35 @@ +package com.nanri.aiimage.modules.pricetrack.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +/** + * 跟价 的任务心跳实现（2026-09 全维度审查 G5）。 + * + *

心跳逻辑（缓存刷新 / 任务缓存回写）从 task 模块收回本模块，task 侧只依赖 SPI 接口， + * 消除 task → 业务模块的编译期依赖。 + */ +@Service +@RequiredArgsConstructor +public class PriceTrackTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { + + private final PriceTrackTaskCacheService cacheService; + + @Override + public String moduleType() { + return "PRICE_TRACK"; + } + + @Override + public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { + cacheService.touchTaskHeartbeat(taskId); + } + + @Override + public void saveTaskCache(FileTaskEntity task) { + cacheService.saveTaskCache(task); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/controller/ProductRiskResolveController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/controller/ProductRiskResolveController.java index 8022f48b..21cfe9f1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/controller/ProductRiskResolveController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/controller/ProductRiskResolveController.java @@ -8,14 +8,14 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCreateTaskRequ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskSubmitResultRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskTaskBatchRequest; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCountryPreferenceVo; import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCreateTaskVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskHistoryVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.common.model.vo.ProductRiskHistoryVo; +import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo; import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskPendingDeleteVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskBatchVo; +import com.nanri.aiimage.common.model.vo.ProductRiskTaskBatchVo; import com.nanri.aiimage.modules.productrisk.service.ProductRiskResolveService; import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService; import io.swagger.v3.oas.annotations.Operation; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/dto/ProductRiskCreateTaskRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/dto/ProductRiskCreateTaskRequest.java index 587ebe31..7149c8fc 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/dto/ProductRiskCreateTaskRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/dto/ProductRiskCreateTaskRequest.java @@ -1,7 +1,7 @@ package com.nanri.aiimage.modules.productrisk.model.dto; import com.fasterxml.jackson.annotation.JsonProperty; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCreateTaskVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCreateTaskVo.java index 8b442f92..9b9cb704 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCreateTaskVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskCreateTaskVo.java @@ -5,6 +5,7 @@ import lombok.Data; import java.util.ArrayList; import java.util.List; +import com.nanri.aiimage.common.model.vo.ProductRiskResultItemVo; @Data @Schema(description = "创建任务响应：含任务 id 与各店初始快照（含 resultId），用于推队列与前端展示") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskResolveService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskResolveService.java index 76d5492c..c0ca2ad8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskResolveService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskResolveService.java @@ -12,10 +12,10 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequ import com.nanri.aiimage.modules.productrisk.model.entity.ProductRiskCountryPrefEntity; import com.nanri.aiimage.modules.productrisk.model.entity.ProductRiskShopCandidateEntity; import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCountryPreferenceVo; +import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo; import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskHeartbeatSpi.java new file mode 100644 index 00000000..454c30a6 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskHeartbeatSpi.java @@ -0,0 +1,35 @@ +package com.nanri.aiimage.modules.productrisk.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +/** + * 商品风险解决 的任务心跳实现（2026-09 全维度审查 G5）。 + * + *

心跳逻辑（缓存刷新 / 任务缓存回写）从 task 模块收回本模块，task 侧只依赖 SPI 接口， + * 消除 task → 业务模块的编译期依赖。 + */ +@Service +@RequiredArgsConstructor +public class ProductRiskTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { + + private final ProductRiskTaskCacheService cacheService; + + @Override + public String moduleType() { + return "PRODUCT_RISK_RESOLVE"; + } + + @Override + public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { + cacheService.touchTaskHeartbeat(taskId); + } + + @Override + public void saveTaskCache(FileTaskEntity task) { + cacheService.saveTaskCache(task); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java index b4456065..456af95c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java @@ -13,14 +13,14 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskShopPayloadDto import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskSubmitResultRequest; import com.nanri.aiimage.modules.productrisk.model.entity.ProductRiskShopCandidateEntity; import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCreateTaskVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskHistoryVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskResultItemVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskBatchVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskDetailVo; +import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.common.model.vo.ProductRiskHistoryVo; +import com.nanri.aiimage.common.model.vo.ProductRiskResultItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskTaskBatchVo; +import com.nanri.aiimage.common.model.vo.ProductRiskTaskDetailVo; import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskPendingDeleteVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskItemVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskTaskItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskHeartbeatSpi.java new file mode 100644 index 00000000..8a7550fe --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskHeartbeatSpi.java @@ -0,0 +1,35 @@ +package com.nanri.aiimage.modules.publish.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +/** + * 上架模块的任务心跳实现（2026-09 全维度审查 G5）。 + * + *

心跳前做任务归属校验（非本实例拥有的任务拒绝续命），心跳刷新走模块自己的进度服务； + * 本模块没有独立的任务缓存回写（历史行为保持一致）。 + */ +@Service +@RequiredArgsConstructor +public class PublishTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { + + private final PublishTaskService publishTaskService; + + @Override + public String moduleType() { + return PublishTaskService.MODULE_TYPE; + } + + @Override + public void ensureOwnership(FileTaskEntity task, String operation) { + publishTaskService.ensureTaskOwnedByCurrentInstance(task, operation); + } + + @Override + public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { + publishTaskService.touchHeartbeat(taskId, request); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java index f11ccc8c..674096d6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java @@ -3,9 +3,9 @@ package com.nanri.aiimage.modules.queryasin.controller; import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCreateTaskRequest; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinSubmitResultRequest; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinTaskBatchRequest; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinResolveService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinResolveService.java index c1b4e181..3e0f4a92 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinResolveService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinResolveService.java @@ -4,9 +4,9 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo; import com.nanri.aiimage.modules.queryasin.mapper.QueryAsinShopCandidateMapper; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto; import com.nanri.aiimage.modules.queryasin.model.entity.QueryAsinShopCandidateEntity; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskHeartbeatSpi.java new file mode 100644 index 00000000..9bebd858 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskHeartbeatSpi.java @@ -0,0 +1,35 @@ +package com.nanri.aiimage.modules.queryasin.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +/** + * 查询ASIN 的任务心跳实现（2026-09 全维度审查 G5）。 + * + *

心跳逻辑（缓存刷新 / 任务缓存回写）从 task 模块收回本模块，task 侧只依赖 SPI 接口， + * 消除 task → 业务模块的编译期依赖。 + */ +@Service +@RequiredArgsConstructor +public class QueryAsinTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { + + private final QueryAsinTaskCacheService cacheService; + + @Override + public String moduleType() { + return "QUERY_ASIN"; + } + + @Override + public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { + cacheService.touchTaskHeartbeat(taskId); + } + + @Override + public void saveTaskCache(FileTaskEntity task) { + cacheService.saveTaskCache(task); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java index f74392ec..0beb8357 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java @@ -21,7 +21,7 @@ import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinTaskBatchVo; import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinCreateTaskVo; import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinHistoryVo; import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinResultItemVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java index 48e0a4bb..fa677d49 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java @@ -4,10 +4,10 @@ import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCountryPreferenceVo; +import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskBatchRequest; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlItemMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlItemMapper.java index c97209c1..486812f8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlItemMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlItemMapper.java @@ -19,6 +19,20 @@ public interface ShopDataCrawlItemMapper extends BaseMapper该表是增长最快的表（店 × 日 × 国 × SKU）且此前**没有任何清理策略**（2026-09 全维度审查）。 + * 撞款扫描只读每店最新批次，历史批次仅用于追溯，过期即可删；分批删避免长事务与大范围行锁， + * 走 idx_shop_data_crawl_item_biz_date 范围扫描。 + */ + @Delete(""" + DELETE FROM biz_shop_data_crawl_item + WHERE business_date < #{cutoff} + LIMIT #{batchSize} + """) + int deleteOlderThanBatch(@Param("cutoff") java.time.LocalDate cutoff, @Param("batchSize") int batchSize); + /** 删除归属某累计档的全部明细行（管理端真删店铺数据记录时清理，避免留下悬空引用）。 */ @Delete(""" DELETE FROM biz_shop_data_crawl_item diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlItemRetentionService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlItemRetentionService.java new file mode 100644 index 00000000..9f9bbbc9 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlItemRetentionService.java @@ -0,0 +1,76 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlItemMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.time.LocalDate; + +/** + * 采集明细表（biz_shop_data_crawl_item）的保留期清理（2026-09 全维度审查）。 + * + *

该表按「店 × 日 × 国 × SKU」增长，是全库增长最快的表，此前只有"按店按日替换当天批次" + * 与"删累计档时连带删除"两条删除路径，历史批次永不清理。撞款扫描只读每店**最新**批次， + * 因此保留期内（默认 30 天）之外的批次可安全删除。 + * + *

双节点用 job 锁保证单实例执行；分批删除并限制单轮批次数，避免一次跑太久占住锁。 + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class ShopDataCrawlItemRetentionService { + + /** 单轮最多删除的批次数（每批 batchSize 行），剩余留给下一轮。 */ + static final int MAX_BATCHES_PER_RUN = 20; + + private final ShopDataCrawlItemMapper itemMapper; + private final DistributedJobLockService distributedJobLockService; + + @Value("${aiimage.shop-data-crawl.item-retention-days:30}") + private int retentionDays = 30; + + @Value("${aiimage.shop-data-crawl.item-retention-batch-size:5000}") + private int retentionBatchSize = 5000; + + @Scheduled(cron = "${aiimage.shop-data-crawl.item-retention-cron:0 30 3 * * *}") + public void purgeExpiredItems() { + int days = Math.max(1, retentionDays); + int batchSize = Math.max(100, retentionBatchSize); + LocalDate cutoff = LocalDate.now().minusDays(days); + + DistributedJobLockService.LockHandle lockHandle = + distributedJobLockService.tryLock("shop-data-crawl:item-retention", Duration.ofMinutes(15)); + if (lockHandle == null) { + log.info("[shop-data-crawl] item retention skipped because another instance holds the lock"); + return; + } + try (lockHandle) { + int totalDeleted = 0; + int batches = 0; + while (batches < MAX_BATCHES_PER_RUN) { + int deleted = itemMapper.deleteOlderThanBatch(cutoff, batchSize); + batches++; + totalDeleted += deleted; + if (deleted < batchSize) { + break; + } + } + if (totalDeleted > 0) { + log.info("[shop-data-crawl] item retention deleted={} cutoff={} retentionDays={} batches={}", + totalDeleted, cutoff, days, batches); + } + } catch (Exception ex) { + log.warn("[shop-data-crawl] item retention failed cutoff={} msg={}", cutoff, ex.getMessage(), ex); + } + } + + /** 供测试读取当前生效的保留天数。 */ + int effectiveRetentionDays() { + return Math.max(1, retentionDays); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlResolveService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlResolveService.java index b1b97cab..e46b9f10 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlResolveService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlResolveService.java @@ -12,10 +12,10 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequ import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlCountryPrefEntity; import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlShopCandidateEntity; import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCountryPreferenceVo; +import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo; import com.nanri.aiimage.modules.shopkey.service.ShopManageService; import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskHeartbeatSpi.java new file mode 100644 index 00000000..6bf687d5 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskHeartbeatSpi.java @@ -0,0 +1,41 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +/** + * 店铺数据采集的任务心跳实现（2026-09 全维度审查 G5）。 + * + *

除常规缓存刷新外，本模块在心跳前要做**任务归属校验**：非本实例拥有的任务拒绝心跳， + * 避免另一实例误把在途任务的 updated_at 续命。 + */ +@Service +@RequiredArgsConstructor +public class ShopDataCrawlTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { + + private final ShopDataCrawlTaskCacheService cacheService; + private final ShopDataCrawlTaskService shopDataCrawlTaskService; + + @Override + public String moduleType() { + return "SHOP_DATA_CRAWL"; + } + + @Override + public void ensureOwnership(FileTaskEntity task, String operation) { + shopDataCrawlTaskService.ensureTaskOwnedByCurrentInstance(task, operation); + } + + @Override + public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { + cacheService.touchTaskHeartbeat(taskId); + } + + @Override + public void saveTaskCache(FileTaskEntity task) { + cacheService.saveTaskCache(task); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java index e96272f6..8a17756d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java @@ -27,7 +27,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo; import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo; import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/mapper/ShopDuplicateCheckItemMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/mapper/ShopDuplicateCheckItemMapper.java index 26e70966..e26a6303 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/mapper/ShopDuplicateCheckItemMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/mapper/ShopDuplicateCheckItemMapper.java @@ -16,7 +16,18 @@ import java.util.List; @Mapper public interface ShopDuplicateCheckItemMapper { - /** 每店最新 business_date 批次的全部明细行（撞款统计直接数据源）。 */ + /** + * 全量店铺名（去重，走 idx_shop_data_crawl_item_shop_date 的索引扫描，不读行数据）。 + * 撞款扫描按店逐批取数用，避免一次性把全部明细拉进堆。 + */ + @Select("SELECT DISTINCT shop_name FROM biz_shop_data_crawl_item") + List selectDistinctShopNames(); + + /** + * 单店最新 business_date 批次的明细行。 + * 内层 MAX(business_date) 走 (shop_name, business_date) 索引（松散索引扫描）， + * 外层按索引前缀取该批次，只读目标店的行，不再全表 GROUP BY + JOIN。 + */ @Select(""" SELECT i.shop_name AS shopName, i.country AS country, @@ -25,14 +36,12 @@ public interface ShopDuplicateCheckItemMapper { i.price AS price, i.brand AS brand FROM biz_shop_data_crawl_item i - JOIN ( - SELECT shop_name, MAX(business_date) AS max_date - FROM biz_shop_data_crawl_item - GROUP BY shop_name - ) latest - ON latest.shop_name = i.shop_name AND latest.max_date = i.business_date + WHERE i.shop_name = #{shopName} + AND i.business_date = ( + SELECT MAX(business_date) FROM biz_shop_data_crawl_item WHERE shop_name = #{shopName} + ) """) - List selectLatestShopItems(); + List selectLatestItemsByShop(@Param("shopName") String shopName); /** 幂等判定：该店该日是否已有明细批次。 */ @Select(""" diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckScanService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckScanService.java index 221382e6..22bc5955 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckScanService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckScanService.java @@ -232,23 +232,39 @@ public class ShopDataDuplicateCheckScanService { * 重复检查基数 = 已采集落库的店（超管应见 8 家）。 */ private List collectShopParsed(DistributedJobLockService.LockHandle lock) { - List rows = itemMapper.selectLatestShopItems(); - if (rows.isEmpty()) { + // 按店逐批取数（2026-09 全维度审查 C7）：此前一条 full-table GROUP BY + JOIN 把全部明细 + // 一次性拉进堆，而明细表随采集天数线性增长（每日每店一批）。改为先走索引扫描拿店名， + // 再逐店按 (shop_name, business_date) 索引取「该店最新批次」， + // 堆内同时只驻留单店行 + 已构建结构。 + List rawShopNames = itemMapper.selectDistinctShopNames(); + if (rawShopNames.isEmpty()) { log.info("[shop-duplicate-check] 明细表暂无采集数据，本次扫描不出库"); return List.of(); } // 店铺名去重（分组标签来自店铺管理 biz_shop_manage，与页面口径一致）。 Set shopNames = new LinkedHashSet<>(); - for (ShopLatestItemRowDto row : rows) { - if (row != null) { - shopNames.add(normalizeShopName(row.shopName())); + for (String rawShopName : rawShopNames) { + if (rawShopName != null && !rawShopName.isBlank()) { + shopNames.add(normalizeShopName(rawShopName)); } } Map groupLabels = new HashMap<>(); for (ShopGroupLabelDto dto : sourceMapper.selectGroupLabels(new ArrayList<>(shopNames))) { groupLabels.put(shopKey(dto.getShopName()), dto.getGroupName() == null ? "" : dto.getGroupName()); } - List parsedShops = buildParsedShops(rows, groupLabels); + List parsedShops = new ArrayList<>(); + for (String rawShopName : rawShopNames) { + if (rawShopName == null || rawShopName.isBlank()) { + continue; + } + List rows = itemMapper.selectLatestItemsByShop(rawShopName); + if (rows.isEmpty()) { + // 该店历史批次存在但最新批次无行：跳过（不影响其它店） + log.info("[shop-duplicate-check] 店铺最新批次无明细行，跳过 shopName={}", rawShopName); + continue; + } + parsedShops.addAll(buildParsedShops(rows, groupLabels)); + } log.info("[shop-duplicate-check] 明细数据源扫描：{} 家店铺，{} 行明细", parsedShops.size(), rawRowsCount(parsedShops)); return parsedShops; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java index 66a40ec4..bfea9b7e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java @@ -5,6 +5,7 @@ import com.baomidou.mybatisplus.core.toolkit.support.SFunction; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import cn.hutool.core.util.IdUtil; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.util.ExcelStreamReader; import com.nanri.aiimage.modules.shopkey.mapper.SkipPriceAsinMapper; import com.nanri.aiimage.modules.shopkey.mapper.ShopManageMapper; import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinCreateRequest; @@ -19,12 +20,8 @@ import com.nanri.aiimage.modules.shopkey.model.vo.SkipPriceAsinPageVo; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.DataFormatter; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; -import org.apache.poi.ss.usermodel.WorkbookFactory; import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @@ -33,7 +30,6 @@ import org.springframework.web.multipart.MultipartFile; import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileInputStream; import java.nio.file.Files; import java.math.BigDecimal; import java.math.RoundingMode; @@ -485,45 +481,136 @@ public class SkipPriceAsinService { private void processImportFile(File tempFile, String filename, Long groupId, String shopName, boolean deleteMode, QueryAsinImportProgressVo progress) throws Exception { validateExcelFilename(filename); - try (FileInputStream inputStream = new FileInputStream(tempFile); - Workbook workbook = WorkbookFactory.create(inputStream)) { - Sheet sheet = workbook.getNumberOfSheets() == 0 ? null : workbook.getSheetAt(0); - if (sheet == null) { - throw new BusinessException("Excel 为空"); - } - HeaderMapping mapping = resolveHeaderMapping(sheet); - if (mapping.asinColumns().isEmpty()) { - throw new BusinessException("Excel 至少需要包含一个国家的 ASIN 列"); + // 2026-09 全维度审查 C5：改逐行流式解析（ExcelStreamReader → EasyExcel SAX）。 + // 此前 WorkbookFactory.create 把整表读成 DOM（50 万行约 1~2GB 堆），且行数上限在整表加载 + // 完之后才校验（等于没有防线）。现在内存只保留表头映射与计数（不随文件行数增长）， + // 行数上限在迭代过程中即时生效，进度总行数用 onSheetTotal 的近似值。 + ImportSession session = new ImportSession(progress, groupId, shopName, deleteMode); + ExcelStreamReader.readFirstSheet(tempFile, new ExcelStreamReader.SheetRowHandler() { + @Override + public void onHeader(String sheetName, Integer sheetNo, Map headerMap) { + session.acceptHeader(headerMap); } - int firstDataRow = mapping.firstDataRowIndex(); - int lastRow = Math.max(sheet.getLastRowNum(), firstDataRow - 1); - int totalRows = Math.max(0, lastRow - firstDataRow + 1); - if (maxImportRows > 0 && totalRows > maxImportRows) { + @Override + public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) { + session.acceptApproximateTotalRows(approximateTotalRows); + } + + @Override + public void onRow(String sheetName, Integer sheetNo, int rowIndex, + Map headerMap, Map rowMap) { + session.acceptRow(rowIndex, rowMap); + } + }); + session.finish(); + } + + /** + * 流式导入的累加器：表头映射 + 计数与进度，内存只与单行/单批写入相关，不随文件行数增长。 + * 行号沿用 EasyExcel 的 sheet 绝对 0 基行号（与旧 POI Row.getRowNum() 口径一致）。 + */ + private final class ImportSession { + + private final QueryAsinImportProgressVo progress; + private final Long groupId; + private final String shopName; + private final boolean deleteMode; + private final ImportCounters counters = new ImportCounters(); + private Map firstHeaderMap = Map.of(); + private HeaderMapping mapping; + private Integer approximateTotalRows; + private boolean headerSeen; + private int processedRows; + + private ImportSession(QueryAsinImportProgressVo progress, Long groupId, String shopName, boolean deleteMode) { + this.progress = progress; + this.groupId = groupId; + this.shopName = shopName; + this.deleteMode = deleteMode; + } + + private void acceptHeader(Map headerMap) { + this.firstHeaderMap = headerMap == null ? Map.of() : headerMap; + this.headerSeen = true; + } + + private void acceptApproximateTotalRows(Integer approximateTotalRows) { + this.approximateTotalRows = approximateTotalRows; + } + + private void acceptRow(int rowIndex, Map rowMap) { + if (!headerSeen) { + // 表头行整行为空时 EasyExcel 不上报表头回调：与旧实现 sheet.getRow(0) == null 一致 + throw new BusinessException("Excel 表头为空"); + } + if (mapping == null) { + // 两行表头格式下，第 1 行（0 基）就是第二行表头，由首个 onRow 回调提供； + // 第 1 行为空行时 EasyExcel 不上报，等价于旧实现 secondHeader 为空行。 + Map secondHeaderMap = rowIndex == 1 ? rowMap : null; + long resolveStartedAt = System.nanoTime(); + mapping = resolveHeaderMapping(firstHeaderMap, secondHeaderMap); + if (mapping.asinColumns().isEmpty()) { + throw new BusinessException("Excel 至少需要包含一个国家的 ASIN 列"); + } + updateTotalRows(); + log.info("[skip-price-asin-import] 流式解析表头 firstDataRowIndex={} asinColumns={} priceColumns={} " + + "approxTotalRows={} totalRows={} resolveMs={} deleteMode={}", + mapping.firstDataRowIndex(), mapping.asinColumns(), mapping.priceColumns(), + approximateTotalRows, progress.getTotalRows(), elapsedMs(resolveStartedAt), deleteMode); + if (mapping.firstDataRowIndex() > rowIndex) { + // 该行是第二行表头（未进入数据区）：不作为数据行处理 + return; + } + } + processedRows++; + if (maxImportRows > 0 && processedRows > maxImportRows) { + // 流式下无法前置知道总行数：超限即中断（旧实现在整表加载后才校验，实际是 OOM 之后才失败） throw new BusinessException("导入行数超过上限: " + maxImportRows); } - progress.setTotalRows(totalRows); - - DataFormatter formatter = new DataFormatter(); - ImportCounters counters = new ImportCounters(); - for (int rowIndex = firstDataRow; rowIndex <= lastRow; rowIndex++) { - Row row = sheet.getRow(rowIndex); - if (row == null || isBlankRow(row, formatter)) { - counters.skippedCount++; - updateImportProgress(progress, counters, rowIndex - firstDataRow + 1); - continue; - } - if (deleteMode) { - consumeDeleteImportRow(row, formatter, mapping, groupId, shopName, counters); - } else { - consumeAddImportRow(row, formatter, mapping, groupId, shopName, counters); - } - updateImportProgress(progress, counters, rowIndex - firstDataRow + 1); + if (isBlankRow(rowMap)) { + counters.skippedCount++; + updateImportProgress(progress, counters, processedRows); + return; } + if (deleteMode) { + consumeDeleteImportRow(rowMap, mapping, groupId, shopName, counters); + } else { + consumeAddImportRow(rowMap, mapping, groupId, shopName, counters); + } + updateImportProgress(progress, counters, processedRows); + } + + /** 近似总行数换算为数据行数（近似值 = 含表头的物理末行行号，减去数据起始行号）。 */ + private void updateTotalRows() { + if (progress == null || approximateTotalRows == null || approximateTotalRows <= 0) { + return; + } + progress.setTotalRows(Math.max(0, approximateTotalRows - mapping.firstDataRowIndex())); + } + + private void finish() { + if (!headerSeen) { + // 空表（无任何行）或表头行缺失：与旧实现 sheet.getRow(0) == null 一致 + throw new BusinessException("Excel 表头为空"); + } + Integer totalRows = progress == null ? null : progress.getTotalRows(); + if (progress != null && (totalRows == null || processedRows > totalRows)) { + // 近似总行数缺失（或与实际物理行数不一致）时用实际处理行数兜底，避免展示成 x/0 + progress.setTotalRows(processedRows); + } + log.info("[skip-price-asin-import] 流式导入完成 processedRows={} totalRows={} asinCount={} " + + "insertedCount={} deletedCount={} skippedCount={} deleteMode={}", + processedRows, progress == null ? null : progress.getTotalRows(), counters.asinCount, + counters.insertedCount, counters.deletedCount, counters.skippedCount, deleteMode); } } - private void consumeAddImportRow(Row row, DataFormatter formatter, HeaderMapping mapping, + private long elapsedMs(long startedAtNanos) { + return (System.nanoTime() - startedAtNanos) / 1_000_000; + } + + private void consumeAddImportRow(Map rowMap, HeaderMapping mapping, Long groupId, String shopName, ImportCounters counters) { boolean acceptedAny = false; boolean sawAnyAsin = false; @@ -532,7 +619,7 @@ public class SkipPriceAsinService { entity.setShopName(shopName); for (Map.Entry entry : mapping.asinColumns().entrySet()) { String country = entry.getKey(); - String asin = normalizeBlank(cellText(row, entry.getValue(), formatter)).toUpperCase(Locale.ROOT); + String asin = normalizeBlank(cellText(rowMap, entry.getValue())).toUpperCase(Locale.ROOT); if (asin.isEmpty()) { continue; } @@ -543,7 +630,7 @@ public class SkipPriceAsinService { continue; } BigDecimal minimumPrice = parseOptionalMinimumPrice( - cellText(row, mapping.priceColumns().get(country), formatter)); + cellText(rowMap, mapping.priceColumns().get(country))); if (existsCountryAsin(groupId, shopName, country, asin)) { counters.skippedCount++; continue; @@ -561,13 +648,13 @@ public class SkipPriceAsinService { skipPriceAsinMapper.insert(entity); } - private void consumeDeleteImportRow(Row row, DataFormatter formatter, HeaderMapping mapping, + private void consumeDeleteImportRow(Map rowMap, HeaderMapping mapping, Long groupId, String shopName, ImportCounters counters) { boolean acceptedAny = false; boolean sawAnyAsin = false; for (Map.Entry entry : mapping.asinColumns().entrySet()) { String country = entry.getKey(); - String asin = normalizeBlank(cellText(row, entry.getValue(), formatter)).toUpperCase(Locale.ROOT); + String asin = normalizeBlank(cellText(rowMap, entry.getValue())).toUpperCase(Locale.ROOT); if (asin.isEmpty()) { continue; } @@ -646,27 +733,43 @@ public class SkipPriceAsinService { return skipPriceAsinMapper.selectOne(query.last("LIMIT 1")); } - private HeaderMapping resolveHeaderMapping(Sheet sheet) { - DataFormatter formatter = new DataFormatter(); - Row firstHeader = sheet.getRow(0); - Row secondHeader = sheet.getRow(1); - if (firstHeader == null) { + /** + * 由流式表头（第 0 行 headerMap）与第二行表头（两行表头格式时的首个 onRow 回调，可为 null）解析列映射。 + * 保持与旧 DOM 实现完全相同的列别名匹配与 firstDataRowIndex 语义： + * 两行表头（国家行 + 字段行）匹配到 ASIN 列时 firstDataRowIndex = 2； + * 否则退回单行表头（国家与字段同列，如「德国ASIN」）匹配，firstDataRowIndex = 1。 + */ + private HeaderMapping resolveHeaderMapping(Map firstHeaderMap, + Map secondHeaderMap) { + if (firstHeaderMap == null || firstHeaderMap.isEmpty()) { throw new BusinessException("Excel 表头为空"); } - int maxColumn = Math.max(lastCellNum(firstHeader), lastCellNum(secondHeader)); + int maxColumn = Math.max(lastColumnCount(firstHeaderMap), lastColumnCount(secondHeaderMap)); Map asinColumns = new LinkedHashMap<>(); Map priceColumns = new LinkedHashMap<>(); + // 兼容合并单元格表头：EasyExcel SAX 不回填被合并覆盖的单元格文本（旧 DOM 的 cellTextWithMerged + // 会读到主单元格），这里用「字段列未显式标注国家时继承左侧最近国家列」还原同样的列映射结果。 + String carriedCountry = ""; + List carriedColumns = new ArrayList<>(); for (int columnIndex = 0; columnIndex < maxColumn; columnIndex++) { - String countryText = cellTextWithMerged(sheet, firstHeader, columnIndex, formatter); - String fieldText = cellText(secondHeader, columnIndex, formatter); + String countryText = cellText(firstHeaderMap, columnIndex); + String fieldText = cellText(secondHeaderMap, columnIndex); String country = normalizeCountryAlias(countryText); String fieldKey = normalizeHeaderKey(fieldText); if (country.isEmpty()) { country = normalizeCountryAlias(fieldText); fieldKey = normalizeHeaderKey(countryText); } - if (country.isEmpty() || !SUPPORTED_COUNTRIES.contains(country)) { + if (country.isEmpty() && !carriedCountry.isEmpty()) { + String inheritedKey = normalizeHeaderKey(fieldText); + if (isAsinHeader(inheritedKey) || isPriceHeader(inheritedKey)) { + country = carriedCountry; + fieldKey = inheritedKey; + carriedColumns.add(columnIndex); + } + } + if (country.isEmpty()) { continue; } if (isAsinHeader(fieldKey)) { @@ -674,10 +777,11 @@ public class SkipPriceAsinService { } else if (isPriceHeader(fieldKey)) { priceColumns.putIfAbsent(country, columnIndex); } + carriedCountry = country; } if (asinColumns.isEmpty()) { for (int columnIndex = 0; columnIndex < maxColumn; columnIndex++) { - String headerText = cellText(firstHeader, columnIndex, formatter); + String headerText = cellText(firstHeaderMap, columnIndex); String key = normalizeHeaderKey(headerText); String country = normalizeCountryAlias(headerText); if (country.isEmpty()) { @@ -687,13 +791,31 @@ public class SkipPriceAsinService { asinColumns.putIfAbsent(country, columnIndex); } } + if (!carriedColumns.isEmpty()) { + log.info("[skip-price-asin-import] 表头按合并单元格口径继承国家列 carriedColumns={} 但未匹配到 ASIN 列", + carriedColumns); + } return new HeaderMapping(asinColumns, priceColumns, 1); } + if (!carriedColumns.isEmpty()) { + log.info("[skip-price-asin-import] 检测到疑似合并单元格表头，被覆盖列按左侧国家列归属 carriedColumns={}", + carriedColumns); + } return new HeaderMapping(asinColumns, priceColumns, 2); } - private int lastCellNum(Row row) { - return row == null || row.getLastCellNum() < 0 ? 0 : row.getLastCellNum(); + /** 流式表头行最大列数（与旧 Row.getLastCellNum() 口径一致：无单元格时为 0）。 */ + private int lastColumnCount(Map rowMap) { + if (rowMap == null || rowMap.isEmpty()) { + return 0; + } + int lastColumnCount = 0; + for (Integer columnIndex : rowMap.keySet()) { + if (columnIndex != null && columnIndex >= lastColumnCount) { + lastColumnCount = columnIndex + 1; + } + } + return lastColumnCount; } private boolean isAsinHeader(String key) { @@ -705,27 +827,12 @@ public class SkipPriceAsinService { || key.contains("minprice") || key.equals("price"); } - private String cellTextWithMerged(Sheet sheet, Row row, int columnIndex, DataFormatter formatter) { - if (row == null) { + /** 流式解析下按列索引取行内文本（归一化口径与旧 cellText(Row, ...) 一致）。 */ + private String cellText(Map rowMap, Integer columnIndex) { + if (rowMap == null || columnIndex == null || columnIndex < 0) { return ""; } - int rowIndex = row.getRowNum(); - for (int i = 0; i < sheet.getNumMergedRegions(); i++) { - var region = sheet.getMergedRegion(i); - if (region.isInRange(rowIndex, columnIndex)) { - Row firstRow = sheet.getRow(region.getFirstRow()); - return cellText(firstRow, region.getFirstColumn(), formatter); - } - } - return cellText(row, columnIndex, formatter); - } - - private String cellText(Row row, Integer columnIndex, DataFormatter formatter) { - if (row == null || columnIndex == null || columnIndex < 0) { - return ""; - } - Cell cell = row.getCell(columnIndex); - return cell == null ? "" : normalizeExcelText(formatter.formatCellValue(cell)); + return normalizeExcelText(rowMap.get(columnIndex)); } private String normalizeExcelText(String value) { @@ -798,12 +905,13 @@ public class SkipPriceAsinService { return left.compareTo(right) == 0; } - private boolean isBlankRow(Row row, DataFormatter formatter) { - if (row == null) { + /** 该行所有单元格文本是否全空白（归一化口径与旧 isBlankRow(Row, DataFormatter) 一致）。 */ + private boolean isBlankRow(Map rowMap) { + if (rowMap == null || rowMap.isEmpty()) { return true; } - for (int i = 0; i < lastCellNum(row); i++) { - if (!cellText(row, i, formatter).isEmpty()) { + for (String value : rowMap.values()) { + if (!normalizeExcelText(value).isEmpty()) { return false; } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/controller/ShopMatchController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/controller/ShopMatchController.java index 943fef38..767875a4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/controller/ShopMatchController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/controller/ShopMatchController.java @@ -5,12 +5,12 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRe import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskTaskBatchRequest; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskHistoryVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskBatchVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCountryPreferenceVo; +import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.common.model.vo.ProductRiskHistoryVo; +import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.common.model.vo.ProductRiskTaskBatchVo; import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchCreateTaskRequest; import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchStageCompleteRequest; import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchSubmitResultRequest; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchResolveService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchResolveService.java index 0cc80148..aa4a2c63 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchResolveService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchResolveService.java @@ -8,10 +8,10 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRe import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCountryPreferenceVo; +import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo; import com.nanri.aiimage.modules.shopmatch.mapper.ShopMatchCountryPrefMapper; import com.nanri.aiimage.modules.shopmatch.mapper.ShopMatchShopCandidateMapper; import com.nanri.aiimage.modules.shopmatch.model.entity.ShopMatchCountryPrefEntity; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskHeartbeatSpi.java new file mode 100644 index 00000000..e7cea8db --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskHeartbeatSpi.java @@ -0,0 +1,35 @@ +package com.nanri.aiimage.modules.shopmatch.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +/** + * 定时匹配 的任务心跳实现（2026-09 全维度审查 G5）。 + * + *

心跳逻辑（缓存刷新 / 任务缓存回写）从 task 模块收回本模块，task 侧只依赖 SPI 接口， + * 消除 task → 业务模块的编译期依赖。 + */ +@Service +@RequiredArgsConstructor +public class ShopMatchTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { + + private final ShopMatchTaskCacheService cacheService; + + @Override + public String moduleType() { + return "SHOP_MATCH"; + } + + @Override + public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { + cacheService.touchTaskHeartbeat(taskId); + } + + @Override + public void saveTaskCache(FileTaskEntity task) { + cacheService.saveTaskCache(task); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java index 4f643fef..c12b0391 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java @@ -9,13 +9,13 @@ import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.config.TaskPressureProperties; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskHistoryVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskResultItemVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskBatchVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskDetailVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.common.model.vo.ProductRiskHistoryVo; +import com.nanri.aiimage.common.model.vo.ProductRiskResultItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskTaskBatchVo; +import com.nanri.aiimage.common.model.vo.ProductRiskTaskDetailVo; +import com.nanri.aiimage.common.model.vo.ProductRiskTaskItemVo; import com.nanri.aiimage.modules.pricetrack.model.vo.SkipPriceAsinPageVo; import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto; import com.nanri.aiimage.modules.shopmatch.mapper.ShopMatchShopCandidateMapper; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskHeartbeatSpi.java new file mode 100644 index 00000000..bdd84587 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskHeartbeatSpi.java @@ -0,0 +1,53 @@ +package com.nanri.aiimage.modules.similarasin.service; + +import com.nanri.aiimage.config.SimilarAsinProperties; +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.time.Duration; + +/** + * 货源查询（相似 ASIN）的任务心跳实现（2026-09 全维度审查 G5）。 + * + *

本模块的心跳有两处特殊： + * 1. 客户端心跳间隔远小于 stale 判定阈值，因此 biz_file_task.updated_at 采用**检查点节流** + * （按 similarAsinProperties 配置的间隔刷新），避免每次心跳都写库； + * 2. 心跳时顺带刷新结果文件组装作业的租约（避免作业被误判卡死）。 + */ +@Service +@RequiredArgsConstructor +public class SimilarAsinTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { + + private final SimilarAsinTaskCacheService cacheService; + private final TaskFileJobService taskFileJobService; + private final SimilarAsinProperties similarAsinProperties; + + @Override + public String moduleType() { + return "SIMILAR_ASIN"; + } + + @Override + public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { + cacheService.touchTaskHeartbeat(taskId); + } + + @Override + public void onHeartbeat(FileTaskEntity task) { + taskFileJobService.touchRunningAssembleJobsIfStale( + task.getId(), moduleType(), checkpointIntervalMillis()); + } + + @Override + public Duration checkpointInterval() { + return Duration.ofMillis(checkpointIntervalMillis()); + } + + private long checkpointIntervalMillis() { + return Math.max(1_000L, similarAsinProperties.getDbTaskTouchIntervalMillis()); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java index a038904b..a20ecd78 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupService.java @@ -4,8 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.nanri.aiimage.common.service.DistributedJobLockService; import com.nanri.aiimage.config.ModuleCleanupProperties; -import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper; -import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity; +import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; @@ -55,7 +54,7 @@ public class ModuleHistoryCleanupService { private final TaskResultPayloadMapper taskResultPayloadMapper; private final TaskScopeStateMapper taskScopeStateMapper; private final TaskChunkMapper taskChunkMapper; - private final CollectDataItemMapper collectDataItemMapper; + private final CollectDataItemCleanupSpi collectDataItemCleanupSpi; private final DistributedJobLockService distributedJobLockService; private final TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator; private final TransactionTemplate transactionTemplate; @@ -76,7 +75,7 @@ public class ModuleHistoryCleanupService { TaskResultPayloadMapper taskResultPayloadMapper, TaskScopeStateMapper taskScopeStateMapper, TaskChunkMapper taskChunkMapper, - CollectDataItemMapper collectDataItemMapper, + CollectDataItemCleanupSpi collectDataItemCleanupSpi, DistributedJobLockService distributedJobLockService, TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator, PlatformTransactionManager platformTransactionManager) { @@ -89,7 +88,7 @@ public class ModuleHistoryCleanupService { this.taskResultPayloadMapper = taskResultPayloadMapper; this.taskScopeStateMapper = taskScopeStateMapper; this.taskChunkMapper = taskChunkMapper; - this.collectDataItemMapper = collectDataItemMapper; + this.collectDataItemCleanupSpi = collectDataItemCleanupSpi; this.distributedJobLockService = distributedJobLockService; this.transientPayloadDeleteOrchestrator = transientPayloadDeleteOrchestrator; this.transactionTemplate = new TransactionTemplate(platformTransactionManager); @@ -218,8 +217,8 @@ public class ModuleHistoryCleanupService { .in(TaskChunkEntity::getTaskId, cleanupTaskIds)); if (!collectDataTaskIds.isEmpty()) { - collectDataItemMapper.delete(new LambdaQueryWrapper() - .in(CollectDataItemEntity::getTaskId, collectDataTaskIds)); + // 采集明细属于 collectdata 模块（G5：消除 task → 业务依赖，走 SPI） + collectDataItemCleanupSpi.deleteItemsByTaskIds(collectDataTaskIds); } fileResultMapper.delete(new LambdaQueryWrapper() diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/StaleTaskRepairService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/StaleTaskRepairService.java index 48cf0587..0da71dce 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/StaleTaskRepairService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/StaleTaskRepairService.java @@ -3,8 +3,7 @@ package com.nanri.aiimage.modules.task.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.nanri.aiimage.common.service.DistributedJobLockService; -import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper; -import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity; +import com.nanri.aiimage.modules.task.spi.BrandTaskStaleRepairSpi; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import lombok.RequiredArgsConstructor; @@ -32,16 +31,15 @@ public class StaleTaskRepairService { private static final String STATUS_RUNNING = "RUNNING"; private static final String STATUS_FAILED = "FAILED"; - private static final String STATUS_BRAND_PENDING = "pending"; - private static final String STATUS_BRAND_RUNNING = "running"; - private static final String STATUS_BRAND_FAILED = "failed"; - private static final String STATUS_BRAND_CANCELLED = "cancelled"; /** 中间态（无人接单）或 RUNNING（心跳残留）超过该时长未更新即判死。真活任务心跳会不断刷新 updatedAt，不会被误杀。 */ private static final long STALE_IDLE_MINUTES = 120; + /** 单轮品牌陈旧任务处理上限（原实现内联 limit 500）。 */ + private static final int MAX_BRAND_STALE_BATCH = 500; + private final FileTaskMapper fileTaskMapper; - private final BrandCrawlTaskMapper brandCrawlTaskMapper; + private final BrandTaskStaleRepairSpi brandTaskStaleRepairSpi; private final DistributedJobLockService distributedJobLockService; /** @@ -117,26 +115,15 @@ public class StaleTaskRepairService { } } - /** brand_crawl_tasks 的 pending/running 陈旧 → failed。 */ + /** + * brand_crawl_tasks 的 pending/running 陈旧 → failed。 + * 品牌表与状态字面量属于 brand 模块，实现放在 BrandTaskStaleRepairSpiImpl（G5：消除 task → 业务依赖）。 + */ private void repairBrandStale() { LocalDateTime cutoff = LocalDateTime.now().minusMinutes(STALE_IDLE_MINUTES); - List stale = brandCrawlTaskMapper.selectList(new LambdaQueryWrapper() - .select(BrandCrawlTaskEntity::getId) - .in(BrandCrawlTaskEntity::getStatus, STATUS_BRAND_PENDING, STATUS_BRAND_RUNNING) - .lt(BrandCrawlTaskEntity::getUpdatedAt, cutoff) - .last("limit 500")); - if (stale.isEmpty()) { - return; - } - int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() - .in(BrandCrawlTaskEntity::getId, stale.stream().map(BrandCrawlTaskEntity::getId).toList()) - .in(BrandCrawlTaskEntity::getStatus, STATUS_BRAND_PENDING, STATUS_BRAND_RUNNING) - .lt(BrandCrawlTaskEntity::getUpdatedAt, cutoff) - .set(BrandCrawlTaskEntity::getStatus, STATUS_BRAND_FAILED) - .set(BrandCrawlTaskEntity::getErrorMessage, "任务长期无心跳，已自动失败")); - if (updated > 0) { - log.warn("[stale-task-repair] brand 陈旧任务已标失败 count={} ids={}", - updated, stale.stream().map(BrandCrawlTaskEntity::getId).toList()); + List failedIds = brandTaskStaleRepairSpi.failStaleBrandTasks(cutoff, MAX_BRAND_STALE_BATCH); + if (!failedIds.isEmpty()) { + log.warn("[stale-task-repair] brand 陈旧任务已标失败 count={} ids={}", failedIds.size(), failedIds); } } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java index 45551aa3..bb633af2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java @@ -2,76 +2,66 @@ package com.nanri.aiimage.modules.task.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; -import com.nanri.aiimage.config.SimilarAsinProperties; -import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService; -import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper; -import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity; -import com.nanri.aiimage.modules.brand.service.BrandTaskProgressCacheService; -import com.nanri.aiimage.modules.collectdata.service.CollectDataService; -import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService; -import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService; -import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService; -import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService; -import com.nanri.aiimage.modules.publish.service.PublishTaskService; -import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService; -import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService; -import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService; -import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService; -import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo; -import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskCacheService; -import lombok.RequiredArgsConstructor; +import com.nanri.aiimage.modules.task.spi.BrandTaskHeartbeatSpi; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import java.time.Duration; import java.time.LocalDateTime; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +/** + * 任务心跳入口。 + * + *

2026-09 全维度审查 G5：本类此前直接注入 12 个业务模块的 CacheService/Service 并用 switch 分发， + * 与"业务模块实现 task 的 Handler SPI"形成 task ↔ 业务的循环依赖。现改为依赖 + * {@link TaskModuleHeartbeatSpi}（各业务模块自己实现）与 {@link BrandTaskHeartbeatSpi}（品牌任务， + * 独立于 biz_file_task），task 侧不再 import 任何业务模块。 + */ @Service -@RequiredArgsConstructor @Slf4j public class TaskHeartbeatService { private static final String STATUS_RUNNING = "RUNNING"; - private static final String BRAND_STATUS_RUNNING = "running"; - - private static final String MODULE_DELETE_BRAND = "DELETE_BRAND"; - private static final String MODULE_PRODUCT_RISK = "PRODUCT_RISK_RESOLVE"; - private static final String MODULE_PRICE_TRACK = "PRICE_TRACK"; - private static final String MODULE_SHOP_MATCH = "SHOP_MATCH"; - private static final String MODULE_PATROL_DELETE = "PATROL_DELETE"; - private static final String MODULE_QUERY_ASIN = "QUERY_ASIN"; - private static final String MODULE_SHOP_DATA_CRAWL = "SHOP_DATA_CRAWL"; - private static final String MODULE_WITHDRAW = "WITHDRAW"; - private static final String MODULE_APPEARANCE_PATENT = "APPEARANCE_PATENT"; - private static final String MODULE_SIMILAR_ASIN = "SIMILAR_ASIN"; - private static final String MODULE_COLLECT_DATA = "COLLECT_DATA"; private static final String MODULE_BRAND = "BRAND"; + private static final String MODULE_SHOP_DATA_CRAWL = "SHOP_DATA_CRAWL"; private final FileTaskMapper fileTaskMapper; private final TaskProgressSnapshotService taskProgressSnapshotService; - private final BrandCrawlTaskMapper brandCrawlTaskMapper; - private final ProductRiskTaskCacheService productRiskTaskCacheService; - private final PublishTaskService publishTaskService; - private final PriceTrackTaskCacheService priceTrackTaskCacheService; - private final ShopMatchTaskCacheService shopMatchTaskCacheService; - private final PatrolDeleteTaskCacheService patrolDeleteTaskCacheService; - private final QueryAsinTaskCacheService queryAsinTaskCacheService; - private final ShopDataCrawlTaskCacheService shopDataCrawlTaskCacheService; - private final ShopDataCrawlTaskService shopDataCrawlTaskService; - private final WithdrawTaskCacheService withdrawTaskCacheService; - private final AppearancePatentTaskCacheService appearancePatentTaskCacheService; - private final SimilarAsinTaskCacheService similarAsinTaskCacheService; - private final SimilarAsinProperties similarAsinProperties; - private final TaskFileJobService taskFileJobService; - private final DeleteBrandTaskCacheService deleteBrandTaskCacheService; - private final BrandTaskProgressCacheService brandTaskProgressCacheService; - private final CollectDataService collectDataService; + private final BrandTaskHeartbeatSpi brandTaskHeartbeatSpi; + /** moduleType → 模块心跳实现（启动时建索引并校验重复注册） */ + private final Map moduleHandlers; + + public TaskHeartbeatService(FileTaskMapper fileTaskMapper, + TaskProgressSnapshotService taskProgressSnapshotService, + BrandTaskHeartbeatSpi brandTaskHeartbeatSpi, + List moduleHeartbeatHandlers) { + this.fileTaskMapper = fileTaskMapper; + this.taskProgressSnapshotService = taskProgressSnapshotService; + this.brandTaskHeartbeatSpi = brandTaskHeartbeatSpi; + Map handlers = new LinkedHashMap<>(); + for (TaskModuleHeartbeatSpi handler : moduleHeartbeatHandlers == null ? List.of() : moduleHeartbeatHandlers) { + String moduleType = handler.moduleType(); + if (moduleType == null || moduleType.isBlank()) { + throw new IllegalStateException("TaskModuleHeartbeatSpi 未声明 moduleType: " + + handler.getClass().getName()); + } + TaskModuleHeartbeatSpi exists = handlers.put(moduleType, handler); + if (exists != null) { + throw new IllegalStateException("moduleType=" + moduleType + " 注册了多个心跳实现: " + + exists.getClass().getName() + " / " + handler.getClass().getName()); + } + } + this.moduleHandlers = Map.copyOf(handlers); + log.info("[task-heartbeat-spi] 模块心跳实现注册完成 count={} modules={}", handlers.size(), handlers.keySet()); + } public TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request) { if (taskId == null || taskId <= 0) { @@ -82,15 +72,12 @@ public class TaskHeartbeatService { String requestedModuleType = request == null || request.getModuleType() == null ? null : request.getModuleType().trim(); - FileTaskEntity fileTask = MODULE_BRAND.equalsIgnoreCase(requestedModuleType) - ? null - : selectFileTask(taskId); - BrandCrawlTaskEntity brandTask = MODULE_SHOP_DATA_CRAWL.equalsIgnoreCase(requestedModuleType) - ? null - : selectBrandTask(taskId); + // 品牌任务在独立表里：客户端声明是 BRAND 时不必再查 biz_file_task（历史行为保持一致） + boolean brandOnly = MODULE_BRAND.equalsIgnoreCase(requestedModuleType); + boolean skipBrandLookup = MODULE_SHOP_DATA_CRAWL.equalsIgnoreCase(requestedModuleType); - TaskHeartbeatVo fileResult = touchFileTaskIfRunning(fileTask, request); - TaskHeartbeatVo brandResult = touchBrandTaskIfRunning(brandTask, request); + TaskHeartbeatVo fileResult = brandOnly ? null : touchFileTaskIfRunning(selectFileTask(taskId), request); + TaskHeartbeatVo brandResult = skipBrandLookup ? null : brandTaskHeartbeatSpi.heartbeat(taskId, request); if (fileResult != null && fileResult.isAlive() && brandResult != null && brandResult.isAlive()) { return TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING); @@ -158,204 +145,72 @@ public class TaskHeartbeatService { syncTerminalState(fileTask, status, fileTask.getErrorMessage()); return TaskHeartbeatVo.notAlive(fileTask.getModuleType(), status, "task is not running"); } - BrandCrawlTaskEntity brandTask = selectBrandTask(taskId); - if (brandTask != null) { - String status = brandTask.getStatus(); - if ("running".equalsIgnoreCase(status) || "pending".equalsIgnoreCase(status)) { - // 与 file 分支统一改为条件更新：整行 updateById 会拿读取快照覆盖并发写入的字段 - // （客户端重启上报与品牌任务自身状态流转同时发生时） - int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() - .eq(BrandCrawlTaskEntity::getId, brandTask.getId()) - .in(BrandCrawlTaskEntity::getStatus, "running", "pending", "RUNNING", "PENDING") - .set(BrandCrawlTaskEntity::getStatus, "cancelled") - .set(BrandCrawlTaskEntity::getErrorMessage, safeReason) - .set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now())); - if (updated > 0) { - log.warn("[task-interrupted] brand task marked cancelled by client restart taskId={} reason={}", - taskId, safeReason); - return TaskHeartbeatVo.notAlive(MODULE_BRAND, "cancelled", "marked cancelled"); - } - } - log.info("[task-interrupted] brand task not in running/pending, skipped taskId={} status={}", taskId, status); - return TaskHeartbeatVo.notAlive(MODULE_BRAND, status, "task is not running"); + TaskHeartbeatVo brandResult = brandTaskHeartbeatSpi.markInterrupted(taskId, safeReason); + if (brandResult != null) { + return brandResult; } log.debug("[task-interrupted] task not found (normal polling) taskId={}", taskId); return TaskHeartbeatVo.notAlive(null, null, "task not found"); } - private BrandCrawlTaskEntity selectBrandTask(Long taskId) { - LambdaQueryWrapper brandQuery = new LambdaQueryWrapper() - .eq(BrandCrawlTaskEntity::getId, taskId) - .last("limit 1"); - return brandCrawlTaskMapper.selectOne(brandQuery); - } - private TaskHeartbeatVo touchFileTaskIfRunning(FileTaskEntity task, TaskHeartbeatRequest request) { if (task == null) { return null; } String moduleType = task.getModuleType() == null ? "" : task.getModuleType(); - if (PublishTaskService.MODULE_TYPE.equals(moduleType)) { - publishTaskService.ensureTaskOwnedByCurrentInstance(task, "publish task heartbeat"); - } - if (MODULE_SHOP_DATA_CRAWL.equals(moduleType)) { - shopDataCrawlTaskService.ensureTaskOwnedByCurrentInstance(task, "shop data crawl task heartbeat"); + TaskModuleHeartbeatSpi handler = moduleHandlers.get(moduleType); + if (handler != null) { + handler.ensureOwnership(task, moduleType + " task heartbeat"); } String status = task.getStatus(); if (!STATUS_RUNNING.equals(status)) { - log.debug("[task-heartbeat] file task is not running (normal polling) taskId={ actualUserId={} moduleType={} status={}", + log.debug("[task-heartbeat] file task is not running (normal polling) taskId={} actualUserId={} moduleType={} status={}", task.getId(), task.getUserId(), moduleType, status); return TaskHeartbeatVo.notAlive(moduleType, status, "task is not running"); } - if (MODULE_SIMILAR_ASIN.equals(moduleType)) { - return touchSimilarAsinHeartbeat(task); + + // updated_at 检查点：默认每次心跳刷新；实现声明了节流间隔的（similar-asin）按间隔刷新， + // 避免客户端密集心跳把库写满 + Duration interval = handler == null ? null : handler.checkpointInterval(); + if (isCheckpointDue(task, interval)) { + LocalDateTime now = LocalDateTime.now(); + int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(FileTaskEntity::getId, task.getId()) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .set(FileTaskEntity::getUpdatedAt, now)); + if (updated <= 0) { + FileTaskEntity latest = fileTaskMapper.selectById(task.getId()); + if (interval != null && latest != null && STATUS_RUNNING.equals(latest.getStatus())) { + // 节流下的条件更新未命中：并发心跳刚刷新过，按最新实体继续（不判死） + task = latest; + } else { + log.warn("[task-heartbeat] file task heartbeat update missed taskId={} actualUserId={} moduleType={} status={} latestStatus={}", + task.getId(), task.getUserId(), moduleType, status, + latest == null ? null : latest.getStatus()); + return TaskHeartbeatVo.notAlive(moduleType, latest == null ? status : latest.getStatus(), + "task is not running"); + } + } else { + task.setUpdatedAt(now); + } } - LocalDateTime now = LocalDateTime.now(); - int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper() - .eq(FileTaskEntity::getId, task.getId()) - .eq(FileTaskEntity::getStatus, STATUS_RUNNING) - .set(FileTaskEntity::getUpdatedAt, now)); - if (updated <= 0) { - FileTaskEntity latest = fileTaskMapper.selectById(task.getId()); - log.warn("[task-heartbeat] file task heartbeat update missed taskId={} actualUserId={} moduleType={} status={} latestStatus={}", - task.getId(), task.getUserId(), moduleType, status, - latest == null ? null : latest.getStatus()); - return TaskHeartbeatVo.notAlive(moduleType, latest == null ? status : latest.getStatus(), "task is not running"); + + if (handler != null) { + handler.touchHeartbeat(task.getId(), request); + handler.onHeartbeat(task); + handler.saveTaskCache(task); } - touchModuleHeartbeat(moduleType, task.getId(), request); - task.setUpdatedAt(now); - saveFileTaskCache(moduleType, task); return TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING); } - private TaskHeartbeatVo touchSimilarAsinHeartbeat(FileTaskEntity task) { - LocalDateTime now = LocalDateTime.now(); - long intervalMillis = Math.max(1_000L, similarAsinProperties.getDbTaskTouchIntervalMillis()); - LocalDateTime cutoff = now.minus(Duration.ofMillis(intervalMillis)); - boolean checkpointDue = task.getUpdatedAt() == null || !task.getUpdatedAt().isAfter(cutoff); - - similarAsinTaskCacheService.touchTaskHeartbeat(task.getId()); - taskFileJobService.touchRunningAssembleJobsIfStale(task.getId(), MODULE_SIMILAR_ASIN, intervalMillis); - if (!checkpointDue) { - saveFileTaskCache(MODULE_SIMILAR_ASIN, task); - return TaskHeartbeatVo.alive(MODULE_SIMILAR_ASIN, STATUS_RUNNING); - } - - int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper() - .eq(FileTaskEntity::getId, task.getId()) - .eq(FileTaskEntity::getStatus, STATUS_RUNNING) - .and(wrapper -> wrapper.isNull(FileTaskEntity::getUpdatedAt) - .or() - .le(FileTaskEntity::getUpdatedAt, cutoff)) - .set(FileTaskEntity::getUpdatedAt, now)); - if (updated <= 0) { - FileTaskEntity latest = fileTaskMapper.selectById(task.getId()); - if (latest == null || !STATUS_RUNNING.equals(latest.getStatus())) { - return TaskHeartbeatVo.notAlive(MODULE_SIMILAR_ASIN, - latest == null ? task.getStatus() : latest.getStatus(), "task is not running"); - } - task = latest; - } else { - task.setUpdatedAt(now); - } - saveFileTaskCache(MODULE_SIMILAR_ASIN, task); - return TaskHeartbeatVo.alive(MODULE_SIMILAR_ASIN, STATUS_RUNNING); - } - - private TaskHeartbeatVo touchBrandTaskIfRunning(BrandCrawlTaskEntity task, TaskHeartbeatRequest request) { - if (task == null) { - return null; - } - String status = task.getStatus(); - if (!BRAND_STATUS_RUNNING.equals(status)) { - log.warn("[task-heartbeat] brand task is not running taskId={} actualUserId={} status={}", - task.getId(), task.getUserId(), status); - return TaskHeartbeatVo.notAlive(MODULE_BRAND, status, "task is not running"); - } - int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() - .eq(BrandCrawlTaskEntity::getId, task.getId()) - .eq(BrandCrawlTaskEntity::getStatus, BRAND_STATUS_RUNNING) - .set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now())); - if (updated <= 0) { - BrandCrawlTaskEntity latest = brandCrawlTaskMapper.selectById(task.getId()); - log.warn("[task-heartbeat] brand task heartbeat update missed taskId={} actualUserId={} status={} latestStatus={}", - task.getId(), task.getUserId(), status, - latest == null ? null : latest.getStatus()); - return TaskHeartbeatVo.notAlive(MODULE_BRAND, latest == null ? status : latest.getStatus(), "task is not running"); - } - brandTaskProgressCacheService.touchHeartbeat( - task.getId(), - request == null ? null : request.getPhase(), - request == null ? null : request.getCurrent(), - request == null ? null : request.getTotal()); - return TaskHeartbeatVo.alive(MODULE_BRAND, BRAND_STATUS_RUNNING); - } - - private void touchModuleHeartbeat(String moduleType, Long taskId, TaskHeartbeatRequest request) { - switch (moduleType) { - case MODULE_PRODUCT_RISK -> { - productRiskTaskCacheService.touchTaskHeartbeat(taskId); - } - case PublishTaskService.MODULE_TYPE -> { - publishTaskService.touchHeartbeat(taskId, request); - } - case MODULE_PRICE_TRACK -> { - priceTrackTaskCacheService.touchTaskHeartbeat(taskId); - } - case MODULE_SHOP_MATCH -> { - shopMatchTaskCacheService.touchTaskHeartbeat(taskId); - } - case MODULE_PATROL_DELETE -> { - patrolDeleteTaskCacheService.touchTaskHeartbeat(taskId); - } - case MODULE_QUERY_ASIN -> { - queryAsinTaskCacheService.touchTaskHeartbeat(taskId); - } - case MODULE_SHOP_DATA_CRAWL -> shopDataCrawlTaskCacheService.touchTaskHeartbeat(taskId); - case MODULE_WITHDRAW -> { - withdrawTaskCacheService.touchTaskHeartbeat(taskId); - } - case MODULE_APPEARANCE_PATENT -> { - appearancePatentTaskCacheService.touchTaskHeartbeat(taskId); - } - case MODULE_SIMILAR_ASIN -> { - similarAsinTaskCacheService.touchTaskHeartbeat(taskId); - } - case MODULE_DELETE_BRAND -> { - deleteBrandTaskCacheService.saveProgress(taskId, buildDeleteBrandHeartbeatProgress(request), true); - } - case MODULE_COLLECT_DATA -> collectDataService.updateProgress(taskId, request); - default -> { - } - } - } - - private Map buildDeleteBrandHeartbeatProgress(TaskHeartbeatRequest request) { - String now = String.valueOf(System.currentTimeMillis()); - Map values = new LinkedHashMap<>(); - values.put("last_heartbeat_at", now); - values.put("updated_at", now); - if (request != null) { - putIfPresent(values, "phase", request.getPhase()); - putIfPresent(values, "current", request.getCurrent()); - putIfPresent(values, "total", request.getTotal()); - } - return values; - } - - private void saveFileTaskCache(String moduleType, FileTaskEntity task) { - switch (moduleType) { - case MODULE_PRODUCT_RISK -> productRiskTaskCacheService.saveTaskCache(task); - case MODULE_PRICE_TRACK -> priceTrackTaskCacheService.saveTaskCache(task); - case MODULE_SHOP_MATCH -> shopMatchTaskCacheService.saveTaskCache(task); - case MODULE_PATROL_DELETE -> patrolDeleteTaskCacheService.saveTaskCache(task); - case MODULE_QUERY_ASIN -> queryAsinTaskCacheService.saveTaskCache(task); - case MODULE_SHOP_DATA_CRAWL -> shopDataCrawlTaskCacheService.saveTaskCache(task); - case MODULE_WITHDRAW -> withdrawTaskCacheService.saveTaskCache(task); - case MODULE_DELETE_BRAND -> deleteBrandTaskCacheService.saveTaskCache(task); - default -> { - } + /** 无节流间隔时永远认为需要刷新（历史行为）；有间隔时按 updated_at 距上次刷新的时长判断。 */ + private boolean isCheckpointDue(FileTaskEntity task, Duration interval) { + if (interval == null) { + return true; } + long intervalMillis = Math.max(1_000L, interval.toMillis()); + LocalDateTime cutoff = LocalDateTime.now().minus(Duration.ofMillis(intervalMillis)); + return task.getUpdatedAt() == null || !task.getUpdatedAt().isAfter(cutoff); } /** 把终态同步到模块缓存与进度快照；失败只告警，不影响中断接口本身的成功语义。 */ @@ -372,10 +227,13 @@ public class TaskHeartbeatService { } } - private void putIfPresent(Map values, String key, Object value) { - if (value != null) { - values.put(key, String.valueOf(value)); + private void saveFileTaskCache(String moduleType, FileTaskEntity task) { + if (moduleType == null) { + return; + } + TaskModuleHeartbeatSpi handler = moduleHandlers.get(moduleType); + if (handler != null) { + handler.saveTaskCache(task); } } - } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadDeleteOrchestrator.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadDeleteOrchestrator.java index f1d038ff..43d898ef 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadDeleteOrchestrator.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadDeleteOrchestrator.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.task.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.util.PendingDeleteJournal; import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService; import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; @@ -10,8 +11,11 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; +import java.nio.file.Path; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -49,6 +53,12 @@ public class TransientPayloadDeleteOrchestrator { @Value("${aiimage.transient-storage.max-pending-deletes:1000}") private long maxPendingDeletes = 1000; + /** + * 待删对象的本地落盘日志：pending 集合重启即丢，未被引用的对象会一直残留在桶里。 + * 入队追加、flush 后按「仍在 pending + 异步删除失败」收敛重写、启动时回放后重新走引用检查。 + */ + private volatile PendingDeleteJournal journal; + public TransientPayloadDeleteOrchestrator(TransientPayloadStorageService transientPayloadStorageService, RustfsObjectStorageService rustfsObjectStorageService, TaskChunkMapper taskChunkMapper, @@ -84,6 +94,7 @@ public class TransientPayloadDeleteOrchestrator { } if (pendingPointers.add(pointer)) { accepted++; + journal().record(pointer); } } return accepted; @@ -164,6 +175,7 @@ public class TransientPayloadDeleteOrchestrator { } private void deleteObjects(List pointers) { + List failed = new ArrayList<>(); for (String pointer : pointers) { if (pointer == null || !pointer.startsWith(RUSTFS_POINTER_PREFIX)) { // 本地/OSS 指针有实例归属与其它清理路径，不在此批量删除范围。 @@ -173,10 +185,50 @@ public class TransientPayloadDeleteOrchestrator { try { rustfsObjectStorageService.deleteObject(objectKey); } catch (Exception ex) { + // 失败对象写回日志：进程重启后由启动回放重新提交，不再随进程消失 + failed.add(pointer); log.warn("[transient-payload] async delete failed objectKey={} err={}", objectKey, ex.getMessage()); } } + synchronized (pendingPointers) { + Set keep = new LinkedHashSet<>(pendingPointers); + keep.addAll(failed); + journal().rewrite(keep); + } + if (!failed.isEmpty()) { + log.warn("[transient-payload] async delete failed, kept in journal count={}", failed.size()); + } + } + + /** + * 启动回放：把上次进程残留的待删对象重新入队，等待下一轮 flush 做引用检查与物理删除 + * （引用检查幂等，重复提交同一对象只入队一次）。 + */ + @EventListener(ApplicationReadyEvent.class) + public void replayJournalOnStartup() { + List pointers = journal().readAll(); + if (pointers.isEmpty()) { + return; + } + int accepted = submitDeletes(pointers); + log.info("[transient-payload] 启动回放待删对象 journalSize={} accepted={} pending={}", + pointers.size(), accepted, pendingCount()); + } + + private PendingDeleteJournal journal() { + PendingDeleteJournal local = journal; + if (local != null) { + return local; + } + synchronized (pendingPointers) { + if (journal == null) { + Path path = Path.of(System.getProperty("java.io.tmpdir"), + "aiimage-delete-journal", "transient-payload-pending.log"); + journal = new PendingDeleteJournal(path, pendingLimit()); + } + return journal; + } } /** 批量 IN 反查两张引用表，返回仍被引用的指针集合（查询异常抛给调用方）。 */ diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/BrandTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/BrandTaskHeartbeatSpi.java new file mode 100644 index 00000000..a7fbccc9 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/BrandTaskHeartbeatSpi.java @@ -0,0 +1,25 @@ +package com.nanri.aiimage.modules.task.spi; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo; + +/** + * 品牌检测任务（brand_crawl_tasks，独立于 biz_file_task）的心跳/中断扩展点。 + * + *

2026-09 全维度审查 G5：品牌任务有自己的表与进度缓存，原先由 {@code TaskHeartbeatService} + * 直接注入 brand 模块的 Mapper 与缓存服务，构成 task → 业务的依赖。这里把品牌侧逻辑整体 + * 交回 brand 模块实现，task 侧只按接口调用。 + */ +public interface BrandTaskHeartbeatSpi { + + /** + * 品牌任务心跳：任务不存在返回 null；非 running 返回 notAlive；running 则条件更新 + * updated_at 并刷新进度缓存。 + */ + TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request); + + /** + * 客户端重启中断：running/pending → cancelled（条件更新），返回心跳结果；任务不存在返回 null。 + */ + TaskHeartbeatVo markInterrupted(Long taskId, String reason); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/BrandTaskStaleRepairSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/BrandTaskStaleRepairSpi.java new file mode 100644 index 00000000..dfc3c960 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/BrandTaskStaleRepairSpi.java @@ -0,0 +1,22 @@ +package com.nanri.aiimage.modules.task.spi; + +import java.time.LocalDateTime; +import java.util.List; + +/** + * 品牌检测任务的陈旧修复扩展点（2026-09 全维度审查 G5）。 + * + *

brand_crawl_tasks 有独立的表与状态字面量（pending/running/failed），原先由 + * {@code StaleTaskRepairService} 直接操作品牌模块的 Mapper，构成 task → 业务的依赖。 + * 现由品牌模块实现本接口，task 侧只按接口调用。 + */ +public interface BrandTaskStaleRepairSpi { + + /** + * 把 pending/running 且 updated_at 早于 cutoff 的品牌任务置为失败（条件更新，幂等）。 + * + * @param limit 单轮处理上限 + * @return 实际标记失败的任务 id（空表示本轮无陈旧任务） + */ + List failStaleBrandTasks(LocalDateTime cutoff, int limit); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/CollectDataItemCleanupSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/CollectDataItemCleanupSpi.java new file mode 100644 index 00000000..42e4bae0 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/CollectDataItemCleanupSpi.java @@ -0,0 +1,20 @@ +package com.nanri.aiimage.modules.task.spi; + +import java.util.List; + +/** + * 采集明细行的历史清理扩展点（2026-09 全维度审查 G5）。 + * + *

任务历史清理时除了 task 自己的表，还要连带删除 biz_collect_data_item 的明细行； + * 该表属于采集模块，原先由 {@code ModuleHistoryCleanupService} 直接使用其 Mapper， + * 构成 task → 业务的依赖。现由采集模块实现本接口。 + */ +public interface CollectDataItemCleanupSpi { + + /** + * 删除这些任务名下的采集明细行。 + * + * @return 删除行数 + */ + int deleteItemsByTaskIds(List taskIds); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/TaskModuleHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/TaskModuleHeartbeatSpi.java new file mode 100644 index 00000000..4e5aaaca --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/spi/TaskModuleHeartbeatSpi.java @@ -0,0 +1,57 @@ +package com.nanri.aiimage.modules.task.spi; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; + +import java.time.Duration; + +/** + * 任务心跳的模块侧扩展点（2026-09 全维度审查 G5：消除 task ↔ 业务模块的双向依赖）。 + * + *

背景：{@code TaskHeartbeatService} 原先直接注入 12 个业务模块的 CacheService 并用 switch 分发， + * 与"业务模块实现 task 的 Handler SPI"形成真实循环依赖。现改为：task 侧只依赖本接口， + * 各业务模块各自实现一个 Bean（Spring 注入 List 后在 task 侧建索引）。 + * + *

新增模块时：实现本接口并在注册表中登记模块类型；心跳契约测试会校验覆盖。 + */ +public interface TaskModuleHeartbeatSpi { + + /** 本实现负责的 moduleType（与 biz_file_task.module_type 一致）。 */ + String moduleType(); + + /** + * 刷新模块自己的进度缓存/心跳（非 DB 写入）。 + * 多数模块实现为 {@code cacheService.touchTaskHeartbeat(taskId)}。 + */ + void touchHeartbeat(Long taskId, TaskHeartbeatRequest request); + + /** + * 心跳时回写模块的任务缓存（部分模块没有独立缓存，保持默认空实现）。 + */ + default void saveTaskCache(FileTaskEntity task) { + } + + /** + * 心跳前的任务归属校验：publish / shop-data-crawl 在非本实例拥有时拒绝心跳， + * 其余模块默认放行。 + */ + default void ensureOwnership(FileTaskEntity task, String operation) { + } + + /** + * 心跳时的额外模块侧动作（如 similar-asin 刷新结果文件组装作业的租约）。 + */ + default void onHeartbeat(FileTaskEntity task) { + } + + /** + * biz_file_task.updated_at 检查点节流间隔。 + * + *

返回非空时，DB 的 updated_at 只在距上次刷新超过该间隔后才更新 + * （客户端心跳间隔远小于 stale 判定阈值时避免每次心跳都写库）； + * 默认 null 表示每次心跳都刷新（与历史行为一致）。 + */ + default Duration checkpointInterval() { + return null; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/controller/WithdrawTaskController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/controller/WithdrawTaskController.java index 0e2bd37d..0176da4f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/controller/WithdrawTaskController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/controller/WithdrawTaskController.java @@ -3,9 +3,9 @@ package com.nanri.aiimage.modules.withdraw.controller; import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo; import com.nanri.aiimage.modules.withdraw.model.dto.WithdrawCandidateClearRequest; import com.nanri.aiimage.modules.withdraw.model.dto.WithdrawCreateTaskRequest; import com.nanri.aiimage.modules.withdraw.model.dto.WithdrawSubmitResultRequest; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawResolveService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawResolveService.java index a40fb201..e0405d0c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawResolveService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawResolveService.java @@ -4,9 +4,9 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo; +import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo; +import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo; +import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo; import com.nanri.aiimage.modules.withdraw.mapper.WithdrawShopCandidateMapper; import com.nanri.aiimage.modules.withdraw.model.entity.WithdrawShopCandidateEntity; import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawTaskHeartbeatSpi.java b/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawTaskHeartbeatSpi.java new file mode 100644 index 00000000..67c77ca3 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawTaskHeartbeatSpi.java @@ -0,0 +1,35 @@ +package com.nanri.aiimage.modules.withdraw.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +/** + * 取款 的任务心跳实现（2026-09 全维度审查 G5）。 + * + *

心跳逻辑（缓存刷新 / 任务缓存回写）从 task 模块收回本模块，task 侧只依赖 SPI 接口， + * 消除 task → 业务模块的编译期依赖。 + */ +@Service +@RequiredArgsConstructor +public class WithdrawTaskHeartbeatSpi implements TaskModuleHeartbeatSpi { + + private final WithdrawTaskCacheService cacheService; + + @Override + public String moduleType() { + return "WITHDRAW"; + } + + @Override + public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) { + cacheService.touchTaskHeartbeat(taskId); + } + + @Override + public void saveTaskCache(FileTaskEntity task) { + cacheService.saveTaskCache(task); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawTaskService.java index 2b783d71..1dac9d3c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/withdraw/service/WithdrawTaskService.java @@ -10,7 +10,7 @@ import com.nanri.aiimage.common.exception.BusinessCodes; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.config.TaskPressureProperties; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; -import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo; +import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; diff --git a/backend-java/src/main/resources/db/V123__drop_prefix_indexes.sql b/backend-java/src/main/resources/db/V123__drop_prefix_indexes.sql new file mode 100644 index 00000000..7024bfc1 --- /dev/null +++ b/backend-java/src/main/resources/db/V123__drop_prefix_indexes.sql @@ -0,0 +1,46 @@ +-- V123: 删除 biz_file_result 上被复合索引完全覆盖的两个单列索引 +-- +-- 判定原则（同 V121）：被删索引的列是另一索引的**严格前缀**， +-- 所有能用它的查询都能被保留的复合索引覆盖，删除只减少写入侧的索引维护与 buffer pool 占用。 +-- +-- 1) biz_file_result.idx_task_id (task_id) +-- 被 V38 的 idx_file_result_task_module_id (task_id, module_type, id) 覆盖。 +-- 2) biz_file_result.idx_module_type (module_type) +-- 被 V34 的 idx_biz_file_result_module_user_created (module_type, user_id, created_at) +-- 与 V42 的 idx_file_result_patrol_history (module_type, user_id, created_at, id) 覆盖。 +-- 两处均为 V1 建表时的遗留索引（当时还没有任何复合索引）。 +-- +-- 风险：DROP INDEX 为 INPLACE 在线操作，但会立即改变执行计划；两者均为「被覆盖」判定。 +-- 回滚： +-- ALTER TABLE biz_file_result ADD INDEX idx_task_id (task_id); +-- ALTER TABLE biz_file_result ADD INDEX idx_module_type (module_type); + +SET @db_name = DATABASE(); + +-- 1) biz_file_result.idx_task_id +SET @idx_exists := ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_result' + AND INDEX_NAME = 'idx_task_id' +); +SET @sql := IF(@idx_exists > 0, + 'ALTER TABLE biz_file_result DROP INDEX idx_task_id', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- 2) biz_file_result.idx_module_type +SET @idx_exists := ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_result' + AND INDEX_NAME = 'idx_module_type' +); +SET @sql := IF(@idx_exists > 0, + 'ALTER TABLE biz_file_result DROP INDEX idx_module_type', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/backend-java/src/main/resources/db/V124__drop_dedupe_total_data_country_index.sql b/backend-java/src/main/resources/db/V124__drop_dedupe_total_data_country_index.sql new file mode 100644 index 00000000..53916629 --- /dev/null +++ b/backend-java/src/main/resources/db/V124__drop_dedupe_total_data_country_index.sql @@ -0,0 +1,27 @@ +-- V124: 删除 biz_dedupe_total_data.idx_country（函数列索引，永不生效） +-- +-- 背景（2026-09 全维度审查 C6）：该表按国家的筛选走 FIND_IN_SET(?, IFNULL(country,''))， +-- 是函数列比较，MySQL 无法使用 country 上的普通索引；V91 建的 idx_country 从未被任何执行计划 +-- 使用过，只增加写入维护与 buffer pool 占用。 +-- +-- 前缀关键字搜索本身就吃 uk_data_value (data_value) 唯一索引（本次同时把关键字由 +-- %kw% 前置通配改为前缀匹配，见 DedupeTotalDataService.page），因此无需新增索引。 +-- +-- 风险：DROP INDEX 为在线 INPLACE 操作；country 仍用于结果展示，仅去掉索引。 +-- 回滚： +-- ALTER TABLE biz_dedupe_total_data ADD INDEX idx_country (country); + +SET @db_name = DATABASE(); + +SET @idx_exists := ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_dedupe_total_data' + AND INDEX_NAME = 'idx_country' +); +SET @sql := IF(@idx_exists > 0, + 'ALTER TABLE biz_dedupe_total_data DROP INDEX idx_country', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/backend-java/src/test/java/com/nanri/aiimage/ArchitectureBoundaryTest.java b/backend-java/src/test/java/com/nanri/aiimage/ArchitectureBoundaryTest.java index 3eb2f425..87385093 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/ArchitectureBoundaryTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/ArchitectureBoundaryTest.java @@ -34,7 +34,9 @@ class ArchitectureBoundaryTest { * 本次以实测值 119 为准恢复告警能力；经 git diff 核验，当日的 task 模块改动 * （引用计数按 task 收敛、快照 upsert 缓存、心跳条件更新、锁续期重试等）未新增任何业务依赖。 */ - private static final int TASK_TO_BUSINESS_BASELINE = 119; + // 2026-09-14 心跳/陈旧修复/历史清理 SPI 化后实测 6（G5：task 只依赖 task.spi 下接口， +// 业务模块各自实现）；继续按棘轮收敛，不允许回升 + private static final int TASK_TO_BUSINESS_BASELINE = 6; private static volatile JavaClasses cached; diff --git a/backend-java/src/test/java/com/nanri/aiimage/common/util/PendingDeleteJournalTest.java b/backend-java/src/test/java/com/nanri/aiimage/common/util/PendingDeleteJournalTest.java new file mode 100644 index 00000000..71fa2f69 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/common/util/PendingDeleteJournalTest.java @@ -0,0 +1,90 @@ +package com.nanri.aiimage.common.util; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * 待删对象日志：入队追加、启动回放、收敛重写。对应审查项 D13（删除补偿队列重启即丢）。 + */ +class PendingDeleteJournalTest { + + @TempDir + Path tempDir; + + @Test + void recordThenReadAllReturnsAdmittedKeysInOrder() { + PendingDeleteJournal journal = new PendingDeleteJournal(tempDir.resolve("nested/queue.log"), 100); + + journal.record("a/1.bin"); + journal.record("b/2.bin"); + journal.record("a/1.bin"); + + assertEquals(List.of("a/1.bin", "b/2.bin"), journal.readAll()); + } + + @Test + void rewriteKeepsOnlyRemainingAndDropsFileWhenEmpty() throws Exception { + Path path = tempDir.resolve("queue.log"); + PendingDeleteJournal journal = new PendingDeleteJournal(path, 100); + journal.record("keep.bin"); + journal.record("done.bin"); + + journal.rewrite(List.of("keep.bin")); + assertEquals(List.of("keep.bin"), journal.readAll()); + + journal.rewrite(List.of()); + assertFalse(Files.exists(path), "队列收敛为空后日志文件应删除"); + assertEquals(List.of(), journal.readAll()); + } + + @Test + void missingFileYieldsEmptyInsteadOfThrowing() { + PendingDeleteJournal journal = new PendingDeleteJournal(tempDir.resolve("not-created.log"), 10); + assertEquals(List.of(), journal.readAll()); + } + + @Test + void blankAndMultilineRecordsAreSanitizedOrIgnored() throws Exception { + Path path = tempDir.resolve("queue.log"); + PendingDeleteJournal journal = new PendingDeleteJournal(path, 10); + + journal.record(null); + journal.record(" "); + journal.record("bad\nkey.bin"); + journal.record(" good.bin "); + + assertEquals(List.of("bad key.bin", "good.bin"), journal.readAll()); + // 单行一条记录：换行被替换，不会破坏行结构 + assertEquals(2, Files.readAllLines(path, StandardCharsets.UTF_8).size()); + } + + @Test + void corruptedLineIsIgnoredWhileValidOnesSurvive() throws Exception { + Path path = tempDir.resolve("queue.log"); + Files.createDirectories(tempDir); + Files.writeString(path, "\n \nvalid.bin\n", StandardCharsets.UTF_8); + + PendingDeleteJournal journal = new PendingDeleteJournal(path, 10); + + assertEquals(List.of("valid.bin"), journal.readAll()); + } + + @Test + void rewriteRespectsMaxEntriesCap() { + PendingDeleteJournal journal = new PendingDeleteJournal(tempDir.resolve("queue.log"), 2); + + journal.rewrite(List.of("k1.bin", "k2.bin", "k3.bin")); + + assertEquals(List.of("k1.bin", "k2.bin"), journal.readAll()); + assertTrue(journal.readAll().size() <= 2); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/deletebrand/service/TaskModuleCoverageTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/deletebrand/service/TaskModuleCoverageTest.java new file mode 100644 index 00000000..dbf772d4 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/deletebrand/service/TaskModuleCoverageTest.java @@ -0,0 +1,112 @@ +package com.nanri.aiimage.modules.deletebrand.service; + +import com.nanri.aiimage.common.module.TaskModuleRegistry; +import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService; +import com.nanri.aiimage.modules.brand.service.BrandTaskService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService; +import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService; +import com.nanri.aiimage.config.ModuleCleanupProperties; +import com.nanri.aiimage.config.TaskFileJobConfig; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * 模块清单覆盖面自检（2026-09 全维度审查 G6）。 + * + * 新增工具模块此前要同时改 5 处枚举，其中 4 处没有自检，漏改表现为"功能静默不生效"。 + * 本测试把「注册表 = 单一来源」这件事钉住：数据驱动的三处必须与注册表完全一致， + * 委派式判死巡检线必须为注册表标记的每个模块登记动作。 + * 新增模块时若漏改任一处，这里会红。 + */ +class TaskModuleCoverageTest { + + /** 当前线上模块清单（新增模块时同时更新：注册表 + 本清单 + 对应 Handler） */ + private static final Set EXPECTED_MODULES = Set.of( + "PUBLISH", "DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", + "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", + "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA", "SHOP_DATA_CRAWL", "BRAND"); + + @Test + void registryCoversEveryKnownModuleWithLabel() { + assertEquals(EXPECTED_MODULES, TaskModuleRegistry.moduleTypes()); + + Map labels = TaskModuleRegistry.labels(); + assertEquals(EXPECTED_MODULES.size(), labels.size()); + for (String moduleType : EXPECTED_MODULES) { + String label = labels.get(moduleType); + assertNotNull(label, "模块缺少中文名: " + moduleType); + assertTrue(!label.isBlank(), "模块中文名不能为空: " + moduleType); + } + assertEquals("跟价", TaskModuleRegistry.labelOf("PRICE_TRACK")); + assertEquals("UNKNOWN_MODULE", TaskModuleRegistry.labelOf("UNKNOWN_MODULE"), "未知模块回退原值"); + } + + @Test + void resultFileJobWhitelistMatchesRegistry() { + assertEquals(TaskModuleRegistry.resultFileJobModuleTypes(), TaskFileJobConfig.RESULT_FILE_JOB_MODULE_TYPES); + // 纯表格处理模块不产出结果文件作业 + assertTrue(!TaskFileJobConfig.RESULT_FILE_JOB_MODULE_TYPES.contains("DEDUPE")); + assertTrue(!TaskFileJobConfig.RESULT_FILE_JOB_MODULE_TYPES.contains("SPLIT")); + assertTrue(!TaskFileJobConfig.RESULT_FILE_JOB_MODULE_TYPES.contains("CONVERT")); + assertTrue(TaskFileJobConfig.RESULT_FILE_JOB_MODULE_TYPES.contains("SHOP_DATA_CRAWL")); + } + + @Test + void ageCleanupListMatchesRegistry() { + ModuleCleanupProperties properties = new ModuleCleanupProperties(); + + assertEquals(TaskModuleRegistry.ageCleanupModuleTypes(), Set.copyOf(properties.getModuleTypes())); + // 店铺数据采集靠每日工作簿自身保留期，不参与按天清理 + assertTrue(!properties.getModuleTypes().contains("SHOP_DATA_CRAWL")); + assertTrue(properties.getModuleTypes().contains("DEDUPE")); + } + + /** + * 只装配 delegatedStaleChecks 需要的四个业务服务，其余依赖留空—— + * 这样测试聚焦"巡检线登记了哪些模块"，不牵扯任务锁/缓存等无关依赖。 + */ + private static DeleteBrandStaleTaskService serviceForDelegatedChecks() { + return new DeleteBrandStaleTaskService( + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, + mock(BrandTaskService.class), + mock(AppearancePatentTaskService.class), + mock(SimilarAsinTaskService.class), + null, null, null, null, + mock(ShopDataCrawlTaskService.class)); + } + + @Test + void delegatedStaleCheckHandlersCoverRegistryFlag() { + DeleteBrandStaleTaskService service = serviceForDelegatedChecks(); + + Map handlers = service.delegatedStaleChecks(); + + assertEquals(TaskModuleRegistry.delegatedStaleCheckModuleTypes(), handlers.keySet(), + "注册表标记了 delegatedStaleCheck 的模块必须都在巡检线登记动作（反之亦然）"); + for (Runnable action : handlers.values()) { + assertNotNull(action); + } + } + + @Test + void delegatedStaleCheckIncludesModulesThatHadOwnScheduler() { + DeleteBrandStaleTaskService service = serviceForDelegatedChecks(); + + List keys = List.copyOf(service.delegatedStaleChecks().keySet()); + + // 商品管理采集 2026-09 从自带 @Scheduled 并入巡检线（P1-8 双实例判死盲区），不得回退 + assertTrue(keys.contains("SHOP_DATA_CRAWL"), "商品管理采集必须留在 stale-check 巡检线内: " + keys); + assertTrue(keys.contains("SIMILAR_ASIN")); + assertTrue(keys.contains("APPEARANCE_PATENT")); + assertTrue(keys.contains("BRAND")); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlItemRetentionServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlItemRetentionServiceTest.java new file mode 100644 index 00000000..4f82f765 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlItemRetentionServiceTest.java @@ -0,0 +1,84 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service; + +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlItemMapper; +import org.junit.jupiter.api.Test; + +import java.time.LocalDate; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 采集明细表保留期清理（2026-09 全维度审查：该表此前无任何清理策略）。 + */ +class ShopDataCrawlItemRetentionServiceTest { + + private final ShopDataCrawlItemMapper itemMapper = mock(ShopDataCrawlItemMapper.class); + private final DistributedJobLockService jobLockService = mock(DistributedJobLockService.class); + + private ShopDataCrawlItemRetentionService service() { + return new ShopDataCrawlItemRetentionService(itemMapper, jobLockService); + } + + private void lockAvailable() { + when(jobLockService.tryLock(anyString(), any())) + .thenReturn(mock(DistributedJobLockService.LockHandle.class)); + } + + @Test + void deletesInBatchesUntilBatchNotFull() { + lockAvailable(); + when(itemMapper.deleteOlderThanBatch(any(LocalDate.class), anyInt())) + .thenReturn(5000, 5000, 120); + + service().purgeExpiredItems(); + + verify(itemMapper, times(3)).deleteOlderThanBatch(any(LocalDate.class), eq(5000)); + } + + @Test + void stopsAtBatchLimitPerRun() { + lockAvailable(); + when(itemMapper.deleteOlderThanBatch(any(LocalDate.class), anyInt())).thenReturn(5000); + + service().purgeExpiredItems(); + + verify(itemMapper, times(ShopDataCrawlItemRetentionService.MAX_BATCHES_PER_RUN)) + .deleteOlderThanBatch(any(LocalDate.class), anyInt()); + } + + @Test + void skipsWhenAnotherInstanceHoldsLock() { + when(jobLockService.tryLock(anyString(), any())).thenReturn(null); + + service().purgeExpiredItems(); + + verify(itemMapper, never()).deleteOlderThanBatch(any(LocalDate.class), anyInt()); + } + + @Test + void deleteFailureIsLoggedAndDoesNotThrow() { + lockAvailable(); + when(itemMapper.deleteOlderThanBatch(any(LocalDate.class), anyInt())) + .thenThrow(new IllegalStateException("db down")); + + service().purgeExpiredItems(); + + verify(itemMapper).deleteOlderThanBatch(any(LocalDate.class), anyInt()); + } + + @Test + void retentionDaysHasLowerBound() { + ShopDataCrawlItemRetentionService service = service(); + assertEquals(30, service.effectiveRetentionDays(), "默认保留 30 天"); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupKeysetPaginationTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupKeysetPaginationTest.java index 7d9465a0..142f12f4 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupKeysetPaginationTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupKeysetPaginationTest.java @@ -9,7 +9,7 @@ import com.nanri.aiimage.config.InstanceMetadata; import com.nanri.aiimage.config.ModuleCleanupProperties; import com.nanri.aiimage.config.StorageProperties; import com.nanri.aiimage.config.TransientStorageProperties; -import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper; +import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi; import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity; import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; @@ -90,7 +90,7 @@ class ModuleHistoryCleanupKeysetPaginationTest { private TaskResultPayloadMapper taskResultPayloadMapper; private TaskScopeStateMapper taskScopeStateMapper; private TaskChunkMapper taskChunkMapper; - private CollectDataItemMapper collectDataItemMapper; + private CollectDataItemCleanupSpi collectDataItemMapper; private DistributedJobLockService lockService; private TransientPayloadStorageService storage; private RustfsObjectStorageService rustfs; @@ -129,7 +129,7 @@ class ModuleHistoryCleanupKeysetPaginationTest { taskResultPayloadMapper = mock(TaskResultPayloadMapper.class); taskScopeStateMapper = mock(TaskScopeStateMapper.class); taskChunkMapper = mock(TaskChunkMapper.class); - collectDataItemMapper = mock(CollectDataItemMapper.class); + collectDataItemMapper = mock(CollectDataItemCleanupSpi.class); StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class); ValueOperations valueOperations = mock(ValueOperations.class); when(redisTemplate.opsForValue()).thenReturn(valueOperations); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupLoggingTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupLoggingTest.java index 0dce452b..bcf579e3 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupLoggingTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupLoggingTest.java @@ -13,7 +13,7 @@ import com.nanri.aiimage.config.InstanceMetadata; import com.nanri.aiimage.config.ModuleCleanupProperties; import com.nanri.aiimage.config.StorageProperties; import com.nanri.aiimage.config.TransientStorageProperties; -import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper; +import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi; import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity; import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; @@ -92,7 +92,7 @@ class ModuleHistoryCleanupLoggingTest { private TaskResultPayloadMapper taskResultPayloadMapper; private TaskScopeStateMapper taskScopeStateMapper; private TaskChunkMapper taskChunkMapper; - private CollectDataItemMapper collectDataItemMapper; + private CollectDataItemCleanupSpi collectDataItemMapper; private DistributedJobLockService lockService; private TransientPayloadStorageService storage; private RustfsObjectStorageService rustfs; @@ -128,7 +128,7 @@ class ModuleHistoryCleanupLoggingTest { taskResultPayloadMapper = mock(TaskResultPayloadMapper.class); taskScopeStateMapper = mock(TaskScopeStateMapper.class); taskChunkMapper = mock(TaskChunkMapper.class); - collectDataItemMapper = mock(CollectDataItemMapper.class); + collectDataItemMapper = mock(CollectDataItemCleanupSpi.class); StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class); ValueOperations valueOperations = mock(ValueOperations.class); when(redisTemplate.opsForValue()).thenReturn(valueOperations); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupPayloadCleanupTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupPayloadCleanupTest.java index 14ecbc61..d1a2b410 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupPayloadCleanupTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/ModuleHistoryCleanupPayloadCleanupTest.java @@ -9,7 +9,7 @@ import com.nanri.aiimage.config.InstanceMetadata; import com.nanri.aiimage.config.ModuleCleanupProperties; import com.nanri.aiimage.config.StorageProperties; import com.nanri.aiimage.config.TransientStorageProperties; -import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper; +import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi; import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity; import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; @@ -86,7 +86,7 @@ class ModuleHistoryCleanupPayloadCleanupTest { private TaskResultPayloadMapper taskResultPayloadMapper; private TaskScopeStateMapper taskScopeStateMapper; private TaskChunkMapper taskChunkMapper; - private CollectDataItemMapper collectDataItemMapper; + private CollectDataItemCleanupSpi collectDataItemMapper; private DistributedJobLockService lockService; private TransientPayloadStorageService storage; private RustfsObjectStorageService rustfs; @@ -122,7 +122,7 @@ class ModuleHistoryCleanupPayloadCleanupTest { taskResultPayloadMapper = mock(TaskResultPayloadMapper.class); taskScopeStateMapper = mock(TaskScopeStateMapper.class); taskChunkMapper = mock(TaskChunkMapper.class); - collectDataItemMapper = mock(CollectDataItemMapper.class); + collectDataItemMapper = mock(CollectDataItemCleanupSpi.class); StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class); ValueOperations valueOperations = mock(ValueOperations.class); when(redisTemplate.opsForValue()).thenReturn(valueOperations); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatServiceTest.java index 6255266e..06fc6afc 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatServiceTest.java @@ -1,50 +1,48 @@ package com.nanri.aiimage.modules.task.service; +import com.baomidou.mybatisplus.core.MybatisConfiguration; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; -import com.baomidou.mybatisplus.core.MybatisConfiguration; import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; import com.nanri.aiimage.config.SimilarAsinProperties; -import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService; -import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper; -import com.nanri.aiimage.modules.brand.service.BrandTaskProgressCacheService; import com.nanri.aiimage.modules.collectdata.service.CollectDataService; -import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService; -import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService; -import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService; -import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService; +import com.nanri.aiimage.modules.collectdata.service.CollectDataTaskHeartbeatSpi; +import com.nanri.aiimage.modules.publish.service.PublishTaskHeartbeatSpi; import com.nanri.aiimage.modules.publish.service.PublishTaskService; -import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService; import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskHeartbeatSpi; import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService; -import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService; import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService; +import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskHeartbeatSpi; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo; -import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskCacheService; +import com.nanri.aiimage.modules.task.spi.BrandTaskHeartbeatSpi; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; import org.apache.ibatis.builder.MapperBuilderAssistant; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; +import org.junit.jupiter.api.Test; import java.time.LocalDateTime; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.isNull; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; -@ExtendWith(MockitoExtension.class) +/** + * 心跳分发契约（G5 SPI 化后）：task 侧只按 moduleType 路由到 {@link TaskModuleHeartbeatSpi}， + * 各模块的行为由各自的 SPI 实现负责，因此这里同时覆盖"路由正确"与"实现行为正确"。 + */ class TaskHeartbeatServiceTest { @BeforeAll @@ -54,35 +52,30 @@ class TaskHeartbeatServiceTest { FileTaskEntity.class); } - @Mock private FileTaskMapper fileTaskMapper; - @Mock private BrandCrawlTaskMapper brandCrawlTaskMapper; - @Mock private ProductRiskTaskCacheService productRiskTaskCacheService; - @Mock private PublishTaskService publishTaskService; - @Mock private PriceTrackTaskCacheService priceTrackTaskCacheService; - @Mock private ShopMatchTaskCacheService shopMatchTaskCacheService; - @Mock private PatrolDeleteTaskCacheService patrolDeleteTaskCacheService; - @Mock private QueryAsinTaskCacheService queryAsinTaskCacheService; - @Mock private ShopDataCrawlTaskCacheService shopDataCrawlTaskCacheService; - @Mock private ShopDataCrawlTaskService shopDataCrawlTaskService; - @Mock private WithdrawTaskCacheService withdrawTaskCacheService; - @Mock private AppearancePatentTaskCacheService appearancePatentTaskCacheService; - @Mock private SimilarAsinTaskCacheService similarAsinTaskCacheService; - @Mock private SimilarAsinProperties similarAsinProperties; - @Mock private TaskFileJobService taskFileJobService; - @Mock private DeleteBrandTaskCacheService deleteBrandTaskCacheService; - @Mock private BrandTaskProgressCacheService brandTaskProgressCacheService; - @Mock private CollectDataService collectDataService; + private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class); + private final TaskProgressSnapshotService taskProgressSnapshotService = mock(TaskProgressSnapshotService.class); + private final BrandTaskHeartbeatSpi brandTaskHeartbeatSpi = mock(BrandTaskHeartbeatSpi.class); - @InjectMocks private TaskHeartbeatService service; + private TaskHeartbeatService serviceWith(TaskModuleHeartbeatSpi... handlers) { + return new TaskHeartbeatService(fileTaskMapper, taskProgressSnapshotService, brandTaskHeartbeatSpi, + List.of(handlers)); + } + + private static FileTaskEntity runningTask(long taskId, String moduleType) { + FileTaskEntity task = new FileTaskEntity(); + task.setId(taskId); + task.setModuleType(moduleType); + task.setStatus("RUNNING"); + return task; + } @Test @SuppressWarnings("unchecked") void publishHeartbeatTouchesGenericTaskAndModuleProgress() { long taskId = 20142L; - FileTaskEntity task = new FileTaskEntity(); - task.setId(taskId); - task.setModuleType(PublishTaskService.MODULE_TYPE); - task.setStatus("RUNNING"); + FileTaskEntity task = runningTask(taskId, PublishTaskService.MODULE_TYPE); + PublishTaskService publishTaskService = mock(PublishTaskService.class); + TaskHeartbeatService service = serviceWith(new PublishTaskHeartbeatSpi(publishTaskService)); TaskHeartbeatRequest request = new TaskHeartbeatRequest(); request.setPhase("dispatching"); @@ -90,7 +83,6 @@ class TaskHeartbeatServiceTest { request.setTotal(100); when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task); - when(brandCrawlTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null); when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); TaskHeartbeatVo result = service.heartbeat(taskId, request); @@ -99,7 +91,7 @@ class TaskHeartbeatServiceTest { assertEquals(PublishTaskService.MODULE_TYPE, result.getModuleType()); InOrder routingBeforeUpdate = inOrder(publishTaskService, fileTaskMapper); routingBeforeUpdate.verify(publishTaskService) - .ensureTaskOwnedByCurrentInstance(task, "publish task heartbeat"); + .ensureTaskOwnedByCurrentInstance(task, PublishTaskService.MODULE_TYPE + " task heartbeat"); routingBeforeUpdate.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class)); verify(publishTaskService).touchHeartbeat(taskId, request); } @@ -108,10 +100,12 @@ class TaskHeartbeatServiceTest { @SuppressWarnings("unchecked") void shopDataCrawlHeartbeatChecksOwnerBeforeDatabaseUpdate() { long taskId = 20143L; - FileTaskEntity task = new FileTaskEntity(); - task.setId(taskId); - task.setModuleType("SHOP_DATA_CRAWL"); - task.setStatus("RUNNING"); + FileTaskEntity task = runningTask(taskId, "SHOP_DATA_CRAWL"); + ShopDataCrawlTaskService shopDataCrawlTaskService = mock(ShopDataCrawlTaskService.class); + ShopDataCrawlTaskCacheService cacheService = mock(ShopDataCrawlTaskCacheService.class); + TaskHeartbeatService service = serviceWith( + new ShopDataCrawlTaskHeartbeatSpi(cacheService, shopDataCrawlTaskService)); + TaskHeartbeatRequest request = new TaskHeartbeatRequest(); request.setModuleType("SHOP_DATA_CRAWL"); when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task); @@ -122,27 +116,27 @@ class TaskHeartbeatServiceTest { assertTrue(result.isAlive()); InOrder order = inOrder(shopDataCrawlTaskService, fileTaskMapper); order.verify(shopDataCrawlTaskService) - .ensureTaskOwnedByCurrentInstance(task, "shop data crawl task heartbeat"); + .ensureTaskOwnedByCurrentInstance(task, "SHOP_DATA_CRAWL task heartbeat"); order.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class)); - verify(shopDataCrawlTaskCacheService).touchTaskHeartbeat(taskId); - verify(shopDataCrawlTaskCacheService).saveTaskCache(task); - verify(brandCrawlTaskMapper, never()).selectOne(any(LambdaQueryWrapper.class)); + verify(cacheService).touchTaskHeartbeat(taskId); + verify(cacheService).saveTaskCache(task); + // SHOP_DATA_CRAWL 不需要查品牌任务表（历史行为：跳过品牌查找） + verifyNoInteractions(brandTaskHeartbeatSpi); } @Test @SuppressWarnings("unchecked") void collectDataHeartbeatForwardsProcessedKeywordProgress() { long taskId = 21016L; - FileTaskEntity task = new FileTaskEntity(); - task.setId(taskId); - task.setModuleType("COLLECT_DATA"); - task.setStatus("RUNNING"); + FileTaskEntity task = runningTask(taskId, "COLLECT_DATA"); + CollectDataService collectDataService = mock(CollectDataService.class); + TaskHeartbeatService service = serviceWith(new CollectDataTaskHeartbeatSpi(collectDataService)); + TaskHeartbeatRequest request = new TaskHeartbeatRequest(); request.setCurrent(4); request.setTotal(19); when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task); - when(brandCrawlTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null); when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); TaskHeartbeatVo result = service.heartbeat(taskId, request); @@ -155,21 +149,39 @@ class TaskHeartbeatServiceTest { @SuppressWarnings("unchecked") void similarAsinHeartbeatUsesRedisWithoutRefreshingRecentDatabaseCheckpoint() { long taskId = 20581L; - FileTaskEntity task = new FileTaskEntity(); - task.setId(taskId); - task.setModuleType("SIMILAR_ASIN"); - task.setStatus("RUNNING"); + FileTaskEntity task = runningTask(taskId, "SIMILAR_ASIN"); task.setUpdatedAt(LocalDateTime.now()); + SimilarAsinTaskCacheService cacheService = mock(SimilarAsinTaskCacheService.class); + TaskFileJobService taskFileJobService = mock(TaskFileJobService.class); + SimilarAsinProperties properties = mock(SimilarAsinProperties.class); + TaskHeartbeatService service = serviceWith( + new SimilarAsinTaskHeartbeatSpi(cacheService, taskFileJobService, properties)); + when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task); - when(brandCrawlTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null); - when(similarAsinProperties.getDbTaskTouchIntervalMillis()).thenReturn(120000L); + when(properties.getDbTaskTouchIntervalMillis()).thenReturn(120000L); TaskHeartbeatVo result = service.heartbeat(taskId, new TaskHeartbeatRequest()); assertTrue(result.isAlive()); assertEquals("SIMILAR_ASIN", result.getModuleType()); - verify(similarAsinTaskCacheService).touchTaskHeartbeat(taskId); + verify(cacheService).touchTaskHeartbeat(taskId); verify(taskFileJobService).touchRunningAssembleJobsIfStale(taskId, "SIMILAR_ASIN", 120000L); + // 检查点未到期：不写 biz_file_task.updated_at verify(fileTaskMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class)); } + + @Test + void unknownModuleStillHeartbeatsFileTaskWithoutModuleHandler() { + long taskId = 20150L; + FileTaskEntity task = runningTask(taskId, "UNREGISTERED_MODULE"); + TaskHeartbeatService service = serviceWith(); + + when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task); + when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + + TaskHeartbeatVo result = service.heartbeat(taskId, new TaskHeartbeatRequest()); + + assertTrue(result.isAlive(), "未注册 SPI 的模块不应影响通用心跳（保持向后兼容）"); + assertEquals("UNREGISTERED_MODULE", result.getModuleType()); + } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskModuleHeartbeatSpiCoverageTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskModuleHeartbeatSpiCoverageTest.java new file mode 100644 index 00000000..224fd5df --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskModuleHeartbeatSpiCoverageTest.java @@ -0,0 +1,96 @@ +package com.nanri.aiimage.modules.task.service; + +import com.nanri.aiimage.config.SimilarAsinProperties; +import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService; +import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskHeartbeatSpi; +import com.nanri.aiimage.modules.collectdata.service.CollectDataService; +import com.nanri.aiimage.modules.collectdata.service.CollectDataTaskHeartbeatSpi; +import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService; +import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskHeartbeatSpi; +import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService; +import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskHeartbeatSpi; +import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService; +import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskHeartbeatSpi; +import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService; +import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskHeartbeatSpi; +import com.nanri.aiimage.modules.publish.service.PublishTaskHeartbeatSpi; +import com.nanri.aiimage.modules.publish.service.PublishTaskService; +import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService; +import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskHeartbeatSpi; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskHeartbeatSpi; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService; +import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService; +import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskHeartbeatSpi; +import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService; +import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskHeartbeatSpi; +import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi; +import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskCacheService; +import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskHeartbeatSpi; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * 模块心跳 SPI 的覆盖面自检（2026-09 全维度审查 G5）。 + * + *

心跳按 moduleType 路由，moduleType 字符串写错会让该模块的心跳静默失效 + * （任务被当成无心跳 → 30 分钟后误判失败）。这里把"哪个实现负责哪个 moduleType" + * 钉死：新增模块漏实现或写错字符串都会红测试。 + */ +class TaskModuleHeartbeatSpiCoverageTest { + + private static Map implementationsByModuleType() { + Map impls = new LinkedHashMap<>(); + register(impls, new AppearancePatentTaskHeartbeatSpi(mock(AppearancePatentTaskCacheService.class))); + register(impls, new CollectDataTaskHeartbeatSpi(mock(CollectDataService.class))); + register(impls, new DeleteBrandTaskHeartbeatSpi(mock(DeleteBrandTaskCacheService.class))); + register(impls, new PatrolDeleteTaskHeartbeatSpi(mock(PatrolDeleteTaskCacheService.class))); + register(impls, new PriceTrackTaskHeartbeatSpi(mock(PriceTrackTaskCacheService.class))); + register(impls, new ProductRiskTaskHeartbeatSpi(mock(ProductRiskTaskCacheService.class))); + register(impls, new PublishTaskHeartbeatSpi(mock(PublishTaskService.class))); + register(impls, new QueryAsinTaskHeartbeatSpi(mock(QueryAsinTaskCacheService.class))); + register(impls, new ShopDataCrawlTaskHeartbeatSpi( + mock(ShopDataCrawlTaskCacheService.class), mock(ShopDataCrawlTaskService.class))); + register(impls, new ShopMatchTaskHeartbeatSpi(mock(ShopMatchTaskCacheService.class))); + register(impls, new SimilarAsinTaskHeartbeatSpi( + mock(SimilarAsinTaskCacheService.class), mock(TaskFileJobService.class), + mock(SimilarAsinProperties.class))); + register(impls, new WithdrawTaskHeartbeatSpi(mock(WithdrawTaskCacheService.class))); + return impls; + } + + private static void register(Map impls, TaskModuleHeartbeatSpi impl) { + TaskModuleHeartbeatSpi exists = impls.put(impl.moduleType(), impl); + if (exists != null) { + throw new IllegalStateException("moduleType=" + impl.moduleType() + " 重复实现: " + + exists.getClass().getName() + " / " + impl.getClass().getName()); + } + } + + @Test + void everyHeartbeatModuleHasImplementationWithExactModuleType() { + Map impls = implementationsByModuleType(); + + assertEquals(12, impls.size(), "有任务心跳的模块都应有 SPI 实现，实际 " + impls.keySet()); + for (String moduleType : new String[]{ + "APPEARANCE_PATENT", "COLLECT_DATA", "DELETE_BRAND", "PATROL_DELETE", "PRICE_TRACK", + "PRODUCT_RISK_RESOLVE", "PUBLISH", "QUERY_ASIN", "SHOP_DATA_CRAWL", "SHOP_MATCH", + "SIMILAR_ASIN", "WITHDRAW"}) { + assertEquals(true, impls.containsKey(moduleType), "缺少心跳实现: " + moduleType); + } + } + + @Test + void moduleTypesAreUppercaseAndUnique() { + for (TaskModuleHeartbeatSpi impl : implementationsByModuleType().values()) { + String moduleType = impl.moduleType(); + assertEquals(moduleType.toUpperCase(), moduleType, + "moduleType 必须与 biz_file_task.module_type 一致（全大写）: " + moduleType); + } + } +}