refactor+perf+fix: G5 模块边界 SPI 化、C5 流式解析、C6/C7 查询优化、D13 队列持久化、A3/A4/A7 鉴权
模块边界(G5 / G7) - 新增 task/spi/TaskModuleHeartbeatSpi + 13 个模块实现:TaskHeartbeatService 不再 import 任何 业务模块(原先注入 12 个 CacheService 并用 switch 分发);启动校验重复注册 - 新增 task/spi/BrandTaskHeartbeatSpi(品牌任务心跳/中断)、BrandTaskStaleRepairSpi(陈旧修复)、 CollectDataItemCleanupSpi(历史清理):跨模块 Mapper 操作收回业务模块 - G7:10 个被跨模块借用的 productrisk VO 迁至 common/model/vo - 架构棘轮收紧:TASK_TO_BUSINESS_BASELINE 119 → 6(实测) - 新增 TaskModuleHeartbeatSpiCoverageTest(moduleType 覆盖与拼写) 性能与容量(C5/C6/C7/D13) - C5 流式解析:SkipPriceAsinService(含两行表头语义)、AppearancePatentExcelParser、 BrandTaskService、DeleteBrandRunService、LocalFileStorageService.getExcelInfo 改 ExcelStreamReader; 行数上限改为迭代中生效 - C6 去重总数据列表:关键字改前缀匹配(命中 uk_data_value);V124 删除永不生效的 idx_country - C7 撞款扫描:按店逐批取数(索引前缀),不再全表 GROUP BY + JOIN + 全量拉内存 - D13 待删对象本地日志 PendingDeleteJournal(启动回放 + 收敛重写),RustfsDeleteRetryService 与 TransientPayloadDeleteOrchestrator 接入;异步删除失败对象写回日志 - V123 删除 biz_file_result 两个被复合索引覆盖的单列索引 安全(A3/A4/A7 + 守卫名单) - A4 数字人版本写操作要求管理员;A7 视频密钥按登录身份(超管例外) - A3 上传接口加危险扩展名黑名单(可配置) - AdminApiGuardFilter 用户态名单补 /api/ziniao(controller 已 requireAdmin,此处为开关打开后的第二层) 容量(明细表保留期) - 新增 ShopDataCrawlItemRetentionService:biz_shop_data_crawl_item 按保留期(默认 30 天)分批清理 (该表此前无任何清理策略,是增长最快的表),job 锁 + 单轮批次上限
This commit is contained in:
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.nanri.aiimage.modules.productrisk.model.vo;
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.nanri.aiimage.modules.productrisk.model.vo;
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.nanri.aiimage.modules.productrisk.model.vo;
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.nanri.aiimage.modules.productrisk.model.vo;
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.nanri.aiimage.modules.productrisk.model.vo;
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.nanri.aiimage.modules.productrisk.model.vo;
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.nanri.aiimage.modules.productrisk.model.vo;
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.nanri.aiimage.modules.productrisk.model.vo;
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.nanri.aiimage.modules.productrisk.model.vo;
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.nanri.aiimage.modules.productrisk.model.vo;
|
||||
package com.nanri.aiimage.common.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.nanri.aiimage.modules.shopmatch.model.vo.ShopMatchTaskStageVo;
|
||||
@@ -0,0 +1,111 @@
|
||||
package com.nanri.aiimage.common.module;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 任务模块清单的**单一来源**(2026-09 全维度审查 G6)。
|
||||
*
|
||||
* 背景:新增一个工具模块此前要同时改 5 处枚举(结果文件 Job 白名单、按天清理清单、
|
||||
* 站内通知模块名、任务心跳分支、陈旧判死巡检线),其中 4 处没有任何自检,
|
||||
* 漏改表现为"功能静默不生效"。现在把**数据驱动的三处**(结果文件 Job、按天清理、通知中文名)
|
||||
* 统一从这里派生,并由 {@code TaskModuleCoverageTest} 守住覆盖面。
|
||||
*
|
||||
* 心跳与陈旧判死两处是代码分支驱动(switch / 逐模块委派),无法纯数据派生,
|
||||
* 新增模块时仍需实现对应 Handler,测试会提示缺失。
|
||||
*/
|
||||
public final class TaskModuleRegistry {
|
||||
|
||||
/**
|
||||
* 单个任务模块的声明。
|
||||
*
|
||||
* @param type moduleType(与 biz_file_task.module_type 一致)
|
||||
* @param label 站内通知/界面用中文名
|
||||
* @param resultFileJob 是否产出结果文件(需要注册 ResultFileJobHandler)
|
||||
* @param ageCleanup 是否参与按天清理(ModuleCleanupProperties)
|
||||
* @param delegatedStaleCheck 是否以"委派"方式并入 stale-check 巡检线
|
||||
* (DeleteBrandStaleTaskService.delegatedStaleChecks,必须逐模块登记动作)
|
||||
* @param selfScheduledStaleCheck 该模块的陈旧判死自带更快的调度(publish 60s / collect-data 30s),
|
||||
* 刻意不并入 2 分钟一轮的巡检线——并入会显著拉长判死时延。
|
||||
* 这类模块的"不在委托名单"是设计差异,不是漏接;由覆盖面测试固定。
|
||||
*/
|
||||
public record Module(String type, String label, boolean resultFileJob, boolean ageCleanup,
|
||||
boolean delegatedStaleCheck, boolean selfScheduledStaleCheck) {
|
||||
}
|
||||
|
||||
private static final List<Module> MODULES = List.of(
|
||||
new Module("PUBLISH", "上架", true, false, false, true),
|
||||
new Module("DEDUPE", "数据去重", false, true, false, false),
|
||||
new Module("SPLIT", "数据拆分", false, true, false, false),
|
||||
new Module("CONVERT", "格式转换", false, true, false, false),
|
||||
new Module("DELETE_BRAND", "删除ASIN", true, true, false, false),
|
||||
new Module("PRODUCT_RISK_RESOLVE", "商品风险解决", true, true, false, false),
|
||||
new Module("PRICE_TRACK", "跟价", true, true, false, false),
|
||||
new Module("SHOP_MATCH", "定时匹配", true, true, false, false),
|
||||
new Module("PATROL_DELETE", "巡店删除", true, true, false, false),
|
||||
new Module("QUERY_ASIN", "查询ASIN", true, true, false, false),
|
||||
new Module("WITHDRAW", "取款", true, true, false, false),
|
||||
new Module("APPEARANCE_PATENT", "外观专利检测", true, true, true, false),
|
||||
new Module("SIMILAR_ASIN", "货源查询", true, true, true, false),
|
||||
new Module("COLLECT_DATA", "采集数据", true, true, false, true),
|
||||
new Module("SHOP_DATA_CRAWL", "店铺数据抓取", true, false, true, false),
|
||||
new Module("BRAND", "品牌检测", true, false, true, false)
|
||||
);
|
||||
|
||||
private TaskModuleRegistry() {
|
||||
}
|
||||
|
||||
public static List<Module> modules() {
|
||||
return MODULES;
|
||||
}
|
||||
|
||||
/** 全部 moduleType。 */
|
||||
public static Set<String> moduleTypes() {
|
||||
return MODULES.stream().map(Module::type).collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
/** moduleType → 中文名(站内通知等展示用)。 */
|
||||
public static Map<String, String> labels() {
|
||||
Map<String, String> labels = new LinkedHashMap<>();
|
||||
for (Module module : MODULES) {
|
||||
labels.put(module.type(), module.label());
|
||||
}
|
||||
return Map.copyOf(labels);
|
||||
}
|
||||
|
||||
/** 产出结果文件的模块(需要注册 Handler 的模块)。 */
|
||||
public static Set<String> resultFileJobModuleTypes() {
|
||||
return MODULES.stream().filter(Module::resultFileJob).map(Module::type)
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
/** 参与按天清理的模块。 */
|
||||
public static Set<String> ageCleanupModuleTypes() {
|
||||
return MODULES.stream().filter(Module::ageCleanup).map(Module::type)
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
/** 自带更快巡检节奏、刻意不并入集中巡检线的模块。 */
|
||||
public static Set<String> selfScheduledStaleCheckModuleTypes() {
|
||||
return MODULES.stream().filter(Module::selfScheduledStaleCheck).map(Module::type)
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
/** 以"委派"方式并入 stale-check 巡检线的模块(DeleteBrandStaleTaskService 必须逐模块登记动作)。 */
|
||||
public static Set<String> delegatedStaleCheckModuleTypes() {
|
||||
return MODULES.stream().filter(Module::delegatedStaleCheck).map(Module::type)
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
public static String labelOf(String moduleType) {
|
||||
for (Module module : MODULES) {
|
||||
if (module.type().equals(moduleType)) {
|
||||
return module.label();
|
||||
}
|
||||
}
|
||||
return moduleType;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package com.nanri.aiimage.common.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* 跨节点的小容量状态存储(2026-09 全维度审查 D2)。
|
||||
*
|
||||
* <p>用途:导入/导出任务的进度、归属用户、分组等状态。原先只存节点本地内存,
|
||||
* 客户端轮询落到另一节点就报"任务不存在"(nginx 的 user_id 亲和只覆盖常态)。
|
||||
* 现在本地 Map 作快路径、Redis 作跨节点真源,任何节点都能读到。
|
||||
*
|
||||
* <p>写节流:调用方可能**逐行**刷新进度(数十万行),逐次写 Redis 不可接受;
|
||||
* 默认 500ms 内只写一次(终态用 {@link #putNow} 立即落库)。
|
||||
*
|
||||
* <p>语义取舍:读-改-写不保证原子(进度类状态可接受);{@code redis}/{@code objectMapper}
|
||||
* 为空时退化为纯本地(单测与未注入场景)。
|
||||
*/
|
||||
@Slf4j
|
||||
public final class NodeSharedStore<K, V> {
|
||||
|
||||
private static final long DEFAULT_WRITE_THROTTLE_MILLIS = 500L;
|
||||
|
||||
private final String keyPrefix;
|
||||
private final Duration ttl;
|
||||
private final Class<V> valueType;
|
||||
private final StringRedisTemplate redis;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final long writeThrottleMillis;
|
||||
|
||||
private final ConcurrentHashMap<K, V> local = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<K, AtomicLong> lastWriteAt = new ConcurrentHashMap<>();
|
||||
|
||||
public NodeSharedStore(String keyPrefix,
|
||||
Duration ttl,
|
||||
Class<V> valueType,
|
||||
StringRedisTemplate redis,
|
||||
ObjectMapper objectMapper) {
|
||||
this(keyPrefix, ttl, valueType, redis, objectMapper, DEFAULT_WRITE_THROTTLE_MILLIS);
|
||||
}
|
||||
|
||||
public NodeSharedStore(String keyPrefix,
|
||||
Duration ttl,
|
||||
Class<V> valueType,
|
||||
StringRedisTemplate redis,
|
||||
ObjectMapper objectMapper,
|
||||
long writeThrottleMillis) {
|
||||
this.keyPrefix = keyPrefix;
|
||||
this.ttl = ttl;
|
||||
this.valueType = valueType;
|
||||
this.redis = redis;
|
||||
this.objectMapper = objectMapper;
|
||||
this.writeThrottleMillis = Math.max(0L, writeThrottleMillis);
|
||||
}
|
||||
|
||||
/** 本地优先;本地没有则读 Redis 并回填本地(跨节点可见)。 */
|
||||
public V get(K key) {
|
||||
if (key == null) {
|
||||
return null;
|
||||
}
|
||||
V cached = local.get(key);
|
||||
if (cached != null) {
|
||||
return cached;
|
||||
}
|
||||
V remote = readRemote(key);
|
||||
if (remote != null) {
|
||||
local.put(key, remote);
|
||||
}
|
||||
return remote;
|
||||
}
|
||||
|
||||
public boolean containsKey(K key) {
|
||||
return get(key) != null;
|
||||
}
|
||||
|
||||
/** 写入并(按节流)同步到 Redis。 */
|
||||
public void put(K key, V value) {
|
||||
if (key == null || value == null) {
|
||||
return;
|
||||
}
|
||||
local.put(key, value);
|
||||
if (!throttleAllowsWrite(key)) {
|
||||
return;
|
||||
}
|
||||
writeRemote(key, value);
|
||||
}
|
||||
|
||||
/** 立即写入 Redis(终态、归属等一次性状态用)。 */
|
||||
public void putNow(K key, V value) {
|
||||
if (key == null || value == null) {
|
||||
return;
|
||||
}
|
||||
local.put(key, value);
|
||||
writeRemote(key, value);
|
||||
}
|
||||
|
||||
/** 删除(本地 + Redis),返回删除前的值(可能为 null)。 */
|
||||
public V remove(K key) {
|
||||
if (key == null) {
|
||||
return null;
|
||||
}
|
||||
V previous = local.remove(key);
|
||||
lastWriteAt.remove(key);
|
||||
if (redis != null) {
|
||||
try {
|
||||
redis.delete(fullKey(key));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[node-shared-store] 删除远端状态失败 key={} msg={}", fullKey(key), ex.getMessage());
|
||||
}
|
||||
}
|
||||
return previous;
|
||||
}
|
||||
|
||||
/** 本地快照(仅用于日志/统计;不含其它节点的写入)。 */
|
||||
public int localSize() {
|
||||
return local.size();
|
||||
}
|
||||
|
||||
private boolean throttleAllowsWrite(K key) {
|
||||
if (writeThrottleMillis <= 0L) {
|
||||
return true;
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
AtomicLong last = lastWriteAt.computeIfAbsent(key, ignored -> new AtomicLong(0L));
|
||||
long previous = last.get();
|
||||
if (now - previous < writeThrottleMillis) {
|
||||
return false;
|
||||
}
|
||||
return last.compareAndSet(previous, now);
|
||||
}
|
||||
|
||||
private V readRemote(K key) {
|
||||
if (redis == null || objectMapper == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
String json = redis.opsForValue().get(fullKey(key));
|
||||
if (json == null || json.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return objectMapper.readValue(json, valueType);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[node-shared-store] 读取远端状态失败 key={} msg={}", fullKey(key), ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void writeRemote(K key, V value) {
|
||||
if (redis == null || objectMapper == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
String json = objectMapper.writeValueAsString(value);
|
||||
redis.opsForValue().set(fullKey(key), json, ttl);
|
||||
} catch (Exception ex) {
|
||||
// 写失败不影响本地进度(下次 put 会重试),只留线索
|
||||
log.warn("[node-shared-store] 写入远端状态失败 key={} msg={}", fullKey(key), ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private String fullKey(K key) {
|
||||
return keyPrefix + ":" + key;
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,13 @@ public final class ExcelStreamReader {
|
||||
default void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) throws Exception {
|
||||
}
|
||||
|
||||
/**
|
||||
* 表头之后回调本次 sheet 的近似总行数(EasyExcel 基于 sheet 尺寸,可能为 null)。
|
||||
* 流式解析无法在读完前得到精确行数,需要展示进度/做前置上限校验的调用方可用它近似。
|
||||
*/
|
||||
default void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) throws Exception {
|
||||
}
|
||||
|
||||
void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception;
|
||||
}
|
||||
|
||||
@@ -77,6 +84,7 @@ public final class ExcelStreamReader {
|
||||
currentHeaderMap = normalizedHeadMap;
|
||||
try {
|
||||
handler.onHeader(sheetName(context), sheetNo(context), currentHeaderMap);
|
||||
handler.onSheetTotal(sheetName(context), sheetNo(context), approximateTotalRows(context));
|
||||
} catch (RuntimeException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
@@ -109,6 +117,15 @@ public final class ExcelStreamReader {
|
||||
public void doAfterAllAnalysed(AnalysisContext context) {
|
||||
}
|
||||
|
||||
private Integer approximateTotalRows(AnalysisContext context) {
|
||||
try {
|
||||
return context.readSheetHolder() == null ? null
|
||||
: context.readSheetHolder().getApproximateTotalRowNumber();
|
||||
} catch (Exception ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String sheetName(AnalysisContext context) {
|
||||
return context.readSheetHolder() == null ? "" : context.readSheetHolder().getSheetName();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package com.nanri.aiimage.common.util;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 待删对象队列的本地落盘日志:进程重启后仍能恢复「待删除对象」清单。
|
||||
*
|
||||
* <p>使用场景:对象存储删除失败后的补偿队列原本只在节点内存里,重启即丢,
|
||||
* 对应对象会一直残留在桶里直到生命周期规则过期。这里用一行一个 objectKey 的
|
||||
* 追加日志做最小持久化——入队追加、队列收敛后整体重写、启动时回放。
|
||||
*
|
||||
* <p>并发:所有方法内部同步,调用方无需额外加锁;文件损坏/读写异常只记日志不抛出,
|
||||
* 保证补偿链路本身不会因为落盘失败而中断业务。
|
||||
*
|
||||
* <p>上限:{@code maxEntries} 用于防止日志在异常堆积时无界增长(超出后丢弃最早的记录,
|
||||
* 与内存队列的容量准入语义一致——丢的是「待删对象」,最坏结果是对象残留)。
|
||||
*/
|
||||
@Slf4j
|
||||
public final class PendingDeleteJournal {
|
||||
|
||||
private final Path path;
|
||||
private final int maxEntries;
|
||||
private final Object lock = new Object();
|
||||
|
||||
public PendingDeleteJournal(Path path, int maxEntries) {
|
||||
this.path = path;
|
||||
this.maxEntries = Math.max(1, maxEntries);
|
||||
}
|
||||
|
||||
public Path getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
/** 追加一条待删对象(重复追加由回放时的 Set 语义去重)。 */
|
||||
public void record(String objectKey) {
|
||||
if (objectKey == null || objectKey.isBlank()) {
|
||||
return;
|
||||
}
|
||||
synchronized (lock) {
|
||||
try {
|
||||
Path parent = path.getParent();
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
Files.writeString(path, sanitize(objectKey) + System.lineSeparator(),
|
||||
StandardCharsets.UTF_8, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[delete-journal] 追加待删对象失败 path={} msg={}", path, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 启动回放:返回日志中的待删对象(按首次出现顺序去重)。 */
|
||||
public List<String> readAll() {
|
||||
synchronized (lock) {
|
||||
if (!Files.isRegularFile(path)) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> lines;
|
||||
try {
|
||||
lines = Files.readAllLines(path, StandardCharsets.UTF_8);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[delete-journal] 读取待删对象日志失败 path={} msg={}", path, ex.getMessage());
|
||||
return List.of();
|
||||
}
|
||||
Set<String> unique = new LinkedHashSet<>();
|
||||
for (String line : lines) {
|
||||
if (line == null) {
|
||||
continue;
|
||||
}
|
||||
String key = line.trim();
|
||||
if (!key.isEmpty()) {
|
||||
unique.add(key);
|
||||
}
|
||||
}
|
||||
return new ArrayList<>(unique);
|
||||
}
|
||||
}
|
||||
|
||||
/** 队列收敛(成功删除/引用仍在)后整体重写为剩余的待删对象;剩余为空则删除日志。 */
|
||||
public void rewrite(Collection<String> remaining) {
|
||||
synchronized (lock) {
|
||||
Set<String> keep = new LinkedHashSet<>();
|
||||
if (remaining != null) {
|
||||
for (String key : remaining) {
|
||||
if (key != null && !key.isBlank()) {
|
||||
keep.add(sanitize(key));
|
||||
if (keep.size() >= maxEntries) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (keep.isEmpty()) {
|
||||
Files.deleteIfExists(path);
|
||||
return;
|
||||
}
|
||||
Path parent = path.getParent();
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
Path tmp = path.resolveSibling(path.getFileName() + ".tmp");
|
||||
StringBuilder content = new StringBuilder();
|
||||
for (String key : keep) {
|
||||
content.append(key).append(System.lineSeparator());
|
||||
}
|
||||
Files.writeString(tmp, content.toString(), StandardCharsets.UTF_8,
|
||||
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
|
||||
try {
|
||||
Files.move(tmp, path, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
|
||||
} catch (IOException atomicUnsupported) {
|
||||
// 少数文件系统不支持 ATOMIC_MOVE,退化为普通替换
|
||||
Files.move(tmp, path, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[delete-journal] 重写待删对象日志失败 path={} msg={}", path, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 单行一条记录:去掉换行避免破坏行结构。 */
|
||||
private static String sanitize(String objectKey) {
|
||||
return objectKey.replace('\n', ' ').replace('\r', ' ').trim();
|
||||
}
|
||||
}
|
||||
@@ -79,6 +79,9 @@ public class AdminApiGuardFilter extends OncePerRequestFilter {
|
||||
"/api/shop-data-crawl",
|
||||
"/api/withdraw",
|
||||
"/api/task-file-jobs",
|
||||
// 2026-09 复核补:该前缀全部接口已在 controller 内 requireAdmin(含可换取员工店铺登录
|
||||
// 令牌的 /shops/open),此处纳入名单是"开关打开后的第二层";开关关闭时行为不变
|
||||
"/api/ziniao",
|
||||
// /api/tasks/{taskId}/interrupted 仅凭 taskId 即可把 RUNNING 任务置为 FAILED,
|
||||
// 匿名遍历 taskId 就能批量打断线上任务
|
||||
"/api/tasks",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import com.nanri.aiimage.common.module.TaskModuleRegistry;
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@@ -19,5 +20,6 @@ public class ModuleCleanupProperties {
|
||||
private int batchSize = 500;
|
||||
// SHOP_DATA_CRAWL keeps one per-shop daily workbook in its task service
|
||||
// and must not be removed by the age-based sweep.
|
||||
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
|
||||
/** 参与按天清理的模块:取自模块注册表(G6)。 */
|
||||
private List<String> moduleTypes = new ArrayList<>(TaskModuleRegistry.ageCleanupModuleTypes());
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ import io.micrometer.core.instrument.MeterRegistry;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import com.nanri.aiimage.common.module.TaskModuleRegistry;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.core.task.TaskExecutor;
|
||||
@@ -52,11 +53,11 @@ import java.util.concurrent.Semaphore;
|
||||
public class TaskFileJobConfig {
|
||||
|
||||
/** 结果文件 Job 支持的全部 moduleType(启动校验枚举源,见 ResultFileJobHandlerRegistry.validateCoverage) */
|
||||
public static final Set<String> RESULT_FILE_JOB_MODULE_TYPES = Set.of(
|
||||
"SHOP_MATCH", "PRICE_TRACK", "PRODUCT_RISK_RESOLVE",
|
||||
"PUBLISH", "QUERY_ASIN", "SHOP_DATA_CRAWL", "WITHDRAW",
|
||||
"PATROL_DELETE", "APPEARANCE_PATENT", "SIMILAR_ASIN",
|
||||
"DELETE_BRAND", "BRAND", "COLLECT_DATA");
|
||||
/**
|
||||
* 结果文件 Job 支持的 moduleType:取自模块注册表(G6),
|
||||
* 与 {@code ResultFileJobHandlerRegistry.validateCoverage} 的启动自检配合使用。
|
||||
*/
|
||||
public static final Set<String> RESULT_FILE_JOB_MODULE_TYPES = TaskModuleRegistry.resultFileJobModuleTypes();
|
||||
|
||||
@Bean("taskFileJobDispatchExecutor")
|
||||
public TaskExecutor taskFileJobDispatchExecutor(
|
||||
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 外观专利检测 的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>心跳逻辑(缓存刷新 / 任务缓存回写)从 task 模块收回本模块,task 侧只依赖 SPI 接口,
|
||||
* 消除 task → 业务模块的编译期依赖。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class AppearancePatentTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final AppearancePatentTaskCacheService cacheService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return "APPEARANCE_PATENT";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
cacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
}
|
||||
+141
-75
@@ -1,17 +1,17 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service.support;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.PushbackInputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
@@ -37,9 +37,8 @@ public class AppearancePatentExcelParser {
|
||||
if (input == null) {
|
||||
throw new IllegalArgumentException("input must not be null");
|
||||
}
|
||||
try (FileInputStream fis = new FileInputStream(input);
|
||||
Workbook workbook = WorkbookFactory.create(fis)) {
|
||||
return parseWorkbook(workbook, maxRows);
|
||||
try (InputStream inputStream = new FileInputStream(input)) {
|
||||
return readStreaming(inputStream, maxRows);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
@@ -52,8 +51,8 @@ public class AppearancePatentExcelParser {
|
||||
if (input == null) {
|
||||
throw new IllegalArgumentException("input must not be null");
|
||||
}
|
||||
try (Workbook workbook = WorkbookFactory.create(input)) {
|
||||
return parseWorkbook(workbook, DEFAULT_MAX_ROWS);
|
||||
try {
|
||||
return readStreaming(input, DEFAULT_MAX_ROWS);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
@@ -62,83 +61,154 @@ public class AppearancePatentExcelParser {
|
||||
}
|
||||
}
|
||||
|
||||
private ParsedSheet parseWorkbook(Workbook workbook, int maxRows) {
|
||||
int safeMaxRows = Math.max(1, maxRows);
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row header = sheet.getRow(0);
|
||||
if (header == null) {
|
||||
/**
|
||||
* 流式解析(EasyExcel SAX),替代 POI WorkbookFactory 全量 DOM 加载:
|
||||
* 大表不再整表驻留堆内存,行数上限在迭代过程中即时生效(超限抛错)。
|
||||
* 语义与原 POI 路径一致:cell 归一化、表头别名匹配、空行跳过、必填表头缺失抛错、无字段截断。
|
||||
* 注意不关闭传入的 InputStream(由调用方负责)。
|
||||
*/
|
||||
private ParsedSheet readStreaming(InputStream inputStream, int maxRows) throws Exception {
|
||||
// EasyExcel 会把非 zip 文本当 CSV 解析成功;原 WorkbookFactory 只认 xlsx/xls,
|
||||
// 这里先做文件魔数校验,保持「垃圾文件→解析 Excel 失败」的语义并拒绝 CSV 误解析。
|
||||
PushbackInputStream pb = new PushbackInputStream(new BufferedInputStream(inputStream, 8192), 8);
|
||||
requireExcelMagic(pb);
|
||||
SheetContext ctx = new SheetContext(Math.max(1, maxRows));
|
||||
ExcelStreamReader.readFirstSheet(pb, new ExcelStreamReader.SheetRowHandler() {
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
ctx.initHeader(headerMap);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
ctx.consumeRow(rowIndex, rowMap);
|
||||
}
|
||||
});
|
||||
if (ctx.headers == null) {
|
||||
// 空表 / 表头行整行为空(EasyExcel 不上报表头回调):与旧实现 header == null 一致
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
|
||||
List<String> headers = readHeaders(header, formatter);
|
||||
int idCol = findRequiredHeader(headerMap, "id");
|
||||
int asinCol = findRequiredHeader(headerMap, "asin");
|
||||
int countryCol = findRequiredHeader(headerMap, "国家", "country");
|
||||
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
|
||||
int skuCol = findOptionalHeaderExact(headerMap,
|
||||
"sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku");
|
||||
int urlCol = findOptionalHeaderExact(headerMap,
|
||||
"url", "rul", "link", "image", "img", "pic", "picture",
|
||||
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
|
||||
int titleCol = findOptionalHeaderExact(headerMap,
|
||||
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
|
||||
|
||||
List<AppearanceExcelRow> rows = new ArrayList<>();
|
||||
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
|
||||
Row row = sheet.getRow(i);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
String id = cell(row, idCol, formatter);
|
||||
String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT);
|
||||
String country = cell(row, countryCol, formatter);
|
||||
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
if (rows.size() >= safeMaxRows) {
|
||||
throw new BusinessException("解析行数超过上限: " + safeMaxRows);
|
||||
}
|
||||
rows.add(new AppearanceExcelRow(
|
||||
i + 1,
|
||||
id,
|
||||
asin,
|
||||
country,
|
||||
priceCol >= 0 ? cell(row, priceCol, formatter) : "",
|
||||
skuCol >= 0 ? cell(row, skuCol, formatter) : "",
|
||||
urlCol >= 0 ? cell(row, urlCol, formatter) : "",
|
||||
titleCol >= 0 ? cell(row, titleCol, formatter) : "",
|
||||
readRowValues(row, headers, formatter)));
|
||||
}
|
||||
return new ParsedSheet(headers, rows);
|
||||
return new ParsedSheet(ctx.headers, ctx.rows);
|
||||
}
|
||||
|
||||
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
|
||||
/** 校验文件头魔数:xlsx=PK(zip)、xls=OLE2(CFD)。判非抛「解析 Excel 失败」;用 unread 回退已读字节。 */
|
||||
private void requireExcelMagic(PushbackInputStream in) throws IOException {
|
||||
byte[] head = new byte[8];
|
||||
int n = 0;
|
||||
while (n < head.length) {
|
||||
int r = in.read(head, n, head.length - n);
|
||||
if (r < 0) {
|
||||
break;
|
||||
}
|
||||
n += r;
|
||||
}
|
||||
if (n > 0) {
|
||||
in.unread(head, 0, n);
|
||||
}
|
||||
boolean isZip = n >= 4 && head[0] == 'P' && head[1] == 'K' && (head[2] == 3 || head[2] == 5 || head[2] == 7);
|
||||
boolean isOle2 = n >= 8
|
||||
&& (head[0] & 0xFF) == 0xD0 && (head[1] & 0xFF) == 0xCF
|
||||
&& (head[2] & 0xFF) == 0x11 && (head[3] & 0xFF) == 0xE0;
|
||||
if (!isZip && !isOle2) {
|
||||
log.warn("[appearance-patent] parse rejected non-excel magic head={}", Arrays.copyOf(head, Math.max(n, 0)));
|
||||
throw new BusinessException("解析 Excel 失败");
|
||||
}
|
||||
}
|
||||
|
||||
/** 单表解析上下文:表头就绪后逐行累积结果行。 */
|
||||
private final class SheetContext {
|
||||
|
||||
private final int maxRows;
|
||||
private final List<AppearanceExcelRow> rows = new ArrayList<>();
|
||||
private List<String> headers;
|
||||
private int idCol;
|
||||
private int asinCol;
|
||||
private int countryCol;
|
||||
private int priceCol;
|
||||
private int skuCol;
|
||||
private int urlCol;
|
||||
private int titleCol;
|
||||
|
||||
SheetContext(int maxRows) {
|
||||
this.maxRows = maxRows;
|
||||
}
|
||||
|
||||
void initHeader(Map<Integer, String> rawHeaderMap) {
|
||||
// EasyExcel 回调给出 列号 → 表头文本,与旧 Row 遍历等价(缺列一般为 null/空串)
|
||||
int lastColumnCount = lastColumnCount(rawHeaderMap);
|
||||
Map<String, Integer> map = new LinkedHashMap<>();
|
||||
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||
List<String> headerNames = new ArrayList<>();
|
||||
for (int i = 0; i < lastColumnCount; i++) {
|
||||
String val = normalize(rawHeaderMap.getOrDefault(i, ""));
|
||||
headerNames.add(val.isBlank() ? "列" + (i + 1) : val);
|
||||
if (!val.isBlank()) {
|
||||
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
this.headers = headerNames;
|
||||
this.idCol = findRequiredHeader(map, "id");
|
||||
this.asinCol = findRequiredHeader(map, "asin");
|
||||
this.countryCol = findRequiredHeader(map, "国家", "country");
|
||||
this.priceCol = findOptionalHeaderExact(map, "价格", "price");
|
||||
this.skuCol = findOptionalHeaderExact(map,
|
||||
"sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku");
|
||||
this.urlCol = findOptionalHeaderExact(map,
|
||||
"url", "rul", "link", "image", "img", "pic", "picture",
|
||||
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
|
||||
this.titleCol = findOptionalHeaderExact(map,
|
||||
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
|
||||
}
|
||||
|
||||
private List<String> readHeaders(Row header, DataFormatter formatter) {
|
||||
List<String> headers = new ArrayList<>();
|
||||
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||
headers.add(val.isBlank() ? "列" + (i + 1) : val);
|
||||
void consumeRow(int rowIndex, Map<Integer, String> rowMap) {
|
||||
if (headers == null) {
|
||||
return;
|
||||
}
|
||||
return headers;
|
||||
// EasyExcel rowIndex 从 0 起(0 为表头),POI 原实现行号同样 0 起并 +1 展示
|
||||
String id = streamCell(rowMap, idCol);
|
||||
String asin = streamCell(rowMap, asinCol).toUpperCase(Locale.ROOT);
|
||||
String country = streamCell(rowMap, countryCol);
|
||||
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
|
||||
return;
|
||||
}
|
||||
if (rows.size() >= maxRows) {
|
||||
throw new BusinessException("解析行数超过上限: " + maxRows);
|
||||
}
|
||||
|
||||
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter) {
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
for (int i = 0; i < headers.size(); i++) {
|
||||
values.put(headers.get(i), cell(row, i, formatter));
|
||||
values.put(headers.get(i), streamCell(rowMap, i));
|
||||
}
|
||||
rows.add(new AppearanceExcelRow(
|
||||
rowIndex + 1,
|
||||
id,
|
||||
asin,
|
||||
country,
|
||||
priceCol >= 0 ? streamCell(rowMap, priceCol) : "",
|
||||
skuCol >= 0 ? streamCell(rowMap, skuCol) : "",
|
||||
urlCol >= 0 ? streamCell(rowMap, urlCol) : "",
|
||||
titleCol >= 0 ? streamCell(rowMap, titleCol) : "",
|
||||
values));
|
||||
}
|
||||
|
||||
private int lastColumnCount(Map<Integer, String> rowMap) {
|
||||
if (rowMap == null || rowMap.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
int lastColumnCount = 0;
|
||||
for (Integer columnIndex : rowMap.keySet()) {
|
||||
if (columnIndex != null && columnIndex >= lastColumnCount) {
|
||||
lastColumnCount = columnIndex + 1;
|
||||
}
|
||||
}
|
||||
return lastColumnCount;
|
||||
}
|
||||
|
||||
private String streamCell(Map<Integer, String> rowMap, int col) {
|
||||
if (col < 0 || rowMap == null) {
|
||||
return "";
|
||||
}
|
||||
return normalize(rowMap.get(col));
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private int findRequiredHeader(Map<String, Integer> map, String... names) {
|
||||
@@ -177,10 +247,6 @@ public class AppearancePatentExcelParser {
|
||||
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
|
||||
}
|
||||
|
||||
private String cell(Row row, int col, DataFormatter formatter) {
|
||||
return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col)));
|
||||
}
|
||||
|
||||
private String normalize(String val) {
|
||||
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
package com.nanri.aiimage.modules.brand.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
|
||||
import com.nanri.aiimage.modules.task.spi.BrandTaskHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 品牌检测任务(brand_crawl_tasks)的心跳/中断实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>品牌任务有独立的表与状态字面量(running/pending/cancelled),原先这段逻辑散在
|
||||
* {@code TaskHeartbeatService} 里并直接依赖本模块的 Mapper 与进度缓存;现整体收回本模块。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class BrandCrawlTaskHeartbeatSpi implements BrandTaskHeartbeatSpi {
|
||||
|
||||
private static final String MODULE_BRAND = "BRAND";
|
||||
private static final String BRAND_STATUS_RUNNING = "running";
|
||||
|
||||
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
|
||||
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||
|
||||
@Override
|
||||
public TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
BrandCrawlTaskEntity task = selectTask(taskId);
|
||||
if (task == null) {
|
||||
return null;
|
||||
}
|
||||
String status = task.getStatus();
|
||||
if (!BRAND_STATUS_RUNNING.equals(status)) {
|
||||
log.warn("[task-heartbeat] brand task is not running taskId={} actualUserId={} status={}",
|
||||
task.getId(), task.getUserId(), status);
|
||||
return TaskHeartbeatVo.notAlive(MODULE_BRAND, status, "task is not running");
|
||||
}
|
||||
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, task.getId())
|
||||
.eq(BrandCrawlTaskEntity::getStatus, BRAND_STATUS_RUNNING)
|
||||
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated <= 0) {
|
||||
BrandCrawlTaskEntity latest = brandCrawlTaskMapper.selectById(task.getId());
|
||||
log.warn("[task-heartbeat] brand task heartbeat update missed taskId={} actualUserId={} status={} latestStatus={}",
|
||||
task.getId(), task.getUserId(), status,
|
||||
latest == null ? null : latest.getStatus());
|
||||
return TaskHeartbeatVo.notAlive(MODULE_BRAND, latest == null ? status : latest.getStatus(),
|
||||
"task is not running");
|
||||
}
|
||||
brandTaskProgressCacheService.touchHeartbeat(
|
||||
task.getId(),
|
||||
request == null ? null : request.getPhase(),
|
||||
request == null ? null : request.getCurrent(),
|
||||
request == null ? null : request.getTotal());
|
||||
return TaskHeartbeatVo.alive(MODULE_BRAND, BRAND_STATUS_RUNNING);
|
||||
}
|
||||
|
||||
@Override
|
||||
public TaskHeartbeatVo markInterrupted(Long taskId, String reason) {
|
||||
BrandCrawlTaskEntity task = selectTask(taskId);
|
||||
if (task == null) {
|
||||
return null;
|
||||
}
|
||||
String status = task.getStatus();
|
||||
if ("running".equalsIgnoreCase(status) || "pending".equalsIgnoreCase(status)) {
|
||||
// 与 file 分支统一改为条件更新:整行 updateById 会拿读取快照覆盖并发写入的字段
|
||||
// (客户端重启上报与品牌任务自身状态流转同时发生时)
|
||||
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, task.getId())
|
||||
.in(BrandCrawlTaskEntity::getStatus, "running", "pending", "RUNNING", "PENDING")
|
||||
.set(BrandCrawlTaskEntity::getStatus, "cancelled")
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, reason)
|
||||
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
log.warn("[task-interrupted] brand task marked cancelled by client restart taskId={} reason={}",
|
||||
taskId, reason);
|
||||
return TaskHeartbeatVo.notAlive(MODULE_BRAND, "cancelled", "marked cancelled");
|
||||
}
|
||||
}
|
||||
log.info("[task-interrupted] brand task not in running/pending, skipped taskId={} status={}", taskId, status);
|
||||
return TaskHeartbeatVo.notAlive(MODULE_BRAND, status, "task is not running");
|
||||
}
|
||||
|
||||
private BrandCrawlTaskEntity selectTask(Long taskId) {
|
||||
LambdaQueryWrapper<BrandCrawlTaskEntity> brandQuery = new LambdaQueryWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||
.last("limit 1");
|
||||
return brandCrawlTaskMapper.selectOne(brandQuery);
|
||||
}
|
||||
}
|
||||
+104
-56
@@ -8,6 +8,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.config.BrandProgressProperties;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
@@ -39,12 +40,7 @@ import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@@ -882,54 +878,60 @@ public class BrandTaskService {
|
||||
}
|
||||
|
||||
private ParsedBrandFile parseBrandFile(File inputFile) {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = WorkbookFactory.create(fis)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
// 2026-09 全维度审查 C5:改逐行流式解析(ExcelStreamReader → EasyExcel SAX)。
|
||||
// 此前 WorkbookFactory.create 把整表读成 DOM,品牌源文件(几十万行)会整表驻留堆内存。
|
||||
// 解析结果(columns/rows/uniqueBrands/行号)与错误文案不变,行号沿用 sheet 绝对 0 基行号 +1。
|
||||
if (!hasExcelMagic(inputFile)) {
|
||||
log.warn("[brand] 源文件不是 Excel(疑似 CSV/文本),拒绝解析 file={}", inputFile.getName());
|
||||
throw new BusinessException("读取 Excel 失败");
|
||||
}
|
||||
List<String> columns = extractHeaders(headerRow, formatter);
|
||||
if (columns.isEmpty()) {
|
||||
throw new BusinessException("未读取到有效表头");
|
||||
}
|
||||
Map<String, Integer> headerIndexes = buildHeaderIndexes(headerRow, formatter, columns);
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
Set<String> uniqueBrands = new LinkedHashSet<>();
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
Map<String, Object> rowData = new LinkedHashMap<>();
|
||||
for (String column : columns) {
|
||||
Integer index = headerIndexes.get(column);
|
||||
String value = index == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(index)));
|
||||
rowData.put(column, value);
|
||||
}
|
||||
String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), ""));
|
||||
if (brand.isBlank()) {
|
||||
rowData.put("__rowIndex", rowNum + 1);
|
||||
rows.add(rowData);
|
||||
continue;
|
||||
}
|
||||
if (uniqueBrands.add(brand)) {
|
||||
rowData.put("__rowIndex", rowNum + 1);
|
||||
rows.add(rowData);
|
||||
}
|
||||
}
|
||||
return new ParsedBrandFile(sheet.getSheetName(), columns, rows, new ArrayList<>(uniqueBrands));
|
||||
BrandFileSheetContext context = new BrandFileSheetContext();
|
||||
try {
|
||||
ExcelStreamReader.readFirstSheet(inputFile, context);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (IOException ex) {
|
||||
throw new BusinessException("读取 Excel 失败");
|
||||
}
|
||||
return context.finish();
|
||||
}
|
||||
|
||||
private List<String> extractHeaders(Row headerRow, DataFormatter formatter) {
|
||||
/**
|
||||
* 文件魔数校验:xlsx=PK(zip)、xls=OLE2(CFD)。
|
||||
* EasyExcel 会把非 zip 文本当 CSV 静默解析成功,而旧 POI WorkbookFactory 只认 xlsx/xls(垃圾文件直接失败),
|
||||
* 这里保持「非 Excel 源文件 → 读取失败」的语义(与 SimilarAsinExcelParser 同口径)。
|
||||
*/
|
||||
private boolean hasExcelMagic(File file) {
|
||||
byte[] head = new byte[8];
|
||||
try (InputStream inputStream = new FileInputStream(file)) {
|
||||
int n = inputStream.readNBytes(head, 0, head.length);
|
||||
boolean isZip = n >= 4 && head[0] == 'P' && head[1] == 'K' && (head[2] == 3 || head[2] == 5 || head[2] == 7);
|
||||
boolean isOle2 = n >= 8
|
||||
&& (head[0] & 0xFF) == 0xD0 && (head[1] & 0xFF) == 0xCF
|
||||
&& (head[2] & 0xFF) == 0x11 && (head[3] & 0xFF) == 0xE0;
|
||||
return isZip || isOle2;
|
||||
} catch (IOException ex) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** 品牌源文件流式解析上下文:表头解析 + 按品牌去重累积结果行(内存不随整表 DOM 放大)。 */
|
||||
private final class BrandFileSheetContext implements ExcelStreamReader.SheetRowHandler {
|
||||
|
||||
private final List<Map<String, Object>> rows = new ArrayList<>();
|
||||
private final Set<String> uniqueBrands = new LinkedHashSet<>();
|
||||
private List<String> columns;
|
||||
private Map<String, Integer> headerIndexes;
|
||||
private String sheetName = "";
|
||||
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
this.sheetName = sheetName == null ? "" : sheetName;
|
||||
int lastColumnCount = lastColumnCount(headerMap);
|
||||
List<String> headers = new ArrayList<>();
|
||||
Set<String> seen = new LinkedHashSet<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeHeaderValue(cell == null ? null : formatter.formatCellValue(cell));
|
||||
for (int i = 0; i < lastColumnCount; i++) {
|
||||
String value = normalizeHeaderValue(headerMap.get(i));
|
||||
if (value.isBlank() || seen.contains(value)) {
|
||||
continue;
|
||||
}
|
||||
@@ -939,24 +941,70 @@ public class BrandTaskService {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return headers;
|
||||
if (headers.isEmpty()) {
|
||||
throw new BusinessException("未读取到有效表头");
|
||||
}
|
||||
|
||||
private Map<String, Integer> buildHeaderIndexes(Row headerRow, DataFormatter formatter, List<String> columns) {
|
||||
Map<String, Integer> headerIndexes = new LinkedHashMap<>();
|
||||
Set<String> allowed = new LinkedHashSet<>(columns);
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeHeaderValue(cell == null ? null : formatter.formatCellValue(cell));
|
||||
if (value.isBlank() || headerIndexes.containsKey(value) || !allowed.contains(value)) {
|
||||
Map<String, Integer> indexes = new LinkedHashMap<>();
|
||||
Set<String> allowed = new LinkedHashSet<>(headers);
|
||||
for (int i = 0; i < lastColumnCount; i++) {
|
||||
String value = normalizeHeaderValue(headerMap.get(i));
|
||||
if (value.isBlank() || indexes.containsKey(value) || !allowed.contains(value)) {
|
||||
continue;
|
||||
}
|
||||
headerIndexes.put(value, i);
|
||||
indexes.put(value, i);
|
||||
if ("缩略图地址8".equals(value)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return headerIndexes;
|
||||
this.columns = headers;
|
||||
this.headerIndexes = indexes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
if (columns == null) {
|
||||
// 表头行整行为空时 EasyExcel 不上报表头回调:与旧实现 headerRow == null 一致
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
Map<String, Object> rowData = new LinkedHashMap<>();
|
||||
for (String column : columns) {
|
||||
Integer index = headerIndexes.get(column);
|
||||
String value = index == null ? "" : normalizeCellText(rowMap.get(index));
|
||||
rowData.put(column, value);
|
||||
}
|
||||
String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), ""));
|
||||
if (brand.isBlank()) {
|
||||
rowData.put("__rowIndex", rowIndex + 1);
|
||||
rows.add(rowData);
|
||||
return;
|
||||
}
|
||||
if (uniqueBrands.add(brand)) {
|
||||
rowData.put("__rowIndex", rowIndex + 1);
|
||||
rows.add(rowData);
|
||||
}
|
||||
}
|
||||
|
||||
private ParsedBrandFile finish() {
|
||||
if (columns == null) {
|
||||
// 空表(无任何行)或表头行缺失:与旧实现 headerRow == null 一致
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
return new ParsedBrandFile(sheetName, columns, rows, new ArrayList<>(uniqueBrands));
|
||||
}
|
||||
|
||||
private int lastColumnCount(Map<Integer, String> headerMap) {
|
||||
if (headerMap == null || headerMap.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
int lastColumnCount = 0;
|
||||
for (Integer columnIndex : headerMap.keySet()) {
|
||||
if (columnIndex != null && columnIndex >= lastColumnCount) {
|
||||
lastColumnCount = columnIndex + 1;
|
||||
}
|
||||
}
|
||||
return lastColumnCount;
|
||||
}
|
||||
}
|
||||
|
||||
private String normalizeHeaderValue(String value) {
|
||||
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
package com.nanri.aiimage.modules.brand.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.BrandTaskStaleRepairSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* brand_crawl_tasks 的陈旧修复实现(2026-09 全维度审查 G5:逻辑从 task 模块收回本模块)。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class BrandTaskStaleRepairSpiImpl implements BrandTaskStaleRepairSpi {
|
||||
|
||||
private static final String STATUS_PENDING = "pending";
|
||||
private static final String STATUS_RUNNING = "running";
|
||||
private static final String STATUS_FAILED = "failed";
|
||||
|
||||
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
|
||||
|
||||
@Override
|
||||
public List<Long> failStaleBrandTasks(LocalDateTime cutoff, int limit) {
|
||||
List<BrandCrawlTaskEntity> stale = brandCrawlTaskMapper.selectList(
|
||||
new LambdaQueryWrapper<BrandCrawlTaskEntity>()
|
||||
.select(BrandCrawlTaskEntity::getId)
|
||||
.in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING)
|
||||
.lt(BrandCrawlTaskEntity::getUpdatedAt, cutoff)
|
||||
.last("limit " + Math.max(1, limit)));
|
||||
if (stale.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<Long> ids = stale.stream().map(BrandCrawlTaskEntity::getId).toList();
|
||||
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.in(BrandCrawlTaskEntity::getId, ids)
|
||||
.in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING)
|
||||
.lt(BrandCrawlTaskEntity::getUpdatedAt, cutoff)
|
||||
.set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, "任务长期无心跳,已自动失败"));
|
||||
return ids;
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 采集明细行的历史清理实现(2026-09 全维度审查 G5:逻辑从 task 模块收回本模块)。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class CollectDataItemCleanupSpiImpl implements CollectDataItemCleanupSpi {
|
||||
|
||||
private final CollectDataItemMapper collectDataItemMapper;
|
||||
|
||||
@Override
|
||||
public int deleteItemsByTaskIds(List<Long> taskIds) {
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
return collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.in(CollectDataItemEntity::getTaskId, taskIds));
|
||||
}
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 采集数据的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>进度由采集模块自己的导入进度表维护,心跳直接把客户端上报的进度写进去。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class CollectDataTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final CollectDataService collectDataService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return "COLLECT_DATA";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
collectDataService.updateProgress(taskId, request);
|
||||
}
|
||||
}
|
||||
+235
-162
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.dedupe.service;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest;
|
||||
@@ -122,7 +123,10 @@ public class DedupeTotalDataService {
|
||||
String safeCountry = country == null ? "" : country.trim();
|
||||
AccessScope scope = resolveAccessScope(operatorId);
|
||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||
.like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
|
||||
// 2026-09 全维度审查 C6:关键字由"包含"改为"前缀"匹配——后台该输入框标注的是
|
||||
// 「输入 ASIN」(相邻的用户名框才是"模糊搜索"),前缀匹配才能命中 data_value 索引;
|
||||
// 此前的 %kw% 前置通配使 idx 永不生效,93 万行表每翻页两次全表扫
|
||||
.likeRight(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
|
||||
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
|
||||
.apply(!safeCountry.isEmpty(),
|
||||
"FIND_IN_SET({0}, IFNULL(country, '')) > 0",
|
||||
@@ -667,101 +671,162 @@ public class DedupeTotalDataService {
|
||||
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
|
||||
}
|
||||
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
// 2026-09 全维度审查 C5:改逐行流式解析(ExcelStreamReader → EasyExcel SAX)。
|
||||
// 此前 WorkbookFactory.create 把整表读成 DOM(50 万行约 1~2GB 堆),且行数上限在
|
||||
// 加载完之后才校验(等于没有防线)。现在内存只保留单批待插入值与文件内去重集合,
|
||||
// 行数上限在迭代过程中即时生效。
|
||||
ImportSession session = new ImportSession(progress, groupId, uploaderUserId, uploaderUsername);
|
||||
try {
|
||||
ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() {
|
||||
private Integer countryColumnIndex;
|
||||
private boolean headerReady;
|
||||
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell));
|
||||
if (!value.isBlank() && !headerMap.containsKey(value)) {
|
||||
headerMap.put(value, i);
|
||||
}
|
||||
}
|
||||
|
||||
Integer asinIndex = headerMap.get("ASIN");
|
||||
if (asinIndex == null) {
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
session.headerIndex = buildHeaderIndex(headerMap);
|
||||
if (session.headerIndex.get("ASIN") == null) {
|
||||
throw new BusinessException("缺少 ASIN 列");
|
||||
}
|
||||
Integer countryIndex = headerMap.get("国家");
|
||||
countryColumnIndex = session.headerIndex.get("国家");
|
||||
headerReady = true;
|
||||
}
|
||||
|
||||
Set<String> seenInFile = new HashSet<>();
|
||||
List<String> pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE);
|
||||
Map<String, String> pendingCountries = new HashMap<>();
|
||||
int totalRows = Math.max(sheet.getLastRowNum(), 0);
|
||||
if (maxImportRows > 0 && totalRows > maxImportRows) {
|
||||
@Override
|
||||
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
|
||||
if (progress != null && approximateTotalRows != null && approximateTotalRows > 0) {
|
||||
progress.setTotalRows(approximateTotalRows);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
if (!headerReady) {
|
||||
// 无表头(空表):与旧 DOM 实现一致,不处理任何数据行
|
||||
return;
|
||||
}
|
||||
session.acceptRow(rowMap, countryColumnIndex);
|
||||
}
|
||||
});
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
log.error("[dedupe-total-data] 导入 Excel 失败 filename={} processedRows={}",
|
||||
filename, session.processedRows(), e);
|
||||
throw new BusinessException("导入 Excel 失败", e);
|
||||
}
|
||||
return session.finish();
|
||||
}
|
||||
|
||||
/** 表头名 → 列索引(首次出现优先、忽略空表头),与旧 DOM 实现口径一致。 */
|
||||
private Map<String, Integer> buildHeaderIndex(Map<Integer, String> headerMap) {
|
||||
Map<String, Integer> index = new HashMap<>();
|
||||
if (headerMap == null) {
|
||||
return index;
|
||||
}
|
||||
for (Map.Entry<Integer, String> entry : headerMap.entrySet()) {
|
||||
if (entry.getKey() == null) {
|
||||
continue;
|
||||
}
|
||||
String value = normalizeExcelText(entry.getValue());
|
||||
if (!value.isBlank() && !index.containsKey(value)) {
|
||||
index.put(value, entry.getKey());
|
||||
}
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
/** 流式解析下按列索引取行内文本。 */
|
||||
private String cellText(Map<Integer, String> rowMap, Integer columnIndex) {
|
||||
if (rowMap == null || columnIndex == null) {
|
||||
return null;
|
||||
}
|
||||
return rowMap.get(columnIndex);
|
||||
}
|
||||
|
||||
/** 流式导入的累加器:单批待插入值 + 计数与进度(内存只随批大小增长,不随文件行数增长)。 */
|
||||
private final class ImportSession {
|
||||
|
||||
private final DedupeTotalDataImportProgressVo progress;
|
||||
private final Long groupId;
|
||||
private final Long uploaderUserId;
|
||||
private final String uploaderUsername;
|
||||
private final Set<String> seenInFile = new HashSet<>();
|
||||
private final List<String> pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE);
|
||||
private final Map<String, String> pendingCountries = new HashMap<>();
|
||||
private Map<String, Integer> headerIndex = Map.of();
|
||||
private int processedRows;
|
||||
private int asinCount;
|
||||
private int insertedCount;
|
||||
private int skippedCount;
|
||||
|
||||
private ImportSession(DedupeTotalDataImportProgressVo progress, Long groupId,
|
||||
Long uploaderUserId, String uploaderUsername) {
|
||||
this.progress = progress;
|
||||
this.groupId = groupId;
|
||||
this.uploaderUserId = uploaderUserId;
|
||||
this.uploaderUsername = uploaderUsername;
|
||||
}
|
||||
|
||||
private void acceptRow(Map<Integer, String> rowMap, Integer countryColumnIndex) {
|
||||
processedRows++;
|
||||
if (maxImportRows > 0 && processedRows > maxImportRows) {
|
||||
// 流式下无法前置知道总行数:超限即中断,已插入批次保留(与旧实现的"截断"语义不同,
|
||||
// 但旧实现在加载后才校验,实际是 OOM 之后才失败)
|
||||
throw new BusinessException("导入行数超过上限: " + maxImportRows);
|
||||
}
|
||||
int asinCount = 0;
|
||||
int insertedCount = 0;
|
||||
int skippedCount = 0;
|
||||
if (progress != null) {
|
||||
progress.setTotalRows(totalRows);
|
||||
progress.setProcessedRows(0);
|
||||
progress.setAsinCount(0);
|
||||
progress.setInsertedCount(0);
|
||||
progress.setSkippedCount(0);
|
||||
progress.setErrorMessage(null);
|
||||
}
|
||||
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
skippedCount++;
|
||||
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
|
||||
continue;
|
||||
}
|
||||
String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex)));
|
||||
Integer asinIndex = headerIndex.get("ASIN");
|
||||
String asin = asinIndex == null ? "" : normalizeComparableValueOrBlank(cellText(rowMap, asinIndex));
|
||||
if (asin.isBlank()) {
|
||||
skippedCount++;
|
||||
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
|
||||
continue;
|
||||
updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount);
|
||||
return;
|
||||
}
|
||||
asinCount++;
|
||||
if (!seenInFile.add(asin)) {
|
||||
skippedCount++;
|
||||
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
|
||||
continue;
|
||||
updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount);
|
||||
return;
|
||||
}
|
||||
pendingValues.add(asin);
|
||||
if (countryIndex != null) {
|
||||
pendingCountries.put(asin, parseCountries(formatter.formatCellValue(row.getCell(countryIndex))));
|
||||
if (countryColumnIndex != null) {
|
||||
pendingCountries.put(asin, parseCountries(cellText(rowMap, countryColumnIndex)));
|
||||
}
|
||||
if (pendingValues.size() >= IMPORT_BATCH_SIZE) {
|
||||
ImportBatchResult batch = insertImportBatch(
|
||||
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
|
||||
insertedCount += batch.insertedCount();
|
||||
skippedCount += batch.skippedCount();
|
||||
pendingValues.clear();
|
||||
pendingCountries.clear();
|
||||
flushBatch();
|
||||
}
|
||||
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
|
||||
updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount);
|
||||
}
|
||||
if (!pendingValues.isEmpty()) {
|
||||
ImportBatchResult batch = insertImportBatch(
|
||||
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
|
||||
insertedCount += batch.insertedCount();
|
||||
skippedCount += batch.skippedCount();
|
||||
pendingValues.clear();
|
||||
pendingCountries.clear();
|
||||
}
|
||||
updateImportProgress(progress, totalRows, asinCount, insertedCount, skippedCount);
|
||||
|
||||
private void flushBatch() {
|
||||
if (pendingValues.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
ImportBatchResult batch = insertImportBatch(
|
||||
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
|
||||
insertedCount += batch.insertedCount();
|
||||
skippedCount += batch.skippedCount();
|
||||
pendingValues.clear();
|
||||
pendingCountries.clear();
|
||||
}
|
||||
|
||||
private int processedRows() {
|
||||
return processedRows;
|
||||
}
|
||||
|
||||
private DedupeTotalDataImportVo finish() {
|
||||
flushBatch();
|
||||
updateImportProgress(progress, processedRows, asinCount, insertedCount, skippedCount);
|
||||
if (progress != null && processedRows > progress.getTotalRows()) {
|
||||
// 近似总行数缺失(或与实际物理行数不一致)时用实际处理行数兜底,避免展示成 x/0
|
||||
progress.setTotalRows(processedRows);
|
||||
}
|
||||
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
|
||||
vo.setTotalRows(totalRows);
|
||||
vo.setTotalRows(processedRows);
|
||||
vo.setAsinCount(asinCount);
|
||||
vo.setInsertedCount(insertedCount);
|
||||
vo.setSkippedCount(skippedCount);
|
||||
return vo;
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new BusinessException("导入 Excel 失败");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -850,112 +915,120 @@ public class DedupeTotalDataService {
|
||||
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
|
||||
}
|
||||
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
// 2026-09 全维度审查 C5:与导入同样改流式解析,避免 50 万行时 DOM 占 1~2GB 堆
|
||||
DeleteSession session = new DeleteSession(progress, scope, groupId);
|
||||
try {
|
||||
ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() {
|
||||
private boolean headerReady;
|
||||
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell));
|
||||
if (!value.isBlank() && !headerMap.containsKey(value)) {
|
||||
headerMap.put(value, i);
|
||||
}
|
||||
}
|
||||
|
||||
Integer asinIndex = headerMap.get("ASIN");
|
||||
if (asinIndex == null) {
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
session.headerIndex = buildHeaderIndex(headerMap);
|
||||
if (session.headerIndex.get("ASIN") == null) {
|
||||
throw new BusinessException("缺少 ASIN 列");
|
||||
}
|
||||
|
||||
Set<String> seenInFile = new HashSet<>();
|
||||
List<String> pendingDeletes = new ArrayList<>();
|
||||
int totalRows = Math.max(sheet.getLastRowNum(), 0);
|
||||
if (maxImportRows > 0 && totalRows > maxImportRows) {
|
||||
throw new BusinessException("导入行数超过上限: " + maxImportRows);
|
||||
}
|
||||
int asinCount = 0;
|
||||
int deletedCount = 0;
|
||||
int skippedCount = 0;
|
||||
if (progress != null) {
|
||||
progress.setTotalRows(totalRows);
|
||||
progress.setProcessedRows(0);
|
||||
progress.setAsinCount(0);
|
||||
progress.setInsertedCount(0);
|
||||
progress.setSkippedCount(0);
|
||||
progress.setErrorMessage(null);
|
||||
headerReady = true;
|
||||
}
|
||||
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
skippedCount++;
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(deletedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
@Override
|
||||
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
|
||||
if (progress != null && approximateTotalRows != null && approximateTotalRows > 0) {
|
||||
progress.setTotalRows(approximateTotalRows);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
String asin = normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinIndex)));
|
||||
if (asin.isBlank()) {
|
||||
skippedCount++;
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(deletedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
asinCount++;
|
||||
String dataValue = asin;
|
||||
if (!seenInFile.add(dataValue)) {
|
||||
skippedCount++;
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(deletedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 批量删除:攒批 + IN 一次删,避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1
|
||||
pendingDeletes.add(dataValue);
|
||||
if (pendingDeletes.size() >= DELETE_BATCH_SIZE) {
|
||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
|
||||
pendingDeletes = new ArrayList<>();
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
if (!headerReady) {
|
||||
return;
|
||||
}
|
||||
if (progress != null) {
|
||||
progress.setProcessedRows(rowNum);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(deletedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
session.acceptRow(rowMap);
|
||||
}
|
||||
}
|
||||
if (!pendingDeletes.isEmpty()) {
|
||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
|
||||
}
|
||||
|
||||
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
|
||||
vo.setTotalRows(totalRows);
|
||||
vo.setAsinCount(asinCount);
|
||||
vo.setInsertedCount(deletedCount);
|
||||
vo.setSkippedCount(skippedCount);
|
||||
return vo;
|
||||
});
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
// 此前丢弃原始异常:排查时只能看到一句「删除 Excel 匹配数据失败」
|
||||
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败", e);
|
||||
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败 filename={} processedRows={}",
|
||||
filename, session.processedRows(), e);
|
||||
throw new BusinessException("删除 Excel 匹配数据失败", e);
|
||||
}
|
||||
return session.finish();
|
||||
}
|
||||
|
||||
/** 流式删除匹配的累加器:单批待删值 + 计数与进度。 */
|
||||
private final class DeleteSession {
|
||||
|
||||
private final DedupeTotalDataImportProgressVo progress;
|
||||
private final AccessScope scope;
|
||||
private final Long groupId;
|
||||
private final Set<String> seenInFile = new HashSet<>();
|
||||
private final List<String> pendingDeletes = new ArrayList<>();
|
||||
private Map<String, Integer> headerIndex = Map.of();
|
||||
private int processedRows;
|
||||
private int asinCount;
|
||||
private int deletedCount;
|
||||
private int skippedCount;
|
||||
|
||||
private DeleteSession(DedupeTotalDataImportProgressVo progress, AccessScope scope, Long groupId) {
|
||||
this.progress = progress;
|
||||
this.scope = scope;
|
||||
this.groupId = groupId;
|
||||
}
|
||||
|
||||
private void acceptRow(Map<Integer, String> rowMap) {
|
||||
processedRows++;
|
||||
if (maxImportRows > 0 && processedRows > maxImportRows) {
|
||||
throw new BusinessException("导入行数超过上限: " + maxImportRows);
|
||||
}
|
||||
Integer asinIndex = headerIndex.get("ASIN");
|
||||
String asin = asinIndex == null ? "" : normalizeComparableValueOrBlank(cellText(rowMap, asinIndex));
|
||||
if (asin.isBlank() || !seenInFile.add(asin)) {
|
||||
skippedCount++;
|
||||
updateProgress();
|
||||
return;
|
||||
}
|
||||
asinCount++;
|
||||
// 批量删除:攒批 + IN 一次删,避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1
|
||||
pendingDeletes.add(asin);
|
||||
if (pendingDeletes.size() >= DELETE_BATCH_SIZE) {
|
||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
|
||||
pendingDeletes.clear();
|
||||
}
|
||||
updateProgress();
|
||||
}
|
||||
|
||||
private void updateProgress() {
|
||||
if (progress == null) {
|
||||
return;
|
||||
}
|
||||
progress.setProcessedRows(processedRows);
|
||||
progress.setAsinCount(asinCount);
|
||||
progress.setInsertedCount(deletedCount);
|
||||
progress.setSkippedCount(skippedCount);
|
||||
}
|
||||
|
||||
private int processedRows() {
|
||||
return processedRows;
|
||||
}
|
||||
|
||||
private DedupeTotalDataImportVo finish() {
|
||||
if (!pendingDeletes.isEmpty()) {
|
||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
|
||||
pendingDeletes.clear();
|
||||
}
|
||||
updateProgress();
|
||||
if (progress != null && processedRows > progress.getTotalRows()) {
|
||||
progress.setTotalRows(processedRows);
|
||||
}
|
||||
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
|
||||
vo.setTotalRows(processedRows);
|
||||
vo.setAsinCount(asinCount);
|
||||
vo.setInsertedCount(deletedCount);
|
||||
vo.setSkippedCount(skippedCount);
|
||||
return vo;
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
|
||||
+115
-30
@@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.config.DeleteBrandProgressProperties;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.deletebrand.model.cache.DeleteBrandParsedFileCacheDto;
|
||||
@@ -44,10 +45,8 @@ import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
@@ -59,6 +58,7 @@ import org.springframework.transaction.support.TransactionTemplate;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
@@ -515,39 +515,102 @@ public class DeleteBrandRunService {
|
||||
}
|
||||
|
||||
private ParsedDeleteBrandFile parseDeleteBrandFile(File inputFile) {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row titleRow = sheet.getRow(0);
|
||||
Row headerRow = sheet.getRow(1);
|
||||
if (titleRow == null || headerRow == null) {
|
||||
throw new BusinessException("Excel 表头不完整");
|
||||
// 2026-09 全维度审查 C5:改逐行流式解析(ExcelStreamReader → EasyExcel SAX)。
|
||||
// 此前 WorkbookFactory.create 把整表读成 DOM,删除品牌源文件(几十万行)会整表驻留堆内存。
|
||||
// 解析结果(国家分组/预览行/行号)与错误文案不变,行号沿用 sheet 绝对 0 基行号 +1。
|
||||
if (!hasExcelMagic(inputFile)) {
|
||||
log.warn("[delete-brand] 源文件不是 Excel(疑似 CSV/文本),拒绝解析 file={}", inputFile.getName());
|
||||
throw new BusinessException("读取删除品牌 Excel 失败");
|
||||
}
|
||||
DeleteBrandSheetContext context = new DeleteBrandSheetContext();
|
||||
try {
|
||||
ExcelStreamReader.readFirstSheet(inputFile, context);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
// 此前该 catch 既不记日志也不带 cause,线上只有一句「读取删除品牌 Excel 失败」
|
||||
log.warn("[delete-brand] 读取删除品牌 Excel 失败 file={} err={}", inputFile.getName(), ex.getMessage(), ex);
|
||||
throw new BusinessException("读取删除品牌 Excel 失败");
|
||||
}
|
||||
return context.finish();
|
||||
}
|
||||
|
||||
List<CountryColumnPair> pairs = resolveCountryPairs(titleRow, headerRow, formatter);
|
||||
/**
|
||||
* 文件魔数校验:xlsx=PK(zip)、xls=OLE2(CFD)。
|
||||
* EasyExcel 会把非 zip 文本当 CSV 静默解析成功,而旧 POI WorkbookFactory 只认 xlsx/xls(垃圾文件直接失败),
|
||||
* 这里保持「非 Excel 源文件 → 读取失败」的语义(与 SimilarAsinExcelParser 同口径)。
|
||||
*/
|
||||
private boolean hasExcelMagic(File file) {
|
||||
byte[] head = new byte[8];
|
||||
try (FileInputStream inputStream = new FileInputStream(file)) {
|
||||
int n = inputStream.readNBytes(head, 0, head.length);
|
||||
boolean isZip = n >= 4 && head[0] == 'P' && head[1] == 'K' && (head[2] == 3 || head[2] == 5 || head[2] == 7);
|
||||
boolean isOle2 = n >= 8
|
||||
&& (head[0] & 0xFF) == 0xD0 && (head[1] & 0xFF) == 0xCF
|
||||
&& (head[2] & 0xFF) == 0x11 && (head[3] & 0xFF) == 0xE0;
|
||||
return isZip || isOle2;
|
||||
} catch (IOException ex) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除品牌源文件流式解析上下文:两行表头(国家标题行 + 字段行)解析列组 + 逐行按国家收集 ASIN。
|
||||
* dataStartRowIndex 语义与旧实现一致(删除ASIN 表头记为 2,其余"状态"表头记为 1)。
|
||||
*/
|
||||
private final class DeleteBrandSheetContext implements ExcelStreamReader.SheetRowHandler {
|
||||
|
||||
private final Map<String, Map<String, DeleteBrandCountryAsinVo>> grouped = new LinkedHashMap<>();
|
||||
private final Map<String, String> displayCountryNames = new LinkedHashMap<>();
|
||||
private Map<Integer, String> titleRowMap;
|
||||
private boolean titleRowSeen;
|
||||
private boolean headerRowSeen;
|
||||
private List<CountryColumnPair> pairs;
|
||||
private int firstDataRow = 2;
|
||||
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
// 第 0 行 = 国家标题行
|
||||
titleRowSeen = true;
|
||||
titleRowMap = headerMap == null ? Map.of() : headerMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
if (!headerRowSeen) {
|
||||
if (rowIndex != 1) {
|
||||
// 第 1 行整行为空(EasyExcel 不上报):与旧实现 headerRow == null 一致
|
||||
throw new BusinessException("Excel 表头不完整");
|
||||
}
|
||||
if (!titleRowSeen) {
|
||||
// 第 0 行整行为空(EasyExcel 不上报表头回调):与旧实现 titleRow == null 一致
|
||||
throw new BusinessException("Excel 表头不完整");
|
||||
}
|
||||
headerRowSeen = true;
|
||||
pairs = resolveCountryPairs(titleRowMap, rowMap);
|
||||
if (pairs.isEmpty()) {
|
||||
throw new BusinessException("未识别到删除品牌表头");
|
||||
}
|
||||
|
||||
Map<String, Map<String, DeleteBrandCountryAsinVo>> grouped = new LinkedHashMap<>();
|
||||
Map<String, String> displayCountryNames = new LinkedHashMap<>();
|
||||
|
||||
int firstDataRow = pairs.stream()
|
||||
firstDataRow = pairs.stream()
|
||||
.mapToInt(CountryColumnPair::dataStartRowIndex)
|
||||
.min()
|
||||
.orElse(2);
|
||||
for (int rowNum = firstDataRow; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
if (rowIndex < firstDataRow) {
|
||||
return;
|
||||
}
|
||||
consumeDataRow(rowIndex, rowMap);
|
||||
}
|
||||
|
||||
private void consumeDataRow(int rowNum, Map<Integer, String> rowMap) {
|
||||
for (CountryColumnPair pair : pairs) {
|
||||
if (rowNum < pair.dataStartRowIndex()) {
|
||||
continue;
|
||||
}
|
||||
String country = pair.country();
|
||||
String asin = normalizeCellText(formatter.formatCellValue(row.getCell(pair.asinColumnIndex())));
|
||||
String status = normalizeCellText(formatter.formatCellValue(row.getCell(pair.statusColumnIndex())));
|
||||
String asin = normalizeCellText(cellText(rowMap, pair.asinColumnIndex()));
|
||||
String status = normalizeCellText(cellText(rowMap, pair.statusColumnIndex()));
|
||||
if (asin.isBlank() && status.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
@@ -571,6 +634,11 @@ public class DeleteBrandRunService {
|
||||
}
|
||||
}
|
||||
|
||||
private ParsedDeleteBrandFile finish() {
|
||||
if (!headerRowSeen) {
|
||||
// 空表 / 表头行缺失:与旧实现 titleRow == null || headerRow == null 一致
|
||||
throw new BusinessException("Excel 表头不完整");
|
||||
}
|
||||
List<DeleteBrandCountryGroupVo> countries = new ArrayList<>();
|
||||
List<DeleteBrandPreviewRowVo> previewRows = new ArrayList<>();
|
||||
int totalRows = 0;
|
||||
@@ -598,10 +666,6 @@ public class DeleteBrandRunService {
|
||||
}
|
||||
|
||||
return new ParsedDeleteBrandFile(totalRows, countries, previewRows);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取删除品牌 Excel 失败");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -612,13 +676,13 @@ public class DeleteBrandRunService {
|
||||
return value.replace(" ", "").trim().toUpperCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private List<CountryColumnPair> resolveCountryPairs(Row titleRow, Row headerRow, DataFormatter formatter) {
|
||||
private List<CountryColumnPair> resolveCountryPairs(Map<Integer, String> titleRowMap, Map<Integer, String> headerRowMap) {
|
||||
List<CountryColumnPair> pairs = new ArrayList<>();
|
||||
int lastCellNum = Math.max(titleRow.getLastCellNum(), headerRow.getLastCellNum());
|
||||
int lastCellNum = Math.max(lastColumnCount(titleRowMap), lastColumnCount(headerRowMap));
|
||||
for (int i = 0; i < lastCellNum; i++) {
|
||||
String title = normalizeCellText(formatter.formatCellValue(titleRow.getCell(i)));
|
||||
String firstHeader = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i)));
|
||||
String secondHeader = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i + 1)));
|
||||
String title = normalizeCellText(cellText(titleRowMap, i));
|
||||
String firstHeader = normalizeCellText(cellText(headerRowMap, i));
|
||||
String secondHeader = normalizeCellText(cellText(headerRowMap, i + 1));
|
||||
if (title.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
@@ -633,6 +697,27 @@ public class DeleteBrandRunService {
|
||||
return pairs;
|
||||
}
|
||||
|
||||
/** 流式解析下按列索引取行内文本(与旧 formatter.formatCellValue(row.getCell(index)) 口径一致)。 */
|
||||
private String cellText(Map<Integer, String> rowMap, Integer columnIndex) {
|
||||
if (rowMap == null || columnIndex == null || columnIndex < 0) {
|
||||
return "";
|
||||
}
|
||||
return rowMap.get(columnIndex);
|
||||
}
|
||||
|
||||
private int lastColumnCount(Map<Integer, String> rowMap) {
|
||||
if (rowMap == null || rowMap.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
int lastColumnCount = 0;
|
||||
for (Integer columnIndex : rowMap.keySet()) {
|
||||
if (columnIndex != null && columnIndex >= lastColumnCount) {
|
||||
lastColumnCount = columnIndex + 1;
|
||||
}
|
||||
}
|
||||
return lastColumnCount;
|
||||
}
|
||||
|
||||
private String resolveShopName(DeleteBrandSourceFileDto sourceFile) {
|
||||
String name = sourceFile.getOriginalFilename();
|
||||
if (name == null || name.isBlank()) {
|
||||
|
||||
+21
-7
@@ -38,6 +38,7 @@ import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
@@ -103,13 +104,9 @@ public class DeleteBrandStaleTaskService {
|
||||
ShopMatchStaleCheckStats patrolDeleteStats = failStalePatrolDeleteTasks();
|
||||
ShopMatchStaleCheckStats queryAsinStats = failStaleQueryAsinTasks();
|
||||
ShopMatchStaleCheckStats withdrawStats = failStaleWithdrawTasks();
|
||||
runModuleStaleCheck("brand", brandTaskService::failStaleRunningTasks);
|
||||
runModuleStaleCheck("appearance-patent", appearancePatentTaskService::finalizeStaleTasks);
|
||||
runModuleStaleCheck("similar-asin", similarAsinTaskService::finalizeStaleTasks);
|
||||
// 商品管理采集并入本巡检线(2026-09):此前它自带 @Scheduled 且按 owner 过滤,
|
||||
// 双实例下 owner 宕机即无人判死(P1-8);并入后由本方法的 job 锁保证单实例扫描,
|
||||
// 它自己改为全局判死 + 任务锁 + status CAS
|
||||
runModuleStaleCheck("shop-data-crawl", shopDataCrawlTaskService::finalizeStaleTasks);
|
||||
for (Map.Entry<String, Runnable> delegated : delegatedStaleChecks().entrySet()) {
|
||||
runModuleStaleCheck(delegated.getKey(), delegated.getValue());
|
||||
}
|
||||
// 周期每 2 分钟一轮,各模块 summary 合并为单行,避免定期刷屏
|
||||
// 注意:每段占位符数量必须与实参一致——此前每段 5 个占位符只传 4 个参数,
|
||||
// 导致 withdraw 之后的取值整体错位、末尾 elapsedMs/thread 打成字面量
|
||||
@@ -125,6 +122,23 @@ public class DeleteBrandStaleTaskService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 委派式陈旧判死:moduleType → 处理动作。
|
||||
* {@code TaskModuleRegistry} 中 delegatedStaleCheck=true 的模块都必须在这里登记,
|
||||
* 由 TaskModuleCoverageTest 守住一致性(新增模块漏登记会红测试)。
|
||||
*/
|
||||
Map<String, Runnable> delegatedStaleChecks() {
|
||||
Map<String, Runnable> handlers = new LinkedHashMap<>();
|
||||
handlers.put("BRAND", brandTaskService::failStaleRunningTasks);
|
||||
handlers.put("APPEARANCE_PATENT", appearancePatentTaskService::finalizeStaleTasks);
|
||||
handlers.put("SIMILAR_ASIN", similarAsinTaskService::finalizeStaleTasks);
|
||||
// 商品管理采集并入本巡检线(2026-09):此前它自带 @Scheduled 且按 owner 过滤,
|
||||
// 双实例下 owner 宕机即无人判死(P1-8);并入后由本方法的 job 锁保证单实例扫描,
|
||||
// 它自己改为全局判死 + 任务锁 + status CAS
|
||||
handlers.put("SHOP_DATA_CRAWL", shopDataCrawlTaskService::finalizeStaleTasks);
|
||||
return handlers;
|
||||
}
|
||||
|
||||
private void runModuleStaleCheck(String moduleName, Runnable action) {
|
||||
long startedAt = System.currentTimeMillis();
|
||||
try {
|
||||
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
package com.nanri.aiimage.modules.deletebrand.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 删除 ASIN 的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>本模块的心跳不写独立的"心跳键",而是整段进度写进任务缓存(saveProgress,forceWrite=true),
|
||||
* 并携带客户端上报的 phase/current/total。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class DeleteBrandTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final DeleteBrandTaskCacheService cacheService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return "DELETE_BRAND";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
cacheService.saveProgress(taskId, buildHeartbeatProgress(request), true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveTaskCache(FileTaskEntity task) {
|
||||
cacheService.saveTaskCache(task);
|
||||
}
|
||||
|
||||
private Map<String, String> buildHeartbeatProgress(TaskHeartbeatRequest request) {
|
||||
String now = String.valueOf(System.currentTimeMillis());
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
values.put("last_heartbeat_at", now);
|
||||
values.put("updated_at", now);
|
||||
if (request != null) {
|
||||
putIfPresent(values, "phase", request.getPhase());
|
||||
putIfPresent(values, "current", request.getCurrent());
|
||||
putIfPresent(values, "total", request.getTotal());
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private void putIfPresent(Map<String, String> values, String key, Object value) {
|
||||
if (value != null) {
|
||||
values.put(key, String.valueOf(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
+24
-4
@@ -1,12 +1,14 @@
|
||||
package com.nanri.aiimage.modules.digitalhuman.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.digitalhuman.model.dto.UploadVersionRequest;
|
||||
import com.nanri.aiimage.modules.digitalhuman.model.vo.DigitalHumanVersionVo;
|
||||
import com.nanri.aiimage.modules.digitalhuman.service.DigitalHumanVersionService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.http.MediaType;
|
||||
@@ -27,10 +29,22 @@ import java.util.List;
|
||||
public class DigitalHumanVersionController {
|
||||
|
||||
private final DigitalHumanVersionService versionService;
|
||||
private final AdminAuthSupport adminAuthSupport;
|
||||
|
||||
/**
|
||||
* 写操作(上传/发布/设为最新/删除)必须管理员身份:2026-09 全维度审查 A4——
|
||||
* 此前整套接口匿名可达,任何人可发布/删除客户端版本。
|
||||
* 读接口(列表/latest/详情/download-url)保持匿名:桌面客户端自身要查版本与下载地址。
|
||||
*/
|
||||
private void requireAdmin(HttpServletRequest request) {
|
||||
adminAuthSupport.requireAdminOrInternal(request);
|
||||
}
|
||||
|
||||
@PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@Operation(summary = "上传新版本", description = "上传数字人程序新版本到 OSS,并记录版本信息")
|
||||
public ApiResponse<DigitalHumanVersionVo> uploadVersion(@Valid @ModelAttribute UploadVersionRequest request) {
|
||||
public ApiResponse<DigitalHumanVersionVo> uploadVersion(@Valid @ModelAttribute UploadVersionRequest request,
|
||||
HttpServletRequest httpRequest) {
|
||||
requireAdmin(httpRequest);
|
||||
DigitalHumanVersionVo vo = versionService.uploadVersion(
|
||||
request.getVersion(),
|
||||
request.getFile(),
|
||||
@@ -63,21 +77,27 @@ public class DigitalHumanVersionController {
|
||||
@PostMapping("/{version}/release")
|
||||
@Operation(summary = "发布版本", description = "将草稿状态的版本发布,状态变更为 RELEASED")
|
||||
public ApiResponse<DigitalHumanVersionVo> releaseVersion(
|
||||
@Parameter(description = "版本号", example = "1.0.0") @PathVariable String version) {
|
||||
@Parameter(description = "版本号", example = "1.0.0") @PathVariable String version,
|
||||
HttpServletRequest request) {
|
||||
requireAdmin(request);
|
||||
return ApiResponse.success(versionService.releaseVersion(version));
|
||||
}
|
||||
|
||||
@PostMapping("/{version}/set-latest")
|
||||
@Operation(summary = "设为最新版本", description = "将指定已发布版本设置为最新版本,清除其他版本的最新标记")
|
||||
public ApiResponse<DigitalHumanVersionVo> setLatest(
|
||||
@Parameter(description = "版本号", example = "1.0.0") @PathVariable String version) {
|
||||
@Parameter(description = "版本号", example = "1.0.0") @PathVariable String version,
|
||||
HttpServletRequest request) {
|
||||
requireAdmin(request);
|
||||
return ApiResponse.success(versionService.setLatest(version));
|
||||
}
|
||||
|
||||
@DeleteMapping("/{version}")
|
||||
@Operation(summary = "删除版本", description = "删除指定版本,同时删除 OSS 上的文件。最新版本不允许删除")
|
||||
public ApiResponse<Void> deleteVersion(
|
||||
@Parameter(description = "版本号", example = "1.0.0") @PathVariable String version) {
|
||||
@Parameter(description = "版本号", example = "1.0.0") @PathVariable String version,
|
||||
HttpServletRequest request) {
|
||||
requireAdmin(request);
|
||||
versionService.deleteVersion(version);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
+118
-29
@@ -3,34 +3,46 @@ package com.nanri.aiimage.modules.file.service;
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.file.model.vo.ExcelInfoVo;
|
||||
import com.nanri.aiimage.modules.file.model.vo.UploadFileVo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class LocalFileStorageService {
|
||||
|
||||
/** fileKey → 文件名 索引容量:超限淘汰最旧条目,避免索引自身无界增长。 */
|
||||
static final int SOURCE_FILE_INDEX_CAPACITY = 1024;
|
||||
|
||||
/**
|
||||
* 禁止上传的可执行/脚本类扩展名(2026-09 全维度审查 A3)。
|
||||
* 上传接口是桌面端文件中转口(临时目录落盘,2GB/次),此前对类型无任何限制;
|
||||
* 这里挡掉"落盘后一旦被其它组件误执行/误引用即可造成危害"的扩展名。
|
||||
* 业务文件(xlsx/xls/csv/txt/json/zip/图片/视频)不受影响;
|
||||
* 如需调整由运维在 application.yml 覆盖 aiimage.files.upload-blocked-extensions。
|
||||
*/
|
||||
private static final String DEFAULT_BLOCKED_EXTENSIONS =
|
||||
"exe,bat,cmd,com,scr,msi,dll,sh,ps1,vbs,vbe,wsf,hta,jsp,jspx,php,jar,so,dylib";
|
||||
|
||||
@Value("${aiimage.files.upload-blocked-extensions:" + DEFAULT_BLOCKED_EXTENSIONS + "}")
|
||||
private String blockedExtensions = DEFAULT_BLOCKED_EXTENSIONS;
|
||||
|
||||
private final StorageProperties storageProperties;
|
||||
|
||||
/** 临时目录标准路径:供调用方做"文件必须落在上传临时目录内"的穿越校验。 */
|
||||
@@ -51,6 +63,7 @@ public class LocalFileStorageService {
|
||||
FileUtil.mkdir(tempDir);
|
||||
String fileKey = IdUtil.fastSimpleUUID();
|
||||
String extName = FileUtil.extName(file.getOriginalFilename());
|
||||
rejectBlockedExtension(file.getOriginalFilename(), extName);
|
||||
String filename = fileKey + (extName.isEmpty() ? "" : "." + extName);
|
||||
File target = FileUtil.file(tempDir, filename);
|
||||
file.transferTo(target);
|
||||
@@ -65,49 +78,125 @@ public class LocalFileStorageService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
/** 扩展名黑名单校验:命中即拒绝落盘(大小写不敏感)。 */
|
||||
private void rejectBlockedExtension(String originalFilename, String extName) {
|
||||
if (extName == null || extName.isBlank()) {
|
||||
return;
|
||||
}
|
||||
String normalized = extName.trim().toLowerCase(java.util.Locale.ROOT);
|
||||
for (String blocked : blockedExtensions.split(",")) {
|
||||
if (!blocked.isBlank() && blocked.trim().toLowerCase(java.util.Locale.ROOT).equals(normalized)) {
|
||||
log.warn("[file-upload] 拒绝上传被禁扩展名 originalFilename={} ext={}", originalFilename, normalized);
|
||||
throw new BusinessException("不支持上传该类型文件:." + normalized);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ExcelInfoVo getExcelInfo(String fileKey) throws IOException {
|
||||
File inputFile = findLocalSourceFile(fileKey);
|
||||
if (inputFile == null || !inputFile.exists()) {
|
||||
throw new BusinessException("源文件不存在");
|
||||
}
|
||||
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
// 2026-09 全维度审查 C5:改流式解析(ExcelStreamReader → EasyExcel SAX)。
|
||||
// 此前 WorkbookFactory.create 把整表读成 DOM,只为取表头与行数却让整表驻留堆内存。
|
||||
// 返回值结构与字段含义不变:headers 同旧口径(去重、空表头跳过、命中「缩略图地址8」截断),
|
||||
// totalRows 仍是"数据行数"(旧实现取 lastRowNum,等于物理末行行号-1)。
|
||||
try {
|
||||
ExcelInfoStreamingReader reader = new ExcelInfoStreamingReader();
|
||||
ExcelStreamReader.readFirstSheet(inputFile, reader);
|
||||
reader.assertHeaderSeen();
|
||||
ExcelInfoVo vo = new ExcelInfoVo();
|
||||
vo.setHeaders(reader.headers());
|
||||
vo.setTotalRows(reader.totalRows());
|
||||
return vo;
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
log.warn("[file-excel-info] 读取 Excel 信息失败 fileKey={} err={}", fileKey, e.getMessage(), e);
|
||||
throw new BusinessException("读取 Excel 信息失败");
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Integer> headerMap = new HashMap<>();
|
||||
List<String> headers = new ArrayList<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
Cell cell = headerRow.getCell(i);
|
||||
String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell));
|
||||
/**
|
||||
* 表头与行数的流式读取:内存不随文件行数增长。
|
||||
* 表头行整行为空(EasyExcel 不上报表头回调)时视为「Excel 表头为空」——旧实现仅在 getRow(0)==null
|
||||
* 时报同一错误,此处口径略严(不存在的表头与空表头同样处理)。
|
||||
*/
|
||||
private final class ExcelInfoStreamingReader implements ExcelStreamReader.SheetRowHandler {
|
||||
|
||||
private final List<String> headers = new ArrayList<>();
|
||||
private final Set<String> seenHeaders = new LinkedHashSet<>();
|
||||
private Integer approximateTotalRows;
|
||||
private boolean headerSeen;
|
||||
private boolean headerTruncated;
|
||||
private int countedRows;
|
||||
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
headerSeen = true;
|
||||
int lastColumnCount = lastHeaderColumnCount(headerMap);
|
||||
for (int i = 0; i < lastColumnCount && !headerTruncated; i++) {
|
||||
String value = normalizeCellText(headerMap.get(i));
|
||||
if (value.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty()
|
||||
? value
|
||||
: value.split("idASIN国家状态价格变体数量", 2)[0].trim();
|
||||
if (headerMap.containsKey(value)) {
|
||||
if (!seenHeaders.add(value)) {
|
||||
continue;
|
||||
}
|
||||
headerMap.put(value, i);
|
||||
headers.add(value);
|
||||
if ("缩略图地址8".equals(value)) {
|
||||
break;
|
||||
headerTruncated = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ExcelInfoVo vo = new ExcelInfoVo();
|
||||
vo.setHeaders(headers);
|
||||
vo.setTotalRows(Math.max(sheet.getLastRowNum(), 0));
|
||||
return vo;
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new BusinessException("读取 Excel 信息失败");
|
||||
@Override
|
||||
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
|
||||
this.approximateTotalRows = approximateTotalRows;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
countedRows++;
|
||||
}
|
||||
|
||||
private void assertHeaderSeen() {
|
||||
if (!headerSeen) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> headers() {
|
||||
return headers;
|
||||
}
|
||||
|
||||
/**
|
||||
* onSheetTotal 的近似值 = 含表头的物理末行行号(POI lastRowNum+1 口径),换算回旧的
|
||||
* "数据行数"口径;近似值缺失(dimension 标签缺失)或偏小时用遍历计数兜底。
|
||||
*/
|
||||
private int totalRows() {
|
||||
int approximateDataRows = approximateTotalRows == null || approximateTotalRows <= 0
|
||||
? 0
|
||||
: approximateTotalRows - 1;
|
||||
return Math.max(approximateDataRows, countedRows);
|
||||
}
|
||||
|
||||
private int lastHeaderColumnCount(Map<Integer, String> headerMap) {
|
||||
if (headerMap == null || headerMap.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
int lastColumnCount = 0;
|
||||
for (Integer columnIndex : headerMap.keySet()) {
|
||||
if (columnIndex != null && columnIndex >= lastColumnCount) {
|
||||
lastColumnCount = columnIndex + 1;
|
||||
}
|
||||
}
|
||||
return lastColumnCount;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+50
@@ -1,11 +1,15 @@
|
||||
package com.nanri.aiimage.modules.file.service.object;
|
||||
|
||||
import com.nanri.aiimage.common.util.PendingDeleteJournal;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
@@ -26,6 +30,48 @@ public class RustfsDeleteRetryService {
|
||||
private final ConcurrentLinkedQueue<RetryItem> queue = new ConcurrentLinkedQueue<>();
|
||||
private final ConcurrentHashMap<String, RetryItem> pending = new ConcurrentHashMap<>();
|
||||
private final Object admissionLock = new Object();
|
||||
/**
|
||||
* 待删对象的本地落盘日志:内存队列重启即丢,对应对象会一直残留在桶里直到生命周期过期。
|
||||
* 入队时追加、每轮重试后按 pending 收敛重写、启动时回放。
|
||||
*/
|
||||
private volatile PendingDeleteJournal journal;
|
||||
|
||||
/**
|
||||
* 启动回放:把上次进程残留的待删对象重新入队(容量准入、去重语义与在线入队一致),
|
||||
* 随后由 {@link #retryPendingDeletes()} 的周期任务继续重试。
|
||||
*/
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void replayJournalOnStartup() {
|
||||
if (!properties.isDeleteRetryEnabled()) {
|
||||
return;
|
||||
}
|
||||
List<String> objectKeys = journal().readAll();
|
||||
if (objectKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
int replayed = 0;
|
||||
for (String objectKey : objectKeys) {
|
||||
enqueue(objectKey, null);
|
||||
replayed++;
|
||||
}
|
||||
log.info("[rustfs] 启动回放待删对象 journalSize={} replayed={} pending={}",
|
||||
objectKeys.size(), replayed, pending.size());
|
||||
}
|
||||
|
||||
private PendingDeleteJournal journal() {
|
||||
PendingDeleteJournal local = journal;
|
||||
if (local != null) {
|
||||
return local;
|
||||
}
|
||||
synchronized (admissionLock) {
|
||||
if (journal == null) {
|
||||
Path path = Path.of(System.getProperty("java.io.tmpdir"),
|
||||
"aiimage-delete-journal", "rustfs-delete-retry.log");
|
||||
journal = new PendingDeleteJournal(path, Math.max(1, properties.getDeleteRetryQueueCapacity()));
|
||||
}
|
||||
return journal;
|
||||
}
|
||||
}
|
||||
|
||||
public void enqueue(String objectKey, Throwable cause) {
|
||||
if (!properties.isDeleteRetryEnabled() || objectKey == null || objectKey.isBlank()) {
|
||||
@@ -97,6 +143,8 @@ public class RustfsDeleteRetryService {
|
||||
}
|
||||
retryNextInvocation.forEach(this::requeue);
|
||||
if (processed > 0) {
|
||||
// 收敛落盘:成功删除的对象从日志移除,仍失败/待重试的保留
|
||||
journal().rewrite(pending.keySet());
|
||||
log.info("[rustfs] delete retry batch completed processed={} success={} failed={} pending={}",
|
||||
processed, success, failed, pending.size());
|
||||
}
|
||||
@@ -124,6 +172,8 @@ public class RustfsDeleteRetryService {
|
||||
}
|
||||
item = new RetryItem(objectKey);
|
||||
pending.put(objectKey, item);
|
||||
// 只在"新准入"时记账:重写以 pending 为准,重复失败无需重复追加
|
||||
journal().record(objectKey);
|
||||
return item;
|
||||
}
|
||||
}
|
||||
|
||||
+28
-3
@@ -1,6 +1,9 @@
|
||||
package com.nanri.aiimage.modules.imagevideo.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
|
||||
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.imagevideo.model.dto.DouyinCopyRequest;
|
||||
import com.nanri.aiimage.modules.imagevideo.model.dto.ImageVideoSecretSaveRequest;
|
||||
import com.nanri.aiimage.modules.imagevideo.model.dto.ImageVideoVoiceCloneRequest;
|
||||
@@ -17,6 +20,7 @@ import com.nanri.aiimage.modules.imagevideo.service.ImageVideoAsyncTaskService;
|
||||
import com.nanri.aiimage.modules.imagevideo.service.ImageVideoSecretService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
@@ -36,21 +40,42 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
public class ImageVideoController {
|
||||
|
||||
private final ImageVideoCozeService imageVideoCozeService;
|
||||
private final AdminAuthSupport adminAuthSupport;
|
||||
private final ImageVideoSecretService imageVideoSecretService;
|
||||
private final ImageVideoAsyncTaskService imageVideoAsyncTaskService;
|
||||
|
||||
@GetMapping("/secrets")
|
||||
@Operation(summary = "查询视频密钥配置状态")
|
||||
public ApiResponse<ImageVideoSecretStatusVo> secretStatus(@RequestParam("user_id") Long userId) {
|
||||
return ApiResponse.success(imageVideoSecretService.status(userId));
|
||||
public ApiResponse<ImageVideoSecretStatusVo> secretStatus(@RequestParam("user_id") Long userId,
|
||||
HttpServletRequest request) {
|
||||
return ApiResponse.success(imageVideoSecretService.status(resolveSelfUserId(request, userId)));
|
||||
}
|
||||
|
||||
@PutMapping("/secrets")
|
||||
@Operation(summary = "保存视频密钥配置")
|
||||
public ApiResponse<ImageVideoSecretStatusVo> saveSecrets(@Valid @RequestBody ImageVideoSecretSaveRequest request) {
|
||||
public ApiResponse<ImageVideoSecretStatusVo> saveSecrets(@Valid @RequestBody ImageVideoSecretSaveRequest request,
|
||||
HttpServletRequest httpRequest) {
|
||||
resolveSelfUserId(httpRequest, request.getUserId());
|
||||
return ApiResponse.success(imageVideoSecretService.save(request));
|
||||
}
|
||||
|
||||
/**
|
||||
* 2026-09 全维度审查 A7:视频密钥接口此前匿名可达,且 userId 取自请求体/查询串
|
||||
* ——任何人可读写他人密钥配置。现要求登录身份(或可信内部代理),
|
||||
* 且只允许操作自己(超管例外,保留后台代查能力)。
|
||||
*/
|
||||
private Long resolveSelfUserId(HttpServletRequest request, Long requestedUserId) {
|
||||
AdminUserEntity me = adminAuthSupport.requireUserOrInternal(request);
|
||||
if (requestedUserId == null || requestedUserId <= 0) {
|
||||
throw new BusinessException("用户 id 不合法");
|
||||
}
|
||||
boolean superAdmin = "super_admin".equals(adminAuthSupport.currentRole(me));
|
||||
if (!superAdmin && !requestedUserId.equals(me.getId())) {
|
||||
throw new BusinessException(403, "只能操作自己的视频密钥配置");
|
||||
}
|
||||
return requestedUserId;
|
||||
}
|
||||
|
||||
@PostMapping("/douyin-copy")
|
||||
@Operation(summary = "抖音文案识别和仿写")
|
||||
public ApiResponse<ImageVideoAsyncTaskVo> douyinCopy(@Valid @RequestBody DouyinCopyRequest request) {
|
||||
|
||||
+6
-18
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.notification.service;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.config.NotificationProperties;
|
||||
import com.nanri.aiimage.common.module.TaskModuleRegistry;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||
@@ -54,24 +55,11 @@ public class NotificationScanScheduler {
|
||||
|
||||
private static final DateTimeFormatter HOUR_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHH");
|
||||
|
||||
/** 任务模块类型 → 中文名(与 TaskHeartbeatService 的模块常量对齐)。 */
|
||||
private static final Map<String, String> MODULE_LABELS = Map.ofEntries(
|
||||
Map.entry("APPEARANCE_PATENT", "外观专利检测"),
|
||||
Map.entry("SIMILAR_ASIN", "货源查询"),
|
||||
Map.entry("PATROL_DELETE", "巡店删除"),
|
||||
Map.entry("PRICE_TRACK", "跟价"),
|
||||
Map.entry("PRODUCT_RISK_RESOLVE", "商品风险解决"),
|
||||
Map.entry("QUERY_ASIN", "查询ASIN"),
|
||||
Map.entry("WITHDRAW", "取款"),
|
||||
Map.entry("SHOP_DATA_CRAWL", "店铺数据抓取"),
|
||||
Map.entry("SHOP_MATCH", "定时匹配"),
|
||||
Map.entry("BRAND", "品牌检测"),
|
||||
Map.entry("COLLECT_DATA", "采集数据"),
|
||||
Map.entry("DELETE_BRAND", "删除ASIN"),
|
||||
Map.entry("SPLIT", "数据拆分"),
|
||||
Map.entry("CONVERT", "格式转换"),
|
||||
Map.entry("PUBLISH", "上架"),
|
||||
Map.entry("DEDUPE", "数据去重"));
|
||||
/**
|
||||
* 任务模块类型 → 中文名:直接取模块注册表(G6),新增模块只需在
|
||||
* {@code TaskModuleRegistry} 追加一行,不再需要同步这份表。
|
||||
*/
|
||||
private static final Map<String, String> MODULE_LABELS = TaskModuleRegistry.labels();
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
|
||||
|
||||
+3
-3
@@ -13,9 +13,9 @@ import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteResolveService
|
||||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.enums.ParameterIn;
|
||||
|
||||
+3
-3
@@ -10,9 +10,9 @@ import com.nanri.aiimage.modules.patroldelete.model.entity.PatrolDeleteShopCandi
|
||||
import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteConditionVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.nanri.aiimage.modules.patroldelete.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 巡店删除 的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>心跳逻辑(缓存刷新 / 任务缓存回写)从 task 模块收回本模块,task 侧只依赖 SPI 接口,
|
||||
* 消除 task → 业务模块的编译期依赖。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PatrolDeleteTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final PatrolDeleteTaskCacheService cacheService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return "PATROL_DELETE";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
cacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveTaskCache(FileTaskEntity task) {
|
||||
cacheService.saveTaskCache(task);
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -21,7 +21,7 @@ import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteCreateTaskVo;
|
||||
import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteHistoryVo;
|
||||
import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteResultItemVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 跟价 的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>心跳逻辑(缓存刷新 / 任务缓存回写)从 task 模块收回本模块,task 侧只依赖 SPI 接口,
|
||||
* 消除 task → 业务模块的编译期依赖。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PriceTrackTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final PriceTrackTaskCacheService cacheService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return "PRICE_TRACK";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
cacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveTaskCache(FileTaskEntity task) {
|
||||
cacheService.saveTaskCache(task);
|
||||
}
|
||||
}
|
||||
+6
-6
@@ -8,14 +8,14 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCreateTaskRequ
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskTaskBatchRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCreateTaskVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskHistoryVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskHistoryVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskPendingDeleteVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskBatchVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskResolveService;
|
||||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
package com.nanri.aiimage.modules.productrisk.model.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
|
||||
+1
@@ -5,6 +5,7 @@ import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskResultItemVo;
|
||||
|
||||
@Data
|
||||
@Schema(description = "创建任务响应:含任务 id 与各店初始快照(含 resultId),用于推队列与前端展示")
|
||||
|
||||
+4
-4
@@ -12,10 +12,10 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequ
|
||||
import com.nanri.aiimage.modules.productrisk.model.entity.ProductRiskCountryPrefEntity;
|
||||
import com.nanri.aiimage.modules.productrisk.model.entity.ProductRiskShopCandidateEntity;
|
||||
import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCountryPreferenceVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.nanri.aiimage.modules.productrisk.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 商品风险解决 的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>心跳逻辑(缓存刷新 / 任务缓存回写)从 task 模块收回本模块,task 侧只依赖 SPI 接口,
|
||||
* 消除 task → 业务模块的编译期依赖。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ProductRiskTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final ProductRiskTaskCacheService cacheService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return "PRODUCT_RISK_RESOLVE";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
cacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveTaskCache(FileTaskEntity task) {
|
||||
cacheService.saveTaskCache(task);
|
||||
}
|
||||
}
|
||||
+7
-7
@@ -13,14 +13,14 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskShopPayloadDto
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.entity.ProductRiskShopCandidateEntity;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCreateTaskVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskHistoryVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskResultItemVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskDetailVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskHistoryVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskResultItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskTaskBatchVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskTaskDetailVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskPendingDeleteVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskItemVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskTaskItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.nanri.aiimage.modules.publish.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 上架模块的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>心跳前做任务归属校验(非本实例拥有的任务拒绝续命),心跳刷新走模块自己的进度服务;
|
||||
* 本模块没有独立的任务缓存回写(历史行为保持一致)。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PublishTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final PublishTaskService publishTaskService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return PublishTaskService.MODULE_TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void ensureOwnership(FileTaskEntity task, String operation) {
|
||||
publishTaskService.ensureTaskOwnedByCurrentInstance(task, operation);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
publishTaskService.touchHeartbeat(taskId, request);
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -3,9 +3,9 @@ package com.nanri.aiimage.modules.queryasin.controller;
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCreateTaskRequest;
|
||||
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinTaskBatchRequest;
|
||||
|
||||
+3
-3
@@ -4,9 +4,9 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.modules.queryasin.mapper.QueryAsinShopCandidateMapper;
|
||||
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto;
|
||||
import com.nanri.aiimage.modules.queryasin.model.entity.QueryAsinShopCandidateEntity;
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.nanri.aiimage.modules.queryasin.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 查询ASIN 的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>心跳逻辑(缓存刷新 / 任务缓存回写)从 task 模块收回本模块,task 侧只依赖 SPI 接口,
|
||||
* 消除 task → 业务模块的编译期依赖。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class QueryAsinTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final QueryAsinTaskCacheService cacheService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return "QUERY_ASIN";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
cacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveTaskCache(FileTaskEntity task) {
|
||||
cacheService.saveTaskCache(task);
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -21,7 +21,7 @@ import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinCreateTaskVo;
|
||||
import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinHistoryVo;
|
||||
import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinResultItemVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
|
||||
+4
-4
@@ -4,10 +4,10 @@ import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCountryPreferenceVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskBatchRequest;
|
||||
|
||||
+14
@@ -19,6 +19,20 @@ public interface ShopDataCrawlItemMapper extends BaseMapper<ShopDataCrawlItemEnt
|
||||
""")
|
||||
int deleteBatch(@Param("shopName") String shopName, @Param("businessDate") java.time.LocalDate businessDate);
|
||||
|
||||
/**
|
||||
* 按批次日期分批清理历史明细(保留期用)。
|
||||
*
|
||||
* <p>该表是增长最快的表(店 × 日 × 国 × SKU)且此前**没有任何清理策略**(2026-09 全维度审查)。
|
||||
* 撞款扫描只读每店最新批次,历史批次仅用于追溯,过期即可删;分批删避免长事务与大范围行锁,
|
||||
* 走 idx_shop_data_crawl_item_biz_date 范围扫描。
|
||||
*/
|
||||
@Delete("""
|
||||
DELETE FROM biz_shop_data_crawl_item
|
||||
WHERE business_date < #{cutoff}
|
||||
LIMIT #{batchSize}
|
||||
""")
|
||||
int deleteOlderThanBatch(@Param("cutoff") java.time.LocalDate cutoff, @Param("batchSize") int batchSize);
|
||||
|
||||
/** 删除归属某累计档的全部明细行(管理端真删店铺数据记录时清理,避免留下悬空引用)。 */
|
||||
@Delete("""
|
||||
DELETE FROM biz_shop_data_crawl_item
|
||||
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlItemMapper;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDate;
|
||||
|
||||
/**
|
||||
* 采集明细表(biz_shop_data_crawl_item)的保留期清理(2026-09 全维度审查)。
|
||||
*
|
||||
* <p>该表按「店 × 日 × 国 × SKU」增长,是全库增长最快的表,此前只有"按店按日替换当天批次"
|
||||
* 与"删累计档时连带删除"两条删除路径,历史批次永不清理。撞款扫描只读每店**最新**批次,
|
||||
* 因此保留期内(默认 30 天)之外的批次可安全删除。
|
||||
*
|
||||
* <p>双节点用 job 锁保证单实例执行;分批删除并限制单轮批次数,避免一次跑太久占住锁。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class ShopDataCrawlItemRetentionService {
|
||||
|
||||
/** 单轮最多删除的批次数(每批 batchSize 行),剩余留给下一轮。 */
|
||||
static final int MAX_BATCHES_PER_RUN = 20;
|
||||
|
||||
private final ShopDataCrawlItemMapper itemMapper;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
@Value("${aiimage.shop-data-crawl.item-retention-days:30}")
|
||||
private int retentionDays = 30;
|
||||
|
||||
@Value("${aiimage.shop-data-crawl.item-retention-batch-size:5000}")
|
||||
private int retentionBatchSize = 5000;
|
||||
|
||||
@Scheduled(cron = "${aiimage.shop-data-crawl.item-retention-cron:0 30 3 * * *}")
|
||||
public void purgeExpiredItems() {
|
||||
int days = Math.max(1, retentionDays);
|
||||
int batchSize = Math.max(100, retentionBatchSize);
|
||||
LocalDate cutoff = LocalDate.now().minusDays(days);
|
||||
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
distributedJobLockService.tryLock("shop-data-crawl:item-retention", Duration.ofMinutes(15));
|
||||
if (lockHandle == null) {
|
||||
log.info("[shop-data-crawl] item retention skipped because another instance holds the lock");
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
int totalDeleted = 0;
|
||||
int batches = 0;
|
||||
while (batches < MAX_BATCHES_PER_RUN) {
|
||||
int deleted = itemMapper.deleteOlderThanBatch(cutoff, batchSize);
|
||||
batches++;
|
||||
totalDeleted += deleted;
|
||||
if (deleted < batchSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (totalDeleted > 0) {
|
||||
log.info("[shop-data-crawl] item retention deleted={} cutoff={} retentionDays={} batches={}",
|
||||
totalDeleted, cutoff, days, batches);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[shop-data-crawl] item retention failed cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/** 供测试读取当前生效的保留天数。 */
|
||||
int effectiveRetentionDays() {
|
||||
return Math.max(1, retentionDays);
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -12,10 +12,10 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequ
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlCountryPrefEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlShopCandidateEntity;
|
||||
import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCountryPreferenceVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.modules.shopkey.service.ShopManageService;
|
||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
|
||||
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 店铺数据采集的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>除常规缓存刷新外,本模块在心跳前要做**任务归属校验**:非本实例拥有的任务拒绝心跳,
|
||||
* 避免另一实例误把在途任务的 updated_at 续命。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ShopDataCrawlTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final ShopDataCrawlTaskCacheService cacheService;
|
||||
private final ShopDataCrawlTaskService shopDataCrawlTaskService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return "SHOP_DATA_CRAWL";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void ensureOwnership(FileTaskEntity task, String operation) {
|
||||
shopDataCrawlTaskService.ensureTaskOwnedByCurrentInstance(task, operation);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
cacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveTaskCache(FileTaskEntity task) {
|
||||
cacheService.saveTaskCache(task);
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -27,7 +27,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
|
||||
+17
-8
@@ -16,7 +16,18 @@ import java.util.List;
|
||||
@Mapper
|
||||
public interface ShopDuplicateCheckItemMapper {
|
||||
|
||||
/** 每店最新 business_date 批次的全部明细行(撞款统计直接数据源)。 */
|
||||
/**
|
||||
* 全量店铺名(去重,走 idx_shop_data_crawl_item_shop_date 的索引扫描,不读行数据)。
|
||||
* 撞款扫描按店逐批取数用,避免一次性把全部明细拉进堆。
|
||||
*/
|
||||
@Select("SELECT DISTINCT shop_name FROM biz_shop_data_crawl_item")
|
||||
List<String> selectDistinctShopNames();
|
||||
|
||||
/**
|
||||
* 单店最新 business_date 批次的明细行。
|
||||
* 内层 MAX(business_date) 走 (shop_name, business_date) 索引(松散索引扫描),
|
||||
* 外层按索引前缀取该批次,只读目标店的行,不再全表 GROUP BY + JOIN。
|
||||
*/
|
||||
@Select("""
|
||||
SELECT i.shop_name AS shopName,
|
||||
i.country AS country,
|
||||
@@ -25,14 +36,12 @@ public interface ShopDuplicateCheckItemMapper {
|
||||
i.price AS price,
|
||||
i.brand AS brand
|
||||
FROM biz_shop_data_crawl_item i
|
||||
JOIN (
|
||||
SELECT shop_name, MAX(business_date) AS max_date
|
||||
FROM biz_shop_data_crawl_item
|
||||
GROUP BY shop_name
|
||||
) latest
|
||||
ON latest.shop_name = i.shop_name AND latest.max_date = i.business_date
|
||||
WHERE i.shop_name = #{shopName}
|
||||
AND i.business_date = (
|
||||
SELECT MAX(business_date) FROM biz_shop_data_crawl_item WHERE shop_name = #{shopName}
|
||||
)
|
||||
""")
|
||||
List<ShopLatestItemRowDto> selectLatestShopItems();
|
||||
List<ShopLatestItemRowDto> selectLatestItemsByShop(@Param("shopName") String shopName);
|
||||
|
||||
/** 幂等判定:该店该日是否已有明细批次。 */
|
||||
@Select("""
|
||||
|
||||
+22
-6
@@ -232,23 +232,39 @@ public class ShopDataDuplicateCheckScanService {
|
||||
* 重复检查基数 = 已采集落库的店(超管应见 8 家)。
|
||||
*/
|
||||
private List<ShopParsed> collectShopParsed(DistributedJobLockService.LockHandle lock) {
|
||||
List<ShopLatestItemRowDto> rows = itemMapper.selectLatestShopItems();
|
||||
if (rows.isEmpty()) {
|
||||
// 按店逐批取数(2026-09 全维度审查 C7):此前一条 full-table GROUP BY + JOIN 把全部明细
|
||||
// 一次性拉进堆,而明细表随采集天数线性增长(每日每店一批)。改为先走索引扫描拿店名,
|
||||
// 再逐店按 (shop_name, business_date) 索引取「该店最新批次」,
|
||||
// 堆内同时只驻留单店行 + 已构建结构。
|
||||
List<String> rawShopNames = itemMapper.selectDistinctShopNames();
|
||||
if (rawShopNames.isEmpty()) {
|
||||
log.info("[shop-duplicate-check] 明细表暂无采集数据,本次扫描不出库");
|
||||
return List.of();
|
||||
}
|
||||
// 店铺名去重(分组标签来自店铺管理 biz_shop_manage,与页面口径一致)。
|
||||
Set<String> shopNames = new LinkedHashSet<>();
|
||||
for (ShopLatestItemRowDto row : rows) {
|
||||
if (row != null) {
|
||||
shopNames.add(normalizeShopName(row.shopName()));
|
||||
for (String rawShopName : rawShopNames) {
|
||||
if (rawShopName != null && !rawShopName.isBlank()) {
|
||||
shopNames.add(normalizeShopName(rawShopName));
|
||||
}
|
||||
}
|
||||
Map<String, String> groupLabels = new HashMap<>();
|
||||
for (ShopGroupLabelDto dto : sourceMapper.selectGroupLabels(new ArrayList<>(shopNames))) {
|
||||
groupLabels.put(shopKey(dto.getShopName()), dto.getGroupName() == null ? "" : dto.getGroupName());
|
||||
}
|
||||
List<ShopParsed> parsedShops = buildParsedShops(rows, groupLabels);
|
||||
List<ShopParsed> parsedShops = new ArrayList<>();
|
||||
for (String rawShopName : rawShopNames) {
|
||||
if (rawShopName == null || rawShopName.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
List<ShopLatestItemRowDto> rows = itemMapper.selectLatestItemsByShop(rawShopName);
|
||||
if (rows.isEmpty()) {
|
||||
// 该店历史批次存在但最新批次无行:跳过(不影响其它店)
|
||||
log.info("[shop-duplicate-check] 店铺最新批次无明细行,跳过 shopName={}", rawShopName);
|
||||
continue;
|
||||
}
|
||||
parsedShops.addAll(buildParsedShops(rows, groupLabels));
|
||||
}
|
||||
log.info("[shop-duplicate-check] 明细数据源扫描:{} 家店铺,{} 行明细", parsedShops.size(), rawRowsCount(parsedShops));
|
||||
return parsedShops;
|
||||
}
|
||||
|
||||
+176
-68
@@ -5,6 +5,7 @@ import com.baomidou.mybatisplus.core.toolkit.support.SFunction;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.modules.shopkey.mapper.SkipPriceAsinMapper;
|
||||
import com.nanri.aiimage.modules.shopkey.mapper.ShopManageMapper;
|
||||
import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinCreateRequest;
|
||||
@@ -19,12 +20,8 @@ import com.nanri.aiimage.modules.shopkey.model.vo.SkipPriceAsinPageVo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -33,7 +30,6 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
@@ -485,45 +481,136 @@ public class SkipPriceAsinService {
|
||||
private void processImportFile(File tempFile, String filename, Long groupId, String shopName,
|
||||
boolean deleteMode, QueryAsinImportProgressVo progress) throws Exception {
|
||||
validateExcelFilename(filename);
|
||||
try (FileInputStream inputStream = new FileInputStream(tempFile);
|
||||
Workbook workbook = WorkbookFactory.create(inputStream)) {
|
||||
Sheet sheet = workbook.getNumberOfSheets() == 0 ? null : workbook.getSheetAt(0);
|
||||
if (sheet == null) {
|
||||
throw new BusinessException("Excel 为空");
|
||||
// 2026-09 全维度审查 C5:改逐行流式解析(ExcelStreamReader → EasyExcel SAX)。
|
||||
// 此前 WorkbookFactory.create 把整表读成 DOM(50 万行约 1~2GB 堆),且行数上限在整表加载
|
||||
// 完之后才校验(等于没有防线)。现在内存只保留表头映射与计数(不随文件行数增长),
|
||||
// 行数上限在迭代过程中即时生效,进度总行数用 onSheetTotal 的近似值。
|
||||
ImportSession session = new ImportSession(progress, groupId, shopName, deleteMode);
|
||||
ExcelStreamReader.readFirstSheet(tempFile, new ExcelStreamReader.SheetRowHandler() {
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
|
||||
session.acceptHeader(headerMap);
|
||||
}
|
||||
HeaderMapping mapping = resolveHeaderMapping(sheet);
|
||||
|
||||
@Override
|
||||
public void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) {
|
||||
session.acceptApproximateTotalRows(approximateTotalRows);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int rowIndex,
|
||||
Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
|
||||
session.acceptRow(rowIndex, rowMap);
|
||||
}
|
||||
});
|
||||
session.finish();
|
||||
}
|
||||
|
||||
/**
|
||||
* 流式导入的累加器:表头映射 + 计数与进度,内存只与单行/单批写入相关,不随文件行数增长。
|
||||
* 行号沿用 EasyExcel 的 sheet 绝对 0 基行号(与旧 POI Row.getRowNum() 口径一致)。
|
||||
*/
|
||||
private final class ImportSession {
|
||||
|
||||
private final QueryAsinImportProgressVo progress;
|
||||
private final Long groupId;
|
||||
private final String shopName;
|
||||
private final boolean deleteMode;
|
||||
private final ImportCounters counters = new ImportCounters();
|
||||
private Map<Integer, String> firstHeaderMap = Map.of();
|
||||
private HeaderMapping mapping;
|
||||
private Integer approximateTotalRows;
|
||||
private boolean headerSeen;
|
||||
private int processedRows;
|
||||
|
||||
private ImportSession(QueryAsinImportProgressVo progress, Long groupId, String shopName, boolean deleteMode) {
|
||||
this.progress = progress;
|
||||
this.groupId = groupId;
|
||||
this.shopName = shopName;
|
||||
this.deleteMode = deleteMode;
|
||||
}
|
||||
|
||||
private void acceptHeader(Map<Integer, String> headerMap) {
|
||||
this.firstHeaderMap = headerMap == null ? Map.of() : headerMap;
|
||||
this.headerSeen = true;
|
||||
}
|
||||
|
||||
private void acceptApproximateTotalRows(Integer approximateTotalRows) {
|
||||
this.approximateTotalRows = approximateTotalRows;
|
||||
}
|
||||
|
||||
private void acceptRow(int rowIndex, Map<Integer, String> rowMap) {
|
||||
if (!headerSeen) {
|
||||
// 表头行整行为空时 EasyExcel 不上报表头回调:与旧实现 sheet.getRow(0) == null 一致
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
if (mapping == null) {
|
||||
// 两行表头格式下,第 1 行(0 基)就是第二行表头,由首个 onRow 回调提供;
|
||||
// 第 1 行为空行时 EasyExcel 不上报,等价于旧实现 secondHeader 为空行。
|
||||
Map<Integer, String> secondHeaderMap = rowIndex == 1 ? rowMap : null;
|
||||
long resolveStartedAt = System.nanoTime();
|
||||
mapping = resolveHeaderMapping(firstHeaderMap, secondHeaderMap);
|
||||
if (mapping.asinColumns().isEmpty()) {
|
||||
throw new BusinessException("Excel 至少需要包含一个国家的 ASIN 列");
|
||||
}
|
||||
|
||||
int firstDataRow = mapping.firstDataRowIndex();
|
||||
int lastRow = Math.max(sheet.getLastRowNum(), firstDataRow - 1);
|
||||
int totalRows = Math.max(0, lastRow - firstDataRow + 1);
|
||||
if (maxImportRows > 0 && totalRows > maxImportRows) {
|
||||
updateTotalRows();
|
||||
log.info("[skip-price-asin-import] 流式解析表头 firstDataRowIndex={} asinColumns={} priceColumns={} "
|
||||
+ "approxTotalRows={} totalRows={} resolveMs={} deleteMode={}",
|
||||
mapping.firstDataRowIndex(), mapping.asinColumns(), mapping.priceColumns(),
|
||||
approximateTotalRows, progress.getTotalRows(), elapsedMs(resolveStartedAt), deleteMode);
|
||||
if (mapping.firstDataRowIndex() > rowIndex) {
|
||||
// 该行是第二行表头(未进入数据区):不作为数据行处理
|
||||
return;
|
||||
}
|
||||
}
|
||||
processedRows++;
|
||||
if (maxImportRows > 0 && processedRows > maxImportRows) {
|
||||
// 流式下无法前置知道总行数:超限即中断(旧实现在整表加载后才校验,实际是 OOM 之后才失败)
|
||||
throw new BusinessException("导入行数超过上限: " + maxImportRows);
|
||||
}
|
||||
progress.setTotalRows(totalRows);
|
||||
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
ImportCounters counters = new ImportCounters();
|
||||
for (int rowIndex = firstDataRow; rowIndex <= lastRow; rowIndex++) {
|
||||
Row row = sheet.getRow(rowIndex);
|
||||
if (row == null || isBlankRow(row, formatter)) {
|
||||
if (isBlankRow(rowMap)) {
|
||||
counters.skippedCount++;
|
||||
updateImportProgress(progress, counters, rowIndex - firstDataRow + 1);
|
||||
continue;
|
||||
updateImportProgress(progress, counters, processedRows);
|
||||
return;
|
||||
}
|
||||
if (deleteMode) {
|
||||
consumeDeleteImportRow(row, formatter, mapping, groupId, shopName, counters);
|
||||
consumeDeleteImportRow(rowMap, mapping, groupId, shopName, counters);
|
||||
} else {
|
||||
consumeAddImportRow(row, formatter, mapping, groupId, shopName, counters);
|
||||
consumeAddImportRow(rowMap, mapping, groupId, shopName, counters);
|
||||
}
|
||||
updateImportProgress(progress, counters, rowIndex - firstDataRow + 1);
|
||||
updateImportProgress(progress, counters, processedRows);
|
||||
}
|
||||
|
||||
/** 近似总行数换算为数据行数(近似值 = 含表头的物理末行行号,减去数据起始行号)。 */
|
||||
private void updateTotalRows() {
|
||||
if (progress == null || approximateTotalRows == null || approximateTotalRows <= 0) {
|
||||
return;
|
||||
}
|
||||
progress.setTotalRows(Math.max(0, approximateTotalRows - mapping.firstDataRowIndex()));
|
||||
}
|
||||
|
||||
private void finish() {
|
||||
if (!headerSeen) {
|
||||
// 空表(无任何行)或表头行缺失:与旧实现 sheet.getRow(0) == null 一致
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
Integer totalRows = progress == null ? null : progress.getTotalRows();
|
||||
if (progress != null && (totalRows == null || processedRows > totalRows)) {
|
||||
// 近似总行数缺失(或与实际物理行数不一致)时用实际处理行数兜底,避免展示成 x/0
|
||||
progress.setTotalRows(processedRows);
|
||||
}
|
||||
log.info("[skip-price-asin-import] 流式导入完成 processedRows={} totalRows={} asinCount={} "
|
||||
+ "insertedCount={} deletedCount={} skippedCount={} deleteMode={}",
|
||||
processedRows, progress == null ? null : progress.getTotalRows(), counters.asinCount,
|
||||
counters.insertedCount, counters.deletedCount, counters.skippedCount, deleteMode);
|
||||
}
|
||||
}
|
||||
|
||||
private void consumeAddImportRow(Row row, DataFormatter formatter, HeaderMapping mapping,
|
||||
private long elapsedMs(long startedAtNanos) {
|
||||
return (System.nanoTime() - startedAtNanos) / 1_000_000;
|
||||
}
|
||||
|
||||
private void consumeAddImportRow(Map<Integer, String> rowMap, HeaderMapping mapping,
|
||||
Long groupId, String shopName, ImportCounters counters) {
|
||||
boolean acceptedAny = false;
|
||||
boolean sawAnyAsin = false;
|
||||
@@ -532,7 +619,7 @@ public class SkipPriceAsinService {
|
||||
entity.setShopName(shopName);
|
||||
for (Map.Entry<String, Integer> entry : mapping.asinColumns().entrySet()) {
|
||||
String country = entry.getKey();
|
||||
String asin = normalizeBlank(cellText(row, entry.getValue(), formatter)).toUpperCase(Locale.ROOT);
|
||||
String asin = normalizeBlank(cellText(rowMap, entry.getValue())).toUpperCase(Locale.ROOT);
|
||||
if (asin.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
@@ -543,7 +630,7 @@ public class SkipPriceAsinService {
|
||||
continue;
|
||||
}
|
||||
BigDecimal minimumPrice = parseOptionalMinimumPrice(
|
||||
cellText(row, mapping.priceColumns().get(country), formatter));
|
||||
cellText(rowMap, mapping.priceColumns().get(country)));
|
||||
if (existsCountryAsin(groupId, shopName, country, asin)) {
|
||||
counters.skippedCount++;
|
||||
continue;
|
||||
@@ -561,13 +648,13 @@ public class SkipPriceAsinService {
|
||||
skipPriceAsinMapper.insert(entity);
|
||||
}
|
||||
|
||||
private void consumeDeleteImportRow(Row row, DataFormatter formatter, HeaderMapping mapping,
|
||||
private void consumeDeleteImportRow(Map<Integer, String> rowMap, HeaderMapping mapping,
|
||||
Long groupId, String shopName, ImportCounters counters) {
|
||||
boolean acceptedAny = false;
|
||||
boolean sawAnyAsin = false;
|
||||
for (Map.Entry<String, Integer> entry : mapping.asinColumns().entrySet()) {
|
||||
String country = entry.getKey();
|
||||
String asin = normalizeBlank(cellText(row, entry.getValue(), formatter)).toUpperCase(Locale.ROOT);
|
||||
String asin = normalizeBlank(cellText(rowMap, entry.getValue())).toUpperCase(Locale.ROOT);
|
||||
if (asin.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
@@ -646,27 +733,43 @@ public class SkipPriceAsinService {
|
||||
return skipPriceAsinMapper.selectOne(query.last("LIMIT 1"));
|
||||
}
|
||||
|
||||
private HeaderMapping resolveHeaderMapping(Sheet sheet) {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
Row firstHeader = sheet.getRow(0);
|
||||
Row secondHeader = sheet.getRow(1);
|
||||
if (firstHeader == null) {
|
||||
/**
|
||||
* 由流式表头(第 0 行 headerMap)与第二行表头(两行表头格式时的首个 onRow 回调,可为 null)解析列映射。
|
||||
* 保持与旧 DOM 实现完全相同的列别名匹配与 firstDataRowIndex 语义:
|
||||
* 两行表头(国家行 + 字段行)匹配到 ASIN 列时 firstDataRowIndex = 2;
|
||||
* 否则退回单行表头(国家与字段同列,如「德国ASIN」)匹配,firstDataRowIndex = 1。
|
||||
*/
|
||||
private HeaderMapping resolveHeaderMapping(Map<Integer, String> firstHeaderMap,
|
||||
Map<Integer, String> secondHeaderMap) {
|
||||
if (firstHeaderMap == null || firstHeaderMap.isEmpty()) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
|
||||
int maxColumn = Math.max(lastCellNum(firstHeader), lastCellNum(secondHeader));
|
||||
int maxColumn = Math.max(lastColumnCount(firstHeaderMap), lastColumnCount(secondHeaderMap));
|
||||
Map<String, Integer> asinColumns = new LinkedHashMap<>();
|
||||
Map<String, Integer> priceColumns = new LinkedHashMap<>();
|
||||
// 兼容合并单元格表头:EasyExcel SAX 不回填被合并覆盖的单元格文本(旧 DOM 的 cellTextWithMerged
|
||||
// 会读到主单元格),这里用「字段列未显式标注国家时继承左侧最近国家列」还原同样的列映射结果。
|
||||
String carriedCountry = "";
|
||||
List<Integer> carriedColumns = new ArrayList<>();
|
||||
for (int columnIndex = 0; columnIndex < maxColumn; columnIndex++) {
|
||||
String countryText = cellTextWithMerged(sheet, firstHeader, columnIndex, formatter);
|
||||
String fieldText = cellText(secondHeader, columnIndex, formatter);
|
||||
String countryText = cellText(firstHeaderMap, columnIndex);
|
||||
String fieldText = cellText(secondHeaderMap, columnIndex);
|
||||
String country = normalizeCountryAlias(countryText);
|
||||
String fieldKey = normalizeHeaderKey(fieldText);
|
||||
if (country.isEmpty()) {
|
||||
country = normalizeCountryAlias(fieldText);
|
||||
fieldKey = normalizeHeaderKey(countryText);
|
||||
}
|
||||
if (country.isEmpty() || !SUPPORTED_COUNTRIES.contains(country)) {
|
||||
if (country.isEmpty() && !carriedCountry.isEmpty()) {
|
||||
String inheritedKey = normalizeHeaderKey(fieldText);
|
||||
if (isAsinHeader(inheritedKey) || isPriceHeader(inheritedKey)) {
|
||||
country = carriedCountry;
|
||||
fieldKey = inheritedKey;
|
||||
carriedColumns.add(columnIndex);
|
||||
}
|
||||
}
|
||||
if (country.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (isAsinHeader(fieldKey)) {
|
||||
@@ -674,10 +777,11 @@ public class SkipPriceAsinService {
|
||||
} else if (isPriceHeader(fieldKey)) {
|
||||
priceColumns.putIfAbsent(country, columnIndex);
|
||||
}
|
||||
carriedCountry = country;
|
||||
}
|
||||
if (asinColumns.isEmpty()) {
|
||||
for (int columnIndex = 0; columnIndex < maxColumn; columnIndex++) {
|
||||
String headerText = cellText(firstHeader, columnIndex, formatter);
|
||||
String headerText = cellText(firstHeaderMap, columnIndex);
|
||||
String key = normalizeHeaderKey(headerText);
|
||||
String country = normalizeCountryAlias(headerText);
|
||||
if (country.isEmpty()) {
|
||||
@@ -687,13 +791,31 @@ public class SkipPriceAsinService {
|
||||
asinColumns.putIfAbsent(country, columnIndex);
|
||||
}
|
||||
}
|
||||
if (!carriedColumns.isEmpty()) {
|
||||
log.info("[skip-price-asin-import] 表头按合并单元格口径继承国家列 carriedColumns={} 但未匹配到 ASIN 列",
|
||||
carriedColumns);
|
||||
}
|
||||
return new HeaderMapping(asinColumns, priceColumns, 1);
|
||||
}
|
||||
if (!carriedColumns.isEmpty()) {
|
||||
log.info("[skip-price-asin-import] 检测到疑似合并单元格表头,被覆盖列按左侧国家列归属 carriedColumns={}",
|
||||
carriedColumns);
|
||||
}
|
||||
return new HeaderMapping(asinColumns, priceColumns, 2);
|
||||
}
|
||||
|
||||
private int lastCellNum(Row row) {
|
||||
return row == null || row.getLastCellNum() < 0 ? 0 : row.getLastCellNum();
|
||||
/** 流式表头行最大列数(与旧 Row.getLastCellNum() 口径一致:无单元格时为 0)。 */
|
||||
private int lastColumnCount(Map<Integer, String> rowMap) {
|
||||
if (rowMap == null || rowMap.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
int lastColumnCount = 0;
|
||||
for (Integer columnIndex : rowMap.keySet()) {
|
||||
if (columnIndex != null && columnIndex >= lastColumnCount) {
|
||||
lastColumnCount = columnIndex + 1;
|
||||
}
|
||||
}
|
||||
return lastColumnCount;
|
||||
}
|
||||
|
||||
private boolean isAsinHeader(String key) {
|
||||
@@ -705,27 +827,12 @@ public class SkipPriceAsinService {
|
||||
|| key.contains("minprice") || key.equals("price");
|
||||
}
|
||||
|
||||
private String cellTextWithMerged(Sheet sheet, Row row, int columnIndex, DataFormatter formatter) {
|
||||
if (row == null) {
|
||||
/** 流式解析下按列索引取行内文本(归一化口径与旧 cellText(Row, ...) 一致)。 */
|
||||
private String cellText(Map<Integer, String> rowMap, Integer columnIndex) {
|
||||
if (rowMap == null || columnIndex == null || columnIndex < 0) {
|
||||
return "";
|
||||
}
|
||||
int rowIndex = row.getRowNum();
|
||||
for (int i = 0; i < sheet.getNumMergedRegions(); i++) {
|
||||
var region = sheet.getMergedRegion(i);
|
||||
if (region.isInRange(rowIndex, columnIndex)) {
|
||||
Row firstRow = sheet.getRow(region.getFirstRow());
|
||||
return cellText(firstRow, region.getFirstColumn(), formatter);
|
||||
}
|
||||
}
|
||||
return cellText(row, columnIndex, formatter);
|
||||
}
|
||||
|
||||
private String cellText(Row row, Integer columnIndex, DataFormatter formatter) {
|
||||
if (row == null || columnIndex == null || columnIndex < 0) {
|
||||
return "";
|
||||
}
|
||||
Cell cell = row.getCell(columnIndex);
|
||||
return cell == null ? "" : normalizeExcelText(formatter.formatCellValue(cell));
|
||||
return normalizeExcelText(rowMap.get(columnIndex));
|
||||
}
|
||||
|
||||
private String normalizeExcelText(String value) {
|
||||
@@ -798,12 +905,13 @@ public class SkipPriceAsinService {
|
||||
return left.compareTo(right) == 0;
|
||||
}
|
||||
|
||||
private boolean isBlankRow(Row row, DataFormatter formatter) {
|
||||
if (row == null) {
|
||||
/** 该行所有单元格文本是否全空白(归一化口径与旧 isBlankRow(Row, DataFormatter) 一致)。 */
|
||||
private boolean isBlankRow(Map<Integer, String> rowMap) {
|
||||
if (rowMap == null || rowMap.isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
for (int i = 0; i < lastCellNum(row); i++) {
|
||||
if (!cellText(row, i, formatter).isEmpty()) {
|
||||
for (String value : rowMap.values()) {
|
||||
if (!normalizeExcelText(value).isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
+6
-6
@@ -5,12 +5,12 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRe
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskTaskBatchRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskHistoryVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskBatchVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCountryPreferenceVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskHistoryVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchCreateTaskRequest;
|
||||
import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchStageCompleteRequest;
|
||||
import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchSubmitResultRequest;
|
||||
|
||||
+4
-4
@@ -8,10 +8,10 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRe
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCountryPreferenceVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.modules.shopmatch.mapper.ShopMatchCountryPrefMapper;
|
||||
import com.nanri.aiimage.modules.shopmatch.mapper.ShopMatchShopCandidateMapper;
|
||||
import com.nanri.aiimage.modules.shopmatch.model.entity.ShopMatchCountryPrefEntity;
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.nanri.aiimage.modules.shopmatch.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 定时匹配 的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>心跳逻辑(缓存刷新 / 任务缓存回写)从 task 模块收回本模块,task 侧只依赖 SPI 接口,
|
||||
* 消除 task → 业务模块的编译期依赖。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ShopMatchTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final ShopMatchTaskCacheService cacheService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return "SHOP_MATCH";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
cacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveTaskCache(FileTaskEntity task) {
|
||||
cacheService.saveTaskCache(task);
|
||||
}
|
||||
}
|
||||
+7
-7
@@ -9,13 +9,13 @@ import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskHistoryVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskResultItemVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskDetailVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskTaskItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskHistoryVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskResultItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskTaskBatchVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskTaskDetailVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskTaskItemVo;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.vo.SkipPriceAsinPageVo;
|
||||
import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinCountryAsinsDto;
|
||||
import com.nanri.aiimage.modules.shopmatch.mapper.ShopMatchShopCandidateMapper;
|
||||
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* 货源查询(相似 ASIN)的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>本模块的心跳有两处特殊:
|
||||
* 1. 客户端心跳间隔远小于 stale 判定阈值,因此 biz_file_task.updated_at 采用**检查点节流**
|
||||
* (按 similarAsinProperties 配置的间隔刷新),避免每次心跳都写库;
|
||||
* 2. 心跳时顺带刷新结果文件组装作业的租约(避免作业被误判卡死)。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class SimilarAsinTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final SimilarAsinTaskCacheService cacheService;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final SimilarAsinProperties similarAsinProperties;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return "SIMILAR_ASIN";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
cacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onHeartbeat(FileTaskEntity task) {
|
||||
taskFileJobService.touchRunningAssembleJobsIfStale(
|
||||
task.getId(), moduleType(), checkpointIntervalMillis());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Duration checkpointInterval() {
|
||||
return Duration.ofMillis(checkpointIntervalMillis());
|
||||
}
|
||||
|
||||
private long checkpointIntervalMillis() {
|
||||
return Math.max(1_000L, similarAsinProperties.getDbTaskTouchIntervalMillis());
|
||||
}
|
||||
}
|
||||
+6
-7
@@ -4,8 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.ModuleCleanupProperties;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -55,7 +54,7 @@ public class ModuleHistoryCleanupService {
|
||||
private final TaskResultPayloadMapper taskResultPayloadMapper;
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final TaskChunkMapper taskChunkMapper;
|
||||
private final CollectDataItemMapper collectDataItemMapper;
|
||||
private final CollectDataItemCleanupSpi collectDataItemCleanupSpi;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator;
|
||||
private final TransactionTemplate transactionTemplate;
|
||||
@@ -76,7 +75,7 @@ public class ModuleHistoryCleanupService {
|
||||
TaskResultPayloadMapper taskResultPayloadMapper,
|
||||
TaskScopeStateMapper taskScopeStateMapper,
|
||||
TaskChunkMapper taskChunkMapper,
|
||||
CollectDataItemMapper collectDataItemMapper,
|
||||
CollectDataItemCleanupSpi collectDataItemCleanupSpi,
|
||||
DistributedJobLockService distributedJobLockService,
|
||||
TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator,
|
||||
PlatformTransactionManager platformTransactionManager) {
|
||||
@@ -89,7 +88,7 @@ public class ModuleHistoryCleanupService {
|
||||
this.taskResultPayloadMapper = taskResultPayloadMapper;
|
||||
this.taskScopeStateMapper = taskScopeStateMapper;
|
||||
this.taskChunkMapper = taskChunkMapper;
|
||||
this.collectDataItemMapper = collectDataItemMapper;
|
||||
this.collectDataItemCleanupSpi = collectDataItemCleanupSpi;
|
||||
this.distributedJobLockService = distributedJobLockService;
|
||||
this.transientPayloadDeleteOrchestrator = transientPayloadDeleteOrchestrator;
|
||||
this.transactionTemplate = new TransactionTemplate(platformTransactionManager);
|
||||
@@ -218,8 +217,8 @@ public class ModuleHistoryCleanupService {
|
||||
.in(TaskChunkEntity::getTaskId, cleanupTaskIds));
|
||||
|
||||
if (!collectDataTaskIds.isEmpty()) {
|
||||
collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.in(CollectDataItemEntity::getTaskId, collectDataTaskIds));
|
||||
// 采集明细属于 collectdata 模块(G5:消除 task → 业务依赖,走 SPI)
|
||||
collectDataItemCleanupSpi.deleteItemsByTaskIds(collectDataTaskIds);
|
||||
}
|
||||
|
||||
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
|
||||
|
||||
+12
-25
@@ -3,8 +3,7 @@ package com.nanri.aiimage.modules.task.service;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.BrandTaskStaleRepairSpi;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@@ -32,16 +31,15 @@ public class StaleTaskRepairService {
|
||||
private static final String STATUS_RUNNING = "RUNNING";
|
||||
private static final String STATUS_FAILED = "FAILED";
|
||||
|
||||
private static final String STATUS_BRAND_PENDING = "pending";
|
||||
private static final String STATUS_BRAND_RUNNING = "running";
|
||||
private static final String STATUS_BRAND_FAILED = "failed";
|
||||
private static final String STATUS_BRAND_CANCELLED = "cancelled";
|
||||
|
||||
/** 中间态(无人接单)或 RUNNING(心跳残留)超过该时长未更新即判死。真活任务心跳会不断刷新 updatedAt,不会被误杀。 */
|
||||
private static final long STALE_IDLE_MINUTES = 120;
|
||||
|
||||
/** 单轮品牌陈旧任务处理上限(原实现内联 limit 500)。 */
|
||||
private static final int MAX_BRAND_STALE_BATCH = 500;
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
|
||||
private final BrandTaskStaleRepairSpi brandTaskStaleRepairSpi;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
/**
|
||||
@@ -117,26 +115,15 @@ public class StaleTaskRepairService {
|
||||
}
|
||||
}
|
||||
|
||||
/** brand_crawl_tasks 的 pending/running 陈旧 → failed。 */
|
||||
/**
|
||||
* brand_crawl_tasks 的 pending/running 陈旧 → failed。
|
||||
* 品牌表与状态字面量属于 brand 模块,实现放在 BrandTaskStaleRepairSpiImpl(G5:消除 task → 业务依赖)。
|
||||
*/
|
||||
private void repairBrandStale() {
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(STALE_IDLE_MINUTES);
|
||||
List<BrandCrawlTaskEntity> stale = brandCrawlTaskMapper.selectList(new LambdaQueryWrapper<BrandCrawlTaskEntity>()
|
||||
.select(BrandCrawlTaskEntity::getId)
|
||||
.in(BrandCrawlTaskEntity::getStatus, STATUS_BRAND_PENDING, STATUS_BRAND_RUNNING)
|
||||
.lt(BrandCrawlTaskEntity::getUpdatedAt, cutoff)
|
||||
.last("limit 500"));
|
||||
if (stale.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.in(BrandCrawlTaskEntity::getId, stale.stream().map(BrandCrawlTaskEntity::getId).toList())
|
||||
.in(BrandCrawlTaskEntity::getStatus, STATUS_BRAND_PENDING, STATUS_BRAND_RUNNING)
|
||||
.lt(BrandCrawlTaskEntity::getUpdatedAt, cutoff)
|
||||
.set(BrandCrawlTaskEntity::getStatus, STATUS_BRAND_FAILED)
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, "任务长期无心跳,已自动失败"));
|
||||
if (updated > 0) {
|
||||
log.warn("[stale-task-repair] brand 陈旧任务已标失败 count={} ids={}",
|
||||
updated, stale.stream().map(BrandCrawlTaskEntity::getId).toList());
|
||||
List<Long> failedIds = brandTaskStaleRepairSpi.failStaleBrandTasks(cutoff, MAX_BRAND_STALE_BATCH);
|
||||
if (!failedIds.isEmpty()) {
|
||||
log.warn("[stale-task-repair] brand 陈旧任务已标失败 count={} ids={}", failedIds.size(), failedIds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+81
-223
@@ -2,76 +2,66 @@ package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||
import com.nanri.aiimage.modules.brand.service.BrandTaskProgressCacheService;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
|
||||
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService;
|
||||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
|
||||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService;
|
||||
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
|
||||
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
|
||||
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
|
||||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskCacheService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import com.nanri.aiimage.modules.task.spi.BrandTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 任务心跳入口。
|
||||
*
|
||||
* <p>2026-09 全维度审查 G5:本类此前直接注入 12 个业务模块的 CacheService/Service 并用 switch 分发,
|
||||
* 与"业务模块实现 task 的 Handler SPI"形成 task ↔ 业务的循环依赖。现改为依赖
|
||||
* {@link TaskModuleHeartbeatSpi}(各业务模块自己实现)与 {@link BrandTaskHeartbeatSpi}(品牌任务,
|
||||
* 独立于 biz_file_task),task 侧不再 import 任何业务模块。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class TaskHeartbeatService {
|
||||
|
||||
private static final String STATUS_RUNNING = "RUNNING";
|
||||
private static final String BRAND_STATUS_RUNNING = "running";
|
||||
|
||||
private static final String MODULE_DELETE_BRAND = "DELETE_BRAND";
|
||||
private static final String MODULE_PRODUCT_RISK = "PRODUCT_RISK_RESOLVE";
|
||||
private static final String MODULE_PRICE_TRACK = "PRICE_TRACK";
|
||||
private static final String MODULE_SHOP_MATCH = "SHOP_MATCH";
|
||||
private static final String MODULE_PATROL_DELETE = "PATROL_DELETE";
|
||||
private static final String MODULE_QUERY_ASIN = "QUERY_ASIN";
|
||||
private static final String MODULE_SHOP_DATA_CRAWL = "SHOP_DATA_CRAWL";
|
||||
private static final String MODULE_WITHDRAW = "WITHDRAW";
|
||||
private static final String MODULE_APPEARANCE_PATENT = "APPEARANCE_PATENT";
|
||||
private static final String MODULE_SIMILAR_ASIN = "SIMILAR_ASIN";
|
||||
private static final String MODULE_COLLECT_DATA = "COLLECT_DATA";
|
||||
private static final String MODULE_BRAND = "BRAND";
|
||||
private static final String MODULE_SHOP_DATA_CRAWL = "SHOP_DATA_CRAWL";
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
|
||||
private final ProductRiskTaskCacheService productRiskTaskCacheService;
|
||||
private final PublishTaskService publishTaskService;
|
||||
private final PriceTrackTaskCacheService priceTrackTaskCacheService;
|
||||
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
|
||||
private final PatrolDeleteTaskCacheService patrolDeleteTaskCacheService;
|
||||
private final QueryAsinTaskCacheService queryAsinTaskCacheService;
|
||||
private final ShopDataCrawlTaskCacheService shopDataCrawlTaskCacheService;
|
||||
private final ShopDataCrawlTaskService shopDataCrawlTaskService;
|
||||
private final WithdrawTaskCacheService withdrawTaskCacheService;
|
||||
private final AppearancePatentTaskCacheService appearancePatentTaskCacheService;
|
||||
private final SimilarAsinTaskCacheService similarAsinTaskCacheService;
|
||||
private final SimilarAsinProperties similarAsinProperties;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
||||
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||
private final CollectDataService collectDataService;
|
||||
private final BrandTaskHeartbeatSpi brandTaskHeartbeatSpi;
|
||||
/** moduleType → 模块心跳实现(启动时建索引并校验重复注册) */
|
||||
private final Map<String, TaskModuleHeartbeatSpi> moduleHandlers;
|
||||
|
||||
public TaskHeartbeatService(FileTaskMapper fileTaskMapper,
|
||||
TaskProgressSnapshotService taskProgressSnapshotService,
|
||||
BrandTaskHeartbeatSpi brandTaskHeartbeatSpi,
|
||||
List<TaskModuleHeartbeatSpi> moduleHeartbeatHandlers) {
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
this.taskProgressSnapshotService = taskProgressSnapshotService;
|
||||
this.brandTaskHeartbeatSpi = brandTaskHeartbeatSpi;
|
||||
Map<String, TaskModuleHeartbeatSpi> handlers = new LinkedHashMap<>();
|
||||
for (TaskModuleHeartbeatSpi handler : moduleHeartbeatHandlers == null ? List.<TaskModuleHeartbeatSpi>of() : moduleHeartbeatHandlers) {
|
||||
String moduleType = handler.moduleType();
|
||||
if (moduleType == null || moduleType.isBlank()) {
|
||||
throw new IllegalStateException("TaskModuleHeartbeatSpi 未声明 moduleType: "
|
||||
+ handler.getClass().getName());
|
||||
}
|
||||
TaskModuleHeartbeatSpi exists = handlers.put(moduleType, handler);
|
||||
if (exists != null) {
|
||||
throw new IllegalStateException("moduleType=" + moduleType + " 注册了多个心跳实现: "
|
||||
+ exists.getClass().getName() + " / " + handler.getClass().getName());
|
||||
}
|
||||
}
|
||||
this.moduleHandlers = Map.copyOf(handlers);
|
||||
log.info("[task-heartbeat-spi] 模块心跳实现注册完成 count={} modules={}", handlers.size(), handlers.keySet());
|
||||
}
|
||||
|
||||
public TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
@@ -82,15 +72,12 @@ public class TaskHeartbeatService {
|
||||
String requestedModuleType = request == null || request.getModuleType() == null
|
||||
? null
|
||||
: request.getModuleType().trim();
|
||||
FileTaskEntity fileTask = MODULE_BRAND.equalsIgnoreCase(requestedModuleType)
|
||||
? null
|
||||
: selectFileTask(taskId);
|
||||
BrandCrawlTaskEntity brandTask = MODULE_SHOP_DATA_CRAWL.equalsIgnoreCase(requestedModuleType)
|
||||
? null
|
||||
: selectBrandTask(taskId);
|
||||
// 品牌任务在独立表里:客户端声明是 BRAND 时不必再查 biz_file_task(历史行为保持一致)
|
||||
boolean brandOnly = MODULE_BRAND.equalsIgnoreCase(requestedModuleType);
|
||||
boolean skipBrandLookup = MODULE_SHOP_DATA_CRAWL.equalsIgnoreCase(requestedModuleType);
|
||||
|
||||
TaskHeartbeatVo fileResult = touchFileTaskIfRunning(fileTask, request);
|
||||
TaskHeartbeatVo brandResult = touchBrandTaskIfRunning(brandTask, request);
|
||||
TaskHeartbeatVo fileResult = brandOnly ? null : touchFileTaskIfRunning(selectFileTask(taskId), request);
|
||||
TaskHeartbeatVo brandResult = skipBrandLookup ? null : brandTaskHeartbeatSpi.heartbeat(taskId, request);
|
||||
|
||||
if (fileResult != null && fileResult.isAlive() && brandResult != null && brandResult.isAlive()) {
|
||||
return TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING);
|
||||
@@ -158,58 +145,34 @@ public class TaskHeartbeatService {
|
||||
syncTerminalState(fileTask, status, fileTask.getErrorMessage());
|
||||
return TaskHeartbeatVo.notAlive(fileTask.getModuleType(), status, "task is not running");
|
||||
}
|
||||
BrandCrawlTaskEntity brandTask = selectBrandTask(taskId);
|
||||
if (brandTask != null) {
|
||||
String status = brandTask.getStatus();
|
||||
if ("running".equalsIgnoreCase(status) || "pending".equalsIgnoreCase(status)) {
|
||||
// 与 file 分支统一改为条件更新:整行 updateById 会拿读取快照覆盖并发写入的字段
|
||||
// (客户端重启上报与品牌任务自身状态流转同时发生时)
|
||||
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, brandTask.getId())
|
||||
.in(BrandCrawlTaskEntity::getStatus, "running", "pending", "RUNNING", "PENDING")
|
||||
.set(BrandCrawlTaskEntity::getStatus, "cancelled")
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, safeReason)
|
||||
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
log.warn("[task-interrupted] brand task marked cancelled by client restart taskId={} reason={}",
|
||||
taskId, safeReason);
|
||||
return TaskHeartbeatVo.notAlive(MODULE_BRAND, "cancelled", "marked cancelled");
|
||||
}
|
||||
}
|
||||
log.info("[task-interrupted] brand task not in running/pending, skipped taskId={} status={}", taskId, status);
|
||||
return TaskHeartbeatVo.notAlive(MODULE_BRAND, status, "task is not running");
|
||||
TaskHeartbeatVo brandResult = brandTaskHeartbeatSpi.markInterrupted(taskId, safeReason);
|
||||
if (brandResult != null) {
|
||||
return brandResult;
|
||||
}
|
||||
log.debug("[task-interrupted] task not found (normal polling) taskId={}", taskId);
|
||||
return TaskHeartbeatVo.notAlive(null, null, "task not found");
|
||||
}
|
||||
|
||||
private BrandCrawlTaskEntity selectBrandTask(Long taskId) {
|
||||
LambdaQueryWrapper<BrandCrawlTaskEntity> brandQuery = new LambdaQueryWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||
.last("limit 1");
|
||||
return brandCrawlTaskMapper.selectOne(brandQuery);
|
||||
}
|
||||
|
||||
private TaskHeartbeatVo touchFileTaskIfRunning(FileTaskEntity task, TaskHeartbeatRequest request) {
|
||||
if (task == null) {
|
||||
return null;
|
||||
}
|
||||
String moduleType = task.getModuleType() == null ? "" : task.getModuleType();
|
||||
if (PublishTaskService.MODULE_TYPE.equals(moduleType)) {
|
||||
publishTaskService.ensureTaskOwnedByCurrentInstance(task, "publish task heartbeat");
|
||||
}
|
||||
if (MODULE_SHOP_DATA_CRAWL.equals(moduleType)) {
|
||||
shopDataCrawlTaskService.ensureTaskOwnedByCurrentInstance(task, "shop data crawl task heartbeat");
|
||||
TaskModuleHeartbeatSpi handler = moduleHandlers.get(moduleType);
|
||||
if (handler != null) {
|
||||
handler.ensureOwnership(task, moduleType + " task heartbeat");
|
||||
}
|
||||
String status = task.getStatus();
|
||||
if (!STATUS_RUNNING.equals(status)) {
|
||||
log.debug("[task-heartbeat] file task is not running (normal polling) taskId={ actualUserId={} moduleType={} status={}",
|
||||
log.debug("[task-heartbeat] file task is not running (normal polling) taskId={} actualUserId={} moduleType={} status={}",
|
||||
task.getId(), task.getUserId(), moduleType, status);
|
||||
return TaskHeartbeatVo.notAlive(moduleType, status, "task is not running");
|
||||
}
|
||||
if (MODULE_SIMILAR_ASIN.equals(moduleType)) {
|
||||
return touchSimilarAsinHeartbeat(task);
|
||||
}
|
||||
|
||||
// updated_at 检查点:默认每次心跳刷新;实现声明了节流间隔的(similar-asin)按间隔刷新,
|
||||
// 避免客户端密集心跳把库写满
|
||||
Duration interval = handler == null ? null : handler.checkpointInterval();
|
||||
if (isCheckpointDue(task, interval)) {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
@@ -217,145 +180,37 @@ public class TaskHeartbeatService {
|
||||
.set(FileTaskEntity::getUpdatedAt, now));
|
||||
if (updated <= 0) {
|
||||
FileTaskEntity latest = fileTaskMapper.selectById(task.getId());
|
||||
if (interval != null && latest != null && STATUS_RUNNING.equals(latest.getStatus())) {
|
||||
// 节流下的条件更新未命中:并发心跳刚刷新过,按最新实体继续(不判死)
|
||||
task = latest;
|
||||
} else {
|
||||
log.warn("[task-heartbeat] file task heartbeat update missed taskId={} actualUserId={} moduleType={} status={} latestStatus={}",
|
||||
task.getId(), task.getUserId(), moduleType, status,
|
||||
latest == null ? null : latest.getStatus());
|
||||
return TaskHeartbeatVo.notAlive(moduleType, latest == null ? status : latest.getStatus(), "task is not running");
|
||||
return TaskHeartbeatVo.notAlive(moduleType, latest == null ? status : latest.getStatus(),
|
||||
"task is not running");
|
||||
}
|
||||
touchModuleHeartbeat(moduleType, task.getId(), request);
|
||||
task.setUpdatedAt(now);
|
||||
saveFileTaskCache(moduleType, task);
|
||||
return TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING);
|
||||
}
|
||||
|
||||
private TaskHeartbeatVo touchSimilarAsinHeartbeat(FileTaskEntity task) {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
long intervalMillis = Math.max(1_000L, similarAsinProperties.getDbTaskTouchIntervalMillis());
|
||||
LocalDateTime cutoff = now.minus(Duration.ofMillis(intervalMillis));
|
||||
boolean checkpointDue = task.getUpdatedAt() == null || !task.getUpdatedAt().isAfter(cutoff);
|
||||
|
||||
similarAsinTaskCacheService.touchTaskHeartbeat(task.getId());
|
||||
taskFileJobService.touchRunningAssembleJobsIfStale(task.getId(), MODULE_SIMILAR_ASIN, intervalMillis);
|
||||
if (!checkpointDue) {
|
||||
saveFileTaskCache(MODULE_SIMILAR_ASIN, task);
|
||||
return TaskHeartbeatVo.alive(MODULE_SIMILAR_ASIN, STATUS_RUNNING);
|
||||
}
|
||||
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.and(wrapper -> wrapper.isNull(FileTaskEntity::getUpdatedAt)
|
||||
.or()
|
||||
.le(FileTaskEntity::getUpdatedAt, cutoff))
|
||||
.set(FileTaskEntity::getUpdatedAt, now));
|
||||
if (updated <= 0) {
|
||||
FileTaskEntity latest = fileTaskMapper.selectById(task.getId());
|
||||
if (latest == null || !STATUS_RUNNING.equals(latest.getStatus())) {
|
||||
return TaskHeartbeatVo.notAlive(MODULE_SIMILAR_ASIN,
|
||||
latest == null ? task.getStatus() : latest.getStatus(), "task is not running");
|
||||
}
|
||||
task = latest;
|
||||
} else {
|
||||
task.setUpdatedAt(now);
|
||||
}
|
||||
saveFileTaskCache(MODULE_SIMILAR_ASIN, task);
|
||||
return TaskHeartbeatVo.alive(MODULE_SIMILAR_ASIN, STATUS_RUNNING);
|
||||
}
|
||||
|
||||
private TaskHeartbeatVo touchBrandTaskIfRunning(BrandCrawlTaskEntity task, TaskHeartbeatRequest request) {
|
||||
if (task == null) {
|
||||
return null;
|
||||
if (handler != null) {
|
||||
handler.touchHeartbeat(task.getId(), request);
|
||||
handler.onHeartbeat(task);
|
||||
handler.saveTaskCache(task);
|
||||
}
|
||||
String status = task.getStatus();
|
||||
if (!BRAND_STATUS_RUNNING.equals(status)) {
|
||||
log.warn("[task-heartbeat] brand task is not running taskId={} actualUserId={} status={}",
|
||||
task.getId(), task.getUserId(), status);
|
||||
return TaskHeartbeatVo.notAlive(MODULE_BRAND, status, "task is not running");
|
||||
}
|
||||
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, task.getId())
|
||||
.eq(BrandCrawlTaskEntity::getStatus, BRAND_STATUS_RUNNING)
|
||||
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated <= 0) {
|
||||
BrandCrawlTaskEntity latest = brandCrawlTaskMapper.selectById(task.getId());
|
||||
log.warn("[task-heartbeat] brand task heartbeat update missed taskId={} actualUserId={} status={} latestStatus={}",
|
||||
task.getId(), task.getUserId(), status,
|
||||
latest == null ? null : latest.getStatus());
|
||||
return TaskHeartbeatVo.notAlive(MODULE_BRAND, latest == null ? status : latest.getStatus(), "task is not running");
|
||||
}
|
||||
brandTaskProgressCacheService.touchHeartbeat(
|
||||
task.getId(),
|
||||
request == null ? null : request.getPhase(),
|
||||
request == null ? null : request.getCurrent(),
|
||||
request == null ? null : request.getTotal());
|
||||
return TaskHeartbeatVo.alive(MODULE_BRAND, BRAND_STATUS_RUNNING);
|
||||
return TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING);
|
||||
}
|
||||
|
||||
private void touchModuleHeartbeat(String moduleType, Long taskId, TaskHeartbeatRequest request) {
|
||||
switch (moduleType) {
|
||||
case MODULE_PRODUCT_RISK -> {
|
||||
productRiskTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case PublishTaskService.MODULE_TYPE -> {
|
||||
publishTaskService.touchHeartbeat(taskId, request);
|
||||
}
|
||||
case MODULE_PRICE_TRACK -> {
|
||||
priceTrackTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_SHOP_MATCH -> {
|
||||
shopMatchTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_PATROL_DELETE -> {
|
||||
patrolDeleteTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_QUERY_ASIN -> {
|
||||
queryAsinTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_SHOP_DATA_CRAWL -> shopDataCrawlTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
case MODULE_WITHDRAW -> {
|
||||
withdrawTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_APPEARANCE_PATENT -> {
|
||||
appearancePatentTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_SIMILAR_ASIN -> {
|
||||
similarAsinTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_DELETE_BRAND -> {
|
||||
deleteBrandTaskCacheService.saveProgress(taskId, buildDeleteBrandHeartbeatProgress(request), true);
|
||||
}
|
||||
case MODULE_COLLECT_DATA -> collectDataService.updateProgress(taskId, request);
|
||||
default -> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> buildDeleteBrandHeartbeatProgress(TaskHeartbeatRequest request) {
|
||||
String now = String.valueOf(System.currentTimeMillis());
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
values.put("last_heartbeat_at", now);
|
||||
values.put("updated_at", now);
|
||||
if (request != null) {
|
||||
putIfPresent(values, "phase", request.getPhase());
|
||||
putIfPresent(values, "current", request.getCurrent());
|
||||
putIfPresent(values, "total", request.getTotal());
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private void saveFileTaskCache(String moduleType, FileTaskEntity task) {
|
||||
switch (moduleType) {
|
||||
case MODULE_PRODUCT_RISK -> productRiskTaskCacheService.saveTaskCache(task);
|
||||
case MODULE_PRICE_TRACK -> priceTrackTaskCacheService.saveTaskCache(task);
|
||||
case MODULE_SHOP_MATCH -> shopMatchTaskCacheService.saveTaskCache(task);
|
||||
case MODULE_PATROL_DELETE -> patrolDeleteTaskCacheService.saveTaskCache(task);
|
||||
case MODULE_QUERY_ASIN -> queryAsinTaskCacheService.saveTaskCache(task);
|
||||
case MODULE_SHOP_DATA_CRAWL -> shopDataCrawlTaskCacheService.saveTaskCache(task);
|
||||
case MODULE_WITHDRAW -> withdrawTaskCacheService.saveTaskCache(task);
|
||||
case MODULE_DELETE_BRAND -> deleteBrandTaskCacheService.saveTaskCache(task);
|
||||
default -> {
|
||||
}
|
||||
/** 无节流间隔时永远认为需要刷新(历史行为);有间隔时按 updated_at 距上次刷新的时长判断。 */
|
||||
private boolean isCheckpointDue(FileTaskEntity task, Duration interval) {
|
||||
if (interval == null) {
|
||||
return true;
|
||||
}
|
||||
long intervalMillis = Math.max(1_000L, interval.toMillis());
|
||||
LocalDateTime cutoff = LocalDateTime.now().minus(Duration.ofMillis(intervalMillis));
|
||||
return task.getUpdatedAt() == null || !task.getUpdatedAt().isAfter(cutoff);
|
||||
}
|
||||
|
||||
/** 把终态同步到模块缓存与进度快照;失败只告警,不影响中断接口本身的成功语义。 */
|
||||
@@ -372,10 +227,13 @@ public class TaskHeartbeatService {
|
||||
}
|
||||
}
|
||||
|
||||
private void putIfPresent(Map<String, String> values, String key, Object value) {
|
||||
if (value != null) {
|
||||
values.put(key, String.valueOf(value));
|
||||
private void saveFileTaskCache(String moduleType, FileTaskEntity task) {
|
||||
if (moduleType == null) {
|
||||
return;
|
||||
}
|
||||
TaskModuleHeartbeatSpi handler = moduleHandlers.get(moduleType);
|
||||
if (handler != null) {
|
||||
handler.saveTaskCache(task);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+52
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.util.PendingDeleteJournal;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
@@ -10,8 +11,11 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
@@ -49,6 +53,12 @@ public class TransientPayloadDeleteOrchestrator {
|
||||
@Value("${aiimage.transient-storage.max-pending-deletes:1000}")
|
||||
private long maxPendingDeletes = 1000;
|
||||
|
||||
/**
|
||||
* 待删对象的本地落盘日志:pending 集合重启即丢,未被引用的对象会一直残留在桶里。
|
||||
* 入队追加、flush 后按「仍在 pending + 异步删除失败」收敛重写、启动时回放后重新走引用检查。
|
||||
*/
|
||||
private volatile PendingDeleteJournal journal;
|
||||
|
||||
public TransientPayloadDeleteOrchestrator(TransientPayloadStorageService transientPayloadStorageService,
|
||||
RustfsObjectStorageService rustfsObjectStorageService,
|
||||
TaskChunkMapper taskChunkMapper,
|
||||
@@ -84,6 +94,7 @@ public class TransientPayloadDeleteOrchestrator {
|
||||
}
|
||||
if (pendingPointers.add(pointer)) {
|
||||
accepted++;
|
||||
journal().record(pointer);
|
||||
}
|
||||
}
|
||||
return accepted;
|
||||
@@ -164,6 +175,7 @@ public class TransientPayloadDeleteOrchestrator {
|
||||
}
|
||||
|
||||
private void deleteObjects(List<String> pointers) {
|
||||
List<String> failed = new ArrayList<>();
|
||||
for (String pointer : pointers) {
|
||||
if (pointer == null || !pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
||||
// 本地/OSS 指针有实例归属与其它清理路径,不在此批量删除范围。
|
||||
@@ -173,10 +185,50 @@ public class TransientPayloadDeleteOrchestrator {
|
||||
try {
|
||||
rustfsObjectStorageService.deleteObject(objectKey);
|
||||
} catch (Exception ex) {
|
||||
// 失败对象写回日志:进程重启后由启动回放重新提交,不再随进程消失
|
||||
failed.add(pointer);
|
||||
log.warn("[transient-payload] async delete failed objectKey={} err={}",
|
||||
objectKey, ex.getMessage());
|
||||
}
|
||||
}
|
||||
synchronized (pendingPointers) {
|
||||
Set<String> keep = new LinkedHashSet<>(pendingPointers);
|
||||
keep.addAll(failed);
|
||||
journal().rewrite(keep);
|
||||
}
|
||||
if (!failed.isEmpty()) {
|
||||
log.warn("[transient-payload] async delete failed, kept in journal count={}", failed.size());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动回放:把上次进程残留的待删对象重新入队,等待下一轮 flush 做引用检查与物理删除
|
||||
* (引用检查幂等,重复提交同一对象只入队一次)。
|
||||
*/
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void replayJournalOnStartup() {
|
||||
List<String> pointers = journal().readAll();
|
||||
if (pointers.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
int accepted = submitDeletes(pointers);
|
||||
log.info("[transient-payload] 启动回放待删对象 journalSize={} accepted={} pending={}",
|
||||
pointers.size(), accepted, pendingCount());
|
||||
}
|
||||
|
||||
private PendingDeleteJournal journal() {
|
||||
PendingDeleteJournal local = journal;
|
||||
if (local != null) {
|
||||
return local;
|
||||
}
|
||||
synchronized (pendingPointers) {
|
||||
if (journal == null) {
|
||||
Path path = Path.of(System.getProperty("java.io.tmpdir"),
|
||||
"aiimage-delete-journal", "transient-payload-pending.log");
|
||||
journal = new PendingDeleteJournal(path, pendingLimit());
|
||||
}
|
||||
return journal;
|
||||
}
|
||||
}
|
||||
|
||||
/** 批量 IN 反查两张引用表,返回仍被引用的指针集合(查询异常抛给调用方)。 */
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
package com.nanri.aiimage.modules.task.spi;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
|
||||
|
||||
/**
|
||||
* 品牌检测任务(brand_crawl_tasks,独立于 biz_file_task)的心跳/中断扩展点。
|
||||
*
|
||||
* <p>2026-09 全维度审查 G5:品牌任务有自己的表与进度缓存,原先由 {@code TaskHeartbeatService}
|
||||
* 直接注入 brand 模块的 Mapper 与缓存服务,构成 task → 业务的依赖。这里把品牌侧逻辑整体
|
||||
* 交回 brand 模块实现,task 侧只按接口调用。
|
||||
*/
|
||||
public interface BrandTaskHeartbeatSpi {
|
||||
|
||||
/**
|
||||
* 品牌任务心跳:任务不存在返回 null;非 running 返回 notAlive;running 则条件更新
|
||||
* updated_at 并刷新进度缓存。
|
||||
*/
|
||||
TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request);
|
||||
|
||||
/**
|
||||
* 客户端重启中断:running/pending → cancelled(条件更新),返回心跳结果;任务不存在返回 null。
|
||||
*/
|
||||
TaskHeartbeatVo markInterrupted(Long taskId, String reason);
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package com.nanri.aiimage.modules.task.spi;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 品牌检测任务的陈旧修复扩展点(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>brand_crawl_tasks 有独立的表与状态字面量(pending/running/failed),原先由
|
||||
* {@code StaleTaskRepairService} 直接操作品牌模块的 Mapper,构成 task → 业务的依赖。
|
||||
* 现由品牌模块实现本接口,task 侧只按接口调用。
|
||||
*/
|
||||
public interface BrandTaskStaleRepairSpi {
|
||||
|
||||
/**
|
||||
* 把 pending/running 且 updated_at 早于 cutoff 的品牌任务置为失败(条件更新,幂等)。
|
||||
*
|
||||
* @param limit 单轮处理上限
|
||||
* @return 实际标记失败的任务 id(空表示本轮无陈旧任务)
|
||||
*/
|
||||
List<Long> failStaleBrandTasks(LocalDateTime cutoff, int limit);
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
package com.nanri.aiimage.modules.task.spi;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 采集明细行的历史清理扩展点(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>任务历史清理时除了 task 自己的表,还要连带删除 biz_collect_data_item 的明细行;
|
||||
* 该表属于采集模块,原先由 {@code ModuleHistoryCleanupService} 直接使用其 Mapper,
|
||||
* 构成 task → 业务的依赖。现由采集模块实现本接口。
|
||||
*/
|
||||
public interface CollectDataItemCleanupSpi {
|
||||
|
||||
/**
|
||||
* 删除这些任务名下的采集明细行。
|
||||
*
|
||||
* @return 删除行数
|
||||
*/
|
||||
int deleteItemsByTaskIds(List<Long> taskIds);
|
||||
}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
package com.nanri.aiimage.modules.task.spi;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* 任务心跳的模块侧扩展点(2026-09 全维度审查 G5:消除 task ↔ 业务模块的双向依赖)。
|
||||
*
|
||||
* <p>背景:{@code TaskHeartbeatService} 原先直接注入 12 个业务模块的 CacheService 并用 switch 分发,
|
||||
* 与"业务模块实现 task 的 Handler SPI"形成真实循环依赖。现改为:task 侧只依赖本接口,
|
||||
* 各业务模块各自实现一个 Bean(Spring 注入 List 后在 task 侧建索引)。
|
||||
*
|
||||
* <p>新增模块时:实现本接口并在注册表中登记模块类型;心跳契约测试会校验覆盖。
|
||||
*/
|
||||
public interface TaskModuleHeartbeatSpi {
|
||||
|
||||
/** 本实现负责的 moduleType(与 biz_file_task.module_type 一致)。 */
|
||||
String moduleType();
|
||||
|
||||
/**
|
||||
* 刷新模块自己的进度缓存/心跳(非 DB 写入)。
|
||||
* 多数模块实现为 {@code cacheService.touchTaskHeartbeat(taskId)}。
|
||||
*/
|
||||
void touchHeartbeat(Long taskId, TaskHeartbeatRequest request);
|
||||
|
||||
/**
|
||||
* 心跳时回写模块的任务缓存(部分模块没有独立缓存,保持默认空实现)。
|
||||
*/
|
||||
default void saveTaskCache(FileTaskEntity task) {
|
||||
}
|
||||
|
||||
/**
|
||||
* 心跳前的任务归属校验:publish / shop-data-crawl 在非本实例拥有时拒绝心跳,
|
||||
* 其余模块默认放行。
|
||||
*/
|
||||
default void ensureOwnership(FileTaskEntity task, String operation) {
|
||||
}
|
||||
|
||||
/**
|
||||
* 心跳时的额外模块侧动作(如 similar-asin 刷新结果文件组装作业的租约)。
|
||||
*/
|
||||
default void onHeartbeat(FileTaskEntity task) {
|
||||
}
|
||||
|
||||
/**
|
||||
* biz_file_task.updated_at 检查点节流间隔。
|
||||
*
|
||||
* <p>返回非空时,DB 的 updated_at 只在距上次刷新超过该间隔后才更新
|
||||
* (客户端心跳间隔远小于 stale 判定阈值时避免每次心跳都写库);
|
||||
* 默认 null 表示每次心跳都刷新(与历史行为一致)。
|
||||
*/
|
||||
default Duration checkpointInterval() {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -3,9 +3,9 @@ package com.nanri.aiimage.modules.withdraw.controller;
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.modules.withdraw.model.dto.WithdrawCandidateClearRequest;
|
||||
import com.nanri.aiimage.modules.withdraw.model.dto.WithdrawCreateTaskRequest;
|
||||
import com.nanri.aiimage.modules.withdraw.model.dto.WithdrawSubmitResultRequest;
|
||||
|
||||
+3
-3
@@ -4,9 +4,9 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskCandidateVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskMatchShopsVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskShopQueueItemVo;
|
||||
import com.nanri.aiimage.modules.withdraw.mapper.WithdrawShopCandidateMapper;
|
||||
import com.nanri.aiimage.modules.withdraw.model.entity.WithdrawShopCandidateEntity;
|
||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.nanri.aiimage.modules.withdraw.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* 取款 的任务心跳实现(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>心跳逻辑(缓存刷新 / 任务缓存回写)从 task 模块收回本模块,task 侧只依赖 SPI 接口,
|
||||
* 消除 task → 业务模块的编译期依赖。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class WithdrawTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
private final WithdrawTaskCacheService cacheService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return "WITHDRAW";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
cacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveTaskCache(FileTaskEntity task) {
|
||||
cacheService.saveTaskCache(task);
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -10,7 +10,7 @@ import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
-- V123: 删除 biz_file_result 上被复合索引完全覆盖的两个单列索引
|
||||
--
|
||||
-- 判定原则(同 V121):被删索引的列是另一索引的**严格前缀**,
|
||||
-- 所有能用它的查询都能被保留的复合索引覆盖,删除只减少写入侧的索引维护与 buffer pool 占用。
|
||||
--
|
||||
-- 1) biz_file_result.idx_task_id (task_id)
|
||||
-- 被 V38 的 idx_file_result_task_module_id (task_id, module_type, id) 覆盖。
|
||||
-- 2) biz_file_result.idx_module_type (module_type)
|
||||
-- 被 V34 的 idx_biz_file_result_module_user_created (module_type, user_id, created_at)
|
||||
-- 与 V42 的 idx_file_result_patrol_history (module_type, user_id, created_at, id) 覆盖。
|
||||
-- 两处均为 V1 建表时的遗留索引(当时还没有任何复合索引)。
|
||||
--
|
||||
-- 风险:DROP INDEX 为 INPLACE 在线操作,但会立即改变执行计划;两者均为「被覆盖」判定。
|
||||
-- 回滚:
|
||||
-- ALTER TABLE biz_file_result ADD INDEX idx_task_id (task_id);
|
||||
-- ALTER TABLE biz_file_result ADD INDEX idx_module_type (module_type);
|
||||
|
||||
SET @db_name = DATABASE();
|
||||
|
||||
-- 1) biz_file_result.idx_task_id
|
||||
SET @idx_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_result'
|
||||
AND INDEX_NAME = 'idx_task_id'
|
||||
);
|
||||
SET @sql := IF(@idx_exists > 0,
|
||||
'ALTER TABLE biz_file_result DROP INDEX idx_task_id',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- 2) biz_file_result.idx_module_type
|
||||
SET @idx_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_result'
|
||||
AND INDEX_NAME = 'idx_module_type'
|
||||
);
|
||||
SET @sql := IF(@idx_exists > 0,
|
||||
'ALTER TABLE biz_file_result DROP INDEX idx_module_type',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
@@ -0,0 +1,27 @@
|
||||
-- V124: 删除 biz_dedupe_total_data.idx_country(函数列索引,永不生效)
|
||||
--
|
||||
-- 背景(2026-09 全维度审查 C6):该表按国家的筛选走 FIND_IN_SET(?, IFNULL(country,'')),
|
||||
-- 是函数列比较,MySQL 无法使用 country 上的普通索引;V91 建的 idx_country 从未被任何执行计划
|
||||
-- 使用过,只增加写入维护与 buffer pool 占用。
|
||||
--
|
||||
-- 前缀关键字搜索本身就吃 uk_data_value (data_value) 唯一索引(本次同时把关键字由
|
||||
-- %kw% 前置通配改为前缀匹配,见 DedupeTotalDataService.page),因此无需新增索引。
|
||||
--
|
||||
-- 风险:DROP INDEX 为在线 INPLACE 操作;country 仍用于结果展示,仅去掉索引。
|
||||
-- 回滚:
|
||||
-- ALTER TABLE biz_dedupe_total_data ADD INDEX idx_country (country);
|
||||
|
||||
SET @db_name = DATABASE();
|
||||
|
||||
SET @idx_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_dedupe_total_data'
|
||||
AND INDEX_NAME = 'idx_country'
|
||||
);
|
||||
SET @sql := IF(@idx_exists > 0,
|
||||
'ALTER TABLE biz_dedupe_total_data DROP INDEX idx_country',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
@@ -34,7 +34,9 @@ class ArchitectureBoundaryTest {
|
||||
* 本次以实测值 119 为准恢复告警能力;经 git diff 核验,当日的 task 模块改动
|
||||
* (引用计数按 task 收敛、快照 upsert 缓存、心跳条件更新、锁续期重试等)未新增任何业务依赖。
|
||||
*/
|
||||
private static final int TASK_TO_BUSINESS_BASELINE = 119;
|
||||
// 2026-09-14 心跳/陈旧修复/历史清理 SPI 化后实测 6(G5:task 只依赖 task.spi 下接口,
|
||||
// 业务模块各自实现);继续按棘轮收敛,不允许回升
|
||||
private static final int TASK_TO_BUSINESS_BASELINE = 6;
|
||||
|
||||
private static volatile JavaClasses cached;
|
||||
|
||||
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
package com.nanri.aiimage.common.util;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* 待删对象日志:入队追加、启动回放、收敛重写。对应审查项 D13(删除补偿队列重启即丢)。
|
||||
*/
|
||||
class PendingDeleteJournalTest {
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
@Test
|
||||
void recordThenReadAllReturnsAdmittedKeysInOrder() {
|
||||
PendingDeleteJournal journal = new PendingDeleteJournal(tempDir.resolve("nested/queue.log"), 100);
|
||||
|
||||
journal.record("a/1.bin");
|
||||
journal.record("b/2.bin");
|
||||
journal.record("a/1.bin");
|
||||
|
||||
assertEquals(List.of("a/1.bin", "b/2.bin"), journal.readAll());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rewriteKeepsOnlyRemainingAndDropsFileWhenEmpty() throws Exception {
|
||||
Path path = tempDir.resolve("queue.log");
|
||||
PendingDeleteJournal journal = new PendingDeleteJournal(path, 100);
|
||||
journal.record("keep.bin");
|
||||
journal.record("done.bin");
|
||||
|
||||
journal.rewrite(List.of("keep.bin"));
|
||||
assertEquals(List.of("keep.bin"), journal.readAll());
|
||||
|
||||
journal.rewrite(List.of());
|
||||
assertFalse(Files.exists(path), "队列收敛为空后日志文件应删除");
|
||||
assertEquals(List.of(), journal.readAll());
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingFileYieldsEmptyInsteadOfThrowing() {
|
||||
PendingDeleteJournal journal = new PendingDeleteJournal(tempDir.resolve("not-created.log"), 10);
|
||||
assertEquals(List.of(), journal.readAll());
|
||||
}
|
||||
|
||||
@Test
|
||||
void blankAndMultilineRecordsAreSanitizedOrIgnored() throws Exception {
|
||||
Path path = tempDir.resolve("queue.log");
|
||||
PendingDeleteJournal journal = new PendingDeleteJournal(path, 10);
|
||||
|
||||
journal.record(null);
|
||||
journal.record(" ");
|
||||
journal.record("bad\nkey.bin");
|
||||
journal.record(" good.bin ");
|
||||
|
||||
assertEquals(List.of("bad key.bin", "good.bin"), journal.readAll());
|
||||
// 单行一条记录:换行被替换,不会破坏行结构
|
||||
assertEquals(2, Files.readAllLines(path, StandardCharsets.UTF_8).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void corruptedLineIsIgnoredWhileValidOnesSurvive() throws Exception {
|
||||
Path path = tempDir.resolve("queue.log");
|
||||
Files.createDirectories(tempDir);
|
||||
Files.writeString(path, "\n \nvalid.bin\n", StandardCharsets.UTF_8);
|
||||
|
||||
PendingDeleteJournal journal = new PendingDeleteJournal(path, 10);
|
||||
|
||||
assertEquals(List.of("valid.bin"), journal.readAll());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rewriteRespectsMaxEntriesCap() {
|
||||
PendingDeleteJournal journal = new PendingDeleteJournal(tempDir.resolve("queue.log"), 2);
|
||||
|
||||
journal.rewrite(List.of("k1.bin", "k2.bin", "k3.bin"));
|
||||
|
||||
assertEquals(List.of("k1.bin", "k2.bin"), journal.readAll());
|
||||
assertTrue(journal.readAll().size() <= 2);
|
||||
}
|
||||
}
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
package com.nanri.aiimage.modules.deletebrand.service;
|
||||
|
||||
import com.nanri.aiimage.common.module.TaskModuleRegistry;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
|
||||
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
||||
import com.nanri.aiimage.config.ModuleCleanupProperties;
|
||||
import com.nanri.aiimage.config.TaskFileJobConfig;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
/**
|
||||
* 模块清单覆盖面自检(2026-09 全维度审查 G6)。
|
||||
*
|
||||
* 新增工具模块此前要同时改 5 处枚举,其中 4 处没有自检,漏改表现为"功能静默不生效"。
|
||||
* 本测试把「注册表 = 单一来源」这件事钉住:数据驱动的三处必须与注册表完全一致,
|
||||
* 委派式判死巡检线必须为注册表标记的每个模块登记动作。
|
||||
* 新增模块时若漏改任一处,这里会红。
|
||||
*/
|
||||
class TaskModuleCoverageTest {
|
||||
|
||||
/** 当前线上模块清单(新增模块时同时更新:注册表 + 本清单 + 对应 Handler) */
|
||||
private static final Set<String> EXPECTED_MODULES = Set.of(
|
||||
"PUBLISH", "DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE",
|
||||
"PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW",
|
||||
"APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA", "SHOP_DATA_CRAWL", "BRAND");
|
||||
|
||||
@Test
|
||||
void registryCoversEveryKnownModuleWithLabel() {
|
||||
assertEquals(EXPECTED_MODULES, TaskModuleRegistry.moduleTypes());
|
||||
|
||||
Map<String, String> labels = TaskModuleRegistry.labels();
|
||||
assertEquals(EXPECTED_MODULES.size(), labels.size());
|
||||
for (String moduleType : EXPECTED_MODULES) {
|
||||
String label = labels.get(moduleType);
|
||||
assertNotNull(label, "模块缺少中文名: " + moduleType);
|
||||
assertTrue(!label.isBlank(), "模块中文名不能为空: " + moduleType);
|
||||
}
|
||||
assertEquals("跟价", TaskModuleRegistry.labelOf("PRICE_TRACK"));
|
||||
assertEquals("UNKNOWN_MODULE", TaskModuleRegistry.labelOf("UNKNOWN_MODULE"), "未知模块回退原值");
|
||||
}
|
||||
|
||||
@Test
|
||||
void resultFileJobWhitelistMatchesRegistry() {
|
||||
assertEquals(TaskModuleRegistry.resultFileJobModuleTypes(), TaskFileJobConfig.RESULT_FILE_JOB_MODULE_TYPES);
|
||||
// 纯表格处理模块不产出结果文件作业
|
||||
assertTrue(!TaskFileJobConfig.RESULT_FILE_JOB_MODULE_TYPES.contains("DEDUPE"));
|
||||
assertTrue(!TaskFileJobConfig.RESULT_FILE_JOB_MODULE_TYPES.contains("SPLIT"));
|
||||
assertTrue(!TaskFileJobConfig.RESULT_FILE_JOB_MODULE_TYPES.contains("CONVERT"));
|
||||
assertTrue(TaskFileJobConfig.RESULT_FILE_JOB_MODULE_TYPES.contains("SHOP_DATA_CRAWL"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ageCleanupListMatchesRegistry() {
|
||||
ModuleCleanupProperties properties = new ModuleCleanupProperties();
|
||||
|
||||
assertEquals(TaskModuleRegistry.ageCleanupModuleTypes(), Set.copyOf(properties.getModuleTypes()));
|
||||
// 店铺数据采集靠每日工作簿自身保留期,不参与按天清理
|
||||
assertTrue(!properties.getModuleTypes().contains("SHOP_DATA_CRAWL"));
|
||||
assertTrue(properties.getModuleTypes().contains("DEDUPE"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 只装配 delegatedStaleChecks 需要的四个业务服务,其余依赖留空——
|
||||
* 这样测试聚焦"巡检线登记了哪些模块",不牵扯任务锁/缓存等无关依赖。
|
||||
*/
|
||||
private static DeleteBrandStaleTaskService serviceForDelegatedChecks() {
|
||||
return new DeleteBrandStaleTaskService(
|
||||
null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
|
||||
null,
|
||||
mock(BrandTaskService.class),
|
||||
mock(AppearancePatentTaskService.class),
|
||||
mock(SimilarAsinTaskService.class),
|
||||
null, null, null, null,
|
||||
mock(ShopDataCrawlTaskService.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void delegatedStaleCheckHandlersCoverRegistryFlag() {
|
||||
DeleteBrandStaleTaskService service = serviceForDelegatedChecks();
|
||||
|
||||
Map<String, Runnable> handlers = service.delegatedStaleChecks();
|
||||
|
||||
assertEquals(TaskModuleRegistry.delegatedStaleCheckModuleTypes(), handlers.keySet(),
|
||||
"注册表标记了 delegatedStaleCheck 的模块必须都在巡检线登记动作(反之亦然)");
|
||||
for (Runnable action : handlers.values()) {
|
||||
assertNotNull(action);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void delegatedStaleCheckIncludesModulesThatHadOwnScheduler() {
|
||||
DeleteBrandStaleTaskService service = serviceForDelegatedChecks();
|
||||
|
||||
List<String> keys = List.copyOf(service.delegatedStaleChecks().keySet());
|
||||
|
||||
// 商品管理采集 2026-09 从自带 @Scheduled 并入巡检线(P1-8 双实例判死盲区),不得回退
|
||||
assertTrue(keys.contains("SHOP_DATA_CRAWL"), "商品管理采集必须留在 stale-check 巡检线内: " + keys);
|
||||
assertTrue(keys.contains("SIMILAR_ASIN"));
|
||||
assertTrue(keys.contains("APPEARANCE_PATENT"));
|
||||
assertTrue(keys.contains("BRAND"));
|
||||
}
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlItemMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDate;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 采集明细表保留期清理(2026-09 全维度审查:该表此前无任何清理策略)。
|
||||
*/
|
||||
class ShopDataCrawlItemRetentionServiceTest {
|
||||
|
||||
private final ShopDataCrawlItemMapper itemMapper = mock(ShopDataCrawlItemMapper.class);
|
||||
private final DistributedJobLockService jobLockService = mock(DistributedJobLockService.class);
|
||||
|
||||
private ShopDataCrawlItemRetentionService service() {
|
||||
return new ShopDataCrawlItemRetentionService(itemMapper, jobLockService);
|
||||
}
|
||||
|
||||
private void lockAvailable() {
|
||||
when(jobLockService.tryLock(anyString(), any()))
|
||||
.thenReturn(mock(DistributedJobLockService.LockHandle.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletesInBatchesUntilBatchNotFull() {
|
||||
lockAvailable();
|
||||
when(itemMapper.deleteOlderThanBatch(any(LocalDate.class), anyInt()))
|
||||
.thenReturn(5000, 5000, 120);
|
||||
|
||||
service().purgeExpiredItems();
|
||||
|
||||
verify(itemMapper, times(3)).deleteOlderThanBatch(any(LocalDate.class), eq(5000));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopsAtBatchLimitPerRun() {
|
||||
lockAvailable();
|
||||
when(itemMapper.deleteOlderThanBatch(any(LocalDate.class), anyInt())).thenReturn(5000);
|
||||
|
||||
service().purgeExpiredItems();
|
||||
|
||||
verify(itemMapper, times(ShopDataCrawlItemRetentionService.MAX_BATCHES_PER_RUN))
|
||||
.deleteOlderThanBatch(any(LocalDate.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void skipsWhenAnotherInstanceHoldsLock() {
|
||||
when(jobLockService.tryLock(anyString(), any())).thenReturn(null);
|
||||
|
||||
service().purgeExpiredItems();
|
||||
|
||||
verify(itemMapper, never()).deleteOlderThanBatch(any(LocalDate.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteFailureIsLoggedAndDoesNotThrow() {
|
||||
lockAvailable();
|
||||
when(itemMapper.deleteOlderThanBatch(any(LocalDate.class), anyInt()))
|
||||
.thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
service().purgeExpiredItems();
|
||||
|
||||
verify(itemMapper).deleteOlderThanBatch(any(LocalDate.class), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void retentionDaysHasLowerBound() {
|
||||
ShopDataCrawlItemRetentionService service = service();
|
||||
assertEquals(30, service.effectiveRetentionDays(), "默认保留 30 天");
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -9,7 +9,7 @@ import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.ModuleCleanupProperties;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
@@ -90,7 +90,7 @@ class ModuleHistoryCleanupKeysetPaginationTest {
|
||||
private TaskResultPayloadMapper taskResultPayloadMapper;
|
||||
private TaskScopeStateMapper taskScopeStateMapper;
|
||||
private TaskChunkMapper taskChunkMapper;
|
||||
private CollectDataItemMapper collectDataItemMapper;
|
||||
private CollectDataItemCleanupSpi collectDataItemMapper;
|
||||
private DistributedJobLockService lockService;
|
||||
private TransientPayloadStorageService storage;
|
||||
private RustfsObjectStorageService rustfs;
|
||||
@@ -129,7 +129,7 @@ class ModuleHistoryCleanupKeysetPaginationTest {
|
||||
taskResultPayloadMapper = mock(TaskResultPayloadMapper.class);
|
||||
taskScopeStateMapper = mock(TaskScopeStateMapper.class);
|
||||
taskChunkMapper = mock(TaskChunkMapper.class);
|
||||
collectDataItemMapper = mock(CollectDataItemMapper.class);
|
||||
collectDataItemMapper = mock(CollectDataItemCleanupSpi.class);
|
||||
StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class);
|
||||
ValueOperations<String, String> valueOperations = mock(ValueOperations.class);
|
||||
when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
|
||||
+3
-3
@@ -13,7 +13,7 @@ import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.ModuleCleanupProperties;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
@@ -92,7 +92,7 @@ class ModuleHistoryCleanupLoggingTest {
|
||||
private TaskResultPayloadMapper taskResultPayloadMapper;
|
||||
private TaskScopeStateMapper taskScopeStateMapper;
|
||||
private TaskChunkMapper taskChunkMapper;
|
||||
private CollectDataItemMapper collectDataItemMapper;
|
||||
private CollectDataItemCleanupSpi collectDataItemMapper;
|
||||
private DistributedJobLockService lockService;
|
||||
private TransientPayloadStorageService storage;
|
||||
private RustfsObjectStorageService rustfs;
|
||||
@@ -128,7 +128,7 @@ class ModuleHistoryCleanupLoggingTest {
|
||||
taskResultPayloadMapper = mock(TaskResultPayloadMapper.class);
|
||||
taskScopeStateMapper = mock(TaskScopeStateMapper.class);
|
||||
taskChunkMapper = mock(TaskChunkMapper.class);
|
||||
collectDataItemMapper = mock(CollectDataItemMapper.class);
|
||||
collectDataItemMapper = mock(CollectDataItemCleanupSpi.class);
|
||||
StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class);
|
||||
ValueOperations<String, String> valueOperations = mock(ValueOperations.class);
|
||||
when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
|
||||
+3
-3
@@ -9,7 +9,7 @@ import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.ModuleCleanupProperties;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
@@ -86,7 +86,7 @@ class ModuleHistoryCleanupPayloadCleanupTest {
|
||||
private TaskResultPayloadMapper taskResultPayloadMapper;
|
||||
private TaskScopeStateMapper taskScopeStateMapper;
|
||||
private TaskChunkMapper taskChunkMapper;
|
||||
private CollectDataItemMapper collectDataItemMapper;
|
||||
private CollectDataItemCleanupSpi collectDataItemMapper;
|
||||
private DistributedJobLockService lockService;
|
||||
private TransientPayloadStorageService storage;
|
||||
private RustfsObjectStorageService rustfs;
|
||||
@@ -122,7 +122,7 @@ class ModuleHistoryCleanupPayloadCleanupTest {
|
||||
taskResultPayloadMapper = mock(TaskResultPayloadMapper.class);
|
||||
taskScopeStateMapper = mock(TaskScopeStateMapper.class);
|
||||
taskChunkMapper = mock(TaskChunkMapper.class);
|
||||
collectDataItemMapper = mock(CollectDataItemMapper.class);
|
||||
collectDataItemMapper = mock(CollectDataItemCleanupSpi.class);
|
||||
StringRedisTemplate redisTemplate = mock(StringRedisTemplate.class);
|
||||
ValueOperations<String, String> valueOperations = mock(ValueOperations.class);
|
||||
when(redisTemplate.opsForValue()).thenReturn(valueOperations);
|
||||
|
||||
+75
-63
@@ -1,50 +1,48 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.service.BrandTaskProgressCacheService;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
|
||||
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService;
|
||||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
|
||||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.publish.service.PublishTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
|
||||
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
|
||||
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
|
||||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskCacheService;
|
||||
import com.nanri.aiimage.modules.task.spi.BrandTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.InOrder;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
/**
|
||||
* 心跳分发契约(G5 SPI 化后):task 侧只按 moduleType 路由到 {@link TaskModuleHeartbeatSpi},
|
||||
* 各模块的行为由各自的 SPI 实现负责,因此这里同时覆盖"路由正确"与"实现行为正确"。
|
||||
*/
|
||||
class TaskHeartbeatServiceTest {
|
||||
|
||||
@BeforeAll
|
||||
@@ -54,35 +52,30 @@ class TaskHeartbeatServiceTest {
|
||||
FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private BrandCrawlTaskMapper brandCrawlTaskMapper;
|
||||
@Mock private ProductRiskTaskCacheService productRiskTaskCacheService;
|
||||
@Mock private PublishTaskService publishTaskService;
|
||||
@Mock private PriceTrackTaskCacheService priceTrackTaskCacheService;
|
||||
@Mock private ShopMatchTaskCacheService shopMatchTaskCacheService;
|
||||
@Mock private PatrolDeleteTaskCacheService patrolDeleteTaskCacheService;
|
||||
@Mock private QueryAsinTaskCacheService queryAsinTaskCacheService;
|
||||
@Mock private ShopDataCrawlTaskCacheService shopDataCrawlTaskCacheService;
|
||||
@Mock private ShopDataCrawlTaskService shopDataCrawlTaskService;
|
||||
@Mock private WithdrawTaskCacheService withdrawTaskCacheService;
|
||||
@Mock private AppearancePatentTaskCacheService appearancePatentTaskCacheService;
|
||||
@Mock private SimilarAsinTaskCacheService similarAsinTaskCacheService;
|
||||
@Mock private SimilarAsinProperties similarAsinProperties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
||||
@Mock private BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||
@Mock private CollectDataService collectDataService;
|
||||
private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
private final TaskProgressSnapshotService taskProgressSnapshotService = mock(TaskProgressSnapshotService.class);
|
||||
private final BrandTaskHeartbeatSpi brandTaskHeartbeatSpi = mock(BrandTaskHeartbeatSpi.class);
|
||||
|
||||
@InjectMocks private TaskHeartbeatService service;
|
||||
private TaskHeartbeatService serviceWith(TaskModuleHeartbeatSpi... handlers) {
|
||||
return new TaskHeartbeatService(fileTaskMapper, taskProgressSnapshotService, brandTaskHeartbeatSpi,
|
||||
List.of(handlers));
|
||||
}
|
||||
|
||||
private static FileTaskEntity runningTask(long taskId, String moduleType) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(taskId);
|
||||
task.setModuleType(moduleType);
|
||||
task.setStatus("RUNNING");
|
||||
return task;
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void publishHeartbeatTouchesGenericTaskAndModuleProgress() {
|
||||
long taskId = 20142L;
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(taskId);
|
||||
task.setModuleType(PublishTaskService.MODULE_TYPE);
|
||||
task.setStatus("RUNNING");
|
||||
FileTaskEntity task = runningTask(taskId, PublishTaskService.MODULE_TYPE);
|
||||
PublishTaskService publishTaskService = mock(PublishTaskService.class);
|
||||
TaskHeartbeatService service = serviceWith(new PublishTaskHeartbeatSpi(publishTaskService));
|
||||
|
||||
TaskHeartbeatRequest request = new TaskHeartbeatRequest();
|
||||
request.setPhase("dispatching");
|
||||
@@ -90,7 +83,6 @@ class TaskHeartbeatServiceTest {
|
||||
request.setTotal(100);
|
||||
|
||||
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task);
|
||||
when(brandCrawlTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null);
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
|
||||
TaskHeartbeatVo result = service.heartbeat(taskId, request);
|
||||
@@ -99,7 +91,7 @@ class TaskHeartbeatServiceTest {
|
||||
assertEquals(PublishTaskService.MODULE_TYPE, result.getModuleType());
|
||||
InOrder routingBeforeUpdate = inOrder(publishTaskService, fileTaskMapper);
|
||||
routingBeforeUpdate.verify(publishTaskService)
|
||||
.ensureTaskOwnedByCurrentInstance(task, "publish task heartbeat");
|
||||
.ensureTaskOwnedByCurrentInstance(task, PublishTaskService.MODULE_TYPE + " task heartbeat");
|
||||
routingBeforeUpdate.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
verify(publishTaskService).touchHeartbeat(taskId, request);
|
||||
}
|
||||
@@ -108,10 +100,12 @@ class TaskHeartbeatServiceTest {
|
||||
@SuppressWarnings("unchecked")
|
||||
void shopDataCrawlHeartbeatChecksOwnerBeforeDatabaseUpdate() {
|
||||
long taskId = 20143L;
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(taskId);
|
||||
task.setModuleType("SHOP_DATA_CRAWL");
|
||||
task.setStatus("RUNNING");
|
||||
FileTaskEntity task = runningTask(taskId, "SHOP_DATA_CRAWL");
|
||||
ShopDataCrawlTaskService shopDataCrawlTaskService = mock(ShopDataCrawlTaskService.class);
|
||||
ShopDataCrawlTaskCacheService cacheService = mock(ShopDataCrawlTaskCacheService.class);
|
||||
TaskHeartbeatService service = serviceWith(
|
||||
new ShopDataCrawlTaskHeartbeatSpi(cacheService, shopDataCrawlTaskService));
|
||||
|
||||
TaskHeartbeatRequest request = new TaskHeartbeatRequest();
|
||||
request.setModuleType("SHOP_DATA_CRAWL");
|
||||
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task);
|
||||
@@ -122,27 +116,27 @@ class TaskHeartbeatServiceTest {
|
||||
assertTrue(result.isAlive());
|
||||
InOrder order = inOrder(shopDataCrawlTaskService, fileTaskMapper);
|
||||
order.verify(shopDataCrawlTaskService)
|
||||
.ensureTaskOwnedByCurrentInstance(task, "shop data crawl task heartbeat");
|
||||
.ensureTaskOwnedByCurrentInstance(task, "SHOP_DATA_CRAWL task heartbeat");
|
||||
order.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
verify(shopDataCrawlTaskCacheService).touchTaskHeartbeat(taskId);
|
||||
verify(shopDataCrawlTaskCacheService).saveTaskCache(task);
|
||||
verify(brandCrawlTaskMapper, never()).selectOne(any(LambdaQueryWrapper.class));
|
||||
verify(cacheService).touchTaskHeartbeat(taskId);
|
||||
verify(cacheService).saveTaskCache(task);
|
||||
// SHOP_DATA_CRAWL 不需要查品牌任务表(历史行为:跳过品牌查找)
|
||||
verifyNoInteractions(brandTaskHeartbeatSpi);
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void collectDataHeartbeatForwardsProcessedKeywordProgress() {
|
||||
long taskId = 21016L;
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(taskId);
|
||||
task.setModuleType("COLLECT_DATA");
|
||||
task.setStatus("RUNNING");
|
||||
FileTaskEntity task = runningTask(taskId, "COLLECT_DATA");
|
||||
CollectDataService collectDataService = mock(CollectDataService.class);
|
||||
TaskHeartbeatService service = serviceWith(new CollectDataTaskHeartbeatSpi(collectDataService));
|
||||
|
||||
TaskHeartbeatRequest request = new TaskHeartbeatRequest();
|
||||
request.setCurrent(4);
|
||||
request.setTotal(19);
|
||||
|
||||
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task);
|
||||
when(brandCrawlTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null);
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
|
||||
TaskHeartbeatVo result = service.heartbeat(taskId, request);
|
||||
@@ -155,21 +149,39 @@ class TaskHeartbeatServiceTest {
|
||||
@SuppressWarnings("unchecked")
|
||||
void similarAsinHeartbeatUsesRedisWithoutRefreshingRecentDatabaseCheckpoint() {
|
||||
long taskId = 20581L;
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(taskId);
|
||||
task.setModuleType("SIMILAR_ASIN");
|
||||
task.setStatus("RUNNING");
|
||||
FileTaskEntity task = runningTask(taskId, "SIMILAR_ASIN");
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
SimilarAsinTaskCacheService cacheService = mock(SimilarAsinTaskCacheService.class);
|
||||
TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
|
||||
SimilarAsinProperties properties = mock(SimilarAsinProperties.class);
|
||||
TaskHeartbeatService service = serviceWith(
|
||||
new SimilarAsinTaskHeartbeatSpi(cacheService, taskFileJobService, properties));
|
||||
|
||||
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task);
|
||||
when(brandCrawlTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null);
|
||||
when(similarAsinProperties.getDbTaskTouchIntervalMillis()).thenReturn(120000L);
|
||||
when(properties.getDbTaskTouchIntervalMillis()).thenReturn(120000L);
|
||||
|
||||
TaskHeartbeatVo result = service.heartbeat(taskId, new TaskHeartbeatRequest());
|
||||
|
||||
assertTrue(result.isAlive());
|
||||
assertEquals("SIMILAR_ASIN", result.getModuleType());
|
||||
verify(similarAsinTaskCacheService).touchTaskHeartbeat(taskId);
|
||||
verify(cacheService).touchTaskHeartbeat(taskId);
|
||||
verify(taskFileJobService).touchRunningAssembleJobsIfStale(taskId, "SIMILAR_ASIN", 120000L);
|
||||
// 检查点未到期:不写 biz_file_task.updated_at
|
||||
verify(fileTaskMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void unknownModuleStillHeartbeatsFileTaskWithoutModuleHandler() {
|
||||
long taskId = 20150L;
|
||||
FileTaskEntity task = runningTask(taskId, "UNREGISTERED_MODULE");
|
||||
TaskHeartbeatService service = serviceWith();
|
||||
|
||||
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task);
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
|
||||
TaskHeartbeatVo result = service.heartbeat(taskId, new TaskHeartbeatRequest());
|
||||
|
||||
assertTrue(result.isAlive(), "未注册 SPI 的模块不应影响通用心跳(保持向后兼容)");
|
||||
assertEquals("UNREGISTERED_MODULE", result.getModuleType());
|
||||
}
|
||||
}
|
||||
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService;
|
||||
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService;
|
||||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService;
|
||||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.publish.service.PublishTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.publish.service.PublishTaskService;
|
||||
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService;
|
||||
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
|
||||
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService;
|
||||
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskCacheService;
|
||||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskHeartbeatSpi;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
/**
|
||||
* 模块心跳 SPI 的覆盖面自检(2026-09 全维度审查 G5)。
|
||||
*
|
||||
* <p>心跳按 moduleType 路由,moduleType 字符串写错会让该模块的心跳静默失效
|
||||
* (任务被当成无心跳 → 30 分钟后误判失败)。这里把"哪个实现负责哪个 moduleType"
|
||||
* 钉死:新增模块漏实现或写错字符串都会红测试。
|
||||
*/
|
||||
class TaskModuleHeartbeatSpiCoverageTest {
|
||||
|
||||
private static Map<String, TaskModuleHeartbeatSpi> implementationsByModuleType() {
|
||||
Map<String, TaskModuleHeartbeatSpi> impls = new LinkedHashMap<>();
|
||||
register(impls, new AppearancePatentTaskHeartbeatSpi(mock(AppearancePatentTaskCacheService.class)));
|
||||
register(impls, new CollectDataTaskHeartbeatSpi(mock(CollectDataService.class)));
|
||||
register(impls, new DeleteBrandTaskHeartbeatSpi(mock(DeleteBrandTaskCacheService.class)));
|
||||
register(impls, new PatrolDeleteTaskHeartbeatSpi(mock(PatrolDeleteTaskCacheService.class)));
|
||||
register(impls, new PriceTrackTaskHeartbeatSpi(mock(PriceTrackTaskCacheService.class)));
|
||||
register(impls, new ProductRiskTaskHeartbeatSpi(mock(ProductRiskTaskCacheService.class)));
|
||||
register(impls, new PublishTaskHeartbeatSpi(mock(PublishTaskService.class)));
|
||||
register(impls, new QueryAsinTaskHeartbeatSpi(mock(QueryAsinTaskCacheService.class)));
|
||||
register(impls, new ShopDataCrawlTaskHeartbeatSpi(
|
||||
mock(ShopDataCrawlTaskCacheService.class), mock(ShopDataCrawlTaskService.class)));
|
||||
register(impls, new ShopMatchTaskHeartbeatSpi(mock(ShopMatchTaskCacheService.class)));
|
||||
register(impls, new SimilarAsinTaskHeartbeatSpi(
|
||||
mock(SimilarAsinTaskCacheService.class), mock(TaskFileJobService.class),
|
||||
mock(SimilarAsinProperties.class)));
|
||||
register(impls, new WithdrawTaskHeartbeatSpi(mock(WithdrawTaskCacheService.class)));
|
||||
return impls;
|
||||
}
|
||||
|
||||
private static void register(Map<String, TaskModuleHeartbeatSpi> impls, TaskModuleHeartbeatSpi impl) {
|
||||
TaskModuleHeartbeatSpi exists = impls.put(impl.moduleType(), impl);
|
||||
if (exists != null) {
|
||||
throw new IllegalStateException("moduleType=" + impl.moduleType() + " 重复实现: "
|
||||
+ exists.getClass().getName() + " / " + impl.getClass().getName());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void everyHeartbeatModuleHasImplementationWithExactModuleType() {
|
||||
Map<String, TaskModuleHeartbeatSpi> impls = implementationsByModuleType();
|
||||
|
||||
assertEquals(12, impls.size(), "有任务心跳的模块都应有 SPI 实现,实际 " + impls.keySet());
|
||||
for (String moduleType : new String[]{
|
||||
"APPEARANCE_PATENT", "COLLECT_DATA", "DELETE_BRAND", "PATROL_DELETE", "PRICE_TRACK",
|
||||
"PRODUCT_RISK_RESOLVE", "PUBLISH", "QUERY_ASIN", "SHOP_DATA_CRAWL", "SHOP_MATCH",
|
||||
"SIMILAR_ASIN", "WITHDRAW"}) {
|
||||
assertEquals(true, impls.containsKey(moduleType), "缺少心跳实现: " + moduleType);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void moduleTypesAreUppercaseAndUnique() {
|
||||
for (TaskModuleHeartbeatSpi impl : implementationsByModuleType().values()) {
|
||||
String moduleType = impl.moduleType();
|
||||
assertEquals(moduleType.toUpperCase(), moduleType,
|
||||
"moduleType 必须与 biz_file_task.module_type 一致(全大写): " + moduleType);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user