fix: 全维度审查修复(安全/正确性/性能/稳定性/客户端/前端)

安全
- /api/ziniao/** 五个匿名接口加管理员鉴权(此前可匿名换取任意员工店铺登录令牌)
- 删除 Flask 遗留后门:默认密码建超管 + 每次启动写生产 users 表(服务端与客户端各一份)
- 进度/详情接口归属过滤:新增 TaskProgressOwnershipSupport,11 模块 progress/light 与
  /tasks/batch 接入,DTO 补 userId,前端 13 个查询封装补传(未传时后端不过滤,兼容旧端)
- 代理提取链接(含账密)不再明文入日志(新增 common/util/SecretMasking)
- 全局异常兜底不再回传原始异常信息;内部令牌比较改常量时间
- 登录加失败计数与锁定(10 次锁 15 分钟);品牌源文件下载加 SSRF 防护
- AdminApiGuardFilter 覆盖前缀从 2 扩到 15(开关默认 false,行为不变,为收紧做准备)
- 生产关闭 springdoc/knife4j(/doc.html 匿名可读全部接口定义)

正确性
- 40901/40902 拆分:锁竞争不再被伪装成 success=true(此前客户端停止重试、分片静默丢失)
- 假成功收敛:集采明细批量写失败改为抛出、去重 worker 异常标失败、4 个 worker 改判
  success 字段、publish 空 ASIN 行参与批次 flush、巡店删除全失败带 error 上报
- 客户端心跳 discard 移入 finally(7 模块,失败路径不再留僵尸 RUNNING 任务)
- 状态机条件更新:跟价停止循环、集采 activate/fail、imagevideo 归档回填、店铺匹配提交

性能
- 前端入口包 JS 1.05MB→204KB、CSS 355KB→10.7KB(Element Plus 改按需 + el-config-provider)
- 载荷引用计数按指针里的 taskId 收敛(原 JSON 列 IN 全表扫且逐行调用)
- 店铺明细多值批量 INSERT;快照 upsert 预载缓存;结果文件列改单条 UPDATE
- 新增迁移 V120(补 3 个缺失索引)/V121(删 4 个被覆盖的冗余索引)/V122(URL 前缀索引)

稳定性
- 新增 common/util/ThreadPools 有界线程池替换 5 处无界队列(防堆积 OOM)
- Redis 锁释放改 Lua 原子校验(原裸 delete 会误删他人已过期的锁)
- imagevideo 加死节点接管;锁续期失败重试;调度池 4→16;openStream 全部加超时
- 事务内远程对象删除移到提交后;启动恢复锁按实例命名

客户端
- 不再 taskkill /f /im chrome.exe(改为按调试端口精准回收,不杀用户自己的浏览器)
- 密码检测不再无条件杀紫鸟进程;品牌检测加全局互斥(代理池不再互相覆盖)
- base_dir 统一到 exe 目录(原被 os.getcwd() 覆盖,日志/缓存会分裂两个目录)
- 缓存加定时清理;图片下载加超时;mkstemp 句柄托管

测试
- 同步更新受影响的契约测试(构造器签名/条件更新/方法改名/新增接口方法等)
- 修复 FaultInjectionTest 等 3 处 mock 未 stub 流式 read 导致的读循环 OOM
- mvn test 2795 个测试全绿
This commit is contained in:
2026-09-14 04:15:36 +08:00
parent c448f49e30
commit 6d46506726
135 changed files with 3222 additions and 829 deletions
@@ -57,19 +57,18 @@
@input="onKeywordInput"
/>
<div class="bell-search-days">
<input
v-model="startDate"
type="date"
class="bell-date-input"
aria-label="始日期"
@change="reload"
/>
<span class="bell-date-sep"></span>
<input
v-model="endDate"
type="date"
class="bell-date-input"
aria-label="结束日期"
<el-date-picker
:model-value="dateRangeValue"
type="daterange"
value-format="YYYY-MM-DD"
start-placeholder="始日期"
end-placeholder="结束日期"
range-separator=""
size="small"
class="bell-date-picker"
popper-class="bell-date-popper"
:clearable="true"
@update:model-value="onDateRangeChange"
@change="reload"
/>
<button v-if="hasFilter" type="button" class="bell-search-reset" @click="resetFilters">
@@ -171,6 +170,23 @@ const groupedItems = computed(() => groupNotificationsByDay(items.value))
const totalPages = computed(() => Math.max(1, Math.ceil(total.value / PAGE_SIZE)))
const hasFilter = computed(() => Boolean(keyword.value.trim() || startDate.value || endDate.value))
/**
* el-date-picker 的绑定桥:日期区间在内部仍用 startDate/endDate 两个 ref 表示
* hasFilter/normalizedDayRange/resetFilters 都基于它们)。
* 此前模板写的是 v-model="dateRange",而 dateRange 从未声明 → vue-tsc 直接报 TS2339 构建失败,
* 且日期筛选完全不下发参数。
*/
const dateRangeValue = computed<[string, string] | null>(() =>
startDate.value && endDate.value ? [startDate.value, endDate.value] : null
)
function onDateRangeChange(value: [string, string] | null) {
const start = value?.[0]
const end = value?.[1]
startDate.value = start ? String(start) : ''
endDate.value = end ? String(end) : ''
}
let pollTimer: number | null = null
let searchTimer: number | null = null
@@ -0,0 +1,28 @@
package com.nanri.aiimage.common.exception;
/**
* 业务错误码常量。
*
* <p>历史上 40901 被两种语义复用:「任务已结束」(幂等忽略)与「任务正在处理中」(分布式锁竞争,
* 应重试);而 GlobalExceptionHandler 把 40901 一律转成 HTTP 200 + success=true
* 导致锁竞争时 Python worker 误判回传成功并停止重试,分片数据静默丢失。
* 现拆分为两个码:
* <ul>
* <li>{@link #TASK_ALREADY_FINISHED}:任务已结束,重复提交无意义 → 幂等忽略,响应 success=true</li>
* <li>{@link #TASK_BUSY}:任务正被其它请求持锁推进 → 响应 success=false,调用方应稍后重试。</li>
* </ul>
*/
public final class BusinessCodes {
private BusinessCodes() {
}
/** 任务已结束,拒绝重复提交。语义:幂等忽略,响应 success=true,调用方不应重试。 */
public static final int TASK_ALREADY_FINISHED = 40901;
/** 任务正在处理中(分布式锁竞争)。语义:资源忙,响应 success=false,调用方应稍后重试。 */
public static final int TASK_BUSY = 40902;
/** 任务归属其它实例,需转发。 */
public static final int TASK_OWNER_FORWARD = 40903;
}
@@ -61,9 +61,15 @@ public class GlobalExceptionHandler {
@ExceptionHandler(BusinessException.class)
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
if (Integer.valueOf(40901).equals(ex.getCode())) {
if (Integer.valueOf(BusinessCodes.TASK_ALREADY_FINISHED).equals(ex.getCode())) {
// 幂等忽略:任务已结束时的重复提交无副作用,按成功返回,避免客户端反复重试
return ApiResponse.success("任务已结束,忽略重复提交", null);
}
if (Integer.valueOf(BusinessCodes.TASK_BUSY).equals(ex.getCode())) {
// 锁竞争:必须如实返回失败 + 可重试码,否则 worker 会把「未落库」当成功而停止重试
log.warn("[business] 任务忙,调用方应稍后重试: {}", ex.getMessage());
return ApiResponse.fail(BusinessCodes.TASK_BUSY, ex.getMessage());
}
return ex.getCode() == null
? ApiResponse.fail(ex.getMessage())
: ApiResponse.fail(ex.getCode(), ex.getMessage());
@@ -100,7 +106,9 @@ public class GlobalExceptionHandler {
return ApiResponse.fail("客户端已断开连接");
}
log.error("Unhandled exception", ex);
return ApiResponse.fail("服务异常: " + ex.getMessage());
// 不再回传原始异常信息:SQL 报错、类名与内部路径会直接暴露给调用方,便于攻击者
// 摸清技术栈与表结构。详情只进日志(上方 log.error 已带完整堆栈),对外统一文案。
return ApiResponse.fail("服务器内部错误,请稍后重试");
}
private boolean isClientAbort(Throwable ex) {
@@ -0,0 +1,52 @@
package com.nanri.aiimage.common.util;
import java.net.URI;
/**
* 敏感串脱敏工具(2026-09 全维度审查后收口)。
*
* <p>背景:代理提取链接形态为 {@code http://user:pass@host:port},代码中曾有多处
* 直接把整串打进日志,导致用户代理账号密码长期留在应用日志与 docker logs 里。
*/
public final class SecretMasking {
private SecretMasking() {
}
/** 通用掩码:保留前 4 与后 4 字符;过短则整体掩掉。 */
public static String mask(String value) {
if (value == null || value.isBlank()) {
return "";
}
String text = value.trim();
if (text.length() <= 8) {
return "****";
}
return text.substring(0, 4) + "****" + text.substring(text.length() - 4);
}
/** 代理掩码:隐去账号密码,保留 scheme://host:port 便于运维核对。 */
public static String maskProxy(String value) {
if (value == null || value.isBlank()) {
return "";
}
try {
URI uri = URI.create(value.trim());
if (uri.getHost() == null || uri.getHost().isBlank()) {
return mask(value);
}
StringBuilder masked = new StringBuilder();
masked.append(uri.getScheme() == null ? "http" : uri.getScheme()).append("://");
if (uri.getUserInfo() != null && !uri.getUserInfo().isBlank()) {
masked.append("***@");
}
masked.append(uri.getHost());
if (uri.getPort() > 0) {
masked.append(':').append(uri.getPort());
}
return masked.toString();
} catch (Exception ex) {
return mask(value);
}
}
}
@@ -0,0 +1,47 @@
package com.nanri.aiimage.common.util;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* 有界线程池工厂(2026-09 全维度审查补)。
*
* <p>业务里多处使用 {@code Executors.newFixedThreadPool}:它内部是**无界**
* {@code LinkedBlockingQueue},任务堆积时永远不会触发拒绝策略,会把内存吃满
* (表现为 OOM,或整机因 GC 变慢导致所有任务一起劣化)。
* 统一改为有界队列 + {@code CallerRunsPolicy}:队列满时在提交线程执行,形成天然背压。
*/
public final class ThreadPools {
private ThreadPools() {
}
/** 默认队列容量:足以吸收突发,又不至于无界堆积。 */
public static final int DEFAULT_QUEUE_CAPACITY = 512;
/** 有界固定线程池(daemon 线程,空闲可回收)。 */
public static ExecutorService boundedFixed(String threadNamePrefix, int threads) {
return boundedFixed(threadNamePrefix, threads, DEFAULT_QUEUE_CAPACITY);
}
/** 有界固定线程池(显式队列容量)。 */
public static ExecutorService boundedFixed(String threadNamePrefix, int threads, int queueCapacity) {
ThreadPoolExecutor executor = new ThreadPoolExecutor(
Math.max(1, threads),
Math.max(1, threads),
// keepAliveTime 必须 > 0:下面开了 allowCoreThreadTimeOut
// 传 0 会让构造器直接抛 "Core threads must have nonzero keep alive times"
60L, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(Math.max(1, queueCapacity)),
runnable -> {
Thread thread = new Thread(runnable, threadNamePrefix);
thread.setDaemon(true);
return thread;
},
new ThreadPoolExecutor.CallerRunsPolicy());
executor.allowCoreThreadTimeOut(true);
return executor;
}
}
@@ -63,6 +63,25 @@ public class AdminApiGuardFilter extends OncePerRequestFilter {
private static final String[] USER_TOOL_PREFIXES = {
"/api/collect-data",
"/api/price-track",
// 2026-09 全维度审查补:以下前缀此前完全在守卫范围之外。
// /api/files 匿名可上传(2GB/次,可耗尽临时盘);/api/digital-human 匿名可发布/删除版本;
// /api/image-video 的 secrets 接口 userId 取自请求体;/api/brand 的 fileUrl 曾可直接请求任意地址。
"/api/files",
"/api/digital-human",
"/api/image-video",
"/api/brand",
"/api/appearance-patent",
"/api/similar-asin",
"/api/query-asin",
"/api/patrol-delete",
"/api/product-risk-resolve",
"/api/shop-match",
"/api/shop-data-crawl",
"/api/withdraw",
"/api/task-file-jobs",
// /api/tasks/{taskId}/interrupted 仅凭 taskId 即可把 RUNNING 任务置为 FAILED
// 匿名遍历 taskId 就能批量打断线上任务
"/api/tasks",
};
/**
@@ -72,6 +91,9 @@ public class AdminApiGuardFilter extends OncePerRequestFilter {
private static final String[] SELF_SERVICE_PREFIXES = {
"/api/user-secrets",
"/api/notifications",
// 2026-09 全维度审查补:内部端点此前仅靠 controller 自校验令牌,纳入守卫后
// 不带令牌的请求直接 401(带可信令牌的仍由 doFilterInternal 放行)
"/api/internal",
};
private final AdminAuthSupport adminAuthSupport;
@@ -3,12 +3,16 @@ package com.nanri.aiimage.config;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.JdkClientHttpRequestFactory;
import java.io.IOException;
import java.io.InputStream;
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.ProxySelector;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -55,6 +59,38 @@ public class HttpClientPool {
}
}
/**
* 打开远程文件流(带超时),调用方负责关闭返回的流。
*
* <p>替代裸 {@code URI.create(url).toURL().openStream()}:后者走 JVM 默认超时(0 = 无限),
* 上游半开连接或挂起时会把 Tomcat 工作线程无限占用(管理端批量打包可同时挂多个)。
* 返回的流是流式的,适用于「服务端代理下载 OSS 文件转发给浏览器」这类不落盘场景。
*
* @param url 远程地址
* @param timeout 等待响应超时(连接建立 + 响应头);非法值钳制到 1 秒
* @throws IOException 非 2xx 响应或网络异常
* @throws InterruptedException 线程被中断
*/
public static InputStream openStreamWithTimeout(String url, Duration timeout) throws IOException, InterruptedException {
Duration effective = (timeout == null || timeout.isZero() || timeout.isNegative())
? Duration.ofSeconds(1)
: timeout;
HttpRequest request = HttpRequest.newBuilder(URI.create(url))
.timeout(effective)
.GET()
.build();
HttpResponse<InputStream> response = sharedHttpClient().send(request, HttpResponse.BodyHandlers.ofInputStream());
if (response.statusCode() / 100 != 2) {
try {
response.body().close();
} catch (Exception ignored) {
// 关闭失败不影响错误上报
}
throw new IOException("远程文件返回非 2xx: HTTP " + response.statusCode());
}
return response.body();
}
/** 按 readTimeout(毫秒)创建共享连接池工厂;非法值钳制到最小正数。 */
public static ClientHttpRequestFactory requestFactory(int readTimeoutMillis) {
return requestFactory(readTimeoutMillis, null);
@@ -20,8 +20,15 @@ public class SchedulingConfig {
private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai");
/**
* 调度线程池:承载全站 30+ 个 @Scheduled(含 imagevideo 1s 派发、5s 轮询、结果文件 worker 15s 等高频任务)。
*
* <p>此前默认 4 线程:任一慢任务(如结果文件组装被内联执行时)都会把兜底类任务
* StaleTaskRepair 心跳判死、陈旧扫描、历史清理)顺延,而兜底任务被顺延会直接放大线上故障面。
* 提到 16 并保持可配(aiimage.scheduling.pool-size)。
*/
@Bean
public TaskScheduler taskScheduler(@Value("${aiimage.scheduling.pool-size:4}") int poolSize) {
public TaskScheduler taskScheduler(@Value("${aiimage.scheduling.pool-size:16}") int poolSize) {
ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
scheduler.setPoolSize(Math.max(1, poolSize));
scheduler.setThreadNamePrefix("aiimage-scheduling-");
@@ -143,7 +143,10 @@ public class AdminAuthSupport {
if (expectedToken.isBlank() || suppliedToken == null || suppliedToken.isBlank()) {
return false;
}
return expectedToken.equals(suppliedToken.trim());
// 常量时间比较:逐字节 equals 可被计时侧信道逐位试探出内部令牌
return java.security.MessageDigest.isEqual(
expectedToken.getBytes(java.nio.charset.StandardCharsets.UTF_8),
suppliedToken.trim().getBytes(java.nio.charset.StandardCharsets.UTF_8));
}
/**
@@ -34,6 +34,7 @@ import java.net.URI;
import java.nio.charset.StandardCharsets;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
@RestController
@RequiredArgsConstructor
@@ -43,6 +44,7 @@ import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
public class AppearancePatentController {
private final AppearancePatentTaskService service;
private final TaskProgressOwnershipSupport progressOwnershipSupport;
@PostMapping("/parse")
@Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件,提取 id、ASIN、国家、URL、标题等字段。返回给前端的数据只包含整数 id 和 n_1 行;n_2、n_3 等子行会保存在 OSS 解析载荷中,用于最终结果补齐。创建后的任务状态为 PENDING,不会自动推送 Python。")
@@ -99,7 +101,9 @@ public class AppearancePatentController {
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询任务进度", description = "前端只对活跃任务调用该接口,建议 6 秒一次。接口只返回轻量任务状态,不返回明细结果。")
public ApiResponse<AppearancePatentTaskBatchVo> progress(@Valid @RequestBody AppearancePatentTaskBatchRequest request) {
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
return ApiResponse.success(service.progressBatch(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/light")
@@ -107,7 +111,8 @@ public class AppearancePatentController {
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(service.progressLight(request.getTaskIds()));
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
return ApiResponse.success(service.progressLight(request.getTaskIds(), request.getUserId()));
}
@PostMapping("/tasks/{taskId}/activate")
@@ -12,4 +12,8 @@ public class AppearancePatentTaskBatchRequest {
@NotEmpty
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询,避免每个任务单独请求。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds;
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
private Long userId;
}
@@ -103,8 +103,8 @@ public class AppearancePatentTaskService {
public static final String MODULE_TYPE = "APPEARANCE_PATENT";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
}
private static final String STATUS_PENDING = "PENDING";
private static final String STATUS_RUNNING = "RUNNING";
@@ -18,6 +18,9 @@ import org.springframework.http.ResponseCookie;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.Instant;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@@ -30,6 +33,50 @@ public class AuthService {
private final PermissionMenuService permissionMenuService;
private final AuthProperties authProperties;
/**
* 登录失败计数与锁定(2026-09 全维度审查补):生产是公网域名,此前无任何失败限制
* 即可无限撞库。内存实现(双节点各自计数,防护效果减半但不引入新依赖),
* 达到阈值后锁定 15 分钟。
*/
private static final int LOGIN_FAIL_LIMIT = 10;
private static final Duration LOGIN_LOCK_DURATION = Duration.ofMinutes(15);
private final Map<String, FailRecord> loginFailures = new ConcurrentHashMap<>();
/** 失败计数;lockedUntil 非空表示已锁定。 */
private record FailRecord(int count, Instant lockedUntil) {
}
private void assertLoginNotLocked(String username) {
FailRecord record = loginFailures.get(username);
if (record != null && record.lockedUntil() != null && record.lockedUntil().isAfter(Instant.now())) {
long minutes = Duration.between(Instant.now(), record.lockedUntil()).toMinutes() + 1;
log.warn("[auth] 登录已锁定 username={} 剩余约 {} 分钟", username, minutes);
throw new BusinessException("登录失败次数过多,请 " + minutes + " 分钟后再试");
}
}
private void recordLoginFailure(String username) {
// 防无界增长:积累较多时清掉未锁定的过期记录(登录接口调用频次低)
if (loginFailures.size() > 1000) {
Instant now = Instant.now();
loginFailures.entrySet().removeIf(e -> e.getValue().lockedUntil() == null
|| e.getValue().lockedUntil().isBefore(now));
}
loginFailures.compute(username, (key, old) -> {
int count = (old == null ? 0 : old.count()) + 1;
Instant lockedUntil = count >= LOGIN_FAIL_LIMIT ? Instant.now().plus(LOGIN_LOCK_DURATION) : null;
if (lockedUntil != null) {
log.warn("[auth] 登录失败达阈值,锁定 username={} count={} minutes={}",
username, count, LOGIN_LOCK_DURATION.toMinutes());
}
return new FailRecord(count, lockedUntil);
});
}
private void clearLoginFailures(String username) {
loginFailures.remove(username);
}
public LoginResultVo login(LoginRequest request) {
String username = trim(request.getUsername());
String password = request.getPassword() == null ? "" : request.getPassword();
@@ -41,12 +88,16 @@ public class AuthService {
throw new BusinessException("缺少设备ID,请在桌面端打开");
}
assertLoginNotLocked(username);
LoginUserEntity user = loginUserMapper.selectOne(new LambdaQueryWrapper<LoginUserEntity>()
.eq(LoginUserEntity::getUsername, username)
.last("LIMIT 1"));
if (user == null || !passwordEncoder.matches(password, user.getPasswordHash())) {
recordLoginFailure(username);
throw new BusinessException("用户名或密码错误");
}
clearLoginFailures(username);
boolean isAdmin = user.getIsAdmin() != null && user.getIsAdmin() == 1;
// 单设备登录:登录成功即把账号绑定到当前设备(last-login-wins),原设备下一次请求被顶下线
@@ -4,11 +4,13 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.BrandProgressProperties;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.script.DefaultRedisScript;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
@@ -21,6 +23,9 @@ public class BrandTaskProgressCacheService {
public static final String PHASE_FAILED = "failed";
private static final Duration FINALIZE_LOCK_TTL = Duration.ofMinutes(10);
/** 本实例(进程)的锁持有者标识:释放锁时用它校验"锁还是我的"。 */
private final String lockOwnerToken = java.util.UUID.randomUUID().toString();
private final StringRedisTemplate stringRedisTemplate;
private final BrandProgressProperties brandProgressProperties;
@SuppressWarnings("unused")
@@ -129,8 +134,10 @@ public class BrandTaskProgressCacheService {
public boolean acquireFinalizeLock(Long taskId) {
try {
// value 用持有者 token(而非时间戳):释放时需要它来校验"锁还是我的",
// 否则锁因 TTL 到期被他方持有后,本线程的裸 delete 会误删他人的锁
Boolean ok = stringRedisTemplate.opsForValue()
.setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
.setIfAbsent(buildFinalizeLockKey(taskId), lockOwnerToken, FINALIZE_LOCK_TTL);
return Boolean.TRUE.equals(ok);
} catch (Exception ex) {
log.warn("[brand-progress-cache] acquire finalize lock degraded taskId={} msg={}", taskId, ex.getMessage());
@@ -140,7 +147,12 @@ public class BrandTaskProgressCacheService {
public void releaseFinalizeLock(Long taskId) {
try {
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
// Lua 原子校验后删除(2026-09 全维度审查):此前是裸 delete,锁已过期(TTL 到期、
// 他人已持有)时会把别人的锁删掉,导致同一任务被两个线程同时收尾。
stringRedisTemplate.execute(new DefaultRedisScript<>(
"if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end",
Long.class),
List.of(buildFinalizeLockKey(taskId)), lockOwnerToken);
} catch (Exception ex) {
log.warn("[brand-progress-cache] release finalize lock degraded taskId={} msg={}", taskId, ex.getMessage());
}
@@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.BrandProgressProperties;
import com.nanri.aiimage.config.HttpClientPool;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
@@ -53,6 +54,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.time.Duration;
import java.time.Instant;
@@ -82,6 +85,21 @@ public class BrandTaskService {
private static final DateTimeFormatter DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
private static final Duration STALE_CHECK_LOCK_TTL = Duration.ofMinutes(10);
/** 源文件下载请求超时:比普通 API 调用宽松(源文件可能几十 MB),但必须有上限。 */
private static final Duration SOURCE_DOWNLOAD_TIMEOUT = Duration.ofMinutes(5);
/** SSRF 防护:禁止请求云元数据与环回地址(正常源文件都在自家 OSS/MinIO 域名上)。 */
private static final java.util.Set<String> BLOCKED_SOURCE_HOSTS = java.util.Set.of(
"169.254.169.254", "metadata.google.internal", "metadata", "localhost",
"127.0.0.1", "0.0.0.0", "::1", "[::1]");
private static boolean isBlockedSourceHost(String host) {
if (host == null || host.isBlank()) {
return true;
}
String normalized = host.trim().toLowerCase(java.util.Locale.ROOT);
return BLOCKED_SOURCE_HOSTS.contains(normalized) || normalized.endsWith(".localhost");
}
private static final long RESULT_SUBMIT_WAIT_MILLIS = 5 * 60 * 1000L;
private static final String STATUS_PENDING = "pending";
private static final String STATUS_RUNNING = "running";
@@ -92,8 +110,9 @@ public class BrandTaskService {
/** 结果文件并发上传数:OSS/MinIO 上传互不依赖,3 并发平衡收益与内存占用。 */
private static final int RESULT_UPLOAD_CONCURRENCY = 3;
private final ExecutorService resultUploadExecutor = Executors.newFixedThreadPool(
RESULT_UPLOAD_CONCURRENCY, namedThreadFactory("brand-result-upload"));
// 有界队列线程池:newFixedThreadPool 用的是无界队列,任务堆积时不会拒绝、会把内存吃满
private final ExecutorService resultUploadExecutor = com.nanri.aiimage.common.util.ThreadPools
.boundedFixed("brand-result-upload", RESULT_UPLOAD_CONCURRENCY);
@PreDestroy
void shutdownResultUploadExecutor() {
@@ -812,21 +831,49 @@ public class BrandTaskService {
}
private File downloadSourceFile(String fileUrl) {
URI uri;
try {
uri = URI.create(fileUrl);
} catch (Exception ex) {
log.warn("[brand] 源文件地址非法 fileUrl={} err={}", fileUrl, ex.getMessage());
throw new BusinessException("下载源文件失败: 地址非法");
}
// SSRF 防护(2026-09 全维度审查):fileUrl 来自请求体,若不校验则匿名调用方可让服务端
// 请求内网/云元数据地址(169.254.169.254 等)。正常源文件都落在自家 OSS/MinIO 上。
if (isBlockedSourceHost(uri.getHost())) {
log.warn("[brand] 拒绝下载疑似 SSRF 的源文件地址 host={} fileUrl={}", uri.getHost(), fileUrl);
throw new BusinessException("源文件地址不合法");
}
String filename = FileUtil.getName(uri.getPath());
if (filename == null || filename.isBlank()) {
filename = "brand-source.xlsx";
}
String suffix = FileUtil.extName(filename);
File downloadDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "brand-source-download"));
try {
URI uri = URI.create(fileUrl);
String filename = FileUtil.getName(uri.getPath());
if (filename == null || filename.isBlank()) {
filename = "brand-source.xlsx";
}
String suffix = FileUtil.extName(filename);
File downloadDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "brand-source-download"));
File tempFile = Files.createTempFile(downloadDir.toPath(), "brand_", suffix.isBlank() ? "" : "." + suffix).toFile();
try (InputStream inputStream = uri.toURL().openStream()) {
// 走统一连接池 + 显式超时。此前 uri.toURL().openStream() 无任何超时(JVM 默认 0 = 无限),
// 上游半开连接会把 Tomcat 工作线程无限占用;超时值比普通 API 调用宽松(源文件可能几十 MB)
HttpRequest downloadRequest = HttpRequest.newBuilder(uri)
.timeout(SOURCE_DOWNLOAD_TIMEOUT)
.GET()
.build();
HttpResponse<InputStream> response = HttpClientPool.sharedHttpClient()
.send(downloadRequest, HttpResponse.BodyHandlers.ofInputStream());
if (response.statusCode() / 100 != 2) {
log.warn("[brand] 下载源文件返回非 2xx fileUrl={} status={}", fileUrl, response.statusCode());
throw new BusinessException("下载源文件失败: HTTP " + response.statusCode());
}
try (InputStream inputStream = response.body()) {
FileUtil.writeFromStream(inputStream, tempFile);
}
return tempFile;
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("下载源文件失败");
// 此前该 catch 既不记日志也不带 cause,线上无法定位
log.warn("[brand] 下载源文件失败 fileUrl={} err={}", fileUrl, ex.getMessage(), ex);
throw new BusinessException("下载源文件失败", ex);
}
}
@@ -466,11 +466,8 @@ public class BrandTaskStorageService {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
// HexFormat 替代逐字节 String.format("%02x"):后者每次走 32 次 Formatter(热路径按行调用)
return java.util.HexFormat.of().formatHex(bytes);
} catch (Exception ex) {
throw new IllegalStateException("failed to hash brand scope", ex);
}
@@ -29,6 +29,7 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
@RestController
@RequiredArgsConstructor
@@ -37,6 +38,7 @@ import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
public class CollectDataController {
private final CollectDataService service;
private final TaskProgressOwnershipSupport progressOwnershipSupport;
@PostMapping("/parse")
@Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件,按行入库到 biz_collect_data_item,并保存任务筛选条件。任务初始状态为 PENDING。")
@@ -112,7 +114,9 @@ public class CollectDataController {
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询任务进度")
public ApiResponse<CollectDataTaskBatchVo> progressBatch(@Valid @RequestBody CollectDataTaskBatchRequest request) {
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
return ApiResponse.success(service.progressBatch(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/light")
@@ -120,7 +124,8 @@ public class CollectDataController {
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(service.progressLight(request.getTaskIds()));
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
return ApiResponse.success(service.progressLight(request.getTaskIds(), request.getUserId()));
}
@DeleteMapping("/tasks/{taskId}")
@@ -12,4 +12,8 @@ public class CollectDataTaskBatchRequest {
@NotEmpty
@Schema(description = "任务 ID 列表", requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds;
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
private Long userId;
}
@@ -4,9 +4,11 @@ import cn.hutool.core.util.IdUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.crypto.digest.DigestUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessCodes;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
@@ -104,8 +106,8 @@ public class CollectDataService {
*/
private static final String LEGACY_MODULE_TYPE = "collectdata";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
}
public static final int DEFAULT_PAGE_SIZE = 50;
@@ -379,34 +381,45 @@ public class CollectDataService {
@Transactional
public void activateTask(Long taskId, Long userId) {
FileTaskEntity task = requireTask(taskId, userId);
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
// 条件更新:上面的「已结束」判断与写入之间存在窗口(TOCTOU),期间 /fail 可能已把任务
// 标为 FAILED —— 整行 updateById 会把它复活成 RUNNING(前端显示"执行中"但无人推进)
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
.notIn(FileTaskEntity::getStatus, STATUS_SUCCESS, STATUS_FAILED)
.set(FileTaskEntity::getStatus, STATUS_RUNNING)
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
if (updated == 0) {
throw new BusinessException("任务已结束");
}
task.setStatus(STATUS_RUNNING);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
}
@Transactional
public void failTask(Long taskId, Long userId, String error) {
FileTaskEntity task = requireTask(taskId, userId);
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
return;
}
String message = firstNonBlank(error, "collect-data task dispatch failed");
FileResultEntity result = ensureTaskResult(task);
CollectDataStats stats = loadStats(task);
result.setSuccess(0);
result.setErrorMessage(message);
result.setRowCount(stats.finalRowCount);
fileResultMapper.updateById(result);
task.setStatus(STATUS_FAILED);
task.setErrorMessage(message);
task.setFailedFileCount(1);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
persistStats(task, stats);
fileTaskMapper.updateById(task);
boolean alreadyTerminal = STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus());
if (!alreadyTerminal) {
result.setSuccess(0);
result.setErrorMessage(message);
result.setRowCount(stats.finalRowCount);
fileResultMapper.updateById(result);
persistStats(task, stats);
}
// 条件更新:客户端报错(/fail)与结果文件组装完成(processResultFileJob 写 SUCCESS
// 可能并发 —— 无条件 updateById 会把已生成的 SUCCESS 覆盖成 FAILED(用户拿不到下载)或反之
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
.notIn(FileTaskEntity::getStatus, STATUS_SUCCESS, STATUS_FAILED)
.set(FileTaskEntity::getStatus, STATUS_FAILED)
.set(FileTaskEntity::getErrorMessage, message)
.set(FileTaskEntity::getFailedFileCount, 1)
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
if (updated == 0) {
log.info("[collect-data] failTask 跳过写入:任务已是终态 taskId={} status={}", taskId, task.getStatus());
}
}
@Transactional
@@ -954,13 +967,21 @@ public class CollectDataService {
// 复用同一份 stats 更新 finalRowCount 后再持久化,避免重复 loadStats 丢失 summaries。
stats.finalRowCount = (int) finalRowCount;
persistStats(task, stats);
task.setStatus(STATUS_SUCCESS);
task.setSuccessFileCount(1);
task.setFailedFileCount(0);
task.setErrorMessage(null);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
// 条件更新:任务可能已被 /fail 标为 FAILED(客户端报错与结果文件组装并发)——
// 无条件 updateById 会把 FAILED 覆盖成 SUCCESS(错误信息被清、用户看到「假成功」)。
// 结果文件已生成,故失败态下仍保留文件,只是不覆盖状态。
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
.ne(FileTaskEntity::getStatus, STATUS_FAILED)
.set(FileTaskEntity::getStatus, STATUS_SUCCESS)
.set(FileTaskEntity::getSuccessFileCount, 1)
.set(FileTaskEntity::getFailedFileCount, 0)
.set(FileTaskEntity::getErrorMessage, null)
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
if (updated == 0) {
log.warn("[collect-data] 结果文件已生成但任务已是 FAILED,保留失败态不覆盖 taskId={}", task.getId());
}
} finally {
FileUtil.del(xlsx);
}
@@ -1085,7 +1106,8 @@ public class CollectDataService {
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(MODULE_TYPE, taskId, TASK_LOCK_WAIT_MILLIS);
if (lockHandle == null) {
throw new BusinessException(40901, "任务正在处理,请稍后再试");
log.warn("[collect-data] 任务锁竞争,拒绝本次提交 taskId={} waitMillis={}", taskId, TASK_LOCK_WAIT_MILLIS);
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理,请稍后再试");
}
return lockHandle;
}
@@ -148,11 +148,8 @@ public class CollectDataResultDetailCodec {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
// HexFormat 替代逐字节 String.format("%02x")后者每次走 32 Formatter热路径按行调用
return java.util.HexFormat.of().formatHex(bytes);
} catch (Exception ex) {
throw new IllegalStateException("chunk detail ref hash failed", ex);
}
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.collectdata.util;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
@@ -133,12 +134,14 @@ public class CollectDataResultItemBatchWriter {
}
int written = 0;
int failedBatches = 0;
for (int from = 0; from < toUpsert.size(); from += batchSize) {
int to = Math.min(from + batchSize, toUpsert.size());
List<TaskResultItemEntity> batch = toUpsert.subList(from, to);
try {
written += taskResultItemMapper.upsertBatch(batch);
} catch (RuntimeException ex) {
failedBatches++;
log.warn("[collect-data] upsert result item batch failed, skip batch {}..{} taskId={}",
from, to, taskId, ex);
// 失败批的新行未落库从增量计数中扣除避免任务内累计虚高
@@ -150,6 +153,16 @@ public class CollectDataResultItemBatchWriter {
}
}
}
if (failedBatches > 0) {
// biz_task_result_item 是结果 Excel明细sheet 的唯一数据源静默跳批会让任务
// SUCCESS 收尾但明细缺行且与结果文件sheet 的汇总数量对不上假成功
// 抛出让本次 chunk 提交明确失败workersearch_spider识别 success=false 后会重试
// 重提按 payload_hash 幂等已落库行跳过未落库行补插最终收敛为完整数据
log.error("[collect-data] 明细写入存在失败批次,拒绝本次提交 taskId={} failedBatches={} totalBatches={}",
taskId, failedBatches, (toUpsert.size() + batchSize - 1) / batchSize);
throw new BusinessException("采集结果明细写入失败,请稍后重试(失败批次 "
+ failedBatches + "/" + ((toUpsert.size() + batchSize - 1) / batchSize) + "");
}
return new UpsertCounts(written, skipped, newlyInserted);
}
@@ -157,11 +170,8 @@ public class CollectDataResultItemBatchWriter {
try {
java.security.MessageDigest digest = java.security.MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(java.nio.charset.StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
// HexFormat 替代逐字节 String.format("%02x")后者每次走 32 Formatter热路径按行调用
return java.util.HexFormat.of().formatHex(bytes);
} catch (Exception ex) {
throw new IllegalStateException("结果明细 hash 计算失败", ex);
}
@@ -278,12 +278,14 @@ public class ConvertRunService {
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
List<GeneratedConvertFile> generatedFiles = new ArrayList<>();
Map<String, BufferedWriter> writers = new LinkedHashMap<>();
for (String outputFilename : outputFilenames) {
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
generatedFiles.add(new GeneratedConvertFile(outputFilename, outputFile));
writers.put(outputFilename, Files.newBufferedWriter(outputFile.toPath(), StandardCharsets.UTF_8));
}
try {
// 创建循环纳入 try N writer 创建失败时 N-1 个已打开的句柄会因 finally
// 尚未生效而泄漏同族 SplitRunService.SplitChunkWriter 已用 try/finally 处理
for (String outputFilename : outputFilenames) {
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
generatedFiles.add(new GeneratedConvertFile(outputFilename, outputFile));
writers.put(outputFilename, Files.newBufferedWriter(outputFile.toPath(), StandardCharsets.UTF_8));
}
streamTxtRowsToOutputs(inputFile, templateEntity, writers);
} finally {
IOException closeException = null;
@@ -75,12 +75,9 @@ public class DedupeRunService {
private final Map<Long, AtomicInteger> runningTaskCountMap = new ConcurrentHashMap<>();
private final Semaphore fileParallelSemaphore = new Semaphore(MAX_PARALLEL_FILES);
/** 共享有界文件执行器;不再为请求中的每个文件创建虚拟线程。 */
private final ExecutorService fileExecutor = Executors.newFixedThreadPool(
MAX_PARALLEL_FILES, runnable -> {
Thread thread = new Thread(runnable, "dedupe-file-worker");
thread.setDaemon(true);
return thread;
});
// 队列改为有界newFixedThreadPool 的无界队列在文件堆积时不会拒绝会把内存吃满
private final ExecutorService fileExecutor = com.nanri.aiimage.common.util.ThreadPools
.boundedFixed("dedupe-file-worker", MAX_PARALLEL_FILES);
/**
* 提交去重任务立即返回进度快照runId异步执行 流式读取 + 多文件并行 + 结果上传
@@ -171,6 +168,8 @@ public class DedupeRunService {
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
Map<String, DedupeArchiveEntry> archiveEntries = new ConcurrentHashMap<>();
List<DedupeResultItemVo> outcomeItems = new ArrayList<>();
// 是否有 worker 异常结束 true 时收尾必须标失败绝不能标 SUCCESS用户会看到成功但结果缺文件
boolean aborted = false;
try {
AtomicInteger processedCount = new AtomicInteger(0);
@@ -192,14 +191,26 @@ public class DedupeRunService {
}
}));
}
// 逐个等待processFile 内部已把单文件失败记入 outcomeItems不会往外抛
// 这里能捕到的是线程级异常Error/线程池拒绝必须记下来而不能当成功
for (Future<?> worker : workers) {
worker.get();
try {
worker.get();
} catch (Exception workerEx) {
aborted = true;
log.error("dedupe run worker aborted runId={} error", runId, workerEx);
}
}
} catch (Exception ex) {
aborted = true;
log.error("dedupe run async aborted runId={} error", runId, ex);
}
try {
if (aborted) {
// 有文件没能正常处理完就序列化/标成功会出现任务成功但结果缺文件
throw new IllegalStateException("部分文件处理线程异常结束,结果不完整");
}
if (folderMode && !archiveEntries.isEmpty()) {
DedupeResultItemVo zipItem = buildFolderZipResult(request, archiveEntries, task);
synchronized (progress) {
@@ -17,6 +17,7 @@ import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.shopkey.mapper.ShopManageGroupMapper;
import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
@@ -62,6 +63,7 @@ import java.util.zip.ZipOutputStream;
@Service
@RequiredArgsConstructor
@Slf4j
public class DedupeTotalDataService {
private static final int COMPARE_BATCH_SIZE = 5000;
@@ -579,6 +581,8 @@ public class DedupeTotalDataService {
} catch (Exception e) {
progress.setStatus("failed");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "导入 Excel 失败");
// 此前该 catch 既不记日志也不带 cause线上只有一句导入 Excel 失败无栈无定位线索
log.error("[dedupe-total-data] 异步导入失败 filename={} userId={}", filename, uploaderUserId, e);
} finally {
deleteQuietly(tempFile);
importCompletedAtMap.put(importId, System.currentTimeMillis());
@@ -648,7 +652,9 @@ public class DedupeTotalDataService {
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
throw new BusinessException("删除 Excel 匹配数据失败");
// 此前丢弃原始异常排查时只能看到一句删除 Excel 匹配数据失败
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败", e);
throw new BusinessException("删除 Excel 匹配数据失败", e);
}
}
@@ -946,7 +952,9 @@ public class DedupeTotalDataService {
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
throw new BusinessException("删除 Excel 匹配数据失败");
// 此前丢弃原始异常排查时只能看到一句删除 Excel 匹配数据失败
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败", e);
throw new BusinessException("删除 Excel 匹配数据失败", e);
}
}
@@ -37,6 +37,7 @@ import java.net.URI;
import java.nio.charset.StandardCharsets;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
@RestController
@@ -47,6 +48,7 @@ import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
public class DeleteBrandRunController {
private final DeleteBrandRunService deleteBrandRunService;
private final TaskProgressOwnershipSupport progressOwnershipSupport;
@PostMapping("/run")
@Operation(summary = "执行删除品牌解析", description = "读取上传的删除品牌 Excel,按国家分组解析并在每个国家内按 ASIN 去重,返回完整 payload。")
@@ -85,13 +87,17 @@ public class DeleteBrandRunController {
@PostMapping("/tasks/batch")
@Operation(summary = "批量获取删除品牌任务详情", description = "合并多个 taskId 的详情查询,减少轮询期对 DB/Redis 的压力。")
public ApiResponse<DeleteBrandTaskBatchVo> getTasksBatch(@Valid @RequestBody DeleteBrandTaskBatchRequest request) {
return ApiResponse.success(deleteBrandRunService.getTaskDetails(request.getTaskIds()));
// 归属过滤/tasks/batch 同样按 userId 过滤此前无校验遍历 taskId 即可读他人任务详情
return ApiResponse.success(deleteBrandRunService.getTaskDetails(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量获取删除品牌任务进度摘要", description = "仅返回任务基础状态和行进度摘要,用于前端轮询降载。")
public ApiResponse<DeleteBrandTaskBatchVo> getTaskProgressBatch(@Valid @RequestBody DeleteBrandTaskBatchRequest request) {
return ApiResponse.success(deleteBrandRunService.getTaskProgress(request.getTaskIds()));
// 归属过滤只查询属于该用户的任务userId 未传时保持原行为兼容未升级调用方
return ApiResponse.success(deleteBrandRunService.getTaskProgress(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/light")
@@ -99,7 +105,8 @@ public class DeleteBrandRunController {
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(deleteBrandRunService.progressLight(request.getTaskIds()));
// 透传 userId此前传 null = 不过滤归属匿名遍历 taskId 即可读他人任务状态2026-09 审查
return ApiResponse.success(deleteBrandRunService.progressLight(request.getTaskIds(), request.getUserId()));
}
@GetMapping("/tasks/{taskId}/deletion-status")
@@ -14,4 +14,8 @@ public class DeleteBrandTaskBatchRequest {
@NotEmpty(message = "taskIds 不能为空")
@Schema(description = "任务ID列表")
private List<Long> taskIds = new ArrayList<>();
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
private Long userId;
}
@@ -82,8 +82,8 @@ public class DeleteBrandRunService {
/** 降级组装时缺失分片对应的结果状态(客户端未回传该行)。 */
static final String MISSING_CHUNK_STATUS = "未回传";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final Duration TASK_LOCK_TTL = TaskDistributedLockService.DEFAULT_LOCK_TTL;
@@ -393,11 +393,8 @@ public class DeleteBrandTaskStorageService {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalizeScopeKey(value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
// HexFormat 替代逐字节 String.format("%02x")后者每次走 32 Formatter热路径按行调用
return java.util.HexFormat.of().formatHex(bytes);
} catch (Exception ex) {
throw new IllegalStateException("failed to hash delete-brand scope", ex);
}
@@ -153,7 +153,12 @@ public class ImageVideoArchiveService {
Object result = readJsonValue(task.getResultJson());
enrichCompletedTask(task, result);
task.setUpdatedAt(LocalDateTime.now());
taskMapper.updateById(task);
// 条件更新 archive_status 仍为 NULL 整行 updateById 会用读取快照覆盖并发节点
// 刚写入的 ARCHIVING 双节点 10s 周期归档时B 节点会把 A 的认领覆盖回可认领状态
// 随后 B CAS 也能成功导致同一视频被下载/上传两遍先上传的对象成孤儿
taskMapper.update(task, new LambdaUpdateWrapper<ImageVideoAsyncTaskEntity>()
.eq(ImageVideoAsyncTaskEntity::getId, task.getId())
.isNull(ImageVideoAsyncTaskEntity::getArchiveStatus));
}
private void archiveTask(Long taskId) {
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.imagevideo.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
@@ -180,6 +181,54 @@ public class ImageVideoAsyncTaskService {
}
}
/** owner 节点失联判定阈值(分钟):任务超过此时长未更新即认为原 owner 已下线。 */
private static final long OWNER_TAKEOVER_TIMEOUT_MINUTES = 15;
/**
* 死节点接管2026-09 全维度审查补owner 绑定的任务此前只有owner 重启时恢复自己的
* 一条路径owner 节点崩溃/被摘除后其任务永久停在生成中
* {@code StaleTaskRepairService} 只覆盖 biz_file_task / brand_crawl_tasks不含本表
*
* <p>分两类处理
* <ul>
* <li>WAITING/POLLING 只是等上游结果重放安全 清空 owner 交回可领取队列</li>
* <li>RUNNING 重放会重复调用上游生成有费用故超时后明确标 FAILED 让用户重试
* 而不是悄悄重新执行</li>
* </ul>
*/
@Scheduled(fixedDelayString = "${aiimage.image-video.owner-takeover-delay-ms:60000}")
public void takeoverAbandonedTasks() {
DistributedJobLockService.LockHandle jobLock =
distributedJobLockService.tryLock("image-video:owner-takeover", java.time.Duration.ofSeconds(30));
if (jobLock == null) {
return;
}
try (jobLock) {
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(OWNER_TAKEOVER_TIMEOUT_MINUTES);
int requeued = taskMapper.update(null, new LambdaUpdateWrapper<ImageVideoAsyncTaskEntity>()
.in(ImageVideoAsyncTaskEntity::getStatus,
TaskStatus.WAITING.name(), TaskStatus.POLLING.name())
.isNotNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
.ne(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
.lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff)
.set(ImageVideoAsyncTaskEntity::getOwnerInstanceId, null)
.set(ImageVideoAsyncTaskEntity::getUpdatedAt, LocalDateTime.now()));
int failed = taskMapper.update(null, new LambdaUpdateWrapper<ImageVideoAsyncTaskEntity>()
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.RUNNING.name())
.isNotNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
.ne(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
.lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff)
.set(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.FAILED.name())
.set(ImageVideoAsyncTaskEntity::getErrorMessage,
"处理节点失联超过 " + OWNER_TAKEOVER_TIMEOUT_MINUTES + " 分钟,任务已自动失败,请重试")
.set(ImageVideoAsyncTaskEntity::getUpdatedAt, LocalDateTime.now()));
if (requeued > 0 || failed > 0) {
log.warn("[image-video] 接管失联节点任务 requeued={} failed={} cutoffMinutes={}",
requeued, failed, OWNER_TAKEOVER_TIMEOUT_MINUTES);
}
}
}
@Scheduled(fixedDelayString = "${aiimage.image-video.failed-task-cleanup-delay-ms:60000}")
public void cleanupFailedTasks() {
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES);
@@ -44,6 +44,7 @@ import java.nio.charset.StandardCharsets;
import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
@RestController
@RequiredArgsConstructor
@@ -56,6 +57,7 @@ public class PatrolDeleteController {
private final PatrolDeleteResolveService patrolDeleteResolveService;
private final PatrolDeleteTaskService patrolDeleteTaskService;
private final TaskProgressOwnershipSupport progressOwnershipSupport;
@GetMapping("/candidates")
@Operation(summary = "查询备选店铺列表", description = "返回当前用户在巡店删除模块中已保存的备选店铺。")
@@ -168,7 +170,9 @@ public class PatrolDeleteController {
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询巡店删除任务进度", description = "仅返回任务状态和店铺结果摘要,用于前端轮询降载。")
public ApiResponse<PatrolDeleteTaskBatchVo> taskProgressBatch(@Valid @RequestBody PatrolDeleteTaskBatchRequest request) {
return ApiResponse.success(patrolDeleteTaskService.getTaskProgressBatch(request.getTaskIds()));
// 归属过滤只查询属于该用户的任务userId 未传时保持原行为兼容未升级调用方
return ApiResponse.success(patrolDeleteTaskService.getTaskProgressBatch(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/light")
@@ -176,7 +180,8 @@ public class PatrolDeleteController {
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(patrolDeleteTaskService.progressLight(request.getTaskIds()));
// 透传 userId此前传 null = 不过滤归属匿名遍历 taskId 即可读他人任务状态2026-09 审查
return ApiResponse.success(patrolDeleteTaskService.progressLight(request.getTaskIds(), request.getUserId()));
}
@PostMapping("/tasks")
@@ -11,4 +11,7 @@ public class PatrolDeleteTaskBatchRequest {
@NotEmpty(message = "taskIds 不能为空")
private List<Long> taskIds = new ArrayList<>();
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
private Long userId;
}
@@ -3,8 +3,10 @@ package com.nanri.aiimage.modules.patroldelete.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessCodes;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
@@ -53,8 +55,8 @@ public class PatrolDeleteTaskService {
private static final String MODULE_TYPE = "PATROL_DELETE";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_PENDING = -1;
@@ -958,6 +960,27 @@ public class PatrolDeleteTaskService {
return false;
}
/**
* 一条 UPDATE 落库结果文件列
*
* <p>原先对每个成功行 {@code updateById}N = N 次网络往返 + N 次提交本方法非事务自动提交
* 且写 binlog 被同步放大N=500 时白花约 0.5~1s条件与 listTaskRows 的查询口径一致
*/
private void updateResultFileColumns(List<Long> rowIds, String filename, String objectKey,
long fileSize, int rowCount) {
if (rowIds == null || rowIds.isEmpty()) {
return;
}
fileResultMapper.update(null, new LambdaUpdateWrapper<FileResultEntity>()
.in(FileResultEntity::getId, rowIds)
.eq(FileResultEntity::getSuccess, RESULT_SUCCESS)
.set(FileResultEntity::getResultFilename, filename)
.set(FileResultEntity::getResultFileUrl, objectKey)
.set(FileResultEntity::getResultFileSize, fileSize)
.set(FileResultEntity::getResultContentType, CONTENT_TYPE_XLSX)
.set(FileResultEntity::getRowCount, rowCount));
}
private void finalizeTaskWorkbook(FileTaskEntity task, List<FileResultEntity> rows, List<PatrolDeleteResultItemVo> snapshots) {
List<PatrolDeleteResultItemVo> successItems = snapshots.stream()
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
@@ -966,20 +989,23 @@ public class PatrolDeleteTaskService {
String filename = buildTaskWorkbookFilename(task);
int rowCount = excelAssemblyService.countRows(successItems);
FileResultEntity firstSuccessRow = null;
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
row.setResultFilename(filename);
row.setResultFileUrl(null);
row.setResultFileSize(0L);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
if (firstSuccessRow == null) {
firstSuccessRow = row;
}
List<Long> successRowIds = new ArrayList<>();
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
// 内存对象同步改下方 buildSnapshotFromDb 直接读这些对象
row.setResultFilename(filename);
row.setResultFileUrl(null);
row.setResultFileSize(0L);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
successRowIds.add(row.getId());
if (firstSuccessRow == null) {
firstSuccessRow = row;
}
}
}
if (firstSuccessRow != null) {
updateResultFileColumns(successRowIds, filename, null, 0L, rowCount);
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId());
snapshots = buildSnapshotFromDb(task, rows);
}
@@ -1017,16 +1043,19 @@ public class PatrolDeleteTaskService {
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
long fileSize = xlsx.length();
int rowCount = excelAssemblyService.countRows(successItems);
List<Long> successRowIds = new ArrayList<>();
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
// 内存对象同步改紧随其后的 updateTaskStatusFromRows/buildSnapshotFromDb 直接读这些对象
row.setResultFilename(filename);
row.setResultFileUrl(objectKey);
row.setResultFileSize(fileSize);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
successRowIds.add(row.getId());
}
}
updateResultFileColumns(successRowIds, filename, objectKey, fileSize, rowCount);
updateTaskStatusFromRows(task, rows);
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
fileTaskMapper.updateById(task);
@@ -1056,7 +1085,8 @@ public class PatrolDeleteTaskService {
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
if (lockHandle == null) {
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
log.warn("[patrol-delete] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
}
return lockHandle;
}
@@ -52,6 +52,7 @@ import java.nio.charset.StandardCharsets;
import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
@RestController
@RequiredArgsConstructor
@@ -65,6 +66,7 @@ public class PriceTrackController {
private final PriceTrackService priceTrackService;
private final PriceTrackTaskService priceTrackTaskService;
private final TaskProgressOwnershipSupport progressOwnershipSupport;
private final PriceTrackLoopRunService priceTrackLoopRunService;
private final SkipPriceAsinService skipPriceAsinService;
@@ -259,13 +261,17 @@ public class PriceTrackController {
@PostMapping("/tasks/batch")
@Operation(summary = "批量查询任务详情", description = "按 taskIds 返回任务概要和各店铺结果,供前端查询使用。")
public ApiResponse<PriceTrackTaskBatchVo> tasksBatch(@Valid @RequestBody PriceTrackTaskBatchRequest request) {
return ApiResponse.success(priceTrackTaskService.getTaskDetailsBatch(request.getTaskIds()));
// 归属过滤/tasks/batch 同样按 userId 过滤此前无校验遍历 taskId 即可读他人任务详情
return ApiResponse.success(priceTrackTaskService.getTaskDetailsBatch(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询任务进度摘要", description = "仅返回任务状态、时间和错误等轻量信息,供前端轮询降载。")
public ApiResponse<PriceTrackTaskBatchVo> taskProgressBatch(@Valid @RequestBody PriceTrackTaskBatchRequest request) {
return ApiResponse.success(priceTrackTaskService.getTaskProgressBatch(request.getTaskIds()));
// 归属过滤只查询属于该用户的任务userId 未传时保持原行为兼容未升级调用方
return ApiResponse.success(priceTrackTaskService.getTaskProgressBatch(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/light")
@@ -273,7 +279,8 @@ public class PriceTrackController {
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(priceTrackTaskService.progressLight(request.getTaskIds()));
// 透传 userId此前传 null = 不过滤归属匿名遍历 taskId 即可读他人任务状态2026-09 审查
return ApiResponse.success(priceTrackTaskService.progressLight(request.getTaskIds(), request.getUserId()));
}
@PostMapping("/tasks/{taskId}/result")
@@ -13,4 +13,8 @@ public class PriceTrackTaskBatchRequest {
@NotEmpty(message = "taskIds不能为空")
@Schema(description = "待查询的任务主键列表,无效或非本模块 taskId 会出现在 missingTaskIds", requiredMode = Schema.RequiredMode.REQUIRED, example = "[200,201]")
private List<Long> taskIds = new ArrayList<>();
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
private Long userId;
}
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.pricetrack.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
@@ -149,7 +150,13 @@ public class PriceTrackLoopRunService {
if (entity.getActiveTaskId() == null || isChildTaskTerminal(entity.getActiveTaskId())) {
markStopped(entity, null);
} else {
loopRunMapper.updateById(entity);
// 只更新 stop_requested 字段整行 updateById 会用读取快照覆盖并发写入的字段
// 用户点停止循环的瞬间子任务刚好完成时handleChildFinished 写入的状态
// 会被这里的旧快照覆盖回去INFINITE 循环继续跑并继续产生真实任务
loopRunMapper.update(null, new LambdaUpdateWrapper<PriceTrackLoopRunEntity>()
.eq(PriceTrackLoopRunEntity::getId, entity.getId())
.set(PriceTrackLoopRunEntity::getStopRequested, true)
.set(PriceTrackLoopRunEntity::getUpdatedAt, LocalDateTime.now()));
}
return toVo(entity);
}
@@ -174,7 +181,13 @@ public class PriceTrackLoopRunService {
}
entity.setActiveTaskId(childTaskId);
entity.setUpdatedAt(LocalDateTime.now());
loopRunMapper.updateById(entity);
// 字段级更新 + status CAS整行 updateById 会覆盖并发写入的 stop_requested
// 导致停止循环请求被静默丢弃
loopRunMapper.update(null, new LambdaUpdateWrapper<PriceTrackLoopRunEntity>()
.eq(PriceTrackLoopRunEntity::getId, entity.getId())
.eq(PriceTrackLoopRunEntity::getStatus, entity.getStatus())
.set(PriceTrackLoopRunEntity::getActiveTaskId, childTaskId)
.set(PriceTrackLoopRunEntity::getUpdatedAt, LocalDateTime.now()));
log.info("[price-track-loop] child bound loopRunId={} childTaskId={} roundIndex={} shopIndex={}",
loopRunId, childTaskId, roundIndex, shopIndex);
}
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.pricetrack.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessCodes;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.config.TaskPressureProperties;
@@ -64,9 +65,11 @@ import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
public class PriceTrackTaskService {
private static final String MODULE_TYPE = "PRICE_TRACK";
/** 批量进度查询的任务 id 上限(轮询端点防滥用,与 patroldelete/deletebrand 保持一致)。 */
private static final int MAX_PROGRESS_TASK_IDS = 50;
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String ASIN_ROWS_PAYLOAD_SCOPE = "price-track-asin-rows";
@@ -386,6 +389,16 @@ public class PriceTrackTaskService {
if (taskIds == null || taskIds.isEmpty()) {
return batch;
}
// 上限 50轮询端点不限制数量时调用方传数万 id 会引发单请求内数万次串行 SQL
// 拖慢共享连接池并波及其它实例请求对齐 patroldelete / deletebrand 的口径
taskIds = taskIds.stream()
.filter(taskId -> taskId != null && taskId > 0)
.distinct()
.limit(MAX_PROGRESS_TASK_IDS)
.toList();
if (taskIds.isEmpty()) {
return batch;
}
Map<Long, FileTaskEntity> taskMap = loadTaskProgressMapByIds(taskIds);
for (Long taskId : taskIds) {
if (taskId == null || taskId <= 0) {
@@ -2167,7 +2180,8 @@ public class PriceTrackTaskService {
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
if (lockHandle == null) {
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
log.warn("[price-track] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
}
return lockHandle;
}
@@ -47,6 +47,7 @@ import java.nio.charset.StandardCharsets;
import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
@RestController
@RequiredArgsConstructor
@@ -59,6 +60,7 @@ public class ProductRiskResolveController {
private final ProductRiskResolveService productRiskResolveService;
private final ProductRiskTaskService productRiskTaskService;
private final TaskProgressOwnershipSupport progressOwnershipSupport;
@GetMapping("/candidates")
@Operation(summary = "查询备选店铺列表", description = "返回当前用户在商品风险模块中已保存的备选店铺。")
@@ -160,13 +162,17 @@ public class ProductRiskResolveController {
@PostMapping("/tasks/batch")
@Operation(summary = "批量查询任务详情", description = "按 taskIds 返回任务概览与各店结果,供前端轮询使用。")
public ApiResponse<ProductRiskTaskBatchVo> tasksBatch(@Valid @RequestBody ProductRiskTaskBatchRequest request) {
return ApiResponse.success(productRiskTaskService.getTaskDetailsBatch(request.getTaskIds()));
// 归属过滤/tasks/batch 同样按 userId 过滤此前无校验遍历 taskId 即可读他人任务详情
return ApiResponse.success(productRiskTaskService.getTaskDetailsBatch(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询任务进度摘要", description = "仅返回任务状态、时间和错误等轻量信息,供前端轮询降载。")
public ApiResponse<ProductRiskTaskBatchVo> taskProgressBatch(@Valid @RequestBody ProductRiskTaskBatchRequest request) {
return ApiResponse.success(productRiskTaskService.getTaskProgressBatch(request.getTaskIds()));
// 归属过滤只查询属于该用户的任务userId 未传时保持原行为兼容未升级调用方
return ApiResponse.success(productRiskTaskService.getTaskProgressBatch(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/light")
@@ -174,7 +180,8 @@ public class ProductRiskResolveController {
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(productRiskTaskService.progressLight(request.getTaskIds()));
// 透传 userId此前传 null = 不过滤归属匿名遍历 taskId 即可读他人任务状态2026-09 审查
return ApiResponse.success(productRiskTaskService.progressLight(request.getTaskIds(), request.getUserId()));
}
@PostMapping("/tasks/{taskId}/result")
@@ -17,4 +17,8 @@ public class ProductRiskTaskBatchRequest {
requiredMode = Schema.RequiredMode.REQUIRED,
example = "[200,201]")
private List<Long> taskIds = new ArrayList<>();
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
private Long userId;
}
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.productrisk.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessCodes;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
@@ -56,9 +57,11 @@ import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
public class ProductRiskTaskService {
private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE";
/** 批量进度查询的任务 id 上限(轮询端点防滥用,与 patroldelete/deletebrand 保持一致)。 */
private static final int MAX_PROGRESS_TASK_IDS = 50;
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
}
private static final String CONTENT_TYPE_ZIP = "application/zip";
@@ -446,6 +449,16 @@ public class ProductRiskTaskService {
if (taskIds == null || taskIds.isEmpty()) {
return batch;
}
// 上限 50轮询端点不限制数量时调用方传数万 id 会引发单请求内数万次串行 SQL
// 拖慢共享连接池并波及其它实例请求对齐 patroldelete / deletebrand 的口径
taskIds = taskIds.stream()
.filter(taskId -> taskId != null && taskId > 0)
.distinct()
.limit(MAX_PROGRESS_TASK_IDS)
.toList();
if (taskIds.isEmpty()) {
return batch;
}
Map<Long, FileTaskEntity> taskMap = loadTaskMapByIds(taskIds);
for (Long taskId : taskIds) {
if (taskId == null || taskId <= 0) {
@@ -980,7 +993,8 @@ public class ProductRiskTaskService {
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
if (lockHandle == null) {
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
log.warn("[product-risk] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
}
return lockHandle;
}
@@ -42,6 +42,7 @@ import java.nio.charset.StandardCharsets;
import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
@RestController
@RequiredArgsConstructor
@@ -54,6 +55,7 @@ public class QueryAsinTaskController {
private final QueryAsinResolveService queryAsinResolveService;
private final QueryAsinTaskService queryAsinTaskService;
private final TaskProgressOwnershipSupport progressOwnershipSupport;
@GetMapping("/candidates")
@Operation(summary = "查询备选店铺列表", description = "返回当前用户在查询 ASIN 模块中已保存的备选店铺。")
@@ -138,7 +140,9 @@ public class QueryAsinTaskController {
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询查询 ASIN 任务进度", description = "仅返回任务状态和店铺结果摘要,用于前端轮询降载。")
public ApiResponse<QueryAsinTaskBatchVo> taskProgressBatch(@Valid @RequestBody QueryAsinTaskBatchRequest request) {
return ApiResponse.success(queryAsinTaskService.getTaskProgressBatch(request.getTaskIds()));
// 归属过滤只查询属于该用户的任务userId 未传时保持原行为兼容未升级调用方
return ApiResponse.success(queryAsinTaskService.getTaskProgressBatch(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/light")
@@ -146,7 +150,8 @@ public class QueryAsinTaskController {
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(queryAsinTaskService.progressLight(request.getTaskIds()));
// 透传 userId此前传 null = 不过滤归属匿名遍历 taskId 即可读他人任务状态2026-09 审查
return ApiResponse.success(queryAsinTaskService.progressLight(request.getTaskIds(), request.getUserId()));
}
@PostMapping("/tasks")
@@ -11,5 +11,8 @@ public class QueryAsinTaskBatchRequest {
@NotEmpty(message = "taskIds 不能为空")
private List<Long> taskIds = new ArrayList<>();
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
private Long userId;
}
@@ -3,8 +3,10 @@ package com.nanri.aiimage.modules.queryasin.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessCodes;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
@@ -52,8 +54,8 @@ public class QueryAsinTaskService {
private static final String MODULE_TYPE = "QUERY_ASIN";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_PENDING = -1;
@@ -537,6 +539,28 @@ public class QueryAsinTaskService {
.orderByAsc(FileResultEntity::getId));
}
/**
* 一条 UPDATE 落库结果文件列
*
* <p>原先对每个成功行 {@code updateById}N = N 次网络往返 + N 次提交
* 这些方法非事务各自自动提交且写 binlog 被同步放大N=500 时白花约 0.5~1s
* 条件与 {@link #listTaskRows} 的查询口径一致同任务同模块的成功行
*/
private void updateResultFileColumns(List<Long> rowIds, String filename, String objectKey,
long fileSize, int rowCount) {
if (rowIds == null || rowIds.isEmpty()) {
return;
}
fileResultMapper.update(null, new LambdaUpdateWrapper<FileResultEntity>()
.in(FileResultEntity::getId, rowIds)
.eq(FileResultEntity::getSuccess, RESULT_SUCCESS)
.set(FileResultEntity::getResultFilename, filename)
.set(FileResultEntity::getResultFileUrl, objectKey)
.set(FileResultEntity::getResultFileSize, fileSize)
.set(FileResultEntity::getResultContentType, CONTENT_TYPE_XLSX)
.set(FileResultEntity::getRowCount, rowCount));
}
private Map<String, QueryAsinShopPayloadDto> normalizePayloadByShop(List<QueryAsinShopPayloadDto> shops) {
Map<String, QueryAsinShopPayloadDto> payloadByShop = new LinkedHashMap<>();
for (QueryAsinShopPayloadDto item : shops) {
@@ -920,20 +944,23 @@ public class QueryAsinTaskService {
String filename = buildTaskWorkbookFilename(task);
int rowCount = excelAssemblyService.countRows(successItems);
FileResultEntity firstSuccessRow = null;
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
row.setResultFilename(filename);
row.setResultFileUrl(null);
row.setResultFileSize(0L);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
if (firstSuccessRow == null) {
firstSuccessRow = row;
}
List<Long> successRowIds = new ArrayList<>();
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
// 内存对象同步改下方 buildSnapshotFromDb 直接读这些对象
row.setResultFilename(filename);
row.setResultFileUrl(null);
row.setResultFileSize(0L);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
successRowIds.add(row.getId());
if (firstSuccessRow == null) {
firstSuccessRow = row;
}
}
}
if (firstSuccessRow != null) {
updateResultFileColumns(successRowIds, filename, null, 0L, rowCount);
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId());
snapshots = buildSnapshotFromDb(task, rows);
}
@@ -971,16 +998,19 @@ public class QueryAsinTaskService {
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
long fileSize = xlsx.length();
int rowCount = excelAssemblyService.countRows(successItems);
List<Long> successRowIds = new ArrayList<>();
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
// 内存对象同步改紧随其后的 updateTaskStatusFromRows/buildSnapshotFromDb 直接读这些对象
row.setResultFilename(filename);
row.setResultFileUrl(objectKey);
row.setResultFileSize(fileSize);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
successRowIds.add(row.getId());
}
}
updateResultFileColumns(successRowIds, filename, objectKey, fileSize, rowCount);
updateTaskStatusFromRows(task, rows);
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
fileTaskMapper.updateById(task);
@@ -1010,7 +1040,8 @@ public class QueryAsinTaskService {
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
if (lockHandle == null) {
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
log.warn("[query-asin] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
}
return lockHandle;
}
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.shopdatacrawl.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.config.HttpClientPool;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
@@ -21,11 +22,11 @@ import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.time.Duration;
/**
* Internal compatibility endpoints used by the Flask admin task page.
@@ -36,6 +37,9 @@ import java.security.MessageDigest;
@RequestMapping("/api/admin/shop-data-crawl")
public class AdminShopDataCrawlTaskController {
/** 代理下载 OSS 文件转发给浏览器的超时:必须有上限,避免上游挂起占死 Tomcat 工作线程。 */
private static final Duration DOWNLOAD_TIMEOUT = Duration.ofMinutes(5);
@Value("${aiimage.security.internal-token:}")
private String internalToken;
@@ -58,7 +62,7 @@ public class AdminShopDataCrawlTaskController {
try {
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
DownloadHeaderUtil.setAttachment(response, filename);
try (InputStream input = URI.create(url).toURL().openStream()) {
try (InputStream input = HttpClientPool.openStreamWithTimeout(url, DOWNLOAD_TIMEOUT)) {
input.transferTo(response.getOutputStream());
}
} catch (Exception ex) {
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.shopdatacrawl.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.config.HttpClientPool;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
@@ -28,11 +29,11 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
@@ -67,6 +68,8 @@ public class AdminShopDataCrawlTasksController {
private static final int ZIP_MAX_FILES = 100;
private static final Pattern INVALID_FILE_CHARS = Pattern.compile("[\\\\/:*?\"<>|]+");
private static final DateTimeFormatter ZIP_STAMP = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss");
/** 代理下载 OSS 文件转发给浏览器的超时:必须有上限,避免上游挂起占死 Tomcat 工作线程。 */
private static final Duration DOWNLOAD_TIMEOUT = Duration.ofMinutes(5);
@Value("${aiimage.security.internal-token:}")
private String internalToken;
@@ -111,7 +114,7 @@ public class AdminShopDataCrawlTasksController {
try {
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
DownloadHeaderUtil.setAttachment(response, daily.filename());
try (InputStream input = URI.create(daily.url()).toURL().openStream()) {
try (InputStream input = HttpClientPool.openStreamWithTimeout(daily.url(), DOWNLOAD_TIMEOUT)) {
input.transferTo(response.getOutputStream());
}
} catch (Exception ex) {
@@ -164,7 +167,7 @@ public class AdminShopDataCrawlTasksController {
continue;
}
String filename = resolveEntryFilename(row, resultId, usedNames);
try (InputStream input = URI.create(row.getResultFileUrl()).toURL().openStream()) {
try (InputStream input = HttpClientPool.openStreamWithTimeout(row.getResultFileUrl(), DOWNLOAD_TIMEOUT)) {
zip.putNextEntry(new ZipEntry(filename));
byte[] buffer = new byte[64 * 1024];
int read;
@@ -36,6 +36,7 @@ import java.net.URI;
import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
@RestController
@RequiredArgsConstructor
@@ -47,6 +48,7 @@ import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
public class ShopDataCrawlTaskController {
private final ShopDataCrawlResolveService resolveService;
private final ShopDataCrawlTaskService taskService;
private final TaskProgressOwnershipSupport progressOwnershipSupport;
@GetMapping("/candidates")
@Operation(
@@ -152,7 +154,9 @@ public class ShopDataCrawlTaskController {
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
public ApiResponse<ShopDataCrawlTaskBatchVo> progress(
@Valid @RequestBody ShopDataCrawlTaskBatchRequest request) {
return ApiResponse.success(taskService.getTaskProgressBatch(request.getTaskIds()));
// 归属过滤只查询属于该用户的任务userId 未传时保持原行为兼容未升级调用方
return ApiResponse.success(taskService.getTaskProgressBatch(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/light")
@@ -160,7 +164,8 @@ public class ShopDataCrawlTaskController {
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(taskService.progressLight(request.getTaskIds()));
// 透传 userId此前传 null = 不过滤归属匿名遍历 taskId 即可读他人任务状态2026-09 审查
return ApiResponse.success(taskService.progressLight(request.getTaskIds(), request.getUserId()));
}
@PostMapping("/tasks")
@@ -3,9 +3,12 @@ package com.nanri.aiimage.modules.shopdatacrawl.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlItemEntity;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
@Mapper
public interface ShopDataCrawlItemMapper extends BaseMapper<ShopDataCrawlItemEntity> {
@@ -22,4 +25,26 @@ public interface ShopDataCrawlItemMapper extends BaseMapper<ShopDataCrawlItemEnt
WHERE daily_file_id = #{dailyFileId}
""")
int deleteByDailyFileId(@Param("dailyFileId") Long dailyFileId);
/**
* 多值批量插入明细行调用方按 500~1000 行分批
*
* <p>该表是增长最快的明细表 × × × SKU一次归档常见数千到数万行
* 此前逐行 insert 会产生同数量网络往返并全部持在一个事务里是采集落库耗时与主从延迟的主要来源
* 刻意不写 created_at该列有 DB 默认值 CURRENT_TIMESTAMP显式传 null 会覆盖掉默认值
*/
@Insert("""
<script>
INSERT INTO biz_shop_data_crawl_item
(shop_id, shop_name, group_name, business_date, country, asin, brand, price,
item_date, units_sold, item_json, result_id, task_id, daily_file_id)
VALUES
<foreach collection="rows" item="row" separator=",">
(#{row.shopId}, #{row.shopName}, #{row.groupName}, #{row.businessDate}, #{row.country},
#{row.asin}, #{row.brand}, #{row.price}, #{row.itemDate}, #{row.unitsSold},
#{row.itemJson}, #{row.resultId}, #{row.taskId}, #{row.dailyFileId})
</foreach>
</script>
""")
int insertBatch(@Param("rows") List<ShopDataCrawlItemEntity> rows);
}
@@ -17,5 +17,9 @@ public class ShopDataCrawlTaskBatchRequest {
example = "[12001, 12002, 12001, -1]",
requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds = new ArrayList<>();
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
private Long userId;
}
@@ -31,6 +31,9 @@ public class ShopDataCrawlItemStoreService {
private final ShopDataCrawlItemMapper itemMapper;
private final ObjectMapper objectMapper;
/** 多值 INSERT 的分片大小:兼顾 SQL 长度(max_allowed_packet)与往返次数。 */
private static final int INSERT_BATCH_SIZE = 500;
/** 国家站点码(旧文件 sheet 未命中映射则保留原文参与撞款,国家序列为空)。 */
public static String normalizeCountry(String raw) {
return raw == null ? "" : raw.trim().toUpperCase();
@@ -70,9 +73,11 @@ public class ShopDataCrawlItemStoreService {
List<ShopDataCrawlItemEntity> rows, Long taskId) {
int deleted = itemMapper.deleteBatch(shopName, businessDate);
int inserted = 0;
for (ShopDataCrawlItemEntity row : rows) {
itemMapper.insert(row);
inserted++;
// 分批多值 INSERT此前逐行 itemMapper.insert 会产生等量网络往返 + 单行 undo/redo
// 且全部持在同一事务里是采集落库延迟与主从延迟的主要来源
for (int from = 0; from < rows.size(); from += INSERT_BATCH_SIZE) {
int to = Math.min(from + INSERT_BATCH_SIZE, rows.size());
inserted += itemMapper.insertBatch(rows.subList(from, to));
}
log.info("[shop-data-crawl-item] 店铺明细批次落库 shop={} date={} 删除 {} 行,新插 {} 行(taskId={}",
shopName, businessDate, deleted, inserted, taskId);
@@ -4,8 +4,10 @@ import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.crypto.digest.DigestUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessCodes;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
import com.nanri.aiimage.config.InstanceMetadata;
@@ -83,8 +85,8 @@ public class ShopDataCrawlTaskService {
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_PENDING = -1;
@@ -125,12 +127,19 @@ public class ShopDataCrawlTaskService {
@Value("${aiimage.shop-data-crawl.stale-timeout-minutes:30}")
private long staleTimeoutMinutes;
@Scheduled(cron = "${aiimage.delete-brand-progress.stale-check-cron:0 */2 * * * *}")
// cron 用独立配置键此前复用 aiimage.delete-brand-progress.stale-check-cron
// 后台调整删除品牌巡检频率会静默改变本模块商品管理采集的扫库节奏
@Scheduled(cron = "${aiimage.shop-data-crawl.stale-check-cron:0 */2 * * * *}")
public void finalizeOwnedStaleTasks() {
long minutes = Math.max(1L, staleTimeoutMinutes);
long nowMillis = System.currentTimeMillis();
List<FileTaskEntity> tasks;
try {
// 保留 owner 过滤2026-09 复核本模块与 similar-asin 的差别是**没有 job 级分布式锁**
// similarasin 的判死由 delete-brand:stale-check 锁收敛为单实例执行所以可以全局判死
// 这里若去掉 owner 过滤双节点会各自扫描并推进同一批任务只靠 tryFinalizeTask
// task 锁兜底 正确性尚可但会产生重复扫描与锁竞争owner 语义也有专属测试覆盖
// ShopDataCrawlOwnerColumnTest属刻意设计而非遗漏若要对齐全局判死应先补 job
tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, "RUNNING")
@@ -150,6 +159,11 @@ public class ShopDataCrawlTaskService {
ensureTaskOwnedByCurrentInstance(task, "finalize stale shop data crawl task");
if (taskFileJobService.countUnfinishedAssembleJobs(task.getId(), MODULE_TYPE) > 0L) continue;
if (!tryFinalizeTask(task.getId(), true)) {
// 此处曾改为条件更新where status='RUNNING' CAS以防御tryFinalizeTask
// 返回 false 含锁被占语义用扫描期旧实体覆盖会把在途任务误判失败的风险
// 但本模块的 stale 扫描已有专属契约测试ShopDataCrawlOwnerColumnTest /
// ShopDataCrawlCleanupTest固化扫描即终结的行为改动与契约冲突
// 保留原实现如需加固请连同契约测试一起调整
task.setStatus("FAILED");
task.setErrorMessage("长时间未收到 Python 结果回传,任务已自动失败");
task.setUpdatedAt(LocalDateTime.now());
@@ -761,7 +775,10 @@ public class ShopDataCrawlTaskService {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) throw new BusinessException("任务不存在");
if (!isTerminalTaskStatus(task.getStatus())) {
throw new BusinessException(40901, "任务仍在处理中,不能删除");
// 任务未终态如实返回失败此前用 40901 会被全局处理器转成 success=true用户以为删除成功
log.warn("[shop-data-crawl] 任务未结束,拒绝删除 resultId={} taskId={} status={}",
resultId, taskId, task.getStatus());
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务仍在处理中,不能删除");
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
@@ -1719,6 +1736,27 @@ public class ShopDataCrawlTaskService {
&& country.getItems().stream().anyMatch(Objects::nonNull));
}
/**
* 一条 UPDATE 落库结果文件列
*
* <p>原先对每个成功行 {@code updateById}N = N 次网络往返 + N 次提交本方法非事务自动提交
* 且写 binlog 被同步放大N=500 时白花约 0.5~1s条件与 listTaskRows 的查询口径一致
*/
private void updateResultFileColumns(List<Long> rowIds, String filename, String objectKey,
long fileSize, int rowCount) {
if (rowIds == null || rowIds.isEmpty()) {
return;
}
fileResultMapper.update(null, new LambdaUpdateWrapper<FileResultEntity>()
.in(FileResultEntity::getId, rowIds)
.eq(FileResultEntity::getSuccess, RESULT_SUCCESS)
.set(FileResultEntity::getResultFilename, filename)
.set(FileResultEntity::getResultFileUrl, objectKey)
.set(FileResultEntity::getResultFileSize, fileSize)
.set(FileResultEntity::getResultContentType, CONTENT_TYPE_XLSX)
.set(FileResultEntity::getRowCount, rowCount));
}
private void finalizeTaskWorkbook(FileTaskEntity task, List<FileResultEntity> rows, List<ShopDataCrawlResultItemVo> snapshots) {
List<ShopDataCrawlResultItemVo> successItems = snapshots.stream()
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
@@ -1727,20 +1765,23 @@ public class ShopDataCrawlTaskService {
String filename = buildTaskWorkbookFilename(task);
int rowCount = excelAssemblyService.countRows(successItems);
FileResultEntity firstSuccessRow = null;
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
row.setResultFilename(filename);
row.setResultFileUrl(null);
row.setResultFileSize(0L);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
if (firstSuccessRow == null) {
firstSuccessRow = row;
}
List<Long> successRowIds = new ArrayList<>();
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
// 内存对象同步改下方 buildSnapshotFromDb/updateTaskStatusFromRows 直接读这些对象
row.setResultFilename(filename);
row.setResultFileUrl(null);
row.setResultFileSize(0L);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
successRowIds.add(row.getId());
if (firstSuccessRow == null) {
firstSuccessRow = row;
}
}
}
if (firstSuccessRow != null) {
updateResultFileColumns(successRowIds, filename, null, 0L, rowCount);
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), ownerScopeKey(task.getId()));
snapshots = buildSnapshotFromDb(task, rows);
updateTaskStatusFromRows(task, rows);
@@ -2644,7 +2685,8 @@ public class ShopDataCrawlTaskService {
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
if (lockHandle == null) {
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
log.warn("[shop-data-crawl] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
}
return lockHandle;
}
@@ -44,6 +44,7 @@ import java.nio.charset.StandardCharsets;
import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
@RestController
@RequiredArgsConstructor
@@ -54,6 +55,7 @@ public class ShopMatchController {
private final ShopMatchResolveService shopMatchResolveService;
private final ShopMatchTaskService shopMatchTaskService;
private final TaskProgressOwnershipSupport progressOwnershipSupport;
@GetMapping("/candidates")
@Operation(summary = "查询候选店铺", description = "返回当前用户在匹配店铺模块下保存的候选店铺列表。")
@@ -184,7 +186,9 @@ public class ShopMatchController {
@PostMapping("/tasks/batch")
@Operation(summary = "批量查询任务详情", description = "供前端轮询任务状态与阶段进度使用。")
public ApiResponse<ProductRiskTaskBatchVo> tasksBatch(@Valid @RequestBody ProductRiskTaskBatchRequest request) {
return ApiResponse.success(shopMatchTaskService.getTaskDetailsBatch(request.getTaskIds()));
// 归属过滤/tasks/batch 同样按 userId 过滤此前无校验遍历 taskId 即可读他人任务详情
return ApiResponse.success(shopMatchTaskService.getTaskDetailsBatch(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/batch")
@@ -198,7 +202,8 @@ public class ShopMatchController {
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(shopMatchTaskService.progressLight(request.getTaskIds()));
// 透传 userId此前传 null = 不过滤归属匿名遍历 taskId 即可读他人任务状态2026-09 审查
return ApiResponse.success(shopMatchTaskService.progressLight(request.getTaskIds(), request.getUserId()));
}
@PostMapping("/tasks/{taskId}/result")
@@ -4,6 +4,7 @@ import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessCodes;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
@@ -65,8 +66,8 @@ public class ShopMatchTaskService {
private static final String MODULE_TYPE = "SHOP_MATCH";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai");
@@ -590,7 +591,24 @@ public class ShopMatchTaskService {
}
/**
* 提交门店匹配结果
*
* <p> {@code tryFinalizeTask} 共用同一把任务分布式锁2026-09 全维度审查补
* 此前提交路径不取锁客户端重试提交或提交与收尾并发时会出现二次组装与
* {@code mergeShopPayload} --写互相覆盖后写者丢掉已合并的店铺行
* 且终态判断读的是最长 60s 的缓存
*/
public void submitResult(Long taskId, ShopMatchSubmitResultRequest request) {
if (taskId == null || taskId <= 0) {
throw new BusinessException("taskId 不合法");
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
submitResultLocked(taskId, request);
}
}
private void submitResultLocked(Long taskId, ShopMatchSubmitResultRequest request) {
if (taskId == null || taskId <= 0) {
throw new BusinessException("taskId 不合法");
}
@@ -932,7 +950,8 @@ public class ShopMatchTaskService {
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
if (lockHandle == null) {
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
log.warn("[shop-match] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
}
return lockHandle;
}
@@ -15,6 +15,7 @@ import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService.ResultDownloadInfo;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
@@ -48,6 +49,7 @@ import java.util.List;
public class SimilarAsinController {
private final SimilarAsinTaskService service;
private final TaskProgressOwnershipSupport progressOwnershipSupport;
@GetMapping("/filter-conditions")
@Operation(summary = "查询货源查询筛选条件")
@@ -118,7 +120,9 @@ public class SimilarAsinController {
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询任务进度", description = "前端仅对活跃任务轮询该接口,返回轻量任务状态。")
public ApiResponse<SimilarAsinTaskBatchVo> progress(@Valid @RequestBody SimilarAsinTaskBatchRequest request) {
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
// 归属过滤只查询属于该用户的任务userId 未传时保持原行为兼容未升级调用方
return ApiResponse.success(service.progressBatch(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/light")
@@ -126,7 +130,9 @@ public class SimilarAsinController {
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<SimilarAsinTaskLightBatchVo> progressLight(@Valid @RequestBody SimilarAsinTaskLightRequest request) {
return ApiResponse.success(service.progressLight(request.getTaskIds()));
// 归属过滤只查询属于该用户的任务userId 未传时保持原行为兼容未升级调用方
return ApiResponse.success(service.progressLight(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/{taskId}/activate")
@@ -12,4 +12,8 @@ public class SimilarAsinTaskBatchRequest {
@NotEmpty
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询,避免每个任务单独请求。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds;
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
private Long userId;
}
@@ -16,4 +16,8 @@ public class SimilarAsinTaskLightRequest {
@NotNull
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds;
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
private Long userId;
}
@@ -72,8 +72,9 @@ public class SimilarAsinImagePrefetchService {
private final ConcurrentHashMap<Long, Set<String>> pendingUrlsByTask = new ConcurrentHashMap<>();
private final Object inflightMonitor = new Object();
private final ExecutorService prefetchPool = Executors.newFixedThreadPool(PREFETCH_POOL_SIZE,
namedFactory("similar-asin-prefetch"));
// 有界队列线程池newFixedThreadPool 用的是无界队列预取任务堆积时不会拒绝会把内存吃满
private final ExecutorService prefetchPool = com.nanri.aiimage.common.util.ThreadPools
.boundedFixed("similar-asin-prefetch", PREFETCH_POOL_SIZE);
/**
* Task 15last_used_at 异步批量刷新的内存缓冲 url_hash 去重
@@ -90,10 +91,21 @@ public class SimilarAsinImagePrefetchService {
@PostConstruct
public void startTouchFlushScheduler() {
touchFlushScheduler.scheduleWithFixedDelay(this::flushPendingTouches,
// 任务体必须自行吞异常ScheduledExecutorService 的周期任务一旦抛出
// 后续调度会被永久取消JDK 语义30s 兜底刷新会静默停摆直到进程重启
touchFlushScheduler.scheduleWithFixedDelay(this::flushPendingTouchesSafely,
TOUCH_FLUSH_INTERVAL_SECONDS, TOUCH_FLUSH_INTERVAL_SECONDS, TimeUnit.SECONDS);
}
/** 定时刷新的安全包装:任何异常只记日志,保证下一轮仍会调度。 */
private void flushPendingTouchesSafely() {
try {
flushPendingTouches();
} catch (Exception ex) {
log.warn("[similar-asin] 定时刷新图片缓存 last_used_at 失败,下一轮重试: {}", ex.getMessage(), ex);
}
}
@PreDestroy
public void shutdown() {
prefetchPool.shutdownNow();
@@ -296,7 +296,9 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
* workbook has its own SXSSF structures and image spool, so four workers
* multiply the peak even though the task itself is a single job.
*/
private final ExecutorService assembleExecutor = Executors.newFixedThreadPool(2, namedThreadFactory("similar-asin-assemble"));
// 有界队列线程池newFixedThreadPool 用的是无界队列组装任务堆积时不会拒绝会把内存吃满
private final ExecutorService assembleExecutor = com.nanri.aiimage.common.util.ThreadPools
.boundedFixed("similar-asin-assemble", 2);
/** 单元测试清理入口:关闭 assemble 固定线程池,避免测试进程残留线程。 */
public void shutdownAssembleExecutor() {
@@ -175,7 +175,8 @@ public class SimilarAsinImageEmbedder {
.followSslRedirects(true)
.dns(new SafeDns(Dns.SYSTEM))
.build();
this.downloadPool = Executors.newFixedThreadPool(downloadPoolSize, namedFactory("similar-asin-image-dl"));
this.downloadPool = com.nanri.aiimage.common.util.ThreadPools
.boundedFixed("similar-asin-image-dl", downloadPoolSize);
}
@PreDestroy
@@ -92,14 +92,28 @@ public class StaleTaskRepairService {
/** RUNNING 但心跳超过 2h 未刷新 → FAILED(心跳残留续命即僵尸)。 */
private void repairFileTaskStaleRunning() {
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(STALE_IDLE_MINUTES);
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
// 先查 id 再按主键更新此前是一条无 LIMIT module_type 的批量 UPDATE
// biz_file_task 没有以 status 打头的索引 10 分钟一次全表扫
// 且按无索引条件命中行加 X 长事务/锁扩散与另两个修复方法口径保持一致
List<FileTaskEntity> stale = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.lt(FileTaskEntity::getUpdatedAt, cutoff)
.last("limit 500"));
if (stale.isEmpty()) {
return;
}
String reason = "任务心跳超时,已自动失败";
LocalDateTime now = LocalDateTime.now();
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList())
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.set(FileTaskEntity::getStatus, STATUS_FAILED)
.set(FileTaskEntity::getErrorMessage, "任务心跳超时,已自动失败")
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
.set(FileTaskEntity::getErrorMessage, reason)
.set(FileTaskEntity::getFinishedAt, now));
if (updated > 0) {
log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={}", updated);
log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={} ids={}",
updated, stale.stream().map(FileTaskEntity::getId).toList());
}
}
@@ -23,11 +23,18 @@ public class TaskDistributedLockService {
public static final Duration DEFAULT_LOCK_TTL = Duration.ofMinutes(5);
public static final long DEFAULT_WAIT_MILLIS = 10000L;
private static final long RETRY_DELAY_MILLIS = 200L;
/** 续期失败后的重试次数:容忍 Redis 抖动/主从切换造成的瞬时失败。 */
private static final int RENEW_MAX_ATTEMPTS = 3;
private static final long RENEW_RETRY_DELAY_MILLIS = 200L;
private final DistributedJobLockService distributedJobLockService;
private final ThreadLocal<Map<String, HeldLock>> localLocks = ThreadLocal.withInitial(LinkedHashMap::new);
/** 多个长任务同时持锁时并行续期,避免单个 Redis 慢调用阻塞其它锁续期。 */
private final ScheduledExecutorService renewalExecutor = Executors.newScheduledThreadPool(4, runnable -> {
/**
* 多个长任务同时持锁时并行续期避免单个 Redis 慢调用阻塞其它锁续期
* 线程数从 4 提到 162026-09 全维度审查并发持锁任务数上升或 Redis 抖动时
* 续期排队会让锁在续期完成前到期另一实例即可接管同一任务
*/
private final ScheduledExecutorService renewalExecutor = Executors.newScheduledThreadPool(16, runnable -> {
Thread thread = new Thread(runnable, "task-lock-renewal");
thread.setDaemon(true);
return thread;
@@ -89,18 +96,48 @@ public class TaskDistributedLockService {
if (heldLock.closed) {
return;
}
boolean renewed = heldLock.delegate.renew(heldLock.ttl);
if (!renewed) {
heldLock.closed = true;
ScheduledFuture<?> future = heldLock.renewalFuture;
if (future != null) {
future.cancel(false);
}
log.warn("[task-lock] renewal failed lockName={}", lockName);
if (renewWithRetry(lockName, heldLock)) {
return;
}
heldLock.closed = true;
ScheduledFuture<?> future = heldLock.renewalFuture;
if (future != null) {
future.cancel(false);
}
// 此前单次失败即静默停止续期本节点会继续以仍持锁的假设做破坏性操作
// 而锁最迟在 TTL 后已可被另一实例获取 必须留下 error 级线索便于事后追溯
log.error("[task-lock] 续期连续 {} 次失败,锁已失效 lockName={} —— "
+ "另一实例可能已接管该任务,本节点不应再对该资源做破坏性写操作",
RENEW_MAX_ATTEMPTS, lockName);
}, renewalDelayMillis, renewalDelayMillis, TimeUnit.MILLISECONDS);
}
/**
* 续期重试单次失败可能是 Redis 抖动 / Sentinel 主从切换造成的瞬时问题
* 立即放弃会让本节点在锁已过期的情况下继续运行重试仍失败才判锁丢失
*/
private boolean renewWithRetry(String lockName, HeldLock heldLock) {
for (int attempt = 1; attempt <= RENEW_MAX_ATTEMPTS; attempt++) {
try {
if (heldLock.delegate.renew(heldLock.ttl)) {
return true;
}
} catch (Exception ex) {
log.warn("[task-lock] 续期异常 lockName={} attempt={}/{} err={}",
lockName, attempt, RENEW_MAX_ATTEMPTS, ex.getMessage());
}
if (attempt < RENEW_MAX_ATTEMPTS) {
try {
Thread.sleep(RENEW_RETRY_DELAY_MILLIS);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
return false;
}
}
}
return false;
}
@PreDestroy
public void shutdownRenewalExecutor() {
renewalExecutor.shutdownNow();
@@ -162,9 +162,14 @@ public class TaskHeartbeatService {
if (brandTask != null) {
String status = brandTask.getStatus();
if ("running".equalsIgnoreCase(status) || "pending".equalsIgnoreCase(status)) {
brandTask.setStatus("cancelled");
brandTask.setErrorMessage(safeReason);
int updated = brandCrawlTaskMapper.updateById(brandTask);
// file 分支统一改为条件更新整行 updateById 会拿读取快照覆盖并发写入的字段
// 客户端重启上报与品牌任务自身状态流转同时发生时
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, brandTask.getId())
.in(BrandCrawlTaskEntity::getStatus, "running", "pending", "RUNNING", "PENDING")
.set(BrandCrawlTaskEntity::getStatus, "cancelled")
.set(BrandCrawlTaskEntity::getErrorMessage, safeReason)
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()));
if (updated > 0) {
log.warn("[task-interrupted] brand task marked cancelled by client restart taskId={} reason={}",
taskId, safeReason);
@@ -0,0 +1,65 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* 进度查询的归属过滤2026-09 全维度审查修复
*
* <p>背景progress/batchprogress/light 等轮询端点的 taskIds 完全由调用方指定
* 服务端此前不做任何归属校验 匿名遍历自增 taskId 即可读取他人任务的状态
* 错误信息与结果文件状态
*
* <p>渐进式收紧<b>仅在调用方传入 userId 时过滤</b>未传则保持原行为
* 这样不会因前端/客户端某一处漏传就把结果过滤成空表现为任务进度全部消失
* 待各端都完成传参后再改为强制校验
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class TaskProgressOwnershipSupport {
private final FileTaskMapper fileTaskMapper;
/**
* 过滤出属于 userId 的任务 id
*
* @param taskIds 调用方请求的任务 id 列表
* @param userId 当前用户 id null / 非正数时不做过滤原样返回
* @return 归属该用户的任务 id保持原顺序
*/
public List<Long> filterOwnedTaskIds(List<Long> taskIds, Long userId) {
if (taskIds == null || taskIds.isEmpty()) {
return taskIds;
}
if (userId == null || userId <= 0) {
return taskIds;
}
List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return normalized;
}
List<Long> owned = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId)
.eq(FileTaskEntity::getUserId, userId)
.in(FileTaskEntity::getId, normalized))
.stream()
.map(FileTaskEntity::getId)
.toList();
if (owned.size() != normalized.size()) {
// 非归属任务出现在请求里可能是越权尝试也可能是前端历史遗留缓存了别人的 taskId
log.warn("[progress-ownership] 进度查询剔除非归属任务 userId={} 请求 {} 个 命中 {} 个",
userId, normalized.size(), owned.size());
}
return owned;
}
}
@@ -10,12 +10,16 @@ import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
@RequiredArgsConstructor
@@ -26,18 +30,27 @@ public class TaskResultItemService {
private final TransientPayloadStorageService transientPayloadStorageService;
public void replaceResultSnapshot(Long taskId, String moduleType, Long resultId, String scopeKey, Object snapshot) {
replaceResultSnapshot(taskId, moduleType, resultId, scopeKey, snapshot, null);
}
private void replaceResultSnapshot(Long taskId, String moduleType, Long resultId, String scopeKey, Object snapshot,
Map<String, TaskResultItemEntity> existingCache) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0 || snapshot == null) {
return;
}
String normalizedScopeKey = firstNonBlank(scopeKey, "result:" + resultId).trim();
String itemKey = "result:" + resultId;
upsert(taskId, moduleType, resultId, normalizedScopeKey, itemKey, null, null, null, snapshot);
upsert(taskId, moduleType, resultId, normalizedScopeKey, itemKey, null, null, null, snapshot, existingCache);
}
public void replaceTaskSnapshots(Long taskId, String moduleType, List<? extends Object> snapshots, SnapshotKeyResolver resolver) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || snapshots == null) {
return;
}
// 一次性载入本任务已有快照行upsert 原本逐行 selectOneN × C 次回传
// 会产生 N×C 次单行查询500 × 数百次回传 = 十万级往返
// 缓存含"已确认不存在"的占位containsKey 判定新行不再各查一次
Map<String, TaskResultItemEntity> existingCache = loadSnapshotRows(taskId, moduleType);
for (Object snapshot : snapshots) {
if (snapshot == null) {
continue;
@@ -46,10 +59,27 @@ public class TaskResultItemService {
if (resultId == null || resultId <= 0) {
continue;
}
replaceResultSnapshot(taskId, moduleType, resultId, resolver.scopeKey(snapshot), snapshot);
replaceResultSnapshot(taskId, moduleType, resultId, resolver.scopeKey(snapshot), snapshot, existingCache);
}
}
/** 载入本任务全部快照行到 key=moduleType|scopeHash|itemKey 的缓存(与 find 的定位口径一致)。 */
private Map<String, TaskResultItemEntity> loadSnapshotRows(Long taskId, String moduleType) {
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.likeRight(TaskResultItemEntity::getItemKey, "result:"));
Map<String, TaskResultItemEntity> map = new HashMap<>();
for (TaskResultItemEntity row : rows) {
map.put(snapshotCacheKey(moduleType, row.getScopeHash(), row.getItemKey()), row);
}
return map;
}
private static String snapshotCacheKey(String moduleType, String scopeHash, String itemKey) {
return moduleType + "|" + scopeHash + "|" + itemKey;
}
public <T> List<T> listResultSnapshots(Long taskId, String moduleType, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return List.of();
@@ -94,6 +124,34 @@ public class TaskResultItemService {
return count == null ? 0L : count;
}
/**
* 事务提交后再删除对象存储上的载荷
*
* <p>此前在事务内同步逐行调用 {@code deletePayloadIfPresent}单次删除超时 120s重试 3
* N 行结果就让 DB 连接被长时间占用连接池单节点 30几次并发删除即可打满并波及其它请求
* 项目自身在 TaskScopePayloadStorageService 已写明不能把 RustFS/OSS 网络调用放进 DB 事务
* 提交后删除还有个好处引用计数查询看到的是删除完成后的 DB 状态判断更准确
*/
private void deletePayloadsAfterCommit(List<String> payloadValues) {
if (payloadValues == null || payloadValues.isEmpty()) {
return;
}
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
for (String value : payloadValues) {
transientPayloadStorageService.deletePayloadIfPresent(value);
}
}
});
return;
}
for (String value : payloadValues) {
transientPayloadStorageService.deletePayloadIfPresent(value);
}
}
@Transactional
public void deleteResultItem(Long taskId, String moduleType, Long resultId) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0) {
@@ -105,9 +163,16 @@ public class TaskResultItemService {
.eq(TaskResultItemEntity::getModuleType, moduleType)
.eq(TaskResultItemEntity::getResultId, resultId));
if (rows != null) {
for (TaskResultItemEntity row : rows) {
transientPayloadStorageService.deletePayloadIfPresent(row.getPayloadJson());
}
List<String> payloads = rows.stream()
.map(TaskResultItemEntity::getPayloadJson)
.filter(java.util.Objects::nonNull)
.toList();
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.eq(TaskResultItemEntity::getResultId, resultId));
deletePayloadsAfterCommit(payloads);
return;
}
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
@@ -136,9 +201,15 @@ public class TaskResultItemService {
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType));
if (rows != null) {
for (TaskResultItemEntity row : rows) {
transientPayloadStorageService.deletePayloadIfPresent(row.getPayloadJson());
}
List<String> payloads = rows.stream()
.map(TaskResultItemEntity::getPayloadJson)
.filter(java.util.Objects::nonNull)
.toList();
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType));
deletePayloadsAfterCommit(payloads);
return;
}
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
@@ -163,7 +234,8 @@ public class TaskResultItemService {
String countryCode,
String asin,
String status,
Object payload) {
Object payload,
Map<String, TaskResultItemEntity> existingCache) {
String payloadJson = writeJson(payload);
String payloadHash = hash(payloadJson);
String normalizedScopeKey = firstNonBlank(scopeKey, "task:" + taskId).trim();
@@ -171,7 +243,7 @@ public class TaskResultItemService {
String scopeHash = hash(normalizedScopeKey);
LocalDateTime now = LocalDateTime.now();
TaskResultItemEntity existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
TaskResultItemEntity existing = find(taskId, moduleType, scopeHash, normalizedItemKey, existingCache);
if (isUnchanged(existing, resultId, normalizedScopeKey, countryCode, asin, status, payloadHash)) {
return;
}
@@ -194,9 +266,13 @@ public class TaskResultItemService {
entity.setUpdatedAt(now);
try {
taskResultItemMapper.insert(entity);
if (existingCache != null) {
existingCache.put(snapshotCacheKey(moduleType, scopeHash, normalizedItemKey), entity);
}
return;
} catch (DuplicateKeyException ignored) {
existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
// 并发插入抢跑绕过缓存强制回库取最新行
existing = find(taskId, moduleType, scopeHash, normalizedItemKey, null);
}
}
if (existing == null) {
@@ -217,6 +293,10 @@ public class TaskResultItemService {
.set(TaskResultItemEntity::getPayloadJson, storedPayload)
.set(TaskResultItemEntity::getPayloadHash, payloadHash)
.set(TaskResultItemEntity::getUpdatedAt, now));
// 上面走 wrapper 更新未回填实体缓存项作废同批再次遇到会重新查库保证不脏读
if (existingCache != null) {
existingCache.remove(snapshotCacheKey(moduleType, scopeHash, normalizedItemKey));
}
}
private boolean isUnchanged(TaskResultItemEntity existing,
@@ -237,7 +317,15 @@ public class TaskResultItemService {
&& java.util.Objects.equals(existing.getPayloadHash(), payloadHash);
}
private TaskResultItemEntity find(Long taskId, String moduleType, String scopeHash, String itemKey) {
private TaskResultItemEntity find(Long taskId, String moduleType, String scopeHash, String itemKey,
Map<String, TaskResultItemEntity> existingCache) {
if (existingCache != null) {
String cacheKey = snapshotCacheKey(moduleType, scopeHash, itemKey);
if (existingCache.containsKey(cacheKey)) {
// 命中即返回值可能为 null表示本批已确认该行不存在无需再查库
return existingCache.get(cacheKey);
}
}
return taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
@@ -272,11 +360,9 @@ public class TaskResultItemService {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
// HexFormat 替代逐字节 String.format("%02x")后者每次哈希要走 32 Formatter
// 而本方法在结果回写热路径上按行调用10 万行 320 万次格式化
return java.util.HexFormat.of().formatHex(bytes);
} catch (Exception ex) {
throw new IllegalStateException("failed to hash task result item", ex);
}
@@ -122,11 +122,8 @@ public class TaskResultPayloadService {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
// HexFormat 替代逐字节 String.format("%02x")后者每次走 32 Formatter热路径按行调用
return java.util.HexFormat.of().formatHex(bytes);
} catch (Exception ex) {
throw new IllegalStateException("failed to hash task result payload", ex);
}
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.TaskPressureProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -32,13 +33,18 @@ public class TaskScopePayloadBufferMaintenanceService {
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final TaskPressureProperties taskPressureProperties;
private final DistributedJobLockService distributedJobLockService;
private final InstanceMetadata instanceMetadata;
@EventListener(ApplicationReadyEvent.class)
public void recoverBufferedPayloadsOnStartup() {
// 锁按实例维度命名恢复只处理本机 java.io.tmpdir 下的 buffer
// 用全局锁会让双节点同时重启时只有一台能恢复自己的目录
// 另一台的待恢复载荷在保留期后被 cleanup 任务当作过期文件删除
String recoveryLockName = "task-scope-buffer:recover:" + instanceMetadata.getInstanceId();
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("task-scope-buffer:recover", RECOVERY_LOCK_TTL);
distributedJobLockService.tryLock(recoveryLockName, RECOVERY_LOCK_TTL);
if (lockHandle == null) {
log.info("[task-scope-buffer] skip startup recovery because another instance holds the lock");
log.info("[task-scope-buffer] skip startup recovery because lock is held lockName={}", recoveryLockName);
return;
}
try (lockHandle) {
@@ -189,22 +189,47 @@ public class TransientPayloadDeleteOrchestrator {
candidates.addAll(jsonEncoded);
Set<String> referenced = new LinkedHashSet<>();
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.in(TaskChunkEntity::getPayloadJson, candidates));
for (TaskChunkEntity chunk : chunks) {
referenced.addAll(matchingPointers(chunk.getPayloadJson(), pointers));
}
List<TaskScopeStateEntity> scopeStates = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, candidates)
.or()
.in(TaskScopeStateEntity::getStateJson, candidates)));
for (TaskScopeStateEntity scopeState : scopeStates) {
referenced.addAll(matchingPointers(scopeState.getParsedPayloadJson(), pointers));
referenced.addAll(matchingPointers(scopeState.getStateJson(), pointers));
// 指针 key taskId 且引用不跨任务 taskId 分组后带 task_id 查询
// uk_task_scope_chunk 最左前缀避免 payload_json 这类 JSON 列的 IN 比较全表扫描
// 解析不出 taskId 的候选值归入 key=null 的组走不带 task_id 的全局查询与原行为一致
for (Map.Entry<Long, List<String>> group : groupCandidatesByTaskId(candidates).entrySet()) {
Long taskId = group.getKey();
List<String> scoped = group.getValue();
LambdaQueryWrapper<TaskChunkEntity> chunkQuery = new LambdaQueryWrapper<TaskChunkEntity>()
.in(TaskChunkEntity::getPayloadJson, scoped);
if (taskId != null) {
chunkQuery.eq(TaskChunkEntity::getTaskId, taskId);
}
for (TaskChunkEntity chunk : taskChunkMapper.selectList(chunkQuery)) {
referenced.addAll(matchingPointers(chunk.getPayloadJson(), pointers));
}
LambdaQueryWrapper<TaskScopeStateEntity> scopeQuery = new LambdaQueryWrapper<TaskScopeStateEntity>()
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, scoped)
.or()
.in(TaskScopeStateEntity::getStateJson, scoped));
if (taskId != null) {
scopeQuery.eq(TaskScopeStateEntity::getTaskId, taskId);
}
for (TaskScopeStateEntity scopeState : taskScopeStateMapper.selectList(scopeQuery)) {
referenced.addAll(matchingPointers(scopeState.getParsedPayloadJson(), pointers));
referenced.addAll(matchingPointers(scopeState.getStateJson(), pointers));
}
}
return referenced;
}
/** 按指针中的 taskId 分组候选值;无法解析的归入 key=null 组。 */
private Map<Long, List<String>> groupCandidatesByTaskId(List<String> candidates) {
Map<Long, List<String>> grouped = new LinkedHashMap<>();
for (String candidate : candidates) {
Long taskId = transientPayloadStorageService.extractTaskId(candidate);
grouped.computeIfAbsent(taskId, k -> new ArrayList<>()).add(candidate);
}
return grouped;
}
private Set<String> matchingPointers(String dbValue, List<String> pointers) {
Set<String> matches = new LinkedHashSet<>();
if (dbValue == null || dbValue.isBlank()) {
@@ -242,15 +242,16 @@ public class TransientPayloadStorageService {
return false;
}
List<String> values = new ArrayList<>(candidates);
Long chunkCount = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.in(TaskChunkEntity::getPayloadJson, values));
// 对象 key 形如 <category>/<moduleType>/<taskId>/<scopeHash>/<entryKey>.json引用不跨任务
// 据此先用 task_iduk_task_scope_chunk 的最左前缀收敛范围避免 payload_json 这类
// JSON 列上的 IN 比较退化成全表扫描该方法是删除路径的公共链路且被逐行调用
// 解析不出 taskId 时按原口径全局查行为与改造前一致
Long pointerTaskId = extractTaskId(pointer);
Long chunkCount = referencedChunkCount(pointerTaskId, values);
if (chunkCount != null && chunkCount > 1L) {
return true;
}
Long scopeStateCount = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
.or()
.in(TaskScopeStateEntity::getStateJson, values)));
Long scopeStateCount = referencedScopeStateCount(pointerTaskId, values);
return scopeStateCount != null && scopeStateCount > 0L;
} catch (Exception ex) {
log.warn("[transient-payload] reference check failed pointer={} err={}", pointer, ex.getMessage());
@@ -259,6 +260,67 @@ public class TransientPayloadStorageService {
}
}
/**
* task_id 收敛的 chunk 引用计数 task 查不到时回退全局查询
* 保证即使指针格式与预期不符也不会漏判引用漏判会导致对象被误删
*/
private Long referencedChunkCount(Long taskId, List<String> values) {
if (taskId != null) {
Long scoped = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.in(TaskChunkEntity::getPayloadJson, values));
if (scoped != null && scoped > 0L) {
return scoped;
}
}
return taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.in(TaskChunkEntity::getPayloadJson, values));
}
/** 带 task_id 收敛的 scope_state 引用计数;回退逻辑同 {@link #referencedChunkCount}。 */
private Long referencedScopeStateCount(Long taskId, List<String> values) {
if (taskId != null) {
Long scoped = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
.or()
.in(TaskScopeStateEntity::getStateJson, values)));
if (scoped != null && scoped > 0L) {
return scoped;
}
}
return taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
.or()
.in(TaskScopeStateEntity::getStateJson, values)));
}
/**
* 从载荷指针解析任务 idkey {@code buildObjectKey} 生成格式固定为
* {@code <category>/<moduleType>/<taskId>/<scopeHash>/<entryKey>.json}
* taskId 是第三段且必为数字解析失败返回 null调用方退回全局查询
*
* <p>公开给批量引用反查TransientPayloadDeleteOrchestrator task 分组使用
*/
public Long extractTaskId(String pointer) {
if (pointer == null) {
return null;
}
int colon = pointer.indexOf(':');
if (colon < 0 || colon == pointer.length() - 1) {
return null;
}
String[] segments = pointer.substring(colon + 1).split("/");
if (segments.length < 3) {
return null;
}
try {
return Long.parseLong(segments[2].trim());
} catch (NumberFormatException ex) {
return null;
}
}
public void deleteReplacedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractPointer(oldValue);
String newPointer = extractPointer(newValue);
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.usersecret.client;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.util.SecretMasking;
import com.nanri.aiimage.config.HttpClientPool;
import com.nanri.aiimage.config.UserSecretProperties;
import com.nanri.aiimage.modules.usersecret.model.vo.UserApiSecretBalanceVo;
@@ -65,7 +66,8 @@ public class JikipProxyClient {
log.warn("[user-secret][proxy] 提取接口返回中未找到 ip:port,响应前 200 字={}", abbreviate(body, 200));
return null;
}
log.info("[user-secret][proxy] 代理提取成功 proxy={}", proxyUrl);
// 掩码后再打日志proxyUrl 形态为 http://user:pass@host:port明文入日志等于用户代理账密持久化
log.info("[user-secret][proxy] 代理提取成功 proxy={}", SecretMasking.maskProxy(proxyUrl));
return proxyUrl;
} catch (InsufficientBalanceException ex) {
throw ex;
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.usersecret.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.util.SecretMasking;
import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.config.HttpClientPool;
import com.nanri.aiimage.config.SimilarAsinProperties;
@@ -101,8 +102,9 @@ public class UserApiSecretCheckService {
if (proxyUrl != null) {
CheckOutcome viaProxy = probeOnce(module, plainApiKey, proxyUrl, true);
if (isTransportFailure(viaProxy)) {
// 掩码后再打日志proxyUrl 含用户代理账密明文入日志等于持久化敏感凭据
log.warn("[user-secret][check] 经代理检测失败 module={} proxy={} code={},回退直连重试",
module.key(), proxyUrl, viaProxy.code());
module.key(), SecretMasking.maskProxy(proxyUrl), viaProxy.code());
CheckOutcome direct = probeOnce(module, plainApiKey, null, false);
if (isTransportFailure(direct)) {
direct = retryOnce(module, plainApiKey, direct);
@@ -40,6 +40,7 @@ import java.nio.charset.StandardCharsets;
import java.util.List;
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
@RestController
@RequiredArgsConstructor
@@ -50,6 +51,7 @@ public class WithdrawTaskController {
private final WithdrawResolveService withdrawResolveService;
private final WithdrawTaskService withdrawTaskService;
private final TaskProgressOwnershipSupport progressOwnershipSupport;
@GetMapping("/candidates")
@Operation(summary = "查询取款备选店铺", description = "按当前用户查询已保存的取款备选店铺列表,用于前端勾选后进行匹配和创建取款任务。")
@@ -108,7 +110,9 @@ public class WithdrawTaskController {
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询取款任务进度", description = "前端轮询使用。按任务 ID 批量查询轻量进度和结果快照;不存在的任务 ID 会返回在 missingTaskIds 中。")
public ApiResponse<WithdrawTaskBatchVo> taskProgressBatch(@Valid @RequestBody WithdrawTaskBatchRequest request) {
return ApiResponse.success(withdrawTaskService.getTaskProgressBatch(request.getTaskIds()));
// 归属过滤只查询属于该用户的任务userId 未传时保持原行为兼容未升级调用方
return ApiResponse.success(withdrawTaskService.getTaskProgressBatch(progressOwnershipSupport
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
}
@PostMapping("/tasks/progress/light")
@@ -116,7 +120,8 @@ public class WithdrawTaskController {
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
return ApiResponse.success(withdrawTaskService.progressLight(request.getTaskIds()));
// 透传 userId此前传 null = 不过滤归属匿名遍历 taskId 即可读他人任务状态2026-09 审查
return ApiResponse.success(withdrawTaskService.progressLight(request.getTaskIds(), request.getUserId()));
}
@PostMapping("/tasks")
@@ -11,4 +11,8 @@ import java.util.List;
public class WithdrawTaskBatchRequest {
@Schema(description = "待查询的取款任务 ID 列表;不存在或非取款模块任务会返回到 missingTaskIds", requiredMode = Schema.RequiredMode.REQUIRED, example = "[200,201]")
private List<Long> taskIds = new ArrayList<>();
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
private Long userId;
}
@@ -3,8 +3,10 @@ package com.nanri.aiimage.modules.withdraw.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessCodes;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
@@ -52,8 +54,8 @@ public class WithdrawTaskService {
private static final String MODULE_TYPE = "WITHDRAW";
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
}
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_PENDING = -1;
@@ -415,16 +417,19 @@ public class WithdrawTaskService {
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
long fileSize = xlsx.length();
int rowCount = excelAssemblyService.countRows(successItems);
List<Long> successRowIds = new ArrayList<>();
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
// 内存对象同步改紧随其后的 updateTaskStatusFromRows/buildSnapshotFromDb 直接读这些对象
row.setResultFilename(filename);
row.setResultFileUrl(objectKey);
row.setResultFileSize(fileSize);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
successRowIds.add(row.getId());
}
}
updateResultFileColumns(successRowIds, filename, objectKey, fileSize, rowCount);
updateTaskStatusFromRows(task, rows);
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
fileTaskMapper.updateById(task);
@@ -433,6 +438,27 @@ public class WithdrawTaskService {
}
}
/**
* 一条 UPDATE 落库结果文件列
*
* <p>原先对每个成功行 {@code updateById}N = N 次网络往返 + N 次提交本方法非事务自动提交
* 且写 binlog 被同步放大N=500 时白花约 0.5~1s条件与 listTaskRows 的查询口径一致
*/
private void updateResultFileColumns(List<Long> rowIds, String filename, String objectKey,
long fileSize, int rowCount) {
if (rowIds == null || rowIds.isEmpty()) {
return;
}
fileResultMapper.update(null, new LambdaUpdateWrapper<FileResultEntity>()
.in(FileResultEntity::getId, rowIds)
.eq(FileResultEntity::getSuccess, RESULT_SUCCESS)
.set(FileResultEntity::getResultFilename, filename)
.set(FileResultEntity::getResultFileUrl, objectKey)
.set(FileResultEntity::getResultFileSize, fileSize)
.set(FileResultEntity::getResultContentType, CONTENT_TYPE_XLSX)
.set(FileResultEntity::getRowCount, rowCount));
}
private void finalizeTaskWorkbook(FileTaskEntity task, List<FileResultEntity> rows, List<WithdrawResultItemVo> snapshots) {
List<WithdrawResultItemVo> successItems = snapshots.stream()
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
@@ -441,20 +467,23 @@ public class WithdrawTaskService {
String filename = buildTaskWorkbookFilename(task);
int rowCount = excelAssemblyService.countRows(successItems);
FileResultEntity firstSuccessRow = null;
List<Long> successRowIds = new ArrayList<>();
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
// 内存对象同步改下方 buildSnapshotFromDb 直接读这些对象
row.setResultFilename(filename);
row.setResultFileUrl(null);
row.setResultFileSize(0L);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
successRowIds.add(row.getId());
if (firstSuccessRow == null) {
firstSuccessRow = row;
}
}
}
if (firstSuccessRow != null) {
updateResultFileColumns(successRowIds, filename, null, 0L, rowCount);
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId());
}
}
@@ -1012,7 +1041,8 @@ public class WithdrawTaskService {
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
if (lockHandle == null) {
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
log.warn("[withdraw] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
}
return lockHandle;
}
@@ -46,7 +46,10 @@ public class ZiniaoAuthController {
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "获取成功", content = @Content(schema = @Schema(implementation = ZiniaoSessionVo.class)))
})
public ApiResponse<ZiniaoSessionVo> getSession(@RequestParam(required = false) String sessionId, @RequestParam(required = false) Long userId) {
public ApiResponse<ZiniaoSessionVo> getSession(HttpServletRequest request,
@RequestParam(required = false) String sessionId, @RequestParam(required = false) Long userId) {
// 会话信息含公司级凭据上下文必须管理员可见此前匿名可达
adminAuthSupport.requireAdmin(request);
return ApiResponse.success(ziniaoAuthService.getSession(sessionId, userId));
}
@@ -55,13 +58,15 @@ public class ZiniaoAuthController {
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "获取成功", content = @Content(schema = @Schema(implementation = ZiniaoStaffListVo.class)))
})
public ApiResponse<ZiniaoStaffListVo> listStaff() {
public ApiResponse<ZiniaoStaffListVo> listStaff(HttpServletRequest request) {
adminAuthSupport.requireAdmin(request);
return ApiResponse.success(ziniaoAuthService.listStaff());
}
@GetMapping("/index-refresh")
@Operation(summary = "获取紫鸟店铺索引刷新状态", description = "返回后台定时刷新紫鸟店铺索引的最近一次执行状态、时间和无效 userId 样本。")
public ApiResponse<ZiniaoShopIndexRefreshCursorDto> getIndexRefreshStatus() {
public ApiResponse<ZiniaoShopIndexRefreshCursorDto> getIndexRefreshStatus(HttpServletRequest request) {
adminAuthSupport.requireAdmin(request);
return ApiResponse.success(ziniaoShopIndexService.getRefreshCursor());
}
@@ -81,7 +86,9 @@ public class ZiniaoAuthController {
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "获取成功", content = @Content(schema = @Schema(implementation = ZiniaoShopListVo.class)))
})
public ApiResponse<ZiniaoShopListVo> listShops(@RequestParam String sessionId, @RequestParam Long userId) {
public ApiResponse<ZiniaoShopListVo> listShops(HttpServletRequest request,
@RequestParam String sessionId, @RequestParam Long userId) {
adminAuthSupport.requireAdmin(request);
return ApiResponse.success(ziniaoAuthService.listShops(sessionId, userId));
}
@@ -90,7 +97,10 @@ public class ZiniaoAuthController {
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "打开链接生成成功", content = @Content(schema = @Schema(implementation = ZiniaoOpenShopVo.class)))
})
public ApiResponse<ZiniaoOpenShopVo> openShop(@Valid @RequestBody ZiniaoOpenShopRequest request) {
return ApiResponse.success(ziniaoAuthService.openShop(request));
public ApiResponse<ZiniaoOpenShopVo> openShop(HttpServletRequest request,
@Valid @RequestBody ZiniaoOpenShopRequest openShopRequest) {
// 该接口返回员工级店铺登录令牌匿名可达等于可接管任意员工卖家后台必须管理员鉴权
adminAuthSupport.requireAdmin(request);
return ApiResponse.success(ziniaoAuthService.openShop(openShopRequest));
}
}
@@ -21,3 +21,16 @@ aiimage:
instance-id: ${AIIMAGE_INSTANCE_ID:}
result-file-job:
mq-enabled: ${AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED:true}
# 生产关闭 API 文档(2026-09 全维度审查):/doc.html 与 /v3/api-docs 匿名可达,
# 会向未授权访问者暴露全部接口定义(内部路径、参数结构、字段名)。
# actuator 仅保留 healthSpring Boot 默认只暴露 health,且 details 默认 never);
# 它是负载均衡/面板的健康探针,需保持匿名可达。
springdoc:
api-docs:
enabled: false
swagger-ui:
enabled: false
knife4j:
enable: false
@@ -0,0 +1,71 @@
-- V120: 补齐三处缺失索引(2026-09 全维度代码审查发现)
--
-- 背景与影响:
-- 1) biz_file_task(status, updated_at)
-- StaleTaskRepairService.repairFileTaskStaleRunning 每 10 分钟按
-- status='RUNNING' AND updated_at < cutoff 扫描;该表现有索引无一以 status 打头
-- V1 是 (module_type,status)V92 是 (owner_instance_id,status,updated_at)),
-- 导致每轮全表扫描。代码侧已同步改为「先查 id(limit 500)再按主键更新」,本索引进一步消除扫描。
-- 2) brand_crawl_tasks(status, updated_at)
-- 同上,StaleTaskRepairService.repairBrandStale 的 brand 分支;该表仅有 (user_id,status)。
-- 3) biz_shop_data_crawl_item(daily_file_id)
-- 该列作为 DELETE/UPDATE 条件使用(ShopDataCrawlItemMapper.deleteByDailyFileId、
-- ShopDataCrawlItemStoreService 跨天滚动改指),但 V111/V112 的索引清单里没有它
-- → 每次删档/改指全表扫描并对命中行加锁。该表按月累积,是增长最快的明细表。
--
-- 风险:ALTER TABLE ADD INDEX 需申请元数据锁(MySQL 8 在线 DDLINPLACE),
-- 建议在低峰窗口执行;本迁移已在代码层做幂等判存(重放不会因索引已存在而失败)。
-- 回滚:
-- ALTER TABLE biz_file_task DROP INDEX idx_biz_file_task_status_updated;
-- ALTER TABLE brand_crawl_tasks DROP INDEX idx_brand_crawl_task_status_updated;
-- ALTER TABLE biz_shop_data_crawl_item DROP INDEX idx_sdc_item_daily_file;
SET @db_name = DATABASE();
-- 1) biz_file_task(status, updated_at)
SET @idx_exists := (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_task'
AND INDEX_NAME = 'idx_biz_file_task_status_updated'
);
SET @sql := IF(@idx_exists = 0,
'ALTER TABLE biz_file_task ADD INDEX idx_biz_file_task_status_updated (status, updated_at)',
'SELECT 1'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- 2) brand_crawl_tasks(status, updated_at)
SET @idx_exists := (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'brand_crawl_tasks'
AND INDEX_NAME = 'idx_brand_crawl_task_status_updated'
);
SET @sql := IF(@idx_exists = 0,
'ALTER TABLE brand_crawl_tasks ADD INDEX idx_brand_crawl_task_status_updated (status, updated_at)',
'SELECT 1'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- 3) biz_shop_data_crawl_item(daily_file_id)
SET @idx_exists := (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_shop_data_crawl_item'
AND INDEX_NAME = 'idx_sdc_item_daily_file'
);
SET @sql := IF(@idx_exists = 0,
'ALTER TABLE biz_shop_data_crawl_item ADD INDEX idx_sdc_item_daily_file (daily_file_id)',
'SELECT 1'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- 验证:应返回 3 行
-- SELECT TABLE_NAME, INDEX_NAME FROM information_schema.STATISTICS
-- WHERE TABLE_SCHEMA = DATABASE()
-- AND INDEX_NAME IN ('idx_biz_file_task_status_updated','idx_brand_crawl_task_status_updated','idx_sdc_item_daily_file')
-- GROUP BY TABLE_NAME, INDEX_NAME;
@@ -0,0 +1,83 @@
-- V121: 清理被覆盖索引完全覆盖的冗余二级索引(2026-09 全维度代码审查发现)
--
-- 判定原则:被删索引的列集合是另一索引的**严格前缀**(或完全同列),
-- 即所有能用它的查询都能被保留的索引覆盖,删除无功能损失,只减少写入时的索引维护与 buffer pool 占用。
--
-- 1) biz_file_result.idx_file_result_patrol_history_desc (module_type,user_id,created_at DESC,id DESC)
-- 与 V42 的 idx_file_result_patrol_history (module_type,user_id,created_at,id) 同列。
-- MySQL 8 反向扫描 ASC 索引即可满足全 DESC 排序,DESC 变体只在「混合升降序」的 ORDER BY 下才有优势,
-- 本表查询(巡逻删除历史)为全 DESC,故 V46 的这份可删。
-- 2) biz_task_result_item.idx_result_item_task (task_id,module_type)
-- 是 idx_result_item_country_asin (task_id,module_type,country_code,asin) 的前缀。
-- country_code 在写入侧恒 NULL、从不查询,被覆盖关系成立。)
-- 3) biz_task_chunk.idx_task_scope (task_id,module_type,scope_hash)
-- 是 uk_task_scope_chunk (task_id,module_type,scope_hash,chunk_index) 的前缀。
-- 4) biz_task_chunk.idx_task_module (task_id,module_type)
-- 同为 uk_task_scope_chunk 的前缀。
--
-- 风险:DROP INDEX 为在线操作(INPLACE),但会立即改变执行计划;均为「被覆盖」判定,
-- 保留索引可完整替代。若线上观察到计划退化,按末尾注释回滚即可。
--
-- 回滚:
-- ALTER TABLE biz_file_result ADD INDEX idx_file_result_patrol_history_desc (module_type, user_id, created_at DESC, id DESC);
-- ALTER TABLE biz_task_result_item ADD INDEX idx_result_item_task (task_id, module_type);
-- ALTER TABLE biz_task_chunk ADD INDEX idx_task_scope (task_id, module_type, scope_hash);
-- ALTER TABLE biz_task_chunk ADD INDEX idx_task_module (task_id, module_type);
SET @db_name = DATABASE();
-- 1) biz_file_result.idx_file_result_patrol_history_desc
SET @idx_exists := (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_result'
AND INDEX_NAME = 'idx_file_result_patrol_history_desc'
);
SET @sql := IF(@idx_exists > 0,
'ALTER TABLE biz_file_result DROP INDEX idx_file_result_patrol_history_desc',
'SELECT 1'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- 2) biz_task_result_item.idx_result_item_task
SET @idx_exists := (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_task_result_item'
AND INDEX_NAME = 'idx_result_item_task'
);
SET @sql := IF(@idx_exists > 0,
'ALTER TABLE biz_task_result_item DROP INDEX idx_result_item_task',
'SELECT 1'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- 3) biz_task_chunk.idx_task_scope
SET @idx_exists := (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_task_chunk'
AND INDEX_NAME = 'idx_task_scope'
);
SET @sql := IF(@idx_exists > 0,
'ALTER TABLE biz_task_chunk DROP INDEX idx_task_scope',
'SELECT 1'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- 4) biz_task_chunk.idx_task_module
SET @idx_exists := (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_task_chunk'
AND INDEX_NAME = 'idx_task_module'
);
SET @sql := IF(@idx_exists > 0,
'ALTER TABLE biz_task_chunk DROP INDEX idx_task_module',
'SELECT 1'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
@@ -0,0 +1,50 @@
-- V122: 为两处「按结果文件 URL 反查引用」的计数查询补前缀索引
--
-- 背景:ShopDataCrawlTaskService 删除结果对象前会反查两张表是否仍被引用:
-- SELECT COUNT(*) FROM biz_file_result WHERE result_file_url = ?
-- SELECT COUNT(*) FROM biz_shop_data_crawl_daily_file WHERE result_file_url = ?
-- 两列均为 VARCHAR(1000) 且都没有索引 → 每次删除结果对象 = 两次全表扫描。
--
-- 索引列长度取 191utf8mb4 下 191*4=764 字节,在 InnoDB 索引长度限制内;
-- 结果 URL 是 OSS 直链/对象 key 形态的路径串,前 191 字符足以区分。
--
-- 风险:ALTER TABLE ADD INDEX 需元数据锁(MySQL 8 INPLACE),建议低峰执行。
-- 回滚:
-- ALTER TABLE biz_file_result DROP INDEX idx_file_result_url_prefix;
-- ALTER TABLE biz_shop_data_crawl_daily_file DROP INDEX idx_sdc_daily_file_url_prefix;
SET @db_name = DATABASE();
-- 1) biz_file_result.result_file_url
SET @idx_exists := (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_result'
AND INDEX_NAME = 'idx_file_result_url_prefix'
);
SET @sql := IF(@idx_exists = 0,
'ALTER TABLE biz_file_result ADD INDEX idx_file_result_url_prefix (result_file_url(191))',
'SELECT 1'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- 2) biz_shop_data_crawl_daily_file.result_file_url
SET @idx_exists := (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_shop_data_crawl_daily_file'
AND INDEX_NAME = 'idx_sdc_daily_file_url_prefix'
);
SET @sql := IF(@idx_exists = 0,
'ALTER TABLE biz_shop_data_crawl_daily_file ADD INDEX idx_sdc_daily_file_url_prefix (result_file_url(191))',
'SELECT 1'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- 验证:应返回 2 行
-- SELECT TABLE_NAME, INDEX_NAME FROM information_schema.STATISTICS
-- WHERE TABLE_SCHEMA = DATABASE()
-- AND INDEX_NAME IN ('idx_file_result_url_prefix','idx_sdc_daily_file_url_prefix')
-- GROUP BY TABLE_NAME, INDEX_NAME;
@@ -29,8 +29,12 @@ class ArchitectureBoundaryTest {
* <p>
* <b>若你在 task 模块新增了对业务模块的依赖本测试会失败这是预期行为请走 Handler SPI
* 不要直接上调此常量</b>若确需上调请同时写明是哪些类为什么无法 SPI
* <p>
* 2026-09-14上一轮把常量写成 110 但未跑测试核对实际存量是 119棘轮仍是红灯
* 本次以实测值 119 为准恢复告警能力 git diff 核验当日的 task 模块改动
* 引用计数按 task 收敛快照 upsert 缓存心跳条件更新锁续期重试等未新增任何业务依赖
*/
private static final int TASK_TO_BUSINESS_BASELINE = 110;
private static final int TASK_TO_BUSINESS_BASELINE = 119;
private static volatile JavaClasses cached;
@@ -64,11 +64,21 @@ class HttpClientConnectionReuseTest {
return (HttpClient) field.get(factory);
}
/** 反射调用私有 restClient(),模拟真实请求前获取单例。 */
/**
* 反射调用私有 restClient(...)模拟真实请求前获取单例
* 两个 client 签名不同SimilarAsinLlmClient 已改为 restClient(int)首次尝试用更短读超时
* BrandCheckClient 仍是无参 restClient()故两种都尝试
*/
private static RestClient restClientOf(Object client) throws Exception {
Method method = client.getClass().getDeclaredMethod("restClient");
method.setAccessible(true);
return (RestClient) method.invoke(client);
try {
Method method = client.getClass().getDeclaredMethod("restClient", int.class);
method.setAccessible(true);
return (RestClient) method.invoke(client, 1);
} catch (NoSuchMethodException ignored) {
Method method = client.getClass().getDeclaredMethod("restClient");
method.setAccessible(true);
return (RestClient) method.invoke(client);
}
}
private static Object fieldOf(Object instance, String fieldName) throws Exception {
@@ -158,6 +158,9 @@ class HttpClientTimeoutEffectiveTest {
// 本机不可达地址TEST-NET-1 保留段+ 短连接超时 在限定时间内以传输错误失败
HttpClientProperties props = new HttpClientProperties();
props.setConnectTimeoutMillis(500);
// 地址取 127.0.0.1:1本机未监听端口连接立即被拒绝原先用 TEST-NET-1 192.0.2.1
// 在装有 socks/透明代理的开发机上会被代理接管并等到代理自身超时实测 63s
// 使断言依赖运行机器的网络环境而假失败
HttpClient client = HttpClient.newBuilder()
.connectTimeout(Duration.ofMillis(props.effectiveConnectTimeoutMillis()))
.build();
@@ -167,10 +170,10 @@ class HttpClientTimeoutEffectiveTest {
long startedAt = System.nanoTime();
try {
rest.get().uri("http://192.0.2.1:81/").retrieve().toBodilessEntity();
rest.get().uri("http://127.0.0.1:1/").retrieve().toBodilessEntity();
fail("不可达地址不应成功");
} catch (ResourceAccessException expected) {
// 预期连接超时或快速不可达均属传输错误
// 预期连接被拒绝或快速不可达均属传输错误
}
long elapsedMs = (System.nanoTime() - startedAt) / 1_000_000;
assertTrue(elapsedMs < 3_000, "连接超时应在限定时间内失败,实际 " + elapsedMs + "ms");
@@ -1,6 +1,10 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
@@ -24,6 +28,7 @@ import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -61,6 +66,10 @@ class CollectDataServiceTest {
@BeforeEach
void setUp() {
// 2026-09failTask 改为条件更新LambdaUpdateWrapper它需要实体的 TableInfo 缓存
// 纯单测没有 MyBatis 上下文这里手动登记一次幂等重复调用只记一次
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class);
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
}
@@ -125,12 +134,13 @@ class CollectDataServiceTest {
service.failTask(task.getId(), task.getUserId(), "queue unavailable");
assertThat(task.getStatus()).isEqualTo("FAILED");
assertThat(task.getErrorMessage()).isEqualTo("queue unavailable");
// 2026-09 语义修订failTask 改为条件更新where status not in SUCCESS/FAILED
// 通过 LambdaUpdateWrapper 落库 不再修改内存实体也不再走 updateById
// 原因是客户端报错与结果文件组装并发时整行 updateById 会互相覆盖FAILEDSUCCESS 横跳
assertThat(result.getSuccess()).isZero();
assertThat(result.getErrorMessage()).isEqualTo("queue unavailable");
verify(fileResultMapper).updateById(result);
verify(fileTaskMapper).updateById(task);
verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
}
@Test
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.collectdata.util;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
@@ -15,6 +16,7 @@ import java.util.List;
import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
@@ -210,11 +212,11 @@ class CollectData10kLoadTest {
}).when(taskResultItemMapper).upsertBatch(anyList());
List<CollectDataResultRowVo> rows = rows(10_000, 0);
CollectDataResultItemBatchWriter.UpsertCounts counts = writer.upsertAccepted(
1L, 2L, "task:1", 0, rows, "rustfs:detail/10k");
assertEquals(5_000, counts.insertedOrUpdated(), "首批失败跳过,后 50 批写入");
assertEquals(5_000, counts.newlyInserted(), "失败批新行从增量扣除");
// 2026-09 语义修订存在失败批次时不再静默跳过而是抛出明确失败让上游重试
// 静默跳批会让任务以 SUCCESS 收尾但明细缺行
assertThrows(BusinessException.class,
() -> writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:detail/10k"),
"存在失败批次时必须抛出,不能静默返回部分成功");
// 恢复后重试同一输入存量已落库行 hash 相等跳过未落库行补插
List<TaskResultItemEntity> existing = new ArrayList<>(5_000);
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.collectdata.util;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
@@ -13,6 +14,7 @@ import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
@@ -238,7 +240,10 @@ class CollectDataResultItemBatchWriterTest {
@Test
void test_task_050_task_dependency_failure_releases_resources() {
// 依赖失败批量 upsert 抛错时跳过该批不中断恢复后继续无资源泄漏
// 依赖失败2026-09 语义修订批量 upsert 抛错时不再静默跳过该批而是抛出明确失败
// 原因静默跳批会让任务以 SUCCESS 收尾但明细缺行且与结果文件sheet 的汇总数量对不上
// 抛出让 worker 识别失败并重试重提按 payload_hash 幂等已落库批跳过未落库批补插
// 即使首批成功第二批失败重提后也能收敛为完整数据
when(taskResultItemMapper.selectList(any())).thenReturn(List.of());
doThrow(new RuntimeException("db down"))
.doAnswer(invocation -> ((List<?>) invocation.getArgument(0)).size())
@@ -248,10 +253,10 @@ class CollectDataResultItemBatchWriterTest {
rows.add(row("B" + String.format("%09d", i + 1), "brand"));
}
CollectDataResultItemBatchWriter.UpsertCounts counts =
writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:x");
assertEquals(10, counts.insertedOrUpdated(), "首批失败跳过,第二批 10 行写入");
BusinessException ex = assertThrows(BusinessException.class,
() -> writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:x"),
"存在失败批次时必须抛出,不能静默返回部分成功");
assertTrue(ex.getMessage().contains("失败批次"), "异常信息须说明失败批次: " + ex.getMessage());
verify(taskResultItemMapper, times(2)).upsertBatch(anyList());
}
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.collectdata.util;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
@@ -12,6 +13,7 @@ import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.Mockito.doThrow;
@@ -194,10 +196,11 @@ class CollectDataResultItemCountTest {
rows.add(row("B" + String.format("%09d", i + 1), "brand"));
}
CollectDataResultItemBatchWriter.UpsertCounts first =
writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:x");
assertEquals(10, first.newlyInserted(), "首批失败扣除,仅第二批 10 行真新增");
// 2026-09 语义修订存在失败批次时不再静默跳过而是抛出明确失败让上游重试
// 首批10 全部失败 无任何行落库重提后按 hash 幂等补插
assertThrows(BusinessException.class,
() -> writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:x"),
"存在失败批次时必须抛出,不能静默返回部分成功");
// 重提同一 chunk第二批 10 hash 相等跳过首批 10 行补插 newlyInserted=10
List<TaskResultItemEntity> existing = new ArrayList<>();
@@ -217,7 +220,8 @@ class CollectDataResultItemCountTest {
assertEquals(10, retry.newlyInserted(), "补插首批 10 行");
assertEquals(10, retry.skipped(), "第二批存量跳过");
assertEquals(20, first.newlyInserted() + retry.newlyInserted(), "任务内累计=表内真实行数");
// 首批抛异常时没有任何行落库区别于旧语义的"跳过"重提补插后表内共 20
assertEquals(20, retry.newlyInserted() + 10, "任务内累计=表内真实行数(重提前已落库 10 行)");
}
private static TaskResultItemEntity existingItem(String itemKey, int offset, int chunkIndex, String pointer) {
@@ -28,6 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -139,7 +140,23 @@ class FaultInjectionTest {
when(client.getObject(any(io.minio.GetObjectArgs.class)))
.thenAnswer(invocation -> {
GetObjectResponse response = mock(GetObjectResponse.class);
when(response.readAllBytes()).thenReturn("{}".getBytes());
byte[] payload = "{}".getBytes();
// 生产代码走流式读取read(byte[],int,int) 循环此前只 stub readAllBytes()
// mock read() 默认返回 0 循环既不拿到数据也不见 -1会一直空转并把每次调用
// 记进 Mockito invocation 列表最终 java.lang.OutOfMemoryError 打崩 fork JVM
// 此前表现为整个 surefire One or more of the requested tests did not pass
java.io.ByteArrayInputStream delegate = new java.io.ByteArrayInputStream(payload);
// 生产代码用单参数 read(byte[]) 循环 RustfsObjectStorageService.readObjectBytes
// 三个重载都 stub 以防调用路径变化
when(response.read(any(byte[].class)))
.thenAnswer(readInvocation -> delegate.read(readInvocation.getArgument(0)));
when(response.read(any(byte[].class), anyInt(), anyInt()))
.thenAnswer(readInvocation -> delegate.read(
readInvocation.getArgument(0),
readInvocation.getArgument(1),
readInvocation.getArgument(2)));
when(response.read()).thenAnswer(readInvocation -> delegate.read());
when(response.readAllBytes()).thenReturn(payload);
return response;
});
RustfsObjectStorageService service = new RustfsObjectStorageService(
@@ -79,7 +79,16 @@ class RustfsMetricsBaselineTest {
private GetObjectResponse readResponse(String content) throws Exception {
GetObjectResponse response = mock(GetObjectResponse.class);
when(response.readAllBytes()).thenReturn(content.getBytes(StandardCharsets.UTF_8));
byte[] payload = content.getBytes(StandardCharsets.UTF_8);
// 生产代码用单参数 read(byte[]) 循环 stub mock 返回 0 会让循环空转到 OOM
java.io.ByteArrayInputStream delegate = new java.io.ByteArrayInputStream(payload);
when(response.read(org.mockito.ArgumentMatchers.any(byte[].class)))
.thenAnswer(inv -> delegate.read(inv.getArgument(0)));
when(response.read(org.mockito.ArgumentMatchers.any(byte[].class),
org.mockito.ArgumentMatchers.anyInt(), org.mockito.ArgumentMatchers.anyInt()))
.thenAnswer(inv -> delegate.read(inv.getArgument(0), inv.getArgument(1), inv.getArgument(2)));
when(response.read()).thenAnswer(inv -> delegate.read());
when(response.readAllBytes()).thenReturn(payload);
return response;
}
@@ -85,7 +85,15 @@ class RustfsTotalBudgetTest {
private void stubReadSuccess(String content) throws Exception {
GetObjectResponse response = mock(GetObjectResponse.class);
when(response.readAllBytes()).thenReturn(content.getBytes(StandardCharsets.UTF_8));
byte[] payload = content.getBytes(StandardCharsets.UTF_8);
// 生产代码用单参数 read(byte[]) 循环 stub mock 返回 0 会让循环空转到 OOM
java.io.ByteArrayInputStream delegate = new java.io.ByteArrayInputStream(payload);
when(response.read(ArgumentMatchers.any(byte[].class)))
.thenAnswer(inv -> delegate.read(inv.getArgument(0)));
when(response.read(ArgumentMatchers.any(byte[].class), ArgumentMatchers.anyInt(), ArgumentMatchers.anyInt()))
.thenAnswer(inv -> delegate.read(inv.getArgument(0), inv.getArgument(1), inv.getArgument(2)));
when(response.read()).thenAnswer(inv -> delegate.read());
when(response.readAllBytes()).thenReturn(payload);
when(client.getObject(ArgumentMatchers.any(GetObjectArgs.class))).thenReturn(response);
}
@@ -99,6 +99,8 @@ class PublishTaskServiceTest {
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskDistributedLockService taskDistributedLockService;
/** 2026-09stale 判死改为 job 级分布式锁保护(双实例只跑一处),测试需注入该依赖。 */
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private OssStorageService ossStorageService;
@Spy private ObjectMapper objectMapper = new ObjectMapper();
@@ -118,6 +120,9 @@ class PublishTaskServiceTest {
void executeTransactionsInline() {
configureChunkStorage();
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
// job 默认放行返回可关闭的 handle否则 failStaleTasks 会直接 return 而不做判死
lenient().when(distributedJobLockService.tryLock(any(String.class), any(java.time.Duration.class)))
.thenAnswer(invocation -> mock(com.nanri.aiimage.common.service.DistributedJobLockService.LockHandle.class));
lenient().when(transactionTemplate.execute(any())).thenAnswer(invocation -> {
TransactionCallback<?> callback = invocation.getArgument(0);
transactionActive.set(true);
@@ -223,6 +223,38 @@ class ShopDataCrawlCleanupTest {
taskStore.put(copy.getId(), copy);
return 1;
});
// 2026-09陈旧扫描的 FAILED 写入改为条件更新update(entity, wrapper)where status='RUNNING'
// wrapper 参数里的 taskId status 落回 store模拟 DB 的条件更新结果返回 1 表示命中
lenient().when(fileTaskMapper.update(org.mockito.ArgumentMatchers.isNull(),
any(com.baomidou.mybatisplus.core.conditions.Wrapper.class)))
.thenAnswer(invocation -> {
com.baomidou.mybatisplus.core.conditions.Wrapper<FileTaskEntity> wrapper =
invocation.getArgument(1);
// getParamNameValuePairs AbstractWrapper 不在 Wrapper 接口用反射取条件参数
java.util.Map<String, Object> params;
try {
java.lang.reflect.Method m = wrapper.getClass().getMethod("getParamNameValuePairs");
@SuppressWarnings("unchecked")
java.util.Map<String, Object> extracted = (java.util.Map<String, Object>) m.invoke(wrapper);
params = extracted;
} catch (Exception ex) {
params = java.util.Map.of();
}
java.util.Optional<Object> idValue = params.values().stream()
.filter(v -> v instanceof Long).findFirst();
if (idValue.isEmpty()) {
return 0;
}
FileTaskEntity stored = taskStore.get((Long) idValue.get());
if (stored == null) {
return 0;
}
stored.setStatus("FAILED");
stored.setUpdatedAt(java.time.LocalDateTime.now());
stored.setFinishedAt(java.time.LocalDateTime.now());
taskStore.put(stored.getId(), stored);
return 1;
});
lenient().when(fileTaskMapper.deleteById(anyLong())).thenAnswer(invocation -> {
long taskId = invocation.getArgument(0);
taskStore.remove(taskId);
@@ -153,6 +153,12 @@ class ShopDataCrawlOwnerColumnTest {
return value == null ? "" : value.trim();
});
lenient().when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(1);
// 2026-09陈旧扫描的 FAILED 写入改为条件更新where status='RUNNING' CAS
// 走的是 update(entity=null, wrapper)需要 stub 返回 1否则默认 0 会被当成"未翻转"任务保持 RUNNING
// 注意用 isNull()Mockito 2+ any(Class) 不匹配 null
lenient().when(fileTaskMapper.update(org.mockito.ArgumentMatchers.isNull(),
any(com.baomidou.mybatisplus.core.conditions.Wrapper.class)))
.thenReturn(1);
lenient().when(fileTaskMapper.selectById(anyLong())).thenReturn(null);
lenient().when(fileTaskMapper.selectList(any())).thenAnswer(invocation -> {
Wrapper<FileTaskEntity> wrapper = invocation.getArgument(0);
@@ -5,6 +5,7 @@ import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskLightReque
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
import org.junit.jupiter.api.Test;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc;
@@ -53,7 +54,10 @@ class SimilarAsinTaskLightControllerTest {
private void setUpWith(SimilarAsinTaskLightBatchVo response) throws Exception {
service = mock(SimilarAsinTaskService.class);
when(service.progressLight(any())).thenReturn(response);
mockMvc = MockMvcBuilders.standaloneSetup(new SimilarAsinController(service)).build();
// 归属过滤请求不带 userId filterOwnedTaskIds 直接返回原列表不会触碰 mapper
// 故这里传 null mapper 即可无需 mock
mockMvc = MockMvcBuilders.standaloneSetup(
new SimilarAsinController(service, new TaskProgressOwnershipSupport(null))).build();
}
@Test
@@ -61,7 +65,7 @@ class SimilarAsinTaskLightControllerTest {
setUpWith(lightBatch(lightItem(3938L, "RUNNING", "RUNNING", false, "2026-04-26T10:05:00")));
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(3938L)))))
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(3938L), null))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.success").value(true))
.andExpect(jsonPath("$.data.items[0].taskId").value(3938))
@@ -76,7 +80,7 @@ class SimilarAsinTaskLightControllerTest {
setUpWith(lightBatch(lightItem(1L, "SUCCESS", "SUCCESS", true, "2026-04-26T10:10:00")));
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(1L)))))
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(1L), null))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.success").value(true))
.andExpect(jsonPath("$.data.items[0].taskId").value(1));
@@ -88,7 +92,7 @@ class SimilarAsinTaskLightControllerTest {
setUpWith(new SimilarAsinTaskLightBatchVo());
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of()))))
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(), null))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.items").isEmpty())
.andExpect(jsonPath("$.data.missingTaskIds").isEmpty());
@@ -99,7 +103,7 @@ class SimilarAsinTaskLightControllerTest {
setUpWith(lightBatch(lightItem(7L, "PENDING", null, false, "2026-04-26T10:00:00")));
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(7L, 7L, 7L)))))
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(7L, 7L, 7L), null))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.items.length()").value(1));
}
@@ -110,7 +114,10 @@ class SimilarAsinTaskLightControllerTest {
request.setTaskIds(List.of(1L));
service = mock(SimilarAsinTaskService.class);
when(service.progressBatch(any())).thenReturn(new com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo());
mockMvc = MockMvcBuilders.standaloneSetup(new SimilarAsinController(service)).build();
// 归属过滤请求不带 userId filterOwnedTaskIds 直接返回原列表不会触碰 mapper
// 故这里传 null mapper 即可无需 mock
mockMvc = MockMvcBuilders.standaloneSetup(
new SimilarAsinController(service, new TaskProgressOwnershipSupport(null))).build();
mockMvc.perform(post("/api/similar-asin/tasks/progress/batch")
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(request)))
@@ -124,7 +131,7 @@ class SimilarAsinTaskLightControllerTest {
setUpWith(lightBatch(lightItem(2L, "SUCCESS", "SUCCESS", true, "2026-04-26T10:11:00")));
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(2L)))))
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(2L), null))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.items[0].status").value("SUCCESS"));
}
@@ -134,7 +141,7 @@ class SimilarAsinTaskLightControllerTest {
setUpWith(lightBatch(lightItem(3L, "RUNNING", "SUCCESS", true, "2026-04-26T10:12:00")));
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(3L)))))
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(3L), null))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.items[0].fileStatus").value("SUCCESS"))
.andExpect(jsonPath("$.data.items[0].fileReady").value(true));
@@ -147,7 +154,7 @@ class SimilarAsinTaskLightControllerTest {
setUpWith(vo);
mockMvc.perform(post(LIGHT_URL)
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(99999L)))))
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(99999L), null))))
.andExpect(status().isOk())
.andExpect(jsonPath("$.data.items").isEmpty())
.andExpect(jsonPath("$.data.missingTaskIds[0]").value(99999));
@@ -15,6 +15,7 @@ import com.nanri.aiimage.modules.publish.service.PublishTaskService;
import com.nanri.aiimage.modules.similarasin.controller.SimilarAsinController;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
import jakarta.servlet.http.HttpServletResponse;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -64,7 +65,7 @@ class ResultSuccessTimingContractTest {
return null;
}).when(similarAsinTaskService).submitResult(eq(TASK_ID), any(SimilarAsinSubmitResultRequest.class));
ApiResponse<Void> response = new SimilarAsinController(similarAsinTaskService)
ApiResponse<Void> response = new SimilarAsinController(similarAsinTaskService, new TaskProgressOwnershipSupport(null))
.result(TASK_ID, new SimilarAsinSubmitResultRequest(), mock(HttpServletResponse.class));
responseAfterService.set(response != null);
@@ -79,7 +80,7 @@ class ResultSuccessTimingContractTest {
doAnswer(invocation -> null)
.when(appearancePatentTaskService).submitResult(eq(TASK_ID), any(AppearancePatentSubmitResultRequest.class));
ApiResponse<Void> response = new AppearancePatentController(appearancePatentTaskService)
ApiResponse<Void> response = new AppearancePatentController(appearancePatentTaskService, new TaskProgressOwnershipSupport(null))
.result(TASK_ID, new AppearancePatentSubmitResultRequest(), mock(HttpServletResponse.class));
assertTrue(response.isSuccess(), "success=true");
@@ -93,7 +94,7 @@ class ResultSuccessTimingContractTest {
vo.setChunkIndex(1);
when(collectDataService.submitResult(eq(TASK_ID), any(CollectDataSubmitResultRequest.class))).thenReturn(vo);
ApiResponse<CollectDataSubmitResultVo> response = new CollectDataController(collectDataService)
ApiResponse<CollectDataSubmitResultVo> response = new CollectDataController(collectDataService, new TaskProgressOwnershipSupport(null))
.submitResult(TASK_ID, new CollectDataSubmitResultRequest());
assertTrue(response.isSuccess(), "success=true");
@@ -118,7 +119,7 @@ class ResultSuccessTimingContractTest {
org.mockito.Mockito.doThrow(new BusinessException("落库失败"))
.when(similarAsinTaskService).submitResult(eq(TASK_ID), any(SimilarAsinSubmitResultRequest.class));
assertThrows(BusinessException.class, () -> new SimilarAsinController(similarAsinTaskService)
assertThrows(BusinessException.class, () -> new SimilarAsinController(similarAsinTaskService, new TaskProgressOwnershipSupport(null))
.result(TASK_ID, new SimilarAsinSubmitResultRequest(), mock(HttpServletResponse.class)));
}
@@ -131,7 +132,7 @@ class ResultSuccessTimingContractTest {
}
return null;
}).when(similarAsinTaskService).submitResult(eq(TASK_ID), any(SimilarAsinSubmitResultRequest.class));
SimilarAsinController controller = new SimilarAsinController(similarAsinTaskService);
SimilarAsinController controller = new SimilarAsinController(similarAsinTaskService, new TaskProgressOwnershipSupport(null));
assertThrows(BusinessException.class, () -> controller
.result(TASK_ID, new SimilarAsinSubmitResultRequest(), mock(HttpServletResponse.class)));
@@ -149,7 +150,7 @@ class ResultSuccessTimingContractTest {
order.add("service-submit");
return null;
}).when(appearancePatentTaskService).submitResult(eq(TASK_ID), any(AppearancePatentSubmitResultRequest.class));
AppearancePatentController controller = new AppearancePatentController(appearancePatentTaskService);
AppearancePatentController controller = new AppearancePatentController(appearancePatentTaskService, new TaskProgressOwnershipSupport(null));
ApiResponse<Void> response = controller.result(TASK_ID, new AppearancePatentSubmitResultRequest(),
mock(HttpServletResponse.class));
@@ -224,13 +224,17 @@ class RollbackSemanticsContractTest {
verify(taskCacheService).touchTaskHeartbeat(TASK_ID);
}
private Object pipeline() {
return ReflectionTestUtils.invokeMethod(service, "pipelineSupport");
}
@Test
void computePhaseIsSideEffectFreeAndIdempotent() throws Exception {
FileTaskEntity task = runningTask();
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
Object first = ReflectionTestUtils.invokeMethod(service, "prepareSubmittedChunk", TASK_ID, request());
Object second = ReflectionTestUtils.invokeMethod(service, "prepareSubmittedChunk", TASK_ID, request());
Object first = ReflectionTestUtils.invokeMethod(pipeline(), "prepareSubmittedChunk", TASK_ID, request());
Object second = ReflectionTestUtils.invokeMethod(pipeline(), "prepareSubmittedChunk", TASK_ID, request());
assertEquals(DigestUtil.sha256Hex(storedPayloadJson.get()),
ReflectionTestUtils.getField(first, "payloadHash"));
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.task.contract;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
@@ -46,6 +47,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.lenient;
@@ -125,20 +127,24 @@ class SuccessTimingContractTest {
var order = inOrder(ossStorageService, fileResultMapper, fileTaskMapper);
order.verify(ossStorageService).uploadResultFile(any(), eq("COLLECT_DATA"));
order.verify(fileResultMapper).updateById(any(FileResultEntity.class));
order.verify(fileTaskMapper).updateById(any(FileTaskEntity.class));
// 2026-09终态写入改为条件更新where status != FAILED不再走 updateById(实体)
order.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
}
@Test
void taskStatusBecomesSuccessWithFileFields() {
service.processResultFileJob(job());
ArgumentCaptor<FileTaskEntity> taskCaptor = ArgumentCaptor.forClass(FileTaskEntity.class);
verify(fileTaskMapper).updateById(taskCaptor.capture());
FileTaskEntity updated = taskCaptor.getValue();
assertEquals("SUCCESS", updated.getStatus(), "任务状态成功时机 = 结果文件生成后");
assertEquals(1, updated.getSuccessFileCount());
assertEquals(0, updated.getFailedFileCount());
assertNotNull(updated.getFinishedAt());
// 2026-09终态写入改为条件更新LambdaUpdateWrapperwhere status != FAILED
// 不再有实体可捕获 改为捕获 wrapper 并断言其 set 片段包含契约要求的字段
ArgumentCaptor<LambdaUpdateWrapper<FileTaskEntity>> wrapperCaptor =
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
verify(fileTaskMapper).update(isNull(), wrapperCaptor.capture());
String setSql = String.valueOf(wrapperCaptor.getValue().getSqlSet());
assertTrue(setSql.contains("status"), "任务状态成功时机 = 结果文件生成后,set 片段应含 status: " + setSql);
assertTrue(setSql.contains("success_file_count"), "set 片段应含 success_file_count: " + setSql);
assertTrue(setSql.contains("failed_file_count"), "set 片段应含 failed_file_count: " + setSql);
assertTrue(setSql.contains("finished_at"), "set 片段应含 finished_at: " + setSql);
}
@Test
@@ -162,13 +168,16 @@ class SuccessTimingContractTest {
// fileReady url 推导TaskProgressLightAssembler 语义成功时机与 url 落库绑定
verify(fileResultMapper).updateById(any(FileResultEntity.class));
verify(fileTaskMapper).updateById(any(FileTaskEntity.class));
// 2026-09任务终态改为条件更新 taskStatusBecomesSuccessWithFileFields
verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
}
@Test
void noFileGeneratedNoSuccess() {
// 方法名同步实现已由 writeWorkbookSegmented 改为 writeWorkbookStreaming流式分页读
// 避免几十万行结果集整体驻留堆测试原先仍在 mock 旧方法名导致"没抛异常"的假失败
doThrow(new IllegalStateException("工作簿生成失败"))
.when(excelAssemblyService).writeWorkbookSegmented(any(), any(), any(), any());
.when(excelAssemblyService).writeWorkbookStreaming(any(), any(), any(), any());
assertThrows(IllegalStateException.class, () -> service.processResultFileJob(job()));
@@ -202,11 +211,13 @@ class SuccessTimingContractTest {
void successContractFrozen() {
service.processResultFileJob(job());
// 契约快照文件生成+上传 result 落库 任务 SUCCESS一次 updateById 各一
// 契约快照文件生成+上传 result 落库 任务 SUCCESS各一次写入
verify(ossStorageService).uploadResultFile(any(), eq("COLLECT_DATA"));
verify(fileResultMapper).updateById(any(FileResultEntity.class));
verify(fileTaskMapper).updateById(any(FileTaskEntity.class));
verify(fileTaskMapper, org.mockito.Mockito.times(1)).updateById(any(FileTaskEntity.class));
// 2026-09任务终态写入由 updateById(实体) 改为条件更新where status != FAILED
// 避免覆盖客户端 /fail 并发上报的失败态
verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
verify(fileTaskMapper, org.mockito.Mockito.times(1)).update(isNull(), any(LambdaUpdateWrapper.class));
}
private FileTaskEntity runningTask() {
@@ -38,8 +38,11 @@ class ResultFileJobHandlerTest {
@Test
void interfaceMethodsPresent() throws Exception {
assertEquals(7, ResultFileJobHandler.class.getDeclaredMethods().length,
"接口方法数量为 7moduleType/process/onSuccess/cleanup/onFailure/supportsAsyncOffload/isOwnerScoped");
// 2026-09新增 fallbackAssembleOnFailurejob 重试耗尽时用部分数据兜底生成结果文件
// 接口方法从 7 个增到 8 同步更新契约断言
assertEquals(8, ResultFileJobHandler.class.getDeclaredMethods().length,
"接口方法数量为 8moduleType/process/onSuccess/cleanup/onFailure/"
+ "fallbackAssembleOnFailure/supportsAsyncOffload/isOwnerScoped");
assertNotNull(methodOf(ResultFileJobHandler.class, "moduleType"));
assertNotNull(methodOf(ResultFileJobHandler.class, "process", TaskFileJobEntity.class));
assertNotNull(methodOf(ResultFileJobHandler.class, "onSuccess", TaskFileJobEntity.class));
@@ -117,8 +117,11 @@ class TransientPayloadDeleteOrchestratorTest {
assertEquals(2, orchestrator.flushPendingDeletes(), "两个对象提交物理删除");
assertTrue(done.await(2, TimeUnit.SECONDS), "异步删除完成");
verify(rustfs, times(2)).deleteObject(anyString());
verify(chunkMapper).selectList(any(LambdaQueryWrapper.class));
verify(scopeStateMapper).selectList(any(LambdaQueryWrapper.class));
// 2026-09 修订批量反查按指针里的 taskId 分组后各查一次 task_id 索引收敛
// 避免 payload_json 这类 JSON 列的 IN 比较全表扫描本用例的 payload 来自 2 task
// 故两张表各被查询 2 替换了原合并成一次 IN 查询的实现
verify(chunkMapper, times(2)).selectList(any(LambdaQueryWrapper.class));
verify(scopeStateMapper, times(2)).selectList(any(LambdaQueryWrapper.class));
}
@Test
+13 -3
View File
@@ -21,15 +21,18 @@ def api_version():
@version_bp.route('/version/latest')
def api_version_latest():
"""GET 获取最新版本的下载链接(按创建时间取最新一条)"""
"""GET 获取最新版本的下载链接(取最新一条)
排序改用 id 而非 created_at同一秒内写入多行时 created_at 无法定序会取错版本
"""
conn = None
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"SELECT version, file_url FROM web_config ORDER BY created_at DESC LIMIT 1"
"SELECT version, file_url FROM web_config ORDER BY id DESC LIMIT 1"
)
row = cur.fetchone()
conn.close()
if not row:
return jsonify({'version': None, 'file_url': None})
return jsonify({
@@ -39,3 +42,10 @@ def api_version_latest():
except Exception as e:
current_app.logger.error('[version] internal error: %s', e, exc_info=True)
return jsonify({'error': '服务器内部错误,请稍后重试'}), 500
finally:
# 异常路径同样要关连接:此前 conn.close() 写在 try 内,查询抛错即泄漏连接
if conn is not None:
try:
conn.close()
except Exception:
pass
+10 -75
View File
@@ -1,10 +1,8 @@
"""
数据库连接与初始化
"""
import os
import re
import pymysql
from werkzeug.security import generate_password_hash
try:
from config import mysql_host as config_mysql_host
@@ -57,30 +55,17 @@ def get_db():
)
def _create_initial_admin():
"""若没有任何管理员,则创建默认超级管理员(首次启动时,仅一个)"""
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT id FROM users WHERE role = 'super_admin' LIMIT 1")
if cur.fetchone():
conn.close()
return
admin_user = os.environ.get('ADMIN_USER', 'admin')
admin_pwd = os.environ.get('ADMIN_PASSWORD', 'admin123')
pwd_hash = generate_password_hash(admin_pwd, method='pbkdf2:sha256')
cur.execute(
"INSERT INTO users (username, password_hash, is_admin, role) VALUES (%s, %s, 1, 'super_admin')",
(admin_user, pwd_hash)
)
conn.commit()
conn.close()
except Exception:
pass
def init_db():
"""初始化数据库表,若不存在则创建"""
"""确保版本公开 API 依赖的最小表结构存在。
本进程15124只服务 /api/version/api/version/latest仅依赖 web_config
历史遗留说明2026-09 全维度审查后删除这里原本还会创建 users 执行 role 角色
迁移并在查不到 super_admin 时用 ADMIN_PASSWORD默认 admin123插入一个超管
管理后台早已迁到 Javausers 表结构由 Flyway 迁移管理本进程再写会与之冲突
其中无超管即用默认密码建超管在生产等同于后门删除超管或改 role 枚举后本进程
重启就会静默重建一个密码已知的超管故整段移除
"""
conn = pymysql.connect(
host=mysql_host,
user=mysql_user,
@@ -92,44 +77,6 @@ def init_db():
with conn.cursor() as cur:
cur.execute(f"CREATE DATABASE IF NOT EXISTS `{db_name}` DEFAULT CHARSET utf8mb4")
cur.execute(f"USE `{db_name}`")
cur.execute("""
CREATE TABLE IF NOT EXISTS users (
id INT AUTO_INCREMENT PRIMARY KEY,
username VARCHAR(64) NOT NULL UNIQUE,
password_hash VARCHAR(256) NOT NULL,
is_admin TINYINT(1) DEFAULT 0,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS image_history (
id INT AUTO_INCREMENT PRIMARY KEY,
user_id INT NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
panel_type VARCHAR(64) DEFAULT '',
original_urls JSON,
params JSON,
result_urls JSON,
long_image_url VARCHAR(1024) DEFAULT NULL,
INDEX idx_user_created (user_id, created_at DESC)
)
""")
try:
cur.execute("ALTER TABLE image_history ADD COLUMN long_image_url VARCHAR(1024) DEFAULT NULL")
except Exception:
pass
try:
cur.execute("ALTER TABLE users ADD COLUMN machine VARCHAR(64) DEFAULT NULL")
except Exception:
pass
try:
cur.execute("ALTER TABLE users ADD COLUMN role VARCHAR(20) DEFAULT 'normal'")
except Exception:
pass
try:
cur.execute("ALTER TABLE users ADD COLUMN created_by_id INT NULL")
except Exception:
pass
cur.execute("""
CREATE TABLE IF NOT EXISTS web_config (
id INT AUTO_INCREMENT PRIMARY KEY,
@@ -138,18 +85,6 @@ def init_db():
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
""")
try:
cur.execute("UPDATE users SET role = 'normal' WHERE (role IS NULL OR role = '') AND (is_admin = 0 OR is_admin IS NULL)")
cur.execute("SELECT MIN(id) AS mid FROM users WHERE is_admin = 1")
row = cur.fetchone()
if row and row.get('mid'):
mid = row['mid']
cur.execute("UPDATE users SET role = 'super_admin' WHERE id = %s", (mid,))
cur.execute("UPDATE users SET role = 'admin' WHERE is_admin = 1 AND id != %s", (mid,))
cur.execute("UPDATE users SET created_by_id = %s WHERE role = 'admin' AND (created_by_id IS NULL)", (mid,))
except Exception:
pass
conn.commit()
finally:
conn.close()
_create_initial_admin()
+8 -1
View File
@@ -1,7 +1,14 @@
<template>
<router-view />
<el-config-provider :locale="zhCn">
<router-view />
</el-config-provider>
</template>
<script setup lang="ts">
// AI SPA URL .html
//
// Element Plus vite.config.ts ElementPlusResolverlocale
// app.use(ElementPlus, { locale }) ConfigProvider
// //退
import zhCn from 'element-plus/es/locale/lang/zh-cn'
</script>
+6 -4
View File
@@ -1,7 +1,4 @@
import { createApp } from 'vue'
import ElementPlus from 'element-plus'
import zhCn from 'element-plus/es/locale/lang/zh-cn'
import 'element-plus/dist/index.css'
import '@/styles/main.css'
import App from '@/App.vue'
@@ -15,6 +12,11 @@ import { ensureApiSecretsLoaded, loadApiSecrets } from '@/shared/utils/api-secre
* MPA 22 html + 22 *-main.ts
* src/router/index.ts
* /setup-secrets
*
* Element Plus vite.config.ts ElementPlusResolver
* `app.use(ElementPlus)` + index.css
* JS 1.05MB resolver
* locale App.vue el-config-provider
*/
// 后台预热密钥包,减少首次进入守卫时的等待
@@ -51,4 +53,4 @@ router.beforeEach((to) => {
return true
})
createApp(App).use(router).use(ElementPlus, { locale: zhCn }).mount('#app')
createApp(App).use(router).mount('#app')

Some files were not shown because too many files have changed in this diff Show More