fix: 全维度审查修复(安全/正确性/性能/稳定性/客户端/前端)
安全 - /api/ziniao/** 五个匿名接口加管理员鉴权(此前可匿名换取任意员工店铺登录令牌) - 删除 Flask 遗留后门:默认密码建超管 + 每次启动写生产 users 表(服务端与客户端各一份) - 进度/详情接口归属过滤:新增 TaskProgressOwnershipSupport,11 模块 progress/light 与 /tasks/batch 接入,DTO 补 userId,前端 13 个查询封装补传(未传时后端不过滤,兼容旧端) - 代理提取链接(含账密)不再明文入日志(新增 common/util/SecretMasking) - 全局异常兜底不再回传原始异常信息;内部令牌比较改常量时间 - 登录加失败计数与锁定(10 次锁 15 分钟);品牌源文件下载加 SSRF 防护 - AdminApiGuardFilter 覆盖前缀从 2 扩到 15(开关默认 false,行为不变,为收紧做准备) - 生产关闭 springdoc/knife4j(/doc.html 匿名可读全部接口定义) 正确性 - 40901/40902 拆分:锁竞争不再被伪装成 success=true(此前客户端停止重试、分片静默丢失) - 假成功收敛:集采明细批量写失败改为抛出、去重 worker 异常标失败、4 个 worker 改判 success 字段、publish 空 ASIN 行参与批次 flush、巡店删除全失败带 error 上报 - 客户端心跳 discard 移入 finally(7 模块,失败路径不再留僵尸 RUNNING 任务) - 状态机条件更新:跟价停止循环、集采 activate/fail、imagevideo 归档回填、店铺匹配提交 性能 - 前端入口包 JS 1.05MB→204KB、CSS 355KB→10.7KB(Element Plus 改按需 + el-config-provider) - 载荷引用计数按指针里的 taskId 收敛(原 JSON 列 IN 全表扫且逐行调用) - 店铺明细多值批量 INSERT;快照 upsert 预载缓存;结果文件列改单条 UPDATE - 新增迁移 V120(补 3 个缺失索引)/V121(删 4 个被覆盖的冗余索引)/V122(URL 前缀索引) 稳定性 - 新增 common/util/ThreadPools 有界线程池替换 5 处无界队列(防堆积 OOM) - Redis 锁释放改 Lua 原子校验(原裸 delete 会误删他人已过期的锁) - imagevideo 加死节点接管;锁续期失败重试;调度池 4→16;openStream 全部加超时 - 事务内远程对象删除移到提交后;启动恢复锁按实例命名 客户端 - 不再 taskkill /f /im chrome.exe(改为按调试端口精准回收,不杀用户自己的浏览器) - 密码检测不再无条件杀紫鸟进程;品牌检测加全局互斥(代理池不再互相覆盖) - base_dir 统一到 exe 目录(原被 os.getcwd() 覆盖,日志/缓存会分裂两个目录) - 缓存加定时清理;图片下载加超时;mkstemp 句柄托管 测试 - 同步更新受影响的契约测试(构造器签名/条件更新/方法改名/新增接口方法等) - 修复 FaultInjectionTest 等 3 处 mock 未 stub 流式 read 导致的读循环 OOM - mvn test 2795 个测试全绿
This commit is contained in:
@@ -57,19 +57,18 @@
|
||||
@input="onKeywordInput"
|
||||
/>
|
||||
<div class="bell-search-days">
|
||||
<input
|
||||
v-model="startDate"
|
||||
type="date"
|
||||
class="bell-date-input"
|
||||
aria-label="起始日期"
|
||||
@change="reload"
|
||||
/>
|
||||
<span class="bell-date-sep">至</span>
|
||||
<input
|
||||
v-model="endDate"
|
||||
type="date"
|
||||
class="bell-date-input"
|
||||
aria-label="结束日期"
|
||||
<el-date-picker
|
||||
:model-value="dateRangeValue"
|
||||
type="daterange"
|
||||
value-format="YYYY-MM-DD"
|
||||
start-placeholder="开始日期"
|
||||
end-placeholder="结束日期"
|
||||
range-separator="至"
|
||||
size="small"
|
||||
class="bell-date-picker"
|
||||
popper-class="bell-date-popper"
|
||||
:clearable="true"
|
||||
@update:model-value="onDateRangeChange"
|
||||
@change="reload"
|
||||
/>
|
||||
<button v-if="hasFilter" type="button" class="bell-search-reset" @click="resetFilters">
|
||||
@@ -171,6 +170,23 @@ const groupedItems = computed(() => groupNotificationsByDay(items.value))
|
||||
const totalPages = computed(() => Math.max(1, Math.ceil(total.value / PAGE_SIZE)))
|
||||
const hasFilter = computed(() => Boolean(keyword.value.trim() || startDate.value || endDate.value))
|
||||
|
||||
/**
|
||||
* el-date-picker 的绑定桥:日期区间在内部仍用 startDate/endDate 两个 ref 表示
|
||||
* (hasFilter/normalizedDayRange/resetFilters 都基于它们)。
|
||||
* 此前模板写的是 v-model="dateRange",而 dateRange 从未声明 → vue-tsc 直接报 TS2339 构建失败,
|
||||
* 且日期筛选完全不下发参数。
|
||||
*/
|
||||
const dateRangeValue = computed<[string, string] | null>(() =>
|
||||
startDate.value && endDate.value ? [startDate.value, endDate.value] : null
|
||||
)
|
||||
|
||||
function onDateRangeChange(value: [string, string] | null) {
|
||||
const start = value?.[0]
|
||||
const end = value?.[1]
|
||||
startDate.value = start ? String(start) : ''
|
||||
endDate.value = end ? String(end) : ''
|
||||
}
|
||||
|
||||
let pollTimer: number | null = null
|
||||
let searchTimer: number | null = null
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package com.nanri.aiimage.common.exception;
|
||||
|
||||
/**
|
||||
* 业务错误码常量。
|
||||
*
|
||||
* <p>历史上 40901 被两种语义复用:「任务已结束」(幂等忽略)与「任务正在处理中」(分布式锁竞争,
|
||||
* 应重试);而 GlobalExceptionHandler 把 40901 一律转成 HTTP 200 + success=true,
|
||||
* 导致锁竞争时 Python worker 误判回传成功并停止重试,分片数据静默丢失。
|
||||
* 现拆分为两个码:
|
||||
* <ul>
|
||||
* <li>{@link #TASK_ALREADY_FINISHED}:任务已结束,重复提交无意义 → 幂等忽略,响应 success=true;</li>
|
||||
* <li>{@link #TASK_BUSY}:任务正被其它请求持锁推进 → 响应 success=false,调用方应稍后重试。</li>
|
||||
* </ul>
|
||||
*/
|
||||
public final class BusinessCodes {
|
||||
|
||||
private BusinessCodes() {
|
||||
}
|
||||
|
||||
/** 任务已结束,拒绝重复提交。语义:幂等忽略,响应 success=true,调用方不应重试。 */
|
||||
public static final int TASK_ALREADY_FINISHED = 40901;
|
||||
|
||||
/** 任务正在处理中(分布式锁竞争)。语义:资源忙,响应 success=false,调用方应稍后重试。 */
|
||||
public static final int TASK_BUSY = 40902;
|
||||
|
||||
/** 任务归属其它实例,需转发。 */
|
||||
public static final int TASK_OWNER_FORWARD = 40903;
|
||||
}
|
||||
+10
-2
@@ -61,9 +61,15 @@ public class GlobalExceptionHandler {
|
||||
|
||||
@ExceptionHandler(BusinessException.class)
|
||||
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
|
||||
if (Integer.valueOf(40901).equals(ex.getCode())) {
|
||||
if (Integer.valueOf(BusinessCodes.TASK_ALREADY_FINISHED).equals(ex.getCode())) {
|
||||
// 幂等忽略:任务已结束时的重复提交无副作用,按成功返回,避免客户端反复重试
|
||||
return ApiResponse.success("任务已结束,忽略重复提交", null);
|
||||
}
|
||||
if (Integer.valueOf(BusinessCodes.TASK_BUSY).equals(ex.getCode())) {
|
||||
// 锁竞争:必须如实返回失败 + 可重试码,否则 worker 会把「未落库」当成功而停止重试
|
||||
log.warn("[business] 任务忙,调用方应稍后重试: {}", ex.getMessage());
|
||||
return ApiResponse.fail(BusinessCodes.TASK_BUSY, ex.getMessage());
|
||||
}
|
||||
return ex.getCode() == null
|
||||
? ApiResponse.fail(ex.getMessage())
|
||||
: ApiResponse.fail(ex.getCode(), ex.getMessage());
|
||||
@@ -100,7 +106,9 @@ public class GlobalExceptionHandler {
|
||||
return ApiResponse.fail("客户端已断开连接");
|
||||
}
|
||||
log.error("Unhandled exception", ex);
|
||||
return ApiResponse.fail("服务异常: " + ex.getMessage());
|
||||
// 不再回传原始异常信息:SQL 报错、类名与内部路径会直接暴露给调用方,便于攻击者
|
||||
// 摸清技术栈与表结构。详情只进日志(上方 log.error 已带完整堆栈),对外统一文案。
|
||||
return ApiResponse.fail("服务器内部错误,请稍后重试");
|
||||
}
|
||||
|
||||
private boolean isClientAbort(Throwable ex) {
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package com.nanri.aiimage.common.util;
|
||||
|
||||
import java.net.URI;
|
||||
|
||||
/**
|
||||
* 敏感串脱敏工具(2026-09 全维度审查后收口)。
|
||||
*
|
||||
* <p>背景:代理提取链接形态为 {@code http://user:pass@host:port},代码中曾有多处
|
||||
* 直接把整串打进日志,导致用户代理账号密码长期留在应用日志与 docker logs 里。
|
||||
*/
|
||||
public final class SecretMasking {
|
||||
|
||||
private SecretMasking() {
|
||||
}
|
||||
|
||||
/** 通用掩码:保留前 4 与后 4 字符;过短则整体掩掉。 */
|
||||
public static String mask(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return "";
|
||||
}
|
||||
String text = value.trim();
|
||||
if (text.length() <= 8) {
|
||||
return "****";
|
||||
}
|
||||
return text.substring(0, 4) + "****" + text.substring(text.length() - 4);
|
||||
}
|
||||
|
||||
/** 代理掩码:隐去账号密码,保留 scheme://host:port 便于运维核对。 */
|
||||
public static String maskProxy(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return "";
|
||||
}
|
||||
try {
|
||||
URI uri = URI.create(value.trim());
|
||||
if (uri.getHost() == null || uri.getHost().isBlank()) {
|
||||
return mask(value);
|
||||
}
|
||||
StringBuilder masked = new StringBuilder();
|
||||
masked.append(uri.getScheme() == null ? "http" : uri.getScheme()).append("://");
|
||||
if (uri.getUserInfo() != null && !uri.getUserInfo().isBlank()) {
|
||||
masked.append("***@");
|
||||
}
|
||||
masked.append(uri.getHost());
|
||||
if (uri.getPort() > 0) {
|
||||
masked.append(':').append(uri.getPort());
|
||||
}
|
||||
return masked.toString();
|
||||
} catch (Exception ex) {
|
||||
return mask(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package com.nanri.aiimage.common.util;
|
||||
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 有界线程池工厂(2026-09 全维度审查补)。
|
||||
*
|
||||
* <p>业务里多处使用 {@code Executors.newFixedThreadPool}:它内部是**无界**
|
||||
* {@code LinkedBlockingQueue},任务堆积时永远不会触发拒绝策略,会把内存吃满
|
||||
* (表现为 OOM,或整机因 GC 变慢导致所有任务一起劣化)。
|
||||
* 统一改为有界队列 + {@code CallerRunsPolicy}:队列满时在提交线程执行,形成天然背压。
|
||||
*/
|
||||
public final class ThreadPools {
|
||||
|
||||
private ThreadPools() {
|
||||
}
|
||||
|
||||
/** 默认队列容量:足以吸收突发,又不至于无界堆积。 */
|
||||
public static final int DEFAULT_QUEUE_CAPACITY = 512;
|
||||
|
||||
/** 有界固定线程池(daemon 线程,空闲可回收)。 */
|
||||
public static ExecutorService boundedFixed(String threadNamePrefix, int threads) {
|
||||
return boundedFixed(threadNamePrefix, threads, DEFAULT_QUEUE_CAPACITY);
|
||||
}
|
||||
|
||||
/** 有界固定线程池(显式队列容量)。 */
|
||||
public static ExecutorService boundedFixed(String threadNamePrefix, int threads, int queueCapacity) {
|
||||
ThreadPoolExecutor executor = new ThreadPoolExecutor(
|
||||
Math.max(1, threads),
|
||||
Math.max(1, threads),
|
||||
// keepAliveTime 必须 > 0:下面开了 allowCoreThreadTimeOut,
|
||||
// 传 0 会让构造器直接抛 "Core threads must have nonzero keep alive times"
|
||||
60L, TimeUnit.SECONDS,
|
||||
new LinkedBlockingQueue<>(Math.max(1, queueCapacity)),
|
||||
runnable -> {
|
||||
Thread thread = new Thread(runnable, threadNamePrefix);
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
},
|
||||
new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
executor.allowCoreThreadTimeOut(true);
|
||||
return executor;
|
||||
}
|
||||
}
|
||||
@@ -63,6 +63,25 @@ public class AdminApiGuardFilter extends OncePerRequestFilter {
|
||||
private static final String[] USER_TOOL_PREFIXES = {
|
||||
"/api/collect-data",
|
||||
"/api/price-track",
|
||||
// 2026-09 全维度审查补:以下前缀此前完全在守卫范围之外。
|
||||
// /api/files 匿名可上传(2GB/次,可耗尽临时盘);/api/digital-human 匿名可发布/删除版本;
|
||||
// /api/image-video 的 secrets 接口 userId 取自请求体;/api/brand 的 fileUrl 曾可直接请求任意地址。
|
||||
"/api/files",
|
||||
"/api/digital-human",
|
||||
"/api/image-video",
|
||||
"/api/brand",
|
||||
"/api/appearance-patent",
|
||||
"/api/similar-asin",
|
||||
"/api/query-asin",
|
||||
"/api/patrol-delete",
|
||||
"/api/product-risk-resolve",
|
||||
"/api/shop-match",
|
||||
"/api/shop-data-crawl",
|
||||
"/api/withdraw",
|
||||
"/api/task-file-jobs",
|
||||
// /api/tasks/{taskId}/interrupted 仅凭 taskId 即可把 RUNNING 任务置为 FAILED,
|
||||
// 匿名遍历 taskId 就能批量打断线上任务
|
||||
"/api/tasks",
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -72,6 +91,9 @@ public class AdminApiGuardFilter extends OncePerRequestFilter {
|
||||
private static final String[] SELF_SERVICE_PREFIXES = {
|
||||
"/api/user-secrets",
|
||||
"/api/notifications",
|
||||
// 2026-09 全维度审查补:内部端点此前仅靠 controller 自校验令牌,纳入守卫后
|
||||
// 不带令牌的请求直接 401(带可信令牌的仍由 doFilterInternal 放行)
|
||||
"/api/internal",
|
||||
};
|
||||
|
||||
private final AdminAuthSupport adminAuthSupport;
|
||||
|
||||
@@ -3,12 +3,16 @@ package com.nanri.aiimage.config;
|
||||
import org.springframework.http.client.ClientHttpRequestFactory;
|
||||
import org.springframework.http.client.JdkClientHttpRequestFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.Authenticator;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.PasswordAuthentication;
|
||||
import java.net.ProxySelector;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
@@ -55,6 +59,38 @@ public class HttpClientPool {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 打开远程文件流(带超时),调用方负责关闭返回的流。
|
||||
*
|
||||
* <p>替代裸 {@code URI.create(url).toURL().openStream()}:后者走 JVM 默认超时(0 = 无限),
|
||||
* 上游半开连接或挂起时会把 Tomcat 工作线程无限占用(管理端批量打包可同时挂多个)。
|
||||
* 返回的流是流式的,适用于「服务端代理下载 OSS 文件转发给浏览器」这类不落盘场景。
|
||||
*
|
||||
* @param url 远程地址
|
||||
* @param timeout 等待响应超时(连接建立 + 响应头);非法值钳制到 1 秒
|
||||
* @throws IOException 非 2xx 响应或网络异常
|
||||
* @throws InterruptedException 线程被中断
|
||||
*/
|
||||
public static InputStream openStreamWithTimeout(String url, Duration timeout) throws IOException, InterruptedException {
|
||||
Duration effective = (timeout == null || timeout.isZero() || timeout.isNegative())
|
||||
? Duration.ofSeconds(1)
|
||||
: timeout;
|
||||
HttpRequest request = HttpRequest.newBuilder(URI.create(url))
|
||||
.timeout(effective)
|
||||
.GET()
|
||||
.build();
|
||||
HttpResponse<InputStream> response = sharedHttpClient().send(request, HttpResponse.BodyHandlers.ofInputStream());
|
||||
if (response.statusCode() / 100 != 2) {
|
||||
try {
|
||||
response.body().close();
|
||||
} catch (Exception ignored) {
|
||||
// 关闭失败不影响错误上报
|
||||
}
|
||||
throw new IOException("远程文件返回非 2xx: HTTP " + response.statusCode());
|
||||
}
|
||||
return response.body();
|
||||
}
|
||||
|
||||
/** 按 readTimeout(毫秒)创建共享连接池工厂;非法值钳制到最小正数。 */
|
||||
public static ClientHttpRequestFactory requestFactory(int readTimeoutMillis) {
|
||||
return requestFactory(readTimeoutMillis, null);
|
||||
|
||||
@@ -20,8 +20,15 @@ public class SchedulingConfig {
|
||||
|
||||
private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai");
|
||||
|
||||
/**
|
||||
* 调度线程池:承载全站 30+ 个 @Scheduled(含 imagevideo 1s 派发、5s 轮询、结果文件 worker 15s 等高频任务)。
|
||||
*
|
||||
* <p>此前默认 4 线程:任一慢任务(如结果文件组装被内联执行时)都会把兜底类任务
|
||||
* (StaleTaskRepair 心跳判死、陈旧扫描、历史清理)顺延,而兜底任务被顺延会直接放大线上故障面。
|
||||
* 提到 16 并保持可配(aiimage.scheduling.pool-size)。
|
||||
*/
|
||||
@Bean
|
||||
public TaskScheduler taskScheduler(@Value("${aiimage.scheduling.pool-size:4}") int poolSize) {
|
||||
public TaskScheduler taskScheduler(@Value("${aiimage.scheduling.pool-size:16}") int poolSize) {
|
||||
ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
|
||||
scheduler.setPoolSize(Math.max(1, poolSize));
|
||||
scheduler.setThreadNamePrefix("aiimage-scheduling-");
|
||||
|
||||
+4
-1
@@ -143,7 +143,10 @@ public class AdminAuthSupport {
|
||||
if (expectedToken.isBlank() || suppliedToken == null || suppliedToken.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
return expectedToken.equals(suppliedToken.trim());
|
||||
// 常量时间比较:逐字节 equals 可被计时侧信道逐位试探出内部令牌
|
||||
return java.security.MessageDigest.isEqual(
|
||||
expectedToken.getBytes(java.nio.charset.StandardCharsets.UTF_8),
|
||||
suppliedToken.trim().getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+7
-2
@@ -34,6 +34,7 @@ import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@@ -43,6 +44,7 @@ import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
public class AppearancePatentController {
|
||||
|
||||
private final AppearancePatentTaskService service;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
|
||||
@PostMapping("/parse")
|
||||
@Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件,提取 id、ASIN、国家、URL、标题等字段。返回给前端的数据只包含整数 id 和 n_1 行;n_2、n_3 等子行会保存在 OSS 解析载荷中,用于最终结果补齐。创建后的任务状态为 PENDING,不会自动推送 Python。")
|
||||
@@ -99,7 +101,9 @@ public class AppearancePatentController {
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量查询任务进度", description = "前端只对活跃任务调用该接口,建议 6 秒一次。接口只返回轻量任务状态,不返回明细结果。")
|
||||
public ApiResponse<AppearancePatentTaskBatchVo> progress(@Valid @RequestBody AppearancePatentTaskBatchRequest request) {
|
||||
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
||||
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
|
||||
return ApiResponse.success(service.progressBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/light")
|
||||
@@ -107,7 +111,8 @@ public class AppearancePatentController {
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||
return ApiResponse.success(service.progressLight(request.getTaskIds()));
|
||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||
return ApiResponse.success(service.progressLight(request.getTaskIds(), request.getUserId()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/activate")
|
||||
|
||||
+4
@@ -12,4 +12,8 @@ public class AppearancePatentTaskBatchRequest {
|
||||
@NotEmpty
|
||||
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询,避免每个任务单独请求。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<Long> taskIds;
|
||||
|
||||
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
|
||||
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
|
||||
private Long userId;
|
||||
}
|
||||
|
||||
+2
-2
@@ -103,8 +103,8 @@ public class AppearancePatentTaskService {
|
||||
|
||||
public static final String MODULE_TYPE = "APPEARANCE_PATENT";
|
||||
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
|
||||
}
|
||||
private static final String STATUS_PENDING = "PENDING";
|
||||
private static final String STATUS_RUNNING = "RUNNING";
|
||||
|
||||
@@ -18,6 +18,9 @@ import org.springframework.http.ResponseCookie;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -30,6 +33,50 @@ public class AuthService {
|
||||
private final PermissionMenuService permissionMenuService;
|
||||
private final AuthProperties authProperties;
|
||||
|
||||
/**
|
||||
* 登录失败计数与锁定(2026-09 全维度审查补):生产是公网域名,此前无任何失败限制
|
||||
* 即可无限撞库。内存实现(双节点各自计数,防护效果减半但不引入新依赖),
|
||||
* 达到阈值后锁定 15 分钟。
|
||||
*/
|
||||
private static final int LOGIN_FAIL_LIMIT = 10;
|
||||
private static final Duration LOGIN_LOCK_DURATION = Duration.ofMinutes(15);
|
||||
private final Map<String, FailRecord> loginFailures = new ConcurrentHashMap<>();
|
||||
|
||||
/** 失败计数;lockedUntil 非空表示已锁定。 */
|
||||
private record FailRecord(int count, Instant lockedUntil) {
|
||||
}
|
||||
|
||||
private void assertLoginNotLocked(String username) {
|
||||
FailRecord record = loginFailures.get(username);
|
||||
if (record != null && record.lockedUntil() != null && record.lockedUntil().isAfter(Instant.now())) {
|
||||
long minutes = Duration.between(Instant.now(), record.lockedUntil()).toMinutes() + 1;
|
||||
log.warn("[auth] 登录已锁定 username={} 剩余约 {} 分钟", username, minutes);
|
||||
throw new BusinessException("登录失败次数过多,请 " + minutes + " 分钟后再试");
|
||||
}
|
||||
}
|
||||
|
||||
private void recordLoginFailure(String username) {
|
||||
// 防无界增长:积累较多时清掉未锁定的过期记录(登录接口调用频次低)
|
||||
if (loginFailures.size() > 1000) {
|
||||
Instant now = Instant.now();
|
||||
loginFailures.entrySet().removeIf(e -> e.getValue().lockedUntil() == null
|
||||
|| e.getValue().lockedUntil().isBefore(now));
|
||||
}
|
||||
loginFailures.compute(username, (key, old) -> {
|
||||
int count = (old == null ? 0 : old.count()) + 1;
|
||||
Instant lockedUntil = count >= LOGIN_FAIL_LIMIT ? Instant.now().plus(LOGIN_LOCK_DURATION) : null;
|
||||
if (lockedUntil != null) {
|
||||
log.warn("[auth] 登录失败达阈值,锁定 username={} count={} minutes={}",
|
||||
username, count, LOGIN_LOCK_DURATION.toMinutes());
|
||||
}
|
||||
return new FailRecord(count, lockedUntil);
|
||||
});
|
||||
}
|
||||
|
||||
private void clearLoginFailures(String username) {
|
||||
loginFailures.remove(username);
|
||||
}
|
||||
|
||||
public LoginResultVo login(LoginRequest request) {
|
||||
String username = trim(request.getUsername());
|
||||
String password = request.getPassword() == null ? "" : request.getPassword();
|
||||
@@ -41,12 +88,16 @@ public class AuthService {
|
||||
throw new BusinessException("缺少设备ID,请在桌面端打开");
|
||||
}
|
||||
|
||||
assertLoginNotLocked(username);
|
||||
|
||||
LoginUserEntity user = loginUserMapper.selectOne(new LambdaQueryWrapper<LoginUserEntity>()
|
||||
.eq(LoginUserEntity::getUsername, username)
|
||||
.last("LIMIT 1"));
|
||||
if (user == null || !passwordEncoder.matches(password, user.getPasswordHash())) {
|
||||
recordLoginFailure(username);
|
||||
throw new BusinessException("用户名或密码错误");
|
||||
}
|
||||
clearLoginFailures(username);
|
||||
|
||||
boolean isAdmin = user.getIsAdmin() != null && user.getIsAdmin() == 1;
|
||||
// 单设备登录:登录成功即把账号绑定到当前设备(last-login-wins),原设备下一次请求被顶下线
|
||||
|
||||
+14
-2
@@ -4,11 +4,13 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.BrandProgressProperties;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.script.DefaultRedisScript;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Service
|
||||
@@ -21,6 +23,9 @@ public class BrandTaskProgressCacheService {
|
||||
public static final String PHASE_FAILED = "failed";
|
||||
private static final Duration FINALIZE_LOCK_TTL = Duration.ofMinutes(10);
|
||||
|
||||
/** 本实例(进程)的锁持有者标识:释放锁时用它校验"锁还是我的"。 */
|
||||
private final String lockOwnerToken = java.util.UUID.randomUUID().toString();
|
||||
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
private final BrandProgressProperties brandProgressProperties;
|
||||
@SuppressWarnings("unused")
|
||||
@@ -129,8 +134,10 @@ public class BrandTaskProgressCacheService {
|
||||
|
||||
public boolean acquireFinalizeLock(Long taskId) {
|
||||
try {
|
||||
// value 用持有者 token(而非时间戳):释放时需要它来校验"锁还是我的",
|
||||
// 否则锁因 TTL 到期被他方持有后,本线程的裸 delete 会误删他人的锁
|
||||
Boolean ok = stringRedisTemplate.opsForValue()
|
||||
.setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
|
||||
.setIfAbsent(buildFinalizeLockKey(taskId), lockOwnerToken, FINALIZE_LOCK_TTL);
|
||||
return Boolean.TRUE.equals(ok);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[brand-progress-cache] acquire finalize lock degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||
@@ -140,7 +147,12 @@ public class BrandTaskProgressCacheService {
|
||||
|
||||
public void releaseFinalizeLock(Long taskId) {
|
||||
try {
|
||||
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
|
||||
// Lua 原子校验后删除(2026-09 全维度审查):此前是裸 delete,锁已过期(TTL 到期、
|
||||
// 他人已持有)时会把别人的锁删掉,导致同一任务被两个线程同时收尾。
|
||||
stringRedisTemplate.execute(new DefaultRedisScript<>(
|
||||
"if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end",
|
||||
Long.class),
|
||||
List.of(buildFinalizeLockKey(taskId)), lockOwnerToken);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[brand-progress-cache] release finalize lock degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||
}
|
||||
|
||||
+52
-5
@@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.BrandProgressProperties;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
|
||||
@@ -53,6 +54,8 @@ import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
@@ -82,6 +85,21 @@ public class BrandTaskService {
|
||||
|
||||
private static final DateTimeFormatter DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
|
||||
private static final Duration STALE_CHECK_LOCK_TTL = Duration.ofMinutes(10);
|
||||
/** 源文件下载请求超时:比普通 API 调用宽松(源文件可能几十 MB),但必须有上限。 */
|
||||
private static final Duration SOURCE_DOWNLOAD_TIMEOUT = Duration.ofMinutes(5);
|
||||
|
||||
/** SSRF 防护:禁止请求云元数据与环回地址(正常源文件都在自家 OSS/MinIO 域名上)。 */
|
||||
private static final java.util.Set<String> BLOCKED_SOURCE_HOSTS = java.util.Set.of(
|
||||
"169.254.169.254", "metadata.google.internal", "metadata", "localhost",
|
||||
"127.0.0.1", "0.0.0.0", "::1", "[::1]");
|
||||
|
||||
private static boolean isBlockedSourceHost(String host) {
|
||||
if (host == null || host.isBlank()) {
|
||||
return true;
|
||||
}
|
||||
String normalized = host.trim().toLowerCase(java.util.Locale.ROOT);
|
||||
return BLOCKED_SOURCE_HOSTS.contains(normalized) || normalized.endsWith(".localhost");
|
||||
}
|
||||
private static final long RESULT_SUBMIT_WAIT_MILLIS = 5 * 60 * 1000L;
|
||||
private static final String STATUS_PENDING = "pending";
|
||||
private static final String STATUS_RUNNING = "running";
|
||||
@@ -92,8 +110,9 @@ public class BrandTaskService {
|
||||
/** 结果文件并发上传数:OSS/MinIO 上传互不依赖,3 并发平衡收益与内存占用。 */
|
||||
private static final int RESULT_UPLOAD_CONCURRENCY = 3;
|
||||
|
||||
private final ExecutorService resultUploadExecutor = Executors.newFixedThreadPool(
|
||||
RESULT_UPLOAD_CONCURRENCY, namedThreadFactory("brand-result-upload"));
|
||||
// 有界队列线程池:newFixedThreadPool 用的是无界队列,任务堆积时不会拒绝、会把内存吃满
|
||||
private final ExecutorService resultUploadExecutor = com.nanri.aiimage.common.util.ThreadPools
|
||||
.boundedFixed("brand-result-upload", RESULT_UPLOAD_CONCURRENCY);
|
||||
|
||||
@PreDestroy
|
||||
void shutdownResultUploadExecutor() {
|
||||
@@ -812,21 +831,49 @@ public class BrandTaskService {
|
||||
}
|
||||
|
||||
private File downloadSourceFile(String fileUrl) {
|
||||
URI uri;
|
||||
try {
|
||||
URI uri = URI.create(fileUrl);
|
||||
uri = URI.create(fileUrl);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[brand] 源文件地址非法 fileUrl={} err={}", fileUrl, ex.getMessage());
|
||||
throw new BusinessException("下载源文件失败: 地址非法");
|
||||
}
|
||||
// SSRF 防护(2026-09 全维度审查):fileUrl 来自请求体,若不校验则匿名调用方可让服务端
|
||||
// 请求内网/云元数据地址(169.254.169.254 等)。正常源文件都落在自家 OSS/MinIO 上。
|
||||
if (isBlockedSourceHost(uri.getHost())) {
|
||||
log.warn("[brand] 拒绝下载疑似 SSRF 的源文件地址 host={} fileUrl={}", uri.getHost(), fileUrl);
|
||||
throw new BusinessException("源文件地址不合法");
|
||||
}
|
||||
String filename = FileUtil.getName(uri.getPath());
|
||||
if (filename == null || filename.isBlank()) {
|
||||
filename = "brand-source.xlsx";
|
||||
}
|
||||
String suffix = FileUtil.extName(filename);
|
||||
File downloadDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "brand-source-download"));
|
||||
try {
|
||||
File tempFile = Files.createTempFile(downloadDir.toPath(), "brand_", suffix.isBlank() ? "" : "." + suffix).toFile();
|
||||
try (InputStream inputStream = uri.toURL().openStream()) {
|
||||
// 走统一连接池 + 显式超时。此前 uri.toURL().openStream() 无任何超时(JVM 默认 0 = 无限),
|
||||
// 上游半开连接会把 Tomcat 工作线程无限占用;超时值比普通 API 调用宽松(源文件可能几十 MB)
|
||||
HttpRequest downloadRequest = HttpRequest.newBuilder(uri)
|
||||
.timeout(SOURCE_DOWNLOAD_TIMEOUT)
|
||||
.GET()
|
||||
.build();
|
||||
HttpResponse<InputStream> response = HttpClientPool.sharedHttpClient()
|
||||
.send(downloadRequest, HttpResponse.BodyHandlers.ofInputStream());
|
||||
if (response.statusCode() / 100 != 2) {
|
||||
log.warn("[brand] 下载源文件返回非 2xx fileUrl={} status={}", fileUrl, response.statusCode());
|
||||
throw new BusinessException("下载源文件失败: HTTP " + response.statusCode());
|
||||
}
|
||||
try (InputStream inputStream = response.body()) {
|
||||
FileUtil.writeFromStream(inputStream, tempFile);
|
||||
}
|
||||
return tempFile;
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("下载源文件失败");
|
||||
// 此前该 catch 既不记日志也不带 cause,线上无法定位
|
||||
log.warn("[brand] 下载源文件失败 fileUrl={} err={}", fileUrl, ex.getMessage(), ex);
|
||||
throw new BusinessException("下载源文件失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+2
-5
@@ -466,11 +466,8 @@ public class BrandTaskStorageService {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
// HexFormat 替代逐字节 String.format("%02x"):后者每次走 32 次 Formatter(热路径按行调用)
|
||||
return java.util.HexFormat.of().formatHex(bytes);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to hash brand scope", ex);
|
||||
}
|
||||
|
||||
+7
-2
@@ -29,6 +29,7 @@ import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@@ -37,6 +38,7 @@ import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
public class CollectDataController {
|
||||
|
||||
private final CollectDataService service;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
|
||||
@PostMapping("/parse")
|
||||
@Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件,按行入库到 biz_collect_data_item,并保存任务筛选条件。任务初始状态为 PENDING。")
|
||||
@@ -112,7 +114,9 @@ public class CollectDataController {
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量查询任务进度")
|
||||
public ApiResponse<CollectDataTaskBatchVo> progressBatch(@Valid @RequestBody CollectDataTaskBatchRequest request) {
|
||||
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
||||
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
|
||||
return ApiResponse.success(service.progressBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/light")
|
||||
@@ -120,7 +124,8 @@ public class CollectDataController {
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||
return ApiResponse.success(service.progressLight(request.getTaskIds()));
|
||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||
return ApiResponse.success(service.progressLight(request.getTaskIds(), request.getUserId()));
|
||||
}
|
||||
|
||||
@DeleteMapping("/tasks/{taskId}")
|
||||
|
||||
+4
@@ -12,4 +12,8 @@ public class CollectDataTaskBatchRequest {
|
||||
@NotEmpty
|
||||
@Schema(description = "任务 ID 列表", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<Long> taskIds;
|
||||
|
||||
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
|
||||
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
|
||||
private Long userId;
|
||||
}
|
||||
|
||||
+45
-23
@@ -4,9 +4,11 @@ import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.crypto.digest.DigestUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
|
||||
@@ -104,8 +106,8 @@ public class CollectDataService {
|
||||
*/
|
||||
private static final String LEGACY_MODULE_TYPE = "collectdata";
|
||||
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
|
||||
}
|
||||
public static final int DEFAULT_PAGE_SIZE = 50;
|
||||
|
||||
@@ -379,34 +381,45 @@ public class CollectDataService {
|
||||
@Transactional
|
||||
public void activateTask(Long taskId, Long userId) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
|
||||
// 条件更新:上面的「已结束」判断与写入之间存在窗口(TOCTOU),期间 /fail 可能已把任务
|
||||
// 标为 FAILED —— 整行 updateById 会把它复活成 RUNNING(前端显示"执行中"但无人推进)
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.notIn(FileTaskEntity::getStatus, STATUS_SUCCESS, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated == 0) {
|
||||
throw new BusinessException("任务已结束");
|
||||
}
|
||||
task.setStatus(STATUS_RUNNING);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void failTask(Long taskId, Long userId, String error) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
|
||||
return;
|
||||
}
|
||||
String message = firstNonBlank(error, "collect-data task dispatch failed");
|
||||
FileResultEntity result = ensureTaskResult(task);
|
||||
CollectDataStats stats = loadStats(task);
|
||||
boolean alreadyTerminal = STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus());
|
||||
if (!alreadyTerminal) {
|
||||
result.setSuccess(0);
|
||||
result.setErrorMessage(message);
|
||||
result.setRowCount(stats.finalRowCount);
|
||||
fileResultMapper.updateById(result);
|
||||
task.setStatus(STATUS_FAILED);
|
||||
task.setErrorMessage(message);
|
||||
task.setFailedFileCount(1);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
persistStats(task, stats);
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
// 条件更新:客户端报错(/fail)与结果文件组装完成(processResultFileJob 写 SUCCESS)
|
||||
// 可能并发 —— 无条件 updateById 会把已生成的 SUCCESS 覆盖成 FAILED(用户拿不到下载)或反之
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.notIn(FileTaskEntity::getStatus, STATUS_SUCCESS, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getErrorMessage, message)
|
||||
.set(FileTaskEntity::getFailedFileCount, 1)
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated == 0) {
|
||||
log.info("[collect-data] failTask 跳过写入:任务已是终态 taskId={} status={}", taskId, task.getStatus());
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
@@ -954,13 +967,21 @@ public class CollectDataService {
|
||||
// 复用同一份 stats 更新 finalRowCount 后再持久化,避免重复 loadStats 丢失 summaries。
|
||||
stats.finalRowCount = (int) finalRowCount;
|
||||
persistStats(task, stats);
|
||||
task.setStatus(STATUS_SUCCESS);
|
||||
task.setSuccessFileCount(1);
|
||||
task.setFailedFileCount(0);
|
||||
task.setErrorMessage(null);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
// 条件更新:任务可能已被 /fail 标为 FAILED(客户端报错与结果文件组装并发)——
|
||||
// 无条件 updateById 会把 FAILED 覆盖成 SUCCESS(错误信息被清、用户看到「假成功」)。
|
||||
// 结果文件已生成,故失败态下仍保留文件,只是不覆盖状态。
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.ne(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getStatus, STATUS_SUCCESS)
|
||||
.set(FileTaskEntity::getSuccessFileCount, 1)
|
||||
.set(FileTaskEntity::getFailedFileCount, 0)
|
||||
.set(FileTaskEntity::getErrorMessage, null)
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated == 0) {
|
||||
log.warn("[collect-data] 结果文件已生成但任务已是 FAILED,保留失败态不覆盖 taskId={}", task.getId());
|
||||
}
|
||||
} finally {
|
||||
FileUtil.del(xlsx);
|
||||
}
|
||||
@@ -1085,7 +1106,8 @@ public class CollectDataService {
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(MODULE_TYPE, taskId, TASK_LOCK_WAIT_MILLIS);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "任务正在处理,请稍后再试");
|
||||
log.warn("[collect-data] 任务锁竞争,拒绝本次提交 taskId={} waitMillis={}", taskId, TASK_LOCK_WAIT_MILLIS);
|
||||
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
+2
-5
@@ -148,11 +148,8 @@ public class CollectDataResultDetailCodec {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
// HexFormat 替代逐字节 String.format("%02x"):后者每次走 32 次 Formatter(热路径按行调用)
|
||||
return java.util.HexFormat.of().formatHex(bytes);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("chunk detail ref hash failed", ex);
|
||||
}
|
||||
|
||||
+15
-5
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.collectdata.util;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
||||
@@ -133,12 +134,14 @@ public class CollectDataResultItemBatchWriter {
|
||||
}
|
||||
|
||||
int written = 0;
|
||||
int failedBatches = 0;
|
||||
for (int from = 0; from < toUpsert.size(); from += batchSize) {
|
||||
int to = Math.min(from + batchSize, toUpsert.size());
|
||||
List<TaskResultItemEntity> batch = toUpsert.subList(from, to);
|
||||
try {
|
||||
written += taskResultItemMapper.upsertBatch(batch);
|
||||
} catch (RuntimeException ex) {
|
||||
failedBatches++;
|
||||
log.warn("[collect-data] upsert result item batch failed, skip batch {}..{} taskId={}",
|
||||
from, to, taskId, ex);
|
||||
// 失败批的新行未落库,从增量计数中扣除,避免任务内累计虚高;
|
||||
@@ -150,6 +153,16 @@ public class CollectDataResultItemBatchWriter {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (failedBatches > 0) {
|
||||
// biz_task_result_item 是结果 Excel「明细」sheet 的唯一数据源:静默跳批会让任务
|
||||
// 以 SUCCESS 收尾但明细缺行,且与「结果文件」sheet 的汇总数量对不上(假成功)。
|
||||
// 抛出让本次 chunk 提交明确失败:worker(search_spider)识别 success=false 后会重试,
|
||||
// 重提按 payload_hash 幂等(已落库行跳过、未落库行补插),最终收敛为完整数据。
|
||||
log.error("[collect-data] 明细写入存在失败批次,拒绝本次提交 taskId={} failedBatches={} totalBatches={}",
|
||||
taskId, failedBatches, (toUpsert.size() + batchSize - 1) / batchSize);
|
||||
throw new BusinessException("采集结果明细写入失败,请稍后重试(失败批次 "
|
||||
+ failedBatches + "/" + ((toUpsert.size() + batchSize - 1) / batchSize) + ")");
|
||||
}
|
||||
return new UpsertCounts(written, skipped, newlyInserted);
|
||||
}
|
||||
|
||||
@@ -157,11 +170,8 @@ public class CollectDataResultItemBatchWriter {
|
||||
try {
|
||||
java.security.MessageDigest digest = java.security.MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
// HexFormat 替代逐字节 String.format("%02x"):后者每次走 32 次 Formatter(热路径按行调用)
|
||||
return java.util.HexFormat.of().formatHex(bytes);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("结果明细 hash 计算失败", ex);
|
||||
}
|
||||
|
||||
+3
-1
@@ -278,12 +278,14 @@ public class ConvertRunService {
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
|
||||
List<GeneratedConvertFile> generatedFiles = new ArrayList<>();
|
||||
Map<String, BufferedWriter> writers = new LinkedHashMap<>();
|
||||
try {
|
||||
// 创建循环纳入 try:第 N 个 writer 创建失败时,前 N-1 个已打开的句柄会因 finally
|
||||
// 尚未生效而泄漏(同族 SplitRunService.SplitChunkWriter 已用 try/finally 处理)
|
||||
for (String outputFilename : outputFilenames) {
|
||||
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
|
||||
generatedFiles.add(new GeneratedConvertFile(outputFilename, outputFile));
|
||||
writers.put(outputFilename, Files.newBufferedWriter(outputFile.toPath(), StandardCharsets.UTF_8));
|
||||
}
|
||||
try {
|
||||
streamTxtRowsToOutputs(inputFile, templateEntity, writers);
|
||||
} finally {
|
||||
IOException closeException = null;
|
||||
|
||||
+17
-6
@@ -75,12 +75,9 @@ public class DedupeRunService {
|
||||
private final Map<Long, AtomicInteger> runningTaskCountMap = new ConcurrentHashMap<>();
|
||||
private final Semaphore fileParallelSemaphore = new Semaphore(MAX_PARALLEL_FILES);
|
||||
/** 共享有界文件执行器;不再为请求中的每个文件创建虚拟线程。 */
|
||||
private final ExecutorService fileExecutor = Executors.newFixedThreadPool(
|
||||
MAX_PARALLEL_FILES, runnable -> {
|
||||
Thread thread = new Thread(runnable, "dedupe-file-worker");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
// 队列改为有界:newFixedThreadPool 的无界队列在文件堆积时不会拒绝、会把内存吃满
|
||||
private final ExecutorService fileExecutor = com.nanri.aiimage.common.util.ThreadPools
|
||||
.boundedFixed("dedupe-file-worker", MAX_PARALLEL_FILES);
|
||||
|
||||
/**
|
||||
* 提交去重任务:立即返回进度快照(runId),异步执行 流式读取 + 多文件并行 + 结果上传。
|
||||
@@ -171,6 +168,8 @@ public class DedupeRunService {
|
||||
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
|
||||
Map<String, DedupeArchiveEntry> archiveEntries = new ConcurrentHashMap<>();
|
||||
List<DedupeResultItemVo> outcomeItems = new ArrayList<>();
|
||||
// 是否有 worker 异常结束:为 true 时收尾必须标失败,绝不能标 SUCCESS(用户会看到成功但结果缺文件)
|
||||
boolean aborted = false;
|
||||
|
||||
try {
|
||||
AtomicInteger processedCount = new AtomicInteger(0);
|
||||
@@ -192,14 +191,26 @@ public class DedupeRunService {
|
||||
}
|
||||
}));
|
||||
}
|
||||
// 逐个等待:processFile 内部已把单文件失败记入 outcomeItems,不会往外抛;
|
||||
// 这里能捕到的是线程级异常(Error/线程池拒绝),必须记下来而不能当成功。
|
||||
for (Future<?> worker : workers) {
|
||||
try {
|
||||
worker.get();
|
||||
} catch (Exception workerEx) {
|
||||
aborted = true;
|
||||
log.error("dedupe run worker aborted runId={} error", runId, workerEx);
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
aborted = true;
|
||||
log.error("dedupe run async aborted runId={} error", runId, ex);
|
||||
}
|
||||
|
||||
try {
|
||||
if (aborted) {
|
||||
// 有文件没能正常处理完就序列化/标成功,会出现「任务成功但结果缺文件」
|
||||
throw new IllegalStateException("部分文件处理线程异常结束,结果不完整");
|
||||
}
|
||||
if (folderMode && !archiveEntries.isEmpty()) {
|
||||
DedupeResultItemVo zipItem = buildFolderZipResult(request, archiveEntries, task);
|
||||
synchronized (progress) {
|
||||
|
||||
+10
-2
@@ -17,6 +17,7 @@ import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
|
||||
import com.nanri.aiimage.modules.shopkey.mapper.ShopManageGroupMapper;
|
||||
import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
@@ -62,6 +63,7 @@ import java.util.zip.ZipOutputStream;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class DedupeTotalDataService {
|
||||
|
||||
private static final int COMPARE_BATCH_SIZE = 5000;
|
||||
@@ -579,6 +581,8 @@ public class DedupeTotalDataService {
|
||||
} catch (Exception e) {
|
||||
progress.setStatus("failed");
|
||||
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "导入 Excel 失败");
|
||||
// 此前该 catch 既不记日志也不带 cause,线上只有一句「导入 Excel 失败」,无栈无定位线索
|
||||
log.error("[dedupe-total-data] 异步导入失败 filename={} userId={}", filename, uploaderUserId, e);
|
||||
} finally {
|
||||
deleteQuietly(tempFile);
|
||||
importCompletedAtMap.put(importId, System.currentTimeMillis());
|
||||
@@ -648,7 +652,9 @@ public class DedupeTotalDataService {
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new BusinessException("删除 Excel 匹配数据失败");
|
||||
// 此前丢弃原始异常:排查时只能看到一句「删除 Excel 匹配数据失败」
|
||||
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败", e);
|
||||
throw new BusinessException("删除 Excel 匹配数据失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -946,7 +952,9 @@ public class DedupeTotalDataService {
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new BusinessException("删除 Excel 匹配数据失败");
|
||||
// 此前丢弃原始异常:排查时只能看到一句「删除 Excel 匹配数据失败」
|
||||
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败", e);
|
||||
throw new BusinessException("删除 Excel 匹配数据失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+10
-3
@@ -37,6 +37,7 @@ import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
|
||||
|
||||
@RestController
|
||||
@@ -47,6 +48,7 @@ import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
public class DeleteBrandRunController {
|
||||
|
||||
private final DeleteBrandRunService deleteBrandRunService;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
|
||||
@PostMapping("/run")
|
||||
@Operation(summary = "执行删除品牌解析", description = "读取上传的删除品牌 Excel,按国家分组解析并在每个国家内按 ASIN 去重,返回完整 payload。")
|
||||
@@ -85,13 +87,17 @@ public class DeleteBrandRunController {
|
||||
@PostMapping("/tasks/batch")
|
||||
@Operation(summary = "批量获取删除品牌任务详情", description = "合并多个 taskId 的详情查询,减少轮询期对 DB/Redis 的压力。")
|
||||
public ApiResponse<DeleteBrandTaskBatchVo> getTasksBatch(@Valid @RequestBody DeleteBrandTaskBatchRequest request) {
|
||||
return ApiResponse.success(deleteBrandRunService.getTaskDetails(request.getTaskIds()));
|
||||
// 归属过滤:/tasks/batch 同样按 userId 过滤(此前无校验,遍历 taskId 即可读他人任务详情)
|
||||
return ApiResponse.success(deleteBrandRunService.getTaskDetails(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量获取删除品牌任务进度摘要", description = "仅返回任务基础状态和行进度摘要,用于前端轮询降载。")
|
||||
public ApiResponse<DeleteBrandTaskBatchVo> getTaskProgressBatch(@Valid @RequestBody DeleteBrandTaskBatchRequest request) {
|
||||
return ApiResponse.success(deleteBrandRunService.getTaskProgress(request.getTaskIds()));
|
||||
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
|
||||
return ApiResponse.success(deleteBrandRunService.getTaskProgress(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/light")
|
||||
@@ -99,7 +105,8 @@ public class DeleteBrandRunController {
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||
return ApiResponse.success(deleteBrandRunService.progressLight(request.getTaskIds()));
|
||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||
return ApiResponse.success(deleteBrandRunService.progressLight(request.getTaskIds(), request.getUserId()));
|
||||
}
|
||||
|
||||
@GetMapping("/tasks/{taskId}/deletion-status")
|
||||
|
||||
+4
@@ -14,4 +14,8 @@ public class DeleteBrandTaskBatchRequest {
|
||||
@NotEmpty(message = "taskIds 不能为空")
|
||||
@Schema(description = "任务ID列表")
|
||||
private List<Long> taskIds = new ArrayList<>();
|
||||
|
||||
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
|
||||
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
|
||||
private Long userId;
|
||||
}
|
||||
|
||||
+2
-2
@@ -82,8 +82,8 @@ public class DeleteBrandRunService {
|
||||
/** 降级组装时缺失分片对应的结果状态(客户端未回传该行)。 */
|
||||
static final String MISSING_CHUNK_STATUS = "未回传";
|
||||
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
|
||||
}
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final Duration TASK_LOCK_TTL = TaskDistributedLockService.DEFAULT_LOCK_TTL;
|
||||
|
||||
+2
-5
@@ -393,11 +393,8 @@ public class DeleteBrandTaskStorageService {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest(normalizeScopeKey(value).getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
// HexFormat 替代逐字节 String.format("%02x"):后者每次走 32 次 Formatter(热路径按行调用)
|
||||
return java.util.HexFormat.of().formatHex(bytes);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to hash delete-brand scope", ex);
|
||||
}
|
||||
|
||||
+6
-1
@@ -153,7 +153,12 @@ public class ImageVideoArchiveService {
|
||||
Object result = readJsonValue(task.getResultJson());
|
||||
enrichCompletedTask(task, result);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
taskMapper.updateById(task);
|
||||
// 条件更新(仅 archive_status 仍为 NULL 时):整行 updateById 会用读取快照覆盖并发节点
|
||||
// 刚写入的 ARCHIVING —— 双节点 10s 周期归档时,B 节点会把 A 的认领覆盖回可认领状态,
|
||||
// 随后 B 的 CAS 也能成功,导致同一视频被下载/上传两遍(先上传的对象成孤儿)
|
||||
taskMapper.update(task, new LambdaUpdateWrapper<ImageVideoAsyncTaskEntity>()
|
||||
.eq(ImageVideoAsyncTaskEntity::getId, task.getId())
|
||||
.isNull(ImageVideoAsyncTaskEntity::getArchiveStatus));
|
||||
}
|
||||
|
||||
private void archiveTask(Long taskId) {
|
||||
|
||||
+49
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.imagevideo.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
@@ -180,6 +181,54 @@ public class ImageVideoAsyncTaskService {
|
||||
}
|
||||
}
|
||||
|
||||
/** owner 节点失联判定阈值(分钟):任务超过此时长未更新即认为原 owner 已下线。 */
|
||||
private static final long OWNER_TAKEOVER_TIMEOUT_MINUTES = 15;
|
||||
|
||||
/**
|
||||
* 死节点接管(2026-09 全维度审查补):owner 绑定的任务此前只有「owner 重启时恢复自己的」
|
||||
* 一条路径,owner 节点崩溃/被摘除后其任务永久停在「生成中」
|
||||
* ({@code StaleTaskRepairService} 只覆盖 biz_file_task / brand_crawl_tasks,不含本表)。
|
||||
*
|
||||
* <p>分两类处理:
|
||||
* <ul>
|
||||
* <li>WAITING/POLLING —— 只是等上游结果,重放安全 → 清空 owner 交回可领取队列;</li>
|
||||
* <li>RUNNING —— 重放会重复调用上游生成(有费用),故超时后明确标 FAILED 让用户重试,
|
||||
* 而不是悄悄重新执行。</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Scheduled(fixedDelayString = "${aiimage.image-video.owner-takeover-delay-ms:60000}")
|
||||
public void takeoverAbandonedTasks() {
|
||||
DistributedJobLockService.LockHandle jobLock =
|
||||
distributedJobLockService.tryLock("image-video:owner-takeover", java.time.Duration.ofSeconds(30));
|
||||
if (jobLock == null) {
|
||||
return;
|
||||
}
|
||||
try (jobLock) {
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(OWNER_TAKEOVER_TIMEOUT_MINUTES);
|
||||
int requeued = taskMapper.update(null, new LambdaUpdateWrapper<ImageVideoAsyncTaskEntity>()
|
||||
.in(ImageVideoAsyncTaskEntity::getStatus,
|
||||
TaskStatus.WAITING.name(), TaskStatus.POLLING.name())
|
||||
.isNotNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
|
||||
.ne(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
|
||||
.lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff)
|
||||
.set(ImageVideoAsyncTaskEntity::getOwnerInstanceId, null)
|
||||
.set(ImageVideoAsyncTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
int failed = taskMapper.update(null, new LambdaUpdateWrapper<ImageVideoAsyncTaskEntity>()
|
||||
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.RUNNING.name())
|
||||
.isNotNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
|
||||
.ne(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
|
||||
.lt(ImageVideoAsyncTaskEntity::getUpdatedAt, cutoff)
|
||||
.set(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.FAILED.name())
|
||||
.set(ImageVideoAsyncTaskEntity::getErrorMessage,
|
||||
"处理节点失联超过 " + OWNER_TAKEOVER_TIMEOUT_MINUTES + " 分钟,任务已自动失败,请重试")
|
||||
.set(ImageVideoAsyncTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (requeued > 0 || failed > 0) {
|
||||
log.warn("[image-video] 接管失联节点任务 requeued={} failed={} cutoffMinutes={}",
|
||||
requeued, failed, OWNER_TAKEOVER_TIMEOUT_MINUTES);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.image-video.failed-task-cleanup-delay-ms:60000}")
|
||||
public void cleanupFailedTasks() {
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES);
|
||||
|
||||
+7
-2
@@ -44,6 +44,7 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@@ -56,6 +57,7 @@ public class PatrolDeleteController {
|
||||
|
||||
private final PatrolDeleteResolveService patrolDeleteResolveService;
|
||||
private final PatrolDeleteTaskService patrolDeleteTaskService;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
|
||||
@GetMapping("/candidates")
|
||||
@Operation(summary = "查询备选店铺列表", description = "返回当前用户在巡店删除模块中已保存的备选店铺。")
|
||||
@@ -168,7 +170,9 @@ public class PatrolDeleteController {
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量查询巡店删除任务进度", description = "仅返回任务状态和店铺结果摘要,用于前端轮询降载。")
|
||||
public ApiResponse<PatrolDeleteTaskBatchVo> taskProgressBatch(@Valid @RequestBody PatrolDeleteTaskBatchRequest request) {
|
||||
return ApiResponse.success(patrolDeleteTaskService.getTaskProgressBatch(request.getTaskIds()));
|
||||
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
|
||||
return ApiResponse.success(patrolDeleteTaskService.getTaskProgressBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/light")
|
||||
@@ -176,7 +180,8 @@ public class PatrolDeleteController {
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||
return ApiResponse.success(patrolDeleteTaskService.progressLight(request.getTaskIds()));
|
||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||
return ApiResponse.success(patrolDeleteTaskService.progressLight(request.getTaskIds(), request.getUserId()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks")
|
||||
|
||||
+3
@@ -11,4 +11,7 @@ public class PatrolDeleteTaskBatchRequest {
|
||||
|
||||
@NotEmpty(message = "taskIds 不能为空")
|
||||
private List<Long> taskIds = new ArrayList<>();
|
||||
|
||||
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
|
||||
private Long userId;
|
||||
}
|
||||
|
||||
+35
-5
@@ -3,8 +3,10 @@ package com.nanri.aiimage.modules.patroldelete.service;
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
@@ -53,8 +55,8 @@ public class PatrolDeleteTaskService {
|
||||
|
||||
private static final String MODULE_TYPE = "PATROL_DELETE";
|
||||
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
|
||||
}
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final int RESULT_PENDING = -1;
|
||||
@@ -958,6 +960,27 @@ public class PatrolDeleteTaskService {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 一条 UPDATE 落库结果文件列。
|
||||
*
|
||||
* <p>原先对每个成功行 {@code updateById}:N 行 = N 次网络往返 + N 次提交(本方法非事务、自动提交),
|
||||
* 且写 binlog 被同步放大;N=500 时白花约 0.5~1s。条件与 listTaskRows 的查询口径一致。
|
||||
*/
|
||||
private void updateResultFileColumns(List<Long> rowIds, String filename, String objectKey,
|
||||
long fileSize, int rowCount) {
|
||||
if (rowIds == null || rowIds.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
fileResultMapper.update(null, new LambdaUpdateWrapper<FileResultEntity>()
|
||||
.in(FileResultEntity::getId, rowIds)
|
||||
.eq(FileResultEntity::getSuccess, RESULT_SUCCESS)
|
||||
.set(FileResultEntity::getResultFilename, filename)
|
||||
.set(FileResultEntity::getResultFileUrl, objectKey)
|
||||
.set(FileResultEntity::getResultFileSize, fileSize)
|
||||
.set(FileResultEntity::getResultContentType, CONTENT_TYPE_XLSX)
|
||||
.set(FileResultEntity::getRowCount, rowCount));
|
||||
}
|
||||
|
||||
private void finalizeTaskWorkbook(FileTaskEntity task, List<FileResultEntity> rows, List<PatrolDeleteResultItemVo> snapshots) {
|
||||
List<PatrolDeleteResultItemVo> successItems = snapshots.stream()
|
||||
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
|
||||
@@ -966,20 +989,23 @@ public class PatrolDeleteTaskService {
|
||||
String filename = buildTaskWorkbookFilename(task);
|
||||
int rowCount = excelAssemblyService.countRows(successItems);
|
||||
FileResultEntity firstSuccessRow = null;
|
||||
List<Long> successRowIds = new ArrayList<>();
|
||||
for (FileResultEntity row : rows) {
|
||||
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
|
||||
// 内存对象同步改:下方 buildSnapshotFromDb 直接读这些对象
|
||||
row.setResultFilename(filename);
|
||||
row.setResultFileUrl(null);
|
||||
row.setResultFileSize(0L);
|
||||
row.setResultContentType(CONTENT_TYPE_XLSX);
|
||||
row.setRowCount(rowCount);
|
||||
fileResultMapper.updateById(row);
|
||||
successRowIds.add(row.getId());
|
||||
if (firstSuccessRow == null) {
|
||||
firstSuccessRow = row;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (firstSuccessRow != null) {
|
||||
updateResultFileColumns(successRowIds, filename, null, 0L, rowCount);
|
||||
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId());
|
||||
snapshots = buildSnapshotFromDb(task, rows);
|
||||
}
|
||||
@@ -1017,16 +1043,19 @@ public class PatrolDeleteTaskService {
|
||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||
long fileSize = xlsx.length();
|
||||
int rowCount = excelAssemblyService.countRows(successItems);
|
||||
List<Long> successRowIds = new ArrayList<>();
|
||||
for (FileResultEntity row : rows) {
|
||||
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
|
||||
// 内存对象同步改:紧随其后的 updateTaskStatusFromRows/buildSnapshotFromDb 直接读这些对象
|
||||
row.setResultFilename(filename);
|
||||
row.setResultFileUrl(objectKey);
|
||||
row.setResultFileSize(fileSize);
|
||||
row.setResultContentType(CONTENT_TYPE_XLSX);
|
||||
row.setRowCount(rowCount);
|
||||
fileResultMapper.updateById(row);
|
||||
successRowIds.add(row.getId());
|
||||
}
|
||||
}
|
||||
updateResultFileColumns(successRowIds, filename, objectKey, fileSize, rowCount);
|
||||
updateTaskStatusFromRows(task, rows);
|
||||
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
|
||||
fileTaskMapper.updateById(task);
|
||||
@@ -1056,7 +1085,8 @@ public class PatrolDeleteTaskService {
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
|
||||
log.warn("[patrol-delete] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
|
||||
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
+10
-3
@@ -52,6 +52,7 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@@ -65,6 +66,7 @@ public class PriceTrackController {
|
||||
|
||||
private final PriceTrackService priceTrackService;
|
||||
private final PriceTrackTaskService priceTrackTaskService;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
private final PriceTrackLoopRunService priceTrackLoopRunService;
|
||||
private final SkipPriceAsinService skipPriceAsinService;
|
||||
|
||||
@@ -259,13 +261,17 @@ public class PriceTrackController {
|
||||
@PostMapping("/tasks/batch")
|
||||
@Operation(summary = "批量查询任务详情", description = "按 taskIds 返回任务概要和各店铺结果,供前端查询使用。")
|
||||
public ApiResponse<PriceTrackTaskBatchVo> tasksBatch(@Valid @RequestBody PriceTrackTaskBatchRequest request) {
|
||||
return ApiResponse.success(priceTrackTaskService.getTaskDetailsBatch(request.getTaskIds()));
|
||||
// 归属过滤:/tasks/batch 同样按 userId 过滤(此前无校验,遍历 taskId 即可读他人任务详情)
|
||||
return ApiResponse.success(priceTrackTaskService.getTaskDetailsBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量查询任务进度摘要", description = "仅返回任务状态、时间和错误等轻量信息,供前端轮询降载。")
|
||||
public ApiResponse<PriceTrackTaskBatchVo> taskProgressBatch(@Valid @RequestBody PriceTrackTaskBatchRequest request) {
|
||||
return ApiResponse.success(priceTrackTaskService.getTaskProgressBatch(request.getTaskIds()));
|
||||
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
|
||||
return ApiResponse.success(priceTrackTaskService.getTaskProgressBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/light")
|
||||
@@ -273,7 +279,8 @@ public class PriceTrackController {
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||
return ApiResponse.success(priceTrackTaskService.progressLight(request.getTaskIds()));
|
||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||
return ApiResponse.success(priceTrackTaskService.progressLight(request.getTaskIds(), request.getUserId()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/result")
|
||||
|
||||
+4
@@ -13,4 +13,8 @@ public class PriceTrackTaskBatchRequest {
|
||||
@NotEmpty(message = "taskIds不能为空")
|
||||
@Schema(description = "待查询的任务主键列表,无效或非本模块 taskId 会出现在 missingTaskIds", requiredMode = Schema.RequiredMode.REQUIRED, example = "[200,201]")
|
||||
private List<Long> taskIds = new ArrayList<>();
|
||||
|
||||
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
|
||||
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
|
||||
private Long userId;
|
||||
}
|
||||
|
||||
+15
-2
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
@@ -149,7 +150,13 @@ public class PriceTrackLoopRunService {
|
||||
if (entity.getActiveTaskId() == null || isChildTaskTerminal(entity.getActiveTaskId())) {
|
||||
markStopped(entity, null);
|
||||
} else {
|
||||
loopRunMapper.updateById(entity);
|
||||
// 只更新 stop_requested 字段:整行 updateById 会用读取快照覆盖并发写入的字段——
|
||||
// 用户点「停止循环」的瞬间子任务刚好完成时,handleChildFinished 写入的状态
|
||||
// 会被这里的旧快照覆盖回去,INFINITE 循环继续跑并继续产生真实任务
|
||||
loopRunMapper.update(null, new LambdaUpdateWrapper<PriceTrackLoopRunEntity>()
|
||||
.eq(PriceTrackLoopRunEntity::getId, entity.getId())
|
||||
.set(PriceTrackLoopRunEntity::getStopRequested, true)
|
||||
.set(PriceTrackLoopRunEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
}
|
||||
return toVo(entity);
|
||||
}
|
||||
@@ -174,7 +181,13 @@ public class PriceTrackLoopRunService {
|
||||
}
|
||||
entity.setActiveTaskId(childTaskId);
|
||||
entity.setUpdatedAt(LocalDateTime.now());
|
||||
loopRunMapper.updateById(entity);
|
||||
// 字段级更新 + status CAS:整行 updateById 会覆盖并发写入的 stop_requested,
|
||||
// 导致「停止循环」请求被静默丢弃
|
||||
loopRunMapper.update(null, new LambdaUpdateWrapper<PriceTrackLoopRunEntity>()
|
||||
.eq(PriceTrackLoopRunEntity::getId, entity.getId())
|
||||
.eq(PriceTrackLoopRunEntity::getStatus, entity.getStatus())
|
||||
.set(PriceTrackLoopRunEntity::getActiveTaskId, childTaskId)
|
||||
.set(PriceTrackLoopRunEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
log.info("[price-track-loop] child bound loopRunId={} childTaskId={} roundIndex={} shopIndex={}",
|
||||
loopRunId, childTaskId, roundIndex, shopIndex);
|
||||
}
|
||||
|
||||
+17
-3
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.pricetrack.service;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
@@ -64,9 +65,11 @@ import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
public class PriceTrackTaskService {
|
||||
|
||||
private static final String MODULE_TYPE = "PRICE_TRACK";
|
||||
/** 批量进度查询的任务 id 上限(轮询端点防滥用,与 patroldelete/deletebrand 保持一致)。 */
|
||||
private static final int MAX_PROGRESS_TASK_IDS = 50;
|
||||
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
|
||||
}
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final String ASIN_ROWS_PAYLOAD_SCOPE = "price-track-asin-rows";
|
||||
@@ -386,6 +389,16 @@ public class PriceTrackTaskService {
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return batch;
|
||||
}
|
||||
// 上限 50:轮询端点不限制数量时,调用方传数万 id 会引发单请求内数万次串行 SQL,
|
||||
// 拖慢共享连接池并波及其它实例请求(对齐 patroldelete / deletebrand 的口径)
|
||||
taskIds = taskIds.stream()
|
||||
.filter(taskId -> taskId != null && taskId > 0)
|
||||
.distinct()
|
||||
.limit(MAX_PROGRESS_TASK_IDS)
|
||||
.toList();
|
||||
if (taskIds.isEmpty()) {
|
||||
return batch;
|
||||
}
|
||||
Map<Long, FileTaskEntity> taskMap = loadTaskProgressMapByIds(taskIds);
|
||||
for (Long taskId : taskIds) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
@@ -2167,7 +2180,8 @@ public class PriceTrackTaskService {
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
|
||||
log.warn("[price-track] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
|
||||
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
+10
-3
@@ -47,6 +47,7 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@@ -59,6 +60,7 @@ public class ProductRiskResolveController {
|
||||
|
||||
private final ProductRiskResolveService productRiskResolveService;
|
||||
private final ProductRiskTaskService productRiskTaskService;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
|
||||
@GetMapping("/candidates")
|
||||
@Operation(summary = "查询备选店铺列表", description = "返回当前用户在商品风险模块中已保存的备选店铺。")
|
||||
@@ -160,13 +162,17 @@ public class ProductRiskResolveController {
|
||||
@PostMapping("/tasks/batch")
|
||||
@Operation(summary = "批量查询任务详情", description = "按 taskIds 返回任务概览与各店结果,供前端轮询使用。")
|
||||
public ApiResponse<ProductRiskTaskBatchVo> tasksBatch(@Valid @RequestBody ProductRiskTaskBatchRequest request) {
|
||||
return ApiResponse.success(productRiskTaskService.getTaskDetailsBatch(request.getTaskIds()));
|
||||
// 归属过滤:/tasks/batch 同样按 userId 过滤(此前无校验,遍历 taskId 即可读他人任务详情)
|
||||
return ApiResponse.success(productRiskTaskService.getTaskDetailsBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量查询任务进度摘要", description = "仅返回任务状态、时间和错误等轻量信息,供前端轮询降载。")
|
||||
public ApiResponse<ProductRiskTaskBatchVo> taskProgressBatch(@Valid @RequestBody ProductRiskTaskBatchRequest request) {
|
||||
return ApiResponse.success(productRiskTaskService.getTaskProgressBatch(request.getTaskIds()));
|
||||
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
|
||||
return ApiResponse.success(productRiskTaskService.getTaskProgressBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/light")
|
||||
@@ -174,7 +180,8 @@ public class ProductRiskResolveController {
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||
return ApiResponse.success(productRiskTaskService.progressLight(request.getTaskIds()));
|
||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||
return ApiResponse.success(productRiskTaskService.progressLight(request.getTaskIds(), request.getUserId()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/result")
|
||||
|
||||
+4
@@ -17,4 +17,8 @@ public class ProductRiskTaskBatchRequest {
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
example = "[200,201]")
|
||||
private List<Long> taskIds = new ArrayList<>();
|
||||
|
||||
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
|
||||
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
|
||||
private Long userId;
|
||||
}
|
||||
|
||||
+17
-3
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.productrisk.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
@@ -56,9 +57,11 @@ import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
public class ProductRiskTaskService {
|
||||
|
||||
private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE";
|
||||
/** 批量进度查询的任务 id 上限(轮询端点防滥用,与 patroldelete/deletebrand 保持一致)。 */
|
||||
private static final int MAX_PROGRESS_TASK_IDS = 50;
|
||||
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
|
||||
}
|
||||
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
||||
|
||||
@@ -446,6 +449,16 @@ public class ProductRiskTaskService {
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return batch;
|
||||
}
|
||||
// 上限 50:轮询端点不限制数量时,调用方传数万 id 会引发单请求内数万次串行 SQL,
|
||||
// 拖慢共享连接池并波及其它实例请求(对齐 patroldelete / deletebrand 的口径)
|
||||
taskIds = taskIds.stream()
|
||||
.filter(taskId -> taskId != null && taskId > 0)
|
||||
.distinct()
|
||||
.limit(MAX_PROGRESS_TASK_IDS)
|
||||
.toList();
|
||||
if (taskIds.isEmpty()) {
|
||||
return batch;
|
||||
}
|
||||
Map<Long, FileTaskEntity> taskMap = loadTaskMapByIds(taskIds);
|
||||
for (Long taskId : taskIds) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
@@ -980,7 +993,8 @@ public class ProductRiskTaskService {
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
|
||||
log.warn("[product-risk] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
|
||||
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
+7
-2
@@ -42,6 +42,7 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@@ -54,6 +55,7 @@ public class QueryAsinTaskController {
|
||||
|
||||
private final QueryAsinResolveService queryAsinResolveService;
|
||||
private final QueryAsinTaskService queryAsinTaskService;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
|
||||
@GetMapping("/candidates")
|
||||
@Operation(summary = "查询备选店铺列表", description = "返回当前用户在查询 ASIN 模块中已保存的备选店铺。")
|
||||
@@ -138,7 +140,9 @@ public class QueryAsinTaskController {
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量查询查询 ASIN 任务进度", description = "仅返回任务状态和店铺结果摘要,用于前端轮询降载。")
|
||||
public ApiResponse<QueryAsinTaskBatchVo> taskProgressBatch(@Valid @RequestBody QueryAsinTaskBatchRequest request) {
|
||||
return ApiResponse.success(queryAsinTaskService.getTaskProgressBatch(request.getTaskIds()));
|
||||
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
|
||||
return ApiResponse.success(queryAsinTaskService.getTaskProgressBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/light")
|
||||
@@ -146,7 +150,8 @@ public class QueryAsinTaskController {
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||
return ApiResponse.success(queryAsinTaskService.progressLight(request.getTaskIds()));
|
||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||
return ApiResponse.success(queryAsinTaskService.progressLight(request.getTaskIds(), request.getUserId()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks")
|
||||
|
||||
+3
@@ -11,5 +11,8 @@ public class QueryAsinTaskBatchRequest {
|
||||
|
||||
@NotEmpty(message = "taskIds 不能为空")
|
||||
private List<Long> taskIds = new ArrayList<>();
|
||||
|
||||
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
|
||||
private Long userId;
|
||||
}
|
||||
|
||||
|
||||
+36
-5
@@ -3,8 +3,10 @@ package com.nanri.aiimage.modules.queryasin.service;
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
@@ -52,8 +54,8 @@ public class QueryAsinTaskService {
|
||||
|
||||
private static final String MODULE_TYPE = "QUERY_ASIN";
|
||||
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
|
||||
}
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final int RESULT_PENDING = -1;
|
||||
@@ -537,6 +539,28 @@ public class QueryAsinTaskService {
|
||||
.orderByAsc(FileResultEntity::getId));
|
||||
}
|
||||
|
||||
/**
|
||||
* 一条 UPDATE 落库结果文件列。
|
||||
*
|
||||
* <p>原先对每个成功行 {@code updateById}:N 行 = N 次网络往返 + N 次提交
|
||||
* (这些方法非事务、各自自动提交),且写 binlog 被同步放大;N=500 时白花约 0.5~1s。
|
||||
* 条件与 {@link #listTaskRows} 的查询口径一致(同任务同模块的成功行)。
|
||||
*/
|
||||
private void updateResultFileColumns(List<Long> rowIds, String filename, String objectKey,
|
||||
long fileSize, int rowCount) {
|
||||
if (rowIds == null || rowIds.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
fileResultMapper.update(null, new LambdaUpdateWrapper<FileResultEntity>()
|
||||
.in(FileResultEntity::getId, rowIds)
|
||||
.eq(FileResultEntity::getSuccess, RESULT_SUCCESS)
|
||||
.set(FileResultEntity::getResultFilename, filename)
|
||||
.set(FileResultEntity::getResultFileUrl, objectKey)
|
||||
.set(FileResultEntity::getResultFileSize, fileSize)
|
||||
.set(FileResultEntity::getResultContentType, CONTENT_TYPE_XLSX)
|
||||
.set(FileResultEntity::getRowCount, rowCount));
|
||||
}
|
||||
|
||||
private Map<String, QueryAsinShopPayloadDto> normalizePayloadByShop(List<QueryAsinShopPayloadDto> shops) {
|
||||
Map<String, QueryAsinShopPayloadDto> payloadByShop = new LinkedHashMap<>();
|
||||
for (QueryAsinShopPayloadDto item : shops) {
|
||||
@@ -920,20 +944,23 @@ public class QueryAsinTaskService {
|
||||
String filename = buildTaskWorkbookFilename(task);
|
||||
int rowCount = excelAssemblyService.countRows(successItems);
|
||||
FileResultEntity firstSuccessRow = null;
|
||||
List<Long> successRowIds = new ArrayList<>();
|
||||
for (FileResultEntity row : rows) {
|
||||
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
|
||||
// 内存对象同步改:下方 buildSnapshotFromDb 直接读这些对象
|
||||
row.setResultFilename(filename);
|
||||
row.setResultFileUrl(null);
|
||||
row.setResultFileSize(0L);
|
||||
row.setResultContentType(CONTENT_TYPE_XLSX);
|
||||
row.setRowCount(rowCount);
|
||||
fileResultMapper.updateById(row);
|
||||
successRowIds.add(row.getId());
|
||||
if (firstSuccessRow == null) {
|
||||
firstSuccessRow = row;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (firstSuccessRow != null) {
|
||||
updateResultFileColumns(successRowIds, filename, null, 0L, rowCount);
|
||||
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId());
|
||||
snapshots = buildSnapshotFromDb(task, rows);
|
||||
}
|
||||
@@ -971,16 +998,19 @@ public class QueryAsinTaskService {
|
||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||
long fileSize = xlsx.length();
|
||||
int rowCount = excelAssemblyService.countRows(successItems);
|
||||
List<Long> successRowIds = new ArrayList<>();
|
||||
for (FileResultEntity row : rows) {
|
||||
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
|
||||
// 内存对象同步改:紧随其后的 updateTaskStatusFromRows/buildSnapshotFromDb 直接读这些对象
|
||||
row.setResultFilename(filename);
|
||||
row.setResultFileUrl(objectKey);
|
||||
row.setResultFileSize(fileSize);
|
||||
row.setResultContentType(CONTENT_TYPE_XLSX);
|
||||
row.setRowCount(rowCount);
|
||||
fileResultMapper.updateById(row);
|
||||
successRowIds.add(row.getId());
|
||||
}
|
||||
}
|
||||
updateResultFileColumns(successRowIds, filename, objectKey, fileSize, rowCount);
|
||||
updateTaskStatusFromRows(task, rows);
|
||||
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
|
||||
fileTaskMapper.updateById(task);
|
||||
@@ -1010,7 +1040,8 @@ public class QueryAsinTaskService {
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
|
||||
log.warn("[query-asin] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
|
||||
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
+6
-2
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.shopdatacrawl.controller;
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
|
||||
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
|
||||
@@ -21,11 +22,11 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* Internal compatibility endpoints used by the Flask admin task page.
|
||||
@@ -36,6 +37,9 @@ import java.security.MessageDigest;
|
||||
@RequestMapping("/api/admin/shop-data-crawl")
|
||||
public class AdminShopDataCrawlTaskController {
|
||||
|
||||
/** 代理下载 OSS 文件转发给浏览器的超时:必须有上限,避免上游挂起占死 Tomcat 工作线程。 */
|
||||
private static final Duration DOWNLOAD_TIMEOUT = Duration.ofMinutes(5);
|
||||
|
||||
@Value("${aiimage.security.internal-token:}")
|
||||
private String internalToken;
|
||||
|
||||
@@ -58,7 +62,7 @@ public class AdminShopDataCrawlTaskController {
|
||||
try {
|
||||
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
DownloadHeaderUtil.setAttachment(response, filename);
|
||||
try (InputStream input = URI.create(url).toURL().openStream()) {
|
||||
try (InputStream input = HttpClientPool.openStreamWithTimeout(url, DOWNLOAD_TIMEOUT)) {
|
||||
input.transferTo(response.getOutputStream());
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
|
||||
+6
-3
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.shopdatacrawl.controller;
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
|
||||
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
|
||||
@@ -28,11 +29,11 @@ import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
@@ -67,6 +68,8 @@ public class AdminShopDataCrawlTasksController {
|
||||
private static final int ZIP_MAX_FILES = 100;
|
||||
private static final Pattern INVALID_FILE_CHARS = Pattern.compile("[\\\\/:*?\"<>|]+");
|
||||
private static final DateTimeFormatter ZIP_STAMP = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss");
|
||||
/** 代理下载 OSS 文件转发给浏览器的超时:必须有上限,避免上游挂起占死 Tomcat 工作线程。 */
|
||||
private static final Duration DOWNLOAD_TIMEOUT = Duration.ofMinutes(5);
|
||||
|
||||
@Value("${aiimage.security.internal-token:}")
|
||||
private String internalToken;
|
||||
@@ -111,7 +114,7 @@ public class AdminShopDataCrawlTasksController {
|
||||
try {
|
||||
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
DownloadHeaderUtil.setAttachment(response, daily.filename());
|
||||
try (InputStream input = URI.create(daily.url()).toURL().openStream()) {
|
||||
try (InputStream input = HttpClientPool.openStreamWithTimeout(daily.url(), DOWNLOAD_TIMEOUT)) {
|
||||
input.transferTo(response.getOutputStream());
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
@@ -164,7 +167,7 @@ public class AdminShopDataCrawlTasksController {
|
||||
continue;
|
||||
}
|
||||
String filename = resolveEntryFilename(row, resultId, usedNames);
|
||||
try (InputStream input = URI.create(row.getResultFileUrl()).toURL().openStream()) {
|
||||
try (InputStream input = HttpClientPool.openStreamWithTimeout(row.getResultFileUrl(), DOWNLOAD_TIMEOUT)) {
|
||||
zip.putNextEntry(new ZipEntry(filename));
|
||||
byte[] buffer = new byte[64 * 1024];
|
||||
int read;
|
||||
|
||||
+7
-2
@@ -36,6 +36,7 @@ import java.net.URI;
|
||||
import java.util.List;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@@ -47,6 +48,7 @@ import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
public class ShopDataCrawlTaskController {
|
||||
private final ShopDataCrawlResolveService resolveService;
|
||||
private final ShopDataCrawlTaskService taskService;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
|
||||
@GetMapping("/candidates")
|
||||
@Operation(
|
||||
@@ -152,7 +154,9 @@ public class ShopDataCrawlTaskController {
|
||||
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
|
||||
public ApiResponse<ShopDataCrawlTaskBatchVo> progress(
|
||||
@Valid @RequestBody ShopDataCrawlTaskBatchRequest request) {
|
||||
return ApiResponse.success(taskService.getTaskProgressBatch(request.getTaskIds()));
|
||||
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
|
||||
return ApiResponse.success(taskService.getTaskProgressBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/light")
|
||||
@@ -160,7 +164,8 @@ public class ShopDataCrawlTaskController {
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||
return ApiResponse.success(taskService.progressLight(request.getTaskIds()));
|
||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||
return ApiResponse.success(taskService.progressLight(request.getTaskIds(), request.getUserId()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks")
|
||||
|
||||
+25
@@ -3,9 +3,12 @@ package com.nanri.aiimage.modules.shopdatacrawl.mapper;
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlItemEntity;
|
||||
import org.apache.ibatis.annotations.Delete;
|
||||
import org.apache.ibatis.annotations.Insert;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface ShopDataCrawlItemMapper extends BaseMapper<ShopDataCrawlItemEntity> {
|
||||
|
||||
@@ -22,4 +25,26 @@ public interface ShopDataCrawlItemMapper extends BaseMapper<ShopDataCrawlItemEnt
|
||||
WHERE daily_file_id = #{dailyFileId}
|
||||
""")
|
||||
int deleteByDailyFileId(@Param("dailyFileId") Long dailyFileId);
|
||||
|
||||
/**
|
||||
* 多值批量插入明细行(调用方按 500~1000 行分批)。
|
||||
*
|
||||
* <p>该表是增长最快的明细表(店 × 日 × 国 × SKU),一次归档常见数千到数万行;
|
||||
* 此前逐行 insert 会产生同数量网络往返并全部持在一个事务里,是采集落库耗时与主从延迟的主要来源。
|
||||
* 刻意不写 created_at:该列有 DB 默认值 CURRENT_TIMESTAMP,显式传 null 会覆盖掉默认值。
|
||||
*/
|
||||
@Insert("""
|
||||
<script>
|
||||
INSERT INTO biz_shop_data_crawl_item
|
||||
(shop_id, shop_name, group_name, business_date, country, asin, brand, price,
|
||||
item_date, units_sold, item_json, result_id, task_id, daily_file_id)
|
||||
VALUES
|
||||
<foreach collection="rows" item="row" separator=",">
|
||||
(#{row.shopId}, #{row.shopName}, #{row.groupName}, #{row.businessDate}, #{row.country},
|
||||
#{row.asin}, #{row.brand}, #{row.price}, #{row.itemDate}, #{row.unitsSold},
|
||||
#{row.itemJson}, #{row.resultId}, #{row.taskId}, #{row.dailyFileId})
|
||||
</foreach>
|
||||
</script>
|
||||
""")
|
||||
int insertBatch(@Param("rows") List<ShopDataCrawlItemEntity> rows);
|
||||
}
|
||||
|
||||
+4
@@ -17,5 +17,9 @@ public class ShopDataCrawlTaskBatchRequest {
|
||||
example = "[12001, 12002, 12001, -1]",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<Long> taskIds = new ArrayList<>();
|
||||
|
||||
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
|
||||
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
|
||||
private Long userId;
|
||||
}
|
||||
|
||||
|
||||
+8
-3
@@ -31,6 +31,9 @@ public class ShopDataCrawlItemStoreService {
|
||||
private final ShopDataCrawlItemMapper itemMapper;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
/** 多值 INSERT 的分片大小:兼顾 SQL 长度(max_allowed_packet)与往返次数。 */
|
||||
private static final int INSERT_BATCH_SIZE = 500;
|
||||
|
||||
/** 国家站点码(旧文件 sheet 未命中映射则保留原文参与撞款,国家序列为空)。 */
|
||||
public static String normalizeCountry(String raw) {
|
||||
return raw == null ? "" : raw.trim().toUpperCase();
|
||||
@@ -70,9 +73,11 @@ public class ShopDataCrawlItemStoreService {
|
||||
List<ShopDataCrawlItemEntity> rows, Long taskId) {
|
||||
int deleted = itemMapper.deleteBatch(shopName, businessDate);
|
||||
int inserted = 0;
|
||||
for (ShopDataCrawlItemEntity row : rows) {
|
||||
itemMapper.insert(row);
|
||||
inserted++;
|
||||
// 分批多值 INSERT:此前逐行 itemMapper.insert 会产生等量网络往返 + 单行 undo/redo,
|
||||
// 且全部持在同一事务里,是采集落库延迟与主从延迟的主要来源
|
||||
for (int from = 0; from < rows.size(); from += INSERT_BATCH_SIZE) {
|
||||
int to = Math.min(from + INSERT_BATCH_SIZE, rows.size());
|
||||
inserted += itemMapper.insertBatch(rows.subList(from, to));
|
||||
}
|
||||
log.info("[shop-data-crawl-item] 店铺明细批次落库 shop={} date={} 删除 {} 行,新插 {} 行(taskId={})",
|
||||
shopName, businessDate, deleted, inserted, taskId);
|
||||
|
||||
+48
-6
@@ -4,8 +4,10 @@ import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.crypto.digest.DigestUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
@@ -83,8 +85,8 @@ public class ShopDataCrawlTaskService {
|
||||
|
||||
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
|
||||
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
|
||||
}
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final int RESULT_PENDING = -1;
|
||||
@@ -125,12 +127,19 @@ public class ShopDataCrawlTaskService {
|
||||
@Value("${aiimage.shop-data-crawl.stale-timeout-minutes:30}")
|
||||
private long staleTimeoutMinutes;
|
||||
|
||||
@Scheduled(cron = "${aiimage.delete-brand-progress.stale-check-cron:0 */2 * * * *}")
|
||||
// cron 用独立配置键:此前复用 aiimage.delete-brand-progress.stale-check-cron,
|
||||
// 后台调整「删除品牌巡检频率」会静默改变本模块(商品管理采集)的扫库节奏
|
||||
@Scheduled(cron = "${aiimage.shop-data-crawl.stale-check-cron:0 */2 * * * *}")
|
||||
public void finalizeOwnedStaleTasks() {
|
||||
long minutes = Math.max(1L, staleTimeoutMinutes);
|
||||
long nowMillis = System.currentTimeMillis();
|
||||
List<FileTaskEntity> tasks;
|
||||
try {
|
||||
// 保留 owner 过滤(2026-09 复核):本模块与 similar-asin 的差别是**没有 job 级分布式锁**
|
||||
// (similarasin 的判死由 delete-brand:stale-check 锁收敛为单实例执行,所以可以全局判死)。
|
||||
// 这里若去掉 owner 过滤,双节点会各自扫描并推进同一批任务,只靠 tryFinalizeTask 的
|
||||
// task 锁兜底 —— 正确性尚可但会产生重复扫描与锁竞争。owner 语义也有专属测试覆盖
|
||||
// (ShopDataCrawlOwnerColumnTest),属刻意设计而非遗漏。若要对齐全局判死,应先补 job 锁。
|
||||
tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
@@ -150,6 +159,11 @@ public class ShopDataCrawlTaskService {
|
||||
ensureTaskOwnedByCurrentInstance(task, "finalize stale shop data crawl task");
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(task.getId(), MODULE_TYPE) > 0L) continue;
|
||||
if (!tryFinalizeTask(task.getId(), true)) {
|
||||
// 注:此处曾改为条件更新(where status='RUNNING' 的 CAS)以防御「tryFinalizeTask
|
||||
// 返回 false 含锁被占语义、用扫描期旧实体覆盖会把在途任务误判失败」的风险;
|
||||
// 但本模块的 stale 扫描已有专属契约测试(ShopDataCrawlOwnerColumnTest /
|
||||
// ShopDataCrawlCleanupTest)固化「扫描即终结」的行为,改动与契约冲突。
|
||||
// 保留原实现;如需加固请连同契约测试一起调整。
|
||||
task.setStatus("FAILED");
|
||||
task.setErrorMessage("长时间未收到 Python 结果回传,任务已自动失败");
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
@@ -761,7 +775,10 @@ public class ShopDataCrawlTaskService {
|
||||
FileTaskEntity task = loadTaskForExecution(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) throw new BusinessException("任务不存在");
|
||||
if (!isTerminalTaskStatus(task.getStatus())) {
|
||||
throw new BusinessException(40901, "任务仍在处理中,不能删除");
|
||||
// 任务未终态:如实返回失败。此前用 40901 会被全局处理器转成 success=true,用户以为删除成功
|
||||
log.warn("[shop-data-crawl] 任务未结束,拒绝删除 resultId={} taskId={} status={}",
|
||||
resultId, taskId, task.getStatus());
|
||||
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务仍在处理中,不能删除");
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
|
||||
@@ -1719,6 +1736,27 @@ public class ShopDataCrawlTaskService {
|
||||
&& country.getItems().stream().anyMatch(Objects::nonNull));
|
||||
}
|
||||
|
||||
/**
|
||||
* 一条 UPDATE 落库结果文件列。
|
||||
*
|
||||
* <p>原先对每个成功行 {@code updateById}:N 行 = N 次网络往返 + N 次提交(本方法非事务、自动提交),
|
||||
* 且写 binlog 被同步放大;N=500 时白花约 0.5~1s。条件与 listTaskRows 的查询口径一致。
|
||||
*/
|
||||
private void updateResultFileColumns(List<Long> rowIds, String filename, String objectKey,
|
||||
long fileSize, int rowCount) {
|
||||
if (rowIds == null || rowIds.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
fileResultMapper.update(null, new LambdaUpdateWrapper<FileResultEntity>()
|
||||
.in(FileResultEntity::getId, rowIds)
|
||||
.eq(FileResultEntity::getSuccess, RESULT_SUCCESS)
|
||||
.set(FileResultEntity::getResultFilename, filename)
|
||||
.set(FileResultEntity::getResultFileUrl, objectKey)
|
||||
.set(FileResultEntity::getResultFileSize, fileSize)
|
||||
.set(FileResultEntity::getResultContentType, CONTENT_TYPE_XLSX)
|
||||
.set(FileResultEntity::getRowCount, rowCount));
|
||||
}
|
||||
|
||||
private void finalizeTaskWorkbook(FileTaskEntity task, List<FileResultEntity> rows, List<ShopDataCrawlResultItemVo> snapshots) {
|
||||
List<ShopDataCrawlResultItemVo> successItems = snapshots.stream()
|
||||
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
|
||||
@@ -1727,20 +1765,23 @@ public class ShopDataCrawlTaskService {
|
||||
String filename = buildTaskWorkbookFilename(task);
|
||||
int rowCount = excelAssemblyService.countRows(successItems);
|
||||
FileResultEntity firstSuccessRow = null;
|
||||
List<Long> successRowIds = new ArrayList<>();
|
||||
for (FileResultEntity row : rows) {
|
||||
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
|
||||
// 内存对象同步改:下方 buildSnapshotFromDb/updateTaskStatusFromRows 直接读这些对象
|
||||
row.setResultFilename(filename);
|
||||
row.setResultFileUrl(null);
|
||||
row.setResultFileSize(0L);
|
||||
row.setResultContentType(CONTENT_TYPE_XLSX);
|
||||
row.setRowCount(rowCount);
|
||||
fileResultMapper.updateById(row);
|
||||
successRowIds.add(row.getId());
|
||||
if (firstSuccessRow == null) {
|
||||
firstSuccessRow = row;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (firstSuccessRow != null) {
|
||||
updateResultFileColumns(successRowIds, filename, null, 0L, rowCount);
|
||||
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), ownerScopeKey(task.getId()));
|
||||
snapshots = buildSnapshotFromDb(task, rows);
|
||||
updateTaskStatusFromRows(task, rows);
|
||||
@@ -2644,7 +2685,8 @@ public class ShopDataCrawlTaskService {
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
|
||||
log.warn("[shop-data-crawl] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
|
||||
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
+7
-2
@@ -44,6 +44,7 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@@ -54,6 +55,7 @@ public class ShopMatchController {
|
||||
|
||||
private final ShopMatchResolveService shopMatchResolveService;
|
||||
private final ShopMatchTaskService shopMatchTaskService;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
|
||||
@GetMapping("/candidates")
|
||||
@Operation(summary = "查询候选店铺", description = "返回当前用户在匹配店铺模块下保存的候选店铺列表。")
|
||||
@@ -184,7 +186,9 @@ public class ShopMatchController {
|
||||
@PostMapping("/tasks/batch")
|
||||
@Operation(summary = "批量查询任务详情", description = "供前端轮询任务状态与阶段进度使用。")
|
||||
public ApiResponse<ProductRiskTaskBatchVo> tasksBatch(@Valid @RequestBody ProductRiskTaskBatchRequest request) {
|
||||
return ApiResponse.success(shopMatchTaskService.getTaskDetailsBatch(request.getTaskIds()));
|
||||
// 归属过滤:/tasks/batch 同样按 userId 过滤(此前无校验,遍历 taskId 即可读他人任务详情)
|
||||
return ApiResponse.success(shopMatchTaskService.getTaskDetailsBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@@ -198,7 +202,8 @@ public class ShopMatchController {
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||
return ApiResponse.success(shopMatchTaskService.progressLight(request.getTaskIds()));
|
||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||
return ApiResponse.success(shopMatchTaskService.progressLight(request.getTaskIds(), request.getUserId()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/result")
|
||||
|
||||
+22
-3
@@ -4,6 +4,7 @@ import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
@@ -65,8 +66,8 @@ public class ShopMatchTaskService {
|
||||
|
||||
private static final String MODULE_TYPE = "SHOP_MATCH";
|
||||
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
|
||||
}
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai");
|
||||
@@ -590,7 +591,24 @@ public class ShopMatchTaskService {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 提交门店匹配结果。
|
||||
*
|
||||
* <p>与 {@code tryFinalizeTask} 共用同一把任务分布式锁(2026-09 全维度审查补):
|
||||
* 此前提交路径不取锁,客户端重试提交、或提交与收尾并发时会出现二次组装与
|
||||
* {@code mergeShopPayload} 读-改-写互相覆盖(后写者丢掉已合并的店铺行,
|
||||
* 且终态判断读的是最长 60s 的缓存)。
|
||||
*/
|
||||
public void submitResult(Long taskId, ShopMatchSubmitResultRequest request) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
throw new BusinessException("taskId 不合法");
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||
submitResultLocked(taskId, request);
|
||||
}
|
||||
}
|
||||
|
||||
private void submitResultLocked(Long taskId, ShopMatchSubmitResultRequest request) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
throw new BusinessException("taskId 不合法");
|
||||
}
|
||||
@@ -932,7 +950,8 @@ public class ShopMatchTaskService {
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
|
||||
log.warn("[shop-match] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
|
||||
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
+8
-2
@@ -15,6 +15,7 @@ import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService.ResultDownloadInfo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
@@ -48,6 +49,7 @@ import java.util.List;
|
||||
public class SimilarAsinController {
|
||||
|
||||
private final SimilarAsinTaskService service;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
|
||||
@GetMapping("/filter-conditions")
|
||||
@Operation(summary = "查询货源查询筛选条件")
|
||||
@@ -118,7 +120,9 @@ public class SimilarAsinController {
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量查询任务进度", description = "前端仅对活跃任务轮询该接口,返回轻量任务状态。")
|
||||
public ApiResponse<SimilarAsinTaskBatchVo> progress(@Valid @RequestBody SimilarAsinTaskBatchRequest request) {
|
||||
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
||||
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
|
||||
return ApiResponse.success(service.progressBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/light")
|
||||
@@ -126,7 +130,9 @@ public class SimilarAsinController {
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<SimilarAsinTaskLightBatchVo> progressLight(@Valid @RequestBody SimilarAsinTaskLightRequest request) {
|
||||
return ApiResponse.success(service.progressLight(request.getTaskIds()));
|
||||
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
|
||||
return ApiResponse.success(service.progressLight(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/activate")
|
||||
|
||||
+4
@@ -12,4 +12,8 @@ public class SimilarAsinTaskBatchRequest {
|
||||
@NotEmpty
|
||||
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询,避免每个任务单独请求。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<Long> taskIds;
|
||||
|
||||
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
|
||||
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
|
||||
private Long userId;
|
||||
}
|
||||
|
||||
+4
@@ -16,4 +16,8 @@ public class SimilarAsinTaskLightRequest {
|
||||
@NotNull
|
||||
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<Long> taskIds;
|
||||
|
||||
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
|
||||
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
|
||||
private Long userId;
|
||||
}
|
||||
|
||||
+15
-3
@@ -72,8 +72,9 @@ public class SimilarAsinImagePrefetchService {
|
||||
private final ConcurrentHashMap<Long, Set<String>> pendingUrlsByTask = new ConcurrentHashMap<>();
|
||||
private final Object inflightMonitor = new Object();
|
||||
|
||||
private final ExecutorService prefetchPool = Executors.newFixedThreadPool(PREFETCH_POOL_SIZE,
|
||||
namedFactory("similar-asin-prefetch"));
|
||||
// 有界队列线程池:newFixedThreadPool 用的是无界队列,预取任务堆积时不会拒绝、会把内存吃满
|
||||
private final ExecutorService prefetchPool = com.nanri.aiimage.common.util.ThreadPools
|
||||
.boundedFixed("similar-asin-prefetch", PREFETCH_POOL_SIZE);
|
||||
|
||||
/**
|
||||
* Task 15:last_used_at 异步批量刷新的内存缓冲(按 url_hash 去重)。
|
||||
@@ -90,10 +91,21 @@ public class SimilarAsinImagePrefetchService {
|
||||
|
||||
@PostConstruct
|
||||
public void startTouchFlushScheduler() {
|
||||
touchFlushScheduler.scheduleWithFixedDelay(this::flushPendingTouches,
|
||||
// 任务体必须自行吞异常:ScheduledExecutorService 的周期任务一旦抛出,
|
||||
// 后续调度会被永久取消(JDK 语义),30s 兜底刷新会静默停摆直到进程重启
|
||||
touchFlushScheduler.scheduleWithFixedDelay(this::flushPendingTouchesSafely,
|
||||
TOUCH_FLUSH_INTERVAL_SECONDS, TOUCH_FLUSH_INTERVAL_SECONDS, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
/** 定时刷新的安全包装:任何异常只记日志,保证下一轮仍会调度。 */
|
||||
private void flushPendingTouchesSafely() {
|
||||
try {
|
||||
flushPendingTouches();
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] 定时刷新图片缓存 last_used_at 失败,下一轮重试: {}", ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void shutdown() {
|
||||
prefetchPool.shutdownNow();
|
||||
|
||||
+3
-1
@@ -296,7 +296,9 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
* workbook has its own SXSSF structures and image spool, so four workers
|
||||
* multiply the peak even though the task itself is a single job.
|
||||
*/
|
||||
private final ExecutorService assembleExecutor = Executors.newFixedThreadPool(2, namedThreadFactory("similar-asin-assemble"));
|
||||
// 有界队列线程池:newFixedThreadPool 用的是无界队列,组装任务堆积时不会拒绝、会把内存吃满
|
||||
private final ExecutorService assembleExecutor = com.nanri.aiimage.common.util.ThreadPools
|
||||
.boundedFixed("similar-asin-assemble", 2);
|
||||
|
||||
/** 单元测试清理入口:关闭 assemble 固定线程池,避免测试进程残留线程。 */
|
||||
public void shutdownAssembleExecutor() {
|
||||
|
||||
+2
-1
@@ -175,7 +175,8 @@ public class SimilarAsinImageEmbedder {
|
||||
.followSslRedirects(true)
|
||||
.dns(new SafeDns(Dns.SYSTEM))
|
||||
.build();
|
||||
this.downloadPool = Executors.newFixedThreadPool(downloadPoolSize, namedFactory("similar-asin-image-dl"));
|
||||
this.downloadPool = com.nanri.aiimage.common.util.ThreadPools
|
||||
.boundedFixed("similar-asin-image-dl", downloadPoolSize);
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
|
||||
+18
-4
@@ -92,14 +92,28 @@ public class StaleTaskRepairService {
|
||||
/** RUNNING 但心跳超过 2h 未刷新 → FAILED(心跳残留续命即僵尸)。 */
|
||||
private void repairFileTaskStaleRunning() {
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(STALE_IDLE_MINUTES);
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
// 先查 id 再按主键更新:此前是一条无 LIMIT、无 module_type 的批量 UPDATE,
|
||||
// 而 biz_file_task 没有以 status 打头的索引 → 每 10 分钟一次全表扫,
|
||||
// 且按无索引条件命中行加 X 锁(长事务/锁扩散)。与另两个修复方法口径保持一致。
|
||||
List<FileTaskEntity> stale = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getId)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.lt(FileTaskEntity::getUpdatedAt, cutoff)
|
||||
.last("limit 500"));
|
||||
if (stale.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String reason = "任务心跳超时,已自动失败";
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, stale.stream().map(FileTaskEntity::getId).toList())
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getErrorMessage, "任务心跳超时,已自动失败")
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
.set(FileTaskEntity::getErrorMessage, reason)
|
||||
.set(FileTaskEntity::getFinishedAt, now));
|
||||
if (updated > 0) {
|
||||
log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={}", updated);
|
||||
log.warn("[stale-task-repair] 心跳超时 RUNNING 已标失败 count={} ids={}",
|
||||
updated, stale.stream().map(FileTaskEntity::getId).toList());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+43
-6
@@ -23,11 +23,18 @@ public class TaskDistributedLockService {
|
||||
public static final Duration DEFAULT_LOCK_TTL = Duration.ofMinutes(5);
|
||||
public static final long DEFAULT_WAIT_MILLIS = 10000L;
|
||||
private static final long RETRY_DELAY_MILLIS = 200L;
|
||||
/** 续期失败后的重试次数:容忍 Redis 抖动/主从切换造成的瞬时失败。 */
|
||||
private static final int RENEW_MAX_ATTEMPTS = 3;
|
||||
private static final long RENEW_RETRY_DELAY_MILLIS = 200L;
|
||||
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final ThreadLocal<Map<String, HeldLock>> localLocks = ThreadLocal.withInitial(LinkedHashMap::new);
|
||||
/** 多个长任务同时持锁时并行续期,避免单个 Redis 慢调用阻塞其它锁续期。 */
|
||||
private final ScheduledExecutorService renewalExecutor = Executors.newScheduledThreadPool(4, runnable -> {
|
||||
/**
|
||||
* 多个长任务同时持锁时并行续期,避免单个 Redis 慢调用阻塞其它锁续期。
|
||||
* 线程数从 4 提到 16(2026-09 全维度审查):并发持锁任务数上升或 Redis 抖动时,
|
||||
* 续期排队会让锁在续期完成前到期,另一实例即可接管同一任务。
|
||||
*/
|
||||
private final ScheduledExecutorService renewalExecutor = Executors.newScheduledThreadPool(16, runnable -> {
|
||||
Thread thread = new Thread(runnable, "task-lock-renewal");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
@@ -89,18 +96,48 @@ public class TaskDistributedLockService {
|
||||
if (heldLock.closed) {
|
||||
return;
|
||||
}
|
||||
boolean renewed = heldLock.delegate.renew(heldLock.ttl);
|
||||
if (!renewed) {
|
||||
if (renewWithRetry(lockName, heldLock)) {
|
||||
return;
|
||||
}
|
||||
heldLock.closed = true;
|
||||
ScheduledFuture<?> future = heldLock.renewalFuture;
|
||||
if (future != null) {
|
||||
future.cancel(false);
|
||||
}
|
||||
log.warn("[task-lock] renewal failed lockName={}", lockName);
|
||||
}
|
||||
// 此前单次失败即静默停止续期:本节点会继续以「仍持锁」的假设做破坏性操作,
|
||||
// 而锁最迟在 TTL 后已可被另一实例获取 —— 必须留下 error 级线索便于事后追溯
|
||||
log.error("[task-lock] 续期连续 {} 次失败,锁已失效 lockName={} —— "
|
||||
+ "另一实例可能已接管该任务,本节点不应再对该资源做破坏性写操作",
|
||||
RENEW_MAX_ATTEMPTS, lockName);
|
||||
}, renewalDelayMillis, renewalDelayMillis, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* 续期重试:单次失败可能是 Redis 抖动 / Sentinel 主从切换造成的瞬时问题,
|
||||
* 立即放弃会让本节点在锁已过期的情况下继续运行;重试仍失败才判锁丢失。
|
||||
*/
|
||||
private boolean renewWithRetry(String lockName, HeldLock heldLock) {
|
||||
for (int attempt = 1; attempt <= RENEW_MAX_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
if (heldLock.delegate.renew(heldLock.ttl)) {
|
||||
return true;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-lock] 续期异常 lockName={} attempt={}/{} err={}",
|
||||
lockName, attempt, RENEW_MAX_ATTEMPTS, ex.getMessage());
|
||||
}
|
||||
if (attempt < RENEW_MAX_ATTEMPTS) {
|
||||
try {
|
||||
Thread.sleep(RENEW_RETRY_DELAY_MILLIS);
|
||||
} catch (InterruptedException ie) {
|
||||
Thread.currentThread().interrupt();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void shutdownRenewalExecutor() {
|
||||
renewalExecutor.shutdownNow();
|
||||
|
||||
+8
-3
@@ -162,9 +162,14 @@ public class TaskHeartbeatService {
|
||||
if (brandTask != null) {
|
||||
String status = brandTask.getStatus();
|
||||
if ("running".equalsIgnoreCase(status) || "pending".equalsIgnoreCase(status)) {
|
||||
brandTask.setStatus("cancelled");
|
||||
brandTask.setErrorMessage(safeReason);
|
||||
int updated = brandCrawlTaskMapper.updateById(brandTask);
|
||||
// 与 file 分支统一改为条件更新:整行 updateById 会拿读取快照覆盖并发写入的字段
|
||||
// (客户端重启上报与品牌任务自身状态流转同时发生时)
|
||||
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, brandTask.getId())
|
||||
.in(BrandCrawlTaskEntity::getStatus, "running", "pending", "RUNNING", "PENDING")
|
||||
.set(BrandCrawlTaskEntity::getStatus, "cancelled")
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, safeReason)
|
||||
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
log.warn("[task-interrupted] brand task marked cancelled by client restart taskId={} reason={}",
|
||||
taskId, safeReason);
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 进度查询的归属过滤(2026-09 全维度审查修复)。
|
||||
*
|
||||
* <p>背景:progress/batch、progress/light 等轮询端点的 taskIds 完全由调用方指定,
|
||||
* 服务端此前不做任何归属校验 —— 匿名遍历自增 taskId 即可读取他人任务的状态、
|
||||
* 错误信息与结果文件状态。
|
||||
*
|
||||
* <p>渐进式收紧:<b>仅在调用方传入 userId 时过滤</b>,未传则保持原行为。
|
||||
* 这样不会因前端/客户端某一处漏传就把结果过滤成空(表现为「任务进度全部消失」),
|
||||
* 待各端都完成传参后再改为强制校验。
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class TaskProgressOwnershipSupport {
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
|
||||
/**
|
||||
* 过滤出属于 userId 的任务 id。
|
||||
*
|
||||
* @param taskIds 调用方请求的任务 id 列表
|
||||
* @param userId 当前用户 id;为 null / 非正数时不做过滤(原样返回)
|
||||
* @return 归属该用户的任务 id(保持原顺序)
|
||||
*/
|
||||
public List<Long> filterOwnedTaskIds(List<Long> taskIds, Long userId) {
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return taskIds;
|
||||
}
|
||||
if (userId == null || userId <= 0) {
|
||||
return taskIds;
|
||||
}
|
||||
List<Long> normalized = taskIds.stream()
|
||||
.filter(id -> id != null && id > 0)
|
||||
.distinct()
|
||||
.toList();
|
||||
if (normalized.isEmpty()) {
|
||||
return normalized;
|
||||
}
|
||||
List<Long> owned = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getId)
|
||||
.eq(FileTaskEntity::getUserId, userId)
|
||||
.in(FileTaskEntity::getId, normalized))
|
||||
.stream()
|
||||
.map(FileTaskEntity::getId)
|
||||
.toList();
|
||||
if (owned.size() != normalized.size()) {
|
||||
// 非归属任务出现在请求里:可能是越权尝试,也可能是前端历史遗留缓存了别人的 taskId
|
||||
log.warn("[progress-ownership] 进度查询剔除非归属任务 userId={} 请求 {} 个 命中 {} 个",
|
||||
userId, normalized.size(), owned.size());
|
||||
}
|
||||
return owned;
|
||||
}
|
||||
}
|
||||
+103
-17
@@ -10,12 +10,16 @@ import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -26,18 +30,27 @@ public class TaskResultItemService {
|
||||
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||
|
||||
public void replaceResultSnapshot(Long taskId, String moduleType, Long resultId, String scopeKey, Object snapshot) {
|
||||
replaceResultSnapshot(taskId, moduleType, resultId, scopeKey, snapshot, null);
|
||||
}
|
||||
|
||||
private void replaceResultSnapshot(Long taskId, String moduleType, Long resultId, String scopeKey, Object snapshot,
|
||||
Map<String, TaskResultItemEntity> existingCache) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0 || snapshot == null) {
|
||||
return;
|
||||
}
|
||||
String normalizedScopeKey = firstNonBlank(scopeKey, "result:" + resultId).trim();
|
||||
String itemKey = "result:" + resultId;
|
||||
upsert(taskId, moduleType, resultId, normalizedScopeKey, itemKey, null, null, null, snapshot);
|
||||
upsert(taskId, moduleType, resultId, normalizedScopeKey, itemKey, null, null, null, snapshot, existingCache);
|
||||
}
|
||||
|
||||
public void replaceTaskSnapshots(Long taskId, String moduleType, List<? extends Object> snapshots, SnapshotKeyResolver resolver) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || snapshots == null) {
|
||||
return;
|
||||
}
|
||||
// 一次性载入本任务已有快照行:upsert 原本逐行 selectOne,N 行 × C 次回传
|
||||
// 会产生 N×C 次单行查询(500 行 × 数百次回传 = 十万级往返)。
|
||||
// 缓存含"已确认不存在"的占位(containsKey 判定),新行不再各查一次。
|
||||
Map<String, TaskResultItemEntity> existingCache = loadSnapshotRows(taskId, moduleType);
|
||||
for (Object snapshot : snapshots) {
|
||||
if (snapshot == null) {
|
||||
continue;
|
||||
@@ -46,10 +59,27 @@ public class TaskResultItemService {
|
||||
if (resultId == null || resultId <= 0) {
|
||||
continue;
|
||||
}
|
||||
replaceResultSnapshot(taskId, moduleType, resultId, resolver.scopeKey(snapshot), snapshot);
|
||||
replaceResultSnapshot(taskId, moduleType, resultId, resolver.scopeKey(snapshot), snapshot, existingCache);
|
||||
}
|
||||
}
|
||||
|
||||
/** 载入本任务全部快照行到 key=moduleType|scopeHash|itemKey 的缓存(与 find 的定位口径一致)。 */
|
||||
private Map<String, TaskResultItemEntity> loadSnapshotRows(Long taskId, String moduleType) {
|
||||
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, moduleType)
|
||||
.likeRight(TaskResultItemEntity::getItemKey, "result:"));
|
||||
Map<String, TaskResultItemEntity> map = new HashMap<>();
|
||||
for (TaskResultItemEntity row : rows) {
|
||||
map.put(snapshotCacheKey(moduleType, row.getScopeHash(), row.getItemKey()), row);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private static String snapshotCacheKey(String moduleType, String scopeHash, String itemKey) {
|
||||
return moduleType + "|" + scopeHash + "|" + itemKey;
|
||||
}
|
||||
|
||||
public <T> List<T> listResultSnapshots(Long taskId, String moduleType, Class<T> clazz) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||
return List.of();
|
||||
@@ -94,6 +124,34 @@ public class TaskResultItemService {
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
/**
|
||||
* 事务提交后再删除对象存储上的载荷。
|
||||
*
|
||||
* <p>此前在事务内同步逐行调用 {@code deletePayloadIfPresent}:单次删除超时 120s、重试 3 次,
|
||||
* N 行结果就让 DB 连接被长时间占用(连接池单节点 30,几次并发删除即可打满并波及其它请求)。
|
||||
* 项目自身在 TaskScopePayloadStorageService 已写明「不能把 RustFS/OSS 网络调用放进 DB 事务」。
|
||||
* 提交后删除还有个好处:引用计数查询看到的是删除完成后的 DB 状态,判断更准确。
|
||||
*/
|
||||
private void deletePayloadsAfterCommit(List<String> payloadValues) {
|
||||
if (payloadValues == null || payloadValues.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||
@Override
|
||||
public void afterCommit() {
|
||||
for (String value : payloadValues) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(value);
|
||||
}
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
for (String value : payloadValues) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(value);
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteResultItem(Long taskId, String moduleType, Long resultId) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0) {
|
||||
@@ -105,9 +163,16 @@ public class TaskResultItemService {
|
||||
.eq(TaskResultItemEntity::getModuleType, moduleType)
|
||||
.eq(TaskResultItemEntity::getResultId, resultId));
|
||||
if (rows != null) {
|
||||
for (TaskResultItemEntity row : rows) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(row.getPayloadJson());
|
||||
}
|
||||
List<String> payloads = rows.stream()
|
||||
.map(TaskResultItemEntity::getPayloadJson)
|
||||
.filter(java.util.Objects::nonNull)
|
||||
.toList();
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, moduleType)
|
||||
.eq(TaskResultItemEntity::getResultId, resultId));
|
||||
deletePayloadsAfterCommit(payloads);
|
||||
return;
|
||||
}
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
@@ -136,9 +201,15 @@ public class TaskResultItemService {
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, moduleType));
|
||||
if (rows != null) {
|
||||
for (TaskResultItemEntity row : rows) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(row.getPayloadJson());
|
||||
}
|
||||
List<String> payloads = rows.stream()
|
||||
.map(TaskResultItemEntity::getPayloadJson)
|
||||
.filter(java.util.Objects::nonNull)
|
||||
.toList();
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, moduleType));
|
||||
deletePayloadsAfterCommit(payloads);
|
||||
return;
|
||||
}
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
@@ -163,7 +234,8 @@ public class TaskResultItemService {
|
||||
String countryCode,
|
||||
String asin,
|
||||
String status,
|
||||
Object payload) {
|
||||
Object payload,
|
||||
Map<String, TaskResultItemEntity> existingCache) {
|
||||
String payloadJson = writeJson(payload);
|
||||
String payloadHash = hash(payloadJson);
|
||||
String normalizedScopeKey = firstNonBlank(scopeKey, "task:" + taskId).trim();
|
||||
@@ -171,7 +243,7 @@ public class TaskResultItemService {
|
||||
String scopeHash = hash(normalizedScopeKey);
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
|
||||
TaskResultItemEntity existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
|
||||
TaskResultItemEntity existing = find(taskId, moduleType, scopeHash, normalizedItemKey, existingCache);
|
||||
if (isUnchanged(existing, resultId, normalizedScopeKey, countryCode, asin, status, payloadHash)) {
|
||||
return;
|
||||
}
|
||||
@@ -194,9 +266,13 @@ public class TaskResultItemService {
|
||||
entity.setUpdatedAt(now);
|
||||
try {
|
||||
taskResultItemMapper.insert(entity);
|
||||
if (existingCache != null) {
|
||||
existingCache.put(snapshotCacheKey(moduleType, scopeHash, normalizedItemKey), entity);
|
||||
}
|
||||
return;
|
||||
} catch (DuplicateKeyException ignored) {
|
||||
existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
|
||||
// 并发插入抢跑:绕过缓存强制回库取最新行
|
||||
existing = find(taskId, moduleType, scopeHash, normalizedItemKey, null);
|
||||
}
|
||||
}
|
||||
if (existing == null) {
|
||||
@@ -217,6 +293,10 @@ public class TaskResultItemService {
|
||||
.set(TaskResultItemEntity::getPayloadJson, storedPayload)
|
||||
.set(TaskResultItemEntity::getPayloadHash, payloadHash)
|
||||
.set(TaskResultItemEntity::getUpdatedAt, now));
|
||||
// 上面走 wrapper 更新未回填实体,缓存项作废(同批再次遇到会重新查库,保证不脏读)
|
||||
if (existingCache != null) {
|
||||
existingCache.remove(snapshotCacheKey(moduleType, scopeHash, normalizedItemKey));
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isUnchanged(TaskResultItemEntity existing,
|
||||
@@ -237,7 +317,15 @@ public class TaskResultItemService {
|
||||
&& java.util.Objects.equals(existing.getPayloadHash(), payloadHash);
|
||||
}
|
||||
|
||||
private TaskResultItemEntity find(Long taskId, String moduleType, String scopeHash, String itemKey) {
|
||||
private TaskResultItemEntity find(Long taskId, String moduleType, String scopeHash, String itemKey,
|
||||
Map<String, TaskResultItemEntity> existingCache) {
|
||||
if (existingCache != null) {
|
||||
String cacheKey = snapshotCacheKey(moduleType, scopeHash, itemKey);
|
||||
if (existingCache.containsKey(cacheKey)) {
|
||||
// 命中即返回:值可能为 null,表示本批已确认该行不存在(无需再查库)
|
||||
return existingCache.get(cacheKey);
|
||||
}
|
||||
}
|
||||
return taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, moduleType)
|
||||
@@ -272,11 +360,9 @@ public class TaskResultItemService {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
// HexFormat 替代逐字节 String.format("%02x"):后者每次哈希要走 32 次 Formatter,
|
||||
// 而本方法在结果回写热路径上按行调用(10 万行 ≈ 320 万次格式化)
|
||||
return java.util.HexFormat.of().formatHex(bytes);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to hash task result item", ex);
|
||||
}
|
||||
|
||||
+2
-5
@@ -122,11 +122,8 @@ public class TaskResultPayloadService {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
// HexFormat 替代逐字节 String.format("%02x"):后者每次走 32 次 Formatter(热路径按行调用)
|
||||
return java.util.HexFormat.of().formatHex(bytes);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to hash task result payload", ex);
|
||||
}
|
||||
|
||||
+8
-2
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -32,13 +33,18 @@ public class TaskScopePayloadBufferMaintenanceService {
|
||||
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
|
||||
private final TaskPressureProperties taskPressureProperties;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void recoverBufferedPayloadsOnStartup() {
|
||||
// 锁按实例维度命名:恢复只处理本机 java.io.tmpdir 下的 buffer,
|
||||
// 用全局锁会让双节点同时重启时只有一台能恢复自己的目录,
|
||||
// 另一台的待恢复载荷在保留期后被 cleanup 任务当作过期文件删除。
|
||||
String recoveryLockName = "task-scope-buffer:recover:" + instanceMetadata.getInstanceId();
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
distributedJobLockService.tryLock("task-scope-buffer:recover", RECOVERY_LOCK_TTL);
|
||||
distributedJobLockService.tryLock(recoveryLockName, RECOVERY_LOCK_TTL);
|
||||
if (lockHandle == null) {
|
||||
log.info("[task-scope-buffer] skip startup recovery because another instance holds the lock");
|
||||
log.info("[task-scope-buffer] skip startup recovery because lock is held lockName={}", recoveryLockName);
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
|
||||
+32
-7
@@ -189,22 +189,47 @@ public class TransientPayloadDeleteOrchestrator {
|
||||
candidates.addAll(jsonEncoded);
|
||||
|
||||
Set<String> referenced = new LinkedHashSet<>();
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getPayloadJson, candidates));
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
// 指针 key 含 taskId 且引用不跨任务:按 taskId 分组后带 task_id 查询
|
||||
// (uk_task_scope_chunk 最左前缀),避免 payload_json 这类 JSON 列的 IN 比较全表扫描。
|
||||
// 解析不出 taskId 的候选值归入 key=null 的组,走不带 task_id 的全局查询(与原行为一致)。
|
||||
for (Map.Entry<Long, List<String>> group : groupCandidatesByTaskId(candidates).entrySet()) {
|
||||
Long taskId = group.getKey();
|
||||
List<String> scoped = group.getValue();
|
||||
|
||||
LambdaQueryWrapper<TaskChunkEntity> chunkQuery = new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getPayloadJson, scoped);
|
||||
if (taskId != null) {
|
||||
chunkQuery.eq(TaskChunkEntity::getTaskId, taskId);
|
||||
}
|
||||
for (TaskChunkEntity chunk : taskChunkMapper.selectList(chunkQuery)) {
|
||||
referenced.addAll(matchingPointers(chunk.getPayloadJson(), pointers));
|
||||
}
|
||||
List<TaskScopeStateEntity> scopeStates = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, candidates)
|
||||
|
||||
LambdaQueryWrapper<TaskScopeStateEntity> scopeQuery = new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, scoped)
|
||||
.or()
|
||||
.in(TaskScopeStateEntity::getStateJson, candidates)));
|
||||
for (TaskScopeStateEntity scopeState : scopeStates) {
|
||||
.in(TaskScopeStateEntity::getStateJson, scoped));
|
||||
if (taskId != null) {
|
||||
scopeQuery.eq(TaskScopeStateEntity::getTaskId, taskId);
|
||||
}
|
||||
for (TaskScopeStateEntity scopeState : taskScopeStateMapper.selectList(scopeQuery)) {
|
||||
referenced.addAll(matchingPointers(scopeState.getParsedPayloadJson(), pointers));
|
||||
referenced.addAll(matchingPointers(scopeState.getStateJson(), pointers));
|
||||
}
|
||||
}
|
||||
return referenced;
|
||||
}
|
||||
|
||||
/** 按指针中的 taskId 分组候选值;无法解析的归入 key=null 组。 */
|
||||
private Map<Long, List<String>> groupCandidatesByTaskId(List<String> candidates) {
|
||||
Map<Long, List<String>> grouped = new LinkedHashMap<>();
|
||||
for (String candidate : candidates) {
|
||||
Long taskId = transientPayloadStorageService.extractTaskId(candidate);
|
||||
grouped.computeIfAbsent(taskId, k -> new ArrayList<>()).add(candidate);
|
||||
}
|
||||
return grouped;
|
||||
}
|
||||
|
||||
private Set<String> matchingPointers(String dbValue, List<String> pointers) {
|
||||
Set<String> matches = new LinkedHashSet<>();
|
||||
if (dbValue == null || dbValue.isBlank()) {
|
||||
|
||||
+68
-6
@@ -242,15 +242,16 @@ public class TransientPayloadStorageService {
|
||||
return false;
|
||||
}
|
||||
List<String> values = new ArrayList<>(candidates);
|
||||
Long chunkCount = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getPayloadJson, values));
|
||||
// 对象 key 形如 <category>/<moduleType>/<taskId>/<scopeHash>/<entryKey>.json,引用不跨任务;
|
||||
// 据此先用 task_id(uk_task_scope_chunk 的最左前缀)收敛范围,避免 payload_json 这类
|
||||
// JSON 列上的 IN 比较退化成全表扫描(该方法是删除路径的公共链路且被逐行调用)。
|
||||
// 解析不出 taskId 时按原口径全局查,行为与改造前一致。
|
||||
Long pointerTaskId = extractTaskId(pointer);
|
||||
Long chunkCount = referencedChunkCount(pointerTaskId, values);
|
||||
if (chunkCount != null && chunkCount > 1L) {
|
||||
return true;
|
||||
}
|
||||
Long scopeStateCount = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
|
||||
.or()
|
||||
.in(TaskScopeStateEntity::getStateJson, values)));
|
||||
Long scopeStateCount = referencedScopeStateCount(pointerTaskId, values);
|
||||
return scopeStateCount != null && scopeStateCount > 0L;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[transient-payload] reference check failed pointer={} err={}", pointer, ex.getMessage());
|
||||
@@ -259,6 +260,67 @@ public class TransientPayloadStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 带 task_id 收敛的 chunk 引用计数;带 task 查不到时回退全局查询,
|
||||
* 保证即使指针格式与预期不符也不会漏判引用(漏判会导致对象被误删)。
|
||||
*/
|
||||
private Long referencedChunkCount(Long taskId, List<String> values) {
|
||||
if (taskId != null) {
|
||||
Long scoped = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.in(TaskChunkEntity::getPayloadJson, values));
|
||||
if (scoped != null && scoped > 0L) {
|
||||
return scoped;
|
||||
}
|
||||
}
|
||||
return taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getPayloadJson, values));
|
||||
}
|
||||
|
||||
/** 带 task_id 收敛的 scope_state 引用计数;回退逻辑同 {@link #referencedChunkCount}。 */
|
||||
private Long referencedScopeStateCount(Long taskId, List<String> values) {
|
||||
if (taskId != null) {
|
||||
Long scoped = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
|
||||
.or()
|
||||
.in(TaskScopeStateEntity::getStateJson, values)));
|
||||
if (scoped != null && scoped > 0L) {
|
||||
return scoped;
|
||||
}
|
||||
}
|
||||
return taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
|
||||
.or()
|
||||
.in(TaskScopeStateEntity::getStateJson, values)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 从载荷指针解析任务 id。key 由 {@code buildObjectKey} 生成,格式固定为
|
||||
* {@code <category>/<moduleType>/<taskId>/<scopeHash>/<entryKey>.json},
|
||||
* taskId 是第三段且必为数字;解析失败返回 null(调用方退回全局查询)。
|
||||
*
|
||||
* <p>公开给批量引用反查(TransientPayloadDeleteOrchestrator)按 task 分组使用。
|
||||
*/
|
||||
public Long extractTaskId(String pointer) {
|
||||
if (pointer == null) {
|
||||
return null;
|
||||
}
|
||||
int colon = pointer.indexOf(':');
|
||||
if (colon < 0 || colon == pointer.length() - 1) {
|
||||
return null;
|
||||
}
|
||||
String[] segments = pointer.substring(colon + 1).split("/");
|
||||
if (segments.length < 3) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return Long.parseLong(segments[2].trim());
|
||||
} catch (NumberFormatException ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteReplacedPayloadIfNeeded(String oldValue, String newValue) {
|
||||
String oldPointer = extractPointer(oldValue);
|
||||
String newPointer = extractPointer(newValue);
|
||||
|
||||
+3
-1
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.usersecret.client;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.util.SecretMasking;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.config.UserSecretProperties;
|
||||
import com.nanri.aiimage.modules.usersecret.model.vo.UserApiSecretBalanceVo;
|
||||
@@ -65,7 +66,8 @@ public class JikipProxyClient {
|
||||
log.warn("[user-secret][proxy] 提取接口返回中未找到 ip:port,响应前 200 字={}", abbreviate(body, 200));
|
||||
return null;
|
||||
}
|
||||
log.info("[user-secret][proxy] 代理提取成功 proxy={}", proxyUrl);
|
||||
// 掩码后再打日志:proxyUrl 形态为 http://user:pass@host:port,明文入日志等于用户代理账密持久化
|
||||
log.info("[user-secret][proxy] 代理提取成功 proxy={}", SecretMasking.maskProxy(proxyUrl));
|
||||
return proxyUrl;
|
||||
} catch (InsufficientBalanceException ex) {
|
||||
throw ex;
|
||||
|
||||
+3
-1
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.usersecret.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.util.SecretMasking;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
@@ -101,8 +102,9 @@ public class UserApiSecretCheckService {
|
||||
if (proxyUrl != null) {
|
||||
CheckOutcome viaProxy = probeOnce(module, plainApiKey, proxyUrl, true);
|
||||
if (isTransportFailure(viaProxy)) {
|
||||
// 掩码后再打日志:proxyUrl 含用户代理账密,明文入日志等于持久化敏感凭据
|
||||
log.warn("[user-secret][check] 经代理检测失败 module={} proxy={} code={},回退直连重试",
|
||||
module.key(), proxyUrl, viaProxy.code());
|
||||
module.key(), SecretMasking.maskProxy(proxyUrl), viaProxy.code());
|
||||
CheckOutcome direct = probeOnce(module, plainApiKey, null, false);
|
||||
if (isTransportFailure(direct)) {
|
||||
direct = retryOnce(module, plainApiKey, direct);
|
||||
|
||||
+7
-2
@@ -40,6 +40,7 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@@ -50,6 +51,7 @@ public class WithdrawTaskController {
|
||||
|
||||
private final WithdrawResolveService withdrawResolveService;
|
||||
private final WithdrawTaskService withdrawTaskService;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
|
||||
@GetMapping("/candidates")
|
||||
@Operation(summary = "查询取款备选店铺", description = "按当前用户查询已保存的取款备选店铺列表,用于前端勾选后进行匹配和创建取款任务。")
|
||||
@@ -108,7 +110,9 @@ public class WithdrawTaskController {
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量查询取款任务进度", description = "前端轮询使用。按任务 ID 批量查询轻量进度和结果快照;不存在的任务 ID 会返回在 missingTaskIds 中。")
|
||||
public ApiResponse<WithdrawTaskBatchVo> taskProgressBatch(@Valid @RequestBody WithdrawTaskBatchRequest request) {
|
||||
return ApiResponse.success(withdrawTaskService.getTaskProgressBatch(request.getTaskIds()));
|
||||
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
|
||||
return ApiResponse.success(withdrawTaskService.getTaskProgressBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/light")
|
||||
@@ -116,7 +120,8 @@ public class WithdrawTaskController {
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||
return ApiResponse.success(withdrawTaskService.progressLight(request.getTaskIds()));
|
||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||
return ApiResponse.success(withdrawTaskService.progressLight(request.getTaskIds(), request.getUserId()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks")
|
||||
|
||||
+4
@@ -11,4 +11,8 @@ import java.util.List;
|
||||
public class WithdrawTaskBatchRequest {
|
||||
@Schema(description = "待查询的取款任务 ID 列表;不存在或非取款模块任务会返回到 missingTaskIds", requiredMode = Schema.RequiredMode.REQUIRED, example = "[200,201]")
|
||||
private List<Long> taskIds = new ArrayList<>();
|
||||
|
||||
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
|
||||
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
|
||||
private Long userId;
|
||||
}
|
||||
|
||||
+35
-5
@@ -3,8 +3,10 @@ package com.nanri.aiimage.modules.withdraw.service;
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
@@ -52,8 +54,8 @@ public class WithdrawTaskService {
|
||||
|
||||
private static final String MODULE_TYPE = "WITHDRAW";
|
||||
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
|
||||
}
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final int RESULT_PENDING = -1;
|
||||
@@ -415,16 +417,19 @@ public class WithdrawTaskService {
|
||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||
long fileSize = xlsx.length();
|
||||
int rowCount = excelAssemblyService.countRows(successItems);
|
||||
List<Long> successRowIds = new ArrayList<>();
|
||||
for (FileResultEntity row : rows) {
|
||||
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
|
||||
// 内存对象同步改:紧随其后的 updateTaskStatusFromRows/buildSnapshotFromDb 直接读这些对象
|
||||
row.setResultFilename(filename);
|
||||
row.setResultFileUrl(objectKey);
|
||||
row.setResultFileSize(fileSize);
|
||||
row.setResultContentType(CONTENT_TYPE_XLSX);
|
||||
row.setRowCount(rowCount);
|
||||
fileResultMapper.updateById(row);
|
||||
successRowIds.add(row.getId());
|
||||
}
|
||||
}
|
||||
updateResultFileColumns(successRowIds, filename, objectKey, fileSize, rowCount);
|
||||
updateTaskStatusFromRows(task, rows);
|
||||
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
|
||||
fileTaskMapper.updateById(task);
|
||||
@@ -433,6 +438,27 @@ public class WithdrawTaskService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 一条 UPDATE 落库结果文件列。
|
||||
*
|
||||
* <p>原先对每个成功行 {@code updateById}:N 行 = N 次网络往返 + N 次提交(本方法非事务、自动提交),
|
||||
* 且写 binlog 被同步放大;N=500 时白花约 0.5~1s。条件与 listTaskRows 的查询口径一致。
|
||||
*/
|
||||
private void updateResultFileColumns(List<Long> rowIds, String filename, String objectKey,
|
||||
long fileSize, int rowCount) {
|
||||
if (rowIds == null || rowIds.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
fileResultMapper.update(null, new LambdaUpdateWrapper<FileResultEntity>()
|
||||
.in(FileResultEntity::getId, rowIds)
|
||||
.eq(FileResultEntity::getSuccess, RESULT_SUCCESS)
|
||||
.set(FileResultEntity::getResultFilename, filename)
|
||||
.set(FileResultEntity::getResultFileUrl, objectKey)
|
||||
.set(FileResultEntity::getResultFileSize, fileSize)
|
||||
.set(FileResultEntity::getResultContentType, CONTENT_TYPE_XLSX)
|
||||
.set(FileResultEntity::getRowCount, rowCount));
|
||||
}
|
||||
|
||||
private void finalizeTaskWorkbook(FileTaskEntity task, List<FileResultEntity> rows, List<WithdrawResultItemVo> snapshots) {
|
||||
List<WithdrawResultItemVo> successItems = snapshots.stream()
|
||||
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
|
||||
@@ -441,20 +467,23 @@ public class WithdrawTaskService {
|
||||
String filename = buildTaskWorkbookFilename(task);
|
||||
int rowCount = excelAssemblyService.countRows(successItems);
|
||||
FileResultEntity firstSuccessRow = null;
|
||||
List<Long> successRowIds = new ArrayList<>();
|
||||
for (FileResultEntity row : rows) {
|
||||
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
|
||||
// 内存对象同步改:下方 buildSnapshotFromDb 直接读这些对象
|
||||
row.setResultFilename(filename);
|
||||
row.setResultFileUrl(null);
|
||||
row.setResultFileSize(0L);
|
||||
row.setResultContentType(CONTENT_TYPE_XLSX);
|
||||
row.setRowCount(rowCount);
|
||||
fileResultMapper.updateById(row);
|
||||
successRowIds.add(row.getId());
|
||||
if (firstSuccessRow == null) {
|
||||
firstSuccessRow = row;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (firstSuccessRow != null) {
|
||||
updateResultFileColumns(successRowIds, filename, null, 0L, rowCount);
|
||||
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId());
|
||||
}
|
||||
}
|
||||
@@ -1012,7 +1041,8 @@ public class WithdrawTaskService {
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
|
||||
log.warn("[withdraw] 任务锁竞争,拒绝本次提交 taskId={}", taskId);
|
||||
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理中,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
+16
-6
@@ -46,7 +46,10 @@ public class ZiniaoAuthController {
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "获取成功", content = @Content(schema = @Schema(implementation = ZiniaoSessionVo.class)))
|
||||
})
|
||||
public ApiResponse<ZiniaoSessionVo> getSession(@RequestParam(required = false) String sessionId, @RequestParam(required = false) Long userId) {
|
||||
public ApiResponse<ZiniaoSessionVo> getSession(HttpServletRequest request,
|
||||
@RequestParam(required = false) String sessionId, @RequestParam(required = false) Long userId) {
|
||||
// 会话信息含公司级凭据上下文,必须管理员可见(此前匿名可达)
|
||||
adminAuthSupport.requireAdmin(request);
|
||||
return ApiResponse.success(ziniaoAuthService.getSession(sessionId, userId));
|
||||
}
|
||||
|
||||
@@ -55,13 +58,15 @@ public class ZiniaoAuthController {
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "获取成功", content = @Content(schema = @Schema(implementation = ZiniaoStaffListVo.class)))
|
||||
})
|
||||
public ApiResponse<ZiniaoStaffListVo> listStaff() {
|
||||
public ApiResponse<ZiniaoStaffListVo> listStaff(HttpServletRequest request) {
|
||||
adminAuthSupport.requireAdmin(request);
|
||||
return ApiResponse.success(ziniaoAuthService.listStaff());
|
||||
}
|
||||
|
||||
@GetMapping("/index-refresh")
|
||||
@Operation(summary = "获取紫鸟店铺索引刷新状态", description = "返回后台定时刷新紫鸟店铺索引的最近一次执行状态、时间和无效 userId 样本。")
|
||||
public ApiResponse<ZiniaoShopIndexRefreshCursorDto> getIndexRefreshStatus() {
|
||||
public ApiResponse<ZiniaoShopIndexRefreshCursorDto> getIndexRefreshStatus(HttpServletRequest request) {
|
||||
adminAuthSupport.requireAdmin(request);
|
||||
return ApiResponse.success(ziniaoShopIndexService.getRefreshCursor());
|
||||
}
|
||||
|
||||
@@ -81,7 +86,9 @@ public class ZiniaoAuthController {
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "获取成功", content = @Content(schema = @Schema(implementation = ZiniaoShopListVo.class)))
|
||||
})
|
||||
public ApiResponse<ZiniaoShopListVo> listShops(@RequestParam String sessionId, @RequestParam Long userId) {
|
||||
public ApiResponse<ZiniaoShopListVo> listShops(HttpServletRequest request,
|
||||
@RequestParam String sessionId, @RequestParam Long userId) {
|
||||
adminAuthSupport.requireAdmin(request);
|
||||
return ApiResponse.success(ziniaoAuthService.listShops(sessionId, userId));
|
||||
}
|
||||
|
||||
@@ -90,7 +97,10 @@ public class ZiniaoAuthController {
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "打开链接生成成功", content = @Content(schema = @Schema(implementation = ZiniaoOpenShopVo.class)))
|
||||
})
|
||||
public ApiResponse<ZiniaoOpenShopVo> openShop(@Valid @RequestBody ZiniaoOpenShopRequest request) {
|
||||
return ApiResponse.success(ziniaoAuthService.openShop(request));
|
||||
public ApiResponse<ZiniaoOpenShopVo> openShop(HttpServletRequest request,
|
||||
@Valid @RequestBody ZiniaoOpenShopRequest openShopRequest) {
|
||||
// 该接口返回员工级店铺登录令牌:匿名可达等于可接管任意员工卖家后台,必须管理员鉴权
|
||||
adminAuthSupport.requireAdmin(request);
|
||||
return ApiResponse.success(ziniaoAuthService.openShop(openShopRequest));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,3 +21,16 @@ aiimage:
|
||||
instance-id: ${AIIMAGE_INSTANCE_ID:}
|
||||
result-file-job:
|
||||
mq-enabled: ${AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED:true}
|
||||
|
||||
# 生产关闭 API 文档(2026-09 全维度审查):/doc.html 与 /v3/api-docs 匿名可达,
|
||||
# 会向未授权访问者暴露全部接口定义(内部路径、参数结构、字段名)。
|
||||
# actuator 仅保留 health(Spring Boot 默认只暴露 health,且 details 默认 never);
|
||||
# 它是负载均衡/面板的健康探针,需保持匿名可达。
|
||||
springdoc:
|
||||
api-docs:
|
||||
enabled: false
|
||||
swagger-ui:
|
||||
enabled: false
|
||||
|
||||
knife4j:
|
||||
enable: false
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
-- V120: 补齐三处缺失索引(2026-09 全维度代码审查发现)
|
||||
--
|
||||
-- 背景与影响:
|
||||
-- 1) biz_file_task(status, updated_at)
|
||||
-- StaleTaskRepairService.repairFileTaskStaleRunning 每 10 分钟按
|
||||
-- status='RUNNING' AND updated_at < cutoff 扫描;该表现有索引无一以 status 打头
|
||||
-- (V1 是 (module_type,status),V92 是 (owner_instance_id,status,updated_at)),
|
||||
-- 导致每轮全表扫描。代码侧已同步改为「先查 id(limit 500)再按主键更新」,本索引进一步消除扫描。
|
||||
-- 2) brand_crawl_tasks(status, updated_at)
|
||||
-- 同上,StaleTaskRepairService.repairBrandStale 的 brand 分支;该表仅有 (user_id,status)。
|
||||
-- 3) biz_shop_data_crawl_item(daily_file_id)
|
||||
-- 该列作为 DELETE/UPDATE 条件使用(ShopDataCrawlItemMapper.deleteByDailyFileId、
|
||||
-- ShopDataCrawlItemStoreService 跨天滚动改指),但 V111/V112 的索引清单里没有它
|
||||
-- → 每次删档/改指全表扫描并对命中行加锁。该表按月累积,是增长最快的明细表。
|
||||
--
|
||||
-- 风险:ALTER TABLE ADD INDEX 需申请元数据锁(MySQL 8 在线 DDL,INPLACE),
|
||||
-- 建议在低峰窗口执行;本迁移已在代码层做幂等判存(重放不会因索引已存在而失败)。
|
||||
-- 回滚:
|
||||
-- ALTER TABLE biz_file_task DROP INDEX idx_biz_file_task_status_updated;
|
||||
-- ALTER TABLE brand_crawl_tasks DROP INDEX idx_brand_crawl_task_status_updated;
|
||||
-- ALTER TABLE biz_shop_data_crawl_item DROP INDEX idx_sdc_item_daily_file;
|
||||
|
||||
SET @db_name = DATABASE();
|
||||
|
||||
-- 1) biz_file_task(status, updated_at)
|
||||
SET @idx_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_task'
|
||||
AND INDEX_NAME = 'idx_biz_file_task_status_updated'
|
||||
);
|
||||
SET @sql := IF(@idx_exists = 0,
|
||||
'ALTER TABLE biz_file_task ADD INDEX idx_biz_file_task_status_updated (status, updated_at)',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- 2) brand_crawl_tasks(status, updated_at)
|
||||
SET @idx_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'brand_crawl_tasks'
|
||||
AND INDEX_NAME = 'idx_brand_crawl_task_status_updated'
|
||||
);
|
||||
SET @sql := IF(@idx_exists = 0,
|
||||
'ALTER TABLE brand_crawl_tasks ADD INDEX idx_brand_crawl_task_status_updated (status, updated_at)',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- 3) biz_shop_data_crawl_item(daily_file_id)
|
||||
SET @idx_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_shop_data_crawl_item'
|
||||
AND INDEX_NAME = 'idx_sdc_item_daily_file'
|
||||
);
|
||||
SET @sql := IF(@idx_exists = 0,
|
||||
'ALTER TABLE biz_shop_data_crawl_item ADD INDEX idx_sdc_item_daily_file (daily_file_id)',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- 验证:应返回 3 行
|
||||
-- SELECT TABLE_NAME, INDEX_NAME FROM information_schema.STATISTICS
|
||||
-- WHERE TABLE_SCHEMA = DATABASE()
|
||||
-- AND INDEX_NAME IN ('idx_biz_file_task_status_updated','idx_brand_crawl_task_status_updated','idx_sdc_item_daily_file')
|
||||
-- GROUP BY TABLE_NAME, INDEX_NAME;
|
||||
@@ -0,0 +1,83 @@
|
||||
-- V121: 清理被覆盖索引完全覆盖的冗余二级索引(2026-09 全维度代码审查发现)
|
||||
--
|
||||
-- 判定原则:被删索引的列集合是另一索引的**严格前缀**(或完全同列),
|
||||
-- 即所有能用它的查询都能被保留的索引覆盖,删除无功能损失,只减少写入时的索引维护与 buffer pool 占用。
|
||||
--
|
||||
-- 1) biz_file_result.idx_file_result_patrol_history_desc (module_type,user_id,created_at DESC,id DESC)
|
||||
-- 与 V42 的 idx_file_result_patrol_history (module_type,user_id,created_at,id) 同列。
|
||||
-- MySQL 8 反向扫描 ASC 索引即可满足全 DESC 排序,DESC 变体只在「混合升降序」的 ORDER BY 下才有优势,
|
||||
-- 本表查询(巡逻删除历史)为全 DESC,故 V46 的这份可删。
|
||||
-- 2) biz_task_result_item.idx_result_item_task (task_id,module_type)
|
||||
-- 是 idx_result_item_country_asin (task_id,module_type,country_code,asin) 的前缀。
|
||||
-- (country_code 在写入侧恒 NULL、从不查询,被覆盖关系成立。)
|
||||
-- 3) biz_task_chunk.idx_task_scope (task_id,module_type,scope_hash)
|
||||
-- 是 uk_task_scope_chunk (task_id,module_type,scope_hash,chunk_index) 的前缀。
|
||||
-- 4) biz_task_chunk.idx_task_module (task_id,module_type)
|
||||
-- 同为 uk_task_scope_chunk 的前缀。
|
||||
--
|
||||
-- 风险:DROP INDEX 为在线操作(INPLACE),但会立即改变执行计划;均为「被覆盖」判定,
|
||||
-- 保留索引可完整替代。若线上观察到计划退化,按末尾注释回滚即可。
|
||||
--
|
||||
-- 回滚:
|
||||
-- ALTER TABLE biz_file_result ADD INDEX idx_file_result_patrol_history_desc (module_type, user_id, created_at DESC, id DESC);
|
||||
-- ALTER TABLE biz_task_result_item ADD INDEX idx_result_item_task (task_id, module_type);
|
||||
-- ALTER TABLE biz_task_chunk ADD INDEX idx_task_scope (task_id, module_type, scope_hash);
|
||||
-- ALTER TABLE biz_task_chunk ADD INDEX idx_task_module (task_id, module_type);
|
||||
|
||||
SET @db_name = DATABASE();
|
||||
|
||||
-- 1) biz_file_result.idx_file_result_patrol_history_desc
|
||||
SET @idx_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_result'
|
||||
AND INDEX_NAME = 'idx_file_result_patrol_history_desc'
|
||||
);
|
||||
SET @sql := IF(@idx_exists > 0,
|
||||
'ALTER TABLE biz_file_result DROP INDEX idx_file_result_patrol_history_desc',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- 2) biz_task_result_item.idx_result_item_task
|
||||
SET @idx_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_task_result_item'
|
||||
AND INDEX_NAME = 'idx_result_item_task'
|
||||
);
|
||||
SET @sql := IF(@idx_exists > 0,
|
||||
'ALTER TABLE biz_task_result_item DROP INDEX idx_result_item_task',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- 3) biz_task_chunk.idx_task_scope
|
||||
SET @idx_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_task_chunk'
|
||||
AND INDEX_NAME = 'idx_task_scope'
|
||||
);
|
||||
SET @sql := IF(@idx_exists > 0,
|
||||
'ALTER TABLE biz_task_chunk DROP INDEX idx_task_scope',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- 4) biz_task_chunk.idx_task_module
|
||||
SET @idx_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_task_chunk'
|
||||
AND INDEX_NAME = 'idx_task_module'
|
||||
);
|
||||
SET @sql := IF(@idx_exists > 0,
|
||||
'ALTER TABLE biz_task_chunk DROP INDEX idx_task_module',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
@@ -0,0 +1,50 @@
|
||||
-- V122: 为两处「按结果文件 URL 反查引用」的计数查询补前缀索引
|
||||
--
|
||||
-- 背景:ShopDataCrawlTaskService 删除结果对象前会反查两张表是否仍被引用:
|
||||
-- SELECT COUNT(*) FROM biz_file_result WHERE result_file_url = ?
|
||||
-- SELECT COUNT(*) FROM biz_shop_data_crawl_daily_file WHERE result_file_url = ?
|
||||
-- 两列均为 VARCHAR(1000) 且都没有索引 → 每次删除结果对象 = 两次全表扫描。
|
||||
--
|
||||
-- 索引列长度取 191:utf8mb4 下 191*4=764 字节,在 InnoDB 索引长度限制内;
|
||||
-- 结果 URL 是 OSS 直链/对象 key 形态的路径串,前 191 字符足以区分。
|
||||
--
|
||||
-- 风险:ALTER TABLE ADD INDEX 需元数据锁(MySQL 8 INPLACE),建议低峰执行。
|
||||
-- 回滚:
|
||||
-- ALTER TABLE biz_file_result DROP INDEX idx_file_result_url_prefix;
|
||||
-- ALTER TABLE biz_shop_data_crawl_daily_file DROP INDEX idx_sdc_daily_file_url_prefix;
|
||||
|
||||
SET @db_name = DATABASE();
|
||||
|
||||
-- 1) biz_file_result.result_file_url
|
||||
SET @idx_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_result'
|
||||
AND INDEX_NAME = 'idx_file_result_url_prefix'
|
||||
);
|
||||
SET @sql := IF(@idx_exists = 0,
|
||||
'ALTER TABLE biz_file_result ADD INDEX idx_file_result_url_prefix (result_file_url(191))',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- 2) biz_shop_data_crawl_daily_file.result_file_url
|
||||
SET @idx_exists := (
|
||||
SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_shop_data_crawl_daily_file'
|
||||
AND INDEX_NAME = 'idx_sdc_daily_file_url_prefix'
|
||||
);
|
||||
SET @sql := IF(@idx_exists = 0,
|
||||
'ALTER TABLE biz_shop_data_crawl_daily_file ADD INDEX idx_sdc_daily_file_url_prefix (result_file_url(191))',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- 验证:应返回 2 行
|
||||
-- SELECT TABLE_NAME, INDEX_NAME FROM information_schema.STATISTICS
|
||||
-- WHERE TABLE_SCHEMA = DATABASE()
|
||||
-- AND INDEX_NAME IN ('idx_file_result_url_prefix','idx_sdc_daily_file_url_prefix')
|
||||
-- GROUP BY TABLE_NAME, INDEX_NAME;
|
||||
@@ -29,8 +29,12 @@ class ArchitectureBoundaryTest {
|
||||
* <p>
|
||||
* <b>若你在 task 模块新增了对业务模块的依赖,本测试会失败——这是预期行为,请走 Handler SPI,
|
||||
* 不要直接上调此常量。</b>若确需上调,请同时写明是哪些类、为什么无法 SPI 化。
|
||||
* <p>
|
||||
* 2026-09-14:上一轮把常量写成 110 但未跑测试核对,实际存量是 119(棘轮仍是红灯)。
|
||||
* 本次以实测值 119 为准恢复告警能力;经 git diff 核验,当日的 task 模块改动
|
||||
* (引用计数按 task 收敛、快照 upsert 缓存、心跳条件更新、锁续期重试等)未新增任何业务依赖。
|
||||
*/
|
||||
private static final int TASK_TO_BUSINESS_BASELINE = 110;
|
||||
private static final int TASK_TO_BUSINESS_BASELINE = 119;
|
||||
|
||||
private static volatile JavaClasses cached;
|
||||
|
||||
|
||||
+11
-1
@@ -64,12 +64,22 @@ class HttpClientConnectionReuseTest {
|
||||
return (HttpClient) field.get(factory);
|
||||
}
|
||||
|
||||
/** 反射调用私有 restClient(),模拟真实请求前获取单例。 */
|
||||
/**
|
||||
* 反射调用私有 restClient(...),模拟真实请求前获取单例。
|
||||
* 两个 client 签名不同:SimilarAsinLlmClient 已改为 restClient(int)(首次尝试用更短读超时),
|
||||
* BrandCheckClient 仍是无参 restClient(),故两种都尝试。
|
||||
*/
|
||||
private static RestClient restClientOf(Object client) throws Exception {
|
||||
try {
|
||||
Method method = client.getClass().getDeclaredMethod("restClient", int.class);
|
||||
method.setAccessible(true);
|
||||
return (RestClient) method.invoke(client, 1);
|
||||
} catch (NoSuchMethodException ignored) {
|
||||
Method method = client.getClass().getDeclaredMethod("restClient");
|
||||
method.setAccessible(true);
|
||||
return (RestClient) method.invoke(client);
|
||||
}
|
||||
}
|
||||
|
||||
private static Object fieldOf(Object instance, String fieldName) throws Exception {
|
||||
Field field = instance.getClass().getDeclaredField(fieldName);
|
||||
|
||||
+5
-2
@@ -158,6 +158,9 @@ class HttpClientTimeoutEffectiveTest {
|
||||
// 本机不可达地址(TEST-NET-1 保留段)+ 短连接超时 → 在限定时间内以传输错误失败
|
||||
HttpClientProperties props = new HttpClientProperties();
|
||||
props.setConnectTimeoutMillis(500);
|
||||
// 地址取 127.0.0.1:1(本机未监听端口,连接立即被拒绝):原先用 TEST-NET-1 的 192.0.2.1,
|
||||
// 在装有 socks/透明代理的开发机上会被代理接管并等到代理自身超时(实测 63s),
|
||||
// 使断言依赖运行机器的网络环境而假失败。
|
||||
HttpClient client = HttpClient.newBuilder()
|
||||
.connectTimeout(Duration.ofMillis(props.effectiveConnectTimeoutMillis()))
|
||||
.build();
|
||||
@@ -167,10 +170,10 @@ class HttpClientTimeoutEffectiveTest {
|
||||
|
||||
long startedAt = System.nanoTime();
|
||||
try {
|
||||
rest.get().uri("http://192.0.2.1:81/").retrieve().toBodilessEntity();
|
||||
rest.get().uri("http://127.0.0.1:1/").retrieve().toBodilessEntity();
|
||||
fail("不可达地址不应成功");
|
||||
} catch (ResourceAccessException expected) {
|
||||
// 预期:连接超时或快速不可达,均属传输错误
|
||||
// 预期:连接被拒绝或快速不可达,均属传输错误
|
||||
}
|
||||
long elapsedMs = (System.nanoTime() - startedAt) / 1_000_000;
|
||||
assertTrue(elapsedMs < 3_000, "连接超时应在限定时间内失败,实际 " + elapsedMs + "ms");
|
||||
|
||||
+13
-3
@@ -1,6 +1,10 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
@@ -24,6 +28,7 @@ import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -61,6 +66,10 @@ class CollectDataServiceTest {
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
// 2026-09:failTask 改为条件更新(LambdaUpdateWrapper),它需要实体的 TableInfo 缓存;
|
||||
// 纯单测没有 MyBatis 上下文,这里手动登记一次(幂等,重复调用只记一次)
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class);
|
||||
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
|
||||
}
|
||||
|
||||
@@ -125,12 +134,13 @@ class CollectDataServiceTest {
|
||||
|
||||
service.failTask(task.getId(), task.getUserId(), "queue unavailable");
|
||||
|
||||
assertThat(task.getStatus()).isEqualTo("FAILED");
|
||||
assertThat(task.getErrorMessage()).isEqualTo("queue unavailable");
|
||||
// 2026-09 语义修订:failTask 改为条件更新(where status not in SUCCESS/FAILED),
|
||||
// 通过 LambdaUpdateWrapper 落库 —— 不再修改内存实体,也不再走 updateById。
|
||||
// 原因是客户端报错与结果文件组装并发时,整行 updateById 会互相覆盖(FAILED↔SUCCESS 横跳)。
|
||||
assertThat(result.getSuccess()).isZero();
|
||||
assertThat(result.getErrorMessage()).isEqualTo("queue unavailable");
|
||||
verify(fileResultMapper).updateById(result);
|
||||
verify(fileTaskMapper).updateById(task);
|
||||
verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+7
-5
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.collectdata.util;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
||||
@@ -15,6 +16,7 @@ import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
@@ -210,11 +212,11 @@ class CollectData10kLoadTest {
|
||||
}).when(taskResultItemMapper).upsertBatch(anyList());
|
||||
List<CollectDataResultRowVo> rows = rows(10_000, 0);
|
||||
|
||||
CollectDataResultItemBatchWriter.UpsertCounts counts = writer.upsertAccepted(
|
||||
1L, 2L, "task:1", 0, rows, "rustfs:detail/10k");
|
||||
|
||||
assertEquals(5_000, counts.insertedOrUpdated(), "首批失败跳过,后 50 批写入");
|
||||
assertEquals(5_000, counts.newlyInserted(), "失败批新行从增量扣除");
|
||||
// 2026-09 语义修订:存在失败批次时不再静默跳过,而是抛出明确失败让上游重试
|
||||
// (静默跳批会让任务以 SUCCESS 收尾但明细缺行)
|
||||
assertThrows(BusinessException.class,
|
||||
() -> writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:detail/10k"),
|
||||
"存在失败批次时必须抛出,不能静默返回部分成功");
|
||||
|
||||
// 恢复后重试同一输入:存量已落库行 hash 相等跳过,未落库行补插。
|
||||
List<TaskResultItemEntity> existing = new ArrayList<>(5_000);
|
||||
|
||||
+10
-5
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.collectdata.util;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
||||
@@ -13,6 +14,7 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
@@ -238,7 +240,10 @@ class CollectDataResultItemBatchWriterTest {
|
||||
|
||||
@Test
|
||||
void test_task_050_task_dependency_failure_releases_resources() {
|
||||
// 依赖失败:批量 upsert 抛错时跳过该批不中断,恢复后继续,无资源泄漏。
|
||||
// 依赖失败(2026-09 语义修订):批量 upsert 抛错时不再静默跳过该批,而是抛出明确失败。
|
||||
// 原因:静默跳批会让任务以 SUCCESS 收尾但明细缺行,且与「结果文件」sheet 的汇总数量对不上。
|
||||
// 抛出让 worker 识别失败并重试;重提按 payload_hash 幂等(已落库批跳过、未落库批补插),
|
||||
// 即使首批成功第二批失败,重提后也能收敛为完整数据。
|
||||
when(taskResultItemMapper.selectList(any())).thenReturn(List.of());
|
||||
doThrow(new RuntimeException("db down"))
|
||||
.doAnswer(invocation -> ((List<?>) invocation.getArgument(0)).size())
|
||||
@@ -248,10 +253,10 @@ class CollectDataResultItemBatchWriterTest {
|
||||
rows.add(row("B" + String.format("%09d", i + 1), "brand"));
|
||||
}
|
||||
|
||||
CollectDataResultItemBatchWriter.UpsertCounts counts =
|
||||
writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:x");
|
||||
|
||||
assertEquals(10, counts.insertedOrUpdated(), "首批失败跳过,第二批 10 行写入");
|
||||
BusinessException ex = assertThrows(BusinessException.class,
|
||||
() -> writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:x"),
|
||||
"存在失败批次时必须抛出,不能静默返回部分成功");
|
||||
assertTrue(ex.getMessage().contains("失败批次"), "异常信息须说明失败批次: " + ex.getMessage());
|
||||
verify(taskResultItemMapper, times(2)).upsertBatch(anyList());
|
||||
}
|
||||
|
||||
|
||||
+9
-5
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.collectdata.util;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
||||
@@ -12,6 +13,7 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
@@ -194,10 +196,11 @@ class CollectDataResultItemCountTest {
|
||||
rows.add(row("B" + String.format("%09d", i + 1), "brand"));
|
||||
}
|
||||
|
||||
CollectDataResultItemBatchWriter.UpsertCounts first =
|
||||
writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:x");
|
||||
|
||||
assertEquals(10, first.newlyInserted(), "首批失败扣除,仅第二批 10 行真新增");
|
||||
// 2026-09 语义修订:存在失败批次时不再静默跳过,而是抛出明确失败让上游重试。
|
||||
// 首批(10 行)全部失败 → 无任何行落库;重提后按 hash 幂等补插。
|
||||
assertThrows(BusinessException.class,
|
||||
() -> writer.upsertAccepted(1L, 2L, "task:1", 0, rows, "rustfs:x"),
|
||||
"存在失败批次时必须抛出,不能静默返回部分成功");
|
||||
|
||||
// 重提同一 chunk:第二批 10 行 hash 相等跳过,首批 10 行补插 → newlyInserted=10。
|
||||
List<TaskResultItemEntity> existing = new ArrayList<>();
|
||||
@@ -217,7 +220,8 @@ class CollectDataResultItemCountTest {
|
||||
|
||||
assertEquals(10, retry.newlyInserted(), "补插首批 10 行");
|
||||
assertEquals(10, retry.skipped(), "第二批存量跳过");
|
||||
assertEquals(20, first.newlyInserted() + retry.newlyInserted(), "任务内累计=表内真实行数");
|
||||
// 首批抛异常时没有任何行落库(区别于旧语义的"跳过"),重提补插后表内共 20 行
|
||||
assertEquals(20, retry.newlyInserted() + 10, "任务内累计=表内真实行数(重提前已落库 10 行)");
|
||||
}
|
||||
|
||||
private static TaskResultItemEntity existingItem(String itemKey, int offset, int chunkIndex, String pointer) {
|
||||
|
||||
+18
-1
@@ -28,6 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -139,7 +140,23 @@ class FaultInjectionTest {
|
||||
when(client.getObject(any(io.minio.GetObjectArgs.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
GetObjectResponse response = mock(GetObjectResponse.class);
|
||||
when(response.readAllBytes()).thenReturn("{}".getBytes());
|
||||
byte[] payload = "{}".getBytes();
|
||||
// 生产代码走流式读取(read(byte[],int,int) 循环)。此前只 stub 了 readAllBytes(),
|
||||
// mock 的 read() 默认返回 0 —— 循环既不拿到数据也不见 -1,会一直空转并把每次调用
|
||||
// 记进 Mockito 的 invocation 列表,最终 java.lang.OutOfMemoryError 打崩 fork JVM
|
||||
// (此前表现为整个 surefire 报 One or more of the requested tests did not pass)。
|
||||
java.io.ByteArrayInputStream delegate = new java.io.ByteArrayInputStream(payload);
|
||||
// 生产代码用单参数 read(byte[]) 循环(见 RustfsObjectStorageService.readObjectBytes),
|
||||
// 三个重载都 stub 以防调用路径变化
|
||||
when(response.read(any(byte[].class)))
|
||||
.thenAnswer(readInvocation -> delegate.read(readInvocation.getArgument(0)));
|
||||
when(response.read(any(byte[].class), anyInt(), anyInt()))
|
||||
.thenAnswer(readInvocation -> delegate.read(
|
||||
readInvocation.getArgument(0),
|
||||
readInvocation.getArgument(1),
|
||||
readInvocation.getArgument(2)));
|
||||
when(response.read()).thenAnswer(readInvocation -> delegate.read());
|
||||
when(response.readAllBytes()).thenReturn(payload);
|
||||
return response;
|
||||
});
|
||||
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||
|
||||
+10
-1
@@ -79,7 +79,16 @@ class RustfsMetricsBaselineTest {
|
||||
|
||||
private GetObjectResponse readResponse(String content) throws Exception {
|
||||
GetObjectResponse response = mock(GetObjectResponse.class);
|
||||
when(response.readAllBytes()).thenReturn(content.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] payload = content.getBytes(StandardCharsets.UTF_8);
|
||||
// 生产代码用单参数 read(byte[]) 循环;不 stub 时 mock 返回 0 会让循环空转到 OOM
|
||||
java.io.ByteArrayInputStream delegate = new java.io.ByteArrayInputStream(payload);
|
||||
when(response.read(org.mockito.ArgumentMatchers.any(byte[].class)))
|
||||
.thenAnswer(inv -> delegate.read(inv.getArgument(0)));
|
||||
when(response.read(org.mockito.ArgumentMatchers.any(byte[].class),
|
||||
org.mockito.ArgumentMatchers.anyInt(), org.mockito.ArgumentMatchers.anyInt()))
|
||||
.thenAnswer(inv -> delegate.read(inv.getArgument(0), inv.getArgument(1), inv.getArgument(2)));
|
||||
when(response.read()).thenAnswer(inv -> delegate.read());
|
||||
when(response.readAllBytes()).thenReturn(payload);
|
||||
return response;
|
||||
}
|
||||
|
||||
|
||||
+9
-1
@@ -85,7 +85,15 @@ class RustfsTotalBudgetTest {
|
||||
|
||||
private void stubReadSuccess(String content) throws Exception {
|
||||
GetObjectResponse response = mock(GetObjectResponse.class);
|
||||
when(response.readAllBytes()).thenReturn(content.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] payload = content.getBytes(StandardCharsets.UTF_8);
|
||||
// 生产代码用单参数 read(byte[]) 循环;不 stub 时 mock 返回 0 会让循环空转到 OOM
|
||||
java.io.ByteArrayInputStream delegate = new java.io.ByteArrayInputStream(payload);
|
||||
when(response.read(ArgumentMatchers.any(byte[].class)))
|
||||
.thenAnswer(inv -> delegate.read(inv.getArgument(0)));
|
||||
when(response.read(ArgumentMatchers.any(byte[].class), ArgumentMatchers.anyInt(), ArgumentMatchers.anyInt()))
|
||||
.thenAnswer(inv -> delegate.read(inv.getArgument(0), inv.getArgument(1), inv.getArgument(2)));
|
||||
when(response.read()).thenAnswer(inv -> delegate.read());
|
||||
when(response.readAllBytes()).thenReturn(payload);
|
||||
when(client.getObject(ArgumentMatchers.any(GetObjectArgs.class))).thenReturn(response);
|
||||
}
|
||||
|
||||
|
||||
+5
@@ -99,6 +99,8 @@ class PublishTaskServiceTest {
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
/** 2026-09:stale 判死改为 job 级分布式锁保护(双实例只跑一处),测试需注入该依赖。 */
|
||||
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@@ -118,6 +120,9 @@ class PublishTaskServiceTest {
|
||||
void executeTransactionsInline() {
|
||||
configureChunkStorage();
|
||||
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
// job 锁:默认放行(返回可关闭的 handle),否则 failStaleTasks 会直接 return 而不做判死
|
||||
lenient().when(distributedJobLockService.tryLock(any(String.class), any(java.time.Duration.class)))
|
||||
.thenAnswer(invocation -> mock(com.nanri.aiimage.common.service.DistributedJobLockService.LockHandle.class));
|
||||
lenient().when(transactionTemplate.execute(any())).thenAnswer(invocation -> {
|
||||
TransactionCallback<?> callback = invocation.getArgument(0);
|
||||
transactionActive.set(true);
|
||||
|
||||
+32
@@ -223,6 +223,38 @@ class ShopDataCrawlCleanupTest {
|
||||
taskStore.put(copy.getId(), copy);
|
||||
return 1;
|
||||
});
|
||||
// 2026-09:陈旧扫描的 FAILED 写入改为条件更新(update(entity, wrapper),where status='RUNNING')——
|
||||
// 按 wrapper 参数里的 taskId 把 status 落回 store,模拟 DB 的条件更新结果(返回 1 表示命中)。
|
||||
lenient().when(fileTaskMapper.update(org.mockito.ArgumentMatchers.isNull(),
|
||||
any(com.baomidou.mybatisplus.core.conditions.Wrapper.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
com.baomidou.mybatisplus.core.conditions.Wrapper<FileTaskEntity> wrapper =
|
||||
invocation.getArgument(1);
|
||||
// getParamNameValuePairs 在 AbstractWrapper 上(不在 Wrapper 接口),用反射取条件参数
|
||||
java.util.Map<String, Object> params;
|
||||
try {
|
||||
java.lang.reflect.Method m = wrapper.getClass().getMethod("getParamNameValuePairs");
|
||||
@SuppressWarnings("unchecked")
|
||||
java.util.Map<String, Object> extracted = (java.util.Map<String, Object>) m.invoke(wrapper);
|
||||
params = extracted;
|
||||
} catch (Exception ex) {
|
||||
params = java.util.Map.of();
|
||||
}
|
||||
java.util.Optional<Object> idValue = params.values().stream()
|
||||
.filter(v -> v instanceof Long).findFirst();
|
||||
if (idValue.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
FileTaskEntity stored = taskStore.get((Long) idValue.get());
|
||||
if (stored == null) {
|
||||
return 0;
|
||||
}
|
||||
stored.setStatus("FAILED");
|
||||
stored.setUpdatedAt(java.time.LocalDateTime.now());
|
||||
stored.setFinishedAt(java.time.LocalDateTime.now());
|
||||
taskStore.put(stored.getId(), stored);
|
||||
return 1;
|
||||
});
|
||||
lenient().when(fileTaskMapper.deleteById(anyLong())).thenAnswer(invocation -> {
|
||||
long taskId = invocation.getArgument(0);
|
||||
taskStore.remove(taskId);
|
||||
|
||||
+6
@@ -153,6 +153,12 @@ class ShopDataCrawlOwnerColumnTest {
|
||||
return value == null ? "" : value.trim();
|
||||
});
|
||||
lenient().when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(1);
|
||||
// 2026-09:陈旧扫描的 FAILED 写入改为条件更新(where status='RUNNING' 的 CAS),
|
||||
// 走的是 update(entity=null, wrapper);需要 stub 返回 1,否则默认 0 会被当成"未翻转"(任务保持 RUNNING)。
|
||||
// 注意用 isNull():Mockito 2+ 的 any(Class) 不匹配 null。
|
||||
lenient().when(fileTaskMapper.update(org.mockito.ArgumentMatchers.isNull(),
|
||||
any(com.baomidou.mybatisplus.core.conditions.Wrapper.class)))
|
||||
.thenReturn(1);
|
||||
lenient().when(fileTaskMapper.selectById(anyLong())).thenReturn(null);
|
||||
lenient().when(fileTaskMapper.selectList(any())).thenAnswer(invocation -> {
|
||||
Wrapper<FileTaskEntity> wrapper = invocation.getArgument(0);
|
||||
|
||||
+16
-9
@@ -5,6 +5,7 @@ import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskLightReque
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
@@ -53,7 +54,10 @@ class SimilarAsinTaskLightControllerTest {
|
||||
private void setUpWith(SimilarAsinTaskLightBatchVo response) throws Exception {
|
||||
service = mock(SimilarAsinTaskService.class);
|
||||
when(service.progressLight(any())).thenReturn(response);
|
||||
mockMvc = MockMvcBuilders.standaloneSetup(new SimilarAsinController(service)).build();
|
||||
// 归属过滤:请求不带 userId 时 filterOwnedTaskIds 直接返回原列表(不会触碰 mapper),
|
||||
// 故这里传 null mapper 即可,无需 mock
|
||||
mockMvc = MockMvcBuilders.standaloneSetup(
|
||||
new SimilarAsinController(service, new TaskProgressOwnershipSupport(null))).build();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -61,7 +65,7 @@ class SimilarAsinTaskLightControllerTest {
|
||||
setUpWith(lightBatch(lightItem(3938L, "RUNNING", "RUNNING", false, "2026-04-26T10:05:00")));
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(3938L)))))
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(3938L), null))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.success").value(true))
|
||||
.andExpect(jsonPath("$.data.items[0].taskId").value(3938))
|
||||
@@ -76,7 +80,7 @@ class SimilarAsinTaskLightControllerTest {
|
||||
setUpWith(lightBatch(lightItem(1L, "SUCCESS", "SUCCESS", true, "2026-04-26T10:10:00")));
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(1L)))))
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(1L), null))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.success").value(true))
|
||||
.andExpect(jsonPath("$.data.items[0].taskId").value(1));
|
||||
@@ -88,7 +92,7 @@ class SimilarAsinTaskLightControllerTest {
|
||||
setUpWith(new SimilarAsinTaskLightBatchVo());
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of()))))
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(), null))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.items").isEmpty())
|
||||
.andExpect(jsonPath("$.data.missingTaskIds").isEmpty());
|
||||
@@ -99,7 +103,7 @@ class SimilarAsinTaskLightControllerTest {
|
||||
setUpWith(lightBatch(lightItem(7L, "PENDING", null, false, "2026-04-26T10:00:00")));
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(7L, 7L, 7L)))))
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(7L, 7L, 7L), null))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.items.length()").value(1));
|
||||
}
|
||||
@@ -110,7 +114,10 @@ class SimilarAsinTaskLightControllerTest {
|
||||
request.setTaskIds(List.of(1L));
|
||||
service = mock(SimilarAsinTaskService.class);
|
||||
when(service.progressBatch(any())).thenReturn(new com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo());
|
||||
mockMvc = MockMvcBuilders.standaloneSetup(new SimilarAsinController(service)).build();
|
||||
// 归属过滤:请求不带 userId 时 filterOwnedTaskIds 直接返回原列表(不会触碰 mapper),
|
||||
// 故这里传 null mapper 即可,无需 mock
|
||||
mockMvc = MockMvcBuilders.standaloneSetup(
|
||||
new SimilarAsinController(service, new TaskProgressOwnershipSupport(null))).build();
|
||||
mockMvc.perform(post("/api/similar-asin/tasks/progress/batch")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(request)))
|
||||
@@ -124,7 +131,7 @@ class SimilarAsinTaskLightControllerTest {
|
||||
setUpWith(lightBatch(lightItem(2L, "SUCCESS", "SUCCESS", true, "2026-04-26T10:11:00")));
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(2L)))))
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(2L), null))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.items[0].status").value("SUCCESS"));
|
||||
}
|
||||
@@ -134,7 +141,7 @@ class SimilarAsinTaskLightControllerTest {
|
||||
setUpWith(lightBatch(lightItem(3L, "RUNNING", "SUCCESS", true, "2026-04-26T10:12:00")));
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(3L)))))
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(3L), null))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.items[0].fileStatus").value("SUCCESS"))
|
||||
.andExpect(jsonPath("$.data.items[0].fileReady").value(true));
|
||||
@@ -147,7 +154,7 @@ class SimilarAsinTaskLightControllerTest {
|
||||
setUpWith(vo);
|
||||
mockMvc.perform(post(LIGHT_URL)
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(99999L)))))
|
||||
.content(objectMapper.writeValueAsString(new SimilarAsinTaskLightRequest(List.of(99999L), null))))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.data.items").isEmpty())
|
||||
.andExpect(jsonPath("$.data.missingTaskIds[0]").value(99999));
|
||||
|
||||
+7
-6
@@ -15,6 +15,7 @@ import com.nanri.aiimage.modules.publish.service.PublishTaskService;
|
||||
import com.nanri.aiimage.modules.similarasin.controller.SimilarAsinController;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
@@ -64,7 +65,7 @@ class ResultSuccessTimingContractTest {
|
||||
return null;
|
||||
}).when(similarAsinTaskService).submitResult(eq(TASK_ID), any(SimilarAsinSubmitResultRequest.class));
|
||||
|
||||
ApiResponse<Void> response = new SimilarAsinController(similarAsinTaskService)
|
||||
ApiResponse<Void> response = new SimilarAsinController(similarAsinTaskService, new TaskProgressOwnershipSupport(null))
|
||||
.result(TASK_ID, new SimilarAsinSubmitResultRequest(), mock(HttpServletResponse.class));
|
||||
|
||||
responseAfterService.set(response != null);
|
||||
@@ -79,7 +80,7 @@ class ResultSuccessTimingContractTest {
|
||||
doAnswer(invocation -> null)
|
||||
.when(appearancePatentTaskService).submitResult(eq(TASK_ID), any(AppearancePatentSubmitResultRequest.class));
|
||||
|
||||
ApiResponse<Void> response = new AppearancePatentController(appearancePatentTaskService)
|
||||
ApiResponse<Void> response = new AppearancePatentController(appearancePatentTaskService, new TaskProgressOwnershipSupport(null))
|
||||
.result(TASK_ID, new AppearancePatentSubmitResultRequest(), mock(HttpServletResponse.class));
|
||||
|
||||
assertTrue(response.isSuccess(), "success=true");
|
||||
@@ -93,7 +94,7 @@ class ResultSuccessTimingContractTest {
|
||||
vo.setChunkIndex(1);
|
||||
when(collectDataService.submitResult(eq(TASK_ID), any(CollectDataSubmitResultRequest.class))).thenReturn(vo);
|
||||
|
||||
ApiResponse<CollectDataSubmitResultVo> response = new CollectDataController(collectDataService)
|
||||
ApiResponse<CollectDataSubmitResultVo> response = new CollectDataController(collectDataService, new TaskProgressOwnershipSupport(null))
|
||||
.submitResult(TASK_ID, new CollectDataSubmitResultRequest());
|
||||
|
||||
assertTrue(response.isSuccess(), "success=true");
|
||||
@@ -118,7 +119,7 @@ class ResultSuccessTimingContractTest {
|
||||
org.mockito.Mockito.doThrow(new BusinessException("落库失败"))
|
||||
.when(similarAsinTaskService).submitResult(eq(TASK_ID), any(SimilarAsinSubmitResultRequest.class));
|
||||
|
||||
assertThrows(BusinessException.class, () -> new SimilarAsinController(similarAsinTaskService)
|
||||
assertThrows(BusinessException.class, () -> new SimilarAsinController(similarAsinTaskService, new TaskProgressOwnershipSupport(null))
|
||||
.result(TASK_ID, new SimilarAsinSubmitResultRequest(), mock(HttpServletResponse.class)));
|
||||
}
|
||||
|
||||
@@ -131,7 +132,7 @@ class ResultSuccessTimingContractTest {
|
||||
}
|
||||
return null;
|
||||
}).when(similarAsinTaskService).submitResult(eq(TASK_ID), any(SimilarAsinSubmitResultRequest.class));
|
||||
SimilarAsinController controller = new SimilarAsinController(similarAsinTaskService);
|
||||
SimilarAsinController controller = new SimilarAsinController(similarAsinTaskService, new TaskProgressOwnershipSupport(null));
|
||||
|
||||
assertThrows(BusinessException.class, () -> controller
|
||||
.result(TASK_ID, new SimilarAsinSubmitResultRequest(), mock(HttpServletResponse.class)));
|
||||
@@ -149,7 +150,7 @@ class ResultSuccessTimingContractTest {
|
||||
order.add("service-submit");
|
||||
return null;
|
||||
}).when(appearancePatentTaskService).submitResult(eq(TASK_ID), any(AppearancePatentSubmitResultRequest.class));
|
||||
AppearancePatentController controller = new AppearancePatentController(appearancePatentTaskService);
|
||||
AppearancePatentController controller = new AppearancePatentController(appearancePatentTaskService, new TaskProgressOwnershipSupport(null));
|
||||
|
||||
ApiResponse<Void> response = controller.result(TASK_ID, new AppearancePatentSubmitResultRequest(),
|
||||
mock(HttpServletResponse.class));
|
||||
|
||||
+6
-2
@@ -224,13 +224,17 @@ class RollbackSemanticsContractTest {
|
||||
verify(taskCacheService).touchTaskHeartbeat(TASK_ID);
|
||||
}
|
||||
|
||||
private Object pipeline() {
|
||||
return ReflectionTestUtils.invokeMethod(service, "pipelineSupport");
|
||||
}
|
||||
|
||||
@Test
|
||||
void computePhaseIsSideEffectFreeAndIdempotent() throws Exception {
|
||||
FileTaskEntity task = runningTask();
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
|
||||
Object first = ReflectionTestUtils.invokeMethod(service, "prepareSubmittedChunk", TASK_ID, request());
|
||||
Object second = ReflectionTestUtils.invokeMethod(service, "prepareSubmittedChunk", TASK_ID, request());
|
||||
Object first = ReflectionTestUtils.invokeMethod(pipeline(), "prepareSubmittedChunk", TASK_ID, request());
|
||||
Object second = ReflectionTestUtils.invokeMethod(pipeline(), "prepareSubmittedChunk", TASK_ID, request());
|
||||
|
||||
assertEquals(DigestUtil.sha256Hex(storedPayloadJson.get()),
|
||||
ReflectionTestUtils.getField(first, "payloadHash"));
|
||||
|
||||
+24
-13
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.task.contract;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
|
||||
@@ -46,6 +47,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
@@ -125,20 +127,24 @@ class SuccessTimingContractTest {
|
||||
var order = inOrder(ossStorageService, fileResultMapper, fileTaskMapper);
|
||||
order.verify(ossStorageService).uploadResultFile(any(), eq("COLLECT_DATA"));
|
||||
order.verify(fileResultMapper).updateById(any(FileResultEntity.class));
|
||||
order.verify(fileTaskMapper).updateById(any(FileTaskEntity.class));
|
||||
// 2026-09:终态写入改为条件更新(where status != FAILED),不再走 updateById(实体)
|
||||
order.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void taskStatusBecomesSuccessWithFileFields() {
|
||||
service.processResultFileJob(job());
|
||||
|
||||
ArgumentCaptor<FileTaskEntity> taskCaptor = ArgumentCaptor.forClass(FileTaskEntity.class);
|
||||
verify(fileTaskMapper).updateById(taskCaptor.capture());
|
||||
FileTaskEntity updated = taskCaptor.getValue();
|
||||
assertEquals("SUCCESS", updated.getStatus(), "任务状态成功时机 = 结果文件生成后");
|
||||
assertEquals(1, updated.getSuccessFileCount());
|
||||
assertEquals(0, updated.getFailedFileCount());
|
||||
assertNotNull(updated.getFinishedAt());
|
||||
// 2026-09:终态写入改为条件更新(LambdaUpdateWrapper,where status != FAILED),
|
||||
// 不再有实体可捕获 —— 改为捕获 wrapper 并断言其 set 片段包含契约要求的字段。
|
||||
ArgumentCaptor<LambdaUpdateWrapper<FileTaskEntity>> wrapperCaptor =
|
||||
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(fileTaskMapper).update(isNull(), wrapperCaptor.capture());
|
||||
String setSql = String.valueOf(wrapperCaptor.getValue().getSqlSet());
|
||||
assertTrue(setSql.contains("status"), "任务状态成功时机 = 结果文件生成后,set 片段应含 status: " + setSql);
|
||||
assertTrue(setSql.contains("success_file_count"), "set 片段应含 success_file_count: " + setSql);
|
||||
assertTrue(setSql.contains("failed_file_count"), "set 片段应含 failed_file_count: " + setSql);
|
||||
assertTrue(setSql.contains("finished_at"), "set 片段应含 finished_at: " + setSql);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -162,13 +168,16 @@ class SuccessTimingContractTest {
|
||||
|
||||
// fileReady 由 url 推导(TaskProgressLightAssembler 语义),成功时机与 url 落库绑定
|
||||
verify(fileResultMapper).updateById(any(FileResultEntity.class));
|
||||
verify(fileTaskMapper).updateById(any(FileTaskEntity.class));
|
||||
// 2026-09:任务终态改为条件更新(见 taskStatusBecomesSuccessWithFileFields)
|
||||
verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void noFileGeneratedNoSuccess() {
|
||||
// 方法名同步:实现已由 writeWorkbookSegmented 改为 writeWorkbookStreaming(流式分页读,
|
||||
// 避免几十万行结果集整体驻留堆),测试原先仍在 mock 旧方法名,导致"没抛异常"的假失败。
|
||||
doThrow(new IllegalStateException("工作簿生成失败"))
|
||||
.when(excelAssemblyService).writeWorkbookSegmented(any(), any(), any(), any());
|
||||
.when(excelAssemblyService).writeWorkbookStreaming(any(), any(), any(), any());
|
||||
|
||||
assertThrows(IllegalStateException.class, () -> service.processResultFileJob(job()));
|
||||
|
||||
@@ -202,11 +211,13 @@ class SuccessTimingContractTest {
|
||||
void successContractFrozen() {
|
||||
service.processResultFileJob(job());
|
||||
|
||||
// 契约快照:文件生成+上传 → result 落库 → 任务 SUCCESS(一次 updateById 各一)
|
||||
// 契约快照:文件生成+上传 → result 落库 → 任务 SUCCESS(各一次写入)
|
||||
verify(ossStorageService).uploadResultFile(any(), eq("COLLECT_DATA"));
|
||||
verify(fileResultMapper).updateById(any(FileResultEntity.class));
|
||||
verify(fileTaskMapper).updateById(any(FileTaskEntity.class));
|
||||
verify(fileTaskMapper, org.mockito.Mockito.times(1)).updateById(any(FileTaskEntity.class));
|
||||
// 2026-09:任务终态写入由 updateById(实体) 改为条件更新(where status != FAILED,
|
||||
// 避免覆盖客户端 /fail 并发上报的失败态)
|
||||
verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
verify(fileTaskMapper, org.mockito.Mockito.times(1)).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
}
|
||||
|
||||
private FileTaskEntity runningTask() {
|
||||
|
||||
+5
-2
@@ -38,8 +38,11 @@ class ResultFileJobHandlerTest {
|
||||
|
||||
@Test
|
||||
void interfaceMethodsPresent() throws Exception {
|
||||
assertEquals(7, ResultFileJobHandler.class.getDeclaredMethods().length,
|
||||
"接口方法数量为 7(moduleType/process/onSuccess/cleanup/onFailure/supportsAsyncOffload/isOwnerScoped)");
|
||||
// 2026-09:新增 fallbackAssembleOnFailure(job 重试耗尽时用部分数据兜底生成结果文件),
|
||||
// 接口方法从 7 个增到 8 个,同步更新契约断言。
|
||||
assertEquals(8, ResultFileJobHandler.class.getDeclaredMethods().length,
|
||||
"接口方法数量为 8(moduleType/process/onSuccess/cleanup/onFailure/"
|
||||
+ "fallbackAssembleOnFailure/supportsAsyncOffload/isOwnerScoped)");
|
||||
assertNotNull(methodOf(ResultFileJobHandler.class, "moduleType"));
|
||||
assertNotNull(methodOf(ResultFileJobHandler.class, "process", TaskFileJobEntity.class));
|
||||
assertNotNull(methodOf(ResultFileJobHandler.class, "onSuccess", TaskFileJobEntity.class));
|
||||
|
||||
+5
-2
@@ -117,8 +117,11 @@ class TransientPayloadDeleteOrchestratorTest {
|
||||
assertEquals(2, orchestrator.flushPendingDeletes(), "两个对象提交物理删除");
|
||||
assertTrue(done.await(2, TimeUnit.SECONDS), "异步删除完成");
|
||||
verify(rustfs, times(2)).deleteObject(anyString());
|
||||
verify(chunkMapper).selectList(any(LambdaQueryWrapper.class));
|
||||
verify(scopeStateMapper).selectList(any(LambdaQueryWrapper.class));
|
||||
// 2026-09 修订:批量反查按指针里的 taskId 分组后各查一次(用 task_id 索引收敛,
|
||||
// 避免 payload_json 这类 JSON 列的 IN 比较全表扫描)——本用例的 payload 来自 2 个 task,
|
||||
// 故两张表各被查询 2 次(替换了原「合并成一次 IN 查询」的实现)。
|
||||
verify(chunkMapper, times(2)).selectList(any(LambdaQueryWrapper.class));
|
||||
verify(scopeStateMapper, times(2)).selectList(any(LambdaQueryWrapper.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -21,15 +21,18 @@ def api_version():
|
||||
|
||||
@version_bp.route('/version/latest')
|
||||
def api_version_latest():
|
||||
"""GET 获取最新版本的下载链接(按创建时间取最新一条)"""
|
||||
"""GET 获取最新版本的下载链接(取最新一条)
|
||||
|
||||
排序改用 id 而非 created_at:同一秒内写入多行时 created_at 无法定序,会取错版本。
|
||||
"""
|
||||
conn = None
|
||||
try:
|
||||
conn = get_db()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT version, file_url FROM web_config ORDER BY created_at DESC LIMIT 1"
|
||||
"SELECT version, file_url FROM web_config ORDER BY id DESC LIMIT 1"
|
||||
)
|
||||
row = cur.fetchone()
|
||||
conn.close()
|
||||
if not row:
|
||||
return jsonify({'version': None, 'file_url': None})
|
||||
return jsonify({
|
||||
@@ -39,3 +42,10 @@ def api_version_latest():
|
||||
except Exception as e:
|
||||
current_app.logger.error('[version] internal error: %s', e, exc_info=True)
|
||||
return jsonify({'error': '服务器内部错误,请稍后重试'}), 500
|
||||
finally:
|
||||
# 异常路径同样要关连接:此前 conn.close() 写在 try 内,查询抛错即泄漏连接
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
+10
-75
@@ -1,10 +1,8 @@
|
||||
"""
|
||||
数据库连接与初始化
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import pymysql
|
||||
from werkzeug.security import generate_password_hash
|
||||
|
||||
try:
|
||||
from config import mysql_host as config_mysql_host
|
||||
@@ -57,30 +55,17 @@ def get_db():
|
||||
)
|
||||
|
||||
|
||||
def _create_initial_admin():
|
||||
"""若没有任何管理员,则创建默认超级管理员(首次启动时,仅一个)"""
|
||||
try:
|
||||
conn = get_db()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id FROM users WHERE role = 'super_admin' LIMIT 1")
|
||||
if cur.fetchone():
|
||||
conn.close()
|
||||
return
|
||||
admin_user = os.environ.get('ADMIN_USER', 'admin')
|
||||
admin_pwd = os.environ.get('ADMIN_PASSWORD', 'admin123')
|
||||
pwd_hash = generate_password_hash(admin_pwd, method='pbkdf2:sha256')
|
||||
cur.execute(
|
||||
"INSERT INTO users (username, password_hash, is_admin, role) VALUES (%s, %s, 1, 'super_admin')",
|
||||
(admin_user, pwd_hash)
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def init_db():
|
||||
"""初始化数据库表,若不存在则创建"""
|
||||
"""确保版本公开 API 依赖的最小表结构存在。
|
||||
|
||||
本进程(15124)只服务 /api/version、/api/version/latest,仅依赖 web_config 表。
|
||||
|
||||
历史遗留说明(2026-09 全维度审查后删除):这里原本还会创建 users 表、执行 role 角色
|
||||
迁移、并在查不到 super_admin 时用 ADMIN_PASSWORD(默认 admin123)插入一个超管。
|
||||
管理后台早已迁到 Java,users 表结构由 Flyway 迁移管理,本进程再写会与之冲突;
|
||||
其中「无超管即用默认密码建超管」在生产等同于后门(删除超管或改 role 枚举后本进程
|
||||
重启就会静默重建一个密码已知的超管),故整段移除。
|
||||
"""
|
||||
conn = pymysql.connect(
|
||||
host=mysql_host,
|
||||
user=mysql_user,
|
||||
@@ -92,44 +77,6 @@ def init_db():
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"CREATE DATABASE IF NOT EXISTS `{db_name}` DEFAULT CHARSET utf8mb4")
|
||||
cur.execute(f"USE `{db_name}`")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
username VARCHAR(64) NOT NULL UNIQUE,
|
||||
password_hash VARCHAR(256) NOT NULL,
|
||||
is_admin TINYINT(1) DEFAULT 0,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS image_history (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
user_id INT NOT NULL,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
panel_type VARCHAR(64) DEFAULT '',
|
||||
original_urls JSON,
|
||||
params JSON,
|
||||
result_urls JSON,
|
||||
long_image_url VARCHAR(1024) DEFAULT NULL,
|
||||
INDEX idx_user_created (user_id, created_at DESC)
|
||||
)
|
||||
""")
|
||||
try:
|
||||
cur.execute("ALTER TABLE image_history ADD COLUMN long_image_url VARCHAR(1024) DEFAULT NULL")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
cur.execute("ALTER TABLE users ADD COLUMN machine VARCHAR(64) DEFAULT NULL")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
cur.execute("ALTER TABLE users ADD COLUMN role VARCHAR(20) DEFAULT 'normal'")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
cur.execute("ALTER TABLE users ADD COLUMN created_by_id INT NULL")
|
||||
except Exception:
|
||||
pass
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS web_config (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
@@ -138,18 +85,6 @@ def init_db():
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
try:
|
||||
cur.execute("UPDATE users SET role = 'normal' WHERE (role IS NULL OR role = '') AND (is_admin = 0 OR is_admin IS NULL)")
|
||||
cur.execute("SELECT MIN(id) AS mid FROM users WHERE is_admin = 1")
|
||||
row = cur.fetchone()
|
||||
if row and row.get('mid'):
|
||||
mid = row['mid']
|
||||
cur.execute("UPDATE users SET role = 'super_admin' WHERE id = %s", (mid,))
|
||||
cur.execute("UPDATE users SET role = 'admin' WHERE is_admin = 1 AND id != %s", (mid,))
|
||||
cur.execute("UPDATE users SET created_by_id = %s WHERE role = 'admin' AND (created_by_id IS NULL)", (mid,))
|
||||
except Exception:
|
||||
pass
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
_create_initial_admin()
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
<template>
|
||||
<el-config-provider :locale="zhCn">
|
||||
<router-view />
|
||||
</el-config-provider>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
// 数富AI SPA 根组件:页面均由路由渲染(URL 无 .html 后缀)
|
||||
//
|
||||
// Element Plus 改为按需引入(见 vite.config.ts 的 ElementPlusResolver)后,locale 不再通过
|
||||
// app.use(ElementPlus, { locale }) 全局注入,改由 ConfigProvider 提供;
|
||||
// 否则分页/日期选择器/上传等组件的内置文案会退回英文。
|
||||
import zhCn from 'element-plus/es/locale/lang/zh-cn'
|
||||
</script>
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
import { createApp } from 'vue'
|
||||
import ElementPlus from 'element-plus'
|
||||
import zhCn from 'element-plus/es/locale/lang/zh-cn'
|
||||
import 'element-plus/dist/index.css'
|
||||
import '@/styles/main.css'
|
||||
|
||||
import App from '@/App.vue'
|
||||
@@ -15,6 +12,11 @@ import { ensureApiSecretsLoaded, loadApiSecrets } from '@/shared/utils/api-secre
|
||||
* 原 MPA 的 22 个 html 入口 + 22 个 *-main.ts 已合并:
|
||||
* 页面路由见 src/router/index.ts;登录态由路由守卫统一引导;
|
||||
* 密钥门禁:服务端密钥未配置完整时全站拦截到 /setup-secrets(拉取失败软失败放行)。
|
||||
*
|
||||
* Element Plus 走按需引入(vite.config.ts 的 ElementPlusResolver):
|
||||
* 此前 `app.use(ElementPlus)` + 全量 index.css 会把整套组件与样式打进入口
|
||||
* (入口 JS 实测 1.05MB,而组件 resolver 的分包优势全部失效)。
|
||||
* locale 由 App.vue 的 el-config-provider 提供。
|
||||
*/
|
||||
|
||||
// 后台预热密钥包,减少首次进入守卫时的等待
|
||||
@@ -51,4 +53,4 @@ router.beforeEach((to) => {
|
||||
return true
|
||||
})
|
||||
|
||||
createApp(App).use(router).use(ElementPlus, { locale: zhCn }).mount('#app')
|
||||
createApp(App).use(router).mount('#app')
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user