fix: 全维度审查修复(安全/正确性/性能/稳定性/客户端/前端)
安全 - /api/ziniao/** 五个匿名接口加管理员鉴权(此前可匿名换取任意员工店铺登录令牌) - 删除 Flask 遗留后门:默认密码建超管 + 每次启动写生产 users 表(服务端与客户端各一份) - 进度/详情接口归属过滤:新增 TaskProgressOwnershipSupport,11 模块 progress/light 与 /tasks/batch 接入,DTO 补 userId,前端 13 个查询封装补传(未传时后端不过滤,兼容旧端) - 代理提取链接(含账密)不再明文入日志(新增 common/util/SecretMasking) - 全局异常兜底不再回传原始异常信息;内部令牌比较改常量时间 - 登录加失败计数与锁定(10 次锁 15 分钟);品牌源文件下载加 SSRF 防护 - AdminApiGuardFilter 覆盖前缀从 2 扩到 15(开关默认 false,行为不变,为收紧做准备) - 生产关闭 springdoc/knife4j(/doc.html 匿名可读全部接口定义) 正确性 - 40901/40902 拆分:锁竞争不再被伪装成 success=true(此前客户端停止重试、分片静默丢失) - 假成功收敛:集采明细批量写失败改为抛出、去重 worker 异常标失败、4 个 worker 改判 success 字段、publish 空 ASIN 行参与批次 flush、巡店删除全失败带 error 上报 - 客户端心跳 discard 移入 finally(7 模块,失败路径不再留僵尸 RUNNING 任务) - 状态机条件更新:跟价停止循环、集采 activate/fail、imagevideo 归档回填、店铺匹配提交 性能 - 前端入口包 JS 1.05MB→204KB、CSS 355KB→10.7KB(Element Plus 改按需 + el-config-provider) - 载荷引用计数按指针里的 taskId 收敛(原 JSON 列 IN 全表扫且逐行调用) - 店铺明细多值批量 INSERT;快照 upsert 预载缓存;结果文件列改单条 UPDATE - 新增迁移 V120(补 3 个缺失索引)/V121(删 4 个被覆盖的冗余索引)/V122(URL 前缀索引) 稳定性 - 新增 common/util/ThreadPools 有界线程池替换 5 处无界队列(防堆积 OOM) - Redis 锁释放改 Lua 原子校验(原裸 delete 会误删他人已过期的锁) - imagevideo 加死节点接管;锁续期失败重试;调度池 4→16;openStream 全部加超时 - 事务内远程对象删除移到提交后;启动恢复锁按实例命名 客户端 - 不再 taskkill /f /im chrome.exe(改为按调试端口精准回收,不杀用户自己的浏览器) - 密码检测不再无条件杀紫鸟进程;品牌检测加全局互斥(代理池不再互相覆盖) - base_dir 统一到 exe 目录(原被 os.getcwd() 覆盖,日志/缓存会分裂两个目录) - 缓存加定时清理;图片下载加超时;mkstemp 句柄托管 测试 - 同步更新受影响的契约测试(构造器签名/条件更新/方法改名/新增接口方法等) - 修复 FaultInjectionTest 等 3 处 mock 未 stub 流式 read 导致的读循环 OOM - mvn test 2795 个测试全绿
This commit is contained in:
+7
-2
@@ -29,6 +29,7 @@ import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskProgressLightRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressOwnershipSupport;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@@ -37,6 +38,7 @@ import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
public class CollectDataController {
|
||||
|
||||
private final CollectDataService service;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
|
||||
@PostMapping("/parse")
|
||||
@Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件,按行入库到 biz_collect_data_item,并保存任务筛选条件。任务初始状态为 PENDING。")
|
||||
@@ -112,7 +114,9 @@ public class CollectDataController {
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量查询任务进度")
|
||||
public ApiResponse<CollectDataTaskBatchVo> progressBatch(@Valid @RequestBody CollectDataTaskBatchRequest request) {
|
||||
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
||||
// 归属过滤:只查询属于该用户的任务(userId 未传时保持原行为,兼容未升级调用方)
|
||||
return ApiResponse.success(service.progressBatch(progressOwnershipSupport
|
||||
.filterOwnedTaskIds(request.getTaskIds(), request.getUserId())));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/light")
|
||||
@@ -120,7 +124,8 @@ public class CollectDataController {
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileError/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<TaskProgressLightBatchVo> progressLight(@Valid @RequestBody TaskProgressLightRequest request) {
|
||||
return ApiResponse.success(service.progressLight(request.getTaskIds()));
|
||||
// 透传 userId:此前传 null = 不过滤归属,匿名遍历 taskId 即可读他人任务状态(2026-09 审查)
|
||||
return ApiResponse.success(service.progressLight(request.getTaskIds(), request.getUserId()));
|
||||
}
|
||||
|
||||
@DeleteMapping("/tasks/{taskId}")
|
||||
|
||||
+4
@@ -12,4 +12,8 @@ public class CollectDataTaskBatchRequest {
|
||||
@NotEmpty
|
||||
@Schema(description = "任务 ID 列表", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<Long> taskIds;
|
||||
|
||||
/** 当前用户 ID(可省略;传入时仅返回该用户的任务,用于进度接口归属过滤)。 */
|
||||
@Schema(description = "当前用户 ID(可省略;传入时仅返回该用户的任务)", example = "1")
|
||||
private Long userId;
|
||||
}
|
||||
|
||||
+50
-28
@@ -4,9 +4,11 @@ import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.crypto.digest.DigestUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
|
||||
@@ -104,8 +106,8 @@ public class CollectDataService {
|
||||
*/
|
||||
private static final String LEGACY_MODULE_TYPE = "collectdata";
|
||||
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds, Long userId) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, userId, taskIds);
|
||||
}
|
||||
public static final int DEFAULT_PAGE_SIZE = 50;
|
||||
|
||||
@@ -379,34 +381,45 @@ public class CollectDataService {
|
||||
@Transactional
|
||||
public void activateTask(Long taskId, Long userId) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
|
||||
// 条件更新:上面的「已结束」判断与写入之间存在窗口(TOCTOU),期间 /fail 可能已把任务
|
||||
// 标为 FAILED —— 整行 updateById 会把它复活成 RUNNING(前端显示"执行中"但无人推进)
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.notIn(FileTaskEntity::getStatus, STATUS_SUCCESS, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated == 0) {
|
||||
throw new BusinessException("任务已结束");
|
||||
}
|
||||
task.setStatus(STATUS_RUNNING);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void failTask(Long taskId, Long userId, String error) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
|
||||
return;
|
||||
}
|
||||
String message = firstNonBlank(error, "collect-data task dispatch failed");
|
||||
FileResultEntity result = ensureTaskResult(task);
|
||||
CollectDataStats stats = loadStats(task);
|
||||
result.setSuccess(0);
|
||||
result.setErrorMessage(message);
|
||||
result.setRowCount(stats.finalRowCount);
|
||||
fileResultMapper.updateById(result);
|
||||
task.setStatus(STATUS_FAILED);
|
||||
task.setErrorMessage(message);
|
||||
task.setFailedFileCount(1);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
persistStats(task, stats);
|
||||
fileTaskMapper.updateById(task);
|
||||
boolean alreadyTerminal = STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus());
|
||||
if (!alreadyTerminal) {
|
||||
result.setSuccess(0);
|
||||
result.setErrorMessage(message);
|
||||
result.setRowCount(stats.finalRowCount);
|
||||
fileResultMapper.updateById(result);
|
||||
persistStats(task, stats);
|
||||
}
|
||||
// 条件更新:客户端报错(/fail)与结果文件组装完成(processResultFileJob 写 SUCCESS)
|
||||
// 可能并发 —— 无条件 updateById 会把已生成的 SUCCESS 覆盖成 FAILED(用户拿不到下载)或反之
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.notIn(FileTaskEntity::getStatus, STATUS_SUCCESS, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getErrorMessage, message)
|
||||
.set(FileTaskEntity::getFailedFileCount, 1)
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated == 0) {
|
||||
log.info("[collect-data] failTask 跳过写入:任务已是终态 taskId={} status={}", taskId, task.getStatus());
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
@@ -954,13 +967,21 @@ public class CollectDataService {
|
||||
// 复用同一份 stats 更新 finalRowCount 后再持久化,避免重复 loadStats 丢失 summaries。
|
||||
stats.finalRowCount = (int) finalRowCount;
|
||||
persistStats(task, stats);
|
||||
task.setStatus(STATUS_SUCCESS);
|
||||
task.setSuccessFileCount(1);
|
||||
task.setFailedFileCount(0);
|
||||
task.setErrorMessage(null);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
// 条件更新:任务可能已被 /fail 标为 FAILED(客户端报错与结果文件组装并发)——
|
||||
// 无条件 updateById 会把 FAILED 覆盖成 SUCCESS(错误信息被清、用户看到「假成功」)。
|
||||
// 结果文件已生成,故失败态下仍保留文件,只是不覆盖状态。
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.ne(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getStatus, STATUS_SUCCESS)
|
||||
.set(FileTaskEntity::getSuccessFileCount, 1)
|
||||
.set(FileTaskEntity::getFailedFileCount, 0)
|
||||
.set(FileTaskEntity::getErrorMessage, null)
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated == 0) {
|
||||
log.warn("[collect-data] 结果文件已生成但任务已是 FAILED,保留失败态不覆盖 taskId={}", task.getId());
|
||||
}
|
||||
} finally {
|
||||
FileUtil.del(xlsx);
|
||||
}
|
||||
@@ -1085,7 +1106,8 @@ public class CollectDataService {
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(MODULE_TYPE, taskId, TASK_LOCK_WAIT_MILLIS);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "任务正在处理,请稍后再试");
|
||||
log.warn("[collect-data] 任务锁竞争,拒绝本次提交 taskId={} waitMillis={}", taskId, TASK_LOCK_WAIT_MILLIS);
|
||||
throw new BusinessException(BusinessCodes.TASK_BUSY, "任务正在处理,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
+2
-5
@@ -148,11 +148,8 @@ public class CollectDataResultDetailCodec {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
// HexFormat 替代逐字节 String.format("%02x"):后者每次走 32 次 Formatter(热路径按行调用)
|
||||
return java.util.HexFormat.of().formatHex(bytes);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("chunk detail ref hash failed", ex);
|
||||
}
|
||||
|
||||
+15
-5
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.collectdata.util;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
||||
@@ -133,12 +134,14 @@ public class CollectDataResultItemBatchWriter {
|
||||
}
|
||||
|
||||
int written = 0;
|
||||
int failedBatches = 0;
|
||||
for (int from = 0; from < toUpsert.size(); from += batchSize) {
|
||||
int to = Math.min(from + batchSize, toUpsert.size());
|
||||
List<TaskResultItemEntity> batch = toUpsert.subList(from, to);
|
||||
try {
|
||||
written += taskResultItemMapper.upsertBatch(batch);
|
||||
} catch (RuntimeException ex) {
|
||||
failedBatches++;
|
||||
log.warn("[collect-data] upsert result item batch failed, skip batch {}..{} taskId={}",
|
||||
from, to, taskId, ex);
|
||||
// 失败批的新行未落库,从增量计数中扣除,避免任务内累计虚高;
|
||||
@@ -150,6 +153,16 @@ public class CollectDataResultItemBatchWriter {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (failedBatches > 0) {
|
||||
// biz_task_result_item 是结果 Excel「明细」sheet 的唯一数据源:静默跳批会让任务
|
||||
// 以 SUCCESS 收尾但明细缺行,且与「结果文件」sheet 的汇总数量对不上(假成功)。
|
||||
// 抛出让本次 chunk 提交明确失败:worker(search_spider)识别 success=false 后会重试,
|
||||
// 重提按 payload_hash 幂等(已落库行跳过、未落库行补插),最终收敛为完整数据。
|
||||
log.error("[collect-data] 明细写入存在失败批次,拒绝本次提交 taskId={} failedBatches={} totalBatches={}",
|
||||
taskId, failedBatches, (toUpsert.size() + batchSize - 1) / batchSize);
|
||||
throw new BusinessException("采集结果明细写入失败,请稍后重试(失败批次 "
|
||||
+ failedBatches + "/" + ((toUpsert.size() + batchSize - 1) / batchSize) + ")");
|
||||
}
|
||||
return new UpsertCounts(written, skipped, newlyInserted);
|
||||
}
|
||||
|
||||
@@ -157,11 +170,8 @@ public class CollectDataResultItemBatchWriter {
|
||||
try {
|
||||
java.security.MessageDigest digest = java.security.MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
// HexFormat 替代逐字节 String.format("%02x"):后者每次走 32 次 Formatter(热路径按行调用)
|
||||
return java.util.HexFormat.of().formatHex(bytes);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("结果明细 hash 计算失败", ex);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user