Compare commits
12 Commits
1fe3368c5a
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 46be044121 | |||
| e0303f9cba | |||
| 9d39705c77 | |||
| 986df86e89 | |||
| bd359411a9 | |||
| 3634ea1d62 | |||
| 3ce0569c59 | |||
| d2d95f0b71 | |||
| 2a51006888 | |||
| a89de129ea | |||
| ddefcbed56 | |||
| 3137299bfe |
+10
-1
@@ -55,10 +55,19 @@ public class GlobalExceptionHandler {
|
|||||||
? ApiResponse.fail(forwardEx.getMessage())
|
? ApiResponse.fail(forwardEx.getMessage())
|
||||||
: ApiResponse.fail(forwardEx.getCode(), forwardEx.getMessage());
|
: ApiResponse.fail(forwardEx.getCode(), forwardEx.getMessage());
|
||||||
} catch (Exception forwardEx) {
|
} catch (Exception forwardEx) {
|
||||||
|
// 转发失败是**瞬时基础设施故障**(归属实例正在滚动重启),不是业务结论,
|
||||||
|
// 更不能表达成「任务不存活」。原先返回 ApiResponse.fail(40903) —— HTTP 200
|
||||||
|
// 加 data:null,而客户端那句 bool((resp.json().get("data") or {}).get("alive"))
|
||||||
|
// 会把「拿不到数据」折叠成 alive=false,于是客户端把**健康的长任务主动停掉**:
|
||||||
|
// 2026-09-18 任务 28616 就是这么死的(归属节点 server-110 重启窗口内,心跳经
|
||||||
|
// nginx 落到 server-121,转发 3 次 Connection refused 后返回空 data)。
|
||||||
|
// 改为 503 + 空 body:新客户端按状态码判为「未知」继续跑;老客户端因 body 不是
|
||||||
|
// JSON、resp.json() 抛异常,同样落到「未知」。顺带让这类故障在 HTTP 指标里可见
|
||||||
|
// (原先记成 200,监控完全看不到滚动重启期间丢了多少心跳)。
|
||||||
log.warn("[instance-routing] forward failed taskId={} operation={} owner={} current={} msg={}",
|
log.warn("[instance-routing] forward failed taskId={} operation={} owner={} current={} msg={}",
|
||||||
ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(),
|
ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(),
|
||||||
forwardEx.getMessage(), forwardEx);
|
forwardEx.getMessage(), forwardEx);
|
||||||
return ApiResponse.fail(40903, "任务归属实例转发失败: " + forwardEx.getMessage());
|
return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import io.jsonwebtoken.Claims;
|
|||||||
import jakarta.servlet.http.Cookie;
|
import jakarta.servlet.http.Cookie;
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.http.HttpHeaders;
|
import org.springframework.http.HttpHeaders;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@@ -19,6 +20,7 @@ import java.nio.charset.StandardCharsets;
|
|||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
@Component
|
@Component
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class AdminAuthSupport {
|
public class AdminAuthSupport {
|
||||||
@@ -62,8 +64,30 @@ public class AdminAuthSupport {
|
|||||||
return user;
|
return user;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 当前用户必须是管理员或超级管理员,否则抛 403。 */
|
/**
|
||||||
public AdminUserEntity requireAdmin(HttpServletRequest request) {
|
* 解析当前请求 JWT 中**签名的**设备标识(deviceId claim);识别不出时返回空串。
|
||||||
|
*
|
||||||
|
* <p>无 token、token 过期/非法、内部令牌通道调用一律返回空串——调用方必须把空串
|
||||||
|
* 当作"来源不明"做保守判定,绝不据此放宽任何限制。只认签名 claim,不接受
|
||||||
|
* X-Device-Id 请求头(头由客户端可控,见 {@link DeviceSessionPolicy} 类注释)。</p>
|
||||||
|
*
|
||||||
|
* <p>本方法只做识别、不做鉴权,因此解析失败不抛异常,仅记日志后返回空串,
|
||||||
|
* 避免把匿名/内部调用直接升级成 401。</p>
|
||||||
|
*/
|
||||||
|
public String currentDeviceId(HttpServletRequest request) {
|
||||||
|
String token = resolveToken(request);
|
||||||
|
if (token == null || token.isBlank()) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return DeviceSessionPolicy.claimDeviceId(jwtService.parse(token));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[auth] 解析 token 取设备标识失败,按来源不明处理: {}", ex.getMessage());
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 当前用户必须是管理员或超级管理员,否则抛 403。 */ public AdminUserEntity requireAdmin(HttpServletRequest request) {
|
||||||
AdminUserEntity user = requireUser(request);
|
AdminUserEntity user = requireUser(request);
|
||||||
String role = currentRole(user);
|
String role = currentRole(user);
|
||||||
if (role == null) {
|
if (role == null) {
|
||||||
|
|||||||
@@ -23,6 +23,14 @@ public class BrandCheckProperties {
|
|||||||
* 10 次尝试合计约 45s,既能覆盖限流窗口又不会让单个品牌长时间占住查询线程。
|
* 10 次尝试合计约 45s,既能覆盖限流窗口又不会让单个品牌长时间占住查询线程。
|
||||||
*/
|
*/
|
||||||
private int retryMaxIntervalMillis = 10000;
|
private int retryMaxIntervalMillis = 10000;
|
||||||
|
/**
|
||||||
|
* 一次批量检查的总耗时上限(毫秒):整批用尽后不再重试,剩余品牌按「查询失败」收尾。
|
||||||
|
* 上限的意义不是省时间,而是给「分片回传」这类同步调用方一个时延上界——上游卡死时
|
||||||
|
* 单品牌 10 次重试曾把一次回传拖到 103.5 秒(taskId 28599),客户端重试预算耗尽后
|
||||||
|
* 中止了整个采集。首轮请求始终执行,故上游只是略慢时不会误降级。
|
||||||
|
* 设为 0 或负数表示不限制。
|
||||||
|
*/
|
||||||
|
private int totalTimeoutMillis = 90000;
|
||||||
private int connectTimeoutMillis = 10000;
|
private int connectTimeoutMillis = 10000;
|
||||||
private int readTimeoutMillis = 60000;
|
private int readTimeoutMillis = 60000;
|
||||||
}
|
}
|
||||||
|
|||||||
+74
-4
@@ -424,6 +424,53 @@ public class AppearancePatentTaskService {
|
|||||||
|
|
||||||
public void submitResult(Long taskId, AppearancePatentSubmitResultRequest request) {
|
public void submitResult(Long taskId, AppearancePatentSubmitResultRequest request) {
|
||||||
submitResultLocked(taskId, request);
|
submitResultLocked(taskId, request);
|
||||||
|
// 分片补传后自动恢复终态失败的组装 job —— 走与删除品牌同一套口径,
|
||||||
|
// 此前外观专利没有接该入口,分片补齐后只能人工重置 job(线上任务 28459 即如此)。
|
||||||
|
maybeRecoverTerminalFailedAssemble(taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 补传恢复:此前因分片缺失导致组装 job 重试耗尽(终态失败),客户端补传缺口后
|
||||||
|
* 把失败的组装 job 重置为 PENDING 重新派发({@code resetTerminalFailedForRecovery} 自行补发 dispatch 事件)。
|
||||||
|
*
|
||||||
|
* <p>best-effort:恢复失败不得影响补传本身——分片已经落库,恢复只是让后续组装继续推进。
|
||||||
|
* 常态(无终态失败 job)下只查两次即返回,不触发分片扫描。
|
||||||
|
*/
|
||||||
|
private void maybeRecoverTerminalFailedAssemble(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
FileResultEntity result = findResultRecord(taskId);
|
||||||
|
if (result == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (taskFileJobService.hasSuccessfulAssembleJob(taskId, MODULE_TYPE, result.getId())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!taskFileJobService.isTerminalFailedAssembleJob(taskId, MODULE_TYPE, result.getId())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!isResultSubmissionComplete(taskId)) {
|
||||||
|
log.info("[appearance-patent] 组装 job 终态失败但分片仍未补传完整,暂不恢复 taskId={} resultId={}",
|
||||||
|
taskId, result.getId());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info("[appearance-patent] 分片已补传完整,恢复终态失败的组装 job taskId={} resultId={}",
|
||||||
|
taskId, result.getId());
|
||||||
|
taskFileJobService.resetTerminalFailedForRecovery(taskId, MODULE_TYPE, result.getId());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[appearance-patent] 补传恢复检查失败(不影响本次补传)taskId={} err={}", taskId, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 按 task 取结果行(不创建);不存在返回 null。 */
|
||||||
|
private FileResultEntity findResultRecord(Long taskId) {
|
||||||
|
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||||
|
.eq(FileResultEntity::getTaskId, taskId)
|
||||||
|
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.last("limit 1"));
|
||||||
|
return rows == null || rows.isEmpty() ? null : rows.getFirst();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void submitResultLocked(Long taskId, AppearancePatentSubmitResultRequest request) {
|
private void submitResultLocked(Long taskId, AppearancePatentSubmitResultRequest request) {
|
||||||
@@ -1223,6 +1270,7 @@ public class AppearancePatentTaskService {
|
|||||||
if (rows == null || rows.isEmpty()) {
|
if (rows == null || rows.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
String conflictDetail = "未发生冲突";
|
||||||
for (int attempt = 1; attempt <= CHUNK_PAYLOAD_MERGE_RETRY_LIMIT; attempt++) {
|
for (int attempt = 1; attempt <= CHUNK_PAYLOAD_MERGE_RETRY_LIMIT; attempt++) {
|
||||||
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
@@ -1264,13 +1312,35 @@ public class AppearancePatentTaskService {
|
|||||||
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
// CAS 冲突:读回行上的当前哈希,否则只有一句 conflict 无从判断是谁改的(线上 28459 至今未能定位)
|
||||||
|
String currentHash = currentPayloadHash(chunk.getId());
|
||||||
|
conflictDetail = "期望hash=" + oldPayloadHash + " 当前hash=" + currentHash;
|
||||||
if (attempt < CHUNK_PAYLOAD_MERGE_RETRY_LIMIT) {
|
if (attempt < CHUNK_PAYLOAD_MERGE_RETRY_LIMIT) {
|
||||||
log.warn("[appearance-patent] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{}",
|
log.warn("[appearance-patent] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{} 期望hash={} 当前hash={}",
|
||||||
taskId, scopeHash, chunkIndex, attempt, CHUNK_PAYLOAD_MERGE_RETRY_LIMIT);
|
taskId, scopeHash, chunkIndex, attempt, CHUNK_PAYLOAD_MERGE_RETRY_LIMIT, oldPayloadHash, currentHash);
|
||||||
|
// 还要重试:这次写的对象会被下次重写,先删掉避免堆积
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||||
|
} else {
|
||||||
|
// 终局失败:保留这次写入的版本化对象,作为读路径「同槽位兄弟对象」兜底的恢复源。
|
||||||
|
// 行没指过去不该让该分片永久判死——删掉它才是线上 28459 丢数据的形态。
|
||||||
|
log.error("[appearance-patent] chunk payload 合并冲突重试耗尽,保留兜底对象 taskId={} scopeHash={} chunk={} 期望hash={} 当前hash={} 保留对象={}",
|
||||||
|
taskId, scopeHash, chunkIndex, oldPayloadHash, currentHash, storedPayload);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
throw new IllegalStateException("appearance patent chunk payload update conflict");
|
throw new IllegalStateException("appearance patent chunk payload update conflict " + conflictDetail);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 读回 chunk 行当前的 payload_hash,用于 CAS 冲突定位(行已不存在/读取失败时返回可读标记)。 */
|
||||||
|
private String currentPayloadHash(Long chunkId) {
|
||||||
|
if (chunkId == null) {
|
||||||
|
return "chunkId 为空";
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
TaskChunkEntity latest = taskChunkMapper.selectById(chunkId);
|
||||||
|
return latest == null ? "行已不存在" : latest.getPayloadHash();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
return "读取失败:" + ex.getMessage();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives,
|
private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives,
|
||||||
|
|||||||
+14
-2
@@ -95,10 +95,14 @@ public class BrandCheckClient {
|
|||||||
|
|
||||||
public BrandCheckBatchResult checkAll(List<String> brands, String strategy) {
|
public BrandCheckBatchResult checkAll(List<String> brands, String strategy) {
|
||||||
List<String> distinctBrands = distinctNonBlank(brands);
|
List<String> distinctBrands = distinctNonBlank(brands);
|
||||||
|
// 整批共用一个耗时预算:上游 16890 卡死时,单品牌 10 次重试曾把一次分片回传拖到
|
||||||
|
// 103.5 秒(taskId 28599),客户端重试预算耗尽后中止了整个采集。预算用尽即停止重试。
|
||||||
|
long budgetMillis = properties.getTotalTimeoutMillis();
|
||||||
|
long deadlineNanos = budgetMillis > 0L ? System.nanoTime() + budgetMillis * 1_000_000L : Long.MAX_VALUE;
|
||||||
List<CompletableFuture<BrandCheckOutcome>> futures = new ArrayList<>(distinctBrands.size());
|
List<CompletableFuture<BrandCheckOutcome>> futures = new ArrayList<>(distinctBrands.size());
|
||||||
for (String brand : distinctBrands) {
|
for (String brand : distinctBrands) {
|
||||||
futures.add(CompletableFuture.supplyAsync(
|
futures.add(CompletableFuture.supplyAsync(
|
||||||
() -> checkOneBrand(brand, strategy), checkExecutor));
|
() -> checkOneBrand(brand, strategy, deadlineNanos), checkExecutor));
|
||||||
}
|
}
|
||||||
List<Object> failedData = new ArrayList<>();
|
List<Object> failedData = new ArrayList<>();
|
||||||
List<Object> queryFailedData = new ArrayList<>();
|
List<Object> queryFailedData = new ArrayList<>();
|
||||||
@@ -110,11 +114,19 @@ public class BrandCheckClient {
|
|||||||
return new BrandCheckBatchResult(distinctBrands, failedData, queryFailedData);
|
return new BrandCheckBatchResult(distinctBrands, failedData, queryFailedData);
|
||||||
}
|
}
|
||||||
|
|
||||||
private BrandCheckOutcome checkOneBrand(String brand, String strategy) {
|
private BrandCheckOutcome checkOneBrand(String brand, String strategy, long deadlineNanos) {
|
||||||
int attempts = Math.max(1, properties.getRetryTimes());
|
int attempts = Math.max(1, properties.getRetryTimes());
|
||||||
BrandCheckResponse response = null;
|
BrandCheckResponse response = null;
|
||||||
Exception lastFailure = null;
|
Exception lastFailure = null;
|
||||||
for (int attempt = 1; attempt <= attempts; attempt++) {
|
for (int attempt = 1; attempt <= attempts; attempt++) {
|
||||||
|
// 预算用尽就不再重试,按查询失败收尾。只掐「重试」不打断已发出的请求,
|
||||||
|
// 故最坏耗时 ≈ 预算 + 一次请求的读超时;首轮始终执行,避免上游只是慢一点时被误降级。
|
||||||
|
if (attempt > 1 && System.nanoTime() >= deadlineNanos) {
|
||||||
|
log.warn("[brand-check] 整批耗时预算用尽,停止重试 brand={} attempt={}/{} lastErr={}",
|
||||||
|
brand, attempt, attempts,
|
||||||
|
lastFailure == null ? "query_faild_data 持续非空" : lastFailure.getMessage());
|
||||||
|
break;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
response = check(brand, strategy);
|
response = check(brand, strategy);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
|
|||||||
+79
-36
@@ -782,6 +782,39 @@ public class CollectDataService {
|
|||||||
throw new BusinessException("request is empty");
|
throw new BusinessException("request is empty");
|
||||||
}
|
}
|
||||||
ensureRustfsPayloadStorageEnabled();
|
ensureRustfsPayloadStorageEnabled();
|
||||||
|
|
||||||
|
// 锁外预检:任务不存在/已结束时立即失败,不为终态任务白跑去重查询与品牌检测。
|
||||||
|
// 只做快速失败,并发正确性仍由锁内的重读复核保证。
|
||||||
|
FileTaskEntity probe = fileTaskMapper.selectById(taskId);
|
||||||
|
if (probe == null || !MODULE_TYPE.equals(probe.getModuleType())) {
|
||||||
|
throw new BusinessException("任务不存在");
|
||||||
|
}
|
||||||
|
if (STATUS_SUCCESS.equals(probe.getStatus()) || STATUS_FAILED.equals(probe.getStatus())) {
|
||||||
|
log.warn("[collect-data] 任务已结束,拒绝重复提交 taskId={} status={}", taskId, probe.getStatus());
|
||||||
|
throw new BusinessException(40901, "任务已结束,拒绝重复提交");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 归一化 / 去重过滤 / 品牌检测放在锁外:品牌检测是同步远程调用,上游 16890 抖动时
|
||||||
|
// 单品牌 10 次重试合计上百秒(taskId 28599 实测:chunk 回传在锁内等品牌检测 103.5 秒,
|
||||||
|
// 期间心跳与客户端重试全部撞 40902 拿不到锁,客户端 5 次重试预算耗尽后中止整个采集)。
|
||||||
|
// 这几步只依赖本批入参、不写任务状态,放锁外不改变 chunk 落库的串行语义。
|
||||||
|
List<CollectDataResultRowVo> rows = normalizeSubmitRows(request.getItems());
|
||||||
|
buildParseLimits().validateChunkRowCount(rows.size());
|
||||||
|
// 「结果文件」sheet 需要 Python 回传的全量数据(不经任何后端过滤),
|
||||||
|
// 因此 chunk payload 写入未筛除空 ASIN 的全量 rows;
|
||||||
|
// 仅当 ASIN 非空时才进入去重 / 无效品牌 / 品牌检测链路并落 biz_task_result_item。
|
||||||
|
List<CollectDataResultRowVo> rowsForFiltering = new ArrayList<>(rows.size());
|
||||||
|
for (CollectDataResultRowVo row : rows) {
|
||||||
|
if (row.getAsin() != null && !row.getAsin().isBlank()) {
|
||||||
|
rowsForFiltering.add(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
long prepareStartAt = System.currentTimeMillis();
|
||||||
|
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(rowsForFiltering);
|
||||||
|
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome = brandBatchFilter.filter(filtered.kept());
|
||||||
|
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms",
|
||||||
|
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt);
|
||||||
|
|
||||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||||
@@ -819,25 +852,23 @@ public class CollectDataService {
|
|||||||
return buildSubmitVo(task, result, chunkIndex, chunkTotal, request, 0);
|
return buildSubmitVo(task, result, chunkIndex, chunkTotal, request, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
List<CollectDataResultRowVo> rows = normalizeSubmitRows(request.getItems());
|
|
||||||
buildParseLimits().validateChunkRowCount(rows.size());
|
|
||||||
CollectDataStats stats = loadStats(task);
|
CollectDataStats stats = loadStats(task);
|
||||||
stats.receivedRows += rows.size();
|
stats.receivedRows += rows.size();
|
||||||
stats.currentChunkRows = rows.size();
|
stats.currentChunkRows = rows.size();
|
||||||
|
|
||||||
// 「结果文件」sheet 需要 Python 回传的全量数据(不经任何后端过滤),
|
|
||||||
// 因此 chunk payload 写入未筛除空 ASIN 的全量 rows;
|
|
||||||
// 仅当 ASIN 非空时才进入去重 / 无效品牌 / 品牌检测链路并落 biz_task_result_item。
|
|
||||||
List<CollectDataResultRowVo> rowsForFiltering = new ArrayList<>(rows.size());
|
|
||||||
for (CollectDataResultRowVo row : rows) {
|
|
||||||
if (row.getAsin() != null && !row.getAsin().isBlank()) {
|
|
||||||
rowsForFiltering.add(row);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(rowsForFiltering);
|
|
||||||
stats.dedupeFilteredCount += filtered.dedupeFilteredCount();
|
stats.dedupeFilteredCount += filtered.dedupeFilteredCount();
|
||||||
stats.invalidFilteredCount += filtered.invalidFilteredCount();
|
stats.invalidFilteredCount += filtered.invalidFilteredCount();
|
||||||
List<CollectDataResultRowVo> accepted = filterByBrandCheck(filtered.kept(), stats);
|
stats.brandRejectedCount += brandOutcome.rejected().size();
|
||||||
|
stats.brandQueryFailedCount += brandOutcome.queryFailed().size();
|
||||||
|
// 只有被服务端判定为无效品牌(rejected)的行才写入无效品牌表;
|
||||||
|
// queryFailed 是品牌检测服务端到端失败(如缺 X-Token 422/超时/限流),
|
||||||
|
// 一并写入会把故障期间被误伤的品牌永久拉黑,后续同品牌商品全部被
|
||||||
|
// invalidFiltered 过滤(task-27265 实测:通过 Python 过滤的 21 行中
|
||||||
|
// 8 行查询失败被写表后误杀,最终结果 Excel 0 行)。rejected 为空时
|
||||||
|
// 不触发任何写入,避免空批次无意义调用。
|
||||||
|
if (!brandOutcome.rejected().isEmpty()) {
|
||||||
|
invalidAsinBatchWriter.writeBatch(brandOutcome.rejected());
|
||||||
|
}
|
||||||
|
List<CollectDataResultRowVo> accepted = brandOutcome.accepted();
|
||||||
// 结果明细改为 chunk 级存储:整个 chunk 的 accepted 行共享一个
|
// 结果明细改为 chunk 级存储:整个 chunk 的 accepted 行共享一个
|
||||||
// RustFS 对象(deterministic key,同 chunk 重提覆盖同一对象),
|
// RustFS 对象(deterministic key,同 chunk 重提覆盖同一对象),
|
||||||
// biz_task_result_item.payload_json 只存 {chunk, offset, payload} 引用。
|
// biz_task_result_item.payload_json 只存 {chunk, offset, payload} 引用。
|
||||||
@@ -872,6 +903,14 @@ public class CollectDataService {
|
|||||||
|
|
||||||
if (request.getError() != null && !request.getError().isBlank()) {
|
if (request.getError() != null && !request.getError().isBlank()) {
|
||||||
markTaskFailed(task, result, request.getError(), stats);
|
markTaskFailed(task, result, request.getError(), stats);
|
||||||
|
// 失败但已收到分片:照常组装结果文件,让用户能下载已采集的数据。
|
||||||
|
// 此前失败分支只标失败不组装,已落库的数据也没有任何结果文件可下载
|
||||||
|
// (taskId 28599:55 个分片全部收到、187 行明细已落库,用户却拿不到文件)。
|
||||||
|
if (hasReceivedChunks(taskId)) {
|
||||||
|
enqueueFinalWorkbook(task, result, stats);
|
||||||
|
log.warn("[collect-data] 任务失败仍组装部分结果 taskId={} error={} finalRows={}",
|
||||||
|
taskId, request.getError(), stats.finalRowCount);
|
||||||
|
}
|
||||||
} else if (Boolean.TRUE.equals(request.getDone())) {
|
} else if (Boolean.TRUE.equals(request.getDone())) {
|
||||||
enqueueFinalWorkbook(task, result, stats);
|
enqueueFinalWorkbook(task, result, stats);
|
||||||
} else {
|
} else {
|
||||||
@@ -933,22 +972,6 @@ public class CollectDataService {
|
|||||||
return rows;
|
return rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<CollectDataResultRowVo> filterByBrandCheck(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
|
|
||||||
CollectDataBrandBatchFilter.BrandBatchOutcome outcome = brandBatchFilter.filter(rows);
|
|
||||||
stats.brandRejectedCount += outcome.rejected().size();
|
|
||||||
stats.brandQueryFailedCount += outcome.queryFailed().size();
|
|
||||||
// 只有被服务端判定为无效品牌(rejected)的行才写入无效品牌表;
|
|
||||||
// queryFailed 是品牌检测服务端到端失败(如缺 X-Token 422/超时/限流),
|
|
||||||
// 一并写入会把故障期间被误伤的品牌永久拉黑,后续同品牌商品全部被
|
|
||||||
// invalidFiltered 过滤(task-27265 实测:通过 Python 过滤的 21 行中
|
|
||||||
// 8 行查询失败被写表后误杀,最终结果 Excel 0 行)。rejected 为空时
|
|
||||||
// 不触发任何写入,避免空批次无意义调用。
|
|
||||||
if (!outcome.rejected().isEmpty()) {
|
|
||||||
invalidAsinBatchWriter.writeBatch(outcome.rejected());
|
|
||||||
}
|
|
||||||
return outcome.accepted();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void persistChunk(Long taskId,
|
private void persistChunk(Long taskId,
|
||||||
String scopeKey,
|
String scopeKey,
|
||||||
String scopeHash,
|
String scopeHash,
|
||||||
@@ -1011,7 +1034,8 @@ public class CollectDataService {
|
|||||||
result.setResultFileSize(0L);
|
result.setResultFileSize(0L);
|
||||||
result.setResultContentType(CONTENT_TYPE_XLSX);
|
result.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
result.setRowCount(stats.finalRowCount);
|
result.setRowCount(stats.finalRowCount);
|
||||||
result.setErrorMessage(null);
|
// 不清 errorMessage:失败任务的部分结果组装也走这里,清掉会让用户看不到真实失败原因
|
||||||
|
// (成功路径的 errorMessage 本来就为 null,无需清理)。
|
||||||
fileResultMapper.updateById(result);
|
fileResultMapper.updateById(result);
|
||||||
task.setUpdatedAt(LocalDateTime.now());
|
task.setUpdatedAt(LocalDateTime.now());
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
@@ -1060,6 +1084,22 @@ public class CollectDataService {
|
|||||||
stats.summaries,
|
stats.summaries,
|
||||||
batch -> streamRawRows(task.getId(), batch));
|
batch -> streamRawRows(task.getId(), batch));
|
||||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||||
|
|
||||||
|
// 复用同一份 stats 更新 finalRowCount 后再持久化,避免重复 loadStats 丢失 summaries。
|
||||||
|
stats.finalRowCount = (int) finalRowCount;
|
||||||
|
persistStats(task, stats);
|
||||||
|
|
||||||
|
// 失败原因先留存:下面的乐观写入会清空 result.errorMessage,任务已被判失败时要用它恢复。
|
||||||
|
String failureReason = result.getErrorMessage();
|
||||||
|
if (failureReason == null || failureReason.isBlank()) {
|
||||||
|
failureReason = task.getErrorMessage();
|
||||||
|
}
|
||||||
|
if (failureReason == null || failureReason.isBlank()) {
|
||||||
|
failureReason = "任务失败,结果文件为已采集的部分数据";
|
||||||
|
}
|
||||||
|
|
||||||
|
// 结果行先按成功乐观写入:保持「结果文件先于任务成功落库」的时序,
|
||||||
|
// 万一进程在这两步之间退出,任务仍是 RUNNING,会被陈旧巡检重新组装(可自愈)。
|
||||||
result.setResultFilename(filename);
|
result.setResultFilename(filename);
|
||||||
result.setResultFileUrl(objectKey);
|
result.setResultFileUrl(objectKey);
|
||||||
result.setResultFileSize(xlsx.length());
|
result.setResultFileSize(xlsx.length());
|
||||||
@@ -1069,10 +1109,7 @@ public class CollectDataService {
|
|||||||
result.setErrorMessage(null);
|
result.setErrorMessage(null);
|
||||||
fileResultMapper.updateById(result);
|
fileResultMapper.updateById(result);
|
||||||
|
|
||||||
// 复用同一份 stats 更新 finalRowCount 后再持久化,避免重复 loadStats 丢失 summaries。
|
// 条件更新:任务可能已被判失败(客户端上报失败 / 陈旧判死与结果文件组装并发)——
|
||||||
stats.finalRowCount = (int) finalRowCount;
|
|
||||||
persistStats(task, stats);
|
|
||||||
// 条件更新:任务可能已被 /fail 标为 FAILED(客户端报错与结果文件组装并发)——
|
|
||||||
// 无条件 updateById 会把 FAILED 覆盖成 SUCCESS(错误信息被清、用户看到「假成功」)。
|
// 无条件 updateById 会把 FAILED 覆盖成 SUCCESS(错误信息被清、用户看到「假成功」)。
|
||||||
// 结果文件已生成,故失败态下仍保留文件,只是不覆盖状态。
|
// 结果文件已生成,故失败态下仍保留文件,只是不覆盖状态。
|
||||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||||
@@ -1085,7 +1122,13 @@ public class CollectDataService {
|
|||||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||||
if (updated == 0) {
|
if (updated == 0) {
|
||||||
log.warn("[collect-data] 结果文件已生成但任务已是 FAILED,保留失败态不覆盖 taskId={}", task.getId());
|
// 任务已是 FAILED:结果记录改回失败语义并保留真实原因,但文件 URL 照常保留,
|
||||||
|
// 用户看到「失败 + 原因」的同时仍能下载已采集的部分结果。
|
||||||
|
result.setSuccess(0);
|
||||||
|
result.setErrorMessage(failureReason);
|
||||||
|
fileResultMapper.updateById(result);
|
||||||
|
log.warn("[collect-data] 结果文件已生成但任务已是 FAILED,保留失败态与原因 taskId={} rows={} reason={}",
|
||||||
|
task.getId(), finalRowCount, failureReason);
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
FileUtil.del(xlsx);
|
FileUtil.del(xlsx);
|
||||||
|
|||||||
+23
-1
@@ -27,6 +27,7 @@ import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskResumeService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
@@ -92,6 +93,8 @@ public class DeleteBrandStaleTaskService {
|
|||||||
private final ShopDataCrawlTaskService shopDataCrawlTaskService;
|
private final ShopDataCrawlTaskService shopDataCrawlTaskService;
|
||||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||||
|
/** 客户端中断任务的自动续跑(保留失败记录 + 重排队续跑任务,V129)。 */
|
||||||
|
private final TaskResumeService taskResumeService;
|
||||||
|
|
||||||
@Value("${aiimage.temp-dir.retention-hours:24}")
|
@Value("${aiimage.temp-dir.retention-hours:24}")
|
||||||
private long tempDirRetentionHours;
|
private long tempDirRetentionHours;
|
||||||
@@ -114,13 +117,16 @@ public class DeleteBrandStaleTaskService {
|
|||||||
ShopMatchStaleCheckStats queryAsinStats = failStaleQueryAsinTasks();
|
ShopMatchStaleCheckStats queryAsinStats = failStaleQueryAsinTasks();
|
||||||
ShopMatchStaleCheckStats withdrawStats = failStaleWithdrawTasks();
|
ShopMatchStaleCheckStats withdrawStats = failStaleWithdrawTasks();
|
||||||
NoResultUploadStaleCheckStats noUploadStats = failNoResultUploadTasks();
|
NoResultUploadStaleCheckStats noUploadStats = failNoResultUploadTasks();
|
||||||
|
// 客户端重启中断的任务:除了标失败(客户端上报,用户能看到原因),还要重新排队
|
||||||
|
// 一条 PENDING 续跑任务交给客户端兜底拉取执行,否则长任务一遇客户端更新就整个白跑
|
||||||
|
TaskResumeService.ResumeStats resumeStats = resumeInterruptedSafely();
|
||||||
for (Map.Entry<String, Runnable> delegated : delegatedStaleChecks().entrySet()) {
|
for (Map.Entry<String, Runnable> delegated : delegatedStaleChecks().entrySet()) {
|
||||||
runModuleStaleCheck(delegated.getKey(), delegated.getValue());
|
runModuleStaleCheck(delegated.getKey(), delegated.getValue());
|
||||||
}
|
}
|
||||||
// 周期每 2 分钟一轮,各模块 summary 合并为单行,避免定期刷屏
|
// 周期每 2 分钟一轮,各模块 summary 合并为单行,避免定期刷屏
|
||||||
// 注意:每段占位符数量必须与实参一致——此前每段 5 个占位符只传 4 个参数,
|
// 注意:每段占位符数量必须与实参一致——此前每段 5 个占位符只传 4 个参数,
|
||||||
// 导致 withdraw 之后的取值整体错位、末尾 elapsedMs/thread 打成字面量
|
// 导致 withdraw 之后的取值整体错位、末尾 elapsedMs/thread 打成字面量
|
||||||
log.info("[stale-check] summary product-risk(s={} f={} x={} p={}) price-track(s={} f={} x={} p={}) shop-match(s={} f={} x={} p={}) patrol-delete(s={} f={} x={} p={}) query-asin(s={} f={} x={} p={}) withdraw(s={} f={} x={} p={}) no-upload(c={} f={} x={}) elapsedMs={} thread={}",
|
log.info("[stale-check] summary product-risk(s={} f={} x={} p={}) price-track(s={} f={} x={} p={}) shop-match(s={} f={} x={} p={}) patrol-delete(s={} f={} x={} p={}) query-asin(s={} f={} x={} p={}) withdraw(s={} f={} x={} p={}) no-upload(c={} f={} x={}) resume(s={} r={} k={} u={}) elapsedMs={} thread={}",
|
||||||
stats.scannedTaskCount, stats.finalizedTaskCount, stats.failedTaskCount, stats.skippedTaskCount,
|
stats.scannedTaskCount, stats.finalizedTaskCount, stats.failedTaskCount, stats.skippedTaskCount,
|
||||||
priceTrackStats.scannedTaskCount, priceTrackStats.finalizedTaskCount, priceTrackStats.failedTaskCount, priceTrackStats.skippedTaskCount,
|
priceTrackStats.scannedTaskCount, priceTrackStats.finalizedTaskCount, priceTrackStats.failedTaskCount, priceTrackStats.skippedTaskCount,
|
||||||
shopMatchStats.scannedTaskCount, shopMatchStats.finalizedTaskCount, shopMatchStats.failedTaskCount, shopMatchStats.skippedTaskCount,
|
shopMatchStats.scannedTaskCount, shopMatchStats.finalizedTaskCount, shopMatchStats.failedTaskCount, shopMatchStats.skippedTaskCount,
|
||||||
@@ -128,11 +134,27 @@ public class DeleteBrandStaleTaskService {
|
|||||||
queryAsinStats.scannedTaskCount, queryAsinStats.finalizedTaskCount, queryAsinStats.failedTaskCount, queryAsinStats.skippedTaskCount,
|
queryAsinStats.scannedTaskCount, queryAsinStats.finalizedTaskCount, queryAsinStats.failedTaskCount, queryAsinStats.skippedTaskCount,
|
||||||
withdrawStats.scannedTaskCount, withdrawStats.finalizedTaskCount, withdrawStats.failedTaskCount, withdrawStats.skippedTaskCount,
|
withdrawStats.scannedTaskCount, withdrawStats.finalizedTaskCount, withdrawStats.failedTaskCount, withdrawStats.skippedTaskCount,
|
||||||
noUploadStats.scannedTaskCount, noUploadStats.failedTaskCount, noUploadStats.skippedTaskCount,
|
noUploadStats.scannedTaskCount, noUploadStats.failedTaskCount, noUploadStats.skippedTaskCount,
|
||||||
|
resumeStats.scannedTaskCount, resumeStats.resumedTaskCount, resumeStats.skippedTaskCount,
|
||||||
|
resumeStats.unsupportedTaskCount,
|
||||||
System.currentTimeMillis() - startedAt,
|
System.currentTimeMillis() - startedAt,
|
||||||
Thread.currentThread().getName());
|
Thread.currentThread().getName());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 自动续跑巡检的隔离壳:续跑失败绝不能拖垮判死主流程。
|
||||||
|
* 判死是任务状态的兜底(不做会留下永远 RUNNING 的孤儿),优先级高于续跑;
|
||||||
|
* 这里失败只记日志并返回空统计,2 分钟后的下一轮自然重试。
|
||||||
|
*/
|
||||||
|
private TaskResumeService.ResumeStats resumeInterruptedSafely() {
|
||||||
|
try {
|
||||||
|
return taskResumeService.resumeInterruptedTasks();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[task-resume] 自动续跑巡检失败(不影响本轮判死): {}", ex.getMessage(), ex);
|
||||||
|
return new TaskResumeService.ResumeStats();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 委派式陈旧判死:moduleType → 处理动作。
|
* 委派式陈旧判死:moduleType → 处理动作。
|
||||||
* {@code TaskModuleRegistry} 中 delegatedStaleCheck=true 的模块都必须在这里登记,
|
* {@code TaskModuleRegistry} 中 delegatedStaleCheck=true 的模块都必须在这里登记,
|
||||||
|
|||||||
+97
-1
@@ -5,11 +5,14 @@ import io.micrometer.core.instrument.DistributionSummary;
|
|||||||
import io.micrometer.core.instrument.MeterRegistry;
|
import io.micrometer.core.instrument.MeterRegistry;
|
||||||
import io.micrometer.core.instrument.Timer;
|
import io.micrometer.core.instrument.Timer;
|
||||||
import io.minio.GetObjectArgs;
|
import io.minio.GetObjectArgs;
|
||||||
|
import io.minio.ListObjectsArgs;
|
||||||
import io.minio.MinioClient;
|
import io.minio.MinioClient;
|
||||||
import io.minio.PutObjectArgs;
|
import io.minio.PutObjectArgs;
|
||||||
import io.minio.RemoveObjectArgs;
|
import io.minio.RemoveObjectArgs;
|
||||||
|
import io.minio.Result;
|
||||||
import io.minio.StatObjectArgs;
|
import io.minio.StatObjectArgs;
|
||||||
import io.minio.errors.ErrorResponseException;
|
import io.minio.errors.ErrorResponseException;
|
||||||
|
import io.minio.messages.Item;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import okhttp3.ConnectionPool;
|
import okhttp3.ConnectionPool;
|
||||||
import okhttp3.Dispatcher;
|
import okhttp3.Dispatcher;
|
||||||
@@ -21,8 +24,11 @@ import org.springframework.stereotype.Service;
|
|||||||
import java.io.ByteArrayInputStream;
|
import java.io.ByteArrayInputStream;
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
import java.util.UUID;
|
||||||
import java.util.concurrent.Semaphore;
|
import java.util.concurrent.Semaphore;
|
||||||
import java.util.concurrent.ThreadLocalRandom;
|
import java.util.concurrent.ThreadLocalRandom;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
@@ -45,6 +51,12 @@ public class RustfsObjectStorageService {
|
|||||||
"NoSuchKey", "NoSuchBucket", "NoSuchVersion",
|
"NoSuchKey", "NoSuchBucket", "NoSuchVersion",
|
||||||
"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "InvalidBucketName");
|
"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "InvalidBucketName");
|
||||||
|
|
||||||
|
/** 标准 UUID 字符串长度(8-4-4-4-12),用于识别版本化对象 key。 */
|
||||||
|
private static final int UUID_STRING_LENGTH = 36;
|
||||||
|
|
||||||
|
/** 兄弟对象兜底列出的上限:只为找回同槽位的版本化对象,不需要列全。 */
|
||||||
|
private static final int MAX_SIBLING_LIST_KEYS = 50;
|
||||||
|
|
||||||
private final TransientStorageProperties properties;
|
private final TransientStorageProperties properties;
|
||||||
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
|
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
|
||||||
private final ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider;
|
private final ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider;
|
||||||
@@ -106,7 +118,22 @@ public class RustfsObjectStorageService {
|
|||||||
return uploadBytes(objectKey, bytes, verifyAfterUpload);
|
return uploadBytes(objectKey, bytes, verifyAfterUpload);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 三参重载:是否做「上传失败补偿删除」按对象 key 形态自动判定。
|
||||||
|
*
|
||||||
|
* <p>只有版本化 key(末段以 UUID 结尾)是本次写入独占的;确定性 key 会被重传重写复用,
|
||||||
|
* 删它就可能删掉别的 DB 行仍在引用的对象(2026-09-17 线上任务 28459 的载荷对象就是这么丢的)。
|
||||||
|
*/
|
||||||
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload) {
|
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload) {
|
||||||
|
return uploadBytes(objectKey, content, verifyAfterUpload, isVersionedObjectKey(objectKey));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param compensateDeleteOnFailure put 已完成、但后续可见性校验失败时,是否把该对象排进删除补偿队列。
|
||||||
|
* 仅当调用方能确认「该对象不会被其它写入复用时」才可传 true。
|
||||||
|
*/
|
||||||
|
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload,
|
||||||
|
boolean compensateDeleteOnFailure) {
|
||||||
long deadlineNanos = operationDeadlineNanos();
|
long deadlineNanos = operationDeadlineNanos();
|
||||||
if (!isConfigured()) {
|
if (!isConfigured()) {
|
||||||
throw new IllegalStateException("transient storage is not configured");
|
throw new IllegalStateException("transient storage is not configured");
|
||||||
@@ -136,13 +163,45 @@ public class RustfsObjectStorageService {
|
|||||||
}
|
}
|
||||||
return uploadedObjectKey;
|
return uploadedObjectKey;
|
||||||
} catch (RuntimeException ex) {
|
} catch (RuntimeException ex) {
|
||||||
if (putCompleted.get()) {
|
if (putCompleted.get() && compensateDeleteOnFailure) {
|
||||||
enqueueDeleteRetry(objectKey, ex);
|
enqueueDeleteRetry(objectKey, ex);
|
||||||
|
} else if (putCompleted.get()) {
|
||||||
|
// 共享 key 会被重传重写:此处删除可能删掉别的行正在引用的对象,交给保留期清理兜底。
|
||||||
|
// 线上任务 28459 的 chunk-462/473/484 就是被这条无条件删除队列删掉的。
|
||||||
|
log.warn("[rustfs] 跳过上传失败补偿删除(对象非本次独占,可能被复用)objectKey={} err={}",
|
||||||
|
objectKey, ex.getMessage());
|
||||||
}
|
}
|
||||||
throw ex;
|
throw ex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 对象 key 是否为「本次写入独占」的版本化 key:末段(去掉 {@code .json} 后缀)以 UUID 结尾。
|
||||||
|
*
|
||||||
|
* <p>只看末段——UUID 出现在中间段(如 scopeHash)不代表该对象被独占;解析失败一律按共享处理
|
||||||
|
* (保守:宁可留下孤儿对象,也不删掉可能仍被引用的对象)。
|
||||||
|
*/
|
||||||
|
static boolean isVersionedObjectKey(String objectKey) {
|
||||||
|
if (objectKey == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String key = objectKey.trim();
|
||||||
|
if (key.endsWith(".json")) {
|
||||||
|
key = key.substring(0, key.length() - ".json".length());
|
||||||
|
}
|
||||||
|
int slash = key.lastIndexOf('/');
|
||||||
|
String lastSegment = slash < 0 ? key : key.substring(slash + 1);
|
||||||
|
if (lastSegment.length() < UUID_STRING_LENGTH) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
UUID.fromString(lastSegment.substring(lastSegment.length() - UUID_STRING_LENGTH));
|
||||||
|
return true;
|
||||||
|
} catch (IllegalArgumentException ex) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public String readObjectAsString(String objectKey) {
|
public String readObjectAsString(String objectKey) {
|
||||||
byte[] bytes = readObjectBytes(objectKey);
|
byte[] bytes = readObjectBytes(objectKey);
|
||||||
return new String(bytes, StandardCharsets.UTF_8);
|
return new String(bytes, StandardCharsets.UTF_8);
|
||||||
@@ -182,6 +241,43 @@ public class RustfsObjectStorageService {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 列出前缀下的对象 key,按最后修改时间倒序(最新在前)。
|
||||||
|
*
|
||||||
|
* <p>只服务于「指针指向的对象已不存在、需要找回同槽位的版本化兄弟对象」这一兜底路径,
|
||||||
|
* 因此刻意不做重试、不参与失败窗口记账:列出失败直接抛错,由调用方按原错误语义处理。
|
||||||
|
*/
|
||||||
|
public List<String> listObjectKeysNewestFirst(String prefix, int limit) {
|
||||||
|
if (!isConfigured()) {
|
||||||
|
throw new IllegalStateException("transient storage is not configured");
|
||||||
|
}
|
||||||
|
int safeLimit = Math.max(1, Math.min(limit, MAX_SIBLING_LIST_KEYS));
|
||||||
|
long deadlineNanos = operationDeadlineNanos();
|
||||||
|
List<String[]> entries = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
Iterable<Result<Item>> results = buildClient(deadlineNanos).listObjects(ListObjectsArgs.builder()
|
||||||
|
.bucket(properties.getBucket())
|
||||||
|
.prefix(prefix == null ? "" : prefix)
|
||||||
|
.recursive(true)
|
||||||
|
.maxKeys(safeLimit)
|
||||||
|
.build());
|
||||||
|
for (Result<Item> result : results) {
|
||||||
|
Item item = result.get();
|
||||||
|
entries.add(new String[]{item.objectName(),
|
||||||
|
item.lastModified() == null ? "" : item.lastModified().toString()});
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("transient storage list failed prefix=" + prefix
|
||||||
|
+ " err=" + ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
entries.sort((left, right) -> right[1].compareTo(left[1]));
|
||||||
|
List<String> keys = new ArrayList<>(entries.size());
|
||||||
|
for (String[] entry : entries) {
|
||||||
|
keys.add(entry[0]);
|
||||||
|
}
|
||||||
|
return keys;
|
||||||
|
}
|
||||||
|
|
||||||
public void deleteObject(String objectKey) {
|
public void deleteObject(String objectKey) {
|
||||||
deleteObject(objectKey, true, operationDeadlineNanos());
|
deleteObject(objectKey, true, operationDeadlineNanos());
|
||||||
}
|
}
|
||||||
|
|||||||
+2
@@ -29,6 +29,8 @@ public class PriceTrackLoopRunEntity {
|
|||||||
@TableField(value = "active_task_id", updateStrategy = FieldStrategy.ALWAYS)
|
@TableField(value = "active_task_id", updateStrategy = FieldStrategy.ALWAYS)
|
||||||
private Long activeTaskId;
|
private Long activeTaskId;
|
||||||
private Boolean stopRequested;
|
private Boolean stopRequested;
|
||||||
|
/** 因客户端中断自动重派当前轮的次数:封顶用,避免会话持续不可用时无限重派(V129)。 */
|
||||||
|
private Integer resumeAttempt;
|
||||||
private String errorMessage;
|
private String errorMessage;
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
|
|||||||
+47
@@ -39,6 +39,14 @@ public class PriceTrackLoopRunService {
|
|||||||
private static final String STATUS_STOPPED = "STOPPED";
|
private static final String STATUS_STOPPED = "STOPPED";
|
||||||
private static final String EXECUTION_MODE_FINITE = "FINITE";
|
private static final String EXECUTION_MODE_FINITE = "FINITE";
|
||||||
private static final String EXECUTION_MODE_INFINITE = "INFINITE";
|
private static final String EXECUTION_MODE_INFINITE = "INFINITE";
|
||||||
|
/** 客户端异常中断的任务 errorMessage 前缀(TaskHeartbeatService.markInterrupted 写入)。 */
|
||||||
|
private static final String CLIENT_INTERRUPT_ERROR_PREFIX = "客户端异常中断";
|
||||||
|
/**
|
||||||
|
* 中断后自动重派当前轮的次数上限。
|
||||||
|
* 会话持续不可用(账号被风控、紫鸟未就绪)时,不封顶会无限重派、无限重开浏览器——
|
||||||
|
* 2026-09-18 任务 28587 就是这样白烧了 5 小时。
|
||||||
|
*/
|
||||||
|
private static final int MAX_AUTO_RESUME_ATTEMPT = 3;
|
||||||
|
|
||||||
private final PriceTrackLoopRunMapper loopRunMapper;
|
private final PriceTrackLoopRunMapper loopRunMapper;
|
||||||
private final FileTaskMapper fileTaskMapper;
|
private final FileTaskMapper fileTaskMapper;
|
||||||
@@ -259,6 +267,33 @@ public class PriceTrackLoopRunService {
|
|||||||
entity.setActiveTaskId(null);
|
entity.setActiveTaskId(null);
|
||||||
entity.setUpdatedAt(LocalDateTime.now());
|
entity.setUpdatedAt(LocalDateTime.now());
|
||||||
if (STATUS_FAILED.equals(task.getStatus())) {
|
if (STATUS_FAILED.equals(task.getStatus())) {
|
||||||
|
// 用户已请求停止时绝不续派:停止意图优先于自动恢复(否则点完停止循环还会自己转起来)
|
||||||
|
if (Boolean.TRUE.equals(entity.getStopRequested())) {
|
||||||
|
markStopped(entity, null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// 客户端重启中断(markInterrupted 写入的前缀)不终止循环:清空 active_task_id 后保持
|
||||||
|
// RUNNING,客户端下次 dispatchNext 会拿到**同一店铺、同一轮次**的 childTaskRequest,
|
||||||
|
// 等于原地续跑——页内已处理的 ASIN 由服务端 skip_asins 去重,不会重复改价。
|
||||||
|
if (isClientInterrupt(task) && currentResumeAttempt(entity) < MAX_AUTO_RESUME_ATTEMPT) {
|
||||||
|
int attempt = currentResumeAttempt(entity) + 1;
|
||||||
|
entity.setStatus(STATUS_RUNNING);
|
||||||
|
entity.setErrorMessage(null);
|
||||||
|
entity.setFinishedAt(null);
|
||||||
|
entity.setResumeAttempt(attempt);
|
||||||
|
loopRunMapper.updateById(entity);
|
||||||
|
log.warn("[price-track-loop] 子任务因客户端中断失败,自动重派当前轮 loopRunId={} childTaskId={} "
|
||||||
|
+ "round={} shopIndex={} resumeAttempt={}/{} error={}",
|
||||||
|
entity.getId(), childTaskId, entity.getCurrentRound(), entity.getCurrentShopIndex(),
|
||||||
|
attempt, MAX_AUTO_RESUME_ATTEMPT, task.getErrorMessage());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (isClientInterrupt(task)) {
|
||||||
|
log.warn("[price-track-loop] 客户端中断续跑已达上限 {} 次,终止循环 loopRunId={} childTaskId={} "
|
||||||
|
+ "round={} shopIndex={}",
|
||||||
|
MAX_AUTO_RESUME_ATTEMPT, entity.getId(), childTaskId,
|
||||||
|
entity.getCurrentRound(), entity.getCurrentShopIndex());
|
||||||
|
}
|
||||||
entity.setStatus(STATUS_FAILED);
|
entity.setStatus(STATUS_FAILED);
|
||||||
entity.setErrorMessage(task.getErrorMessage() == null || task.getErrorMessage().isBlank()
|
entity.setErrorMessage(task.getErrorMessage() == null || task.getErrorMessage().isBlank()
|
||||||
? "子任务执行失败"
|
? "子任务执行失败"
|
||||||
@@ -269,6 +304,8 @@ public class PriceTrackLoopRunService {
|
|||||||
entity.getId(), childTaskId, entity.getCurrentRound(), entity.getCurrentShopIndex(), entity.getErrorMessage());
|
entity.getId(), childTaskId, entity.getCurrentRound(), entity.getCurrentShopIndex(), entity.getErrorMessage());
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
// 子任务成功 → 中断续跑计数归零(否则历史上的中断会一直占用封顶额度)
|
||||||
|
entity.setResumeAttempt(0);
|
||||||
List<PriceTrackMatchShopsVo.PriceTrackShopQueueItem> items = parseShops(entity);
|
List<PriceTrackMatchShopsVo.PriceTrackShopQueueItem> items = parseShops(entity);
|
||||||
if (items.isEmpty()) {
|
if (items.isEmpty()) {
|
||||||
entity.setStatus(STATUS_FAILED);
|
entity.setStatus(STATUS_FAILED);
|
||||||
@@ -300,6 +337,16 @@ public class PriceTrackLoopRunService {
|
|||||||
entity.getId(), childTaskId, entity.getStatus(), entity.getCurrentRound(), entity.getCurrentShopIndex(), entity.getActiveTaskId());
|
entity.getId(), childTaskId, entity.getStatus(), entity.getCurrentRound(), entity.getCurrentShopIndex(), entity.getActiveTaskId());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 子任务失败原因是否为「客户端异常中断」(客户端重启上报,可自动重派续跑)。 */
|
||||||
|
private static boolean isClientInterrupt(FileTaskEntity task) {
|
||||||
|
String message = task == null ? null : task.getErrorMessage();
|
||||||
|
return message != null && message.startsWith(CLIENT_INTERRUPT_ERROR_PREFIX);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int currentResumeAttempt(PriceTrackLoopRunEntity entity) {
|
||||||
|
return entity.getResumeAttempt() == null ? 0 : entity.getResumeAttempt();
|
||||||
|
}
|
||||||
|
|
||||||
private void reconcileWithTerminalChild(PriceTrackLoopRunEntity entity) {
|
private void reconcileWithTerminalChild(PriceTrackLoopRunEntity entity) {
|
||||||
if (entity == null || entity.getActiveTaskId() == null || isTerminal(entity.getStatus())) {
|
if (entity == null || entity.getActiveTaskId() == null || isTerminal(entity.getStatus())) {
|
||||||
return;
|
return;
|
||||||
|
|||||||
+68
-12
@@ -652,9 +652,8 @@ public class ProductRiskTaskService {
|
|||||||
}
|
}
|
||||||
matchedShopCount++;
|
matchedShopCount++;
|
||||||
if (payload.getError() != null && !payload.getError().isBlank()) {
|
if (payload.getError() != null && !payload.getError().isBlank()) {
|
||||||
markResultFailed(fr, payload.getError());
|
finalizeFailedShop(fr, shopKey, mergeShopPayload(taskId, shopKey, payload),
|
||||||
batchErrors.add(shopKey + ": " + payload.getError());
|
payload.getError(), batchErrors);
|
||||||
productRiskTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -759,9 +758,8 @@ public class ProductRiskTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (cachedPayload.getError() != null && !cachedPayload.getError().isBlank()) {
|
if (cachedPayload.getError() != null && !cachedPayload.getError().isBlank()) {
|
||||||
markResultFailed(fr, cachedPayload.getError());
|
// 陈旧收尾同样保留已跑出来的行:失败原因照常回显,文件顺手组装
|
||||||
batchErrors.add(shopKey + ": " + cachedPayload.getError());
|
finalizeFailedShop(fr, shopKey, cachedPayload, cachedPayload.getError(), batchErrors);
|
||||||
productRiskTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
|
||||||
changed = true;
|
changed = true;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -852,6 +850,9 @@ public class ProductRiskTaskService {
|
|||||||
String stem = safeFileStem(displayName);
|
String stem = safeFileStem(displayName);
|
||||||
File xlsx = FileUtil.file(workRoot, stem + ".xlsx");
|
File xlsx = FileUtil.file(workRoot, stem + ".xlsx");
|
||||||
File zip = FileUtil.file(workRoot, stem + ".zip");
|
File zip = FileUtil.file(workRoot, stem + ".zip");
|
||||||
|
// 失败店铺的部分结果(errorMessage 已写明原因):组装完成时保留失败态,只挂文件。
|
||||||
|
// 否则组装一落地就把行"洗成成功",用户再也看不到「这个店铺其实没跑完」。
|
||||||
|
boolean partialFailure = fr.getErrorMessage() != null && !fr.getErrorMessage().isBlank();
|
||||||
try {
|
try {
|
||||||
excelAssemblyService.writeWorkbook(xlsx, displayName, countries);
|
excelAssemblyService.writeWorkbook(xlsx, displayName, countries);
|
||||||
ZipUtil.zip(zip, false, xlsx);
|
ZipUtil.zip(zip, false, xlsx);
|
||||||
@@ -861,8 +862,12 @@ public class ProductRiskTaskService {
|
|||||||
fr.setResultFileSize(zip.length());
|
fr.setResultFileSize(zip.length());
|
||||||
fr.setResultContentType(CONTENT_TYPE_ZIP);
|
fr.setResultContentType(CONTENT_TYPE_ZIP);
|
||||||
fr.setRowCount(excelAssemblyService.countRows(countries));
|
fr.setRowCount(excelAssemblyService.countRows(countries));
|
||||||
fr.setSuccess(1);
|
if (partialFailure) {
|
||||||
fr.setErrorMessage(null);
|
fr.setSuccess(0);
|
||||||
|
} else {
|
||||||
|
fr.setSuccess(1);
|
||||||
|
fr.setErrorMessage(null);
|
||||||
|
}
|
||||||
fileResultMapper.updateById(fr);
|
fileResultMapper.updateById(fr);
|
||||||
} finally {
|
} finally {
|
||||||
FileUtil.del(xlsx);
|
FileUtil.del(xlsx);
|
||||||
@@ -899,13 +904,56 @@ public class ProductRiskTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) {
|
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) {
|
||||||
|
enqueueResultFileAssembly(result, shopKey, payload, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param preserveFailure 失败店铺的部分结果:保留 success=0 + 失败原因,只把文件挂上去,
|
||||||
|
* 任务状态不变(仍 FAILED),用户仍能下载已跑出来的行
|
||||||
|
*/
|
||||||
|
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey,
|
||||||
|
ProductRiskShopPayloadDto payload, boolean preserveFailure) {
|
||||||
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
||||||
taskResultItemService.replaceResultSnapshot(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey, payload);
|
taskResultItemService.replaceResultSnapshot(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey, payload);
|
||||||
markResultFilePending(result, shopKey, payload);
|
markResultFilePending(result, shopKey, payload, preserveFailure);
|
||||||
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void markResultFilePending(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) {
|
/**
|
||||||
|
* 失败店铺的收尾:任务照旧标 FAILED(原因回显给用户),但已经把跑出来的行组装成
|
||||||
|
* 部分结果文件随任务交付——否则用户只看到「失败」,却拿不到「哪些 ASIN 实际已被处理」
|
||||||
|
* 的记录(会话掉线这类"跑了几个国家才断"的场景尤其需要)。一行可用数据都没有时才退化成纯失败。
|
||||||
|
*/
|
||||||
|
private void finalizeFailedShop(FileResultEntity fr, String shopKey,
|
||||||
|
ProductRiskShopPayloadDto mergedPayload, String errorMessage,
|
||||||
|
List<String> batchErrors) {
|
||||||
|
int rows = mergedPayload == null ? 0 : countPayloadRows(mergedPayload);
|
||||||
|
String message = errorMessage;
|
||||||
|
if (rows > 0) {
|
||||||
|
markResultFailed(fr, errorMessage);
|
||||||
|
try {
|
||||||
|
enqueueResultFileAssembly(fr, shopKey, mergedPayload, true);
|
||||||
|
batchErrors.add(shopKey + ": " + errorMessage);
|
||||||
|
productRiskTaskCacheService.removeShopMergedPayload(fr.getTaskId(), shopKey);
|
||||||
|
log.info("[product-risk] 失败店铺仍产出部分结果 taskId={} shop={} rows={} error={}",
|
||||||
|
fr.getTaskId(), shopKey, rows, errorMessage);
|
||||||
|
return;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
message = errorMessage + "(部分结果组装排队失败:" + (ex.getMessage() == null ? "未知错误" : ex.getMessage()) + ")";
|
||||||
|
log.warn("[product-risk] 失败店铺部分结果排队失败,仅标记失败 taskId={} shop={} msg={}",
|
||||||
|
fr.getTaskId(), shopKey, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.info("[product-risk] 失败店铺无可用行,不产出结果文件 taskId={} shop={} error={}",
|
||||||
|
fr.getTaskId(), shopKey, errorMessage);
|
||||||
|
}
|
||||||
|
markResultFailed(fr, message);
|
||||||
|
batchErrors.add(shopKey + ": " + message);
|
||||||
|
productRiskTaskCacheService.removeShopMergedPayload(fr.getTaskId(), shopKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void markResultFilePending(FileResultEntity result, String shopKey,
|
||||||
|
ProductRiskShopPayloadDto payload, boolean preserveFailure) {
|
||||||
Map<String, List<ProductRiskRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
Map<String, List<ProductRiskRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
||||||
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank()
|
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank()
|
||||||
? payload.getShopName().trim()
|
? payload.getShopName().trim()
|
||||||
@@ -916,8 +964,16 @@ public class ProductRiskTaskService {
|
|||||||
result.setResultFileSize(0L);
|
result.setResultFileSize(0L);
|
||||||
result.setResultContentType(CONTENT_TYPE_ZIP);
|
result.setResultContentType(CONTENT_TYPE_ZIP);
|
||||||
result.setRowCount(excelAssemblyService.countRows(countries));
|
result.setRowCount(excelAssemblyService.countRows(countries));
|
||||||
result.setSuccess(1);
|
if (preserveFailure) {
|
||||||
result.setErrorMessage(null);
|
// 失败店铺的部分结果:成功态与失败原因都不能动,只标「文件名已定、文件待组装」
|
||||||
|
result.setSuccess(0);
|
||||||
|
if (result.getErrorMessage() == null || result.getErrorMessage().isBlank()) {
|
||||||
|
result.setErrorMessage("店铺未跑完,仅产出部分结果");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
result.setSuccess(1);
|
||||||
|
result.setErrorMessage(null);
|
||||||
|
}
|
||||||
fileResultMapper.updateById(result);
|
fileResultMapper.updateById(result);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+9
-3
@@ -1,6 +1,7 @@
|
|||||||
package com.nanri.aiimage.modules.publish.controller;
|
package com.nanri.aiimage.modules.publish.controller;
|
||||||
|
|
||||||
import com.nanri.aiimage.common.api.ApiResponse;
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||||
import com.nanri.aiimage.modules.publish.model.dto.PublishParseRequest;
|
import com.nanri.aiimage.modules.publish.model.dto.PublishParseRequest;
|
||||||
import com.nanri.aiimage.modules.publish.model.dto.PublishSubmitResultRequest;
|
import com.nanri.aiimage.modules.publish.model.dto.PublishSubmitResultRequest;
|
||||||
import com.nanri.aiimage.modules.publish.model.dto.PublishTaskBatchRequest;
|
import com.nanri.aiimage.modules.publish.model.dto.PublishTaskBatchRequest;
|
||||||
@@ -14,6 +15,7 @@ import com.nanri.aiimage.modules.publish.service.PublishTaskService;
|
|||||||
import io.swagger.v3.oas.annotations.Operation;
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
import io.swagger.v3.oas.annotations.Parameter;
|
import io.swagger.v3.oas.annotations.Parameter;
|
||||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||||
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import jakarta.validation.Valid;
|
import jakarta.validation.Valid;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||||
@@ -34,6 +36,7 @@ import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
|||||||
public class PublishController {
|
public class PublishController {
|
||||||
|
|
||||||
private final PublishTaskService publishTaskService;
|
private final PublishTaskService publishTaskService;
|
||||||
|
private final AdminAuthSupport adminAuthSupport;
|
||||||
|
|
||||||
@PostMapping("/parse")
|
@PostMapping("/parse")
|
||||||
@Operation(
|
@Operation(
|
||||||
@@ -57,15 +60,18 @@ public class PublishController {
|
|||||||
@PostMapping("/tasks/{taskId}/files/{fileId}/activate")
|
@PostMapping("/tasks/{taskId}/files/{fileId}/activate")
|
||||||
@Operation(
|
@Operation(
|
||||||
summary = "激活任务中的单个文件",
|
summary = "激活任务中的单个文件",
|
||||||
description = "前端派发 Python 队列前调用。taskId 和 fileId 用于定位文件,user_id 可省略;如传入则校验任务归属。同一任务同一时间只允许一个 RUNNING 文件;重复激活当前 RUNNING 文件为幂等操作。")
|
description = "前端派发 Python 队列前调用。taskId 和 fileId 用于定位文件,user_id 可省略;如传入则校验任务归属。同一任务同一时间只允许一个 RUNNING 文件;重复激活当前 RUNNING 文件为幂等操作。"
|
||||||
|
+ "店铺互斥按 (发起方设备, 店铺) 判定:同一台机器上同一店铺只允许一个任务在跑(该机器上该店铺只有一个紫鸟浏览器会话,并发会互相切换国家);不同设备各自持有独立会话,允许同一店铺并行跑不同国家。"
|
||||||
|
+ "设备号取自 JWT 签名的 deviceId claim,缺失时退回按店铺全局互斥。")
|
||||||
public ApiResponse<Void> activateFile(
|
public ApiResponse<Void> activateFile(
|
||||||
@Parameter(description = "上架任务 ID", required = true, example = "9001")
|
@Parameter(description = "上架任务 ID", required = true, example = "9001")
|
||||||
@PathVariable Long taskId,
|
@PathVariable Long taskId,
|
||||||
@Parameter(description = "任务内文件 ID", required = true, example = "9101")
|
@Parameter(description = "任务内文件 ID", required = true, example = "9101")
|
||||||
@PathVariable Long fileId,
|
@PathVariable Long fileId,
|
||||||
@Parameter(description = "可选的任务所属用户 ID;省略时由 taskId 反查", required = false, example = "1")
|
@Parameter(description = "可选的任务所属用户 ID;省略时由 taskId 反查", required = false, example = "1")
|
||||||
@RequestParam(value = "user_id", required = false) Long userId) {
|
@RequestParam(value = "user_id", required = false) Long userId,
|
||||||
publishTaskService.activateFile(taskId, fileId, userId);
|
HttpServletRequest request) {
|
||||||
|
publishTaskService.activateFile(taskId, fileId, userId, adminAuthSupport.currentDeviceId(request));
|
||||||
return ApiResponse.success(null);
|
return ApiResponse.success(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
@@ -20,6 +20,8 @@ public class PublishFileEntity {
|
|||||||
private Integer matched;
|
private Integer matched;
|
||||||
private String shopId;
|
private String shopId;
|
||||||
private Long matchedUserId;
|
private Long matchedUserId;
|
||||||
|
/** 激活该文件时客户端所在设备(JWT 签名的 deviceId);空表示来源不明,按全局店铺互斥保守处理。 */
|
||||||
|
private String deviceId;
|
||||||
private String platform;
|
private String platform;
|
||||||
private String companyName;
|
private String companyName;
|
||||||
private String matchStatus;
|
private String matchStatus;
|
||||||
|
|||||||
+53
-1
@@ -174,7 +174,14 @@ public class PublishTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void activateFile(Long taskId, Long fileId, Long userId) {
|
/**
|
||||||
|
* 激活任务中的单个文件。
|
||||||
|
*
|
||||||
|
* @param deviceId 发起方设备标识(JWT 签名的 deviceId claim);空串/空白表示来源不明
|
||||||
|
* (旧客户端 token 无该 claim、内部令牌调用),此时退回全局店铺互斥
|
||||||
|
*/
|
||||||
|
public void activateFile(Long taskId, Long fileId, Long userId, String deviceId) {
|
||||||
|
String device = deviceId == null ? "" : deviceId.trim();
|
||||||
try (TaskDistributedLockService.LockHandle lock =
|
try (TaskDistributedLockService.LockHandle lock =
|
||||||
taskDistributedLockService.acquire(MODULE_TYPE, taskId)) {
|
taskDistributedLockService.acquire(MODULE_TYPE, taskId)) {
|
||||||
if (lock == null) {
|
if (lock == null) {
|
||||||
@@ -198,16 +205,61 @@ public class PublishTaskService {
|
|||||||
if (runningFile != null) {
|
if (runningFile != null) {
|
||||||
throw new BusinessException("同一任务已有文件正在执行: " + runningFile.getSourceFilename());
|
throw new BusinessException("同一任务已有文件正在执行: " + runningFile.getSourceFilename());
|
||||||
}
|
}
|
||||||
|
// 店铺级互斥:同一台设备上同一店铺同一时刻只允许一个上架任务在跑。
|
||||||
|
// 2026-09-17 事故(28519/28520/28521 相继提交同一店铺「林洪武」):同一台机器上同一
|
||||||
|
// 店铺被多个任务并发打开,客户端 startBrowser 全部返回 -10000,三个任务一起失败。
|
||||||
|
// 激活是任务真正开跑的唯一入口,在这里挡掉并带出占用中的任务号,用户才知道要等谁。
|
||||||
|
//
|
||||||
|
// 互斥键是 (设备, 店铺) 而不是店铺:紫鸟浏览器会话是**每台机器一份**,不同客户端
|
||||||
|
// 各自持有独立会话,同一家店可以在两台机器上并行跑不同国家(2026-09-18 任务 28624
|
||||||
|
// 在另一台机器上被 28616 误挡)。真正必须串行的是同一台设备——那里只有一个会话,
|
||||||
|
// 两个任务会互相切换国家(add_product.py 的 SwitchingCountries 是会话级状态,任务
|
||||||
|
// 中途断线重连还会再切一次),轻则失败,重则把 A 国家的商品提交进 B 国家的店铺。
|
||||||
|
// 设备号取自 JWT 签名的 deviceId claim;为空(旧客户端 token 无该 claim / 内部令牌
|
||||||
|
// 调用)时退回改动前的全局店铺互斥,保守不放宽。
|
||||||
|
//
|
||||||
|
// 注意:本校验与随后的状态更新之间仍有极小竞态窗口(两个请求恰好同时通过校验);
|
||||||
|
// 真正的串行由客户端店铺锁保证,这一层的目的是尽早给出明确提示,避免白传文件与重复执行。
|
||||||
|
String shopName = file.getShopName();
|
||||||
|
if (shopName != null && !shopName.isBlank()) {
|
||||||
|
LambdaQueryWrapper<PublishFileEntity> shopRunningQuery = new LambdaQueryWrapper<PublishFileEntity>()
|
||||||
|
.eq(PublishFileEntity::getShopName, shopName)
|
||||||
|
.eq(PublishFileEntity::getStatus, STATUS_RUNNING)
|
||||||
|
.ne(PublishFileEntity::getTaskId, taskId)
|
||||||
|
.orderByAsc(PublishFileEntity::getId)
|
||||||
|
.last("limit 1");
|
||||||
|
if (device.isEmpty()) {
|
||||||
|
log.info("[publish] 激活无设备标识,按全局店铺互斥判定 taskId={} fileId={} shop={}",
|
||||||
|
taskId, fileId, shopName);
|
||||||
|
} else {
|
||||||
|
// 本设备的 RUNNING 行,以及设备未知的存量行(旧客户端/内部调用,NULL 或空串)
|
||||||
|
// ——后者无法判断落在哪台机器上,一律保守视为可能同机。
|
||||||
|
shopRunningQuery.and(wrapper -> wrapper
|
||||||
|
.eq(PublishFileEntity::getDeviceId, device)
|
||||||
|
.or().isNull(PublishFileEntity::getDeviceId)
|
||||||
|
.or().eq(PublishFileEntity::getDeviceId, ""));
|
||||||
|
}
|
||||||
|
PublishFileEntity shopRunning = publishFileMapper.selectOne(shopRunningQuery);
|
||||||
|
if (shopRunning != null) {
|
||||||
|
log.warn("[publish] 店铺互斥拦截 taskId={} fileId={} shop={} device={} 占用任务={} 占用设备={}",
|
||||||
|
taskId, fileId, shopName, device,
|
||||||
|
shopRunning.getTaskId(), shopRunning.getDeviceId());
|
||||||
|
throw new BusinessException("店铺「" + shopName + "」已有上架任务正在执行(任务 "
|
||||||
|
+ shopRunning.getTaskId() + "),请等它完成后再提交");
|
||||||
|
}
|
||||||
|
}
|
||||||
int updated = publishFileMapper.update(null, new LambdaUpdateWrapper<PublishFileEntity>()
|
int updated = publishFileMapper.update(null, new LambdaUpdateWrapper<PublishFileEntity>()
|
||||||
.eq(PublishFileEntity::getId, fileId)
|
.eq(PublishFileEntity::getId, fileId)
|
||||||
.eq(PublishFileEntity::getTaskId, taskId)
|
.eq(PublishFileEntity::getTaskId, taskId)
|
||||||
.eq(PublishFileEntity::getStatus, STATUS_PENDING)
|
.eq(PublishFileEntity::getStatus, STATUS_PENDING)
|
||||||
.set(PublishFileEntity::getStatus, STATUS_RUNNING)
|
.set(PublishFileEntity::getStatus, STATUS_RUNNING)
|
||||||
|
.set(PublishFileEntity::getDeviceId, device.isEmpty() ? null : device)
|
||||||
.set(PublishFileEntity::getUpdatedAt, LocalDateTime.now())
|
.set(PublishFileEntity::getUpdatedAt, LocalDateTime.now())
|
||||||
.set(PublishFileEntity::getErrorMessage, null));
|
.set(PublishFileEntity::getErrorMessage, null));
|
||||||
if (updated <= 0) {
|
if (updated <= 0) {
|
||||||
throw new BusinessException("文件激活失败,请刷新后重试");
|
throw new BusinessException("文件激活失败,请刷新后重试");
|
||||||
}
|
}
|
||||||
|
log.info("[publish] 文件激活成功 taskId={} fileId={} shop={} device={}", taskId, fileId, shopName, device);
|
||||||
if (STATUS_PENDING.equals(task.getStatus())) {
|
if (STATUS_PENDING.equals(task.getStatus())) {
|
||||||
fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getId, taskId)
|
.eq(FileTaskEntity::getId, taskId)
|
||||||
|
|||||||
+9
@@ -27,6 +27,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
@@ -121,6 +122,7 @@ public class ShopDataCrawlTaskService {
|
|||||||
private final InstanceMetadata instanceMetadata;
|
private final InstanceMetadata instanceMetadata;
|
||||||
private final ShopDataCrawlDailyFileService dailyFileService;
|
private final ShopDataCrawlDailyFileService dailyFileService;
|
||||||
private final ShopDataCrawlItemStoreService shopDataCrawlItemStoreService;
|
private final ShopDataCrawlItemStoreService shopDataCrawlItemStoreService;
|
||||||
|
private final DuplicateCheckRefreshPort duplicateCheckRefreshPort;
|
||||||
private final PlatformTransactionManager transactionManager;
|
private final PlatformTransactionManager transactionManager;
|
||||||
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
@@ -2116,6 +2118,13 @@ public class ShopDataCrawlTaskService {
|
|||||||
shopDataCrawlItemStoreService.saveShopBatchFromSnapshot(
|
shopDataCrawlItemStoreService.saveShopBatchFromSnapshot(
|
||||||
snapshot.getShopName(), itemBatchDate, accumulatedItems,
|
snapshot.getShopName(), itemBatchDate, accumulatedItems,
|
||||||
snapshot.getResultId(), task.getId(), baseDailyFile == null ? null : baseDailyFile.getId());
|
snapshot.getResultId(), task.getId(), baseDailyFile == null ? null : baseDailyFile.getId());
|
||||||
|
// 明细已落库:请求撞款重扫(异步合并执行,不阻塞归档;端口契约保证不抛错)
|
||||||
|
try {
|
||||||
|
duplicateCheckRefreshPort.requestRefresh("shop-data-crawl:" + snapshot.getShopName());
|
||||||
|
} catch (RuntimeException ex) {
|
||||||
|
log.warn("[shop-data-crawl] 请求撞款重扫失败(忽略,不影响归档) shop={} msg={}",
|
||||||
|
snapshot.getShopName(), ex.getMessage());
|
||||||
|
}
|
||||||
int rowCount = excelAssemblyService.writeWorkbook(outputXlsx, accumulatedItems);
|
int rowCount = excelAssemblyService.writeWorkbook(outputXlsx, accumulatedItems);
|
||||||
String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
|
String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
|
||||||
if (blank(objectKey)) {
|
if (blank(objectKey)) {
|
||||||
|
|||||||
+17
@@ -0,0 +1,17 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopdatacrawl.spi;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 采集明细就绪后的撞款重扫触发端口(2026-09:店铺数据采集落库后即时刷新重复检查)。
|
||||||
|
*
|
||||||
|
* <p>实现方在 shopduplicatecheck 模块({@code ShopDataDuplicateCheckScanService})。
|
||||||
|
* 契约:实现必须异步执行、去抖合并,不得阻塞调用方、不得向外抛出异常。
|
||||||
|
*/
|
||||||
|
public interface DuplicateCheckRefreshPort {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 请求一次撞款重扫(异步;合并窗口内的多次触发聚合为一次扫描)。
|
||||||
|
*
|
||||||
|
* @param reason 触发来源,仅用于日志排查
|
||||||
|
*/
|
||||||
|
void requestRefresh(String reason);
|
||||||
|
}
|
||||||
+32
-2
@@ -9,6 +9,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlItemStoreService;
|
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlItemStoreService;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckItemMapper;
|
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckItemMapper;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckSourceMapper;
|
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckSourceMapper;
|
||||||
@@ -21,6 +22,7 @@ import com.nanri.aiimage.modules.shopduplicatecheck.model.entity.ShopDataDuplica
|
|||||||
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanPayload;
|
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanPayload;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanSummary;
|
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanSummary;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckAggregator;
|
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckAggregator;
|
||||||
|
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckRefreshScheduler;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckWorkbookParser;
|
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckWorkbookParser;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.RawRow;
|
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.RawRow;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.ShopParsed;
|
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.ShopParsed;
|
||||||
@@ -43,14 +45,14 @@ import java.util.Set;
|
|||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 撞款扫描:每日 00:00 定时 + force 同步全量重扫。
|
* 撞款扫描:每日 00:00 定时 + force 同步全量重扫 + 采集落库触发的异步合并重扫。
|
||||||
* 数据源 = 采集明细表(biz_shop_data_crawl_item,采集先落库再更新文件),
|
* 数据源 = 采集明细表(biz_shop_data_crawl_item,采集先落库再更新文件),
|
||||||
* 直接查库聚合 shops/items/summary 落库 shop_data_duplicate_scan(输出契约不变)。
|
* 直接查库聚合 shops/items/summary 落库 shop_data_duplicate_scan(输出契约不变)。
|
||||||
* 双实例通过 Redis 分布式锁防重;扫描失败落 FAILED 行并上抛,force 场景由端点转 409/500。
|
* 双实例通过 Redis 分布式锁防重;扫描失败落 FAILED 行并上抛,force 场景由端点转 409/500。
|
||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class ShopDataDuplicateCheckScanService {
|
public class ShopDataDuplicateCheckScanService implements DuplicateCheckRefreshPort {
|
||||||
|
|
||||||
public static final String SCAN_LOCK = "shop-data-duplicate-check:scan";
|
public static final String SCAN_LOCK = "shop-data-duplicate-check:scan";
|
||||||
static final DateTimeFormatter SCANNED_AT_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
static final DateTimeFormatter SCANNED_AT_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||||
@@ -71,6 +73,9 @@ public class ShopDataDuplicateCheckScanService {
|
|||||||
private final AtomicLong cachedRowId = new AtomicLong(-1L);
|
private final AtomicLong cachedRowId = new AtomicLong(-1L);
|
||||||
private volatile CachedScan cachedScan;
|
private volatile CachedScan cachedScan;
|
||||||
|
|
||||||
|
/** 采集落库触发的异步合并重扫调度器(单飞 + 去抖 + 锁忙重试)。 */
|
||||||
|
private final DuplicateCheckRefreshScheduler refreshScheduler;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
public ShopDataDuplicateCheckScanService(ShopDataDuplicateScanMapper scanMapper,
|
public ShopDataDuplicateCheckScanService(ShopDataDuplicateScanMapper scanMapper,
|
||||||
ShopDuplicateCheckSourceMapper sourceMapper,
|
ShopDuplicateCheckSourceMapper sourceMapper,
|
||||||
@@ -86,6 +91,7 @@ public class ShopDataDuplicateCheckScanService {
|
|||||||
this.objectMapper = objectMapper;
|
this.objectMapper = objectMapper;
|
||||||
this.itemMapper = itemMapper;
|
this.itemMapper = itemMapper;
|
||||||
this.itemStoreService = itemStoreService;
|
this.itemStoreService = itemStoreService;
|
||||||
|
this.refreshScheduler = new DuplicateCheckRefreshScheduler(this::runRefreshOnce);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 读侧视图:scanned_at 为最新 SUCCESS 行 created_at(yyyy-MM-dd HH:mm:ss)。 */
|
/** 读侧视图:scanned_at 为最新 SUCCESS 行 created_at(yyyy-MM-dd HH:mm:ss)。 */
|
||||||
@@ -122,6 +128,30 @@ public class ShopDataDuplicateCheckScanService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 采集明细落库后的重扫请求(端口实现):异步合并执行,不阻塞、不抛错。 */
|
||||||
|
@Override
|
||||||
|
public void requestRefresh(String reason) {
|
||||||
|
refreshScheduler.request(reason);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 调度器单次扫描动作:锁被占返回 LOCK_BUSY 供其重试;失败只记日志(FAILED 行已落库)。 */
|
||||||
|
private DuplicateCheckRefreshScheduler.Outcome runRefreshOnce() {
|
||||||
|
try {
|
||||||
|
scanNow();
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.DONE;
|
||||||
|
} catch (BusinessException ex) {
|
||||||
|
if (ex.getCode() != null && ex.getCode() == 409) {
|
||||||
|
log.info("[shop-duplicate-check] 自动重扫未执行:其它扫描进行中 msg={}", ex.getMessage());
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.LOCK_BUSY;
|
||||||
|
}
|
||||||
|
log.warn("[shop-duplicate-check] 自动重扫失败 code={} msg={}", ex.getCode(), ex.getMessage());
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.FAILED;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.error("[shop-duplicate-check] 自动重扫异常", ex);
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.FAILED;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** 最新 SUCCESS 扫描视图;无扫描结果返回 null。 */
|
/** 最新 SUCCESS 扫描视图;无扫描结果返回 null。 */
|
||||||
public DuplicateScanView loadLatest() {
|
public DuplicateScanView loadLatest() {
|
||||||
ScanLightRowDto light = scanMapper.selectLatestLightRow();
|
ScanLightRowDto light = scanMapper.selectLatestLightRow();
|
||||||
|
|||||||
+138
@@ -0,0 +1,138 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopduplicatecheck.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.util.ThreadPools;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
import java.util.function.Supplier;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 采集落库触发的撞款重扫调度器:合并窗口去抖 + 单飞 + 锁忙重试。
|
||||||
|
*
|
||||||
|
* <p>语义:{@link #request} 只置位并异步执行,永不阻塞调用方、永不向外抛错;
|
||||||
|
* 合并窗口内的多次触发聚合为一次扫描;扫描动作执行期间到达的触发在下一轮执行;
|
||||||
|
* 扫描因分布式锁被占用未执行({@link Outcome#LOCK_BUSY})时按固定间隔重试有限次。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
public class DuplicateCheckRefreshScheduler {
|
||||||
|
|
||||||
|
/** 单次扫描动作的终态:完成 / 锁被占(可重试)/ 失败(不重试,等下次触发或定时扫描)。 */
|
||||||
|
public enum Outcome {
|
||||||
|
DONE, LOCK_BUSY, FAILED
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final long DEFAULT_DEBOUNCE_MILLIS = 10_000L;
|
||||||
|
private static final long DEFAULT_LOCK_RETRY_MILLIS = 20_000L;
|
||||||
|
private static final int DEFAULT_MAX_LOCK_RETRIES = 6;
|
||||||
|
|
||||||
|
private final Supplier<Outcome> scanAction;
|
||||||
|
private final long debounceMillis;
|
||||||
|
private final long lockRetryMillis;
|
||||||
|
private final int maxLockRetries;
|
||||||
|
private final ExecutorService executor;
|
||||||
|
private final AtomicBoolean pending = new AtomicBoolean(false);
|
||||||
|
private final AtomicBoolean running = new AtomicBoolean(false);
|
||||||
|
|
||||||
|
public DuplicateCheckRefreshScheduler(Supplier<Outcome> scanAction) {
|
||||||
|
this(scanAction, DEFAULT_DEBOUNCE_MILLIS, DEFAULT_LOCK_RETRY_MILLIS, DEFAULT_MAX_LOCK_RETRIES);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 测试用:注入更短的窗口与重试参数。 */
|
||||||
|
DuplicateCheckRefreshScheduler(Supplier<Outcome> scanAction, long debounceMillis,
|
||||||
|
long lockRetryMillis, int maxLockRetries) {
|
||||||
|
this.scanAction = scanAction;
|
||||||
|
this.debounceMillis = Math.max(0L, debounceMillis);
|
||||||
|
this.lockRetryMillis = Math.max(0L, lockRetryMillis);
|
||||||
|
this.maxLockRetries = Math.max(0, maxLockRetries);
|
||||||
|
this.executor = ThreadPools.boundedFixed("shop-dup-refresh", 1, 8);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 请求一次重扫(异步、去抖合并)。调用方不被阻塞,也不会收到异常。 */
|
||||||
|
public void request(String reason) {
|
||||||
|
pending.set(true);
|
||||||
|
if (running.compareAndSet(false, true)) {
|
||||||
|
submit(reason);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void submit(String reason) {
|
||||||
|
try {
|
||||||
|
log.info("[shop-duplicate-check] 触发撞款重扫(异步合并执行,窗口={}ms) reason={}", debounceMillis, reason);
|
||||||
|
executor.execute(this::drain);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
// 提交失败(如线程池拒绝)时复位单飞标记,避免后续触发被永久吞掉
|
||||||
|
running.set(false);
|
||||||
|
log.warn("[shop-duplicate-check] 撞款重扫任务提交失败 reason={} msg={}", reason, ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void drain() {
|
||||||
|
try {
|
||||||
|
while (true) {
|
||||||
|
// 合并窗口:窗口内到达的多次触发聚合为同一轮扫描
|
||||||
|
if (!sleepQuietly(debounceMillis)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!pending.compareAndSet(true, false)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int lockRetries = 0;
|
||||||
|
while (true) {
|
||||||
|
Outcome outcome = runOnceSafely();
|
||||||
|
if (outcome != Outcome.LOCK_BUSY) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (lockRetries >= maxLockRetries) {
|
||||||
|
log.warn("[shop-duplicate-check] 撞款重扫连续 {} 次未取得扫描锁,放弃本轮(等待下次触发或定时扫描)",
|
||||||
|
lockRetries + 1);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
lockRetries++;
|
||||||
|
log.info("[shop-duplicate-check] 撞款重扫未取得扫描锁,{}ms 后重试(第 {}/{} 次)",
|
||||||
|
lockRetryMillis, lockRetries, maxLockRetries);
|
||||||
|
if (!sleepQuietly(lockRetryMillis)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
running.set(false);
|
||||||
|
// 竞态兜底:running 复位前到达的触发可能没能提交,补一次
|
||||||
|
if (pending.get() && running.compareAndSet(false, true)) {
|
||||||
|
submit("race-guard");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 执行一次扫描动作;动作自身异常也被吸收(调度器对外零抛出)。 */
|
||||||
|
private Outcome runOnceSafely() {
|
||||||
|
long startedAt = System.currentTimeMillis();
|
||||||
|
try {
|
||||||
|
Outcome outcome = scanAction.get();
|
||||||
|
long elapsed = System.currentTimeMillis() - startedAt;
|
||||||
|
if (outcome == Outcome.DONE) {
|
||||||
|
log.info("[shop-duplicate-check] 采集后自动重扫完成 耗时={}ms", elapsed);
|
||||||
|
} else if (outcome == Outcome.FAILED) {
|
||||||
|
log.warn("[shop-duplicate-check] 采集后自动重扫失败 耗时={}ms", elapsed);
|
||||||
|
}
|
||||||
|
return outcome == null ? Outcome.FAILED : outcome;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.error("[shop-duplicate-check] 采集后自动重扫异常", ex);
|
||||||
|
return Outcome.FAILED;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static boolean sleepQuietly(long millis) {
|
||||||
|
if (millis <= 0) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Thread.sleep(millis);
|
||||||
|
return true;
|
||||||
|
} catch (InterruptedException ex) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+67
-11
@@ -676,8 +676,7 @@ public class ShopMatchTaskService {
|
|||||||
changed = true;
|
changed = true;
|
||||||
ShopMatchShopPayloadDto merged = mergeShopPayload(taskId, shopKey, incoming);
|
ShopMatchShopPayloadDto merged = mergeShopPayload(taskId, shopKey, incoming);
|
||||||
if (incoming.getError() != null && !incoming.getError().isBlank()) {
|
if (incoming.getError() != null && !incoming.getError().isBlank()) {
|
||||||
markResultFailed(result, incoming.getError().trim());
|
finalizeFailedShop(result, shopKey, merged, incoming.getError().trim());
|
||||||
shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (!isShopPayloadCompleted(merged)) {
|
if (!isShopPayloadCompleted(merged)) {
|
||||||
@@ -756,9 +755,8 @@ public class ShopMatchTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (cachedPayload.getError() != null && !cachedPayload.getError().isBlank()) {
|
if (cachedPayload.getError() != null && !cachedPayload.getError().isBlank()) {
|
||||||
markResultFailed(result, cachedPayload.getError());
|
// 陈旧收尾同样保留已跑出来的行:失败原因照常回显,文件顺手组装
|
||||||
batchErrors.add(shopKey + ": " + cachedPayload.getError());
|
batchErrors.add(shopKey + ": " + finalizeFailedShop(result, shopKey, cachedPayload, cachedPayload.getError()));
|
||||||
shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
|
||||||
changed = true;
|
changed = true;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -819,6 +817,9 @@ public class ShopMatchTaskService {
|
|||||||
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() ? payload.getShopName().trim() : shopKey;
|
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() ? payload.getShopName().trim() : shopKey;
|
||||||
String stem = safeFileStem(displayName);
|
String stem = safeFileStem(displayName);
|
||||||
File xlsx = FileUtil.file(workRoot, stem + ".xlsx");
|
File xlsx = FileUtil.file(workRoot, stem + ".xlsx");
|
||||||
|
// 失败店铺的部分结果(errorMessage 已写明原因):组装完成时保留失败态,只挂文件。
|
||||||
|
// 否则组装一落地就把行"洗成成功",用户再也看不到「这个店铺其实没跑完」。
|
||||||
|
boolean partialFailure = result.getErrorMessage() != null && !result.getErrorMessage().isBlank();
|
||||||
try {
|
try {
|
||||||
excelAssemblyService.writeWorkbook(xlsx, countries);
|
excelAssemblyService.writeWorkbook(xlsx, countries);
|
||||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||||
@@ -827,8 +828,12 @@ public class ShopMatchTaskService {
|
|||||||
result.setResultFileSize(xlsx.length());
|
result.setResultFileSize(xlsx.length());
|
||||||
result.setResultContentType(CONTENT_TYPE_XLSX);
|
result.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
result.setRowCount(excelAssemblyService.countRows(countries));
|
result.setRowCount(excelAssemblyService.countRows(countries));
|
||||||
result.setSuccess(1);
|
if (partialFailure) {
|
||||||
result.setErrorMessage(null);
|
result.setSuccess(0);
|
||||||
|
} else {
|
||||||
|
result.setSuccess(1);
|
||||||
|
result.setErrorMessage(null);
|
||||||
|
}
|
||||||
fileResultMapper.updateById(result);
|
fileResultMapper.updateById(result);
|
||||||
} finally {
|
} finally {
|
||||||
FileUtil.del(xlsx);
|
FileUtil.del(xlsx);
|
||||||
@@ -861,12 +866,55 @@ public class ShopMatchTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) {
|
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) {
|
||||||
|
enqueueResultFileAssembly(result, shopKey, payload, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param preserveFailure 失败店铺的部分结果:保留 success=0 + 失败原因,只把文件挂上去,
|
||||||
|
* 任务状态不变(仍 FAILED),用户仍能下载已跑出来的行
|
||||||
|
*/
|
||||||
|
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey,
|
||||||
|
ShopMatchShopPayloadDto payload, boolean preserveFailure) {
|
||||||
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
||||||
markResultFilePending(result, shopKey, payload);
|
markResultFilePending(result, shopKey, payload, preserveFailure);
|
||||||
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void markResultFilePending(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) {
|
/**
|
||||||
|
* 失败店铺的收尾:任务照旧标 FAILED(原因回显给用户),但已经把跑出来的行组装成
|
||||||
|
* 部分结果文件随任务交付——否则用户只看到「失败」,却拿不到「哪些 ASIN 实际已被匹配」
|
||||||
|
* 的记录(会话掉线这类"跑了几个国家才断"的场景尤其需要)。一行可用数据都没有时才退化成纯失败。
|
||||||
|
*
|
||||||
|
* @return 最终写入结果记录的失败原因(组装排队失败时会附上原因)
|
||||||
|
*/
|
||||||
|
private String finalizeFailedShop(FileResultEntity result, String shopKey,
|
||||||
|
ShopMatchShopPayloadDto mergedPayload, String errorMessage) {
|
||||||
|
int rows = mergedPayload == null ? 0 : countPayloadRows(mergedPayload);
|
||||||
|
String message = errorMessage;
|
||||||
|
if (rows > 0) {
|
||||||
|
markResultFailed(result, errorMessage);
|
||||||
|
try {
|
||||||
|
enqueueResultFileAssembly(result, shopKey, mergedPayload, true);
|
||||||
|
shopMatchTaskCacheService.removeShopMergedPayload(result.getTaskId(), shopKey);
|
||||||
|
log.info("[shop-match] 失败店铺仍产出部分结果 taskId={} shop={} rows={} error={}",
|
||||||
|
result.getTaskId(), shopKey, rows, errorMessage);
|
||||||
|
return errorMessage;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
message = errorMessage + "(部分结果组装排队失败:" + (ex.getMessage() == null ? "未知错误" : ex.getMessage()) + ")";
|
||||||
|
log.warn("[shop-match] 失败店铺部分结果排队失败,仅标记失败 taskId={} shop={} msg={}",
|
||||||
|
result.getTaskId(), shopKey, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.info("[shop-match] 失败店铺无可用行,不产出结果文件 taskId={} shop={} error={}",
|
||||||
|
result.getTaskId(), shopKey, errorMessage);
|
||||||
|
}
|
||||||
|
markResultFailed(result, message);
|
||||||
|
shopMatchTaskCacheService.removeShopMergedPayload(result.getTaskId(), shopKey);
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void markResultFilePending(FileResultEntity result, String shopKey,
|
||||||
|
ShopMatchShopPayloadDto payload, boolean preserveFailure) {
|
||||||
Map<String, List<ShopMatchRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
Map<String, List<ShopMatchRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
||||||
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() ? payload.getShopName().trim() : shopKey;
|
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() ? payload.getShopName().trim() : shopKey;
|
||||||
String stem = safeFileStem(displayName);
|
String stem = safeFileStem(displayName);
|
||||||
@@ -875,8 +923,16 @@ public class ShopMatchTaskService {
|
|||||||
result.setResultFileSize(0L);
|
result.setResultFileSize(0L);
|
||||||
result.setResultContentType(CONTENT_TYPE_XLSX);
|
result.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
result.setRowCount(excelAssemblyService.countRows(countries));
|
result.setRowCount(excelAssemblyService.countRows(countries));
|
||||||
result.setSuccess(1);
|
if (preserveFailure) {
|
||||||
result.setErrorMessage(null);
|
// 失败店铺的部分结果:成功态与失败原因都不能动,只标「文件名已定、文件待组装」
|
||||||
|
result.setSuccess(0);
|
||||||
|
if (result.getErrorMessage() == null || result.getErrorMessage().isBlank()) {
|
||||||
|
result.setErrorMessage("店铺未跑完,仅产出部分结果");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
result.setSuccess(1);
|
||||||
|
result.setErrorMessage(null);
|
||||||
|
}
|
||||||
fileResultMapper.updateById(result);
|
fileResultMapper.updateById(result);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+27
-4
@@ -214,6 +214,7 @@ public class SimilarAsinPipelineSupport {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
int maxAttempts = 3;
|
int maxAttempts = 3;
|
||||||
|
String conflictDetail = "未发生冲突";
|
||||||
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
|
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||||
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
@@ -275,13 +276,35 @@ public class SimilarAsinPipelineSupport {
|
|||||||
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
// CAS 冲突:读回行上的当前哈希,否则只有一句 conflict 无从判断是谁改的
|
||||||
|
String currentHash = currentPayloadHash(chunk.getId());
|
||||||
|
conflictDetail = "期望hash=" + oldPayloadHash + " 当前hash=" + currentHash;
|
||||||
if (attempt < maxAttempts) {
|
if (attempt < maxAttempts) {
|
||||||
log.warn("[similar-asin] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{}",
|
log.warn("[similar-asin] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{} 期望hash={} 当前hash={}",
|
||||||
taskId, scopeHash, chunkIndex, attempt, maxAttempts);
|
taskId, scopeHash, chunkIndex, attempt, maxAttempts, oldPayloadHash, currentHash);
|
||||||
|
// 还要重试:这次写的对象会被下次重写,先删掉避免堆积
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||||
|
} else {
|
||||||
|
// 终局失败:保留这次写入的版本化对象,作为读路径「同槽位兄弟对象」兜底的恢复源。
|
||||||
|
// 与外观专利同一口径——行没指过去不该让该分片永久判死。
|
||||||
|
log.error("[similar-asin] chunk payload 合并冲突重试耗尽,保留兜底对象 taskId={} scopeHash={} chunk={} 期望hash={} 当前hash={} 保留对象={}",
|
||||||
|
taskId, scopeHash, chunkIndex, oldPayloadHash, currentHash, storedPayload);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
throw new IllegalStateException("相似ASIN分片载荷更新失败");
|
throw new IllegalStateException("相似ASIN分片载荷更新失败 " + conflictDetail);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 读回 chunk 行当前的 payload_hash,用于 CAS 冲突定位(行已不存在/读取失败时返回可读标记)。 */
|
||||||
|
private String currentPayloadHash(Long chunkId) {
|
||||||
|
if (chunkId == null) {
|
||||||
|
return "chunkId 为空";
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
TaskChunkEntity latest = taskChunkMapper.selectById(chunkId);
|
||||||
|
return latest == null ? "行已不存在" : latest.getPayloadHash();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
return "读取失败:" + ex.getMessage();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+4
@@ -26,6 +26,10 @@ public class FileTaskEntity {
|
|||||||
private String createdBy;
|
private String createdBy;
|
||||||
private Long userId;
|
private Long userId;
|
||||||
private String ownerInstanceId;
|
private String ownerInstanceId;
|
||||||
|
/** 续跑来源任务 ID:本行是客户端中断后由服务端自动重排队的续跑任务时非空(V129)。 */
|
||||||
|
private Long resumeOfTaskId;
|
||||||
|
/** 续跑代数:0=原始任务,N=第 N 次自动续跑(封顶用,V129)。 */
|
||||||
|
private Integer resumeAttempt;
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
private LocalDateTime finishedAt;
|
private LocalDateTime finishedAt;
|
||||||
|
|||||||
+190
@@ -0,0 +1,190 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import cn.hutool.core.util.IdUtil;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.spi.ClientTaskPullSpi;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 客户端中断任务的自动续跑:**保留失败记录 + 重新排队一条 PENDING 续跑任务**。
|
||||||
|
*
|
||||||
|
* <p>场景:客户端被更新脚本/任务管理器杀掉时,{@code TaskHeartbeatService.markInterrupted} 会把在跑的任务
|
||||||
|
* 标 FAILED(原因「客户端异常中断: …」),用户能看到发生过什么;但长任务(采集/相似ASIN/外观专利/单次跟价)
|
||||||
|
* 一遇客户端重启就整个白跑,需要有人把它重新放回队列。本服务负责这件事。
|
||||||
|
*
|
||||||
|
* <p>续跑方式刻意复用已有链路:新建的任务落 PENDING,由 {@link TaskClientPullService} 的兜底拉取
|
||||||
|
* (客户端每分钟 {@code GET /api/tasks/pull-pending})领走执行——不再新造第二套派发机制。
|
||||||
|
* 只有注册了 {@link ClientTaskPullSpi} 的模块才会被续跑(这些模块的载荷能由服务端自行组装);
|
||||||
|
* 上架/改价等写操作模块不在其中,避免盲目重跑。
|
||||||
|
*
|
||||||
|
* <p>封顶 {@code max-attempt}:会话持续不可用(账号被风控、紫鸟未就绪)时,不封顶会无限重排队。
|
||||||
|
* 计数写在 {@code biz_file_task.resume_attempt}(V129),随续跑任务代代递增。
|
||||||
|
*
|
||||||
|
* <p>幂等:同一原任务已存在续跑任务({@code resume_of_task_id} 反查)则跳过,重复扫描安全。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
public class TaskResumeService {
|
||||||
|
|
||||||
|
private static final String STATUS_FAILED = "FAILED";
|
||||||
|
private static final String STATUS_PENDING = "PENDING";
|
||||||
|
/** 客户端重启中断的失败原因前缀,由 TaskHeartbeatService.markInterrupted 写入。 */
|
||||||
|
private static final String CLIENT_INTERRUPT_PREFIX = "客户端异常中断";
|
||||||
|
|
||||||
|
private final FileTaskMapper fileTaskMapper;
|
||||||
|
/** moduleType → 模块兜底载荷实现;只有注册了 SPI 的模块才能被自动续跑。 */
|
||||||
|
private final Map<String, ClientTaskPullSpi> resumeHandlers;
|
||||||
|
|
||||||
|
@Value("${aiimage.task-resume.enabled:true}")
|
||||||
|
private boolean enabled;
|
||||||
|
|
||||||
|
/** 续跑代数上限:达到后不再重排队,任务保持 FAILED 等人工介入。 */
|
||||||
|
@Value("${aiimage.task-resume.max-attempt:3}")
|
||||||
|
private int maxAttempt;
|
||||||
|
|
||||||
|
/** 只处理最近这段时间内中断的任务,避免开机扫描历史积压。 */
|
||||||
|
@Value("${aiimage.task-resume.window-minutes:30}")
|
||||||
|
private long windowMinutes;
|
||||||
|
|
||||||
|
@Value("${aiimage.task-resume.limit:20}")
|
||||||
|
private int limit;
|
||||||
|
|
||||||
|
public TaskResumeService(FileTaskMapper fileTaskMapper, List<ClientTaskPullSpi> pullSpiHandlers) {
|
||||||
|
this.fileTaskMapper = fileTaskMapper;
|
||||||
|
Map<String, ClientTaskPullSpi> index = new LinkedHashMap<>();
|
||||||
|
if (pullSpiHandlers != null) {
|
||||||
|
for (ClientTaskPullSpi handler : pullSpiHandlers) {
|
||||||
|
String moduleType = handler.moduleType();
|
||||||
|
if (moduleType == null || moduleType.isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
index.put(moduleType.trim().toUpperCase(Locale.ROOT), handler);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.resumeHandlers = Map.copyOf(index);
|
||||||
|
log.info("[task-resume] 可自动续跑模块注册完成 count={} modules={}", index.size(), index.keySet());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 扫描一轮:把最近因客户端中断而失败、且未续跑过的任务重新排队。 */
|
||||||
|
public ResumeStats resumeInterruptedTasks() {
|
||||||
|
ResumeStats stats = new ResumeStats();
|
||||||
|
if (!enabled) {
|
||||||
|
return stats;
|
||||||
|
}
|
||||||
|
if (resumeHandlers.isEmpty()) {
|
||||||
|
log.warn("[task-resume] 没有注册任何 ClientTaskPullSpi,跳过本轮续跑");
|
||||||
|
return stats;
|
||||||
|
}
|
||||||
|
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(Math.max(1L, windowMinutes));
|
||||||
|
int safeLimit = Math.max(1, limit);
|
||||||
|
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||||
|
.likeRight(FileTaskEntity::getErrorMessage, CLIENT_INTERRUPT_PREFIX)
|
||||||
|
.in(FileTaskEntity::getModuleType, resumeHandlers.keySet())
|
||||||
|
.lt(FileTaskEntity::getResumeAttempt, maxAttempt)
|
||||||
|
.ge(FileTaskEntity::getFinishedAt, cutoff)
|
||||||
|
.orderByAsc(FileTaskEntity::getId)
|
||||||
|
.last("limit " + safeLimit));
|
||||||
|
if (candidates.isEmpty()) {
|
||||||
|
stats.unsupportedTaskCount = countUnsupportedInterrupts(cutoff);
|
||||||
|
return stats;
|
||||||
|
}
|
||||||
|
stats.unsupportedTaskCount = countUnsupportedInterrupts(cutoff);
|
||||||
|
stats.scannedTaskCount = candidates.size();
|
||||||
|
for (FileTaskEntity original : candidates) {
|
||||||
|
if (original.getUserId() == null || original.getUserId() <= 0) {
|
||||||
|
log.warn("[task-resume] 原任务没有归属用户,跳过 taskId={}", original.getId());
|
||||||
|
stats.skippedTaskCount++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// 幂等:同一原任务已经排过一次续跑就跳过(重复扫描 / 双实例竞态下不会重复建单)
|
||||||
|
if (hasResumeChild(original.getId())) {
|
||||||
|
stats.skippedTaskCount++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
FileTaskEntity resume = buildResumeTask(original);
|
||||||
|
try {
|
||||||
|
fileTaskMapper.insert(resume);
|
||||||
|
stats.resumedTaskCount++;
|
||||||
|
log.warn("[task-resume] 已自动重新排队 taskId={} moduleType={} userId={} 续跑任务={} 代数={}/{} 中断原因={}",
|
||||||
|
original.getId(), original.getModuleType(), original.getUserId(),
|
||||||
|
resume.getId(), resume.getResumeAttempt(), maxAttempt, original.getErrorMessage());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
stats.skippedTaskCount++;
|
||||||
|
log.error("[task-resume] 重新排队失败 taskId={} moduleType={} err={}",
|
||||||
|
original.getId(), original.getModuleType(), ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return stats;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 统计「因客户端中断而失败、但模块不支持自动续跑」的任务数。
|
||||||
|
*
|
||||||
|
* <p>这些模块(上架/改价/审批/商品管理采集/跟价的非循环任务等)的续跑载荷需要**用户在页面上选的
|
||||||
|
* 执行参数**(如 ziniao_version),而这份选择只存在于派发那一刻的浏览器里、没落到服务端
|
||||||
|
* request_json —— 自动重排队会用错参数。因此它们只做**可见**:数量进巡检 summary,
|
||||||
|
* 运维据此人工重跑;将来把这类参数回写落库后即可纳入续跑白名单。
|
||||||
|
*/
|
||||||
|
private int countUnsupportedInterrupts(LocalDateTime cutoff) {
|
||||||
|
try {
|
||||||
|
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||||
|
.likeRight(FileTaskEntity::getErrorMessage, CLIENT_INTERRUPT_PREFIX)
|
||||||
|
.notIn(FileTaskEntity::getModuleType, resumeHandlers.keySet())
|
||||||
|
.ge(FileTaskEntity::getFinishedAt, cutoff));
|
||||||
|
return count == null ? 0 : count.intValue();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[task-resume] 统计不支持续跑的中断任务失败(忽略): {}", ex.getMessage());
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 该原任务是否已经有续跑任务(反查 resume_of_task_id)。 */
|
||||||
|
private boolean hasResumeChild(Long originalTaskId) {
|
||||||
|
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getResumeOfTaskId, originalTaskId));
|
||||||
|
return count != null && count > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private FileTaskEntity buildResumeTask(FileTaskEntity original) {
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
FileTaskEntity resume = new FileTaskEntity();
|
||||||
|
resume.setTaskNo(original.getModuleType() + "-" + IdUtil.getSnowflakeNextIdStr());
|
||||||
|
resume.setModuleType(original.getModuleType());
|
||||||
|
resume.setTaskMode(original.getTaskMode());
|
||||||
|
resume.setStatus(STATUS_PENDING);
|
||||||
|
resume.setSourceFileCount(original.getSourceFileCount());
|
||||||
|
resume.setSuccessFileCount(0);
|
||||||
|
resume.setFailedFileCount(0);
|
||||||
|
// 原任务的请求参数整体带走:各模块的 ClientTaskPullSpi 从 request_json / 关联表还原执行参数
|
||||||
|
resume.setRequestJson(original.getRequestJson());
|
||||||
|
resume.setCreatedBy(original.getCreatedBy());
|
||||||
|
resume.setUserId(original.getUserId());
|
||||||
|
resume.setResumeOfTaskId(original.getId());
|
||||||
|
int attempt = original.getResumeAttempt() == null ? 0 : original.getResumeAttempt();
|
||||||
|
resume.setResumeAttempt(attempt + 1);
|
||||||
|
resume.setCreatedAt(now);
|
||||||
|
resume.setUpdatedAt(now);
|
||||||
|
return resume;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 单轮统计(合并进 stale-check 的 summary 日志,避免定期刷屏)。 */
|
||||||
|
public static final class ResumeStats {
|
||||||
|
public int scannedTaskCount;
|
||||||
|
public int resumedTaskCount;
|
||||||
|
public int skippedTaskCount;
|
||||||
|
/** 模块不支持自动续跑的中断任务数(仅计数,供运维人工重跑)。 */
|
||||||
|
public int unsupportedTaskCount;
|
||||||
|
}
|
||||||
|
}
|
||||||
+67
-5
@@ -30,6 +30,7 @@ import java.util.Locale;
|
|||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
import java.util.zip.GZIPInputStream;
|
import java.util.zip.GZIPInputStream;
|
||||||
import java.util.zip.GZIPOutputStream;
|
import java.util.zip.GZIPOutputStream;
|
||||||
|
|
||||||
@@ -150,8 +151,20 @@ public class TransientPayloadStorageService {
|
|||||||
return decodeStoredPayloadBytes(readLocalPayloadBytes(stripLocalInstanceId(localKey)));
|
return decodeStoredPayloadBytes(readLocalPayloadBytes(stripLocalInstanceId(localKey)));
|
||||||
}
|
}
|
||||||
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
||||||
return decodeStoredPayloadBytes(
|
String objectKey = pointer.substring(RUSTFS_POINTER_PREFIX.length());
|
||||||
rustfsObjectStorageService.readObjectBytes(pointer.substring(RUSTFS_POINTER_PREFIX.length())));
|
try {
|
||||||
|
return decodeStoredPayloadBytes(rustfsObjectStorageService.readObjectBytes(objectKey));
|
||||||
|
} catch (RuntimeException readException) {
|
||||||
|
String sibling = findVersionedChunkSibling(objectKey, readException);
|
||||||
|
if (sibling == null) {
|
||||||
|
throw readException;
|
||||||
|
}
|
||||||
|
// 2026-09-17 线上任务 28459:行指向的普通 key 被误删,但同一分片槽位的版本化对象还在。
|
||||||
|
// 读出它即可让任务按已有数据出结果,不必整单失败。
|
||||||
|
log.warn("[transient-payload] chunk 载荷对象不存在,回退同槽位版本化对象 pointer={} sibling={}",
|
||||||
|
pointer, sibling);
|
||||||
|
return decodeStoredPayloadBytes(rustfsObjectStorageService.readObjectBytes(sibling));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
|
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
|
||||||
return decodeStoredPayloadBytes(
|
return decodeStoredPayloadBytes(
|
||||||
@@ -164,6 +177,53 @@ public class TransientPayloadStorageService {
|
|||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** chunk 载荷槽位的 entryKey 形态:{@code chunk-<index>}(版本化写入则形如 {@code chunk-<index>-<uuid>})。 */
|
||||||
|
private static final Pattern CHUNK_ENTRY_KEY_PATTERN = Pattern.compile("chunk-\\d+");
|
||||||
|
|
||||||
|
/** 兄弟对象查找上限:只为找回同槽位对象,不需要列全。 */
|
||||||
|
private static final int MAX_SIBLING_LOOKUP_KEYS = 50;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 指针对象已不存在时,尝试找回同一分片槽位的版本化兄弟对象。
|
||||||
|
*
|
||||||
|
* <p>两个条件同时满足才兜底,避免读到无关对象或掩盖真实故障:
|
||||||
|
* <ol>
|
||||||
|
* <li>末段 entryKey 是 {@code chunk-<index>} 形态——只有这种槽位才有「版本化兄弟」语义;</li>
|
||||||
|
* <li>失败原因是对象确实不存在(NoSuchKey)——权限/网络类失败照旧上抛以便重试。</li>
|
||||||
|
* </ol>
|
||||||
|
*/
|
||||||
|
private String findVersionedChunkSibling(String objectKey, Throwable cause) {
|
||||||
|
if (!isObjectMissing(cause)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
int slash = objectKey.lastIndexOf('/');
|
||||||
|
String directory = slash < 0 ? "" : objectKey.substring(0, slash + 1);
|
||||||
|
String fileName = slash < 0 ? objectKey : objectKey.substring(slash + 1);
|
||||||
|
if (!fileName.endsWith(".json")) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String entryKey = fileName.substring(0, fileName.length() - ".json".length());
|
||||||
|
if (!CHUNK_ENTRY_KEY_PATTERN.matcher(entryKey).matches()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
List<String> candidates = rustfsObjectStorageService.listObjectKeysNewestFirst(
|
||||||
|
directory + entryKey + "-", MAX_SIBLING_LOOKUP_KEYS);
|
||||||
|
return candidates.isEmpty() ? null : candidates.getFirst();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 对象确已不存在:RustFS 返回 NoSuchKey,message 为 "The specified key does not exist."。 */
|
||||||
|
private static boolean isObjectMissing(Throwable error) {
|
||||||
|
Throwable cursor = error;
|
||||||
|
while (cursor != null) {
|
||||||
|
String message = cursor.getMessage();
|
||||||
|
if (message != null && message.contains("does not exist")) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
cursor = cursor.getCause();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
public void deletePayloadIfPresent(String value) {
|
public void deletePayloadIfPresent(String value) {
|
||||||
String pointer = extractPointer(value);
|
String pointer = extractPointer(value);
|
||||||
if (pointer == null) {
|
if (pointer == null) {
|
||||||
@@ -209,8 +269,10 @@ public class TransientPayloadStorageService {
|
|||||||
*
|
*
|
||||||
* <p>判断口径:
|
* <p>判断口径:
|
||||||
* <ul>
|
* <ul>
|
||||||
* <li>{@code biz_task_chunk.payload_json} 命中 > 1 行(> 1 表示除了 caller 视角下
|
* <li>{@code biz_task_chunk.payload_json} 命中任意行(≥ 1)→ 视为仍被引用。
|
||||||
* 自己即将释放的那一行之外,至少还有别的 chunk 行也指向同一对象)→ 视为仍被引用。</li>
|
* 曾用 {@code > 1} 作判据,等于放行「恰好还有 1 行引用」的情况,会把对方仍在用的对象
|
||||||
|
* 删掉(2026-09-17 线上任务 28459:合并成功后指针未落库 + 旧对象被删 → 该分片永久读不到)。
|
||||||
|
* 物理删除本就约定在 DB 行删除之后执行,故调用方正常路径下引用数必然为 0。</li>
|
||||||
* <li>{@code biz_task_scope_state.parsed_payload_json}/{@code state_json} 命中 > 0 行 →
|
* <li>{@code biz_task_scope_state.parsed_payload_json}/{@code state_json} 命中 > 0 行 →
|
||||||
* 视为仍被引用(这两个字段不是 caller 自身行的常见持有者,命中即非自我引用)。</li>
|
* 视为仍被引用(这两个字段不是 caller 自身行的常见持有者,命中即非自我引用)。</li>
|
||||||
* </ul>
|
* </ul>
|
||||||
@@ -248,7 +310,7 @@ public class TransientPayloadStorageService {
|
|||||||
// 解析不出 taskId 时按原口径全局查,行为与改造前一致。
|
// 解析不出 taskId 时按原口径全局查,行为与改造前一致。
|
||||||
Long pointerTaskId = extractTaskId(pointer);
|
Long pointerTaskId = extractTaskId(pointer);
|
||||||
Long chunkCount = referencedChunkCount(pointerTaskId, values);
|
Long chunkCount = referencedChunkCount(pointerTaskId, values);
|
||||||
if (chunkCount != null && chunkCount > 1L) {
|
if (chunkCount != null && chunkCount > 0L) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
Long scopeStateCount = referencedScopeStateCount(pointerTaskId, values);
|
Long scopeStateCount = referencedScopeStateCount(pointerTaskId, values);
|
||||||
|
|||||||
@@ -279,6 +279,17 @@ aiimage:
|
|||||||
module-types: ${AIIMAGE_CLIENT_TASK_PULL_MODULE_TYPES:SIMILAR_ASIN,COLLECT_DATA,APPEARANCE_PATENT}
|
module-types: ${AIIMAGE_CLIENT_TASK_PULL_MODULE_TYPES:SIMILAR_ASIN,COLLECT_DATA,APPEARANCE_PATENT}
|
||||||
min-pending-minutes: ${AIIMAGE_CLIENT_TASK_PULL_MIN_PENDING_MINUTES:5}
|
min-pending-minutes: ${AIIMAGE_CLIENT_TASK_PULL_MIN_PENDING_MINUTES:5}
|
||||||
limit: ${AIIMAGE_CLIENT_TASK_PULL_LIMIT:5}
|
limit: ${AIIMAGE_CLIENT_TASK_PULL_LIMIT:5}
|
||||||
|
# 客户端中断任务的自动续跑(V129):客户端被更新脚本/任务管理器杀掉时,在跑的任务会被
|
||||||
|
# 客户端启动时上报中断并标 FAILED(用户能看到真实原因),这里把这些任务重新排队成 PENDING,
|
||||||
|
# 由上面的兜底拉取通道交给在线客户端继续跑——长任务不再因为一次客户端更新就整个白跑。
|
||||||
|
# 只有注册了 ClientTaskPullSpi 的模块会被续跑(相似ASIN/采集/外观专利),
|
||||||
|
# 上架/改价等写操作模块刻意不在内,避免盲目重跑。
|
||||||
|
# enabled 默认跟随兜底拉取开关:拉了没人领的话,续跑任务只会积压并被 stale 判死。
|
||||||
|
task-resume:
|
||||||
|
enabled: ${AIIMAGE_TASK_RESUME_ENABLED:${AIIMAGE_CLIENT_TASK_PULL_ENABLED:false}}
|
||||||
|
max-attempt: ${AIIMAGE_TASK_RESUME_MAX_ATTEMPT:3}
|
||||||
|
window-minutes: ${AIIMAGE_TASK_RESUME_WINDOW_MINUTES:30}
|
||||||
|
limit: ${AIIMAGE_TASK_RESUME_LIMIT:20}
|
||||||
coze-task:
|
coze-task:
|
||||||
max-concurrent: ${AIIMAGE_COZE_TASK_MAX_CONCURRENT:12}
|
max-concurrent: ${AIIMAGE_COZE_TASK_MAX_CONCURRENT:12}
|
||||||
brand-check:
|
brand-check:
|
||||||
@@ -289,6 +300,7 @@ aiimage:
|
|||||||
retry-times: ${AIIMAGE_BRAND_CHECK_RETRY_TIMES:10}
|
retry-times: ${AIIMAGE_BRAND_CHECK_RETRY_TIMES:10}
|
||||||
retry-interval-millis: ${AIIMAGE_BRAND_CHECK_RETRY_INTERVAL_MILLIS:1000}
|
retry-interval-millis: ${AIIMAGE_BRAND_CHECK_RETRY_INTERVAL_MILLIS:1000}
|
||||||
retry-max-interval-millis: ${AIIMAGE_BRAND_CHECK_RETRY_MAX_INTERVAL_MILLIS:10000}
|
retry-max-interval-millis: ${AIIMAGE_BRAND_CHECK_RETRY_MAX_INTERVAL_MILLIS:10000}
|
||||||
|
total-timeout-millis: ${AIIMAGE_BRAND_CHECK_TOTAL_TIMEOUT_MILLIS:90000}
|
||||||
connect-timeout-millis: ${AIIMAGE_BRAND_CHECK_CONNECT_TIMEOUT_MILLIS:10000}
|
connect-timeout-millis: ${AIIMAGE_BRAND_CHECK_CONNECT_TIMEOUT_MILLIS:10000}
|
||||||
read-timeout-millis: ${AIIMAGE_BRAND_CHECK_READ_TIMEOUT_MILLIS:60000}
|
read-timeout-millis: ${AIIMAGE_BRAND_CHECK_READ_TIMEOUT_MILLIS:60000}
|
||||||
appearance-patent:
|
appearance-patent:
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
-- V129: 客户端中断后的自动续跑(保留失败记录 + 服务端重新排队)
|
||||||
|
--
|
||||||
|
-- 背景:2026-09-18 任务 28587(跟价,uid 977 店铺「张美莺」)在客户端 09:51/09:57 被重启后中断,
|
||||||
|
-- 旧行为是任务一直挂 RUNNING 到 stale 兜底判死(最长 2 小时),长任务一旦撞上客户端更新就白跑。
|
||||||
|
-- 新契约:中断任务**保留失败记录**(用户能看到「因客户端重启中断」),同时由服务端自动重新排队续跑。
|
||||||
|
--
|
||||||
|
-- 两条续跑路径各需要一个计数:
|
||||||
|
-- 1) biz_file_task.resume_of_task_id + resume_attempt —— 通用模块(相似ASIN/采集/外观专利/跟价单次任务)
|
||||||
|
-- 由 TaskResumeService 复制出一条 PENDING 续跑任务,交给客户端兜底拉取;代数用于封顶。
|
||||||
|
-- 2) biz_price_track_loop_run.resume_attempt —— 跟价循环由 loop_run 驱动,中断时不让循环终止,
|
||||||
|
-- 而是清掉 active_task_id 重新派发当前轮(客户端下次 dispatch 即拿到同一店铺/轮次),同样封顶。
|
||||||
|
--
|
||||||
|
-- 封顶的意义:会话持续不可用(如账号被风控)时,不封顶会无限重排队、无限重开浏览器。
|
||||||
|
--
|
||||||
|
-- 风险:ADD COLUMN 走 INSTANT,两张表均为小表,秒级完成;建议低峰执行。
|
||||||
|
-- 回滚:ALTER TABLE biz_file_task DROP COLUMN resume_of_task_id, DROP COLUMN resume_attempt;
|
||||||
|
-- ALTER TABLE biz_price_track_loop_run DROP COLUMN resume_attempt;
|
||||||
|
|
||||||
|
SET @db_name = DATABASE();
|
||||||
|
|
||||||
|
SET @col_exists := (
|
||||||
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_task' AND COLUMN_NAME = 'resume_of_task_id'
|
||||||
|
);
|
||||||
|
SET @sql := IF(@col_exists = 0,
|
||||||
|
'ALTER TABLE biz_file_task ADD COLUMN resume_of_task_id BIGINT NULL COMMENT ''续跑来源任务ID(客户端中断后自动重排队)'' AFTER owner_instance_id',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
|
SET @col_exists := (
|
||||||
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_task' AND COLUMN_NAME = 'resume_attempt'
|
||||||
|
);
|
||||||
|
SET @sql := IF(@col_exists = 0,
|
||||||
|
'ALTER TABLE biz_file_task ADD COLUMN resume_attempt INT NOT NULL DEFAULT 0 COMMENT ''续跑代数:0=原始任务,N=第N次自动续跑'' AFTER resume_of_task_id',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
|
SET @col_exists := (
|
||||||
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_price_track_loop_run' AND COLUMN_NAME = 'resume_attempt'
|
||||||
|
);
|
||||||
|
SET @sql := IF(@col_exists = 0,
|
||||||
|
'ALTER TABLE biz_price_track_loop_run ADD COLUMN resume_attempt INT NOT NULL DEFAULT 0 COMMENT ''中断后自动重派当前轮的次数(用于封顶)'' AFTER stop_requested',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
-- V130: biz_publish_file 增加 device_id 列(激活该文件时客户端所在设备)
|
||||||
|
--
|
||||||
|
-- 背景:同店铺互斥原本只按 shop_name 全局判定(PublishTaskService.activateFile)。
|
||||||
|
-- 但紫鸟浏览器会话是**每台机器一份**:不同客户端各自持有独立的店铺会话,同一家店
|
||||||
|
-- 完全可以在两台机器上并行跑不同国家。原来的全局判定把这种合法的跨机器并行也挡了
|
||||||
|
-- (2026-09-18 任务 28624 被 28616 误挡:两条在不同机器上)。
|
||||||
|
-- 真正必须串行的是「同一台机器上的同一家店」——那里只有一个浏览器会话,两个任务会
|
||||||
|
-- 互相切换国家(add_product.py 的 SwitchingCountries 是会话级状态,任务中途断线重连
|
||||||
|
-- 还会再切一次),轻则失败,重则把 A 国家的商品提交进 B 国家的店铺。
|
||||||
|
--
|
||||||
|
-- 因此互斥键由 shop_name 改为 (device_id, shop_name)。device 取自 JWT 里**签名的**
|
||||||
|
-- deviceId claim(绝不使用客户端可控的 X-Device-Id 请求头,见 DeviceSessionPolicy 注释)。
|
||||||
|
--
|
||||||
|
-- 空值语义:NULL/空串表示"来源不明"——旧客户端(未升级、token 无 claim)或内部令牌调用。
|
||||||
|
-- 该情形退回改动前的全局店铺互斥,保持保守,不因迁移把风险放开。存量 RUNNING 行均为 NULL,
|
||||||
|
-- 因此会继续全挡直到跑完,随后新激活的行都带设备号,跨机器并行自然生效。
|
||||||
|
--
|
||||||
|
-- 风险:ADD COLUMN 走 INSTANT/INPLACE,生产该表仅数百行,秒级完成;建议低峰执行。
|
||||||
|
-- 回滚:ALTER TABLE biz_publish_file DROP COLUMN device_id;
|
||||||
|
|
||||||
|
SET @db_name = DATABASE();
|
||||||
|
|
||||||
|
SET @col_exists := (
|
||||||
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_publish_file' AND COLUMN_NAME = 'device_id'
|
||||||
|
);
|
||||||
|
SET @sql := IF(@col_exists = 0,
|
||||||
|
'ALTER TABLE biz_publish_file ADD COLUMN device_id VARCHAR(128) NULL COMMENT ''device that activated this file, from signed JWT deviceId claim'' AFTER matched_user_id',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
+85
@@ -0,0 +1,85 @@
|
|||||||
|
package com.nanri.aiimage.common.exception;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.common.service.TaskOwnerForwardService;
|
||||||
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.mockito.InjectMocks;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
import org.springframework.http.HttpStatus;
|
||||||
|
import org.springframework.http.ResponseEntity;
|
||||||
|
import org.springframework.web.client.ResourceAccessException;
|
||||||
|
|
||||||
|
import java.net.ConnectException;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 跨实例转发的失败语义。
|
||||||
|
*
|
||||||
|
* <p>2026-09-18 任务 28616:归属节点 server-110 滚动重启期间,客户端心跳经 nginx 落到
|
||||||
|
* server-121,转发 3 次 Connection refused。当时这里返回 {@code ApiResponse.fail(40903)}
|
||||||
|
* ——HTTP 200 + {@code data:null},而客户端用
|
||||||
|
* {@code bool((resp.json().get("data") or {}).get("alive"))} 解析,把「拿不到数据」折叠成
|
||||||
|
* {@code alive=false},于是客户端把一个跑到 66/253 的健康上架任务主动停掉、关闭店铺。
|
||||||
|
* 修复后转发失败返回 503 + 空 body:新客户端按状态码判为「未知」,老客户端因 body 不是
|
||||||
|
* JSON、解析抛异常同样落到「未知」,两边都不会再自杀。</p>
|
||||||
|
*/
|
||||||
|
@ExtendWith(MockitoExtension.class)
|
||||||
|
class GlobalExceptionHandlerTest {
|
||||||
|
|
||||||
|
@Mock private TaskOwnerForwardService taskOwnerForwardService;
|
||||||
|
|
||||||
|
@InjectMocks private GlobalExceptionHandler handler;
|
||||||
|
|
||||||
|
private static TaskOwnerMismatchException ownerMismatch() {
|
||||||
|
return new TaskOwnerMismatchException(28616L, "PUBLISH task heartbeat", "server-110", "server-121");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void forwardConnectFailureReturns503WithEmptyBody() {
|
||||||
|
when(taskOwnerForwardService.forwardCurrentRequest(any(), any()))
|
||||||
|
.thenThrow(new ResourceAccessException("Connection refused",
|
||||||
|
new ConnectException("Connection refused")));
|
||||||
|
|
||||||
|
Object result = handler.handleTaskOwnerMismatchException(ownerMismatch(), mock(HttpServletRequest.class));
|
||||||
|
|
||||||
|
ResponseEntity<?> response = assertInstanceOf(ResponseEntity.class, result);
|
||||||
|
assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
|
||||||
|
// 空 body 是关键:一旦带上 JSON,老客户端的 bool((data or {}).get("alive")) 又会判成「死」
|
||||||
|
assertNull(response.getBody(), "转发失败必须无响应体,否则老客户端会把未知当成任务已死");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void successfulForwardPassesThroughUpstreamStatusAndBody() {
|
||||||
|
byte[] upstream = "{\"success\":true,\"data\":{\"alive\":true}}".getBytes(java.nio.charset.StandardCharsets.UTF_8);
|
||||||
|
when(taskOwnerForwardService.forwardCurrentRequest(any(), any()))
|
||||||
|
.thenReturn(ResponseEntity.ok(upstream));
|
||||||
|
|
||||||
|
Object result = handler.handleTaskOwnerMismatchException(ownerMismatch(), mock(HttpServletRequest.class));
|
||||||
|
|
||||||
|
ResponseEntity<?> response = assertInstanceOf(ResponseEntity.class, result);
|
||||||
|
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||||
|
assertEquals(upstream, response.getBody(), "转发成功时上游响应体必须原样透传");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void configErrorKeepsBusinessEnvelopeInsteadOfServiceUnavailable() {
|
||||||
|
// 路由未配置 / 检测到转发循环属于配置错误,不是瞬时故障:保留业务信封,不回 503
|
||||||
|
when(taskOwnerForwardService.forwardCurrentRequest(any(), any()))
|
||||||
|
.thenThrow(new BusinessException(40903, "任务归属实例未配置服务路由:server-110"));
|
||||||
|
|
||||||
|
Object result = handler.handleTaskOwnerMismatchException(ownerMismatch(), mock(HttpServletRequest.class));
|
||||||
|
|
||||||
|
ApiResponse<?> response = assertInstanceOf(ApiResponse.class, result);
|
||||||
|
assertFalse(response.isSuccess(), "配置错误仍按业务失败返回");
|
||||||
|
}
|
||||||
|
}
|
||||||
+218
@@ -0,0 +1,218 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||||
|
import com.nanri.aiimage.config.InstanceMetadata;
|
||||||
|
import com.nanri.aiimage.config.StorageProperties;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||||
|
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.mockito.InjectMocks;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.Spy;
|
||||||
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
import org.springframework.test.util.ReflectionTestUtils;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyLong;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
|
import static org.mockito.Mockito.lenient;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.times;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 外观专利分片载荷合并的 CAS 冲突处理(2026-09-17 线上任务 28459 的遗留项)。
|
||||||
|
*
|
||||||
|
* <p>原实现在每次 CAS 冲突后都删掉刚写入的版本化对象,重试耗尽即抛异常、行仍指向旧指针——
|
||||||
|
* 一旦旧对象也不在,该分片就永久读不到(28459 的 chunk-462/473/484 正是这个形态)。
|
||||||
|
* 现在:冲突时读回行上的当前哈希以便定位;**终局失败保留最后一个兜底对象**,
|
||||||
|
* 让读路径的「同槽位兄弟对象」兜底仍有数据可取。
|
||||||
|
*/
|
||||||
|
@ExtendWith(MockitoExtension.class)
|
||||||
|
class AppearancePatentChunkMergeConflictTest {
|
||||||
|
|
||||||
|
private static final Long TASK_ID = 28459L;
|
||||||
|
private static final String SCOPE_HASH = "2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e";
|
||||||
|
private static final int CHUNK_INDEX = 462;
|
||||||
|
|
||||||
|
@Mock private LocalFileStorageService localFileStorageService;
|
||||||
|
@Mock private OssStorageService ossStorageService;
|
||||||
|
@Mock private StorageProperties storageProperties;
|
||||||
|
@Mock private FileTaskMapper fileTaskMapper;
|
||||||
|
@Mock private FileResultMapper fileResultMapper;
|
||||||
|
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||||
|
@Mock private TaskChunkMapper taskChunkMapper;
|
||||||
|
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
@Mock private AppearancePatentLlmClient llmClient;
|
||||||
|
@Mock private AppearancePatentTaskCacheService taskCacheService;
|
||||||
|
@Mock private AppearancePatentProperties properties;
|
||||||
|
@Mock private TaskFileJobService taskFileJobService;
|
||||||
|
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
|
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||||
|
@Mock private PlatformTransactionManager transactionManager;
|
||||||
|
@Mock private DistributedJobLockService distributedJobLockService;
|
||||||
|
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||||
|
@Mock private InstanceMetadata instanceMetadata;
|
||||||
|
@Mock private TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
|
@InjectMocks private AppearancePatentTaskService service;
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void initializeMybatisMetadata() {
|
||||||
|
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||||
|
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() {
|
||||||
|
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||||
|
// 共享写开启才会走版本化对象存储;否则落到本地兜底路径直接报「RustFS 未配置」
|
||||||
|
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 首次冲突后重试成功:中间那次写的对象要删(会被下次重写),且最终行被改到新对象。 */
|
||||||
|
@Test
|
||||||
|
void retryAfterConflictDeletesSupersededObject() {
|
||||||
|
stubChunk("hash-old", "ptr-chunk-462.json");
|
||||||
|
AtomicInteger stores = new AtomicInteger();
|
||||||
|
stubVersionedStore(stores);
|
||||||
|
when(taskChunkMapper.update(any(), any())).thenReturn(0, 1);
|
||||||
|
when(taskChunkMapper.selectById(anyLong())).thenReturn(chunk("hash-other", "ptr-chunk-462.json"));
|
||||||
|
|
||||||
|
invokeMerge();
|
||||||
|
|
||||||
|
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||||
|
// 第 1 次冲突写的对象被删;第 2 次成功,走的是「替换旧对象」而不是删新对象
|
||||||
|
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-1");
|
||||||
|
verify(transientPayloadStorageService, never()).deletePayloadIfPresent("sibling-2");
|
||||||
|
verify(transientPayloadStorageService).deleteReplacedPayloadIfNeeded(eq("ptr-chunk-462.json"), eq("sibling-2"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 冲突重试耗尽:**保留**最后一次写入的兜底对象(本次要修的形态),并抛异常带出两个哈希。 */
|
||||||
|
@Test
|
||||||
|
void exhaustedConflictKeepsLastStoredObjectAsFallback() {
|
||||||
|
stubChunk("hash-old", "ptr-chunk-462.json");
|
||||||
|
AtomicInteger stores = new AtomicInteger();
|
||||||
|
stubVersionedStore(stores);
|
||||||
|
when(taskChunkMapper.update(any(), any())).thenReturn(0);
|
||||||
|
when(taskChunkMapper.selectById(anyLong())).thenReturn(chunk("hash-other", "ptr-chunk-462.json"));
|
||||||
|
|
||||||
|
IllegalStateException ex = assertThrows(IllegalStateException.class, this::invokeMerge);
|
||||||
|
|
||||||
|
verify(transientPayloadStorageService, times(3)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||||
|
// 前两次冲突的对象照旧删除;第三次(终局)的对象必须保留
|
||||||
|
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-1");
|
||||||
|
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-2");
|
||||||
|
verify(transientPayloadStorageService, never()).deletePayloadIfPresent("sibling-3");
|
||||||
|
assertTrue(ex.getMessage().contains("hash-old"), "异常需带出期望哈希,实际: " + ex.getMessage());
|
||||||
|
assertTrue(ex.getMessage().contains("hash-other"), "异常需带出当前哈希,实际: " + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 冲突时读回行上的当前哈希,供定位(此前只有一句 update conflict,线上无法定位)。 */
|
||||||
|
@Test
|
||||||
|
void conflictReadsBackCurrentHashForDiagnostics() {
|
||||||
|
stubChunk("hash-old", "ptr-chunk-462.json");
|
||||||
|
stubVersionedStore(new AtomicInteger());
|
||||||
|
when(taskChunkMapper.update(any(), any())).thenReturn(0);
|
||||||
|
when(taskChunkMapper.selectById(7L)).thenReturn(chunk("hash-changed-by-other-writer", "ptr-chunk-462.json"));
|
||||||
|
|
||||||
|
assertThrows(IllegalStateException.class, this::invokeMerge);
|
||||||
|
|
||||||
|
verify(taskChunkMapper, times(3)).selectById(7L);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 读回行失败不得掩盖原始冲突:异常信息里给出可读标记。 */
|
||||||
|
@Test
|
||||||
|
void readBackFailureDoesNotMaskConflict() {
|
||||||
|
stubChunk("hash-old", "ptr-chunk-462.json");
|
||||||
|
stubVersionedStore(new AtomicInteger());
|
||||||
|
when(taskChunkMapper.update(any(), any())).thenReturn(0);
|
||||||
|
when(taskChunkMapper.selectById(anyLong())).thenThrow(new IllegalStateException("db down"));
|
||||||
|
|
||||||
|
IllegalStateException ex = assertThrows(IllegalStateException.class, this::invokeMerge);
|
||||||
|
|
||||||
|
assertTrue(ex.getMessage().contains("读取失败"), "实际: " + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 辅助 =====
|
||||||
|
|
||||||
|
private void invokeMerge() {
|
||||||
|
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
|
||||||
|
row.setRowToken("r1");
|
||||||
|
row.setAsin("B0A0000001");
|
||||||
|
ReflectionTestUtils.invokeMethod(service, "mergeChunkPayload", TASK_ID, SCOPE_HASH, CHUNK_INDEX, List.of(row));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void stubChunk(String payloadHash, String payloadPointer) {
|
||||||
|
when(taskChunkMapper.selectOne(any())).thenReturn(chunk(payloadHash, payloadPointer));
|
||||||
|
lenient().when(transientPayloadStorageService.resolvePayload(eq(payloadPointer), anyString()))
|
||||||
|
.thenReturn("[{\"rowToken\":\"r1\",\"asin\":\"B0A0000001\"}]");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void stubVersionedStore(AtomicInteger stores) {
|
||||||
|
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||||
|
anyString(), anyLong(), anyString(), any(), anyString()))
|
||||||
|
.thenAnswer(inv -> "sibling-" + stores.incrementAndGet());
|
||||||
|
}
|
||||||
|
|
||||||
|
private TaskChunkEntity chunk(String payloadHash, String payloadPointer) {
|
||||||
|
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||||
|
chunk.setId(7L);
|
||||||
|
chunk.setTaskId(TASK_ID);
|
||||||
|
chunk.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
|
||||||
|
chunk.setScopeHash(SCOPE_HASH);
|
||||||
|
chunk.setChunkIndex(CHUNK_INDEX);
|
||||||
|
chunk.setPayloadJson(payloadPointer);
|
||||||
|
chunk.setPayloadHash(payloadHash);
|
||||||
|
return chunk;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 断言辅助:确认存的对象数(避免误用未使用的 import)。 */
|
||||||
|
@Test
|
||||||
|
void storeIsCalledOnceWhenUpdateSucceeds() {
|
||||||
|
stubChunk("hash-old", "ptr-chunk-462.json");
|
||||||
|
AtomicInteger stores = new AtomicInteger();
|
||||||
|
stubVersionedStore(stores);
|
||||||
|
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||||
|
|
||||||
|
invokeMerge();
|
||||||
|
|
||||||
|
assertEquals(1, stores.get());
|
||||||
|
verify(taskChunkMapper, never()).selectById(anyLong());
|
||||||
|
}
|
||||||
|
}
|
||||||
+243
@@ -0,0 +1,243 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||||
|
import com.nanri.aiimage.config.InstanceMetadata;
|
||||||
|
import com.nanri.aiimage.config.StorageProperties;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
|
||||||
|
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.mockito.InjectMocks;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.Spy;
|
||||||
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
import org.springframework.test.util.ReflectionTestUtils;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
import org.springframework.transaction.TransactionDefinition;
|
||||||
|
import org.springframework.transaction.TransactionStatus;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyLong;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
|
import static org.mockito.Mockito.doAnswer;
|
||||||
|
import static org.mockito.Mockito.lenient;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 外观专利「补传恢复」入口(2026-09-17 线上任务 28459 的遗留项)。
|
||||||
|
*
|
||||||
|
* <p>分片缺失导致组装 job 重试耗尽后,客户端补传缺口应能自动把该 job 重置重跑。
|
||||||
|
* 该能力在 {@code TaskFileJobService} 里早就有了,但只有删除品牌模块接了入口,
|
||||||
|
* 外观专利没有——28459 补齐分片后仍需人工重置 job 才能出结果。
|
||||||
|
*/
|
||||||
|
@ExtendWith(MockitoExtension.class)
|
||||||
|
class AppearancePatentTerminalFailedRecoveryTest {
|
||||||
|
|
||||||
|
private static final Long TASK_ID = 28459L;
|
||||||
|
private static final Long RESULT_ID = 31490L;
|
||||||
|
private static final String MODULE_TYPE = "APPEARANCE_PATENT";
|
||||||
|
|
||||||
|
@Mock private LocalFileStorageService localFileStorageService;
|
||||||
|
@Mock private OssStorageService ossStorageService;
|
||||||
|
@Mock private StorageProperties storageProperties;
|
||||||
|
@Mock private FileTaskMapper fileTaskMapper;
|
||||||
|
@Mock private FileResultMapper fileResultMapper;
|
||||||
|
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||||
|
@Mock private TaskChunkMapper taskChunkMapper;
|
||||||
|
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
@Mock private AppearancePatentLlmClient llmClient;
|
||||||
|
@Mock private AppearancePatentTaskCacheService taskCacheService;
|
||||||
|
@Mock private AppearancePatentProperties properties;
|
||||||
|
@Mock private TaskFileJobService taskFileJobService;
|
||||||
|
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
|
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||||
|
@Mock private PlatformTransactionManager transactionManager;
|
||||||
|
@Mock private DistributedJobLockService distributedJobLockService;
|
||||||
|
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||||
|
@Mock private InstanceMetadata instanceMetadata;
|
||||||
|
@Mock private TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
@Mock private TransactionStatus transactionStatus;
|
||||||
|
|
||||||
|
@InjectMocks private AppearancePatentTaskService service;
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void initializeMybatisMetadata() {
|
||||||
|
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||||
|
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() {
|
||||||
|
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||||
|
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class))).thenReturn(transactionStatus);
|
||||||
|
lenient().doAnswer(inv -> null).when(transactionManager).commit(transactionStatus);
|
||||||
|
lenient().doAnswer(inv -> null).when(transactionManager).rollback(transactionStatus);
|
||||||
|
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
lenient().when(taskChunkMapper.selectCount(any())).thenReturn(0L);
|
||||||
|
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(null);
|
||||||
|
lenient().when(taskScopeStateMapper.selectOne(any())).thenReturn(null);
|
||||||
|
lenient().when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(1);
|
||||||
|
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
|
||||||
|
lenient().when(transientPayloadStorageService.storeChunkPayload(
|
||||||
|
eq(MODULE_TYPE), eq(TASK_ID), anyString(), any(), anyString()))
|
||||||
|
.thenReturn("\"rustfs:task-chunk/appearance_patent/28459/hash/chunk-11.json\"");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 直接覆盖恢复判定 =====
|
||||||
|
|
||||||
|
/** 已有成功的组装 job → 不恢复。 */
|
||||||
|
@Test
|
||||||
|
void successfulAssembleJobSkipsRecovery() {
|
||||||
|
stubResultRow();
|
||||||
|
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(true);
|
||||||
|
|
||||||
|
invokeRecovery();
|
||||||
|
|
||||||
|
verify(taskFileJobService, never()).resetTerminalFailedForRecovery(anyLong(), anyString(), anyLong());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 组装 job 不是「重试耗尽的终态失败」→ 不恢复。 */
|
||||||
|
@Test
|
||||||
|
void nonTerminalFailedJobSkipsRecovery() {
|
||||||
|
stubResultRow();
|
||||||
|
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(false);
|
||||||
|
when(taskFileJobService.isTerminalFailedAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(false);
|
||||||
|
|
||||||
|
invokeRecovery();
|
||||||
|
|
||||||
|
verify(taskFileJobService, never()).resetTerminalFailedForRecovery(anyLong(), anyString(), anyLong());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 终态失败 + 分片已补传完整 → 重置 job 重新派发(本次要修的场景)。 */
|
||||||
|
@Test
|
||||||
|
void terminalFailedJobWithCompleteChunksIsReset() {
|
||||||
|
stubResultRow();
|
||||||
|
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(false);
|
||||||
|
when(taskFileJobService.isTerminalFailedAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(true);
|
||||||
|
when(taskScopeStateMapper.selectCount(any())).thenReturn(1L);
|
||||||
|
|
||||||
|
invokeRecovery();
|
||||||
|
|
||||||
|
verify(taskFileJobService).resetTerminalFailedForRecovery(TASK_ID, MODULE_TYPE, RESULT_ID);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 终态失败但分片尚未补齐 → 不恢复(否则又会读到缺失分片再失败一次)。 */
|
||||||
|
@Test
|
||||||
|
void terminalFailedJobWithIncompleteChunksIsNotReset() {
|
||||||
|
stubResultRow();
|
||||||
|
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(false);
|
||||||
|
when(taskFileJobService.isTerminalFailedAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(true);
|
||||||
|
when(taskScopeStateMapper.selectCount(any())).thenReturn(0L);
|
||||||
|
|
||||||
|
invokeRecovery();
|
||||||
|
|
||||||
|
verify(taskFileJobService, never()).resetTerminalFailedForRecovery(anyLong(), anyString(), anyLong());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 还没有结果行 → 不恢复。 */
|
||||||
|
@Test
|
||||||
|
void missingResultRowSkipsRecovery() {
|
||||||
|
when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
|
||||||
|
invokeRecovery();
|
||||||
|
|
||||||
|
verify(taskFileJobService, never()).isTerminalFailedAssembleJob(anyLong(), anyString(), anyLong());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** taskId 非法 → 直接返回,不查库。 */
|
||||||
|
@Test
|
||||||
|
void invalidTaskIdSkipsLookup() {
|
||||||
|
ReflectionTestUtils.invokeMethod(service, "maybeRecoverTerminalFailedAssemble", 0L);
|
||||||
|
ReflectionTestUtils.invokeMethod(service, "maybeRecoverTerminalFailedAssemble", (Object) null);
|
||||||
|
|
||||||
|
verify(fileResultMapper, never()).selectList(any());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 恢复检查自身抛异常 → 吞掉,不影响补传结果(best-effort)。 */
|
||||||
|
@Test
|
||||||
|
void recoveryFailureDoesNotPropagate() {
|
||||||
|
when(fileResultMapper.selectList(any())).thenThrow(new IllegalStateException("db down"));
|
||||||
|
|
||||||
|
assertDoesNotThrow(this::invokeRecovery);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 走完整提交路径 =====
|
||||||
|
|
||||||
|
/** 分片提交成功后触发恢复检查(接线正确)。 */
|
||||||
|
@Test
|
||||||
|
void submitResultTriggersRecoveryCheck() {
|
||||||
|
stubRunningTask();
|
||||||
|
stubResultRow();
|
||||||
|
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(false);
|
||||||
|
when(taskFileJobService.isTerminalFailedAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(true);
|
||||||
|
when(taskScopeStateMapper.selectCount(any())).thenReturn(1L);
|
||||||
|
|
||||||
|
service.submitResult(TASK_ID, request());
|
||||||
|
|
||||||
|
verify(taskFileJobService).resetTerminalFailedForRecovery(TASK_ID, MODULE_TYPE, RESULT_ID);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 辅助 =====
|
||||||
|
|
||||||
|
private void invokeRecovery() {
|
||||||
|
ReflectionTestUtils.invokeMethod(service, "maybeRecoverTerminalFailedAssemble", TASK_ID);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void stubResultRow() {
|
||||||
|
FileResultEntity result = new FileResultEntity();
|
||||||
|
result.setId(RESULT_ID);
|
||||||
|
result.setTaskId(TASK_ID);
|
||||||
|
result.setModuleType(MODULE_TYPE);
|
||||||
|
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of(result));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void stubRunningTask() {
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(TASK_ID);
|
||||||
|
task.setModuleType(MODULE_TYPE);
|
||||||
|
task.setStatus("RUNNING");
|
||||||
|
task.setUserId(1121L);
|
||||||
|
task.setResultJson("{\"parsedPayloadRef\":\"rustfs:task-parsed/x.json\",\"ownerInstanceId\":\"instance-a\"}");
|
||||||
|
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||||
|
}
|
||||||
|
|
||||||
|
private AppearancePatentSubmitResultRequest request() {
|
||||||
|
AppearancePatentSubmitResultRequest request = new AppearancePatentSubmitResultRequest();
|
||||||
|
request.setSubmissionId("appearance-patent-" + TASK_ID);
|
||||||
|
request.setChunkIndex(11);
|
||||||
|
request.setChunkTotal(500);
|
||||||
|
request.setDone(false);
|
||||||
|
return request;
|
||||||
|
}
|
||||||
|
}
|
||||||
+24
@@ -264,6 +264,30 @@ class CollectDataServiceTxBoundaryTest {
|
|||||||
verify(transactionTemplate, never()).execute(any());
|
verify(transactionTemplate, never()).execute(any());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** taskId 28599:失败时已收到的分片仍要组装成结果文件,让用户能下载已采集的数据。 */
|
||||||
|
@Test
|
||||||
|
void failedSubmitWithReceivedChunksEnqueuesPartialWorkbook() {
|
||||||
|
when(taskChunkMapper.selectCount(any())).thenReturn(170L);
|
||||||
|
CollectDataSubmitResultRequest request = submitRequest();
|
||||||
|
request.setError("中间分批回传失败,终止本次采集以避免服务端数据残缺");
|
||||||
|
|
||||||
|
service.submitResult(TASK_ID, request);
|
||||||
|
|
||||||
|
verify(taskFileJobService).enqueueAssembleResult(
|
||||||
|
eq(TASK_ID), eq("COLLECT_DATA"), eq(5001L), anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void failedSubmitWithoutReceivedChunksEnqueuesNothing() {
|
||||||
|
CollectDataSubmitResultRequest request = submitRequest();
|
||||||
|
request.setError("采集端启动失败");
|
||||||
|
|
||||||
|
service.submitResult(TASK_ID, request);
|
||||||
|
|
||||||
|
verify(taskFileJobService, never())
|
||||||
|
.enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
|
||||||
|
}
|
||||||
|
|
||||||
private CollectDataSubmitResultRequest submitRequest() {
|
private CollectDataSubmitResultRequest submitRequest() {
|
||||||
CollectDataSubmitRowDto row = new CollectDataSubmitRowDto();
|
CollectDataSubmitRowDto row = new CollectDataSubmitRowDto();
|
||||||
row.setAsin("B0COLLECT1");
|
row.setAsin("B0COLLECT1");
|
||||||
|
|||||||
+6
-1
@@ -14,6 +14,7 @@ import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskResumeService;
|
||||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
import org.junit.jupiter.api.BeforeAll;
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
@@ -59,6 +60,7 @@ class DeleteBrandStaleTaskServiceTest {
|
|||||||
@Mock private ProductRiskTaskService productRiskTaskService;
|
@Mock private ProductRiskTaskService productRiskTaskService;
|
||||||
@Mock private ProductRiskTaskCacheService productRiskTaskCacheService;
|
@Mock private ProductRiskTaskCacheService productRiskTaskCacheService;
|
||||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||||
|
@Mock private TaskResumeService taskResumeService;
|
||||||
|
|
||||||
@BeforeAll
|
@BeforeAll
|
||||||
static void initializeTableInfo() {
|
static void initializeTableInfo() {
|
||||||
@@ -374,6 +376,9 @@ class DeleteBrandStaleTaskServiceTest {
|
|||||||
private void noUploadEnabled() {
|
private void noUploadEnabled() {
|
||||||
when(deleteBrandProgressProperties.isNoResultUploadCheckEnabled()).thenReturn(true);
|
when(deleteBrandProgressProperties.isNoResultUploadCheckEnabled()).thenReturn(true);
|
||||||
when(deleteBrandProgressProperties.getNoResultUploadTimeoutMinutes()).thenReturn(180L);
|
when(deleteBrandProgressProperties.getNoResultUploadTimeoutMinutes()).thenReturn(180L);
|
||||||
|
// 续跑巡检在判死主流程里被调用:默认返回空统计(无中断任务可续跑);
|
||||||
|
// 只有跑到 failStaleRunningTasks 的用例会用到,故 lenient
|
||||||
|
lenient().when(taskResumeService.resumeInterruptedTasks()).thenReturn(new TaskResumeService.ResumeStats());
|
||||||
}
|
}
|
||||||
|
|
||||||
private void noUploadTaskLockAvailable() {
|
private void noUploadTaskLockAvailable() {
|
||||||
@@ -398,7 +403,7 @@ class DeleteBrandStaleTaskServiceTest {
|
|||||||
null, null, null, null, null, null, null, null, null, null,
|
null, null, null, null, null, null, null, null, null, null,
|
||||||
null, null, null,
|
null, null, null,
|
||||||
deleteBrandProgressProperties, null, taskDistributedLockService, taskFileJobService, null,
|
deleteBrandProgressProperties, null, taskDistributedLockService, taskFileJobService, null,
|
||||||
taskScopeStateMapper, taskHeartbeatPositionService);
|
taskScopeStateMapper, taskHeartbeatPositionService, taskResumeService);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void lockAvailable() {
|
private void lockAvailable() {
|
||||||
|
|||||||
+1
-1
@@ -82,7 +82,7 @@ class TaskModuleCoverageTest {
|
|||||||
mock(SimilarAsinTaskService.class),
|
mock(SimilarAsinTaskService.class),
|
||||||
null, null, null, null,
|
null, null, null, null,
|
||||||
mock(ShopDataCrawlTaskService.class),
|
mock(ShopDataCrawlTaskService.class),
|
||||||
null, null);
|
null, null, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
+12
-6
@@ -297,16 +297,19 @@ class RustfsObjectStorageServiceTest {
|
|||||||
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||||
properties, emptyProvider(), provider(retryService), () -> client);
|
properties, emptyProvider(), provider(retryService), () -> client);
|
||||||
|
|
||||||
|
// 补偿删除只对版本化 key(本次写入独占)生效,故这里用 versioned key 覆盖该路径
|
||||||
|
String firstKey = "task/a-11111111-2222-3333-4444-555555555555.json";
|
||||||
|
String secondKey = "task/b-66666666-7777-8888-9999-000000000000.json";
|
||||||
IllegalStateException firstFailure = assertThrows(IllegalStateException.class,
|
IllegalStateException firstFailure = assertThrows(IllegalStateException.class,
|
||||||
() -> service.uploadText("task/a.json", "{}", true));
|
() -> service.uploadText(firstKey, "{}", true));
|
||||||
IllegalStateException secondFailure = assertThrows(IllegalStateException.class,
|
IllegalStateException secondFailure = assertThrows(IllegalStateException.class,
|
||||||
() -> service.uploadText("task/b.json", "{}", true));
|
() -> service.uploadText(secondKey, "{}", true));
|
||||||
|
|
||||||
assertTrue(firstFailure.getMessage().contains("not visible"));
|
assertTrue(firstFailure.getMessage().contains("not visible"));
|
||||||
assertTrue(secondFailure.getMessage().contains("not visible"));
|
assertTrue(secondFailure.getMessage().contains("not visible"));
|
||||||
verify(client, times(5)).statObject(any(StatObjectArgs.class));
|
verify(client, times(5)).statObject(any(StatObjectArgs.class));
|
||||||
verify(retryService).enqueue(eq("task/a.json"), same(firstFailure));
|
verify(retryService).enqueue(eq(firstKey), same(firstFailure));
|
||||||
verify(retryService).enqueue(eq("task/b.json"), same(secondFailure));
|
verify(retryService).enqueue(eq(secondKey), same(secondFailure));
|
||||||
IllegalStateException rejected = assertThrows(IllegalStateException.class,
|
IllegalStateException rejected = assertThrows(IllegalStateException.class,
|
||||||
() -> service.uploadText("task/c.json", "{}", false));
|
() -> service.uploadText("task/c.json", "{}", false));
|
||||||
assertTrue(rejected.getMessage().contains("cooldown active"));
|
assertTrue(rejected.getMessage().contains("cooldown active"));
|
||||||
@@ -360,6 +363,9 @@ class RustfsObjectStorageServiceTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
void completedUploadIsEnqueuedWhenDeadlineExpiresAfterPut() throws Exception {
|
void completedUploadIsEnqueuedWhenDeadlineExpiresAfterPut() throws Exception {
|
||||||
|
// 只有版本化 key(本次写入独占)才允许补偿删除;共享 key 的守卫见
|
||||||
|
// RustfsUploadCompensationGuardTest#deterministicKeyDerivedFromShapeDoesNotEnqueueCompensation
|
||||||
|
String versionedKey = "task/chunk-1-0ffc254b-afad-4279-aab2-e85e3ff955e9.json";
|
||||||
TransientStorageProperties properties = configuredProperties();
|
TransientStorageProperties properties = configuredProperties();
|
||||||
properties.setOperationTimeoutSeconds(1);
|
properties.setOperationTimeoutSeconds(1);
|
||||||
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||||
@@ -372,11 +378,11 @@ class RustfsObjectStorageServiceTest {
|
|||||||
properties, emptyProvider(), provider(retryService), () -> client);
|
properties, emptyProvider(), provider(retryService), () -> client);
|
||||||
|
|
||||||
IllegalStateException failure = assertThrows(IllegalStateException.class,
|
IllegalStateException failure = assertThrows(IllegalStateException.class,
|
||||||
() -> service.uploadText("task/a.json", "{}", false));
|
() -> service.uploadText(versionedKey, "{}", false));
|
||||||
|
|
||||||
assertTrue(failure.getMessage().contains("operation timeout"));
|
assertTrue(failure.getMessage().contains("operation timeout"));
|
||||||
verify(client).putObject(any(PutObjectArgs.class));
|
verify(client).putObject(any(PutObjectArgs.class));
|
||||||
verify(retryService).enqueue(eq("task/a.json"), same(failure));
|
verify(retryService).enqueue(eq(versionedKey), same(failure));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static TransientStorageProperties configuredProperties() {
|
private static TransientStorageProperties configuredProperties() {
|
||||||
|
|||||||
+217
@@ -0,0 +1,217 @@
|
|||||||
|
package com.nanri.aiimage.modules.file.service.object;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||||
|
import io.minio.MinioClient;
|
||||||
|
import io.minio.PutObjectArgs;
|
||||||
|
import io.minio.StatObjectArgs;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.springframework.beans.factory.ObjectProvider;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 上传失败补偿删除的准入守卫(2026-09-17 线上任务 28459 事故)。
|
||||||
|
*
|
||||||
|
* <p>事故形态:客户端重传复用**确定性 key**({@code chunk-462.json})。第一次上传 put 已成功、
|
||||||
|
* 但随后的 {@code verifyObjectVisible} 抖动 → 旧代码无条件
|
||||||
|
* {@code enqueueDeleteRetry(objectKey)};而删除重试队列对该 key 不再做任何引用校验
|
||||||
|
* ({@code RustfsDeleteRetryService} → {@code deleteObjectFromRetry} → {@code removeObject}),
|
||||||
|
* 于是把重传成功后 DB 行仍指向的对象删掉,下游读 chunk 直接 404,整个任务组装失败。
|
||||||
|
*
|
||||||
|
* <p>口径:只有**本次写入独占的对象**(版本化 key,末段以 UUID 结尾)才允许补偿删除;
|
||||||
|
* 确定性 key 会被重写复用,删它就可能删掉别的行正在引用的对象。
|
||||||
|
*/
|
||||||
|
class RustfsUploadCompensationGuardTest {
|
||||||
|
|
||||||
|
// ===== 正常路径 =====
|
||||||
|
|
||||||
|
/** 唯一 key(版本化)+ 验证失败 + 显式允许 → 补偿删除入队。 */
|
||||||
|
@Test
|
||||||
|
void uniqueKeyVerifyFailureEnqueuesCompensation() throws Exception {
|
||||||
|
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||||
|
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||||
|
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
|
||||||
|
|
||||||
|
assertThrows(IllegalStateException.class,
|
||||||
|
() -> service.uploadBytes("task/chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json",
|
||||||
|
"{}".getBytes(), true, true));
|
||||||
|
|
||||||
|
verify(retryService).enqueue(eq("task/chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json"),
|
||||||
|
any(Throwable.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 三参重载按 key 形态判定:版本化 key 自动允许补偿删除。 */
|
||||||
|
@Test
|
||||||
|
void versionedKeyDerivedFromShapeEnqueuesCompensation() throws Exception {
|
||||||
|
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||||
|
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||||
|
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
|
||||||
|
|
||||||
|
assertThrows(IllegalStateException.class,
|
||||||
|
() -> service.uploadBytes("task/chunk-77-d03bf3b0-abbb-469d-9bb8-a7c01c309477.json",
|
||||||
|
"{}".getBytes(), true));
|
||||||
|
|
||||||
|
verify(retryService).enqueue(anyString(), any(Throwable.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 事故根因:共享 key 必须不删 =====
|
||||||
|
|
||||||
|
/** 确定性(共享)key + 显式禁止 → 不入队(异常仍照旧抛出,失败归属不变)。 */
|
||||||
|
@Test
|
||||||
|
void sharedKeyVerifyFailureDoesNotEnqueueCompensation() throws Exception {
|
||||||
|
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||||
|
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||||
|
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
|
||||||
|
|
||||||
|
assertThrows(IllegalStateException.class,
|
||||||
|
() -> service.uploadBytes("task/chunk-462.json", "{}".getBytes(), true, false));
|
||||||
|
|
||||||
|
verify(retryService, never()).enqueue(anyString(), any(Throwable.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 三参重载按 key 形态判定:确定性 key 自动禁止补偿删除(本次事故的直接修复点)。 */
|
||||||
|
@Test
|
||||||
|
void deterministicKeyDerivedFromShapeDoesNotEnqueueCompensation() throws Exception {
|
||||||
|
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||||
|
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||||
|
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
|
||||||
|
|
||||||
|
assertThrows(IllegalStateException.class,
|
||||||
|
() -> service.uploadBytes("task/chunk-462.json", "{}".getBytes(), true));
|
||||||
|
|
||||||
|
verify(retryService, never()).enqueue(anyString(), any(Throwable.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** verify=false 上传即成功,不得产生任何补偿删除。 */
|
||||||
|
@Test
|
||||||
|
void verifyDisabledNeverEnqueuesCompensation() throws Exception {
|
||||||
|
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||||
|
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||||
|
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
|
||||||
|
|
||||||
|
String key = service.uploadBytes("task/chunk-462-uuid-x.json", "{}".getBytes(), false, true);
|
||||||
|
|
||||||
|
assertEquals("task/chunk-462-uuid-x.json", key);
|
||||||
|
verify(retryService, never()).enqueue(anyString(), any(Throwable.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 异常路径 =====
|
||||||
|
|
||||||
|
/** put 从未成功过:对象本就不存在,不该产生删除动作(与开关无关)。 */
|
||||||
|
@Test
|
||||||
|
void putNeverCompletedDoesNotEnqueueCompensation() throws Exception {
|
||||||
|
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||||
|
MinioClient client = mock(MinioClient.class);
|
||||||
|
when(client.putObject(any(PutObjectArgs.class))).thenThrow(new IOException("put failed"));
|
||||||
|
TransientStorageProperties properties = configuredProperties();
|
||||||
|
properties.setUploadMaxRetries(1);
|
||||||
|
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||||
|
properties, emptyProvider(), provider(retryService), () -> client);
|
||||||
|
|
||||||
|
assertThrows(RuntimeException.class,
|
||||||
|
() -> service.uploadBytes("task/chunk-462-uuid-x.json", "{}".getBytes(), true, true));
|
||||||
|
|
||||||
|
verify(retryService, never()).enqueue(anyString(), any(Throwable.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 补偿删除入队本身失败:不得把上传失败改写成入队异常,原异常必须继续上抛。 */
|
||||||
|
@Test
|
||||||
|
void enqueueFailureDoesNotMaskUploadFailure() throws Exception {
|
||||||
|
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||||
|
org.mockito.Mockito.doThrow(new IllegalStateException("queue down"))
|
||||||
|
.when(retryService).enqueue(anyString(), any(Throwable.class));
|
||||||
|
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||||
|
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
|
||||||
|
|
||||||
|
IllegalStateException ex = assertThrows(IllegalStateException.class,
|
||||||
|
() -> service.uploadBytes("task/chunk-9-11111111-2222-3333-4444-555555555555.json",
|
||||||
|
"{}".getBytes(), true, true));
|
||||||
|
|
||||||
|
assertFalse(ex.getMessage() == null || ex.getMessage().contains("queue down"),
|
||||||
|
"入队异常不应成为对外错误;实际=" + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== key 形态判定的边界 =====
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void versionedKeyShapeIsRecognized() {
|
||||||
|
assertTrue(RustfsObjectStorageService.isVersionedObjectKey(
|
||||||
|
"task-chunk/appearance_patent/28459/hash/chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json"));
|
||||||
|
assertTrue(RustfsObjectStorageService.isVersionedObjectKey(
|
||||||
|
"task-chunk/x/1/hash/0ffc254b-afad-4279-aab2-e85e3ff955e9.json"),
|
||||||
|
"整个 entryKey 就是 UUID 时也应识别为唯一 key");
|
||||||
|
assertTrue(RustfsObjectStorageService.isVersionedObjectKey(
|
||||||
|
"task-chunk/x/1/hash/chunk-1-0ffc254b-afad-4279-aab2-e85e3ff955e9"),
|
||||||
|
"无 .json 后缀但末段是 UUID,同样视为唯一");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void deterministicKeyShapeIsNotRecognized() {
|
||||||
|
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("task-chunk/x/1/hash/chunk-462.json"));
|
||||||
|
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("task-parsed/x/1/hash/latest.json"));
|
||||||
|
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("task-result-payload/x/1/hash/28459.json"),
|
||||||
|
"提交号这类 key 会被重写复用,不能当成唯一 key");
|
||||||
|
assertFalse(RustfsObjectStorageService.isVersionedObjectKey(
|
||||||
|
"task-chunk/x/1/0ffc254b-afad-4279-aab2-e85e3ff955e9/chunk-462.json"),
|
||||||
|
"UUID 出现在中间段不算唯一——重写复用的判据只看末段");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void malformedKeyShapeDoesNotThrowAndIsTreatedAsShared() {
|
||||||
|
assertFalse(RustfsObjectStorageService.isVersionedObjectKey(null));
|
||||||
|
assertFalse(RustfsObjectStorageService.isVersionedObjectKey(""));
|
||||||
|
assertFalse(RustfsObjectStorageService.isVersionedObjectKey(" "));
|
||||||
|
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("chunk-462.json"));
|
||||||
|
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("-afad-4279-aab2-e85e3ff955e9.json".substring(0, 20)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 辅助 =====
|
||||||
|
|
||||||
|
private static MinioClient putOkButStatFails() {
|
||||||
|
MinioClient client = mock(MinioClient.class);
|
||||||
|
try {
|
||||||
|
when(client.putObject(any(PutObjectArgs.class))).thenReturn(null);
|
||||||
|
when(client.statObject(any(StatObjectArgs.class))).thenThrow(new IOException("stat 抖动"));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException(ex);
|
||||||
|
}
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TransientStorageProperties configuredProperties() {
|
||||||
|
TransientStorageProperties properties = new TransientStorageProperties();
|
||||||
|
properties.setEndpoint("http://127.0.0.1:9000");
|
||||||
|
properties.setBucket("bucket");
|
||||||
|
properties.setAccessKeyId("ak");
|
||||||
|
properties.setAccessKeySecret("sk");
|
||||||
|
properties.setBaseRetryDelayMillis(0);
|
||||||
|
properties.setRetryJitterMillis(0);
|
||||||
|
return properties;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static <T> ObjectProvider<T> provider(T value) {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
ObjectProvider<T> provider = mock(ObjectProvider.class);
|
||||||
|
when(provider.getIfAvailable()).thenReturn(value);
|
||||||
|
return provider;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static <T> ObjectProvider<T> emptyProvider() {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
ObjectProvider<T> provider = mock(ObjectProvider.class);
|
||||||
|
when(provider.getIfAvailable()).thenReturn(null);
|
||||||
|
return provider;
|
||||||
|
}
|
||||||
|
}
|
||||||
+195
@@ -14,6 +14,7 @@ import java.util.Map;
|
|||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
@@ -120,6 +121,200 @@ class PriceTrackLoopRunServiceTest {
|
|||||||
verify(loopRunMapper, never()).updateById(loop);
|
verify(loopRunMapper, never()).updateById(loop);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void 子任务因客户端中断失败时循环保持运行并重派当前轮() throws Exception {
|
||||||
|
ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class);
|
||||||
|
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||||
|
PriceTrackLoopRunService service = new PriceTrackLoopRunService(
|
||||||
|
loopRunMapper, fileTaskMapper, objectMapper, mock(ZiniaoShopSwitchService.class));
|
||||||
|
|
||||||
|
PriceTrackLoopRunEntity loop = new PriceTrackLoopRunEntity();
|
||||||
|
loop.setId(2699L);
|
||||||
|
loop.setUserId(977L);
|
||||||
|
loop.setStatus("RUNNING");
|
||||||
|
loop.setExecutionMode("INFINITE");
|
||||||
|
loop.setCurrentRound(1);
|
||||||
|
loop.setCurrentShopIndex(0);
|
||||||
|
loop.setActiveTaskId(28587L);
|
||||||
|
loop.setResumeAttempt(0);
|
||||||
|
loop.setStopRequested(false);
|
||||||
|
loop.setShopsJson(objectMapper.writeValueAsString(List.of(shop("张美莺"))));
|
||||||
|
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(28587L);
|
||||||
|
task.setUserId(977L);
|
||||||
|
task.setModuleType("PRICE_TRACK");
|
||||||
|
task.setStatus("FAILED");
|
||||||
|
task.setErrorMessage("客户端异常中断: 客户端重启恢复上报(上次任务类型: price-track-run)");
|
||||||
|
task.setRequestJson(objectMapper.writeValueAsString(Map.of(
|
||||||
|
"loopRunId", 2699, "roundIndex", 1, "shopIndex", 0)));
|
||||||
|
|
||||||
|
when(loopRunMapper.selectList(any())).thenReturn(List.of(loop));
|
||||||
|
when(fileTaskMapper.selectById(28587L)).thenReturn(task);
|
||||||
|
|
||||||
|
service.syncLoopRunAfterChildTerminal(28587L);
|
||||||
|
|
||||||
|
assertEquals("RUNNING", loop.getStatus(), "中断不该终止循环");
|
||||||
|
assertNull(loop.getActiveTaskId(), "必须清空 active_task_id 才能重派当前轮");
|
||||||
|
assertNull(loop.getFinishedAt());
|
||||||
|
assertNull(loop.getErrorMessage());
|
||||||
|
assertEquals(1, loop.getCurrentRound(), "轮次不变:原地续跑");
|
||||||
|
assertEquals(0, loop.getCurrentShopIndex(), "店铺不变:原地续跑");
|
||||||
|
assertEquals(1, loop.getResumeAttempt(), "中断计数递增");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void 中断重派达上限后循环判失败() throws Exception {
|
||||||
|
ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class);
|
||||||
|
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||||
|
PriceTrackLoopRunService service = new PriceTrackLoopRunService(
|
||||||
|
loopRunMapper, fileTaskMapper, objectMapper, mock(ZiniaoShopSwitchService.class));
|
||||||
|
|
||||||
|
PriceTrackLoopRunEntity loop = new PriceTrackLoopRunEntity();
|
||||||
|
loop.setId(2699L);
|
||||||
|
loop.setUserId(977L);
|
||||||
|
loop.setStatus("RUNNING");
|
||||||
|
loop.setExecutionMode("INFINITE");
|
||||||
|
loop.setCurrentRound(1);
|
||||||
|
loop.setCurrentShopIndex(0);
|
||||||
|
loop.setActiveTaskId(28587L);
|
||||||
|
loop.setResumeAttempt(3);
|
||||||
|
loop.setStopRequested(false);
|
||||||
|
loop.setShopsJson(objectMapper.writeValueAsString(List.of(shop("张美莺"))));
|
||||||
|
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(28587L);
|
||||||
|
task.setUserId(977L);
|
||||||
|
task.setModuleType("PRICE_TRACK");
|
||||||
|
task.setStatus("FAILED");
|
||||||
|
task.setErrorMessage("客户端异常中断: 客户端重启恢复上报");
|
||||||
|
task.setRequestJson(objectMapper.writeValueAsString(Map.of(
|
||||||
|
"loopRunId", 2699, "roundIndex", 1, "shopIndex", 0)));
|
||||||
|
|
||||||
|
when(loopRunMapper.selectList(any())).thenReturn(List.of(loop));
|
||||||
|
when(fileTaskMapper.selectById(28587L)).thenReturn(task);
|
||||||
|
|
||||||
|
service.syncLoopRunAfterChildTerminal(28587L);
|
||||||
|
|
||||||
|
assertEquals("FAILED", loop.getStatus(), "达上限后不再重派,交人工介入");
|
||||||
|
assertNotNull(loop.getErrorMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void 非中断原因的子任务失败仍终止循环() throws Exception {
|
||||||
|
ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class);
|
||||||
|
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||||
|
PriceTrackLoopRunService service = new PriceTrackLoopRunService(
|
||||||
|
loopRunMapper, fileTaskMapper, objectMapper, mock(ZiniaoShopSwitchService.class));
|
||||||
|
|
||||||
|
PriceTrackLoopRunEntity loop = new PriceTrackLoopRunEntity();
|
||||||
|
loop.setId(2699L);
|
||||||
|
loop.setUserId(977L);
|
||||||
|
loop.setStatus("RUNNING");
|
||||||
|
loop.setExecutionMode("INFINITE");
|
||||||
|
loop.setCurrentRound(1);
|
||||||
|
loop.setCurrentShopIndex(0);
|
||||||
|
loop.setActiveTaskId(28587L);
|
||||||
|
loop.setResumeAttempt(0);
|
||||||
|
loop.setStopRequested(false);
|
||||||
|
loop.setShopsJson(objectMapper.writeValueAsString(List.of(shop("张美莺"))));
|
||||||
|
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(28587L);
|
||||||
|
task.setUserId(977L);
|
||||||
|
task.setModuleType("PRICE_TRACK");
|
||||||
|
task.setStatus("FAILED");
|
||||||
|
task.setErrorMessage("张美莺: 国家 德国 未跑完:重试后仍未获取到页码信息");
|
||||||
|
task.setRequestJson(objectMapper.writeValueAsString(Map.of(
|
||||||
|
"loopRunId", 2699, "roundIndex", 1, "shopIndex", 0)));
|
||||||
|
|
||||||
|
when(loopRunMapper.selectList(any())).thenReturn(List.of(loop));
|
||||||
|
when(fileTaskMapper.selectById(28587L)).thenReturn(task);
|
||||||
|
|
||||||
|
service.syncLoopRunAfterChildTerminal(28587L);
|
||||||
|
|
||||||
|
assertEquals("FAILED", loop.getStatus(), "业务失败照旧终止循环,只有中断才续跑");
|
||||||
|
assertEquals(0, loop.getResumeAttempt());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void 子任务成功后中断计数归零() throws Exception {
|
||||||
|
ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class);
|
||||||
|
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||||
|
PriceTrackLoopRunService service = new PriceTrackLoopRunService(
|
||||||
|
loopRunMapper, fileTaskMapper, objectMapper, mock(ZiniaoShopSwitchService.class));
|
||||||
|
|
||||||
|
PriceTrackLoopRunEntity loop = new PriceTrackLoopRunEntity();
|
||||||
|
loop.setId(2699L);
|
||||||
|
loop.setUserId(977L);
|
||||||
|
loop.setStatus("RUNNING");
|
||||||
|
loop.setExecutionMode("FINITE");
|
||||||
|
loop.setTargetRounds(1);
|
||||||
|
loop.setCurrentRound(1);
|
||||||
|
loop.setCurrentShopIndex(0);
|
||||||
|
loop.setActiveTaskId(28587L);
|
||||||
|
loop.setResumeAttempt(2);
|
||||||
|
loop.setStopRequested(false);
|
||||||
|
loop.setShopsJson(objectMapper.writeValueAsString(List.of(shop("张美莺"))));
|
||||||
|
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(28587L);
|
||||||
|
task.setUserId(977L);
|
||||||
|
task.setModuleType("PRICE_TRACK");
|
||||||
|
task.setStatus("SUCCESS");
|
||||||
|
task.setRequestJson(objectMapper.writeValueAsString(Map.of(
|
||||||
|
"loopRunId", 2699, "roundIndex", 1, "shopIndex", 0)));
|
||||||
|
|
||||||
|
when(loopRunMapper.selectList(any())).thenReturn(List.of(loop));
|
||||||
|
when(fileTaskMapper.selectById(28587L)).thenReturn(task);
|
||||||
|
|
||||||
|
service.syncLoopRunAfterChildTerminal(28587L);
|
||||||
|
|
||||||
|
assertEquals(0, loop.getResumeAttempt(), "成功一轮后中断计数归零,避免历史中断占用额度");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void 已请求停止时中断失败不续派而是停止() throws Exception {
|
||||||
|
ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
PriceTrackLoopRunMapper loopRunMapper = mock(PriceTrackLoopRunMapper.class);
|
||||||
|
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||||
|
PriceTrackLoopRunService service = new PriceTrackLoopRunService(
|
||||||
|
loopRunMapper, fileTaskMapper, objectMapper, mock(ZiniaoShopSwitchService.class));
|
||||||
|
|
||||||
|
PriceTrackLoopRunEntity loop = new PriceTrackLoopRunEntity();
|
||||||
|
loop.setId(2699L);
|
||||||
|
loop.setUserId(977L);
|
||||||
|
loop.setStatus("RUNNING");
|
||||||
|
loop.setExecutionMode("INFINITE");
|
||||||
|
loop.setCurrentRound(1);
|
||||||
|
loop.setCurrentShopIndex(0);
|
||||||
|
loop.setActiveTaskId(28587L);
|
||||||
|
loop.setResumeAttempt(0);
|
||||||
|
loop.setStopRequested(true);
|
||||||
|
loop.setShopsJson(objectMapper.writeValueAsString(List.of(shop("张美莺"))));
|
||||||
|
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(28587L);
|
||||||
|
task.setUserId(977L);
|
||||||
|
task.setModuleType("PRICE_TRACK");
|
||||||
|
task.setStatus("FAILED");
|
||||||
|
task.setErrorMessage("客户端异常中断: 客户端重启恢复上报");
|
||||||
|
task.setRequestJson(objectMapper.writeValueAsString(Map.of(
|
||||||
|
"loopRunId", 2699, "roundIndex", 1, "shopIndex", 0)));
|
||||||
|
|
||||||
|
when(loopRunMapper.selectList(any())).thenReturn(List.of(loop));
|
||||||
|
when(fileTaskMapper.selectById(28587L)).thenReturn(task);
|
||||||
|
|
||||||
|
service.syncLoopRunAfterChildTerminal(28587L);
|
||||||
|
|
||||||
|
assertEquals("STOPPED", loop.getStatus(), "停止意图优先于自动续跑");
|
||||||
|
assertEquals(0, loop.getResumeAttempt());
|
||||||
|
}
|
||||||
|
|
||||||
private PriceTrackMatchShopsVo.PriceTrackShopQueueItem shop(String name) {
|
private PriceTrackMatchShopsVo.PriceTrackShopQueueItem shop(String name) {
|
||||||
PriceTrackMatchShopsVo.PriceTrackShopQueueItem item = new PriceTrackMatchShopsVo.PriceTrackShopQueueItem();
|
PriceTrackMatchShopsVo.PriceTrackShopQueueItem item = new PriceTrackMatchShopsVo.PriceTrackShopQueueItem();
|
||||||
item.setShopName(name);
|
item.setShopName(name);
|
||||||
|
|||||||
+159
-2
@@ -37,6 +37,7 @@ import org.junit.jupiter.api.BeforeAll;
|
|||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
import org.mockito.InjectMocks;
|
import org.mockito.InjectMocks;
|
||||||
import org.mockito.Mock;
|
import org.mockito.Mock;
|
||||||
import org.mockito.Spy;
|
import org.mockito.Spy;
|
||||||
@@ -109,6 +110,10 @@ class PublishTaskServiceTest {
|
|||||||
|
|
||||||
@InjectMocks private PublishTaskService service;
|
@InjectMocks private PublishTaskService service;
|
||||||
|
|
||||||
|
/** 店铺互斥按 (设备, 店铺) 判定:两台机器各自的设备标识。 */
|
||||||
|
private static final String DEVICE_A = "device-aaaa";
|
||||||
|
private static final String DEVICE_B = "device-bbbb";
|
||||||
|
|
||||||
private final List<TaskChunkEntity> storedChunks = new ArrayList<>();
|
private final List<TaskChunkEntity> storedChunks = new ArrayList<>();
|
||||||
private final List<TaskScopeStateEntity> storedScopes = new ArrayList<>();
|
private final List<TaskScopeStateEntity> storedScopes = new ArrayList<>();
|
||||||
private final Map<String, String> rustfsPayloads = new LinkedHashMap<>();
|
private final Map<String, String> rustfsPayloads = new LinkedHashMap<>();
|
||||||
@@ -744,13 +749,165 @@ class PublishTaskServiceTest {
|
|||||||
when(publishFileMapper.selectOne(any())).thenReturn(running);
|
when(publishFileMapper.selectOne(any())).thenReturn(running);
|
||||||
|
|
||||||
BusinessException error = assertThrows(BusinessException.class,
|
BusinessException error = assertThrows(BusinessException.class,
|
||||||
() -> service.activateFile(taskId, fileId, 7L));
|
() -> service.activateFile(taskId, fileId, 7L, DEVICE_A));
|
||||||
|
|
||||||
assertTrue(error.getMessage().contains("已有文件正在执行"));
|
assertTrue(error.getMessage().contains("已有文件正在执行"));
|
||||||
verify(publishFileMapper, never()).update(isNull(), any());
|
verify(publishFileMapper, never()).update(isNull(), any());
|
||||||
verify(lock).close();
|
verify(lock).close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void activateFileRejectsShopAlreadyRunningOnSameDevice() {
|
||||||
|
long taskId = 106L;
|
||||||
|
long fileId = 206L;
|
||||||
|
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||||
|
PublishFileEntity target = file(taskId, fileId, "PENDING", "林洪武.xlsx");
|
||||||
|
target.setShopName("林洪武");
|
||||||
|
// 同一台设备上另一个任务(28520)正跑同一店铺 —— 2026-09-17 事故形态
|
||||||
|
PublishFileEntity otherTaskRunning = file(105L, 205L, "RUNNING", "林洪武.xlsx");
|
||||||
|
otherTaskRunning.setShopName("林洪武");
|
||||||
|
otherTaskRunning.setDeviceId(DEVICE_A);
|
||||||
|
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||||
|
|
||||||
|
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
|
||||||
|
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
||||||
|
when(publishFileMapper.selectById(fileId)).thenReturn(target);
|
||||||
|
// 第一次 selectOne:同任务其它 RUNNING 文件(无);第二次:同设备同店铺跨任务(有)
|
||||||
|
when(publishFileMapper.selectOne(any())).thenReturn(null, otherTaskRunning);
|
||||||
|
|
||||||
|
BusinessException error = assertThrows(BusinessException.class,
|
||||||
|
() -> service.activateFile(taskId, fileId, 7L, DEVICE_A));
|
||||||
|
|
||||||
|
assertTrue(error.getMessage().contains("林洪武"), "提示要带店铺名: " + error.getMessage());
|
||||||
|
assertTrue(error.getMessage().contains("105"), "提示要带占用中的任务号: " + error.getMessage());
|
||||||
|
verify(publishFileMapper, never()).update(isNull(), any());
|
||||||
|
verify(lock).close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 用户需求:同一家店允许在不同客户端(不同机器)上并行跑不同国家。 */
|
||||||
|
@Test
|
||||||
|
void activateFileAllowsShopRunningOnAnotherDevice() {
|
||||||
|
long taskId = 109L;
|
||||||
|
long fileId = 209L;
|
||||||
|
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||||
|
PublishFileEntity target = file(taskId, fileId, "PENDING", "林芳.xlsx");
|
||||||
|
target.setShopName("林芳");
|
||||||
|
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||||
|
|
||||||
|
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
|
||||||
|
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
||||||
|
when(publishFileMapper.selectById(fileId)).thenReturn(target);
|
||||||
|
// 另一台设备的占用行不会命中本设备的查询,因此这里返回 null
|
||||||
|
when(publishFileMapper.selectOne(any())).thenReturn(null);
|
||||||
|
when(publishFileMapper.update(isNull(), any())).thenReturn(1);
|
||||||
|
|
||||||
|
service.activateFile(taskId, fileId, 7L, DEVICE_B);
|
||||||
|
|
||||||
|
verify(publishFileMapper).update(isNull(), any());
|
||||||
|
// 店铺互斥查询必须真正带上设备维度,否则跨机器并行会被重新挡住
|
||||||
|
ArgumentCaptor<LambdaQueryWrapper> captor = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||||
|
verify(publishFileMapper, times(2)).selectOne(captor.capture());
|
||||||
|
String shopSql = captor.getAllValues().get(1).getSqlSegment();
|
||||||
|
assertTrue(shopSql.contains("device_id"), "店铺互斥查询要带设备维度: " + shopSql);
|
||||||
|
assertTrue(shopSql.contains("shop_name"), "设备维度不能替代店铺维度: " + shopSql);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 存量行(改动前激活、device_id 为空)无法判断落在哪台机器上,一律保守视为可能同机。 */
|
||||||
|
@Test
|
||||||
|
void activateFileRejectsWhenRunningRowHasUnknownDevice() {
|
||||||
|
long taskId = 110L;
|
||||||
|
long fileId = 210L;
|
||||||
|
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||||
|
PublishFileEntity target = file(taskId, fileId, "PENDING", "林芳.xlsx");
|
||||||
|
target.setShopName("林芳");
|
||||||
|
PublishFileEntity legacyRunning = file(105L, 205L, "RUNNING", "林芳.xlsx");
|
||||||
|
legacyRunning.setShopName("林芳");
|
||||||
|
legacyRunning.setDeviceId(null);
|
||||||
|
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||||
|
|
||||||
|
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
|
||||||
|
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
||||||
|
when(publishFileMapper.selectById(fileId)).thenReturn(target);
|
||||||
|
when(publishFileMapper.selectOne(any())).thenReturn(null, legacyRunning);
|
||||||
|
|
||||||
|
BusinessException error = assertThrows(BusinessException.class,
|
||||||
|
() -> service.activateFile(taskId, fileId, 7L, DEVICE_B));
|
||||||
|
|
||||||
|
assertTrue(error.getMessage().contains("105"), "存量未知设备行必须继续拦住: " + error.getMessage());
|
||||||
|
verify(publishFileMapper, never()).update(isNull(), any());
|
||||||
|
verify(lock).close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 无设备标识(旧客户端 token 无 deviceId claim / 内部令牌调用)→ 退回改动前的全局店铺互斥。 */
|
||||||
|
@Test
|
||||||
|
void activateFileFallsBackToGlobalShopCheckWhenDeviceMissing() {
|
||||||
|
long taskId = 111L;
|
||||||
|
long fileId = 211L;
|
||||||
|
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||||
|
PublishFileEntity target = file(taskId, fileId, "PENDING", "林芳.xlsx");
|
||||||
|
target.setShopName("林芳");
|
||||||
|
PublishFileEntity otherDeviceRunning = file(105L, 205L, "RUNNING", "林芳.xlsx");
|
||||||
|
otherDeviceRunning.setShopName("林芳");
|
||||||
|
otherDeviceRunning.setDeviceId(DEVICE_A);
|
||||||
|
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||||
|
|
||||||
|
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
|
||||||
|
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
||||||
|
when(publishFileMapper.selectById(fileId)).thenReturn(target);
|
||||||
|
when(publishFileMapper.selectOne(any())).thenReturn(null, otherDeviceRunning);
|
||||||
|
|
||||||
|
BusinessException error = assertThrows(BusinessException.class,
|
||||||
|
() -> service.activateFile(taskId, fileId, 7L, ""));
|
||||||
|
|
||||||
|
assertTrue(error.getMessage().contains("105"), "来源不明时必须保持全局互斥: " + error.getMessage());
|
||||||
|
ArgumentCaptor<LambdaQueryWrapper> captor = ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||||
|
verify(publishFileMapper, times(2)).selectOne(captor.capture());
|
||||||
|
assertFalse(captor.getAllValues().get(1).getSqlSegment().contains("device_id"),
|
||||||
|
"来源不明时不应按设备收窄,否则等于放宽互斥");
|
||||||
|
verify(publishFileMapper, never()).update(isNull(), any());
|
||||||
|
verify(lock).close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void activateFileAllowsWhenShopHasNoOtherRunningTask() {
|
||||||
|
long taskId = 107L;
|
||||||
|
long fileId = 207L;
|
||||||
|
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||||
|
PublishFileEntity target = file(taskId, fileId, "PENDING", "林洪武.xlsx");
|
||||||
|
target.setShopName("林洪武");
|
||||||
|
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||||
|
|
||||||
|
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
|
||||||
|
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
||||||
|
when(publishFileMapper.selectById(fileId)).thenReturn(target);
|
||||||
|
when(publishFileMapper.selectOne(any())).thenReturn(null);
|
||||||
|
when(publishFileMapper.update(isNull(), any())).thenReturn(1);
|
||||||
|
|
||||||
|
service.activateFile(taskId, fileId, 7L, DEVICE_A);
|
||||||
|
|
||||||
|
verify(publishFileMapper).update(isNull(), any());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void activateFileSkipsShopCheckWhenShopNameBlank() {
|
||||||
|
long taskId = 108L;
|
||||||
|
long fileId = 208L;
|
||||||
|
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||||
|
// 未匹配到店铺的文件:没有店铺标识就不做店铺维度校验(不能因此阻断激活)
|
||||||
|
PublishFileEntity target = file(taskId, fileId, "PENDING", "未匹配.xlsx");
|
||||||
|
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||||
|
|
||||||
|
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
|
||||||
|
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
||||||
|
when(publishFileMapper.selectById(fileId)).thenReturn(target);
|
||||||
|
when(publishFileMapper.selectOne(any())).thenReturn(null);
|
||||||
|
when(publishFileMapper.update(isNull(), any())).thenReturn(1);
|
||||||
|
|
||||||
|
service.activateFile(taskId, fileId, 7L, DEVICE_A);
|
||||||
|
|
||||||
|
verify(publishFileMapper, times(1)).selectOne(any());
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void activateFileIsIdempotentForTheSameRunningFile() {
|
void activateFileIsIdempotentForTheSameRunningFile() {
|
||||||
long taskId = 105L;
|
long taskId = 105L;
|
||||||
@@ -763,7 +920,7 @@ class PublishTaskServiceTest {
|
|||||||
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
||||||
when(publishFileMapper.selectById(fileId)).thenReturn(running);
|
when(publishFileMapper.selectById(fileId)).thenReturn(running);
|
||||||
|
|
||||||
service.activateFile(taskId, fileId, 7L);
|
service.activateFile(taskId, fileId, 7L, DEVICE_A);
|
||||||
|
|
||||||
verify(publishFileMapper, never()).selectOne(any());
|
verify(publishFileMapper, never()).selectOne(any());
|
||||||
verify(publishFileMapper, never()).update(isNull(), any());
|
verify(publishFileMapper, never()).update(isNull(), any());
|
||||||
|
|||||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
@@ -137,6 +138,7 @@ class ShopDataCrawlChunkUpsertTest {
|
|||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
mock(ShopDataCrawlItemStoreService.class),
|
mock(ShopDataCrawlItemStoreService.class),
|
||||||
|
mock(DuplicateCheckRefreshPort.class),
|
||||||
null,
|
null,
|
||||||
mock(TaskProgressLightAssembler.class));
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
|
|||||||
+2
@@ -16,6 +16,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResu
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
@@ -174,6 +175,7 @@ class ShopDataCrawlCleanupTest {
|
|||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
mock(ShopDataCrawlItemStoreService.class),
|
mock(ShopDataCrawlItemStoreService.class),
|
||||||
|
mock(DuplicateCheckRefreshPort.class),
|
||||||
null,
|
null,
|
||||||
mock(TaskProgressLightAssembler.class));
|
mock(TaskProgressLightAssembler.class));
|
||||||
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
|
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
|
||||||
|
|||||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
@@ -146,6 +147,7 @@ class ShopDataCrawlDailyFileIncrementalTest {
|
|||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
mock(ShopDataCrawlItemStoreService.class),
|
mock(ShopDataCrawlItemStoreService.class),
|
||||||
|
mock(DuplicateCheckRefreshPort.class),
|
||||||
null,
|
null,
|
||||||
mock(TaskProgressLightAssembler.class));
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
|
|||||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
@@ -152,6 +153,7 @@ class ShopDataCrawlDailyFileJobSplitTest {
|
|||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
mock(ShopDataCrawlItemStoreService.class),
|
mock(ShopDataCrawlItemStoreService.class),
|
||||||
|
mock(DuplicateCheckRefreshPort.class),
|
||||||
null,
|
null,
|
||||||
mock(TaskProgressLightAssembler.class));
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
|
|||||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
@@ -152,6 +153,7 @@ class ShopDataCrawlDailyFileLockTest {
|
|||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
mock(ShopDataCrawlItemStoreService.class),
|
mock(ShopDataCrawlItemStoreService.class),
|
||||||
|
mock(DuplicateCheckRefreshPort.class),
|
||||||
null,
|
null,
|
||||||
mock(TaskProgressLightAssembler.class));
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
|
|||||||
+2
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
@@ -134,6 +135,7 @@ class ShopDataCrawlLightweightProgressTest {
|
|||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
mock(ShopDataCrawlItemStoreService.class),
|
mock(ShopDataCrawlItemStoreService.class),
|
||||||
|
mock(DuplicateCheckRefreshPort.class),
|
||||||
null,
|
null,
|
||||||
mock(TaskProgressLightAssembler.class));
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
|
|||||||
+2
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
@@ -139,6 +140,7 @@ class ShopDataCrawlOwnerColumnTest {
|
|||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
mock(ShopDataCrawlItemStoreService.class),
|
mock(ShopDataCrawlItemStoreService.class),
|
||||||
|
mock(DuplicateCheckRefreshPort.class),
|
||||||
null,
|
null,
|
||||||
mock(TaskProgressLightAssembler.class));
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
|
|||||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
@@ -146,6 +147,7 @@ class ShopDataCrawlProgressQueryTest {
|
|||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
mock(ShopDataCrawlItemStoreService.class),
|
mock(ShopDataCrawlItemStoreService.class),
|
||||||
|
mock(DuplicateCheckRefreshPort.class),
|
||||||
null,
|
null,
|
||||||
mock(TaskProgressLightAssembler.class));
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
|
|||||||
+2
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
@@ -133,6 +134,7 @@ class ShopDataCrawlRowDedupKeyTest {
|
|||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
mock(ShopDataCrawlItemStoreService.class),
|
mock(ShopDataCrawlItemStoreService.class),
|
||||||
|
mock(DuplicateCheckRefreshPort.class),
|
||||||
null,
|
null,
|
||||||
mock(TaskProgressLightAssembler.class));
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
|
|||||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
@@ -134,6 +135,7 @@ class ShopDataCrawlScopeCounterTest {
|
|||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
mock(ShopDataCrawlItemStoreService.class),
|
mock(ShopDataCrawlItemStoreService.class),
|
||||||
|
mock(DuplicateCheckRefreshPort.class),
|
||||||
null,
|
null,
|
||||||
mock(TaskProgressLightAssembler.class));
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
|
|||||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
@@ -135,6 +136,7 @@ class ShopDataCrawlScopeMergeTest {
|
|||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
mock(ShopDataCrawlItemStoreService.class),
|
mock(ShopDataCrawlItemStoreService.class),
|
||||||
|
mock(DuplicateCheckRefreshPort.class),
|
||||||
null,
|
null,
|
||||||
mock(TaskProgressLightAssembler.class));
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
|
|||||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
@@ -129,6 +130,7 @@ class ShopDataCrawlTaskServiceChunkTest {
|
|||||||
instanceMetadata,
|
instanceMetadata,
|
||||||
dailyFileService,
|
dailyFileService,
|
||||||
mock(ShopDataCrawlItemStoreService.class),
|
mock(ShopDataCrawlItemStoreService.class),
|
||||||
|
mock(DuplicateCheckRefreshPort.class),
|
||||||
null,
|
null,
|
||||||
mock(TaskProgressLightAssembler.class));
|
mock(TaskProgressLightAssembler.class));
|
||||||
|
|
||||||
|
|||||||
+123
@@ -0,0 +1,123 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopduplicatecheck.service.support;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
import java.util.function.BooleanSupplier;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.junit.jupiter.api.Assertions.fail;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 撞款重扫调度器:合并窗口去抖(突发触发聚为一次扫描)、单飞、
|
||||||
|
* 锁忙有限重试、动作异常吸收(对外零抛出)。
|
||||||
|
*/
|
||||||
|
class DuplicateCheckRefreshSchedulerTest {
|
||||||
|
|
||||||
|
private static final long AWAIT_TIMEOUT_MILLIS = 5_000L;
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void request_mergesBurstIntoSingleScan() throws Exception {
|
||||||
|
AtomicInteger scans = new AtomicInteger();
|
||||||
|
CountDownLatch firstScan = new CountDownLatch(1);
|
||||||
|
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||||
|
scans.incrementAndGet();
|
||||||
|
firstScan.countDown();
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.DONE;
|
||||||
|
}, 150L, 30L, 3);
|
||||||
|
|
||||||
|
scheduler.request("burst-1");
|
||||||
|
Thread.sleep(10L);
|
||||||
|
scheduler.request("burst-2");
|
||||||
|
Thread.sleep(10L);
|
||||||
|
scheduler.request("burst-3");
|
||||||
|
|
||||||
|
assertTrue(firstScan.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), "合并窗口内的触发应执行扫描");
|
||||||
|
Thread.sleep(300L);
|
||||||
|
assertEquals(1, scans.get(), "合并窗口内多次触发只执行一次扫描");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void request_retriesWhileLockBusyThenSucceeds() throws Exception {
|
||||||
|
AtomicInteger attempts = new AtomicInteger();
|
||||||
|
CountDownLatch succeeded = new CountDownLatch(1);
|
||||||
|
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||||
|
if (attempts.incrementAndGet() <= 2) {
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.LOCK_BUSY;
|
||||||
|
}
|
||||||
|
succeeded.countDown();
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.DONE;
|
||||||
|
}, 10L, 30L, 5);
|
||||||
|
|
||||||
|
scheduler.request("retry");
|
||||||
|
|
||||||
|
assertTrue(succeeded.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), "锁忙重试后应成功执行");
|
||||||
|
Thread.sleep(100L);
|
||||||
|
assertEquals(3, attempts.get(), "2 次锁忙 + 1 次成功");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void request_givesUpAfterMaxLockRetries() throws Exception {
|
||||||
|
AtomicInteger attempts = new AtomicInteger();
|
||||||
|
CountDownLatch firstAttempt = new CountDownLatch(1);
|
||||||
|
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||||
|
attempts.incrementAndGet();
|
||||||
|
firstAttempt.countDown();
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.LOCK_BUSY;
|
||||||
|
}, 10L, 20L, 2);
|
||||||
|
|
||||||
|
scheduler.request("always-busy");
|
||||||
|
|
||||||
|
assertTrue(firstAttempt.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS));
|
||||||
|
awaitUntil(() -> attempts.get() >= 3, "应完成初试 + 2 次重试");
|
||||||
|
Thread.sleep(200L);
|
||||||
|
assertEquals(3, attempts.get(), "超过重试上限后放弃,不再执行");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void request_absorbsActionFailure() throws Exception {
|
||||||
|
AtomicInteger attempts = new AtomicInteger();
|
||||||
|
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||||
|
attempts.incrementAndGet();
|
||||||
|
throw new IllegalStateException("模拟扫描动作异常");
|
||||||
|
}, 10L, 20L, 1);
|
||||||
|
|
||||||
|
assertDoesNotThrow(() -> scheduler.request("boom"), "request 不得向调用方抛错");
|
||||||
|
awaitUntil(() -> attempts.get() >= 1, "动作应被执行");
|
||||||
|
Thread.sleep(100L);
|
||||||
|
assertEquals(1, attempts.get(), "动作异常视为失败,不做锁忙重试");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void request_afterPreviousCycleAllowsNewScan() throws Exception {
|
||||||
|
AtomicInteger scans = new AtomicInteger();
|
||||||
|
CountDownLatch twoScans = new CountDownLatch(2);
|
||||||
|
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||||
|
scans.incrementAndGet();
|
||||||
|
twoScans.countDown();
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.DONE;
|
||||||
|
}, 20L, 20L, 2);
|
||||||
|
|
||||||
|
scheduler.request("first");
|
||||||
|
awaitUntil(() -> scans.get() >= 1, "首轮扫描应执行");
|
||||||
|
scheduler.request("second");
|
||||||
|
|
||||||
|
assertTrue(twoScans.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), "新触发应再执行一次扫描");
|
||||||
|
assertEquals(2, scans.get());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void awaitUntil(BooleanSupplier condition, String message) throws InterruptedException {
|
||||||
|
long deadline = System.currentTimeMillis() + AWAIT_TIMEOUT_MILLIS;
|
||||||
|
while (System.currentTimeMillis() < deadline) {
|
||||||
|
if (condition.getAsBoolean()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Thread.sleep(10L);
|
||||||
|
}
|
||||||
|
fail(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
+97
@@ -33,6 +33,7 @@ import java.nio.charset.StandardCharsets;
|
|||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
import java.util.concurrent.atomic.AtomicReference;
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
@@ -42,10 +43,12 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyLong;
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
import static org.mockito.ArgumentMatchers.eq;
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
import static org.mockito.Mockito.doAnswer;
|
import static org.mockito.Mockito.doAnswer;
|
||||||
import static org.mockito.Mockito.lenient;
|
import static org.mockito.Mockito.lenient;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
import static org.mockito.Mockito.times;
|
import static org.mockito.Mockito.times;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
@@ -397,4 +400,98 @@ class SimilarAsinTaskServiceChunkMergeLimitTest {
|
|||||||
verify(taskChunkMapper, times(1)).update(any(), any());
|
verify(taskChunkMapper, times(1)).update(any(), any());
|
||||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ===== CAS 冲突处理(2026-09-17 线上任务 28459 遗留项,与外观专利同一口径) =====
|
||||||
|
|
||||||
|
/** 冲突重试耗尽:**保留**最后一次写入的对象作读兜底,异常带出期望/当前哈希。 */
|
||||||
|
@Test
|
||||||
|
void casConflictExhaustedKeepsLastStoredObjectAsFallback() throws Exception {
|
||||||
|
TaskChunkEntity chunk = chunk(7L, "hashA", 1, "ptr:chunk-A");
|
||||||
|
chunk.setPayloadHash("hashA");
|
||||||
|
stubConflictMerge(chunk, 0);
|
||||||
|
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(9004L);
|
||||||
|
Throwable ex = invokeMergeExpectingFailure(task, List.of(row("r1", "B0A0000001", "标题1")));
|
||||||
|
|
||||||
|
assertTrue(ex instanceof IllegalStateException, "实际: " + ex);
|
||||||
|
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-1");
|
||||||
|
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-2");
|
||||||
|
verify(transientPayloadStorageService, never()).deletePayloadIfPresent("sibling-3");
|
||||||
|
assertTrue(ex.getMessage().contains("hashA"), "需带出期望哈希,实际: " + ex.getMessage());
|
||||||
|
assertTrue(ex.getMessage().contains("hashB"), "需带出当前哈希,实际: " + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 冲突后重试成功:被顶替的中间对象删除,最终对象写入行(不删)。 */
|
||||||
|
@Test
|
||||||
|
void casConflictThenSuccessDeletesSupersededObjects() throws Exception {
|
||||||
|
TaskChunkEntity chunk = chunk(7L, "hashA", 1, "ptr:chunk-A");
|
||||||
|
chunk.setPayloadHash("hashA");
|
||||||
|
stubConflictMerge(chunk, 0, 0, 1);
|
||||||
|
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(9004L);
|
||||||
|
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
||||||
|
|
||||||
|
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-1");
|
||||||
|
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-2");
|
||||||
|
verify(transientPayloadStorageService, never()).deletePayloadIfPresent("sibling-3");
|
||||||
|
verify(transientPayloadStorageService).deleteReplacedPayloadIfNeeded(eq("ptr:chunk-A"), eq("sibling-3"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 冲突时读回行上的当前哈希供定位(此前只有一句 conflict,线上无法定位)。 */
|
||||||
|
@Test
|
||||||
|
void casConflictReadsBackCurrentHashForDiagnostics() throws Exception {
|
||||||
|
TaskChunkEntity chunk = chunk(7L, "hashA", 1, "ptr:chunk-A");
|
||||||
|
chunk.setPayloadHash("hashA");
|
||||||
|
stubConflictMerge(chunk, 0);
|
||||||
|
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(9004L);
|
||||||
|
invokeMergeExpectingFailure(task, List.of(row("r1", "B0A0000001", "标题1")));
|
||||||
|
|
||||||
|
verify(taskChunkMapper, times(3)).selectById(7L);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 读回行失败不得掩盖原始冲突:异常信息里给出可读标记。 */
|
||||||
|
@Test
|
||||||
|
void casConflictReadBackFailureDoesNotMaskConflict() throws Exception {
|
||||||
|
TaskChunkEntity chunk = chunk(7L, "hashA", 1, "ptr:chunk-A");
|
||||||
|
chunk.setPayloadHash("hashA");
|
||||||
|
stubConflictMerge(chunk, 0);
|
||||||
|
when(taskChunkMapper.selectById(anyLong())).thenThrow(new IllegalStateException("db down"));
|
||||||
|
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(9004L);
|
||||||
|
Throwable ex = invokeMergeExpectingFailure(task, List.of(row("r1", "B0A0000001", "标题1")));
|
||||||
|
|
||||||
|
assertTrue(ex.getMessage().contains("读取失败"), "实际: " + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 反射调用会包一层 InvocationTargetException,取根因以便断言业务异常。 */
|
||||||
|
private Throwable invokeMergeExpectingFailure(FileTaskEntity task, List<SimilarAsinResultRowDto> rows) throws Exception {
|
||||||
|
try {
|
||||||
|
invokeMerge(service, task, "hashA", 1, rows);
|
||||||
|
return null;
|
||||||
|
} catch (java.lang.reflect.InvocationTargetException ex) {
|
||||||
|
return ex.getCause();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 冲突场景公共 stub:每次尝试写一个不同的版本化对象,update 按传入序列返回。 */
|
||||||
|
private void stubConflictMerge(TaskChunkEntity chunk, Integer... updateResults) throws Exception {
|
||||||
|
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||||
|
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||||
|
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||||
|
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
|
||||||
|
AtomicInteger stores = new AtomicInteger();
|
||||||
|
when(transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||||
|
anyString(), any(), anyString(), any(), anyString()))
|
||||||
|
.thenAnswer(inv -> "sibling-" + stores.incrementAndGet());
|
||||||
|
when(taskChunkMapper.update(any(), any())).thenReturn(updateResults[0], java.util.Arrays.copyOfRange(updateResults, 1, updateResults.length));
|
||||||
|
|
||||||
|
TaskChunkEntity current = chunk(7L, "hashB", 1, "ptr:chunk-A");
|
||||||
|
current.setPayloadHash("hashB");
|
||||||
|
lenient().when(taskChunkMapper.selectById(7L)).thenReturn(current);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-1
@@ -107,7 +107,9 @@ class ResultSuccessTimingContractTest {
|
|||||||
doAnswer(invocation -> null)
|
doAnswer(invocation -> null)
|
||||||
.when(publishTaskService).submitResult(eq(TASK_ID), any(PublishSubmitResultRequest.class));
|
.when(publishTaskService).submitResult(eq(TASK_ID), any(PublishSubmitResultRequest.class));
|
||||||
|
|
||||||
ApiResponse<Void> response = new PublishController(publishTaskService)
|
// 第二个构造参数是 AdminAuthSupport(activateFile 取设备号用),本用例只测 submitResult,
|
||||||
|
// 不经过该依赖,故传 null(同文件其它 controller 用例亦有传 null 的先例)。
|
||||||
|
ApiResponse<Void> response = new PublishController(publishTaskService, null)
|
||||||
.submitResult(TASK_ID, new PublishSubmitResultRequest());
|
.submitResult(TASK_ID, new PublishSubmitResultRequest());
|
||||||
|
|
||||||
assertTrue(response.isSuccess(), "success=true");
|
assertTrue(response.isSuccess(), "success=true");
|
||||||
|
|||||||
+33
@@ -107,6 +107,8 @@ class SuccessTimingContractTest {
|
|||||||
.thenReturn("oss://shufuai/collect-data/6868.xlsx");
|
.thenReturn("oss://shufuai/collect-data/6868.xlsx");
|
||||||
lenient().when(fileResultMapper.updateById(any(FileResultEntity.class))).thenReturn(1);
|
lenient().when(fileResultMapper.updateById(any(FileResultEntity.class))).thenReturn(1);
|
||||||
lenient().when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(1);
|
lenient().when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(1);
|
||||||
|
// 终态条件更新的返回值为「匹配行数」:任务非 FAILED 时匹配 1 行(默认用例都基于 RUNNING 任务)。
|
||||||
|
lenient().when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
private TaskFileJobEntity job() {
|
private TaskFileJobEntity job() {
|
||||||
@@ -124,6 +126,8 @@ class SuccessTimingContractTest {
|
|||||||
void successHappensOnlyAfterFileGeneratedAndUploaded() {
|
void successHappensOnlyAfterFileGeneratedAndUploaded() {
|
||||||
service.processResultFileJob(job());
|
service.processResultFileJob(job());
|
||||||
|
|
||||||
|
// 契约核心:结果文件「生成并上传」必须先于任何落库(结果行与任务终态)。
|
||||||
|
// 保持结果行先写:若进程在这两步之间退出,任务仍是 RUNNING,陈旧巡检可重新组装(自愈)。
|
||||||
var order = inOrder(ossStorageService, fileResultMapper, fileTaskMapper);
|
var order = inOrder(ossStorageService, fileResultMapper, fileTaskMapper);
|
||||||
order.verify(ossStorageService).uploadResultFile(any(), eq("COLLECT_DATA"));
|
order.verify(ossStorageService).uploadResultFile(any(), eq("COLLECT_DATA"));
|
||||||
order.verify(fileResultMapper).updateById(any(FileResultEntity.class));
|
order.verify(fileResultMapper).updateById(any(FileResultEntity.class));
|
||||||
@@ -220,6 +224,35 @@ class SuccessTimingContractTest {
|
|||||||
verify(fileTaskMapper, org.mockito.Mockito.times(1)).update(isNull(), any(LambdaUpdateWrapper.class));
|
verify(fileTaskMapper, org.mockito.Mockito.times(1)).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* taskId 28599:组装完成时任务已是 FAILED(客户端上报失败或陈旧判死在前)——
|
||||||
|
* 条件更新不匹配,任务保持失败态与真实原因,但结果文件的 url 照常落库,
|
||||||
|
* 用户「看到失败」的同时仍能下载已采集的部分结果。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void failedTaskKeepsFailureButStillCarriesDownloadUrl() {
|
||||||
|
String reason = "中间分批回传失败,终止本次采集以避免服务端数据残缺";
|
||||||
|
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
|
||||||
|
FileTaskEntity failedTask = runningTask();
|
||||||
|
failedTask.setStatus("FAILED");
|
||||||
|
failedTask.setErrorMessage(reason);
|
||||||
|
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(failedTask);
|
||||||
|
FileResultEntity failedResult = runningResult();
|
||||||
|
failedResult.setErrorMessage(reason);
|
||||||
|
when(fileResultMapper.selectById(RESULT_ID)).thenReturn(failedResult);
|
||||||
|
|
||||||
|
service.processResultFileJob(job());
|
||||||
|
|
||||||
|
ArgumentCaptor<FileResultEntity> resultCaptor = ArgumentCaptor.forClass(FileResultEntity.class);
|
||||||
|
// 失败态是补偿写:先乐观写成功,条件更新不匹配后再写回失败语义
|
||||||
|
verify(fileResultMapper, org.mockito.Mockito.times(2)).updateById(resultCaptor.capture());
|
||||||
|
FileResultEntity result = resultCaptor.getAllValues().get(1);
|
||||||
|
assertEquals("oss://shufuai/collect-data/6868.xlsx", result.getResultFileUrl(),
|
||||||
|
"失败任务也要有可下载的结果文件");
|
||||||
|
assertEquals(0, result.getSuccess(), "任务失败则结果记录保持失败语义,前端显示「失败」而非「已完成」");
|
||||||
|
assertEquals(reason, result.getErrorMessage(), "真实失败原因不能被组装流程清掉");
|
||||||
|
}
|
||||||
|
|
||||||
private FileTaskEntity runningTask() {
|
private FileTaskEntity runningTask() {
|
||||||
FileTaskEntity task = new FileTaskEntity();
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
task.setId(TASK_ID);
|
task.setId(TASK_ID);
|
||||||
|
|||||||
+9
-3
@@ -33,8 +33,13 @@ import static org.mockito.Mockito.when;
|
|||||||
/**
|
/**
|
||||||
* task-150:清理前引用检查契约(plan 09)。
|
* task-150:清理前引用检查契约(plan 09)。
|
||||||
* payload 被 biz_task_chunk / biz_task_scope_state 引用则不清理;
|
* payload 被 biz_task_chunk / biz_task_scope_state 引用则不清理;
|
||||||
* chunk 仅剩自身一行(count=1)不算共享引用;查询异常保守保留;
|
* chunk 命中任意行(含仅剩 1 行)都不算「可安全清理」;查询异常保守保留;
|
||||||
* 检查只读无副作用;候选值批量反查。
|
* 检查只读无副作用;候选值批量反查。
|
||||||
|
*
|
||||||
|
* <p>2026-09-17 线上任务 28459 之后收紧:原契约把 count==1 当作「调用方自己那行」放行删除,
|
||||||
|
* 但调用方无法证明那一行就是自己(如合并已把自己那行指向新对象、另一行仍指向旧对象时 count 恰为 1),
|
||||||
|
* 删除即造成该分片永久读不到 → 整单失败。物理删除本就约定在 DB 行删除之后执行,正常路径引用数必然为 0;
|
||||||
|
* 宁可留下孤儿对象(有保留期清理兜底),也不删掉可能仍被引用的对象。
|
||||||
*/
|
*/
|
||||||
@ExtendWith(MockitoExtension.class)
|
@ExtendWith(MockitoExtension.class)
|
||||||
class TaskPayloadReferenceCheckTest {
|
class TaskPayloadReferenceCheckTest {
|
||||||
@@ -100,13 +105,14 @@ class TaskPayloadReferenceCheckTest {
|
|||||||
verify(rustfsObjectStorageService, never()).deleteObject(any());
|
verify(rustfsObjectStorageService, never()).deleteObject(any());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** count==1 无法证明那一行就是调用方自己(可能正是别的行仍在用)→ 保守不删。 */
|
||||||
@Test
|
@Test
|
||||||
void singleChunkRowIsOwnRowNotSharedReference() {
|
void singleChunkRowBlocksDeleteBecauseOwnershipCannotBeProven() {
|
||||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||||
|
|
||||||
service.deletePayloadIfPresent(RUSTFS_VALUE);
|
service.deletePayloadIfPresent(RUSTFS_VALUE);
|
||||||
|
|
||||||
verify(rustfsObjectStorageService).deleteObject("payload-key");
|
verify(rustfsObjectStorageService, never()).deleteObject(any());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
+168
@@ -0,0 +1,168 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.spi.ClientTaskPullSpi;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
|
import org.springframework.test.util.ReflectionTestUtils;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.Mockito.doThrow;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 客户端中断任务的自动续跑:保留失败记录 + 重新排队 PENDING 续跑任务。
|
||||||
|
*
|
||||||
|
* <p>背景:2026-09-18 任务 28587 因客户端被更新脚本重启而中断后一直挂 RUNNING;
|
||||||
|
* 中断上报把任务标 FAILED 之后,还需要有人把长任务放回队列,否则一遇客户端更新就整个白跑。
|
||||||
|
*/
|
||||||
|
class TaskResumeServiceTest {
|
||||||
|
|
||||||
|
private static ClientTaskPullSpi spi(String moduleType) {
|
||||||
|
return new ClientTaskPullSpi() {
|
||||||
|
@Override
|
||||||
|
public String moduleType() {
|
||||||
|
return moduleType;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, Object> buildQueuePayload(FileTaskEntity task) {
|
||||||
|
return Map.of("type", "x-run", "data", Map.of("taskId", task.getId()));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TaskResumeService service(FileTaskMapper mapper, List<ClientTaskPullSpi> handlers,
|
||||||
|
boolean enabled, int maxAttempt) {
|
||||||
|
TaskResumeService service = new TaskResumeService(mapper, handlers);
|
||||||
|
ReflectionTestUtils.setField(service, "enabled", enabled);
|
||||||
|
ReflectionTestUtils.setField(service, "maxAttempt", maxAttempt);
|
||||||
|
ReflectionTestUtils.setField(service, "windowMinutes", 30L);
|
||||||
|
ReflectionTestUtils.setField(service, "limit", 20);
|
||||||
|
return service;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileTaskEntity interruptedTask(Long id, String moduleType) {
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(id);
|
||||||
|
task.setModuleType(moduleType);
|
||||||
|
task.setUserId(977L);
|
||||||
|
task.setTaskMode("PYTHON_QUEUE");
|
||||||
|
task.setStatus("FAILED");
|
||||||
|
task.setErrorMessage("客户端异常中断: 客户端重启恢复上报(上次任务类型: similar-asin-run)");
|
||||||
|
task.setRequestJson("{\"items\":[]}");
|
||||||
|
task.setResumeAttempt(0);
|
||||||
|
task.setFinishedAt(LocalDateTime.now());
|
||||||
|
return task;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void 中断任务会被重新排队为待执行的续跑任务() {
|
||||||
|
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||||
|
when(mapper.selectList(any())).thenReturn(List.of(interruptedTask(28587L, "SIMILAR_ASIN")));
|
||||||
|
when(mapper.selectCount(any())).thenReturn(0L);
|
||||||
|
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||||
|
|
||||||
|
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||||
|
|
||||||
|
assertEquals(1, stats.scannedTaskCount);
|
||||||
|
assertEquals(1, stats.resumedTaskCount);
|
||||||
|
ArgumentCaptor<FileTaskEntity> captor = ArgumentCaptor.forClass(FileTaskEntity.class);
|
||||||
|
verify(mapper).insert(captor.capture());
|
||||||
|
FileTaskEntity resumed = captor.getValue();
|
||||||
|
assertEquals("PENDING", resumed.getStatus(), "续跑任务必须是待领取状态,交给客户端兜底拉取");
|
||||||
|
assertEquals(28587L, resumed.getResumeOfTaskId(), "必须回指原任务,用于追溯与幂等");
|
||||||
|
assertEquals(1, resumed.getResumeAttempt());
|
||||||
|
assertEquals(977L, resumed.getUserId());
|
||||||
|
assertEquals("SIMILAR_ASIN", resumed.getModuleType());
|
||||||
|
assertEquals("{\"items\":[]}", resumed.getRequestJson(), "原任务参数必须整体带走");
|
||||||
|
assertTrue(resumed.getTaskNo().startsWith("SIMILAR_ASIN-"), "任务号沿用模块前缀");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void 已有续跑任务时跳过避免重复排队() {
|
||||||
|
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||||
|
when(mapper.selectList(any())).thenReturn(List.of(interruptedTask(28587L, "SIMILAR_ASIN")));
|
||||||
|
when(mapper.selectCount(any())).thenReturn(1L, 1L);
|
||||||
|
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||||
|
|
||||||
|
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||||
|
|
||||||
|
assertEquals(1, stats.skippedTaskCount);
|
||||||
|
assertEquals(0, stats.resumedTaskCount);
|
||||||
|
verify(mapper, never()).insert(any(FileTaskEntity.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void 开关关闭时不扫描() {
|
||||||
|
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||||
|
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), false, 3);
|
||||||
|
|
||||||
|
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||||
|
|
||||||
|
assertEquals(0, stats.scannedTaskCount);
|
||||||
|
verify(mapper, never()).selectList(any());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void 没有注册模块兜底实现时不扫描() {
|
||||||
|
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||||
|
TaskResumeService service = service(mapper, List.of(), true, 3);
|
||||||
|
|
||||||
|
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||||
|
|
||||||
|
assertEquals(0, stats.scannedTaskCount);
|
||||||
|
verify(mapper, never()).selectList(any());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void 插入失败只计入跳过不抛出() {
|
||||||
|
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||||
|
when(mapper.selectList(any())).thenReturn(List.of(interruptedTask(28587L, "SIMILAR_ASIN")));
|
||||||
|
when(mapper.selectCount(any())).thenReturn(0L);
|
||||||
|
doThrow(new RuntimeException("库挂了")).when(mapper).insert(any(FileTaskEntity.class));
|
||||||
|
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||||
|
|
||||||
|
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||||
|
|
||||||
|
assertEquals(1, stats.skippedTaskCount);
|
||||||
|
assertEquals(0, stats.resumedTaskCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void 无候选时统计不支持续跑的中断任务数() {
|
||||||
|
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||||
|
when(mapper.selectList(any())).thenReturn(List.of());
|
||||||
|
when(mapper.selectCount(any())).thenReturn(3L);
|
||||||
|
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||||
|
|
||||||
|
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||||
|
|
||||||
|
assertEquals(0, stats.resumedTaskCount);
|
||||||
|
assertEquals(3, stats.unsupportedTaskCount, "不支持续跑的模块也要可见,供运维人工重跑");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void 没有归属用户的中断任务跳过() {
|
||||||
|
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||||
|
FileTaskEntity orphan = interruptedTask(28587L, "SIMILAR_ASIN");
|
||||||
|
orphan.setUserId(null);
|
||||||
|
when(mapper.selectList(any())).thenReturn(List.of(orphan));
|
||||||
|
TaskResumeService service = service(mapper, List.of(spi("SIMILAR_ASIN")), true, 3);
|
||||||
|
|
||||||
|
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||||
|
|
||||||
|
assertEquals(1, stats.skippedTaskCount);
|
||||||
|
verify(mapper, never()).insert(any(FileTaskEntity.class));
|
||||||
|
}
|
||||||
|
}
|
||||||
+230
@@ -0,0 +1,230 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.config.InstanceMetadata;
|
||||||
|
import com.nanri.aiimage.config.StorageProperties;
|
||||||
|
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||||
|
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyInt;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* chunk 载荷对象缺失时的「同槽位版本化兄弟对象」读兜底(2026-09-17 线上任务 28459)。
|
||||||
|
*
|
||||||
|
* <p>事故后 DB 行指向的普通 key({@code chunk-462.json})已被误删,但同槽位的版本化对象
|
||||||
|
* ({@code chunk-462-<uuid>.json})还在。兜底让读路径自动改用兄弟对象,任务即可按已有数据出结果,
|
||||||
|
* 而不是整单 FAILED——这是防止「删错对象 → 任务永久失败」的最后一道网。
|
||||||
|
*/
|
||||||
|
class TransientPayloadChunkSiblingFallbackTest {
|
||||||
|
|
||||||
|
private static final String DIR = "task-chunk/appearance_patent/28459/"
|
||||||
|
+ "2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e/";
|
||||||
|
private static final String CHUNK_KEY = DIR + "chunk-462.json";
|
||||||
|
private static final String CHUNK_POINTER = "rustfs:" + CHUNK_KEY;
|
||||||
|
private static final String SIBLING_KEY = DIR + "chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json";
|
||||||
|
|
||||||
|
// ===== 正常路径 =====
|
||||||
|
|
||||||
|
/** 对象在:直接读,不得触发列对象(正常读路径不能因为兜底变慢)。 */
|
||||||
|
@Test
|
||||||
|
void presentObjectIsReadWithoutListing() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
when(rustfs.readObjectBytes(CHUNK_KEY)).thenReturn(bytes("[\"原始\"]"));
|
||||||
|
|
||||||
|
String payload = service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed");
|
||||||
|
|
||||||
|
assertEquals("[\"原始\"]", payload);
|
||||||
|
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 对象缺失 + 存在兄弟对象 → 读兄弟对象(本次事故的自动恢复路径)。 */
|
||||||
|
@Test
|
||||||
|
void missingChunkObjectFallsBackToVersionedSibling() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
|
||||||
|
when(rustfs.listObjectKeysNewestFirst(DIR + "chunk-462-", 50)).thenReturn(List.of(SIBLING_KEY));
|
||||||
|
when(rustfs.readObjectBytes(SIBLING_KEY)).thenReturn(bytes("[\"合并后\"]"));
|
||||||
|
|
||||||
|
String payload = service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed");
|
||||||
|
|
||||||
|
assertEquals("[\"合并后\"]", payload);
|
||||||
|
verify(rustfs).listObjectKeysNewestFirst(DIR + "chunk-462-", 50);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 多个兄弟对象:取列表首个(实现约定按最后修改时间倒序,最新在前)。 */
|
||||||
|
@Test
|
||||||
|
void newestSiblingIsPreferred() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
String newest = DIR + "chunk-462-ffffffff-ffff-ffff-ffff-ffffffffffff.json";
|
||||||
|
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
|
||||||
|
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of(newest, SIBLING_KEY));
|
||||||
|
when(rustfs.readObjectBytes(newest)).thenReturn(bytes("[\"最新\"]"));
|
||||||
|
|
||||||
|
assertEquals("[\"最新\"]", service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
|
||||||
|
verify(rustfs).readObjectBytes(newest);
|
||||||
|
verify(rustfs, never()).readObjectBytes(SIBLING_KEY);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 指针是 JSON 编码形式(DB 里的常见写法)时兜底同样生效。 */
|
||||||
|
@Test
|
||||||
|
void jsonQuotedPointerStillFallsBack() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
|
||||||
|
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of(SIBLING_KEY));
|
||||||
|
when(rustfs.readObjectBytes(SIBLING_KEY)).thenReturn(bytes("[\"带引号\"]"));
|
||||||
|
|
||||||
|
assertEquals("[\"带引号\"]",
|
||||||
|
service(rustfs).resolvePayload('"' + CHUNK_POINTER + '"', "read chunk failed"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 非指针内容(直接就是载荷 JSON)原样返回,不触碰对象存储。 */
|
||||||
|
@Test
|
||||||
|
void rawContentIsReturnedAsIs() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
|
||||||
|
assertEquals("[1,2,3]", service(rustfs).resolvePayload("[1,2,3]", "read chunk failed"));
|
||||||
|
verify(rustfs, never()).readObjectBytes(anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 边界:不该兜底的场景 =====
|
||||||
|
|
||||||
|
/** 非 chunk 槽位(如 row-1.json)缺失 → 不兜底,照旧抛错(避免误配无关对象)。 */
|
||||||
|
@Test
|
||||||
|
void nonChunkSlotDoesNotFallBack() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
String key = "task-result-item/similar-asin/9/hash/row-1.json";
|
||||||
|
when(rustfs.readObjectBytes(key)).thenThrow(missingKey());
|
||||||
|
|
||||||
|
assertThrows(IllegalStateException.class,
|
||||||
|
() -> service(rustfs).resolvePayload("rustfs:" + key, "read item failed"));
|
||||||
|
|
||||||
|
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** parsed 槽位(latest.json)缺失 → 不兜底(该槽位没有版本化兄弟语义)。 */
|
||||||
|
@Test
|
||||||
|
void parsedSlotDoesNotFallBack() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
String key = "task-parsed/appearance_patent/28459/hash/latest.json";
|
||||||
|
when(rustfs.readObjectBytes(key)).thenThrow(missingKey());
|
||||||
|
|
||||||
|
assertThrows(IllegalStateException.class,
|
||||||
|
() -> service(rustfs).resolvePayload("rustfs:" + key, "read parsed failed"));
|
||||||
|
|
||||||
|
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 非缺失类失败(权限/网络)→ 不兜底,照旧上抛以便重试。 */
|
||||||
|
@Test
|
||||||
|
void nonMissingFailureDoesNotFallBack() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
when(rustfs.readObjectBytes(CHUNK_KEY))
|
||||||
|
.thenThrow(new IllegalStateException("Access Denied"));
|
||||||
|
|
||||||
|
assertThrows(IllegalStateException.class,
|
||||||
|
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
|
||||||
|
|
||||||
|
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 异常路径 =====
|
||||||
|
|
||||||
|
/** 没有兄弟对象 → 抛错,且错误信息必须保留原 errorMessage 与原指针(便于定位)。 */
|
||||||
|
@Test
|
||||||
|
void noSiblingRethrowsWithOriginalPointer() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
|
||||||
|
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of());
|
||||||
|
|
||||||
|
IllegalStateException ex = assertThrows(IllegalStateException.class,
|
||||||
|
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
|
||||||
|
|
||||||
|
assertTrue(ex.getMessage().contains("read chunk failed"), "错误信息=" + ex.getMessage());
|
||||||
|
assertTrue(ex.getMessage().contains("chunk-462.json"), "错误信息=" + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 兄弟对象读取也失败 → 异常上抛,不得静默返回空内容。 */
|
||||||
|
@Test
|
||||||
|
void siblingReadFailurePropagates() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
|
||||||
|
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of(SIBLING_KEY));
|
||||||
|
when(rustfs.readObjectBytes(SIBLING_KEY)).thenThrow(new IllegalStateException("read timeout"));
|
||||||
|
|
||||||
|
IllegalStateException ex = assertThrows(IllegalStateException.class,
|
||||||
|
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
|
||||||
|
|
||||||
|
assertTrue(ex.getMessage().contains("read chunk failed"), "错误信息=" + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 列对象本身失败(桶不可用)→ 仍按原错误抛出,错误信息不含兄弟查找细节。 */
|
||||||
|
@Test
|
||||||
|
void listFailureDoesNotMaskOriginalError() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
|
||||||
|
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt()))
|
||||||
|
.thenThrow(new IllegalStateException("bucket unavailable"));
|
||||||
|
|
||||||
|
IllegalStateException ex = assertThrows(IllegalStateException.class,
|
||||||
|
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
|
||||||
|
|
||||||
|
assertTrue(ex.getMessage().contains("read chunk failed"), "错误信息=" + ex.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** local 指针不走 rustfs 兄弟兜底。 */
|
||||||
|
@Test
|
||||||
|
void localPointerDoesNotUseChunkSiblingFallback() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
|
||||||
|
assertThrows(IllegalStateException.class,
|
||||||
|
() -> service(rustfs).resolvePayload("local:i/server-121/" + CHUNK_KEY, "read chunk failed"));
|
||||||
|
|
||||||
|
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 辅助 =====
|
||||||
|
|
||||||
|
private static IllegalStateException missingKey() {
|
||||||
|
return new IllegalStateException("The specified key does not exist.");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] bytes(String value) {
|
||||||
|
return value.getBytes(StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
|
||||||
|
private TransientPayloadStorageService service(RustfsObjectStorageService rustfs) {
|
||||||
|
TransientStorageProperties transientProperties = new TransientStorageProperties();
|
||||||
|
transientProperties.setEnabled(true);
|
||||||
|
transientProperties.setEndpoint("http://127.0.0.1:9000");
|
||||||
|
transientProperties.setBucket("json-server");
|
||||||
|
transientProperties.setAccessKeyId("ak");
|
||||||
|
transientProperties.setAccessKeySecret("sk");
|
||||||
|
StorageProperties storageProperties = new StorageProperties();
|
||||||
|
storageProperties.setLocalTempDir(System.getProperty("java.io.tmpdir"));
|
||||||
|
return new TransientPayloadStorageService(
|
||||||
|
transientProperties,
|
||||||
|
storageProperties,
|
||||||
|
rustfs,
|
||||||
|
mock(OssStorageService.class),
|
||||||
|
new ObjectMapper(),
|
||||||
|
new InstanceMetadata("server-121"),
|
||||||
|
mock(TaskChunkMapper.class),
|
||||||
|
mock(TaskScopeStateMapper.class));
|
||||||
|
}
|
||||||
|
}
|
||||||
+180
@@ -0,0 +1,180 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.config.InstanceMetadata;
|
||||||
|
import com.nanri.aiimage.config.StorageProperties;
|
||||||
|
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 临时载荷删除前的引用守卫(2026-09-17 线上任务 28459 事故的第二条同族缺陷)。
|
||||||
|
*
|
||||||
|
* <p>守卫本意是「还有别的行引用同一 pointer 就不物理删」。原实现用
|
||||||
|
* {@code chunkCount > 1} 作判据,等价于「恰好还有 1 行引用时可以删」——
|
||||||
|
* 这在 caller 已经把自己那行挪走、但**另一行**仍指向旧对象的场景下,会把
|
||||||
|
* 对方仍在用的对象删掉。docs 里 CollectDataDeleteConsistencyTest 已明确口径是
|
||||||
|
* 「跨任务共享指针仍引用 → 跳过删除」,故判据收紧为「命中任意一行即不删」。
|
||||||
|
*/
|
||||||
|
class TransientPayloadDeleteReferenceGuardTest {
|
||||||
|
|
||||||
|
private static final String RUSTFS_CHUNK_POINTER =
|
||||||
|
"rustfs:task-chunk/appearance_patent/28459/2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e/chunk-462.json";
|
||||||
|
|
||||||
|
// ===== 正常路径 =====
|
||||||
|
|
||||||
|
/** 没有任何行引用 → 允许物理删除。 */
|
||||||
|
@Test
|
||||||
|
void zeroReferencesAllowsPhysicalDelete() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
TaskScopeStateMapper scopeMapper = mock(TaskScopeStateMapper.class);
|
||||||
|
when(chunkMapper.selectCount(any())).thenReturn(0L);
|
||||||
|
when(scopeMapper.selectCount(any())).thenReturn(0L);
|
||||||
|
|
||||||
|
service(rustfs, chunkMapper, scopeMapper).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
|
||||||
|
|
||||||
|
verify(rustfs).deleteObject(eq("task-chunk/appearance_patent/28459/2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e/chunk-462.json"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 多条行引用 → 不删(原有行为必须保持)。 */
|
||||||
|
@Test
|
||||||
|
void multipleReferencesBlockDelete() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
when(chunkMapper.selectCount(any())).thenReturn(3L);
|
||||||
|
|
||||||
|
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class)).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
|
||||||
|
|
||||||
|
verify(rustfs, never()).deleteObject(anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 本次修复点 =====
|
||||||
|
|
||||||
|
/** 恰好 1 行仍引用 → 必须不删(原实现会删掉这唯一一行仍在用的对象)。 */
|
||||||
|
@Test
|
||||||
|
void singleReferencingChunkRowBlocksDelete() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
when(chunkMapper.selectCount(any())).thenReturn(1L);
|
||||||
|
|
||||||
|
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class)).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
|
||||||
|
|
||||||
|
verify(rustfs, never()).deleteObject(anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** scope_state 单行引用同样阻断删除(该字段命中即非自我引用)。 */
|
||||||
|
@Test
|
||||||
|
void singleScopeStateReferenceBlocksDelete() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
TaskScopeStateMapper scopeMapper = mock(TaskScopeStateMapper.class);
|
||||||
|
when(chunkMapper.selectCount(any())).thenReturn(0L);
|
||||||
|
when(scopeMapper.selectCount(any())).thenReturn(1L);
|
||||||
|
|
||||||
|
service(rustfs, chunkMapper, scopeMapper).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
|
||||||
|
|
||||||
|
verify(rustfs, never()).deleteObject(anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 边界 =====
|
||||||
|
|
||||||
|
/** 传入 JSON 编码(带引号)的指针——DB 里常见写法,必须能解析并同样受守卫约束。 */
|
||||||
|
@Test
|
||||||
|
void jsonQuotedPointerIsParsedAndStillGuarded() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
when(chunkMapper.selectCount(any())).thenReturn(1L);
|
||||||
|
|
||||||
|
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class))
|
||||||
|
.deletePayloadIfPresent('"' + RUSTFS_CHUNK_POINTER + '"');
|
||||||
|
|
||||||
|
verify(rustfs, never()).deleteObject(anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 非指针值原样传入 → 不解析、不查库、不删。 */
|
||||||
|
@Test
|
||||||
|
void nonPointerValueIsIgnoredWithoutLookup() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
|
||||||
|
TransientPayloadStorageService service = service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class));
|
||||||
|
assertDoesNotThrow(() -> service.deletePayloadIfPresent("{\"rows\":[1,2,3]}"));
|
||||||
|
|
||||||
|
verify(chunkMapper, never()).selectCount(any());
|
||||||
|
verify(rustfs, never()).deleteObject(anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 空值不得抛异常、不得触发任何删除。 */
|
||||||
|
@Test
|
||||||
|
void nullOrBlankValueIsIgnored() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
TransientPayloadStorageService service = service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class));
|
||||||
|
|
||||||
|
assertDoesNotThrow(() -> service.deletePayloadIfPresent(null));
|
||||||
|
assertDoesNotThrow(() -> service.deletePayloadIfPresent(" "));
|
||||||
|
|
||||||
|
verify(chunkMapper, never()).selectCount(any());
|
||||||
|
verify(rustfs, never()).deleteObject(anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 异常路径 =====
|
||||||
|
|
||||||
|
/** 反查引用的查询抛异常 → 保守不删(原实现即如此,必须保持)。 */
|
||||||
|
@Test
|
||||||
|
void referenceQueryFailureIsConservativeAndSkipsDelete() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
when(chunkMapper.selectCount(any())).thenThrow(new IllegalStateException("db down"));
|
||||||
|
|
||||||
|
assertDoesNotThrow(() ->
|
||||||
|
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class))
|
||||||
|
.deletePayloadIfPresent(RUSTFS_CHUNK_POINTER));
|
||||||
|
|
||||||
|
verify(rustfs, never()).deleteObject(anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 跨实例的 local 指针:不物理删(留给持有实例的清理任务)。 */
|
||||||
|
@Test
|
||||||
|
void crossInstanceLocalPointerIsNotDeleted() {
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
when(chunkMapper.selectCount(any())).thenReturn(0L);
|
||||||
|
|
||||||
|
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class))
|
||||||
|
.deletePayloadIfPresent("local:i/server-110/task-chunk/x/1/hash/chunk-1.json");
|
||||||
|
|
||||||
|
verify(rustfs, never()).deleteObject(anyString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private TransientPayloadStorageService service(RustfsObjectStorageService rustfs,
|
||||||
|
TaskChunkMapper chunkMapper,
|
||||||
|
TaskScopeStateMapper scopeMapper) {
|
||||||
|
TransientStorageProperties transientProperties = new TransientStorageProperties();
|
||||||
|
transientProperties.setEnabled(true);
|
||||||
|
StorageProperties storageProperties = new StorageProperties();
|
||||||
|
storageProperties.setLocalTempDir(System.getProperty("java.io.tmpdir"));
|
||||||
|
return new TransientPayloadStorageService(
|
||||||
|
transientProperties,
|
||||||
|
storageProperties,
|
||||||
|
rustfs,
|
||||||
|
mock(OssStorageService.class),
|
||||||
|
new ObjectMapper(),
|
||||||
|
new InstanceMetadata("server-121"),
|
||||||
|
chunkMapper,
|
||||||
|
scopeMapper);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,10 +11,14 @@ import router from '@/router'
|
|||||||
import { ensureAuth } from '@/shared/auth/ensure-auth'
|
import { ensureAuth } from '@/shared/auth/ensure-auth'
|
||||||
import { ensureApiSecretsLoaded, loadApiSecrets } from '@/shared/utils/api-secret-store'
|
import { ensureApiSecretsLoaded, loadApiSecrets } from '@/shared/utils/api-secret-store'
|
||||||
import { installUserTokenSync } from '@/shared/auth/user-token-sync.ts'
|
import { installUserTokenSync } from '@/shared/auth/user-token-sync.ts'
|
||||||
|
import { installSubmitGuard } from '@/shared/utils/submit-guard'
|
||||||
|
|
||||||
// 登录态令牌同步给桌面端 Python(A1/A3):无桥环境静默跳过
|
// 登录态令牌同步给桌面端 Python(A1/A3):无桥环境静默跳过
|
||||||
installUserTokenSync()
|
installUserTokenSync()
|
||||||
|
|
||||||
|
// 提交按钮防连点:捕获阶段拦下同一按钮的连点(各工具页 .btn-run 通用)
|
||||||
|
installSubmitGuard()
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 数富AI 前端统一入口(SPA,URL 无 .html 后缀)
|
* 数富AI 前端统一入口(SPA,URL 无 .html 后缀)
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -596,7 +596,9 @@ function taskKeywordProgress(item: CollectDataHistoryItem) {
|
|||||||
|
|
||||||
function canDownload(item: CollectDataHistoryItem) {
|
function canDownload(item: CollectDataHistoryItem) {
|
||||||
const status = normalizeTaskStatus(item)
|
const status = normalizeTaskStatus(item)
|
||||||
return Boolean(item.downloadUrl && (item.success || status === 'SUCCESS'))
|
// 失败任务只要已生成结果文件也允许下载:失败时后端按已收到的分片组装部分结果,
|
||||||
|
// 已采集的数据不应随任务失败一起不可用(taskId 28599 事故)。
|
||||||
|
return Boolean(item.downloadUrl && (item.success || status === 'SUCCESS' || status === 'FAILED'))
|
||||||
}
|
}
|
||||||
|
|
||||||
async function downloadResult(item: CollectDataHistoryItem) {
|
async function downloadResult(item: CollectDataHistoryItem) {
|
||||||
|
|||||||
@@ -166,7 +166,7 @@ import {
|
|||||||
} from '@/shared/api/java-modules'
|
} from '@/shared/api/java-modules'
|
||||||
import { getPywebviewApi, type UploadedJavaFile } from '@/shared/bridges/pywebview'
|
import { getPywebviewApi, type UploadedJavaFile } from '@/shared/bridges/pywebview'
|
||||||
import { saveUrlWithProgress } from '@/shared/utils/download-progress'
|
import { saveUrlWithProgress } from '@/shared/utils/download-progress'
|
||||||
import { EXCEL_EXTENSIONS, checkSelectedFiles, guardBlocked } from '@/shared/dispatch-guard'
|
import { EXCEL_EXTENSIONS, checkSelectedFiles, collectDistinctErrors, guardBlocked } from '@/shared/dispatch-guard'
|
||||||
import { passGuard } from '@/shared/dispatch-guard-ui'
|
import { passGuard } from '@/shared/dispatch-guard-ui'
|
||||||
import { formatDateTime } from '@/shared/utils/datetime'
|
import { formatDateTime } from '@/shared/utils/datetime'
|
||||||
import { uploadPathsToJava } from '@/shared/utils/upload-to-java'
|
import { uploadPathsToJava } from '@/shared/utils/upload-to-java'
|
||||||
@@ -349,11 +349,15 @@ async function submitConvertRun() {
|
|||||||
convertResultItems.value = result.items || []
|
convertResultItems.value = result.items || []
|
||||||
await loadConvertHistory()
|
await loadConvertHistory()
|
||||||
if (result.total > 0 && result.successCount === 0) {
|
if (result.total > 0 && result.successCount === 0) {
|
||||||
|
// 逐文件原因就在 items[].error 里,优先直接列出,不再让用户对着「常见原因」猜
|
||||||
|
const details = collectDistinctErrors((result.items || []).map((item) => item.error)).slice(0, 5)
|
||||||
await passGuard(
|
await passGuard(
|
||||||
guardBlocked(
|
guardBlocked(
|
||||||
'格式转换未成功',
|
'格式转换未成功',
|
||||||
`本次提交的 ${result.total} 个文件全部转换失败。\n` +
|
`本次提交的 ${result.total} 个文件全部转换失败。\n` +
|
||||||
'常见原因是源文件表头与所选模板不匹配,或文件内没有数据行。\n' +
|
(details.length
|
||||||
|
? `失败原因:\n${details.map((message) => `· ${message}`).join('\n')}\n`
|
||||||
|
: '常见原因是源文件表头与所选模板不匹配,或文件内没有数据行。\n') +
|
||||||
'请在右侧结果列表查看每个文件的失败原因,或换一个模板后重试。',
|
'请在右侧结果列表查看每个文件的失败原因,或换一个模板后重试。',
|
||||||
'convert.all-failed',
|
'convert.all-failed',
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -137,7 +137,7 @@ import { expandBrandFolderRecursive } from '@/shared/api/brand'
|
|||||||
import { deleteDedupeHistory, getDedupeHistory, getDedupeResultDownloadUrl, getDedupeRunProgress, getExcelInfo, runDedupe, type DedupeResultItem, type DedupeRunProgressVo, type DedupeRunVo } from '@/shared/api/java-modules'
|
import { deleteDedupeHistory, getDedupeHistory, getDedupeResultDownloadUrl, getDedupeRunProgress, getExcelInfo, runDedupe, type DedupeResultItem, type DedupeRunProgressVo, type DedupeRunVo } from '@/shared/api/java-modules'
|
||||||
import { getPywebviewApi, type UploadedJavaFile } from '@/shared/bridges/pywebview'
|
import { getPywebviewApi, type UploadedJavaFile } from '@/shared/bridges/pywebview'
|
||||||
import { saveUrlWithProgress } from '@/shared/utils/download-progress'
|
import { saveUrlWithProgress } from '@/shared/utils/download-progress'
|
||||||
import { EXCEL_EXTENSIONS, checkSelectedFiles, guardBlocked } from '@/shared/dispatch-guard'
|
import { EXCEL_EXTENSIONS, checkSelectedFiles, collectDistinctErrors, guardBlocked } from '@/shared/dispatch-guard'
|
||||||
import { passGuard } from '@/shared/dispatch-guard-ui'
|
import { passGuard } from '@/shared/dispatch-guard-ui'
|
||||||
import { formatDateTime } from '@/shared/utils/datetime'
|
import { formatDateTime } from '@/shared/utils/datetime'
|
||||||
import { uploadPathsToJava } from '@/shared/utils/upload-to-java'
|
import { uploadPathsToJava } from '@/shared/utils/upload-to-java'
|
||||||
@@ -237,7 +237,8 @@ function clearAllCleanColumns() {
|
|||||||
async function loadCleanHeaders(fileKey: string) {
|
async function loadCleanHeaders(fileKey: string) {
|
||||||
const result = await getExcelInfo(fileKey)
|
const result = await getExcelInfo(fileKey)
|
||||||
if (!result.headers?.length) {
|
if (!result.headers?.length) {
|
||||||
ElMessage.error('读取 Excel 表头失败')
|
// 接口本身成功、只是没解析出表头,说明文件内容有问题而非「读取失败」,别让用户反复重选文件
|
||||||
|
ElMessage.error('未读到表头行:请确认文件不是空表、且首行是表头')
|
||||||
cleanAvailableColumns.value = []
|
cleanAvailableColumns.value = []
|
||||||
cleanSelectedColumns.value = []
|
cleanSelectedColumns.value = []
|
||||||
return
|
return
|
||||||
@@ -359,11 +360,15 @@ async function submitCleanRun() {
|
|||||||
await loadCleanHistory()
|
await loadCleanHistory()
|
||||||
// 全部文件都失败时不该报「完成」,让用户看清是哪一批没处理成功
|
// 全部文件都失败时不该报「完成」,让用户看清是哪一批没处理成功
|
||||||
if (result.total > 0 && result.successCount === 0) {
|
if (result.total > 0 && result.successCount === 0) {
|
||||||
|
// 逐文件原因就在 items[].error 里,优先直接列出,不再让用户对着「常见原因」猜
|
||||||
|
const details = collectDistinctErrors((result.items || []).map((item) => item.error)).slice(0, 5)
|
||||||
await passGuard(
|
await passGuard(
|
||||||
guardBlocked(
|
guardBlocked(
|
||||||
'去重未成功',
|
'去重未成功',
|
||||||
`本次提交的 ${result.total} 个文件全部处理失败。\n` +
|
`本次提交的 ${result.total} 个文件全部处理失败。\n` +
|
||||||
'常见原因是表头列名与所选保留列对不上,或文件内没有数据行。\n' +
|
(details.length
|
||||||
|
? `失败原因:\n${details.map((message) => `· ${message}`).join('\n')}\n`
|
||||||
|
: '常见原因是表头列名与所选保留列对不上,或文件内没有数据行。\n') +
|
||||||
'请在右侧结果列表查看每个文件的失败原因后重试。',
|
'请在右侧结果列表查看每个文件的失败原因后重试。',
|
||||||
'dedupe.all-failed',
|
'dedupe.all-failed',
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -1105,7 +1105,13 @@ async function submitRun() {
|
|||||||
syncResultState()
|
syncResultState()
|
||||||
|
|
||||||
if (hasBlockedItems) {
|
if (hasBlockedItems) {
|
||||||
ElMessage.warning('部分文件尚未匹配到店铺,已保留状态信息,请稍后重试。')
|
// 具体原因(店铺未录入 / 索引未命中 / 表头错误等)在各文件项里,直接带出来;
|
||||||
|
// 只报「请稍后重试」会把「需去后台添加店铺」误导成「等一等就好」
|
||||||
|
const firstBlocked = normalizedItems.find((item) => !isUsableMatchedItem(item))
|
||||||
|
const detail = firstBlocked ? getDisplayError(firstBlocked) : ''
|
||||||
|
ElMessage.warning(detail
|
||||||
|
? `部分文件不可启动:${detail}`
|
||||||
|
: '部分文件尚未匹配到店铺,已保留状态信息,请稍后重试。')
|
||||||
} else if (hasStaleMatchedItems) {
|
} else if (hasStaleMatchedItems) {
|
||||||
ElMessage.warning('部分文件匹配到的店铺信息已过期,仍可启动任务;后台会自动重新匹配。')
|
ElMessage.warning('部分文件匹配到的店铺信息已过期,仍可启动任务;后台会自动重新匹配。')
|
||||||
} else if (hasRunnableItems) {
|
} else if (hasRunnableItems) {
|
||||||
|
|||||||
@@ -511,6 +511,12 @@ function formatMatchRemark(row: PatrolDeleteShopQueueItem) {
|
|||||||
if (row.matched) {
|
if (row.matched) {
|
||||||
return "已匹配成功,请查看状态确认";
|
return "已匹配成功,请查看状态确认";
|
||||||
}
|
}
|
||||||
|
if (row.matchStatus === "CONFLICT") {
|
||||||
|
return "存在多个同名店铺,请人工确认";
|
||||||
|
}
|
||||||
|
if (row.matchStatus === "PENDING") {
|
||||||
|
return "店铺尚未匹配完成,请稍后查看";
|
||||||
|
}
|
||||||
return "未匹配成功,请检查店铺名";
|
return "未匹配成功,请检查店铺名";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -848,6 +848,8 @@ function formatMatchRemark(row: PriceTrackShopQueueItem) {
|
|||||||
if (msg) return msg
|
if (msg) return msg
|
||||||
if (row.matched && row.matchStatus === 'MATCHED') return '已匹配成功,可启动任务'
|
if (row.matched && row.matchStatus === 'MATCHED') return '已匹配成功,可启动任务'
|
||||||
if (row.matched) return '已关联店铺,请查看状态确认'
|
if (row.matched) return '已关联店铺,请查看状态确认'
|
||||||
|
if (row.matchStatus === 'CONFLICT') return '存在多个同名店铺,请人工确认'
|
||||||
|
if (row.matchStatus === 'PENDING') return '店铺尚未匹配完成,请稍后查看'
|
||||||
return '未匹配成功,请检查店铺名'
|
return '未匹配成功,请检查店铺名'
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1656,7 +1658,8 @@ function statusClass(item: PriceTrackHistoryItem) {
|
|||||||
function canDownload(item: PriceTrackHistoryItem) {
|
function canDownload(item: PriceTrackHistoryItem) {
|
||||||
if (!item.resultId || (!item.fileReady && !item.downloadUrl)) return false
|
if (!item.resultId || (!item.fileReady && !item.downloadUrl)) return false
|
||||||
const st = resolvedTaskStatus(item)
|
const st = resolvedTaskStatus(item)
|
||||||
return st === 'SUCCESS' || item.success === true
|
// 失败任务也可能有已跑出来的部分结果文件(后端按已收分片组装),有文件就允许下载
|
||||||
|
return st === 'SUCCESS' || item.success === true || st === 'FAILED'
|
||||||
}
|
}
|
||||||
|
|
||||||
async function downloadResult(item: PriceTrackHistoryItem) {
|
async function downloadResult(item: PriceTrackHistoryItem) {
|
||||||
|
|||||||
@@ -1018,7 +1018,8 @@ function statusClass(item: ProductRiskHistoryItem) {
|
|||||||
function canDownload(item: ProductRiskHistoryItem) {
|
function canDownload(item: ProductRiskHistoryItem) {
|
||||||
if (!item.resultId || (!item.fileReady && !item.downloadUrl)) return false
|
if (!item.resultId || (!item.fileReady && !item.downloadUrl)) return false
|
||||||
const st = resolvedTaskStatus(item)
|
const st = resolvedTaskStatus(item)
|
||||||
return st === 'SUCCESS' || item.success === true
|
// 失败任务也可能有已跑出来的部分结果文件(后端按已收分片组装),有文件就允许下载
|
||||||
|
return st === 'SUCCESS' || item.success === true || st === 'FAILED'
|
||||||
}
|
}
|
||||||
|
|
||||||
async function downloadResult(item: ProductRiskHistoryItem) {
|
async function downloadResult(item: ProductRiskHistoryItem) {
|
||||||
@@ -1056,6 +1057,12 @@ function formatMatchRemark(row: ProductRiskShopQueueItem) {
|
|||||||
if (row.matched) {
|
if (row.matched) {
|
||||||
return '已关联店铺,请查看状态确认'
|
return '已关联店铺,请查看状态确认'
|
||||||
}
|
}
|
||||||
|
if (row.matchStatus === 'CONFLICT') {
|
||||||
|
return '存在多个同名店铺,请人工确认'
|
||||||
|
}
|
||||||
|
if (row.matchStatus === 'PENDING') {
|
||||||
|
return '店铺尚未匹配完成,请稍后查看'
|
||||||
|
}
|
||||||
return '未匹配成功,请检查店铺名'
|
return '未匹配成功,请检查店铺名'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -216,6 +216,8 @@ import {
|
|||||||
checkParseResult,
|
checkParseResult,
|
||||||
checkQueuePayload,
|
checkQueuePayload,
|
||||||
checkSelectedFiles,
|
checkSelectedFiles,
|
||||||
|
collectDistinctErrors,
|
||||||
|
guardBlocked,
|
||||||
} from '@/shared/dispatch-guard'
|
} from '@/shared/dispatch-guard'
|
||||||
import { passGuard } from '@/shared/dispatch-guard-ui'
|
import { passGuard } from '@/shared/dispatch-guard-ui'
|
||||||
import { useZiniaoVersion, type ZiniaoVersion } from '@/shared/utils/ziniao-version'
|
import { useZiniaoVersion, type ZiniaoVersion } from '@/shared/utils/ziniao-version'
|
||||||
@@ -563,10 +565,25 @@ async function submitRun() {
|
|||||||
sync_countries: syncCountries.value.filter((country) => country !== publishCountry.value),
|
sync_countries: syncCountries.value.filter((country) => country !== publishCountry.value),
|
||||||
})
|
})
|
||||||
if (!parsed.taskId) throw new Error('后端未返回有效任务标识')
|
if (!parsed.taskId) throw new Error('后端未返回有效任务标识')
|
||||||
// publish 的 Vo 用 files 承载明细,没有 acceptedRows;这里先卡住 taskId 与总行数
|
const fileErrors = collectDistinctErrors(
|
||||||
|
(parsed.files || []).map((file) => file.errorMessage || file.error),
|
||||||
|
)
|
||||||
|
// 店铺未录入后台是最高频的解析失败原因(2026-09-17 客户因此连试 7 次),
|
||||||
|
// 单独用「店铺未找到」弹窗直给后端原因,避免被通用文案淹没
|
||||||
|
const missingShopErrors = fileErrors.filter((message) => message.includes('未找到店铺'))
|
||||||
|
if (!parsed.totalRows && missingShopErrors.length) {
|
||||||
|
await passGuard(guardBlocked('店铺未找到', missingShopErrors.join('\n'), 'publish.shop-missing'))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// publish 的 Vo 用 files 承载明细,没有 acceptedRows;这里先卡住 taskId 与总行数。
|
||||||
|
// 文件级失败(店铺未匹配等)时 Java 不解析 Excel、totalRows 同样为 0,把具体
|
||||||
|
// 原因一并交给 guard 优先展示,避免只报「没有读到任何数据行」误导排查方向。
|
||||||
const guard = checkParseResult(
|
const guard = checkParseResult(
|
||||||
{ taskId: parsed.taskId, totalRows: parsed.totalRows, acceptedRows: parsed.totalRows },
|
{ taskId: parsed.taskId, totalRows: parsed.totalRows, acceptedRows: parsed.totalRows },
|
||||||
{ requiredColumnsHint: '店铺名 / 商品行' },
|
{
|
||||||
|
requiredColumnsHint: '店铺名 / 商品行',
|
||||||
|
fileErrors,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
if (!(await passGuard(guard))) return
|
if (!(await passGuard(guard))) return
|
||||||
|
|
||||||
@@ -604,7 +621,9 @@ async function submitRun() {
|
|||||||
|
|
||||||
await Promise.all([loadDashboard(), loadHistory()])
|
await Promise.all([loadDashboard(), loadHistory()])
|
||||||
if (!batch.pendingFileIds.length) {
|
if (!batch.pendingFileIds.length) {
|
||||||
queueMessage.value = '解析完成,当前没有匹配成功且可上架的文件。'
|
queueMessage.value = fileErrors.length
|
||||||
|
? `解析完成,当前没有匹配成功且可上架的文件:${fileErrors[0]}`
|
||||||
|
: '解析完成,当前没有匹配成功且可上架的文件。'
|
||||||
ElMessage.warning(queueMessage.value)
|
ElMessage.warning(queueMessage.value)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -917,7 +936,10 @@ async function processQueue() {
|
|||||||
saveQueueState()
|
saveQueueState()
|
||||||
throw new Error(queueMessage.value)
|
throw new Error(queueMessage.value)
|
||||||
}
|
}
|
||||||
queueMessage.value = `文件 ${file.sourceFilename || nextFileId} 启动失败,已记录并继续下一个文件。`
|
// 原因必须回显给用户:后端会因「该店铺已有上架任务在执行」直接拒绝激活,
|
||||||
|
// 只写日志的话用户只看到"启动失败",会以为是文件问题而反复重传
|
||||||
|
queueMessage.value = `文件 ${file.sourceFilename || nextFileId} 启动失败:${reason}`
|
||||||
|
ElMessage.warning(queueMessage.value)
|
||||||
activeFileId.value = null
|
activeFileId.value = null
|
||||||
saveQueueState()
|
saveQueueState()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -439,6 +439,12 @@ function formatMatchRemark(row: QueryAsinShopQueueItem) {
|
|||||||
if (row.matched) {
|
if (row.matched) {
|
||||||
return "已匹配成功,请查看状态确认";
|
return "已匹配成功,请查看状态确认";
|
||||||
}
|
}
|
||||||
|
if (row.matchStatus === "CONFLICT") {
|
||||||
|
return "存在多个同名店铺,请人工确认";
|
||||||
|
}
|
||||||
|
if (row.matchStatus === "PENDING") {
|
||||||
|
return "店铺尚未匹配完成,请稍后查看";
|
||||||
|
}
|
||||||
return "未匹配成功,请检查店铺名";
|
return "未匹配成功,请检查店铺名";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -531,7 +531,8 @@ function currentTaskStageText(item: ShopMatchHistoryItem) { const snapshot = tas
|
|||||||
function nextScheduledDisplay(item: ShopMatchHistoryItem) { const snapshot = taskSnapshotOf(item); const task = snapshot?.task; const stages = task?.scheduleStages || []; if (typeof task?.currentStageIndex === 'number') { const stage = stages.find((entry) => entry.stageIndex === task.currentStageIndex); if (stage?.scheduledAt) return formatMonthDayTime(stage.scheduledAt) } if (item.scheduledAt) return formatMonthDayTime(item.scheduledAt); return '' }
|
function nextScheduledDisplay(item: ShopMatchHistoryItem) { const snapshot = taskSnapshotOf(item); const task = snapshot?.task; const stages = task?.scheduleStages || []; if (typeof task?.currentStageIndex === 'number') { const stage = stages.find((entry) => entry.stageIndex === task.currentStageIndex); if (stage?.scheduledAt) return formatMonthDayTime(stage.scheduledAt) } if (item.scheduledAt) return formatMonthDayTime(item.scheduledAt); return '' }
|
||||||
function statusText(item: ShopMatchHistoryItem) { const status = resolvedTaskStatus(item); if (status === 'SCHEDULED') return '待执行'; if (status === 'RUNNING') return '执行中'; if (status === 'SUCCESS' || status === 'COMPLETED') return '已完成'; if (status === 'FAILED') return '失败'; return item.success ? '已完成' : '未知' }
|
function statusText(item: ShopMatchHistoryItem) { const status = resolvedTaskStatus(item); if (status === 'SCHEDULED') return '待执行'; if (status === 'RUNNING') return '执行中'; if (status === 'SUCCESS' || status === 'COMPLETED') return '已完成'; if (status === 'FAILED') return '失败'; return item.success ? '已完成' : '未知' }
|
||||||
function statusClass(item: ShopMatchHistoryItem) { const status = resolvedTaskStatus(item); return status === 'SUCCESS' || status === 'COMPLETED' ? 'success' : status === 'FAILED' ? 'failed' : 'running' }
|
function statusClass(item: ShopMatchHistoryItem) { const status = resolvedTaskStatus(item); return status === 'SUCCESS' || status === 'COMPLETED' ? 'success' : status === 'FAILED' ? 'failed' : 'running' }
|
||||||
function canDownload(item: ShopMatchHistoryItem) { const status = resolvedTaskStatus(item); return !!item.resultId && (!!item.fileReady || !!item.downloadUrl) && (status === 'SUCCESS' || status === 'COMPLETED') }
|
// 失败任务也可能有已跑出来的部分结果文件(后端按已收分片组装),有文件就允许下载
|
||||||
|
function canDownload(item: ShopMatchHistoryItem) { const status = resolvedTaskStatus(item); return !!item.resultId && (!!item.fileReady || !!item.downloadUrl) && (status === 'SUCCESS' || status === 'COMPLETED' || status === 'FAILED') }
|
||||||
async function downloadResult(item: ShopMatchHistoryItem) { if (!item.resultId) return; const url = getShopMatchResultDownloadUrl(item.resultId); const filename = item.outputFilename || `${item.shopName || 'result'}.xlsx`; const result = await saveUrlWithProgress(url, filename, `shop-match:${item.resultId}`); if (result.success) ElMessage.success(`已保存: ${result.path || filename}`); else if (result.error && result.error !== '用户取消') ElMessage.error(result.error) }
|
async function downloadResult(item: ShopMatchHistoryItem) { if (!item.resultId) return; const url = getShopMatchResultDownloadUrl(item.resultId); const filename = item.outputFilename || `${item.shopName || 'result'}.xlsx`; const result = await saveUrlWithProgress(url, filename, `shop-match:${item.resultId}`); if (result.success) ElMessage.success(`已保存: ${result.path || filename}`); else if (result.error && result.error !== '用户取消') ElMessage.error(result.error) }
|
||||||
async function deleteTaskRecord(item: ShopMatchHistoryItem) {
|
async function deleteTaskRecord(item: ShopMatchHistoryItem) {
|
||||||
const taskId = normalizeTaskId(item.taskId)
|
const taskId = normalizeTaskId(item.taskId)
|
||||||
@@ -565,7 +566,7 @@ async function deleteTaskRecord(item: ShopMatchHistoryItem) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
function formatMatchStatus(status?: string) { const value = (status || '').trim(); return { MATCHED: '已匹配', PENDING: '待匹配', CONFLICT: '需人工确认', INDEX_STALE: '匹配已过期' }[value] || value || '—' }
|
function formatMatchStatus(status?: string) { const value = (status || '').trim(); return { MATCHED: '已匹配', PENDING: '待匹配', CONFLICT: '需人工确认', INDEX_STALE: '匹配已过期' }[value] || value || '—' }
|
||||||
function formatMatchRemark(row: ShopMatchShopQueueItem) { const message = (row.matchMessage || '').trim(); if (message) return message; if (row.matched && row.matchStatus === 'MATCHED') return '已匹配成功,可启动任务'; if (row.matched) return '已关联店铺,请查看状态确认'; return '未匹配成功,请检查店铺名' }
|
function formatMatchRemark(row: ShopMatchShopQueueItem) { const message = (row.matchMessage || '').trim(); if (message) return message; if (row.matched && row.matchStatus === 'MATCHED') return '已匹配成功,可启动任务'; if (row.matched) return '已关联店铺,请查看状态确认'; if (row.matchStatus === 'CONFLICT') return '存在多个同名店铺,请人工确认'; if (row.matchStatus === 'PENDING') return '店铺尚未匹配完成,请稍后查看'; return '未匹配成功,请检查店铺名' }
|
||||||
async function pushToPythonQueue() { autoQueueEnabled.value = true; await processMatchedQueue() }
|
async function pushToPythonQueue() { autoQueueEnabled.value = true; await processMatchedQueue() }
|
||||||
function nextMatchedQueueItem() { return matchedItems.value.find((item) => item.matched) }
|
function nextMatchedQueueItem() { return matchedItems.value.find((item) => item.matched) }
|
||||||
async function processMatchedQueue() { if (disposed || queueWorkerRunning.value) return; const api = getPywebviewApi(); if (!api?.enqueue_json) { ElMessage.error('当前客户端未提供 enqueue_json'); return } const matched = matchedItems.value.filter((item) => item.matched); if (!matched.length) { ElMessage.warning('请先匹配可用店铺'); return } if (!orderedCountryCodes.value.length) { ElMessage.warning('请至少勾选一个国家'); return } let scheduleValues: string[] | undefined; try { scheduleValues = parseScheduleValues() } catch (error) { ElMessage.error(error instanceof Error ? error.message : '定时配置无效'); return } queueWorkerRunning.value = true; pushing.value = true; queuePayloadText.value = ''; let successCount = 0; let failedCount = 0; let index = 0; try { while (!disposed && autoQueueEnabled.value) { const item = nextMatchedQueueItem(); if (!item) break; index += 1; try { const created = await withTransientRetry(() => createShopMatchTask([buildTaskCreateItem(item)], orderedCountryCodes.value, scheduleValues), (attempt, maxAttempts) => { queuePushResult.value = `服务暂时不可用,正在重试创建任务(${attempt}/${maxAttempts})...` }); const taskId = created.taskId; const initialStatus = scheduleValues?.length ? 'SCHEDULED' : 'RUNNING'; const snapshot: ShopMatchTaskDetailVo = { task: { id: taskId, status: initialStatus, scheduledAt: scheduleValues?.[0], countryCodes: [...orderedCountryCodes.value], currentStageIndex: scheduleValues?.length ? 0 : undefined, activeStageIndex: undefined, scheduleStages: (scheduleValues || []).map((value, stageIndex) => ({ stageIndex, scheduledAt: value, status: stageIndex === 0 ? 'SCHEDULED' : 'PENDING' })) }, items: created.items }; taskSnapshots.value = { ...taskSnapshots.value, [taskId]: snapshot }; taskDetails.value = { ...taskDetails.value, [taskId]: initialStatus }; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); const createdItem = created.items?.[0]; if (!createdItem) { removeMatchedRowsLocally([item]); continue } if (scheduleValues?.length) { successCount += 1; restoreScheduledDispatches(); queuePushResult.value = `任务 ${taskId} 已创建,共 ${scheduleValues.length} 个执行时间点`; removeMatchedRowsLocally([item]); continue } const payload = buildQueuePayload(taskId, item, orderedCountryCodes.value); queuePayloadText.value = JSON.stringify(payload, null, 2); const result = await api.enqueue_json(payload); if (!result?.success) { failedCount += 1; queuePushResult.value = `任务 ${taskId} 启动失败,已自动继续下一条:${result?.error || '未知错误'}`; ElMessage.error(queuePushResult.value); removeMatchedRowsLocally([item]); continue } addPollingTask(taskId); ensurePolling(true); removeMatchedRowsLocally([item]); queuePushResult.value = `任务 ${taskId} 已提交,等待执行完成`; const finalStatus = await waitForTaskTerminal(taskId); if (finalStatus !== 'SUCCESS') { failedCount += 1; queuePushResult.value = `任务 ${taskId} 执行失败,已自动继续下一条`; ElMessage.error(queuePushResult.value); continue } successCount += 1; queuePushResult.value = `任务 ${taskId} 已完成` } catch (error) { if (disposed) break; failedCount += 1; queuePushResult.value = `第 ${index} 条任务处理失败,已自动继续下一条:${error instanceof Error ? error.message : '未知错误'}`; ElMessage.error(queuePushResult.value); removeMatchedRowsLocally([item]); continue } } restoreScheduledDispatches(); if (!scheduleValues?.length) ensurePolling(true); if (!disposed && (successCount > 0 || failedCount > 0)) ElMessage.success(scheduleValues?.length ? `定时任务已创建:成功 ${successCount} 条,失败 ${failedCount} 条` : `店铺启动已完成:成功 ${successCount} 条,失败 ${failedCount} 条`) } catch (error) { if (disposed) return; const message = error instanceof Error ? error.message : '启动失败'; queuePushResult.value = message; ElMessage.error(message) } finally { queueWorkerRunning.value = false; pushing.value = false } }
|
async function processMatchedQueue() { if (disposed || queueWorkerRunning.value) return; const api = getPywebviewApi(); if (!api?.enqueue_json) { ElMessage.error('当前客户端未提供 enqueue_json'); return } const matched = matchedItems.value.filter((item) => item.matched); if (!matched.length) { ElMessage.warning('请先匹配可用店铺'); return } if (!orderedCountryCodes.value.length) { ElMessage.warning('请至少勾选一个国家'); return } let scheduleValues: string[] | undefined; try { scheduleValues = parseScheduleValues() } catch (error) { ElMessage.error(error instanceof Error ? error.message : '定时配置无效'); return } queueWorkerRunning.value = true; pushing.value = true; queuePayloadText.value = ''; let successCount = 0; let failedCount = 0; let index = 0; try { while (!disposed && autoQueueEnabled.value) { const item = nextMatchedQueueItem(); if (!item) break; index += 1; try { const created = await withTransientRetry(() => createShopMatchTask([buildTaskCreateItem(item)], orderedCountryCodes.value, scheduleValues), (attempt, maxAttempts) => { queuePushResult.value = `服务暂时不可用,正在重试创建任务(${attempt}/${maxAttempts})...` }); const taskId = created.taskId; const initialStatus = scheduleValues?.length ? 'SCHEDULED' : 'RUNNING'; const snapshot: ShopMatchTaskDetailVo = { task: { id: taskId, status: initialStatus, scheduledAt: scheduleValues?.[0], countryCodes: [...orderedCountryCodes.value], currentStageIndex: scheduleValues?.length ? 0 : undefined, activeStageIndex: undefined, scheduleStages: (scheduleValues || []).map((value, stageIndex) => ({ stageIndex, scheduledAt: value, status: stageIndex === 0 ? 'SCHEDULED' : 'PENDING' })) }, items: created.items }; taskSnapshots.value = { ...taskSnapshots.value, [taskId]: snapshot }; taskDetails.value = { ...taskDetails.value, [taskId]: initialStatus }; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); const createdItem = created.items?.[0]; if (!createdItem) { removeMatchedRowsLocally([item]); continue } if (scheduleValues?.length) { successCount += 1; restoreScheduledDispatches(); queuePushResult.value = `任务 ${taskId} 已创建,共 ${scheduleValues.length} 个执行时间点`; removeMatchedRowsLocally([item]); continue } const payload = buildQueuePayload(taskId, item, orderedCountryCodes.value); queuePayloadText.value = JSON.stringify(payload, null, 2); const result = await api.enqueue_json(payload); if (!result?.success) { failedCount += 1; queuePushResult.value = `任务 ${taskId} 启动失败,已自动继续下一条:${result?.error || '未知错误'}`; ElMessage.error(queuePushResult.value); removeMatchedRowsLocally([item]); continue } addPollingTask(taskId); ensurePolling(true); removeMatchedRowsLocally([item]); queuePushResult.value = `任务 ${taskId} 已提交,等待执行完成`; const finalStatus = await waitForTaskTerminal(taskId); if (finalStatus !== 'SUCCESS') { failedCount += 1; queuePushResult.value = `任务 ${taskId} 执行失败,已自动继续下一条`; ElMessage.error(queuePushResult.value); continue } successCount += 1; queuePushResult.value = `任务 ${taskId} 已完成` } catch (error) { if (disposed) break; failedCount += 1; queuePushResult.value = `第 ${index} 条任务处理失败,已自动继续下一条:${error instanceof Error ? error.message : '未知错误'}`; ElMessage.error(queuePushResult.value); removeMatchedRowsLocally([item]); continue } } restoreScheduledDispatches(); if (!scheduleValues?.length) ensurePolling(true); if (!disposed && (successCount > 0 || failedCount > 0)) ElMessage.success(scheduleValues?.length ? `定时任务已创建:成功 ${successCount} 条,失败 ${failedCount} 条` : `店铺启动已完成:成功 ${successCount} 条,失败 ${failedCount} 条`) } catch (error) { if (disposed) return; const message = error instanceof Error ? error.message : '启动失败'; queuePushResult.value = message; ElMessage.error(message) } finally { queueWorkerRunning.value = false; pushing.value = false } }
|
||||||
|
|||||||
@@ -145,7 +145,7 @@ import { expandBrandFolderRecursive } from '@/shared/api/brand'
|
|||||||
import { deleteSplitHistory, getExcelInfo, getSplitHistory, getSplitResultDownloadUrl, runSplit, type SplitResultItem } from '@/shared/api/java-modules'
|
import { deleteSplitHistory, getExcelInfo, getSplitHistory, getSplitResultDownloadUrl, runSplit, type SplitResultItem } from '@/shared/api/java-modules'
|
||||||
import { getPywebviewApi, type UploadedJavaFile } from '@/shared/bridges/pywebview'
|
import { getPywebviewApi, type UploadedJavaFile } from '@/shared/bridges/pywebview'
|
||||||
import { saveUrlWithProgress } from '@/shared/utils/download-progress'
|
import { saveUrlWithProgress } from '@/shared/utils/download-progress'
|
||||||
import { EXCEL_EXTENSIONS, checkSelectedFiles, guardBlocked } from '@/shared/dispatch-guard'
|
import { EXCEL_EXTENSIONS, checkSelectedFiles, collectDistinctErrors, guardBlocked } from '@/shared/dispatch-guard'
|
||||||
import { passGuard } from '@/shared/dispatch-guard-ui'
|
import { passGuard } from '@/shared/dispatch-guard-ui'
|
||||||
import { formatDateTime } from '@/shared/utils/datetime'
|
import { formatDateTime } from '@/shared/utils/datetime'
|
||||||
import { uploadPathsToJava } from '@/shared/utils/upload-to-java'
|
import { uploadPathsToJava } from '@/shared/utils/upload-to-java'
|
||||||
@@ -327,11 +327,15 @@ async function submitSplitRun() {
|
|||||||
splitResultItems.value = result.items || []
|
splitResultItems.value = result.items || []
|
||||||
await loadSplitHistory()
|
await loadSplitHistory()
|
||||||
if (result.total > 0 && result.successCount === 0) {
|
if (result.total > 0 && result.successCount === 0) {
|
||||||
|
// 逐文件原因就在 items[].error 里,优先直接列出,不再让用户对着「常见原因」猜
|
||||||
|
const details = collectDistinctErrors((result.items || []).map((item) => item.error)).slice(0, 5)
|
||||||
await passGuard(
|
await passGuard(
|
||||||
guardBlocked(
|
guardBlocked(
|
||||||
'拆分未成功',
|
'拆分未成功',
|
||||||
`本次提交的 ${result.total} 个文件全部处理失败。\n` +
|
`本次提交的 ${result.total} 个文件全部处理失败。\n` +
|
||||||
'常见原因是表头列名与所选保留列对不上,或文件内没有数据行。\n' +
|
(details.length
|
||||||
|
? `失败原因:\n${details.map((message) => `· ${message}`).join('\n')}\n`
|
||||||
|
: '常见原因是表头列名与所选保留列对不上,或文件内没有数据行。\n') +
|
||||||
'请在右侧结果列表查看每个文件的失败原因后重试。',
|
'请在右侧结果列表查看每个文件的失败原因后重试。',
|
||||||
'split.all-failed',
|
'split.all-failed',
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -570,6 +570,12 @@ function formatMatchRemark(row: WithdrawShopQueueItem) {
|
|||||||
if (row.matched) {
|
if (row.matched) {
|
||||||
return "已匹配成功,请查看状态确认";
|
return "已匹配成功,请查看状态确认";
|
||||||
}
|
}
|
||||||
|
if (row.matchStatus === "CONFLICT") {
|
||||||
|
return "存在多个同名店铺,请人工确认";
|
||||||
|
}
|
||||||
|
if (row.matchStatus === "PENDING") {
|
||||||
|
return "店铺尚未匹配完成,请稍后查看";
|
||||||
|
}
|
||||||
return "未匹配成功,请检查店铺名";
|
return "未匹配成功,请检查店铺名";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1015,12 +1015,12 @@ async function waitForImageVideoTask(ticket: ImageVideoAsyncTaskVo): Promise<unk
|
|||||||
}
|
}
|
||||||
if (task.status === 'SUCCESS') return task.result
|
if (task.status === 'SUCCESS') return task.result
|
||||||
if (isTerminalImageVideoTask(task)) {
|
if (isTerminalImageVideoTask(task)) {
|
||||||
throw new Error(task.errorMessage || 'Coze task failed')
|
throw new Error(task.errorMessage || 'Coze 任务执行失败')
|
||||||
}
|
}
|
||||||
await sleep(IMAGE_VIDEO_TASK_POLL_DELAY_MS)
|
await sleep(IMAGE_VIDEO_TASK_POLL_DELAY_MS)
|
||||||
task = await getImageVideoAsyncTask(task.taskId)
|
task = await getImageVideoAsyncTask(task.taskId)
|
||||||
}
|
}
|
||||||
throw new Error('Coze task polling timed out')
|
throw new Error('Coze 任务查询超时,请稍后重试')
|
||||||
}
|
}
|
||||||
|
|
||||||
async function rewriteScriptFromSource() {
|
async function rewriteScriptFromSource() {
|
||||||
@@ -1296,7 +1296,8 @@ async function pollAssemblyResult(tab: WorkspaceTab, taskId: number) {
|
|||||||
if (isTerminalImageVideoTask(task)) {
|
if (isTerminalImageVideoTask(task)) {
|
||||||
assembly.polling = false
|
assembly.polling = false
|
||||||
if (task.status === 'FAILED') {
|
if (task.status === 'FAILED') {
|
||||||
ElMessage.error('Coze 工作流执行失败')
|
// 后端 errorMessage 带具体原因(内容违规/超时/额度等),只报「执行失败」用户无从下手
|
||||||
|
ElMessage.error(task.errorMessage ? `Coze 工作流执行失败:${task.errorMessage}` : 'Coze 工作流执行失败')
|
||||||
} else {
|
} else {
|
||||||
ElMessage.success(videoUrl || assembly.videoUrl ? '视频生成完成' : 'Coze 工作流执行完成,未解析到视频地址')
|
ElMessage.success(videoUrl || assembly.videoUrl ? '视频生成完成' : 'Coze 工作流执行完成,未解析到视频地址')
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,6 +24,15 @@ export interface ClientChangelogEntry {
|
|||||||
|
|
||||||
/** 更新日志数据(新版本在前;发版时在数组最前追加一条) */
|
/** 更新日志数据(新版本在前;发版时在数组最前追加一条) */
|
||||||
export const CLIENT_CHANGELOG: ClientChangelogEntry[] = [
|
export const CLIENT_CHANGELOG: ClientChangelogEntry[] = [
|
||||||
|
{
|
||||||
|
version: '4.0.28',
|
||||||
|
date: '2026-09-17',
|
||||||
|
items: [
|
||||||
|
'修复多个任务同时操作同一店铺导致「打开店铺失败」的问题',
|
||||||
|
'同一店铺的任务改为排队执行,不会再互相打断',
|
||||||
|
'紫鸟更新内核期间不再直接报「打开店铺失败」,而是等待完成',
|
||||||
|
],
|
||||||
|
},
|
||||||
{
|
{
|
||||||
version: '4.0.27',
|
version: '4.0.27',
|
||||||
date: '2026-09-17',
|
date: '2026-09-17',
|
||||||
|
|||||||
@@ -267,6 +267,13 @@ export interface ParseResultOptions {
|
|||||||
requireRows?: boolean
|
requireRows?: boolean
|
||||||
/** 必要字段名,用于拼「缺什么」的提示,如 'ASIN / 国家' */
|
/** 必要字段名,用于拼「缺什么」的提示,如 'ASIN / 国家' */
|
||||||
requiredColumnsHint?: string
|
requiredColumnsHint?: string
|
||||||
|
/**
|
||||||
|
* 文件级失败原因(后端 files[].errorMessage)。0 有效行时优先展示这些具体
|
||||||
|
* 原因:店铺未录入、表头不匹配等都会让文件级解析提前失败,totalRows 同样是
|
||||||
|
* 0,只报通用「空文件」文案会把用户引向错误的排查方向(2026-09-17 用户因
|
||||||
|
* 「店铺未录入」被误导反复重传同一个 Excel,连试 7 次)。
|
||||||
|
*/
|
||||||
|
fileErrors?: readonly string[]
|
||||||
title?: string
|
title?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -333,7 +340,21 @@ export function checkParseResult(
|
|||||||
: ''
|
: ''
|
||||||
|
|
||||||
if (requireRows && acceptedRows === 0) {
|
if (requireRows && acceptedRows === 0) {
|
||||||
if (!totalRows) {
|
const fileErrors = collectDistinctErrors(options.fileErrors)
|
||||||
|
if (fileErrors.length) {
|
||||||
|
const shown = fileErrors.slice(0, 8)
|
||||||
|
issues.push({
|
||||||
|
code: 'parse.file-failed',
|
||||||
|
severity: 'block',
|
||||||
|
message:
|
||||||
|
'以下文件解析未通过,没有可执行的数据行:\n' +
|
||||||
|
shown.map((item) => `· ${item}`).join('\n') +
|
||||||
|
(fileErrors.length > shown.length
|
||||||
|
? `\n· 另有 ${fileErrors.length - shown.length} 条不同原因`
|
||||||
|
: '') +
|
||||||
|
'\n请按上述原因处理对应文件后重新上传解析。',
|
||||||
|
})
|
||||||
|
} else if (!totalRows) {
|
||||||
issues.push({
|
issues.push({
|
||||||
code: 'parse.empty-file',
|
code: 'parse.empty-file',
|
||||||
severity: 'block',
|
severity: 'block',
|
||||||
@@ -387,6 +408,20 @@ function normalizeCount(value: unknown): number | null {
|
|||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 汇总一组错误原因:去空、去重(同一原因多条只留一条),保持出现顺序。 */
|
||||||
|
export function collectDistinctErrors(values: readonly unknown[] | undefined): string[] {
|
||||||
|
if (!values || !values.length) return []
|
||||||
|
const seen = new Set<string>()
|
||||||
|
const result: string[] = []
|
||||||
|
for (const value of values) {
|
||||||
|
const trimmed = typeof value === 'string' ? value.trim() : ''
|
||||||
|
if (!trimmed || seen.has(trimmed)) continue
|
||||||
|
seen.add(trimmed)
|
||||||
|
result.push(trimmed)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
export interface QueuePayloadOptions {
|
export interface QueuePayloadOptions {
|
||||||
/** data 下必须存在且非空的字段名 */
|
/** data 下必须存在且非空的字段名 */
|
||||||
requiredDataKeys?: readonly string[]
|
requiredDataKeys?: readonly string[]
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
/**
|
||||||
|
* 提交按钮防连点(纯逻辑 + 全局安装器,供 main.ts 与单测复用)。
|
||||||
|
*
|
||||||
|
* 为什么需要:各工具页的「开始上架 / 启动任务 / 匹配店铺」按钮在提交逻辑跑完后
|
||||||
|
* 立刻恢复可点,用户手快连点就会重复发起。2026-09-17 上架事故里同一店铺被接连
|
||||||
|
* 提交三次、服务端并存多个同店铺任务,客户端并发打开同一店铺时全部失败。
|
||||||
|
*
|
||||||
|
* 语义:
|
||||||
|
* - 冷却按**按钮元素**各自计算(WeakMap),点 A 按钮不会影响 B 按钮;
|
||||||
|
* - 从"首次有效点击"起算,被拦的点击不会把冷却越拖越长(否则用户越急越点不开);
|
||||||
|
* - 在**捕获阶段**拦截,抢在 Vue 的 @click 之前,被拦的点击不触发任何提交逻辑。
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** 提交按钮的统一类名(17 个工具页的主按钮都用它) */
|
||||||
|
export const SUBMIT_BUTTON_SELECTOR = '.btn-run'
|
||||||
|
|
||||||
|
/** 默认冷却时长:够挡住手快连点,又不至于让正常操作感到卡顿 */
|
||||||
|
export const DEFAULT_SUBMIT_COOLDOWN_MS = 1500
|
||||||
|
|
||||||
|
export interface ClickGate {
|
||||||
|
/** 本次点击是否应被拦下(被拦时不会刷新冷却) */
|
||||||
|
shouldBlock(target: object): boolean
|
||||||
|
/** 解除某个目标的冷却(例如提交失败要允许用户立刻重试) */
|
||||||
|
reset(target: object): void
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createClickGate(
|
||||||
|
cooldownMs: number = DEFAULT_SUBMIT_COOLDOWN_MS,
|
||||||
|
now: () => number = () => Date.now(),
|
||||||
|
): ClickGate {
|
||||||
|
// 非法时长(NaN/0/负数)回落默认值:宁可多挡一下,也不能等同"不设防"
|
||||||
|
const cooldown = Number.isFinite(cooldownMs) && cooldownMs > 0
|
||||||
|
? cooldownMs
|
||||||
|
: DEFAULT_SUBMIT_COOLDOWN_MS
|
||||||
|
|
||||||
|
const lastClickAt = new WeakMap<object, number>()
|
||||||
|
|
||||||
|
return {
|
||||||
|
shouldBlock(target: object): boolean {
|
||||||
|
const current = now()
|
||||||
|
const last = lastClickAt.get(target)
|
||||||
|
// 用 undefined(而非 0)表示"从未点击过":哨兵值参与减法会跟时钟起点耦合,
|
||||||
|
// 在 now() 起点较小(测试假时钟/单调时钟)时会误判首次点击为连点
|
||||||
|
if (last !== undefined && current - last < cooldown) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
lastClickAt.set(target, current)
|
||||||
|
return false
|
||||||
|
},
|
||||||
|
reset(target: object): void {
|
||||||
|
lastClickAt.delete(target)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SubmitGuardOptions {
|
||||||
|
/** 生效的按钮选择器,默认 SUBMIT_BUTTON_SELECTOR */
|
||||||
|
selector?: string
|
||||||
|
cooldownMs?: number
|
||||||
|
now?: () => number
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 全局安装防连点(在 main.ts 调用一次即可,覆盖所有工具页的提交按钮)。
|
||||||
|
*
|
||||||
|
* @returns 卸载函数(测试与热更新用)
|
||||||
|
*/
|
||||||
|
export function installSubmitGuard(options: SubmitGuardOptions = {}): () => void {
|
||||||
|
const selector = options.selector ?? SUBMIT_BUTTON_SELECTOR
|
||||||
|
const gate = createClickGate(options.cooldownMs, options.now)
|
||||||
|
|
||||||
|
const handler = (event: Event): void => {
|
||||||
|
const target = event.target as Element | null
|
||||||
|
const button = target && typeof target.closest === 'function'
|
||||||
|
? target.closest(selector)
|
||||||
|
: null
|
||||||
|
if (!button) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (gate.shouldBlock(button)) {
|
||||||
|
event.stopImmediatePropagation()
|
||||||
|
event.preventDefault()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('click', handler, true)
|
||||||
|
return () => document.removeEventListener('click', handler, true)
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import {
|
|||||||
checkParseResult,
|
checkParseResult,
|
||||||
checkQueuePayload,
|
checkQueuePayload,
|
||||||
checkSelectedFiles,
|
checkSelectedFiles,
|
||||||
|
collectDistinctErrors,
|
||||||
extensionOf,
|
extensionOf,
|
||||||
findUnsafeJsonPaths,
|
findUnsafeJsonPaths,
|
||||||
guardPassed,
|
guardPassed,
|
||||||
@@ -165,6 +166,55 @@ test('test_task_101_dispatch_guard_boundary_parse_result_zero_rows_blocked', ()
|
|||||||
assert.match(emptyFile.message, /没有读到任何数据行/)
|
assert.match(emptyFile.message, /没有读到任何数据行/)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test('test_task_101_dispatch_guard_parse_result_file_errors_shown_instead_of_empty_file', () => {
|
||||||
|
// 2026-09-17:店铺未录入后台导致文件级失败,用户被「空文件」文案误导反复重传
|
||||||
|
const result = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0 }), {
|
||||||
|
fileErrors: ['后台店铺管理中未找到店铺:林清斌,请先添加店铺信息'],
|
||||||
|
})
|
||||||
|
assert.equal(result.ok, false)
|
||||||
|
assert.deepEqual(codes(result), ['parse.file-failed'])
|
||||||
|
assert.match(result.message, /林清斌/)
|
||||||
|
assert.ok(!result.message.includes('没有读到任何数据行'), '有具体原因时不展示通用空文件文案')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('test_task_101_dispatch_guard_parse_result_file_errors_dedupe_blank_and_limit', () => {
|
||||||
|
const deduped = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0 }), {
|
||||||
|
fileErrors: ['店铺未匹配', ' ', '店铺未匹配', '', '第二条原因'],
|
||||||
|
})
|
||||||
|
assert.deepEqual(codes(deduped), ['parse.file-failed'])
|
||||||
|
assert.equal(
|
||||||
|
deduped.message.split('\n').filter((line) => line.startsWith('· ')).length,
|
||||||
|
2,
|
||||||
|
'空串与重复原因不应重复展示',
|
||||||
|
)
|
||||||
|
|
||||||
|
const many = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0 }), {
|
||||||
|
fileErrors: Array.from({ length: 10 }, (_, index) => `原因 ${index}`),
|
||||||
|
})
|
||||||
|
assert.match(many.message, /另有 2 条不同原因/)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('test_task_101_dispatch_guard_parse_result_blank_file_errors_fallback_to_empty_file', () => {
|
||||||
|
const result = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0 }), {
|
||||||
|
fileErrors: [' ', ''],
|
||||||
|
})
|
||||||
|
assert.deepEqual(codes(result), ['parse.empty-file'])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('test_task_101_dispatch_guard_collect_distinct_errors', () => {
|
||||||
|
assert.deepEqual(collectDistinctErrors(['a', ' ', 'a', '', 'b']), ['a', 'b'], '去空去重且保持顺序')
|
||||||
|
assert.deepEqual(collectDistinctErrors([' 店铺未找到 ']), ['店铺未找到'], '首尾空白裁剪')
|
||||||
|
assert.deepEqual(collectDistinctErrors([]), [])
|
||||||
|
assert.deepEqual(collectDistinctErrors(undefined), [])
|
||||||
|
assert.deepEqual(collectDistinctErrors([null, 7, undefined]), [], '非字符串项忽略')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('test_task_101_dispatch_guard_parse_result_file_errors_ignored_when_rows_present', () => {
|
||||||
|
const result = checkParseResult(parseVo(), { fileErrors: ['某文件失败'] })
|
||||||
|
assert.equal(result.ok, true)
|
||||||
|
assert.equal(result.needsConfirm, false)
|
||||||
|
})
|
||||||
|
|
||||||
test('test_task_101_dispatch_guard_parse_result_zero_rows_allowed_when_not_required', () => {
|
test('test_task_101_dispatch_guard_parse_result_zero_rows_allowed_when_not_required', () => {
|
||||||
const result = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0 }), { requireRows: false })
|
const result = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0 }), { requireRows: false })
|
||||||
assert.equal(result.ok, true)
|
assert.equal(result.ok, true)
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
/**
|
||||||
|
* 提交按钮防连点(shared/utils/submit-guard)行为测试。
|
||||||
|
*
|
||||||
|
* 背景(2026-09-17 上架事故):前端「开始上架 / 启动任务」按钮在提交完成后立刻
|
||||||
|
* 恢复可点,手快连点会重复创建任务;同一店铺并存多个任务后,客户端并发打开
|
||||||
|
* 同一店铺时全部失败。这里把冷却逻辑抽成纯函数以便单测。
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { test } from 'node:test'
|
||||||
|
import assert from 'node:assert/strict'
|
||||||
|
import {
|
||||||
|
createClickGate,
|
||||||
|
installSubmitGuard,
|
||||||
|
DEFAULT_SUBMIT_COOLDOWN_MS,
|
||||||
|
SUBMIT_BUTTON_SELECTOR,
|
||||||
|
} from '../src/shared/utils/submit-guard.ts'
|
||||||
|
|
||||||
|
function fakeClock(start = 1_000) {
|
||||||
|
let current = start
|
||||||
|
return {
|
||||||
|
now: () => current,
|
||||||
|
advance: (ms: number) => {
|
||||||
|
current += ms
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
test('默认冷却时长是正数秒级', () => {
|
||||||
|
assert.ok(DEFAULT_SUBMIT_COOLDOWN_MS >= 1000, '冷却至少 1 秒,否则挡不住连点')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('默认选择器覆盖各工具页的主按钮', () => {
|
||||||
|
assert.equal(SUBMIT_BUTTON_SELECTOR, '.btn-run')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('首次点击放行', () => {
|
||||||
|
const clock = fakeClock()
|
||||||
|
const gate = createClickGate(1500, clock.now)
|
||||||
|
assert.equal(gate.shouldBlock({}), false)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('冷却期内重复点击被拦截', () => {
|
||||||
|
const clock = fakeClock()
|
||||||
|
const gate = createClickGate(1500, clock.now)
|
||||||
|
const button = {}
|
||||||
|
|
||||||
|
assert.equal(gate.shouldBlock(button), false)
|
||||||
|
clock.advance(100)
|
||||||
|
assert.equal(gate.shouldBlock(button), true)
|
||||||
|
clock.advance(1399)
|
||||||
|
assert.equal(gate.shouldBlock(button), true)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('冷却结束时(边界)放行', () => {
|
||||||
|
const clock = fakeClock()
|
||||||
|
const gate = createClickGate(1500, clock.now)
|
||||||
|
const button = {}
|
||||||
|
|
||||||
|
assert.equal(gate.shouldBlock(button), false)
|
||||||
|
clock.advance(1500)
|
||||||
|
assert.equal(gate.shouldBlock(button), false, '恰好到达冷却终点应放行')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('不同按钮互不影响', () => {
|
||||||
|
const clock = fakeClock()
|
||||||
|
const gate = createClickGate(1500, clock.now)
|
||||||
|
const startButton = {}
|
||||||
|
const matchButton = {}
|
||||||
|
|
||||||
|
assert.equal(gate.shouldBlock(startButton), false)
|
||||||
|
assert.equal(gate.shouldBlock(matchButton), false, '另一个按钮不该被前一个的冷却波及')
|
||||||
|
clock.advance(100)
|
||||||
|
assert.equal(gate.shouldBlock(startButton), true)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('自定义冷却时长生效', () => {
|
||||||
|
const clock = fakeClock()
|
||||||
|
const gate = createClickGate(300, clock.now)
|
||||||
|
const button = {}
|
||||||
|
|
||||||
|
assert.equal(gate.shouldBlock(button), false)
|
||||||
|
clock.advance(299)
|
||||||
|
assert.equal(gate.shouldBlock(button), true)
|
||||||
|
clock.advance(2)
|
||||||
|
assert.equal(gate.shouldBlock(button), false)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('被拦截的点击不会延长冷却', () => {
|
||||||
|
const clock = fakeClock()
|
||||||
|
const gate = createClickGate(1000, clock.now)
|
||||||
|
const button = {}
|
||||||
|
|
||||||
|
assert.equal(gate.shouldBlock(button), false)
|
||||||
|
for (let i = 0; i < 5; i += 1) {
|
||||||
|
clock.advance(100)
|
||||||
|
assert.equal(gate.shouldBlock(button), true)
|
||||||
|
}
|
||||||
|
// 从首次点击起算 1000ms 后就该放行,而不是被连点拖长
|
||||||
|
clock.advance(500)
|
||||||
|
assert.equal(gate.shouldBlock(button), false)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('同一时刻的两次点击只有第一次放行', () => {
|
||||||
|
const clock = fakeClock()
|
||||||
|
const gate = createClickGate(1500, clock.now)
|
||||||
|
const button = {}
|
||||||
|
|
||||||
|
assert.equal(gate.shouldBlock(button), false)
|
||||||
|
assert.equal(gate.shouldBlock(button), true, '同一 tick 的第二次点击必须被拦')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('reset 后立即放行', () => {
|
||||||
|
const clock = fakeClock()
|
||||||
|
const gate = createClickGate(1500, clock.now)
|
||||||
|
const button = {}
|
||||||
|
|
||||||
|
assert.equal(gate.shouldBlock(button), false)
|
||||||
|
assert.equal(gate.shouldBlock(button), true)
|
||||||
|
gate.reset(button)
|
||||||
|
assert.equal(gate.shouldBlock(button), false)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('非法冷却时长回落到默认值', () => {
|
||||||
|
const clock = fakeClock()
|
||||||
|
const gate = createClickGate(Number.NaN, clock.now)
|
||||||
|
const button = {}
|
||||||
|
|
||||||
|
assert.equal(gate.shouldBlock(button), false)
|
||||||
|
clock.advance(DEFAULT_SUBMIT_COOLDOWN_MS - 1)
|
||||||
|
assert.equal(gate.shouldBlock(button), true, 'NaN 应回落为默认冷却而不是立刻放行')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('全局安装:捕获阶段注册且可卸载', () => {
|
||||||
|
const registered: Array<{ type: string; handler: unknown; capture: boolean }> = []
|
||||||
|
const removed: string[] = []
|
||||||
|
const originalDocument = (globalThis as Record<string, unknown>).document
|
||||||
|
;(globalThis as Record<string, unknown>).document = {
|
||||||
|
addEventListener: (type: string, handler: unknown, capture: boolean) => {
|
||||||
|
registered.push({ type, handler, capture })
|
||||||
|
},
|
||||||
|
removeEventListener: (type: string) => {
|
||||||
|
removed.push(type)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const uninstall = installSubmitGuard({ cooldownMs: 1000, now: () => 0 })
|
||||||
|
assert.equal(registered.length, 1)
|
||||||
|
assert.equal(registered[0].type, 'click')
|
||||||
|
assert.equal(registered[0].capture, true, '必须捕获阶段,否则 Vue 的 @click 已先执行')
|
||||||
|
uninstall()
|
||||||
|
assert.deepEqual(removed, ['click'])
|
||||||
|
} finally {
|
||||||
|
;(globalThis as Record<string, unknown>).document = originalDocument
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test('全局安装:只拦 .btn-run 的连点,且第二次点击阻断冒泡', () => {
|
||||||
|
const registered: Array<{ handler: (event: unknown) => void }> = []
|
||||||
|
const originalDocument = (globalThis as Record<string, unknown>).document
|
||||||
|
;(globalThis as Record<string, unknown>).document = {
|
||||||
|
addEventListener: (_type: string, handler: (event: unknown) => void) => {
|
||||||
|
registered.push({ handler })
|
||||||
|
},
|
||||||
|
removeEventListener: () => undefined,
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
installSubmitGuard({ cooldownMs: 1000, now: () => 5000 })
|
||||||
|
const handler = registered[0].handler
|
||||||
|
|
||||||
|
const button = { closest: (selector: string) => (selector === '.btn-run' ? button : null) }
|
||||||
|
const elsewhere = { closest: () => null }
|
||||||
|
const makeEvent = (target: unknown) => {
|
||||||
|
const calls: string[] = []
|
||||||
|
return {
|
||||||
|
calls,
|
||||||
|
target,
|
||||||
|
stopImmediatePropagation: () => calls.push('stop'),
|
||||||
|
preventDefault: () => calls.push('prevent'),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const first = makeEvent(button)
|
||||||
|
handler(first)
|
||||||
|
assert.deepEqual(first.calls, [], '首次点击不应被拦')
|
||||||
|
|
||||||
|
const second = makeEvent(button)
|
||||||
|
handler(second)
|
||||||
|
assert.deepEqual(second.calls, ['stop', 'prevent'], '连点必须被拦下')
|
||||||
|
|
||||||
|
const other = makeEvent(elsewhere)
|
||||||
|
handler(other)
|
||||||
|
assert.deepEqual(other.calls, [], '非提交按钮的点击不受影响')
|
||||||
|
} finally {
|
||||||
|
;(globalThis as Record<string, unknown>).document = originalDocument
|
||||||
|
}
|
||||||
|
})
|
||||||
Reference in New Issue
Block a user