Compare commits
11 Commits
6e1689dfe5
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 3634ea1d62 | |||
| 3ce0569c59 | |||
| d2d95f0b71 | |||
| 2a51006888 | |||
| a89de129ea | |||
| ddefcbed56 | |||
| 3137299bfe | |||
| 1fe3368c5a | |||
| 7aea3a0a50 | |||
| 188aedec84 | |||
| d5952945dd |
+19
-5
@@ -29,7 +29,14 @@ public class TransientStorageProperties {
|
||||
*/
|
||||
private long maxTotalConcurrentOperations = 0;
|
||||
private long acquirePermitTimeoutMillis = 2000;
|
||||
private long baseRetryDelayMillis = 500;
|
||||
/**
|
||||
* 首次重试前的基础退避。
|
||||
*
|
||||
* <p>线上高频的重试诱因是 `unexpected end of stream`——那是**立即失败**(连接被 RustFS
|
||||
* 重置后 OkHttp 读响应即报错),不是等超时,所以 500ms 基本是白等:每天上千次累计十几分钟。
|
||||
* 降到 200ms 保留退避语义(真遇到服务端过载仍会退让),又不至于让用户等太久。
|
||||
*/
|
||||
private long baseRetryDelayMillis = 200;
|
||||
private long maxRetryDelayMillis = 5000;
|
||||
private long retryJitterMillis = 250;
|
||||
private long failureWindowSeconds = 60;
|
||||
@@ -37,13 +44,20 @@ public class TransientStorageProperties {
|
||||
private long failureCooldownMillis = 10000;
|
||||
private int dispatcherMaxRequests = 56;
|
||||
private int dispatcherMaxRequestsPerHost = 56;
|
||||
/**
|
||||
* 空闲连接保留数。
|
||||
*
|
||||
* <p>2026-09-17 曾试过设 0(彻底不复用)来验证"unexpected end of stream 是复用死连接导致的"
|
||||
* 这一假设——**实测照旧失败**(新容器起来后第一次请求就中招)。至此已排除公网链路、
|
||||
* keepAlive 过长、连接复用三项;用 mc 并发压 200 个小对象也全部成功,说明服务端没问题。
|
||||
* 剩余方向指向 MinIO Java SDK / OkHttp 与 RustFS 的协议细节,故恢复默认的连接复用。
|
||||
*/
|
||||
private int connectionPoolMaxIdle = 5;
|
||||
/**
|
||||
* 空闲连接在池里的保留时长。默认 5 分钟(OkHttp 原值)会让客户端复用"已被 RustFS
|
||||
* 或中间设备关掉的空闲连接",表现为 unexpected end of stream —— 线上每天上千次,
|
||||
* 全靠重试兜底。对象存储访问是突发型,连接复用率本就低,缩短保活几乎没有代价。
|
||||
* 空闲连接在池里的保留时长。曾由 300000 调到 30000 试图减少 unexpected end of stream,
|
||||
* 实测无改善(该现象与连接复用无关,见 {@link #connectionPoolMaxIdle} 的排查记录),故恢复原值。
|
||||
*/
|
||||
private long connectionPoolKeepAliveMillis = 30000;
|
||||
private long connectionPoolKeepAliveMillis = 300000;
|
||||
private long warnPayloadBytes = 5L * 1024 * 1024;
|
||||
private long maxPayloadBytes = 50L * 1024 * 1024;
|
||||
private long maxStoredPayloadBytes = 50L * 1024 * 1024;
|
||||
|
||||
+105
-4
@@ -424,6 +424,53 @@ public class AppearancePatentTaskService {
|
||||
|
||||
public void submitResult(Long taskId, AppearancePatentSubmitResultRequest request) {
|
||||
submitResultLocked(taskId, request);
|
||||
// 分片补传后自动恢复终态失败的组装 job —— 走与删除品牌同一套口径,
|
||||
// 此前外观专利没有接该入口,分片补齐后只能人工重置 job(线上任务 28459 即如此)。
|
||||
maybeRecoverTerminalFailedAssemble(taskId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 补传恢复:此前因分片缺失导致组装 job 重试耗尽(终态失败),客户端补传缺口后
|
||||
* 把失败的组装 job 重置为 PENDING 重新派发({@code resetTerminalFailedForRecovery} 自行补发 dispatch 事件)。
|
||||
*
|
||||
* <p>best-effort:恢复失败不得影响补传本身——分片已经落库,恢复只是让后续组装继续推进。
|
||||
* 常态(无终态失败 job)下只查两次即返回,不触发分片扫描。
|
||||
*/
|
||||
private void maybeRecoverTerminalFailedAssemble(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
FileResultEntity result = findResultRecord(taskId);
|
||||
if (result == null) {
|
||||
return;
|
||||
}
|
||||
if (taskFileJobService.hasSuccessfulAssembleJob(taskId, MODULE_TYPE, result.getId())) {
|
||||
return;
|
||||
}
|
||||
if (!taskFileJobService.isTerminalFailedAssembleJob(taskId, MODULE_TYPE, result.getId())) {
|
||||
return;
|
||||
}
|
||||
if (!isResultSubmissionComplete(taskId)) {
|
||||
log.info("[appearance-patent] 组装 job 终态失败但分片仍未补传完整,暂不恢复 taskId={} resultId={}",
|
||||
taskId, result.getId());
|
||||
return;
|
||||
}
|
||||
log.info("[appearance-patent] 分片已补传完整,恢复终态失败的组装 job taskId={} resultId={}",
|
||||
taskId, result.getId());
|
||||
taskFileJobService.resetTerminalFailedForRecovery(taskId, MODULE_TYPE, result.getId());
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent] 补传恢复检查失败(不影响本次补传)taskId={} err={}", taskId, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/** 按 task 取结果行(不创建);不存在返回 null。 */
|
||||
private FileResultEntity findResultRecord(Long taskId) {
|
||||
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, taskId)
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||
.last("limit 1"));
|
||||
return rows == null || rows.isEmpty() ? null : rows.getFirst();
|
||||
}
|
||||
|
||||
private void submitResultLocked(Long taskId, AppearancePatentSubmitResultRequest request) {
|
||||
@@ -1065,6 +1112,14 @@ public class AppearancePatentTaskService {
|
||||
long activeAssembleJobs = taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE);
|
||||
log.info("[appearance-patent] stale recovery probe taskId={} uploadComplete={} activeAssembleJobs={} persistedRows={}",
|
||||
taskId, uploadComplete, activeAssembleJobs, hasPersistedResultRows(taskId));
|
||||
// 已有「重试耗尽且已终态收尾」的 assemble job:说明恢复已经试过、缺失是永久的。
|
||||
// 再重建只会每 30 秒空转一轮,而且恢复过程刷新任务心跳会让任务永远 RUNNING
|
||||
// (线上任务 28459 实测:48 分钟里每隔 30 秒重建一次 job)。返回 false 交给
|
||||
// finalizeStaleTask 按失败收尾,用户看到明确失败而不是无限等待。
|
||||
if (taskFileJobService.hasExhaustedAssembleJob(taskId, MODULE_TYPE)) {
|
||||
log.warn("[appearance-patent] stale recovery 放弃:已有重试耗尽的 assemble job,按失败收尾 taskId={}", taskId);
|
||||
return false;
|
||||
}
|
||||
if (!hasPersistedResultRows(taskId)) {
|
||||
log.info("[appearance-patent] stale recovery aborted because no persisted rows taskId={}", taskId);
|
||||
return false;
|
||||
@@ -1215,6 +1270,7 @@ public class AppearancePatentTaskService {
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String conflictDetail = "未发生冲突";
|
||||
for (int attempt = 1; attempt <= CHUNK_PAYLOAD_MERGE_RETRY_LIMIT; attempt++) {
|
||||
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
@@ -1256,13 +1312,35 @@ public class AppearancePatentTaskService {
|
||||
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
||||
return;
|
||||
}
|
||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||
// CAS 冲突:读回行上的当前哈希,否则只有一句 conflict 无从判断是谁改的(线上 28459 至今未能定位)
|
||||
String currentHash = currentPayloadHash(chunk.getId());
|
||||
conflictDetail = "期望hash=" + oldPayloadHash + " 当前hash=" + currentHash;
|
||||
if (attempt < CHUNK_PAYLOAD_MERGE_RETRY_LIMIT) {
|
||||
log.warn("[appearance-patent] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{}",
|
||||
taskId, scopeHash, chunkIndex, attempt, CHUNK_PAYLOAD_MERGE_RETRY_LIMIT);
|
||||
log.warn("[appearance-patent] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{} 期望hash={} 当前hash={}",
|
||||
taskId, scopeHash, chunkIndex, attempt, CHUNK_PAYLOAD_MERGE_RETRY_LIMIT, oldPayloadHash, currentHash);
|
||||
// 还要重试:这次写的对象会被下次重写,先删掉避免堆积
|
||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||
} else {
|
||||
// 终局失败:保留这次写入的版本化对象,作为读路径「同槽位兄弟对象」兜底的恢复源。
|
||||
// 行没指过去不该让该分片永久判死——删掉它才是线上 28459 丢数据的形态。
|
||||
log.error("[appearance-patent] chunk payload 合并冲突重试耗尽,保留兜底对象 taskId={} scopeHash={} chunk={} 期望hash={} 当前hash={} 保留对象={}",
|
||||
taskId, scopeHash, chunkIndex, oldPayloadHash, currentHash, storedPayload);
|
||||
}
|
||||
}
|
||||
throw new IllegalStateException("appearance patent chunk payload update conflict");
|
||||
throw new IllegalStateException("appearance patent chunk payload update conflict " + conflictDetail);
|
||||
}
|
||||
|
||||
/** 读回 chunk 行当前的 payload_hash,用于 CAS 冲突定位(行已不存在/读取失败时返回可读标记)。 */
|
||||
private String currentPayloadHash(Long chunkId) {
|
||||
if (chunkId == null) {
|
||||
return "chunkId 为空";
|
||||
}
|
||||
try {
|
||||
TaskChunkEntity latest = taskChunkMapper.selectById(chunkId);
|
||||
return latest == null ? "行已不存在" : latest.getPayloadHash();
|
||||
} catch (Exception ex) {
|
||||
return "读取失败:" + ex.getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives,
|
||||
@@ -2915,12 +2993,35 @@ public class AppearancePatentTaskService {
|
||||
} catch (Exception ex) {
|
||||
log.warn("[appearance-patent] read chunk payload failed taskId={} chunk={} err={}",
|
||||
chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage());
|
||||
if (isPayloadMissing(ex)) {
|
||||
// payload 对象已不在(被清理或从未写入):重试多少次都读不回来。继续抛会让
|
||||
// ASSEMBLE_RESULT job 的终态回调每轮重跑兜底组装 → 再读同一个缺失对象 → 无限循环
|
||||
// (线上任务 28459 每 10~30 秒重试一次)。跳过该分片,让任务按已有分片出部分结果,
|
||||
// 与品牌/相似ASIN「失败也产出可下载的部分结果」同一口径。
|
||||
log.warn("[appearance-patent] chunk payload 已不存在,跳过该分片(任务按已有分片出结果)"
|
||||
+ " taskId={} chunk={}", chunk.getTaskId(), chunk.getChunkIndex());
|
||||
return rows;
|
||||
}
|
||||
throw new BusinessException("appearance patent chunk payload read failed chunk="
|
||||
+ chunk.getChunkIndex() + ": " + ex.getMessage(), ex);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** payload 对象已不存在(RustFS 返回 NoSuchKey:message 为 "The specified key does not exist.")。
|
||||
* 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。 */
|
||||
private static boolean isPayloadMissing(Throwable error) {
|
||||
Throwable cursor = error;
|
||||
while (cursor != null) {
|
||||
String message = cursor.getMessage();
|
||||
if (message != null && message.contains("does not exist")) {
|
||||
return true;
|
||||
}
|
||||
cursor = cursor.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private String rowKey(AppearancePatentParsedRowVo row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
|
||||
+150
-1
@@ -5,10 +5,14 @@ import io.micrometer.core.instrument.DistributionSummary;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import io.micrometer.core.instrument.Timer;
|
||||
import io.minio.GetObjectArgs;
|
||||
import io.minio.ListObjectsArgs;
|
||||
import io.minio.MinioClient;
|
||||
import io.minio.PutObjectArgs;
|
||||
import io.minio.RemoveObjectArgs;
|
||||
import io.minio.Result;
|
||||
import io.minio.StatObjectArgs;
|
||||
import io.minio.errors.ErrorResponseException;
|
||||
import io.minio.messages.Item;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import okhttp3.ConnectionPool;
|
||||
import okhttp3.Dispatcher;
|
||||
@@ -20,7 +24,11 @@ import org.springframework.stereotype.Service;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
@@ -38,6 +46,17 @@ public class RustfsObjectStorageService {
|
||||
private static final String OP_STAT = "stat";
|
||||
private static final String OP_TOTAL = "total";
|
||||
|
||||
/** S3 确定性错误码:重试不会改变结果,应立即失败而不是白打两次请求。 */
|
||||
private static final Set<String> NON_RETRYABLE_S3_CODES = Set.of(
|
||||
"NoSuchKey", "NoSuchBucket", "NoSuchVersion",
|
||||
"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "InvalidBucketName");
|
||||
|
||||
/** 标准 UUID 字符串长度(8-4-4-4-12),用于识别版本化对象 key。 */
|
||||
private static final int UUID_STRING_LENGTH = 36;
|
||||
|
||||
/** 兄弟对象兜底列出的上限:只为找回同槽位的版本化对象,不需要列全。 */
|
||||
private static final int MAX_SIBLING_LIST_KEYS = 50;
|
||||
|
||||
private final TransientStorageProperties properties;
|
||||
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
|
||||
private final ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider;
|
||||
@@ -99,7 +118,22 @@ public class RustfsObjectStorageService {
|
||||
return uploadBytes(objectKey, bytes, verifyAfterUpload);
|
||||
}
|
||||
|
||||
/**
|
||||
* 三参重载:是否做「上传失败补偿删除」按对象 key 形态自动判定。
|
||||
*
|
||||
* <p>只有版本化 key(末段以 UUID 结尾)是本次写入独占的;确定性 key 会被重传重写复用,
|
||||
* 删它就可能删掉别的 DB 行仍在引用的对象(2026-09-17 线上任务 28459 的载荷对象就是这么丢的)。
|
||||
*/
|
||||
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload) {
|
||||
return uploadBytes(objectKey, content, verifyAfterUpload, isVersionedObjectKey(objectKey));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param compensateDeleteOnFailure put 已完成、但后续可见性校验失败时,是否把该对象排进删除补偿队列。
|
||||
* 仅当调用方能确认「该对象不会被其它写入复用时」才可传 true。
|
||||
*/
|
||||
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload,
|
||||
boolean compensateDeleteOnFailure) {
|
||||
long deadlineNanos = operationDeadlineNanos();
|
||||
if (!isConfigured()) {
|
||||
throw new IllegalStateException("transient storage is not configured");
|
||||
@@ -129,13 +163,45 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
return uploadedObjectKey;
|
||||
} catch (RuntimeException ex) {
|
||||
if (putCompleted.get()) {
|
||||
if (putCompleted.get() && compensateDeleteOnFailure) {
|
||||
enqueueDeleteRetry(objectKey, ex);
|
||||
} else if (putCompleted.get()) {
|
||||
// 共享 key 会被重传重写:此处删除可能删掉别的行正在引用的对象,交给保留期清理兜底。
|
||||
// 线上任务 28459 的 chunk-462/473/484 就是被这条无条件删除队列删掉的。
|
||||
log.warn("[rustfs] 跳过上传失败补偿删除(对象非本次独占,可能被复用)objectKey={} err={}",
|
||||
objectKey, ex.getMessage());
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 对象 key 是否为「本次写入独占」的版本化 key:末段(去掉 {@code .json} 后缀)以 UUID 结尾。
|
||||
*
|
||||
* <p>只看末段——UUID 出现在中间段(如 scopeHash)不代表该对象被独占;解析失败一律按共享处理
|
||||
* (保守:宁可留下孤儿对象,也不删掉可能仍被引用的对象)。
|
||||
*/
|
||||
static boolean isVersionedObjectKey(String objectKey) {
|
||||
if (objectKey == null) {
|
||||
return false;
|
||||
}
|
||||
String key = objectKey.trim();
|
||||
if (key.endsWith(".json")) {
|
||||
key = key.substring(0, key.length() - ".json".length());
|
||||
}
|
||||
int slash = key.lastIndexOf('/');
|
||||
String lastSegment = slash < 0 ? key : key.substring(slash + 1);
|
||||
if (lastSegment.length() < UUID_STRING_LENGTH) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
UUID.fromString(lastSegment.substring(lastSegment.length() - UUID_STRING_LENGTH));
|
||||
return true;
|
||||
} catch (IllegalArgumentException ex) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public String readObjectAsString(String objectKey) {
|
||||
byte[] bytes = readObjectBytes(objectKey);
|
||||
return new String(bytes, StandardCharsets.UTF_8);
|
||||
@@ -175,6 +241,43 @@ public class RustfsObjectStorageService {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 列出前缀下的对象 key,按最后修改时间倒序(最新在前)。
|
||||
*
|
||||
* <p>只服务于「指针指向的对象已不存在、需要找回同槽位的版本化兄弟对象」这一兜底路径,
|
||||
* 因此刻意不做重试、不参与失败窗口记账:列出失败直接抛错,由调用方按原错误语义处理。
|
||||
*/
|
||||
public List<String> listObjectKeysNewestFirst(String prefix, int limit) {
|
||||
if (!isConfigured()) {
|
||||
throw new IllegalStateException("transient storage is not configured");
|
||||
}
|
||||
int safeLimit = Math.max(1, Math.min(limit, MAX_SIBLING_LIST_KEYS));
|
||||
long deadlineNanos = operationDeadlineNanos();
|
||||
List<String[]> entries = new ArrayList<>();
|
||||
try {
|
||||
Iterable<Result<Item>> results = buildClient(deadlineNanos).listObjects(ListObjectsArgs.builder()
|
||||
.bucket(properties.getBucket())
|
||||
.prefix(prefix == null ? "" : prefix)
|
||||
.recursive(true)
|
||||
.maxKeys(safeLimit)
|
||||
.build());
|
||||
for (Result<Item> result : results) {
|
||||
Item item = result.get();
|
||||
entries.add(new String[]{item.objectName(),
|
||||
item.lastModified() == null ? "" : item.lastModified().toString()});
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("transient storage list failed prefix=" + prefix
|
||||
+ " err=" + ex.getMessage(), ex);
|
||||
}
|
||||
entries.sort((left, right) -> right[1].compareTo(left[1]));
|
||||
List<String> keys = new ArrayList<>(entries.size());
|
||||
for (String[] entry : entries) {
|
||||
keys.add(entry[0]);
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
public void deleteObject(String objectKey) {
|
||||
deleteObject(objectKey, true, operationDeadlineNanos());
|
||||
}
|
||||
@@ -251,6 +354,15 @@ public class RustfsObjectStorageService {
|
||||
last = ex;
|
||||
recordOperation(operation, attempt < maxRetries ? "retry" : "failure", elapsedNanos(startedAt));
|
||||
recordFailure(operation, objectKey, ex);
|
||||
if (isNonRetryable(ex)) {
|
||||
// 确定性错误(NoSuchKey / AccessDenied…):重试多少次结果都一样。
|
||||
// 线上 read 一个已被清理的 chunk 就会连打 3 次请求、还被记成 ERROR。
|
||||
log.warn("[rustfs] 确定性错误,不重试 operation={} objectKey={} err={}",
|
||||
operation, objectKey, ex.getMessage());
|
||||
throw ex instanceof RuntimeException runtimeException
|
||||
? runtimeException
|
||||
: new IllegalStateException(ex);
|
||||
}
|
||||
if (attempt < maxRetries) {
|
||||
delayMillis = retryDelayMillis(attempt);
|
||||
log.warn("[rustfs] operation failed, retrying operation={} objectKey={} attempt={}/{} delayMs={} err={}",
|
||||
@@ -279,6 +391,27 @@ public class RustfsObjectStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 是否为「重试也没用」的确定性错误(NoSuchKey / AccessDenied / 签名错误…)。
|
||||
*
|
||||
* <p>只有网络类与 5xx 类失败才值得重试;确定性错误重试多少次结果都一样。
|
||||
*/
|
||||
private static boolean isNonRetryable(Throwable error) {
|
||||
Throwable cursor = error;
|
||||
while (cursor != null) {
|
||||
if (cursor instanceof ErrorResponseException responseException) {
|
||||
String code = responseException.errorResponse() == null
|
||||
? null
|
||||
: responseException.errorResponse().code();
|
||||
if (code != null && NON_RETRYABLE_S3_CODES.contains(code)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
cursor = cursor.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void acquirePermit(String operation, String objectKey, Semaphore semaphore, long deadlineNanos) {
|
||||
try {
|
||||
checkDeadline(operation, objectKey, deadlineNanos);
|
||||
@@ -345,6 +478,22 @@ public class RustfsObjectStorageService {
|
||||
Math.max(1L, properties.getConnectionPoolKeepAliveMillis()),
|
||||
TimeUnit.MILLISECONDS))
|
||||
.retryOnConnectionFailure(true)
|
||||
.addNetworkInterceptor(chain -> {
|
||||
okhttp3.Request request = chain.request();
|
||||
okhttp3.RequestBody body = request.body();
|
||||
// OkHttp 对「长度为 0 的请求体」不会写 Content-Length,于是请求既无
|
||||
// Content-Length 也无 Transfer-Encoding(HTTP/1.1 不允许这样)。
|
||||
// RustFS 对此直接回 411 Length Required,客户端读到不完整响应就报
|
||||
// unexpected end of stream,进而重试——线上抓包实测 90 秒内 515 次 411,
|
||||
// 而上传空内容(content 为 null/空)在业务里是常态。
|
||||
// 用 network interceptor 在协议层补上该头(普通 interceptor 会被
|
||||
// BridgeInterceptor 按 body 长度覆盖掉,加了也不生效)。
|
||||
if (body != null && body.contentLength() == 0L
|
||||
&& request.header("Content-Length") == null) {
|
||||
request = request.newBuilder().header("Content-Length", "0").build();
|
||||
}
|
||||
return chain.proceed(request);
|
||||
})
|
||||
.build();
|
||||
}
|
||||
return httpClient;
|
||||
|
||||
+21
@@ -198,6 +198,27 @@ public class PublishTaskService {
|
||||
if (runningFile != null) {
|
||||
throw new BusinessException("同一任务已有文件正在执行: " + runningFile.getSourceFilename());
|
||||
}
|
||||
// 店铺级互斥:同一店铺同一时刻只允许一个上架任务在跑。
|
||||
// 2026-09-17 事故(28519/28520/28521 相继提交同一店铺「林洪武」):同一店铺被多个
|
||||
// 任务并发打开,客户端 startBrowser 全部返回 -10000,三个任务一起失败。激活是任务
|
||||
// 真正开跑的唯一入口,在这里挡掉并带出占用中的任务号,用户才知道要等谁。
|
||||
//
|
||||
// 注意:本校验与随后的状态更新之间仍有极小竞态窗口(两个请求恰好同时通过校验);
|
||||
// 真正的串行由客户端店铺锁保证,这一层的目的是尽早给出明确提示,避免白传文件与重复执行。
|
||||
String shopName = file.getShopName();
|
||||
if (shopName != null && !shopName.isBlank()) {
|
||||
PublishFileEntity shopRunning = publishFileMapper.selectOne(
|
||||
new LambdaQueryWrapper<PublishFileEntity>()
|
||||
.eq(PublishFileEntity::getShopName, shopName)
|
||||
.eq(PublishFileEntity::getStatus, STATUS_RUNNING)
|
||||
.ne(PublishFileEntity::getTaskId, taskId)
|
||||
.orderByAsc(PublishFileEntity::getId)
|
||||
.last("limit 1"));
|
||||
if (shopRunning != null) {
|
||||
throw new BusinessException("店铺「" + shopName + "」已有上架任务正在执行(任务 "
|
||||
+ shopRunning.getTaskId() + "),请等它完成后再提交");
|
||||
}
|
||||
}
|
||||
int updated = publishFileMapper.update(null, new LambdaUpdateWrapper<PublishFileEntity>()
|
||||
.eq(PublishFileEntity::getId, fileId)
|
||||
.eq(PublishFileEntity::getTaskId, taskId)
|
||||
|
||||
+9
@@ -27,6 +27,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
@@ -121,6 +122,7 @@ public class ShopDataCrawlTaskService {
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
private final ShopDataCrawlDailyFileService dailyFileService;
|
||||
private final ShopDataCrawlItemStoreService shopDataCrawlItemStoreService;
|
||||
private final DuplicateCheckRefreshPort duplicateCheckRefreshPort;
|
||||
private final PlatformTransactionManager transactionManager;
|
||||
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
|
||||
@@ -2116,6 +2118,13 @@ public class ShopDataCrawlTaskService {
|
||||
shopDataCrawlItemStoreService.saveShopBatchFromSnapshot(
|
||||
snapshot.getShopName(), itemBatchDate, accumulatedItems,
|
||||
snapshot.getResultId(), task.getId(), baseDailyFile == null ? null : baseDailyFile.getId());
|
||||
// 明细已落库:请求撞款重扫(异步合并执行,不阻塞归档;端口契约保证不抛错)
|
||||
try {
|
||||
duplicateCheckRefreshPort.requestRefresh("shop-data-crawl:" + snapshot.getShopName());
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[shop-data-crawl] 请求撞款重扫失败(忽略,不影响归档) shop={} msg={}",
|
||||
snapshot.getShopName(), ex.getMessage());
|
||||
}
|
||||
int rowCount = excelAssemblyService.writeWorkbook(outputXlsx, accumulatedItems);
|
||||
String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
|
||||
if (blank(objectKey)) {
|
||||
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.spi;
|
||||
|
||||
/**
|
||||
* 采集明细就绪后的撞款重扫触发端口(2026-09:店铺数据采集落库后即时刷新重复检查)。
|
||||
*
|
||||
* <p>实现方在 shopduplicatecheck 模块({@code ShopDataDuplicateCheckScanService})。
|
||||
* 契约:实现必须异步执行、去抖合并,不得阻塞调用方、不得向外抛出异常。
|
||||
*/
|
||||
public interface DuplicateCheckRefreshPort {
|
||||
|
||||
/**
|
||||
* 请求一次撞款重扫(异步;合并窗口内的多次触发聚合为一次扫描)。
|
||||
*
|
||||
* @param reason 触发来源,仅用于日志排查
|
||||
*/
|
||||
void requestRefresh(String reason);
|
||||
}
|
||||
+32
-2
@@ -9,6 +9,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlItemStoreService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckItemMapper;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckSourceMapper;
|
||||
@@ -21,6 +22,7 @@ import com.nanri.aiimage.modules.shopduplicatecheck.model.entity.ShopDataDuplica
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanPayload;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanSummary;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckAggregator;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckRefreshScheduler;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckWorkbookParser;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.RawRow;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.ShopParsed;
|
||||
@@ -43,14 +45,14 @@ import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* 撞款扫描:每日 00:00 定时 + force 同步全量重扫。
|
||||
* 撞款扫描:每日 00:00 定时 + force 同步全量重扫 + 采集落库触发的异步合并重扫。
|
||||
* 数据源 = 采集明细表(biz_shop_data_crawl_item,采集先落库再更新文件),
|
||||
* 直接查库聚合 shops/items/summary 落库 shop_data_duplicate_scan(输出契约不变)。
|
||||
* 双实例通过 Redis 分布式锁防重;扫描失败落 FAILED 行并上抛,force 场景由端点转 409/500。
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ShopDataDuplicateCheckScanService {
|
||||
public class ShopDataDuplicateCheckScanService implements DuplicateCheckRefreshPort {
|
||||
|
||||
public static final String SCAN_LOCK = "shop-data-duplicate-check:scan";
|
||||
static final DateTimeFormatter SCANNED_AT_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||
@@ -71,6 +73,9 @@ public class ShopDataDuplicateCheckScanService {
|
||||
private final AtomicLong cachedRowId = new AtomicLong(-1L);
|
||||
private volatile CachedScan cachedScan;
|
||||
|
||||
/** 采集落库触发的异步合并重扫调度器(单飞 + 去抖 + 锁忙重试)。 */
|
||||
private final DuplicateCheckRefreshScheduler refreshScheduler;
|
||||
|
||||
@Autowired
|
||||
public ShopDataDuplicateCheckScanService(ShopDataDuplicateScanMapper scanMapper,
|
||||
ShopDuplicateCheckSourceMapper sourceMapper,
|
||||
@@ -86,6 +91,7 @@ public class ShopDataDuplicateCheckScanService {
|
||||
this.objectMapper = objectMapper;
|
||||
this.itemMapper = itemMapper;
|
||||
this.itemStoreService = itemStoreService;
|
||||
this.refreshScheduler = new DuplicateCheckRefreshScheduler(this::runRefreshOnce);
|
||||
}
|
||||
|
||||
/** 读侧视图:scanned_at 为最新 SUCCESS 行 created_at(yyyy-MM-dd HH:mm:ss)。 */
|
||||
@@ -122,6 +128,30 @@ public class ShopDataDuplicateCheckScanService {
|
||||
}
|
||||
}
|
||||
|
||||
/** 采集明细落库后的重扫请求(端口实现):异步合并执行,不阻塞、不抛错。 */
|
||||
@Override
|
||||
public void requestRefresh(String reason) {
|
||||
refreshScheduler.request(reason);
|
||||
}
|
||||
|
||||
/** 调度器单次扫描动作:锁被占返回 LOCK_BUSY 供其重试;失败只记日志(FAILED 行已落库)。 */
|
||||
private DuplicateCheckRefreshScheduler.Outcome runRefreshOnce() {
|
||||
try {
|
||||
scanNow();
|
||||
return DuplicateCheckRefreshScheduler.Outcome.DONE;
|
||||
} catch (BusinessException ex) {
|
||||
if (ex.getCode() != null && ex.getCode() == 409) {
|
||||
log.info("[shop-duplicate-check] 自动重扫未执行:其它扫描进行中 msg={}", ex.getMessage());
|
||||
return DuplicateCheckRefreshScheduler.Outcome.LOCK_BUSY;
|
||||
}
|
||||
log.warn("[shop-duplicate-check] 自动重扫失败 code={} msg={}", ex.getCode(), ex.getMessage());
|
||||
return DuplicateCheckRefreshScheduler.Outcome.FAILED;
|
||||
} catch (Exception ex) {
|
||||
log.error("[shop-duplicate-check] 自动重扫异常", ex);
|
||||
return DuplicateCheckRefreshScheduler.Outcome.FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
/** 最新 SUCCESS 扫描视图;无扫描结果返回 null。 */
|
||||
public DuplicateScanView loadLatest() {
|
||||
ScanLightRowDto light = scanMapper.selectLatestLightRow();
|
||||
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
package com.nanri.aiimage.modules.shopduplicatecheck.service.support;
|
||||
|
||||
import com.nanri.aiimage.common.util.ThreadPools;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* 采集落库触发的撞款重扫调度器:合并窗口去抖 + 单飞 + 锁忙重试。
|
||||
*
|
||||
* <p>语义:{@link #request} 只置位并异步执行,永不阻塞调用方、永不向外抛错;
|
||||
* 合并窗口内的多次触发聚合为一次扫描;扫描动作执行期间到达的触发在下一轮执行;
|
||||
* 扫描因分布式锁被占用未执行({@link Outcome#LOCK_BUSY})时按固定间隔重试有限次。
|
||||
*/
|
||||
@Slf4j
|
||||
public class DuplicateCheckRefreshScheduler {
|
||||
|
||||
/** 单次扫描动作的终态:完成 / 锁被占(可重试)/ 失败(不重试,等下次触发或定时扫描)。 */
|
||||
public enum Outcome {
|
||||
DONE, LOCK_BUSY, FAILED
|
||||
}
|
||||
|
||||
private static final long DEFAULT_DEBOUNCE_MILLIS = 10_000L;
|
||||
private static final long DEFAULT_LOCK_RETRY_MILLIS = 20_000L;
|
||||
private static final int DEFAULT_MAX_LOCK_RETRIES = 6;
|
||||
|
||||
private final Supplier<Outcome> scanAction;
|
||||
private final long debounceMillis;
|
||||
private final long lockRetryMillis;
|
||||
private final int maxLockRetries;
|
||||
private final ExecutorService executor;
|
||||
private final AtomicBoolean pending = new AtomicBoolean(false);
|
||||
private final AtomicBoolean running = new AtomicBoolean(false);
|
||||
|
||||
public DuplicateCheckRefreshScheduler(Supplier<Outcome> scanAction) {
|
||||
this(scanAction, DEFAULT_DEBOUNCE_MILLIS, DEFAULT_LOCK_RETRY_MILLIS, DEFAULT_MAX_LOCK_RETRIES);
|
||||
}
|
||||
|
||||
/** 测试用:注入更短的窗口与重试参数。 */
|
||||
DuplicateCheckRefreshScheduler(Supplier<Outcome> scanAction, long debounceMillis,
|
||||
long lockRetryMillis, int maxLockRetries) {
|
||||
this.scanAction = scanAction;
|
||||
this.debounceMillis = Math.max(0L, debounceMillis);
|
||||
this.lockRetryMillis = Math.max(0L, lockRetryMillis);
|
||||
this.maxLockRetries = Math.max(0, maxLockRetries);
|
||||
this.executor = ThreadPools.boundedFixed("shop-dup-refresh", 1, 8);
|
||||
}
|
||||
|
||||
/** 请求一次重扫(异步、去抖合并)。调用方不被阻塞,也不会收到异常。 */
|
||||
public void request(String reason) {
|
||||
pending.set(true);
|
||||
if (running.compareAndSet(false, true)) {
|
||||
submit(reason);
|
||||
}
|
||||
}
|
||||
|
||||
private void submit(String reason) {
|
||||
try {
|
||||
log.info("[shop-duplicate-check] 触发撞款重扫(异步合并执行,窗口={}ms) reason={}", debounceMillis, reason);
|
||||
executor.execute(this::drain);
|
||||
} catch (Exception ex) {
|
||||
// 提交失败(如线程池拒绝)时复位单飞标记,避免后续触发被永久吞掉
|
||||
running.set(false);
|
||||
log.warn("[shop-duplicate-check] 撞款重扫任务提交失败 reason={} msg={}", reason, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void drain() {
|
||||
try {
|
||||
while (true) {
|
||||
// 合并窗口:窗口内到达的多次触发聚合为同一轮扫描
|
||||
if (!sleepQuietly(debounceMillis)) {
|
||||
return;
|
||||
}
|
||||
if (!pending.compareAndSet(true, false)) {
|
||||
return;
|
||||
}
|
||||
int lockRetries = 0;
|
||||
while (true) {
|
||||
Outcome outcome = runOnceSafely();
|
||||
if (outcome != Outcome.LOCK_BUSY) {
|
||||
break;
|
||||
}
|
||||
if (lockRetries >= maxLockRetries) {
|
||||
log.warn("[shop-duplicate-check] 撞款重扫连续 {} 次未取得扫描锁,放弃本轮(等待下次触发或定时扫描)",
|
||||
lockRetries + 1);
|
||||
break;
|
||||
}
|
||||
lockRetries++;
|
||||
log.info("[shop-duplicate-check] 撞款重扫未取得扫描锁,{}ms 后重试(第 {}/{} 次)",
|
||||
lockRetryMillis, lockRetries, maxLockRetries);
|
||||
if (!sleepQuietly(lockRetryMillis)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
running.set(false);
|
||||
// 竞态兜底:running 复位前到达的触发可能没能提交,补一次
|
||||
if (pending.get() && running.compareAndSet(false, true)) {
|
||||
submit("race-guard");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 执行一次扫描动作;动作自身异常也被吸收(调度器对外零抛出)。 */
|
||||
private Outcome runOnceSafely() {
|
||||
long startedAt = System.currentTimeMillis();
|
||||
try {
|
||||
Outcome outcome = scanAction.get();
|
||||
long elapsed = System.currentTimeMillis() - startedAt;
|
||||
if (outcome == Outcome.DONE) {
|
||||
log.info("[shop-duplicate-check] 采集后自动重扫完成 耗时={}ms", elapsed);
|
||||
} else if (outcome == Outcome.FAILED) {
|
||||
log.warn("[shop-duplicate-check] 采集后自动重扫失败 耗时={}ms", elapsed);
|
||||
}
|
||||
return outcome == null ? Outcome.FAILED : outcome;
|
||||
} catch (Exception ex) {
|
||||
log.error("[shop-duplicate-check] 采集后自动重扫异常", ex);
|
||||
return Outcome.FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean sleepQuietly(long millis) {
|
||||
if (millis <= 0) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
Thread.sleep(millis);
|
||||
return true;
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
+9
@@ -87,6 +87,15 @@ public class SimilarAsinChunkPayloadSupport {
|
||||
recordChunkReadFailure(chunk, false, msg);
|
||||
return rows;
|
||||
}
|
||||
// payload 对象已不存在(RustFS 返回 NoSuchKey):重试多少次都读不回来,跳过而不是
|
||||
// 把组装/收尾永久拖死——与上面的 typeMismatch 分支、以及 collect-data 的降级口径一致。
|
||||
// 线上 appearance-patent 28459 就因同类场景每 10~30 秒重试一次(见同批修复)。
|
||||
if (msg.contains("does not exist")) {
|
||||
log.warn("[similar-asin] chunk payload 已不存在,跳过该分片 taskId={} chunk={} err={}",
|
||||
chunk.getTaskId(), chunk.getChunkIndex(), msg);
|
||||
recordChunkReadFailure(chunk, false, msg);
|
||||
return rows;
|
||||
}
|
||||
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
|
||||
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
|
||||
recordChunkReadFailure(chunk, crossInstance, msg);
|
||||
|
||||
+27
-4
@@ -214,6 +214,7 @@ public class SimilarAsinPipelineSupport {
|
||||
return;
|
||||
}
|
||||
int maxAttempts = 3;
|
||||
String conflictDetail = "未发生冲突";
|
||||
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
@@ -275,13 +276,35 @@ public class SimilarAsinPipelineSupport {
|
||||
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
||||
return;
|
||||
}
|
||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||
// CAS 冲突:读回行上的当前哈希,否则只有一句 conflict 无从判断是谁改的
|
||||
String currentHash = currentPayloadHash(chunk.getId());
|
||||
conflictDetail = "期望hash=" + oldPayloadHash + " 当前hash=" + currentHash;
|
||||
if (attempt < maxAttempts) {
|
||||
log.warn("[similar-asin] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{}",
|
||||
taskId, scopeHash, chunkIndex, attempt, maxAttempts);
|
||||
log.warn("[similar-asin] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{} 期望hash={} 当前hash={}",
|
||||
taskId, scopeHash, chunkIndex, attempt, maxAttempts, oldPayloadHash, currentHash);
|
||||
// 还要重试:这次写的对象会被下次重写,先删掉避免堆积
|
||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||
} else {
|
||||
// 终局失败:保留这次写入的版本化对象,作为读路径「同槽位兄弟对象」兜底的恢复源。
|
||||
// 与外观专利同一口径——行没指过去不该让该分片永久判死。
|
||||
log.error("[similar-asin] chunk payload 合并冲突重试耗尽,保留兜底对象 taskId={} scopeHash={} chunk={} 期望hash={} 当前hash={} 保留对象={}",
|
||||
taskId, scopeHash, chunkIndex, oldPayloadHash, currentHash, storedPayload);
|
||||
}
|
||||
}
|
||||
throw new IllegalStateException("相似ASIN分片载荷更新失败");
|
||||
throw new IllegalStateException("相似ASIN分片载荷更新失败 " + conflictDetail);
|
||||
}
|
||||
|
||||
/** 读回 chunk 行当前的 payload_hash,用于 CAS 冲突定位(行已不存在/读取失败时返回可读标记)。 */
|
||||
private String currentPayloadHash(Long chunkId) {
|
||||
if (chunkId == null) {
|
||||
return "chunkId 为空";
|
||||
}
|
||||
try {
|
||||
TaskChunkEntity latest = taskChunkMapper.selectById(chunkId);
|
||||
return latest == null ? "行已不存在" : latest.getPayloadHash();
|
||||
} catch (Exception ex) {
|
||||
return "读取失败:" + ex.getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
+19
@@ -643,6 +643,25 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, LocalDateTime.now()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 该任务是否已有「重试耗尽且已走完终态回调」的组装 job。
|
||||
*
|
||||
* <p>用于卡死恢复(stale recovery)判断"再重建一次还有没有意义":数据永久缺失时,
|
||||
* 每轮重建只会再失败一次,而恢复过程又会刷新任务心跳,导致任务永远 RUNNING、
|
||||
* 恢复每 30 秒空转一轮(线上任务 28459 实测)。
|
||||
*/
|
||||
public boolean hasExhaustedAssembleJob(Long taskId, String moduleType) {
|
||||
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
return taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getTaskId, taskId)
|
||||
.eq(TaskFileJobEntity::getModuleType, moduleType)
|
||||
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
|
||||
.ge(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.isNotNull(TaskFileJobEntity::getTerminalCallbackAt)) > 0;
|
||||
}
|
||||
|
||||
public TaskFileJobEntity findAssembleJob(Long taskId, String moduleType, Long resultId) {
|
||||
return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
|
||||
}
|
||||
|
||||
+67
-5
@@ -30,6 +30,7 @@ import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
@@ -150,8 +151,20 @@ public class TransientPayloadStorageService {
|
||||
return decodeStoredPayloadBytes(readLocalPayloadBytes(stripLocalInstanceId(localKey)));
|
||||
}
|
||||
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
||||
return decodeStoredPayloadBytes(
|
||||
rustfsObjectStorageService.readObjectBytes(pointer.substring(RUSTFS_POINTER_PREFIX.length())));
|
||||
String objectKey = pointer.substring(RUSTFS_POINTER_PREFIX.length());
|
||||
try {
|
||||
return decodeStoredPayloadBytes(rustfsObjectStorageService.readObjectBytes(objectKey));
|
||||
} catch (RuntimeException readException) {
|
||||
String sibling = findVersionedChunkSibling(objectKey, readException);
|
||||
if (sibling == null) {
|
||||
throw readException;
|
||||
}
|
||||
// 2026-09-17 线上任务 28459:行指向的普通 key 被误删,但同一分片槽位的版本化对象还在。
|
||||
// 读出它即可让任务按已有数据出结果,不必整单失败。
|
||||
log.warn("[transient-payload] chunk 载荷对象不存在,回退同槽位版本化对象 pointer={} sibling={}",
|
||||
pointer, sibling);
|
||||
return decodeStoredPayloadBytes(rustfsObjectStorageService.readObjectBytes(sibling));
|
||||
}
|
||||
}
|
||||
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
|
||||
return decodeStoredPayloadBytes(
|
||||
@@ -164,6 +177,53 @@ public class TransientPayloadStorageService {
|
||||
return value;
|
||||
}
|
||||
|
||||
/** chunk 载荷槽位的 entryKey 形态:{@code chunk-<index>}(版本化写入则形如 {@code chunk-<index>-<uuid>})。 */
|
||||
private static final Pattern CHUNK_ENTRY_KEY_PATTERN = Pattern.compile("chunk-\\d+");
|
||||
|
||||
/** 兄弟对象查找上限:只为找回同槽位对象,不需要列全。 */
|
||||
private static final int MAX_SIBLING_LOOKUP_KEYS = 50;
|
||||
|
||||
/**
|
||||
* 指针对象已不存在时,尝试找回同一分片槽位的版本化兄弟对象。
|
||||
*
|
||||
* <p>两个条件同时满足才兜底,避免读到无关对象或掩盖真实故障:
|
||||
* <ol>
|
||||
* <li>末段 entryKey 是 {@code chunk-<index>} 形态——只有这种槽位才有「版本化兄弟」语义;</li>
|
||||
* <li>失败原因是对象确实不存在(NoSuchKey)——权限/网络类失败照旧上抛以便重试。</li>
|
||||
* </ol>
|
||||
*/
|
||||
private String findVersionedChunkSibling(String objectKey, Throwable cause) {
|
||||
if (!isObjectMissing(cause)) {
|
||||
return null;
|
||||
}
|
||||
int slash = objectKey.lastIndexOf('/');
|
||||
String directory = slash < 0 ? "" : objectKey.substring(0, slash + 1);
|
||||
String fileName = slash < 0 ? objectKey : objectKey.substring(slash + 1);
|
||||
if (!fileName.endsWith(".json")) {
|
||||
return null;
|
||||
}
|
||||
String entryKey = fileName.substring(0, fileName.length() - ".json".length());
|
||||
if (!CHUNK_ENTRY_KEY_PATTERN.matcher(entryKey).matches()) {
|
||||
return null;
|
||||
}
|
||||
List<String> candidates = rustfsObjectStorageService.listObjectKeysNewestFirst(
|
||||
directory + entryKey + "-", MAX_SIBLING_LOOKUP_KEYS);
|
||||
return candidates.isEmpty() ? null : candidates.getFirst();
|
||||
}
|
||||
|
||||
/** 对象确已不存在:RustFS 返回 NoSuchKey,message 为 "The specified key does not exist."。 */
|
||||
private static boolean isObjectMissing(Throwable error) {
|
||||
Throwable cursor = error;
|
||||
while (cursor != null) {
|
||||
String message = cursor.getMessage();
|
||||
if (message != null && message.contains("does not exist")) {
|
||||
return true;
|
||||
}
|
||||
cursor = cursor.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public void deletePayloadIfPresent(String value) {
|
||||
String pointer = extractPointer(value);
|
||||
if (pointer == null) {
|
||||
@@ -209,8 +269,10 @@ public class TransientPayloadStorageService {
|
||||
*
|
||||
* <p>判断口径:
|
||||
* <ul>
|
||||
* <li>{@code biz_task_chunk.payload_json} 命中 > 1 行(> 1 表示除了 caller 视角下
|
||||
* 自己即将释放的那一行之外,至少还有别的 chunk 行也指向同一对象)→ 视为仍被引用。</li>
|
||||
* <li>{@code biz_task_chunk.payload_json} 命中任意行(≥ 1)→ 视为仍被引用。
|
||||
* 曾用 {@code > 1} 作判据,等于放行「恰好还有 1 行引用」的情况,会把对方仍在用的对象
|
||||
* 删掉(2026-09-17 线上任务 28459:合并成功后指针未落库 + 旧对象被删 → 该分片永久读不到)。
|
||||
* 物理删除本就约定在 DB 行删除之后执行,故调用方正常路径下引用数必然为 0。</li>
|
||||
* <li>{@code biz_task_scope_state.parsed_payload_json}/{@code state_json} 命中 > 0 行 →
|
||||
* 视为仍被引用(这两个字段不是 caller 自身行的常见持有者,命中即非自我引用)。</li>
|
||||
* </ul>
|
||||
@@ -248,7 +310,7 @@ public class TransientPayloadStorageService {
|
||||
// 解析不出 taskId 时按原口径全局查,行为与改造前一致。
|
||||
Long pointerTaskId = extractTaskId(pointer);
|
||||
Long chunkCount = referencedChunkCount(pointerTaskId, values);
|
||||
if (chunkCount != null && chunkCount > 1L) {
|
||||
if (chunkCount != null && chunkCount > 0L) {
|
||||
return true;
|
||||
}
|
||||
Long scopeStateCount = referencedScopeStateCount(pointerTaskId, values);
|
||||
|
||||
+218
@@ -0,0 +1,218 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 外观专利分片载荷合并的 CAS 冲突处理(2026-09-17 线上任务 28459 的遗留项)。
|
||||
*
|
||||
* <p>原实现在每次 CAS 冲突后都删掉刚写入的版本化对象,重试耗尽即抛异常、行仍指向旧指针——
|
||||
* 一旦旧对象也不在,该分片就永久读不到(28459 的 chunk-462/473/484 正是这个形态)。
|
||||
* 现在:冲突时读回行上的当前哈希以便定位;**终局失败保留最后一个兜底对象**,
|
||||
* 让读路径的「同槽位兄弟对象」兜底仍有数据可取。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class AppearancePatentChunkMergeConflictTest {
|
||||
|
||||
private static final Long TASK_ID = 28459L;
|
||||
private static final String SCOPE_HASH = "2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e";
|
||||
private static final int CHUNK_INDEX = 462;
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private AppearancePatentLlmClient llmClient;
|
||||
@Mock private AppearancePatentTaskCacheService taskCacheService;
|
||||
@Mock private AppearancePatentProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private PlatformTransactionManager transactionManager;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
|
||||
@InjectMocks private AppearancePatentTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
// 共享写开启才会走版本化对象存储;否则落到本地兜底路径直接报「RustFS 未配置」
|
||||
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
|
||||
}
|
||||
|
||||
/** 首次冲突后重试成功:中间那次写的对象要删(会被下次重写),且最终行被改到新对象。 */
|
||||
@Test
|
||||
void retryAfterConflictDeletesSupersededObject() {
|
||||
stubChunk("hash-old", "ptr-chunk-462.json");
|
||||
AtomicInteger stores = new AtomicInteger();
|
||||
stubVersionedStore(stores);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(0, 1);
|
||||
when(taskChunkMapper.selectById(anyLong())).thenReturn(chunk("hash-other", "ptr-chunk-462.json"));
|
||||
|
||||
invokeMerge();
|
||||
|
||||
verify(transientPayloadStorageService, times(2)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
// 第 1 次冲突写的对象被删;第 2 次成功,走的是「替换旧对象」而不是删新对象
|
||||
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-1");
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent("sibling-2");
|
||||
verify(transientPayloadStorageService).deleteReplacedPayloadIfNeeded(eq("ptr-chunk-462.json"), eq("sibling-2"));
|
||||
}
|
||||
|
||||
/** 冲突重试耗尽:**保留**最后一次写入的兜底对象(本次要修的形态),并抛异常带出两个哈希。 */
|
||||
@Test
|
||||
void exhaustedConflictKeepsLastStoredObjectAsFallback() {
|
||||
stubChunk("hash-old", "ptr-chunk-462.json");
|
||||
AtomicInteger stores = new AtomicInteger();
|
||||
stubVersionedStore(stores);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(0);
|
||||
when(taskChunkMapper.selectById(anyLong())).thenReturn(chunk("hash-other", "ptr-chunk-462.json"));
|
||||
|
||||
IllegalStateException ex = assertThrows(IllegalStateException.class, this::invokeMerge);
|
||||
|
||||
verify(transientPayloadStorageService, times(3)).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
// 前两次冲突的对象照旧删除;第三次(终局)的对象必须保留
|
||||
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-1");
|
||||
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-2");
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent("sibling-3");
|
||||
assertTrue(ex.getMessage().contains("hash-old"), "异常需带出期望哈希,实际: " + ex.getMessage());
|
||||
assertTrue(ex.getMessage().contains("hash-other"), "异常需带出当前哈希,实际: " + ex.getMessage());
|
||||
}
|
||||
|
||||
/** 冲突时读回行上的当前哈希,供定位(此前只有一句 update conflict,线上无法定位)。 */
|
||||
@Test
|
||||
void conflictReadsBackCurrentHashForDiagnostics() {
|
||||
stubChunk("hash-old", "ptr-chunk-462.json");
|
||||
stubVersionedStore(new AtomicInteger());
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(0);
|
||||
when(taskChunkMapper.selectById(7L)).thenReturn(chunk("hash-changed-by-other-writer", "ptr-chunk-462.json"));
|
||||
|
||||
assertThrows(IllegalStateException.class, this::invokeMerge);
|
||||
|
||||
verify(taskChunkMapper, times(3)).selectById(7L);
|
||||
}
|
||||
|
||||
/** 读回行失败不得掩盖原始冲突:异常信息里给出可读标记。 */
|
||||
@Test
|
||||
void readBackFailureDoesNotMaskConflict() {
|
||||
stubChunk("hash-old", "ptr-chunk-462.json");
|
||||
stubVersionedStore(new AtomicInteger());
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(0);
|
||||
when(taskChunkMapper.selectById(anyLong())).thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
IllegalStateException ex = assertThrows(IllegalStateException.class, this::invokeMerge);
|
||||
|
||||
assertTrue(ex.getMessage().contains("读取失败"), "实际: " + ex.getMessage());
|
||||
}
|
||||
|
||||
// ===== 辅助 =====
|
||||
|
||||
private void invokeMerge() {
|
||||
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
|
||||
row.setRowToken("r1");
|
||||
row.setAsin("B0A0000001");
|
||||
ReflectionTestUtils.invokeMethod(service, "mergeChunkPayload", TASK_ID, SCOPE_HASH, CHUNK_INDEX, List.of(row));
|
||||
}
|
||||
|
||||
private void stubChunk(String payloadHash, String payloadPointer) {
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk(payloadHash, payloadPointer));
|
||||
lenient().when(transientPayloadStorageService.resolvePayload(eq(payloadPointer), anyString()))
|
||||
.thenReturn("[{\"rowToken\":\"r1\",\"asin\":\"B0A0000001\"}]");
|
||||
}
|
||||
|
||||
private void stubVersionedStore(AtomicInteger stores) {
|
||||
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||
anyString(), anyLong(), anyString(), any(), anyString()))
|
||||
.thenAnswer(inv -> "sibling-" + stores.incrementAndGet());
|
||||
}
|
||||
|
||||
private TaskChunkEntity chunk(String payloadHash, String payloadPointer) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(7L);
|
||||
chunk.setTaskId(TASK_ID);
|
||||
chunk.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash(SCOPE_HASH);
|
||||
chunk.setChunkIndex(CHUNK_INDEX);
|
||||
chunk.setPayloadJson(payloadPointer);
|
||||
chunk.setPayloadHash(payloadHash);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
/** 断言辅助:确认存的对象数(避免误用未使用的 import)。 */
|
||||
@Test
|
||||
void storeIsCalledOnceWhenUpdateSucceeds() {
|
||||
stubChunk("hash-old", "ptr-chunk-462.json");
|
||||
AtomicInteger stores = new AtomicInteger();
|
||||
stubVersionedStore(stores);
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
invokeMerge();
|
||||
|
||||
assertEquals(1, stores.get());
|
||||
verify(taskChunkMapper, never()).selectById(anyLong());
|
||||
}
|
||||
}
|
||||
+243
@@ -0,0 +1,243 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.TransactionDefinition;
|
||||
import org.springframework.transaction.TransactionStatus;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 外观专利「补传恢复」入口(2026-09-17 线上任务 28459 的遗留项)。
|
||||
*
|
||||
* <p>分片缺失导致组装 job 重试耗尽后,客户端补传缺口应能自动把该 job 重置重跑。
|
||||
* 该能力在 {@code TaskFileJobService} 里早就有了,但只有删除品牌模块接了入口,
|
||||
* 外观专利没有——28459 补齐分片后仍需人工重置 job 才能出结果。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class AppearancePatentTerminalFailedRecoveryTest {
|
||||
|
||||
private static final Long TASK_ID = 28459L;
|
||||
private static final Long RESULT_ID = 31490L;
|
||||
private static final String MODULE_TYPE = "APPEARANCE_PATENT";
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private AppearancePatentLlmClient llmClient;
|
||||
@Mock private AppearancePatentTaskCacheService taskCacheService;
|
||||
@Mock private AppearancePatentProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private PlatformTransactionManager transactionManager;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private TaskProgressLightAssembler taskProgressLightAssembler;
|
||||
@Mock private TransactionStatus transactionStatus;
|
||||
|
||||
@InjectMocks private AppearancePatentTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class))).thenReturn(transactionStatus);
|
||||
lenient().doAnswer(inv -> null).when(transactionManager).commit(transactionStatus);
|
||||
lenient().doAnswer(inv -> null).when(transactionManager).rollback(transactionStatus);
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(taskChunkMapper.selectCount(any())).thenReturn(0L);
|
||||
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(null);
|
||||
lenient().when(taskScopeStateMapper.selectOne(any())).thenReturn(null);
|
||||
lenient().when(fileTaskMapper.updateById(any(FileTaskEntity.class))).thenReturn(1);
|
||||
lenient().when(transientPayloadStorageService.isSharedWriteEnabled()).thenReturn(true);
|
||||
lenient().when(transientPayloadStorageService.storeChunkPayload(
|
||||
eq(MODULE_TYPE), eq(TASK_ID), anyString(), any(), anyString()))
|
||||
.thenReturn("\"rustfs:task-chunk/appearance_patent/28459/hash/chunk-11.json\"");
|
||||
}
|
||||
|
||||
// ===== 直接覆盖恢复判定 =====
|
||||
|
||||
/** 已有成功的组装 job → 不恢复。 */
|
||||
@Test
|
||||
void successfulAssembleJobSkipsRecovery() {
|
||||
stubResultRow();
|
||||
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(true);
|
||||
|
||||
invokeRecovery();
|
||||
|
||||
verify(taskFileJobService, never()).resetTerminalFailedForRecovery(anyLong(), anyString(), anyLong());
|
||||
}
|
||||
|
||||
/** 组装 job 不是「重试耗尽的终态失败」→ 不恢复。 */
|
||||
@Test
|
||||
void nonTerminalFailedJobSkipsRecovery() {
|
||||
stubResultRow();
|
||||
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(false);
|
||||
when(taskFileJobService.isTerminalFailedAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(false);
|
||||
|
||||
invokeRecovery();
|
||||
|
||||
verify(taskFileJobService, never()).resetTerminalFailedForRecovery(anyLong(), anyString(), anyLong());
|
||||
}
|
||||
|
||||
/** 终态失败 + 分片已补传完整 → 重置 job 重新派发(本次要修的场景)。 */
|
||||
@Test
|
||||
void terminalFailedJobWithCompleteChunksIsReset() {
|
||||
stubResultRow();
|
||||
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(false);
|
||||
when(taskFileJobService.isTerminalFailedAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(true);
|
||||
when(taskScopeStateMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
invokeRecovery();
|
||||
|
||||
verify(taskFileJobService).resetTerminalFailedForRecovery(TASK_ID, MODULE_TYPE, RESULT_ID);
|
||||
}
|
||||
|
||||
/** 终态失败但分片尚未补齐 → 不恢复(否则又会读到缺失分片再失败一次)。 */
|
||||
@Test
|
||||
void terminalFailedJobWithIncompleteChunksIsNotReset() {
|
||||
stubResultRow();
|
||||
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(false);
|
||||
when(taskFileJobService.isTerminalFailedAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(true);
|
||||
when(taskScopeStateMapper.selectCount(any())).thenReturn(0L);
|
||||
|
||||
invokeRecovery();
|
||||
|
||||
verify(taskFileJobService, never()).resetTerminalFailedForRecovery(anyLong(), anyString(), anyLong());
|
||||
}
|
||||
|
||||
/** 还没有结果行 → 不恢复。 */
|
||||
@Test
|
||||
void missingResultRowSkipsRecovery() {
|
||||
when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
|
||||
invokeRecovery();
|
||||
|
||||
verify(taskFileJobService, never()).isTerminalFailedAssembleJob(anyLong(), anyString(), anyLong());
|
||||
}
|
||||
|
||||
/** taskId 非法 → 直接返回,不查库。 */
|
||||
@Test
|
||||
void invalidTaskIdSkipsLookup() {
|
||||
ReflectionTestUtils.invokeMethod(service, "maybeRecoverTerminalFailedAssemble", 0L);
|
||||
ReflectionTestUtils.invokeMethod(service, "maybeRecoverTerminalFailedAssemble", (Object) null);
|
||||
|
||||
verify(fileResultMapper, never()).selectList(any());
|
||||
}
|
||||
|
||||
/** 恢复检查自身抛异常 → 吞掉,不影响补传结果(best-effort)。 */
|
||||
@Test
|
||||
void recoveryFailureDoesNotPropagate() {
|
||||
when(fileResultMapper.selectList(any())).thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
assertDoesNotThrow(this::invokeRecovery);
|
||||
}
|
||||
|
||||
// ===== 走完整提交路径 =====
|
||||
|
||||
/** 分片提交成功后触发恢复检查(接线正确)。 */
|
||||
@Test
|
||||
void submitResultTriggersRecoveryCheck() {
|
||||
stubRunningTask();
|
||||
stubResultRow();
|
||||
when(taskFileJobService.hasSuccessfulAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(false);
|
||||
when(taskFileJobService.isTerminalFailedAssembleJob(TASK_ID, MODULE_TYPE, RESULT_ID)).thenReturn(true);
|
||||
when(taskScopeStateMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
service.submitResult(TASK_ID, request());
|
||||
|
||||
verify(taskFileJobService).resetTerminalFailedForRecovery(TASK_ID, MODULE_TYPE, RESULT_ID);
|
||||
}
|
||||
|
||||
// ===== 辅助 =====
|
||||
|
||||
private void invokeRecovery() {
|
||||
ReflectionTestUtils.invokeMethod(service, "maybeRecoverTerminalFailedAssemble", TASK_ID);
|
||||
}
|
||||
|
||||
private void stubResultRow() {
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(RESULT_ID);
|
||||
result.setTaskId(TASK_ID);
|
||||
result.setModuleType(MODULE_TYPE);
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of(result));
|
||||
}
|
||||
|
||||
private void stubRunningTask() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType(MODULE_TYPE);
|
||||
task.setStatus("RUNNING");
|
||||
task.setUserId(1121L);
|
||||
task.setResultJson("{\"parsedPayloadRef\":\"rustfs:task-parsed/x.json\",\"ownerInstanceId\":\"instance-a\"}");
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
}
|
||||
|
||||
private AppearancePatentSubmitResultRequest request() {
|
||||
AppearancePatentSubmitResultRequest request = new AppearancePatentSubmitResultRequest();
|
||||
request.setSubmissionId("appearance-patent-" + TASK_ID);
|
||||
request.setChunkIndex(11);
|
||||
request.setChunkTotal(500);
|
||||
request.setDone(false);
|
||||
return request;
|
||||
}
|
||||
}
|
||||
+12
-6
@@ -297,16 +297,19 @@ class RustfsObjectStorageServiceTest {
|
||||
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||
properties, emptyProvider(), provider(retryService), () -> client);
|
||||
|
||||
// 补偿删除只对版本化 key(本次写入独占)生效,故这里用 versioned key 覆盖该路径
|
||||
String firstKey = "task/a-11111111-2222-3333-4444-555555555555.json";
|
||||
String secondKey = "task/b-66666666-7777-8888-9999-000000000000.json";
|
||||
IllegalStateException firstFailure = assertThrows(IllegalStateException.class,
|
||||
() -> service.uploadText("task/a.json", "{}", true));
|
||||
() -> service.uploadText(firstKey, "{}", true));
|
||||
IllegalStateException secondFailure = assertThrows(IllegalStateException.class,
|
||||
() -> service.uploadText("task/b.json", "{}", true));
|
||||
() -> service.uploadText(secondKey, "{}", true));
|
||||
|
||||
assertTrue(firstFailure.getMessage().contains("not visible"));
|
||||
assertTrue(secondFailure.getMessage().contains("not visible"));
|
||||
verify(client, times(5)).statObject(any(StatObjectArgs.class));
|
||||
verify(retryService).enqueue(eq("task/a.json"), same(firstFailure));
|
||||
verify(retryService).enqueue(eq("task/b.json"), same(secondFailure));
|
||||
verify(retryService).enqueue(eq(firstKey), same(firstFailure));
|
||||
verify(retryService).enqueue(eq(secondKey), same(secondFailure));
|
||||
IllegalStateException rejected = assertThrows(IllegalStateException.class,
|
||||
() -> service.uploadText("task/c.json", "{}", false));
|
||||
assertTrue(rejected.getMessage().contains("cooldown active"));
|
||||
@@ -360,6 +363,9 @@ class RustfsObjectStorageServiceTest {
|
||||
|
||||
@Test
|
||||
void completedUploadIsEnqueuedWhenDeadlineExpiresAfterPut() throws Exception {
|
||||
// 只有版本化 key(本次写入独占)才允许补偿删除;共享 key 的守卫见
|
||||
// RustfsUploadCompensationGuardTest#deterministicKeyDerivedFromShapeDoesNotEnqueueCompensation
|
||||
String versionedKey = "task/chunk-1-0ffc254b-afad-4279-aab2-e85e3ff955e9.json";
|
||||
TransientStorageProperties properties = configuredProperties();
|
||||
properties.setOperationTimeoutSeconds(1);
|
||||
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||
@@ -372,11 +378,11 @@ class RustfsObjectStorageServiceTest {
|
||||
properties, emptyProvider(), provider(retryService), () -> client);
|
||||
|
||||
IllegalStateException failure = assertThrows(IllegalStateException.class,
|
||||
() -> service.uploadText("task/a.json", "{}", false));
|
||||
() -> service.uploadText(versionedKey, "{}", false));
|
||||
|
||||
assertTrue(failure.getMessage().contains("operation timeout"));
|
||||
verify(client).putObject(any(PutObjectArgs.class));
|
||||
verify(retryService).enqueue(eq("task/a.json"), same(failure));
|
||||
verify(retryService).enqueue(eq(versionedKey), same(failure));
|
||||
}
|
||||
|
||||
private static TransientStorageProperties configuredProperties() {
|
||||
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
package com.nanri.aiimage.modules.file.service.object;
|
||||
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import io.minio.MinioClient;
|
||||
import io.minio.PutObjectArgs;
|
||||
import io.minio.StatObjectArgs;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 上传失败补偿删除的准入守卫(2026-09-17 线上任务 28459 事故)。
|
||||
*
|
||||
* <p>事故形态:客户端重传复用**确定性 key**({@code chunk-462.json})。第一次上传 put 已成功、
|
||||
* 但随后的 {@code verifyObjectVisible} 抖动 → 旧代码无条件
|
||||
* {@code enqueueDeleteRetry(objectKey)};而删除重试队列对该 key 不再做任何引用校验
|
||||
* ({@code RustfsDeleteRetryService} → {@code deleteObjectFromRetry} → {@code removeObject}),
|
||||
* 于是把重传成功后 DB 行仍指向的对象删掉,下游读 chunk 直接 404,整个任务组装失败。
|
||||
*
|
||||
* <p>口径:只有**本次写入独占的对象**(版本化 key,末段以 UUID 结尾)才允许补偿删除;
|
||||
* 确定性 key 会被重写复用,删它就可能删掉别的行正在引用的对象。
|
||||
*/
|
||||
class RustfsUploadCompensationGuardTest {
|
||||
|
||||
// ===== 正常路径 =====
|
||||
|
||||
/** 唯一 key(版本化)+ 验证失败 + 显式允许 → 补偿删除入队。 */
|
||||
@Test
|
||||
void uniqueKeyVerifyFailureEnqueuesCompensation() throws Exception {
|
||||
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
|
||||
|
||||
assertThrows(IllegalStateException.class,
|
||||
() -> service.uploadBytes("task/chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json",
|
||||
"{}".getBytes(), true, true));
|
||||
|
||||
verify(retryService).enqueue(eq("task/chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json"),
|
||||
any(Throwable.class));
|
||||
}
|
||||
|
||||
/** 三参重载按 key 形态判定:版本化 key 自动允许补偿删除。 */
|
||||
@Test
|
||||
void versionedKeyDerivedFromShapeEnqueuesCompensation() throws Exception {
|
||||
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
|
||||
|
||||
assertThrows(IllegalStateException.class,
|
||||
() -> service.uploadBytes("task/chunk-77-d03bf3b0-abbb-469d-9bb8-a7c01c309477.json",
|
||||
"{}".getBytes(), true));
|
||||
|
||||
verify(retryService).enqueue(anyString(), any(Throwable.class));
|
||||
}
|
||||
|
||||
// ===== 事故根因:共享 key 必须不删 =====
|
||||
|
||||
/** 确定性(共享)key + 显式禁止 → 不入队(异常仍照旧抛出,失败归属不变)。 */
|
||||
@Test
|
||||
void sharedKeyVerifyFailureDoesNotEnqueueCompensation() throws Exception {
|
||||
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
|
||||
|
||||
assertThrows(IllegalStateException.class,
|
||||
() -> service.uploadBytes("task/chunk-462.json", "{}".getBytes(), true, false));
|
||||
|
||||
verify(retryService, never()).enqueue(anyString(), any(Throwable.class));
|
||||
}
|
||||
|
||||
/** 三参重载按 key 形态判定:确定性 key 自动禁止补偿删除(本次事故的直接修复点)。 */
|
||||
@Test
|
||||
void deterministicKeyDerivedFromShapeDoesNotEnqueueCompensation() throws Exception {
|
||||
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
|
||||
|
||||
assertThrows(IllegalStateException.class,
|
||||
() -> service.uploadBytes("task/chunk-462.json", "{}".getBytes(), true));
|
||||
|
||||
verify(retryService, never()).enqueue(anyString(), any(Throwable.class));
|
||||
}
|
||||
|
||||
/** verify=false 上传即成功,不得产生任何补偿删除。 */
|
||||
@Test
|
||||
void verifyDisabledNeverEnqueuesCompensation() throws Exception {
|
||||
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
|
||||
|
||||
String key = service.uploadBytes("task/chunk-462-uuid-x.json", "{}".getBytes(), false, true);
|
||||
|
||||
assertEquals("task/chunk-462-uuid-x.json", key);
|
||||
verify(retryService, never()).enqueue(anyString(), any(Throwable.class));
|
||||
}
|
||||
|
||||
// ===== 异常路径 =====
|
||||
|
||||
/** put 从未成功过:对象本就不存在,不该产生删除动作(与开关无关)。 */
|
||||
@Test
|
||||
void putNeverCompletedDoesNotEnqueueCompensation() throws Exception {
|
||||
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||
MinioClient client = mock(MinioClient.class);
|
||||
when(client.putObject(any(PutObjectArgs.class))).thenThrow(new IOException("put failed"));
|
||||
TransientStorageProperties properties = configuredProperties();
|
||||
properties.setUploadMaxRetries(1);
|
||||
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||
properties, emptyProvider(), provider(retryService), () -> client);
|
||||
|
||||
assertThrows(RuntimeException.class,
|
||||
() -> service.uploadBytes("task/chunk-462-uuid-x.json", "{}".getBytes(), true, true));
|
||||
|
||||
verify(retryService, never()).enqueue(anyString(), any(Throwable.class));
|
||||
}
|
||||
|
||||
/** 补偿删除入队本身失败:不得把上传失败改写成入队异常,原异常必须继续上抛。 */
|
||||
@Test
|
||||
void enqueueFailureDoesNotMaskUploadFailure() throws Exception {
|
||||
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
|
||||
org.mockito.Mockito.doThrow(new IllegalStateException("queue down"))
|
||||
.when(retryService).enqueue(anyString(), any(Throwable.class));
|
||||
RustfsObjectStorageService service = new RustfsObjectStorageService(
|
||||
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
|
||||
|
||||
IllegalStateException ex = assertThrows(IllegalStateException.class,
|
||||
() -> service.uploadBytes("task/chunk-9-11111111-2222-3333-4444-555555555555.json",
|
||||
"{}".getBytes(), true, true));
|
||||
|
||||
assertFalse(ex.getMessage() == null || ex.getMessage().contains("queue down"),
|
||||
"入队异常不应成为对外错误;实际=" + ex.getMessage());
|
||||
}
|
||||
|
||||
// ===== key 形态判定的边界 =====
|
||||
|
||||
@Test
|
||||
void versionedKeyShapeIsRecognized() {
|
||||
assertTrue(RustfsObjectStorageService.isVersionedObjectKey(
|
||||
"task-chunk/appearance_patent/28459/hash/chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json"));
|
||||
assertTrue(RustfsObjectStorageService.isVersionedObjectKey(
|
||||
"task-chunk/x/1/hash/0ffc254b-afad-4279-aab2-e85e3ff955e9.json"),
|
||||
"整个 entryKey 就是 UUID 时也应识别为唯一 key");
|
||||
assertTrue(RustfsObjectStorageService.isVersionedObjectKey(
|
||||
"task-chunk/x/1/hash/chunk-1-0ffc254b-afad-4279-aab2-e85e3ff955e9"),
|
||||
"无 .json 后缀但末段是 UUID,同样视为唯一");
|
||||
}
|
||||
|
||||
@Test
|
||||
void deterministicKeyShapeIsNotRecognized() {
|
||||
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("task-chunk/x/1/hash/chunk-462.json"));
|
||||
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("task-parsed/x/1/hash/latest.json"));
|
||||
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("task-result-payload/x/1/hash/28459.json"),
|
||||
"提交号这类 key 会被重写复用,不能当成唯一 key");
|
||||
assertFalse(RustfsObjectStorageService.isVersionedObjectKey(
|
||||
"task-chunk/x/1/0ffc254b-afad-4279-aab2-e85e3ff955e9/chunk-462.json"),
|
||||
"UUID 出现在中间段不算唯一——重写复用的判据只看末段");
|
||||
}
|
||||
|
||||
@Test
|
||||
void malformedKeyShapeDoesNotThrowAndIsTreatedAsShared() {
|
||||
assertFalse(RustfsObjectStorageService.isVersionedObjectKey(null));
|
||||
assertFalse(RustfsObjectStorageService.isVersionedObjectKey(""));
|
||||
assertFalse(RustfsObjectStorageService.isVersionedObjectKey(" "));
|
||||
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("chunk-462.json"));
|
||||
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("-afad-4279-aab2-e85e3ff955e9.json".substring(0, 20)));
|
||||
}
|
||||
|
||||
// ===== 辅助 =====
|
||||
|
||||
private static MinioClient putOkButStatFails() {
|
||||
MinioClient client = mock(MinioClient.class);
|
||||
try {
|
||||
when(client.putObject(any(PutObjectArgs.class))).thenReturn(null);
|
||||
when(client.statObject(any(StatObjectArgs.class))).thenThrow(new IOException("stat 抖动"));
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException(ex);
|
||||
}
|
||||
return client;
|
||||
}
|
||||
|
||||
private static TransientStorageProperties configuredProperties() {
|
||||
TransientStorageProperties properties = new TransientStorageProperties();
|
||||
properties.setEndpoint("http://127.0.0.1:9000");
|
||||
properties.setBucket("bucket");
|
||||
properties.setAccessKeyId("ak");
|
||||
properties.setAccessKeySecret("sk");
|
||||
properties.setBaseRetryDelayMillis(0);
|
||||
properties.setRetryJitterMillis(0);
|
||||
return properties;
|
||||
}
|
||||
|
||||
private static <T> ObjectProvider<T> provider(T value) {
|
||||
@SuppressWarnings("unchecked")
|
||||
ObjectProvider<T> provider = mock(ObjectProvider.class);
|
||||
when(provider.getIfAvailable()).thenReturn(value);
|
||||
return provider;
|
||||
}
|
||||
|
||||
private static <T> ObjectProvider<T> emptyProvider() {
|
||||
@SuppressWarnings("unchecked")
|
||||
ObjectProvider<T> provider = mock(ObjectProvider.class);
|
||||
when(provider.getIfAvailable()).thenReturn(null);
|
||||
return provider;
|
||||
}
|
||||
}
|
||||
+67
@@ -751,6 +751,73 @@ class PublishTaskServiceTest {
|
||||
verify(lock).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void activateFileRejectsShopAlreadyRunningInAnotherTask() {
|
||||
long taskId = 106L;
|
||||
long fileId = 206L;
|
||||
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||
PublishFileEntity target = file(taskId, fileId, "PENDING", "林洪武.xlsx");
|
||||
target.setShopName("林洪武");
|
||||
// 另一个任务(28520)正跑同一店铺 —— 2026-09-17 事故形态
|
||||
PublishFileEntity otherTaskRunning = file(105L, 205L, "RUNNING", "林洪武.xlsx");
|
||||
otherTaskRunning.setShopName("林洪武");
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
|
||||
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
|
||||
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
||||
when(publishFileMapper.selectById(fileId)).thenReturn(target);
|
||||
// 第一次 selectOne:同任务其它 RUNNING 文件(无);第二次:同店铺跨任务(有)
|
||||
when(publishFileMapper.selectOne(any())).thenReturn(null, otherTaskRunning);
|
||||
|
||||
BusinessException error = assertThrows(BusinessException.class,
|
||||
() -> service.activateFile(taskId, fileId, 7L));
|
||||
|
||||
assertTrue(error.getMessage().contains("林洪武"), "提示要带店铺名: " + error.getMessage());
|
||||
assertTrue(error.getMessage().contains("105"), "提示要带占用中的任务号: " + error.getMessage());
|
||||
verify(publishFileMapper, never()).update(isNull(), any());
|
||||
verify(lock).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void activateFileAllowsWhenShopHasNoOtherRunningTask() {
|
||||
long taskId = 107L;
|
||||
long fileId = 207L;
|
||||
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||
PublishFileEntity target = file(taskId, fileId, "PENDING", "林洪武.xlsx");
|
||||
target.setShopName("林洪武");
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
|
||||
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
|
||||
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
||||
when(publishFileMapper.selectById(fileId)).thenReturn(target);
|
||||
when(publishFileMapper.selectOne(any())).thenReturn(null);
|
||||
when(publishFileMapper.update(isNull(), any())).thenReturn(1);
|
||||
|
||||
service.activateFile(taskId, fileId, 7L);
|
||||
|
||||
verify(publishFileMapper).update(isNull(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void activateFileSkipsShopCheckWhenShopNameBlank() {
|
||||
long taskId = 108L;
|
||||
long fileId = 208L;
|
||||
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||
// 未匹配到店铺的文件:没有店铺标识就不做店铺维度校验(不能因此阻断激活)
|
||||
PublishFileEntity target = file(taskId, fileId, "PENDING", "未匹配.xlsx");
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
|
||||
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
|
||||
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
||||
when(publishFileMapper.selectById(fileId)).thenReturn(target);
|
||||
when(publishFileMapper.selectOne(any())).thenReturn(null);
|
||||
when(publishFileMapper.update(isNull(), any())).thenReturn(1);
|
||||
|
||||
service.activateFile(taskId, fileId, 7L);
|
||||
|
||||
verify(publishFileMapper, times(1)).selectOne(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void activateFileIsIdempotentForTheSameRunningFile() {
|
||||
long taskId = 105L;
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -137,6 +138,7 @@ class ShopDataCrawlChunkUpsertTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -16,6 +16,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResu
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -174,6 +175,7 @@ class ShopDataCrawlCleanupTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -146,6 +147,7 @@ class ShopDataCrawlDailyFileIncrementalTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -152,6 +153,7 @@ class ShopDataCrawlDailyFileJobSplitTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -152,6 +153,7 @@ class ShopDataCrawlDailyFileLockTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -134,6 +135,7 @@ class ShopDataCrawlLightweightProgressTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -139,6 +140,7 @@ class ShopDataCrawlOwnerColumnTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -146,6 +147,7 @@ class ShopDataCrawlProgressQueryTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -12,6 +12,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -133,6 +134,7 @@ class ShopDataCrawlRowDedupKeyTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -134,6 +135,7 @@ class ShopDataCrawlScopeCounterTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -135,6 +136,7 @@ class ShopDataCrawlScopeMergeTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+2
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
@@ -129,6 +130,7 @@ class ShopDataCrawlTaskServiceChunkTest {
|
||||
instanceMetadata,
|
||||
dailyFileService,
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class));
|
||||
|
||||
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
package com.nanri.aiimage.modules.shopduplicatecheck.service.support;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.BooleanSupplier;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
/**
|
||||
* 撞款重扫调度器:合并窗口去抖(突发触发聚为一次扫描)、单飞、
|
||||
* 锁忙有限重试、动作异常吸收(对外零抛出)。
|
||||
*/
|
||||
class DuplicateCheckRefreshSchedulerTest {
|
||||
|
||||
private static final long AWAIT_TIMEOUT_MILLIS = 5_000L;
|
||||
|
||||
@Test
|
||||
void request_mergesBurstIntoSingleScan() throws Exception {
|
||||
AtomicInteger scans = new AtomicInteger();
|
||||
CountDownLatch firstScan = new CountDownLatch(1);
|
||||
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||
scans.incrementAndGet();
|
||||
firstScan.countDown();
|
||||
return DuplicateCheckRefreshScheduler.Outcome.DONE;
|
||||
}, 150L, 30L, 3);
|
||||
|
||||
scheduler.request("burst-1");
|
||||
Thread.sleep(10L);
|
||||
scheduler.request("burst-2");
|
||||
Thread.sleep(10L);
|
||||
scheduler.request("burst-3");
|
||||
|
||||
assertTrue(firstScan.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), "合并窗口内的触发应执行扫描");
|
||||
Thread.sleep(300L);
|
||||
assertEquals(1, scans.get(), "合并窗口内多次触发只执行一次扫描");
|
||||
}
|
||||
|
||||
@Test
|
||||
void request_retriesWhileLockBusyThenSucceeds() throws Exception {
|
||||
AtomicInteger attempts = new AtomicInteger();
|
||||
CountDownLatch succeeded = new CountDownLatch(1);
|
||||
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||
if (attempts.incrementAndGet() <= 2) {
|
||||
return DuplicateCheckRefreshScheduler.Outcome.LOCK_BUSY;
|
||||
}
|
||||
succeeded.countDown();
|
||||
return DuplicateCheckRefreshScheduler.Outcome.DONE;
|
||||
}, 10L, 30L, 5);
|
||||
|
||||
scheduler.request("retry");
|
||||
|
||||
assertTrue(succeeded.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), "锁忙重试后应成功执行");
|
||||
Thread.sleep(100L);
|
||||
assertEquals(3, attempts.get(), "2 次锁忙 + 1 次成功");
|
||||
}
|
||||
|
||||
@Test
|
||||
void request_givesUpAfterMaxLockRetries() throws Exception {
|
||||
AtomicInteger attempts = new AtomicInteger();
|
||||
CountDownLatch firstAttempt = new CountDownLatch(1);
|
||||
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||
attempts.incrementAndGet();
|
||||
firstAttempt.countDown();
|
||||
return DuplicateCheckRefreshScheduler.Outcome.LOCK_BUSY;
|
||||
}, 10L, 20L, 2);
|
||||
|
||||
scheduler.request("always-busy");
|
||||
|
||||
assertTrue(firstAttempt.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS));
|
||||
awaitUntil(() -> attempts.get() >= 3, "应完成初试 + 2 次重试");
|
||||
Thread.sleep(200L);
|
||||
assertEquals(3, attempts.get(), "超过重试上限后放弃,不再执行");
|
||||
}
|
||||
|
||||
@Test
|
||||
void request_absorbsActionFailure() throws Exception {
|
||||
AtomicInteger attempts = new AtomicInteger();
|
||||
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||
attempts.incrementAndGet();
|
||||
throw new IllegalStateException("模拟扫描动作异常");
|
||||
}, 10L, 20L, 1);
|
||||
|
||||
assertDoesNotThrow(() -> scheduler.request("boom"), "request 不得向调用方抛错");
|
||||
awaitUntil(() -> attempts.get() >= 1, "动作应被执行");
|
||||
Thread.sleep(100L);
|
||||
assertEquals(1, attempts.get(), "动作异常视为失败,不做锁忙重试");
|
||||
}
|
||||
|
||||
@Test
|
||||
void request_afterPreviousCycleAllowsNewScan() throws Exception {
|
||||
AtomicInteger scans = new AtomicInteger();
|
||||
CountDownLatch twoScans = new CountDownLatch(2);
|
||||
DuplicateCheckRefreshScheduler scheduler = new DuplicateCheckRefreshScheduler(() -> {
|
||||
scans.incrementAndGet();
|
||||
twoScans.countDown();
|
||||
return DuplicateCheckRefreshScheduler.Outcome.DONE;
|
||||
}, 20L, 20L, 2);
|
||||
|
||||
scheduler.request("first");
|
||||
awaitUntil(() -> scans.get() >= 1, "首轮扫描应执行");
|
||||
scheduler.request("second");
|
||||
|
||||
assertTrue(twoScans.await(AWAIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS), "新触发应再执行一次扫描");
|
||||
assertEquals(2, scans.get());
|
||||
}
|
||||
|
||||
private static void awaitUntil(BooleanSupplier condition, String message) throws InterruptedException {
|
||||
long deadline = System.currentTimeMillis() + AWAIT_TIMEOUT_MILLIS;
|
||||
while (System.currentTimeMillis() < deadline) {
|
||||
if (condition.getAsBoolean()) {
|
||||
return;
|
||||
}
|
||||
Thread.sleep(10L);
|
||||
}
|
||||
fail(message);
|
||||
}
|
||||
}
|
||||
+97
@@ -33,6 +33,7 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
@@ -42,10 +43,12 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -397,4 +400,98 @@ class SimilarAsinTaskServiceChunkMergeLimitTest {
|
||||
verify(taskChunkMapper, times(1)).update(any(), any());
|
||||
verify(taskScopeStateMapper, times(0)).insert(any(TaskScopeStateEntity.class));
|
||||
}
|
||||
|
||||
// ===== CAS 冲突处理(2026-09-17 线上任务 28459 遗留项,与外观专利同一口径) =====
|
||||
|
||||
/** 冲突重试耗尽:**保留**最后一次写入的对象作读兜底,异常带出期望/当前哈希。 */
|
||||
@Test
|
||||
void casConflictExhaustedKeepsLastStoredObjectAsFallback() throws Exception {
|
||||
TaskChunkEntity chunk = chunk(7L, "hashA", 1, "ptr:chunk-A");
|
||||
chunk.setPayloadHash("hashA");
|
||||
stubConflictMerge(chunk, 0);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
Throwable ex = invokeMergeExpectingFailure(task, List.of(row("r1", "B0A0000001", "标题1")));
|
||||
|
||||
assertTrue(ex instanceof IllegalStateException, "实际: " + ex);
|
||||
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-1");
|
||||
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-2");
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent("sibling-3");
|
||||
assertTrue(ex.getMessage().contains("hashA"), "需带出期望哈希,实际: " + ex.getMessage());
|
||||
assertTrue(ex.getMessage().contains("hashB"), "需带出当前哈希,实际: " + ex.getMessage());
|
||||
}
|
||||
|
||||
/** 冲突后重试成功:被顶替的中间对象删除,最终对象写入行(不删)。 */
|
||||
@Test
|
||||
void casConflictThenSuccessDeletesSupersededObjects() throws Exception {
|
||||
TaskChunkEntity chunk = chunk(7L, "hashA", 1, "ptr:chunk-A");
|
||||
chunk.setPayloadHash("hashA");
|
||||
stubConflictMerge(chunk, 0, 0, 1);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
invokeMerge(service, task, "hashA", 1, List.of(row("r1", "B0A0000001", "标题1")));
|
||||
|
||||
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-1");
|
||||
verify(transientPayloadStorageService).deletePayloadIfPresent("sibling-2");
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent("sibling-3");
|
||||
verify(transientPayloadStorageService).deleteReplacedPayloadIfNeeded(eq("ptr:chunk-A"), eq("sibling-3"));
|
||||
}
|
||||
|
||||
/** 冲突时读回行上的当前哈希供定位(此前只有一句 conflict,线上无法定位)。 */
|
||||
@Test
|
||||
void casConflictReadsBackCurrentHashForDiagnostics() throws Exception {
|
||||
TaskChunkEntity chunk = chunk(7L, "hashA", 1, "ptr:chunk-A");
|
||||
chunk.setPayloadHash("hashA");
|
||||
stubConflictMerge(chunk, 0);
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
invokeMergeExpectingFailure(task, List.of(row("r1", "B0A0000001", "标题1")));
|
||||
|
||||
verify(taskChunkMapper, times(3)).selectById(7L);
|
||||
}
|
||||
|
||||
/** 读回行失败不得掩盖原始冲突:异常信息里给出可读标记。 */
|
||||
@Test
|
||||
void casConflictReadBackFailureDoesNotMaskConflict() throws Exception {
|
||||
TaskChunkEntity chunk = chunk(7L, "hashA", 1, "ptr:chunk-A");
|
||||
chunk.setPayloadHash("hashA");
|
||||
stubConflictMerge(chunk, 0);
|
||||
when(taskChunkMapper.selectById(anyLong())).thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(9004L);
|
||||
Throwable ex = invokeMergeExpectingFailure(task, List.of(row("r1", "B0A0000001", "标题1")));
|
||||
|
||||
assertTrue(ex.getMessage().contains("读取失败"), "实际: " + ex.getMessage());
|
||||
}
|
||||
|
||||
/** 反射调用会包一层 InvocationTargetException,取根因以便断言业务异常。 */
|
||||
private Throwable invokeMergeExpectingFailure(FileTaskEntity task, List<SimilarAsinResultRowDto> rows) throws Exception {
|
||||
try {
|
||||
invokeMerge(service, task, "hashA", 1, rows);
|
||||
return null;
|
||||
} catch (java.lang.reflect.InvocationTargetException ex) {
|
||||
return ex.getCause();
|
||||
}
|
||||
}
|
||||
|
||||
/** 冲突场景公共 stub:每次尝试写一个不同的版本化对象,update 按传入序列返回。 */
|
||||
private void stubConflictMerge(TaskChunkEntity chunk, Integer... updateResults) throws Exception {
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
when(transientPayloadStorageService.resolvePayload(eq("ptr:chunk-A"), anyString()))
|
||||
.thenReturn(rowsJson(List.of(row("r0", "B0A0000000", "存量行"))));
|
||||
AtomicInteger stores = new AtomicInteger();
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||
anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(inv -> "sibling-" + stores.incrementAndGet());
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(updateResults[0], java.util.Arrays.copyOfRange(updateResults, 1, updateResults.length));
|
||||
|
||||
TaskChunkEntity current = chunk(7L, "hashB", 1, "ptr:chunk-A");
|
||||
current.setPayloadHash("hashB");
|
||||
lenient().when(taskChunkMapper.selectById(7L)).thenReturn(current);
|
||||
}
|
||||
}
|
||||
|
||||
+9
-3
@@ -33,8 +33,13 @@ import static org.mockito.Mockito.when;
|
||||
/**
|
||||
* task-150:清理前引用检查契约(plan 09)。
|
||||
* payload 被 biz_task_chunk / biz_task_scope_state 引用则不清理;
|
||||
* chunk 仅剩自身一行(count=1)不算共享引用;查询异常保守保留;
|
||||
* chunk 命中任意行(含仅剩 1 行)都不算「可安全清理」;查询异常保守保留;
|
||||
* 检查只读无副作用;候选值批量反查。
|
||||
*
|
||||
* <p>2026-09-17 线上任务 28459 之后收紧:原契约把 count==1 当作「调用方自己那行」放行删除,
|
||||
* 但调用方无法证明那一行就是自己(如合并已把自己那行指向新对象、另一行仍指向旧对象时 count 恰为 1),
|
||||
* 删除即造成该分片永久读不到 → 整单失败。物理删除本就约定在 DB 行删除之后执行,正常路径引用数必然为 0;
|
||||
* 宁可留下孤儿对象(有保留期清理兜底),也不删掉可能仍被引用的对象。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class TaskPayloadReferenceCheckTest {
|
||||
@@ -100,13 +105,14 @@ class TaskPayloadReferenceCheckTest {
|
||||
verify(rustfsObjectStorageService, never()).deleteObject(any());
|
||||
}
|
||||
|
||||
/** count==1 无法证明那一行就是调用方自己(可能正是别的行仍在用)→ 保守不删。 */
|
||||
@Test
|
||||
void singleChunkRowIsOwnRowNotSharedReference() {
|
||||
void singleChunkRowBlocksDeleteBecauseOwnershipCannotBeProven() {
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
service.deletePayloadIfPresent(RUSTFS_VALUE);
|
||||
|
||||
verify(rustfsObjectStorageService).deleteObject("payload-key");
|
||||
verify(rustfsObjectStorageService, never()).deleteObject(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+230
@@ -0,0 +1,230 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* chunk 载荷对象缺失时的「同槽位版本化兄弟对象」读兜底(2026-09-17 线上任务 28459)。
|
||||
*
|
||||
* <p>事故后 DB 行指向的普通 key({@code chunk-462.json})已被误删,但同槽位的版本化对象
|
||||
* ({@code chunk-462-<uuid>.json})还在。兜底让读路径自动改用兄弟对象,任务即可按已有数据出结果,
|
||||
* 而不是整单 FAILED——这是防止「删错对象 → 任务永久失败」的最后一道网。
|
||||
*/
|
||||
class TransientPayloadChunkSiblingFallbackTest {
|
||||
|
||||
private static final String DIR = "task-chunk/appearance_patent/28459/"
|
||||
+ "2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e/";
|
||||
private static final String CHUNK_KEY = DIR + "chunk-462.json";
|
||||
private static final String CHUNK_POINTER = "rustfs:" + CHUNK_KEY;
|
||||
private static final String SIBLING_KEY = DIR + "chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json";
|
||||
|
||||
// ===== 正常路径 =====
|
||||
|
||||
/** 对象在:直接读,不得触发列对象(正常读路径不能因为兜底变慢)。 */
|
||||
@Test
|
||||
void presentObjectIsReadWithoutListing() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
when(rustfs.readObjectBytes(CHUNK_KEY)).thenReturn(bytes("[\"原始\"]"));
|
||||
|
||||
String payload = service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed");
|
||||
|
||||
assertEquals("[\"原始\"]", payload);
|
||||
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
|
||||
}
|
||||
|
||||
/** 对象缺失 + 存在兄弟对象 → 读兄弟对象(本次事故的自动恢复路径)。 */
|
||||
@Test
|
||||
void missingChunkObjectFallsBackToVersionedSibling() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
|
||||
when(rustfs.listObjectKeysNewestFirst(DIR + "chunk-462-", 50)).thenReturn(List.of(SIBLING_KEY));
|
||||
when(rustfs.readObjectBytes(SIBLING_KEY)).thenReturn(bytes("[\"合并后\"]"));
|
||||
|
||||
String payload = service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed");
|
||||
|
||||
assertEquals("[\"合并后\"]", payload);
|
||||
verify(rustfs).listObjectKeysNewestFirst(DIR + "chunk-462-", 50);
|
||||
}
|
||||
|
||||
/** 多个兄弟对象:取列表首个(实现约定按最后修改时间倒序,最新在前)。 */
|
||||
@Test
|
||||
void newestSiblingIsPreferred() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
String newest = DIR + "chunk-462-ffffffff-ffff-ffff-ffff-ffffffffffff.json";
|
||||
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
|
||||
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of(newest, SIBLING_KEY));
|
||||
when(rustfs.readObjectBytes(newest)).thenReturn(bytes("[\"最新\"]"));
|
||||
|
||||
assertEquals("[\"最新\"]", service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
|
||||
verify(rustfs).readObjectBytes(newest);
|
||||
verify(rustfs, never()).readObjectBytes(SIBLING_KEY);
|
||||
}
|
||||
|
||||
/** 指针是 JSON 编码形式(DB 里的常见写法)时兜底同样生效。 */
|
||||
@Test
|
||||
void jsonQuotedPointerStillFallsBack() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
|
||||
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of(SIBLING_KEY));
|
||||
when(rustfs.readObjectBytes(SIBLING_KEY)).thenReturn(bytes("[\"带引号\"]"));
|
||||
|
||||
assertEquals("[\"带引号\"]",
|
||||
service(rustfs).resolvePayload('"' + CHUNK_POINTER + '"', "read chunk failed"));
|
||||
}
|
||||
|
||||
/** 非指针内容(直接就是载荷 JSON)原样返回,不触碰对象存储。 */
|
||||
@Test
|
||||
void rawContentIsReturnedAsIs() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
|
||||
assertEquals("[1,2,3]", service(rustfs).resolvePayload("[1,2,3]", "read chunk failed"));
|
||||
verify(rustfs, never()).readObjectBytes(anyString());
|
||||
}
|
||||
|
||||
// ===== 边界:不该兜底的场景 =====
|
||||
|
||||
/** 非 chunk 槽位(如 row-1.json)缺失 → 不兜底,照旧抛错(避免误配无关对象)。 */
|
||||
@Test
|
||||
void nonChunkSlotDoesNotFallBack() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
String key = "task-result-item/similar-asin/9/hash/row-1.json";
|
||||
when(rustfs.readObjectBytes(key)).thenThrow(missingKey());
|
||||
|
||||
assertThrows(IllegalStateException.class,
|
||||
() -> service(rustfs).resolvePayload("rustfs:" + key, "read item failed"));
|
||||
|
||||
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
|
||||
}
|
||||
|
||||
/** parsed 槽位(latest.json)缺失 → 不兜底(该槽位没有版本化兄弟语义)。 */
|
||||
@Test
|
||||
void parsedSlotDoesNotFallBack() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
String key = "task-parsed/appearance_patent/28459/hash/latest.json";
|
||||
when(rustfs.readObjectBytes(key)).thenThrow(missingKey());
|
||||
|
||||
assertThrows(IllegalStateException.class,
|
||||
() -> service(rustfs).resolvePayload("rustfs:" + key, "read parsed failed"));
|
||||
|
||||
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
|
||||
}
|
||||
|
||||
/** 非缺失类失败(权限/网络)→ 不兜底,照旧上抛以便重试。 */
|
||||
@Test
|
||||
void nonMissingFailureDoesNotFallBack() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
when(rustfs.readObjectBytes(CHUNK_KEY))
|
||||
.thenThrow(new IllegalStateException("Access Denied"));
|
||||
|
||||
assertThrows(IllegalStateException.class,
|
||||
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
|
||||
|
||||
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
|
||||
}
|
||||
|
||||
// ===== 异常路径 =====
|
||||
|
||||
/** 没有兄弟对象 → 抛错,且错误信息必须保留原 errorMessage 与原指针(便于定位)。 */
|
||||
@Test
|
||||
void noSiblingRethrowsWithOriginalPointer() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
|
||||
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of());
|
||||
|
||||
IllegalStateException ex = assertThrows(IllegalStateException.class,
|
||||
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
|
||||
|
||||
assertTrue(ex.getMessage().contains("read chunk failed"), "错误信息=" + ex.getMessage());
|
||||
assertTrue(ex.getMessage().contains("chunk-462.json"), "错误信息=" + ex.getMessage());
|
||||
}
|
||||
|
||||
/** 兄弟对象读取也失败 → 异常上抛,不得静默返回空内容。 */
|
||||
@Test
|
||||
void siblingReadFailurePropagates() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
|
||||
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of(SIBLING_KEY));
|
||||
when(rustfs.readObjectBytes(SIBLING_KEY)).thenThrow(new IllegalStateException("read timeout"));
|
||||
|
||||
IllegalStateException ex = assertThrows(IllegalStateException.class,
|
||||
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
|
||||
|
||||
assertTrue(ex.getMessage().contains("read chunk failed"), "错误信息=" + ex.getMessage());
|
||||
}
|
||||
|
||||
/** 列对象本身失败(桶不可用)→ 仍按原错误抛出,错误信息不含兄弟查找细节。 */
|
||||
@Test
|
||||
void listFailureDoesNotMaskOriginalError() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
|
||||
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt()))
|
||||
.thenThrow(new IllegalStateException("bucket unavailable"));
|
||||
|
||||
IllegalStateException ex = assertThrows(IllegalStateException.class,
|
||||
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
|
||||
|
||||
assertTrue(ex.getMessage().contains("read chunk failed"), "错误信息=" + ex.getMessage());
|
||||
}
|
||||
|
||||
/** local 指针不走 rustfs 兄弟兜底。 */
|
||||
@Test
|
||||
void localPointerDoesNotUseChunkSiblingFallback() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
|
||||
assertThrows(IllegalStateException.class,
|
||||
() -> service(rustfs).resolvePayload("local:i/server-121/" + CHUNK_KEY, "read chunk failed"));
|
||||
|
||||
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
|
||||
}
|
||||
|
||||
// ===== 辅助 =====
|
||||
|
||||
private static IllegalStateException missingKey() {
|
||||
return new IllegalStateException("The specified key does not exist.");
|
||||
}
|
||||
|
||||
private static byte[] bytes(String value) {
|
||||
return value.getBytes(StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private TransientPayloadStorageService service(RustfsObjectStorageService rustfs) {
|
||||
TransientStorageProperties transientProperties = new TransientStorageProperties();
|
||||
transientProperties.setEnabled(true);
|
||||
transientProperties.setEndpoint("http://127.0.0.1:9000");
|
||||
transientProperties.setBucket("json-server");
|
||||
transientProperties.setAccessKeyId("ak");
|
||||
transientProperties.setAccessKeySecret("sk");
|
||||
StorageProperties storageProperties = new StorageProperties();
|
||||
storageProperties.setLocalTempDir(System.getProperty("java.io.tmpdir"));
|
||||
return new TransientPayloadStorageService(
|
||||
transientProperties,
|
||||
storageProperties,
|
||||
rustfs,
|
||||
mock(OssStorageService.class),
|
||||
new ObjectMapper(),
|
||||
new InstanceMetadata("server-121"),
|
||||
mock(TaskChunkMapper.class),
|
||||
mock(TaskScopeStateMapper.class));
|
||||
}
|
||||
}
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 临时载荷删除前的引用守卫(2026-09-17 线上任务 28459 事故的第二条同族缺陷)。
|
||||
*
|
||||
* <p>守卫本意是「还有别的行引用同一 pointer 就不物理删」。原实现用
|
||||
* {@code chunkCount > 1} 作判据,等价于「恰好还有 1 行引用时可以删」——
|
||||
* 这在 caller 已经把自己那行挪走、但**另一行**仍指向旧对象的场景下,会把
|
||||
* 对方仍在用的对象删掉。docs 里 CollectDataDeleteConsistencyTest 已明确口径是
|
||||
* 「跨任务共享指针仍引用 → 跳过删除」,故判据收紧为「命中任意一行即不删」。
|
||||
*/
|
||||
class TransientPayloadDeleteReferenceGuardTest {
|
||||
|
||||
private static final String RUSTFS_CHUNK_POINTER =
|
||||
"rustfs:task-chunk/appearance_patent/28459/2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e/chunk-462.json";
|
||||
|
||||
// ===== 正常路径 =====
|
||||
|
||||
/** 没有任何行引用 → 允许物理删除。 */
|
||||
@Test
|
||||
void zeroReferencesAllowsPhysicalDelete() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||
TaskScopeStateMapper scopeMapper = mock(TaskScopeStateMapper.class);
|
||||
when(chunkMapper.selectCount(any())).thenReturn(0L);
|
||||
when(scopeMapper.selectCount(any())).thenReturn(0L);
|
||||
|
||||
service(rustfs, chunkMapper, scopeMapper).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
|
||||
|
||||
verify(rustfs).deleteObject(eq("task-chunk/appearance_patent/28459/2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e/chunk-462.json"));
|
||||
}
|
||||
|
||||
/** 多条行引用 → 不删(原有行为必须保持)。 */
|
||||
@Test
|
||||
void multipleReferencesBlockDelete() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||
when(chunkMapper.selectCount(any())).thenReturn(3L);
|
||||
|
||||
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class)).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
|
||||
|
||||
verify(rustfs, never()).deleteObject(anyString());
|
||||
}
|
||||
|
||||
// ===== 本次修复点 =====
|
||||
|
||||
/** 恰好 1 行仍引用 → 必须不删(原实现会删掉这唯一一行仍在用的对象)。 */
|
||||
@Test
|
||||
void singleReferencingChunkRowBlocksDelete() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||
when(chunkMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class)).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
|
||||
|
||||
verify(rustfs, never()).deleteObject(anyString());
|
||||
}
|
||||
|
||||
/** scope_state 单行引用同样阻断删除(该字段命中即非自我引用)。 */
|
||||
@Test
|
||||
void singleScopeStateReferenceBlocksDelete() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||
TaskScopeStateMapper scopeMapper = mock(TaskScopeStateMapper.class);
|
||||
when(chunkMapper.selectCount(any())).thenReturn(0L);
|
||||
when(scopeMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
service(rustfs, chunkMapper, scopeMapper).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
|
||||
|
||||
verify(rustfs, never()).deleteObject(anyString());
|
||||
}
|
||||
|
||||
// ===== 边界 =====
|
||||
|
||||
/** 传入 JSON 编码(带引号)的指针——DB 里常见写法,必须能解析并同样受守卫约束。 */
|
||||
@Test
|
||||
void jsonQuotedPointerIsParsedAndStillGuarded() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||
when(chunkMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class))
|
||||
.deletePayloadIfPresent('"' + RUSTFS_CHUNK_POINTER + '"');
|
||||
|
||||
verify(rustfs, never()).deleteObject(anyString());
|
||||
}
|
||||
|
||||
/** 非指针值原样传入 → 不解析、不查库、不删。 */
|
||||
@Test
|
||||
void nonPointerValueIsIgnoredWithoutLookup() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||
|
||||
TransientPayloadStorageService service = service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class));
|
||||
assertDoesNotThrow(() -> service.deletePayloadIfPresent("{\"rows\":[1,2,3]}"));
|
||||
|
||||
verify(chunkMapper, never()).selectCount(any());
|
||||
verify(rustfs, never()).deleteObject(anyString());
|
||||
}
|
||||
|
||||
/** 空值不得抛异常、不得触发任何删除。 */
|
||||
@Test
|
||||
void nullOrBlankValueIsIgnored() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||
TransientPayloadStorageService service = service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class));
|
||||
|
||||
assertDoesNotThrow(() -> service.deletePayloadIfPresent(null));
|
||||
assertDoesNotThrow(() -> service.deletePayloadIfPresent(" "));
|
||||
|
||||
verify(chunkMapper, never()).selectCount(any());
|
||||
verify(rustfs, never()).deleteObject(anyString());
|
||||
}
|
||||
|
||||
// ===== 异常路径 =====
|
||||
|
||||
/** 反查引用的查询抛异常 → 保守不删(原实现即如此,必须保持)。 */
|
||||
@Test
|
||||
void referenceQueryFailureIsConservativeAndSkipsDelete() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||
when(chunkMapper.selectCount(any())).thenThrow(new IllegalStateException("db down"));
|
||||
|
||||
assertDoesNotThrow(() ->
|
||||
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class))
|
||||
.deletePayloadIfPresent(RUSTFS_CHUNK_POINTER));
|
||||
|
||||
verify(rustfs, never()).deleteObject(anyString());
|
||||
}
|
||||
|
||||
/** 跨实例的 local 指针:不物理删(留给持有实例的清理任务)。 */
|
||||
@Test
|
||||
void crossInstanceLocalPointerIsNotDeleted() {
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||
when(chunkMapper.selectCount(any())).thenReturn(0L);
|
||||
|
||||
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class))
|
||||
.deletePayloadIfPresent("local:i/server-110/task-chunk/x/1/hash/chunk-1.json");
|
||||
|
||||
verify(rustfs, never()).deleteObject(anyString());
|
||||
}
|
||||
|
||||
private TransientPayloadStorageService service(RustfsObjectStorageService rustfs,
|
||||
TaskChunkMapper chunkMapper,
|
||||
TaskScopeStateMapper scopeMapper) {
|
||||
TransientStorageProperties transientProperties = new TransientStorageProperties();
|
||||
transientProperties.setEnabled(true);
|
||||
StorageProperties storageProperties = new StorageProperties();
|
||||
storageProperties.setLocalTempDir(System.getProperty("java.io.tmpdir"));
|
||||
return new TransientPayloadStorageService(
|
||||
transientProperties,
|
||||
storageProperties,
|
||||
rustfs,
|
||||
mock(OssStorageService.class),
|
||||
new ObjectMapper(),
|
||||
new InstanceMetadata("server-121"),
|
||||
chunkMapper,
|
||||
scopeMapper);
|
||||
}
|
||||
}
|
||||
@@ -11,10 +11,14 @@ import router from '@/router'
|
||||
import { ensureAuth } from '@/shared/auth/ensure-auth'
|
||||
import { ensureApiSecretsLoaded, loadApiSecrets } from '@/shared/utils/api-secret-store'
|
||||
import { installUserTokenSync } from '@/shared/auth/user-token-sync.ts'
|
||||
import { installSubmitGuard } from '@/shared/utils/submit-guard'
|
||||
|
||||
// 登录态令牌同步给桌面端 Python(A1/A3):无桥环境静默跳过
|
||||
installUserTokenSync()
|
||||
|
||||
// 提交按钮防连点:捕获阶段拦下同一按钮的连点(各工具页 .btn-run 通用)
|
||||
installSubmitGuard()
|
||||
|
||||
/**
|
||||
* 数富AI 前端统一入口(SPA,URL 无 .html 后缀)
|
||||
*
|
||||
|
||||
@@ -166,7 +166,7 @@ import {
|
||||
} from '@/shared/api/java-modules'
|
||||
import { getPywebviewApi, type UploadedJavaFile } from '@/shared/bridges/pywebview'
|
||||
import { saveUrlWithProgress } from '@/shared/utils/download-progress'
|
||||
import { EXCEL_EXTENSIONS, checkSelectedFiles, guardBlocked } from '@/shared/dispatch-guard'
|
||||
import { EXCEL_EXTENSIONS, checkSelectedFiles, collectDistinctErrors, guardBlocked } from '@/shared/dispatch-guard'
|
||||
import { passGuard } from '@/shared/dispatch-guard-ui'
|
||||
import { formatDateTime } from '@/shared/utils/datetime'
|
||||
import { uploadPathsToJava } from '@/shared/utils/upload-to-java'
|
||||
@@ -349,11 +349,15 @@ async function submitConvertRun() {
|
||||
convertResultItems.value = result.items || []
|
||||
await loadConvertHistory()
|
||||
if (result.total > 0 && result.successCount === 0) {
|
||||
// 逐文件原因就在 items[].error 里,优先直接列出,不再让用户对着「常见原因」猜
|
||||
const details = collectDistinctErrors((result.items || []).map((item) => item.error)).slice(0, 5)
|
||||
await passGuard(
|
||||
guardBlocked(
|
||||
'格式转换未成功',
|
||||
`本次提交的 ${result.total} 个文件全部转换失败。\n` +
|
||||
'常见原因是源文件表头与所选模板不匹配,或文件内没有数据行。\n' +
|
||||
(details.length
|
||||
? `失败原因:\n${details.map((message) => `· ${message}`).join('\n')}\n`
|
||||
: '常见原因是源文件表头与所选模板不匹配,或文件内没有数据行。\n') +
|
||||
'请在右侧结果列表查看每个文件的失败原因,或换一个模板后重试。',
|
||||
'convert.all-failed',
|
||||
),
|
||||
|
||||
@@ -137,7 +137,7 @@ import { expandBrandFolderRecursive } from '@/shared/api/brand'
|
||||
import { deleteDedupeHistory, getDedupeHistory, getDedupeResultDownloadUrl, getDedupeRunProgress, getExcelInfo, runDedupe, type DedupeResultItem, type DedupeRunProgressVo, type DedupeRunVo } from '@/shared/api/java-modules'
|
||||
import { getPywebviewApi, type UploadedJavaFile } from '@/shared/bridges/pywebview'
|
||||
import { saveUrlWithProgress } from '@/shared/utils/download-progress'
|
||||
import { EXCEL_EXTENSIONS, checkSelectedFiles, guardBlocked } from '@/shared/dispatch-guard'
|
||||
import { EXCEL_EXTENSIONS, checkSelectedFiles, collectDistinctErrors, guardBlocked } from '@/shared/dispatch-guard'
|
||||
import { passGuard } from '@/shared/dispatch-guard-ui'
|
||||
import { formatDateTime } from '@/shared/utils/datetime'
|
||||
import { uploadPathsToJava } from '@/shared/utils/upload-to-java'
|
||||
@@ -237,7 +237,8 @@ function clearAllCleanColumns() {
|
||||
async function loadCleanHeaders(fileKey: string) {
|
||||
const result = await getExcelInfo(fileKey)
|
||||
if (!result.headers?.length) {
|
||||
ElMessage.error('读取 Excel 表头失败')
|
||||
// 接口本身成功、只是没解析出表头,说明文件内容有问题而非「读取失败」,别让用户反复重选文件
|
||||
ElMessage.error('未读到表头行:请确认文件不是空表、且首行是表头')
|
||||
cleanAvailableColumns.value = []
|
||||
cleanSelectedColumns.value = []
|
||||
return
|
||||
@@ -359,11 +360,15 @@ async function submitCleanRun() {
|
||||
await loadCleanHistory()
|
||||
// 全部文件都失败时不该报「完成」,让用户看清是哪一批没处理成功
|
||||
if (result.total > 0 && result.successCount === 0) {
|
||||
// 逐文件原因就在 items[].error 里,优先直接列出,不再让用户对着「常见原因」猜
|
||||
const details = collectDistinctErrors((result.items || []).map((item) => item.error)).slice(0, 5)
|
||||
await passGuard(
|
||||
guardBlocked(
|
||||
'去重未成功',
|
||||
`本次提交的 ${result.total} 个文件全部处理失败。\n` +
|
||||
'常见原因是表头列名与所选保留列对不上,或文件内没有数据行。\n' +
|
||||
(details.length
|
||||
? `失败原因:\n${details.map((message) => `· ${message}`).join('\n')}\n`
|
||||
: '常见原因是表头列名与所选保留列对不上,或文件内没有数据行。\n') +
|
||||
'请在右侧结果列表查看每个文件的失败原因后重试。',
|
||||
'dedupe.all-failed',
|
||||
),
|
||||
|
||||
@@ -1105,7 +1105,13 @@ async function submitRun() {
|
||||
syncResultState()
|
||||
|
||||
if (hasBlockedItems) {
|
||||
ElMessage.warning('部分文件尚未匹配到店铺,已保留状态信息,请稍后重试。')
|
||||
// 具体原因(店铺未录入 / 索引未命中 / 表头错误等)在各文件项里,直接带出来;
|
||||
// 只报「请稍后重试」会把「需去后台添加店铺」误导成「等一等就好」
|
||||
const firstBlocked = normalizedItems.find((item) => !isUsableMatchedItem(item))
|
||||
const detail = firstBlocked ? getDisplayError(firstBlocked) : ''
|
||||
ElMessage.warning(detail
|
||||
? `部分文件不可启动:${detail}`
|
||||
: '部分文件尚未匹配到店铺,已保留状态信息,请稍后重试。')
|
||||
} else if (hasStaleMatchedItems) {
|
||||
ElMessage.warning('部分文件匹配到的店铺信息已过期,仍可启动任务;后台会自动重新匹配。')
|
||||
} else if (hasRunnableItems) {
|
||||
|
||||
@@ -511,6 +511,12 @@ function formatMatchRemark(row: PatrolDeleteShopQueueItem) {
|
||||
if (row.matched) {
|
||||
return "已匹配成功,请查看状态确认";
|
||||
}
|
||||
if (row.matchStatus === "CONFLICT") {
|
||||
return "存在多个同名店铺,请人工确认";
|
||||
}
|
||||
if (row.matchStatus === "PENDING") {
|
||||
return "店铺尚未匹配完成,请稍后查看";
|
||||
}
|
||||
return "未匹配成功,请检查店铺名";
|
||||
}
|
||||
|
||||
|
||||
@@ -848,6 +848,8 @@ function formatMatchRemark(row: PriceTrackShopQueueItem) {
|
||||
if (msg) return msg
|
||||
if (row.matched && row.matchStatus === 'MATCHED') return '已匹配成功,可启动任务'
|
||||
if (row.matched) return '已关联店铺,请查看状态确认'
|
||||
if (row.matchStatus === 'CONFLICT') return '存在多个同名店铺,请人工确认'
|
||||
if (row.matchStatus === 'PENDING') return '店铺尚未匹配完成,请稍后查看'
|
||||
return '未匹配成功,请检查店铺名'
|
||||
}
|
||||
|
||||
|
||||
@@ -1056,6 +1056,12 @@ function formatMatchRemark(row: ProductRiskShopQueueItem) {
|
||||
if (row.matched) {
|
||||
return '已关联店铺,请查看状态确认'
|
||||
}
|
||||
if (row.matchStatus === 'CONFLICT') {
|
||||
return '存在多个同名店铺,请人工确认'
|
||||
}
|
||||
if (row.matchStatus === 'PENDING') {
|
||||
return '店铺尚未匹配完成,请稍后查看'
|
||||
}
|
||||
return '未匹配成功,请检查店铺名'
|
||||
}
|
||||
|
||||
|
||||
@@ -216,6 +216,8 @@ import {
|
||||
checkParseResult,
|
||||
checkQueuePayload,
|
||||
checkSelectedFiles,
|
||||
collectDistinctErrors,
|
||||
guardBlocked,
|
||||
} from '@/shared/dispatch-guard'
|
||||
import { passGuard } from '@/shared/dispatch-guard-ui'
|
||||
import { useZiniaoVersion, type ZiniaoVersion } from '@/shared/utils/ziniao-version'
|
||||
@@ -563,10 +565,25 @@ async function submitRun() {
|
||||
sync_countries: syncCountries.value.filter((country) => country !== publishCountry.value),
|
||||
})
|
||||
if (!parsed.taskId) throw new Error('后端未返回有效任务标识')
|
||||
// publish 的 Vo 用 files 承载明细,没有 acceptedRows;这里先卡住 taskId 与总行数
|
||||
const fileErrors = collectDistinctErrors(
|
||||
(parsed.files || []).map((file) => file.errorMessage || file.error),
|
||||
)
|
||||
// 店铺未录入后台是最高频的解析失败原因(2026-09-17 客户因此连试 7 次),
|
||||
// 单独用「店铺未找到」弹窗直给后端原因,避免被通用文案淹没
|
||||
const missingShopErrors = fileErrors.filter((message) => message.includes('未找到店铺'))
|
||||
if (!parsed.totalRows && missingShopErrors.length) {
|
||||
await passGuard(guardBlocked('店铺未找到', missingShopErrors.join('\n'), 'publish.shop-missing'))
|
||||
return
|
||||
}
|
||||
// publish 的 Vo 用 files 承载明细,没有 acceptedRows;这里先卡住 taskId 与总行数。
|
||||
// 文件级失败(店铺未匹配等)时 Java 不解析 Excel、totalRows 同样为 0,把具体
|
||||
// 原因一并交给 guard 优先展示,避免只报「没有读到任何数据行」误导排查方向。
|
||||
const guard = checkParseResult(
|
||||
{ taskId: parsed.taskId, totalRows: parsed.totalRows, acceptedRows: parsed.totalRows },
|
||||
{ requiredColumnsHint: '店铺名 / 商品行' },
|
||||
{
|
||||
requiredColumnsHint: '店铺名 / 商品行',
|
||||
fileErrors,
|
||||
},
|
||||
)
|
||||
if (!(await passGuard(guard))) return
|
||||
|
||||
@@ -604,7 +621,9 @@ async function submitRun() {
|
||||
|
||||
await Promise.all([loadDashboard(), loadHistory()])
|
||||
if (!batch.pendingFileIds.length) {
|
||||
queueMessage.value = '解析完成,当前没有匹配成功且可上架的文件。'
|
||||
queueMessage.value = fileErrors.length
|
||||
? `解析完成,当前没有匹配成功且可上架的文件:${fileErrors[0]}`
|
||||
: '解析完成,当前没有匹配成功且可上架的文件。'
|
||||
ElMessage.warning(queueMessage.value)
|
||||
return
|
||||
}
|
||||
@@ -917,7 +936,10 @@ async function processQueue() {
|
||||
saveQueueState()
|
||||
throw new Error(queueMessage.value)
|
||||
}
|
||||
queueMessage.value = `文件 ${file.sourceFilename || nextFileId} 启动失败,已记录并继续下一个文件。`
|
||||
// 原因必须回显给用户:后端会因「该店铺已有上架任务在执行」直接拒绝激活,
|
||||
// 只写日志的话用户只看到"启动失败",会以为是文件问题而反复重传
|
||||
queueMessage.value = `文件 ${file.sourceFilename || nextFileId} 启动失败:${reason}`
|
||||
ElMessage.warning(queueMessage.value)
|
||||
activeFileId.value = null
|
||||
saveQueueState()
|
||||
}
|
||||
|
||||
@@ -439,6 +439,12 @@ function formatMatchRemark(row: QueryAsinShopQueueItem) {
|
||||
if (row.matched) {
|
||||
return "已匹配成功,请查看状态确认";
|
||||
}
|
||||
if (row.matchStatus === "CONFLICT") {
|
||||
return "存在多个同名店铺,请人工确认";
|
||||
}
|
||||
if (row.matchStatus === "PENDING") {
|
||||
return "店铺尚未匹配完成,请稍后查看";
|
||||
}
|
||||
return "未匹配成功,请检查店铺名";
|
||||
}
|
||||
|
||||
|
||||
@@ -565,7 +565,7 @@ async function deleteTaskRecord(item: ShopMatchHistoryItem) {
|
||||
}
|
||||
}
|
||||
function formatMatchStatus(status?: string) { const value = (status || '').trim(); return { MATCHED: '已匹配', PENDING: '待匹配', CONFLICT: '需人工确认', INDEX_STALE: '匹配已过期' }[value] || value || '—' }
|
||||
function formatMatchRemark(row: ShopMatchShopQueueItem) { const message = (row.matchMessage || '').trim(); if (message) return message; if (row.matched && row.matchStatus === 'MATCHED') return '已匹配成功,可启动任务'; if (row.matched) return '已关联店铺,请查看状态确认'; return '未匹配成功,请检查店铺名' }
|
||||
function formatMatchRemark(row: ShopMatchShopQueueItem) { const message = (row.matchMessage || '').trim(); if (message) return message; if (row.matched && row.matchStatus === 'MATCHED') return '已匹配成功,可启动任务'; if (row.matched) return '已关联店铺,请查看状态确认'; if (row.matchStatus === 'CONFLICT') return '存在多个同名店铺,请人工确认'; if (row.matchStatus === 'PENDING') return '店铺尚未匹配完成,请稍后查看'; return '未匹配成功,请检查店铺名' }
|
||||
async function pushToPythonQueue() { autoQueueEnabled.value = true; await processMatchedQueue() }
|
||||
function nextMatchedQueueItem() { return matchedItems.value.find((item) => item.matched) }
|
||||
async function processMatchedQueue() { if (disposed || queueWorkerRunning.value) return; const api = getPywebviewApi(); if (!api?.enqueue_json) { ElMessage.error('当前客户端未提供 enqueue_json'); return } const matched = matchedItems.value.filter((item) => item.matched); if (!matched.length) { ElMessage.warning('请先匹配可用店铺'); return } if (!orderedCountryCodes.value.length) { ElMessage.warning('请至少勾选一个国家'); return } let scheduleValues: string[] | undefined; try { scheduleValues = parseScheduleValues() } catch (error) { ElMessage.error(error instanceof Error ? error.message : '定时配置无效'); return } queueWorkerRunning.value = true; pushing.value = true; queuePayloadText.value = ''; let successCount = 0; let failedCount = 0; let index = 0; try { while (!disposed && autoQueueEnabled.value) { const item = nextMatchedQueueItem(); if (!item) break; index += 1; try { const created = await withTransientRetry(() => createShopMatchTask([buildTaskCreateItem(item)], orderedCountryCodes.value, scheduleValues), (attempt, maxAttempts) => { queuePushResult.value = `服务暂时不可用,正在重试创建任务(${attempt}/${maxAttempts})...` }); const taskId = created.taskId; const initialStatus = scheduleValues?.length ? 'SCHEDULED' : 'RUNNING'; const snapshot: ShopMatchTaskDetailVo = { task: { id: taskId, status: initialStatus, scheduledAt: scheduleValues?.[0], countryCodes: [...orderedCountryCodes.value], currentStageIndex: scheduleValues?.length ? 0 : undefined, activeStageIndex: undefined, scheduleStages: (scheduleValues || []).map((value, stageIndex) => ({ stageIndex, scheduledAt: value, status: stageIndex === 0 ? 'SCHEDULED' : 'PENDING' })) }, items: created.items }; taskSnapshots.value = { ...taskSnapshots.value, [taskId]: snapshot }; taskDetails.value = { ...taskDetails.value, [taskId]: initialStatus }; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); const createdItem = created.items?.[0]; if (!createdItem) { removeMatchedRowsLocally([item]); continue } if (scheduleValues?.length) { successCount += 1; restoreScheduledDispatches(); queuePushResult.value = `任务 ${taskId} 已创建,共 ${scheduleValues.length} 个执行时间点`; removeMatchedRowsLocally([item]); continue } const payload = buildQueuePayload(taskId, item, orderedCountryCodes.value); queuePayloadText.value = JSON.stringify(payload, null, 2); const result = await api.enqueue_json(payload); if (!result?.success) { failedCount += 1; queuePushResult.value = `任务 ${taskId} 启动失败,已自动继续下一条:${result?.error || '未知错误'}`; ElMessage.error(queuePushResult.value); removeMatchedRowsLocally([item]); continue } addPollingTask(taskId); ensurePolling(true); removeMatchedRowsLocally([item]); queuePushResult.value = `任务 ${taskId} 已提交,等待执行完成`; const finalStatus = await waitForTaskTerminal(taskId); if (finalStatus !== 'SUCCESS') { failedCount += 1; queuePushResult.value = `任务 ${taskId} 执行失败,已自动继续下一条`; ElMessage.error(queuePushResult.value); continue } successCount += 1; queuePushResult.value = `任务 ${taskId} 已完成` } catch (error) { if (disposed) break; failedCount += 1; queuePushResult.value = `第 ${index} 条任务处理失败,已自动继续下一条:${error instanceof Error ? error.message : '未知错误'}`; ElMessage.error(queuePushResult.value); removeMatchedRowsLocally([item]); continue } } restoreScheduledDispatches(); if (!scheduleValues?.length) ensurePolling(true); if (!disposed && (successCount > 0 || failedCount > 0)) ElMessage.success(scheduleValues?.length ? `定时任务已创建:成功 ${successCount} 条,失败 ${failedCount} 条` : `店铺启动已完成:成功 ${successCount} 条,失败 ${failedCount} 条`) } catch (error) { if (disposed) return; const message = error instanceof Error ? error.message : '启动失败'; queuePushResult.value = message; ElMessage.error(message) } finally { queueWorkerRunning.value = false; pushing.value = false } }
|
||||
|
||||
@@ -145,7 +145,7 @@ import { expandBrandFolderRecursive } from '@/shared/api/brand'
|
||||
import { deleteSplitHistory, getExcelInfo, getSplitHistory, getSplitResultDownloadUrl, runSplit, type SplitResultItem } from '@/shared/api/java-modules'
|
||||
import { getPywebviewApi, type UploadedJavaFile } from '@/shared/bridges/pywebview'
|
||||
import { saveUrlWithProgress } from '@/shared/utils/download-progress'
|
||||
import { EXCEL_EXTENSIONS, checkSelectedFiles, guardBlocked } from '@/shared/dispatch-guard'
|
||||
import { EXCEL_EXTENSIONS, checkSelectedFiles, collectDistinctErrors, guardBlocked } from '@/shared/dispatch-guard'
|
||||
import { passGuard } from '@/shared/dispatch-guard-ui'
|
||||
import { formatDateTime } from '@/shared/utils/datetime'
|
||||
import { uploadPathsToJava } from '@/shared/utils/upload-to-java'
|
||||
@@ -327,11 +327,15 @@ async function submitSplitRun() {
|
||||
splitResultItems.value = result.items || []
|
||||
await loadSplitHistory()
|
||||
if (result.total > 0 && result.successCount === 0) {
|
||||
// 逐文件原因就在 items[].error 里,优先直接列出,不再让用户对着「常见原因」猜
|
||||
const details = collectDistinctErrors((result.items || []).map((item) => item.error)).slice(0, 5)
|
||||
await passGuard(
|
||||
guardBlocked(
|
||||
'拆分未成功',
|
||||
`本次提交的 ${result.total} 个文件全部处理失败。\n` +
|
||||
'常见原因是表头列名与所选保留列对不上,或文件内没有数据行。\n' +
|
||||
(details.length
|
||||
? `失败原因:\n${details.map((message) => `· ${message}`).join('\n')}\n`
|
||||
: '常见原因是表头列名与所选保留列对不上,或文件内没有数据行。\n') +
|
||||
'请在右侧结果列表查看每个文件的失败原因后重试。',
|
||||
'split.all-failed',
|
||||
),
|
||||
|
||||
@@ -570,6 +570,12 @@ function formatMatchRemark(row: WithdrawShopQueueItem) {
|
||||
if (row.matched) {
|
||||
return "已匹配成功,请查看状态确认";
|
||||
}
|
||||
if (row.matchStatus === "CONFLICT") {
|
||||
return "存在多个同名店铺,请人工确认";
|
||||
}
|
||||
if (row.matchStatus === "PENDING") {
|
||||
return "店铺尚未匹配完成,请稍后查看";
|
||||
}
|
||||
return "未匹配成功,请检查店铺名";
|
||||
}
|
||||
|
||||
|
||||
@@ -1015,12 +1015,12 @@ async function waitForImageVideoTask(ticket: ImageVideoAsyncTaskVo): Promise<unk
|
||||
}
|
||||
if (task.status === 'SUCCESS') return task.result
|
||||
if (isTerminalImageVideoTask(task)) {
|
||||
throw new Error(task.errorMessage || 'Coze task failed')
|
||||
throw new Error(task.errorMessage || 'Coze 任务执行失败')
|
||||
}
|
||||
await sleep(IMAGE_VIDEO_TASK_POLL_DELAY_MS)
|
||||
task = await getImageVideoAsyncTask(task.taskId)
|
||||
}
|
||||
throw new Error('Coze task polling timed out')
|
||||
throw new Error('Coze 任务查询超时,请稍后重试')
|
||||
}
|
||||
|
||||
async function rewriteScriptFromSource() {
|
||||
@@ -1296,7 +1296,8 @@ async function pollAssemblyResult(tab: WorkspaceTab, taskId: number) {
|
||||
if (isTerminalImageVideoTask(task)) {
|
||||
assembly.polling = false
|
||||
if (task.status === 'FAILED') {
|
||||
ElMessage.error('Coze 工作流执行失败')
|
||||
// 后端 errorMessage 带具体原因(内容违规/超时/额度等),只报「执行失败」用户无从下手
|
||||
ElMessage.error(task.errorMessage ? `Coze 工作流执行失败:${task.errorMessage}` : 'Coze 工作流执行失败')
|
||||
} else {
|
||||
ElMessage.success(videoUrl || assembly.videoUrl ? '视频生成完成' : 'Coze 工作流执行完成,未解析到视频地址')
|
||||
}
|
||||
|
||||
@@ -24,6 +24,15 @@ export interface ClientChangelogEntry {
|
||||
|
||||
/** 更新日志数据(新版本在前;发版时在数组最前追加一条) */
|
||||
export const CLIENT_CHANGELOG: ClientChangelogEntry[] = [
|
||||
{
|
||||
version: '4.0.28',
|
||||
date: '2026-09-17',
|
||||
items: [
|
||||
'修复多个任务同时操作同一店铺导致「打开店铺失败」的问题',
|
||||
'同一店铺的任务改为排队执行,不会再互相打断',
|
||||
'紫鸟更新内核期间不再直接报「打开店铺失败」,而是等待完成',
|
||||
],
|
||||
},
|
||||
{
|
||||
version: '4.0.27',
|
||||
date: '2026-09-17',
|
||||
|
||||
@@ -267,6 +267,13 @@ export interface ParseResultOptions {
|
||||
requireRows?: boolean
|
||||
/** 必要字段名,用于拼「缺什么」的提示,如 'ASIN / 国家' */
|
||||
requiredColumnsHint?: string
|
||||
/**
|
||||
* 文件级失败原因(后端 files[].errorMessage)。0 有效行时优先展示这些具体
|
||||
* 原因:店铺未录入、表头不匹配等都会让文件级解析提前失败,totalRows 同样是
|
||||
* 0,只报通用「空文件」文案会把用户引向错误的排查方向(2026-09-17 用户因
|
||||
* 「店铺未录入」被误导反复重传同一个 Excel,连试 7 次)。
|
||||
*/
|
||||
fileErrors?: readonly string[]
|
||||
title?: string
|
||||
}
|
||||
|
||||
@@ -333,7 +340,21 @@ export function checkParseResult(
|
||||
: ''
|
||||
|
||||
if (requireRows && acceptedRows === 0) {
|
||||
if (!totalRows) {
|
||||
const fileErrors = collectDistinctErrors(options.fileErrors)
|
||||
if (fileErrors.length) {
|
||||
const shown = fileErrors.slice(0, 8)
|
||||
issues.push({
|
||||
code: 'parse.file-failed',
|
||||
severity: 'block',
|
||||
message:
|
||||
'以下文件解析未通过,没有可执行的数据行:\n' +
|
||||
shown.map((item) => `· ${item}`).join('\n') +
|
||||
(fileErrors.length > shown.length
|
||||
? `\n· 另有 ${fileErrors.length - shown.length} 条不同原因`
|
||||
: '') +
|
||||
'\n请按上述原因处理对应文件后重新上传解析。',
|
||||
})
|
||||
} else if (!totalRows) {
|
||||
issues.push({
|
||||
code: 'parse.empty-file',
|
||||
severity: 'block',
|
||||
@@ -387,6 +408,20 @@ function normalizeCount(value: unknown): number | null {
|
||||
return null
|
||||
}
|
||||
|
||||
/** 汇总一组错误原因:去空、去重(同一原因多条只留一条),保持出现顺序。 */
|
||||
export function collectDistinctErrors(values: readonly unknown[] | undefined): string[] {
|
||||
if (!values || !values.length) return []
|
||||
const seen = new Set<string>()
|
||||
const result: string[] = []
|
||||
for (const value of values) {
|
||||
const trimmed = typeof value === 'string' ? value.trim() : ''
|
||||
if (!trimmed || seen.has(trimmed)) continue
|
||||
seen.add(trimmed)
|
||||
result.push(trimmed)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export interface QueuePayloadOptions {
|
||||
/** data 下必须存在且非空的字段名 */
|
||||
requiredDataKeys?: readonly string[]
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* 提交按钮防连点(纯逻辑 + 全局安装器,供 main.ts 与单测复用)。
|
||||
*
|
||||
* 为什么需要:各工具页的「开始上架 / 启动任务 / 匹配店铺」按钮在提交逻辑跑完后
|
||||
* 立刻恢复可点,用户手快连点就会重复发起。2026-09-17 上架事故里同一店铺被接连
|
||||
* 提交三次、服务端并存多个同店铺任务,客户端并发打开同一店铺时全部失败。
|
||||
*
|
||||
* 语义:
|
||||
* - 冷却按**按钮元素**各自计算(WeakMap),点 A 按钮不会影响 B 按钮;
|
||||
* - 从"首次有效点击"起算,被拦的点击不会把冷却越拖越长(否则用户越急越点不开);
|
||||
* - 在**捕获阶段**拦截,抢在 Vue 的 @click 之前,被拦的点击不触发任何提交逻辑。
|
||||
*/
|
||||
|
||||
/** 提交按钮的统一类名(17 个工具页的主按钮都用它) */
|
||||
export const SUBMIT_BUTTON_SELECTOR = '.btn-run'
|
||||
|
||||
/** 默认冷却时长:够挡住手快连点,又不至于让正常操作感到卡顿 */
|
||||
export const DEFAULT_SUBMIT_COOLDOWN_MS = 1500
|
||||
|
||||
export interface ClickGate {
|
||||
/** 本次点击是否应被拦下(被拦时不会刷新冷却) */
|
||||
shouldBlock(target: object): boolean
|
||||
/** 解除某个目标的冷却(例如提交失败要允许用户立刻重试) */
|
||||
reset(target: object): void
|
||||
}
|
||||
|
||||
export function createClickGate(
|
||||
cooldownMs: number = DEFAULT_SUBMIT_COOLDOWN_MS,
|
||||
now: () => number = () => Date.now(),
|
||||
): ClickGate {
|
||||
// 非法时长(NaN/0/负数)回落默认值:宁可多挡一下,也不能等同"不设防"
|
||||
const cooldown = Number.isFinite(cooldownMs) && cooldownMs > 0
|
||||
? cooldownMs
|
||||
: DEFAULT_SUBMIT_COOLDOWN_MS
|
||||
|
||||
const lastClickAt = new WeakMap<object, number>()
|
||||
|
||||
return {
|
||||
shouldBlock(target: object): boolean {
|
||||
const current = now()
|
||||
const last = lastClickAt.get(target)
|
||||
// 用 undefined(而非 0)表示"从未点击过":哨兵值参与减法会跟时钟起点耦合,
|
||||
// 在 now() 起点较小(测试假时钟/单调时钟)时会误判首次点击为连点
|
||||
if (last !== undefined && current - last < cooldown) {
|
||||
return true
|
||||
}
|
||||
lastClickAt.set(target, current)
|
||||
return false
|
||||
},
|
||||
reset(target: object): void {
|
||||
lastClickAt.delete(target)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export interface SubmitGuardOptions {
|
||||
/** 生效的按钮选择器,默认 SUBMIT_BUTTON_SELECTOR */
|
||||
selector?: string
|
||||
cooldownMs?: number
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
/**
|
||||
* 全局安装防连点(在 main.ts 调用一次即可,覆盖所有工具页的提交按钮)。
|
||||
*
|
||||
* @returns 卸载函数(测试与热更新用)
|
||||
*/
|
||||
export function installSubmitGuard(options: SubmitGuardOptions = {}): () => void {
|
||||
const selector = options.selector ?? SUBMIT_BUTTON_SELECTOR
|
||||
const gate = createClickGate(options.cooldownMs, options.now)
|
||||
|
||||
const handler = (event: Event): void => {
|
||||
const target = event.target as Element | null
|
||||
const button = target && typeof target.closest === 'function'
|
||||
? target.closest(selector)
|
||||
: null
|
||||
if (!button) {
|
||||
return
|
||||
}
|
||||
if (gate.shouldBlock(button)) {
|
||||
event.stopImmediatePropagation()
|
||||
event.preventDefault()
|
||||
}
|
||||
}
|
||||
|
||||
document.addEventListener('click', handler, true)
|
||||
return () => document.removeEventListener('click', handler, true)
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
checkParseResult,
|
||||
checkQueuePayload,
|
||||
checkSelectedFiles,
|
||||
collectDistinctErrors,
|
||||
extensionOf,
|
||||
findUnsafeJsonPaths,
|
||||
guardPassed,
|
||||
@@ -165,6 +166,55 @@ test('test_task_101_dispatch_guard_boundary_parse_result_zero_rows_blocked', ()
|
||||
assert.match(emptyFile.message, /没有读到任何数据行/)
|
||||
})
|
||||
|
||||
test('test_task_101_dispatch_guard_parse_result_file_errors_shown_instead_of_empty_file', () => {
|
||||
// 2026-09-17:店铺未录入后台导致文件级失败,用户被「空文件」文案误导反复重传
|
||||
const result = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0 }), {
|
||||
fileErrors: ['后台店铺管理中未找到店铺:林清斌,请先添加店铺信息'],
|
||||
})
|
||||
assert.equal(result.ok, false)
|
||||
assert.deepEqual(codes(result), ['parse.file-failed'])
|
||||
assert.match(result.message, /林清斌/)
|
||||
assert.ok(!result.message.includes('没有读到任何数据行'), '有具体原因时不展示通用空文件文案')
|
||||
})
|
||||
|
||||
test('test_task_101_dispatch_guard_parse_result_file_errors_dedupe_blank_and_limit', () => {
|
||||
const deduped = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0 }), {
|
||||
fileErrors: ['店铺未匹配', ' ', '店铺未匹配', '', '第二条原因'],
|
||||
})
|
||||
assert.deepEqual(codes(deduped), ['parse.file-failed'])
|
||||
assert.equal(
|
||||
deduped.message.split('\n').filter((line) => line.startsWith('· ')).length,
|
||||
2,
|
||||
'空串与重复原因不应重复展示',
|
||||
)
|
||||
|
||||
const many = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0 }), {
|
||||
fileErrors: Array.from({ length: 10 }, (_, index) => `原因 ${index}`),
|
||||
})
|
||||
assert.match(many.message, /另有 2 条不同原因/)
|
||||
})
|
||||
|
||||
test('test_task_101_dispatch_guard_parse_result_blank_file_errors_fallback_to_empty_file', () => {
|
||||
const result = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0 }), {
|
||||
fileErrors: [' ', ''],
|
||||
})
|
||||
assert.deepEqual(codes(result), ['parse.empty-file'])
|
||||
})
|
||||
|
||||
test('test_task_101_dispatch_guard_collect_distinct_errors', () => {
|
||||
assert.deepEqual(collectDistinctErrors(['a', ' ', 'a', '', 'b']), ['a', 'b'], '去空去重且保持顺序')
|
||||
assert.deepEqual(collectDistinctErrors([' 店铺未找到 ']), ['店铺未找到'], '首尾空白裁剪')
|
||||
assert.deepEqual(collectDistinctErrors([]), [])
|
||||
assert.deepEqual(collectDistinctErrors(undefined), [])
|
||||
assert.deepEqual(collectDistinctErrors([null, 7, undefined]), [], '非字符串项忽略')
|
||||
})
|
||||
|
||||
test('test_task_101_dispatch_guard_parse_result_file_errors_ignored_when_rows_present', () => {
|
||||
const result = checkParseResult(parseVo(), { fileErrors: ['某文件失败'] })
|
||||
assert.equal(result.ok, true)
|
||||
assert.equal(result.needsConfirm, false)
|
||||
})
|
||||
|
||||
test('test_task_101_dispatch_guard_parse_result_zero_rows_allowed_when_not_required', () => {
|
||||
const result = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0 }), { requireRows: false })
|
||||
assert.equal(result.ok, true)
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
/**
|
||||
* 提交按钮防连点(shared/utils/submit-guard)行为测试。
|
||||
*
|
||||
* 背景(2026-09-17 上架事故):前端「开始上架 / 启动任务」按钮在提交完成后立刻
|
||||
* 恢复可点,手快连点会重复创建任务;同一店铺并存多个任务后,客户端并发打开
|
||||
* 同一店铺时全部失败。这里把冷却逻辑抽成纯函数以便单测。
|
||||
*/
|
||||
|
||||
import { test } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import {
|
||||
createClickGate,
|
||||
installSubmitGuard,
|
||||
DEFAULT_SUBMIT_COOLDOWN_MS,
|
||||
SUBMIT_BUTTON_SELECTOR,
|
||||
} from '../src/shared/utils/submit-guard.ts'
|
||||
|
||||
function fakeClock(start = 1_000) {
|
||||
let current = start
|
||||
return {
|
||||
now: () => current,
|
||||
advance: (ms: number) => {
|
||||
current += ms
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
test('默认冷却时长是正数秒级', () => {
|
||||
assert.ok(DEFAULT_SUBMIT_COOLDOWN_MS >= 1000, '冷却至少 1 秒,否则挡不住连点')
|
||||
})
|
||||
|
||||
test('默认选择器覆盖各工具页的主按钮', () => {
|
||||
assert.equal(SUBMIT_BUTTON_SELECTOR, '.btn-run')
|
||||
})
|
||||
|
||||
test('首次点击放行', () => {
|
||||
const clock = fakeClock()
|
||||
const gate = createClickGate(1500, clock.now)
|
||||
assert.equal(gate.shouldBlock({}), false)
|
||||
})
|
||||
|
||||
test('冷却期内重复点击被拦截', () => {
|
||||
const clock = fakeClock()
|
||||
const gate = createClickGate(1500, clock.now)
|
||||
const button = {}
|
||||
|
||||
assert.equal(gate.shouldBlock(button), false)
|
||||
clock.advance(100)
|
||||
assert.equal(gate.shouldBlock(button), true)
|
||||
clock.advance(1399)
|
||||
assert.equal(gate.shouldBlock(button), true)
|
||||
})
|
||||
|
||||
test('冷却结束时(边界)放行', () => {
|
||||
const clock = fakeClock()
|
||||
const gate = createClickGate(1500, clock.now)
|
||||
const button = {}
|
||||
|
||||
assert.equal(gate.shouldBlock(button), false)
|
||||
clock.advance(1500)
|
||||
assert.equal(gate.shouldBlock(button), false, '恰好到达冷却终点应放行')
|
||||
})
|
||||
|
||||
test('不同按钮互不影响', () => {
|
||||
const clock = fakeClock()
|
||||
const gate = createClickGate(1500, clock.now)
|
||||
const startButton = {}
|
||||
const matchButton = {}
|
||||
|
||||
assert.equal(gate.shouldBlock(startButton), false)
|
||||
assert.equal(gate.shouldBlock(matchButton), false, '另一个按钮不该被前一个的冷却波及')
|
||||
clock.advance(100)
|
||||
assert.equal(gate.shouldBlock(startButton), true)
|
||||
})
|
||||
|
||||
test('自定义冷却时长生效', () => {
|
||||
const clock = fakeClock()
|
||||
const gate = createClickGate(300, clock.now)
|
||||
const button = {}
|
||||
|
||||
assert.equal(gate.shouldBlock(button), false)
|
||||
clock.advance(299)
|
||||
assert.equal(gate.shouldBlock(button), true)
|
||||
clock.advance(2)
|
||||
assert.equal(gate.shouldBlock(button), false)
|
||||
})
|
||||
|
||||
test('被拦截的点击不会延长冷却', () => {
|
||||
const clock = fakeClock()
|
||||
const gate = createClickGate(1000, clock.now)
|
||||
const button = {}
|
||||
|
||||
assert.equal(gate.shouldBlock(button), false)
|
||||
for (let i = 0; i < 5; i += 1) {
|
||||
clock.advance(100)
|
||||
assert.equal(gate.shouldBlock(button), true)
|
||||
}
|
||||
// 从首次点击起算 1000ms 后就该放行,而不是被连点拖长
|
||||
clock.advance(500)
|
||||
assert.equal(gate.shouldBlock(button), false)
|
||||
})
|
||||
|
||||
test('同一时刻的两次点击只有第一次放行', () => {
|
||||
const clock = fakeClock()
|
||||
const gate = createClickGate(1500, clock.now)
|
||||
const button = {}
|
||||
|
||||
assert.equal(gate.shouldBlock(button), false)
|
||||
assert.equal(gate.shouldBlock(button), true, '同一 tick 的第二次点击必须被拦')
|
||||
})
|
||||
|
||||
test('reset 后立即放行', () => {
|
||||
const clock = fakeClock()
|
||||
const gate = createClickGate(1500, clock.now)
|
||||
const button = {}
|
||||
|
||||
assert.equal(gate.shouldBlock(button), false)
|
||||
assert.equal(gate.shouldBlock(button), true)
|
||||
gate.reset(button)
|
||||
assert.equal(gate.shouldBlock(button), false)
|
||||
})
|
||||
|
||||
test('非法冷却时长回落到默认值', () => {
|
||||
const clock = fakeClock()
|
||||
const gate = createClickGate(Number.NaN, clock.now)
|
||||
const button = {}
|
||||
|
||||
assert.equal(gate.shouldBlock(button), false)
|
||||
clock.advance(DEFAULT_SUBMIT_COOLDOWN_MS - 1)
|
||||
assert.equal(gate.shouldBlock(button), true, 'NaN 应回落为默认冷却而不是立刻放行')
|
||||
})
|
||||
|
||||
test('全局安装:捕获阶段注册且可卸载', () => {
|
||||
const registered: Array<{ type: string; handler: unknown; capture: boolean }> = []
|
||||
const removed: string[] = []
|
||||
const originalDocument = (globalThis as Record<string, unknown>).document
|
||||
;(globalThis as Record<string, unknown>).document = {
|
||||
addEventListener: (type: string, handler: unknown, capture: boolean) => {
|
||||
registered.push({ type, handler, capture })
|
||||
},
|
||||
removeEventListener: (type: string) => {
|
||||
removed.push(type)
|
||||
},
|
||||
}
|
||||
|
||||
try {
|
||||
const uninstall = installSubmitGuard({ cooldownMs: 1000, now: () => 0 })
|
||||
assert.equal(registered.length, 1)
|
||||
assert.equal(registered[0].type, 'click')
|
||||
assert.equal(registered[0].capture, true, '必须捕获阶段,否则 Vue 的 @click 已先执行')
|
||||
uninstall()
|
||||
assert.deepEqual(removed, ['click'])
|
||||
} finally {
|
||||
;(globalThis as Record<string, unknown>).document = originalDocument
|
||||
}
|
||||
})
|
||||
|
||||
test('全局安装:只拦 .btn-run 的连点,且第二次点击阻断冒泡', () => {
|
||||
const registered: Array<{ handler: (event: unknown) => void }> = []
|
||||
const originalDocument = (globalThis as Record<string, unknown>).document
|
||||
;(globalThis as Record<string, unknown>).document = {
|
||||
addEventListener: (_type: string, handler: (event: unknown) => void) => {
|
||||
registered.push({ handler })
|
||||
},
|
||||
removeEventListener: () => undefined,
|
||||
}
|
||||
|
||||
try {
|
||||
installSubmitGuard({ cooldownMs: 1000, now: () => 5000 })
|
||||
const handler = registered[0].handler
|
||||
|
||||
const button = { closest: (selector: string) => (selector === '.btn-run' ? button : null) }
|
||||
const elsewhere = { closest: () => null }
|
||||
const makeEvent = (target: unknown) => {
|
||||
const calls: string[] = []
|
||||
return {
|
||||
calls,
|
||||
target,
|
||||
stopImmediatePropagation: () => calls.push('stop'),
|
||||
preventDefault: () => calls.push('prevent'),
|
||||
}
|
||||
}
|
||||
|
||||
const first = makeEvent(button)
|
||||
handler(first)
|
||||
assert.deepEqual(first.calls, [], '首次点击不应被拦')
|
||||
|
||||
const second = makeEvent(button)
|
||||
handler(second)
|
||||
assert.deepEqual(second.calls, ['stop', 'prevent'], '连点必须被拦下')
|
||||
|
||||
const other = makeEvent(elsewhere)
|
||||
handler(other)
|
||||
assert.deepEqual(other.calls, [], '非提交按钮的点击不受影响')
|
||||
} finally {
|
||||
;(globalThis as Record<string, unknown>).document = originalDocument
|
||||
}
|
||||
})
|
||||
Reference in New Issue
Block a user