fix(tasks): 单片/单文件读不到跳过它,缺数据不许冒充已收齐
三处"一颗坏项拖垮整批"与一处假绿本体: - BrandTaskStorageService:一片载荷读不到就整份聚合丢掉,而三个调用方 (finalizeTask/abort/tryRecoverCompletedRunningTask)都没有就地 try;改逐片跳过长 WARN,并把 receivedChunkCount/completed 的判据从 chunks.size()(DB 行数)改成 真正读到的片数——拿行数比 chunkTotal 正是"118/119 行没数据仍判已完成"的形状; 解析缓存与汇总状态两处同型"整批抛出"一并改成逐条跳过。 - AppearancePatentLlmBatchPipeline:候选收集跑在批循环之前,一抛就走不到循环体内 那道 finally releaseRowInspection,本轮已申领的行要挂满处理中 TTL(15 分钟), 且 terminalizeRoundExhaustedRows 跟着不执行(轮次到顶的行因此永不判死);改逐片容错。 - TransientPayloadStorageService:跨实例判据枚成 public 常量并提供 cause 链走法。 resolvePayload 会把真实原因 rewrap 成「<errorMessage>: <pointer>」,消费端只摸顶层 消息时「only exists on instance=」「does not exist」两处判定恒 false——跨实例留痕 从未下过笔、"跳过该分片"分支是死代码。消费端(ChunkPayloadSupport/ HistoryAssembler/ResultRowReader)全部改走共享常量+cause 链。 - SimilarAsinHistoryAssemblerTest 的假绿桩删除:改前它桩的是生产者早已不再产出的 英文旧串,断言照旧绿;现在跑真实 resolvePayload 取异常(只 mock RustFS 边界), 改前 15 跑 1 红、改后 15/15 绿。 - SimilarAsinStaleReopenSupport:客户端卡死尾句枚成常量、整句文案唯一出口,生产者 与重开白名单共用;此前只是碰巧被前缀命中,生产者一改前缀就会"永不重开"。
This commit is contained in:
+19
-3
@@ -296,8 +296,23 @@ public class AppearancePatentLlmBatchPipeline {
|
||||
int judgedSkipped = 0;
|
||||
int noDataSkipped = 0;
|
||||
int chunkRows = 0;
|
||||
int unreadableChunks = 0;
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
|
||||
Map<String, AppearancePatentResultRowDto> persistedRows;
|
||||
try {
|
||||
persistedRows = readChunkRows(chunk);
|
||||
} catch (Exception ex) {
|
||||
// 单片载荷读不到不许把整轮候选收集炸掉:本方法跑在批循环**之前**,而释放处理中标记的
|
||||
// finally 在循环体内——这里一抛就永远走不到那道 release,本轮已申领的行要挂满
|
||||
// IN_FLIGHT_TTL(15 分钟)才回来,且 terminalizeRoundExhaustedRows 收的是本方法的返回值,
|
||||
// 跟着一起不执行(轮次到顶的行因此永不判死)。跳过这一片、继续读其余片,
|
||||
// 与 AppearancePatentResultRowReader#readChunkRows/SimilarAsinChunkPayloadSupport 同口径。
|
||||
unreadableChunks++;
|
||||
log.warn("[appearance-patent] 单片载荷读取失败,跳过该分片继续收集候选"
|
||||
+ " taskId={} scopeHash={} chunk={} err={}",
|
||||
chunk.getTaskId(), chunk.getScopeHash(), chunk.getChunkIndex(), ex.getMessage());
|
||||
continue;
|
||||
}
|
||||
if (persistedRows.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
@@ -328,8 +343,9 @@ public class AppearancePatentLlmBatchPipeline {
|
||||
candidates.add(row);
|
||||
}
|
||||
}
|
||||
log.info("[appearance-patent] 候选收集逐行申领 taskId={} chunkRows={} judgedSkipped={} noDataSkipped={} claimBlocked={} candidates={}",
|
||||
chunks.get(0).getTaskId(), chunkRows, judgedSkipped, noDataSkipped, claimBlocked, candidates.size());
|
||||
log.info("[appearance-patent] 候选收集逐行申领 taskId={} chunkRows={} judgedSkipped={} noDataSkipped={} claimBlocked={} unreadableChunks={} candidates={}",
|
||||
chunks.get(0).getTaskId(), chunkRows, judgedSkipped, noDataSkipped, claimBlocked,
|
||||
unreadableChunks, candidates.size());
|
||||
return candidates;
|
||||
}
|
||||
|
||||
|
||||
+4
-10
@@ -273,17 +273,11 @@ public class AppearancePatentResultRowReader {
|
||||
}
|
||||
|
||||
/** payload 对象已不存在(RustFS 返回 NoSuchKey:message 为 "The specified key does not exist.")。
|
||||
* 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。 */
|
||||
* 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。
|
||||
* 判据与 cause 链走法都收口在 {@link TransientPayloadStorageService#isObjectMissing}——
|
||||
* 抛点与判定端各抄一遍字面量时,改一处就静默失效(货源侧那两处就是这么废掉的)。 */
|
||||
private static boolean isPayloadMissing(Throwable error) {
|
||||
Throwable cursor = error;
|
||||
while (cursor != null) {
|
||||
String message = cursor.getMessage();
|
||||
if (message != null && message.contains("does not exist")) {
|
||||
return true;
|
||||
}
|
||||
cursor = cursor.getCause();
|
||||
}
|
||||
return false;
|
||||
return TransientPayloadStorageService.isObjectMissing(error);
|
||||
}
|
||||
|
||||
public String rowKey(AppearancePatentParsedRowVo row) {
|
||||
|
||||
+41
-15
@@ -90,7 +90,7 @@ public class BrandTaskStorageService {
|
||||
return List.of();
|
||||
}
|
||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.select(TaskScopeStateEntity::getParsedPayloadJson)
|
||||
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getScopeHash)
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
|
||||
@@ -99,14 +99,17 @@ public class BrandTaskStorageService {
|
||||
}
|
||||
List<BrandParsedFileCacheDto> result = new ArrayList<>();
|
||||
for (TaskScopeStateEntity state : states) {
|
||||
String payloadJson = resolveParsedPayload(state.getParsedPayloadJson());
|
||||
if (isBlank(payloadJson)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
String payloadJson = resolveParsedPayload(state.getParsedPayloadJson());
|
||||
if (isBlank(payloadJson)) {
|
||||
continue;
|
||||
}
|
||||
result.add(objectMapper.readValue(payloadJson, BrandParsedFileCacheDto.class));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取品牌任务解析结果失败");
|
||||
// 单个文件的解析缓存读不到不许把整份缓存一起丢掉:跳过它,其余文件照常出结果。
|
||||
// 整批抛出会让 BrandTaskService 的 finalizeTask / abort 两处调用全单失败(那里没有就地 try)。
|
||||
log.warn("[brand-storage] 解析缓存读取失败,跳过该文件 taskId={} scopeHash={} err={}",
|
||||
taskId, state.getScopeHash(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
@@ -207,7 +210,10 @@ public class BrandTaskStorageService {
|
||||
String aggregateJson = transientPayloadStorageService.resolvePayload(state.getStateJson(), "read brand task aggregate failed");
|
||||
result.put(state.getScopeKey(), objectMapper.readValue(aggregateJson, BrandFileAggregateCacheDto.class));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取品牌任务汇总状态失败");
|
||||
// 单个文件的汇总载荷读不到:跳过它,其余文件的聚合照常返回。
|
||||
// 少一个文件会让调用方按「尚未收齐」处理(可重试),比整批抛出把已读到的文件一起抹掉好。
|
||||
log.warn("[brand-storage] 汇总状态读取失败,跳过该文件 taskId={} scopeKey={} err={}",
|
||||
taskId, state.getScopeKey(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
@@ -444,32 +450,52 @@ public class BrandTaskStorageService {
|
||||
}
|
||||
|
||||
BrandFileAggregateCacheDto aggregate = null;
|
||||
int readableChunks = 0;
|
||||
List<Integer> skippedChunkIndexes = new ArrayList<>();
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
BrandCrawlResultFileDto chunkPayload = readChunkPayload(chunk.getPayloadJson());
|
||||
BrandCrawlResultFileDto chunkPayload;
|
||||
try {
|
||||
chunkPayload = readChunkPayload(chunk);
|
||||
} catch (Exception ex) {
|
||||
// 单片载荷读不到(对象被清理 / 跨实例 local 指针 / JSON 坏)不许把整份聚合一起丢掉:
|
||||
// 跳过这一片、继续读其余片,与 AppearancePatentResultRowReader#readChunkRows、
|
||||
// SimilarAsinChunkPayloadSupport#readChunkRows 同口径。调用方(BrandTaskService 的
|
||||
// finalizeTask / abort / tryRecoverCompletedRunningTask)都没有就地 try,
|
||||
// 这里一抛就是整任务不产出结果。
|
||||
skippedChunkIndexes.add(chunk.getChunkIndex());
|
||||
log.warn("[brand-storage] 单片载荷读取失败,跳过该分片继续聚合 taskId={} scopeHash={} chunk={} err={}",
|
||||
taskId, scopeHash, chunk.getChunkIndex(), ex.getMessage());
|
||||
continue;
|
||||
}
|
||||
if (aggregate == null) {
|
||||
aggregate = createAggregate(chunkPayload);
|
||||
} else {
|
||||
ensureChunkConsistency(aggregate, chunkPayload);
|
||||
}
|
||||
mergeAggregate(aggregate, chunkPayload);
|
||||
readableChunks++;
|
||||
}
|
||||
if (aggregate == null) {
|
||||
throw new BusinessException("读取品牌任务汇总数据失败");
|
||||
throw new BusinessException("读取品牌任务汇总数据失败:全部分片载荷均不可读,分片=" + skippedChunkIndexes);
|
||||
}
|
||||
aggregate.setFileUrl(scopeKey);
|
||||
aggregate.setReceivedChunkCount(chunks.size());
|
||||
aggregate.setCompleted(aggregate.getChunkTotal() != null
|
||||
// 已收齐的判据只数**真正读到**的片:拿 chunks.size()(DB 行数)比 chunkTotal,
|
||||
// 会在载荷缺失时把缺数据的文件判成 completed——正是「118/119 行没数据仍报已完成」的形状。
|
||||
aggregate.setReceivedChunkCount(readableChunks);
|
||||
aggregate.setCompleted(skippedChunkIndexes.isEmpty()
|
||||
&& aggregate.getChunkTotal() != null
|
||||
&& aggregate.getChunkTotal() > 0
|
||||
&& chunks.size() >= aggregate.getChunkTotal());
|
||||
&& readableChunks >= aggregate.getChunkTotal());
|
||||
return aggregate;
|
||||
}
|
||||
|
||||
private BrandCrawlResultFileDto readChunkPayload(String payloadJson) {
|
||||
private BrandCrawlResultFileDto readChunkPayload(TaskChunkEntity chunk) {
|
||||
try {
|
||||
String resolvedPayload = transientPayloadStorageService.resolvePayload(payloadJson, "read brand task chunk failed");
|
||||
String resolvedPayload = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read brand task chunk failed");
|
||||
return objectMapper.readValue(resolvedPayload, BrandCrawlResultFileDto.class);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取品牌任务分片失败");
|
||||
// 带上分片索引与 cause:跳过之后排查只剩这一行日志,裸文案对不上是哪一片
|
||||
throw new BusinessException("读取品牌任务分片失败,分片=" + chunk.getChunkIndex(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -1109,8 +1109,8 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
log.warn("[similar-asin] 从未上报任何结果分片且处理位置长时间停在「{}」未推进,按卡死收尾 taskId={} minutes={}",
|
||||
neverReportedPosition, task.getId(), minutes);
|
||||
finalizeStaleTask(task.getId(),
|
||||
"任务创建至今超过 " + minutes + " 分钟无任何结果回传,处理位置长时间停在「"
|
||||
+ neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败");
|
||||
SimilarAsinStaleReopenSupport.clientStallFailureMessage(
|
||||
minutes, neverReportedPosition));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
+8
-3
@@ -78,11 +78,15 @@ public class SimilarAsinChunkPayloadSupport {
|
||||
String msg = ex.getMessage() == null ? "" : ex.getMessage();
|
||||
// P1-4:识别跨实例 local 指针读不到的场景,把"chunk 在另一实例"的元信息
|
||||
// 通过 task_scope_state.last_error 留痕,便于排查"为何 owner 切换后 chunk 读不到"。
|
||||
boolean crossInstance = msg.contains("only exists on instance=");
|
||||
// 判据必须走 cause 链并与生产者共用同一枚常量:resolvePayload 抛给这里的顶层消息是
|
||||
// "<errorMessage>: <pointer>",「临时载荷仅存在于实例 …」只在 cause 上——
|
||||
// 旧的 contains("only exists on instance=") 认的是 TransientPayloadStorageService 早已
|
||||
// 不再产出的英文串,crossInstance 恒 false,@cross-instance 标记从未下过笔(2026-10-27 复验)。
|
||||
boolean crossInstance = TransientPayloadStorageService.isCrossInstanceLocalPointer(ex);
|
||||
// 类型不兼容(如历史版本序列化数据含当前类加载器不认识的类型标记):
|
||||
// 该 chunk 永久不可读,跳过而不是把组装/收尾永久拖死,留痕便于排查。
|
||||
boolean typeMismatch = ex instanceof TypeNotPresentException
|
||||
|| msg.contains("not present");
|
||||
|| TransientPayloadStorageService.causeChainContains(ex, "not present");
|
||||
if (typeMismatch) {
|
||||
log.warn("[similar-asin] chunk payload type mismatch, skip chunk taskId={} chunk={} err={}",
|
||||
chunk.getTaskId(), chunk.getChunkIndex(), msg);
|
||||
@@ -92,7 +96,8 @@ public class SimilarAsinChunkPayloadSupport {
|
||||
// payload 对象已不存在(RustFS 返回 NoSuchKey):重试多少次都读不回来,跳过而不是
|
||||
// 把组装/收尾永久拖死——与上面的 typeMismatch 分支、以及 collect-data 的降级口径一致。
|
||||
// 线上 appearance-patent 28459 就因同类场景每 10~30 秒重试一次(见同批修复)。
|
||||
if (msg.contains("does not exist")) {
|
||||
// 同样只认 cause 链:rewrap 后顶层消息里没有 "does not exist",这一分支此前也是死的。
|
||||
if (TransientPayloadStorageService.isObjectMissing(ex)) {
|
||||
log.warn("[similar-asin] chunk payload 已不存在,跳过该分片 taskId={} chunk={} err={}",
|
||||
chunk.getTaskId(), chunk.getChunkIndex(), msg);
|
||||
recordChunkReadFailure(chunk, false, msg);
|
||||
|
||||
+3
-1
@@ -439,7 +439,9 @@ public class SimilarAsinHistoryAssembler {
|
||||
String msg = ex.getMessage() == null ? "" : ex.getMessage();
|
||||
// P1-4:识别跨实例 local 指针读不到的场景,把"chunk 在另一实例"的元信息
|
||||
// 通过 task_scope_state.last_error 留痕,便于排查"为何 owner 切换后 chunk 读不到"。
|
||||
boolean crossInstance = msg.contains("only exists on instance=");
|
||||
// 与 SimilarAsinChunkPayloadSupport 同一枚判据:走 cause 链+生产者自己的常量,
|
||||
// 只摸顶层消息(rewrap 后是 "<errorMessage>: <pointer>")永远认不出中文原文。
|
||||
boolean crossInstance = TransientPayloadStorageService.isCrossInstanceLocalPointer(ex);
|
||||
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
|
||||
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
|
||||
recordChunkReadFailure(chunk, crossInstance, msg);
|
||||
|
||||
+15
@@ -28,12 +28,27 @@ import java.util.List;
|
||||
public class SimilarAsinStaleReopenSupport {
|
||||
|
||||
private static final String MODULE_TYPE = "SIMILAR_ASIN";
|
||||
/**
|
||||
* 客户端卡死判死文案的固定尾句,重开判定与生产者共用(口径同 {@code TaskResumeService.DUPLICATE_STOP_REASON}:
|
||||
* 两处各写一遍,改一处就静默失效——这里失效的方向是「迟到分片被拒、用户已采集的数据整批丢掉」)。
|
||||
*/
|
||||
public static final String CLIENT_STALL_FAILURE_SUFFIX = "疑似客户端任务卡死,任务已自动失败";
|
||||
/**
|
||||
* 无结果回传判死的整句文案——唯一出口,生产者({@code SimilarAsinTaskService} 的二次判死)
|
||||
* 与本类的重开判定都从它取数,前缀与尾句不会再各漂一半。
|
||||
*/
|
||||
public static String clientStallFailureMessage(long minutes, String neverReportedPosition) {
|
||||
return "任务创建至今超过 " + minutes + " 分钟无任何结果回传,处理位置长时间停在「"
|
||||
+ neverReportedPosition + "」未推进," + CLIENT_STALL_FAILURE_SUFFIX;
|
||||
}
|
||||
private static final List<String> STALE_FAILURE_MARKERS = List.of(
|
||||
"任务心跳超时,已自动失败",
|
||||
"长时间未收到 Python 心跳,任务已自动失败",
|
||||
"客户端未回传任何检测结果",
|
||||
"客户端回传中断",
|
||||
"任务创建至今超过",
|
||||
// 尾句单独进白名单:上面那枚前缀只是碰巧同句出现,生产者一改前缀重开就整条失效
|
||||
CLIENT_STALL_FAILURE_SUFFIX,
|
||||
"Python heartbeat alive but no result chunk uploaded");
|
||||
private static final Duration REOPEN_WINDOW = Duration.ofHours(24);
|
||||
|
||||
|
||||
+44
-5
@@ -45,6 +45,20 @@ public class TransientPayloadStorageService {
|
||||
private static final String OSS_POINTER_PREFIX = "oss:";
|
||||
private static final String LOCAL_PAYLOAD_DIR = "transient-payload";
|
||||
|
||||
/**
|
||||
* 跨实例消费 local 兜底指针时的判据原文(抛点见 {@link #resolvePayload})。
|
||||
*
|
||||
* <p>枚成 public 常量是因为消费端(货源 {@code SimilarAsinChunkPayloadSupport}/
|
||||
* {@code SimilarAsinHistoryAssembler} 的 {@code chunk-read-failed[N@cross-instance]} 留痕)
|
||||
* 要认的就是这句话。消费端此前各抄一份英文旧串 {@code "only exists on instance="}——
|
||||
* 生产者早就改成中文文案,两处判定恒 false,跨实例标记永不下笔(2026-10-27 复验定性为假绿)。
|
||||
* 现在两端共用这一枚常量,改文案只会让两边一起改,不会静默失效。
|
||||
*/
|
||||
public static final String CROSS_INSTANCE_LOCAL_POINTER_MARKER = "临时载荷仅存在于实例 ";
|
||||
|
||||
/** 对象确已不存在(RustFS NoSuchKey)的判据原文:抛点在 {@code RustfsObjectStorageService}。 */
|
||||
private static final String MISSING_OBJECT_MARKER = "does not exist";
|
||||
|
||||
/**
|
||||
* P0-3:记录"最近一次 store 是否因 RustFS 失败回落到 local"。
|
||||
* 调用方(如 SimilarAsinTaskService.submitResult)拿到 storedPayload 后可以查询
|
||||
@@ -144,7 +158,7 @@ public class TransientPayloadStorageService {
|
||||
if (ownerInstance != null && !ownerInstance.equals(instanceMetadata.getInstanceId())) {
|
||||
// 跨实例消费 local 指针时立即报错,避免悄悄读出空内容造成下游错位。
|
||||
throw new IllegalStateException(
|
||||
"临时载荷仅存在于实例 " + ownerInstance + "(当前实例 "
|
||||
CROSS_INSTANCE_LOCAL_POINTER_MARKER + ownerInstance + "(当前实例 "
|
||||
+ instanceMetadata.getInstanceId() + "),本地兜底指针跨节点不可读; objectKey="
|
||||
+ stripLocalInstanceId(localKey));
|
||||
}
|
||||
@@ -211,15 +225,40 @@ public class TransientPayloadStorageService {
|
||||
return candidates.isEmpty() ? null : candidates.getFirst();
|
||||
}
|
||||
|
||||
/** 对象确已不存在:RustFS 返回 NoSuchKey,message 为 "The specified key does not exist."。 */
|
||||
private static boolean isObjectMissing(Throwable error) {
|
||||
/**
|
||||
* 对象确已不存在:RustFS 返回 NoSuchKey,message 为 "The specified key does not exist."。
|
||||
*
|
||||
* <p>公开给消费端(分片载荷读取)复用。{@link #resolvePayload} 末尾会把真实原因 rewrap 成
|
||||
* 「{@code <errorMessage>}: {@code <pointer>}」,NoSuchKey 与跨实例原文都只剩在 cause 上,
|
||||
* 所以判据一律走 cause 链——只摸顶层消息的话,「跳过该分片」与「跨实例留痕」两个分支
|
||||
* 会一起变成死代码(货源侧 2026-10-27 复验就是这一形状)。
|
||||
*/
|
||||
public static boolean isObjectMissing(Throwable error) {
|
||||
return causeChainContains(error, MISSING_OBJECT_MARKER);
|
||||
}
|
||||
|
||||
/** 载荷归另一台实例(local 兜底指针跨节点不可读);同样必须走 cause 链。 */
|
||||
public static boolean isCrossInstanceLocalPointer(Throwable error) {
|
||||
return causeChainContains(error, CROSS_INSTANCE_LOCAL_POINTER_MARKER);
|
||||
}
|
||||
|
||||
/**
|
||||
* 沿 cause 链找字面量:包装层不许掩盖真实原因。
|
||||
*
|
||||
* <p>{@code cause == cursor} 的自引用按链尾处理,免得异常被自己当 cause 时把调用线程钉死。
|
||||
*/
|
||||
public static boolean causeChainContains(Throwable error, String marker) {
|
||||
if (error == null || marker == null || marker.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
Throwable cursor = error;
|
||||
while (cursor != null) {
|
||||
String message = cursor.getMessage();
|
||||
if (message != null && message.contains("does not exist")) {
|
||||
if (message != null && message.contains(marker)) {
|
||||
return true;
|
||||
}
|
||||
cursor = cursor.getCause();
|
||||
Throwable next = cursor.getCause();
|
||||
cursor = next == cursor ? null : next;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -51,7 +51,8 @@ class CatchSilenceRatchetTest {
|
||||
// 10-25 B15 join 无界收口抬 564 -> 569,逐处出处见 STRICT_NOT_RETHROW 上方注释。
|
||||
// 10-26 第二轮 CR-Fix 2 抬 569 -> 573,逐处出处见 STRICT_NOT_RETHROW 上方注释(4 处必要 fail-open)。
|
||||
static final int GIVEN_TOTAL = 1000;
|
||||
static final int GIVEN_NOT_RETHROW = 573;
|
||||
// 10-27 「整表都要跑完」专项抬 573 -> 577(2 枚新增不重抛 + 2 枚既有整批抛出改逐条跳过,逐处记名见 STRICT_TOTAL 上方)。
|
||||
static final int GIVEN_NOT_RETHROW = 577;
|
||||
static final int GIVEN_LOG_ONLY = 249;
|
||||
static final int GIVEN_EMPTY = 56;
|
||||
|
||||
@@ -141,8 +142,30 @@ class CatchSilenceRatchetTest {
|
||||
* </ul>
|
||||
* 9 处均为 CR-02/05/10/20 的必要出口,不是把已有异常改回吞掉;同批实测 total 1024、
|
||||
* notRethrow 573、logOnly 179、empty 56。
|
||||
*
|
||||
* <p>10-27 整表跑完专项抬 1024 -> 1026(用户口径:整表都要跑完,单项坏了跳过它继续下一项,
|
||||
* 只有 LLM 密钥欠费/失效那一类才允许直接判任务失败),逐处记名:
|
||||
* <ul>
|
||||
* <li>{@code BrandTaskStorageService.rebuildAggregate} 一枚:单片载荷读不到时
|
||||
* log.warn+记跳片号+{@code continue}(NOT_RETHROW)。此前一片抛就把整份聚合丢掉,
|
||||
* 而三个调用方(finalizeTask/abort/tryRecoverCompletedRunningTask)都没有就地 try,
|
||||
* 一个坏分片=整任务不产出;同批把 {@code completed} 的判据从 {@code chunks.size()}
|
||||
* (DB 行数)改成「真正读到的片数」,缺数据不再被判成已收齐。</li>
|
||||
* <li>{@code AppearancePatentLlmBatchPipeline.collectPendingLlmCandidates} 一枚:
|
||||
* 同为 log.warn+计数+{@code continue}(NOT_RETHROW)。collect 跑在批循环<b>之前</b>,
|
||||
* 一抛就走不到循环体内那道 {@code finally releaseRowInspection},本轮已申领的行要挂满
|
||||
* 处理中 TTL(15 分钟),且 {@code terminalizeRoundExhaustedRows} 跟着一起不执行。</li>
|
||||
* </ul>
|
||||
* 另有两枚<b>既有</b> RETHROW 出口按同一口径改成「跳过该项」,形状从 RETHROW 迁入气味两格
|
||||
* (total 不变、rethrow −2、notRethrow +2、logOnly +2):{@code BrandTaskStorageService.getParsedPayload}
|
||||
* 与 {@code BrandTaskStorageService.getAllFileAggregates}——两处此前一个文件的缓存/聚合读不到
|
||||
* 就整批抛,把已读到的其余文件一起抹掉;改后逐条 WARN 跳过,调用方按「尚未收齐」可重试。
|
||||
* ⚠ 这两处不是把异常静默吞掉:跳过都带 {@code [brand-storage]} 锚点 WARN,且
|
||||
* {@code BrandTaskStorageServiceFaultToleranceTest} 钉住「其余项照常出」与
|
||||
* 「全项坏掉仍大声抛」两条,缺数据一律不得冒充已完成。
|
||||
* 同批实测 total 1026、rethrow 393(仍高于下限 380,本格不动)、notRethrow 577、logOnly 181、empty 56。
|
||||
*/
|
||||
static final int STRICT_TOTAL = 1024;
|
||||
static final int STRICT_TOTAL = 1026;
|
||||
/**
|
||||
* 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。
|
||||
* 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。
|
||||
@@ -234,8 +257,13 @@ class CatchSilenceRatchetTest {
|
||||
* {@code OssStorageService.downloadWorkbenchClientObjectToTempFile} 内层清理(同上)、
|
||||
* {@code CostGuardScheduler}(平台自费告警推送失败 log+return,保留基线待下轮重试)。
|
||||
* 四处的误差方向都是「不让清理/告警失败掩盖主错误」,不是把异常静默吞掉。
|
||||
* 10-27 抬 573 -> 577(「整表都要跑完」专项,4 处不重抛,逐处记名见 STRICT_TOTAL 上方注释):
|
||||
* 新增 {@code BrandTaskStorageService.rebuildAggregate} 与
|
||||
* {@code AppearancePatentLlmBatchPipeline.collectPendingLlmCandidates} 两枚逐片跳过;
|
||||
* 另有 {@code BrandTaskStorageService.getParsedPayload}/{@code getAllFileAggregates} 两枚
|
||||
* 既有「整批抛出」出口按同口径改成逐条跳过(自 RETHROW 迁入本格)。
|
||||
*/
|
||||
static final int STRICT_NOT_RETHROW = 573;
|
||||
static final int STRICT_NOT_RETHROW = 577;
|
||||
/**
|
||||
* 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}:
|
||||
* 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出——
|
||||
@@ -251,8 +279,12 @@ class CatchSilenceRatchetTest {
|
||||
* 10-26 抬 176 -> 179(第二轮 CR-Fix 2,3 处新增 log-only):{@code WorkbenchClientPackageService}
|
||||
* 的 removeRejectedObject/deleteTempQuietly 与 {@code OssStorageService} 的临时文件清理内层,
|
||||
* 三处都只 WARN 且带 path/objectKey 锚点;主错误由外层重抛,清理失败不得盖掉它。
|
||||
* 10-27 抬 179 -> 181(「整表都要跑完」专项,2 处):{@code BrandTaskStorageService.getParsedPayload}
|
||||
* 与 {@code getAllFileAggregates} 的逐条跳过出口——块体各只有一句带 {@code [brand-storage]}
|
||||
* 锚点的 WARN。两处此前是「一个文件坏 ⇒ 整批抛出」,改后其余文件照常返回,坏的那条点名可查;
|
||||
* 同批 {@code rebuildAggregate} 与外观专利候选收集因块内另有计数/continue 只进 NOT_RETHROW。
|
||||
*/
|
||||
static final int STRICT_LOG_ONLY = 179; // 严格口径:块内每条顶层语句都是 log 调用才算
|
||||
static final int STRICT_LOG_ONLY = 181; // 严格口径:块内每条顶层语句都是 log 调用才算
|
||||
static final int STRICT_EMPTY = 56;
|
||||
|
||||
/**
|
||||
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentLlmBatchPipeline;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentResultRowReader;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.atLeast;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 候选收集阶段的「一片坏了不许拖垮整轮」(2026-10-27 生产误判后的用户口径:整表都要跑完,
|
||||
* 单项有问题就跳过它继续下一项,不能因为一个坏项影响全部)。
|
||||
*
|
||||
* <p>{@code collectPendingLlmCandidates} 跑在批循环<b>之前</b>,而批循环里的
|
||||
* {@code finally releaseRowInspection} 只在循环体内——collect 一抛就永远走不到那道释放:
|
||||
* <ul>
|
||||
* <li>本轮已申领的行挂在处理中标记上,要等 {@code IN_FLIGHT_TTL}(15 分钟)自己过期;</li>
|
||||
* <li>{@code terminalizeRoundExhaustedRows} 收的是 collect 的返回值,collect 抛出时它也一并跳过,
|
||||
* 轮次到顶的行因此永不判死。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>读失败的异常由<b>真实</b>的 {@link AppearancePatentResultRowReader} +
|
||||
* {@link TransientPayloadStorageService#resolvePayload} 产出,不在测试里手写文案——
|
||||
* 手写桩造的正是生产者早已不再产出的旧字面量(货源侧那支假绿用例的成因)。
|
||||
*/
|
||||
class AppearancePatentCandidateCollectionFaultToleranceTest {
|
||||
|
||||
private static final Long TASK_ID = 31610L;
|
||||
private static final String SCOPE_HASH = "scope-a";
|
||||
private static final String READABLE_HEAD =
|
||||
"[{\"rowToken\":\"r1\",\"id\":\"1\",\"asin\":\"B0A0000001\",\"country\":\"US\","
|
||||
+ "\"title\":\"RUNNING SHOE\",\"url\":\"http://img/1.jpg\",\"groupKey\":\"g1\"}]";
|
||||
private static final String READABLE_TAIL =
|
||||
"[{\"rowToken\":\"r3\",\"id\":\"3\",\"asin\":\"B0A0000003\",\"country\":\"US\","
|
||||
+ "\"title\":\"COTTON BAG\",\"url\":\"http://img/3.jpg\",\"groupKey\":\"g3\"}]";
|
||||
/** 归另一台实例的 local 指针:resolvePayload 抛中文「临时载荷仅存在于实例 …」,reader 再包成 BusinessException。 */
|
||||
private static final String CROSS_INSTANCE_CHUNK =
|
||||
"local:i/server-110/task-chunk/APPEARANCE_PATENT/" + TASK_ID + "/" + SCOPE_HASH + "/chunk-2.json";
|
||||
private static final String MISSING_OBJECT_KEY =
|
||||
"task-chunk/APPEARANCE_PATENT/" + TASK_ID + "/" + SCOPE_HASH + "/chunk-2.json";
|
||||
|
||||
private AppearancePatentTaskCacheService taskCacheService;
|
||||
private AppearancePatentResultRowReader reader;
|
||||
private AppearancePatentLlmBatchPipeline pipeline;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
taskCacheService = mock(AppearancePatentTaskCacheService.class);
|
||||
// 载荷读不到的那一片走真实存储层(跨实例中文原文由它产出),其余两片按裸 JSON 直读
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
when(rustfs.isConfigured()).thenReturn(true);
|
||||
when(rustfs.readObjectBytes(MISSING_OBJECT_KEY)).thenThrow(new IllegalStateException(
|
||||
"The specified key does not exist. (Service: S3, Status Code: 404)"));
|
||||
when(rustfs.listObjectKeysNewestFirst(anyString(), org.mockito.ArgumentMatchers.anyInt())).thenReturn(List.of());
|
||||
TransientPayloadStorageService storage = new TransientPayloadStorageService(
|
||||
mock(TransientStorageProperties.class), mock(StorageProperties.class), rustfs,
|
||||
mock(OssStorageService.class), new ObjectMapper(), instanceMetadata("server-1"),
|
||||
mock(TaskChunkMapper.class), mock(TaskScopeStateMapper.class));
|
||||
reader = new AppearancePatentResultRowReader(
|
||||
AppearancePatentTaskService.MODULE_TYPE, new ObjectMapper(), storage, mock(TaskChunkMapper.class),
|
||||
value -> value == null ? "" : value.trim(), task -> new AppearancePatentParsedPayloadDto());
|
||||
pipeline = pipeline(reader);
|
||||
}
|
||||
|
||||
/**
|
||||
* 中间那片读不到:跳过它,第三片仍要进候选。改前 collect 直接抛 BusinessException,
|
||||
* 于是第一片已申领的 r1 挂在处理中标记上(15 分钟)、第三片根本没被读过。
|
||||
*/
|
||||
@Test
|
||||
void unreadableChunkDoesNotAbortCandidateCollection() {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
|
||||
List<AppearancePatentResultRowDto> candidates = pipeline.collectPendingLlmCandidates(
|
||||
List.of(chunk(1, READABLE_HEAD), chunk(2, CROSS_INSTANCE_CHUNK), chunk(3, READABLE_TAIL)),
|
||||
new AppearancePatentLlmBatchPipeline.ChunkRowKeyIndex());
|
||||
|
||||
assertEquals(List.of("r1", "r3"), rowTokens(candidates),
|
||||
"坏的那一片要跳过,其余两片照常进候选");
|
||||
}
|
||||
|
||||
/**
|
||||
* 核心不变量(逐片容错与「异常路径统一 release」两种合法修法都满足,唯独现状不满足):
|
||||
* 凡是申领成功的行,要么进候选、要么被显式 release,**不许**挂在处理中标记上等 TTL 自己过期。
|
||||
*/
|
||||
@Test
|
||||
void claimedRowsAreNeverLeftInFlightWhenAChunkFails() {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
|
||||
List<String> collected;
|
||||
String thrown = null;
|
||||
try {
|
||||
collected = rowTokens(pipeline.collectPendingLlmCandidates(
|
||||
List.of(chunk(1, READABLE_HEAD), chunk(2, CROSS_INSTANCE_CHUNK)),
|
||||
new AppearancePatentLlmBatchPipeline.ChunkRowKeyIndex()));
|
||||
} catch (RuntimeException ex) {
|
||||
thrown = ex.getClass().getSimpleName() + ": " + ex.getMessage();
|
||||
collected = List.of();
|
||||
}
|
||||
|
||||
ArgumentCaptor<String> claimed = ArgumentCaptor.forClass(String.class);
|
||||
verify(taskCacheService, atLeastOnce()).beginRowInspection(eq(TASK_ID), claimed.capture());
|
||||
Set<String> released = releasedKeys();
|
||||
assertTrue(claimed.getAllValues().contains("r1"), "夹具没能让第一片真的申领到行");
|
||||
for (String key : claimed.getAllValues()) {
|
||||
assertTrue(collected.contains(key) || released.contains(key),
|
||||
"行 " + key + " 已申领却既没进候选也没释放——会挂满处理中 TTL 15 分钟;"
|
||||
+ "抛出=" + thrown + ",候选=" + collected + ",已释放=" + released);
|
||||
}
|
||||
}
|
||||
|
||||
/** 对象缺失(NoSuchKey)那一种 reader 自己就跳了,collect 不得因它丢其余分片。 */
|
||||
@Test
|
||||
void missingObjectChunkIsSkippedAndCollectionContinues() {
|
||||
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||
|
||||
List<AppearancePatentResultRowDto> candidates = pipeline.collectPendingLlmCandidates(
|
||||
List.of(chunk(1, READABLE_HEAD), chunk(2, "rustfs:" + MISSING_OBJECT_KEY), chunk(3, READABLE_TAIL)),
|
||||
new AppearancePatentLlmBatchPipeline.ChunkRowKeyIndex());
|
||||
|
||||
assertEquals(List.of("r1", "r3"), rowTokens(candidates));
|
||||
}
|
||||
|
||||
/**
|
||||
* reader 侧契约不变:跨实例失败照旧抛出(它只放行「重试也没用」的对象缺失)。
|
||||
* 这一支钉的是「容错做在 collect 那一边,没有把 reader 的判据一起放宽」。
|
||||
*/
|
||||
@Test
|
||||
void readerStillThrowsForCrossInstanceChunkPayload() {
|
||||
BusinessException thrown = assertThrows(BusinessException.class,
|
||||
() -> reader.readChunkRows(chunk(2, CROSS_INSTANCE_CHUNK)));
|
||||
assertTrue(thrown.getMessage().contains("读取外观专利结果分片失败"), "实际: " + thrown.getMessage());
|
||||
}
|
||||
|
||||
// ===== 夹具 =====
|
||||
|
||||
private AppearancePatentLlmBatchPipeline pipeline(AppearancePatentResultRowReader rowReader) {
|
||||
return new AppearancePatentLlmBatchPipeline(
|
||||
mock(AppearancePatentProperties.class),
|
||||
mock(AppearancePatentLlmClient.class),
|
||||
taskCacheService,
|
||||
mock(TaskFileJobService.class),
|
||||
row -> row == null || row.getRowToken() == null ? "" : row.getRowToken(),
|
||||
row -> true,
|
||||
rowReader::readChunkRows,
|
||||
taskId -> List.of(),
|
||||
taskId -> false,
|
||||
taskId -> {
|
||||
},
|
||||
task -> "prompt",
|
||||
task -> "sk-test",
|
||||
(taskId, scopeHash, chunkIndex, rows) -> List.of(),
|
||||
(preferred, fallback) -> preferred == null || preferred.isBlank() ? fallback : preferred);
|
||||
}
|
||||
|
||||
private Set<String> releasedKeys() {
|
||||
ArgumentCaptor<List<String>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(taskCacheService, atLeast(0)).releaseRowInspection(anyLong(), captor.capture());
|
||||
return captor.getAllValues().stream().flatMap(List::stream).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
private static List<String> rowTokens(List<AppearancePatentResultRowDto> rows) {
|
||||
return rows.stream().map(AppearancePatentResultRowDto::getRowToken).toList();
|
||||
}
|
||||
|
||||
private static InstanceMetadata instanceMetadata(String instanceId) {
|
||||
InstanceMetadata metadata = mock(InstanceMetadata.class);
|
||||
when(metadata.getInstanceId()).thenReturn(instanceId);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private static TaskChunkEntity chunk(int index, String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId((long) index);
|
||||
chunk.setTaskId(TASK_ID);
|
||||
chunk.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
|
||||
chunk.setScopeHash(SCOPE_HASH);
|
||||
chunk.setChunkIndex(index);
|
||||
chunk.setPayloadJson(payloadJson);
|
||||
chunk.setPayloadHash("hash-" + index);
|
||||
return chunk;
|
||||
}
|
||||
}
|
||||
+268
@@ -0,0 +1,268 @@
|
||||
package com.nanri.aiimage.modules.brand.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
|
||||
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
|
||||
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyBoolean;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 品牌任务存储层的「单项坏了不许拖垮整批」口径(2026-10-27 生产误判后的用户三条硬口径:
|
||||
* 整表都要跑完,读不到的一项跳过并留痕、继续下一项;只有 LLM 密钥欠费/失效那一类才允许直接判任务失败)。
|
||||
*
|
||||
* <p>此前 {@code rebuildAggregate} 逐片读时任一片抛出就把整份聚合丢掉,而三个调用方
|
||||
* (finalizeTask/abort/tryRecoverCompletedRunningTask)都没有就地 try,于是:
|
||||
* <ul>
|
||||
* <li>一个坏分片 ⇒ 整个文件甚至整个任务不产出结果;</li>
|
||||
* <li>{@code completed} 拿 {@code chunks.size()}(DB 行数)比 {@code chunkTotal}
|
||||
* ⇒ 载荷缺失的行照样算「已收齐」,正是「118/119 行没数据仍报已完成」的形状。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>读失败的异常一律由<b>真实</b> {@link TransientPayloadStorageService#resolvePayload} 产出
|
||||
* (在测试里桩造字面量正是货源侧那支假绿用例的成因,见 {@code SimilarAsinChunkPayloadSupport} 同批修复)。
|
||||
* 跨实例 local 指针与 RustFS NoSuchKey 两种真实形状各测一次。
|
||||
*/
|
||||
class BrandTaskStorageServiceFaultToleranceTest {
|
||||
|
||||
private static final Long TASK_ID = 31600L;
|
||||
private static final String FILE_URL = "https://oss/brand/source.xlsx";
|
||||
private static final ObjectMapper JSON = new ObjectMapper();
|
||||
|
||||
private TaskChunkMapper taskChunkMapper;
|
||||
private TaskScopeStateMapper taskScopeStateMapper;
|
||||
private TransientPayloadStorageService transientPayloadStorage;
|
||||
private RustfsObjectStorageService rustfs;
|
||||
private BrandTaskStorageService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
taskChunkMapper = mock(TaskChunkMapper.class);
|
||||
taskScopeStateMapper = mock(TaskScopeStateMapper.class);
|
||||
rustfs = mock(RustfsObjectStorageService.class);
|
||||
// producer 是真实存储层(负责抛真实的中文/NoSuchKey 原文),service 侧只是把调用转给它,
|
||||
// 于是「读到什么异常」与「判定端认不认得出」用的是同一份产出。
|
||||
TransientPayloadStorageService producer = new TransientPayloadStorageService(
|
||||
mock(TransientStorageProperties.class),
|
||||
mock(StorageProperties.class),
|
||||
rustfs,
|
||||
mock(OssStorageService.class),
|
||||
JSON,
|
||||
instanceMetadata("server-1"),
|
||||
taskChunkMapper,
|
||||
taskScopeStateMapper);
|
||||
transientPayloadStorage = mock(TransientPayloadStorageService.class);
|
||||
when(transientPayloadStorage.resolvePayload(anyString(), anyString())).thenAnswer(invocation ->
|
||||
producer.resolvePayload(invocation.getArgument(0), invocation.getArgument(1)));
|
||||
service = new BrandTaskStorageService(taskChunkMapper, taskScopeStateMapper, JSON, transientPayloadStorage);
|
||||
}
|
||||
|
||||
/**
|
||||
* 三片里中间那片对象被删(RustFS NoSuchKey):跳过它,其余两片的数据必须照样聚合出来,
|
||||
* 且**不许**因为「DB 里有 3 行 = chunkTotal 3」就判成已收齐。
|
||||
*/
|
||||
@Test
|
||||
void missingChunkPayloadIsSkippedAndNeverCountedAsComplete() throws Exception {
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(
|
||||
chunk(1, chunkJson(1, 3, "品牌A")),
|
||||
chunk(2, "rustfs:task-chunk/BRAND/" + TASK_ID + "/scope-a/chunk-2.json"),
|
||||
chunk(3, chunkJson(3, 3, "品牌C"))));
|
||||
stubMissingObject("task-chunk/BRAND/" + TASK_ID + "/scope-a/chunk-2.json");
|
||||
stubAggregateWrite();
|
||||
|
||||
BrandTaskStorageService.ChunkStoreResult result = service.refreshFileAggregate(TASK_ID, FILE_URL);
|
||||
BrandFileAggregateCacheDto aggregate = result.aggregate();
|
||||
|
||||
assertNotNull(aggregate, "单个坏分片不得让整份聚合消失");
|
||||
assertEquals(List.of("品牌A", "品牌C"), aggregate.getKeptBrands(), "其余分片的数据必须照常聚合");
|
||||
assertEquals(2, aggregate.getReceivedChunkCount(), "已收数只认真正读到的片");
|
||||
assertFalse(Boolean.TRUE.equals(aggregate.getCompleted()), "缺数据的文件不许被判成已完成");
|
||||
assertFalse(result.newlyCompletedFile(), "缺口未补上时不得回报「本文件刚完成」");
|
||||
}
|
||||
|
||||
/**
|
||||
* 跨实例 local 指针(载荷只在另一台机器上)同样按「单项跳过」处理:
|
||||
* 这一形状此前抛出后被 {@code tryRecoverCompletedRunningTask} 的外层 catch 变成
|
||||
* {@code return false}——放弃整任务恢复,而不是跳过这一片。
|
||||
*/
|
||||
@Test
|
||||
void crossInstanceLocalPointerIsSkippedAndOtherChunksSurvive() throws Exception {
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(
|
||||
chunk(1, chunkJson(1, 2, "品牌A")),
|
||||
chunk(2, "local:i/server-110/task-chunk/BRAND/" + TASK_ID + "/scope-a/chunk-2.json")));
|
||||
stubAggregateWrite();
|
||||
|
||||
BrandFileAggregateCacheDto aggregate = service.refreshFileAggregate(TASK_ID, FILE_URL).aggregate();
|
||||
|
||||
assertNotNull(aggregate);
|
||||
assertEquals(List.of("品牌A"), aggregate.getKeptBrands());
|
||||
assertEquals(1, aggregate.getReceivedChunkCount());
|
||||
assertFalse(Boolean.TRUE.equals(aggregate.getCompleted()));
|
||||
}
|
||||
|
||||
/** 反向对账:一片都不坏时,收齐判据必须照旧成立(别让容错把「已完成」判成永不完成)。 */
|
||||
@Test
|
||||
void allChunksReadableStillMarksTheFileComplete() throws Exception {
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(
|
||||
chunk(1, chunkJson(1, 2, "品牌A")),
|
||||
chunk(2, chunkJson(2, 2, "品牌B"))));
|
||||
stubAggregateWrite();
|
||||
|
||||
BrandTaskStorageService.ChunkStoreResult result = service.refreshFileAggregate(TASK_ID, FILE_URL);
|
||||
|
||||
assertEquals(2, result.aggregate().getReceivedChunkCount());
|
||||
assertTrue(Boolean.TRUE.equals(result.aggregate().getCompleted()), "两片齐了就该判已完成");
|
||||
assertTrue(result.newlyCompletedFile());
|
||||
}
|
||||
|
||||
/** 全部分片都读不到时才允许抛出——这时没有任何数据可交付,跳过也变不出结果。 */
|
||||
@Test
|
||||
void everyChunkUnreadableStillFailsLoudly() throws Exception {
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(
|
||||
chunk(1, "rustfs:task-chunk/BRAND/" + TASK_ID + "/scope-a/chunk-1.json")));
|
||||
stubMissingObject("task-chunk/BRAND/" + TASK_ID + "/scope-a/chunk-1.json");
|
||||
|
||||
BusinessException thrown = assertThrows(BusinessException.class,
|
||||
() -> service.refreshFileAggregate(TASK_ID, FILE_URL));
|
||||
assertTrue(thrown.getMessage().contains("全部分片载荷均不可读"),
|
||||
"抛出文案要点名是整体缺失,实际: " + thrown.getMessage());
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析缓存:一个文件的缓存载荷读不到不许把整份缓存抹掉——
|
||||
* 抛出会让 {@code BrandTaskService.finalizeTask/abort} 整单失败(那两处没有就地 try)。
|
||||
*/
|
||||
@Test
|
||||
void parsedPayloadSkipsUnreadableFileAndKeepsTheRest() throws Exception {
|
||||
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(
|
||||
parsedState("rustfs:task-parsed/BRAND/" + TASK_ID + "/scope-a/latest.json"),
|
||||
parsedState("{\"fileUrl\":\"" + FILE_URL + "\",\"originalFilename\":\"source.xlsx\"}")));
|
||||
stubMissingObject("task-parsed/BRAND/" + TASK_ID + "/scope-a/latest.json");
|
||||
|
||||
List<BrandParsedFileCacheDto> payload = service.getParsedPayload(TASK_ID);
|
||||
|
||||
assertEquals(1, payload.size(), "坏的那条跳过,好的那条必须还在");
|
||||
assertEquals(FILE_URL, payload.get(0).getFileUrl());
|
||||
}
|
||||
|
||||
/** 汇总状态:同口径,坏的那一个文件缺席,其余文件照常返回(调用方按「尚未收齐」重试而不是整单 500)。 */
|
||||
@Test
|
||||
void allFileAggregatesSkipsUnreadableEntryAndKeepsTheRest() throws Exception {
|
||||
TaskScopeStateEntity broken = new TaskScopeStateEntity();
|
||||
broken.setTaskId(TASK_ID);
|
||||
broken.setModuleType("BRAND");
|
||||
broken.setScopeKey("https://oss/brand/broken.xlsx");
|
||||
broken.setStateJson("rustfs:task-scope/BRAND/" + TASK_ID + "/scope-broken/latest.json");
|
||||
TaskScopeStateEntity healthy = new TaskScopeStateEntity();
|
||||
healthy.setTaskId(TASK_ID);
|
||||
healthy.setModuleType("BRAND");
|
||||
healthy.setScopeKey(FILE_URL);
|
||||
healthy.setStateJson("{\"fileUrl\":\"" + FILE_URL + "\",\"chunkTotal\":1,\"receivedChunkCount\":1,\"completed\":true}");
|
||||
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(broken, healthy));
|
||||
stubMissingObject("task-scope/BRAND/" + TASK_ID + "/scope-broken/latest.json");
|
||||
|
||||
Map<String, BrandFileAggregateCacheDto> aggregates = service.getAllFileAggregates(TASK_ID);
|
||||
|
||||
assertEquals(1, aggregates.size(), "坏的那个文件缺席,好的那个必须还在");
|
||||
assertTrue(Boolean.TRUE.equals(aggregates.get(FILE_URL).getCompleted()));
|
||||
}
|
||||
|
||||
// ===== 夹具 =====
|
||||
|
||||
private static InstanceMetadata instanceMetadata(String instanceId) {
|
||||
InstanceMetadata metadata = mock(InstanceMetadata.class);
|
||||
when(metadata.getInstanceId()).thenReturn(instanceId);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private static TaskChunkEntity chunk(int index, String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId((long) index);
|
||||
chunk.setTaskId(TASK_ID);
|
||||
chunk.setModuleType("BRAND");
|
||||
chunk.setScopeKey(FILE_URL);
|
||||
chunk.setScopeHash("scope-a");
|
||||
chunk.setChunkIndex(index);
|
||||
chunk.setChunkTotal(3);
|
||||
chunk.setPayloadJson(payloadJson);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private static String chunkJson(int index, int total, String brand) throws Exception {
|
||||
BrandCrawlResultFileDto file = new BrandCrawlResultFileDto();
|
||||
file.setFileUrl(FILE_URL);
|
||||
file.setOriginalFilename("source.xlsx");
|
||||
file.setChunkIndex(index);
|
||||
file.setChunkTotal(total);
|
||||
file.setTotalLines(119);
|
||||
file.setKeptRows(List.of(brand));
|
||||
return JSON.writeValueAsString(file);
|
||||
}
|
||||
|
||||
private static TaskScopeStateEntity parsedState(String payloadJson) {
|
||||
TaskScopeStateEntity state = new TaskScopeStateEntity();
|
||||
state.setTaskId(TASK_ID);
|
||||
state.setModuleType("BRAND");
|
||||
state.setScopeHash("scope-a");
|
||||
state.setParsedPayloadJson(payloadJson);
|
||||
return state;
|
||||
}
|
||||
|
||||
private void stubMissingObject(String objectKey) {
|
||||
when(rustfs.isConfigured()).thenReturn(true);
|
||||
when(rustfs.readObjectBytes(objectKey)).thenThrow(new IllegalStateException(
|
||||
"The specified key does not exist. (Service: S3, Status Code: 404)"));
|
||||
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of());
|
||||
}
|
||||
|
||||
private void stubAggregateWrite() {
|
||||
TaskScopeStateEntity state = new TaskScopeStateEntity();
|
||||
state.setId(9L);
|
||||
state.setTaskId(TASK_ID);
|
||||
state.setModuleType("BRAND");
|
||||
state.setScopeKey(FILE_URL);
|
||||
state.setStateJson("\"rustfs:task-scope/BRAND/" + TASK_ID + "/scope-a/latest.json\"");
|
||||
when(taskScopeStateMapper.selectOne(any())).thenReturn(state);
|
||||
when(taskScopeStateMapper.update(any(), any())).thenReturn(1);
|
||||
when(taskScopeStateMapper.selectCount(any())).thenReturn(0L);
|
||||
when(transientPayloadStorage.storeScopePayloadVersioned(anyString(), any(), anyString(), anyString(), anyBoolean()))
|
||||
.thenReturn("\"rustfs:task-scope/BRAND/" + TASK_ID + "/scope-a/new.json\"");
|
||||
}
|
||||
}
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 货源分片载荷读取的降级判据(2026-10-27 复验定性为假绿的那两处)。
|
||||
*
|
||||
* <p>判定端此前匹英文 {@code only exists on instance=},而 {@code TransientPayloadStorageService}
|
||||
* 真正抛出的是中文「临时载荷仅存在于实例 X(当前实例 Y)…」,且 {@code resolvePayload} 还会把它
|
||||
* rewrap 成「{@code <errorMessage>}: {@code <pointer>}」(中文只在 cause)——于是
|
||||
* <ul>
|
||||
* <li>{@code crossInstance} 恒 false,{@code chunk-read-failed[N]@cross-instance} 永不下笔;</li>
|
||||
* <li>{@code does not exist} 同理只摸顶层消息,「对象已删就跳过该分片」那条降级分支也是死的,
|
||||
* 一个坏分片会把整单组装/收尾拖成抛错。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>这里不再桩造任何文案:异常一律由<b>真实</b>的 {@link TransientPayloadStorageService#resolvePayload}
|
||||
* 产出(本地磁盘/对象存储那两条真实读取路径),判定端不沿 cause 链匹配就判不出来。
|
||||
*/
|
||||
class SimilarAsinChunkPayloadSupportDegradedReaderTest {
|
||||
|
||||
private static final Long TASK_ID = 31577L;
|
||||
private static final String LOCAL_OBJECT_KEY =
|
||||
"task-chunk/SIMILAR_ASIN/" + TASK_ID + "/scope-a/chunk-2.json";
|
||||
private static final String MISSING_OBJECT_KEY =
|
||||
"task-chunk/SIMILAR_ASIN/" + TASK_ID + "/scope-a/chunk-9.json";
|
||||
private static final String ROWS_JSON =
|
||||
"[{\"rowToken\":\"r1\",\"id\":\"1\",\"asin\":\"B0A0000001\",\"country\":\"英国\"}]";
|
||||
|
||||
private TaskScopeStateMapper taskScopeStateMapper;
|
||||
private TransientPayloadStorageService storage;
|
||||
private SimilarAsinChunkPayloadSupport support;
|
||||
|
||||
@BeforeAll
|
||||
static void initMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
taskScopeStateMapper = mock(TaskScopeStateMapper.class);
|
||||
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||
when(rustfs.isConfigured()).thenReturn(true);
|
||||
when(rustfs.readObjectBytes(MISSING_OBJECT_KEY)).thenThrow(new IllegalStateException(
|
||||
"The specified key does not exist. (Service: S3, Status Code: 404)"));
|
||||
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of());
|
||||
storage = new TransientPayloadStorageService(
|
||||
mock(TransientStorageProperties.class), mock(StorageProperties.class), rustfs,
|
||||
mock(OssStorageService.class), new ObjectMapper(), instanceMetadata("server-1"),
|
||||
mock(TaskChunkMapper.class), taskScopeStateMapper);
|
||||
support = new SimilarAsinChunkPayloadSupport(new ObjectMapper(), storage, taskScopeStateMapper);
|
||||
TaskScopeStateEntity scope = new TaskScopeStateEntity();
|
||||
scope.setId(55L);
|
||||
scope.setLastError("");
|
||||
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope);
|
||||
}
|
||||
|
||||
/** 跨实例 local 指针:仍抛出(调用方要据此重试/换实例),但留痕必须带 {@code @cross-instance}。 */
|
||||
@Test
|
||||
void crossInstanceLocalPointerStaysFatalButIsTaggedAsCrossInstance() {
|
||||
TaskChunkEntity chunk = chunk(2, "local:i/server-110/" + LOCAL_OBJECT_KEY);
|
||||
|
||||
BusinessException thrown = assertThrows(BusinessException.class, () -> support.readChunkRows(chunk));
|
||||
assertTrue(thrown.getMessage().contains("分片=2"), "异常要点名分片,实际: " + thrown.getMessage());
|
||||
String written = writtenTag();
|
||||
assertTrue(written.contains("chunk-read-failed[2@cross-instance]"),
|
||||
"跨实例必须留 @cross-instance 痕,实际写进 last_error 的是: " + written);
|
||||
}
|
||||
|
||||
/** 对象已被删:重试也没用,跳过该分片出空行、只留普通痕,不许把整单组装拖死。 */
|
||||
@Test
|
||||
void missingObjectChunkIsSkippedInsteadOfKillingTheAssembly() {
|
||||
TaskChunkEntity chunk = chunk(9, "rustfs:" + MISSING_OBJECT_KEY);
|
||||
|
||||
Map<String, SimilarAsinResultRowDto> rows = support.readChunkRows(chunk);
|
||||
|
||||
assertTrue(rows.isEmpty(), "缺失载荷按空片跳过");
|
||||
String written = writtenTag();
|
||||
assertTrue(written.contains("chunk-read-failed[9]"), "跳过也要留痕,实际: " + written);
|
||||
assertFalse(written.contains("@cross-instance"), "对象缺失不是跨实例,不许冒充跨实例标记,实际: " + written);
|
||||
}
|
||||
|
||||
/** 载荷健在时必须照常出行(容错不许把正常路径一起判空)。 */
|
||||
@Test
|
||||
void readableChunkStillYieldsItsRows() {
|
||||
Map<String, SimilarAsinResultRowDto> rows = support.readChunkRows(chunk(3, ROWS_JSON));
|
||||
|
||||
assertEquals(1, rows.size());
|
||||
assertTrue(rows.containsKey("r1"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 留痕与读回是一件事的两半:读回端({@code :153} 的 {@code endsWith("@cross-instance")})此前
|
||||
* 拿到的永远是裸标记,因为写端认不出跨实例。这里把<b>真实写进去的那枚标记</b>原样喂回读端,
|
||||
* 断言用户可见摘要会说「跨实例」——写端一改回旧英文串,这支就红。
|
||||
*/
|
||||
@Test
|
||||
void writtenCrossInstanceTagIsRenderedAsCrossInstanceInSummary() {
|
||||
assertThrows(BusinessException.class,
|
||||
() -> support.readChunkRows(chunk(2, "local:i/server-110/" + LOCAL_OBJECT_KEY)),
|
||||
"跨实例载荷仍要抛出(由调用方重试/换实例)");
|
||||
String written = writtenTagOfLastUpdate();
|
||||
TaskScopeStateEntity scope = new TaskScopeStateEntity();
|
||||
scope.setTaskId(TASK_ID);
|
||||
scope.setModuleType("SIMILAR_ASIN");
|
||||
scope.setLastError(written);
|
||||
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(scope));
|
||||
|
||||
String summary = support.collectChunkReadFailureSummary(TASK_ID);
|
||||
|
||||
assertTrue(summary.contains("跨实例"), "摘要必须把跨实例与一般读失败分开,实际写痕="
|
||||
+ written + ",摘要=" + summary);
|
||||
}
|
||||
|
||||
/** 取上一次 update 真正写进 last_error 的那枚标记(时间戳参数按前缀滤掉)。 */
|
||||
private String writtenTagOfLastUpdate() {
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskScopeStateEntity>> captor =
|
||||
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(taskScopeStateMapper).update(isNull(), captor.capture());
|
||||
return captor.getValue().getParamNameValuePairs().values().stream()
|
||||
.map(String::valueOf)
|
||||
.filter(value -> value.startsWith("chunk-read-failed["))
|
||||
.findFirst()
|
||||
.orElse("");
|
||||
}
|
||||
|
||||
// ===== 夹具 =====
|
||||
|
||||
private String writtenTag() {
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskScopeStateEntity>> captor =
|
||||
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(taskScopeStateMapper).update(isNull(), captor.capture());
|
||||
return String.join("|", captor.getValue().getParamNameValuePairs().values().stream()
|
||||
.map(String::valueOf).toList());
|
||||
}
|
||||
|
||||
private static InstanceMetadata instanceMetadata(String instanceId) {
|
||||
InstanceMetadata metadata = mock(InstanceMetadata.class);
|
||||
when(metadata.getInstanceId()).thenReturn(instanceId);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private static TaskChunkEntity chunk(int index, String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId((long) index);
|
||||
chunk.setTaskId(TASK_ID);
|
||||
chunk.setModuleType("SIMILAR_ASIN");
|
||||
chunk.setScopeKey("https://oss/similar-asin/source.xlsx");
|
||||
chunk.setScopeHash("scope-a");
|
||||
chunk.setChunkIndex(index);
|
||||
chunk.setPayloadJson(payloadJson);
|
||||
return chunk;
|
||||
}
|
||||
}
|
||||
+43
-1
@@ -34,6 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -356,6 +357,16 @@ class SimilarAsinHistoryAssemblerTest {
|
||||
assertTrue(item.getFileProgressMessage().endsWith(" 行"), "无 task 无 chunk 仍按行回退,实际: " + item.getFileProgressMessage());
|
||||
}
|
||||
|
||||
/**
|
||||
* 跨实例 local 指针的留痕标记必须是 `chunk-read-failed[N@cross-instance]`。
|
||||
*
|
||||
* <p><b>异常原文一律由真实生产者给出,测试里不许手写文案。</b>此前这支用例桩的是
|
||||
* {@code "payload only exists on instance=server-110"} —— 那是 {@code TransientPayloadStorageService}
|
||||
* 早已不再产出的旧英文串,而真实抛出的是中文「临时载荷仅存在于实例 X(当前实例 Y)…」,
|
||||
* 且 {@code resolvePayload} 还会把它 rewrap 成「<errorMessage>: <pointer>」(中文只在 cause)。
|
||||
* 判定端只摸顶层消息 ⇒ {@code crossInstance} 恒 false ⇒ 断言照旧绿,是假绿本体。
|
||||
* 现在这里跑真实 {@code resolvePayload} 取异常,判定端不沿 cause 链就判不出跨实例。
|
||||
*/
|
||||
@Test
|
||||
void test_assembler_chunk_read_failure_records_error() {
|
||||
// 回退路径读 chunk 载荷失败 → 抛 BusinessException 且 last_error 留痕
|
||||
@@ -378,9 +389,12 @@ class SimilarAsinHistoryAssemblerTest {
|
||||
scope.setLastError("");
|
||||
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
|
||||
|
||||
// 先由真实生产者产出异常原文,再做桩——在 when(...) 的参数里造 mock 会触发 UnfinishedStubbing
|
||||
IllegalStateException crossInstanceFailure = crossInstanceFailureFromRealProducer();
|
||||
|
||||
when(fileTaskMapper.selectById(10L)).thenReturn(t);
|
||||
when(transientPayloadStorageService.resolvePayload("ref:missing", "read similar ASIN chunk failed"))
|
||||
.thenThrow(new IllegalStateException("payload only exists on instance=server-110"));
|
||||
.thenThrow(crossInstanceFailure);
|
||||
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(state));
|
||||
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope);
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
@@ -396,4 +410,32 @@ class SimilarAsinHistoryAssemblerTest {
|
||||
.map(String::valueOf).toList());
|
||||
assertTrue(written.contains("chunk-read-failed[2@cross-instance]"), "跨实例标记留痕,实际: " + written);
|
||||
}
|
||||
|
||||
/**
|
||||
* 用**真实**的 {@link TransientPayloadStorageService#resolvePayload} 产出跨实例失败异常,
|
||||
* 而不是在测试里抄一份字面量:抄的那份会随生产者改文案而悄悄失效(就是本支用例原来的样子)。
|
||||
*/
|
||||
private IllegalStateException crossInstanceFailureFromRealProducer() {
|
||||
com.nanri.aiimage.config.TransientStorageProperties transientProps =
|
||||
mock(com.nanri.aiimage.config.TransientStorageProperties.class);
|
||||
com.nanri.aiimage.config.StorageProperties storageProps =
|
||||
mock(com.nanri.aiimage.config.StorageProperties.class);
|
||||
com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService rustfs =
|
||||
mock(com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService.class);
|
||||
com.nanri.aiimage.modules.file.service.oss.OssStorageService oss =
|
||||
mock(com.nanri.aiimage.modules.file.service.oss.OssStorageService.class);
|
||||
com.nanri.aiimage.config.InstanceMetadata instanceMetadata =
|
||||
mock(com.nanri.aiimage.config.InstanceMetadata.class);
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("server-1");
|
||||
TransientPayloadStorageService producer = new TransientPayloadStorageService(
|
||||
transientProps, storageProps, rustfs, oss, new ObjectMapper(), instanceMetadata,
|
||||
mock(com.nanri.aiimage.modules.task.mapper.TaskChunkMapper.class),
|
||||
mock(com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper.class));
|
||||
// 指针归另一台实例(i/server-110),当前实例 server-1 —— 真实抛点 TransientPayloadStorageService:146
|
||||
IllegalStateException failure = assertThrows(IllegalStateException.class, () -> producer.resolvePayload(
|
||||
"local:i/server-110/task-chunk/SIMILAR_ASIN/10/s1/chunk-2.json", "read similar ASIN chunk failed"));
|
||||
assertTrue(failure.getMessage().startsWith("read similar ASIN chunk failed: local:"),
|
||||
"生产者会把原因 rewrap 成「<errorMessage>: <pointer>」,实际: " + failure.getMessage());
|
||||
return failure;
|
||||
}
|
||||
}
|
||||
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.common.task.TaskStatuses;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 迟到回传重开的判定面(2026-10-27 复验第 3 条:白名单里认不出自家无回传判死生产者写的文案)。
|
||||
*
|
||||
* <p>客户端还在补发时,判死巡检只把 {@code biz_file_task} 改成 FAILED,旧逻辑按终态回 40901,
|
||||
* 客户端拿到 {@code success=true} 就删本地副本 ⇒ 已采集的数据整批丢掉(生产 31577/31472)。
|
||||
* 判据此前是一串各自手抄的 error_message 字面量,与生产者之间没有任何编译期约束:
|
||||
* 生产者改一个字,重开就静默失效。现在文案由
|
||||
* {@link SimilarAsinStaleReopenSupport#clientStallFailureMessage(long, String)} 单点产出,
|
||||
* 生产者与判定端都从它取数。
|
||||
*/
|
||||
class SimilarAsinStaleReopenSupportTest {
|
||||
|
||||
private static final Long TASK_ID = 31577L;
|
||||
|
||||
private FileTaskMapper fileTaskMapper;
|
||||
private SimilarAsinTaskOwnershipSupport ownershipSupport;
|
||||
private SimilarAsinStaleReopenSupport support;
|
||||
|
||||
@BeforeAll
|
||||
static void initMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
fileTaskMapper = mock(FileTaskMapper.class);
|
||||
ownershipSupport = mock(SimilarAsinTaskOwnershipSupport.class);
|
||||
support = new SimilarAsinStaleReopenSupport(
|
||||
fileTaskMapper, mock(FileResultMapper.class), mock(TaskFileJobService.class),
|
||||
mock(SimilarAsinTaskCacheService.class), ownershipSupport);
|
||||
when(ownershipSupport.isOwnerCurrent(any())).thenReturn(true);
|
||||
when(ownershipSupport.ownerFromTask(any())).thenReturn("server-1");
|
||||
when(fileTaskMapper.update(any(), any())).thenReturn(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判死生产者那句话必须能重开。这一支单独钉「尾句」——它此前只靠白名单里的
|
||||
* 「任务创建至今超过」那一枚前缀碰巧命中,生产者一改前缀(例如改成「建单至今」)
|
||||
* 重开就整条失效,而现场看不出来。改前本用例红(只含尾句的文案认不出)。
|
||||
*/
|
||||
@Test
|
||||
void clientStallDeathIsReopenedByItsOwnTailPhrase() {
|
||||
String driftedPrefix = "建单至今超过 120 分钟无任何结果回传,处理位置长时间停在「正在采集第 118/119 行」未推进,"
|
||||
+ SimilarAsinStaleReopenSupport.CLIENT_STALL_FAILURE_SUFFIX;
|
||||
stubFailedTask(driftedPrefix);
|
||||
|
||||
support.reopenIfStaleFailed(TASK_ID);
|
||||
|
||||
verify(fileTaskMapper).update(any(), any());
|
||||
}
|
||||
|
||||
/** 生产者与判定端共用一枚出口:拿工厂造出来的文案,重开必须发生。 */
|
||||
@Test
|
||||
void producerShapedDeathIsReopened() {
|
||||
stubFailedTask(SimilarAsinStaleReopenSupport.clientStallFailureMessage(
|
||||
120, "正在采集第 118/119 行"));
|
||||
|
||||
support.reopenIfStaleFailed(TASK_ID);
|
||||
|
||||
verify(fileTaskMapper).update(any(), any());
|
||||
}
|
||||
|
||||
/** 业务性失败不许重开(结果文件生成本身失败,重开只是让客户端继续往坏管道里灌数据)。 */
|
||||
@Test
|
||||
void businessFailureIsNotReopened() {
|
||||
stubFailedTask("已收到 3 个结果分片,但结果文件生成多次失败,任务已自动结束");
|
||||
|
||||
support.reopenIfStaleFailed(TASK_ID);
|
||||
|
||||
verify(fileTaskMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 字面量只许有一处:生产者({@code SimilarAsinTaskService} 的无回传判死)必须走
|
||||
* {@code clientStallFailureMessage},不许再自己拼一遍——两处各写一遍正是本批要消掉的形状。
|
||||
*/
|
||||
@Test
|
||||
void producerTakesItsDeathMessageFromTheSharedFactory() throws IOException {
|
||||
Path producer = moduleRoot().resolve(
|
||||
"src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java");
|
||||
String source = Files.readString(producer, StandardCharsets.UTF_8);
|
||||
assertTrue(source.contains("SimilarAsinStaleReopenSupport.clientStallFailureMessage("),
|
||||
"生产者不再走共享文案出口,重开判定与判死文案又要各自漂移:" + producer);
|
||||
assertFalse(source.contains("」未推进,疑似客户端任务卡死"),
|
||||
"生产者里不许再出现第二份卡死判死文案字面量");
|
||||
}
|
||||
|
||||
private void stubFailedTask(String errorMessage) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType("SIMILAR_ASIN");
|
||||
task.setStatus(TaskStatuses.FileTask.FAILED);
|
||||
task.setErrorMessage(errorMessage);
|
||||
task.setFinishedAt(LocalDateTime.now().minusMinutes(5));
|
||||
task.setUpdatedAt(LocalDateTime.now().minusMinutes(5));
|
||||
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||
}
|
||||
|
||||
private static Path moduleRoot() {
|
||||
Path here = Path.of("").toAbsolutePath();
|
||||
if (Files.isDirectory(here.resolve("src/main/java"))) {
|
||||
return here;
|
||||
}
|
||||
Path nested = here.resolve("backend-java");
|
||||
if (Files.isDirectory(nested.resolve("src/main/java"))) {
|
||||
return nested;
|
||||
}
|
||||
throw new IllegalStateException("无法定位 backend-java 模块目录(cwd=" + here + ")");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user