fix(tasks): 单片/单文件读不到跳过它,缺数据不许冒充已收齐
三处"一颗坏项拖垮整批"与一处假绿本体: - BrandTaskStorageService:一片载荷读不到就整份聚合丢掉,而三个调用方 (finalizeTask/abort/tryRecoverCompletedRunningTask)都没有就地 try;改逐片跳过长 WARN,并把 receivedChunkCount/completed 的判据从 chunks.size()(DB 行数)改成 真正读到的片数——拿行数比 chunkTotal 正是"118/119 行没数据仍判已完成"的形状; 解析缓存与汇总状态两处同型"整批抛出"一并改成逐条跳过。 - AppearancePatentLlmBatchPipeline:候选收集跑在批循环之前,一抛就走不到循环体内 那道 finally releaseRowInspection,本轮已申领的行要挂满处理中 TTL(15 分钟), 且 terminalizeRoundExhaustedRows 跟着不执行(轮次到顶的行因此永不判死);改逐片容错。 - TransientPayloadStorageService:跨实例判据枚成 public 常量并提供 cause 链走法。 resolvePayload 会把真实原因 rewrap 成「<errorMessage>: <pointer>」,消费端只摸顶层 消息时「only exists on instance=」「does not exist」两处判定恒 false——跨实例留痕 从未下过笔、"跳过该分片"分支是死代码。消费端(ChunkPayloadSupport/ HistoryAssembler/ResultRowReader)全部改走共享常量+cause 链。 - SimilarAsinHistoryAssemblerTest 的假绿桩删除:改前它桩的是生产者早已不再产出的 英文旧串,断言照旧绿;现在跑真实 resolvePayload 取异常(只 mock RustFS 边界), 改前 15 跑 1 红、改后 15/15 绿。 - SimilarAsinStaleReopenSupport:客户端卡死尾句枚成常量、整句文案唯一出口,生产者 与重开白名单共用;此前只是碰巧被前缀命中,生产者一改前缀就会"永不重开"。
This commit is contained in:
+19
-3
@@ -296,8 +296,23 @@ public class AppearancePatentLlmBatchPipeline {
|
|||||||
int judgedSkipped = 0;
|
int judgedSkipped = 0;
|
||||||
int noDataSkipped = 0;
|
int noDataSkipped = 0;
|
||||||
int chunkRows = 0;
|
int chunkRows = 0;
|
||||||
|
int unreadableChunks = 0;
|
||||||
for (TaskChunkEntity chunk : chunks) {
|
for (TaskChunkEntity chunk : chunks) {
|
||||||
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
|
Map<String, AppearancePatentResultRowDto> persistedRows;
|
||||||
|
try {
|
||||||
|
persistedRows = readChunkRows(chunk);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
// 单片载荷读不到不许把整轮候选收集炸掉:本方法跑在批循环**之前**,而释放处理中标记的
|
||||||
|
// finally 在循环体内——这里一抛就永远走不到那道 release,本轮已申领的行要挂满
|
||||||
|
// IN_FLIGHT_TTL(15 分钟)才回来,且 terminalizeRoundExhaustedRows 收的是本方法的返回值,
|
||||||
|
// 跟着一起不执行(轮次到顶的行因此永不判死)。跳过这一片、继续读其余片,
|
||||||
|
// 与 AppearancePatentResultRowReader#readChunkRows/SimilarAsinChunkPayloadSupport 同口径。
|
||||||
|
unreadableChunks++;
|
||||||
|
log.warn("[appearance-patent] 单片载荷读取失败,跳过该分片继续收集候选"
|
||||||
|
+ " taskId={} scopeHash={} chunk={} err={}",
|
||||||
|
chunk.getTaskId(), chunk.getScopeHash(), chunk.getChunkIndex(), ex.getMessage());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (persistedRows.isEmpty()) {
|
if (persistedRows.isEmpty()) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -328,8 +343,9 @@ public class AppearancePatentLlmBatchPipeline {
|
|||||||
candidates.add(row);
|
candidates.add(row);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
log.info("[appearance-patent] 候选收集逐行申领 taskId={} chunkRows={} judgedSkipped={} noDataSkipped={} claimBlocked={} candidates={}",
|
log.info("[appearance-patent] 候选收集逐行申领 taskId={} chunkRows={} judgedSkipped={} noDataSkipped={} claimBlocked={} unreadableChunks={} candidates={}",
|
||||||
chunks.get(0).getTaskId(), chunkRows, judgedSkipped, noDataSkipped, claimBlocked, candidates.size());
|
chunks.get(0).getTaskId(), chunkRows, judgedSkipped, noDataSkipped, claimBlocked,
|
||||||
|
unreadableChunks, candidates.size());
|
||||||
return candidates;
|
return candidates;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+4
-10
@@ -273,17 +273,11 @@ public class AppearancePatentResultRowReader {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** payload 对象已不存在(RustFS 返回 NoSuchKey:message 为 "The specified key does not exist.")。
|
/** payload 对象已不存在(RustFS 返回 NoSuchKey:message 为 "The specified key does not exist.")。
|
||||||
* 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。 */
|
* 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。
|
||||||
|
* 判据与 cause 链走法都收口在 {@link TransientPayloadStorageService#isObjectMissing}——
|
||||||
|
* 抛点与判定端各抄一遍字面量时,改一处就静默失效(货源侧那两处就是这么废掉的)。 */
|
||||||
private static boolean isPayloadMissing(Throwable error) {
|
private static boolean isPayloadMissing(Throwable error) {
|
||||||
Throwable cursor = error;
|
return TransientPayloadStorageService.isObjectMissing(error);
|
||||||
while (cursor != null) {
|
|
||||||
String message = cursor.getMessage();
|
|
||||||
if (message != null && message.contains("does not exist")) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
cursor = cursor.getCause();
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public String rowKey(AppearancePatentParsedRowVo row) {
|
public String rowKey(AppearancePatentParsedRowVo row) {
|
||||||
|
|||||||
+41
-15
@@ -90,7 +90,7 @@ public class BrandTaskStorageService {
|
|||||||
return List.of();
|
return List.of();
|
||||||
}
|
}
|
||||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
.select(TaskScopeStateEntity::getParsedPayloadJson)
|
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getScopeHash)
|
||||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
|
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
|
||||||
@@ -99,14 +99,17 @@ public class BrandTaskStorageService {
|
|||||||
}
|
}
|
||||||
List<BrandParsedFileCacheDto> result = new ArrayList<>();
|
List<BrandParsedFileCacheDto> result = new ArrayList<>();
|
||||||
for (TaskScopeStateEntity state : states) {
|
for (TaskScopeStateEntity state : states) {
|
||||||
String payloadJson = resolveParsedPayload(state.getParsedPayloadJson());
|
|
||||||
if (isBlank(payloadJson)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
|
String payloadJson = resolveParsedPayload(state.getParsedPayloadJson());
|
||||||
|
if (isBlank(payloadJson)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
result.add(objectMapper.readValue(payloadJson, BrandParsedFileCacheDto.class));
|
result.add(objectMapper.readValue(payloadJson, BrandParsedFileCacheDto.class));
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("读取品牌任务解析结果失败");
|
// 单个文件的解析缓存读不到不许把整份缓存一起丢掉:跳过它,其余文件照常出结果。
|
||||||
|
// 整批抛出会让 BrandTaskService 的 finalizeTask / abort 两处调用全单失败(那里没有就地 try)。
|
||||||
|
log.warn("[brand-storage] 解析缓存读取失败,跳过该文件 taskId={} scopeHash={} err={}",
|
||||||
|
taskId, state.getScopeHash(), ex.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
@@ -207,7 +210,10 @@ public class BrandTaskStorageService {
|
|||||||
String aggregateJson = transientPayloadStorageService.resolvePayload(state.getStateJson(), "read brand task aggregate failed");
|
String aggregateJson = transientPayloadStorageService.resolvePayload(state.getStateJson(), "read brand task aggregate failed");
|
||||||
result.put(state.getScopeKey(), objectMapper.readValue(aggregateJson, BrandFileAggregateCacheDto.class));
|
result.put(state.getScopeKey(), objectMapper.readValue(aggregateJson, BrandFileAggregateCacheDto.class));
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("读取品牌任务汇总状态失败");
|
// 单个文件的汇总载荷读不到:跳过它,其余文件的聚合照常返回。
|
||||||
|
// 少一个文件会让调用方按「尚未收齐」处理(可重试),比整批抛出把已读到的文件一起抹掉好。
|
||||||
|
log.warn("[brand-storage] 汇总状态读取失败,跳过该文件 taskId={} scopeKey={} err={}",
|
||||||
|
taskId, state.getScopeKey(), ex.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
@@ -444,32 +450,52 @@ public class BrandTaskStorageService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
BrandFileAggregateCacheDto aggregate = null;
|
BrandFileAggregateCacheDto aggregate = null;
|
||||||
|
int readableChunks = 0;
|
||||||
|
List<Integer> skippedChunkIndexes = new ArrayList<>();
|
||||||
for (TaskChunkEntity chunk : chunks) {
|
for (TaskChunkEntity chunk : chunks) {
|
||||||
BrandCrawlResultFileDto chunkPayload = readChunkPayload(chunk.getPayloadJson());
|
BrandCrawlResultFileDto chunkPayload;
|
||||||
|
try {
|
||||||
|
chunkPayload = readChunkPayload(chunk);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
// 单片载荷读不到(对象被清理 / 跨实例 local 指针 / JSON 坏)不许把整份聚合一起丢掉:
|
||||||
|
// 跳过这一片、继续读其余片,与 AppearancePatentResultRowReader#readChunkRows、
|
||||||
|
// SimilarAsinChunkPayloadSupport#readChunkRows 同口径。调用方(BrandTaskService 的
|
||||||
|
// finalizeTask / abort / tryRecoverCompletedRunningTask)都没有就地 try,
|
||||||
|
// 这里一抛就是整任务不产出结果。
|
||||||
|
skippedChunkIndexes.add(chunk.getChunkIndex());
|
||||||
|
log.warn("[brand-storage] 单片载荷读取失败,跳过该分片继续聚合 taskId={} scopeHash={} chunk={} err={}",
|
||||||
|
taskId, scopeHash, chunk.getChunkIndex(), ex.getMessage());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (aggregate == null) {
|
if (aggregate == null) {
|
||||||
aggregate = createAggregate(chunkPayload);
|
aggregate = createAggregate(chunkPayload);
|
||||||
} else {
|
} else {
|
||||||
ensureChunkConsistency(aggregate, chunkPayload);
|
ensureChunkConsistency(aggregate, chunkPayload);
|
||||||
}
|
}
|
||||||
mergeAggregate(aggregate, chunkPayload);
|
mergeAggregate(aggregate, chunkPayload);
|
||||||
|
readableChunks++;
|
||||||
}
|
}
|
||||||
if (aggregate == null) {
|
if (aggregate == null) {
|
||||||
throw new BusinessException("读取品牌任务汇总数据失败");
|
throw new BusinessException("读取品牌任务汇总数据失败:全部分片载荷均不可读,分片=" + skippedChunkIndexes);
|
||||||
}
|
}
|
||||||
aggregate.setFileUrl(scopeKey);
|
aggregate.setFileUrl(scopeKey);
|
||||||
aggregate.setReceivedChunkCount(chunks.size());
|
// 已收齐的判据只数**真正读到**的片:拿 chunks.size()(DB 行数)比 chunkTotal,
|
||||||
aggregate.setCompleted(aggregate.getChunkTotal() != null
|
// 会在载荷缺失时把缺数据的文件判成 completed——正是「118/119 行没数据仍报已完成」的形状。
|
||||||
|
aggregate.setReceivedChunkCount(readableChunks);
|
||||||
|
aggregate.setCompleted(skippedChunkIndexes.isEmpty()
|
||||||
|
&& aggregate.getChunkTotal() != null
|
||||||
&& aggregate.getChunkTotal() > 0
|
&& aggregate.getChunkTotal() > 0
|
||||||
&& chunks.size() >= aggregate.getChunkTotal());
|
&& readableChunks >= aggregate.getChunkTotal());
|
||||||
return aggregate;
|
return aggregate;
|
||||||
}
|
}
|
||||||
|
|
||||||
private BrandCrawlResultFileDto readChunkPayload(String payloadJson) {
|
private BrandCrawlResultFileDto readChunkPayload(TaskChunkEntity chunk) {
|
||||||
try {
|
try {
|
||||||
String resolvedPayload = transientPayloadStorageService.resolvePayload(payloadJson, "read brand task chunk failed");
|
String resolvedPayload = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read brand task chunk failed");
|
||||||
return objectMapper.readValue(resolvedPayload, BrandCrawlResultFileDto.class);
|
return objectMapper.readValue(resolvedPayload, BrandCrawlResultFileDto.class);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("读取品牌任务分片失败");
|
// 带上分片索引与 cause:跳过之后排查只剩这一行日志,裸文案对不上是哪一片
|
||||||
|
throw new BusinessException("读取品牌任务分片失败,分片=" + chunk.getChunkIndex(), ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -1109,8 +1109,8 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
|||||||
log.warn("[similar-asin] 从未上报任何结果分片且处理位置长时间停在「{}」未推进,按卡死收尾 taskId={} minutes={}",
|
log.warn("[similar-asin] 从未上报任何结果分片且处理位置长时间停在「{}」未推进,按卡死收尾 taskId={} minutes={}",
|
||||||
neverReportedPosition, task.getId(), minutes);
|
neverReportedPosition, task.getId(), minutes);
|
||||||
finalizeStaleTask(task.getId(),
|
finalizeStaleTask(task.getId(),
|
||||||
"任务创建至今超过 " + minutes + " 分钟无任何结果回传,处理位置长时间停在「"
|
SimilarAsinStaleReopenSupport.clientStallFailureMessage(
|
||||||
+ neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败");
|
minutes, neverReportedPosition));
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
+8
-3
@@ -78,11 +78,15 @@ public class SimilarAsinChunkPayloadSupport {
|
|||||||
String msg = ex.getMessage() == null ? "" : ex.getMessage();
|
String msg = ex.getMessage() == null ? "" : ex.getMessage();
|
||||||
// P1-4:识别跨实例 local 指针读不到的场景,把"chunk 在另一实例"的元信息
|
// P1-4:识别跨实例 local 指针读不到的场景,把"chunk 在另一实例"的元信息
|
||||||
// 通过 task_scope_state.last_error 留痕,便于排查"为何 owner 切换后 chunk 读不到"。
|
// 通过 task_scope_state.last_error 留痕,便于排查"为何 owner 切换后 chunk 读不到"。
|
||||||
boolean crossInstance = msg.contains("only exists on instance=");
|
// 判据必须走 cause 链并与生产者共用同一枚常量:resolvePayload 抛给这里的顶层消息是
|
||||||
|
// "<errorMessage>: <pointer>",「临时载荷仅存在于实例 …」只在 cause 上——
|
||||||
|
// 旧的 contains("only exists on instance=") 认的是 TransientPayloadStorageService 早已
|
||||||
|
// 不再产出的英文串,crossInstance 恒 false,@cross-instance 标记从未下过笔(2026-10-27 复验)。
|
||||||
|
boolean crossInstance = TransientPayloadStorageService.isCrossInstanceLocalPointer(ex);
|
||||||
// 类型不兼容(如历史版本序列化数据含当前类加载器不认识的类型标记):
|
// 类型不兼容(如历史版本序列化数据含当前类加载器不认识的类型标记):
|
||||||
// 该 chunk 永久不可读,跳过而不是把组装/收尾永久拖死,留痕便于排查。
|
// 该 chunk 永久不可读,跳过而不是把组装/收尾永久拖死,留痕便于排查。
|
||||||
boolean typeMismatch = ex instanceof TypeNotPresentException
|
boolean typeMismatch = ex instanceof TypeNotPresentException
|
||||||
|| msg.contains("not present");
|
|| TransientPayloadStorageService.causeChainContains(ex, "not present");
|
||||||
if (typeMismatch) {
|
if (typeMismatch) {
|
||||||
log.warn("[similar-asin] chunk payload type mismatch, skip chunk taskId={} chunk={} err={}",
|
log.warn("[similar-asin] chunk payload type mismatch, skip chunk taskId={} chunk={} err={}",
|
||||||
chunk.getTaskId(), chunk.getChunkIndex(), msg);
|
chunk.getTaskId(), chunk.getChunkIndex(), msg);
|
||||||
@@ -92,7 +96,8 @@ public class SimilarAsinChunkPayloadSupport {
|
|||||||
// payload 对象已不存在(RustFS 返回 NoSuchKey):重试多少次都读不回来,跳过而不是
|
// payload 对象已不存在(RustFS 返回 NoSuchKey):重试多少次都读不回来,跳过而不是
|
||||||
// 把组装/收尾永久拖死——与上面的 typeMismatch 分支、以及 collect-data 的降级口径一致。
|
// 把组装/收尾永久拖死——与上面的 typeMismatch 分支、以及 collect-data 的降级口径一致。
|
||||||
// 线上 appearance-patent 28459 就因同类场景每 10~30 秒重试一次(见同批修复)。
|
// 线上 appearance-patent 28459 就因同类场景每 10~30 秒重试一次(见同批修复)。
|
||||||
if (msg.contains("does not exist")) {
|
// 同样只认 cause 链:rewrap 后顶层消息里没有 "does not exist",这一分支此前也是死的。
|
||||||
|
if (TransientPayloadStorageService.isObjectMissing(ex)) {
|
||||||
log.warn("[similar-asin] chunk payload 已不存在,跳过该分片 taskId={} chunk={} err={}",
|
log.warn("[similar-asin] chunk payload 已不存在,跳过该分片 taskId={} chunk={} err={}",
|
||||||
chunk.getTaskId(), chunk.getChunkIndex(), msg);
|
chunk.getTaskId(), chunk.getChunkIndex(), msg);
|
||||||
recordChunkReadFailure(chunk, false, msg);
|
recordChunkReadFailure(chunk, false, msg);
|
||||||
|
|||||||
+3
-1
@@ -439,7 +439,9 @@ public class SimilarAsinHistoryAssembler {
|
|||||||
String msg = ex.getMessage() == null ? "" : ex.getMessage();
|
String msg = ex.getMessage() == null ? "" : ex.getMessage();
|
||||||
// P1-4:识别跨实例 local 指针读不到的场景,把"chunk 在另一实例"的元信息
|
// P1-4:识别跨实例 local 指针读不到的场景,把"chunk 在另一实例"的元信息
|
||||||
// 通过 task_scope_state.last_error 留痕,便于排查"为何 owner 切换后 chunk 读不到"。
|
// 通过 task_scope_state.last_error 留痕,便于排查"为何 owner 切换后 chunk 读不到"。
|
||||||
boolean crossInstance = msg.contains("only exists on instance=");
|
// 与 SimilarAsinChunkPayloadSupport 同一枚判据:走 cause 链+生产者自己的常量,
|
||||||
|
// 只摸顶层消息(rewrap 后是 "<errorMessage>: <pointer>")永远认不出中文原文。
|
||||||
|
boolean crossInstance = TransientPayloadStorageService.isCrossInstanceLocalPointer(ex);
|
||||||
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
|
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
|
||||||
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
|
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
|
||||||
recordChunkReadFailure(chunk, crossInstance, msg);
|
recordChunkReadFailure(chunk, crossInstance, msg);
|
||||||
|
|||||||
+15
@@ -28,12 +28,27 @@ import java.util.List;
|
|||||||
public class SimilarAsinStaleReopenSupport {
|
public class SimilarAsinStaleReopenSupport {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "SIMILAR_ASIN";
|
private static final String MODULE_TYPE = "SIMILAR_ASIN";
|
||||||
|
/**
|
||||||
|
* 客户端卡死判死文案的固定尾句,重开判定与生产者共用(口径同 {@code TaskResumeService.DUPLICATE_STOP_REASON}:
|
||||||
|
* 两处各写一遍,改一处就静默失效——这里失效的方向是「迟到分片被拒、用户已采集的数据整批丢掉」)。
|
||||||
|
*/
|
||||||
|
public static final String CLIENT_STALL_FAILURE_SUFFIX = "疑似客户端任务卡死,任务已自动失败";
|
||||||
|
/**
|
||||||
|
* 无结果回传判死的整句文案——唯一出口,生产者({@code SimilarAsinTaskService} 的二次判死)
|
||||||
|
* 与本类的重开判定都从它取数,前缀与尾句不会再各漂一半。
|
||||||
|
*/
|
||||||
|
public static String clientStallFailureMessage(long minutes, String neverReportedPosition) {
|
||||||
|
return "任务创建至今超过 " + minutes + " 分钟无任何结果回传,处理位置长时间停在「"
|
||||||
|
+ neverReportedPosition + "」未推进," + CLIENT_STALL_FAILURE_SUFFIX;
|
||||||
|
}
|
||||||
private static final List<String> STALE_FAILURE_MARKERS = List.of(
|
private static final List<String> STALE_FAILURE_MARKERS = List.of(
|
||||||
"任务心跳超时,已自动失败",
|
"任务心跳超时,已自动失败",
|
||||||
"长时间未收到 Python 心跳,任务已自动失败",
|
"长时间未收到 Python 心跳,任务已自动失败",
|
||||||
"客户端未回传任何检测结果",
|
"客户端未回传任何检测结果",
|
||||||
"客户端回传中断",
|
"客户端回传中断",
|
||||||
"任务创建至今超过",
|
"任务创建至今超过",
|
||||||
|
// 尾句单独进白名单:上面那枚前缀只是碰巧同句出现,生产者一改前缀重开就整条失效
|
||||||
|
CLIENT_STALL_FAILURE_SUFFIX,
|
||||||
"Python heartbeat alive but no result chunk uploaded");
|
"Python heartbeat alive but no result chunk uploaded");
|
||||||
private static final Duration REOPEN_WINDOW = Duration.ofHours(24);
|
private static final Duration REOPEN_WINDOW = Duration.ofHours(24);
|
||||||
|
|
||||||
|
|||||||
+44
-5
@@ -45,6 +45,20 @@ public class TransientPayloadStorageService {
|
|||||||
private static final String OSS_POINTER_PREFIX = "oss:";
|
private static final String OSS_POINTER_PREFIX = "oss:";
|
||||||
private static final String LOCAL_PAYLOAD_DIR = "transient-payload";
|
private static final String LOCAL_PAYLOAD_DIR = "transient-payload";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 跨实例消费 local 兜底指针时的判据原文(抛点见 {@link #resolvePayload})。
|
||||||
|
*
|
||||||
|
* <p>枚成 public 常量是因为消费端(货源 {@code SimilarAsinChunkPayloadSupport}/
|
||||||
|
* {@code SimilarAsinHistoryAssembler} 的 {@code chunk-read-failed[N@cross-instance]} 留痕)
|
||||||
|
* 要认的就是这句话。消费端此前各抄一份英文旧串 {@code "only exists on instance="}——
|
||||||
|
* 生产者早就改成中文文案,两处判定恒 false,跨实例标记永不下笔(2026-10-27 复验定性为假绿)。
|
||||||
|
* 现在两端共用这一枚常量,改文案只会让两边一起改,不会静默失效。
|
||||||
|
*/
|
||||||
|
public static final String CROSS_INSTANCE_LOCAL_POINTER_MARKER = "临时载荷仅存在于实例 ";
|
||||||
|
|
||||||
|
/** 对象确已不存在(RustFS NoSuchKey)的判据原文:抛点在 {@code RustfsObjectStorageService}。 */
|
||||||
|
private static final String MISSING_OBJECT_MARKER = "does not exist";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* P0-3:记录"最近一次 store 是否因 RustFS 失败回落到 local"。
|
* P0-3:记录"最近一次 store 是否因 RustFS 失败回落到 local"。
|
||||||
* 调用方(如 SimilarAsinTaskService.submitResult)拿到 storedPayload 后可以查询
|
* 调用方(如 SimilarAsinTaskService.submitResult)拿到 storedPayload 后可以查询
|
||||||
@@ -144,7 +158,7 @@ public class TransientPayloadStorageService {
|
|||||||
if (ownerInstance != null && !ownerInstance.equals(instanceMetadata.getInstanceId())) {
|
if (ownerInstance != null && !ownerInstance.equals(instanceMetadata.getInstanceId())) {
|
||||||
// 跨实例消费 local 指针时立即报错,避免悄悄读出空内容造成下游错位。
|
// 跨实例消费 local 指针时立即报错,避免悄悄读出空内容造成下游错位。
|
||||||
throw new IllegalStateException(
|
throw new IllegalStateException(
|
||||||
"临时载荷仅存在于实例 " + ownerInstance + "(当前实例 "
|
CROSS_INSTANCE_LOCAL_POINTER_MARKER + ownerInstance + "(当前实例 "
|
||||||
+ instanceMetadata.getInstanceId() + "),本地兜底指针跨节点不可读; objectKey="
|
+ instanceMetadata.getInstanceId() + "),本地兜底指针跨节点不可读; objectKey="
|
||||||
+ stripLocalInstanceId(localKey));
|
+ stripLocalInstanceId(localKey));
|
||||||
}
|
}
|
||||||
@@ -211,15 +225,40 @@ public class TransientPayloadStorageService {
|
|||||||
return candidates.isEmpty() ? null : candidates.getFirst();
|
return candidates.isEmpty() ? null : candidates.getFirst();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 对象确已不存在:RustFS 返回 NoSuchKey,message 为 "The specified key does not exist."。 */
|
/**
|
||||||
private static boolean isObjectMissing(Throwable error) {
|
* 对象确已不存在:RustFS 返回 NoSuchKey,message 为 "The specified key does not exist."。
|
||||||
|
*
|
||||||
|
* <p>公开给消费端(分片载荷读取)复用。{@link #resolvePayload} 末尾会把真实原因 rewrap 成
|
||||||
|
* 「{@code <errorMessage>}: {@code <pointer>}」,NoSuchKey 与跨实例原文都只剩在 cause 上,
|
||||||
|
* 所以判据一律走 cause 链——只摸顶层消息的话,「跳过该分片」与「跨实例留痕」两个分支
|
||||||
|
* 会一起变成死代码(货源侧 2026-10-27 复验就是这一形状)。
|
||||||
|
*/
|
||||||
|
public static boolean isObjectMissing(Throwable error) {
|
||||||
|
return causeChainContains(error, MISSING_OBJECT_MARKER);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 载荷归另一台实例(local 兜底指针跨节点不可读);同样必须走 cause 链。 */
|
||||||
|
public static boolean isCrossInstanceLocalPointer(Throwable error) {
|
||||||
|
return causeChainContains(error, CROSS_INSTANCE_LOCAL_POINTER_MARKER);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 沿 cause 链找字面量:包装层不许掩盖真实原因。
|
||||||
|
*
|
||||||
|
* <p>{@code cause == cursor} 的自引用按链尾处理,免得异常被自己当 cause 时把调用线程钉死。
|
||||||
|
*/
|
||||||
|
public static boolean causeChainContains(Throwable error, String marker) {
|
||||||
|
if (error == null || marker == null || marker.isEmpty()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
Throwable cursor = error;
|
Throwable cursor = error;
|
||||||
while (cursor != null) {
|
while (cursor != null) {
|
||||||
String message = cursor.getMessage();
|
String message = cursor.getMessage();
|
||||||
if (message != null && message.contains("does not exist")) {
|
if (message != null && message.contains(marker)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
cursor = cursor.getCause();
|
Throwable next = cursor.getCause();
|
||||||
|
cursor = next == cursor ? null : next;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -51,7 +51,8 @@ class CatchSilenceRatchetTest {
|
|||||||
// 10-25 B15 join 无界收口抬 564 -> 569,逐处出处见 STRICT_NOT_RETHROW 上方注释。
|
// 10-25 B15 join 无界收口抬 564 -> 569,逐处出处见 STRICT_NOT_RETHROW 上方注释。
|
||||||
// 10-26 第二轮 CR-Fix 2 抬 569 -> 573,逐处出处见 STRICT_NOT_RETHROW 上方注释(4 处必要 fail-open)。
|
// 10-26 第二轮 CR-Fix 2 抬 569 -> 573,逐处出处见 STRICT_NOT_RETHROW 上方注释(4 处必要 fail-open)。
|
||||||
static final int GIVEN_TOTAL = 1000;
|
static final int GIVEN_TOTAL = 1000;
|
||||||
static final int GIVEN_NOT_RETHROW = 573;
|
// 10-27 「整表都要跑完」专项抬 573 -> 577(2 枚新增不重抛 + 2 枚既有整批抛出改逐条跳过,逐处记名见 STRICT_TOTAL 上方)。
|
||||||
|
static final int GIVEN_NOT_RETHROW = 577;
|
||||||
static final int GIVEN_LOG_ONLY = 249;
|
static final int GIVEN_LOG_ONLY = 249;
|
||||||
static final int GIVEN_EMPTY = 56;
|
static final int GIVEN_EMPTY = 56;
|
||||||
|
|
||||||
@@ -141,8 +142,30 @@ class CatchSilenceRatchetTest {
|
|||||||
* </ul>
|
* </ul>
|
||||||
* 9 处均为 CR-02/05/10/20 的必要出口,不是把已有异常改回吞掉;同批实测 total 1024、
|
* 9 处均为 CR-02/05/10/20 的必要出口,不是把已有异常改回吞掉;同批实测 total 1024、
|
||||||
* notRethrow 573、logOnly 179、empty 56。
|
* notRethrow 573、logOnly 179、empty 56。
|
||||||
|
*
|
||||||
|
* <p>10-27 整表跑完专项抬 1024 -> 1026(用户口径:整表都要跑完,单项坏了跳过它继续下一项,
|
||||||
|
* 只有 LLM 密钥欠费/失效那一类才允许直接判任务失败),逐处记名:
|
||||||
|
* <ul>
|
||||||
|
* <li>{@code BrandTaskStorageService.rebuildAggregate} 一枚:单片载荷读不到时
|
||||||
|
* log.warn+记跳片号+{@code continue}(NOT_RETHROW)。此前一片抛就把整份聚合丢掉,
|
||||||
|
* 而三个调用方(finalizeTask/abort/tryRecoverCompletedRunningTask)都没有就地 try,
|
||||||
|
* 一个坏分片=整任务不产出;同批把 {@code completed} 的判据从 {@code chunks.size()}
|
||||||
|
* (DB 行数)改成「真正读到的片数」,缺数据不再被判成已收齐。</li>
|
||||||
|
* <li>{@code AppearancePatentLlmBatchPipeline.collectPendingLlmCandidates} 一枚:
|
||||||
|
* 同为 log.warn+计数+{@code continue}(NOT_RETHROW)。collect 跑在批循环<b>之前</b>,
|
||||||
|
* 一抛就走不到循环体内那道 {@code finally releaseRowInspection},本轮已申领的行要挂满
|
||||||
|
* 处理中 TTL(15 分钟),且 {@code terminalizeRoundExhaustedRows} 跟着一起不执行。</li>
|
||||||
|
* </ul>
|
||||||
|
* 另有两枚<b>既有</b> RETHROW 出口按同一口径改成「跳过该项」,形状从 RETHROW 迁入气味两格
|
||||||
|
* (total 不变、rethrow −2、notRethrow +2、logOnly +2):{@code BrandTaskStorageService.getParsedPayload}
|
||||||
|
* 与 {@code BrandTaskStorageService.getAllFileAggregates}——两处此前一个文件的缓存/聚合读不到
|
||||||
|
* 就整批抛,把已读到的其余文件一起抹掉;改后逐条 WARN 跳过,调用方按「尚未收齐」可重试。
|
||||||
|
* ⚠ 这两处不是把异常静默吞掉:跳过都带 {@code [brand-storage]} 锚点 WARN,且
|
||||||
|
* {@code BrandTaskStorageServiceFaultToleranceTest} 钉住「其余项照常出」与
|
||||||
|
* 「全项坏掉仍大声抛」两条,缺数据一律不得冒充已完成。
|
||||||
|
* 同批实测 total 1026、rethrow 393(仍高于下限 380,本格不动)、notRethrow 577、logOnly 181、empty 56。
|
||||||
*/
|
*/
|
||||||
static final int STRICT_TOTAL = 1024;
|
static final int STRICT_TOTAL = 1026;
|
||||||
/**
|
/**
|
||||||
* 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。
|
* 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。
|
||||||
* 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。
|
* 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。
|
||||||
@@ -234,8 +257,13 @@ class CatchSilenceRatchetTest {
|
|||||||
* {@code OssStorageService.downloadWorkbenchClientObjectToTempFile} 内层清理(同上)、
|
* {@code OssStorageService.downloadWorkbenchClientObjectToTempFile} 内层清理(同上)、
|
||||||
* {@code CostGuardScheduler}(平台自费告警推送失败 log+return,保留基线待下轮重试)。
|
* {@code CostGuardScheduler}(平台自费告警推送失败 log+return,保留基线待下轮重试)。
|
||||||
* 四处的误差方向都是「不让清理/告警失败掩盖主错误」,不是把异常静默吞掉。
|
* 四处的误差方向都是「不让清理/告警失败掩盖主错误」,不是把异常静默吞掉。
|
||||||
|
* 10-27 抬 573 -> 577(「整表都要跑完」专项,4 处不重抛,逐处记名见 STRICT_TOTAL 上方注释):
|
||||||
|
* 新增 {@code BrandTaskStorageService.rebuildAggregate} 与
|
||||||
|
* {@code AppearancePatentLlmBatchPipeline.collectPendingLlmCandidates} 两枚逐片跳过;
|
||||||
|
* 另有 {@code BrandTaskStorageService.getParsedPayload}/{@code getAllFileAggregates} 两枚
|
||||||
|
* 既有「整批抛出」出口按同口径改成逐条跳过(自 RETHROW 迁入本格)。
|
||||||
*/
|
*/
|
||||||
static final int STRICT_NOT_RETHROW = 573;
|
static final int STRICT_NOT_RETHROW = 577;
|
||||||
/**
|
/**
|
||||||
* 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}:
|
* 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}:
|
||||||
* 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出——
|
* 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出——
|
||||||
@@ -251,8 +279,12 @@ class CatchSilenceRatchetTest {
|
|||||||
* 10-26 抬 176 -> 179(第二轮 CR-Fix 2,3 处新增 log-only):{@code WorkbenchClientPackageService}
|
* 10-26 抬 176 -> 179(第二轮 CR-Fix 2,3 处新增 log-only):{@code WorkbenchClientPackageService}
|
||||||
* 的 removeRejectedObject/deleteTempQuietly 与 {@code OssStorageService} 的临时文件清理内层,
|
* 的 removeRejectedObject/deleteTempQuietly 与 {@code OssStorageService} 的临时文件清理内层,
|
||||||
* 三处都只 WARN 且带 path/objectKey 锚点;主错误由外层重抛,清理失败不得盖掉它。
|
* 三处都只 WARN 且带 path/objectKey 锚点;主错误由外层重抛,清理失败不得盖掉它。
|
||||||
|
* 10-27 抬 179 -> 181(「整表都要跑完」专项,2 处):{@code BrandTaskStorageService.getParsedPayload}
|
||||||
|
* 与 {@code getAllFileAggregates} 的逐条跳过出口——块体各只有一句带 {@code [brand-storage]}
|
||||||
|
* 锚点的 WARN。两处此前是「一个文件坏 ⇒ 整批抛出」,改后其余文件照常返回,坏的那条点名可查;
|
||||||
|
* 同批 {@code rebuildAggregate} 与外观专利候选收集因块内另有计数/continue 只进 NOT_RETHROW。
|
||||||
*/
|
*/
|
||||||
static final int STRICT_LOG_ONLY = 179; // 严格口径:块内每条顶层语句都是 log 调用才算
|
static final int STRICT_LOG_ONLY = 181; // 严格口径:块内每条顶层语句都是 log 调用才算
|
||||||
static final int STRICT_EMPTY = 56;
|
static final int STRICT_EMPTY = 56;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
+213
@@ -0,0 +1,213 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||||
|
import com.nanri.aiimage.config.InstanceMetadata;
|
||||||
|
import com.nanri.aiimage.config.StorageProperties;
|
||||||
|
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentLlmBatchPipeline;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.service.support.AppearancePatentResultRowReader;
|
||||||
|
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyLong;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
|
import static org.mockito.Mockito.atLeast;
|
||||||
|
import static org.mockito.Mockito.atLeastOnce;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 候选收集阶段的「一片坏了不许拖垮整轮」(2026-10-27 生产误判后的用户口径:整表都要跑完,
|
||||||
|
* 单项有问题就跳过它继续下一项,不能因为一个坏项影响全部)。
|
||||||
|
*
|
||||||
|
* <p>{@code collectPendingLlmCandidates} 跑在批循环<b>之前</b>,而批循环里的
|
||||||
|
* {@code finally releaseRowInspection} 只在循环体内——collect 一抛就永远走不到那道释放:
|
||||||
|
* <ul>
|
||||||
|
* <li>本轮已申领的行挂在处理中标记上,要等 {@code IN_FLIGHT_TTL}(15 分钟)自己过期;</li>
|
||||||
|
* <li>{@code terminalizeRoundExhaustedRows} 收的是 collect 的返回值,collect 抛出时它也一并跳过,
|
||||||
|
* 轮次到顶的行因此永不判死。</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* <p>读失败的异常由<b>真实</b>的 {@link AppearancePatentResultRowReader} +
|
||||||
|
* {@link TransientPayloadStorageService#resolvePayload} 产出,不在测试里手写文案——
|
||||||
|
* 手写桩造的正是生产者早已不再产出的旧字面量(货源侧那支假绿用例的成因)。
|
||||||
|
*/
|
||||||
|
class AppearancePatentCandidateCollectionFaultToleranceTest {
|
||||||
|
|
||||||
|
private static final Long TASK_ID = 31610L;
|
||||||
|
private static final String SCOPE_HASH = "scope-a";
|
||||||
|
private static final String READABLE_HEAD =
|
||||||
|
"[{\"rowToken\":\"r1\",\"id\":\"1\",\"asin\":\"B0A0000001\",\"country\":\"US\","
|
||||||
|
+ "\"title\":\"RUNNING SHOE\",\"url\":\"http://img/1.jpg\",\"groupKey\":\"g1\"}]";
|
||||||
|
private static final String READABLE_TAIL =
|
||||||
|
"[{\"rowToken\":\"r3\",\"id\":\"3\",\"asin\":\"B0A0000003\",\"country\":\"US\","
|
||||||
|
+ "\"title\":\"COTTON BAG\",\"url\":\"http://img/3.jpg\",\"groupKey\":\"g3\"}]";
|
||||||
|
/** 归另一台实例的 local 指针:resolvePayload 抛中文「临时载荷仅存在于实例 …」,reader 再包成 BusinessException。 */
|
||||||
|
private static final String CROSS_INSTANCE_CHUNK =
|
||||||
|
"local:i/server-110/task-chunk/APPEARANCE_PATENT/" + TASK_ID + "/" + SCOPE_HASH + "/chunk-2.json";
|
||||||
|
private static final String MISSING_OBJECT_KEY =
|
||||||
|
"task-chunk/APPEARANCE_PATENT/" + TASK_ID + "/" + SCOPE_HASH + "/chunk-2.json";
|
||||||
|
|
||||||
|
private AppearancePatentTaskCacheService taskCacheService;
|
||||||
|
private AppearancePatentResultRowReader reader;
|
||||||
|
private AppearancePatentLlmBatchPipeline pipeline;
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() {
|
||||||
|
taskCacheService = mock(AppearancePatentTaskCacheService.class);
|
||||||
|
// 载荷读不到的那一片走真实存储层(跨实例中文原文由它产出),其余两片按裸 JSON 直读
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
when(rustfs.isConfigured()).thenReturn(true);
|
||||||
|
when(rustfs.readObjectBytes(MISSING_OBJECT_KEY)).thenThrow(new IllegalStateException(
|
||||||
|
"The specified key does not exist. (Service: S3, Status Code: 404)"));
|
||||||
|
when(rustfs.listObjectKeysNewestFirst(anyString(), org.mockito.ArgumentMatchers.anyInt())).thenReturn(List.of());
|
||||||
|
TransientPayloadStorageService storage = new TransientPayloadStorageService(
|
||||||
|
mock(TransientStorageProperties.class), mock(StorageProperties.class), rustfs,
|
||||||
|
mock(OssStorageService.class), new ObjectMapper(), instanceMetadata("server-1"),
|
||||||
|
mock(TaskChunkMapper.class), mock(TaskScopeStateMapper.class));
|
||||||
|
reader = new AppearancePatentResultRowReader(
|
||||||
|
AppearancePatentTaskService.MODULE_TYPE, new ObjectMapper(), storage, mock(TaskChunkMapper.class),
|
||||||
|
value -> value == null ? "" : value.trim(), task -> new AppearancePatentParsedPayloadDto());
|
||||||
|
pipeline = pipeline(reader);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 中间那片读不到:跳过它,第三片仍要进候选。改前 collect 直接抛 BusinessException,
|
||||||
|
* 于是第一片已申领的 r1 挂在处理中标记上(15 分钟)、第三片根本没被读过。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void unreadableChunkDoesNotAbortCandidateCollection() {
|
||||||
|
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||||
|
|
||||||
|
List<AppearancePatentResultRowDto> candidates = pipeline.collectPendingLlmCandidates(
|
||||||
|
List.of(chunk(1, READABLE_HEAD), chunk(2, CROSS_INSTANCE_CHUNK), chunk(3, READABLE_TAIL)),
|
||||||
|
new AppearancePatentLlmBatchPipeline.ChunkRowKeyIndex());
|
||||||
|
|
||||||
|
assertEquals(List.of("r1", "r3"), rowTokens(candidates),
|
||||||
|
"坏的那一片要跳过,其余两片照常进候选");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 核心不变量(逐片容错与「异常路径统一 release」两种合法修法都满足,唯独现状不满足):
|
||||||
|
* 凡是申领成功的行,要么进候选、要么被显式 release,**不许**挂在处理中标记上等 TTL 自己过期。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void claimedRowsAreNeverLeftInFlightWhenAChunkFails() {
|
||||||
|
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||||
|
|
||||||
|
List<String> collected;
|
||||||
|
String thrown = null;
|
||||||
|
try {
|
||||||
|
collected = rowTokens(pipeline.collectPendingLlmCandidates(
|
||||||
|
List.of(chunk(1, READABLE_HEAD), chunk(2, CROSS_INSTANCE_CHUNK)),
|
||||||
|
new AppearancePatentLlmBatchPipeline.ChunkRowKeyIndex()));
|
||||||
|
} catch (RuntimeException ex) {
|
||||||
|
thrown = ex.getClass().getSimpleName() + ": " + ex.getMessage();
|
||||||
|
collected = List.of();
|
||||||
|
}
|
||||||
|
|
||||||
|
ArgumentCaptor<String> claimed = ArgumentCaptor.forClass(String.class);
|
||||||
|
verify(taskCacheService, atLeastOnce()).beginRowInspection(eq(TASK_ID), claimed.capture());
|
||||||
|
Set<String> released = releasedKeys();
|
||||||
|
assertTrue(claimed.getAllValues().contains("r1"), "夹具没能让第一片真的申领到行");
|
||||||
|
for (String key : claimed.getAllValues()) {
|
||||||
|
assertTrue(collected.contains(key) || released.contains(key),
|
||||||
|
"行 " + key + " 已申领却既没进候选也没释放——会挂满处理中 TTL 15 分钟;"
|
||||||
|
+ "抛出=" + thrown + ",候选=" + collected + ",已释放=" + released);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 对象缺失(NoSuchKey)那一种 reader 自己就跳了,collect 不得因它丢其余分片。 */
|
||||||
|
@Test
|
||||||
|
void missingObjectChunkIsSkippedAndCollectionContinues() {
|
||||||
|
when(taskCacheService.beginRowInspection(eq(TASK_ID), anyString())).thenReturn(true);
|
||||||
|
|
||||||
|
List<AppearancePatentResultRowDto> candidates = pipeline.collectPendingLlmCandidates(
|
||||||
|
List.of(chunk(1, READABLE_HEAD), chunk(2, "rustfs:" + MISSING_OBJECT_KEY), chunk(3, READABLE_TAIL)),
|
||||||
|
new AppearancePatentLlmBatchPipeline.ChunkRowKeyIndex());
|
||||||
|
|
||||||
|
assertEquals(List.of("r1", "r3"), rowTokens(candidates));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* reader 侧契约不变:跨实例失败照旧抛出(它只放行「重试也没用」的对象缺失)。
|
||||||
|
* 这一支钉的是「容错做在 collect 那一边,没有把 reader 的判据一起放宽」。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void readerStillThrowsForCrossInstanceChunkPayload() {
|
||||||
|
BusinessException thrown = assertThrows(BusinessException.class,
|
||||||
|
() -> reader.readChunkRows(chunk(2, CROSS_INSTANCE_CHUNK)));
|
||||||
|
assertTrue(thrown.getMessage().contains("读取外观专利结果分片失败"), "实际: " + thrown.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 夹具 =====
|
||||||
|
|
||||||
|
private AppearancePatentLlmBatchPipeline pipeline(AppearancePatentResultRowReader rowReader) {
|
||||||
|
return new AppearancePatentLlmBatchPipeline(
|
||||||
|
mock(AppearancePatentProperties.class),
|
||||||
|
mock(AppearancePatentLlmClient.class),
|
||||||
|
taskCacheService,
|
||||||
|
mock(TaskFileJobService.class),
|
||||||
|
row -> row == null || row.getRowToken() == null ? "" : row.getRowToken(),
|
||||||
|
row -> true,
|
||||||
|
rowReader::readChunkRows,
|
||||||
|
taskId -> List.of(),
|
||||||
|
taskId -> false,
|
||||||
|
taskId -> {
|
||||||
|
},
|
||||||
|
task -> "prompt",
|
||||||
|
task -> "sk-test",
|
||||||
|
(taskId, scopeHash, chunkIndex, rows) -> List.of(),
|
||||||
|
(preferred, fallback) -> preferred == null || preferred.isBlank() ? fallback : preferred);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Set<String> releasedKeys() {
|
||||||
|
ArgumentCaptor<List<String>> captor = ArgumentCaptor.forClass(List.class);
|
||||||
|
verify(taskCacheService, atLeast(0)).releaseRowInspection(anyLong(), captor.capture());
|
||||||
|
return captor.getAllValues().stream().flatMap(List::stream).collect(Collectors.toSet());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<String> rowTokens(List<AppearancePatentResultRowDto> rows) {
|
||||||
|
return rows.stream().map(AppearancePatentResultRowDto::getRowToken).toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static InstanceMetadata instanceMetadata(String instanceId) {
|
||||||
|
InstanceMetadata metadata = mock(InstanceMetadata.class);
|
||||||
|
when(metadata.getInstanceId()).thenReturn(instanceId);
|
||||||
|
return metadata;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TaskChunkEntity chunk(int index, String payloadJson) {
|
||||||
|
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||||
|
chunk.setId((long) index);
|
||||||
|
chunk.setTaskId(TASK_ID);
|
||||||
|
chunk.setModuleType(AppearancePatentTaskService.MODULE_TYPE);
|
||||||
|
chunk.setScopeHash(SCOPE_HASH);
|
||||||
|
chunk.setChunkIndex(index);
|
||||||
|
chunk.setPayloadJson(payloadJson);
|
||||||
|
chunk.setPayloadHash("hash-" + index);
|
||||||
|
return chunk;
|
||||||
|
}
|
||||||
|
}
|
||||||
+268
@@ -0,0 +1,268 @@
|
|||||||
|
package com.nanri.aiimage.modules.brand.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.config.InstanceMetadata;
|
||||||
|
import com.nanri.aiimage.config.StorageProperties;
|
||||||
|
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||||
|
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
|
||||||
|
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
|
||||||
|
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
|
||||||
|
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyBoolean;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyInt;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 品牌任务存储层的「单项坏了不许拖垮整批」口径(2026-10-27 生产误判后的用户三条硬口径:
|
||||||
|
* 整表都要跑完,读不到的一项跳过并留痕、继续下一项;只有 LLM 密钥欠费/失效那一类才允许直接判任务失败)。
|
||||||
|
*
|
||||||
|
* <p>此前 {@code rebuildAggregate} 逐片读时任一片抛出就把整份聚合丢掉,而三个调用方
|
||||||
|
* (finalizeTask/abort/tryRecoverCompletedRunningTask)都没有就地 try,于是:
|
||||||
|
* <ul>
|
||||||
|
* <li>一个坏分片 ⇒ 整个文件甚至整个任务不产出结果;</li>
|
||||||
|
* <li>{@code completed} 拿 {@code chunks.size()}(DB 行数)比 {@code chunkTotal}
|
||||||
|
* ⇒ 载荷缺失的行照样算「已收齐」,正是「118/119 行没数据仍报已完成」的形状。</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* <p>读失败的异常一律由<b>真实</b> {@link TransientPayloadStorageService#resolvePayload} 产出
|
||||||
|
* (在测试里桩造字面量正是货源侧那支假绿用例的成因,见 {@code SimilarAsinChunkPayloadSupport} 同批修复)。
|
||||||
|
* 跨实例 local 指针与 RustFS NoSuchKey 两种真实形状各测一次。
|
||||||
|
*/
|
||||||
|
class BrandTaskStorageServiceFaultToleranceTest {
|
||||||
|
|
||||||
|
private static final Long TASK_ID = 31600L;
|
||||||
|
private static final String FILE_URL = "https://oss/brand/source.xlsx";
|
||||||
|
private static final ObjectMapper JSON = new ObjectMapper();
|
||||||
|
|
||||||
|
private TaskChunkMapper taskChunkMapper;
|
||||||
|
private TaskScopeStateMapper taskScopeStateMapper;
|
||||||
|
private TransientPayloadStorageService transientPayloadStorage;
|
||||||
|
private RustfsObjectStorageService rustfs;
|
||||||
|
private BrandTaskStorageService service;
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void initMybatisMetadata() {
|
||||||
|
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() {
|
||||||
|
taskChunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
taskScopeStateMapper = mock(TaskScopeStateMapper.class);
|
||||||
|
rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
// producer 是真实存储层(负责抛真实的中文/NoSuchKey 原文),service 侧只是把调用转给它,
|
||||||
|
// 于是「读到什么异常」与「判定端认不认得出」用的是同一份产出。
|
||||||
|
TransientPayloadStorageService producer = new TransientPayloadStorageService(
|
||||||
|
mock(TransientStorageProperties.class),
|
||||||
|
mock(StorageProperties.class),
|
||||||
|
rustfs,
|
||||||
|
mock(OssStorageService.class),
|
||||||
|
JSON,
|
||||||
|
instanceMetadata("server-1"),
|
||||||
|
taskChunkMapper,
|
||||||
|
taskScopeStateMapper);
|
||||||
|
transientPayloadStorage = mock(TransientPayloadStorageService.class);
|
||||||
|
when(transientPayloadStorage.resolvePayload(anyString(), anyString())).thenAnswer(invocation ->
|
||||||
|
producer.resolvePayload(invocation.getArgument(0), invocation.getArgument(1)));
|
||||||
|
service = new BrandTaskStorageService(taskChunkMapper, taskScopeStateMapper, JSON, transientPayloadStorage);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 三片里中间那片对象被删(RustFS NoSuchKey):跳过它,其余两片的数据必须照样聚合出来,
|
||||||
|
* 且**不许**因为「DB 里有 3 行 = chunkTotal 3」就判成已收齐。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void missingChunkPayloadIsSkippedAndNeverCountedAsComplete() throws Exception {
|
||||||
|
when(taskChunkMapper.selectList(any())).thenReturn(List.of(
|
||||||
|
chunk(1, chunkJson(1, 3, "品牌A")),
|
||||||
|
chunk(2, "rustfs:task-chunk/BRAND/" + TASK_ID + "/scope-a/chunk-2.json"),
|
||||||
|
chunk(3, chunkJson(3, 3, "品牌C"))));
|
||||||
|
stubMissingObject("task-chunk/BRAND/" + TASK_ID + "/scope-a/chunk-2.json");
|
||||||
|
stubAggregateWrite();
|
||||||
|
|
||||||
|
BrandTaskStorageService.ChunkStoreResult result = service.refreshFileAggregate(TASK_ID, FILE_URL);
|
||||||
|
BrandFileAggregateCacheDto aggregate = result.aggregate();
|
||||||
|
|
||||||
|
assertNotNull(aggregate, "单个坏分片不得让整份聚合消失");
|
||||||
|
assertEquals(List.of("品牌A", "品牌C"), aggregate.getKeptBrands(), "其余分片的数据必须照常聚合");
|
||||||
|
assertEquals(2, aggregate.getReceivedChunkCount(), "已收数只认真正读到的片");
|
||||||
|
assertFalse(Boolean.TRUE.equals(aggregate.getCompleted()), "缺数据的文件不许被判成已完成");
|
||||||
|
assertFalse(result.newlyCompletedFile(), "缺口未补上时不得回报「本文件刚完成」");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 跨实例 local 指针(载荷只在另一台机器上)同样按「单项跳过」处理:
|
||||||
|
* 这一形状此前抛出后被 {@code tryRecoverCompletedRunningTask} 的外层 catch 变成
|
||||||
|
* {@code return false}——放弃整任务恢复,而不是跳过这一片。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void crossInstanceLocalPointerIsSkippedAndOtherChunksSurvive() throws Exception {
|
||||||
|
when(taskChunkMapper.selectList(any())).thenReturn(List.of(
|
||||||
|
chunk(1, chunkJson(1, 2, "品牌A")),
|
||||||
|
chunk(2, "local:i/server-110/task-chunk/BRAND/" + TASK_ID + "/scope-a/chunk-2.json")));
|
||||||
|
stubAggregateWrite();
|
||||||
|
|
||||||
|
BrandFileAggregateCacheDto aggregate = service.refreshFileAggregate(TASK_ID, FILE_URL).aggregate();
|
||||||
|
|
||||||
|
assertNotNull(aggregate);
|
||||||
|
assertEquals(List.of("品牌A"), aggregate.getKeptBrands());
|
||||||
|
assertEquals(1, aggregate.getReceivedChunkCount());
|
||||||
|
assertFalse(Boolean.TRUE.equals(aggregate.getCompleted()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 反向对账:一片都不坏时,收齐判据必须照旧成立(别让容错把「已完成」判成永不完成)。 */
|
||||||
|
@Test
|
||||||
|
void allChunksReadableStillMarksTheFileComplete() throws Exception {
|
||||||
|
when(taskChunkMapper.selectList(any())).thenReturn(List.of(
|
||||||
|
chunk(1, chunkJson(1, 2, "品牌A")),
|
||||||
|
chunk(2, chunkJson(2, 2, "品牌B"))));
|
||||||
|
stubAggregateWrite();
|
||||||
|
|
||||||
|
BrandTaskStorageService.ChunkStoreResult result = service.refreshFileAggregate(TASK_ID, FILE_URL);
|
||||||
|
|
||||||
|
assertEquals(2, result.aggregate().getReceivedChunkCount());
|
||||||
|
assertTrue(Boolean.TRUE.equals(result.aggregate().getCompleted()), "两片齐了就该判已完成");
|
||||||
|
assertTrue(result.newlyCompletedFile());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 全部分片都读不到时才允许抛出——这时没有任何数据可交付,跳过也变不出结果。 */
|
||||||
|
@Test
|
||||||
|
void everyChunkUnreadableStillFailsLoudly() throws Exception {
|
||||||
|
when(taskChunkMapper.selectList(any())).thenReturn(List.of(
|
||||||
|
chunk(1, "rustfs:task-chunk/BRAND/" + TASK_ID + "/scope-a/chunk-1.json")));
|
||||||
|
stubMissingObject("task-chunk/BRAND/" + TASK_ID + "/scope-a/chunk-1.json");
|
||||||
|
|
||||||
|
BusinessException thrown = assertThrows(BusinessException.class,
|
||||||
|
() -> service.refreshFileAggregate(TASK_ID, FILE_URL));
|
||||||
|
assertTrue(thrown.getMessage().contains("全部分片载荷均不可读"),
|
||||||
|
"抛出文案要点名是整体缺失,实际: " + thrown.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 解析缓存:一个文件的缓存载荷读不到不许把整份缓存抹掉——
|
||||||
|
* 抛出会让 {@code BrandTaskService.finalizeTask/abort} 整单失败(那两处没有就地 try)。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void parsedPayloadSkipsUnreadableFileAndKeepsTheRest() throws Exception {
|
||||||
|
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(
|
||||||
|
parsedState("rustfs:task-parsed/BRAND/" + TASK_ID + "/scope-a/latest.json"),
|
||||||
|
parsedState("{\"fileUrl\":\"" + FILE_URL + "\",\"originalFilename\":\"source.xlsx\"}")));
|
||||||
|
stubMissingObject("task-parsed/BRAND/" + TASK_ID + "/scope-a/latest.json");
|
||||||
|
|
||||||
|
List<BrandParsedFileCacheDto> payload = service.getParsedPayload(TASK_ID);
|
||||||
|
|
||||||
|
assertEquals(1, payload.size(), "坏的那条跳过,好的那条必须还在");
|
||||||
|
assertEquals(FILE_URL, payload.get(0).getFileUrl());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 汇总状态:同口径,坏的那一个文件缺席,其余文件照常返回(调用方按「尚未收齐」重试而不是整单 500)。 */
|
||||||
|
@Test
|
||||||
|
void allFileAggregatesSkipsUnreadableEntryAndKeepsTheRest() throws Exception {
|
||||||
|
TaskScopeStateEntity broken = new TaskScopeStateEntity();
|
||||||
|
broken.setTaskId(TASK_ID);
|
||||||
|
broken.setModuleType("BRAND");
|
||||||
|
broken.setScopeKey("https://oss/brand/broken.xlsx");
|
||||||
|
broken.setStateJson("rustfs:task-scope/BRAND/" + TASK_ID + "/scope-broken/latest.json");
|
||||||
|
TaskScopeStateEntity healthy = new TaskScopeStateEntity();
|
||||||
|
healthy.setTaskId(TASK_ID);
|
||||||
|
healthy.setModuleType("BRAND");
|
||||||
|
healthy.setScopeKey(FILE_URL);
|
||||||
|
healthy.setStateJson("{\"fileUrl\":\"" + FILE_URL + "\",\"chunkTotal\":1,\"receivedChunkCount\":1,\"completed\":true}");
|
||||||
|
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(broken, healthy));
|
||||||
|
stubMissingObject("task-scope/BRAND/" + TASK_ID + "/scope-broken/latest.json");
|
||||||
|
|
||||||
|
Map<String, BrandFileAggregateCacheDto> aggregates = service.getAllFileAggregates(TASK_ID);
|
||||||
|
|
||||||
|
assertEquals(1, aggregates.size(), "坏的那个文件缺席,好的那个必须还在");
|
||||||
|
assertTrue(Boolean.TRUE.equals(aggregates.get(FILE_URL).getCompleted()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 夹具 =====
|
||||||
|
|
||||||
|
private static InstanceMetadata instanceMetadata(String instanceId) {
|
||||||
|
InstanceMetadata metadata = mock(InstanceMetadata.class);
|
||||||
|
when(metadata.getInstanceId()).thenReturn(instanceId);
|
||||||
|
return metadata;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TaskChunkEntity chunk(int index, String payloadJson) {
|
||||||
|
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||||
|
chunk.setId((long) index);
|
||||||
|
chunk.setTaskId(TASK_ID);
|
||||||
|
chunk.setModuleType("BRAND");
|
||||||
|
chunk.setScopeKey(FILE_URL);
|
||||||
|
chunk.setScopeHash("scope-a");
|
||||||
|
chunk.setChunkIndex(index);
|
||||||
|
chunk.setChunkTotal(3);
|
||||||
|
chunk.setPayloadJson(payloadJson);
|
||||||
|
return chunk;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String chunkJson(int index, int total, String brand) throws Exception {
|
||||||
|
BrandCrawlResultFileDto file = new BrandCrawlResultFileDto();
|
||||||
|
file.setFileUrl(FILE_URL);
|
||||||
|
file.setOriginalFilename("source.xlsx");
|
||||||
|
file.setChunkIndex(index);
|
||||||
|
file.setChunkTotal(total);
|
||||||
|
file.setTotalLines(119);
|
||||||
|
file.setKeptRows(List.of(brand));
|
||||||
|
return JSON.writeValueAsString(file);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TaskScopeStateEntity parsedState(String payloadJson) {
|
||||||
|
TaskScopeStateEntity state = new TaskScopeStateEntity();
|
||||||
|
state.setTaskId(TASK_ID);
|
||||||
|
state.setModuleType("BRAND");
|
||||||
|
state.setScopeHash("scope-a");
|
||||||
|
state.setParsedPayloadJson(payloadJson);
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void stubMissingObject(String objectKey) {
|
||||||
|
when(rustfs.isConfigured()).thenReturn(true);
|
||||||
|
when(rustfs.readObjectBytes(objectKey)).thenThrow(new IllegalStateException(
|
||||||
|
"The specified key does not exist. (Service: S3, Status Code: 404)"));
|
||||||
|
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void stubAggregateWrite() {
|
||||||
|
TaskScopeStateEntity state = new TaskScopeStateEntity();
|
||||||
|
state.setId(9L);
|
||||||
|
state.setTaskId(TASK_ID);
|
||||||
|
state.setModuleType("BRAND");
|
||||||
|
state.setScopeKey(FILE_URL);
|
||||||
|
state.setStateJson("\"rustfs:task-scope/BRAND/" + TASK_ID + "/scope-a/latest.json\"");
|
||||||
|
when(taskScopeStateMapper.selectOne(any())).thenReturn(state);
|
||||||
|
when(taskScopeStateMapper.update(any(), any())).thenReturn(1);
|
||||||
|
when(taskScopeStateMapper.selectCount(any())).thenReturn(0L);
|
||||||
|
when(transientPayloadStorage.storeScopePayloadVersioned(anyString(), any(), anyString(), anyString(), anyBoolean()))
|
||||||
|
.thenReturn("\"rustfs:task-scope/BRAND/" + TASK_ID + "/scope-a/new.json\"");
|
||||||
|
}
|
||||||
|
}
|
||||||
+191
@@ -0,0 +1,191 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.config.InstanceMetadata;
|
||||||
|
import com.nanri.aiimage.config.StorageProperties;
|
||||||
|
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||||
|
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyInt;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.ArgumentMatchers.isNull;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 货源分片载荷读取的降级判据(2026-10-27 复验定性为假绿的那两处)。
|
||||||
|
*
|
||||||
|
* <p>判定端此前匹英文 {@code only exists on instance=},而 {@code TransientPayloadStorageService}
|
||||||
|
* 真正抛出的是中文「临时载荷仅存在于实例 X(当前实例 Y)…」,且 {@code resolvePayload} 还会把它
|
||||||
|
* rewrap 成「{@code <errorMessage>}: {@code <pointer>}」(中文只在 cause)——于是
|
||||||
|
* <ul>
|
||||||
|
* <li>{@code crossInstance} 恒 false,{@code chunk-read-failed[N]@cross-instance} 永不下笔;</li>
|
||||||
|
* <li>{@code does not exist} 同理只摸顶层消息,「对象已删就跳过该分片」那条降级分支也是死的,
|
||||||
|
* 一个坏分片会把整单组装/收尾拖成抛错。</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* <p>这里不再桩造任何文案:异常一律由<b>真实</b>的 {@link TransientPayloadStorageService#resolvePayload}
|
||||||
|
* 产出(本地磁盘/对象存储那两条真实读取路径),判定端不沿 cause 链匹配就判不出来。
|
||||||
|
*/
|
||||||
|
class SimilarAsinChunkPayloadSupportDegradedReaderTest {
|
||||||
|
|
||||||
|
private static final Long TASK_ID = 31577L;
|
||||||
|
private static final String LOCAL_OBJECT_KEY =
|
||||||
|
"task-chunk/SIMILAR_ASIN/" + TASK_ID + "/scope-a/chunk-2.json";
|
||||||
|
private static final String MISSING_OBJECT_KEY =
|
||||||
|
"task-chunk/SIMILAR_ASIN/" + TASK_ID + "/scope-a/chunk-9.json";
|
||||||
|
private static final String ROWS_JSON =
|
||||||
|
"[{\"rowToken\":\"r1\",\"id\":\"1\",\"asin\":\"B0A0000001\",\"country\":\"英国\"}]";
|
||||||
|
|
||||||
|
private TaskScopeStateMapper taskScopeStateMapper;
|
||||||
|
private TransientPayloadStorageService storage;
|
||||||
|
private SimilarAsinChunkPayloadSupport support;
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void initMybatisMetadata() {
|
||||||
|
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() {
|
||||||
|
taskScopeStateMapper = mock(TaskScopeStateMapper.class);
|
||||||
|
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
|
||||||
|
when(rustfs.isConfigured()).thenReturn(true);
|
||||||
|
when(rustfs.readObjectBytes(MISSING_OBJECT_KEY)).thenThrow(new IllegalStateException(
|
||||||
|
"The specified key does not exist. (Service: S3, Status Code: 404)"));
|
||||||
|
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of());
|
||||||
|
storage = new TransientPayloadStorageService(
|
||||||
|
mock(TransientStorageProperties.class), mock(StorageProperties.class), rustfs,
|
||||||
|
mock(OssStorageService.class), new ObjectMapper(), instanceMetadata("server-1"),
|
||||||
|
mock(TaskChunkMapper.class), taskScopeStateMapper);
|
||||||
|
support = new SimilarAsinChunkPayloadSupport(new ObjectMapper(), storage, taskScopeStateMapper);
|
||||||
|
TaskScopeStateEntity scope = new TaskScopeStateEntity();
|
||||||
|
scope.setId(55L);
|
||||||
|
scope.setLastError("");
|
||||||
|
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 跨实例 local 指针:仍抛出(调用方要据此重试/换实例),但留痕必须带 {@code @cross-instance}。 */
|
||||||
|
@Test
|
||||||
|
void crossInstanceLocalPointerStaysFatalButIsTaggedAsCrossInstance() {
|
||||||
|
TaskChunkEntity chunk = chunk(2, "local:i/server-110/" + LOCAL_OBJECT_KEY);
|
||||||
|
|
||||||
|
BusinessException thrown = assertThrows(BusinessException.class, () -> support.readChunkRows(chunk));
|
||||||
|
assertTrue(thrown.getMessage().contains("分片=2"), "异常要点名分片,实际: " + thrown.getMessage());
|
||||||
|
String written = writtenTag();
|
||||||
|
assertTrue(written.contains("chunk-read-failed[2@cross-instance]"),
|
||||||
|
"跨实例必须留 @cross-instance 痕,实际写进 last_error 的是: " + written);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 对象已被删:重试也没用,跳过该分片出空行、只留普通痕,不许把整单组装拖死。 */
|
||||||
|
@Test
|
||||||
|
void missingObjectChunkIsSkippedInsteadOfKillingTheAssembly() {
|
||||||
|
TaskChunkEntity chunk = chunk(9, "rustfs:" + MISSING_OBJECT_KEY);
|
||||||
|
|
||||||
|
Map<String, SimilarAsinResultRowDto> rows = support.readChunkRows(chunk);
|
||||||
|
|
||||||
|
assertTrue(rows.isEmpty(), "缺失载荷按空片跳过");
|
||||||
|
String written = writtenTag();
|
||||||
|
assertTrue(written.contains("chunk-read-failed[9]"), "跳过也要留痕,实际: " + written);
|
||||||
|
assertFalse(written.contains("@cross-instance"), "对象缺失不是跨实例,不许冒充跨实例标记,实际: " + written);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 载荷健在时必须照常出行(容错不许把正常路径一起判空)。 */
|
||||||
|
@Test
|
||||||
|
void readableChunkStillYieldsItsRows() {
|
||||||
|
Map<String, SimilarAsinResultRowDto> rows = support.readChunkRows(chunk(3, ROWS_JSON));
|
||||||
|
|
||||||
|
assertEquals(1, rows.size());
|
||||||
|
assertTrue(rows.containsKey("r1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 留痕与读回是一件事的两半:读回端({@code :153} 的 {@code endsWith("@cross-instance")})此前
|
||||||
|
* 拿到的永远是裸标记,因为写端认不出跨实例。这里把<b>真实写进去的那枚标记</b>原样喂回读端,
|
||||||
|
* 断言用户可见摘要会说「跨实例」——写端一改回旧英文串,这支就红。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void writtenCrossInstanceTagIsRenderedAsCrossInstanceInSummary() {
|
||||||
|
assertThrows(BusinessException.class,
|
||||||
|
() -> support.readChunkRows(chunk(2, "local:i/server-110/" + LOCAL_OBJECT_KEY)),
|
||||||
|
"跨实例载荷仍要抛出(由调用方重试/换实例)");
|
||||||
|
String written = writtenTagOfLastUpdate();
|
||||||
|
TaskScopeStateEntity scope = new TaskScopeStateEntity();
|
||||||
|
scope.setTaskId(TASK_ID);
|
||||||
|
scope.setModuleType("SIMILAR_ASIN");
|
||||||
|
scope.setLastError(written);
|
||||||
|
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(scope));
|
||||||
|
|
||||||
|
String summary = support.collectChunkReadFailureSummary(TASK_ID);
|
||||||
|
|
||||||
|
assertTrue(summary.contains("跨实例"), "摘要必须把跨实例与一般读失败分开,实际写痕="
|
||||||
|
+ written + ",摘要=" + summary);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 取上一次 update 真正写进 last_error 的那枚标记(时间戳参数按前缀滤掉)。 */
|
||||||
|
private String writtenTagOfLastUpdate() {
|
||||||
|
ArgumentCaptor<LambdaUpdateWrapper<TaskScopeStateEntity>> captor =
|
||||||
|
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||||
|
verify(taskScopeStateMapper).update(isNull(), captor.capture());
|
||||||
|
return captor.getValue().getParamNameValuePairs().values().stream()
|
||||||
|
.map(String::valueOf)
|
||||||
|
.filter(value -> value.startsWith("chunk-read-failed["))
|
||||||
|
.findFirst()
|
||||||
|
.orElse("");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== 夹具 =====
|
||||||
|
|
||||||
|
private String writtenTag() {
|
||||||
|
ArgumentCaptor<LambdaUpdateWrapper<TaskScopeStateEntity>> captor =
|
||||||
|
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||||
|
verify(taskScopeStateMapper).update(isNull(), captor.capture());
|
||||||
|
return String.join("|", captor.getValue().getParamNameValuePairs().values().stream()
|
||||||
|
.map(String::valueOf).toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static InstanceMetadata instanceMetadata(String instanceId) {
|
||||||
|
InstanceMetadata metadata = mock(InstanceMetadata.class);
|
||||||
|
when(metadata.getInstanceId()).thenReturn(instanceId);
|
||||||
|
return metadata;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TaskChunkEntity chunk(int index, String payloadJson) {
|
||||||
|
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||||
|
chunk.setId((long) index);
|
||||||
|
chunk.setTaskId(TASK_ID);
|
||||||
|
chunk.setModuleType("SIMILAR_ASIN");
|
||||||
|
chunk.setScopeKey("https://oss/similar-asin/source.xlsx");
|
||||||
|
chunk.setScopeHash("scope-a");
|
||||||
|
chunk.setChunkIndex(index);
|
||||||
|
chunk.setPayloadJson(payloadJson);
|
||||||
|
return chunk;
|
||||||
|
}
|
||||||
|
}
|
||||||
+43
-1
@@ -34,6 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
|||||||
import static org.mockito.ArgumentMatchers.any;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
import static org.mockito.Mockito.lenient;
|
import static org.mockito.Mockito.lenient;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
import static org.mockito.Mockito.never;
|
import static org.mockito.Mockito.never;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
@@ -356,6 +357,16 @@ class SimilarAsinHistoryAssemblerTest {
|
|||||||
assertTrue(item.getFileProgressMessage().endsWith(" 行"), "无 task 无 chunk 仍按行回退,实际: " + item.getFileProgressMessage());
|
assertTrue(item.getFileProgressMessage().endsWith(" 行"), "无 task 无 chunk 仍按行回退,实际: " + item.getFileProgressMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 跨实例 local 指针的留痕标记必须是 `chunk-read-failed[N@cross-instance]`。
|
||||||
|
*
|
||||||
|
* <p><b>异常原文一律由真实生产者给出,测试里不许手写文案。</b>此前这支用例桩的是
|
||||||
|
* {@code "payload only exists on instance=server-110"} —— 那是 {@code TransientPayloadStorageService}
|
||||||
|
* 早已不再产出的旧英文串,而真实抛出的是中文「临时载荷仅存在于实例 X(当前实例 Y)…」,
|
||||||
|
* 且 {@code resolvePayload} 还会把它 rewrap 成「<errorMessage>: <pointer>」(中文只在 cause)。
|
||||||
|
* 判定端只摸顶层消息 ⇒ {@code crossInstance} 恒 false ⇒ 断言照旧绿,是假绿本体。
|
||||||
|
* 现在这里跑真实 {@code resolvePayload} 取异常,判定端不沿 cause 链就判不出跨实例。
|
||||||
|
*/
|
||||||
@Test
|
@Test
|
||||||
void test_assembler_chunk_read_failure_records_error() {
|
void test_assembler_chunk_read_failure_records_error() {
|
||||||
// 回退路径读 chunk 载荷失败 → 抛 BusinessException 且 last_error 留痕
|
// 回退路径读 chunk 载荷失败 → 抛 BusinessException 且 last_error 留痕
|
||||||
@@ -378,9 +389,12 @@ class SimilarAsinHistoryAssemblerTest {
|
|||||||
scope.setLastError("");
|
scope.setLastError("");
|
||||||
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
|
FileResultEntity row = result(100L, 10L, "a.xlsx", null, 0);
|
||||||
|
|
||||||
|
// 先由真实生产者产出异常原文,再做桩——在 when(...) 的参数里造 mock 会触发 UnfinishedStubbing
|
||||||
|
IllegalStateException crossInstanceFailure = crossInstanceFailureFromRealProducer();
|
||||||
|
|
||||||
when(fileTaskMapper.selectById(10L)).thenReturn(t);
|
when(fileTaskMapper.selectById(10L)).thenReturn(t);
|
||||||
when(transientPayloadStorageService.resolvePayload("ref:missing", "read similar ASIN chunk failed"))
|
when(transientPayloadStorageService.resolvePayload("ref:missing", "read similar ASIN chunk failed"))
|
||||||
.thenThrow(new IllegalStateException("payload only exists on instance=server-110"));
|
.thenThrow(crossInstanceFailure);
|
||||||
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(state));
|
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(state));
|
||||||
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope);
|
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope);
|
||||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||||
@@ -396,4 +410,32 @@ class SimilarAsinHistoryAssemblerTest {
|
|||||||
.map(String::valueOf).toList());
|
.map(String::valueOf).toList());
|
||||||
assertTrue(written.contains("chunk-read-failed[2@cross-instance]"), "跨实例标记留痕,实际: " + written);
|
assertTrue(written.contains("chunk-read-failed[2@cross-instance]"), "跨实例标记留痕,实际: " + written);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 用**真实**的 {@link TransientPayloadStorageService#resolvePayload} 产出跨实例失败异常,
|
||||||
|
* 而不是在测试里抄一份字面量:抄的那份会随生产者改文案而悄悄失效(就是本支用例原来的样子)。
|
||||||
|
*/
|
||||||
|
private IllegalStateException crossInstanceFailureFromRealProducer() {
|
||||||
|
com.nanri.aiimage.config.TransientStorageProperties transientProps =
|
||||||
|
mock(com.nanri.aiimage.config.TransientStorageProperties.class);
|
||||||
|
com.nanri.aiimage.config.StorageProperties storageProps =
|
||||||
|
mock(com.nanri.aiimage.config.StorageProperties.class);
|
||||||
|
com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService rustfs =
|
||||||
|
mock(com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService.class);
|
||||||
|
com.nanri.aiimage.modules.file.service.oss.OssStorageService oss =
|
||||||
|
mock(com.nanri.aiimage.modules.file.service.oss.OssStorageService.class);
|
||||||
|
com.nanri.aiimage.config.InstanceMetadata instanceMetadata =
|
||||||
|
mock(com.nanri.aiimage.config.InstanceMetadata.class);
|
||||||
|
when(instanceMetadata.getInstanceId()).thenReturn("server-1");
|
||||||
|
TransientPayloadStorageService producer = new TransientPayloadStorageService(
|
||||||
|
transientProps, storageProps, rustfs, oss, new ObjectMapper(), instanceMetadata,
|
||||||
|
mock(com.nanri.aiimage.modules.task.mapper.TaskChunkMapper.class),
|
||||||
|
mock(com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper.class));
|
||||||
|
// 指针归另一台实例(i/server-110),当前实例 server-1 —— 真实抛点 TransientPayloadStorageService:146
|
||||||
|
IllegalStateException failure = assertThrows(IllegalStateException.class, () -> producer.resolvePayload(
|
||||||
|
"local:i/server-110/task-chunk/SIMILAR_ASIN/10/s1/chunk-2.json", "read similar ASIN chunk failed"));
|
||||||
|
assertTrue(failure.getMessage().startsWith("read similar ASIN chunk failed: local:"),
|
||||||
|
"生产者会把原因 rewrap 成「<errorMessage>: <pointer>」,实际: " + failure.getMessage());
|
||||||
|
return failure;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+140
@@ -0,0 +1,140 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import com.nanri.aiimage.common.task.TaskStatuses;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 迟到回传重开的判定面(2026-10-27 复验第 3 条:白名单里认不出自家无回传判死生产者写的文案)。
|
||||||
|
*
|
||||||
|
* <p>客户端还在补发时,判死巡检只把 {@code biz_file_task} 改成 FAILED,旧逻辑按终态回 40901,
|
||||||
|
* 客户端拿到 {@code success=true} 就删本地副本 ⇒ 已采集的数据整批丢掉(生产 31577/31472)。
|
||||||
|
* 判据此前是一串各自手抄的 error_message 字面量,与生产者之间没有任何编译期约束:
|
||||||
|
* 生产者改一个字,重开就静默失效。现在文案由
|
||||||
|
* {@link SimilarAsinStaleReopenSupport#clientStallFailureMessage(long, String)} 单点产出,
|
||||||
|
* 生产者与判定端都从它取数。
|
||||||
|
*/
|
||||||
|
class SimilarAsinStaleReopenSupportTest {
|
||||||
|
|
||||||
|
private static final Long TASK_ID = 31577L;
|
||||||
|
|
||||||
|
private FileTaskMapper fileTaskMapper;
|
||||||
|
private SimilarAsinTaskOwnershipSupport ownershipSupport;
|
||||||
|
private SimilarAsinStaleReopenSupport support;
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void initMybatisMetadata() {
|
||||||
|
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||||
|
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void setUp() {
|
||||||
|
fileTaskMapper = mock(FileTaskMapper.class);
|
||||||
|
ownershipSupport = mock(SimilarAsinTaskOwnershipSupport.class);
|
||||||
|
support = new SimilarAsinStaleReopenSupport(
|
||||||
|
fileTaskMapper, mock(FileResultMapper.class), mock(TaskFileJobService.class),
|
||||||
|
mock(SimilarAsinTaskCacheService.class), ownershipSupport);
|
||||||
|
when(ownershipSupport.isOwnerCurrent(any())).thenReturn(true);
|
||||||
|
when(ownershipSupport.ownerFromTask(any())).thenReturn("server-1");
|
||||||
|
when(fileTaskMapper.update(any(), any())).thenReturn(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 判死生产者那句话必须能重开。这一支单独钉「尾句」——它此前只靠白名单里的
|
||||||
|
* 「任务创建至今超过」那一枚前缀碰巧命中,生产者一改前缀(例如改成「建单至今」)
|
||||||
|
* 重开就整条失效,而现场看不出来。改前本用例红(只含尾句的文案认不出)。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void clientStallDeathIsReopenedByItsOwnTailPhrase() {
|
||||||
|
String driftedPrefix = "建单至今超过 120 分钟无任何结果回传,处理位置长时间停在「正在采集第 118/119 行」未推进,"
|
||||||
|
+ SimilarAsinStaleReopenSupport.CLIENT_STALL_FAILURE_SUFFIX;
|
||||||
|
stubFailedTask(driftedPrefix);
|
||||||
|
|
||||||
|
support.reopenIfStaleFailed(TASK_ID);
|
||||||
|
|
||||||
|
verify(fileTaskMapper).update(any(), any());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 生产者与判定端共用一枚出口:拿工厂造出来的文案,重开必须发生。 */
|
||||||
|
@Test
|
||||||
|
void producerShapedDeathIsReopened() {
|
||||||
|
stubFailedTask(SimilarAsinStaleReopenSupport.clientStallFailureMessage(
|
||||||
|
120, "正在采集第 118/119 行"));
|
||||||
|
|
||||||
|
support.reopenIfStaleFailed(TASK_ID);
|
||||||
|
|
||||||
|
verify(fileTaskMapper).update(any(), any());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 业务性失败不许重开(结果文件生成本身失败,重开只是让客户端继续往坏管道里灌数据)。 */
|
||||||
|
@Test
|
||||||
|
void businessFailureIsNotReopened() {
|
||||||
|
stubFailedTask("已收到 3 个结果分片,但结果文件生成多次失败,任务已自动结束");
|
||||||
|
|
||||||
|
support.reopenIfStaleFailed(TASK_ID);
|
||||||
|
|
||||||
|
verify(fileTaskMapper, never()).update(any(), any());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 字面量只许有一处:生产者({@code SimilarAsinTaskService} 的无回传判死)必须走
|
||||||
|
* {@code clientStallFailureMessage},不许再自己拼一遍——两处各写一遍正是本批要消掉的形状。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void producerTakesItsDeathMessageFromTheSharedFactory() throws IOException {
|
||||||
|
Path producer = moduleRoot().resolve(
|
||||||
|
"src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java");
|
||||||
|
String source = Files.readString(producer, StandardCharsets.UTF_8);
|
||||||
|
assertTrue(source.contains("SimilarAsinStaleReopenSupport.clientStallFailureMessage("),
|
||||||
|
"生产者不再走共享文案出口,重开判定与判死文案又要各自漂移:" + producer);
|
||||||
|
assertFalse(source.contains("」未推进,疑似客户端任务卡死"),
|
||||||
|
"生产者里不许再出现第二份卡死判死文案字面量");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void stubFailedTask(String errorMessage) {
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(TASK_ID);
|
||||||
|
task.setModuleType("SIMILAR_ASIN");
|
||||||
|
task.setStatus(TaskStatuses.FileTask.FAILED);
|
||||||
|
task.setErrorMessage(errorMessage);
|
||||||
|
task.setFinishedAt(LocalDateTime.now().minusMinutes(5));
|
||||||
|
task.setUpdatedAt(LocalDateTime.now().minusMinutes(5));
|
||||||
|
when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Path moduleRoot() {
|
||||||
|
Path here = Path.of("").toAbsolutePath();
|
||||||
|
if (Files.isDirectory(here.resolve("src/main/java"))) {
|
||||||
|
return here;
|
||||||
|
}
|
||||||
|
Path nested = here.resolve("backend-java");
|
||||||
|
if (Files.isDirectory(nested.resolve("src/main/java"))) {
|
||||||
|
return nested;
|
||||||
|
}
|
||||||
|
throw new IllegalStateException("无法定位 backend-java 模块目录(cwd=" + here + ")");
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user