fix(tasks): 单片/单文件读不到跳过它,缺数据不许冒充已收齐
三处"一颗坏项拖垮整批"与一处假绿本体: - BrandTaskStorageService:一片载荷读不到就整份聚合丢掉,而三个调用方 (finalizeTask/abort/tryRecoverCompletedRunningTask)都没有就地 try;改逐片跳过长 WARN,并把 receivedChunkCount/completed 的判据从 chunks.size()(DB 行数)改成 真正读到的片数——拿行数比 chunkTotal 正是"118/119 行没数据仍判已完成"的形状; 解析缓存与汇总状态两处同型"整批抛出"一并改成逐条跳过。 - AppearancePatentLlmBatchPipeline:候选收集跑在批循环之前,一抛就走不到循环体内 那道 finally releaseRowInspection,本轮已申领的行要挂满处理中 TTL(15 分钟), 且 terminalizeRoundExhaustedRows 跟着不执行(轮次到顶的行因此永不判死);改逐片容错。 - TransientPayloadStorageService:跨实例判据枚成 public 常量并提供 cause 链走法。 resolvePayload 会把真实原因 rewrap 成「<errorMessage>: <pointer>」,消费端只摸顶层 消息时「only exists on instance=」「does not exist」两处判定恒 false——跨实例留痕 从未下过笔、"跳过该分片"分支是死代码。消费端(ChunkPayloadSupport/ HistoryAssembler/ResultRowReader)全部改走共享常量+cause 链。 - SimilarAsinHistoryAssemblerTest 的假绿桩删除:改前它桩的是生产者早已不再产出的 英文旧串,断言照旧绿;现在跑真实 resolvePayload 取异常(只 mock RustFS 边界), 改前 15 跑 1 红、改后 15/15 绿。 - SimilarAsinStaleReopenSupport:客户端卡死尾句枚成常量、整句文案唯一出口,生产者 与重开白名单共用;此前只是碰巧被前缀命中,生产者一改前缀就会"永不重开"。
This commit is contained in:
+19
-3
@@ -296,8 +296,23 @@ public class AppearancePatentLlmBatchPipeline {
|
||||
int judgedSkipped = 0;
|
||||
int noDataSkipped = 0;
|
||||
int chunkRows = 0;
|
||||
int unreadableChunks = 0;
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
|
||||
Map<String, AppearancePatentResultRowDto> persistedRows;
|
||||
try {
|
||||
persistedRows = readChunkRows(chunk);
|
||||
} catch (Exception ex) {
|
||||
// 单片载荷读不到不许把整轮候选收集炸掉:本方法跑在批循环**之前**,而释放处理中标记的
|
||||
// finally 在循环体内——这里一抛就永远走不到那道 release,本轮已申领的行要挂满
|
||||
// IN_FLIGHT_TTL(15 分钟)才回来,且 terminalizeRoundExhaustedRows 收的是本方法的返回值,
|
||||
// 跟着一起不执行(轮次到顶的行因此永不判死)。跳过这一片、继续读其余片,
|
||||
// 与 AppearancePatentResultRowReader#readChunkRows/SimilarAsinChunkPayloadSupport 同口径。
|
||||
unreadableChunks++;
|
||||
log.warn("[appearance-patent] 单片载荷读取失败,跳过该分片继续收集候选"
|
||||
+ " taskId={} scopeHash={} chunk={} err={}",
|
||||
chunk.getTaskId(), chunk.getScopeHash(), chunk.getChunkIndex(), ex.getMessage());
|
||||
continue;
|
||||
}
|
||||
if (persistedRows.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
@@ -328,8 +343,9 @@ public class AppearancePatentLlmBatchPipeline {
|
||||
candidates.add(row);
|
||||
}
|
||||
}
|
||||
log.info("[appearance-patent] 候选收集逐行申领 taskId={} chunkRows={} judgedSkipped={} noDataSkipped={} claimBlocked={} candidates={}",
|
||||
chunks.get(0).getTaskId(), chunkRows, judgedSkipped, noDataSkipped, claimBlocked, candidates.size());
|
||||
log.info("[appearance-patent] 候选收集逐行申领 taskId={} chunkRows={} judgedSkipped={} noDataSkipped={} claimBlocked={} unreadableChunks={} candidates={}",
|
||||
chunks.get(0).getTaskId(), chunkRows, judgedSkipped, noDataSkipped, claimBlocked,
|
||||
unreadableChunks, candidates.size());
|
||||
return candidates;
|
||||
}
|
||||
|
||||
|
||||
+4
-10
@@ -273,17 +273,11 @@ public class AppearancePatentResultRowReader {
|
||||
}
|
||||
|
||||
/** payload 对象已不存在(RustFS 返回 NoSuchKey:message 为 "The specified key does not exist.")。
|
||||
* 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。 */
|
||||
* 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。
|
||||
* 判据与 cause 链走法都收口在 {@link TransientPayloadStorageService#isObjectMissing}——
|
||||
* 抛点与判定端各抄一遍字面量时,改一处就静默失效(货源侧那两处就是这么废掉的)。 */
|
||||
private static boolean isPayloadMissing(Throwable error) {
|
||||
Throwable cursor = error;
|
||||
while (cursor != null) {
|
||||
String message = cursor.getMessage();
|
||||
if (message != null && message.contains("does not exist")) {
|
||||
return true;
|
||||
}
|
||||
cursor = cursor.getCause();
|
||||
}
|
||||
return false;
|
||||
return TransientPayloadStorageService.isObjectMissing(error);
|
||||
}
|
||||
|
||||
public String rowKey(AppearancePatentParsedRowVo row) {
|
||||
|
||||
+41
-15
@@ -90,7 +90,7 @@ public class BrandTaskStorageService {
|
||||
return List.of();
|
||||
}
|
||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.select(TaskScopeStateEntity::getParsedPayloadJson)
|
||||
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getScopeHash)
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
|
||||
@@ -99,14 +99,17 @@ public class BrandTaskStorageService {
|
||||
}
|
||||
List<BrandParsedFileCacheDto> result = new ArrayList<>();
|
||||
for (TaskScopeStateEntity state : states) {
|
||||
String payloadJson = resolveParsedPayload(state.getParsedPayloadJson());
|
||||
if (isBlank(payloadJson)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
String payloadJson = resolveParsedPayload(state.getParsedPayloadJson());
|
||||
if (isBlank(payloadJson)) {
|
||||
continue;
|
||||
}
|
||||
result.add(objectMapper.readValue(payloadJson, BrandParsedFileCacheDto.class));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取品牌任务解析结果失败");
|
||||
// 单个文件的解析缓存读不到不许把整份缓存一起丢掉:跳过它,其余文件照常出结果。
|
||||
// 整批抛出会让 BrandTaskService 的 finalizeTask / abort 两处调用全单失败(那里没有就地 try)。
|
||||
log.warn("[brand-storage] 解析缓存读取失败,跳过该文件 taskId={} scopeHash={} err={}",
|
||||
taskId, state.getScopeHash(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
@@ -207,7 +210,10 @@ public class BrandTaskStorageService {
|
||||
String aggregateJson = transientPayloadStorageService.resolvePayload(state.getStateJson(), "read brand task aggregate failed");
|
||||
result.put(state.getScopeKey(), objectMapper.readValue(aggregateJson, BrandFileAggregateCacheDto.class));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取品牌任务汇总状态失败");
|
||||
// 单个文件的汇总载荷读不到:跳过它,其余文件的聚合照常返回。
|
||||
// 少一个文件会让调用方按「尚未收齐」处理(可重试),比整批抛出把已读到的文件一起抹掉好。
|
||||
log.warn("[brand-storage] 汇总状态读取失败,跳过该文件 taskId={} scopeKey={} err={}",
|
||||
taskId, state.getScopeKey(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
@@ -444,32 +450,52 @@ public class BrandTaskStorageService {
|
||||
}
|
||||
|
||||
BrandFileAggregateCacheDto aggregate = null;
|
||||
int readableChunks = 0;
|
||||
List<Integer> skippedChunkIndexes = new ArrayList<>();
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
BrandCrawlResultFileDto chunkPayload = readChunkPayload(chunk.getPayloadJson());
|
||||
BrandCrawlResultFileDto chunkPayload;
|
||||
try {
|
||||
chunkPayload = readChunkPayload(chunk);
|
||||
} catch (Exception ex) {
|
||||
// 单片载荷读不到(对象被清理 / 跨实例 local 指针 / JSON 坏)不许把整份聚合一起丢掉:
|
||||
// 跳过这一片、继续读其余片,与 AppearancePatentResultRowReader#readChunkRows、
|
||||
// SimilarAsinChunkPayloadSupport#readChunkRows 同口径。调用方(BrandTaskService 的
|
||||
// finalizeTask / abort / tryRecoverCompletedRunningTask)都没有就地 try,
|
||||
// 这里一抛就是整任务不产出结果。
|
||||
skippedChunkIndexes.add(chunk.getChunkIndex());
|
||||
log.warn("[brand-storage] 单片载荷读取失败,跳过该分片继续聚合 taskId={} scopeHash={} chunk={} err={}",
|
||||
taskId, scopeHash, chunk.getChunkIndex(), ex.getMessage());
|
||||
continue;
|
||||
}
|
||||
if (aggregate == null) {
|
||||
aggregate = createAggregate(chunkPayload);
|
||||
} else {
|
||||
ensureChunkConsistency(aggregate, chunkPayload);
|
||||
}
|
||||
mergeAggregate(aggregate, chunkPayload);
|
||||
readableChunks++;
|
||||
}
|
||||
if (aggregate == null) {
|
||||
throw new BusinessException("读取品牌任务汇总数据失败");
|
||||
throw new BusinessException("读取品牌任务汇总数据失败:全部分片载荷均不可读,分片=" + skippedChunkIndexes);
|
||||
}
|
||||
aggregate.setFileUrl(scopeKey);
|
||||
aggregate.setReceivedChunkCount(chunks.size());
|
||||
aggregate.setCompleted(aggregate.getChunkTotal() != null
|
||||
// 已收齐的判据只数**真正读到**的片:拿 chunks.size()(DB 行数)比 chunkTotal,
|
||||
// 会在载荷缺失时把缺数据的文件判成 completed——正是「118/119 行没数据仍报已完成」的形状。
|
||||
aggregate.setReceivedChunkCount(readableChunks);
|
||||
aggregate.setCompleted(skippedChunkIndexes.isEmpty()
|
||||
&& aggregate.getChunkTotal() != null
|
||||
&& aggregate.getChunkTotal() > 0
|
||||
&& chunks.size() >= aggregate.getChunkTotal());
|
||||
&& readableChunks >= aggregate.getChunkTotal());
|
||||
return aggregate;
|
||||
}
|
||||
|
||||
private BrandCrawlResultFileDto readChunkPayload(String payloadJson) {
|
||||
private BrandCrawlResultFileDto readChunkPayload(TaskChunkEntity chunk) {
|
||||
try {
|
||||
String resolvedPayload = transientPayloadStorageService.resolvePayload(payloadJson, "read brand task chunk failed");
|
||||
String resolvedPayload = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read brand task chunk failed");
|
||||
return objectMapper.readValue(resolvedPayload, BrandCrawlResultFileDto.class);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取品牌任务分片失败");
|
||||
// 带上分片索引与 cause:跳过之后排查只剩这一行日志,裸文案对不上是哪一片
|
||||
throw new BusinessException("读取品牌任务分片失败,分片=" + chunk.getChunkIndex(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -1109,8 +1109,8 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
log.warn("[similar-asin] 从未上报任何结果分片且处理位置长时间停在「{}」未推进,按卡死收尾 taskId={} minutes={}",
|
||||
neverReportedPosition, task.getId(), minutes);
|
||||
finalizeStaleTask(task.getId(),
|
||||
"任务创建至今超过 " + minutes + " 分钟无任何结果回传,处理位置长时间停在「"
|
||||
+ neverReportedPosition + "」未推进,疑似客户端任务卡死,任务已自动失败");
|
||||
SimilarAsinStaleReopenSupport.clientStallFailureMessage(
|
||||
minutes, neverReportedPosition));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
+8
-3
@@ -78,11 +78,15 @@ public class SimilarAsinChunkPayloadSupport {
|
||||
String msg = ex.getMessage() == null ? "" : ex.getMessage();
|
||||
// P1-4:识别跨实例 local 指针读不到的场景,把"chunk 在另一实例"的元信息
|
||||
// 通过 task_scope_state.last_error 留痕,便于排查"为何 owner 切换后 chunk 读不到"。
|
||||
boolean crossInstance = msg.contains("only exists on instance=");
|
||||
// 判据必须走 cause 链并与生产者共用同一枚常量:resolvePayload 抛给这里的顶层消息是
|
||||
// "<errorMessage>: <pointer>",「临时载荷仅存在于实例 …」只在 cause 上——
|
||||
// 旧的 contains("only exists on instance=") 认的是 TransientPayloadStorageService 早已
|
||||
// 不再产出的英文串,crossInstance 恒 false,@cross-instance 标记从未下过笔(2026-10-27 复验)。
|
||||
boolean crossInstance = TransientPayloadStorageService.isCrossInstanceLocalPointer(ex);
|
||||
// 类型不兼容(如历史版本序列化数据含当前类加载器不认识的类型标记):
|
||||
// 该 chunk 永久不可读,跳过而不是把组装/收尾永久拖死,留痕便于排查。
|
||||
boolean typeMismatch = ex instanceof TypeNotPresentException
|
||||
|| msg.contains("not present");
|
||||
|| TransientPayloadStorageService.causeChainContains(ex, "not present");
|
||||
if (typeMismatch) {
|
||||
log.warn("[similar-asin] chunk payload type mismatch, skip chunk taskId={} chunk={} err={}",
|
||||
chunk.getTaskId(), chunk.getChunkIndex(), msg);
|
||||
@@ -92,7 +96,8 @@ public class SimilarAsinChunkPayloadSupport {
|
||||
// payload 对象已不存在(RustFS 返回 NoSuchKey):重试多少次都读不回来,跳过而不是
|
||||
// 把组装/收尾永久拖死——与上面的 typeMismatch 分支、以及 collect-data 的降级口径一致。
|
||||
// 线上 appearance-patent 28459 就因同类场景每 10~30 秒重试一次(见同批修复)。
|
||||
if (msg.contains("does not exist")) {
|
||||
// 同样只认 cause 链:rewrap 后顶层消息里没有 "does not exist",这一分支此前也是死的。
|
||||
if (TransientPayloadStorageService.isObjectMissing(ex)) {
|
||||
log.warn("[similar-asin] chunk payload 已不存在,跳过该分片 taskId={} chunk={} err={}",
|
||||
chunk.getTaskId(), chunk.getChunkIndex(), msg);
|
||||
recordChunkReadFailure(chunk, false, msg);
|
||||
|
||||
+3
-1
@@ -439,7 +439,9 @@ public class SimilarAsinHistoryAssembler {
|
||||
String msg = ex.getMessage() == null ? "" : ex.getMessage();
|
||||
// P1-4:识别跨实例 local 指针读不到的场景,把"chunk 在另一实例"的元信息
|
||||
// 通过 task_scope_state.last_error 留痕,便于排查"为何 owner 切换后 chunk 读不到"。
|
||||
boolean crossInstance = msg.contains("only exists on instance=");
|
||||
// 与 SimilarAsinChunkPayloadSupport 同一枚判据:走 cause 链+生产者自己的常量,
|
||||
// 只摸顶层消息(rewrap 后是 "<errorMessage>: <pointer>")永远认不出中文原文。
|
||||
boolean crossInstance = TransientPayloadStorageService.isCrossInstanceLocalPointer(ex);
|
||||
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
|
||||
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
|
||||
recordChunkReadFailure(chunk, crossInstance, msg);
|
||||
|
||||
+15
@@ -28,12 +28,27 @@ import java.util.List;
|
||||
public class SimilarAsinStaleReopenSupport {
|
||||
|
||||
private static final String MODULE_TYPE = "SIMILAR_ASIN";
|
||||
/**
|
||||
* 客户端卡死判死文案的固定尾句,重开判定与生产者共用(口径同 {@code TaskResumeService.DUPLICATE_STOP_REASON}:
|
||||
* 两处各写一遍,改一处就静默失效——这里失效的方向是「迟到分片被拒、用户已采集的数据整批丢掉」)。
|
||||
*/
|
||||
public static final String CLIENT_STALL_FAILURE_SUFFIX = "疑似客户端任务卡死,任务已自动失败";
|
||||
/**
|
||||
* 无结果回传判死的整句文案——唯一出口,生产者({@code SimilarAsinTaskService} 的二次判死)
|
||||
* 与本类的重开判定都从它取数,前缀与尾句不会再各漂一半。
|
||||
*/
|
||||
public static String clientStallFailureMessage(long minutes, String neverReportedPosition) {
|
||||
return "任务创建至今超过 " + minutes + " 分钟无任何结果回传,处理位置长时间停在「"
|
||||
+ neverReportedPosition + "」未推进," + CLIENT_STALL_FAILURE_SUFFIX;
|
||||
}
|
||||
private static final List<String> STALE_FAILURE_MARKERS = List.of(
|
||||
"任务心跳超时,已自动失败",
|
||||
"长时间未收到 Python 心跳,任务已自动失败",
|
||||
"客户端未回传任何检测结果",
|
||||
"客户端回传中断",
|
||||
"任务创建至今超过",
|
||||
// 尾句单独进白名单:上面那枚前缀只是碰巧同句出现,生产者一改前缀重开就整条失效
|
||||
CLIENT_STALL_FAILURE_SUFFIX,
|
||||
"Python heartbeat alive but no result chunk uploaded");
|
||||
private static final Duration REOPEN_WINDOW = Duration.ofHours(24);
|
||||
|
||||
|
||||
+44
-5
@@ -45,6 +45,20 @@ public class TransientPayloadStorageService {
|
||||
private static final String OSS_POINTER_PREFIX = "oss:";
|
||||
private static final String LOCAL_PAYLOAD_DIR = "transient-payload";
|
||||
|
||||
/**
|
||||
* 跨实例消费 local 兜底指针时的判据原文(抛点见 {@link #resolvePayload})。
|
||||
*
|
||||
* <p>枚成 public 常量是因为消费端(货源 {@code SimilarAsinChunkPayloadSupport}/
|
||||
* {@code SimilarAsinHistoryAssembler} 的 {@code chunk-read-failed[N@cross-instance]} 留痕)
|
||||
* 要认的就是这句话。消费端此前各抄一份英文旧串 {@code "only exists on instance="}——
|
||||
* 生产者早就改成中文文案,两处判定恒 false,跨实例标记永不下笔(2026-10-27 复验定性为假绿)。
|
||||
* 现在两端共用这一枚常量,改文案只会让两边一起改,不会静默失效。
|
||||
*/
|
||||
public static final String CROSS_INSTANCE_LOCAL_POINTER_MARKER = "临时载荷仅存在于实例 ";
|
||||
|
||||
/** 对象确已不存在(RustFS NoSuchKey)的判据原文:抛点在 {@code RustfsObjectStorageService}。 */
|
||||
private static final String MISSING_OBJECT_MARKER = "does not exist";
|
||||
|
||||
/**
|
||||
* P0-3:记录"最近一次 store 是否因 RustFS 失败回落到 local"。
|
||||
* 调用方(如 SimilarAsinTaskService.submitResult)拿到 storedPayload 后可以查询
|
||||
@@ -144,7 +158,7 @@ public class TransientPayloadStorageService {
|
||||
if (ownerInstance != null && !ownerInstance.equals(instanceMetadata.getInstanceId())) {
|
||||
// 跨实例消费 local 指针时立即报错,避免悄悄读出空内容造成下游错位。
|
||||
throw new IllegalStateException(
|
||||
"临时载荷仅存在于实例 " + ownerInstance + "(当前实例 "
|
||||
CROSS_INSTANCE_LOCAL_POINTER_MARKER + ownerInstance + "(当前实例 "
|
||||
+ instanceMetadata.getInstanceId() + "),本地兜底指针跨节点不可读; objectKey="
|
||||
+ stripLocalInstanceId(localKey));
|
||||
}
|
||||
@@ -211,15 +225,40 @@ public class TransientPayloadStorageService {
|
||||
return candidates.isEmpty() ? null : candidates.getFirst();
|
||||
}
|
||||
|
||||
/** 对象确已不存在:RustFS 返回 NoSuchKey,message 为 "The specified key does not exist."。 */
|
||||
private static boolean isObjectMissing(Throwable error) {
|
||||
/**
|
||||
* 对象确已不存在:RustFS 返回 NoSuchKey,message 为 "The specified key does not exist."。
|
||||
*
|
||||
* <p>公开给消费端(分片载荷读取)复用。{@link #resolvePayload} 末尾会把真实原因 rewrap 成
|
||||
* 「{@code <errorMessage>}: {@code <pointer>}」,NoSuchKey 与跨实例原文都只剩在 cause 上,
|
||||
* 所以判据一律走 cause 链——只摸顶层消息的话,「跳过该分片」与「跨实例留痕」两个分支
|
||||
* 会一起变成死代码(货源侧 2026-10-27 复验就是这一形状)。
|
||||
*/
|
||||
public static boolean isObjectMissing(Throwable error) {
|
||||
return causeChainContains(error, MISSING_OBJECT_MARKER);
|
||||
}
|
||||
|
||||
/** 载荷归另一台实例(local 兜底指针跨节点不可读);同样必须走 cause 链。 */
|
||||
public static boolean isCrossInstanceLocalPointer(Throwable error) {
|
||||
return causeChainContains(error, CROSS_INSTANCE_LOCAL_POINTER_MARKER);
|
||||
}
|
||||
|
||||
/**
|
||||
* 沿 cause 链找字面量:包装层不许掩盖真实原因。
|
||||
*
|
||||
* <p>{@code cause == cursor} 的自引用按链尾处理,免得异常被自己当 cause 时把调用线程钉死。
|
||||
*/
|
||||
public static boolean causeChainContains(Throwable error, String marker) {
|
||||
if (error == null || marker == null || marker.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
Throwable cursor = error;
|
||||
while (cursor != null) {
|
||||
String message = cursor.getMessage();
|
||||
if (message != null && message.contains("does not exist")) {
|
||||
if (message != null && message.contains(marker)) {
|
||||
return true;
|
||||
}
|
||||
cursor = cursor.getCause();
|
||||
Throwable next = cursor.getCause();
|
||||
cursor = next == cursor ? null : next;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user