fix(长任务判死): 封口前增加「客户端仍在推进」豁免,避免还在采集就被结案丢数据

服务端二次判死线(心跳正常但 N 分钟无结果分片上报)只看分片间隔,客户端攒批回传一慢
就把仍在采集的任务判成卡死、封口结案,后续分片全被拒(生产 28919:1300 行只出 395 行,
丢 905 行)。改为封口前用 TaskHeartbeatPositionService 的位置快照判定客户端是否仍在推进:
ADVANCING(位置在变)→ 本轮不封口;STALLED(长时间不变)→ 按原判据;UNKNOWN(老客户端
无位置信息)→ 保留原行为。接入 similar-asin / appearance-patent / brand / delete-brand /
collect-data 五条判死线。
This commit is contained in:
2026-09-23 10:52:45 +08:00
parent 8d73377b25
commit afa45d51c3
16 changed files with 348 additions and 6 deletions
@@ -25,10 +25,25 @@ import java.util.Map;
public class TaskHeartbeatPositionService {
private static final String KEY_PREFIX = "task:heartbeat:position:";
private static final String STALL_KEY_PREFIX = "task:progress:stall:";
private static final Duration POSITION_TTL = Duration.ofHours(24);
private final StringRedisTemplate stringRedisTemplate;
/**
* 位置推进状态。
*
* <p>用于「封口/判死前的豁免」:{@link #ADVANCING} 表示客户端主线程仍在推进,
* 调用方不得据超时封口(否则后续回传被拒、数据丢失);{@link #STALLED} 才是
* 「心跳照发但主线程真的不动了」;{@link #UNKNOWN} 表示没有位置信息可用,
* 调用方保留原有判据——它<b>不是</b>判死证据,只是没有豁免依据。
*/
public enum StallState {
ADVANCING,
STALLED,
UNKNOWN
}
public void record(Long taskId, TaskHeartbeatRequest request) {
if (taskId == null || taskId <= 0 || request == null) {
return;
@@ -91,6 +106,88 @@ public class TaskHeartbeatPositionService {
return sb.length() == 0 ? null : sb.toString();
}
/**
* 判定客户端是否「仍在推进」,供封口/判死前的豁免使用。
*
* <p>把当前位置文本与上次快照比对:
* <ul>
* <li>文本变了 → {@link StallState#ADVANCING}(客户端主线程在干活,只是分片还没回传);</li>
* <li>文本相同且已持续 {@code staleWindowMinutes} → {@link StallState#STALLED}(真卡死);</li>
* <li>无位置信息(老客户端不传 progressText) → {@link StallState#UNKNOWN}。</li>
* </ul>
*
* <p>为什么需要它:既有二次判死线只看「多久没收到结果分片」(biz_task_scope_state.last_chunk_at),
* 客户端攒批回传间隔一旦超过阈值(生产 28919:1300 行每 1.5~3 小时才回传一次),
* 服务端就会把仍在采集的任务判成卡死、封口结案,后续分片全部被拒、数据丢失。
* 位置文本由客户端主线程维护:它在变就证明采集在推进,此时不得封口。
*
* @param staleWindowMinutes 判定停滞所需的最短「无变化」时长(分钟),&lt;=0 时返回 UNKNOWN
*/
public StallState evaluateStall(Long taskId, long staleWindowMinutes) {
if (taskId == null || taskId <= 0 || staleWindowMinutes <= 0) {
return StallState.UNKNOWN;
}
String position = describe(taskId);
if (position == null || position.isBlank()) {
return StallState.UNKNOWN;
}
String stallKey = STALL_KEY_PREFIX + taskId;
Map<Object, Object> stored;
try {
stored = stringRedisTemplate.opsForHash().entries(stallKey);
} catch (Exception ex) {
log.warn("[heartbeat-position] 读取停滞快照失败 taskId={} msg={}", taskId, ex.getMessage());
return StallState.UNKNOWN;
}
long now = Instant.now().toEpochMilli();
String storedPosition = stored == null ? "" : value(stored, "pos");
long unchangedSince = stored == null ? 0L : parseMillis(stored.get("at"));
if (!position.equals(storedPosition) || unchangedSince <= 0L) {
writeStallSnapshot(stallKey, position, now, staleWindowMinutes);
return StallState.ADVANCING;
}
long unchangedMillis = now - unchangedSince;
if (unchangedMillis >= Duration.ofMinutes(staleWindowMinutes).toMillis()) {
return StallState.STALLED;
}
log.debug("[heartbeat-position] 位置未变化但在豁免窗口内 taskId={} unchangedMs={} position={}",
taskId, unchangedMillis, position);
return StallState.ADVANCING;
}
/**
* 便捷判定:客户端是否仍在推进(位置文本仍在变)。
*
* <p>各模块「无结果上报」判死线在封口前统一调它:返回 true 表示采集主线程还在干活
* (分片只是攒着没回传),此时<b>不得</b>封口/结案——否则后续分片会被拒收、数据丢失。
*/
public boolean isClientAdvancing(Long taskId, long stallWindowMinutes) {
return evaluateStall(taskId, stallWindowMinutes) == StallState.ADVANCING;
}
private void writeStallSnapshot(String key, String position, long now, long staleWindowMinutes) {
try {
Map<String, String> values = new HashMap<>();
values.put("pos", position);
values.put("at", String.valueOf(now));
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, Duration.ofMinutes(Math.max(60L, staleWindowMinutes * 3L)));
} catch (Exception ex) {
log.warn("[heartbeat-position] 写入停滞快照失败 key={} msg={}", key, ex.getMessage());
}
}
private long parseMillis(Object raw) {
if (raw == null) {
return 0L;
}
try {
return Long.parseLong(String.valueOf(raw).trim());
} catch (NumberFormatException ex) {
return 0L;
}
}
private String value(Map<Object, Object> entries, String key) {
Object raw = entries.get(key);
return raw == null ? "" : String.valueOf(raw).trim();