fix(长任务判死): 封口前增加「客户端仍在推进」豁免,避免还在采集就被结案丢数据
服务端二次判死线(心跳正常但 N 分钟无结果分片上报)只看分片间隔,客户端攒批回传一慢 就把仍在采集的任务判成卡死、封口结案,后续分片全被拒(生产 28919:1300 行只出 395 行, 丢 905 行)。改为封口前用 TaskHeartbeatPositionService 的位置快照判定客户端是否仍在推进: ADVANCING(位置在变)→ 本轮不封口;STALLED(长时间不变)→ 按原判据;UNKNOWN(老客户端 无位置信息)→ 保留原行为。接入 similar-asin / appearance-patent / brand / delete-brand / collect-data 五条判死线。
This commit is contained in:
+97
@@ -25,10 +25,25 @@ import java.util.Map;
|
||||
public class TaskHeartbeatPositionService {
|
||||
|
||||
private static final String KEY_PREFIX = "task:heartbeat:position:";
|
||||
private static final String STALL_KEY_PREFIX = "task:progress:stall:";
|
||||
private static final Duration POSITION_TTL = Duration.ofHours(24);
|
||||
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
|
||||
/**
|
||||
* 位置推进状态。
|
||||
*
|
||||
* <p>用于「封口/判死前的豁免」:{@link #ADVANCING} 表示客户端主线程仍在推进,
|
||||
* 调用方不得据超时封口(否则后续回传被拒、数据丢失);{@link #STALLED} 才是
|
||||
* 「心跳照发但主线程真的不动了」;{@link #UNKNOWN} 表示没有位置信息可用,
|
||||
* 调用方保留原有判据——它<b>不是</b>判死证据,只是没有豁免依据。
|
||||
*/
|
||||
public enum StallState {
|
||||
ADVANCING,
|
||||
STALLED,
|
||||
UNKNOWN
|
||||
}
|
||||
|
||||
public void record(Long taskId, TaskHeartbeatRequest request) {
|
||||
if (taskId == null || taskId <= 0 || request == null) {
|
||||
return;
|
||||
@@ -91,6 +106,88 @@ public class TaskHeartbeatPositionService {
|
||||
return sb.length() == 0 ? null : sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 判定客户端是否「仍在推进」,供封口/判死前的豁免使用。
|
||||
*
|
||||
* <p>把当前位置文本与上次快照比对:
|
||||
* <ul>
|
||||
* <li>文本变了 → {@link StallState#ADVANCING}(客户端主线程在干活,只是分片还没回传);</li>
|
||||
* <li>文本相同且已持续 {@code staleWindowMinutes} → {@link StallState#STALLED}(真卡死);</li>
|
||||
* <li>无位置信息(老客户端不传 progressText) → {@link StallState#UNKNOWN}。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>为什么需要它:既有二次判死线只看「多久没收到结果分片」(biz_task_scope_state.last_chunk_at),
|
||||
* 客户端攒批回传间隔一旦超过阈值(生产 28919:1300 行每 1.5~3 小时才回传一次),
|
||||
* 服务端就会把仍在采集的任务判成卡死、封口结案,后续分片全部被拒、数据丢失。
|
||||
* 位置文本由客户端主线程维护:它在变就证明采集在推进,此时不得封口。
|
||||
*
|
||||
* @param staleWindowMinutes 判定停滞所需的最短「无变化」时长(分钟),<=0 时返回 UNKNOWN
|
||||
*/
|
||||
public StallState evaluateStall(Long taskId, long staleWindowMinutes) {
|
||||
if (taskId == null || taskId <= 0 || staleWindowMinutes <= 0) {
|
||||
return StallState.UNKNOWN;
|
||||
}
|
||||
String position = describe(taskId);
|
||||
if (position == null || position.isBlank()) {
|
||||
return StallState.UNKNOWN;
|
||||
}
|
||||
String stallKey = STALL_KEY_PREFIX + taskId;
|
||||
Map<Object, Object> stored;
|
||||
try {
|
||||
stored = stringRedisTemplate.opsForHash().entries(stallKey);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[heartbeat-position] 读取停滞快照失败 taskId={} msg={}", taskId, ex.getMessage());
|
||||
return StallState.UNKNOWN;
|
||||
}
|
||||
long now = Instant.now().toEpochMilli();
|
||||
String storedPosition = stored == null ? "" : value(stored, "pos");
|
||||
long unchangedSince = stored == null ? 0L : parseMillis(stored.get("at"));
|
||||
if (!position.equals(storedPosition) || unchangedSince <= 0L) {
|
||||
writeStallSnapshot(stallKey, position, now, staleWindowMinutes);
|
||||
return StallState.ADVANCING;
|
||||
}
|
||||
long unchangedMillis = now - unchangedSince;
|
||||
if (unchangedMillis >= Duration.ofMinutes(staleWindowMinutes).toMillis()) {
|
||||
return StallState.STALLED;
|
||||
}
|
||||
log.debug("[heartbeat-position] 位置未变化但在豁免窗口内 taskId={} unchangedMs={} position={}",
|
||||
taskId, unchangedMillis, position);
|
||||
return StallState.ADVANCING;
|
||||
}
|
||||
|
||||
/**
|
||||
* 便捷判定:客户端是否仍在推进(位置文本仍在变)。
|
||||
*
|
||||
* <p>各模块「无结果上报」判死线在封口前统一调它:返回 true 表示采集主线程还在干活
|
||||
* (分片只是攒着没回传),此时<b>不得</b>封口/结案——否则后续分片会被拒收、数据丢失。
|
||||
*/
|
||||
public boolean isClientAdvancing(Long taskId, long stallWindowMinutes) {
|
||||
return evaluateStall(taskId, stallWindowMinutes) == StallState.ADVANCING;
|
||||
}
|
||||
|
||||
private void writeStallSnapshot(String key, String position, long now, long staleWindowMinutes) {
|
||||
try {
|
||||
Map<String, String> values = new HashMap<>();
|
||||
values.put("pos", position);
|
||||
values.put("at", String.valueOf(now));
|
||||
stringRedisTemplate.opsForHash().putAll(key, values);
|
||||
stringRedisTemplate.expire(key, Duration.ofMinutes(Math.max(60L, staleWindowMinutes * 3L)));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[heartbeat-position] 写入停滞快照失败 key={} msg={}", key, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private long parseMillis(Object raw) {
|
||||
if (raw == null) {
|
||||
return 0L;
|
||||
}
|
||||
try {
|
||||
return Long.parseLong(String.valueOf(raw).trim());
|
||||
} catch (NumberFormatException ex) {
|
||||
return 0L;
|
||||
}
|
||||
}
|
||||
|
||||
private String value(Map<Object, Object> entries, String key) {
|
||||
Object raw = entries.get(key);
|
||||
return raw == null ? "" : String.valueOf(raw).trim();
|
||||
|
||||
Reference in New Issue
Block a user