fix(长任务判死): 封口前增加「客户端仍在推进」豁免,避免还在采集就被结案丢数据

服务端二次判死线(心跳正常但 N 分钟无结果分片上报)只看分片间隔,客户端攒批回传一慢
就把仍在采集的任务判成卡死、封口结案,后续分片全被拒(生产 28919:1300 行只出 395 行,
丢 905 行)。改为封口前用 TaskHeartbeatPositionService 的位置快照判定客户端是否仍在推进:
ADVANCING(位置在变)→ 本轮不封口;STALLED(长时间不变)→ 按原判据;UNKNOWN(老客户端
无位置信息)→ 保留原行为。接入 similar-asin / appearance-patent / brand / delete-brand /
collect-data 五条判死线。
This commit is contained in:
2026-09-23 10:52:45 +08:00
parent 8d73377b25
commit afa45d51c3
16 changed files with 348 additions and 6 deletions
@@ -49,6 +49,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
@@ -139,6 +140,7 @@ public class AppearancePatentTaskService {
private final AppearancePatentProperties properties;
private final TaskFileJobService taskFileJobService;
private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
private final TransientPayloadStorageService transientPayloadStorageService;
private final PlatformTransactionManager transactionManager;
private final DistributedJobLockService distributedJobLockService;
@@ -759,6 +761,14 @@ public class AppearancePatentTaskService {
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
continue;
}
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
// 此时封口会把后续分片全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
log.info("[appearance-patent] 客户端仍在推进,本轮不封口 taskId={} lastResultAt={} position={}",
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
continue;
}
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L);
if (lockHandle == null) {
continue;
@@ -36,6 +36,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
@@ -129,6 +130,7 @@ public class BrandTaskService {
private final StorageProperties storageProperties;
private final BrandProgressProperties brandProgressProperties;
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
private final BrandTaskStorageService brandTaskStorageService;
private final DistributedJobLockService distributedJobLockService;
private final TaskDistributedLockService taskDistributedLockService;
@@ -1581,6 +1583,13 @@ public class BrandTaskService {
if (lastResult.isAfter(cutoff)) {
continue;
}
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是结果攒着没回传。
// 此时判失败会把后续回传全部拒收、数据丢失。位置长时间不变或无位置信息(老客户端)才继续。
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
log.info("[brand-stale-check] no-upload skip because client still advancing taskId={} lastResultAt={} position={}",
task.getId(), lastResult, taskHeartbeatPositionService.describe(task.getId()));
continue;
}
log.warn("[brand-stale-check] no-upload failing taskId={} lastResultAt={} timeoutMinutes={}",
task.getId(), lastResult, minutes);
failStaleRunningTask(task.getId(),
@@ -60,6 +60,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
import com.nanri.aiimage.common.service.DistributedJobLockService;
@@ -151,6 +152,7 @@ public class CollectDataService {
private final TaskResultItemMapper taskResultItemMapper;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskFileJobService taskFileJobService;
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
private final TransientPayloadStorageService transientPayloadStorageService;
private final CollectDataExcelAssemblyService excelAssemblyService;
private final OssStorageService ossStorageService;
@@ -623,6 +625,14 @@ public class CollectDataService {
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
continue;
}
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
// 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
log.info("[collect-data] no-upload skip because client still advancing taskId={} lastResultAt={} position={}",
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
continue;
}
finalizeNoUploadStaleTask(task.getId(), lastResultAt, minutes);
}
}
@@ -791,6 +791,15 @@ public class DeleteBrandStaleTaskService {
continue;
}
String moduleType = task.getModuleType();
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
// 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
stats.skippedTaskCount++;
log.info("[stale-check] no-upload skip because client still advancing taskId={} moduleType={} lastResultAt={} position={}",
task.getId(), moduleType, lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
continue;
}
if (hasPendingAssembleJobs(task.getId(), moduleType)) {
stats.skippedTaskCount++;
log.info("[stale-check] no-upload skip pending-assemble-jobs taskId={} moduleType={}", task.getId(), moduleType);
@@ -72,6 +72,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
@@ -199,6 +200,7 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
private final TaskFileJobService taskFileJobService;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
private final TransientPayloadStorageService transientPayloadStorageService;
private final PlatformTransactionManager transactionManager;
private final DistributedJobLockService distributedJobLockService;
@@ -952,13 +954,23 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
continue;
}
// 「客户端仍在推进」豁免:位置文本还在变说明采集主线程在干活,只是分片攒着没回传
// (生产 28919:1300 行每 1.5~3 小时才回传一次)。此时封口会把后续分片全部拒掉、数据丢失。
// 位置无变化满 N 分钟(STALLED)才按原判据封口;无位置信息(老客户端)保持原行为。
TaskHeartbeatPositionService.StallState stallState =
taskHeartbeatPositionService.evaluateStall(task.getId(), minutes);
if (stallState == TaskHeartbeatPositionService.StallState.ADVANCING) {
log.info("[similar-asin] 客户端仍在推进,本轮不封口 taskId={} lastResultAt={} position={}",
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
continue;
}
TaskDistributedLockService.LockHandle taskLockHandle = ownershipSupport().acquireTaskLock(task.getId(), 0L);
if (taskLockHandle == null) {
continue;
}
try (taskLockHandle) {
log.warn("[similar-asin] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={}",
minutes, task.getId(), lastResultAt);
log.warn("[similar-asin] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={} stallState={}",
minutes, task.getId(), lastResultAt, stallState);
finalizeStaleTask(task.getId(),
"Python heartbeat alive but no result chunk uploaded for " + minutes + " minutes");
}
@@ -25,10 +25,25 @@ import java.util.Map;
public class TaskHeartbeatPositionService {
private static final String KEY_PREFIX = "task:heartbeat:position:";
private static final String STALL_KEY_PREFIX = "task:progress:stall:";
private static final Duration POSITION_TTL = Duration.ofHours(24);
private final StringRedisTemplate stringRedisTemplate;
/**
* 位置推进状态。
*
* <p>用于「封口/判死前的豁免」:{@link #ADVANCING} 表示客户端主线程仍在推进,
* 调用方不得据超时封口(否则后续回传被拒、数据丢失);{@link #STALLED} 才是
* 「心跳照发但主线程真的不动了」;{@link #UNKNOWN} 表示没有位置信息可用,
* 调用方保留原有判据——它<b>不是</b>判死证据,只是没有豁免依据。
*/
public enum StallState {
ADVANCING,
STALLED,
UNKNOWN
}
public void record(Long taskId, TaskHeartbeatRequest request) {
if (taskId == null || taskId <= 0 || request == null) {
return;
@@ -91,6 +106,88 @@ public class TaskHeartbeatPositionService {
return sb.length() == 0 ? null : sb.toString();
}
/**
* 判定客户端是否「仍在推进」,供封口/判死前的豁免使用。
*
* <p>把当前位置文本与上次快照比对:
* <ul>
* <li>文本变了 → {@link StallState#ADVANCING}(客户端主线程在干活,只是分片还没回传);</li>
* <li>文本相同且已持续 {@code staleWindowMinutes} → {@link StallState#STALLED}(真卡死);</li>
* <li>无位置信息(老客户端不传 progressText) → {@link StallState#UNKNOWN}。</li>
* </ul>
*
* <p>为什么需要它:既有二次判死线只看「多久没收到结果分片」(biz_task_scope_state.last_chunk_at),
* 客户端攒批回传间隔一旦超过阈值(生产 28919:1300 行每 1.5~3 小时才回传一次),
* 服务端就会把仍在采集的任务判成卡死、封口结案,后续分片全部被拒、数据丢失。
* 位置文本由客户端主线程维护:它在变就证明采集在推进,此时不得封口。
*
* @param staleWindowMinutes 判定停滞所需的最短「无变化」时长(分钟),&lt;=0 时返回 UNKNOWN
*/
public StallState evaluateStall(Long taskId, long staleWindowMinutes) {
if (taskId == null || taskId <= 0 || staleWindowMinutes <= 0) {
return StallState.UNKNOWN;
}
String position = describe(taskId);
if (position == null || position.isBlank()) {
return StallState.UNKNOWN;
}
String stallKey = STALL_KEY_PREFIX + taskId;
Map<Object, Object> stored;
try {
stored = stringRedisTemplate.opsForHash().entries(stallKey);
} catch (Exception ex) {
log.warn("[heartbeat-position] 读取停滞快照失败 taskId={} msg={}", taskId, ex.getMessage());
return StallState.UNKNOWN;
}
long now = Instant.now().toEpochMilli();
String storedPosition = stored == null ? "" : value(stored, "pos");
long unchangedSince = stored == null ? 0L : parseMillis(stored.get("at"));
if (!position.equals(storedPosition) || unchangedSince <= 0L) {
writeStallSnapshot(stallKey, position, now, staleWindowMinutes);
return StallState.ADVANCING;
}
long unchangedMillis = now - unchangedSince;
if (unchangedMillis >= Duration.ofMinutes(staleWindowMinutes).toMillis()) {
return StallState.STALLED;
}
log.debug("[heartbeat-position] 位置未变化但在豁免窗口内 taskId={} unchangedMs={} position={}",
taskId, unchangedMillis, position);
return StallState.ADVANCING;
}
/**
* 便捷判定:客户端是否仍在推进(位置文本仍在变)。
*
* <p>各模块「无结果上报」判死线在封口前统一调它:返回 true 表示采集主线程还在干活
* (分片只是攒着没回传),此时<b>不得</b>封口/结案——否则后续分片会被拒收、数据丢失。
*/
public boolean isClientAdvancing(Long taskId, long stallWindowMinutes) {
return evaluateStall(taskId, stallWindowMinutes) == StallState.ADVANCING;
}
private void writeStallSnapshot(String key, String position, long now, long staleWindowMinutes) {
try {
Map<String, String> values = new HashMap<>();
values.put("pos", position);
values.put("at", String.valueOf(now));
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, Duration.ofMinutes(Math.max(60L, staleWindowMinutes * 3L)));
} catch (Exception ex) {
log.warn("[heartbeat-position] 写入停滞快照失败 key={} msg={}", key, ex.getMessage());
}
}
private long parseMillis(Object raw) {
if (raw == null) {
return 0L;
}
try {
return Long.parseLong(String.valueOf(raw).trim());
} catch (NumberFormatException ex) {
return 0L;
}
}
private String value(Map<Object, Object> entries, String key) {
Object raw = entries.get(key);
return raw == null ? "" : String.valueOf(raw).trim();