From afa45d51c31491f705cca28847550e4a186334a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com>
Date: Wed, 23 Sep 2026 10:52:45 +0800
Subject: [PATCH] =?UTF-8?q?fix(=E9=95=BF=E4=BB=BB=E5=8A=A1=E5=88=A4?=
=?UTF-8?q?=E6=AD=BB):=20=E5=B0=81=E5=8F=A3=E5=89=8D=E5=A2=9E=E5=8A=A0?=
=?UTF-8?q?=E3=80=8C=E5=AE=A2=E6=88=B7=E7=AB=AF=E4=BB=8D=E5=9C=A8=E6=8E=A8?=
=?UTF-8?q?=E8=BF=9B=E3=80=8D=E8=B1=81=E5=85=8D=EF=BC=8C=E9=81=BF=E5=85=8D?=
=?UTF-8?q?=E8=BF=98=E5=9C=A8=E9=87=87=E9=9B=86=E5=B0=B1=E8=A2=AB=E7=BB=93?=
=?UTF-8?q?=E6=A1=88=E4=B8=A2=E6=95=B0=E6=8D=AE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
服务端二次判死线(心跳正常但 N 分钟无结果分片上报)只看分片间隔,客户端攒批回传一慢
就把仍在采集的任务判成卡死、封口结案,后续分片全被拒(生产 28919:1300 行只出 395 行,
丢 905 行)。改为封口前用 TaskHeartbeatPositionService 的位置快照判定客户端是否仍在推进:
ADVANCING(位置在变)→ 本轮不封口;STALLED(长时间不变)→ 按原判据;UNKNOWN(老客户端
无位置信息)→ 保留原行为。接入 similar-asin / appearance-patent / brand / delete-brand /
collect-data 五条判死线。
---
.../service/AppearancePatentTaskService.java | 10 ++
.../brand/service/BrandTaskService.java | 9 ++
.../service/CollectDataService.java | 10 ++
.../service/DeleteBrandStaleTaskService.java | 9 ++
.../service/SimilarAsinTaskService.java | 16 ++-
.../service/TaskHeartbeatPositionService.java | 97 +++++++++++++++++
...arancePatentTaskServiceDelegationTest.java | 2 +
...ancePatentTaskServiceHistoryBatchTest.java | 1 +
.../service/BrandTaskAbortServiceTest.java | 4 +-
.../service/CollectDataActivateGuardTest.java | 1 +
.../CollectDataDeleteConsistencyTest.java | 4 +-
.../service/CollectDataNoUploadStaleTest.java | 3 +
.../CollectDataStorageCallCountTest.java | 4 +-
.../service/SimilarAsinNoUploadStaleTest.java | 101 ++++++++++++++++++
.../RollbackSemanticsContractTest.java | 4 +-
.../TaskHeartbeatPositionServiceTest.java | 79 ++++++++++++++
16 files changed, 348 insertions(+), 6 deletions(-)
create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinNoUploadStaleTest.java
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java
index 267615db2..c604cd2ff 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java
@@ -49,6 +49,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
+import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
@@ -139,6 +140,7 @@ public class AppearancePatentTaskService {
private final AppearancePatentProperties properties;
private final TaskFileJobService taskFileJobService;
private final TaskProgressSnapshotService taskProgressSnapshotService;
+ private final TaskHeartbeatPositionService taskHeartbeatPositionService;
private final TransientPayloadStorageService transientPayloadStorageService;
private final PlatformTransactionManager transactionManager;
private final DistributedJobLockService distributedJobLockService;
@@ -759,6 +761,14 @@ public class AppearancePatentTaskService {
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
continue;
}
+ // 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
+ // 此时封口会把后续分片全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
+ // 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
+ if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
+ log.info("[appearance-patent] 客户端仍在推进,本轮不封口 taskId={} lastResultAt={} position={}",
+ task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
+ continue;
+ }
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L);
if (lockHandle == null) {
continue;
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java
index 92ccabc13..08062d151 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java
@@ -36,6 +36,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
+import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
@@ -129,6 +130,7 @@ public class BrandTaskService {
private final StorageProperties storageProperties;
private final BrandProgressProperties brandProgressProperties;
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
+ private final TaskHeartbeatPositionService taskHeartbeatPositionService;
private final BrandTaskStorageService brandTaskStorageService;
private final DistributedJobLockService distributedJobLockService;
private final TaskDistributedLockService taskDistributedLockService;
@@ -1581,6 +1583,13 @@ public class BrandTaskService {
if (lastResult.isAfter(cutoff)) {
continue;
}
+ // 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是结果攒着没回传。
+ // 此时判失败会把后续回传全部拒收、数据丢失。位置长时间不变或无位置信息(老客户端)才继续。
+ if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
+ log.info("[brand-stale-check] no-upload skip because client still advancing taskId={} lastResultAt={} position={}",
+ task.getId(), lastResult, taskHeartbeatPositionService.describe(task.getId()));
+ continue;
+ }
log.warn("[brand-stale-check] no-upload failing taskId={} lastResultAt={} timeoutMinutes={}",
task.getId(), lastResult, minutes);
failStaleRunningTask(task.getId(),
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java
index 7bfe6cf98..6640c69d0 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java
@@ -60,6 +60,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
+import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
import com.nanri.aiimage.common.service.DistributedJobLockService;
@@ -151,6 +152,7 @@ public class CollectDataService {
private final TaskResultItemMapper taskResultItemMapper;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskFileJobService taskFileJobService;
+ private final TaskHeartbeatPositionService taskHeartbeatPositionService;
private final TransientPayloadStorageService transientPayloadStorageService;
private final CollectDataExcelAssemblyService excelAssemblyService;
private final OssStorageService ossStorageService;
@@ -623,6 +625,14 @@ public class CollectDataService {
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
continue;
}
+ // 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
+ // 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
+ // 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
+ if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
+ log.info("[collect-data] no-upload skip because client still advancing taskId={} lastResultAt={} position={}",
+ task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
+ continue;
+ }
finalizeNoUploadStaleTask(task.getId(), lastResultAt, minutes);
}
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java
index 8890b9a52..fd0b880cd 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java
@@ -791,6 +791,15 @@ public class DeleteBrandStaleTaskService {
continue;
}
String moduleType = task.getModuleType();
+ // 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
+ // 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
+ // 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
+ if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
+ stats.skippedTaskCount++;
+ log.info("[stale-check] no-upload skip because client still advancing taskId={} moduleType={} lastResultAt={} position={}",
+ task.getId(), moduleType, lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
+ continue;
+ }
if (hasPendingAssembleJobs(task.getId(), moduleType)) {
stats.skippedTaskCount++;
log.info("[stale-check] no-upload skip pending-assemble-jobs taskId={} moduleType={}", task.getId(), moduleType);
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
index 7bf52ee32..a8b03d6a9 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
@@ -72,6 +72,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
+import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
@@ -199,6 +200,7 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
private final TaskFileJobService taskFileJobService;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskProgressSnapshotService taskProgressSnapshotService;
+ private final TaskHeartbeatPositionService taskHeartbeatPositionService;
private final TransientPayloadStorageService transientPayloadStorageService;
private final PlatformTransactionManager transactionManager;
private final DistributedJobLockService distributedJobLockService;
@@ -952,13 +954,23 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
continue;
}
+ // 「客户端仍在推进」豁免:位置文本还在变说明采集主线程在干活,只是分片攒着没回传
+ // (生产 28919:1300 行每 1.5~3 小时才回传一次)。此时封口会把后续分片全部拒掉、数据丢失。
+ // 位置无变化满 N 分钟(STALLED)才按原判据封口;无位置信息(老客户端)保持原行为。
+ TaskHeartbeatPositionService.StallState stallState =
+ taskHeartbeatPositionService.evaluateStall(task.getId(), minutes);
+ if (stallState == TaskHeartbeatPositionService.StallState.ADVANCING) {
+ log.info("[similar-asin] 客户端仍在推进,本轮不封口 taskId={} lastResultAt={} position={}",
+ task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
+ continue;
+ }
TaskDistributedLockService.LockHandle taskLockHandle = ownershipSupport().acquireTaskLock(task.getId(), 0L);
if (taskLockHandle == null) {
continue;
}
try (taskLockHandle) {
- log.warn("[similar-asin] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={}",
- minutes, task.getId(), lastResultAt);
+ log.warn("[similar-asin] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={} stallState={}",
+ minutes, task.getId(), lastResultAt, stallState);
finalizeStaleTask(task.getId(),
"Python heartbeat alive but no result chunk uploaded for " + minutes + " minutes");
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java
index 58fcc38ad..bcdee66f9 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java
@@ -25,10 +25,25 @@ import java.util.Map;
public class TaskHeartbeatPositionService {
private static final String KEY_PREFIX = "task:heartbeat:position:";
+ private static final String STALL_KEY_PREFIX = "task:progress:stall:";
private static final Duration POSITION_TTL = Duration.ofHours(24);
private final StringRedisTemplate stringRedisTemplate;
+ /**
+ * 位置推进状态。
+ *
+ *
用于「封口/判死前的豁免」:{@link #ADVANCING} 表示客户端主线程仍在推进,
+ * 调用方不得据超时封口(否则后续回传被拒、数据丢失);{@link #STALLED} 才是
+ * 「心跳照发但主线程真的不动了」;{@link #UNKNOWN} 表示没有位置信息可用,
+ * 调用方保留原有判据——它不是判死证据,只是没有豁免依据。
+ */
+ public enum StallState {
+ ADVANCING,
+ STALLED,
+ UNKNOWN
+ }
+
public void record(Long taskId, TaskHeartbeatRequest request) {
if (taskId == null || taskId <= 0 || request == null) {
return;
@@ -91,6 +106,88 @@ public class TaskHeartbeatPositionService {
return sb.length() == 0 ? null : sb.toString();
}
+ /**
+ * 判定客户端是否「仍在推进」,供封口/判死前的豁免使用。
+ *
+ *
把当前位置文本与上次快照比对:
+ *
+ * - 文本变了 → {@link StallState#ADVANCING}(客户端主线程在干活,只是分片还没回传);
+ * - 文本相同且已持续 {@code staleWindowMinutes} → {@link StallState#STALLED}(真卡死);
+ * - 无位置信息(老客户端不传 progressText) → {@link StallState#UNKNOWN}。
+ *
+ *
+ * 为什么需要它:既有二次判死线只看「多久没收到结果分片」(biz_task_scope_state.last_chunk_at),
+ * 客户端攒批回传间隔一旦超过阈值(生产 28919:1300 行每 1.5~3 小时才回传一次),
+ * 服务端就会把仍在采集的任务判成卡死、封口结案,后续分片全部被拒、数据丢失。
+ * 位置文本由客户端主线程维护:它在变就证明采集在推进,此时不得封口。
+ *
+ * @param staleWindowMinutes 判定停滞所需的最短「无变化」时长(分钟),<=0 时返回 UNKNOWN
+ */
+ public StallState evaluateStall(Long taskId, long staleWindowMinutes) {
+ if (taskId == null || taskId <= 0 || staleWindowMinutes <= 0) {
+ return StallState.UNKNOWN;
+ }
+ String position = describe(taskId);
+ if (position == null || position.isBlank()) {
+ return StallState.UNKNOWN;
+ }
+ String stallKey = STALL_KEY_PREFIX + taskId;
+ Map