From afa45d51c31491f705cca28847550e4a186334a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Wed, 23 Sep 2026 10:52:45 +0800 Subject: [PATCH] =?UTF-8?q?fix(=E9=95=BF=E4=BB=BB=E5=8A=A1=E5=88=A4?= =?UTF-8?q?=E6=AD=BB):=20=E5=B0=81=E5=8F=A3=E5=89=8D=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E3=80=8C=E5=AE=A2=E6=88=B7=E7=AB=AF=E4=BB=8D=E5=9C=A8=E6=8E=A8?= =?UTF-8?q?=E8=BF=9B=E3=80=8D=E8=B1=81=E5=85=8D=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E8=BF=98=E5=9C=A8=E9=87=87=E9=9B=86=E5=B0=B1=E8=A2=AB=E7=BB=93?= =?UTF-8?q?=E6=A1=88=E4=B8=A2=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 服务端二次判死线(心跳正常但 N 分钟无结果分片上报)只看分片间隔,客户端攒批回传一慢 就把仍在采集的任务判成卡死、封口结案,后续分片全被拒(生产 28919:1300 行只出 395 行, 丢 905 行)。改为封口前用 TaskHeartbeatPositionService 的位置快照判定客户端是否仍在推进: ADVANCING(位置在变)→ 本轮不封口;STALLED(长时间不变)→ 按原判据;UNKNOWN(老客户端 无位置信息)→ 保留原行为。接入 similar-asin / appearance-patent / brand / delete-brand / collect-data 五条判死线。 --- .../service/AppearancePatentTaskService.java | 10 ++ .../brand/service/BrandTaskService.java | 9 ++ .../service/CollectDataService.java | 10 ++ .../service/DeleteBrandStaleTaskService.java | 9 ++ .../service/SimilarAsinTaskService.java | 16 ++- .../service/TaskHeartbeatPositionService.java | 97 +++++++++++++++++ ...arancePatentTaskServiceDelegationTest.java | 2 + ...ancePatentTaskServiceHistoryBatchTest.java | 1 + .../service/BrandTaskAbortServiceTest.java | 4 +- .../service/CollectDataActivateGuardTest.java | 1 + .../CollectDataDeleteConsistencyTest.java | 4 +- .../service/CollectDataNoUploadStaleTest.java | 3 + .../CollectDataStorageCallCountTest.java | 4 +- .../service/SimilarAsinNoUploadStaleTest.java | 101 ++++++++++++++++++ .../RollbackSemanticsContractTest.java | 4 +- .../TaskHeartbeatPositionServiceTest.java | 79 ++++++++++++++ 16 files changed, 348 insertions(+), 6 deletions(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinNoUploadStaleTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index 267615db2..c604cd2ff 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -49,6 +49,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity; import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import lombok.RequiredArgsConstructor; @@ -139,6 +140,7 @@ public class AppearancePatentTaskService { private final AppearancePatentProperties properties; private final TaskFileJobService taskFileJobService; private final TaskProgressSnapshotService taskProgressSnapshotService; + private final TaskHeartbeatPositionService taskHeartbeatPositionService; private final TransientPayloadStorageService transientPayloadStorageService; private final PlatformTransactionManager transactionManager; private final DistributedJobLockService distributedJobLockService; @@ -759,6 +761,14 @@ public class AppearancePatentTaskService { if (lastResultAt == null || lastResultAt.isAfter(cutoff)) { continue; } + // 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。 + // 此时封口会把后续分片全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。 + // 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。 + if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { + log.info("[appearance-patent] 客户端仍在推进,本轮不封口 taskId={} lastResultAt={} position={}", + task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId())); + continue; + } TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L); if (lockHandle == null) { continue; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java index 92ccabc13..08062d151 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java @@ -36,6 +36,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; @@ -129,6 +130,7 @@ public class BrandTaskService { private final StorageProperties storageProperties; private final BrandProgressProperties brandProgressProperties; private final BrandTaskProgressCacheService brandTaskProgressCacheService; + private final TaskHeartbeatPositionService taskHeartbeatPositionService; private final BrandTaskStorageService brandTaskStorageService; private final DistributedJobLockService distributedJobLockService; private final TaskDistributedLockService taskDistributedLockService; @@ -1581,6 +1583,13 @@ public class BrandTaskService { if (lastResult.isAfter(cutoff)) { continue; } + // 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是结果攒着没回传。 + // 此时判失败会把后续回传全部拒收、数据丢失。位置长时间不变或无位置信息(老客户端)才继续。 + if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { + log.info("[brand-stale-check] no-upload skip because client still advancing taskId={} lastResultAt={} position={}", + task.getId(), lastResult, taskHeartbeatPositionService.describe(task.getId())); + continue; + } log.warn("[brand-stale-check] no-upload failing taskId={} lastResultAt={} timeoutMinutes={}", task.getId(), lastResult, minutes); failStaleRunningTask(task.getId(), diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java index 7bfe6cf98..6640c69d0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java @@ -60,6 +60,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity; import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import lombok.RequiredArgsConstructor; import com.nanri.aiimage.common.service.DistributedJobLockService; @@ -151,6 +152,7 @@ public class CollectDataService { private final TaskResultItemMapper taskResultItemMapper; private final TaskDistributedLockService taskDistributedLockService; private final TaskFileJobService taskFileJobService; + private final TaskHeartbeatPositionService taskHeartbeatPositionService; private final TransientPayloadStorageService transientPayloadStorageService; private final CollectDataExcelAssemblyService excelAssemblyService; private final OssStorageService ossStorageService; @@ -623,6 +625,14 @@ public class CollectDataService { if (lastResultAt == null || lastResultAt.isAfter(cutoff)) { continue; } + // 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。 + // 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。 + // 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。 + if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { + log.info("[collect-data] no-upload skip because client still advancing taskId={} lastResultAt={} position={}", + task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId())); + continue; + } finalizeNoUploadStaleTask(task.getId(), lastResultAt, minutes); } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java index 8890b9a52..fd0b880cd 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java @@ -791,6 +791,15 @@ public class DeleteBrandStaleTaskService { continue; } String moduleType = task.getModuleType(); + // 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。 + // 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。 + // 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。 + if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) { + stats.skippedTaskCount++; + log.info("[stale-check] no-upload skip because client still advancing taskId={} moduleType={} lastResultAt={} position={}", + task.getId(), moduleType, lastResultAt, taskHeartbeatPositionService.describe(task.getId())); + continue; + } if (hasPendingAssembleJobs(task.getId(), moduleType)) { stats.skippedTaskCount++; log.info("[stale-check] no-upload skip pending-assemble-jobs taskId={} moduleType={}", task.getId(), moduleType); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java index 7bf52ee32..a8b03d6a9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -72,6 +72,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity; import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService; @@ -199,6 +200,7 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost { private final TaskFileJobService taskFileJobService; private final TaskDistributedLockService taskDistributedLockService; private final TaskProgressSnapshotService taskProgressSnapshotService; + private final TaskHeartbeatPositionService taskHeartbeatPositionService; private final TransientPayloadStorageService transientPayloadStorageService; private final PlatformTransactionManager transactionManager; private final DistributedJobLockService distributedJobLockService; @@ -952,13 +954,23 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost { if (lastResultAt == null || lastResultAt.isAfter(cutoff)) { continue; } + // 「客户端仍在推进」豁免:位置文本还在变说明采集主线程在干活,只是分片攒着没回传 + // (生产 28919:1300 行每 1.5~3 小时才回传一次)。此时封口会把后续分片全部拒掉、数据丢失。 + // 位置无变化满 N 分钟(STALLED)才按原判据封口;无位置信息(老客户端)保持原行为。 + TaskHeartbeatPositionService.StallState stallState = + taskHeartbeatPositionService.evaluateStall(task.getId(), minutes); + if (stallState == TaskHeartbeatPositionService.StallState.ADVANCING) { + log.info("[similar-asin] 客户端仍在推进,本轮不封口 taskId={} lastResultAt={} position={}", + task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId())); + continue; + } TaskDistributedLockService.LockHandle taskLockHandle = ownershipSupport().acquireTaskLock(task.getId(), 0L); if (taskLockHandle == null) { continue; } try (taskLockHandle) { - log.warn("[similar-asin] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={}", - minutes, task.getId(), lastResultAt); + log.warn("[similar-asin] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={} stallState={}", + minutes, task.getId(), lastResultAt, stallState); finalizeStaleTask(task.getId(), "Python heartbeat alive but no result chunk uploaded for " + minutes + " minutes"); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java index 58fcc38ad..bcdee66f9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java @@ -25,10 +25,25 @@ import java.util.Map; public class TaskHeartbeatPositionService { private static final String KEY_PREFIX = "task:heartbeat:position:"; + private static final String STALL_KEY_PREFIX = "task:progress:stall:"; private static final Duration POSITION_TTL = Duration.ofHours(24); private final StringRedisTemplate stringRedisTemplate; + /** + * 位置推进状态。 + * + *

用于「封口/判死前的豁免」:{@link #ADVANCING} 表示客户端主线程仍在推进, + * 调用方不得据超时封口(否则后续回传被拒、数据丢失);{@link #STALLED} 才是 + * 「心跳照发但主线程真的不动了」;{@link #UNKNOWN} 表示没有位置信息可用, + * 调用方保留原有判据——它不是判死证据,只是没有豁免依据。 + */ + public enum StallState { + ADVANCING, + STALLED, + UNKNOWN + } + public void record(Long taskId, TaskHeartbeatRequest request) { if (taskId == null || taskId <= 0 || request == null) { return; @@ -91,6 +106,88 @@ public class TaskHeartbeatPositionService { return sb.length() == 0 ? null : sb.toString(); } + /** + * 判定客户端是否「仍在推进」,供封口/判死前的豁免使用。 + * + *

把当前位置文本与上次快照比对: + *

+ * + *

为什么需要它:既有二次判死线只看「多久没收到结果分片」(biz_task_scope_state.last_chunk_at), + * 客户端攒批回传间隔一旦超过阈值(生产 28919:1300 行每 1.5~3 小时才回传一次), + * 服务端就会把仍在采集的任务判成卡死、封口结案,后续分片全部被拒、数据丢失。 + * 位置文本由客户端主线程维护:它在变就证明采集在推进,此时不得封口。 + * + * @param staleWindowMinutes 判定停滞所需的最短「无变化」时长(分钟),<=0 时返回 UNKNOWN + */ + public StallState evaluateStall(Long taskId, long staleWindowMinutes) { + if (taskId == null || taskId <= 0 || staleWindowMinutes <= 0) { + return StallState.UNKNOWN; + } + String position = describe(taskId); + if (position == null || position.isBlank()) { + return StallState.UNKNOWN; + } + String stallKey = STALL_KEY_PREFIX + taskId; + Map stored; + try { + stored = stringRedisTemplate.opsForHash().entries(stallKey); + } catch (Exception ex) { + log.warn("[heartbeat-position] 读取停滞快照失败 taskId={} msg={}", taskId, ex.getMessage()); + return StallState.UNKNOWN; + } + long now = Instant.now().toEpochMilli(); + String storedPosition = stored == null ? "" : value(stored, "pos"); + long unchangedSince = stored == null ? 0L : parseMillis(stored.get("at")); + if (!position.equals(storedPosition) || unchangedSince <= 0L) { + writeStallSnapshot(stallKey, position, now, staleWindowMinutes); + return StallState.ADVANCING; + } + long unchangedMillis = now - unchangedSince; + if (unchangedMillis >= Duration.ofMinutes(staleWindowMinutes).toMillis()) { + return StallState.STALLED; + } + log.debug("[heartbeat-position] 位置未变化但在豁免窗口内 taskId={} unchangedMs={} position={}", + taskId, unchangedMillis, position); + return StallState.ADVANCING; + } + + /** + * 便捷判定:客户端是否仍在推进(位置文本仍在变)。 + * + *

各模块「无结果上报」判死线在封口前统一调它:返回 true 表示采集主线程还在干活 + * (分片只是攒着没回传),此时不得封口/结案——否则后续分片会被拒收、数据丢失。 + */ + public boolean isClientAdvancing(Long taskId, long stallWindowMinutes) { + return evaluateStall(taskId, stallWindowMinutes) == StallState.ADVANCING; + } + + private void writeStallSnapshot(String key, String position, long now, long staleWindowMinutes) { + try { + Map values = new HashMap<>(); + values.put("pos", position); + values.put("at", String.valueOf(now)); + stringRedisTemplate.opsForHash().putAll(key, values); + stringRedisTemplate.expire(key, Duration.ofMinutes(Math.max(60L, staleWindowMinutes * 3L))); + } catch (Exception ex) { + log.warn("[heartbeat-position] 写入停滞快照失败 key={} msg={}", key, ex.getMessage()); + } + } + + private long parseMillis(Object raw) { + if (raw == null) { + return 0L; + } + try { + return Long.parseLong(String.valueOf(raw).trim()); + } catch (NumberFormatException ex) { + return 0L; + } + } + private String value(Map entries, String key) { Object raw = entries.get(key); return raw == null ? "" : String.valueOf(raw).trim(); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskServiceDelegationTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskServiceDelegationTest.java index 97aac4157..7f8d27013 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskServiceDelegationTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskServiceDelegationTest.java @@ -104,6 +104,7 @@ class AppearancePatentTaskServiceDelegationTest { storageProperties, fileTaskMapper, fileResultMapper, taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService, properties, taskFileJobService, taskProgressSnapshotService, + mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class), transientPayloadStorageService, transactionManager, distributedJobLockService, taskDistributedLockService, instanceMetadata, mock(TaskProgressLightAssembler.class), @@ -117,6 +118,7 @@ class AppearancePatentTaskServiceDelegationTest { storageProperties, fileTaskMapper, fileResultMapper, taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService, properties, taskFileJobService, taskProgressSnapshotService, + mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class), transientPayloadStorageService, null, distributedJobLockService, taskDistributedLockService, instanceMetadata, mock(TaskProgressLightAssembler.class), diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskServiceHistoryBatchTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskServiceHistoryBatchTest.java index 543088ea4..e5d9f2155 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskServiceHistoryBatchTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskServiceHistoryBatchTest.java @@ -213,6 +213,7 @@ class AppearancePatentTaskServiceHistoryBatchTest { storageProperties, fileTaskMapper, fileResultMapper, taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService, properties, taskFileJobService, taskProgressSnapshotService, + mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class), transientPayloadStorageService, transactionManager, distributedJobLockService, taskDistributedLockService, instanceMetadata, mock(TaskProgressLightAssembler.class), diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskAbortServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskAbortServiceTest.java index 4533d3b72..3e218837f 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskAbortServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/brand/service/BrandTaskAbortServiceTest.java @@ -79,7 +79,9 @@ class BrandTaskAbortServiceTest { private BrandTaskService service() { return new BrandTaskService(brandCrawlTaskMapper, ossStorageService, storageProperties, - brandProgressProperties, brandTaskProgressCacheService, brandTaskStorageService, + brandProgressProperties, brandTaskProgressCacheService, + mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class), + brandTaskStorageService, distributedJobLockService, taskDistributedLockService, new ObjectMapper(), taskFileJobService, taskProgressSnapshotService); } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataActivateGuardTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataActivateGuardTest.java index a2b3d74a6..1909d9859 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataActivateGuardTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataActivateGuardTest.java @@ -136,6 +136,7 @@ class CollectDataActivateGuardTest { mock(InvalidAsinDataMapper.class), mock(TaskChunkMapper.class), mock(TaskScopeStateMapper.class), mock(TaskResultItemMapper.class), mock(TaskDistributedLockService.class), mock(TaskFileJobService.class), + mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class), mock(TransientPayloadStorageService.class), mock(CollectDataExcelAssemblyService.class), mock(OssStorageService.class), new ObjectMapper(), new TransactionTemplate(transactionManager), diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataDeleteConsistencyTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataDeleteConsistencyTest.java index 3966624d7..f820bc0d0 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataDeleteConsistencyTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataDeleteConsistencyTest.java @@ -104,7 +104,9 @@ class CollectDataDeleteConsistencyTest { mock(com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper.class), mock(com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper.class), taskChunkMapper, taskScopeStateMapper, taskResultItemMapper, - taskDistributedLockService, taskFileJobService, transientPayloadStorageService, + taskDistributedLockService, taskFileJobService, + mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class), + transientPayloadStorageService, mock(com.nanri.aiimage.modules.collectdata.service.CollectDataExcelAssemblyService.class), mock(com.nanri.aiimage.modules.file.service.oss.OssStorageService.class), objectMapper, txTemplate, diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataNoUploadStaleTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataNoUploadStaleTest.java index 601de0b9c..f59bc9dba 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataNoUploadStaleTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataNoUploadStaleTest.java @@ -8,6 +8,7 @@ import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -45,6 +46,8 @@ class CollectDataNoUploadStaleTest { @Mock private TaskScopeStateMapper taskScopeStateMapper; @Mock private TaskDistributedLockService taskDistributedLockService; @Mock private TaskFileJobService taskFileJobService; + /** 默认(Mockito 布尔默认值 false)= 不构成「客户端仍在推进」豁免,与原判据行为一致。 */ + @Mock private TaskHeartbeatPositionService taskHeartbeatPositionService; @Mock private TaskDistributedLockService.LockHandle lockHandle; @Spy diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataStorageCallCountTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataStorageCallCountTest.java index 378340582..507d8dbc3 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataStorageCallCountTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/collectdata/service/CollectDataStorageCallCountTest.java @@ -103,7 +103,9 @@ class CollectDataStorageCallCountTest { mock(com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper.class), mock(com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper.class), taskChunkMapper, taskScopeStateMapper, taskResultItemMapper, - taskDistributedLockService, taskFileJobService, transientPayloadStorageService, + taskDistributedLockService, taskFileJobService, + mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class), + transientPayloadStorageService, mock(com.nanri.aiimage.modules.collectdata.service.CollectDataExcelAssemblyService.class), mock(com.nanri.aiimage.modules.file.service.oss.OssStorageService.class), objectMapper, txTemplate, diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinNoUploadStaleTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinNoUploadStaleTest.java new file mode 100644 index 000000000..719aba18a --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinNoUploadStaleTest.java @@ -0,0 +1,101 @@ +package com.nanri.aiimage.modules.similarasin.service; + +import com.nanri.aiimage.config.SimilarAsinProperties; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 货源二次判死线的「客户端仍在推进」豁免。 + * + *

事故背景(2026-09-22 任务 28919):客户端每 1.5~3 小时才回传一次分片(1300 行 / 12 小时), + * 服务端按「180 分钟无结果分片上报」把仍在采集的任务封口结案,后面到的 22 个分片全被拒收, + * 905 行数据丢失。修复后:位置文本仍在变(客户端在推进)就不封口;位置长时间不变或无位置信息 + * 才走原判据。 + */ +@ExtendWith(MockitoExtension.class) +class SimilarAsinNoUploadStaleTest { + + private static final Long TASK_ID = 28919L; + + @Mock private FileTaskMapper fileTaskMapper; + @Mock private TaskScopeStateMapper taskScopeStateMapper; + @Mock private TaskDistributedLockService taskDistributedLockService; + @Mock private SimilarAsinTaskCacheService taskCacheService; + @Mock private SimilarAsinProperties properties; + @Mock private TaskHeartbeatPositionService taskHeartbeatPositionService; + + @InjectMocks + private SimilarAsinTaskService service; + + @BeforeEach + void setUp() { + when(properties.getNoResultUploadTimeoutMinutes()).thenReturn(180); + when(properties.getStaleTimeoutMinutes()).thenReturn(30); + } + + /** 心跳新鲜(30 分钟内有心跳)+ 分片已 200 分钟没上传 = 命中二次判死线的候选条件。 */ + private void givenStaleChunkUpload() { + FileTaskEntity task = new FileTaskEntity(); + task.setId(TASK_ID); + task.setModuleType("SIMILAR_ASIN"); + task.setStatus("RUNNING"); + task.setCreatedAt(LocalDateTime.now().minusDays(1)); + task.setUpdatedAt(LocalDateTime.now().minusMinutes(1)); + when(fileTaskMapper.selectList(any())).thenReturn(List.of(task)); + when(taskCacheService.getTaskHeartbeatMillis(TASK_ID)).thenReturn(System.currentTimeMillis()); + when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any())) + .thenReturn(List.of(new TaskScopeLastChunkDto(TASK_ID, LocalDateTime.now().minusMinutes(200)))); + } + + private void invokeNoUploadCheck() { + ReflectionTestUtils.invokeMethod(service, "finalizeNoUploadStaleTasks"); + } + + @Test + void 客户端仍在推进时不封口() { + givenStaleChunkUpload(); + when(taskHeartbeatPositionService.evaluateStall(TASK_ID, 180L)) + .thenReturn(TaskHeartbeatPositionService.StallState.ADVANCING); + + invokeNoUploadCheck(); + + // 豁免生效:连任务锁都不去抢,更不会封口结案(也就不会拒收后续分片) + verify(taskDistributedLockService, never()).acquire(anyString(), anyLong(), any(Duration.class), anyLong()); + } + + @Test + void 客户端停滞时保留原封口行为() { + givenStaleChunkUpload(); + when(taskHeartbeatPositionService.evaluateStall(TASK_ID, 180L)) + .thenReturn(TaskHeartbeatPositionService.StallState.STALLED); + when(taskDistributedLockService.acquire(anyString(), anyLong(), any(Duration.class), anyLong())) + .thenReturn(null); + + invokeNoUploadCheck(); + + // 位置长时间不变(真卡死)仍按原判据走:进入锁内封口流程 + verify(taskDistributedLockService).acquire(anyString(), anyLong(), any(Duration.class), anyLong()); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/RollbackSemanticsContractTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/RollbackSemanticsContractTest.java index c3b668f38..18aa013e8 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/RollbackSemanticsContractTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/contract/RollbackSemanticsContractTest.java @@ -306,7 +306,9 @@ class RollbackSemanticsContractTest { storageProperties, fileTaskMapper, fileResultMapper, taskScopeStateMapper, taskChunkMapper, objectMapper, mock(AppearancePatentLlmClient.class), mock(AppearancePatentTaskCacheService.class), mock(com.nanri.aiimage.config.AppearancePatentProperties.class), - taskFileJobService, taskProgressSnapshotService, transientPayloadStorageService, transactionManager, + taskFileJobService, taskProgressSnapshotService, + mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class), + transientPayloadStorageService, transactionManager, distributedJobLockService, taskDistributedLockService, instanceMetadata, mock(TaskProgressLightAssembler.class), mock(UserApiSecretService.class)); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionServiceTest.java index 58bfd9774..837b78723 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionServiceTest.java @@ -13,7 +13,9 @@ import java.time.Duration; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; @@ -111,4 +113,81 @@ class TaskHeartbeatPositionServiceTest { assertNull(service().describe(123L)); } + + // ===== 停滞判定(封口前的豁免依据)===== + + @Test + void stallUnknownWhenNoPosition() { + when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); + when(hashOperations.entries("task:heartbeat:position:123")).thenReturn(Map.of()); + + assertEquals(TaskHeartbeatPositionService.StallState.UNKNOWN, service().evaluateStall(123L, 180L)); + } + + @Test + void stallUnknownOnRedisError() { + when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); + when(hashOperations.entries("task:heartbeat:position:123")) + .thenThrow(new RuntimeException("redis down")); + + assertEquals(TaskHeartbeatPositionService.StallState.UNKNOWN, service().evaluateStall(123L, 180L)); + } + + @Test + void stallAdvancingWhenPositionChangedAndSnapshotRefreshed() { + when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); + when(hashOperations.entries("task:heartbeat:position:123")) + .thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)")); + when(hashOperations.entries("task:progress:stall:123")) + .thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(400/1300)", "at", String.valueOf(Long.MAX_VALUE))); + + assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING, service().evaluateStall(123L, 180L)); + verify(hashOperations).putAll(eq("task:progress:stall:123"), any()); + } + + @Test + void stallAdvancingWhenPositionUnchangedWithinWindow() { + // 位置没变但还没到停滞窗口:仍按「在推进」处理,不得封口 + when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); + when(hashOperations.entries("task:heartbeat:position:123")) + .thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)")); + when(hashOperations.entries("task:progress:stall:123")) + .thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(401/1300)", + "at", String.valueOf(System.currentTimeMillis() - Duration.ofMinutes(5).toMillis()))); + + assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING, service().evaluateStall(123L, 180L)); + } + + @Test + void stallStalledWhenPositionUnchangedBeyondWindow() { + // 位置长时间不动才是真卡死,允许调用方按原判据封口 + when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); + when(hashOperations.entries("task:heartbeat:position:123")) + .thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)")); + when(hashOperations.entries("task:progress:stall:123")) + .thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(401/1300)", + "at", String.valueOf(System.currentTimeMillis() - Duration.ofMinutes(181).toMillis()))); + + assertEquals(TaskHeartbeatPositionService.StallState.STALLED, service().evaluateStall(123L, 180L)); + } + + @Test + void clientAdvancingOnlyForAdvancingState() { + when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); + // 停滞(位置长时间未变)不构成豁免 + when(hashOperations.entries("task:heartbeat:position:123")) + .thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)")); + when(hashOperations.entries("task:progress:stall:123")) + .thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(401/1300)", + "at", String.valueOf(System.currentTimeMillis() - Duration.ofMinutes(181).toMillis()))); + assertFalse(service().isClientAdvancing(123L, 180L)); + + // 无位置信息(老客户端)同样不构成豁免,调用方保留原判据 + assertFalse(service().isClientAdvancing(null, 180L)); + + // 位置在变才是豁免 + when(hashOperations.entries("task:progress:stall:123")) + .thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(1/1300)", "at", String.valueOf(Long.MAX_VALUE))); + assertTrue(service().isClientAdvancing(123L, 180L)); + } }