fix(长任务判死): 封口前增加「客户端仍在推进」豁免,避免还在采集就被结案丢数据
服务端二次判死线(心跳正常但 N 分钟无结果分片上报)只看分片间隔,客户端攒批回传一慢 就把仍在采集的任务判成卡死、封口结案,后续分片全被拒(生产 28919:1300 行只出 395 行, 丢 905 行)。改为封口前用 TaskHeartbeatPositionService 的位置快照判定客户端是否仍在推进: ADVANCING(位置在变)→ 本轮不封口;STALLED(长时间不变)→ 按原判据;UNKNOWN(老客户端 无位置信息)→ 保留原行为。接入 similar-asin / appearance-patent / brand / delete-brand / collect-data 五条判死线。
This commit is contained in:
+10
@@ -49,6 +49,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@@ -139,6 +140,7 @@ public class AppearancePatentTaskService {
|
||||
private final AppearancePatentProperties properties;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||
private final PlatformTransactionManager transactionManager;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
@@ -759,6 +761,14 @@ public class AppearancePatentTaskService {
|
||||
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
|
||||
// 此时封口会把后续分片全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
|
||||
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[appearance-patent] 客户端仍在推进,本轮不封口 taskId={} lastResultAt={} position={}",
|
||||
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
|
||||
continue;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L);
|
||||
if (lockHandle == null) {
|
||||
continue;
|
||||
|
||||
+9
@@ -36,6 +36,7 @@ import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@@ -129,6 +130,7 @@ public class BrandTaskService {
|
||||
private final StorageProperties storageProperties;
|
||||
private final BrandProgressProperties brandProgressProperties;
|
||||
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
private final BrandTaskStorageService brandTaskStorageService;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final TaskDistributedLockService taskDistributedLockService;
|
||||
@@ -1581,6 +1583,13 @@ public class BrandTaskService {
|
||||
if (lastResult.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是结果攒着没回传。
|
||||
// 此时判失败会把后续回传全部拒收、数据丢失。位置长时间不变或无位置信息(老客户端)才继续。
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[brand-stale-check] no-upload skip because client still advancing taskId={} lastResultAt={} position={}",
|
||||
task.getId(), lastResult, taskHeartbeatPositionService.describe(task.getId()));
|
||||
continue;
|
||||
}
|
||||
log.warn("[brand-stale-check] no-upload failing taskId={} lastResultAt={} timeoutMinutes={}",
|
||||
task.getId(), lastResult, minutes);
|
||||
failStaleRunningTask(task.getId(),
|
||||
|
||||
+10
@@ -60,6 +60,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
@@ -151,6 +152,7 @@ public class CollectDataService {
|
||||
private final TaskResultItemMapper taskResultItemMapper;
|
||||
private final TaskDistributedLockService taskDistributedLockService;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||
private final CollectDataExcelAssemblyService excelAssemblyService;
|
||||
private final OssStorageService ossStorageService;
|
||||
@@ -623,6 +625,14 @@ public class CollectDataService {
|
||||
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
|
||||
// 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
|
||||
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
log.info("[collect-data] no-upload skip because client still advancing taskId={} lastResultAt={} position={}",
|
||||
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
|
||||
continue;
|
||||
}
|
||||
finalizeNoUploadStaleTask(task.getId(), lastResultAt, minutes);
|
||||
}
|
||||
}
|
||||
|
||||
+9
@@ -791,6 +791,15 @@ public class DeleteBrandStaleTaskService {
|
||||
continue;
|
||||
}
|
||||
String moduleType = task.getModuleType();
|
||||
// 「客户端仍在推进」豁免:心跳位置文本还在变,说明采集主线程在干活,只是分片攒着没回传。
|
||||
// 此时判死会把后续回传全部拒收、数据丢失(similar-asin 生产 28919:905 行丢失)。
|
||||
// 位置长时间不变(真卡死)或无位置信息(老客户端)才继续走原判据。
|
||||
if (taskHeartbeatPositionService.isClientAdvancing(task.getId(), minutes)) {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[stale-check] no-upload skip because client still advancing taskId={} moduleType={} lastResultAt={} position={}",
|
||||
task.getId(), moduleType, lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
|
||||
continue;
|
||||
}
|
||||
if (hasPendingAssembleJobs(task.getId(), moduleType)) {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[stale-check] no-upload skip pending-assemble-jobs taskId={} moduleType={}", task.getId(), moduleType);
|
||||
|
||||
+14
-2
@@ -72,6 +72,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.usersecret.service.UserApiSecretService;
|
||||
@@ -199,6 +200,7 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final TaskDistributedLockService taskDistributedLockService;
|
||||
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||
private final PlatformTransactionManager transactionManager;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
@@ -952,13 +954,23 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
if (lastResultAt == null || lastResultAt.isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
// 「客户端仍在推进」豁免:位置文本还在变说明采集主线程在干活,只是分片攒着没回传
|
||||
// (生产 28919:1300 行每 1.5~3 小时才回传一次)。此时封口会把后续分片全部拒掉、数据丢失。
|
||||
// 位置无变化满 N 分钟(STALLED)才按原判据封口;无位置信息(老客户端)保持原行为。
|
||||
TaskHeartbeatPositionService.StallState stallState =
|
||||
taskHeartbeatPositionService.evaluateStall(task.getId(), minutes);
|
||||
if (stallState == TaskHeartbeatPositionService.StallState.ADVANCING) {
|
||||
log.info("[similar-asin] 客户端仍在推进,本轮不封口 taskId={} lastResultAt={} position={}",
|
||||
task.getId(), lastResultAt, taskHeartbeatPositionService.describe(task.getId()));
|
||||
continue;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle taskLockHandle = ownershipSupport().acquireTaskLock(task.getId(), 0L);
|
||||
if (taskLockHandle == null) {
|
||||
continue;
|
||||
}
|
||||
try (taskLockHandle) {
|
||||
log.warn("[similar-asin] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={}",
|
||||
minutes, task.getId(), lastResultAt);
|
||||
log.warn("[similar-asin] 心跳正常但 {} 分钟无结果分片上报,按卡死收尾 taskId={} lastResultAt={} stallState={}",
|
||||
minutes, task.getId(), lastResultAt, stallState);
|
||||
finalizeStaleTask(task.getId(),
|
||||
"Python heartbeat alive but no result chunk uploaded for " + minutes + " minutes");
|
||||
}
|
||||
|
||||
+97
@@ -25,10 +25,25 @@ import java.util.Map;
|
||||
public class TaskHeartbeatPositionService {
|
||||
|
||||
private static final String KEY_PREFIX = "task:heartbeat:position:";
|
||||
private static final String STALL_KEY_PREFIX = "task:progress:stall:";
|
||||
private static final Duration POSITION_TTL = Duration.ofHours(24);
|
||||
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
|
||||
/**
|
||||
* 位置推进状态。
|
||||
*
|
||||
* <p>用于「封口/判死前的豁免」:{@link #ADVANCING} 表示客户端主线程仍在推进,
|
||||
* 调用方不得据超时封口(否则后续回传被拒、数据丢失);{@link #STALLED} 才是
|
||||
* 「心跳照发但主线程真的不动了」;{@link #UNKNOWN} 表示没有位置信息可用,
|
||||
* 调用方保留原有判据——它<b>不是</b>判死证据,只是没有豁免依据。
|
||||
*/
|
||||
public enum StallState {
|
||||
ADVANCING,
|
||||
STALLED,
|
||||
UNKNOWN
|
||||
}
|
||||
|
||||
public void record(Long taskId, TaskHeartbeatRequest request) {
|
||||
if (taskId == null || taskId <= 0 || request == null) {
|
||||
return;
|
||||
@@ -91,6 +106,88 @@ public class TaskHeartbeatPositionService {
|
||||
return sb.length() == 0 ? null : sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 判定客户端是否「仍在推进」,供封口/判死前的豁免使用。
|
||||
*
|
||||
* <p>把当前位置文本与上次快照比对:
|
||||
* <ul>
|
||||
* <li>文本变了 → {@link StallState#ADVANCING}(客户端主线程在干活,只是分片还没回传);</li>
|
||||
* <li>文本相同且已持续 {@code staleWindowMinutes} → {@link StallState#STALLED}(真卡死);</li>
|
||||
* <li>无位置信息(老客户端不传 progressText) → {@link StallState#UNKNOWN}。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>为什么需要它:既有二次判死线只看「多久没收到结果分片」(biz_task_scope_state.last_chunk_at),
|
||||
* 客户端攒批回传间隔一旦超过阈值(生产 28919:1300 行每 1.5~3 小时才回传一次),
|
||||
* 服务端就会把仍在采集的任务判成卡死、封口结案,后续分片全部被拒、数据丢失。
|
||||
* 位置文本由客户端主线程维护:它在变就证明采集在推进,此时不得封口。
|
||||
*
|
||||
* @param staleWindowMinutes 判定停滞所需的最短「无变化」时长(分钟),<=0 时返回 UNKNOWN
|
||||
*/
|
||||
public StallState evaluateStall(Long taskId, long staleWindowMinutes) {
|
||||
if (taskId == null || taskId <= 0 || staleWindowMinutes <= 0) {
|
||||
return StallState.UNKNOWN;
|
||||
}
|
||||
String position = describe(taskId);
|
||||
if (position == null || position.isBlank()) {
|
||||
return StallState.UNKNOWN;
|
||||
}
|
||||
String stallKey = STALL_KEY_PREFIX + taskId;
|
||||
Map<Object, Object> stored;
|
||||
try {
|
||||
stored = stringRedisTemplate.opsForHash().entries(stallKey);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[heartbeat-position] 读取停滞快照失败 taskId={} msg={}", taskId, ex.getMessage());
|
||||
return StallState.UNKNOWN;
|
||||
}
|
||||
long now = Instant.now().toEpochMilli();
|
||||
String storedPosition = stored == null ? "" : value(stored, "pos");
|
||||
long unchangedSince = stored == null ? 0L : parseMillis(stored.get("at"));
|
||||
if (!position.equals(storedPosition) || unchangedSince <= 0L) {
|
||||
writeStallSnapshot(stallKey, position, now, staleWindowMinutes);
|
||||
return StallState.ADVANCING;
|
||||
}
|
||||
long unchangedMillis = now - unchangedSince;
|
||||
if (unchangedMillis >= Duration.ofMinutes(staleWindowMinutes).toMillis()) {
|
||||
return StallState.STALLED;
|
||||
}
|
||||
log.debug("[heartbeat-position] 位置未变化但在豁免窗口内 taskId={} unchangedMs={} position={}",
|
||||
taskId, unchangedMillis, position);
|
||||
return StallState.ADVANCING;
|
||||
}
|
||||
|
||||
/**
|
||||
* 便捷判定:客户端是否仍在推进(位置文本仍在变)。
|
||||
*
|
||||
* <p>各模块「无结果上报」判死线在封口前统一调它:返回 true 表示采集主线程还在干活
|
||||
* (分片只是攒着没回传),此时<b>不得</b>封口/结案——否则后续分片会被拒收、数据丢失。
|
||||
*/
|
||||
public boolean isClientAdvancing(Long taskId, long stallWindowMinutes) {
|
||||
return evaluateStall(taskId, stallWindowMinutes) == StallState.ADVANCING;
|
||||
}
|
||||
|
||||
private void writeStallSnapshot(String key, String position, long now, long staleWindowMinutes) {
|
||||
try {
|
||||
Map<String, String> values = new HashMap<>();
|
||||
values.put("pos", position);
|
||||
values.put("at", String.valueOf(now));
|
||||
stringRedisTemplate.opsForHash().putAll(key, values);
|
||||
stringRedisTemplate.expire(key, Duration.ofMinutes(Math.max(60L, staleWindowMinutes * 3L)));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[heartbeat-position] 写入停滞快照失败 key={} msg={}", key, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private long parseMillis(Object raw) {
|
||||
if (raw == null) {
|
||||
return 0L;
|
||||
}
|
||||
try {
|
||||
return Long.parseLong(String.valueOf(raw).trim());
|
||||
} catch (NumberFormatException ex) {
|
||||
return 0L;
|
||||
}
|
||||
}
|
||||
|
||||
private String value(Map<Object, Object> entries, String key) {
|
||||
Object raw = entries.get(key);
|
||||
return raw == null ? "" : String.valueOf(raw).trim();
|
||||
|
||||
+2
@@ -104,6 +104,7 @@ class AppearancePatentTaskServiceDelegationTest {
|
||||
storageProperties, fileTaskMapper, fileResultMapper,
|
||||
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
|
||||
properties, taskFileJobService, taskProgressSnapshotService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
transientPayloadStorageService, transactionManager, distributedJobLockService,
|
||||
taskDistributedLockService, instanceMetadata,
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
@@ -117,6 +118,7 @@ class AppearancePatentTaskServiceDelegationTest {
|
||||
storageProperties, fileTaskMapper, fileResultMapper,
|
||||
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
|
||||
properties, taskFileJobService, taskProgressSnapshotService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
transientPayloadStorageService, null, distributedJobLockService,
|
||||
taskDistributedLockService, instanceMetadata,
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
|
||||
+1
@@ -213,6 +213,7 @@ class AppearancePatentTaskServiceHistoryBatchTest {
|
||||
storageProperties, fileTaskMapper, fileResultMapper,
|
||||
taskScopeStateMapper, taskChunkMapper, new ObjectMapper(), llmClient, taskCacheService,
|
||||
properties, taskFileJobService, taskProgressSnapshotService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
transientPayloadStorageService, transactionManager, distributedJobLockService,
|
||||
taskDistributedLockService, instanceMetadata,
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
|
||||
+3
-1
@@ -79,7 +79,9 @@ class BrandTaskAbortServiceTest {
|
||||
|
||||
private BrandTaskService service() {
|
||||
return new BrandTaskService(brandCrawlTaskMapper, ossStorageService, storageProperties,
|
||||
brandProgressProperties, brandTaskProgressCacheService, brandTaskStorageService,
|
||||
brandProgressProperties, brandTaskProgressCacheService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
brandTaskStorageService,
|
||||
distributedJobLockService, taskDistributedLockService, new ObjectMapper(),
|
||||
taskFileJobService, taskProgressSnapshotService);
|
||||
}
|
||||
|
||||
+1
@@ -136,6 +136,7 @@ class CollectDataActivateGuardTest {
|
||||
mock(InvalidAsinDataMapper.class),
|
||||
mock(TaskChunkMapper.class), mock(TaskScopeStateMapper.class), mock(TaskResultItemMapper.class),
|
||||
mock(TaskDistributedLockService.class), mock(TaskFileJobService.class),
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
mock(TransientPayloadStorageService.class),
|
||||
mock(CollectDataExcelAssemblyService.class), mock(OssStorageService.class),
|
||||
new ObjectMapper(), new TransactionTemplate(transactionManager),
|
||||
|
||||
+3
-1
@@ -104,7 +104,9 @@ class CollectDataDeleteConsistencyTest {
|
||||
mock(com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper.class),
|
||||
mock(com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper.class),
|
||||
taskChunkMapper, taskScopeStateMapper, taskResultItemMapper,
|
||||
taskDistributedLockService, taskFileJobService, transientPayloadStorageService,
|
||||
taskDistributedLockService, taskFileJobService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
transientPayloadStorageService,
|
||||
mock(com.nanri.aiimage.modules.collectdata.service.CollectDataExcelAssemblyService.class),
|
||||
mock(com.nanri.aiimage.modules.file.service.oss.OssStorageService.class),
|
||||
objectMapper, txTemplate,
|
||||
|
||||
+3
@@ -8,6 +8,7 @@ import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
@@ -45,6 +46,8 @@ class CollectDataNoUploadStaleTest {
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
/** 默认(Mockito 布尔默认值 false)= 不构成「客户端仍在推进」豁免,与原判据行为一致。 */
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
@Mock private TaskDistributedLockService.LockHandle lockHandle;
|
||||
|
||||
@Spy
|
||||
|
||||
+3
-1
@@ -103,7 +103,9 @@ class CollectDataStorageCallCountTest {
|
||||
mock(com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper.class),
|
||||
mock(com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper.class),
|
||||
taskChunkMapper, taskScopeStateMapper, taskResultItemMapper,
|
||||
taskDistributedLockService, taskFileJobService, transientPayloadStorageService,
|
||||
taskDistributedLockService, taskFileJobService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
transientPayloadStorageService,
|
||||
mock(com.nanri.aiimage.modules.collectdata.service.CollectDataExcelAssemblyService.class),
|
||||
mock(com.nanri.aiimage.modules.file.service.oss.OssStorageService.class),
|
||||
objectMapper, txTemplate,
|
||||
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskScopeLastChunkDto;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 货源二次判死线的「客户端仍在推进」豁免。
|
||||
*
|
||||
* <p>事故背景(2026-09-22 任务 28919):客户端每 1.5~3 小时才回传一次分片(1300 行 / 12 小时),
|
||||
* 服务端按「180 分钟无结果分片上报」把仍在采集的任务封口结案,后面到的 22 个分片全被拒收,
|
||||
* 905 行数据丢失。修复后:位置文本仍在变(客户端在推进)就不封口;位置长时间不变或无位置信息
|
||||
* 才走原判据。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinNoUploadStaleTest {
|
||||
|
||||
private static final Long TASK_ID = 28919L;
|
||||
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
|
||||
@InjectMocks
|
||||
private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
when(properties.getNoResultUploadTimeoutMinutes()).thenReturn(180);
|
||||
when(properties.getStaleTimeoutMinutes()).thenReturn(30);
|
||||
}
|
||||
|
||||
/** 心跳新鲜(30 分钟内有心跳)+ 分片已 200 分钟没上传 = 命中二次判死线的候选条件。 */
|
||||
private void givenStaleChunkUpload() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType("SIMILAR_ASIN");
|
||||
task.setStatus("RUNNING");
|
||||
task.setCreatedAt(LocalDateTime.now().minusDays(1));
|
||||
task.setUpdatedAt(LocalDateTime.now().minusMinutes(1));
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(taskCacheService.getTaskHeartbeatMillis(TASK_ID)).thenReturn(System.currentTimeMillis());
|
||||
when(taskScopeStateMapper.selectLastChunkAtByTaskIds(any()))
|
||||
.thenReturn(List.of(new TaskScopeLastChunkDto(TASK_ID, LocalDateTime.now().minusMinutes(200))));
|
||||
}
|
||||
|
||||
private void invokeNoUploadCheck() {
|
||||
ReflectionTestUtils.invokeMethod(service, "finalizeNoUploadStaleTasks");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 客户端仍在推进时不封口() {
|
||||
givenStaleChunkUpload();
|
||||
when(taskHeartbeatPositionService.evaluateStall(TASK_ID, 180L))
|
||||
.thenReturn(TaskHeartbeatPositionService.StallState.ADVANCING);
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
// 豁免生效:连任务锁都不去抢,更不会封口结案(也就不会拒收后续分片)
|
||||
verify(taskDistributedLockService, never()).acquire(anyString(), anyLong(), any(Duration.class), anyLong());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 客户端停滞时保留原封口行为() {
|
||||
givenStaleChunkUpload();
|
||||
when(taskHeartbeatPositionService.evaluateStall(TASK_ID, 180L))
|
||||
.thenReturn(TaskHeartbeatPositionService.StallState.STALLED);
|
||||
when(taskDistributedLockService.acquire(anyString(), anyLong(), any(Duration.class), anyLong()))
|
||||
.thenReturn(null);
|
||||
|
||||
invokeNoUploadCheck();
|
||||
|
||||
// 位置长时间不变(真卡死)仍按原判据走:进入锁内封口流程
|
||||
verify(taskDistributedLockService).acquire(anyString(), anyLong(), any(Duration.class), anyLong());
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -306,7 +306,9 @@ class RollbackSemanticsContractTest {
|
||||
storageProperties, fileTaskMapper, fileResultMapper,
|
||||
taskScopeStateMapper, taskChunkMapper, objectMapper, mock(AppearancePatentLlmClient.class),
|
||||
mock(AppearancePatentTaskCacheService.class), mock(com.nanri.aiimage.config.AppearancePatentProperties.class),
|
||||
taskFileJobService, taskProgressSnapshotService, transientPayloadStorageService, transactionManager,
|
||||
taskFileJobService, taskProgressSnapshotService,
|
||||
mock(com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.class),
|
||||
transientPayloadStorageService, transactionManager,
|
||||
distributedJobLockService, taskDistributedLockService, instanceMetadata,
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(UserApiSecretService.class));
|
||||
|
||||
+79
@@ -13,7 +13,9 @@ import java.time.Duration;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
@@ -111,4 +113,81 @@ class TaskHeartbeatPositionServiceTest {
|
||||
|
||||
assertNull(service().describe(123L));
|
||||
}
|
||||
|
||||
// ===== 停滞判定(封口前的豁免依据)=====
|
||||
|
||||
@Test
|
||||
void stallUnknownWhenNoPosition() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123")).thenReturn(Map.of());
|
||||
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.UNKNOWN, service().evaluateStall(123L, 180L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stallUnknownOnRedisError() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenThrow(new RuntimeException("redis down"));
|
||||
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.UNKNOWN, service().evaluateStall(123L, 180L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stallAdvancingWhenPositionChangedAndSnapshotRefreshed() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)"));
|
||||
when(hashOperations.entries("task:progress:stall:123"))
|
||||
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(400/1300)", "at", String.valueOf(Long.MAX_VALUE)));
|
||||
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING, service().evaluateStall(123L, 180L));
|
||||
verify(hashOperations).putAll(eq("task:progress:stall:123"), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void stallAdvancingWhenPositionUnchangedWithinWindow() {
|
||||
// 位置没变但还没到停滞窗口:仍按「在推进」处理,不得封口
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)"));
|
||||
when(hashOperations.entries("task:progress:stall:123"))
|
||||
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(401/1300)",
|
||||
"at", String.valueOf(System.currentTimeMillis() - Duration.ofMinutes(5).toMillis())));
|
||||
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.ADVANCING, service().evaluateStall(123L, 180L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stallStalledWhenPositionUnchangedBeyondWindow() {
|
||||
// 位置长时间不动才是真卡死,允许调用方按原判据封口
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)"));
|
||||
when(hashOperations.entries("task:progress:stall:123"))
|
||||
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(401/1300)",
|
||||
"at", String.valueOf(System.currentTimeMillis() - Duration.ofMinutes(181).toMillis())));
|
||||
|
||||
assertEquals(TaskHeartbeatPositionService.StallState.STALLED, service().evaluateStall(123L, 180L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void clientAdvancingOnlyForAdvancingState() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
// 停滞(位置长时间未变)不构成豁免
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("progressText", "ASIN B0F44Y3YYQ(401/1300)"));
|
||||
when(hashOperations.entries("task:progress:stall:123"))
|
||||
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(401/1300)",
|
||||
"at", String.valueOf(System.currentTimeMillis() - Duration.ofMinutes(181).toMillis())));
|
||||
assertFalse(service().isClientAdvancing(123L, 180L));
|
||||
|
||||
// 无位置信息(老客户端)同样不构成豁免,调用方保留原判据
|
||||
assertFalse(service().isClientAdvancing(null, 180L));
|
||||
|
||||
// 位置在变才是豁免
|
||||
when(hashOperations.entries("task:progress:stall:123"))
|
||||
.thenReturn(Map.of("pos", "ASIN B0F44Y3YYQ(1/1300)", "at", String.valueOf(Long.MAX_VALUE)));
|
||||
assertTrue(service().isClientAdvancing(123L, 180L));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user