fix(品牌任务): 无结果回传判死线去掉静默 continue,改按处理位置三分

`failNoUploadStaleRunningTasks` 里 `last_result_at <= 0`(从未上报结果分片)原先无条件
continue 且零日志——这是 2026-10-06 任务 2626 挂到 9 小时不被判死的出口之一:客户端在 30 轮
失败重跑里心跳照打,`updated_at` 恒新鲜,30 分钟心跳线 / 180 分钟无结果线 / 120 分钟 repair
三条全部躲过。

改为用 TaskHeartbeatPositionService 区分三种情形:
- 位置仍在推进 ⇒ 只是攒着没回传(首批较慢的正常任务),INFO 后跳过;
- 位置长时间停在原地 ⇒ 判死,errorMessage 写进「停在哪个位置」;
- 完全没有位置信息(老客户端不传 progressText)⇒ 证据不足,保守跳过但留 WARN。

BrandTaskStaleCheckTest 8 例(隔离 worktree 跑,避开并发 mvn):品牌包 36 例绿;
把 BrandTaskService 换回本笔前版本后 3 例红(Wanted but not invoked: markFailed)⇒ 用例有牙。
同形状的静默出口还剩 4 处未接上:AppearancePatentTaskService / CollectDataService /
DeleteBrandStaleTaskService / PublishTaskService(各模块回传节奏不同,需逐模块核窗口)。
This commit is contained in:
2026-10-07 01:57:30 +08:00
parent 34113aff8b
commit 98bf6f63a6
2 changed files with 233 additions and 2 deletions
@@ -0,0 +1,206 @@
package com.nanri.aiimage.modules.brand.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.config.BrandProgressProperties;
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 品牌「无结果回传」判死线的从未上报分支(2026-10-06 任务 2626 的静默出口)。
*
* <p>2626:1915 个品牌,客户端因响应解密失败在 30 轮重跑里空转,一次结果都没回传,
* 但 15 秒心跳照打 ⇒ updated_at 永远新鲜、last_result_at 恒为 0。原实现在
* {@code lastResultAt <= 0} 处无条件 continue,任务因此挂 9 小时不判死、前端只看得到 0/1915。
* 现按「处理位置是否仍在推进」区分:在推进 = 首批较慢,跳过;停在原地 = 判死并带位置;
* 无位置信息(老客户端)= 证据不足,保守跳过。
*/
@ExtendWith(MockitoExtension.class)
class BrandTaskStaleCheckTest {
private static final long NO_RESULT_TIMEOUT_MINUTES = 180L;
@Mock private BrandCrawlTaskMapper brandCrawlTaskMapper;
@Mock private BrandProgressProperties brandProgressProperties;
@Mock private BrandTaskProgressCacheService brandTaskProgressCacheService;
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
@Mock private TaskDistributedLockService taskDistributedLockService;
private BrandTaskService service;
@BeforeAll
static void initializeTableInfo() {
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
BrandCrawlTaskEntity.class);
}
@BeforeEach
void setUp() {
// lenient:timeoutDisabledByConfigSkipsScan 会改配置,且它压根不读这两个值
lenient().when(brandProgressProperties.getNoResultUploadTimeoutMinutes())
.thenReturn(NO_RESULT_TIMEOUT_MINUTES);
lenient().when(brandProgressProperties.getHeartbeatTimeoutMinutes()).thenReturn(30L);
lenient().when(taskDistributedLockService.acquire(any(), any(), anyLong()))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
service = newService();
}
private BrandTaskService newService() {
return new BrandTaskService(brandCrawlTaskMapper, null, null, brandProgressProperties,
brandTaskProgressCacheService, taskHeartbeatPositionService, null, null,
taskDistributedLockService, null, null, null, null);
}
private void runNoUploadLine(BrandCrawlTaskEntity task, Map<Object, Object> progress) {
when(brandCrawlTaskMapper.selectList(any(LambdaQueryWrapper.class)))
.thenReturn(List.of(task));
// lenient:拿不到任务锁的用例在读 progress 之前就已经 continue
lenient().when(brandTaskProgressCacheService.getProgress(task.getId())).thenReturn(progress);
ReflectionTestUtils.invokeMethod(service, "failNoUploadStaleRunningTasks");
}
private static BrandCrawlTaskEntity oldRunningTask(long id) {
BrandCrawlTaskEntity task = new BrandCrawlTaskEntity();
task.setId(id);
task.setStatus("running");
task.setCreatedAt(LocalDateTime.now().minusHours(9));
task.setUpdatedAt(LocalDateTime.now().minusSeconds(20));
return task;
}
// ---------- 从未上报结果(last_result_at 缺失):本次修复的分支 ----------
@Test
void neverReportedStalledPositionFailsTaskWithPositionInReason() {
BrandCrawlTaskEntity task = oldRunningTask(2626L);
when(taskHeartbeatPositionService.describe(2626L)).thenReturn("已处理品牌 0/1915");
when(taskHeartbeatPositionService.isClientAdvancing(2626L, NO_RESULT_TIMEOUT_MINUTES))
.thenReturn(false);
runNoUploadLine(task, Map.of());
verify(brandTaskProgressCacheService).markFailed(eq(2626L), any(String.class));
String reason = captureFailReason();
assertTrue(reason.contains("无任何结果回传"), reason);
assertTrue(reason.contains("已处理品牌 0/1915"), "判死文案要带上卡住的位置:" + reason);
}
@Test
void neverReportedButClientAdvancingIsSkipped() {
// 反误杀:首批较慢、位置还在变的正常任务不得被判死(判死后回传会被全部拒收)
BrandCrawlTaskEntity task = oldRunningTask(2627L);
when(taskHeartbeatPositionService.describe(2627L)).thenReturn("已处理品牌 120/1915");
when(taskHeartbeatPositionService.isClientAdvancing(2627L, NO_RESULT_TIMEOUT_MINUTES))
.thenReturn(true);
runNoUploadLine(task, Map.of());
verify(brandTaskProgressCacheService, never()).markFailed(anyLong(), any());
verify(brandCrawlTaskMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class));
}
@Test
void neverReportedWithoutAnyPositionKeepsTaskAlive() {
// UNKNOWN 不是判死证据:老客户端不传 progressText,位置读不到时只能保守跳过
BrandCrawlTaskEntity task = oldRunningTask(2628L);
when(taskHeartbeatPositionService.describe(2628L)).thenReturn(null);
when(taskHeartbeatPositionService.isClientAdvancing(2628L, NO_RESULT_TIMEOUT_MINUTES))
.thenReturn(false);
runNoUploadLine(task, Map.of());
verify(brandTaskProgressCacheService, never()).markFailed(anyLong(), any());
}
@Test
void busyTaskLockNeverReports() {
BrandCrawlTaskEntity task = oldRunningTask(2629L);
when(taskDistributedLockService.acquire(any(), any(), anyLong())).thenReturn(null);
runNoUploadLine(task, Map.of());
verify(brandTaskProgressCacheService, never()).markFailed(anyLong(), any());
}
// ---------- 已有上报但长时间没新分片:原有语义不得回退 ----------
@Test
void reportedStaleAndNotAdvancingStillFails() {
BrandCrawlTaskEntity task = oldRunningTask(2630L);
long fourHoursAgo = System.currentTimeMillis() - 4 * 60 * 60_000L;
when(taskHeartbeatPositionService.isClientAdvancing(2630L, NO_RESULT_TIMEOUT_MINUTES))
.thenReturn(false);
runNoUploadLine(task, Map.of("last_result_at", String.valueOf(fourHoursAgo)));
String reason = captureFailReason();
assertTrue(reason.contains("无结果回传"), reason);
}
@Test
void reportedStaleButAdvancingIsSkipped() {
BrandCrawlTaskEntity task = oldRunningTask(2631L);
long fourHoursAgo = System.currentTimeMillis() - 4 * 60 * 60_000L;
when(taskHeartbeatPositionService.isClientAdvancing(2631L, NO_RESULT_TIMEOUT_MINUTES))
.thenReturn(true);
when(taskHeartbeatPositionService.describe(2631L)).thenReturn("已处理品牌 901/1915");
runNoUploadLine(task, Map.of("last_result_at", String.valueOf(fourHoursAgo)));
verify(brandTaskProgressCacheService, never()).markFailed(anyLong(), any());
}
@Test
void reportedRecentlyIsSkippedWithoutAskingPosition() {
BrandCrawlTaskEntity task = oldRunningTask(2632L);
long fiveMinutesAgo = System.currentTimeMillis() - 5 * 60_000L;
runNoUploadLine(task, Map.of("last_result_at", String.valueOf(fiveMinutesAgo)));
verify(taskHeartbeatPositionService, never()).isClientAdvancing(anyLong(), anyLong());
verify(brandTaskProgressCacheService, never()).markFailed(anyLong(), any());
}
@Test
void timeoutDisabledByConfigSkipsScan() {
BrandTaskService disabled = newService();
when(brandProgressProperties.getNoResultUploadTimeoutMinutes()).thenReturn(0L);
ReflectionTestUtils.invokeMethod(disabled, "failNoUploadStaleRunningTasks");
verify(brandCrawlTaskMapper, never()).selectList(any());
}
/** 取 markFailed 的实际文案:CAS 的 set 值只有占位符,真文本在缓存侧入参里。 */
private String captureFailReason() {
org.mockito.ArgumentCaptor<String> captor = org.mockito.ArgumentCaptor.forClass(String.class);
verify(brandTaskProgressCacheService).markFailed(anyLong(), captor.capture());
return captor.getValue();
}
}