fix(品牌任务): 无结果回传判死线去掉静默 continue,改按处理位置三分
`failNoUploadStaleRunningTasks` 里 `last_result_at <= 0`(从未上报结果分片)原先无条件 continue 且零日志——这是 2026-10-06 任务 2626 挂到 9 小时不被判死的出口之一:客户端在 30 轮 失败重跑里心跳照打,`updated_at` 恒新鲜,30 分钟心跳线 / 180 分钟无结果线 / 120 分钟 repair 三条全部躲过。 改为用 TaskHeartbeatPositionService 区分三种情形: - 位置仍在推进 ⇒ 只是攒着没回传(首批较慢的正常任务),INFO 后跳过; - 位置长时间停在原地 ⇒ 判死,errorMessage 写进「停在哪个位置」; - 完全没有位置信息(老客户端不传 progressText)⇒ 证据不足,保守跳过但留 WARN。 BrandTaskStaleCheckTest 8 例(隔离 worktree 跑,避开并发 mvn):品牌包 36 例绿; 把 BrandTaskService 换回本笔前版本后 3 例红(Wanted but not invoked: markFailed)⇒ 用例有牙。 同形状的静默出口还剩 4 处未接上:AppearancePatentTaskService / CollectDataService / DeleteBrandStaleTaskService / PublishTaskService(各模块回传节奏不同,需逐模块核窗口)。
This commit is contained in:
+206
@@ -0,0 +1,206 @@
|
||||
package com.nanri.aiimage.modules.brand.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.config.BrandProgressProperties;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 品牌「无结果回传」判死线的从未上报分支(2026-10-06 任务 2626 的静默出口)。
|
||||
*
|
||||
* <p>2626:1915 个品牌,客户端因响应解密失败在 30 轮重跑里空转,一次结果都没回传,
|
||||
* 但 15 秒心跳照打 ⇒ updated_at 永远新鲜、last_result_at 恒为 0。原实现在
|
||||
* {@code lastResultAt <= 0} 处无条件 continue,任务因此挂 9 小时不判死、前端只看得到 0/1915。
|
||||
* 现按「处理位置是否仍在推进」区分:在推进 = 首批较慢,跳过;停在原地 = 判死并带位置;
|
||||
* 无位置信息(老客户端)= 证据不足,保守跳过。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class BrandTaskStaleCheckTest {
|
||||
|
||||
private static final long NO_RESULT_TIMEOUT_MINUTES = 180L;
|
||||
|
||||
@Mock private BrandCrawlTaskMapper brandCrawlTaskMapper;
|
||||
@Mock private BrandProgressProperties brandProgressProperties;
|
||||
@Mock private BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||
@Mock private TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
|
||||
private BrandTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeTableInfo() {
|
||||
TableInfoHelper.initTableInfo(
|
||||
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
BrandCrawlTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
// lenient:timeoutDisabledByConfigSkipsScan 会改配置,且它压根不读这两个值
|
||||
lenient().when(brandProgressProperties.getNoResultUploadTimeoutMinutes())
|
||||
.thenReturn(NO_RESULT_TIMEOUT_MINUTES);
|
||||
lenient().when(brandProgressProperties.getHeartbeatTimeoutMinutes()).thenReturn(30L);
|
||||
lenient().when(taskDistributedLockService.acquire(any(), any(), anyLong()))
|
||||
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
|
||||
service = newService();
|
||||
}
|
||||
|
||||
private BrandTaskService newService() {
|
||||
return new BrandTaskService(brandCrawlTaskMapper, null, null, brandProgressProperties,
|
||||
brandTaskProgressCacheService, taskHeartbeatPositionService, null, null,
|
||||
taskDistributedLockService, null, null, null, null);
|
||||
}
|
||||
|
||||
private void runNoUploadLine(BrandCrawlTaskEntity task, Map<Object, Object> progress) {
|
||||
when(brandCrawlTaskMapper.selectList(any(LambdaQueryWrapper.class)))
|
||||
.thenReturn(List.of(task));
|
||||
// lenient:拿不到任务锁的用例在读 progress 之前就已经 continue
|
||||
lenient().when(brandTaskProgressCacheService.getProgress(task.getId())).thenReturn(progress);
|
||||
ReflectionTestUtils.invokeMethod(service, "failNoUploadStaleRunningTasks");
|
||||
}
|
||||
|
||||
private static BrandCrawlTaskEntity oldRunningTask(long id) {
|
||||
BrandCrawlTaskEntity task = new BrandCrawlTaskEntity();
|
||||
task.setId(id);
|
||||
task.setStatus("running");
|
||||
task.setCreatedAt(LocalDateTime.now().minusHours(9));
|
||||
task.setUpdatedAt(LocalDateTime.now().minusSeconds(20));
|
||||
return task;
|
||||
}
|
||||
|
||||
// ---------- 从未上报结果(last_result_at 缺失):本次修复的分支 ----------
|
||||
|
||||
@Test
|
||||
void neverReportedStalledPositionFailsTaskWithPositionInReason() {
|
||||
BrandCrawlTaskEntity task = oldRunningTask(2626L);
|
||||
when(taskHeartbeatPositionService.describe(2626L)).thenReturn("已处理品牌 0/1915");
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(2626L, NO_RESULT_TIMEOUT_MINUTES))
|
||||
.thenReturn(false);
|
||||
|
||||
runNoUploadLine(task, Map.of());
|
||||
|
||||
verify(brandTaskProgressCacheService).markFailed(eq(2626L), any(String.class));
|
||||
String reason = captureFailReason();
|
||||
assertTrue(reason.contains("无任何结果回传"), reason);
|
||||
assertTrue(reason.contains("已处理品牌 0/1915"), "判死文案要带上卡住的位置:" + reason);
|
||||
}
|
||||
|
||||
@Test
|
||||
void neverReportedButClientAdvancingIsSkipped() {
|
||||
// 反误杀:首批较慢、位置还在变的正常任务不得被判死(判死后回传会被全部拒收)
|
||||
BrandCrawlTaskEntity task = oldRunningTask(2627L);
|
||||
when(taskHeartbeatPositionService.describe(2627L)).thenReturn("已处理品牌 120/1915");
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(2627L, NO_RESULT_TIMEOUT_MINUTES))
|
||||
.thenReturn(true);
|
||||
|
||||
runNoUploadLine(task, Map.of());
|
||||
|
||||
verify(brandTaskProgressCacheService, never()).markFailed(anyLong(), any());
|
||||
verify(brandCrawlTaskMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void neverReportedWithoutAnyPositionKeepsTaskAlive() {
|
||||
// UNKNOWN 不是判死证据:老客户端不传 progressText,位置读不到时只能保守跳过
|
||||
BrandCrawlTaskEntity task = oldRunningTask(2628L);
|
||||
when(taskHeartbeatPositionService.describe(2628L)).thenReturn(null);
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(2628L, NO_RESULT_TIMEOUT_MINUTES))
|
||||
.thenReturn(false);
|
||||
|
||||
runNoUploadLine(task, Map.of());
|
||||
|
||||
verify(brandTaskProgressCacheService, never()).markFailed(anyLong(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void busyTaskLockNeverReports() {
|
||||
BrandCrawlTaskEntity task = oldRunningTask(2629L);
|
||||
when(taskDistributedLockService.acquire(any(), any(), anyLong())).thenReturn(null);
|
||||
runNoUploadLine(task, Map.of());
|
||||
verify(brandTaskProgressCacheService, never()).markFailed(anyLong(), any());
|
||||
}
|
||||
|
||||
// ---------- 已有上报但长时间没新分片:原有语义不得回退 ----------
|
||||
|
||||
@Test
|
||||
void reportedStaleAndNotAdvancingStillFails() {
|
||||
BrandCrawlTaskEntity task = oldRunningTask(2630L);
|
||||
long fourHoursAgo = System.currentTimeMillis() - 4 * 60 * 60_000L;
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(2630L, NO_RESULT_TIMEOUT_MINUTES))
|
||||
.thenReturn(false);
|
||||
|
||||
runNoUploadLine(task, Map.of("last_result_at", String.valueOf(fourHoursAgo)));
|
||||
|
||||
String reason = captureFailReason();
|
||||
assertTrue(reason.contains("无结果回传"), reason);
|
||||
}
|
||||
|
||||
@Test
|
||||
void reportedStaleButAdvancingIsSkipped() {
|
||||
BrandCrawlTaskEntity task = oldRunningTask(2631L);
|
||||
long fourHoursAgo = System.currentTimeMillis() - 4 * 60 * 60_000L;
|
||||
when(taskHeartbeatPositionService.isClientAdvancing(2631L, NO_RESULT_TIMEOUT_MINUTES))
|
||||
.thenReturn(true);
|
||||
when(taskHeartbeatPositionService.describe(2631L)).thenReturn("已处理品牌 901/1915");
|
||||
|
||||
runNoUploadLine(task, Map.of("last_result_at", String.valueOf(fourHoursAgo)));
|
||||
|
||||
verify(brandTaskProgressCacheService, never()).markFailed(anyLong(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void reportedRecentlyIsSkippedWithoutAskingPosition() {
|
||||
BrandCrawlTaskEntity task = oldRunningTask(2632L);
|
||||
long fiveMinutesAgo = System.currentTimeMillis() - 5 * 60_000L;
|
||||
|
||||
runNoUploadLine(task, Map.of("last_result_at", String.valueOf(fiveMinutesAgo)));
|
||||
|
||||
verify(taskHeartbeatPositionService, never()).isClientAdvancing(anyLong(), anyLong());
|
||||
verify(brandTaskProgressCacheService, never()).markFailed(anyLong(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void timeoutDisabledByConfigSkipsScan() {
|
||||
BrandTaskService disabled = newService();
|
||||
when(brandProgressProperties.getNoResultUploadTimeoutMinutes()).thenReturn(0L);
|
||||
|
||||
ReflectionTestUtils.invokeMethod(disabled, "failNoUploadStaleRunningTasks");
|
||||
|
||||
verify(brandCrawlTaskMapper, never()).selectList(any());
|
||||
}
|
||||
|
||||
/** 取 markFailed 的实际文案:CAS 的 set 值只有占位符,真文本在缓存侧入参里。 */
|
||||
private String captureFailReason() {
|
||||
org.mockito.ArgumentCaptor<String> captor = org.mockito.ArgumentCaptor.forClass(String.class);
|
||||
verify(brandTaskProgressCacheService).markFailed(anyLong(), captor.capture());
|
||||
return captor.getValue();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user