进度(light透传): 把心跳已存的 current/total 交给前端,为客户端任务进度条打通服务端一段
客户端心跳一直在发 current/total,Java 也已存进 Redis 位置 Hash,但除判死豁免外没人读它, 所以客户端界面上解决风控/跟价这些任务画不出进度条。本次只在 light 出口上透传,不新建表、不写迁移。 - TaskProgressLightVo + progressCurrent/progressTotal/progressLabel(三个同生同灭) - 装配器只对非终态任务读位置;读不到或分母非正数一律留 null,由前端决定不画(0/0 会被读成"进度 0%",与"没有进度"是两件事) - 位置服务加 readDrawablePositions:单次轮询封顶 20 次 Redis 往返,按尝试次数计(老客户端一个都不发时命中数为 0 但往返数仍是全量) - 标签按「,」整段截到 60 字,定长硬切会截出半截店铺名 - 相似ASIN 那份私有 LightVo 同步(它是历史遗留的第二套实现,不同步会让该模块进度随轮询闪烁) - 顺带修该私有实现的投影缺陷:select 少 module_type 一列,真实查询下所有任务被判 missing, 与共享装配器 2026-09-14 那次线上事故同形态(那边有守卫测试、这份没有),现补守卫
This commit is contained in:
+40
@@ -88,6 +88,7 @@ class SimilarAsinTaskServiceLightTest {
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
@@ -191,6 +192,45 @@ class SimilarAsinTaskServiceLightTest {
|
||||
jobDb.add(job(12L, 3L, "FAILED", null));
|
||||
}
|
||||
|
||||
/**
|
||||
* 守卫:light 的任务查询必须选中 module_type。
|
||||
*
|
||||
* <p>下面 634 行用 {@code MODULE_TYPE.equals(task.getModuleType())} 做模块过滤,投影里少这一列
|
||||
* 真实查询就取不到它 → 所有任务被判 missing,这一模块的轮询整面失明。共享装配器 2026-09-14
|
||||
* 就栽在同一个形态上(那边有守卫,这份私有实现没有)。mock 返回完整实体,所以只有断言 SQL
|
||||
* 投影才能抓到它。
|
||||
*/
|
||||
@Test
|
||||
void lightTaskQueryMustSelectModuleType() {
|
||||
seed();
|
||||
service.progressLight(List.of(1L));
|
||||
org.mockito.ArgumentCaptor<Wrapper> captor = org.mockito.ArgumentCaptor.forClass(Wrapper.class);
|
||||
verify(fileTaskMapper).selectList(captor.capture());
|
||||
String select = ((LambdaQueryWrapper<?>) captor.getValue()).getSqlSelect();
|
||||
assertTrue(select != null && select.contains("module_type"),
|
||||
"light 投影缺 module_type 会把所有任务判成 missing,实际=" + select);
|
||||
}
|
||||
|
||||
/** 心跳位置透传:非终态任务带进度,终态条目不带。 */
|
||||
@Test
|
||||
void lightCarriesHeartbeatProgressOnlyForActiveTask() {
|
||||
seed();
|
||||
lenient().when(taskHeartbeatPositionService.readDrawablePositions(anyList())).thenAnswer(invocation -> {
|
||||
List<Long> ids = invocation.getArgument(0);
|
||||
Map<Long, com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.Position> map = new HashMap<>();
|
||||
for (Long id : ids) {
|
||||
map.put(id, new com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService.Position(3, 4, "asin 3/4"));
|
||||
}
|
||||
return map;
|
||||
});
|
||||
|
||||
SimilarAsinTaskLightBatchVo vo = service.progressLight(List.of(1L, 2L));
|
||||
|
||||
assertEquals(4, vo.getItems().get(0).getProgressTotal(), "RUNNING 任务带心跳进度");
|
||||
assertEquals(3, vo.getItems().get(0).getProgressCurrent());
|
||||
assertNull(vo.getItems().get(1).getProgressTotal(), "SUCCESS 任务不得带中途百分比");
|
||||
}
|
||||
|
||||
@Test
|
||||
void lightReturnsLightFields() {
|
||||
seed();
|
||||
|
||||
+1
@@ -119,6 +119,7 @@ class SimilarAsinTaskServicePerf500Test {
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private DistributedJobLockService distributedJobLockService;
|
||||
|
||||
+29
-2
@@ -26,7 +26,9 @@ class TaskProgressLightContractTest {
|
||||
|
||||
private static final Set<String> WHITELIST = new TreeSet<>(Set.of(
|
||||
// rowsVersion:结果行版本(「行数据按需拉取」信号,2026-09-14 加入),仍是轻量字段、不含行内容
|
||||
"taskId", "status", "statusCode", "fileStatus", "fileError", "fileReady", "updatedAt", "rowsVersion"));
|
||||
"taskId", "status", "statusCode", "fileStatus", "fileError", "fileReady", "updatedAt", "rowsVersion",
|
||||
// progress*:客户端心跳上报的执行进度(2026-10-10 加入,三个字段同生同灭,用于画进度条)
|
||||
"progressCurrent", "progressTotal", "progressLabel"));
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@@ -81,6 +83,30 @@ class TaskProgressLightContractTest {
|
||||
assertTrue(bytes.length < 1024, "单条 light 响应 <1KB,实际 " + bytes.length + "B");
|
||||
}
|
||||
|
||||
/**
|
||||
* 体积闸门:加了 progress* 三字段后,带满长度标签的单条 light 响应仍须 <1KB。
|
||||
*
|
||||
* <p>light 是每 10 秒一次、单次最多 200 个任务的轮询面,标签一旦不封顶就会把心跳全文
|
||||
* (200 字,含店铺名/关键词/ASIN)按任务数放大成带宽成本。这里用中文标签喂,
|
||||
* 因为 UTF-8 下一个汉字 3 字节——用 ASCII 喂这条断言等于没测。
|
||||
*/
|
||||
@Test
|
||||
void commonShapeWithFullProgressLabelStillSmall() throws Exception {
|
||||
TaskProgressLightVo vo = fullItem();
|
||||
vo.setProgressCurrent(120);
|
||||
vo.setProgressTotal(150);
|
||||
StringBuilder filler = new StringBuilder();
|
||||
while (filler.length() < 60) {
|
||||
filler.append("店铺=shop-us-east,国家=US,ASIN=B0ABCD1234(3/10),");
|
||||
}
|
||||
vo.setProgressLabel(filler.substring(0, 60));
|
||||
TaskProgressLightBatchVo batch = new TaskProgressLightBatchVo();
|
||||
batch.getItems().add(vo);
|
||||
byte[] bytes = objectMapper.writeValueAsBytes(ApiResponse.success(batch));
|
||||
assertTrue(bytes.length < 1024,
|
||||
"带 60 字中文标签的单条 light 响应仍须 <1KB,实际 " + bytes.length + "B");
|
||||
}
|
||||
|
||||
@Test
|
||||
void commonShapeBatchHasOnlyItemsAndMissing() throws Exception {
|
||||
TaskProgressLightBatchVo batch = new TaskProgressLightBatchVo();
|
||||
@@ -115,7 +141,8 @@ class TaskProgressLightContractTest {
|
||||
void commonShapeWhitelistOrderedDocumented() {
|
||||
// 回归门禁:与 spec 06 §2 白名单一致(taskId/status/statusCode?/fileStatus?/fileError?/fileReady?/updatedAt
|
||||
// + 2026-09-14 新增 rowsVersion:结果行版本,行数据按需拉取的信号,仍是轻量字段)
|
||||
assertEquals(List.of("fileError", "fileReady", "fileStatus", "rowsVersion", "status", "statusCode", "taskId", "updatedAt"),
|
||||
assertEquals(List.of("fileError", "fileReady", "fileStatus", "progressCurrent", "progressLabel",
|
||||
"progressTotal", "rowsVersion", "status", "statusCode", "taskId", "updatedAt"),
|
||||
new java.util.ArrayList<>(WHITELIST));
|
||||
}
|
||||
|
||||
|
||||
+77
@@ -58,6 +58,83 @@ class TaskHeartbeatPositionServiceTest {
|
||||
verify(stringRedisTemplate).expire(eq("task:heartbeat:position:123"), any(Duration.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void readPositionReturnsStructuredNumbers() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123")).thenReturn(Map.of(
|
||||
"current", "2", "total", "10", "progressText", "US(2/10)"));
|
||||
|
||||
TaskHeartbeatPositionService.Position position = service().readPosition(123L);
|
||||
|
||||
assertEquals(2, position.current());
|
||||
assertEquals(10, position.total());
|
||||
assertEquals("US(2/10)", position.label());
|
||||
}
|
||||
|
||||
/** 没有位置(老客户端、或已过期)必须返回 null,调用方按「没有进度」处理。 */
|
||||
@Test
|
||||
void readPositionReturnsNullWhenHashAbsent() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123")).thenReturn(Map.of());
|
||||
|
||||
assertNull(service().readPosition(123L));
|
||||
}
|
||||
|
||||
/**
|
||||
* 只有文本、没有计数时也要返回 null:前端画的是百分比,光有文案算不出分母。
|
||||
* 若把这种位置透下去,前端只能画「未知/总」的空轨。
|
||||
*/
|
||||
@Test
|
||||
void readPositionReturnsNullWhenCountersAbsent() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("progressText", "shop A"));
|
||||
|
||||
assertNull(service().readPosition(123L));
|
||||
}
|
||||
|
||||
/** 计数字段是 Redis 里的裸字符串,脏值/非数字不得把整次轮询带崩。 */
|
||||
@Test
|
||||
void readPositionToleratesUnparsableCurrent() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("current", "abc", "total", "10"));
|
||||
|
||||
TaskHeartbeatPositionService.Position position = service().readPosition(123L);
|
||||
|
||||
assertNull(position.current());
|
||||
assertEquals(10, position.total());
|
||||
assertEquals(0, position.drawable().current(), "分子脏值按 0 计,分母仍可用");
|
||||
}
|
||||
|
||||
@Test
|
||||
void drawableClampsAndRejectsNonPositiveTotal() {
|
||||
assertNull(new TaskHeartbeatPositionService.Position(5, 0, "x").drawable());
|
||||
assertNull(new TaskHeartbeatPositionService.Position(5, null, "x").drawable());
|
||||
assertEquals(50, new TaskHeartbeatPositionService.Position(57, 50, "x").drawable().total());
|
||||
assertEquals(50, new TaskHeartbeatPositionService.Position(57, 50, "x").drawable().current());
|
||||
assertEquals(0, new TaskHeartbeatPositionService.Position(-3, 50, "x").drawable().current());
|
||||
}
|
||||
|
||||
/**
|
||||
* 标签按「,」整段收口:定长硬切会截出「国家 U…」这种半截标签,
|
||||
* 而 light 的响应体积闸门(单条 <1KB)又是必须守的,所以只能按分隔符截。
|
||||
*/
|
||||
@Test
|
||||
void labelTrimmedAtSeparatorNotMidSegment() {
|
||||
when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations);
|
||||
String longLabel = "阶段=crawling,店铺 魏振峰(2/10),国家 US(3/5),关键词 血糖测试仪,ASIN B0ABCDEFGH(4/10)";
|
||||
when(hashOperations.entries("task:heartbeat:position:123"))
|
||||
.thenReturn(Map.of("current", "4", "total", "10", "progressText", longLabel));
|
||||
|
||||
String label = service().readPosition(123L).label();
|
||||
|
||||
assertTrue(label.length() <= 60, "标签须封顶 60 字,实际 " + label.length());
|
||||
assertTrue(longLabel.startsWith(label), "截短结果必须是原文前缀,不能重排");
|
||||
assertFalse(label.endsWith("国") || label.endsWith("家"), "不得截在半段中间");
|
||||
assertEquals("阶段=crawling,店铺 魏振峰(2/10),国家 US(3/5),关键词 血糖测试仪", label);
|
||||
}
|
||||
|
||||
@Test
|
||||
void recordSkipsWhenNoPosition() {
|
||||
TaskHeartbeatRequest request = new TaskHeartbeatRequest();
|
||||
|
||||
+88
-1
@@ -18,6 +18,7 @@ import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
@@ -49,8 +50,9 @@ class TaskProgressLightAssemblerTest {
|
||||
private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
private final FileResultMapper fileResultMapper = mock(FileResultMapper.class);
|
||||
private final TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
|
||||
private final TaskHeartbeatPositionService positionService = mock(TaskHeartbeatPositionService.class);
|
||||
private final TaskProgressLightAssembler assembler =
|
||||
new TaskProgressLightAssembler(fileTaskMapper, fileResultMapper, taskFileJobService);
|
||||
new TaskProgressLightAssembler(fileTaskMapper, fileResultMapper, taskFileJobService, positionService);
|
||||
|
||||
private static FileTaskEntity task(long id, String moduleType, String status, LocalDateTime updatedAt) {
|
||||
FileTaskEntity t = new FileTaskEntity();
|
||||
@@ -78,6 +80,91 @@ class TaskProgressLightAssemblerTest {
|
||||
when(taskFileJobService.findAssembleJobsByTaskIds(any(), anyList())).thenReturn(jobs);
|
||||
}
|
||||
|
||||
@Test
|
||||
void runningTaskCarriesHeartbeatProgress() {
|
||||
stubTasks(List.of(task(1L, "publish", "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 1))));
|
||||
stubJobs(Map.of());
|
||||
when(positionService.readDrawablePositions(anyList()))
|
||||
.thenReturn(Map.of(1L, new TaskHeartbeatPositionService.Position(12, 50, "shop US(2/5)")));
|
||||
|
||||
TaskProgressLightVo item = assembler.assemble("publish", null, List.of(1L)).getItems().get(0);
|
||||
|
||||
assertEquals(12, item.getProgressCurrent());
|
||||
assertEquals(50, item.getProgressTotal());
|
||||
assertEquals("shop US(2/5)", item.getProgressLabel());
|
||||
}
|
||||
|
||||
/**
|
||||
* 终态任务一律不读心跳位置:位置 Hash 只有 24h TTL,隔天就读不到了;而且跑完的任务
|
||||
* 该按状态显示「已完成」,不该显示某个中途百分比。
|
||||
*/
|
||||
@Test
|
||||
void terminalTaskMustNotReadHeartbeatPosition() {
|
||||
stubTasks(List.of(
|
||||
task(1L, "publish", "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 1)),
|
||||
task(2L, "publish", "FAILED", LocalDateTime.of(2026, 8, 1, 10, 2)),
|
||||
task(3L, "publish", "CANCELLED", LocalDateTime.of(2026, 8, 1, 10, 3)),
|
||||
task(4L, "publish", "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 4))));
|
||||
stubJobs(Map.of());
|
||||
|
||||
TaskProgressLightBatchVo vo = assembler.assemble("publish", null, List.of(1L, 2L, 3L, 4L));
|
||||
|
||||
assertEquals(4, vo.getItems().size());
|
||||
for (TaskProgressLightVo item : vo.getItems()) {
|
||||
assertNull(item.getProgressTotal());
|
||||
assertNull(item.getProgressCurrent());
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
ArgumentCaptor<List<Long>> queried = forClass(List.class);
|
||||
verify(positionService).readDrawablePositions(queried.capture());
|
||||
assertEquals(List.of(4L), queried.getValue(), "位置读取列表只应含非终态任务");
|
||||
}
|
||||
|
||||
/**
|
||||
* 终态条目即便 Redis 里还留着位置也必须不给进度。
|
||||
*
|
||||
* <p>位置 Hash 有 24h TTL,任务跑完当天仍能读到最后一次心跳的位置;照它画会把一个
|
||||
* 已完成的任务显示成「87%」,而正确读法是状态徽标+100%。
|
||||
*/
|
||||
@Test
|
||||
void terminalItemsCarryNoProgressEvenWhenPositionStillExists() {
|
||||
stubTasks(List.of(
|
||||
task(1L, "publish", "SUCCESS", LocalDateTime.of(2026, 8, 1, 10, 1)),
|
||||
task(2L, "publish", "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 2))));
|
||||
stubJobs(Map.of());
|
||||
when(positionService.readDrawablePositions(anyList())).thenAnswer(invocation -> {
|
||||
List<Long> ids = invocation.getArgument(0);
|
||||
Map<Long, TaskHeartbeatPositionService.Position> map = new LinkedHashMap<>();
|
||||
for (Long id : ids) {
|
||||
map.put(id, new TaskHeartbeatPositionService.Position(7, 8, "at 7 of 8"));
|
||||
}
|
||||
return map;
|
||||
});
|
||||
|
||||
List<TaskProgressLightVo> items = assembler.assemble("publish", null, List.of(1L, 2L)).getItems();
|
||||
|
||||
assertNull(items.get(0).getProgressTotal(), "SUCCESS 条目不得带进度");
|
||||
assertNull(items.get(0).getProgressCurrent());
|
||||
assertEquals(8, items.get(1).getProgressTotal(), "RUNNING 条目带进度");
|
||||
assertEquals(7, items.get(1).getProgressCurrent());
|
||||
}
|
||||
|
||||
/** 老客户端不发位置:三个字段必须一起留 null,前端据此不画进度轨(而不是画一条 0%)。 */
|
||||
@Test
|
||||
void absentHeartbeatPositionLeavesProgressNull() {
|
||||
stubTasks(List.of(task(1L, "publish", "RUNNING", LocalDateTime.of(2026, 8, 1, 10, 1))));
|
||||
stubJobs(Map.of());
|
||||
when(positionService.readDrawablePositions(anyList())).thenReturn(Map.of());
|
||||
|
||||
TaskProgressLightVo item = assembler.assemble("publish", null, List.of(1L)).getItems().get(0);
|
||||
|
||||
assertNull(item.getProgressCurrent());
|
||||
assertNull(item.getProgressTotal());
|
||||
assertNull(item.getProgressLabel());
|
||||
}
|
||||
|
||||
/**
|
||||
/** 分子超过分母(客户端计数与分母不同源时会发生)要钳住,不能给出 114% 的进度条。 */
|
||||
@Test
|
||||
void statusMappingPreserved() {
|
||||
stubTasks(List.of(
|
||||
|
||||
+2
-1
@@ -36,8 +36,9 @@ class TaskProgressLightPermissionTest {
|
||||
private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
private final FileResultMapper fileResultMapper = mock(FileResultMapper.class);
|
||||
private final TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
|
||||
private final TaskHeartbeatPositionService positionService = mock(TaskHeartbeatPositionService.class);
|
||||
private final TaskProgressLightAssembler assembler =
|
||||
new TaskProgressLightAssembler(fileTaskMapper, fileResultMapper, taskFileJobService);
|
||||
new TaskProgressLightAssembler(fileTaskMapper, fileResultMapper, taskFileJobService, positionService);
|
||||
|
||||
private final List<FileTaskEntity> taskDb = new ArrayList<>();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user