diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskHeartbeatRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskHeartbeatRequest.java index 105c13737..0357c1169 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskHeartbeatRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskHeartbeatRequest.java @@ -42,4 +42,8 @@ public class TaskHeartbeatRequest { @Schema(description = "客户端归一化后的当前处理位置描述(如「店铺 张三(2/10),国家 德国(1/3)」)," + "用于判死日志与排查;老客户端不传,服务端不得以位置为空作为判死依据", example = "店铺 魏振峰(2/10)") private String progressText; + + @Schema(description = "客户端实际采用的进度粒度:asins/shops/countries/items/rows/units。" + + "units 表示 current/total 是「国家数×1000」的摊平当量,展示前要折算", example = "countries") + private String progressGrain; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java index 0728f4dfe..3ffba31b1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionService.java @@ -32,6 +32,8 @@ public class TaskHeartbeatPositionService { private static final int POSITION_LABEL_MAX_LEN = 60; /** 单次 light 轮询最多读多少个任务的心跳位置(Redis 往返上限,见 {@link #readDrawablePositions})。 */ public static final int MAX_POSITION_LOOKUPS_PER_CALL = 20; + /** 客户端 {@code config.PROGRESS_UNIT_SCALE} 的服务端镜像:units 粒度下 1 个自然单位 = 1000。 */ + public static final int PROGRESS_UNIT_SCALE = 1000; private final StringRedisTemplate stringRedisTemplate; @@ -53,7 +55,12 @@ public class TaskHeartbeatPositionService { * 结构化位置读数。{@code current}/{@code total} 可能为 null——客户端只发了文本、 * 没发计数(老版本按四级 key 归一化时 total 全为 0 就不发)。 */ - public record Position(Integer current, Integer total, String label) { + public record Position(Integer current, Integer total, String label, String grain) { + + /** 老客户端没报粒度:grain 留空,调用方按「自然计数」处理。 */ + public Position(Integer current, Integer total, String label) { + this(current, total, label, ""); + } /** * 归一化成「能画」的读数:total 非正数即视为没有进度(返回 null), @@ -67,7 +74,26 @@ public class TaskHeartbeatPositionService { return null; } int safeCurrent = current == null || current < 0 ? 0 : Math.min(current, total); - return new Position(safeCurrent, total, label); + return new Position(safeCurrent, total, label, grain); + } + + /** + * 折算成「可以直接显示」的当量,供后台任务中心看板使用。 + * + *

看板那一列的文案是「已完成 X / 总 Y」(2026-10-04 定的口径),而 {@code units} 这一档 + * 的 current/total 是客户端为「翻页进度单调不减」摊平出来的 国家数×1000——照原样显示会读成 + * 「已完成 1222 / 总 5000」,管理员只会问 5000 是什么。折算回国家当量后数字可读, + * 代价是看板的条子比客户端粗一档(20% 对 24%),这个取舍是值得的。 + * + *

客户端进度条不吃这个方法:它走 progress/light 拿原始千分位,要的是平滑。 + */ + public Position displayUnits() { + if (!"units".equals(grain) || total == null || total <= 0) { + return this; + } + int shownTotal = Math.max(1, total / PROGRESS_UNIT_SCALE); + int shownCurrent = current == null ? 0 : Math.min(shownTotal, current / PROGRESS_UNIT_SCALE); + return new Position(shownCurrent, shownTotal, label, grain); } } @@ -89,6 +115,7 @@ public class TaskHeartbeatPositionService { values.put("current", request.getCurrent() == null ? "" : String.valueOf(request.getCurrent())); values.put("total", request.getTotal() == null ? "" : String.valueOf(request.getTotal())); values.put("progressText", progressText); + values.put("grain", request.getProgressGrain() == null ? "" : request.getProgressGrain().trim()); values.put("at", String.valueOf(Instant.now().toEpochMilli())); String key = KEY_PREFIX + taskId; stringRedisTemplate.opsForHash().putAll(key, values); @@ -161,7 +188,7 @@ public class TaskHeartbeatPositionService { if (current == null && total == null) { return null; } - return new Position(current, total, trimLabel(value(entries, "progressText"))); + return new Position(current, total, trimLabel(value(entries, "progressText")), value(entries, "grain")); } /** @@ -176,13 +203,24 @@ public class TaskHeartbeatPositionService { * @return taskId → 可画读数;读不到位置或分母非正数的任务不出现在结果里 */ public Map readDrawablePositions(Collection taskIds) { - if (taskIds == null || taskIds.isEmpty()) { + return readDrawablePositions(taskIds, MAX_POSITION_LOOKUPS_PER_CALL); + } + + /** + * 同上,但由调用方给封顶值。 + * + *

封顶必须跟着调用面的规模走:progress/light 一次最多 200 个 id、每 10 秒一轮,20 次够; + * 后台看板一页可到 {@code MAX_PAGE_SIZE=100}、每 30 秒一轮,沿用 20 就会让第 21 行之后的任务 + * 永远读不到进度(还是那个「安静地没有条」的老形态)。 + */ + public Map readDrawablePositions(Collection taskIds, int maxLookups) { + if (taskIds == null || taskIds.isEmpty() || maxLookups <= 0) { return Map.of(); } Map result = new LinkedHashMap<>(); int attempts = 0; for (Long taskId : taskIds) { - if (attempts >= MAX_POSITION_LOOKUPS_PER_CALL) { + if (attempts >= maxLookups) { break; } attempts++; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/taskcenter/service/TaskCenterQueryService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/taskcenter/service/TaskCenterQueryService.java index be8d3ed9c..a895c1964 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/taskcenter/service/TaskCenterQueryService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/taskcenter/service/TaskCenterQueryService.java @@ -1,7 +1,9 @@ package com.nanri.aiimage.modules.taskcenter.service; +import com.nanri.aiimage.common.task.TaskStatuses; import com.nanri.aiimage.common.util.UserMessages; import com.nanri.aiimage.common.module.TaskModuleRegistry; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; import com.nanri.aiimage.modules.taskcenter.mapper.TaskCenterQueryMapper; import com.nanri.aiimage.modules.taskcenter.model.dto.TaskCenterCollectStatsRow; import com.nanri.aiimage.modules.taskcenter.model.dto.TaskCenterCountryRow; @@ -49,11 +51,17 @@ public class TaskCenterQueryService { private static final long ACTIVE_TASK_SCAN_CAP = 500; private static final long DEFAULT_PAGE_SIZE = 20; private static final long MAX_PAGE_SIZE = 100; + /** + * 心跳位置读取封顶,与页大小同源:看板一页最多 100 行、30 秒一轮。 + * 沿用 light 侧那个 20 的话,第 21 行之后的在跑任务永远读不到进度。 + */ + private static final int BOARD_POSITION_LOOKUPS = (int) MAX_PAGE_SIZE; /** 进行中状态与 mapper SQL 同源,避免「计数说在跑、卡住判定说不在跑」。 */ private static final Set ACTIVE_STATUSES = TaskCenterQueryMapper.ACTIVE_STATUS_SET; private final TaskCenterQueryMapper queryMapper; + private final TaskHeartbeatPositionService taskHeartbeatPositionService; public TaskCenterOverviewVo overview(TaskCenterScopeService.ScopeContext scope) { LocalDateTime now = LocalDateTime.now(); @@ -204,7 +212,8 @@ public class TaskCenterQueryService { : selectCollectStats(List.of(row), snapshots); TaskCenterTaskDetailVo vo = fillRow(new TaskCenterTaskDetailVo(), row, usernames, deviceNames, - snapshots, collectStats, collectShops(bizIds), collectCountries(bizIds), LocalDateTime.now()); + snapshots, collectStats, collectShops(bizIds), collectCountries(bizIds), + readHeartbeatPositions(List.of(row), snapshots), LocalDateTime.now()); vo.setShops(shopStats(row)); vo.setErrorMessage(row.getErrorMessage()); return vo; @@ -257,16 +266,35 @@ public class TaskCenterQueryService { Map collectStats = selectCollectStats(rows, snapshots); Map> shops = collectShops(bizIds); Map countries = collectCountries(bizIds); + Map positions = readHeartbeatPositions(rows, snapshots); LocalDateTime now = LocalDateTime.now(); List items = new ArrayList<>(rows.size()); for (TaskCenterTaskRow row : rows) { items.add(fillRow(new TaskCenterTaskRowVo(), row, usernames, Map.of(), snapshots, - collectStats, shops, countries, now)); + collectStats, shops, countries, positions, now)); } return items; } + /** + * 批量读「还在跑、又没有快照」那几行的心跳位置。 + * + *

封顶按页大小给(看板一页最多 {@code MAX_PAGE_SIZE} 行、30 秒一轮):若沿用 light 的 20, + * 第 21 行之后的任务永远读不到进度——又退成「安静地没有条」那个最难发现的形态。 + */ + private Map readHeartbeatPositions( + List rows, Map snapshots) { + List ids = new ArrayList<>(); + for (TaskCenterTaskRow row : rows) { + if (wantsHeartbeatPosition(row, snapshots)) { + ids.add(row.getId()); + } + } + return ids.isEmpty() ? Map.of() + : taskHeartbeatPositionService.readDrawablePositions(ids, BOARD_POSITION_LOOKUPS); + } + private static Map toUserNames(List rows) { Map map = new HashMap<>(); for (TaskCenterUserRow row : rows) { @@ -382,6 +410,7 @@ public class TaskCenterQueryService { Map collectStats, Map> shops, Map countries, + Map positions, LocalDateTime now) { vo.setId(row.getId()); vo.setTaskNo(row.getTaskNo()); @@ -406,7 +435,7 @@ public class TaskCenterQueryService { vo.setCurrentScope(snapshot.getCurrentScopeKey()); vo.setMessage(UserMessages.orFallback(snapshot.getMessage(), "")); } else { - fillFallbackProgress(vo, row, collectStats.get(row.getId())); + fillFallbackProgress(vo, row, collectStats.get(row.getId()), positions.get(row.getId())); } // 品牌检测按关键字执行、不落店铺;品牌行的 current_scope_key 是品牌名,不能当店铺用。 @@ -431,8 +460,29 @@ public class TaskCenterQueryService { * 旧的「输入文件数」回退让单文件任务恒显示 0/1,是误导。 * */ + /** 这三个模块的行内计数/关键词统计已足够画出进度,不用心跳位置再补一层。 */ + private static final Set IN_ROW_PROGRESS_MODULES = Set.of("PUBLISH", "DELETE_BRAND", "COLLECT_DATA"); + + /** + * 哪些行该用心跳位置补进度:无快照、不是上面那三个模块、且还在跑。 + * + *

「收集要读的 id」与「填充时决定用不用」共用这一个判据。两处各写一遍的话,改一处 + * 就会让另一处静默拿到 null,表现又是最难发现的那种形态——安静地没有进度条。 + */ + static boolean wantsHeartbeatPosition(TaskCenterTaskRow row, Map snapshots) { + if (row.isBrand() || row.getId() == null + || snapshots.containsKey(snapshotKey(row.getId(), row.getModuleType()))) { + return false; + } + if (IN_ROW_PROGRESS_MODULES.contains(row.getModuleType())) { + return false; + } + return row.getStatus() != null && !TaskStatuses.FileTask.TERMINAL.contains(row.getStatus()); + } + private static void fillFallbackProgress(TaskCenterTaskRowVo vo, TaskCenterTaskRow row, - TaskCenterCollectStatsRow collectStat) { + TaskCenterCollectStatsRow collectStat, + TaskHeartbeatPositionService.Position position) { String module = row.getModuleType(); if (row.isBrand() || "PUBLISH".equals(module) || "DELETE_BRAND".equals(module)) { vo.setProgressTotal(nullToZero(row.getTotalCount())); @@ -449,6 +499,15 @@ public class TaskCenterQueryService { vo.setProgressFailed(0); return; } + // 客户端心跳位置:跟价、定时匹配这些「执行器原本没有进度条」的模块,现在把上报的 + // 读数折算成自然当量画出来(units 那一档要除以 1000,否则看板会显示「已完成 1222 / 总 5000」)。 + if (position != null) { + TaskHeartbeatPositionService.Position shown = position.displayUnits(); + vo.setProgressTotal(shown.total()); + vo.setProgressSuccess(shown.current() == null ? 0 : shown.current()); + vo.setProgressFailed(0); + return; + } vo.setProgressTotal(0); vo.setProgressSuccess(0); vo.setProgressFailed(0); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionServiceTest.java index f69017a08..09828a099 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatPositionServiceTest.java @@ -58,6 +58,71 @@ class TaskHeartbeatPositionServiceTest { verify(stringRedisTemplate).expire(eq("task:heartbeat:position:123"), any(Duration.class)); } + private static TaskHeartbeatPositionService.Position pos(int current, int total, String grain) { + return new TaskHeartbeatPositionService.Position(current, total, "label", grain); + } + + /** 粒度名必须跟着位置一起存:看板要靠它判断这是「国家数×1000」的摊平当量还是自然计数。 */ + @Test + void recordStoresGrainAlongsideCounts() { + when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); + TaskHeartbeatRequest request = new TaskHeartbeatRequest(); + request.setCurrent(1222); + request.setTotal(5000); + request.setProgressGrain("units"); + + service().record(123L, request); + + @SuppressWarnings("unchecked") + ArgumentCaptor> values = ArgumentCaptor.forClass(Map.class); + verify(hashOperations).putAll(eq("task:heartbeat:position:123"), values.capture()); + assertEquals("units", values.getValue().get("grain"), "粒度丢了,看板就会把 5000 当自然总数显示"); + } + + @Test + void readPositionCarriesGrain() { + when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); + when(hashOperations.entries("task:heartbeat:position:123")) + .thenReturn(Map.of("current", "1222", "total", "5000", "grain", "units")); + + assertEquals("units", service().readPosition(123L).grain()); + } + + /** 老客户端没报粒度:grain 为空串,displayUnits 就必须原样返回,不许把 3/9 折成 0/1。 */ + @Test + void absentGrainIsTreatedAsNaturalCount() { + when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); + when(hashOperations.entries("task:heartbeat:position:123")) + .thenReturn(Map.of("current", "3", "total", "9")); + + TaskHeartbeatPositionService.Position position = service().readPosition(123L); + + assertEquals("", position.grain()); + assertEquals(9, position.displayUnits().total()); + assertEquals(3, position.displayUnits().current()); + } + + /** + * 摊平当量与自然计数的折算分别走两条路:看板除以 1000,客户端进度条仍拿原始千分位。 + * + *

这两条必须同时成立——只测折算会漏掉「客户端那条被误伤」,只测原始值会漏掉看板显示 1222/5000。 + */ + @Test + void displayUnitsCollapsesFlattenedScaleOnly() { + assertEquals(5, pos(1222, 5000, "units").displayUnits().total()); + assertEquals(1, pos(1222, 5000, "units").displayUnits().current()); + assertEquals(1222, pos(1222, 5000, "units").drawable().current(), "客户端侧仍是千分位,不能一起折"); + assertEquals(9, pos(3, 9, "countries").displayUnits().total(), "国家计数原样用"); + assertEquals(50, pos(57000, 50000, "units").displayUnits().current(), "折算后也不许超过分母"); + } + + /** 分母不足一个当量时至少给 1 格:0 会被前端读成「没有进度」,那是另一种说谎。 */ + @Test + void displayUnitsNeverCollapsesTotalToZero() { + assertEquals(1, pos(0, 999, "units").displayUnits().total()); + assertEquals(1, pos(998, 999, "units").displayUnits().total()); + } + @Test void readPositionReturnsStructuredNumbers() { when(stringRedisTemplate.opsForHash()).thenReturn(hashOperations); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/taskcenter/service/TaskCenterQueryServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/taskcenter/service/TaskCenterQueryServiceTest.java index 60d49bd95..ee5a102f4 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/taskcenter/service/TaskCenterQueryServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/taskcenter/service/TaskCenterQueryServiceTest.java @@ -1,5 +1,6 @@ package com.nanri.aiimage.modules.taskcenter.service; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; import com.nanri.aiimage.modules.taskcenter.mapper.TaskCenterQueryMapper; import com.nanri.aiimage.modules.taskcenter.model.dto.TaskCenterCollectStatsRow; import com.nanri.aiimage.modules.taskcenter.model.dto.TaskCenterCountryRow; @@ -57,6 +58,7 @@ import static org.mockito.Mockito.when; class TaskCenterQueryServiceTest { @Mock private TaskCenterQueryMapper queryMapper; + @Mock private TaskHeartbeatPositionService positionService; @InjectMocks private TaskCenterQueryService service; private static TaskCenterScopeService.ScopeContext superScope() { @@ -496,6 +498,85 @@ class TaskCenterQueryServiceTest { assertEquals(0L, vo.getProgressFailed()); } + /** + * 「执行器原本没进度条」的模块现在用心跳位置画进度。 + * + *

units 那一档的读数是「国家数×1000」的摊平当量,看板必须折算——照原样显示会读出 + * 「已完成 1222 / 总 5000」,管理员只会问 5000 是什么。countries 是自然计数,原样用。 + */ + @Test + void runningRowWithoutSnapshotTakesProgressFromHeartbeatPosition() { + LocalDateTime now = LocalDateTime.now(); + TaskCenterTaskRow priceTrack = taskRow(1L, "PRICE_TRACK", "RUNNING", now, now); + priceTrack.setTotalCount(1L); + TaskCenterTaskRow shopMatch = taskRow(2L, "SHOP_MATCH", "RUNNING", now, now); + shopMatch.setTotalCount(1L); + + stubPage(List.of(priceTrack, shopMatch)); + when(queryMapper.selectSnapshots(any())).thenReturn(List.of()); + stubPublishShops(List.of()); + stubFileResultShops(List.of()); + when(positionService.readDrawablePositions(any(), org.mockito.ArgumentMatchers.anyInt())) + .thenReturn(java.util.Map.of( + 1L, new TaskHeartbeatPositionService.Position(1222, 5000, "US 2/5", "units"), + 2L, new TaskHeartbeatPositionService.Position(3, 9, "shop A", "countries"))); + + TaskCenterTaskPageVo page = service.tasks(superScope(), new TaskCenterTaskQuery()); + + assertEquals(5L, item(page, 1L).getProgressTotal(), "units 读数必须折算回国家当量"); + assertEquals(1L, item(page, 1L).getProgressSuccess()); + assertEquals(9L, item(page, 2L).getProgressTotal(), "countries 是自然计数,不许再被折算"); + assertEquals(3L, item(page, 2L).getProgressSuccess()); + assertEquals(0L, item(page, 1L).getProgressFailed(), "心跳位置没有失败口径,failed 必须留 0 而不是猜一个"); + } + + /** + * 终态行一次都不读心跳位置。 + * + *

位置 Hash 只有 24h TTL,隔天就读不到;而且跑完的行该按状态收尾,不该显示某个中途百分比 + * (跟价页那种「成功 0 / 总 1 + 失败 1」的旧读法就是被这个形态坑出来的)。 + */ + @Test + void terminalRowsMustNotReadHeartbeatPosition() { + LocalDateTime now = LocalDateTime.now(); + TaskCenterTaskRow done = taskRow(1L, "PRICE_TRACK", "SUCCESS", now, now); + done.setTotalCount(1L); + TaskCenterTaskRow failed = taskRow(2L, "SHOP_MATCH", "FAILED", now, now); + + stubPage(List.of(done, failed)); + when(queryMapper.selectSnapshots(any())).thenReturn(List.of()); + stubPublishShops(List.of()); + stubFileResultShops(List.of()); + + TaskCenterTaskPageVo page = service.tasks(superScope(), new TaskCenterTaskQuery()); + + verify(positionService, org.mockito.Mockito.never()).readDrawablePositions(any(), + org.mockito.ArgumentMatchers.anyInt()); + assertEquals(0L, item(page, 1L).getProgressTotal(), "终态行不回退输入文件数"); + assertEquals(0L, item(page, 2L).getProgressTotal()); + } + + /** 已经有快照的行不用心跳位置:快照是服务端算出来的权威读数,位置只是执行器的自述。 */ + @Test + void snapshotWinsOverHeartbeatPosition() { + LocalDateTime now = LocalDateTime.now(); + TaskCenterTaskRow row = taskRow(1L, "QUERY_ASIN", "RUNNING", now, now); + row.setTotalCount(9L); + + stubPage(List.of(row)); + when(queryMapper.selectSnapshots(any())).thenReturn(List.of( + snapshotRow(1L, "QUERY_ASIN", 40L, 30L, 2L, "shop-a", "在跑"))); + stubPublishShops(List.of()); + stubFileResultShops(List.of()); + + TaskCenterTaskPageVo page = service.tasks(superScope(), new TaskCenterTaskQuery()); + + assertEquals(40L, item(page, 1L).getProgressTotal(), "快照优先,不能被心跳位置覆盖"); + assertEquals(30L, item(page, 1L).getProgressSuccess()); + verify(positionService, org.mockito.Mockito.never()).readDrawablePositions(any(), + org.mockito.ArgumentMatchers.anyInt()); + } + @Test void modulesWithoutClientProgressBarHaveNoFallbackCount() { LocalDateTime now = LocalDateTime.now();