feat(jd): 首页一级入口「京东」+工作台两菜单,获取店铺数据一键采集带进度条
首页在亚马逊/视频右侧新增「京东」一级入口(恒显示),落地页 /jd-console 按亚马逊控制台样式做两张工具卡:获取店铺数据(可用)、商品上架(占位,点开只提示「暂未开通」)。
获取店铺数据改成一键:建单 → 推 jd-shop-collect-run 到本机队列 → 轮询服务端进度。进度只读两处真源——任务行状态 + scope 行合并写游标里的 itemsReceived/cursor.total(客户端抓列表第一页时把 total 写进游标),没有分母就退回不定态动画,不编造百分比;终态即停轮询。后端新增 GET /api/jd-collect/tasks/{taskId}/progress(按 taskId+userId 校归属)。
判据:JdCollectProgressQueryTest(5)、jd-console-tool-catalog.test.ts(5)、jd-console-pending-menu.mount(3)、jd-collect-progress.mount(6);catch 棘轮 blocks 314→315 按名登记(轮询 catch 非静默)。
This commit is contained in:
+17
@@ -4,6 +4,7 @@ import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdCollectTaskCreateRequest;
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdIngestChunkRequest;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdCollectProgressVo;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdCollectTaskVo;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdIngestChunkVo;
|
||||
import com.nanri.aiimage.modules.jd.service.JdCollectTaskService;
|
||||
@@ -19,10 +20,12 @@ import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
@@ -58,6 +61,20 @@ public class JdCollectController {
|
||||
adminAuthSupport.currentDeviceId(httpRequest)));
|
||||
}
|
||||
|
||||
@GetMapping("/tasks/{taskId}/progress")
|
||||
@Operation(summary = "查询京东采集进度", description = "用户端「获取店铺数据」进度条数据源:"
|
||||
+ "任务状态 + 已回传商品数 + 列表 total(客户端抓首页时写入游标) + 当前页/Tab + 已收批次数。"
|
||||
+ "进度查询与建单同口径鉴权:uid 按会话身份回填,任务 owner 与 uid 不一致即 403。")
|
||||
public ApiResponse<JdCollectProgressVo> progress(
|
||||
@Parameter(description = "任务 ID", required = true, example = "33890")
|
||||
@PathVariable Long taskId,
|
||||
@Parameter(description = "任务所属用户 ID;省略时取当前登录身份", required = false, example = "1")
|
||||
@RequestParam(value = "user_id", required = false) Long userId,
|
||||
HttpServletRequest httpRequest) {
|
||||
return ApiResponse.success(collectTaskService.queryProgress(taskId,
|
||||
adminAuthSupport.requireOwnershipUserId(httpRequest, userId)));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/chunk")
|
||||
@Operation(summary = "提交采集分片回传", description = "采集器按批次回传列表/详情/图片清单;"
|
||||
+ "游标合并写(缺件沿用前值)、行 upsert-only;isEnd=true 任务落 SUCCESS;"
|
||||
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package com.nanri.aiimage.modules.jd.model.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* 京东店铺采集进度(v1.5 用户端"获取店铺数据"进度条数据源)。
|
||||
*
|
||||
* <p>数据只取两处真源:{@code biz_file_task}(任务状态/终态原因)与
|
||||
* {@code biz_task_scope_state}(合并写游标:{@code itemsReceived}、{@code cursor.total}、
|
||||
* {@code receivedChunkCount})。不在服务端另算一套百分比——客户端抓完一页就把本页
|
||||
* total 写进游标(合并写,缺键沿用前值,见铁律 30),页面据此算进度。
|
||||
*/
|
||||
@Data
|
||||
public class JdCollectProgressVo {
|
||||
|
||||
private Long taskId;
|
||||
|
||||
/** PENDING / RUNNING / SUCCESS / FAILED(biz_file_task.status 原值)。 */
|
||||
private String status;
|
||||
|
||||
/** 已回传落库的商品数(scope.state_json.itemsReceived 累计值)。 */
|
||||
private Integer itemsReceived;
|
||||
|
||||
/**
|
||||
* 列表总数:客户端抓第一页时从京麦列表接口拿到的 total,没拿到过就为 {@code null}。
|
||||
* 页面据此决定是"百分比进度条"还是"不定态进度条"。
|
||||
*/
|
||||
private Integer total;
|
||||
|
||||
/** 当前采集页(游标,断点续跑会从这里接着跑)。 */
|
||||
private Integer page;
|
||||
|
||||
/** 当前采集 Tab(ONSALE / OFFSALE)。 */
|
||||
private String listTab;
|
||||
|
||||
/** 已收到的分片批次数。 */
|
||||
private Integer batches;
|
||||
|
||||
/** 终态原因(FAILED 时为客户端上报的真实原因)。 */
|
||||
private String message;
|
||||
|
||||
/** 最近一次回传/合并写的时间。 */
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
+103
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.CollectRoutingProperties;
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdCollectTaskCreateRequest;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdCollectProgressVo;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdCollectTaskVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
@@ -20,7 +21,10 @@ import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
@@ -159,4 +163,103 @@ public class JdCollectTaskService {
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.last("limit 1"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询采集进度(v1.5 用户端「获取店铺数据」进度条)。
|
||||
*
|
||||
* <p>只读两处真源:任务行(状态/终态原因)与 scope 行(合并写游标里的
|
||||
* {@code itemsReceived}/{@code cursor.total}/{@code cursor.page})。这里不重算百分比——
|
||||
* 分母来自客户端抓列表第一页时拿到的 total,没拿到过就返回 {@code null},
|
||||
* 由页面降级成不定态进度条(不编造一个假百分比)。
|
||||
*
|
||||
* <p>归属校验按 {@code (taskId, userId)}(铁律 11/22):管理员代客时 userId 来自
|
||||
* 请求参数,若与任务 owner 不一致一律拒,不让 taskId 裸查越权。
|
||||
*/
|
||||
public JdCollectProgressVo queryProgress(Long taskId, Long userId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
throw new BusinessException("任务 ID 不合法");
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
throw new BusinessException("任务不存在: " + taskId);
|
||||
}
|
||||
if (userId != null && task.getUserId() != null && !Objects.equals(userId, task.getUserId())) {
|
||||
throw new BusinessException(403, "无权查看该任务");
|
||||
}
|
||||
|
||||
JdCollectProgressVo vo = new JdCollectProgressVo();
|
||||
vo.setTaskId(task.getId());
|
||||
vo.setStatus(task.getStatus());
|
||||
vo.setMessage(task.getErrorMessage());
|
||||
vo.setUpdatedAt(task.getUpdatedAt());
|
||||
|
||||
TaskScopeStateEntity scope = findScopeRow(taskId);
|
||||
if (scope == null) {
|
||||
// scope 行缺失(历史/异常建单)时只回任务级状态,进度按 0 计,不抛错——
|
||||
// 进度查询是展示面,不该因为一行辅助表缺失把页面打成红条。
|
||||
vo.setItemsReceived(nz(task.getSuccessFileCount()));
|
||||
vo.setBatches(0);
|
||||
return vo;
|
||||
}
|
||||
Map<String, Object> state = parseScopeState(scope.getStateJson());
|
||||
Map<String, Object> cursor = asMap(state.get("cursor"));
|
||||
vo.setItemsReceived(Math.max(nz(task.getSuccessFileCount()), asInt(state.get("itemsReceived"))));
|
||||
vo.setTotal(asNullableInt(cursor.get("total")));
|
||||
vo.setPage(asNullableInt(cursor.get("page")));
|
||||
Object tab = cursor.get("tab");
|
||||
vo.setListTab(tab == null ? null : String.valueOf(tab));
|
||||
vo.setBatches(scope.getReceivedChunkCount() == null ? 0 : scope.getReceivedChunkCount());
|
||||
if (scope.getUpdatedAt() != null) {
|
||||
vo.setUpdatedAt(scope.getUpdatedAt());
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
/** 读 scope 状态 JSON;损坏时返回空对象并留痕(查询面不因一行脏 JSON 抛错)。 */
|
||||
private Map<String, Object> parseScopeState(String stateJson) {
|
||||
if (stateJson == null || stateJson.isBlank()) {
|
||||
return new LinkedHashMap<>();
|
||||
}
|
||||
try {
|
||||
Map<String, Object> parsed = objectMapper.readValue(stateJson,
|
||||
objectMapper.getTypeFactory().constructMapType(LinkedHashMap.class, String.class, Object.class));
|
||||
return parsed == null ? new LinkedHashMap<>() : parsed;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[jd-collect] scope 状态 JSON 解析失败,进度按 0 展示 taskId 相关行: {}", ex.getMessage());
|
||||
return new LinkedHashMap<>();
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static Map<String, Object> asMap(Object raw) {
|
||||
return raw instanceof Map ? new LinkedHashMap<>((Map<String, Object>) raw) : new LinkedHashMap<>();
|
||||
}
|
||||
|
||||
private static int asInt(Object raw) {
|
||||
Integer value = asNullableInt(raw);
|
||||
return value == null ? 0 : value;
|
||||
}
|
||||
|
||||
/**
|
||||
* 宽松读整数:数字类型直接用,字符串只认 1~9 位十进制(int 范围内必然不溢出)。
|
||||
* 不拿 {@code NumberFormatException} 当控制流——脏值在进度展示面是正常的空结果,
|
||||
* 不该为它添一个 catch(catch 棘轮同形判据)。
|
||||
*/
|
||||
private static Integer asNullableInt(Object raw) {
|
||||
if (raw instanceof Number number) {
|
||||
return number.intValue();
|
||||
}
|
||||
if (raw == null) {
|
||||
return null;
|
||||
}
|
||||
String text = String.valueOf(raw).trim();
|
||||
if (!text.matches("-?\\d{1,9}")) {
|
||||
return null;
|
||||
}
|
||||
return Integer.valueOf(text);
|
||||
}
|
||||
|
||||
private static int nz(Integer value) {
|
||||
return value == null ? 0 : Math.max(0, value);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,7 +73,9 @@ class CatchSilenceRatchetTest {
|
||||
// + parseAttrGroups 的 attr_json 兜底;Phase 2/4 的 7 枚全是 RETHROW、Phase 2 的 scope 行
|
||||
// DuplicateKeyException 是 LOG_ONLY,都不占本格),逐处记名见 STRICT_TOTAL 上方注释;
|
||||
// LOG_ONLY 宽口径格 251 没动(宽口径实测 194,离上限很远)。
|
||||
static final int GIVEN_NOT_RETHROW = 615;
|
||||
// 11-14 v1.5「获取店铺数据」进度条抬 615 -> 616:一枚不重抛(进度查询读 scope.state_json
|
||||
// 失败 log.warn + 空对象),逐处记名与误差方向见 STRICT_TOTAL 上方注释;LOG_ONLY 一格没动。
|
||||
static final int GIVEN_NOT_RETHROW = 616;
|
||||
static final int GIVEN_LOG_ONLY = 251;
|
||||
static final int GIVEN_EMPTY = 56;
|
||||
|
||||
@@ -357,7 +359,14 @@ class CatchSilenceRatchetTest {
|
||||
// 「京东时间格式不识别(应为 yyyy-MM-dd HH:mm:ss): <原文>」(RETHROW)。误差方向:
|
||||
// 客户端回传脏时间戳之前直接 DateTimeParseException 裸抛 = 回传口 500 裸堆栈(P1-3);
|
||||
// 改成吞掉置 null = 登记时间被悄悄抹掉。
|
||||
static final int STRICT_TOTAL = 1091;
|
||||
// 11-14 v1.5「获取店铺数据」进度条抬 1091 -> 1092,一枚逐处记名:
|
||||
// `JdCollectTaskService.parseScopeState`:进度查询读 scope.state_json 时
|
||||
// `catch Exception` ⇒ log.warn 带原因 + 返回空对象(NOT_RETHROW)。误差方向:
|
||||
// 改成往外抛 = 一行脏 JSON 把进度查询打成 500、进度条直接红条;改成静默返回空对象
|
||||
// 不记日志 = 游标损坏永远没人知道。演示面只降级不抛错,但必须有那一行 WARN。
|
||||
// (相邻的数值解析刻意不用 NumberFormatException 当控制流:改为 1~9 位十进制正则前置,
|
||||
// 省掉两枚 catch,EMPTY 一格没涨。)
|
||||
static final int STRICT_TOTAL = 1092;
|
||||
/**
|
||||
* 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。
|
||||
* 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。
|
||||
@@ -491,7 +500,10 @@ class CatchSilenceRatchetTest {
|
||||
// 本格含 LOG_ONLY(incl 口径):同批 Phase 2 建单 scope 行那枚 DuplicateKeyException 复用
|
||||
// 先到行把 LOG_ONLY 抬 193 -> 194 时连带占本格一枚额度 ⇒ 614 -> 615(出处同 STRICT_TOTAL
|
||||
// 上方注释;这是本格**含 LOG_ONLY** 口径的连带,不是又一枚新的不重抛)。
|
||||
static final int STRICT_NOT_RETHROW = 615;
|
||||
// 11-14 v1.5「获取店铺数据」进度条:严格口径实测 615 -> 616(一枚不重抛:
|
||||
// JdCollectTaskService.parseScopeState 读 scope.state_json 失败 log.warn + 空对象,
|
||||
// 出处与误差方向见 STRICT_TOTAL 上方注释;logOnly 一格没动)。
|
||||
static final int STRICT_NOT_RETHROW = 616;
|
||||
/**
|
||||
* 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}:
|
||||
* 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出——
|
||||
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
package com.nanri.aiimage.modules.jd.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdCollectProgressVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 京东采集进度查询(v1.5「获取店铺数据」进度条数据源)。
|
||||
*
|
||||
* <p>判据三件:① 进度只读真源(任务行 + scope 合并写游标),不编造百分比;
|
||||
* ② 归属按 {@code (taskId, userId)} 校验,taskId 裸查不能越权;③ scope 行缺失或 JSON
|
||||
* 损坏时降级展示而不是把页面打成红条。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class JdCollectProgressQueryTest {
|
||||
|
||||
@Mock
|
||||
private FileTaskMapper fileTaskMapper;
|
||||
@Mock
|
||||
private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Spy
|
||||
private ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@InjectMocks
|
||||
private JdCollectTaskService service;
|
||||
|
||||
private FileTaskEntity runningTask(long userId) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(33890L);
|
||||
task.setModuleType(JdCollectTaskService.MODULE_TYPE);
|
||||
task.setUserId(userId);
|
||||
task.setStatus("RUNNING");
|
||||
task.setSuccessFileCount(0);
|
||||
task.setUpdatedAt(LocalDateTime.of(2026, 11, 14, 10, 0));
|
||||
return task;
|
||||
}
|
||||
|
||||
private TaskScopeStateEntity scope(String stateJson, Integer batches) {
|
||||
TaskScopeStateEntity scope = new TaskScopeStateEntity();
|
||||
scope.setTaskId(33890L);
|
||||
scope.setModuleType(JdCollectTaskService.MODULE_TYPE);
|
||||
scope.setStateJson(stateJson);
|
||||
scope.setReceivedChunkCount(batches);
|
||||
scope.setUpdatedAt(LocalDateTime.of(2026, 11, 14, 10, 1));
|
||||
return scope;
|
||||
}
|
||||
|
||||
@Test
|
||||
void 进度按合并写游标读出商品数与总数() {
|
||||
when(fileTaskMapper.selectById(33890L)).thenReturn(runningTask(7L));
|
||||
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope(
|
||||
"{\"cursor\":{\"tab\":\"ONSALE\",\"page\":3,\"total\":300},\"itemsReceived\":120}",
|
||||
4));
|
||||
|
||||
JdCollectProgressVo vo = service.queryProgress(33890L, 7L);
|
||||
|
||||
assertThat(vo.getStatus()).isEqualTo("RUNNING");
|
||||
assertThat(vo.getItemsReceived()).isEqualTo(120);
|
||||
assertThat(vo.getTotal()).as("分母只来自客户端写入游标的 total,服务端不另算")
|
||||
.isEqualTo(300);
|
||||
assertThat(vo.getPage()).isEqualTo(3);
|
||||
assertThat(vo.getListTab()).isEqualTo("ONSALE");
|
||||
assertThat(vo.getBatches()).isEqualTo(4);
|
||||
}
|
||||
|
||||
@Test
|
||||
void 归属不符拒绝查询() {
|
||||
when(fileTaskMapper.selectById(33890L)).thenReturn(runningTask(7L));
|
||||
|
||||
assertThatThrownBy(() -> service.queryProgress(33890L, 99L))
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessageContaining("无权查看该任务");
|
||||
}
|
||||
|
||||
@Test
|
||||
void scope缺失时回退任务成功数且总数留空() {
|
||||
FileTaskEntity task = runningTask(7L);
|
||||
task.setStatus("SUCCESS");
|
||||
task.setSuccessFileCount(58);
|
||||
when(fileTaskMapper.selectById(33890L)).thenReturn(task);
|
||||
when(taskScopeStateMapper.selectOne(any())).thenReturn(null);
|
||||
|
||||
JdCollectProgressVo vo = service.queryProgress(33890L, 7L);
|
||||
|
||||
assertThat(vo.getItemsReceived()).isEqualTo(58);
|
||||
assertThat(vo.getTotal()).isNull();
|
||||
assertThat(vo.getBatches()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void 游标脏JSON降级为零进度不抛错() {
|
||||
when(fileTaskMapper.selectById(33890L)).thenReturn(runningTask(7L));
|
||||
when(taskScopeStateMapper.selectOne(any())).thenReturn(scope("{\"cursor\":{半个", 1));
|
||||
|
||||
JdCollectProgressVo vo = service.queryProgress(33890L, 7L);
|
||||
|
||||
assertThat(vo.getItemsReceived()).isZero();
|
||||
assertThat(vo.getTotal()).isNull();
|
||||
assertThat(vo.getBatches()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void 非京东模块任务按不存在拒绝() {
|
||||
FileTaskEntity task = runningTask(7L);
|
||||
task.setModuleType("SIMILAR_ASIN");
|
||||
when(fileTaskMapper.selectById(33890L)).thenReturn(task);
|
||||
|
||||
assertThatThrownBy(() -> service.queryProgress(33890L, 7L))
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessageContaining("任务不存在");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user