feat(jd): 采集建单支持按京东类目圈范围(多选末级类目,客户端按 categoryPath 等值过滤)——JdCollectTaskCreateRequest 加 categoryPaths(null/空=全店采集,现行为不变),validate 归一化写回请求体再进 request_json(trim/去空/保序去重,上限 50 个/单名 100 字,超限当场点名拒);JdTaskPullSpiImpl.buildQueuePayload 载荷加 categoryPaths 键(缺省空列表,键集稳定,兜底拉取与页面推送同源);JdCollectBoardService 历史行同源带回 categoryPaths(重试/继续从历史行重建推送载荷不丢类目);JdCollectController 新增 GET /api/jd-collect/categories 用户域类目下拉端点(requireOwnershipUserId fail-closed,复用 JdProductQueryService.categories(null) 公共库全量口径,SQL 零新增;从未采集的店铺类目不在集内,先全店采一次即有)。判据:JdCollectTaskServiceTest +3(归一化落 requestJson/超上限拒/全空白视为不过滤)、JdTaskPullSpiImplTest 键集断言加 categoryPaths+透传用例、JdCollectBoardServiceTest 历史行带回断言;三支测试类 16 例全绿,mvn -o test-compile 零错。客户端读参/过滤与用户端建单页另笔(多仓边界)
This commit is contained in:
+16
@@ -12,8 +12,10 @@ import com.nanri.aiimage.modules.jd.model.vo.JdCollectProgressVo;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdCollectTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdCollectTaskVo;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdIngestChunkVo;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdProductCategoryVo;
|
||||
import com.nanri.aiimage.modules.jd.service.JdCollectBoardService;
|
||||
import com.nanri.aiimage.modules.jd.service.JdCollectTaskService;
|
||||
import com.nanri.aiimage.modules.jd.service.JdProductQueryService;
|
||||
import com.nanri.aiimage.modules.jd.service.JdResultIngestService;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
|
||||
@@ -36,6 +38,8 @@ import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 京东店铺采集任务面(v1.4 Phase 2,蓝图 §3.2)。
|
||||
*
|
||||
@@ -55,11 +59,23 @@ public class JdCollectController {
|
||||
private final JdCollectTaskService collectTaskService;
|
||||
private final JdCollectBoardService boardService;
|
||||
private final JdResultIngestService ingestService;
|
||||
private final JdProductQueryService jdProductQueryService;
|
||||
private final TaskHeartbeatService taskHeartbeatService;
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
private final TaskProgressOwnershipSupport progressOwnershipSupport;
|
||||
private final AdminAuthSupport adminAuthSupport;
|
||||
|
||||
@GetMapping("/categories")
|
||||
@Operation(summary = "采集类目下拉数据源", description = "建单页类目多选面板的选项集:公共库已落库行"
|
||||
+ " DISTINCT 的 category_path(京麦末级类目名)+ 行数,按类目名排序。与后管京东数据库"
|
||||
+ "同一条 SQL(JdProductQueryService.categories),公共库无公司维度——传 null 全量口径。"
|
||||
+ "边界:从未采集过的店铺/新类目不在集内(先全店采一次即有),页面选项空态不阻塞建单。")
|
||||
public ApiResponse<List<JdProductCategoryVo>> categories(HttpServletRequest httpRequest) {
|
||||
// 登录用户即可读(公共库口径):解析不出会话身份就 401(fail-closed,与建单同口径)。
|
||||
adminAuthSupport.requireOwnershipUserId(httpRequest, null);
|
||||
return ApiResponse.success(jdProductQueryService.categories(null));
|
||||
}
|
||||
|
||||
@GetMapping("/dashboard")
|
||||
@Operation(summary = "查询京东采集任务总览", description = "当前用户的运行中、成功、失败任务数。"
|
||||
+ "任务面板统计卡数据源,页面进入时请求一次即可,不需要持续轮询。")
|
||||
|
||||
+7
@@ -26,4 +26,11 @@ public class JdCollectTaskCreateRequest {
|
||||
|
||||
/** 采集范围上限(商品数);0/缺省=不设上限,采完整 Tab。 */
|
||||
private Integer maxProducts;
|
||||
|
||||
/**
|
||||
* 要采集的京麦末级类目名(多选,客户端按行 {@code categoryPath} 等值过滤,只对命中商品
|
||||
* 开详情采集);null/空=全店采集。服务端建单口归一化(trim/去空/去重)后写进
|
||||
* {@code request_json},与派发载荷、客户端过滤同源。
|
||||
*/
|
||||
private List<String> categoryPaths;
|
||||
}
|
||||
|
||||
+4
@@ -42,6 +42,10 @@ public class JdCollectHistoryItemVo {
|
||||
@Schema(description = "建单时选择的商品 Tab(ONSALE/OFFSALE)")
|
||||
private List<String> listTabs;
|
||||
|
||||
@Schema(description = "建单时选择的采集类目(京麦末级类目名多选;空=全店采集)。"
|
||||
+ "「继续/重试」从历史行重建推送载荷时同源带回")
|
||||
private List<String> categoryPaths;
|
||||
|
||||
@Schema(description = "已落京东数据库的商品行数(biz_jd_product 按 task_id 计数)", example = "1407")
|
||||
private Integer rowCount;
|
||||
|
||||
|
||||
+4
-1
@@ -149,7 +149,7 @@ public class JdCollectBoardService {
|
||||
return scopes;
|
||||
}
|
||||
|
||||
/** 组装历史行:任务头 + 建单载荷(店铺名/Tab,宽松读) + 行数 + 游标进度。 */
|
||||
/** 组装历史行:任务头 + 建单载荷(店铺名/Tab/类目,宽松读) + 行数 + 游标进度。 */
|
||||
private JdCollectHistoryItemVo toHistoryItem(FileTaskEntity task, Integer rowCount,
|
||||
TaskScopeStateEntity scope) {
|
||||
JdCollectHistoryItemVo item = new JdCollectHistoryItemVo();
|
||||
@@ -162,6 +162,9 @@ public class JdCollectBoardService {
|
||||
if (request != null) {
|
||||
item.setShopName(request.getShopName());
|
||||
item.setListTabs(request.getListTabs());
|
||||
// 类目随建单载荷同源带回:重试/继续时页面从历史行重建推送载荷,缺了它
|
||||
// 客户端会按全店采集跑(兜底拉取路径读 request_json 不受影响)。
|
||||
item.setCategoryPaths(request.getCategoryPaths());
|
||||
}
|
||||
item.setRowCount(rowCount == null ? 0 : rowCount);
|
||||
fillProgress(item, task, scope);
|
||||
|
||||
+40
@@ -22,6 +22,7 @@ import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
@@ -52,6 +53,12 @@ public class JdCollectTaskService {
|
||||
/** 京麦商品列表 Tab 词表(与 V193 list_tab 列同词表)。 */
|
||||
private static final Set<String> VALID_LIST_TABS = Set.of("ONSALE", "OFFSALE");
|
||||
|
||||
/** 类目多选上限(防 request_json/派发载荷膨胀;建单页多选面板正常远到不了这个数)。 */
|
||||
private static final int MAX_CATEGORY_PATHS = 50;
|
||||
|
||||
/** 单个类目名长度上限(京麦末级类目名实际远短于此;拦异常输入)。 */
|
||||
private static final int MAX_CATEGORY_PATH_LENGTH = 100;
|
||||
|
||||
/** scope 状态 JSON 的出厂形状:游标为空对象(合并写的"前值"从这里起步)。 */
|
||||
private static final String INITIAL_SCOPE_STATE = "{\"cursor\":{},\"itemsReceived\":0,\"lastBatchId\":null}";
|
||||
|
||||
@@ -149,6 +156,39 @@ public class JdCollectTaskService {
|
||||
if (request.getMaxProducts() != null && request.getMaxProducts() < 0) {
|
||||
throw new BusinessException("采集范围上限不能为负数");
|
||||
}
|
||||
// 类目归一化写回请求体:request_json 里的值必须就是闸门认可的值(铁律 27a 同源),
|
||||
// 派发载荷与客户端过滤都从这里读,不再各归各的。
|
||||
request.setCategoryPaths(normalizeCategoryPaths(request.getCategoryPaths()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 类目多选归一化:trim/去空/保序去重;全空白归 null(=不过滤,全店采集)。
|
||||
* 超限当场点名拒绝(建单口必须拦注定失败的输入),不静默截断。
|
||||
*/
|
||||
private List<String> normalizeCategoryPaths(List<String> categoryPaths) {
|
||||
if (categoryPaths == null || categoryPaths.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
Set<String> normalized = new java.util.LinkedHashSet<>();
|
||||
for (String raw : categoryPaths) {
|
||||
if (raw == null) {
|
||||
continue;
|
||||
}
|
||||
String path = raw.trim();
|
||||
if (path.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (path.length() > MAX_CATEGORY_PATH_LENGTH) {
|
||||
throw new BusinessException("采集类目名过长(上限 " + MAX_CATEGORY_PATH_LENGTH
|
||||
+ " 字): " + path.substring(0, 50) + "…");
|
||||
}
|
||||
normalized.add(path);
|
||||
}
|
||||
if (normalized.size() > MAX_CATEGORY_PATHS) {
|
||||
throw new BusinessException("采集类目最多选 " + MAX_CATEGORY_PATHS + " 个,当前 "
|
||||
+ normalized.size() + " 个");
|
||||
}
|
||||
return normalized.isEmpty() ? null : List.copyOf(normalized);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+10
-4
@@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
@@ -16,8 +17,9 @@ import java.util.Map;
|
||||
* <p>页面推送缺失(只建单没推送 / 推送前关页面 / 纯浏览器打开)时任务会停在 PENDING,
|
||||
* {@code TaskClientPullService} 认领后由本实现组装可直接入队的载荷。{@code data} 的键集
|
||||
* 与客户端 {@code jd_base.process_task} 的读取契约同键({@code taskId / shopName /
|
||||
* listTabs / maxProducts},{@code QUEUE_TYPE} 与客户端 {@code amazon.jd.jd_base.QUEUE_TYPE}
|
||||
* 同值 {@code jd-shop-collect-run})——两处任意一处漂移,兜底拉下来的任务客户端不识别。</p>
|
||||
* listTabs / maxProducts / categoryPaths},{@code QUEUE_TYPE} 与客户端
|
||||
* {@code amazon.jd.jd_base.QUEUE_TYPE} 同值 {@code jd-shop-collect-run})——两处任意一处
|
||||
* 漂移,兜底拉下来的任务客户端不识别(categoryPaths 客户端读不到=不过滤,全店采集)。</p>
|
||||
*
|
||||
* <p>shopName/listTabs/maxProducts 从建单载荷({@code request_json})回读,与页面推送
|
||||
* 同源;载荷解析失败按 SPI 语义返回 null(调用方会把任务标失败并回写原因),不静默出空载荷。</p>
|
||||
@@ -45,8 +47,12 @@ public class JdTaskPullSpiImpl implements ClientTaskPullSpi {
|
||||
data.put("shopName", create.getShopName());
|
||||
data.put("listTabs", create.getListTabs());
|
||||
data.put("maxProducts", create.getMaxProducts());
|
||||
log.info("[jd-collect] 兜底载荷已组装 taskId={} shopName={} tabs={}",
|
||||
task.getId(), create.getShopName(), create.getListTabs());
|
||||
// 缺省放空列表(不是 null):键集稳定,客户端读空集合=不过滤,语义与缺键一致。
|
||||
data.put("categoryPaths", create.getCategoryPaths() == null
|
||||
? List.of() : create.getCategoryPaths());
|
||||
log.info("[jd-collect] 兜底载荷已组装 taskId={} shopName={} tabs={} categories={}",
|
||||
task.getId(), create.getShopName(), create.getListTabs(),
|
||||
create.getCategoryPaths() == null ? 0 : create.getCategoryPaths().size());
|
||||
return Map.of("type", QUEUE_TYPE, "data", data);
|
||||
}
|
||||
}
|
||||
|
||||
+4
-1
@@ -84,7 +84,7 @@ class JdCollectBoardServiceTest {
|
||||
task.setStatus(status);
|
||||
task.setSuccessFileCount(0);
|
||||
task.setRequestJson("{\"userId\":" + userId + ",\"shopName\":\"花屿巷旗舰店\","
|
||||
+ "\"listTabs\":[\"ONSALE\"],\"maxProducts\":0}");
|
||||
+ "\"listTabs\":[\"ONSALE\"],\"maxProducts\":0,\"categoryPaths\":[\"女装\",\"男装\"]}");
|
||||
task.setCreatedAt(LocalDateTime.of(2026, 11, 15, 10, 0));
|
||||
task.setUpdatedAt(LocalDateTime.of(2026, 11, 15, 10, 1));
|
||||
return task;
|
||||
@@ -132,6 +132,9 @@ class JdCollectBoardServiceTest {
|
||||
assertThat(vo.getItems().get(0).getTaskId()).isEqualTo(34398L);
|
||||
JdCollectHistoryItemVo first = vo.getItems().get(0);
|
||||
assertThat(first.getShopName()).as("店铺名来自建单载荷").isEqualTo("花屿巷旗舰店");
|
||||
assertThat(first.getCategoryPaths())
|
||||
.as("类目随建单载荷同源带回(重试/继续从历史行重建推送载荷用)")
|
||||
.containsExactly("女装", "男装");
|
||||
assertThat(first.getRowCount()).as("行数只来自 biz_jd_product 计数").isEqualTo(120);
|
||||
assertThat(first.getItemsReceived()).isEqualTo(120);
|
||||
assertThat(first.getTotal()).isEqualTo(300);
|
||||
|
||||
+57
@@ -105,4 +105,61 @@ class JdCollectTaskServiceTest {
|
||||
.hasMessageContaining("ONSALE/OFFSALE");
|
||||
verify(fileTaskMapper, never()).insert(any(FileTaskEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 类目多选归一化后落requestJson() throws Exception {
|
||||
when(fileTaskMapper.insert(any(FileTaskEntity.class))).thenAnswer(invocation -> {
|
||||
invocation.getArgument(0, FileTaskEntity.class).setId(33891L);
|
||||
return 1;
|
||||
});
|
||||
JdCollectTaskCreateRequest request = validRequest();
|
||||
// List.of 不收 null:空白元素/重复项用 Arrays.asList 喂(建单页不会发这些形状,
|
||||
// 这里钉的是服务端归一化的兜底)。
|
||||
request.setCategoryPaths(new java.util.ArrayList<>(java.util.Arrays.asList(
|
||||
" 女装 ", "男装", "女装", " ")));
|
||||
|
||||
service.createTask(request, "device-abc");
|
||||
|
||||
ArgumentCaptor<FileTaskEntity> taskCaptor = ArgumentCaptor.forClass(FileTaskEntity.class);
|
||||
verify(fileTaskMapper).insert(taskCaptor.capture());
|
||||
JdCollectTaskCreateRequest stored = new ObjectMapper().readValue(
|
||||
taskCaptor.getValue().getRequestJson(), JdCollectTaskCreateRequest.class);
|
||||
assertThat(stored.getCategoryPaths())
|
||||
.as("request_json 里的类目=归一化后的值(trim/去空/保序去重,铁律 27a 同源)")
|
||||
.containsExactly("女装", "男装");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 超过类目上限建单被拒并点名() {
|
||||
JdCollectTaskCreateRequest request = validRequest();
|
||||
java.util.List<String> tooMany = new java.util.ArrayList<>();
|
||||
for (int i = 0; i < 51; i++) {
|
||||
tooMany.add("类目" + i);
|
||||
}
|
||||
request.setCategoryPaths(tooMany);
|
||||
|
||||
assertThatThrownBy(() -> service.createTask(request, null))
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessageContaining("采集类目最多选 50 个");
|
||||
verify(fileTaskMapper, never()).insert(any(FileTaskEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 类目全空白视为不过滤() throws Exception {
|
||||
when(fileTaskMapper.insert(any(FileTaskEntity.class))).thenAnswer(invocation -> {
|
||||
invocation.getArgument(0, FileTaskEntity.class).setId(33892L);
|
||||
return 1;
|
||||
});
|
||||
JdCollectTaskCreateRequest request = validRequest();
|
||||
request.setCategoryPaths(new java.util.ArrayList<>(java.util.Arrays.asList(" ", "")));
|
||||
|
||||
service.createTask(request, "device-abc");
|
||||
|
||||
ArgumentCaptor<FileTaskEntity> taskCaptor = ArgumentCaptor.forClass(FileTaskEntity.class);
|
||||
verify(fileTaskMapper).insert(taskCaptor.capture());
|
||||
JdCollectTaskCreateRequest stored = new ObjectMapper().readValue(
|
||||
taskCaptor.getValue().getRequestJson(), JdCollectTaskCreateRequest.class);
|
||||
assertThat(stored.getCategoryPaths())
|
||||
.as("全空白类目归 null=全店采集(既有行为,不因新参数收紧)").isNull();
|
||||
}
|
||||
}
|
||||
|
||||
+26
-3
@@ -68,10 +68,33 @@ class JdTaskPullSpiImplTest {
|
||||
assertThat(data.get("shopName")).isEqualTo("测试旗舰店");
|
||||
assertThat(data.get("listTabs")).isEqualTo(List.of("ONSALE", "OFFSALE"));
|
||||
assertThat(data.get("maxProducts")).isEqualTo(0);
|
||||
// 客户端 process_task 只读这四键:多键无害,缺键即空跑——所以逐键钉
|
||||
assertThat(data.get("categoryPaths"))
|
||||
.as("建单没选类目时派空列表(键集稳定,客户端读空=不过滤全店采集)")
|
||||
.isEqualTo(List.of());
|
||||
// 客户端 process_task 只读这五键:多键无害,缺键即空跑——所以逐键钉
|
||||
assertThat(data.keySet())
|
||||
.as("data 键集与客户端 jd_base.process_task 读取契约同键(taskId/shopName/listTabs/maxProducts)")
|
||||
.containsExactlyInAnyOrder("taskId", "shopName", "listTabs", "maxProducts");
|
||||
.as("data 键集与客户端 jd_base.process_task 读取契约同键(taskId/shopName/listTabs/maxProducts/categoryPaths)")
|
||||
.containsExactlyInAnyOrder("taskId", "shopName", "listTabs", "maxProducts", "categoryPaths");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 类目多选随载荷透传客户端() throws Exception {
|
||||
FileTaskEntity withCategories = task();
|
||||
withCategories.setRequestJson("{\"userId\":7,\"shopName\":\"测试旗舰店\","
|
||||
+ "\"listTabs\":[\"ONSALE\"],\"maxProducts\":0,\"categoryPaths\":[\"女装\",\"男装\"]}");
|
||||
when(collectTaskService.readCreateRequest(any(FileTaskEntity.class)))
|
||||
.thenAnswer(invocation -> {
|
||||
FileTaskEntity t = invocation.getArgument(0);
|
||||
return new ObjectMapper().readValue(t.getRequestJson(), JdCollectTaskCreateRequest.class);
|
||||
});
|
||||
|
||||
Map<String, Object> payload = spi.buildQueuePayload(withCategories);
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> data = (Map<String, Object>) payload.get("data");
|
||||
assertThat(data.get("categoryPaths"))
|
||||
.as("类目多选原样透传(客户端按 categoryPath 等值过滤,兜底拉取与页面推送同源)")
|
||||
.isEqualTo(List.of("女装", "男装"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Reference in New Issue
Block a user