feat(phase-2): 京东采集任务域——建单/分片回传落库/判活接线
This commit is contained in:
@@ -70,9 +70,10 @@ public final class TaskModuleRegistry {
|
||||
new Module("BRAND", "品牌检测", true, false, true, false, Optional.empty()),
|
||||
// v1.4 京东店铺采集(蓝图 §4 Phase 1):resultFileJob=false——不出 Excel 结果文件,
|
||||
// 商品数据直接落 biz_jd_product(V193),导出由后管做;ageCleanup=true——任务行
|
||||
// 照常按天清理(商品行在 biz_jd_product 独立保留,不被清);判死巡检暂不并入
|
||||
// (Phase 2 落 JdStaleTaskService 时按其调度形态登记,登录态失效判死由它自管)。
|
||||
new Module("JD_SHOP_COLLECT", "京东店铺采集", false, true, false, false, Optional.empty())
|
||||
// 照常按天清理(商品行在 biz_jd_product 独立保留,不被清);Phase 2 判死登记定型:
|
||||
// JdStaleTaskService 自带 @Scheduled(60 分钟超时线,selfScheduledStaleCheck=true,
|
||||
// 同 publish/collect-data 先例),京麦登录态失效判死由回传侧经它落终态。
|
||||
new Module("JD_SHOP_COLLECT", "京东店铺采集", false, true, false, true, Optional.empty())
|
||||
);
|
||||
|
||||
private TaskModuleRegistry() {
|
||||
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
package com.nanri.aiimage.modules.jd.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdCollectTaskCreateRequest;
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdIngestChunkRequest;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdCollectTaskVo;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdIngestChunkVo;
|
||||
import com.nanri.aiimage.modules.jd.service.JdCollectTaskService;
|
||||
import com.nanri.aiimage.modules.jd.service.JdResultIngestService;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* 京东店铺采集任务面(v1.4 Phase 2,蓝图 §3.2)。
|
||||
*
|
||||
* <p>三个入口:建单(用户端登录用户,uid 按会话身份回填)、分片回传(本机采集器直调,
|
||||
* 与货源查询 {@code /result} 同形无鉴权——采集端按 user_id 拉单,行归属以任务 owner 投影)、
|
||||
* 心跳(委托 {@code TaskHeartbeatService} 全局通道,父任务终态时回 {@code alive=false},
|
||||
* 沿用现有心跳语义;客户端下一跳收手)。全局 {@code /api/tasks/{taskId}/heartbeat} 同时保留可用,
|
||||
* 本端点只是给采集器一个模块内聚的 URL。
|
||||
*/
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
@RequestMapping("/api/jd-collect")
|
||||
@Tag(name = "京东店铺采集", description = "京麦后台店铺商品采集:建单 → 本机紫鸟窗口采集 → 分片回传落京东数据库。")
|
||||
public class JdCollectController {
|
||||
|
||||
private final JdCollectTaskService collectTaskService;
|
||||
private final JdResultIngestService ingestService;
|
||||
private final TaskHeartbeatService taskHeartbeatService;
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
private final AdminAuthSupport adminAuthSupport;
|
||||
|
||||
@PostMapping("/tasks")
|
||||
@Operation(summary = "创建京东店铺采集任务", description = "选择店铺 + 商品 Tab(在售/已下架)+ 采集范围建单;"
|
||||
+ "任务落 PENDING、执行通道显式 LOCAL(本机紫鸟执行)。返回 taskId/taskNo/collectChannel。")
|
||||
public ApiResponse<JdCollectTaskVo> createTask(@Valid @RequestBody JdCollectTaskCreateRequest request,
|
||||
HttpServletRequest httpRequest) {
|
||||
// uid 不由请求体说了算:解析不出会话身份就 401(fail-closed,与货源查询同口径)。
|
||||
request.setUserId(adminAuthSupport.requireOwnershipUserId(httpRequest, request.getUserId()));
|
||||
return ApiResponse.success(collectTaskService.createTask(request,
|
||||
adminAuthSupport.currentDeviceId(httpRequest)));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/chunk")
|
||||
@Operation(summary = "提交采集分片回传", description = "采集器按批次回传列表/详情/图片清单;"
|
||||
+ "游标合并写(缺件沿用前值)、行 upsert-only;isEnd=true 任务落 SUCCESS;"
|
||||
+ "error 字段按铁律 31 分档(京麦登录态失效整单 FAILED,非致命只记录)。")
|
||||
public ApiResponse<JdIngestChunkVo> submitChunk(
|
||||
@Parameter(description = "任务 ID", required = true, example = "33890")
|
||||
@PathVariable Long taskId,
|
||||
@RequestBody JdIngestChunkRequest request) {
|
||||
return ApiResponse.success(ingestService.submitChunk(taskId, request));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/heartbeat")
|
||||
@Operation(summary = "上报采集心跳", description = "沿用全局任务心跳语义:任务终态时回 alive=false,"
|
||||
+ "采集端下一跳收手。处理位置一并记入心跳位置表供判死文案使用。")
|
||||
public ApiResponse<TaskHeartbeatVo> heartbeat(
|
||||
@Parameter(description = "任务 ID", required = true, example = "33890")
|
||||
@PathVariable Long taskId,
|
||||
@RequestBody(required = false) TaskHeartbeatRequest request) {
|
||||
taskHeartbeatPositionService.record(taskId, request);
|
||||
return ApiResponse.success(taskHeartbeatService.heartbeat(taskId, request));
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.nanri.aiimage.modules.jd.model.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 京东店铺采集建单请求(蓝图 §3.2 建单契约)。
|
||||
*
|
||||
* <p>用户在用户端「京东采集」页选择店铺 + 商品 Tab(在售/已下架)+ 采集范围并提交。
|
||||
* {@code shopName} 即京麦商家后台的店铺名,采集执行时客户端在已登录京麦的紫鸟窗口内
|
||||
* 按它定位列表页;{@code listTabs} 对应京麦列表页的 Tab({@code ONSALE} 在售中 /
|
||||
* {@code OFFSALE} 已下架,与 V193 {@code list_tab} 列同词表)。
|
||||
*/
|
||||
@Data
|
||||
public class JdCollectTaskCreateRequest {
|
||||
|
||||
/** 提交人 uid(Controller 侧按会话身份回填,请求体自带值不采信)。 */
|
||||
private Long userId;
|
||||
|
||||
/** 京麦店铺名(AC-1:未选店铺时建单被拒并点名原因)。 */
|
||||
private String shopName;
|
||||
|
||||
/** 要采集的商品 Tab(ONSALE/OFFSALE,至少一个)。 */
|
||||
private List<String> listTabs;
|
||||
|
||||
/** 采集范围上限(商品数);0/缺省=不设上限,采完整 Tab。 */
|
||||
private Integer maxProducts;
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
package com.nanri.aiimage.modules.jd.model.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import lombok.Data;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 京东采集分片回传请求(蓝图 §3.2 回传契约)。
|
||||
*
|
||||
* <p>顶层键集 {@code batchId / cursor / items / isEnd}(外加 {@code error} 失败上报),
|
||||
* 行内键集 {@code productCode / summary / detail / images}——键名契约由
|
||||
* {@code JdIngestPayloadContractTest} 以源码绊线钉住,改名必须同轮改判据并在蓝图 §5 登记。
|
||||
*
|
||||
* <p>{@code summary} 是列表页字段(键集见 {@link JdProductSummaryDto});
|
||||
* {@code detail} 是详情页属性全集,服务端<b>原样透传入 {@code attr_json}</b>,不裁剪不转写(AC-3);
|
||||
* {@code images} 只登记键名与去向(tempKey 待位,Phase 4 预签名直传 + 登记对撞时才落
|
||||
* {@code biz_jd_product_image}),本 Phase 不进图片字节。
|
||||
*/
|
||||
@Data
|
||||
public class JdIngestChunkRequest {
|
||||
|
||||
/** 批次号(客户端生成,如 JD20261114-0001),写进行的 collect_batch_id 供断点与差异核对。 */
|
||||
private String batchId;
|
||||
|
||||
/**
|
||||
* 断点游标(如 {@code {"tab":"ONSALE","page":3,"detailDone":120}})。
|
||||
* 可缺:一次请求没带 cursor ≠ 没有游标,服务端<b>沿用前值</b>合并写(铁律 30 / B84-01)。
|
||||
*/
|
||||
private Map<String, Object> cursor;
|
||||
|
||||
/** 本批商品行。 */
|
||||
private List<JdIngestItemDto> items;
|
||||
|
||||
/** true=本任务全部回传完成,任务落 SUCCESS。 */
|
||||
private Boolean isEnd;
|
||||
|
||||
/**
|
||||
* 客户端失败上报(非空时按铁律 31 分档:京麦登录态失效→整单 FAILED;
|
||||
* 代理/LLM 凭据致命词→按 {@code CredentialFatalClassifier} 落终态;
|
||||
* 其余(限流/超时/页面抖动)只记录不判死)。
|
||||
*/
|
||||
private String error;
|
||||
|
||||
/** 单条商品回传行。 */
|
||||
@Data
|
||||
public static class JdIngestItemDto {
|
||||
|
||||
/** 京东商品编码(uk_company_shop_code 三元组之一,缺失的行跳过并计数,不落库)。 */
|
||||
private String productCode;
|
||||
|
||||
/** 列表页字段(typed 键集见 {@link JdProductSummaryDto})。 */
|
||||
private Map<String, Object> summary;
|
||||
|
||||
/** 详情页属性全集(原始 key/label/value,原样入 attr_json)。 */
|
||||
private Map<String, Object> detail;
|
||||
|
||||
/** 图片清单(本 Phase 只消费键名与数量,字节面在 Phase 4 接)。 */
|
||||
private List<JdIngestImageDto> images;
|
||||
}
|
||||
|
||||
/** 图片清单项(对象 key 由服务端预签名面生成,客户端不得自造,Phase 4 生效)。 */
|
||||
@Data
|
||||
public static class JdIngestImageDto {
|
||||
|
||||
/** MAIN / ATTR / DETAIL(V194 image_type 同词表)。 */
|
||||
private String type;
|
||||
|
||||
/** 销售属性名(ATTR 图专用,如「上衣 / 半裙」)。 */
|
||||
private String attrName;
|
||||
|
||||
private Integer sortNo;
|
||||
|
||||
private String objectKey;
|
||||
|
||||
private String sha256;
|
||||
|
||||
private Long bytes;
|
||||
}
|
||||
|
||||
/** 列表页 summary 的 typed 键集(客户端按这些键名回传;未知键忽略不炸批,不在这里展开)。 */
|
||||
@Data
|
||||
@com.fasterxml.jackson.annotation.JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public static class JdProductSummaryDto {
|
||||
|
||||
private String shopId;
|
||||
|
||||
private String shopName;
|
||||
|
||||
private String title;
|
||||
|
||||
private String skuId;
|
||||
|
||||
private String brand;
|
||||
|
||||
private String brandId;
|
||||
|
||||
private String categoryPath;
|
||||
|
||||
private BigDecimal priceMin;
|
||||
|
||||
private BigDecimal priceMax;
|
||||
|
||||
private Integer stockTotal;
|
||||
|
||||
/** 上架中 / 已下架 / 审核中 / 草稿(京麦原值)。 */
|
||||
private String saleStatus;
|
||||
|
||||
/** 采集来源 Tab(ONSALE/OFFSALE)。 */
|
||||
private String listTab;
|
||||
|
||||
/** 京东侧创建时间(yyyy-MM-dd HH:mm:ss)。 */
|
||||
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
|
||||
private String jdCreatedAt;
|
||||
|
||||
/** 京东侧修改时间(yyyy-MM-dd HH:mm:ss)。 */
|
||||
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
|
||||
private String jdUpdatedAt;
|
||||
|
||||
/** 京东详情页 URL(仅作溯源,不作展示源)。 */
|
||||
private String detailUrl;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package com.nanri.aiimage.modules.jd.model.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* 京东店铺采集建单回执(蓝图 §3.2:{@code → 200 { taskId, taskNo, collectChannel }})。
|
||||
*/
|
||||
@Data
|
||||
public class JdCollectTaskVo {
|
||||
|
||||
private Long taskId;
|
||||
|
||||
private String taskNo;
|
||||
|
||||
/**
|
||||
* 执行通道,恒为 {@code LOCAL}——京东采集只能在本机 Windows + 紫鸟窗口执行,
|
||||
* 建单时显式写库(铁律 19:/parse 建单入口必须写通道,不依赖默认值兜底)。
|
||||
*/
|
||||
private String collectChannel;
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.nanri.aiimage.modules.jd.model.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* 京东采集分片回传回执(蓝图 §3.2:{@code → 200 { accept, deduped, upserted, imagesRegistered }})。
|
||||
*
|
||||
* <p>{@code deduped}=本批里缺商品编码/标题无法构成唯一键而跳过的行(服务端留日志点名声);
|
||||
* {@code upserted}=进 {@code biz_jd_product} upsert 的行数(同一商品重复采集刷新原行,行数不增,AC-5);
|
||||
* {@code imagesRegistered}=图片登记数,Phase 2 图片只记 tempKey 待位,恒 0,Phase 4 接预签名直传后生效。
|
||||
*/
|
||||
@Data
|
||||
public class JdIngestChunkVo {
|
||||
|
||||
/** false=本批未被受理(归属缺失 / 凭据致命已落终态),客户端应停止继续提交。 */
|
||||
private boolean accept;
|
||||
|
||||
private int deduped;
|
||||
|
||||
private int upserted;
|
||||
|
||||
private int imagesRegistered;
|
||||
|
||||
/** 点名原因(accept=false 时给采集端与日志看)。 */
|
||||
private String message;
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.nanri.aiimage.modules.jd.service;
|
||||
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.spi.TaskModuleHeartbeatSpi;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 京东店铺采集的心跳 SPI 接线(v1.4 Phase 2)。
|
||||
*
|
||||
* <p>京东模块没有独立的任务缓存(进度由全局心跳位置表、{@code biz_task_scope_state} 台账与
|
||||
* {@code biz_task_progress_snapshot} 承载),所以 {@link #touchHeartbeat} 是语义上的空实现——
|
||||
* 挂这个 Bean 的目的有两个:① 心跳契约测试按注册表校验每个 moduleType 都有 SPI,缺了会红;
|
||||
* ② 显式声明 {@code heartbeatNeedsFullTaskEntity()=false},心跳热路径不捞
|
||||
* {@code request_json}/{@code result_json} 整行(AC-19 同款省钱口径)。
|
||||
* 「父任务终态回 {@code alive=false}」由 {@code TaskHeartbeatService.touchFileTaskIfRunning}
|
||||
* 的 {@code status != RUNNING} 分支免费提供,本类不重复实现。
|
||||
*/
|
||||
@Component
|
||||
public class JdCollectTaskHeartbeatSpi implements TaskModuleHeartbeatSpi {
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return JdCollectTaskService.MODULE_TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touchHeartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
// 京东无模块缓存可刷:进度快照由回传侧与判死侧写,心跳只走 task 侧的通用通道。
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean heartbeatNeedsFullTaskEntity() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
package com.nanri.aiimage.modules.jd.service;
|
||||
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.crypto.digest.DigestUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.CollectRoutingProperties;
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdCollectTaskCreateRequest;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdCollectTaskVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.CollectChannel;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 京东店铺采集建单(v1.4 Phase 2,蓝图 §3.2 建单契约)。
|
||||
*
|
||||
* <p>形状抄货源查询 {@code SimilarAsinTaskService#parseAndCreateTask} 的落库段:
|
||||
* Mapper {@code insert} + taskNo={@code MODULE_TYPE-Snowflake} + 载荷整体进 {@code request_json}。
|
||||
* 三处京东特有:① 执行通道恒 {@code LOCAL} 且<b>建单显式写库</b>(铁律 19:/parse 建单入口必须写
|
||||
* 通道,不依赖 {@code CollectChannel#defaultFor} 的默认兜底);② 建单口<b>不接</b>
|
||||
* {@code requireSubmitSecret}——本模块在 {@code TaskModuleRegistry} 登记为
|
||||
* {@code Optional.empty()}(缺代理在本机执行类是「直连降级」不是注定失败,铁律 27 边界 c,
|
||||
* 黄金测试 {@code JdCollectChannelGoldenTest} 钉死);③ 建单同时落一行
|
||||
* {@code biz_task_scope_state}(scopeKey 绑店铺),回传落库侧的断点游标与批次台账合并写在这行上。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class JdCollectTaskService {
|
||||
|
||||
public static final String MODULE_TYPE = CollectRoutingProperties.MODULE_JD_SHOP_COLLECT;
|
||||
|
||||
private static final String STATUS_PENDING = com.nanri.aiimage.common.task.TaskStatuses.FileTask.PENDING;
|
||||
|
||||
/** 京麦商品列表 Tab 词表(与 V193 list_tab 列同词表)。 */
|
||||
private static final Set<String> VALID_LIST_TABS = Set.of("ONSALE", "OFFSALE");
|
||||
|
||||
/** scope 状态 JSON 的出厂形状:游标为空对象(合并写的"前值"从这里起步)。 */
|
||||
private static final String INITIAL_SCOPE_STATE = "{\"cursor\":{},\"itemsReceived\":0,\"lastBatchId\":null}";
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public JdCollectTaskVo createTask(JdCollectTaskCreateRequest request, String deviceId) {
|
||||
validate(request);
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType(MODULE_TYPE);
|
||||
task.setTaskMode("PYTHON_QUEUE");
|
||||
// 铁律 19:建单入口显式写通道。京东采集只能在本机 Windows + 紫鸟窗口执行,写 LOCAL
|
||||
// 而不是 persistedValue(request.getCollectChannel())——这一类的执行方没有"界面选 ONLINE"的自由度。
|
||||
task.setCollectChannel(CollectChannel.LOCAL.name());
|
||||
task.setStatus(STATUS_PENDING);
|
||||
task.setSourceFileCount(0);
|
||||
task.setSuccessFileCount(0);
|
||||
task.setFailedFileCount(0);
|
||||
task.setCreatedBy("user:" + request.getUserId());
|
||||
task.setUserId(request.getUserId());
|
||||
task.setDeviceId(AdminAuthSupport.normalizeDeviceId(deviceId));
|
||||
task.setCreatedAt(now);
|
||||
task.setUpdatedAt(now);
|
||||
try {
|
||||
task.setRequestJson(objectMapper.writeValueAsString(request));
|
||||
task.setResultJson("{}");
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("序列化建单载荷失败", ex);
|
||||
}
|
||||
fileTaskMapper.insert(task);
|
||||
insertScopeRow(task, request.getShopName(), now);
|
||||
|
||||
JdCollectTaskVo vo = new JdCollectTaskVo();
|
||||
vo.setTaskId(task.getId());
|
||||
vo.setTaskNo(task.getTaskNo());
|
||||
vo.setCollectChannel(task.getCollectChannel());
|
||||
log.info("[jd-collect] 建单成功 taskId={} taskNo={} shopName={} tabs={} userId={} channel={}",
|
||||
task.getId(), task.getTaskNo(), request.getShopName(), request.getListTabs(),
|
||||
request.getUserId(), task.getCollectChannel());
|
||||
return vo;
|
||||
}
|
||||
|
||||
private void validate(JdCollectTaskCreateRequest request) {
|
||||
if (request.getUserId() == null || request.getUserId() <= 0) {
|
||||
throw new BusinessException("user_id 不合法");
|
||||
}
|
||||
if (request.getShopName() == null || request.getShopName().isBlank()) {
|
||||
throw new BusinessException("请选择要采集的京东店铺");
|
||||
}
|
||||
if (request.getListTabs() == null || request.getListTabs().isEmpty()) {
|
||||
throw new BusinessException("请选择要采集的商品 Tab(在售中/已下架)");
|
||||
}
|
||||
for (String tab : request.getListTabs()) {
|
||||
if (tab == null || !VALID_LIST_TABS.contains(tab.trim().toUpperCase(Locale.ROOT))) {
|
||||
throw new BusinessException("商品 Tab 不识别: " + tab + "(只支持 ONSALE/OFFSALE)");
|
||||
}
|
||||
}
|
||||
if (request.getMaxProducts() != null && request.getMaxProducts() < 0) {
|
||||
throw new BusinessException("采集范围上限不能为负数");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 落回传侧要用的 scope 行(幂等:并发建单撞唯一键时复用先到的那行,不顶掉)。
|
||||
* scopeKey 绑店铺,回传侧按 taskId 单 scope 读写断点游标与批次台账。
|
||||
*/
|
||||
private void insertScopeRow(FileTaskEntity task, String shopName, LocalDateTime now) {
|
||||
String scopeKey = "jd:" + shopName.trim();
|
||||
TaskScopeStateEntity scope = new TaskScopeStateEntity();
|
||||
scope.setTaskId(task.getId());
|
||||
scope.setModuleType(MODULE_TYPE);
|
||||
scope.setScopeKey(scopeKey);
|
||||
scope.setScopeHash(DigestUtil.sha256Hex(scopeKey));
|
||||
scope.setStateJson(INITIAL_SCOPE_STATE);
|
||||
scope.setChunkTotal(0);
|
||||
scope.setReceivedChunkCount(0);
|
||||
scope.setCompleted(0);
|
||||
scope.setCreatedAt(now);
|
||||
scope.setUpdatedAt(now);
|
||||
try {
|
||||
taskScopeStateMapper.insert(scope);
|
||||
} catch (DuplicateKeyException ex) {
|
||||
log.info("[jd-collect] scope 行已存在(并发建单/重试),复用先到行 taskId={}", task.getId());
|
||||
}
|
||||
}
|
||||
|
||||
/** 回传侧读建单载荷(拿 shopName 落 shop_name 列);载荷缺失/损坏按异常拒绝回传,不静默兜。 */
|
||||
public JdCollectTaskCreateRequest readCreateRequest(FileTaskEntity task) {
|
||||
try {
|
||||
JdCollectTaskCreateRequest request = objectMapper.readValue(task.getRequestJson(),
|
||||
JdCollectTaskCreateRequest.class);
|
||||
if (request == null || request.getShopName() == null || request.getShopName().isBlank()) {
|
||||
throw new BusinessException("任务载荷缺少 shopName,无法定位店铺");
|
||||
}
|
||||
return request;
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("任务载荷解析失败 taskId=" + task.getId(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/** 供测试与回传侧确认 scope 行形状(按 taskId 单 scope)。 */
|
||||
public TaskScopeStateEntity findScopeRow(Long taskId) {
|
||||
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.last("limit 1"));
|
||||
}
|
||||
}
|
||||
+361
@@ -0,0 +1,361 @@
|
||||
package com.nanri.aiimage.modules.jd.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.mapper.AdminUserMapper;
|
||||
import com.nanri.aiimage.common.model.entity.AdminUserEntity;
|
||||
import com.nanri.aiimage.common.retry.CredentialFatalClassifier;
|
||||
import com.nanri.aiimage.common.task.TaskStatuses;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.jd.mapper.JdProductMapper;
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdCollectTaskCreateRequest;
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdIngestChunkRequest;
|
||||
import com.nanri.aiimage.modules.jd.model.entity.JdProductEntity;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdIngestChunkVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.CredentialFatalTaskCloser;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 京东采集分片回传落库(v1.4 Phase 2,蓝图 §3.2 回传契约)。
|
||||
*
|
||||
* <p>链路:客户端回传 chunk → 归属解析({@code task.user_id -> users.company_id},读不出跳过
|
||||
* 不回填默认 1,铁律 30)→ 行 upsert(唯一出口 {@link JdProductMapper#upsertBatch},
|
||||
* upsert-only 不 delete,B94-01)→ scope 台账合并写(断点游标 + 批次号 + 累计行数)→
|
||||
* {@code isEnd=true} 时任务落 SUCCESS。图片只消费键名与数量(tempKey 待位),
|
||||
* 预签名直传与登记对撞在 Phase 4 接 {@code JdProductImageIngestService}。
|
||||
*
|
||||
* <p>回传 payload 键集契约({@code JdIngestPayloadContractTest} 绊线钉住):顶层
|
||||
* {@code batchId / cursor / items / isEnd}(+ {@code error} 失败上报),行内
|
||||
* {@code productCode / summary / detail / images}——本服务逐键消费,改名必须同轮改判据。
|
||||
*
|
||||
* <p>失败上报按铁律 31 分档:京麦登录态失效(客户端报「京麦未登录 / 登录态失效」)→
|
||||
* 整单 FAILED,心跳对采集端回 {@code alive=false} 停止为注定失败的行继续跑;代理/LLM 凭据
|
||||
* 致命词 → {@link CredentialFatalTaskCloser} 落终态;限流/超时/页面抖动等非致命只记录不判死。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class JdResultIngestService {
|
||||
|
||||
public static final String MODULE_TYPE = JdCollectTaskService.MODULE_TYPE;
|
||||
|
||||
private static final String STATUS_RUNNING = TaskStatuses.FileTask.RUNNING;
|
||||
private static final String STATUS_SUCCESS = TaskStatuses.FileTask.SUCCESS;
|
||||
private static final String LOG_TAG = "jd-collect";
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final JdProductMapper jdProductMapper;
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final AdminUserMapper adminUserMapper;
|
||||
private final JdCollectTaskService collectTaskService;
|
||||
private final JdStaleTaskService jdStaleTaskService;
|
||||
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public JdIngestChunkVo submitChunk(Long taskId, JdIngestChunkRequest request) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
throw new BusinessException("taskId 不合法");
|
||||
}
|
||||
if (request == null) {
|
||||
throw new BusinessException("回传请求体为空");
|
||||
}
|
||||
FileTaskEntity task = requireJdTask(taskId);
|
||||
if (TaskStatuses.FileTask.TERMINAL.contains(task.getStatus())) {
|
||||
throw new BusinessException("任务已结束(" + task.getStatus() + "),回传被拒");
|
||||
}
|
||||
if (request.getError() != null && !request.getError().isBlank()) {
|
||||
return handleErrorReport(task, request.getError().trim());
|
||||
}
|
||||
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||
throw new BusinessException("任务不在运行中(" + task.getStatus() + "),回传被拒");
|
||||
}
|
||||
Long companyId = resolveCompanyId(task);
|
||||
if (companyId == null) {
|
||||
// 铁律 30:读不出归属不得回落默认 1——跳过该任务落库并留日志,让人工先修归属。
|
||||
log.warn("[{}] 归属解析失败(owner userId={} 无 company_id),跳过落库 taskId={}",
|
||||
LOG_TAG, task.getUserId(), taskId);
|
||||
return reject("任务归属缺失,已跳过落库");
|
||||
}
|
||||
JdCollectTaskCreateRequest createRequest = collectTaskService.readCreateRequest(task);
|
||||
TaskScopeStateEntity scope = requireScopeRow(taskId);
|
||||
|
||||
int deduped = 0;
|
||||
int imageCount = 0;
|
||||
List<JdProductEntity> rows = new ArrayList<>();
|
||||
if (request.getItems() != null) {
|
||||
for (JdIngestChunkRequest.JdIngestItemDto item : request.getItems()) {
|
||||
if (item == null || isBlank(item.getProductCode())) {
|
||||
deduped++;
|
||||
log.warn("[{}] 回传行缺商品编码,跳过 taskId={} batchId={}", LOG_TAG, taskId, request.getBatchId());
|
||||
continue;
|
||||
}
|
||||
JdIngestChunkRequest.JdProductSummaryDto summary = toSummary(item.getSummary());
|
||||
if (summary == null || isBlank(summary.getTitle())) {
|
||||
deduped++;
|
||||
log.warn("[{}] 回传行缺列表页标题,跳过 taskId={} productCode={}", LOG_TAG, taskId, item.getProductCode());
|
||||
continue;
|
||||
}
|
||||
rows.add(buildRow(task, companyId, createRequest, item, summary, request.getBatchId(), scope));
|
||||
// 图片本 Phase 只计数占位(tempKey 待位):预签名直传与登记对撞在 Phase 4 落 biz_jd_product_image。
|
||||
imageCount += item.getImages() == null ? 0 : item.getImages().size();
|
||||
}
|
||||
}
|
||||
if (!rows.isEmpty()) {
|
||||
jdProductMapper.upsertBatch(rows);
|
||||
}
|
||||
int itemsReceived = mergeScopeState(scope, request, request.getItems() == null ? 0 : request.getItems().size());
|
||||
if (Boolean.TRUE.equals(request.getIsEnd())) {
|
||||
finalizeSuccess(task, itemsReceived);
|
||||
}
|
||||
JdIngestChunkVo vo = new JdIngestChunkVo();
|
||||
vo.setAccept(true);
|
||||
vo.setDeduped(deduped);
|
||||
vo.setUpserted(rows.size());
|
||||
vo.setImagesRegistered(0);
|
||||
return vo;
|
||||
}
|
||||
|
||||
/** 铁律 31 分档:京麦登录态失效 → 整单终态;代理/LLM 致命 → CredentialFatalTaskCloser;其余只记录。 */
|
||||
private JdIngestChunkVo handleErrorReport(FileTaskEntity task, String error) {
|
||||
if (JdStaleTaskService.isJdLoginStateFailure(error)) {
|
||||
jdStaleTaskService.terminalizeOnJdLoginFatal(task, error);
|
||||
return reject("京麦登录态失效,任务已终止,请在客户端重新登录京麦");
|
||||
}
|
||||
if (CredentialFatalClassifier.isFatal(error)) {
|
||||
new CredentialFatalTaskCloser(LOG_TAG, fileTaskMapper, null).closeIfFatal(task, error);
|
||||
return reject("凭据不可自愈(详见任务失败原因),任务已终止");
|
||||
}
|
||||
// 非致命(限流/超时/页面抖动):只补测不判死,记入 scope 台账后让客户端继续。
|
||||
log.warn("[{}] 非致命失败上报,不判死 taskId={} err={}", LOG_TAG, task.getId(), error);
|
||||
recordLastError(task.getId(), error);
|
||||
JdIngestChunkVo vo = new JdIngestChunkVo();
|
||||
vo.setAccept(true);
|
||||
vo.setMessage("失败上报已记录(非致命)");
|
||||
return vo;
|
||||
}
|
||||
|
||||
private void recordLastError(Long taskId, String error) {
|
||||
TaskScopeStateEntity scope = collectTaskService.findScopeRow(taskId);
|
||||
if (scope == null) {
|
||||
log.warn("[{}] scope 行缺失,失败上报无法记账 taskId={}", LOG_TAG, taskId);
|
||||
return;
|
||||
}
|
||||
Map<String, Object> state = parseState(scope.getStateJson());
|
||||
state.put("lastError", error);
|
||||
scope.setStateJson(writeJson(state));
|
||||
scope.setUpdatedAt(LocalDateTime.now());
|
||||
taskScopeStateMapper.updateById(scope);
|
||||
}
|
||||
|
||||
private void finalizeSuccess(FileTaskEntity task, int itemsReceived) {
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getStatus, STATUS_SUCCESS)
|
||||
.set(FileTaskEntity::getSuccessFileCount, itemsReceived)
|
||||
.set(FileTaskEntity::getFailedFileCount, 0)
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
taskProgressSnapshotService.markTerminal(task.getId(), MODULE_TYPE, STATUS_SUCCESS, "京东采集完成");
|
||||
log.info("[{}] 任务回传完成落 SUCCESS taskId={} itemsReceived={}", LOG_TAG, task.getId(), itemsReceived);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- scope 台账:断点游标合并写(铁律 30 / B84-01) ----------
|
||||
|
||||
/**
|
||||
* 合并写 scope 台账:游标<b>合并</b>不整份覆写(本次请求没带 cursor = 沿用前值),
|
||||
* 批次号与累计行数在此基础上推进。返回累计收到的 items 数(isEnd 时写 successFileCount)。
|
||||
*/
|
||||
private int mergeScopeState(TaskScopeStateEntity scope, JdIngestChunkRequest request, int itemsInBatch) {
|
||||
Map<String, Object> state = parseState(scope.getStateJson());
|
||||
Map<String, Object> previousCursor = asMap(state.get("cursor"));
|
||||
state.put("cursor", mergeCursor(previousCursor, request.getCursor()));
|
||||
if (!isBlank(request.getBatchId())) {
|
||||
state.put("lastBatchId", request.getBatchId().trim());
|
||||
}
|
||||
int itemsReceived = intVal(state.get("itemsReceived")) + itemsInBatch;
|
||||
state.put("itemsReceived", itemsReceived);
|
||||
scope.setStateJson(writeJson(state));
|
||||
scope.setLastChunkAt(LocalDateTime.now());
|
||||
scope.setReceivedChunkCount((scope.getReceivedChunkCount() == null ? 0 : scope.getReceivedChunkCount()) + 1);
|
||||
scope.setUpdatedAt(LocalDateTime.now());
|
||||
taskScopeStateMapper.updateById(scope);
|
||||
return itemsReceived;
|
||||
}
|
||||
|
||||
/**
|
||||
* 游标合并(铁律 30 / B84-01 的判据本体):一次请求没带的键<b>沿用前值</b>,带了按键合并
|
||||
* (同键以本次为准,嵌套 Map 递归合并);值为 null 的键视为「本次没带」,不抹掉前值。
|
||||
* 包私有供 {@code JdProductSyncServiceTest} 直测合并语义。
|
||||
*/
|
||||
static Map<String, Object> mergeCursor(Map<String, Object> previous, Map<String, Object> incoming) {
|
||||
Map<String, Object> merged = previous == null ? new LinkedHashMap<>() : new LinkedHashMap<>(previous);
|
||||
if (incoming == null || incoming.isEmpty()) {
|
||||
return merged;
|
||||
}
|
||||
for (Map.Entry<String, Object> entry : incoming.entrySet()) {
|
||||
if (entry.getValue() == null) {
|
||||
continue;
|
||||
}
|
||||
Object prev = merged.get(entry.getKey());
|
||||
if (entry.getValue() instanceof Map && prev instanceof Map) {
|
||||
merged.put(entry.getKey(), mergeCursor(asMap(prev), asMap(entry.getValue())));
|
||||
} else {
|
||||
merged.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
private Map<String, Object> parseState(String stateJson) {
|
||||
if (stateJson == null || stateJson.isBlank()) {
|
||||
return new LinkedHashMap<>();
|
||||
}
|
||||
try {
|
||||
Map<String, Object> state = objectMapper.readValue(stateJson, STATE_TYPE);
|
||||
return state == null ? new LinkedHashMap<>() : state;
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("scope 台账解析失败(stateJson 损坏)", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private String writeJson(Map<String, Object> value) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(value);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("JSON 序列化失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- 行构建 ----------
|
||||
|
||||
private JdProductEntity buildRow(FileTaskEntity task, Long companyId,
|
||||
JdCollectTaskCreateRequest createRequest,
|
||||
JdIngestChunkRequest.JdIngestItemDto item,
|
||||
JdIngestChunkRequest.JdProductSummaryDto summary,
|
||||
String batchId,
|
||||
TaskScopeStateEntity scope) {
|
||||
JdProductEntity row = new JdProductEntity();
|
||||
row.setCompanyId(companyId);
|
||||
row.setUserId(task.getUserId());
|
||||
row.setTaskId(task.getId());
|
||||
String shopName = firstNonBlank(summary.getShopName(), createRequest.getShopName());
|
||||
row.setShopId(firstNonBlank(summary.getShopId(), shopName));
|
||||
row.setShopName(shopName);
|
||||
row.setProductCode(item.getProductCode().trim());
|
||||
row.setSkuId(summary.getSkuId());
|
||||
row.setTitle(summary.getTitle().trim());
|
||||
row.setBrand(summary.getBrand());
|
||||
row.setBrandId(summary.getBrandId());
|
||||
row.setCategoryPath(summary.getCategoryPath());
|
||||
row.setPriceMin(summary.getPriceMin());
|
||||
row.setPriceMax(summary.getPriceMax());
|
||||
row.setStockTotal(summary.getStockTotal());
|
||||
row.setSaleStatus(summary.getSaleStatus());
|
||||
row.setListTab(firstNonBlank(summary.getListTab(), cursorTab(scope)));
|
||||
row.setJdCreatedAt(parseJdTime(summary.getJdCreatedAt()));
|
||||
row.setJdUpdatedAt(parseJdTime(summary.getJdUpdatedAt()));
|
||||
row.setDetailUrl(summary.getDetailUrl());
|
||||
// detail 原样透传入 attr_json(AC-3 保真防丢键),不裁剪不转写;键数进 attr_count 供质量核查。
|
||||
row.setAttrJson(item.getDetail() == null ? null : writeJson(item.getDetail()));
|
||||
row.setAttrCount(item.getDetail() == null ? 0 : item.getDetail().size());
|
||||
row.setImageCount(0);
|
||||
row.setCollectBatchId(batchId);
|
||||
return row;
|
||||
}
|
||||
|
||||
private JdIngestChunkRequest.JdProductSummaryDto toSummary(Map<String, Object> summary) {
|
||||
if (summary == null || summary.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return objectMapper.convertValue(summary, JdIngestChunkRequest.JdProductSummaryDto.class);
|
||||
} catch (IllegalArgumentException ex) {
|
||||
throw new BusinessException("回传 summary 字段形状不合法: " + ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/** 京东侧时间口径 yyyy-MM-dd HH:mm:ss(容 ISO 的 T 分隔,归一后单格式解析;解析失败直接拒批,不静默置空)。 */
|
||||
private static LocalDateTime parseJdTime(String raw) {
|
||||
if (raw == null || raw.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return LocalDateTime.parse(raw.trim().replace('T', ' '),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
|
||||
}
|
||||
|
||||
private String cursorTab(TaskScopeStateEntity scope) {
|
||||
Map<String, Object> state = parseState(scope.getStateJson());
|
||||
Object tab = asMap(state.get("cursor")).get("tab");
|
||||
return tab == null ? null : String.valueOf(tab);
|
||||
}
|
||||
|
||||
// ---------- 公共小件 ----------
|
||||
|
||||
private FileTaskEntity requireJdTask(Long taskId) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
private TaskScopeStateEntity requireScopeRow(Long taskId) {
|
||||
TaskScopeStateEntity scope = collectTaskService.findScopeRow(taskId);
|
||||
if (scope == null) {
|
||||
throw new BusinessException("任务 scope 台账缺失,无法记账 taskId=" + taskId);
|
||||
}
|
||||
return scope;
|
||||
}
|
||||
|
||||
/** 归属物化:task.user_id -> users.company_id;读不出返回 null(不回填默认 1,铁律 30)。 */
|
||||
private Long resolveCompanyId(FileTaskEntity task) {
|
||||
AdminUserEntity owner = task.getUserId() == null ? null : adminUserMapper.selectById(task.getUserId());
|
||||
return owner == null ? null : owner.getCompanyId();
|
||||
}
|
||||
|
||||
private static JdIngestChunkVo reject(String message) {
|
||||
JdIngestChunkVo vo = new JdIngestChunkVo();
|
||||
vo.setAccept(false);
|
||||
vo.setMessage(message);
|
||||
return vo;
|
||||
}
|
||||
|
||||
private static boolean isBlank(String value) {
|
||||
return value == null || value.isBlank();
|
||||
}
|
||||
|
||||
private static String firstNonBlank(String primary, String fallback) {
|
||||
return isBlank(primary) ? fallback : primary;
|
||||
}
|
||||
|
||||
private static int intVal(Object value) {
|
||||
return value instanceof Number number ? number.intValue() : 0;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static Map<String, Object> asMap(Object value) {
|
||||
return value instanceof Map ? (Map<String, Object>) value : new LinkedHashMap<>();
|
||||
}
|
||||
|
||||
private static final com.fasterxml.jackson.core.type.TypeReference<Map<String, Object>> STATE_TYPE =
|
||||
new com.fasterxml.jackson.core.type.TypeReference<>() {
|
||||
};
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
package com.nanri.aiimage.modules.jd.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.task.TaskStatuses;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.support.TaskFailureLedger;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 京东店铺采集判活(v1.4 Phase 2,蓝图 §4 Phase 2 + 注册表 Phase 1 预留)。
|
||||
*
|
||||
* <p>两件事,登记形态不同:① <b>心跳超时兜底判死</b>——自带 {@code @Scheduled} 巡检
|
||||
* (注册表 {@code selfScheduledStaleCheck=true},同 publish/collect-data 先例:京东单店
|
||||
* 采集节奏慢,60 分钟超时线({@code aiimage.jd.collect.stale-minutes})并入 2 分钟集中巡检线
|
||||
* 没有额外收益,自管判死时延即可);② <b>京麦登录态失效判死</b>——客户端在回传
|
||||
* {@code error} 里报「京麦未登录 / 登录态失效」时由回传侧({@code JdResultIngestService})
|
||||
* 调 {@link #terminalizeOnJdLoginFatal} 整单落终态(铁律 31:凭据不可自愈 ⇒ 行失败 + 整单
|
||||
* 终态,让心跳对采集端回 {@code alive=false} 收手),判死出口集中在本类。
|
||||
*
|
||||
* <p>代理/LLM 凭据致命词的分档判据不在本类——那是 {@code CredentialFatalClassifier} 的公共
|
||||
* 词表,回传侧直接用;本类只认「京麦登录态」这一族京东专属凭据词。
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class JdStaleTaskService {
|
||||
|
||||
public static final String MODULE_TYPE = JdCollectTaskService.MODULE_TYPE;
|
||||
|
||||
private static final String STATUS_RUNNING = TaskStatuses.FileTask.RUNNING;
|
||||
private static final String STATUS_FAILED = TaskStatuses.FileTask.FAILED;
|
||||
private static final String LOG_TAG = "jd-collect";
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final TaskDistributedLockService taskDistributedLockService;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
|
||||
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
|
||||
/** 心跳超时线(分钟),与现有 ONLINE 兜底同口径;下限钳 1。 */
|
||||
@Value("${aiimage.jd.collect.stale-minutes:60}")
|
||||
private long staleMinutes;
|
||||
|
||||
/** 心跳超时兜底判死(自带调度;分布式锁保证 A/B 双实例单跑)。 */
|
||||
@Scheduled(cron = "${aiimage.jd.collect.stale-check-cron:40 */2 * * * *}")
|
||||
public void finalizeStaleTasks() {
|
||||
DistributedJobLockService.LockHandle lock =
|
||||
distributedJobLockService.tryLock("jd-collect:stale-check", java.time.Duration.ofMinutes(10));
|
||||
if (lock == null) {
|
||||
return;
|
||||
}
|
||||
try (lock) {
|
||||
long minutes = Math.max(1L, staleMinutes);
|
||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(minutes);
|
||||
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||
.last("limit 200"));
|
||||
for (FileTaskEntity task : candidates) {
|
||||
if (staleTaskHeartbeatPolicy.isStillQueuedBeforeFirstHeartbeat(
|
||||
task.getId(), 0L, task.getUpdatedAt(), LocalDateTime.now())) {
|
||||
// 首次心跳之前不判死:任务还在客户端队列排队,updated_at 停在认领时刻不是卡死证据。
|
||||
log.info("[{}] 未收到首次心跳、排队宽限内暂不判死 taskId={} updatedAt={} timeoutMinutes={}",
|
||||
LOG_TAG, task.getId(), task.getUpdatedAt(), minutes);
|
||||
continue;
|
||||
}
|
||||
TaskDistributedLockService.LockHandle taskLock =
|
||||
taskDistributedLockService.acquire(MODULE_TYPE, task.getId(), 0L);
|
||||
if (taskLock == null) {
|
||||
continue;
|
||||
}
|
||||
try (taskLock) {
|
||||
failStale(task, minutes);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void failStale(FileTaskEntity task, long minutes) {
|
||||
String reason = "京东采集心跳超时超过 " + minutes + " 分钟,任务已自动失败";
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getErrorMessage, reason)
|
||||
.set(FileTaskEntity::getFailureCategory, TaskFailureLedger.categoryOf(reason))
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
taskProgressSnapshotService.markTerminal(task.getId(), MODULE_TYPE, STATUS_FAILED, reason);
|
||||
log.warn("[{}] 心跳超时改判死 taskId={} updatedAt={} timeoutMinutes={}",
|
||||
LOG_TAG, task.getId(), task.getUpdatedAt(), minutes);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 京麦登录态失效整单判死(铁律 31):CAS RUNNING→FAILED,终态文案点名「京麦未登录」与
|
||||
* 「登录态失效」并给动作句(不写「稍后重试」)。已终态的任务 CAS 自然不命中,可重入。
|
||||
*
|
||||
* @return true=本次真的把任务写成了终态
|
||||
*/
|
||||
public boolean terminalizeOnJdLoginFatal(FileTaskEntity task, String rawError) {
|
||||
if (task == null || task.getId() == null) {
|
||||
return false;
|
||||
}
|
||||
String message = terminalLoginMessage(rawError);
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getSuccessFileCount, 0)
|
||||
.set(FileTaskEntity::getFailedFileCount, 1)
|
||||
.set(FileTaskEntity::getErrorMessage, message)
|
||||
.set(FileTaskEntity::getFailureCategory, TaskFailureLedger.categoryOf(message))
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
taskProgressSnapshotService.markTerminal(task.getId(), MODULE_TYPE, STATUS_FAILED, message);
|
||||
log.warn("[{}] 京麦登录态失效,任务落终态 FAILED taskId={} err={}", LOG_TAG, task.getId(), message);
|
||||
}
|
||||
return updated > 0;
|
||||
}
|
||||
|
||||
/** 终态文案:生产者原文 + 点名两句判据词(「京麦未登录」「登录态失效」都在,AC-10 口径)+ 动作句。 */
|
||||
static String terminalLoginMessage(String rawError) {
|
||||
String raw = rawError == null ? "" : rawError.trim();
|
||||
StringBuilder message = new StringBuilder();
|
||||
if (!raw.isEmpty()) {
|
||||
message.append(raw).append(";");
|
||||
}
|
||||
message.append("京麦未登录(登录态失效),请在客户端重新登录京麦后重试");
|
||||
return message.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 「京麦未登录」这一族京东专属凭据词(客户端回传 error 与判死文案共用同一判据;
|
||||
* 代理/LLM 凭据词不在这里——那两族归 {@code CredentialFatalClassifier})。
|
||||
*/
|
||||
static boolean isJdLoginStateFailure(String message) {
|
||||
if (message == null || message.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
return message.contains("京麦未登录")
|
||||
|| message.contains("登录态失效")
|
||||
|| message.contains("未登录京麦")
|
||||
|| message.contains("京麦登录态");
|
||||
}
|
||||
}
|
||||
-2
@@ -1,6 +1,5 @@
|
||||
package com.nanri.aiimage.modules.jd;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
@@ -25,7 +24,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
* <p>形态为源码绊线:读实现文件断言契约字面量在位。激活时实现类与 Mapper XML
|
||||
* 已存在;键改名必须同轮改本判据并在蓝图 §5 登记。
|
||||
*/
|
||||
@Disabled("Pending Phase 2: JdResultIngestService 与 JdProductMapper.xml 落地后激活")
|
||||
class JdIngestPayloadContractTest {
|
||||
|
||||
private static final Path JAVA_ROOT = Path.of("src", "main", "java", "com", "nanri", "aiimage");
|
||||
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
package com.nanri.aiimage.modules.jd.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdCollectTaskCreateRequest;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdCollectTaskVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* v1.4 Phase 2 建单行为测试:AC-1 的拒绝分支点名 + 铁律 19 的「建单显式写 LOCAL」判据
|
||||
* + 黄金测试钉死的「本模块建单不接 submitSecret 闸门」形状(服务里没有那一调用点,
|
||||
* 由 TaskModuleCoverageTest 的源码扫描兜底)。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class JdCollectTaskServiceTest {
|
||||
|
||||
@Mock
|
||||
private FileTaskMapper fileTaskMapper;
|
||||
@Mock
|
||||
private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Spy
|
||||
private ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@InjectMocks
|
||||
private JdCollectTaskService service;
|
||||
|
||||
private JdCollectTaskCreateRequest validRequest() {
|
||||
JdCollectTaskCreateRequest request = new JdCollectTaskCreateRequest();
|
||||
request.setUserId(7L);
|
||||
request.setShopName("测试旗舰店");
|
||||
request.setListTabs(List.of("ONSALE", "OFFSALE"));
|
||||
request.setMaxProducts(0);
|
||||
return request;
|
||||
}
|
||||
|
||||
@Test
|
||||
void 建单成功且通道显式写LOCAL() {
|
||||
// MP insert 由 KeyGenerator 回填自增 id;mock 里用 Answer 复刻同一行为
|
||||
when(fileTaskMapper.insert(any(FileTaskEntity.class))).thenAnswer(invocation -> {
|
||||
invocation.getArgument(0, FileTaskEntity.class).setId(33890L);
|
||||
return 1;
|
||||
});
|
||||
JdCollectTaskVo vo = service.createTask(validRequest(), "device-abc");
|
||||
|
||||
assertThat(vo.getTaskId()).isNotNull();
|
||||
assertThat(vo.getTaskNo()).startsWith("JD_SHOP_COLLECT-");
|
||||
assertThat(vo.getCollectChannel()).as("铁律 19:建单入口显式写通道,不依赖 defaultFor 兜底")
|
||||
.isEqualTo("LOCAL");
|
||||
ArgumentCaptor<FileTaskEntity> taskCaptor = ArgumentCaptor.forClass(FileTaskEntity.class);
|
||||
verify(fileTaskMapper).insert(taskCaptor.capture());
|
||||
FileTaskEntity task = taskCaptor.getValue();
|
||||
assertThat(task.getModuleType()).isEqualTo("JD_SHOP_COLLECT");
|
||||
assertThat(task.getStatus()).isEqualTo("PENDING");
|
||||
assertThat(task.getUserId()).isEqualTo(7L);
|
||||
assertThat(task.getRequestJson()).contains("测试旗舰店").contains("ONSALE");
|
||||
ArgumentCaptor<TaskScopeStateEntity> scopeCaptor = ArgumentCaptor.forClass(TaskScopeStateEntity.class);
|
||||
verify(taskScopeStateMapper).insert(scopeCaptor.capture());
|
||||
assertThat(scopeCaptor.getValue().getScopeKey()).isEqualTo("jd:测试旗舰店");
|
||||
assertThat(scopeCaptor.getValue().getStateJson()).contains("\"cursor\":{}");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 未选店铺建单被拒并点名() {
|
||||
JdCollectTaskCreateRequest request = validRequest();
|
||||
request.setShopName(" ");
|
||||
assertThatThrownBy(() -> service.createTask(request, null))
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessageContaining("请选择要采集的京东店铺");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 未选Tab建单被拒并点名() {
|
||||
JdCollectTaskCreateRequest request = validRequest();
|
||||
request.setListTabs(List.of());
|
||||
assertThatThrownBy(() -> service.createTask(request, null))
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessageContaining("请选择要采集的商品 Tab");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 不识别的Tab建单被拒() {
|
||||
JdCollectTaskCreateRequest request = validRequest();
|
||||
request.setListTabs(List.of("ALL"));
|
||||
assertThatThrownBy(() -> service.createTask(request, null))
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessageContaining("ONSALE/OFFSALE");
|
||||
verify(fileTaskMapper, never()).insert(any(FileTaskEntity.class));
|
||||
}
|
||||
}
|
||||
+319
@@ -0,0 +1,319 @@
|
||||
package com.nanri.aiimage.modules.jd.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.mapper.AdminUserMapper;
|
||||
import com.nanri.aiimage.common.model.entity.AdminUserEntity;
|
||||
import com.nanri.aiimage.modules.jd.mapper.JdProductMapper;
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdCollectTaskCreateRequest;
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdIngestChunkRequest;
|
||||
import com.nanri.aiimage.modules.jd.model.entity.JdProductEntity;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdIngestChunkVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
/**
|
||||
* v1.4 Phase 2 回传落库行为测试(蓝图 §4 Phase 2 待激活清单:JdProductSyncServiceTest)。
|
||||
*
|
||||
* <p>覆盖蓝图点名的五条判据:覆盖幂等(同一批两次提交行数不增、无 delete)、
|
||||
* synced_at 刷新由 XML UPDATE 段保证、归属读不出跳过且不回填默认 1(铁律 30)、
|
||||
* 空 items 不清空既有行、游标合并写(缺件沿用前值,铁律 30 / B84-01);
|
||||
* 另盖 isEnd 落 SUCCESS 与京麦登录态失效整单判死(铁律 31)。
|
||||
* 夹具走生产接线 {@code submitChunk},不反射私有方法。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class JdProductSyncServiceTest {
|
||||
|
||||
private static final Long TASK_ID = 33890L;
|
||||
private static final Long USER_ID = 7L;
|
||||
private static final Long COMPANY_ID = 2L;
|
||||
|
||||
@Mock
|
||||
private FileTaskMapper fileTaskMapper;
|
||||
@Mock
|
||||
private JdProductMapper jdProductMapper;
|
||||
@Mock
|
||||
private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock
|
||||
private AdminUserMapper adminUserMapper;
|
||||
@Mock
|
||||
private JdCollectTaskService collectTaskService;
|
||||
@Mock
|
||||
private JdStaleTaskService jdStaleTaskService;
|
||||
@Mock
|
||||
private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Spy
|
||||
private ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@InjectMocks
|
||||
private JdResultIngestService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initTableInfo() {
|
||||
// MP 条件构造器在纯单测里必须先初始化实体 TableInfo(CollectDataCredentialFatalTest 同款)
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class);
|
||||
}
|
||||
|
||||
// ---------- 夹具 ----------
|
||||
|
||||
private FileTaskEntity runningTask() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(TASK_ID);
|
||||
task.setModuleType("JD_SHOP_COLLECT");
|
||||
task.setStatus("RUNNING");
|
||||
task.setUserId(USER_ID);
|
||||
task.setRequestJson("{\"userId\":7,\"shopName\":\"测试旗舰店\",\"listTabs\":[\"ONSALE\"],\"maxProducts\":0}");
|
||||
return task;
|
||||
}
|
||||
|
||||
private AdminUserEntity owner(Long companyId) {
|
||||
AdminUserEntity owner = new AdminUserEntity();
|
||||
owner.setId(USER_ID);
|
||||
owner.setCompanyId(companyId);
|
||||
return owner;
|
||||
}
|
||||
|
||||
private TaskScopeStateEntity scopeRow() {
|
||||
TaskScopeStateEntity scope = new TaskScopeStateEntity();
|
||||
scope.setId(1L);
|
||||
scope.setTaskId(TASK_ID);
|
||||
scope.setModuleType("JD_SHOP_COLLECT");
|
||||
scope.setScopeKey("jd:测试旗舰店");
|
||||
scope.setStateJson("{\"cursor\":{\"tab\":\"ONSALE\",\"page\":1},\"itemsReceived\":2,\"lastBatchId\":null}");
|
||||
scope.setReceivedChunkCount(1);
|
||||
return scope;
|
||||
}
|
||||
|
||||
private JdCollectTaskCreateRequest createRequest() {
|
||||
JdCollectTaskCreateRequest request = new JdCollectTaskCreateRequest();
|
||||
request.setUserId(USER_ID);
|
||||
request.setShopName("测试旗舰店");
|
||||
request.setListTabs(List.of("ONSALE"));
|
||||
return request;
|
||||
}
|
||||
|
||||
private JdIngestChunkRequest.JdIngestItemDto item(String productCode, String title) {
|
||||
JdIngestChunkRequest.JdIngestItemDto item = new JdIngestChunkRequest.JdIngestItemDto();
|
||||
item.setProductCode(productCode);
|
||||
Map<String, Object> summary = new LinkedHashMap<>();
|
||||
summary.put("title", title);
|
||||
summary.put("shopId", "shop-001");
|
||||
summary.put("saleStatus", "上架中");
|
||||
summary.put("stockTotal", 2400);
|
||||
summary.put("jdCreatedAt", "2026-11-13 14:06:07");
|
||||
summary.put("unknownFutureKey", "ignored");
|
||||
item.setSummary(summary);
|
||||
Map<String, Object> detail = new LinkedHashMap<>();
|
||||
detail.put("商品属性", Map.of("key", "brand", "label", "品牌", "value", "花屿巷"));
|
||||
detail.put("规格", 2);
|
||||
item.setDetail(detail);
|
||||
JdIngestChunkRequest.JdIngestImageDto image = new JdIngestChunkRequest.JdIngestImageDto();
|
||||
image.setType("MAIN");
|
||||
image.setObjectKey("jd-products/2/33890/abc.jpg");
|
||||
item.setImages(List.of(image));
|
||||
return item;
|
||||
}
|
||||
|
||||
private JdIngestChunkRequest chunk(JdIngestChunkRequest.JdIngestItemDto... items) {
|
||||
JdIngestChunkRequest request = new JdIngestChunkRequest();
|
||||
request.setBatchId("JD20261114-0001");
|
||||
Map<String, Object> cursor = new LinkedHashMap<>();
|
||||
cursor.put("tab", "ONSALE");
|
||||
cursor.put("page", 2);
|
||||
request.setCursor(cursor);
|
||||
request.setItems(List.of(items));
|
||||
return request;
|
||||
}
|
||||
|
||||
private void stubHappyPath() {
|
||||
lenient().when(fileTaskMapper.selectById(TASK_ID)).thenReturn(runningTask());
|
||||
lenient().when(adminUserMapper.selectById(USER_ID)).thenReturn(owner(COMPANY_ID));
|
||||
lenient().when(collectTaskService.readCreateRequest(any(FileTaskEntity.class))).thenReturn(createRequest());
|
||||
lenient().when(collectTaskService.findScopeRow(TASK_ID)).thenReturn(scopeRow());
|
||||
}
|
||||
|
||||
// ---------- 判据 ----------
|
||||
|
||||
@Test
|
||||
void 正常回传按归属落库且不回填默认1() {
|
||||
stubHappyPath();
|
||||
|
||||
JdIngestChunkVo vo = service.submitChunk(TASK_ID,
|
||||
chunk(item("10036442942137", "艺考形体服女"), item("10036442942138", "艺考形体服男")));
|
||||
|
||||
assertThat(vo.isAccept()).isTrue();
|
||||
assertThat(vo.getUpserted()).isEqualTo(2);
|
||||
assertThat(vo.getImagesRegistered()).isZero();
|
||||
@SuppressWarnings("unchecked")
|
||||
ArgumentCaptor<List<JdProductEntity>> captor = ArgumentCaptor.forClass((Class) List.class);
|
||||
verify(jdProductMapper).upsertBatch(captor.capture());
|
||||
List<JdProductEntity> rows = captor.getValue();
|
||||
assertThat(rows).hasSize(2);
|
||||
JdProductEntity first = rows.get(0);
|
||||
assertThat(first.getCompanyId()).as("归属物化按 owner 的 company_id,不回填默认 1(铁律 30)").isEqualTo(COMPANY_ID);
|
||||
assertThat(first.getUserId()).isEqualTo(USER_ID);
|
||||
assertThat(first.getTaskId()).isEqualTo(TASK_ID);
|
||||
assertThat(first.getShopId()).isEqualTo("shop-001");
|
||||
assertThat(first.getShopName()).isEqualTo("测试旗舰店");
|
||||
assertThat(first.getJdCreatedAt()).isEqualTo(LocalDateTime.of(2026, 11, 13, 14, 6, 7));
|
||||
assertThat(first.getAttrCount()).as("detail 键数进 attr_count").isEqualTo(2);
|
||||
assertThat(first.getAttrJson()).as("detail 原样透传,不裁剪").contains("品牌").contains("花屿巷");
|
||||
assertThat(first.getCollectBatchId()).isEqualTo("JD20261114-0001");
|
||||
assertThat(first.getImageCount()).isZero().as("Phase 2 图片只记 tempKey 待位");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 同一批两次提交行数不增且无delete() {
|
||||
stubHappyPath();
|
||||
JdIngestChunkRequest request = chunk(item("10036442942137", "艺考形体服女"));
|
||||
|
||||
service.submitChunk(TASK_ID, request);
|
||||
service.submitChunk(TASK_ID, request);
|
||||
|
||||
verify(jdProductMapper, times(2)).upsertBatch(anyList());
|
||||
verify(jdProductMapper, never()).delete(any(Wrapper.class));
|
||||
verify(jdProductMapper, never()).deleteById(any(java.io.Serializable.class));
|
||||
// 两批进 upsert 的行同键同形(uk_company_shop_code 覆盖最新,AC-5 行数不增)
|
||||
@SuppressWarnings("unchecked")
|
||||
ArgumentCaptor<List<JdProductEntity>> captor = ArgumentCaptor.forClass((Class) List.class);
|
||||
verify(jdProductMapper, times(2)).upsertBatch(captor.capture());
|
||||
List<List<JdProductEntity>> bothBatches = captor.getAllValues();
|
||||
assertThat(bothBatches.get(0)).hasSameSizeAs(bothBatches.get(1));
|
||||
assertThat(bothBatches.get(0).get(0).getProductCode())
|
||||
.as("同一商品重复回传仍是同一唯一键行(覆盖最新)")
|
||||
.isEqualTo(bothBatches.get(1).get(0).getProductCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 二次采集synced_at刷新由XML更新段保证() throws Exception {
|
||||
stubHappyPath();
|
||||
// 二次采集命中的是 upsert 的 UPDATE 分支:XML 显式刷 synced_at = CURRENT_TIMESTAMP
|
||||
//(INSERT 段走列默认值),这条断言钉住刷新语义不被静默删掉。
|
||||
String xml = Files.readString(Path.of("src/main/resources/mapper/jd/JdProductMapper.xml"));
|
||||
assertThat(xml).contains("synced_at = CURRENT_TIMESTAMP");
|
||||
|
||||
service.submitChunk(TASK_ID, chunk(item("10036442942137", "艺考形体服女")));
|
||||
service.submitChunk(TASK_ID, chunk(item("10036442942137", "艺考形体服女")));
|
||||
|
||||
verify(jdProductMapper, times(2)).upsertBatch(anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 归属读不出跳过落库且回执拒绝() {
|
||||
lenient().when(fileTaskMapper.selectById(TASK_ID)).thenReturn(runningTask());
|
||||
lenient().when(adminUserMapper.selectById(USER_ID)).thenReturn(owner(null));
|
||||
|
||||
JdIngestChunkVo vo = service.submitChunk(TASK_ID, chunk(item("10036442942137", "艺考形体服女")));
|
||||
|
||||
assertThat(vo.isAccept()).isFalse();
|
||||
assertThat(vo.getMessage()).contains("归属");
|
||||
verify(jdProductMapper, never()).upsertBatch(anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 空items不清空既有行且游标沿用前值() {
|
||||
stubHappyPath();
|
||||
JdIngestChunkRequest request = chunk();
|
||||
request.setItems(List.of());
|
||||
request.setCursor(null);
|
||||
|
||||
JdIngestChunkVo vo = service.submitChunk(TASK_ID, request);
|
||||
|
||||
assertThat(vo.isAccept()).isTrue();
|
||||
assertThat(vo.getUpserted()).isZero();
|
||||
verify(jdProductMapper, never()).upsertBatch(anyList());
|
||||
ArgumentCaptor<TaskScopeStateEntity> scopeCaptor = ArgumentCaptor.forClass(TaskScopeStateEntity.class);
|
||||
verify(taskScopeStateMapper).updateById(scopeCaptor.capture());
|
||||
assertThat(scopeCaptor.getValue().getStateJson())
|
||||
.contains("\"page\":1")
|
||||
.as("没带 cursor 的请求不得把既有游标抹掉(铁律 30 / B84-01)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 游标合并写_缺件沿用前值_同键以本次为准_嵌套递归() {
|
||||
Map<String, Object> previous = new LinkedHashMap<>();
|
||||
previous.put("tab", "ONSALE");
|
||||
previous.put("page", 1);
|
||||
previous.put("detailDone", 120);
|
||||
|
||||
// 本次没带 cursor = 沿用前值,不是"没有"
|
||||
assertThat(JdResultIngestService.mergeCursor(previous, null))
|
||||
.containsEntry("page", 1).containsEntry("detailDone", 120);
|
||||
// 带了按键合并:同键以本次为准,没带的键保留前值,null 值视为没带
|
||||
Map<String, Object> incoming = new LinkedHashMap<>();
|
||||
incoming.put("page", 3);
|
||||
incoming.put("note", null);
|
||||
assertThat(JdResultIngestService.mergeCursor(previous, incoming))
|
||||
.containsEntry("page", 3).containsEntry("detailDone", 120).doesNotContainKey("note");
|
||||
// 嵌套 Map 递归合并
|
||||
Map<String, Object> prevNested = Map.of("cursor", Map.of("tab", "ONSALE"));
|
||||
Map<String, Object> inNested = Map.of("cursor", Map.of("page", 5));
|
||||
assertThat(JdResultIngestService.mergeCursor(prevNested, inNested))
|
||||
.containsEntry("cursor", Map.of("tab", "ONSALE", "page", 5));
|
||||
}
|
||||
|
||||
@Test
|
||||
void isEnd为真任务落SUCCESS() {
|
||||
stubHappyPath();
|
||||
// CAS 置位成功(mock 默认返 0,这里复刻条件 UPDATE 命中 1 行)
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
JdIngestChunkRequest request = chunk(item("10036442942137", "艺考形体服女"));
|
||||
request.setIsEnd(true);
|
||||
|
||||
service.submitChunk(TASK_ID, request);
|
||||
|
||||
verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
verify(taskProgressSnapshotService).markTerminal(eq(TASK_ID), eq("JD_SHOP_COLLECT"), eq("SUCCESS"), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 京麦登录态失效整单落FAILED且不落库() {
|
||||
lenient().when(fileTaskMapper.selectById(TASK_ID)).thenReturn(runningTask());
|
||||
lenient().when(jdStaleTaskService.terminalizeOnJdLoginFatal(any(FileTaskEntity.class), anyString()))
|
||||
.thenReturn(true);
|
||||
JdIngestChunkRequest request = new JdIngestChunkRequest();
|
||||
request.setBatchId("JD20261114-0002");
|
||||
request.setError("京麦未登录,请重新登录");
|
||||
|
||||
JdIngestChunkVo vo = service.submitChunk(TASK_ID, request);
|
||||
|
||||
assertThat(vo.isAccept()).isFalse();
|
||||
assertThat(vo.getMessage()).contains("京麦");
|
||||
verify(jdProductMapper, never()).upsertBatch(anyList());
|
||||
verify(jdStaleTaskService).terminalizeOnJdLoginFatal(any(FileTaskEntity.class), anyString());
|
||||
}
|
||||
}
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
package com.nanri.aiimage.modules.jd.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatLivenessService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* v1.4 Phase 2 判活行为测试:京麦登录态失效判死(铁律 31,AC-10 文案口径)+
|
||||
* 心跳超时兜底判死的调度入口(分布式锁被占时直接跳过——A/B 双实例单跑契约)。
|
||||
* CAS 落终态用 {@code fileTaskMapper.update} 返回值驱动,不反射私有方法。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class JdStaleTaskServiceTest {
|
||||
|
||||
@Mock
|
||||
private FileTaskMapper fileTaskMapper;
|
||||
@Mock
|
||||
private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock
|
||||
private DistributedJobLockService distributedJobLockService;
|
||||
@Spy
|
||||
private StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy =
|
||||
new StaleTaskHeartbeatPolicy(mock(TaskHeartbeatLivenessService.class));
|
||||
@Mock
|
||||
private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
|
||||
@InjectMocks
|
||||
private JdStaleTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initTableInfo() {
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class);
|
||||
}
|
||||
|
||||
private FileTaskEntity runningTask() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(33890L);
|
||||
task.setModuleType("JD_SHOP_COLLECT");
|
||||
task.setStatus("RUNNING");
|
||||
task.setUserId(7L);
|
||||
task.setUpdatedAt(LocalDateTime.now().minusHours(2));
|
||||
return task;
|
||||
}
|
||||
|
||||
@Test
|
||||
void 京麦登录态词表只认京麦这一族() {
|
||||
assertThat(JdStaleTaskService.isJdLoginStateFailure("京麦未登录,请重新登录")).isTrue();
|
||||
assertThat(JdStaleTaskService.isJdLoginStateFailure("店铺登录态失效")).isTrue();
|
||||
assertThat(JdStaleTaskService.isJdLoginStateFailure("channel_busy")).isFalse();
|
||||
assertThat(JdStaleTaskService.isJdLoginStateFailure("LLM 余额不足")).isFalse();
|
||||
assertThat(JdStaleTaskService.isJdLoginStateFailure(null)).isFalse();
|
||||
assertThat(JdStaleTaskService.isJdLoginStateFailure(" ")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void 终态文案点名京麦并给动作句() {
|
||||
String withPhrase = JdStaleTaskService.terminalLoginMessage("京麦未登录,无法打开商品列表");
|
||||
assertThat(withPhrase).contains("京麦未登录").contains("登录态失效").contains("重新登录京麦");
|
||||
assertThat(withPhrase).doesNotContain("稍后重试");
|
||||
|
||||
// 原文缺判据词时两句判据词仍然点名(AC-10 的文案口径:error_message 含「京麦未登录 / 登录态失效」)
|
||||
String withoutPhrase = JdStaleTaskService.terminalLoginMessage("Cookie 已过期");
|
||||
assertThat(withoutPhrase).contains("Cookie 已过期").contains("京麦未登录").contains("登录态失效");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 登录态失效CAS落终态并同步进度快照() {
|
||||
FileTaskEntity task = runningTask();
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
|
||||
boolean closed = service.terminalizeOnJdLoginFatal(task, "京麦未登录,请重新登录");
|
||||
|
||||
assertThat(closed).isTrue();
|
||||
verify(taskProgressSnapshotService).markTerminal(eq(33890L), eq("JD_SHOP_COLLECT"), eq("FAILED"), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 已终态或CAS不命中时不重复判死() {
|
||||
FileTaskEntity task = runningTask();
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
|
||||
|
||||
boolean closed = service.terminalizeOnJdLoginFatal(task, "京麦未登录");
|
||||
|
||||
assertThat(closed).isFalse();
|
||||
verify(taskProgressSnapshotService, never()).markTerminal(any(), anyString(), anyString(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 分布式锁被占时本轮直接跳过() {
|
||||
when(distributedJobLockService.tryLock(anyString(), any())).thenReturn(null);
|
||||
|
||||
service.finalizeStaleTasks();
|
||||
|
||||
verify(fileTaskMapper, never()).selectList(any());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user