fix(phase-6.1): CR 六项修复——派发链三件套/登记顺序/判死口凭据化/脏时间可读/UPDATE 段摘死列
This commit is contained in:
+6
-2
@@ -65,8 +65,12 @@ public class JdCollectController {
|
||||
public ApiResponse<JdIngestChunkVo> submitChunk(
|
||||
@Parameter(description = "任务 ID", required = true, example = "33890")
|
||||
@PathVariable Long taskId,
|
||||
@RequestBody JdIngestChunkRequest request) {
|
||||
return ApiResponse.success(ingestService.submitChunk(taskId, request));
|
||||
@RequestBody JdIngestChunkRequest request,
|
||||
HttpServletRequest httpRequest) {
|
||||
// P1-2 判死口凭据化:error 上报是唯一能触发整单判死的入口,匿名一律拒绝判死
|
||||
// (items 落库仍走 PENDING_CONVERGENCE 既有欠账形状)。可信判定单一出口 isTrustedInternalToken。
|
||||
boolean trustedErrorReporter = adminAuthSupport.isTrustedInternalToken(httpRequest);
|
||||
return ApiResponse.success(ingestService.submitChunk(taskId, request, trustedErrorReporter));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/heartbeat")
|
||||
|
||||
+39
-12
@@ -72,7 +72,7 @@ public class JdResultIngestService {
|
||||
private final JdProductImageIngestService jdProductImageIngestService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public JdIngestChunkVo submitChunk(Long taskId, JdIngestChunkRequest request) {
|
||||
public JdIngestChunkVo submitChunk(Long taskId, JdIngestChunkRequest request, boolean trustedErrorReporter) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
throw new BusinessException("taskId 不合法");
|
||||
}
|
||||
@@ -84,7 +84,7 @@ public class JdResultIngestService {
|
||||
throw new BusinessException("任务已结束(" + task.getStatus() + "),回传被拒");
|
||||
}
|
||||
if (request.getError() != null && !request.getError().isBlank()) {
|
||||
return handleErrorReport(task, request.getError().trim());
|
||||
return handleErrorReport(task, request.getError().trim(), trustedErrorReporter);
|
||||
}
|
||||
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||
throw new BusinessException("任务不在运行中(" + task.getStatus() + "),回传被拒");
|
||||
@@ -102,6 +102,10 @@ public class JdResultIngestService {
|
||||
int deduped = 0;
|
||||
int imageCount = 0;
|
||||
List<JdProductEntity> rows = new ArrayList<>();
|
||||
// 图片登记延后到 upsert 之后(P0-2):registerTaskImages 按 companyId+taskId+productCode
|
||||
// 查商品行,首采行在 upsert 前必然不存在 → 登记会被 100% 跳过、图片永远落不了库。
|
||||
// 这里先收集,行落库后再逐商品登记。
|
||||
List<PendingImageRegistration> pendingImages = new ArrayList<>();
|
||||
if (request.getItems() != null) {
|
||||
for (JdIngestChunkRequest.JdIngestItemDto item : request.getItems()) {
|
||||
if (item == null || isBlank(item.getProductCode())) {
|
||||
@@ -116,15 +120,19 @@ public class JdResultIngestService {
|
||||
continue;
|
||||
}
|
||||
rows.add(buildRow(task, companyId, createRequest, item, summary, request.getBatchId(), scope));
|
||||
// Phase 4:直传完成的图片清单按 statObject 对撞落 biz_jd_product_image(登记链唯一出口
|
||||
// JdProductImageIngestService,uk_product_image 幂等;商品行未落库时跳过不落孤儿行)。
|
||||
imageCount += jdProductImageIngestService.registerTaskImages(taskId, companyId,
|
||||
item.getProductCode().trim(), toImageSpecs(item.getImages()));
|
||||
pendingImages.add(new PendingImageRegistration(item.getProductCode().trim(),
|
||||
toImageSpecs(item.getImages())));
|
||||
}
|
||||
}
|
||||
if (!rows.isEmpty()) {
|
||||
jdProductMapper.upsertBatch(rows);
|
||||
}
|
||||
// Phase 4:直传完成的图片清单按 statObject 对撞落 biz_jd_product_image(登记链唯一出口
|
||||
// JdProductImageIngestService,uk_product_image 幂等)——必须在商品行落库之后执行。
|
||||
for (PendingImageRegistration pending : pendingImages) {
|
||||
imageCount += jdProductImageIngestService.registerTaskImages(taskId, companyId,
|
||||
pending.productCode(), pending.specs());
|
||||
}
|
||||
int itemsReceived = mergeScopeState(scope, request, request.getItems() == null ? 0 : request.getItems().size());
|
||||
if (Boolean.TRUE.equals(request.getIsEnd())) {
|
||||
finalizeSuccess(task, itemsReceived);
|
||||
@@ -137,8 +145,17 @@ public class JdResultIngestService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
/** 铁律 31 分档:京麦登录态失效 → 整单终态;代理/LLM 致命 → CredentialFatalTaskCloser;其余只记录。 */
|
||||
private JdIngestChunkVo handleErrorReport(FileTaskEntity task, String error) {
|
||||
/** 铁律 31 分档:京麦登录态失效 → 整单终态;代理/LLM 致命 → CredentialFatalTaskCloser;其余只记录。
|
||||
*
|
||||
* <p>P1-2:error 上报是唯一能触发整单判死的入口,只认可信内部凭据(X-Internal-Token,
|
||||
* 判定单一出口 isTrustedInternalToken)——匿名 error 一律拒绝判死(不碰任务状态),
|
||||
* 防止枚举 taskId 即可把任意 RUNNING 京东任务判死。</p>
|
||||
*/
|
||||
private JdIngestChunkVo handleErrorReport(FileTaskEntity task, String error, boolean trustedErrorReporter) {
|
||||
if (!trustedErrorReporter) {
|
||||
log.warn("[{}] 匿名失败上报拒绝判死(缺可信内部凭据)taskId={} err={}", LOG_TAG, task.getId(), error);
|
||||
return reject("失败上报需要可信内部凭据(X-Internal-Token),已拒绝判死");
|
||||
}
|
||||
if (JdStaleTaskService.isJdLoginStateFailure(error)) {
|
||||
jdStaleTaskService.terminalizeOnJdLoginFatal(task, error);
|
||||
return reject("京麦登录态失效,任务已终止,请在客户端重新登录京麦");
|
||||
@@ -316,13 +333,18 @@ public class JdResultIngestService {
|
||||
}
|
||||
}
|
||||
|
||||
/** 京东侧时间口径 yyyy-MM-dd HH:mm:ss(容 ISO 的 T 分隔,归一后单格式解析;解析失败直接拒批,不静默置空)。 */
|
||||
private static LocalDateTime parseJdTime(String raw) {
|
||||
/** 京东侧时间口径 yyyy-MM-dd HH:mm:ss(容 ISO 的 T 分隔,归一后单格式解析;解析失败可读拒批,不静默置空)。 */
|
||||
static LocalDateTime parseJdTime(String raw) {
|
||||
if (raw == null || raw.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return LocalDateTime.parse(raw.trim().replace('T', ' '),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
|
||||
String normalized = raw.trim().replace('T', ' ');
|
||||
try {
|
||||
return LocalDateTime.parse(normalized, DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
|
||||
} catch (java.time.format.DateTimeParseException ex) {
|
||||
// 一行脏时间不许让整批回传 500 裸堆栈:抛可读拒绝(原文进 message,不进堆栈页)。
|
||||
throw new BusinessException("京东时间格式不识别(应为 yyyy-MM-dd HH:mm:ss): " + normalized);
|
||||
}
|
||||
}
|
||||
|
||||
private String cursorTab(TaskScopeStateEntity scope) {
|
||||
@@ -383,4 +405,9 @@ public class JdResultIngestService {
|
||||
private static final com.fasterxml.jackson.core.type.TypeReference<Map<String, Object>> STATE_TYPE =
|
||||
new com.fasterxml.jackson.core.type.TypeReference<>() {
|
||||
};
|
||||
|
||||
/** 延后登记的图片明细(P0-2:商品行 upsert 之后才查得到行,登记必须延后)。 */
|
||||
private record PendingImageRegistration(String productCode,
|
||||
List<JdProductImageIngestService.JdIngestImageSpec> specs) {
|
||||
}
|
||||
}
|
||||
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package com.nanri.aiimage.modules.jd.service;
|
||||
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdCollectTaskCreateRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.spi.ClientTaskPullSpi;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 京东店铺采集的客户端兜底拉取实现(v1.4 Phase 6.1 修 P0-1:派发链断头)。
|
||||
*
|
||||
* <p>页面推送缺失(只建单没推送 / 推送前关页面 / 纯浏览器打开)时任务会停在 PENDING,
|
||||
* {@code TaskClientPullService} 认领后由本实现组装可直接入队的载荷。{@code data} 的键集
|
||||
* 与客户端 {@code jd_base.process_task} 的读取契约同键({@code taskId / shopName /
|
||||
* listTabs / maxProducts},{@code QUEUE_TYPE} 与客户端 {@code amazon.jd.jd_base.QUEUE_TYPE}
|
||||
* 同值 {@code jd-shop-collect-run})——两处任意一处漂移,兜底拉下来的任务客户端不识别。</p>
|
||||
*
|
||||
* <p>shopName/listTabs/maxProducts 从建单载荷({@code request_json})回读,与页面推送
|
||||
* 同源;载荷解析失败按 SPI 语义返回 null(调用方会把任务标失败并回写原因),不静默出空载荷。</p>
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class JdTaskPullSpiImpl implements ClientTaskPullSpi {
|
||||
|
||||
/** 与客户端 amazon/jd/jd_base.py 的 QUEUE_TYPE 逐字同值(改这里必须同轮改客户端)。 */
|
||||
public static final String QUEUE_TYPE = "jd-shop-collect-run";
|
||||
|
||||
private final JdCollectTaskService collectTaskService;
|
||||
|
||||
@Override
|
||||
public String moduleType() {
|
||||
return JdCollectTaskService.MODULE_TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> buildQueuePayload(FileTaskEntity task) {
|
||||
JdCollectTaskCreateRequest create = collectTaskService.readCreateRequest(task);
|
||||
Map<String, Object> data = new LinkedHashMap<>();
|
||||
data.put("taskId", task.getId());
|
||||
data.put("shopName", create.getShopName());
|
||||
data.put("listTabs", create.getListTabs());
|
||||
data.put("maxProducts", create.getMaxProducts());
|
||||
log.info("[jd-collect] 兜底载荷已组装 taskId={} shopName={} tabs={}",
|
||||
task.getId(), create.getShopName(), create.getListTabs());
|
||||
return Map.of("type", QUEUE_TYPE, "data", data);
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,10 @@ AIIMAGE_JD_COLLECT_PAGE_SIZE=50
|
||||
AIIMAGE_JD_COLLECT_DETAIL_CONCURRENCY=1
|
||||
AIIMAGE_JD_COLLECT_IMAGE_MAX_PER_PRODUCT=200
|
||||
AIIMAGE_JD_COLLECT_STALE_MINUTES=60
|
||||
# 京东采集的客户端兜底拉取白名单(v1.4 P0-1):默认值已含 JD_SHOP_COLLECT(见 application.yml)。
|
||||
# ⚠ 只有真的要覆盖整个白名单时才设 AIIMAGE_CLIENT_TASK_PULL_MODULE_TYPES——env 变量存在即生效,
|
||||
# 设成空串会把白名单清空(兜底拉取对 JD 形同关闭)。打开 AIIMAGE_CLIENT_TASK_PULL_ENABLED
|
||||
# 前先确认客户端版本带兜底拉取线程。
|
||||
|
||||
# 任务异常现场证据件(v1.2 Phase 3):独立桶与保留期必须显式配置;桶不得复用 nanri-ai-images。
|
||||
AIIMAGE_EVIDENCE_OSS_BUCKET=task-evidence
|
||||
|
||||
@@ -414,7 +414,10 @@ aiimage:
|
||||
enabled: ${AIIMAGE_CLIENT_TASK_PULL_ENABLED:false}
|
||||
# 2026-09-30 外观专利已迁移 Linux 后台执行(内部认领),不再由客户端浏览器兜底执行,
|
||||
# 因此默认白名单移除 APPEARANCE_PATENT,防止未来误开 enabled 时客户端又抢着抓取。
|
||||
module-types: ${AIIMAGE_CLIENT_TASK_PULL_MODULE_TYPES:SIMILAR_ASIN,COLLECT_DATA}
|
||||
# 2026-11-14 v1.4 加 JD_SHOP_COLLECT(P0-1):京东采集只能在本机客户端执行(紫鸟/京麦
|
||||
# 登录态在桌面机),页面推送缺失时任务会永久停在 PENDING——兜底拉取是它的唯一出路;
|
||||
# queuePayload 由 JdTaskPullSpiImpl 组装(type=jd-shop-collect-run,与客户端同值)。
|
||||
module-types: ${AIIMAGE_CLIENT_TASK_PULL_MODULE_TYPES:SIMILAR_ASIN,COLLECT_DATA,JD_SHOP_COLLECT}
|
||||
min-pending-minutes: ${AIIMAGE_CLIENT_TASK_PULL_MIN_PENDING_MINUTES:5}
|
||||
limit: ${AIIMAGE_CLIENT_TASK_PULL_LIMIT:5}
|
||||
# 服务端内部认领(/api/internal/tasks/pull-pending,Linux 后台 worker 专用),
|
||||
|
||||
@@ -7,12 +7,16 @@
|
||||
(R6/AC-5)。后管只读页的分页/筛选 SQL(Phase 5)落同份 XML。 -->
|
||||
|
||||
<!-- 同步链唯一写入口(Phase 2 落 JdResultIngestService):多值批量 upsert,
|
||||
AppearancePatentCleanItemMapper#upsertBatch 同形。刻意不写 company_id/shop_id/
|
||||
AppearancePatentCleanItemMapper#upsertBatch 同形。刻意不写 company_id/
|
||||
product_code 的 UPDATE 段(唯一键三元组自身不随更新漂移);
|
||||
synced_at 在 UPDATE 段显式刷 CURRENT_TIMESTAMP(INSERT 路径走列默认值,
|
||||
显式传 null 会覆盖掉默认值,故两段口径分开)。
|
||||
⚠ upsert-only:回传读出 0 行=什么都不写,不得 delete-then-insert 覆盖式重写
|
||||
(重投递载荷读不准时会抹掉存量,AppearancePatentCleanItemMapper 2026-11-11 实录)。 -->
|
||||
(重投递载荷读不准时会抹掉存量,AppearancePatentCleanItemMapper 2026-11-11 实录)。
|
||||
⚠ UPDATE 段不写 image_count / main_image_url(P1-1):buildRow 恒 0/恒 null,
|
||||
写进 UPDATE 会把登记链 refreshProductImageCount 刷的真实计数抹回 0
|
||||
(后管「图片数」恒 0);image_count 只由登记侧按表实数刷,main_image_url
|
||||
是 P2 死列(BACKLOG B98-01)不随更新。 -->
|
||||
<insert id="upsertBatch">
|
||||
INSERT INTO biz_jd_product
|
||||
(company_id, user_id, task_id, shop_id, shop_name, product_code, sku_id,
|
||||
@@ -44,11 +48,9 @@
|
||||
list_tab = VALUES(list_tab),
|
||||
jd_created_at = VALUES(jd_created_at),
|
||||
jd_updated_at = VALUES(jd_updated_at),
|
||||
main_image_url = VALUES(main_image_url),
|
||||
detail_url = VALUES(detail_url),
|
||||
attr_json = VALUES(attr_json),
|
||||
attr_count = VALUES(attr_count),
|
||||
image_count = VALUES(image_count),
|
||||
collect_batch_id = VALUES(collect_batch_id),
|
||||
synced_at = CURRENT_TIMESTAMP
|
||||
</insert>
|
||||
|
||||
Reference in New Issue
Block a user