diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/jd/mapper/JdProductMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/jd/mapper/JdProductMapper.java index 368402a1c..a9ca955ab 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/jd/mapper/JdProductMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/jd/mapper/JdProductMapper.java @@ -45,6 +45,24 @@ public interface JdProductMapper extends BaseMapper { @Param("syncedFrom") LocalDateTime syncedFrom, @Param("syncedTo") LocalDateTime syncedTo); + /** + * 后管只读页分页查询(synced_at 倒序)。 + * + *

{@code offset}/{@code limit} 必须是服务侧钳制后的值——page 钳制防 + * {@code (page-1)*page_size} 溢出(铁律 15),mapper 不做第二次换算,避免两处各算各的。

+ */ + /** + * 覆盖清理候选行(重爬覆盖语义,2026-11-15):本任务成功收尾时,同一公司、同一批店铺 + * (shop_id 取本任务落过的行)、不是本任务写的、且所属任务已不在在飞 + * (非 PENDING/RUNNING)的商品行 id。删除走 {@code JdProductRowDeleteService#deleteRows} + * (SKU/图片行随行级联),本查询只出 id 不删。 + * + *

排除在飞任务行:同店铺并发两单时,先收尾的单不得把仍在跑的单已落的行抹掉 + * (后收尾的单会在自己的收尾时清);空结果(本次全采到)返回空表不删。

+ */ + List selectSupersededProductIds(@Param("companyId") Long companyId, + @Param("taskId") Long taskId); + /** * 后管只读页分页查询(synced_at 倒序)。 * diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/jd/service/JdResultIngestService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/jd/service/JdResultIngestService.java index 90674e6aa..3ecdc9494 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/jd/service/JdResultIngestService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/jd/service/JdResultIngestService.java @@ -36,12 +36,16 @@ import java.util.Map; * 京东采集分片回传落库(v1.4 Phase 2,蓝图 §3.2 回传契约)。 * *

链路:客户端回传 chunk → 归属解析({@code task.user_id -> users.company_id},读不出跳过 - * 不回填默认 1,铁律 30)→ 行 upsert(唯一出口 {@link JdProductMapper#upsertBatch}, - * upsert-only 不 delete,B94-01)→ 图片清单登记(Phase 4:直传完成的 objectKey 逐图 + * 不回填默认 1,铁律 30)→ 行 upsert(唯一出口 {@link JdProductMapper#upsertBatch})→ + * 图片清单登记(Phase 4:直传完成的 objectKey 逐图 * statObject 对撞落 {@code biz_jd_product_image},唯一出口 * {@link JdProductImageIngestService#registerTaskImages})→ * scope 台账合并写(断点游标 + 批次号 + 累计行数)→ - * {@code isEnd=true} 时任务落 SUCCESS。 + * {@code isEnd=true} 时任务落 SUCCESS 并执行覆盖清理:同一店铺里本次没采到、 + * 所属任务已不在在飞的旧行整行删掉(用户拍板 2026-11-15:重爬同一店铺直接覆盖旧数据)。 + * 清理只在收尾时做且本次 {@code itemsReceived=0} 不清(B94-01 教训:载荷读不准的空回传 + * 不得抹掉存量),在飞任务(PENDING/RUNNING)的行不动,删除本体走 + * {@link JdProductRowDeleteService#deleteRows}(SKU/图片行级联)。

* *

回传 payload 键集契约({@code JdIngestPayloadContractTest} 绊线钉住):顶层 * {@code batchId / cursor / items / isEnd}(+ {@code error} 失败上报),行内 @@ -71,6 +75,7 @@ public class JdResultIngestService { private final TaskProgressSnapshotService taskProgressSnapshotService; private final JdProductImageIngestService jdProductImageIngestService; private final JdProductSkuIngestService jdProductSkuIngestService; + private final JdProductRowDeleteService jdProductRowDeleteService; private final ObjectMapper objectMapper; public JdIngestChunkVo submitChunk(Long taskId, JdIngestChunkRequest request, boolean trustedErrorReporter) { @@ -147,6 +152,7 @@ public class JdResultIngestService { int itemsReceived = mergeScopeState(scope, request, request.getItems() == null ? 0 : request.getItems().size()); if (Boolean.TRUE.equals(request.getIsEnd())) { finalizeSuccess(task, itemsReceived); + purgeSupersededRows(task, companyId, itemsReceived); } JdIngestChunkVo vo = new JdIngestChunkVo(); vo.setAccept(true); @@ -213,6 +219,39 @@ public class JdResultIngestService { } } + /** + * 覆盖清理(用户拍板 2026-11-15:重爬同一店铺直接覆盖旧数据,采集完成时清)。 + * + *

只清「本次没采到」的旧行:同公司、同店铺(shop_id 取本任务落过的行)、 + * task_id 不是本任务、且所属任务已不在在飞的行(在飞任务如 PENDING/RUNNING 的行不动, + * 同店铺并发两单时先收尾的单不得抹掉仍在跑的单的行,后收尾的单会在自己的收尾时清)。 + * 删除本体走 {@link JdProductRowDeleteService#deleteRows}(SKU/图片行级联,同一事务)。

+ * + *

两条护栏:① 本次 {@code itemsReceived=0} 不清(B94-01 教训——载荷读不准的空回传 + * 一旦清库=存量整库抹掉);② 清理失败不推翻采集结果(新数据已完整落库且任务已落 + * SUCCESS),但必须留痕不静默:error 日志+scope 台账记 lastError。

+ */ + private void purgeSupersededRows(FileTaskEntity task, Long companyId, int itemsReceived) { + if (itemsReceived <= 0) { + log.warn("[{}] 收尾回传 0 行,跳过覆盖清理(B94-01:不清旧) taskId={}", LOG_TAG, task.getId()); + return; + } + try { + List staleIds = jdProductMapper.selectSupersededProductIds(companyId, task.getId()); + if (staleIds.isEmpty()) { + log.info("[{}] 覆盖清理无候选行(本次已覆盖全部旧行) taskId={}", LOG_TAG, task.getId()); + return; + } + int deleted = jdProductRowDeleteService.deleteRows(staleIds); + log.info("[{}] 覆盖清理完成:本次没采到的旧行已删 taskId={} candidates={} deleted={}", + LOG_TAG, task.getId(), staleIds.size(), deleted); + } catch (Exception ex) { + log.error("[{}] 覆盖清理失败(本次采集数据已完整落库,仅旧数据未清,可重爬一次补清) taskId={}", + LOG_TAG, task.getId(), ex); + recordLastError(task.getId(), "覆盖清理失败(本次采集数据已完整落库,仅旧数据未清,可重爬一次补清)"); + } + } + // ---------- scope 台账:断点游标合并写(铁律 30 / B84-01) ---------- /** diff --git a/backend-java/src/main/resources/mapper/jd/JdProductMapper.xml b/backend-java/src/main/resources/mapper/jd/JdProductMapper.xml index c762942b7..7645fa90e 100644 --- a/backend-java/src/main/resources/mapper/jd/JdProductMapper.xml +++ b/backend-java/src/main/resources/mapper/jd/JdProductMapper.xml @@ -60,6 +60,26 @@ synced_at = CURRENT_TIMESTAMP + + + diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/jd/service/JdProductSyncServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/jd/service/JdProductSyncServiceTest.java index 8d44d1dda..96973171b 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/jd/service/JdProductSyncServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/jd/service/JdProductSyncServiceTest.java @@ -82,6 +82,8 @@ class JdProductSyncServiceTest { private JdProductImageIngestService jdProductImageIngestService; @Mock private JdProductSkuIngestService jdProductSkuIngestService; + @Mock + private JdProductRowDeleteService jdProductRowDeleteService; @Spy private ObjectMapper objectMapper = new ObjectMapper(); @@ -317,6 +319,91 @@ class JdProductSyncServiceTest { verify(taskProgressSnapshotService).markTerminal(eq(TASK_ID), eq("JD_SHOP_COLLECT"), eq("SUCCESS"), anyString()); } + // ---------- 覆盖清理(2026-11-15 用户拍板:重爬同一店铺,采集完成时清掉旧数据) ---------- + + @Test + void 成功收尾后清掉本次没采到的旧行() { + stubHappyPath(); + when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + // 同店铺旧行:上次采集落的、本次没采到的商品行 + when(jdProductMapper.selectSupersededProductIds(COMPANY_ID, TASK_ID)).thenReturn(List.of(901L, 902L)); + when(jdProductRowDeleteService.deleteRows(anyList())).thenReturn(2); + JdIngestChunkRequest request = chunk(item("10036442942137", "艺考形体服女")); + request.setIsEnd(true); + + JdIngestChunkVo vo = service.submitChunk(TASK_ID, request, true); + + assertThat(vo.isAccept()).isTrue(); + // 删除走行级联唯一出口(SKU/图片行随删),候选 id 原样透传 + verify(jdProductRowDeleteService).deleteRows(List.of(901L, 902L)); + // 删除只发生在收尾之后:先落库,后清理 + org.mockito.InOrder order = inOrder(jdProductMapper, jdProductRowDeleteService); + order.verify(jdProductMapper).upsertBatch(anyList()); + order.verify(jdProductMapper).selectSupersededProductIds(COMPANY_ID, TASK_ID); + order.verify(jdProductRowDeleteService).deleteRows(anyList()); + } + + @Test + void 收尾回传0行不清旧_B94_01() { + stubHappyPath(); + when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + // 全程累计 0 行的收尾(载荷读不准的空回传):不清旧——清了=存量整库抹掉(B94-01 实录) + TaskScopeStateEntity emptyScope = scopeRow(); + emptyScope.setStateJson("{\"cursor\":{},\"itemsReceived\":0,\"lastBatchId\":null}"); + when(collectTaskService.findScopeRow(TASK_ID)).thenReturn(emptyScope); + JdIngestChunkRequest request = chunk(); + request.setItems(List.of()); + request.setIsEnd(true); + + JdIngestChunkVo vo = service.submitChunk(TASK_ID, request, true); + + assertThat(vo.isAccept()).isTrue(); + verify(jdProductMapper, never()).selectSupersededProductIds(any(), any()); + verify(jdProductRowDeleteService, never()).deleteRows(anyList()); + } + + @Test + void 非isEnd分片不触发覆盖清理() { + stubHappyPath(); + + service.submitChunk(TASK_ID, chunk(item("10036442942137", "艺考形体服女")), true); + + verify(jdProductMapper, never()).selectSupersededProductIds(any(), any()); + verify(jdProductRowDeleteService, never()).deleteRows(anyList()); + } + + @Test + void 覆盖清理失败不推翻回执且台账留痕() { + stubHappyPath(); + when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + when(jdProductMapper.selectSupersededProductIds(COMPANY_ID, TASK_ID)) + .thenThrow(new RuntimeException("db down")); + JdIngestChunkRequest request = chunk(item("10036442942137", "艺考形体服女")); + request.setIsEnd(true); + + JdIngestChunkVo vo = service.submitChunk(TASK_ID, request, true); + + // 新数据已完整落库且任务已 SUCCESS:清理失败不改回执、不炸整批,但必须留痕不静默 + assertThat(vo.isAccept()).isTrue(); + verify(jdProductRowDeleteService, never()).deleteRows(anyList()); + ArgumentCaptor scopeCaptor = ArgumentCaptor.forClass(TaskScopeStateEntity.class); + verify(taskScopeStateMapper, times(2)).updateById(scopeCaptor.capture()); + assertThat(scopeCaptor.getAllValues().stream().map(TaskScopeStateEntity::getStateJson)) + .anySatisfy(json -> assertThat(json).contains("覆盖清理失败")); + } + + @Test + void 覆盖清理SQL必须排除在飞任务行_绊线() throws Exception { + // 同店铺并发两单:先收尾的单不得把仍在跑(PENDING/RUNNING)的单已落的行抹掉, + // 该排除必须钉在 SQL 侧(铁律 20),改回 Java 过滤或删掉守卫即红。 + String xml = Files.readString(Path.of("src/main/resources/mapper/jd/JdProductMapper.xml")); + String purgeSql = xml.substring(xml.indexOf("selectSupersededProductIds")); + assertThat(purgeSql) + .contains("status IN ('PENDING', 'RUNNING')") + .contains("p.task_id != #{taskId}") + .contains("SELECT DISTINCT shop_id FROM biz_jd_product"); + } + @Test void 京麦登录态失效整单落FAILED且不落库() { lenient().when(fileTaskMapper.selectById(TASK_ID)).thenReturn(runningTask());