feat(jd): 京东店铺重爬覆盖旧数据——isEnd 落 SUCCESS 后清理本次没采到的旧行(SKU/图片级联,0 行不清 B94-01,在飞任务行不动)

This commit is contained in:
2026-11-15 11:15:44 +08:00
parent 9c8926068d
commit ab6e49b5e2
4 changed files with 167 additions and 3 deletions
@@ -82,6 +82,8 @@ class JdProductSyncServiceTest {
private JdProductImageIngestService jdProductImageIngestService;
@Mock
private JdProductSkuIngestService jdProductSkuIngestService;
@Mock
private JdProductRowDeleteService jdProductRowDeleteService;
@Spy
private ObjectMapper objectMapper = new ObjectMapper();
@@ -317,6 +319,91 @@ class JdProductSyncServiceTest {
verify(taskProgressSnapshotService).markTerminal(eq(TASK_ID), eq("JD_SHOP_COLLECT"), eq("SUCCESS"), anyString());
}
// ---------- 覆盖清理(2026-11-15 用户拍板:重爬同一店铺,采集完成时清掉旧数据) ----------
@Test
void 成功收尾后清掉本次没采到的旧行() {
stubHappyPath();
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
// 同店铺旧行:上次采集落的、本次没采到的商品行
when(jdProductMapper.selectSupersededProductIds(COMPANY_ID, TASK_ID)).thenReturn(List.of(901L, 902L));
when(jdProductRowDeleteService.deleteRows(anyList())).thenReturn(2);
JdIngestChunkRequest request = chunk(item("10036442942137", "艺考形体服女"));
request.setIsEnd(true);
JdIngestChunkVo vo = service.submitChunk(TASK_ID, request, true);
assertThat(vo.isAccept()).isTrue();
// 删除走行级联唯一出口(SKU/图片行随删),候选 id 原样透传
verify(jdProductRowDeleteService).deleteRows(List.of(901L, 902L));
// 删除只发生在收尾之后:先落库,后清理
org.mockito.InOrder order = inOrder(jdProductMapper, jdProductRowDeleteService);
order.verify(jdProductMapper).upsertBatch(anyList());
order.verify(jdProductMapper).selectSupersededProductIds(COMPANY_ID, TASK_ID);
order.verify(jdProductRowDeleteService).deleteRows(anyList());
}
@Test
void 收尾回传0行不清旧_B94_01() {
stubHappyPath();
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
// 全程累计 0 行的收尾(载荷读不准的空回传):不清旧——清了=存量整库抹掉(B94-01 实录)
TaskScopeStateEntity emptyScope = scopeRow();
emptyScope.setStateJson("{\"cursor\":{},\"itemsReceived\":0,\"lastBatchId\":null}");
when(collectTaskService.findScopeRow(TASK_ID)).thenReturn(emptyScope);
JdIngestChunkRequest request = chunk();
request.setItems(List.of());
request.setIsEnd(true);
JdIngestChunkVo vo = service.submitChunk(TASK_ID, request, true);
assertThat(vo.isAccept()).isTrue();
verify(jdProductMapper, never()).selectSupersededProductIds(any(), any());
verify(jdProductRowDeleteService, never()).deleteRows(anyList());
}
@Test
void 非isEnd分片不触发覆盖清理() {
stubHappyPath();
service.submitChunk(TASK_ID, chunk(item("10036442942137", "艺考形体服女")), true);
verify(jdProductMapper, never()).selectSupersededProductIds(any(), any());
verify(jdProductRowDeleteService, never()).deleteRows(anyList());
}
@Test
void 覆盖清理失败不推翻回执且台账留痕() {
stubHappyPath();
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
when(jdProductMapper.selectSupersededProductIds(COMPANY_ID, TASK_ID))
.thenThrow(new RuntimeException("db down"));
JdIngestChunkRequest request = chunk(item("10036442942137", "艺考形体服女"));
request.setIsEnd(true);
JdIngestChunkVo vo = service.submitChunk(TASK_ID, request, true);
// 新数据已完整落库且任务已 SUCCESS:清理失败不改回执、不炸整批,但必须留痕不静默
assertThat(vo.isAccept()).isTrue();
verify(jdProductRowDeleteService, never()).deleteRows(anyList());
ArgumentCaptor<TaskScopeStateEntity> scopeCaptor = ArgumentCaptor.forClass(TaskScopeStateEntity.class);
verify(taskScopeStateMapper, times(2)).updateById(scopeCaptor.capture());
assertThat(scopeCaptor.getAllValues().stream().map(TaskScopeStateEntity::getStateJson))
.anySatisfy(json -> assertThat(json).contains("覆盖清理失败"));
}
@Test
void 覆盖清理SQL必须排除在飞任务行_绊线() throws Exception {
// 同店铺并发两单:先收尾的单不得把仍在跑(PENDING/RUNNING)的单已落的行抹掉,
// 该排除必须钉在 SQL 侧(铁律 20),改回 Java 过滤或删掉守卫即红。
String xml = Files.readString(Path.of("src/main/resources/mapper/jd/JdProductMapper.xml"));
String purgeSql = xml.substring(xml.indexOf("selectSupersededProductIds"));
assertThat(purgeSql)
.contains("status IN ('PENDING', 'RUNNING')")
.contains("p.task_id != #{taskId}")
.contains("SELECT DISTINCT shop_id FROM biz_jd_product");
}
@Test
void 京麦登录态失效整单落FAILED且不落库() {
lenient().when(fileTaskMapper.selectById(TASK_ID)).thenReturn(runningTask());