feat(jd): 京东店铺重爬覆盖旧数据——isEnd 落 SUCCESS 后清理本次没采到的旧行(SKU/图片级联,0 行不清 B94-01,在飞任务行不动)
This commit is contained in:
@@ -45,6 +45,24 @@ public interface JdProductMapper extends BaseMapper<JdProductEntity> {
|
||||
@Param("syncedFrom") LocalDateTime syncedFrom,
|
||||
@Param("syncedTo") LocalDateTime syncedTo);
|
||||
|
||||
/**
|
||||
* 后管只读页分页查询(synced_at 倒序)。
|
||||
*
|
||||
* <p>{@code offset}/{@code limit} 必须是服务侧钳制后的值——page 钳制防
|
||||
* {@code (page-1)*page_size} 溢出(铁律 15),mapper 不做第二次换算,避免两处各算各的。</p>
|
||||
*/
|
||||
/**
|
||||
* 覆盖清理候选行(重爬覆盖语义,2026-11-15):本任务成功收尾时,同一公司、同一批店铺
|
||||
* (shop_id 取本任务落过的行)、<b>不是本任务写的</b>、且所属任务已不在在飞
|
||||
* (非 PENDING/RUNNING)的商品行 id。删除走 {@code JdProductRowDeleteService#deleteRows}
|
||||
* (SKU/图片行随行级联),本查询只出 id 不删。
|
||||
*
|
||||
* <p>排除在飞任务行:同店铺并发两单时,先收尾的单不得把仍在跑的单已落的行抹掉
|
||||
* (后收尾的单会在自己的收尾时清);空结果(本次全采到)返回空表不删。</p>
|
||||
*/
|
||||
List<Long> selectSupersededProductIds(@Param("companyId") Long companyId,
|
||||
@Param("taskId") Long taskId);
|
||||
|
||||
/**
|
||||
* 后管只读页分页查询(synced_at 倒序)。
|
||||
*
|
||||
|
||||
+42
-3
@@ -36,12 +36,16 @@ import java.util.Map;
|
||||
* 京东采集分片回传落库(v1.4 Phase 2,蓝图 §3.2 回传契约)。
|
||||
*
|
||||
* <p>链路:客户端回传 chunk → 归属解析({@code task.user_id -> users.company_id},读不出跳过
|
||||
* 不回填默认 1,铁律 30)→ 行 upsert(唯一出口 {@link JdProductMapper#upsertBatch},
|
||||
* upsert-only 不 delete,B94-01)→ 图片清单登记(Phase 4:直传完成的 objectKey 逐图
|
||||
* 不回填默认 1,铁律 30)→ 行 upsert(唯一出口 {@link JdProductMapper#upsertBatch})→
|
||||
* 图片清单登记(Phase 4:直传完成的 objectKey 逐图
|
||||
* statObject 对撞落 {@code biz_jd_product_image},唯一出口
|
||||
* {@link JdProductImageIngestService#registerTaskImages})→
|
||||
* scope 台账合并写(断点游标 + 批次号 + 累计行数)→
|
||||
* {@code isEnd=true} 时任务落 SUCCESS。
|
||||
* {@code isEnd=true} 时任务落 SUCCESS 并执行<b>覆盖清理</b>:同一店铺里本次没采到、
|
||||
* 所属任务已不在在飞的旧行整行删掉(用户拍板 2026-11-15:重爬同一店铺直接覆盖旧数据)。
|
||||
* 清理只在收尾时做且本次 {@code itemsReceived=0} 不清(B94-01 教训:载荷读不准的空回传
|
||||
* 不得抹掉存量),在飞任务(PENDING/RUNNING)的行不动,删除本体走
|
||||
* {@link JdProductRowDeleteService#deleteRows}(SKU/图片行级联)。</p>
|
||||
*
|
||||
* <p>回传 payload 键集契约({@code JdIngestPayloadContractTest} 绊线钉住):顶层
|
||||
* {@code batchId / cursor / items / isEnd}(+ {@code error} 失败上报),行内
|
||||
@@ -71,6 +75,7 @@ public class JdResultIngestService {
|
||||
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
private final JdProductImageIngestService jdProductImageIngestService;
|
||||
private final JdProductSkuIngestService jdProductSkuIngestService;
|
||||
private final JdProductRowDeleteService jdProductRowDeleteService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public JdIngestChunkVo submitChunk(Long taskId, JdIngestChunkRequest request, boolean trustedErrorReporter) {
|
||||
@@ -147,6 +152,7 @@ public class JdResultIngestService {
|
||||
int itemsReceived = mergeScopeState(scope, request, request.getItems() == null ? 0 : request.getItems().size());
|
||||
if (Boolean.TRUE.equals(request.getIsEnd())) {
|
||||
finalizeSuccess(task, itemsReceived);
|
||||
purgeSupersededRows(task, companyId, itemsReceived);
|
||||
}
|
||||
JdIngestChunkVo vo = new JdIngestChunkVo();
|
||||
vo.setAccept(true);
|
||||
@@ -213,6 +219,39 @@ public class JdResultIngestService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 覆盖清理(用户拍板 2026-11-15:重爬同一店铺直接覆盖旧数据,采集完成时清)。
|
||||
*
|
||||
* <p>只清「本次没采到」的旧行:同公司、同店铺(shop_id 取本任务落过的行)、
|
||||
* task_id 不是本任务、且所属任务已不在在飞的行(在飞任务如 PENDING/RUNNING 的行不动,
|
||||
* 同店铺并发两单时先收尾的单不得抹掉仍在跑的单的行,后收尾的单会在自己的收尾时清)。
|
||||
* 删除本体走 {@link JdProductRowDeleteService#deleteRows}(SKU/图片行级联,同一事务)。</p>
|
||||
*
|
||||
* <p>两条护栏:① 本次 {@code itemsReceived=0} 不清(B94-01 教训——载荷读不准的空回传
|
||||
* 一旦清库=存量整库抹掉);② 清理失败不推翻采集结果(新数据已完整落库且任务已落
|
||||
* SUCCESS),但必须留痕不静默:error 日志+scope 台账记 lastError。</p>
|
||||
*/
|
||||
private void purgeSupersededRows(FileTaskEntity task, Long companyId, int itemsReceived) {
|
||||
if (itemsReceived <= 0) {
|
||||
log.warn("[{}] 收尾回传 0 行,跳过覆盖清理(B94-01:不清旧) taskId={}", LOG_TAG, task.getId());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
List<Long> staleIds = jdProductMapper.selectSupersededProductIds(companyId, task.getId());
|
||||
if (staleIds.isEmpty()) {
|
||||
log.info("[{}] 覆盖清理无候选行(本次已覆盖全部旧行) taskId={}", LOG_TAG, task.getId());
|
||||
return;
|
||||
}
|
||||
int deleted = jdProductRowDeleteService.deleteRows(staleIds);
|
||||
log.info("[{}] 覆盖清理完成:本次没采到的旧行已删 taskId={} candidates={} deleted={}",
|
||||
LOG_TAG, task.getId(), staleIds.size(), deleted);
|
||||
} catch (Exception ex) {
|
||||
log.error("[{}] 覆盖清理失败(本次采集数据已完整落库,仅旧数据未清,可重爬一次补清) taskId={}",
|
||||
LOG_TAG, task.getId(), ex);
|
||||
recordLastError(task.getId(), "覆盖清理失败(本次采集数据已完整落库,仅旧数据未清,可重爬一次补清)");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- scope 台账:断点游标合并写(铁律 30 / B84-01) ----------
|
||||
|
||||
/**
|
||||
|
||||
@@ -60,6 +60,26 @@
|
||||
synced_at = CURRENT_TIMESTAMP
|
||||
</insert>
|
||||
|
||||
<!-- 覆盖清理候选行(重爬覆盖语义,2026-11-15):本任务成功收尾后,同一公司同一批店铺里
|
||||
不是本任务写、且所属任务已不在在飞(PENDING/RUNNING 不删——同店铺并发两单先收尾的
|
||||
单不得抹掉仍在跑的单的行,后收尾的单会在自己的收尾时清)的商品行。
|
||||
shop_id 取本任务落过的行的 DISTINCT 值(shop_id 缺省回落 shop_name,与 buildRow 同源),
|
||||
删除本体走 JdProductRowDeleteService#deleteRows(SKU/图片行级联),本查询只出 id。 -->
|
||||
<select id="selectSupersededProductIds" resultType="long">
|
||||
SELECT p.id
|
||||
FROM biz_jd_product p
|
||||
WHERE p.company_id = #{companyId}
|
||||
AND p.shop_id IN (
|
||||
SELECT DISTINCT shop_id FROM biz_jd_product
|
||||
WHERE company_id = #{companyId} AND task_id = #{taskId}
|
||||
)
|
||||
AND p.task_id != #{taskId}
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM biz_file_task t
|
||||
WHERE t.id = p.task_id AND t.status IN ('PENDING', 'RUNNING')
|
||||
)
|
||||
</select>
|
||||
|
||||
<!-- 后管只读页(Phase 5,v1.4)过滤面:公司(物化列 eq)、店铺名/商品编码/标题关键词
|
||||
(contains LIKE)、商品状态 eq、分组 eq、同步起止(datetime 闭区间)。
|
||||
全部走 #{} 占位符;offset/limit 必须是服务侧钳制后的值(铁律 15),mapper 不做第二次换算。 -->
|
||||
|
||||
+87
@@ -82,6 +82,8 @@ class JdProductSyncServiceTest {
|
||||
private JdProductImageIngestService jdProductImageIngestService;
|
||||
@Mock
|
||||
private JdProductSkuIngestService jdProductSkuIngestService;
|
||||
@Mock
|
||||
private JdProductRowDeleteService jdProductRowDeleteService;
|
||||
@Spy
|
||||
private ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@@ -317,6 +319,91 @@ class JdProductSyncServiceTest {
|
||||
verify(taskProgressSnapshotService).markTerminal(eq(TASK_ID), eq("JD_SHOP_COLLECT"), eq("SUCCESS"), anyString());
|
||||
}
|
||||
|
||||
// ---------- 覆盖清理(2026-11-15 用户拍板:重爬同一店铺,采集完成时清掉旧数据) ----------
|
||||
|
||||
@Test
|
||||
void 成功收尾后清掉本次没采到的旧行() {
|
||||
stubHappyPath();
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
// 同店铺旧行:上次采集落的、本次没采到的商品行
|
||||
when(jdProductMapper.selectSupersededProductIds(COMPANY_ID, TASK_ID)).thenReturn(List.of(901L, 902L));
|
||||
when(jdProductRowDeleteService.deleteRows(anyList())).thenReturn(2);
|
||||
JdIngestChunkRequest request = chunk(item("10036442942137", "艺考形体服女"));
|
||||
request.setIsEnd(true);
|
||||
|
||||
JdIngestChunkVo vo = service.submitChunk(TASK_ID, request, true);
|
||||
|
||||
assertThat(vo.isAccept()).isTrue();
|
||||
// 删除走行级联唯一出口(SKU/图片行随删),候选 id 原样透传
|
||||
verify(jdProductRowDeleteService).deleteRows(List.of(901L, 902L));
|
||||
// 删除只发生在收尾之后:先落库,后清理
|
||||
org.mockito.InOrder order = inOrder(jdProductMapper, jdProductRowDeleteService);
|
||||
order.verify(jdProductMapper).upsertBatch(anyList());
|
||||
order.verify(jdProductMapper).selectSupersededProductIds(COMPANY_ID, TASK_ID);
|
||||
order.verify(jdProductRowDeleteService).deleteRows(anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 收尾回传0行不清旧_B94_01() {
|
||||
stubHappyPath();
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
// 全程累计 0 行的收尾(载荷读不准的空回传):不清旧——清了=存量整库抹掉(B94-01 实录)
|
||||
TaskScopeStateEntity emptyScope = scopeRow();
|
||||
emptyScope.setStateJson("{\"cursor\":{},\"itemsReceived\":0,\"lastBatchId\":null}");
|
||||
when(collectTaskService.findScopeRow(TASK_ID)).thenReturn(emptyScope);
|
||||
JdIngestChunkRequest request = chunk();
|
||||
request.setItems(List.of());
|
||||
request.setIsEnd(true);
|
||||
|
||||
JdIngestChunkVo vo = service.submitChunk(TASK_ID, request, true);
|
||||
|
||||
assertThat(vo.isAccept()).isTrue();
|
||||
verify(jdProductMapper, never()).selectSupersededProductIds(any(), any());
|
||||
verify(jdProductRowDeleteService, never()).deleteRows(anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 非isEnd分片不触发覆盖清理() {
|
||||
stubHappyPath();
|
||||
|
||||
service.submitChunk(TASK_ID, chunk(item("10036442942137", "艺考形体服女")), true);
|
||||
|
||||
verify(jdProductMapper, never()).selectSupersededProductIds(any(), any());
|
||||
verify(jdProductRowDeleteService, never()).deleteRows(anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 覆盖清理失败不推翻回执且台账留痕() {
|
||||
stubHappyPath();
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
when(jdProductMapper.selectSupersededProductIds(COMPANY_ID, TASK_ID))
|
||||
.thenThrow(new RuntimeException("db down"));
|
||||
JdIngestChunkRequest request = chunk(item("10036442942137", "艺考形体服女"));
|
||||
request.setIsEnd(true);
|
||||
|
||||
JdIngestChunkVo vo = service.submitChunk(TASK_ID, request, true);
|
||||
|
||||
// 新数据已完整落库且任务已 SUCCESS:清理失败不改回执、不炸整批,但必须留痕不静默
|
||||
assertThat(vo.isAccept()).isTrue();
|
||||
verify(jdProductRowDeleteService, never()).deleteRows(anyList());
|
||||
ArgumentCaptor<TaskScopeStateEntity> scopeCaptor = ArgumentCaptor.forClass(TaskScopeStateEntity.class);
|
||||
verify(taskScopeStateMapper, times(2)).updateById(scopeCaptor.capture());
|
||||
assertThat(scopeCaptor.getAllValues().stream().map(TaskScopeStateEntity::getStateJson))
|
||||
.anySatisfy(json -> assertThat(json).contains("覆盖清理失败"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 覆盖清理SQL必须排除在飞任务行_绊线() throws Exception {
|
||||
// 同店铺并发两单:先收尾的单不得把仍在跑(PENDING/RUNNING)的单已落的行抹掉,
|
||||
// 该排除必须钉在 SQL 侧(铁律 20),改回 Java 过滤或删掉守卫即红。
|
||||
String xml = Files.readString(Path.of("src/main/resources/mapper/jd/JdProductMapper.xml"));
|
||||
String purgeSql = xml.substring(xml.indexOf("selectSupersededProductIds"));
|
||||
assertThat(purgeSql)
|
||||
.contains("status IN ('PENDING', 'RUNNING')")
|
||||
.contains("p.task_id != #{taskId}")
|
||||
.contains("SELECT DISTINCT shop_id FROM biz_jd_product");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 京麦登录态失效整单落FAILED且不落库() {
|
||||
lenient().when(fileTaskMapper.selectById(TASK_ID)).thenReturn(runningTask());
|
||||
|
||||
Reference in New Issue
Block a user