From aeaa7f64f47a6991e266ce1c7e50eb40ded86d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Thu, 19 Nov 2026 13:17:56 +0800 Subject: [PATCH] =?UTF-8?q?fix(jd):=20=E5=9B=9E=E4=BC=A0=E9=93=BE=E4=B8=A4?= =?UTF-8?q?=E5=A4=84=200=20=E8=A1=8C=E5=88=A4=E6=AD=BB=E2=80=94=E2=80=94?= =?UTF-8?q?=E7=A9=BA=E5=AE=8C=E6=88=90=E4=B8=8D=E5=86=8D=E8=90=BD=20SUCCES?= =?UTF-8?q?S=E3=80=81=E7=BB=93=E6=9E=84=E5=8F=98=E6=9B=B4=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E7=99=BD=E7=AD=89=2060=20=E5=88=86=E9=92=9F=EF=BC=882?= =?UTF-8?q?026-11-19=2034900/34901/34902=20=E5=AE=9E=E5=BD=95=EF=BC=8C?= =?UTF-8?q?=E9=93=81=E5=BE=8B=2031=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jd/service/JdResultIngestService.java | 56 ++++- .../jd/service/JdStaleTaskService.java | 55 +++++ .../jd/service/JdResultIngestServiceTest.java | 197 ++++++++++++++++++ .../jd/service/JdStaleTaskServiceTest.java | 42 ++++ 4 files changed, 346 insertions(+), 4 deletions(-) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/jd/service/JdResultIngestServiceTest.java diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/jd/service/JdResultIngestService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/jd/service/JdResultIngestService.java index c8df8c144..6d9031491 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/jd/service/JdResultIngestService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/jd/service/JdResultIngestService.java @@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import com.nanri.aiimage.modules.task.service.CredentialFatalTaskCloser; import com.nanri.aiimage.modules.task.service.TaskFinalizeSupport; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import com.nanri.aiimage.modules.task.service.support.TaskFailureLedger; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -164,10 +165,16 @@ public class JdResultIngestService { } int itemsReceived = mergeScopeState(scope, request, request.getItems() == null ? 0 : request.getItems().size()); if (Boolean.TRUE.equals(request.getIsEnd())) { - finalizeSuccess(task, itemsReceived); - // 客户端缺少可验证的完整分页回执,不能据此删除未回传的旧商品。 - log.warn("[{}] 缺少全店分页完整性证明,跳过覆盖清理 taskId={} itemsReceived={}", - LOG_TAG, task.getId(), itemsReceived); + if (itemsReceived <= 0) { + // 空完成判死(2026-11-19 34902 实录):isEnd 且累计 0 行=列表页根本没读出来 + //(没报结构错的空跑同样落这里),照旧落 SUCCESS 会让界面显示「成功」却没有数据。 + finalizeEmptyFailure(task); + } else { + finalizeSuccess(task, itemsReceived); + // 客户端缺少可验证的完整分页回执,不能据此删除未回传的旧商品。 + log.warn("[{}] 缺少全店分页完整性证明,跳过覆盖清理 taskId={} itemsReceived={}", + LOG_TAG, task.getId(), itemsReceived); + } } JdIngestChunkVo vo = new JdIngestChunkVo(); vo.setAccept(true); @@ -197,6 +204,16 @@ public class JdResultIngestService { new CredentialFatalTaskCloser(LOG_TAG, fileTaskMapper, null).closeIfFatal(task, error); return reject("凭据不可自愈(详见任务失败原因),任务已终止"); } + // 非致命档在 0 行时收窄(2026-11-19 34900/34901 实录):「页面结构变更」命中且累计 0 行 + // ⇒ worker 报完即停、心跳随之注销,再挂到心跳超时只会让用户白看 60 分钟「处理中」—— + // 立即判 FAILED(有数据的中途抖动仍走下面的非致命档)。 + if (JdStaleTaskService.isJdPageStructureFailure(error) && receivedRows(task.getId()) == 0) { + jdStaleTaskService.terminalizeOnJdStructureFatal(task, error); + JdIngestChunkVo vo = new JdIngestChunkVo(); + vo.setAccept(true); + vo.setMessage("页面结构变更且未采到任何商品,任务已终止"); + return vo; + } // 非致命(限流/超时/页面抖动):只补测不判死,记入 scope 台账后让客户端继续。 log.warn("[{}] 非致命失败上报,不判死 taskId={} err={}", LOG_TAG, task.getId(), error); recordLastError(task.getId(), error); @@ -206,6 +223,15 @@ public class JdResultIngestService { return vo; } + /** 累计已收行数(scope 台账 itemsReceived);台账缺失按 0 读——首包前就报错的失败正是要判死的形状。 */ + private int receivedRows(Long taskId) { + TaskScopeStateEntity scope = collectTaskService.findScopeRow(taskId); + if (scope == null) { + return 0; + } + return intVal(parseState(scope.getStateJson()).get("itemsReceived")); + } + private void recordLastError(Long taskId, String error) { TaskScopeStateEntity scope = collectTaskService.findScopeRow(taskId); if (scope == null) { @@ -234,6 +260,28 @@ public class JdResultIngestService { } } + private static final String STATUS_FAILED = TaskStatuses.FileTask.FAILED; + + /** 空完成判死:isEnd 且累计 0 行,CAS RUNNING→FAILED 可重入;文案给动作句不写「稍后重试」。 */ + private void finalizeEmptyFailure(FileTaskEntity task) { + String message = "京东采集结束但未采到任何商品数据(0 行):通常是京麦未登录或商品列表页未能打开," + + "请核对采集浏览器里的京麦登录态后重新采集"; + int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(FileTaskEntity::getId, task.getId()) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .set(FileTaskEntity::getStatus, STATUS_FAILED) + .set(FileTaskEntity::getSuccessFileCount, 0) + .set(FileTaskEntity::getFailedFileCount, 1) + .set(FileTaskEntity::getErrorMessage, message) + .set(FileTaskEntity::getFailureCategory, TaskFailureLedger.categoryOf(message)) + .set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()) + .set(FileTaskEntity::getFinishedAt, LocalDateTime.now())); + if (updated > 0) { + taskProgressSnapshotService.markTerminal(task.getId(), MODULE_TYPE, STATUS_FAILED, message); + log.warn("[{}] 回传结束但未采到任何商品,落 FAILED taskId={}", LOG_TAG, task.getId()); + } + } + // ---------- V199 保护闸:人工编辑锁与删除忽略 ---------- /** diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/jd/service/JdStaleTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/jd/service/JdStaleTaskService.java index f8a52b1c1..7b726df21 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/jd/service/JdStaleTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/jd/service/JdStaleTaskService.java @@ -164,4 +164,59 @@ public class JdStaleTaskService { || message.contains("未登录京麦") || message.contains("京麦登录态"); } + + /** + * 京东页面结构变更整单判死(铁律 31,2026-11-19 34900/34901 实录):「列表页点不到 Tab」 + * 发生在首包之前时任务一件数据都没有,worker 报完 error 即停、心跳随之注销 ⇒ 原来只能挂到 + * 心跳超时(60 分钟)才死,用户全程只看「处理中」。回传侧 + * ({@code JdResultIngestService})判「结构变更词命中且累计 0 行」时调本出口立即落 FAILED; + * 有数据的中途抖动仍走非致命档。CAS RUNNING→FAILED 可重入。 + * + * @return true=本次真的把任务写成了终态 + */ + public boolean terminalizeOnJdStructureFatal(FileTaskEntity task, String rawError) { + if (task == null || task.getId() == null) { + return false; + } + String message = terminalStructureMessage(rawError); + int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(FileTaskEntity::getId, task.getId()) + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .set(FileTaskEntity::getStatus, STATUS_FAILED) + .set(FileTaskEntity::getSuccessFileCount, 0) + .set(FileTaskEntity::getFailedFileCount, 1) + .set(FileTaskEntity::getErrorMessage, message) + .set(FileTaskEntity::getFailureCategory, TaskFailureLedger.categoryOf(message)) + .set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()) + .set(FileTaskEntity::getFinishedAt, LocalDateTime.now())); + if (updated > 0) { + taskProgressSnapshotService.markTerminal(task.getId(), MODULE_TYPE, STATUS_FAILED, message); + log.warn("[{}] 页面结构变更且未采到任何数据,任务落终态 FAILED taskId={} err={}", + LOG_TAG, task.getId(), message); + } + return updated > 0; + } + + /** 结构变更终态文案:生产者原文 + 动作句(不写「稍后重试」)。 */ + static String terminalStructureMessage(String rawError) { + String raw = rawError == null ? "" : rawError.trim(); + StringBuilder message = new StringBuilder(); + if (!raw.isEmpty()) { + message.append(raw).append(";"); + } + message.append("请确认采集浏览器里京麦已登录、商品列表页能正常打开后重新采集"); + return message.toString(); + } + + /** + * 「京东页面结构变更」这一族(唯一出处=客户端 {@code structure_change_message} 的固定前缀, + * 回传 error 与判死共用同一判据;登录态/代理/LLM 凭据词各有自己的族,不在这里)。 + */ + static boolean isJdPageStructureFailure(String message) { + if (message == null || message.isBlank()) { + return false; + } + return message.contains("京东页面结构变更"); + } } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/jd/service/JdResultIngestServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/jd/service/JdResultIngestServiceTest.java new file mode 100644 index 000000000..c1ee7b241 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/jd/service/JdResultIngestServiceTest.java @@ -0,0 +1,197 @@ +package com.nanri.aiimage.modules.jd.service; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.mapper.AdminUserMapper; +import com.nanri.aiimage.common.model.entity.AdminUserEntity; +import com.nanri.aiimage.modules.jd.mapper.JdProductIgnoreMapper; +import com.nanri.aiimage.modules.jd.mapper.JdProductMapper; +import com.nanri.aiimage.modules.jd.model.dto.JdIngestChunkRequest; +import com.nanri.aiimage.modules.jd.model.vo.JdIngestChunkVo; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 京东采集回传判定行为测试(2026-11-19 34900/34901/34902 实录的两档收口): + * + *

① 「页面结构变更」失败上报在累计 0 行时立即判 FAILED(原非致命档会让 worker 停住 + * 后挂到心跳超时 60 分钟,用户全程只看「处理中」);有数据的中途抖动保持非致命。 + * ② {@code isEnd=true} 且累计 0 行落 FAILED 不再落 SUCCESS(34902 形状:界面显示「成功」 + * 却没有任何商品数据)。匿名 error 上报照旧拒绝判死。

+ * + *

CAS 落终态用 {@code fileTaskMapper.update} 返回值驱动,不反射私有方法。

+ */ +@ExtendWith(MockitoExtension.class) +class JdResultIngestServiceTest { + + private static final String STRUCTURE_ERROR = + "京东页面结构变更(京东采集):列表页点不到「售卖中」Tab(30 秒内未渲染或点击后元素失效)" + + ";请重跑京东采集取证并校准解析层(docs/jd-collect-api-contract.md)"; + + @Mock + private com.nanri.aiimage.modules.task.mapper.FileTaskMapper fileTaskMapper; + @Mock + private JdProductMapper jdProductMapper; + @Mock + private JdProductIgnoreMapper jdProductIgnoreMapper; + @Mock + private TaskScopeStateMapper taskScopeStateMapper; + @Mock + private AdminUserMapper adminUserMapper; + @Mock + private JdCollectTaskService collectTaskService; + @Mock + private JdStaleTaskService jdStaleTaskService; + @Mock + private TaskProgressSnapshotService taskProgressSnapshotService; + @Mock + private JdProductImageIngestService jdProductImageIngestService; + @Mock + private JdProductSkuIngestService jdProductSkuIngestService; + @Mock + private JdProductRowDeleteService jdProductRowDeleteService; + + private JdResultIngestService service; + + @BeforeAll + static void initTableInfo() { + TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class); + } + + @BeforeEach + void setUp() { + service = new JdResultIngestService(fileTaskMapper, jdProductMapper, jdProductIgnoreMapper, + taskScopeStateMapper, adminUserMapper, collectTaskService, jdStaleTaskService, + taskProgressSnapshotService, jdProductImageIngestService, jdProductSkuIngestService, + jdProductRowDeleteService, new ObjectMapper()); + } + + private FileTaskEntity runningTask() { + FileTaskEntity task = new FileTaskEntity(); + task.setId(88L); + task.setModuleType("JD_SHOP_COLLECT"); + task.setStatus("RUNNING"); + task.setUserId(7L); + task.setUpdatedAt(LocalDateTime.now()); + return task; + } + + private TaskScopeStateEntity scopeWithItems(int itemsReceived) throws Exception { + TaskScopeStateEntity scope = new TaskScopeStateEntity(); + scope.setTaskId(88L); + Map state = new HashMap<>(); + state.put("itemsReceived", itemsReceived); + scope.setStateJson(new ObjectMapper().writeValueAsString(state)); + return scope; + } + + private AdminUserEntity ownerWithCompany() { + AdminUserEntity owner = new AdminUserEntity(); + owner.setId(7L); + owner.setCompanyId(2L); + return owner; + } + + private JdIngestChunkRequest errorRequest() { + JdIngestChunkRequest request = new JdIngestChunkRequest(); + request.setError(STRUCTURE_ERROR); + return request; + } + + @Test + void 结构变更且零行_立即判死不走非致命记录() throws Exception { + FileTaskEntity task = runningTask(); + when(fileTaskMapper.selectById(88L)).thenReturn(task); + when(collectTaskService.findScopeRow(88L)).thenReturn(scopeWithItems(0)); + + JdIngestChunkVo vo = service.submitChunk(88L, errorRequest(), true); + + verify(jdStaleTaskService).terminalizeOnJdStructureFatal(eq(task), contains("售卖中")); + // 判死即终态:不再走非致命档的 scope 台账记账 + verify(taskScopeStateMapper, never()).updateById(any(TaskScopeStateEntity.class)); + assertThat(vo.isAccept()).isTrue(); + assertThat(vo.getMessage()).contains("任务已终止"); + } + + @Test + void 结构变更但已有数据_保持非致命只记录() throws Exception { + FileTaskEntity task = runningTask(); + when(fileTaskMapper.selectById(88L)).thenReturn(task); + when(collectTaskService.findScopeRow(88L)).thenReturn(scopeWithItems(3)); + + JdIngestChunkVo vo = service.submitChunk(88L, errorRequest(), true); + + verify(jdStaleTaskService, never()).terminalizeOnJdStructureFatal(any(), anyString()); + // 非致命档:lastError 记入 scope 台账 + verify(taskScopeStateMapper).updateById(any(TaskScopeStateEntity.class)); + assertThat(vo.isAccept()).isTrue(); + assertThat(vo.getMessage()).contains("非致命"); + } + + @Test + void 匿名结构变更上报_照旧拒绝判死() { + when(fileTaskMapper.selectById(88L)).thenReturn(runningTask()); + + JdIngestChunkVo vo = service.submitChunk(88L, errorRequest(), false); + + verify(jdStaleTaskService, never()).terminalizeOnJdStructureFatal(any(), anyString()); + assertThat(vo.isAccept()).isFalse(); + } + + @Test + void 回传结束但零行_落FAILED不再落SUCCESS() throws Exception { + FileTaskEntity task = runningTask(); + when(fileTaskMapper.selectById(88L)).thenReturn(task); + when(collectTaskService.findScopeRow(88L)).thenReturn(scopeWithItems(0)); + when(adminUserMapper.selectById(7L)).thenReturn(ownerWithCompany()); + when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + JdIngestChunkRequest request = new JdIngestChunkRequest(); + request.setIsEnd(true); + + service.submitChunk(88L, request, true); + + verify(taskProgressSnapshotService).markTerminal(eq(88L), eq("JD_SHOP_COLLECT"), + eq("FAILED"), contains("未采到任何商品")); + } + + @Test + void 回传结束有数据_仍落SUCCESS() throws Exception { + FileTaskEntity task = runningTask(); + when(fileTaskMapper.selectById(88L)).thenReturn(task); + when(collectTaskService.findScopeRow(88L)).thenReturn(scopeWithItems(5)); + when(adminUserMapper.selectById(7L)).thenReturn(ownerWithCompany()); + when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + JdIngestChunkRequest request = new JdIngestChunkRequest(); + request.setIsEnd(true); + + service.submitChunk(88L, request, true); + + verify(taskProgressSnapshotService).markTerminal(eq(88L), eq("JD_SHOP_COLLECT"), + eq("SUCCESS"), anyString()); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/jd/service/JdStaleTaskServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/jd/service/JdStaleTaskServiceTest.java index 72d047f5e..203eec98b 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/jd/service/JdStaleTaskServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/jd/service/JdStaleTaskServiceTest.java @@ -101,6 +101,48 @@ class JdStaleTaskServiceTest { verify(taskProgressSnapshotService).markTerminal(eq(33890L), eq("JD_SHOP_COLLECT"), eq("FAILED"), anyString()); } + @Test + void 结构变更词表只认京东这一族() { + assertThat(JdStaleTaskService.isJdPageStructureFailure( + "京东页面结构变更(京东采集):列表页点不到「售卖中」Tab(30 秒内未渲染或点击后元素失效)")).isTrue(); + assertThat(JdStaleTaskService.isJdPageStructureFailure("京麦未登录,请重新登录")).isFalse(); + assertThat(JdStaleTaskService.isJdPageStructureFailure("channel_busy")).isFalse(); + assertThat(JdStaleTaskService.isJdPageStructureFailure("京东采集异常: ReadTimeout")).isFalse(); + assertThat(JdStaleTaskService.isJdPageStructureFailure(null)).isFalse(); + assertThat(JdStaleTaskService.isJdPageStructureFailure(" ")).isFalse(); + } + + @Test + void 结构变更终态文案带原文与动作句() { + String message = JdStaleTaskService.terminalStructureMessage( + "京东页面结构变更(京东采集):列表页点不到「售卖中」Tab(30 秒内未渲染或点击后元素失效)"); + assertThat(message).contains("京东页面结构变更").contains("售卖中").contains("重新采集"); + assertThat(message).doesNotContain("稍后重试"); + } + + @Test + void 结构变更CAS落终态并同步进度快照() { + FileTaskEntity task = runningTask(); + when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + + boolean closed = service.terminalizeOnJdStructureFatal(task, + "京东页面结构变更(京东采集):列表页点不到「售卖中」Tab"); + + assertThat(closed).isTrue(); + verify(taskProgressSnapshotService).markTerminal(eq(33890L), eq("JD_SHOP_COLLECT"), eq("FAILED"), anyString()); + } + + @Test + void 结构变更CAS不命中时不重复判死() { + FileTaskEntity task = runningTask(); + when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0); + + boolean closed = service.terminalizeOnJdStructureFatal(task, "京东页面结构变更(京东采集):点不到 Tab"); + + assertThat(closed).isFalse(); + verify(taskProgressSnapshotService, never()).markTerminal(any(), anyString(), anyString(), anyString()); + } + @Test void 已终态或CAS不命中时不重复判死() { FileTaskEntity task = runningTask();