fix(jd): 回传链两处 0 行判死——空完成不再落 SUCCESS、结构变更不再白等 60 分钟(2026-11-19 34900/34901/34902 实录,铁律 31)
This commit is contained in:
+52
-4
@@ -23,6 +23,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.CredentialFatalTaskCloser;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFinalizeSupport;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.support.TaskFailureLedger;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -164,10 +165,16 @@ public class JdResultIngestService {
|
||||
}
|
||||
int itemsReceived = mergeScopeState(scope, request, request.getItems() == null ? 0 : request.getItems().size());
|
||||
if (Boolean.TRUE.equals(request.getIsEnd())) {
|
||||
finalizeSuccess(task, itemsReceived);
|
||||
// 客户端缺少可验证的完整分页回执,不能据此删除未回传的旧商品。
|
||||
log.warn("[{}] 缺少全店分页完整性证明,跳过覆盖清理 taskId={} itemsReceived={}",
|
||||
LOG_TAG, task.getId(), itemsReceived);
|
||||
if (itemsReceived <= 0) {
|
||||
// 空完成判死(2026-11-19 34902 实录):isEnd 且累计 0 行=列表页根本没读出来
|
||||
//(没报结构错的空跑同样落这里),照旧落 SUCCESS 会让界面显示「成功」却没有数据。
|
||||
finalizeEmptyFailure(task);
|
||||
} else {
|
||||
finalizeSuccess(task, itemsReceived);
|
||||
// 客户端缺少可验证的完整分页回执,不能据此删除未回传的旧商品。
|
||||
log.warn("[{}] 缺少全店分页完整性证明,跳过覆盖清理 taskId={} itemsReceived={}",
|
||||
LOG_TAG, task.getId(), itemsReceived);
|
||||
}
|
||||
}
|
||||
JdIngestChunkVo vo = new JdIngestChunkVo();
|
||||
vo.setAccept(true);
|
||||
@@ -197,6 +204,16 @@ public class JdResultIngestService {
|
||||
new CredentialFatalTaskCloser(LOG_TAG, fileTaskMapper, null).closeIfFatal(task, error);
|
||||
return reject("凭据不可自愈(详见任务失败原因),任务已终止");
|
||||
}
|
||||
// 非致命档在 0 行时收窄(2026-11-19 34900/34901 实录):「页面结构变更」命中且累计 0 行
|
||||
// ⇒ worker 报完即停、心跳随之注销,再挂到心跳超时只会让用户白看 60 分钟「处理中」——
|
||||
// 立即判 FAILED(有数据的中途抖动仍走下面的非致命档)。
|
||||
if (JdStaleTaskService.isJdPageStructureFailure(error) && receivedRows(task.getId()) == 0) {
|
||||
jdStaleTaskService.terminalizeOnJdStructureFatal(task, error);
|
||||
JdIngestChunkVo vo = new JdIngestChunkVo();
|
||||
vo.setAccept(true);
|
||||
vo.setMessage("页面结构变更且未采到任何商品,任务已终止");
|
||||
return vo;
|
||||
}
|
||||
// 非致命(限流/超时/页面抖动):只补测不判死,记入 scope 台账后让客户端继续。
|
||||
log.warn("[{}] 非致命失败上报,不判死 taskId={} err={}", LOG_TAG, task.getId(), error);
|
||||
recordLastError(task.getId(), error);
|
||||
@@ -206,6 +223,15 @@ public class JdResultIngestService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
/** 累计已收行数(scope 台账 itemsReceived);台账缺失按 0 读——首包前就报错的失败正是要判死的形状。 */
|
||||
private int receivedRows(Long taskId) {
|
||||
TaskScopeStateEntity scope = collectTaskService.findScopeRow(taskId);
|
||||
if (scope == null) {
|
||||
return 0;
|
||||
}
|
||||
return intVal(parseState(scope.getStateJson()).get("itemsReceived"));
|
||||
}
|
||||
|
||||
private void recordLastError(Long taskId, String error) {
|
||||
TaskScopeStateEntity scope = collectTaskService.findScopeRow(taskId);
|
||||
if (scope == null) {
|
||||
@@ -234,6 +260,28 @@ public class JdResultIngestService {
|
||||
}
|
||||
}
|
||||
|
||||
private static final String STATUS_FAILED = TaskStatuses.FileTask.FAILED;
|
||||
|
||||
/** 空完成判死:isEnd 且累计 0 行,CAS RUNNING→FAILED 可重入;文案给动作句不写「稍后重试」。 */
|
||||
private void finalizeEmptyFailure(FileTaskEntity task) {
|
||||
String message = "京东采集结束但未采到任何商品数据(0 行):通常是京麦未登录或商品列表页未能打开,"
|
||||
+ "请核对采集浏览器里的京麦登录态后重新采集";
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getSuccessFileCount, 0)
|
||||
.set(FileTaskEntity::getFailedFileCount, 1)
|
||||
.set(FileTaskEntity::getErrorMessage, message)
|
||||
.set(FileTaskEntity::getFailureCategory, TaskFailureLedger.categoryOf(message))
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
taskProgressSnapshotService.markTerminal(task.getId(), MODULE_TYPE, STATUS_FAILED, message);
|
||||
log.warn("[{}] 回传结束但未采到任何商品,落 FAILED taskId={}", LOG_TAG, task.getId());
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- V199 保护闸:人工编辑锁与删除忽略 ----------
|
||||
|
||||
/**
|
||||
|
||||
+55
@@ -164,4 +164,59 @@ public class JdStaleTaskService {
|
||||
|| message.contains("未登录京麦")
|
||||
|| message.contains("京麦登录态");
|
||||
}
|
||||
|
||||
/**
|
||||
* 京东页面结构变更整单判死(铁律 31,2026-11-19 34900/34901 实录):「列表页点不到 Tab」
|
||||
* 发生在首包之前时任务一件数据都没有,worker 报完 error 即停、心跳随之注销 ⇒ 原来只能挂到
|
||||
* 心跳超时(60 分钟)才死,用户全程只看「处理中」。回传侧
|
||||
* ({@code JdResultIngestService})判「结构变更词命中且累计 0 行」时调本出口立即落 FAILED;
|
||||
* 有数据的中途抖动仍走非致命档。CAS RUNNING→FAILED 可重入。
|
||||
*
|
||||
* @return true=本次真的把任务写成了终态
|
||||
*/
|
||||
public boolean terminalizeOnJdStructureFatal(FileTaskEntity task, String rawError) {
|
||||
if (task == null || task.getId() == null) {
|
||||
return false;
|
||||
}
|
||||
String message = terminalStructureMessage(rawError);
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(FileTaskEntity::getSuccessFileCount, 0)
|
||||
.set(FileTaskEntity::getFailedFileCount, 1)
|
||||
.set(FileTaskEntity::getErrorMessage, message)
|
||||
.set(FileTaskEntity::getFailureCategory, TaskFailureLedger.categoryOf(message))
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
taskProgressSnapshotService.markTerminal(task.getId(), MODULE_TYPE, STATUS_FAILED, message);
|
||||
log.warn("[{}] 页面结构变更且未采到任何数据,任务落终态 FAILED taskId={} err={}",
|
||||
LOG_TAG, task.getId(), message);
|
||||
}
|
||||
return updated > 0;
|
||||
}
|
||||
|
||||
/** 结构变更终态文案:生产者原文 + 动作句(不写「稍后重试」)。 */
|
||||
static String terminalStructureMessage(String rawError) {
|
||||
String raw = rawError == null ? "" : rawError.trim();
|
||||
StringBuilder message = new StringBuilder();
|
||||
if (!raw.isEmpty()) {
|
||||
message.append(raw).append(";");
|
||||
}
|
||||
message.append("请确认采集浏览器里京麦已登录、商品列表页能正常打开后重新采集");
|
||||
return message.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 「京东页面结构变更」这一族(唯一出处=客户端 {@code structure_change_message} 的固定前缀,
|
||||
* 回传 error 与判死共用同一判据;登录态/代理/LLM 凭据词各有自己的族,不在这里)。
|
||||
*/
|
||||
static boolean isJdPageStructureFailure(String message) {
|
||||
if (message == null || message.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
return message.contains("京东页面结构变更");
|
||||
}
|
||||
}
|
||||
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
package com.nanri.aiimage.modules.jd.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.mapper.AdminUserMapper;
|
||||
import com.nanri.aiimage.common.model.entity.AdminUserEntity;
|
||||
import com.nanri.aiimage.modules.jd.mapper.JdProductIgnoreMapper;
|
||||
import com.nanri.aiimage.modules.jd.mapper.JdProductMapper;
|
||||
import com.nanri.aiimage.modules.jd.model.dto.JdIngestChunkRequest;
|
||||
import com.nanri.aiimage.modules.jd.model.vo.JdIngestChunkVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.contains;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 京东采集回传判定行为测试(2026-11-19 34900/34901/34902 实录的两档收口):
|
||||
*
|
||||
* <p>① 「页面结构变更」失败上报在<b>累计 0 行</b>时立即判 FAILED(原非致命档会让 worker 停住
|
||||
* 后挂到心跳超时 60 分钟,用户全程只看「处理中」);有数据的中途抖动保持非致命。
|
||||
* ② {@code isEnd=true} 且累计 0 行落 FAILED 不再落 SUCCESS(34902 形状:界面显示「成功」
|
||||
* 却没有任何商品数据)。匿名 error 上报照旧拒绝判死。</p>
|
||||
*
|
||||
* <p>CAS 落终态用 {@code fileTaskMapper.update} 返回值驱动,不反射私有方法。</p>
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class JdResultIngestServiceTest {
|
||||
|
||||
private static final String STRUCTURE_ERROR =
|
||||
"京东页面结构变更(京东采集):列表页点不到「售卖中」Tab(30 秒内未渲染或点击后元素失效)"
|
||||
+ ";请重跑京东采集取证并校准解析层(docs/jd-collect-api-contract.md)";
|
||||
|
||||
@Mock
|
||||
private com.nanri.aiimage.modules.task.mapper.FileTaskMapper fileTaskMapper;
|
||||
@Mock
|
||||
private JdProductMapper jdProductMapper;
|
||||
@Mock
|
||||
private JdProductIgnoreMapper jdProductIgnoreMapper;
|
||||
@Mock
|
||||
private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock
|
||||
private AdminUserMapper adminUserMapper;
|
||||
@Mock
|
||||
private JdCollectTaskService collectTaskService;
|
||||
@Mock
|
||||
private JdStaleTaskService jdStaleTaskService;
|
||||
@Mock
|
||||
private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock
|
||||
private JdProductImageIngestService jdProductImageIngestService;
|
||||
@Mock
|
||||
private JdProductSkuIngestService jdProductSkuIngestService;
|
||||
@Mock
|
||||
private JdProductRowDeleteService jdProductRowDeleteService;
|
||||
|
||||
private JdResultIngestService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initTableInfo() {
|
||||
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), FileTaskEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service = new JdResultIngestService(fileTaskMapper, jdProductMapper, jdProductIgnoreMapper,
|
||||
taskScopeStateMapper, adminUserMapper, collectTaskService, jdStaleTaskService,
|
||||
taskProgressSnapshotService, jdProductImageIngestService, jdProductSkuIngestService,
|
||||
jdProductRowDeleteService, new ObjectMapper());
|
||||
}
|
||||
|
||||
private FileTaskEntity runningTask() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(88L);
|
||||
task.setModuleType("JD_SHOP_COLLECT");
|
||||
task.setStatus("RUNNING");
|
||||
task.setUserId(7L);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
return task;
|
||||
}
|
||||
|
||||
private TaskScopeStateEntity scopeWithItems(int itemsReceived) throws Exception {
|
||||
TaskScopeStateEntity scope = new TaskScopeStateEntity();
|
||||
scope.setTaskId(88L);
|
||||
Map<String, Object> state = new HashMap<>();
|
||||
state.put("itemsReceived", itemsReceived);
|
||||
scope.setStateJson(new ObjectMapper().writeValueAsString(state));
|
||||
return scope;
|
||||
}
|
||||
|
||||
private AdminUserEntity ownerWithCompany() {
|
||||
AdminUserEntity owner = new AdminUserEntity();
|
||||
owner.setId(7L);
|
||||
owner.setCompanyId(2L);
|
||||
return owner;
|
||||
}
|
||||
|
||||
private JdIngestChunkRequest errorRequest() {
|
||||
JdIngestChunkRequest request = new JdIngestChunkRequest();
|
||||
request.setError(STRUCTURE_ERROR);
|
||||
return request;
|
||||
}
|
||||
|
||||
@Test
|
||||
void 结构变更且零行_立即判死不走非致命记录() throws Exception {
|
||||
FileTaskEntity task = runningTask();
|
||||
when(fileTaskMapper.selectById(88L)).thenReturn(task);
|
||||
when(collectTaskService.findScopeRow(88L)).thenReturn(scopeWithItems(0));
|
||||
|
||||
JdIngestChunkVo vo = service.submitChunk(88L, errorRequest(), true);
|
||||
|
||||
verify(jdStaleTaskService).terminalizeOnJdStructureFatal(eq(task), contains("售卖中"));
|
||||
// 判死即终态:不再走非致命档的 scope 台账记账
|
||||
verify(taskScopeStateMapper, never()).updateById(any(TaskScopeStateEntity.class));
|
||||
assertThat(vo.isAccept()).isTrue();
|
||||
assertThat(vo.getMessage()).contains("任务已终止");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 结构变更但已有数据_保持非致命只记录() throws Exception {
|
||||
FileTaskEntity task = runningTask();
|
||||
when(fileTaskMapper.selectById(88L)).thenReturn(task);
|
||||
when(collectTaskService.findScopeRow(88L)).thenReturn(scopeWithItems(3));
|
||||
|
||||
JdIngestChunkVo vo = service.submitChunk(88L, errorRequest(), true);
|
||||
|
||||
verify(jdStaleTaskService, never()).terminalizeOnJdStructureFatal(any(), anyString());
|
||||
// 非致命档:lastError 记入 scope 台账
|
||||
verify(taskScopeStateMapper).updateById(any(TaskScopeStateEntity.class));
|
||||
assertThat(vo.isAccept()).isTrue();
|
||||
assertThat(vo.getMessage()).contains("非致命");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 匿名结构变更上报_照旧拒绝判死() {
|
||||
when(fileTaskMapper.selectById(88L)).thenReturn(runningTask());
|
||||
|
||||
JdIngestChunkVo vo = service.submitChunk(88L, errorRequest(), false);
|
||||
|
||||
verify(jdStaleTaskService, never()).terminalizeOnJdStructureFatal(any(), anyString());
|
||||
assertThat(vo.isAccept()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void 回传结束但零行_落FAILED不再落SUCCESS() throws Exception {
|
||||
FileTaskEntity task = runningTask();
|
||||
when(fileTaskMapper.selectById(88L)).thenReturn(task);
|
||||
when(collectTaskService.findScopeRow(88L)).thenReturn(scopeWithItems(0));
|
||||
when(adminUserMapper.selectById(7L)).thenReturn(ownerWithCompany());
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
JdIngestChunkRequest request = new JdIngestChunkRequest();
|
||||
request.setIsEnd(true);
|
||||
|
||||
service.submitChunk(88L, request, true);
|
||||
|
||||
verify(taskProgressSnapshotService).markTerminal(eq(88L), eq("JD_SHOP_COLLECT"),
|
||||
eq("FAILED"), contains("未采到任何商品"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void 回传结束有数据_仍落SUCCESS() throws Exception {
|
||||
FileTaskEntity task = runningTask();
|
||||
when(fileTaskMapper.selectById(88L)).thenReturn(task);
|
||||
when(collectTaskService.findScopeRow(88L)).thenReturn(scopeWithItems(5));
|
||||
when(adminUserMapper.selectById(7L)).thenReturn(ownerWithCompany());
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
JdIngestChunkRequest request = new JdIngestChunkRequest();
|
||||
request.setIsEnd(true);
|
||||
|
||||
service.submitChunk(88L, request, true);
|
||||
|
||||
verify(taskProgressSnapshotService).markTerminal(eq(88L), eq("JD_SHOP_COLLECT"),
|
||||
eq("SUCCESS"), anyString());
|
||||
}
|
||||
}
|
||||
+42
@@ -101,6 +101,48 @@ class JdStaleTaskServiceTest {
|
||||
verify(taskProgressSnapshotService).markTerminal(eq(33890L), eq("JD_SHOP_COLLECT"), eq("FAILED"), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 结构变更词表只认京东这一族() {
|
||||
assertThat(JdStaleTaskService.isJdPageStructureFailure(
|
||||
"京东页面结构变更(京东采集):列表页点不到「售卖中」Tab(30 秒内未渲染或点击后元素失效)")).isTrue();
|
||||
assertThat(JdStaleTaskService.isJdPageStructureFailure("京麦未登录,请重新登录")).isFalse();
|
||||
assertThat(JdStaleTaskService.isJdPageStructureFailure("channel_busy")).isFalse();
|
||||
assertThat(JdStaleTaskService.isJdPageStructureFailure("京东采集异常: ReadTimeout")).isFalse();
|
||||
assertThat(JdStaleTaskService.isJdPageStructureFailure(null)).isFalse();
|
||||
assertThat(JdStaleTaskService.isJdPageStructureFailure(" ")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void 结构变更终态文案带原文与动作句() {
|
||||
String message = JdStaleTaskService.terminalStructureMessage(
|
||||
"京东页面结构变更(京东采集):列表页点不到「售卖中」Tab(30 秒内未渲染或点击后元素失效)");
|
||||
assertThat(message).contains("京东页面结构变更").contains("售卖中").contains("重新采集");
|
||||
assertThat(message).doesNotContain("稍后重试");
|
||||
}
|
||||
|
||||
@Test
|
||||
void 结构变更CAS落终态并同步进度快照() {
|
||||
FileTaskEntity task = runningTask();
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
|
||||
boolean closed = service.terminalizeOnJdStructureFatal(task,
|
||||
"京东页面结构变更(京东采集):列表页点不到「售卖中」Tab");
|
||||
|
||||
assertThat(closed).isTrue();
|
||||
verify(taskProgressSnapshotService).markTerminal(eq(33890L), eq("JD_SHOP_COLLECT"), eq("FAILED"), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 结构变更CAS不命中时不重复判死() {
|
||||
FileTaskEntity task = runningTask();
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
|
||||
|
||||
boolean closed = service.terminalizeOnJdStructureFatal(task, "京东页面结构变更(京东采集):点不到 Tab");
|
||||
|
||||
assertThat(closed).isFalse();
|
||||
verify(taskProgressSnapshotService, never()).markTerminal(any(), anyString(), anyString(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 已终态或CAS不命中时不重复判死() {
|
||||
FileTaskEntity task = runningTask();
|
||||
|
||||
Reference in New Issue
Block a user