fix(owner): 跨实例归属背离改为转交作业,不再无限 defer 空转
领取按 biz_task_file_job.owner、执行时校验按 biz_file_task.owner_instance_id, 任务转交后两列一旦背离就是互斥死锁:能领到这行的实例必被校验拒,校验放过的 实例必领不到这行;而拒绝分支 deferRunning 既不计 retry_count 也不计 stuck_requeue_count 还每轮刷新 updated_at,卡死重排与 PENDING 遗骸两条兜底 因此永不触发。现网外观专利 33179/33149 停在 PENDING 每 15 秒被拒一次,单机 12 小时 6381 条拒绝,进度快照冻结 4 小时 19 分,既不失败也不完成。 - TaskFileJobService: 新增 handOffToOwner(owner 列与 scope_key 尾段一起改挂, CAS 按观测到的旧 owner 回撞)与 withOwnerScopeKey(ownerFromScopeKey 的逆操作); enqueueAssembleResult 的 PENDING 分支改为按入队 scopeKey 对账归属——Python 超时恢复每 2 分钟走这一格一次,原先只补发事件不对账等于每 2 分钟确认死锁还在。 - TaskResultFileJobWorker: 归属拒绝改为先改挂再退回。转交后两列重合,拒绝条件 按构造不再成立,因此不需要额外计数兜底。 - 相似ASIN/发布 ownerFromTask 改为 owner_instance_id 列优先:原先只读 resultJson 快照(转交后不再同步),转交目标会算错、死锁只是换个方向。店铺采集本就列优先,未动。 判据:TaskFileJobOwnerHandOffTest(11) + TaskResultFileJobWorkerOwnerHandOffTest(3) + TaskOwnerReadSourceGuardrailTest(5,含四台校验源同源的源码绊线,已反向验证 "改回只读快照即红")。SourceScanTest 台账按设计登记新消费方:裸名 22→23、 调用方 16→17。后端整量 5206 支 0 失败 19 skipped。
This commit is contained in:
+225
@@ -0,0 +1,225 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 跨实例 owner 死锁的防回归判据(2026-11-02 现网任务 33179/33149)。
|
||||
*
|
||||
* <p>形状:领取按 {@code biz_task_file_job.owner},执行时的归属校验按
|
||||
* {@code biz_file_task.owner_instance_id}。任务转交后两列背离——能领到这行的实例必被校验拒,
|
||||
* 校验放过的实例必领不到这行;拒绝走的 {@code deferRunning} 既不计 retry_count 也不计
|
||||
* stuck_requeue_count 还每轮刷新 updated_at,两条兜底因此永不触发,作业既不失败也不完成
|
||||
* (单机 12 小时 6381 条拒绝)。
|
||||
*
|
||||
* <p>把 {@link TaskFileJobService#handOffToOwner} 改回「只退回 PENDING 不改挂」、
|
||||
* 或把 {@link TaskFileJobService#enqueueAssembleResult} 的 PENDING 分支改回「只补发事件不对账」,
|
||||
* 本类即红。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class TaskFileJobOwnerHandOffTest {
|
||||
|
||||
private static final String STALE_OWNER = "server-110";
|
||||
private static final String TASK_OWNER = "server-121";
|
||||
|
||||
@Mock private TaskFileJobMapper taskFileJobMapper;
|
||||
@Mock private ApplicationEventPublisher applicationEventPublisher;
|
||||
|
||||
private TaskFileJobService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeTableInfo() {
|
||||
TableInfoHelper.initTableInfo(
|
||||
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
TaskFileJobEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
}
|
||||
|
||||
private static TaskFileJobEntity job(Long id, Long taskId, String status, String owner) {
|
||||
TaskFileJobEntity entity = new TaskFileJobEntity();
|
||||
entity.setId(id);
|
||||
entity.setTaskId(taskId);
|
||||
entity.setModuleType("APPEARANCE_PATENT");
|
||||
entity.setResultId(36102L);
|
||||
entity.setJobType("ASSEMBLE_RESULT");
|
||||
entity.setStatus(status);
|
||||
entity.setOwner(owner);
|
||||
entity.setScopeKey(owner == null ? "task:" + taskId : "task:" + taskId + ":owner:" + owner);
|
||||
entity.setRetryCount(0);
|
||||
entity.setUpdatedAt(LocalDateTime.now());
|
||||
return entity;
|
||||
}
|
||||
|
||||
// ==================== withOwnerInScopeKey:ownerFromScopeKey 的逆操作 ====================
|
||||
|
||||
@Test
|
||||
void test_scope_key_只换最后一个_owner_尾段() {
|
||||
assertEquals("task:33179:owner:" + TASK_OWNER,
|
||||
TaskFileJobService.withOwnerInScopeKey("task:33179:owner:" + STALE_OWNER, TASK_OWNER));
|
||||
// Java 侧 ownerFromScopeKey 用 lastIndexOf,所以逆操作也必须只改最后一段
|
||||
assertEquals("task:1:owner:x:owner:" + TASK_OWNER,
|
||||
TaskFileJobService.withOwnerInScopeKey("task:1:owner:x:owner:" + STALE_OWNER, TASK_OWNER));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_scope_key_无标记或目标为空时不得凭空造_owner_段() {
|
||||
assertEquals("task:1:plain", TaskFileJobService.withOwnerInScopeKey("task:1:plain", TASK_OWNER));
|
||||
assertEquals("task:1:owner:" + STALE_OWNER,
|
||||
TaskFileJobService.withOwnerInScopeKey("task:1:owner:" + STALE_OWNER, " "));
|
||||
assertNull(TaskFileJobService.withOwnerInScopeKey(null, TASK_OWNER));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_逆操作与提取函数必须互逆() {
|
||||
String rebuilt = TaskFileJobService.withOwnerInScopeKey(
|
||||
"task:33179:owner:" + STALE_OWNER, TASK_OWNER);
|
||||
assertEquals(TASK_OWNER, TaskFileJobService.ownerFromScopeKey(rebuilt),
|
||||
"改挂之后按同一枚谓词读回来必须是目标实例,否则转交等于没转");
|
||||
}
|
||||
|
||||
// ==================== handOffToOwner ====================
|
||||
|
||||
@Test
|
||||
void test_改挂必须同时落_owner_列与_scope_key_且按旧_owner_做_cas() {
|
||||
when(taskFileJobMapper.selectById(23815L))
|
||||
.thenReturn(job(23815L, 33179L, "RUNNING", STALE_OWNER));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
|
||||
assertTrue(service.handOffToOwner(23815L, STALE_OWNER, TASK_OWNER));
|
||||
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> captor =
|
||||
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(taskFileJobMapper).update(isNull(), captor.capture());
|
||||
LambdaUpdateWrapper<TaskFileJobEntity> wrapper = captor.getValue();
|
||||
String setClause = wrapper.getSqlSet();
|
||||
assertTrue(setClause.contains("owner"), "必须改 owner 列: " + setClause);
|
||||
assertTrue(setClause.contains("scope_key"), "必须同步改 scope_key,否则 lastIndexOf 读回的还是旧实例: " + setClause);
|
||||
assertTrue(setClause.contains("error_message"), "必须清掉那条误导性的归属拒绝文案: " + setClause);
|
||||
assertTrue(wrapper.getCustomSqlSegment().contains("status = "), "只能改挂自己手里的 RUNNING 行: " + wrapper.getCustomSqlSegment());
|
||||
assertTrue(wrapper.getCustomSqlSegment().contains("owner = "), "必须按观测到的旧 owner 回撞: " + wrapper.getCustomSqlSegment());
|
||||
assertTrue(wrapper.getParamNameValuePairs().containsValue("task:33179:owner:" + TASK_OWNER),
|
||||
"scope_key 参数必须是换过尾段的值: " + wrapper.getParamNameValuePairs());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_已经挂在目标实例上时不得再发更新() {
|
||||
// 空转的代价就是每轮刷 updated_at:两列重合后必须一口都不动
|
||||
when(taskFileJobMapper.selectById(23815L))
|
||||
.thenReturn(job(23815L, 33179L, "RUNNING", TASK_OWNER));
|
||||
|
||||
assertFalse(service.handOffToOwner(23815L, STALE_OWNER, TASK_OWNER));
|
||||
verify(taskFileJobMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_目标实例为空或行已消失时拒绝改挂() {
|
||||
assertFalse(service.handOffToOwner(23815L, STALE_OWNER, null));
|
||||
assertFalse(service.handOffToOwner(23815L, STALE_OWNER, " "));
|
||||
assertFalse(service.handOffToOwner(null, STALE_OWNER, TASK_OWNER));
|
||||
verify(taskFileJobMapper, never()).selectById(any());
|
||||
|
||||
when(taskFileJobMapper.selectById(23815L)).thenReturn(null);
|
||||
assertFalse(service.handOffToOwner(23815L, STALE_OWNER, TASK_OWNER));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_cas_未命中时返回失败而不抛() {
|
||||
// 期间任务又转交、或行被别的实例改挂:update 打空,由下一轮拒绝重新转交
|
||||
when(taskFileJobMapper.selectById(23815L))
|
||||
.thenReturn(job(23815L, 33179L, "RUNNING", STALE_OWNER));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
|
||||
|
||||
assertFalse(service.handOffToOwner(23815L, STALE_OWNER, TASK_OWNER));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_旧_owner_为空时_cas_必须走_is_null_而不是空串等值() {
|
||||
when(taskFileJobMapper.selectById(1L)).thenReturn(job(1L, 900L, "RUNNING", null));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
|
||||
assertTrue(service.handOffToOwner(1L, null, TASK_OWNER));
|
||||
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> captor =
|
||||
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(taskFileJobMapper).update(isNull(), captor.capture());
|
||||
assertTrue(captor.getValue().getCustomSqlSegment().contains("owner IS NULL"),
|
||||
"存量行 owner 为 NULL,等值空串永远打不中: " + captor.getValue().getCustomSqlSegment());
|
||||
}
|
||||
|
||||
// ==================== enqueueAssembleResult 的 PENDING 对账 ====================
|
||||
|
||||
@Test
|
||||
void test_pending_作业重新入队时必须按入队_scope_key_对账归属() {
|
||||
// Python 超时恢复每 2 分钟就会走到这一格一次。只补发事件不对账,等于每 2 分钟确认一遍死锁还在。
|
||||
TaskFileJobEntity pending = job(23815L, 33179L, "PENDING", STALE_OWNER);
|
||||
when(taskFileJobMapper.selectOne(any())).thenReturn(pending);
|
||||
when(taskFileJobMapper.selectById(23815L))
|
||||
.thenReturn(job(23815L, 33179L, "PENDING", TASK_OWNER));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
|
||||
TaskFileJobEntity result = service.enqueueAssembleResult(
|
||||
33179L, "APPEARANCE_PATENT", 36102L, "task:33179:owner:" + TASK_OWNER);
|
||||
|
||||
assertEquals(TASK_OWNER, result.getOwner(), "返回的必须是对账后的行");
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> captor =
|
||||
ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
|
||||
verify(taskFileJobMapper, times(1)).update(isNull(), captor.capture());
|
||||
assertTrue(captor.getValue().getSqlSet().contains("owner"), "对账必须写 owner 列");
|
||||
assertTrue(captor.getValue().getCustomSqlSegment().contains("status = "),
|
||||
"只能对账仍在 PENDING 的行,别抢正在跑的那台: " + captor.getValue().getCustomSqlSegment());
|
||||
verify(applicationEventPublisher).publishEvent(any(TaskFileJobDispatchEvent.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_pending_作业归属已重合时不得发更新() {
|
||||
TaskFileJobEntity pending = job(23815L, 33179L, "PENDING", TASK_OWNER);
|
||||
when(taskFileJobMapper.selectOne(any())).thenReturn(pending);
|
||||
|
||||
TaskFileJobEntity result = service.enqueueAssembleResult(
|
||||
33179L, "APPEARANCE_PATENT", 36102L, "task:33179:owner:" + TASK_OWNER);
|
||||
|
||||
assertEquals(23815L, result.getId());
|
||||
verify(taskFileJobMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
verify(applicationEventPublisher).publishEvent(any(TaskFileJobDispatchEvent.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_pending_作业入队_scope_key_不带_owner_段时不得凭空对账() {
|
||||
TaskFileJobEntity pending = job(23815L, 33179L, "PENDING", STALE_OWNER);
|
||||
when(taskFileJobMapper.selectOne(any())).thenReturn(pending);
|
||||
|
||||
service.enqueueAssembleResult(33179L, "APPEARANCE_PATENT", 36102L, "task:33179");
|
||||
|
||||
verify(taskFileJobMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
}
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.testutil.SourceScan;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.CsvSource;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* 「任务归属实例」校验源必须四台同源的绊线(2026-11-02)。
|
||||
*
|
||||
* <p>起因:现网任务 33179/33149 的跨实例死锁。领取按 {@code biz_task_file_job.owner}、
|
||||
* 执行时校验按 {@code biz_file_task.owner_instance_id},两列一旦背离就是互斥死锁
|
||||
* (能领到这行的实例必被校验拒、校验放过的实例必领不到这行)。而四台的校验源当时是三样:
|
||||
* 外观专利读列(正确)、相似ASIN 与发布<b>只读 {@code resultJson} 快照</b>、
|
||||
* 店铺采集回落 {@code requestJson}。快照是建单那一刻写死的、任务转交后不再同步,
|
||||
* 于是 worker 侧按异常携带的 owner 转交作业时会转交给已经不管这行的实例——死锁只是换了个方向。
|
||||
*
|
||||
* <p>判据形状:每台的归属读取方法都必须<b>先读 {@code owner_instance_id} 列</b>,
|
||||
* 只在列缺失时才回落 JSON 快照。把任何一台改回「只读快照」,本类即红。
|
||||
* 之所以打在源码上而不是四份单测夹具:这条谓词的正确性在于「读哪个字段」,
|
||||
* 用 mock 造四份任务实体各测一遍,改回读快照时那四份夹具照样全绿。
|
||||
*/
|
||||
class TaskOwnerReadSourceGuardrailTest {
|
||||
|
||||
/** 从方法签名起做括号配对,取出方法体(注释已由 withoutComments 剥掉)。 */
|
||||
private static String methodBody(String source, String signature) {
|
||||
int start = source.indexOf(signature);
|
||||
assertThat(start).as("必须存在方法 " + signature).isGreaterThan(-1);
|
||||
int open = source.indexOf('{', start);
|
||||
assertThat(open).as(signature + " 的方法体起点").isGreaterThan(-1);
|
||||
int depth = 0;
|
||||
for (int i = open; i < source.length(); i++) {
|
||||
char c = source.charAt(i);
|
||||
if (c == '{') {
|
||||
depth++;
|
||||
} else if (c == '}') {
|
||||
depth--;
|
||||
if (depth == 0) {
|
||||
return source.substring(open + 1, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new AssertionError(signature + " 括号未闭合");
|
||||
}
|
||||
|
||||
@ParameterizedTest(name = "{0} 的 {1} 必须先读 owner_instance_id 列")
|
||||
@CsvSource({
|
||||
"AppearancePatentTaskOwnership.java, ownerFromTask(FileTaskEntity task)",
|
||||
"SimilarAsinTaskOwnershipSupport.java, ownerFromTask(FileTaskEntity task)",
|
||||
"PublishTaskService.java, ownerFromTask(FileTaskEntity task)",
|
||||
"ShopDataCrawlTaskService.java, ownerInstanceIdOf(FileTaskEntity task)",
|
||||
})
|
||||
void ownershipReadMustStartFromTheExplicitColumn(String fileName, String signature) {
|
||||
String source = SourceScan.withoutComments(SourceScan.sourceText(fileName));
|
||||
String body = methodBody(source, signature);
|
||||
|
||||
assertThat(body)
|
||||
.as(fileName + " 的 " + signature + " 必须读显式列,而不是只读 JSON 快照")
|
||||
.contains("getOwnerInstanceId()");
|
||||
int columnRead = body.indexOf("getOwnerInstanceId()");
|
||||
int jsonRead = body.indexOf("readTree(");
|
||||
if (jsonRead >= 0) {
|
||||
assertThat(columnRead)
|
||||
.as(fileName + " 必须先读列再读 JSON 快照(列是转交后唯一被更新的来源)")
|
||||
.isLessThan(jsonRead);
|
||||
}
|
||||
// jsonRead < 0 也合法:那一台已经把快照回落后移到别的方法里,列就是唯一来源。
|
||||
// 真正不允许的是反过来——只读快照,那正是相似ASIN 与发布当时的形状。
|
||||
}
|
||||
|
||||
@Test
|
||||
void 转交作业的目标实例必须来自异常携带的任务当前_owner() {
|
||||
String source = SourceScan.withoutComments(SourceScan.sourceText("TaskResultFileJobWorker.java"));
|
||||
String body = methodBody(source, "catch (TaskOwnerMismatchException ex)");
|
||||
|
||||
assertThat(body)
|
||||
.as("只 deferRunning 就是 2026-11-02 那 6381 条空转拒绝的形状")
|
||||
.contains("handOffToOwner");
|
||||
assertThat(body)
|
||||
.as("改挂目标必须取任务当前 owner,取作业自己的旧 owner 等于原地不动")
|
||||
.contains("ex.getOwnerInstanceId()");
|
||||
assertThat(body.indexOf("handOffToOwner"))
|
||||
.as("必须先改挂再退回:deferRunning 把行翻成 PENDING,而改挂的 CAS 只认 RUNNING")
|
||||
.isLessThan(body.indexOf("deferRunning"));
|
||||
}
|
||||
}
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentResultFileJobHandler;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InOrder;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* worker 侧归属拒绝的处理顺序判据(2026-11-02 现网任务 33179/33149)。
|
||||
*
|
||||
* <p>被 {@code TaskOwnerMismatchException} 拒掉时,必须<b>先</b>把作业改挂到任务当前 owner、
|
||||
* <b>再</b>退回 PENDING。顺序反了就不生效:{@code handOffToOwner} 的 CAS 只认 RUNNING 行,
|
||||
* 而 {@code deferRunning} 恰好把行翻成 PENDING——先 defer 的话改挂永远打不中,
|
||||
* 于是回到「每 15 秒领一次、每次都被拒、永不终态」的死锁原状。
|
||||
*
|
||||
* <p>改挂目标必须取异常携带的 {@code ownerInstanceId}(校验刚读到的任务当前归属),
|
||||
* 不能取作业自己的旧 owner——那是原地不动,等于没修。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class TaskResultFileJobWorkerOwnerHandOffTest {
|
||||
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Mock private AppearancePatentTaskService appearancePatentTaskService;
|
||||
|
||||
private TaskResultFileJobWorker worker;
|
||||
private TaskFileJobEntity job;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
ResultFileJobHandlerRegistry registry = new ResultFileJobHandlerRegistry(
|
||||
List.of(new AppearancePatentResultFileJobHandler(appearancePatentTaskService)));
|
||||
worker = new TaskResultFileJobWorker(
|
||||
taskFileJobService,
|
||||
taskDistributedLockService,
|
||||
mock(FileResultMapper.class),
|
||||
mock(TaskFileJobLocalDispatcher.class),
|
||||
instanceMetadata,
|
||||
registry);
|
||||
job = new TaskFileJobEntity();
|
||||
job.setId(23815L);
|
||||
job.setTaskId(33179L);
|
||||
job.setResultId(36102L);
|
||||
job.setModuleType("APPEARANCE_PATENT");
|
||||
job.setJobType("ASSEMBLE_RESULT");
|
||||
job.setStatus("RUNNING");
|
||||
job.setOwner("server-110");
|
||||
job.setScopeKey("task:33179:owner:server-110");
|
||||
job.setRetryCount(1);
|
||||
job.setUpdatedAt(LocalDateTime.now());
|
||||
}
|
||||
|
||||
private void stubRejectedByTaskOwner() throws Exception {
|
||||
when(taskFileJobService.findById(23815L)).thenReturn(null);
|
||||
when(taskDistributedLockService.acquire(eq("APPEARANCE_PATENT"), eq(33179L), anyLong()))
|
||||
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
|
||||
when(appearancePatentTaskService.processResultFileJob(job)).thenThrow(
|
||||
new TaskOwnerMismatchException(33179L, "assemble result file", "server-121", "server-110"));
|
||||
}
|
||||
|
||||
private void processInternal() throws Exception {
|
||||
Method method = TaskResultFileJobWorker.class
|
||||
.getDeclaredMethod("processInternal", TaskFileJobEntity.class);
|
||||
method.setAccessible(true);
|
||||
try {
|
||||
method.invoke(worker, job);
|
||||
} catch (InvocationTargetException ex) {
|
||||
throw (Exception) ex.getCause();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_归属被拒时必须先改挂再退回() throws Exception {
|
||||
stubRejectedByTaskOwner();
|
||||
|
||||
assertDoesNotThrow(this::processInternal);
|
||||
|
||||
InOrder order = inOrder(taskFileJobService);
|
||||
order.verify(taskFileJobService).handOffToOwner(23815L, "server-110", "server-121");
|
||||
order.verify(taskFileJobService).deferRunning(eq(23815L), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_改挂目标必须取任务当前_owner_而不是作业旧_owner() throws Exception {
|
||||
stubRejectedByTaskOwner();
|
||||
|
||||
processInternal();
|
||||
|
||||
ArgumentCaptor<String> target = ArgumentCaptor.forClass(String.class);
|
||||
verify(taskFileJobService).handOffToOwner(eq(23815L), anyString(), target.capture());
|
||||
assertEquals("server-121", target.getValue(),
|
||||
"取成 server-110 就是原地转交,死锁只是换了个方向");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_归属被拒不得计入_retry_预算() throws Exception {
|
||||
// 计入 retry_count 会让任务在归属机一切正常时被本机重试耗尽判死(原注释的语义,改动后仍须成立)
|
||||
stubRejectedByTaskOwner();
|
||||
|
||||
processInternal();
|
||||
|
||||
verify(taskFileJobService, never()).markFailed(any(TaskFileJobEntity.class), anyString());
|
||||
verify(taskFileJobService, never()).requeue(anyLong(), anyString());
|
||||
}
|
||||
}
|
||||
@@ -39,7 +39,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
* 那条数是 {@code grep -rl 'sourceText('} 数出来的,而第 9 支是 {@code SourceScan.java:22} 里
|
||||
* {@code public static String sourceText(String fileName)} 这行<b>被调方自己的声明</b>——它不是调用点。
|
||||
* 所以真实形状是 <b>调用点/调用方/引用方三类数各自成立</b>(2026-10-28 v1.1 Phase 0 起调用方 16 支、
|
||||
* 裸名 22 个,增量逐笔登记在 {@code REGISTERED_BARE_NAMES} 的注释里)。下面第 3 格把这些数同时钉住,
|
||||
* 裸名 23 个,增量逐笔登记在 {@code REGISTERED_BARE_NAMES} 的注释里)。下面第 3 格把这些数同时钉住,
|
||||
* 免得下一个人拿 {@code grep} 再数一遍又把声明算进去。</p>
|
||||
*/
|
||||
class SourceScanTest {
|
||||
@@ -77,7 +77,12 @@ class SourceScanTest {
|
||||
"TaskCenterQueryService.java",
|
||||
"TaskHeartbeatService.java",
|
||||
"AppearancePatentDeliverySupport.java",
|
||||
"RequestTraceFilter.java");
|
||||
"RequestTraceFilter.java",
|
||||
// —— 2026-11-02 跨实例归属死锁根治追加(1 名)——
|
||||
// TaskOwnerReadSourceGuardrailTest 要钉「被拒时必须先改挂再退回」这一枚顺序,
|
||||
// 顺序错在 deferRunning 把行翻成 PENDING 之后 handOffToOwner 的 CAS 就打不中,
|
||||
// 用 mock 造不出这个先后(它要跑的正是 catch 块里两行的次序),只能打在源码连线上。
|
||||
"TaskResultFileJobWorker.java");
|
||||
|
||||
@TempDir
|
||||
Path tmp;
|
||||
@@ -93,7 +98,7 @@ class SourceScanTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void census_唯一裸名恰为22且与已登记集合相等() {
|
||||
void census_唯一裸名恰为23且与已登记集合相等() {
|
||||
Set<String> found = SourceScan.censusSourceTextCallSites().stream()
|
||||
.map(SourceScan.CallSite::bareName)
|
||||
.collect(Collectors.toSet());
|
||||
@@ -101,11 +106,11 @@ class SourceScanTest {
|
||||
assertThat(found)
|
||||
.as("唯一裸名集合必须与类头台账逐字相等——多一个说明有人新引了单文件口径,少一个说明尺子漏扫")
|
||||
.isEqualTo(REGISTERED_BARE_NAMES)
|
||||
.hasSize(22);
|
||||
.hasSize(23);
|
||||
}
|
||||
|
||||
@Test
|
||||
void census_调用方文件恰为16而声明件在外() {
|
||||
void census_调用方文件恰为17而声明件在外() {
|
||||
List<SourceScan.CallSite> sites = SourceScan.censusSourceTextCallSites();
|
||||
Set<String> callerFiles = sites.stream()
|
||||
.map(SourceScan.CallSite::relativeFile)
|
||||
@@ -114,8 +119,9 @@ class SourceScanTest {
|
||||
String declarationFile = "src/test/java/com/nanri/aiimage/testutil/SourceScan.java";
|
||||
|
||||
assertThat(callerFiles)
|
||||
.as("真实字面量调用点落在 16 支调用方文件里(8 支既有 + 2026-10-28 v1.1 Phase 0 预置契约 8 支)")
|
||||
.hasSize(16);
|
||||
.as("真实字面量调用点落在 17 支调用方文件里(8 支既有 + 2026-10-28 v1.1 Phase 0 预置契约 8 支"
|
||||
+ " + 2026-11-02 归属死锁绊线 1 支)")
|
||||
.hasSize(17);
|
||||
assertThat(callerFiles)
|
||||
.as("plan 原文记的「9 支」= grep -rl 'sourceText(' 的条数,第 9 支是被调方自己的声明行"
|
||||
+ "(SourceScan.java:22 的 `public static String sourceText(String fileName)`),"
|
||||
|
||||
Reference in New Issue
Block a user