Compare commits
3 Commits
ccb30aa99e
...
dff0658b32
| Author | SHA1 | Date | |
|---|---|---|---|
| dff0658b32 | |||
| bc818eb3ea | |||
| 5f1dc3ac23 |
+32
-17
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.task.TaskStatuses;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
@@ -102,6 +103,25 @@ public class TaskFileJobService {
|
||||
return retry >= MAX_RETRY_COUNT;
|
||||
}
|
||||
|
||||
/**
|
||||
* 该任务名下还有没有**未终态**的组装作业(PENDING/RUNNING)。
|
||||
*
|
||||
* <p>交付缺口触发的补检轮就是把作业排回这两态({@link #requeue}),所以"有活着的组装作业"
|
||||
* ="这单还在补检"。而任务行可能早就落成 FAILED(心跳超时或交付判死先落),此时按任务状态
|
||||
* 去克隆重跑,等于同一份活开两份:现网 2026-10-24 一枚 760 行的外观专利因此被克隆成 4 枚、
|
||||
* 每枚都全量重跑一遍,四枚卡片的开始时间还都是同一秒(克隆件继承原件 created_at)。
|
||||
*/
|
||||
public boolean hasUnfinishedAssembleJob(Long taskId) {
|
||||
if (taskId == null) {
|
||||
return false;
|
||||
}
|
||||
Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getTaskId, taskId)
|
||||
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
|
||||
.in(TaskFileJobEntity::getStatus, TaskStatuses.Job.PENDING, TaskStatuses.Job.RUNNING));
|
||||
return count != null && count > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* 补传完整后恢复终态失败的组装 job:重置为 PENDING(retryCount 清零)重新派发。
|
||||
* 幂等:job 不存在、非 FAILED 或未达重试上限时不动作,返回当前 job。
|
||||
@@ -310,6 +330,10 @@ public class TaskFileJobService {
|
||||
* 每天上百次 "Lock wait timeout exceeded"(biz_task_file_job)。
|
||||
* 这里的每行更新本就带 status + updatedAt 的 CAS 条件(幂等),逐行独立提交更安全:
|
||||
* 锁即时释放,CAS 不匹配的一方返回 0 行即可,也不会因中途异常回滚掉已修好的行。
|
||||
*
|
||||
* <p>RUNNING 行的重排**不消耗** retry_count(2026-10-24 分账口径):扫描只负责把卡住的行
|
||||
* 放回队列,预算只由真失败({@link #markFailed})推进——"补检一轮要几十分钟"的慢执行不该
|
||||
* 被记成失败次数。PENDING 行的累加保留:那条防的是 dispatch 事件无限重发,与执行快慢无关。
|
||||
*/
|
||||
public StuckJobResetResult resetStuckRunningJobsDetailed(int stuckMinutes, int limit) {
|
||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes));
|
||||
@@ -367,30 +391,21 @@ public class TaskFileJobService {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
||||
int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1);
|
||||
// 重排**不消耗**重试预算(2026-10-24 拍板的分账口径):卡死扫描只负责把卡住的
|
||||
// RUNNING 行放回队列,retry_count 只由真失败(markFailed)推进。之前这里 +1 并在
|
||||
// MAX 时就地判死,把"补检一轮要几十分钟"的慢执行误记成一次失败——现网一枚 760 行
|
||||
// 的外观专利被扫描连扣 4 次,逼到「文件生成任务运行超时,已达到最大重试次数」后
|
||||
// 0 字节收场。代价(拍板时已接受):每次重跑都无异常挂住的真卡死会循环重排、
|
||||
// 不再有扫描侧终态判死;判死只剩 markFailed 这一条真失败的路。
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LambdaUpdateWrapper<TaskFileJobEntity> update = new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now);
|
||||
if (nextRetryCount >= MAX_RETRY_COUNT) {
|
||||
int updated = taskFileJobMapper.update(null, update
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已达到最大重试次数")
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (updated > 0) {
|
||||
exhaustedJobIds.add(job.getId());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
int updated = taskFileJobMapper.update(null, update
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (updated > 0) {
|
||||
|
||||
+17
@@ -65,6 +65,11 @@ public class TaskResumeService {
|
||||
private static final int DUPLICATE_SWEEP_LIMIT = 200;
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
/**
|
||||
* 查"这单名下还有没有活着的组装作业"——补检轮还活着的意思,见
|
||||
* {@link TaskFileJobService#hasUnfinishedAssembleJob(Long)}。
|
||||
*/
|
||||
private final TaskFileJobService taskFileJobs;
|
||||
/** moduleType → 克隆重试实现;只有注册了本 SPI 的模块才会自动重试。 */
|
||||
private final Map<String, ClientTaskRetrySpi> retryHandlers;
|
||||
|
||||
@@ -87,8 +92,10 @@ public class TaskResumeService {
|
||||
private int limit;
|
||||
|
||||
public TaskResumeService(FileTaskMapper fileTaskMapper,
|
||||
TaskFileJobService taskFileJobs,
|
||||
List<ClientTaskRetrySpi> retrySpiHandlers) {
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
this.taskFileJobs = taskFileJobs;
|
||||
Map<String, ClientTaskRetrySpi> retryIndex = new LinkedHashMap<>();
|
||||
if (retrySpiHandlers != null) {
|
||||
for (ClientTaskRetrySpi handler : retrySpiHandlers) {
|
||||
@@ -216,6 +223,16 @@ public class TaskResumeService {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
// 任务行是 FAILED,不等于这份活干完了:交付缺口会把组装作业排回 PENDING/RUNNING 去补检
|
||||
// (见 AppearancePatentDeliverySupport.requestRecheckForMissingRows),此时克隆=同一份活
|
||||
// 开两份、还各按全量重跑一遍。作业真卡死另有 TaskResultFileJobWorker 的 stuck 扫描兜底,
|
||||
// 不会因为这条例外就永远没人管。
|
||||
if (taskFileJobs.hasUnfinishedAssembleJob(original.getId())) {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[task-resume] 不重试 taskId={} moduleType={} 原因=组装作业仍在补检(PENDING/RUNNING)",
|
||||
original.getId(), original.getModuleType());
|
||||
continue;
|
||||
}
|
||||
ClientTaskRetrySpi handler = retryHandlers.get(normalize(original.getModuleType()));
|
||||
if (handler == null) {
|
||||
stats.skippedTaskCount++;
|
||||
|
||||
@@ -67,7 +67,7 @@ class MigrationInventoryTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* 不许再被占用的整数版本洞。两条来历不同,得分开记:
|
||||
* 不许再被占用的整数版本洞。三条来历不同,得分开记:
|
||||
*
|
||||
* <p><b>132/133</b>:已在生产 {@code flyway_schema_history} 中占位、但本地文件被改名让位的版本号。
|
||||
* 生产 131 之后直接装了 134/135(2026-09-25 20:46:16),132/133 从未落库;而 Flyway 未开
|
||||
@@ -82,9 +82,14 @@ class MigrationInventoryTest {
|
||||
* 而同号两个文件会让 Flyway 启动期报 "Found more than one migration with version 148",
|
||||
* 后果是两台 java 同时起不来。宁可让本用例红,也不把撞号留到运行期发现。</p>
|
||||
*
|
||||
* <p><b>168/169</b>:从未有文件占用过的纯跳号,归 cost-guard 那一路({@code V170__user_task_guard.sql},
|
||||
* commit 9c4905c9)——写 170 时直接越过 168/169,全 git 历史核对零命中,两号无人落盘。
|
||||
* 目录最大值升到 170 之后若回填新内容,已应用 170 的环境在 Flyway out-of-order 关闭下会把它当
|
||||
* ignored migration 静默跳过,各环境 Schema 悄悄漂移。与 148 不同这不是并发护栏,是既成跳号的封账。</p>
|
||||
*
|
||||
* <p>共同后果:新迁移只能取当前最大值之上的号(详见 docs/migration-inventory.md)。</p>
|
||||
*/
|
||||
private static final Set<Integer> RETIRED_HOLES = Set.of(132, 133, 148);
|
||||
private static final Set<Integer> RETIRED_HOLES = Set.of(132, 133, 148, 168, 169);
|
||||
|
||||
@Test
|
||||
void integerVersionsSequentialFromOne() throws IOException {
|
||||
|
||||
+26
-16
@@ -44,9 +44,9 @@ class TaskFileJobServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void stuckJobIsRequeuedAndRetryCountIsIncremented() {
|
||||
void stuckJobIsRequeuedWithoutConsumingRetryBudget() {
|
||||
TaskFileJobEntity running = runningJob(101L, 3, LocalDateTime.now().minusHours(1));
|
||||
TaskFileJobEntity pending = runningJob(101L, 4, LocalDateTime.now());
|
||||
TaskFileJobEntity pending = runningJob(101L, 3, LocalDateTime.now());
|
||||
pending.setStatus("PENDING");
|
||||
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
@@ -58,29 +58,29 @@ class TaskFileJobServiceTest {
|
||||
assertEquals(1, result.resetCount());
|
||||
assertTrue(result.exhaustedJobs().isEmpty());
|
||||
verify(applicationEventPublisher).publishEvent(any(TaskFileJobDispatchEvent.class));
|
||||
assertUpdateContains(4, running.getUpdatedAt());
|
||||
assertUpdateLeavesRetryCountIntact(running.getUpdatedAt());
|
||||
}
|
||||
|
||||
/**
|
||||
* 现网 2026-10-24:一枚 760 行的外观专利组装作业在补检轮里被卡死扫描连扣预算,
|
||||
* RUNNING + retryCount=4 一进扫描就被就地判死「文件生成任务运行超时,已达到最大重试次数」,
|
||||
* 结果文件 0 字节收场。改口径后扫描只放回队列:到达 MAX 的终态只能由真失败(markFailed)走出。
|
||||
*/
|
||||
@Test
|
||||
void stuckJobAtLastRetryWaitsForDurableFailureFinalization() {
|
||||
void stuckJobAtLastRetryIsRequeuedWithoutTerminalKill() {
|
||||
TaskFileJobEntity running = runningJob(102L, 4, LocalDateTime.now().minusHours(1));
|
||||
TaskFileJobEntity failed = runningJob(102L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
|
||||
failed.setStatus("FAILED");
|
||||
/*
|
||||
failed.setErrorMessage("文件生成任务运行超时,已达到最大重试次数");
|
||||
*/
|
||||
failed.setErrorMessage("result file job timeout");
|
||||
TaskFileJobEntity pending = runningJob(102L, 4, LocalDateTime.now());
|
||||
pending.setStatus("PENDING");
|
||||
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(failed));
|
||||
when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(pending));
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
|
||||
|
||||
assertEquals(0, result.resetCount());
|
||||
assertEquals(List.of(failed), result.exhaustedJobs());
|
||||
verify(applicationEventPublisher, never()).publishEvent(any());
|
||||
assertUpdateContains(TaskFileJobService.MAX_RETRY_COUNT, running.getUpdatedAt());
|
||||
assertEquals(1, result.resetCount());
|
||||
assertTrue(result.exhaustedJobs().isEmpty());
|
||||
assertUpdateLeavesRetryCountIntact(running.getUpdatedAt());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -125,7 +125,7 @@ class TaskFileJobServiceTest {
|
||||
assertTrue(result.exhaustedJobs().isEmpty());
|
||||
verify(taskFileJobMapper, never()).selectById(any());
|
||||
verify(applicationEventPublisher, never()).publishEvent(any());
|
||||
assertUpdateContains(TaskFileJobService.MAX_RETRY_COUNT, running.getUpdatedAt());
|
||||
assertUpdateLeavesRetryCountIntact(running.getUpdatedAt());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -308,6 +308,16 @@ class TaskFileJobServiceTest {
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue(nextRetry));
|
||||
}
|
||||
|
||||
/** 卡死重排的分账口径:SET 片段里不许出现 retry_count,预算只由真失败(markFailed)推进。 */
|
||||
private void assertUpdateLeavesRetryCountIntact(LocalDateTime updatedAt) {
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||
assertTrue(update.getValue().getSqlSegment().contains("updated_at"));
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue(updatedAt));
|
||||
assertFalse(update.getValue().getSqlSet().contains("retry_count"),
|
||||
"卡死重排不许碰 retry_count");
|
||||
}
|
||||
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
private static ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> updateCaptor() {
|
||||
return ArgumentCaptor.forClass((Class) LambdaUpdateWrapper.class);
|
||||
|
||||
+35
-1
@@ -60,7 +60,15 @@ class TaskResumeServiceTest {
|
||||
|
||||
private static TaskResumeService service(FileTaskMapper mapper, List<ClientTaskRetrySpi> handlers,
|
||||
boolean enabled, int maxAttempt) {
|
||||
TaskResumeService service = new TaskResumeService(mapper, handlers);
|
||||
// 默认"名下没有活着的组装作业"(Mockito 对 boolean 返 false):既有用例判的都是克隆本体,
|
||||
// 补检豁免那条单独用 serviceWithLiveAssembleJob 覆盖。
|
||||
return service(mapper, mock(TaskFileJobService.class), handlers, enabled, maxAttempt);
|
||||
}
|
||||
|
||||
private static TaskResumeService service(FileTaskMapper mapper, TaskFileJobService jobs,
|
||||
List<ClientTaskRetrySpi> handlers,
|
||||
boolean enabled, int maxAttempt) {
|
||||
TaskResumeService service = new TaskResumeService(mapper, jobs, handlers);
|
||||
ReflectionTestUtils.setField(service, "enabled", enabled);
|
||||
ReflectionTestUtils.setField(service, "maxAttempt", maxAttempt);
|
||||
ReflectionTestUtils.setField(service, "windowMinutes", 30L);
|
||||
@@ -138,6 +146,32 @@ class TaskResumeServiceTest {
|
||||
verify(spi, never()).cloneForRetry(any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 现网 2026-10-24:一枚 760 行的外观专利任务因交付缺口把组装作业排回补检(作业还在
|
||||
* PENDING/RUNNING),任务行却已是 FAILED,于是被巡检连克隆成 4 枚、每枚全量重跑一遍,
|
||||
* 四枚卡片开始时间还同秒(克隆件继承原件 created_at)。非零对照=上面那条
|
||||
* 「失败任务会被克隆重试」:作业已终态时照旧克隆。
|
||||
*/
|
||||
@Test
|
||||
void 组装作业仍在补检时不克隆() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(
|
||||
failedTask(32201L, "APPEARANCE_PATENT", "有 3 行抓取失败(未取得商品页数据):B07P5TR7Q5")));
|
||||
when(mapper.selectCount(any())).thenReturn(0L);
|
||||
TaskFileJobService jobs = mock(TaskFileJobService.class);
|
||||
when(jobs.hasUnfinishedAssembleJob(32201L)).thenReturn(true);
|
||||
ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class);
|
||||
when(spi.moduleType()).thenReturn("APPEARANCE_PATENT");
|
||||
TaskResumeService service = service(mapper, jobs, List.of(spi), true, 3);
|
||||
|
||||
TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks();
|
||||
|
||||
assertEquals(1, stats.scannedTaskCount);
|
||||
assertEquals(1, stats.skippedTaskCount);
|
||||
assertEquals(0, stats.resumedTaskCount);
|
||||
verify(spi, never()).cloneForRetry(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void 重试代数到上限后跳过() {
|
||||
FileTaskMapper mapper = mock(FileTaskMapper.class);
|
||||
|
||||
@@ -53,6 +53,8 @@
|
||||
<div v-if="pendingResultHint(itemSource(item))" class="files result-hint">{{ pendingResultHint(itemSource(item)) }}</div>
|
||||
</template>
|
||||
<template #item-actions="{ item }">
|
||||
<button v-if="canDownload(itemSource(item))" type="button" class="download"
|
||||
@click="downloadResult(itemSource(item))">下载</button>
|
||||
<button type="button" class="btn-delete" @click="deleteTaskRecord(itemSource(item))">删除</button>
|
||||
</template>
|
||||
<template #history-item-actions="{ item }">
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* 「当前任务」那一行必须有下载入口的用例(2026-10-24 现网)。
|
||||
*
|
||||
* 症状:外观专利任务的结果文件已经生成(result_file_url 非空),用户在「当前任务」卡片上
|
||||
* 仍然点不到下载——因为 `#item-actions` 里只挂了「删除」,下载只在 `#history-item-actions`。
|
||||
* 用户因此以为"失败了不能下载",实际是这一格入口缺失(另一格是后端补检期间故意不出表,
|
||||
* 那条判据见 result-build-failed-when-downloadable.test.ts,两件事别混)。
|
||||
*
|
||||
* 判据与历史区共用同一个 canDownload:只看文件到没到,不看任务状态。
|
||||
*
|
||||
* 跑法:`node --test tests/appearance-patent-current-task-download.test.ts`
|
||||
*/
|
||||
import { test } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { resolve, dirname } from 'node:path'
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url))
|
||||
const tabSource = readFileSync(
|
||||
resolve(here, '../src/pages/brand/components/BrandAppearancePatentTab.vue'),
|
||||
'utf-8',
|
||||
)
|
||||
|
||||
function slot(name: string): string {
|
||||
const match = tabSource.match(
|
||||
new RegExp(`<template #${name}="\\{ item \\}">([\\s\\S]*?)</template>`),
|
||||
)
|
||||
assert.ok(match, `找不到 #${name} 插槽`)
|
||||
return match[1]
|
||||
}
|
||||
|
||||
test('当前任务区必须有下载按钮', () => {
|
||||
const current = slot('item-actions')
|
||||
assert.match(
|
||||
current,
|
||||
/v-if="canDownload\(itemSource\(item\)\)"[^>]*>\s*下载\s*<\/button>/,
|
||||
'「当前任务」那一行要按 canDownload 出下载按钮,不能只有删除',
|
||||
)
|
||||
})
|
||||
|
||||
test('当前任务与历史区的下载判据必须是同一个', () => {
|
||||
const current = slot('item-actions')
|
||||
const history = slot('history-item-actions')
|
||||
assert.equal(
|
||||
/canDownload\(itemSource\(item\)\)/.test(current),
|
||||
/canDownload\(itemSource\(item\)\)/.test(history),
|
||||
'两处都得到判据;判据不一致等于同一枚任务在两个位置说法不同',
|
||||
)
|
||||
assert.match(current, /deleteTaskRecord\(itemSource\(item\)\)/, '删除按钮不能顺手丢掉')
|
||||
})
|
||||
|
||||
test('下载判据只看文件,不看任务状态', () => {
|
||||
const match = tabSource.match(/function canDownload\([\s\S]*?\n\}/)
|
||||
assert.ok(match, '找不到 canDownload')
|
||||
const body = match[0]
|
||||
assert.match(body, /item\.resultId/, '没有结果行就不该给下载')
|
||||
assert.match(body, /item\.fileReady \|\| item\.downloadUrl/, '文件到了才给下载')
|
||||
assert.doesNotMatch(
|
||||
body,
|
||||
/taskStatus|fileStatus|FAILED/,
|
||||
'判据里出现任务状态就是回归:失败态同样要能下载已生成的文件',
|
||||
)
|
||||
})
|
||||
Reference in New Issue
Block a user