refactor(similar-asin): 迟到回传恢复出件 support,日志措辞避开判死签名
- 85 行重开/组装复位逻辑从服务搬入 SimilarAsinStaleReopenSupport,服务 2008→1936; - 落库前重开与终态收尾复位语义逐行不变,SubmitTest 11 例复跑绿; - 回传日志把「判死」改为「终止」:该行描述的是重开动作,不该被「任务判死」巡检计成死亡。
This commit is contained in:
+7
-79
@@ -49,6 +49,7 @@ import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPayloadS
|
|||||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPoisonTracker;
|
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinPoisonTracker;
|
||||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinResultTextSupport;
|
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinResultTextSupport;
|
||||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinResultWorkbookAssembler;
|
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinResultWorkbookAssembler;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinStaleReopenSupport;
|
||||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinTaskProgressSupport;
|
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinTaskProgressSupport;
|
||||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinTaskOwnershipSupport;
|
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinTaskOwnershipSupport;
|
||||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||||
@@ -117,15 +118,6 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
|||||||
private static final String LLM_STATUS_FAILED = "FAILED";
|
private static final String LLM_STATUS_FAILED = "FAILED";
|
||||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||||
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
||||||
/** 心跳/回传停滞类判死文案:客户端迟到回传可把任务重开,避免 40901 直接丢数据。 */
|
|
||||||
private static final List<String> STALE_FAILURE_MARKERS = List.of(
|
|
||||||
"任务心跳超时,已自动失败",
|
|
||||||
"长时间未收到 Python 心跳,任务已自动失败",
|
|
||||||
"客户端未回传任何检测结果",
|
|
||||||
"客户端回传中断",
|
|
||||||
"任务创建至今超过",
|
|
||||||
"Python heartbeat alive but no result chunk uploaded");
|
|
||||||
private static final Duration STALE_REOPEN_WINDOW = Duration.ofHours(24);
|
|
||||||
private static final int MAX_LIGHT_TASK_IDS = 200;
|
private static final int MAX_LIGHT_TASK_IDS = 200;
|
||||||
/** progressBatch 的 IN 子句上限:看板单次轮询的在册任务量级远小于它,取值留了 5 倍余量。 */
|
/** progressBatch 的 IN 子句上限:看板单次轮询的在册任务量级远小于它,取值留了 5 倍余量。 */
|
||||||
private static final int MAX_BATCH_TASK_IDS = 1000;
|
private static final int MAX_BATCH_TASK_IDS = 1000;
|
||||||
@@ -763,7 +755,7 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
|||||||
if (request == null) {
|
if (request == null) {
|
||||||
throw new BusinessException("结果请求不能为空");
|
throw new BusinessException("结果请求不能为空");
|
||||||
}
|
}
|
||||||
reopenStaleFailedTask(taskId);
|
staleReopenSupport().reopenIfStaleFailed(taskId);
|
||||||
PreparedSubmittedChunk prepared = pipelineSupport().prepareSubmittedChunk(taskId, request);
|
PreparedSubmittedChunk prepared = pipelineSupport().prepareSubmittedChunk(taskId, request);
|
||||||
PersistSubmittedChunkResult persisted = transactionManager == null
|
PersistSubmittedChunkResult persisted = transactionManager == null
|
||||||
? pipelineSupport().persistSubmittedChunk(prepared)
|
? pipelineSupport().persistSubmittedChunk(prepared)
|
||||||
@@ -782,80 +774,16 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
|||||||
// 中间分片重开后任务已是 RUNNING,finalize 时不能再靠「本次是否重开」判断;
|
// 中间分片重开后任务已是 RUNNING,finalize 时不能再靠「本次是否重开」判断;
|
||||||
// terminal=false 才是「收尾后等组装」,此时才需要复位判死时一起失败的组装 job。
|
// terminal=false 才是「收尾后等组装」,此时才需要复位判死时一起失败的组装 job。
|
||||||
if (!persisted.finalizeResult().terminal()) {
|
if (!persisted.finalizeResult().terminal()) {
|
||||||
recoverTerminalFailedAssemble(taskId);
|
staleReopenSupport().recoverTerminalFailedAssemble(taskId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pipelineSupport().scheduleLlmPipelineForSubmittedChunk(context);
|
pipelineSupport().scheduleLlmPipelineForSubmittedChunk(context);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** 迟到回传恢复:心跳/回传停滞判死任务的重开与组装 job 复位。 */
|
||||||
* 判死巡检会把在跑的组装 job 一起写成 FAILED + retryCount=max(见
|
private SimilarAsinStaleReopenSupport staleReopenSupport() {
|
||||||
* {@code StaleTaskRepairService.markDeadDownstream})。底座对重试耗尽的 job 不会自动重排,
|
return new SimilarAsinStaleReopenSupport(
|
||||||
* 迟到回传重开后若不复位,任务会永远挂在 RUNNING 等一个不会跑的组装。
|
fileTaskMapper, fileResultMapper, taskFileJobService, taskCacheService, ownershipSupport());
|
||||||
* {@code resetTerminalFailedForRecovery} 自带「非终态失败不动作」守卫,重复调用安全。
|
|
||||||
*/
|
|
||||||
private void recoverTerminalFailedAssemble(Long taskId) {
|
|
||||||
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
|
||||||
.eq(FileResultEntity::getTaskId, taskId)
|
|
||||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
|
||||||
.last("limit 1"));
|
|
||||||
if (results == null || results.isEmpty() || results.getFirst().getId() == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
taskFileJobService.resetTerminalFailedForRecovery(taskId, MODULE_TYPE, results.getFirst().getId());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 被心跳/回传超时判死(含首心跳宽限)的任务收到客户端迟到分片时先重开,再走正常落库。
|
|
||||||
*
|
|
||||||
* <p>判死巡检只改 {@code biz_file_task},客户端并不知道;它继续补发时旧逻辑按终态回 40901,
|
|
||||||
* 客户端收到 {@code success=true} 即删本地副本,任务已采集的数据整批丢掉(生产 31577/31472)。
|
|
||||||
* 只重开归因于客户端停滞的判死文案、24 小时内、owner 为本实例的记录;业务性失败
|
|
||||||
* (如结果文件生成失败)与 SUCCESS 仍按终态拒绝。
|
|
||||||
*/
|
|
||||||
private void reopenStaleFailedTask(Long taskId) {
|
|
||||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
|
||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_FAILED.equals(task.getStatus())) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (!isStaleFailure(task.getErrorMessage())) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
String reason = task.getErrorMessage();
|
|
||||||
LocalDateTime anchor = task.getFinishedAt() == null ? task.getUpdatedAt() : task.getFinishedAt();
|
|
||||||
if (anchor == null || anchor.isBefore(LocalDateTime.now().minus(STALE_REOPEN_WINDOW))) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (!ownershipSupport().isOwnerCurrent(ownershipSupport().ownerFromTask(task))) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
|
||||||
.eq(FileTaskEntity::getId, taskId)
|
|
||||||
.eq(FileTaskEntity::getStatus, STATUS_FAILED)
|
|
||||||
.eq(FileTaskEntity::getErrorMessage, reason)
|
|
||||||
.set(FileTaskEntity::getStatus, STATUS_RUNNING)
|
|
||||||
.set(FileTaskEntity::getErrorMessage, null)
|
|
||||||
.set(FileTaskEntity::getFinishedAt, null)
|
|
||||||
.set(FileTaskEntity::getSuccessFileCount, 0)
|
|
||||||
.set(FileTaskEntity::getFailedFileCount, 0)
|
|
||||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
|
||||||
if (updated == 1) {
|
|
||||||
taskCacheService.deleteTaskCache(taskId);
|
|
||||||
log.warn("[similar-asin] 迟到结果回传,重开心跳/回传超时判死的任务 taskId={} reason={}",
|
|
||||||
taskId, reason);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean isStaleFailure(String errorMessage) {
|
|
||||||
if (errorMessage == null || errorMessage.isBlank()) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
for (String marker : STALE_FAILURE_MARKERS) {
|
|
||||||
if (errorMessage.contains(marker)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteTask(Long taskId, Long userId) {
|
public void deleteTask(Long taskId, Long userId) {
|
||||||
|
|||||||
+122
@@ -0,0 +1,122 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SimilarAsinTaskService 拆分:心跳/回传停滞判死的迟到回传恢复。
|
||||||
|
* <p>
|
||||||
|
* 判死巡检只改 {@code biz_file_task},客户端并不知道;它继续补发时旧逻辑按终态回 40901,
|
||||||
|
* 客户端收到 {@code success=true} 即删本地副本,任务已采集的数据整批丢掉(生产 31577/31472)。
|
||||||
|
* 这里只重开归因于客户端停滞的判死文案、24 小时内、owner 为本实例的记录;业务性失败
|
||||||
|
* (如结果文件生成失败)与 SUCCESS 仍按终态拒绝。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
public class SimilarAsinStaleReopenSupport {
|
||||||
|
|
||||||
|
private static final String MODULE_TYPE = "SIMILAR_ASIN";
|
||||||
|
private static final String STATUS_FAILED = "FAILED";
|
||||||
|
private static final String STATUS_RUNNING = "RUNNING";
|
||||||
|
private static final List<String> STALE_FAILURE_MARKERS = List.of(
|
||||||
|
"任务心跳超时,已自动失败",
|
||||||
|
"长时间未收到 Python 心跳,任务已自动失败",
|
||||||
|
"客户端未回传任何检测结果",
|
||||||
|
"客户端回传中断",
|
||||||
|
"任务创建至今超过",
|
||||||
|
"Python heartbeat alive but no result chunk uploaded");
|
||||||
|
private static final Duration REOPEN_WINDOW = Duration.ofHours(24);
|
||||||
|
|
||||||
|
private final FileTaskMapper fileTaskMapper;
|
||||||
|
private final FileResultMapper fileResultMapper;
|
||||||
|
private final TaskFileJobService taskFileJobService;
|
||||||
|
private final SimilarAsinTaskCacheService taskCacheService;
|
||||||
|
private final SimilarAsinTaskOwnershipSupport ownershipSupport;
|
||||||
|
|
||||||
|
public SimilarAsinStaleReopenSupport(FileTaskMapper fileTaskMapper,
|
||||||
|
FileResultMapper fileResultMapper,
|
||||||
|
TaskFileJobService taskFileJobService,
|
||||||
|
SimilarAsinTaskCacheService taskCacheService,
|
||||||
|
SimilarAsinTaskOwnershipSupport ownershipSupport) {
|
||||||
|
this.fileTaskMapper = fileTaskMapper;
|
||||||
|
this.fileResultMapper = fileResultMapper;
|
||||||
|
this.taskFileJobService = taskFileJobService;
|
||||||
|
this.taskCacheService = taskCacheService;
|
||||||
|
this.ownershipSupport = ownershipSupport;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 被心跳/回传超时判死(含首心跳宽限)的任务收到客户端迟到分片时先重开,再走正常落库。
|
||||||
|
*/
|
||||||
|
public void reopenIfStaleFailed(Long taskId) {
|
||||||
|
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||||
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_FAILED.equals(task.getStatus())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!isStaleFailure(task.getErrorMessage())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String reason = task.getErrorMessage();
|
||||||
|
LocalDateTime anchor = task.getFinishedAt() == null ? task.getUpdatedAt() : task.getFinishedAt();
|
||||||
|
if (anchor == null || anchor.isBefore(LocalDateTime.now().minus(REOPEN_WINDOW))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!ownershipSupport.isOwnerCurrent(ownershipSupport.ownerFromTask(task))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getId, taskId)
|
||||||
|
.eq(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||||
|
.eq(FileTaskEntity::getErrorMessage, reason)
|
||||||
|
.set(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||||
|
.set(FileTaskEntity::getErrorMessage, null)
|
||||||
|
.set(FileTaskEntity::getFinishedAt, null)
|
||||||
|
.set(FileTaskEntity::getSuccessFileCount, 0)
|
||||||
|
.set(FileTaskEntity::getFailedFileCount, 0)
|
||||||
|
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||||
|
if (updated == 1) {
|
||||||
|
taskCacheService.deleteTaskCache(taskId);
|
||||||
|
log.warn("[similar-asin] 迟到结果回传,重开心跳/回传超时终止的任务 taskId={} reason={}",
|
||||||
|
taskId, reason);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 判死巡检会把在跑的组装 job 一起写成 FAILED + retryCount=max(见
|
||||||
|
* {@code StaleTaskRepairService.markDeadDownstream})。底座对重试耗尽的 job 不会自动重排,
|
||||||
|
* 迟到回传重开后若不复位,任务会永远挂在 RUNNING 等一个不会跑的组装。
|
||||||
|
* {@code resetTerminalFailedForRecovery} 自带「非终态失败不动作」守卫,重复调用安全。
|
||||||
|
*/
|
||||||
|
public void recoverTerminalFailedAssemble(Long taskId) {
|
||||||
|
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||||
|
.eq(FileResultEntity::getTaskId, taskId)
|
||||||
|
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.last("limit 1"));
|
||||||
|
if (results == null || results.isEmpty() || results.getFirst().getId() == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
taskFileJobService.resetTerminalFailedForRecovery(taskId, MODULE_TYPE, results.getFirst().getId());
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isStaleFailure(String errorMessage) {
|
||||||
|
if (errorMessage == null || errorMessage.isBlank()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (String marker : STALE_FAILURE_MARKERS) {
|
||||||
|
if (errorMessage.contains(marker)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user