后端架构更新

This commit is contained in:
super
2026-04-27 09:18:10 +08:00
parent 8ea7e949c2
commit 7e1974a688
141 changed files with 6957 additions and 772 deletions
@@ -0,0 +1,52 @@
package com.nanri.aiimage.modules.task.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.Map;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/task-file-jobs")
@Tag(name = "任务结果文件 Job")
public class TaskFileJobController {
private final TaskFileJobService taskFileJobService;
@GetMapping
@Operation(summary = "查询结果文件生成 Job")
public ApiResponse<List<TaskFileJobEntity>> list(
@RequestParam(value = "status", required = false) String status,
@RequestParam(value = "module_type", required = false) String moduleType,
@RequestParam(value = "task_id", required = false) Long taskId,
@RequestParam(value = "limit", defaultValue = "100") int limit) {
return ApiResponse.success(taskFileJobService.listJobs(status, moduleType, taskId, limit));
}
@PostMapping("/{jobId}/retry")
@Operation(summary = "手动重试结果文件生成 Job")
public ApiResponse<Map<String, Object>> retry(@PathVariable Long jobId) {
boolean updated = taskFileJobService.retry(jobId);
return ApiResponse.success(Map.of("jobId", jobId, "updated", updated));
}
@PostMapping("/stuck/reset")
@Operation(summary = "手动扫描并复位卡住的 RUNNING Job")
public ApiResponse<Map<String, Object>> resetStuck(
@RequestParam(value = "timeout_minutes", defaultValue = "30") int timeoutMinutes,
@RequestParam(value = "limit", defaultValue = "100") int limit) {
int reset = taskFileJobService.resetStuckRunningJobs(timeoutMinutes, limit);
return ApiResponse.success(Map.of("reset", reset));
}
}
@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskFileJobMapper extends BaseMapper<TaskFileJobEntity> {
}
@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskProgressSnapshotMapper extends BaseMapper<TaskProgressSnapshotEntity> {
}
@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskResultItemMapper extends BaseMapper<TaskResultItemEntity> {
}
@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskResultPayloadMapper extends BaseMapper<TaskResultPayloadEntity> {
}
@@ -0,0 +1,14 @@
package com.nanri.aiimage.modules.task.model.dto;
import java.time.LocalDateTime;
public record TaskFileJobDispatchEvent(
Long jobId,
Long taskId,
String moduleType,
Long resultId,
String scopeKey,
String jobType,
LocalDateTime createdAt
) {
}
@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.task.model.dto;
import lombok.Data;
import java.time.LocalDateTime;
@Data
public class TaskFileJobMessage {
private Long jobId;
private Long taskId;
private String moduleType;
private Long resultId;
private String scopeKey;
private String jobType;
private LocalDateTime createdAt;
}
@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_file_job")
public class TaskFileJobEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private Long resultId;
private String scopeKey;
private String jobType;
private String status;
private Integer retryCount;
private String errorMessage;
private String resultFileUrl;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
private LocalDateTime finishedAt;
}
@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_progress_snapshot")
public class TaskProgressSnapshotEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private String status;
private Integer totalCount;
private Integer successCount;
private Integer failedCount;
private Integer pendingCount;
private String currentScopeKey;
private String message;
private String snapshotJson;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,29 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_result_item")
public class TaskResultItemEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private Long resultId;
private String scopeKey;
private String scopeHash;
private String itemKey;
private String countryCode;
private String asin;
private String status;
private String payloadJson;
private String payloadHash;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,26 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_result_payload")
public class TaskResultPayloadEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private String scopeKey;
private String scopeHash;
private String submissionId;
private String payloadJson;
private String payloadHash;
private Long version;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,39 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.spring.annotation.RocketMQMessageListener;
import org.apache.rocketmq.spring.core.RocketMQListener;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "aiimage.result-file-job", name = "mq-enabled", havingValue = "true")
@RocketMQMessageListener(
topic = "${aiimage.result-file-job.topic:aiimage-result-file-job}",
consumerGroup = "${aiimage.result-file-job.consumer-group:aiimage-result-file-job-consumer}"
)
public class TaskFileJobConsumer implements RocketMQListener<TaskFileJobMessage> {
private final TaskFileJobMapper taskFileJobMapper;
private final TaskResultFileJobWorker taskResultFileJobWorker;
@Override
public void onMessage(TaskFileJobMessage message) {
if (message == null || message.getJobId() == null) {
return;
}
TaskFileJobEntity job = taskFileJobMapper.selectById(message.getJobId());
if (job == null) {
log.warn("[task-file-job] mq message ignored, job not found jobId={} taskId={} moduleType={}",
message.getJobId(), message.getTaskId(), message.getModuleType());
return;
}
taskResultFileJobWorker.process(job);
}
}
@@ -0,0 +1,42 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.transaction.event.TransactionPhase;
import org.springframework.transaction.event.TransactionalEventListener;
@Slf4j
@Component
@RequiredArgsConstructor
public class TaskFileJobDispatchCoordinator {
private final TaskFileJobPublisher taskFileJobPublisher;
private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher;
@TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT, fallbackExecution = true)
public void onTaskFileJobDispatch(TaskFileJobDispatchEvent event) {
if (event == null || event.jobId() == null) {
return;
}
TaskFileJobMessage message = new TaskFileJobMessage();
message.setJobId(event.jobId());
message.setTaskId(event.taskId());
message.setModuleType(event.moduleType());
message.setResultId(event.resultId());
message.setScopeKey(event.scopeKey());
message.setJobType(event.jobType());
message.setCreatedAt(event.createdAt());
boolean published = taskFileJobPublisher.publish(message);
if (published) {
return;
}
boolean dispatched = taskFileJobLocalDispatcher.dispatch(event.jobId(), event.taskId(), event.moduleType());
if (dispatched) {
log.info("[task-file-job] local async dispatch scheduled jobId={} taskId={} moduleType={}",
event.jobId(), event.taskId(), event.moduleType());
}
}
}
@@ -0,0 +1,53 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.task.TaskExecutor;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskFileJobLocalDispatcher {
private final TaskFileJobMapper taskFileJobMapper;
private final ObjectProvider<TaskResultFileJobWorker> taskResultFileJobWorkerProvider;
@Qualifier("taskFileJobDispatchExecutor")
private final TaskExecutor taskFileJobDispatchExecutor;
@Value("${aiimage.result-file-job.local-dispatch-enabled:true}")
private boolean localDispatchEnabled;
public boolean dispatch(Long jobId, Long taskId, String moduleType) {
if (!localDispatchEnabled || jobId == null || jobId <= 0) {
return false;
}
taskFileJobDispatchExecutor.execute(() -> {
try {
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
if (job == null) {
log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}",
jobId, taskId, moduleType);
return;
}
TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable();
if (taskResultFileJobWorker == null) {
log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}",
jobId, taskId, moduleType);
return;
}
taskResultFileJobWorker.process(job);
} catch (Exception ex) {
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
jobId, taskId, moduleType, ex.getMessage(), ex);
}
});
return true;
}
}
@@ -0,0 +1,50 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.spring.core.RocketMQTemplate;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskFileJobPublisher {
private final ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider;
@Value("${aiimage.result-file-job.mq-enabled:false}")
private boolean mqEnabled;
@Value("${aiimage.result-file-job.topic:aiimage-result-file-job}")
private String topic;
public boolean publish(TaskFileJobMessage message) {
if (message == null || message.getJobId() == null) {
return false;
}
if (!mqEnabled) {
log.info("[task-file-job] mq disabled, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={}",
message.getJobId(), message.getTaskId(), message.getModuleType());
return false;
}
RocketMQTemplate template = rocketMQTemplateProvider.getIfAvailable();
if (template == null) {
log.warn("[task-file-job] RocketMQTemplate unavailable, jobId={} taskId={} moduleType={}",
message.getJobId(), message.getTaskId(), message.getModuleType());
return false;
}
try {
template.convertAndSend(topic, message);
log.info("[task-file-job] mq published jobId={} taskId={} moduleType={} topic={}",
message.getJobId(), message.getTaskId(), message.getModuleType(), topic);
return true;
} catch (Exception ex) {
log.warn("[task-file-job] publish RocketMQ failed, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} msg={}",
message.getJobId(), message.getTaskId(), message.getModuleType(), ex.getMessage());
return false;
}
}
}
@@ -0,0 +1,225 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.util.List;
@Service
@RequiredArgsConstructor
public class TaskFileJobService {
public static final String JOB_TYPE_ASSEMBLE_RESULT = "ASSEMBLE_RESULT";
private final TaskFileJobMapper taskFileJobMapper;
private final ApplicationEventPublisher applicationEventPublisher;
@Transactional
public TaskFileJobEntity enqueueAssembleResult(Long taskId, String moduleType, Long resultId, String scopeKey) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank() || resultId == null || resultId <= 0) {
return null;
}
LocalDateTime now = LocalDateTime.now();
TaskFileJobEntity existing = findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
if (existing == null) {
TaskFileJobEntity entity = new TaskFileJobEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setResultId(resultId);
entity.setScopeKey(scopeKey);
entity.setJobType(JOB_TYPE_ASSEMBLE_RESULT);
entity.setStatus("PENDING");
entity.setRetryCount(0);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskFileJobMapper.insert(entity);
publishDispatchEvent(entity);
return entity;
} catch (DuplicateKeyException ignored) {
existing = findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
}
}
if (existing == null) {
return null;
}
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus())) {
return existing;
}
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, existing.getId())
.set(TaskFileJobEntity::getScopeKey, scopeKey)
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getUpdatedAt, now)
.set(TaskFileJobEntity::getFinishedAt, null));
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(existing.getId());
publishDispatchEvent(refreshed);
return refreshed;
}
public List<TaskFileJobEntity> listRunnableJobs(int limit) {
return taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.lt(TaskFileJobEntity::getRetryCount, 5)
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + Math.max(1, Math.min(limit, 100))));
}
public List<TaskFileJobEntity> listJobs(String status, String moduleType, Long taskId, int limit) {
LambdaQueryWrapper<TaskFileJobEntity> wrapper = new LambdaQueryWrapper<TaskFileJobEntity>()
.orderByDesc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + Math.max(1, Math.min(limit, 500)));
if (status != null && !status.isBlank()) {
wrapper.eq(TaskFileJobEntity::getStatus, status.trim());
}
if (moduleType != null && !moduleType.isBlank()) {
wrapper.eq(TaskFileJobEntity::getModuleType, moduleType.trim());
}
if (taskId != null && taskId > 0) {
wrapper.eq(TaskFileJobEntity::getTaskId, taskId);
}
return taskFileJobMapper.selectList(wrapper);
}
@Transactional
public boolean retry(Long jobId) {
if (jobId == null || jobId <= 0) {
return false;
}
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, jobId)
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getRetryCount, 0)
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, null));
if (updated > 0) {
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(jobId);
publishDispatchEvent(refreshed);
return true;
}
return false;
}
@Transactional
public int resetStuckRunningJobs(int stuckMinutes, int limit) {
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes));
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.lt(TaskFileJobEntity::getUpdatedAt, threshold)
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + Math.max(1, Math.min(limit, 200))));
int reset = 0;
for (TaskFileJobEntity job : jobs) {
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
if (retryCount >= 5) {
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
continue;
}
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
if (updated > 0) {
reset++;
publishDispatchEvent(taskFileJobMapper.selectById(job.getId()));
}
}
return reset;
}
@Transactional
public boolean markRunning(Long jobId) {
if (jobId == null || jobId <= 0) {
return false;
}
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, jobId)
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.set(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())) > 0;
}
@Transactional
public void markSuccess(TaskFileJobEntity job, String resultFileUrl) {
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.set(TaskFileJobEntity::getStatus, "SUCCESS")
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getResultFileUrl, resultFileUrl)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
}
@Transactional
public void markFailed(TaskFileJobEntity job, String message) {
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount() + 1;
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getRetryCount, retryCount)
.set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
}
public TaskFileJobEntity findAssembleJob(Long taskId, String moduleType, Long resultId) {
return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
}
public long countUnfinishedAssembleJobs(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
return 0L;
}
Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getTaskId, taskId)
.eq(TaskFileJobEntity::getModuleType, moduleType)
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
.ne(TaskFileJobEntity::getStatus, "SUCCESS"));
return count == null ? 0L : count;
}
private TaskFileJobEntity findJob(Long taskId, String moduleType, Long resultId, String jobType) {
if (taskId == null || moduleType == null || moduleType.isBlank() || resultId == null || jobType == null || jobType.isBlank()) {
return null;
}
return taskFileJobMapper.selectOne(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getTaskId, taskId)
.eq(TaskFileJobEntity::getModuleType, moduleType)
.eq(TaskFileJobEntity::getResultId, resultId)
.eq(TaskFileJobEntity::getJobType, jobType)
.last("limit 1"));
}
private void publishDispatchEvent(TaskFileJobEntity entity) {
if (entity == null || entity.getId() == null) {
return;
}
applicationEventPublisher.publishEvent(new TaskFileJobDispatchEvent(
entity.getId(),
entity.getTaskId(),
entity.getModuleType(),
entity.getResultId(),
entity.getScopeKey(),
entity.getJobType(),
LocalDateTime.now()
));
}
}
@@ -0,0 +1,98 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.task.mapper.TaskProgressSnapshotMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
@Service
@RequiredArgsConstructor
public class TaskProgressSnapshotService {
private final TaskProgressSnapshotMapper taskProgressSnapshotMapper;
private final ObjectMapper objectMapper;
@Transactional
public void save(Long taskId,
String moduleType,
String status,
int totalCount,
int successCount,
int failedCount,
String currentScopeKey,
String message,
Object snapshot) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(status)) {
return;
}
int pendingCount = Math.max(0, totalCount - successCount - failedCount);
String snapshotJson = snapshot == null ? null : writeJson(snapshot);
LocalDateTime now = LocalDateTime.now();
TaskProgressSnapshotEntity existing = find(taskId, moduleType);
if (existing == null) {
TaskProgressSnapshotEntity entity = new TaskProgressSnapshotEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setStatus(status);
entity.setTotalCount(totalCount);
entity.setSuccessCount(successCount);
entity.setFailedCount(failedCount);
entity.setPendingCount(pendingCount);
entity.setCurrentScopeKey(currentScopeKey);
entity.setMessage(message);
entity.setSnapshotJson(snapshotJson);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskProgressSnapshotMapper.insert(entity);
return;
} catch (DuplicateKeyException ignored) {
existing = find(taskId, moduleType);
}
}
if (existing == null) {
throw new BusinessException("保存任务进度快照失败");
}
taskProgressSnapshotMapper.update(null, new LambdaUpdateWrapper<TaskProgressSnapshotEntity>()
.eq(TaskProgressSnapshotEntity::getId, existing.getId())
.set(TaskProgressSnapshotEntity::getStatus, status)
.set(TaskProgressSnapshotEntity::getTotalCount, totalCount)
.set(TaskProgressSnapshotEntity::getSuccessCount, successCount)
.set(TaskProgressSnapshotEntity::getFailedCount, failedCount)
.set(TaskProgressSnapshotEntity::getPendingCount, pendingCount)
.set(TaskProgressSnapshotEntity::getCurrentScopeKey, currentScopeKey)
.set(TaskProgressSnapshotEntity::getMessage, message)
.set(TaskProgressSnapshotEntity::getSnapshotJson, snapshotJson)
.set(TaskProgressSnapshotEntity::getUpdatedAt, now));
}
public TaskProgressSnapshotEntity find(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return null;
}
return taskProgressSnapshotMapper.selectOne(new LambdaQueryWrapper<TaskProgressSnapshotEntity>()
.eq(TaskProgressSnapshotEntity::getTaskId, taskId)
.eq(TaskProgressSnapshotEntity::getModuleType, moduleType)
.last("limit 1"));
}
private String writeJson(Object value) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException("序列化任务进度快照失败");
}
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
}
@@ -0,0 +1,166 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandRunService;
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.util.List;
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskResultFileJobWorker {
private final TaskFileJobService taskFileJobService;
private final TaskResultPayloadService taskResultPayloadService;
private final FileResultMapper fileResultMapper;
private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher;
private final ShopMatchTaskService shopMatchTaskService;
private final PriceTrackTaskService priceTrackTaskService;
private final ProductRiskTaskService productRiskTaskService;
private final QueryAsinTaskService queryAsinTaskService;
private final PatrolDeleteTaskService patrolDeleteTaskService;
private final AppearancePatentTaskService appearancePatentTaskService;
private final DeleteBrandRunService deleteBrandRunService;
private final BrandTaskService brandTaskService;
@Value("${aiimage.result-file-job.local-worker-enabled:true}")
private boolean localWorkerEnabled;
@Value("${aiimage.result-file-job.batch-size:20}")
private int batchSize;
@Value("${aiimage.result-file-job.stuck-timeout-minutes:30}")
private int stuckTimeoutMinutes;
@Scheduled(fixedDelayString = "${aiimage.result-file-job.local-worker-delay-ms:15000}")
public void runPendingJobs() {
if (!localWorkerEnabled) {
return;
}
List<TaskFileJobEntity> jobs = taskFileJobService.listRunnableJobs(batchSize);
if (jobs == null || jobs.isEmpty()) {
return;
}
log.info("[task-file-job] scheduled worker picked jobs count={}", jobs.size());
for (TaskFileJobEntity job : jobs) {
boolean dispatched = taskFileJobLocalDispatcher.dispatch(
job.getId(),
job.getTaskId(),
job.getModuleType()
);
if (!dispatched) {
process(job);
}
}
}
@Scheduled(fixedDelayString = "${aiimage.result-file-job.stuck-scan-delay-ms:60000}")
public void resetStuckJobs() {
int reset = taskFileJobService.resetStuckRunningJobs(stuckTimeoutMinutes, batchSize);
if (reset > 0) {
log.warn("[task-file-job] reset stuck running jobs count={} timeoutMinutes={}", reset, stuckTimeoutMinutes);
}
}
public void process(TaskFileJobEntity job) {
if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) {
return;
}
long startedAt = System.currentTimeMillis();
try {
dispatch(job);
String resultFileUrl = resolveResultFileUrl(job);
taskFileJobService.markSuccess(job, resultFileUrl);
cleanupAfterSuccess(job);
log.info("[task-file-job] process success jobId={} taskId={} moduleType={} resultId={} elapsedMs={} resultFileUrl={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
System.currentTimeMillis() - startedAt, resultFileUrl);
} catch (Exception ex) {
String message = ex.getMessage() == null ? "结果文件生成失败" : ex.getMessage();
log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), message);
taskFileJobService.markFailed(job, message);
}
}
private String resolveResultFileUrl(TaskFileJobEntity job) {
if ("BRAND".equals(job.getModuleType())) {
return brandTaskService.resolveResultObjectKey(job.getTaskId());
}
if (job.getResultId() == null) {
return null;
}
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
return result == null ? null : result.getResultFileUrl();
}
private void dispatch(TaskFileJobEntity job) {
String moduleType = job.getModuleType();
if ("SHOP_MATCH".equals(moduleType)) {
shopMatchTaskService.processResultFileJob(job);
return;
}
if ("PRICE_TRACK".equals(moduleType)) {
priceTrackTaskService.processResultFileJob(job);
return;
}
if ("PRODUCT_RISK_RESOLVE".equals(moduleType)) {
productRiskTaskService.processResultFileJob(job);
return;
}
if ("QUERY_ASIN".equals(moduleType)) {
queryAsinTaskService.processResultFileJob(job);
return;
}
if ("PATROL_DELETE".equals(moduleType)) {
patrolDeleteTaskService.processResultFileJob(job);
return;
}
if ("APPEARANCE_PATENT".equals(moduleType)) {
appearancePatentTaskService.processResultFileJob(job);
return;
}
if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.processResultFileJob(job);
return;
}
if ("BRAND".equals(moduleType)) {
brandTaskService.processResultFileJob(job);
return;
}
throw new IllegalArgumentException("unsupported result file job module: " + moduleType);
}
private void cleanupAfterSuccess(TaskFileJobEntity job) {
String moduleType = job.getModuleType();
if ("SHOP_MATCH".equals(moduleType)
|| "PRICE_TRACK".equals(moduleType)
|| "PRODUCT_RISK_RESOLVE".equals(moduleType)
|| "QUERY_ASIN".equals(moduleType)
|| "PATROL_DELETE".equals(moduleType)) {
taskResultPayloadService.deleteLatest(job.getTaskId(), moduleType, job.getScopeKey());
return;
}
if ("APPEARANCE_PATENT".equals(moduleType)) {
appearancePatentTaskService.cleanupResultFileJob(job);
return;
}
if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.cleanupResultFileJob(job);
}
}
}
@@ -0,0 +1,233 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
@Service
@RequiredArgsConstructor
public class TaskResultItemService {
private final TaskResultItemMapper taskResultItemMapper;
private final ObjectMapper objectMapper;
private final TransientPayloadStorageService transientPayloadStorageService;
@Transactional
public void replaceResultSnapshot(Long taskId, String moduleType, Long resultId, String scopeKey, Object snapshot) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0 || snapshot == null) {
return;
}
String normalizedScopeKey = firstNonBlank(scopeKey, "result:" + resultId).trim();
String itemKey = "result:" + resultId;
upsert(taskId, moduleType, resultId, normalizedScopeKey, itemKey, null, null, null, snapshot);
}
@Transactional
public void replaceTaskSnapshots(Long taskId, String moduleType, List<? extends Object> snapshots, SnapshotKeyResolver resolver) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || snapshots == null) {
return;
}
for (Object snapshot : snapshots) {
if (snapshot == null) {
continue;
}
Long resultId = resolver == null ? null : resolver.resultId(snapshot);
if (resultId == null || resultId <= 0) {
continue;
}
replaceResultSnapshot(taskId, moduleType, resultId, resolver.scopeKey(snapshot), snapshot);
}
}
public <T> List<T> listResultSnapshots(Long taskId, String moduleType, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return List.of();
}
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.likeRight(TaskResultItemEntity::getItemKey, "result:")
.orderByAsc(TaskResultItemEntity::getResultId)
.orderByAsc(TaskResultItemEntity::getId));
List<T> out = new ArrayList<>();
for (TaskResultItemEntity row : rows) {
T value = readPayload(row, clazz);
if (value != null) {
out.add(value);
}
}
return out;
}
public <T> T getResultSnapshot(Long taskId, String moduleType, Long resultId, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0) {
return null;
}
TaskResultItemEntity row = taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.eq(TaskResultItemEntity::getResultId, resultId)
.eq(TaskResultItemEntity::getItemKey, "result:" + resultId)
.last("limit 1"));
return readPayload(row, clazz);
}
public long countResultSnapshots(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return 0L;
}
Long count = taskResultItemMapper.selectCount(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.likeRight(TaskResultItemEntity::getItemKey, "result:"));
return count == null ? 0L : count;
}
@Transactional
public void deleteTaskItems(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return;
}
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.select(TaskResultItemEntity::getId, TaskResultItemEntity::getPayloadJson)
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType));
if (rows != null) {
for (TaskResultItemEntity row : rows) {
transientPayloadStorageService.deletePayloadIfPresent(row.getPayloadJson());
}
}
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType));
}
private void upsert(Long taskId,
String moduleType,
Long resultId,
String scopeKey,
String itemKey,
String countryCode,
String asin,
String status,
Object payload) {
String payloadJson = writeJson(payload);
String normalizedScopeKey = firstNonBlank(scopeKey, "task:" + taskId).trim();
String normalizedItemKey = firstNonBlank(itemKey, "result:" + resultId).trim();
String scopeHash = hash(normalizedScopeKey);
String storedPayload = transientPayloadStorageService.storeResultItemPayload(
moduleType, taskId, scopeHash, normalizedItemKey, payloadJson);
String payloadHash = hash(payloadJson);
LocalDateTime now = LocalDateTime.now();
TaskResultItemEntity existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
if (existing == null) {
TaskResultItemEntity entity = new TaskResultItemEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setResultId(resultId);
entity.setScopeKey(normalizedScopeKey);
entity.setScopeHash(scopeHash);
entity.setItemKey(normalizedItemKey);
entity.setCountryCode(countryCode);
entity.setAsin(asin);
entity.setStatus(status);
entity.setPayloadJson(storedPayload);
entity.setPayloadHash(payloadHash);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskResultItemMapper.insert(entity);
return;
} catch (DuplicateKeyException ignored) {
existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
}
}
if (existing == null) {
throw new BusinessException("保存任务结果明细失败");
}
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getPayloadJson(), storedPayload);
taskResultItemMapper.update(null, new LambdaUpdateWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getId, existing.getId())
.set(TaskResultItemEntity::getResultId, resultId)
.set(TaskResultItemEntity::getScopeKey, normalizedScopeKey)
.set(TaskResultItemEntity::getCountryCode, countryCode)
.set(TaskResultItemEntity::getAsin, asin)
.set(TaskResultItemEntity::getStatus, status)
.set(TaskResultItemEntity::getPayloadJson, storedPayload)
.set(TaskResultItemEntity::getPayloadHash, payloadHash)
.set(TaskResultItemEntity::getUpdatedAt, now));
}
private TaskResultItemEntity find(Long taskId, String moduleType, String scopeHash, String itemKey) {
return taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.eq(TaskResultItemEntity::getScopeHash, scopeHash)
.eq(TaskResultItemEntity::getItemKey, itemKey)
.last("limit 1"));
}
private <T> T readPayload(TaskResultItemEntity row, Class<T> clazz) {
if (row == null || isBlank(row.getPayloadJson())) {
return null;
}
try {
String payloadJson = transientPayloadStorageService.resolvePayload(row.getPayloadJson(), "read task result item failed");
return objectMapper.readValue(payloadJson, clazz);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("读取任务结果明细失败");
}
}
private String writeJson(Object value) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException("序列化任务结果明细失败");
}
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new IllegalStateException("failed to hash task result item", ex);
}
}
private String firstNonBlank(String primary, String fallback) {
return primary == null || primary.isBlank() ? fallback : primary;
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
public interface SnapshotKeyResolver {
Long resultId(Object snapshot);
String scopeKey(Object snapshot);
}
}
@@ -0,0 +1,139 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.task.mapper.TaskResultPayloadMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
@Service
@RequiredArgsConstructor
public class TaskResultPayloadService {
private static final String LATEST_SUBMISSION_ID = "latest";
private final TaskResultPayloadMapper taskResultPayloadMapper;
private final ObjectMapper objectMapper;
private final TransientPayloadStorageService transientPayloadStorageService;
@Transactional
public void saveLatest(Long taskId, String moduleType, String scopeKey, Object payload) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey) || payload == null) {
return;
}
String normalizedScopeKey = scopeKey.trim();
String scopeHash = hash(normalizedScopeKey);
String payloadJson = writeJson(payload);
String storedPayload = transientPayloadStorageService.storeResultPayload(
moduleType, taskId, scopeHash, LATEST_SUBMISSION_ID, payloadJson);
String payloadHash = hash(payloadJson);
LocalDateTime now = LocalDateTime.now();
TaskResultPayloadEntity existing = findLatest(taskId, moduleType, scopeHash);
if (existing == null) {
TaskResultPayloadEntity entity = new TaskResultPayloadEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setScopeKey(normalizedScopeKey);
entity.setScopeHash(scopeHash);
entity.setSubmissionId(LATEST_SUBMISSION_ID);
entity.setPayloadJson(storedPayload);
entity.setPayloadHash(payloadHash);
entity.setVersion(1L);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskResultPayloadMapper.insert(entity);
return;
} catch (DuplicateKeyException ignored) {
existing = findLatest(taskId, moduleType, scopeHash);
}
}
if (existing == null) {
throw new BusinessException("保存任务结果载荷失败");
}
long nextVersion = existing.getVersion() == null ? 1L : existing.getVersion() + 1L;
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getPayloadJson(), storedPayload);
taskResultPayloadMapper.update(null, new LambdaUpdateWrapper<TaskResultPayloadEntity>()
.eq(TaskResultPayloadEntity::getId, existing.getId())
.set(TaskResultPayloadEntity::getScopeKey, normalizedScopeKey)
.set(TaskResultPayloadEntity::getPayloadJson, storedPayload)
.set(TaskResultPayloadEntity::getPayloadHash, payloadHash)
.set(TaskResultPayloadEntity::getVersion, nextVersion)
.set(TaskResultPayloadEntity::getUpdatedAt, now));
}
public <T> T getLatest(Long taskId, String moduleType, String scopeKey, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
return null;
}
TaskResultPayloadEntity entity = findLatest(taskId, moduleType, hash(scopeKey.trim()));
if (entity == null || isBlank(entity.getPayloadJson())) {
return null;
}
try {
String payloadJson = transientPayloadStorageService.resolvePayload(entity.getPayloadJson(), "read task result payload failed");
return objectMapper.readValue(payloadJson, clazz);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("读取任务结果载荷失败");
}
}
@Transactional
public void deleteLatest(Long taskId, String moduleType, String scopeKey) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
return;
}
TaskResultPayloadEntity existing = findLatest(taskId, moduleType, hash(scopeKey.trim()));
if (existing != null) {
transientPayloadStorageService.deletePayloadIfPresent(existing.getPayloadJson());
taskResultPayloadMapper.deleteById(existing.getId());
}
}
private TaskResultPayloadEntity findLatest(Long taskId, String moduleType, String scopeHash) {
return taskResultPayloadMapper.selectOne(new LambdaQueryWrapper<TaskResultPayloadEntity>()
.eq(TaskResultPayloadEntity::getTaskId, taskId)
.eq(TaskResultPayloadEntity::getModuleType, moduleType)
.eq(TaskResultPayloadEntity::getScopeHash, scopeHash)
.eq(TaskResultPayloadEntity::getSubmissionId, LATEST_SUBMISSION_ID)
.last("limit 1"));
}
private String writeJson(Object value) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException("序列化任务结果载荷失败");
}
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new IllegalStateException("failed to hash task result payload", ex);
}
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
}
@@ -3,9 +3,7 @@ package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import lombok.RequiredArgsConstructor;
@@ -13,37 +11,23 @@ import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class TaskScopePayloadStorageService {
private static final Set<String> OSS_BACKED_STATE_MODULES = new HashSet<>(Set.of(
"SHOP_MATCH",
"PRODUCT_RISK_RESOLVE",
"PRICE_TRACK",
"PATROL_DELETE",
"QUERY_ASIN"
));
private static final String OSS_POINTER_PREFIX = "oss:";
private static final String LOCAL_BUFFER_DIR = "task-scope-buffer";
private final TaskScopeStateMapper taskScopeStateMapper;
private final ObjectMapper objectMapper;
private final OssStorageService ossStorageService;
private final TaskPressureProperties taskPressureProperties;
private final TransientPayloadStorageService transientPayloadStorageService;
@Transactional
public <T> void saveScopePayload(Long taskId, String moduleType, String scopeKey, T payload) {
@@ -53,13 +37,11 @@ public class TaskScopePayloadStorageService {
String normalizedScopeKey = normalize(scopeKey);
String scopeHash = hash(normalizedScopeKey);
String payloadJson = writeJson(payload, "写入任务范围载荷失败");
String stateJson = transientPayloadStorageService.storeScopePayload(
moduleType, taskId, scopeHash, payloadJson, true);
LocalDateTime now = LocalDateTime.now();
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
if (shouldStoreStatePayloadInOss(moduleType)) {
saveOssBackedScopePayload(taskId, moduleType, normalizedScopeKey, scopeHash, payloadJson, now, state);
return;
}
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
if (state == null) {
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
@@ -84,7 +66,7 @@ public class TaskScopePayloadStorageService {
if (state == null) {
throw new BusinessException("写入任务范围载荷失败");
}
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getStateJson(), stateJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
@@ -145,8 +127,7 @@ public class TaskScopePayloadStorageService {
if (state == null) {
return;
}
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
deleteStatePayloadIfNeeded(state.getStateJson());
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
boolean hasParsedPayload = !isBlank(state.getParsedPayloadJson());
if (hasParsedPayload) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
@@ -195,8 +176,7 @@ public class TaskScopePayloadStorageService {
if (state == null || state.getId() == null) {
continue;
}
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
deleteStatePayloadIfNeeded(state.getStateJson());
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
if (!isBlank(state.getParsedPayloadJson())) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
@@ -221,6 +201,9 @@ public class TaskScopePayloadStorageService {
String scopeKey = normalize(entry.getKey());
String scopeHash = hash(scopeKey);
String parsedPayloadJson = writeJson(entry.getValue(), "写入任务解析载荷失败");
String storedParsedPayload = transientPayloadStorageService.storeParsedPayload(
moduleType, taskId, scopeHash, parsedPayloadJson, true);
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
if (state == null) {
TaskScopeStateEntity entity = new TaskScopeStateEntity();
@@ -228,7 +211,7 @@ public class TaskScopePayloadStorageService {
entity.setModuleType(moduleType);
entity.setScopeKey(scopeKey);
entity.setScopeHash(scopeHash);
entity.setParsedPayloadJson(parsedPayloadJson);
entity.setParsedPayloadJson(storedParsedPayload);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
@@ -244,11 +227,11 @@ public class TaskScopePayloadStorageService {
if (state == null) {
throw new BusinessException("写入任务解析载荷失败");
}
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedPayloadJson);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getParsedPayloadJson(), storedParsedPayload);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadJson)
.set(TaskScopeStateEntity::getParsedPayloadJson, storedParsedPayload)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
}
@@ -271,7 +254,9 @@ public class TaskScopePayloadStorageService {
continue;
}
try {
result.put(state.getScopeKey(), objectMapper.readValue(resolveParsedPayload(state.getParsedPayloadJson()), clazz));
String payloadJson = transientPayloadStorageService.resolvePayload(
state.getParsedPayloadJson(), "read parsed payload failed");
result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, clazz));
} catch (Exception ex) {
throw new BusinessException("读取任务原始载荷失败");
}
@@ -281,32 +266,7 @@ public class TaskScopePayloadStorageService {
@Transactional
public boolean recoverBufferedScopePayload(Long taskId, String moduleType, String scopeHash) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeHash)) {
return false;
}
String payloadJson = readBufferedPayload(taskId, moduleType, scopeHash);
if (payloadJson == null) {
return false;
}
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
if (state == null) {
deleteBufferedPayload(taskId, moduleType, scopeHash);
return false;
}
if (!shouldStoreStatePayloadInOss(moduleType)) {
deleteBufferedPayload(taskId, moduleType, scopeHash);
return false;
}
LocalDateTime now = LocalDateTime.now();
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getStateJson, stateJson)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now));
deleteBufferedPayload(taskId, moduleType, scopeHash);
return true;
return false;
}
public Path getBufferedPayloadRoot() {
@@ -321,126 +281,17 @@ public class TaskScopePayloadStorageService {
.last("limit 1"));
}
private String storeStatePayload(String moduleType, Long taskId, String scopeHash, String payloadJson) {
if (!shouldStoreStatePayloadInOss(moduleType)) {
return payloadJson;
}
try {
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskScopePayload(moduleType, taskId, scopeHash, payloadJson);
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new BusinessException("写入任务范围载荷失败");
}
}
private String resolveStatePayload(TaskScopeStateEntity state) {
if (state == null || isBlank(state.getStateJson())) {
return null;
}
String bufferedPayload = readBufferedPayload(state.getTaskId(), state.getModuleType(), state.getScopeHash());
if (bufferedPayload != null) {
return bufferedPayload;
}
String ossPointer = extractOssPointer(state.getStateJson());
if (ossPointer == null) {
return state.getStateJson();
}
try {
return ossStorageService.readObjectAsString(stripOssPointerPrefix(ossPointer));
return transientPayloadStorageService.resolvePayload(state.getStateJson(), "read task scope payload failed");
} catch (Exception ex) {
throw new BusinessException("读取任务范围载荷失败");
}
}
private void deleteStatePayloadIfNeeded(String value) {
String ossPointer = extractOssPointer(value);
if (ossPointer == null) {
return;
}
try {
ossStorageService.deleteObject(stripOssPointerPrefix(ossPointer));
} catch (Exception ignored) {
}
}
private String resolveParsedPayload(String value) {
if (isBlank(value) || !isOssPointer(value)) {
return value;
}
try {
return ossStorageService.readObjectAsString(stripOssPointerPrefix(value));
} catch (Exception ex) {
throw new BusinessException("读取任务解析载荷失败");
}
}
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
if (isBlank(oldValue) || oldValue.equals(newValue) || !isOssPointer(oldValue)) {
return;
}
try {
ossStorageService.deleteObject(stripOssPointerPrefix(oldValue));
} catch (Exception ignored) {
}
}
private void deleteReplacedStatePayloadIfNeeded(String oldValue, String newValue) {
if (isBlank(oldValue) || oldValue.equals(newValue)) {
return;
}
deleteStatePayloadIfNeeded(oldValue);
}
private void saveOssBackedScopePayload(Long taskId,
String moduleType,
String normalizedScopeKey,
String scopeHash,
String payloadJson,
LocalDateTime now,
TaskScopeStateEntity state) {
if (state == null) {
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setScopeKey(normalizedScopeKey);
entity.setScopeHash(scopeHash);
entity.setStateJson(stateJson);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
entity.setLastChunkAt(now);
entity.setLastError(null);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(entity);
deleteBufferedPayload(taskId, moduleType, scopeHash);
return;
} catch (DuplicateKeyException ignored) {
state = getScopeState(taskId, moduleType, scopeHash);
}
}
if (state == null) {
throw new BusinessException("刷新任务范围缓冲载荷失败");
}
writeBufferedPayload(taskId, moduleType, scopeHash, payloadJson);
if (!shouldFlushBufferedPayload(state, now)) {
return;
}
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
.set(TaskScopeStateEntity::getStateJson, stateJson)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now));
deleteBufferedPayload(taskId, moduleType, scopeHash);
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);
@@ -457,76 +308,6 @@ public class TaskScopePayloadStorageService {
return value == null || value.isBlank();
}
private boolean shouldStoreStatePayloadInOss(String moduleType) {
return OSS_BACKED_STATE_MODULES.contains(normalize(moduleType).toUpperCase());
}
private boolean isOssPointer(String value) {
return !isBlank(value) && value.startsWith(OSS_POINTER_PREFIX);
}
private String extractOssPointer(String value) {
if (isBlank(value)) {
return null;
}
if (isOssPointer(value)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return isOssPointer(decoded) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private String stripOssPointerPrefix(String value) {
return value.substring(OSS_POINTER_PREFIX.length());
}
private boolean shouldFlushBufferedPayload(TaskScopeStateEntity state, LocalDateTime now) {
long flushIntervalMillis = Math.max(0L, taskPressureProperties.getScopePayloadFlushIntervalMillis());
if (flushIntervalMillis <= 0L || state.getUpdatedAt() == null) {
return true;
}
return java.time.Duration.between(state.getUpdatedAt(), now).toMillis() >= flushIntervalMillis;
}
private void writeBufferedPayload(Long taskId, String moduleType, String scopeHash, String payloadJson) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
Files.createDirectories(path.getParent());
Files.writeString(path, payloadJson, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
} catch (IOException ex) {
throw new BusinessException("写入任务范围缓冲载荷失败");
}
}
private String readBufferedPayload(Long taskId, String moduleType, String scopeHash) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
if (!Files.isRegularFile(path)) {
return null;
}
return Files.readString(path);
} catch (IOException ex) {
return null;
}
}
private void deleteBufferedPayload(Long taskId, String moduleType, String scopeHash) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
Files.deleteIfExists(path);
} catch (IOException ignored) {
}
}
private Path buildBufferedPayloadPath(Long taskId, String moduleType, String scopeHash) {
String normalizedModuleType = normalize(moduleType).toLowerCase();
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR, normalizedModuleType, String.valueOf(taskId), scopeHash + ".json");
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
@@ -0,0 +1,149 @@
package com.nanri.aiimage.modules.task.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.TransientStorageProperties;
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.Locale;
import java.util.UUID;
@Service
@RequiredArgsConstructor
public class TransientPayloadStorageService {
private static final String RUSTFS_POINTER_PREFIX = "rustfs:";
private static final String OSS_POINTER_PREFIX = "oss:";
private final TransientStorageProperties properties;
private final RustfsObjectStorageService rustfsObjectStorageService;
private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper;
public boolean isWriteEnabled() {
return properties.isEnabled() && rustfsObjectStorageService.isConfigured();
}
public String storeScopePayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
return store("task-scope", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString);
}
public String storeParsedPayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
return store("task-parsed", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString);
}
public String storeResultPayload(String moduleType, Long taskId, String scopeHash, String submissionId, String content) {
return store("task-result-payload", moduleType, taskId, scopeHash, submissionId, content, true);
}
public String storeResultItemPayload(String moduleType, Long taskId, String scopeHash, String itemKey, String content) {
return store("task-result-item", moduleType, taskId, scopeHash, itemKey, content, true);
}
public String storeChunkPayload(String moduleType, Long taskId, String scopeHash, Integer chunkIndex, String content) {
String entryKey = chunkIndex == null ? UUID.randomUUID().toString() : "chunk-" + chunkIndex;
return store("task-chunk", moduleType, taskId, scopeHash, entryKey, content, true);
}
public String resolvePayload(String value, String errorMessage) {
String pointer = extractPointer(value);
if (pointer == null) {
return value;
}
try {
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
return rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
}
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
return ossStorageService.readObjectAsString(pointer.substring(OSS_POINTER_PREFIX.length()));
}
} catch (Exception ex) {
throw new IllegalStateException(errorMessage, ex);
}
return value;
}
public void deletePayloadIfPresent(String value) {
String pointer = extractPointer(value);
if (pointer == null) {
return;
}
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
rustfsObjectStorageService.deleteObject(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
return;
}
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
ossStorageService.deleteObject(pointer.substring(OSS_POINTER_PREFIX.length()));
}
}
public void deleteReplacedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractPointer(oldValue);
String newPointer = extractPointer(newValue);
if (oldPointer == null || oldPointer.equals(newPointer)) {
return;
}
deletePayloadIfPresent(oldPointer);
}
public String extractPointer(String value) {
if (value == null || value.isBlank()) {
return null;
}
if (isPointer(value)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return isPointer(decoded) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private boolean isPointer(String value) {
return value != null && (value.startsWith(RUSTFS_POINTER_PREFIX) || value.startsWith(OSS_POINTER_PREFIX));
}
private String store(String category,
String moduleType,
Long taskId,
String scopeHash,
String entryKey,
String content,
boolean encodeAsJsonString) {
if (!isWriteEnabled()) {
return content;
}
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
String pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content);
if (!encodeAsJsonString) {
return pointer;
}
try {
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new IllegalStateException("failed to encode transient storage pointer", ex);
}
}
private String buildObjectKey(String category, String moduleType, Long taskId, String scopeHash, String entryKey) {
String normalizedCategory = normalize(category, "unknown");
String normalizedModuleType = normalize(moduleType, "unknown");
String normalizedScopeHash = normalize(scopeHash, UUID.randomUUID().toString());
String normalizedEntryKey = normalize(entryKey, "latest");
return String.format("%s/%s/%s/%s/%s.json",
normalizedCategory,
normalizedModuleType,
taskId == null ? 0L : taskId,
normalizedScopeHash,
normalizedEntryKey);
}
private String normalize(String value, String fallback) {
String normalized = value == null ? "" : value.trim().toLowerCase(Locale.ROOT);
return normalized.isBlank() ? fallback : normalized.replaceAll("[^a-z0-9._-]+", "_");
}
}