异常记录检测修改

This commit is contained in:
super
2026-05-04 19:04:55 +08:00
parent 27f1c07b7f
commit b274def6ba
70 changed files with 7182 additions and 252 deletions
@@ -19,6 +19,13 @@ public class TaskScopeStateEntity {
private String scopeHash;
private String parsedPayloadJson;
private String stateJson;
private String cozeExecuteId;
private String cozeStatus;
private LocalDateTime cozeSubmittedAt;
private LocalDateTime cozeLastPolledAt;
private LocalDateTime cozeCompletedAt;
private Integer cozeAttemptCount;
private String cozeError;
private Integer chunkTotal;
private Integer receivedChunkCount;
private Integer completed;
@@ -4,6 +4,7 @@ import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
@@ -18,8 +19,9 @@ public class TaskFileJobLocalDispatcher {
private final TaskFileJobMapper taskFileJobMapper;
private final ObjectProvider<TaskResultFileJobWorker> taskResultFileJobWorkerProvider;
@Autowired
@Qualifier("taskFileJobDispatchExecutor")
private final TaskExecutor taskFileJobDispatchExecutor;
private TaskExecutor taskFileJobDispatchExecutor;
@Value("${aiimage.result-file-job.local-dispatch-enabled:true}")
private boolean localDispatchEnabled;
@@ -28,26 +30,32 @@ public class TaskFileJobLocalDispatcher {
if (!localDispatchEnabled || jobId == null || jobId <= 0) {
return false;
}
taskFileJobDispatchExecutor.execute(() -> {
try {
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
if (job == null) {
log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}",
jobId, taskId, moduleType);
return;
try {
taskFileJobDispatchExecutor.execute(() -> {
try {
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
if (job == null) {
log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}",
jobId, taskId, moduleType);
return;
}
TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable();
if (taskResultFileJobWorker == null) {
log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}",
jobId, taskId, moduleType);
return;
}
taskResultFileJobWorker.process(job);
} catch (Exception ex) {
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
jobId, taskId, moduleType, ex.getMessage(), ex);
}
TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable();
if (taskResultFileJobWorker == null) {
log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}",
jobId, taskId, moduleType);
return;
}
taskResultFileJobWorker.process(job);
} catch (Exception ex) {
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
jobId, taskId, moduleType, ex.getMessage(), ex);
}
});
return true;
});
return true;
} catch (RuntimeException ex) {
log.warn("[task-file-job] local dispatch executor rejected jobId={} taskId={} moduleType={} msg={}",
jobId, taskId, moduleType, ex.getMessage(), ex);
return false;
}
}
}
@@ -53,7 +53,7 @@ public class TaskFileJobService {
if (existing == null) {
return null;
}
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus())) {
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus()) || "PENDING".equals(existing.getStatus())) {
return existing;
}
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
@@ -192,6 +192,11 @@ public class TaskFileJobService {
return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
}
public boolean hasSuccessfulAssembleJob(Long taskId, String moduleType, Long resultId) {
TaskFileJobEntity job = findAssembleJob(taskId, moduleType, resultId);
return job != null && "SUCCESS".equals(job.getStatus());
}
public Map<Long, TaskFileJobEntity> findAssembleJobsByResultIds(String moduleType, List<Long> resultIds) {
List<Long> normalizedIds = resultIds == null ? List.of() : resultIds.stream()
.filter(id -> id != null && id > 0)
@@ -225,6 +230,18 @@ public class TaskFileJobService {
return count == null ? 0L : count;
}
public long countActiveAssembleJobs(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
return 0L;
}
Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getTaskId, taskId)
.eq(TaskFileJobEntity::getModuleType, moduleType)
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "RUNNING")));
return count == null ? 0L : count;
}
@Transactional
public void deleteTaskJobs(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
@@ -8,12 +8,16 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
@@ -34,8 +38,12 @@ public class TaskResultFileJobWorker {
private final QueryAsinTaskService queryAsinTaskService;
private final PatrolDeleteTaskService patrolDeleteTaskService;
private final AppearancePatentTaskService appearancePatentTaskService;
private final SimilarAsinTaskService similarAsinTaskService;
private final DeleteBrandRunService deleteBrandRunService;
private final BrandTaskService brandTaskService;
@Autowired
@Qualifier("cozeTaskExecutor")
private TaskExecutor cozeTaskExecutor;
@Value("${aiimage.result-file-job.local-worker-enabled:true}")
private boolean localWorkerEnabled;
@@ -80,9 +88,29 @@ public class TaskResultFileJobWorker {
if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) {
return;
}
if ("APPEARANCE_PATENT".equals(job.getModuleType()) || "SIMILAR_ASIN".equals(job.getModuleType())) {
try {
cozeTaskExecutor.execute(() -> processInternal(job));
return;
} catch (RuntimeException ex) {
log.warn("[task-file-job] coze module offload failed, fallback inline jobId={} taskId={} moduleType={} msg={}",
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex);
}
}
processInternal(job);
}
private void processInternal(TaskFileJobEntity job) {
long startedAt = System.currentTimeMillis();
try {
dispatch(job);
boolean completed = dispatch(job);
if (!completed) {
taskFileJobService.touchRunning(job.getId());
log.info("[task-file-job] process deferred jobId={} taskId={} moduleType={} resultId={} elapsedMs={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
System.currentTimeMillis() - startedAt);
return;
}
String resultFileUrl = resolveResultFileUrl(job);
taskFileJobService.markSuccess(job, resultFileUrl);
cleanupAfterSuccess(job);
@@ -108,39 +136,41 @@ public class TaskResultFileJobWorker {
return result == null ? null : result.getResultFileUrl();
}
private void dispatch(TaskFileJobEntity job) {
private boolean dispatch(TaskFileJobEntity job) {
String moduleType = job.getModuleType();
if ("SHOP_MATCH".equals(moduleType)) {
shopMatchTaskService.processResultFileJob(job);
return;
return true;
}
if ("PRICE_TRACK".equals(moduleType)) {
priceTrackTaskService.processResultFileJob(job);
return;
return true;
}
if ("PRODUCT_RISK_RESOLVE".equals(moduleType)) {
productRiskTaskService.processResultFileJob(job);
return;
return true;
}
if ("QUERY_ASIN".equals(moduleType)) {
queryAsinTaskService.processResultFileJob(job);
return;
return true;
}
if ("PATROL_DELETE".equals(moduleType)) {
patrolDeleteTaskService.processResultFileJob(job);
return;
return true;
}
if ("APPEARANCE_PATENT".equals(moduleType)) {
appearancePatentTaskService.processResultFileJob(job);
return;
return appearancePatentTaskService.processResultFileJob(job);
}
if ("SIMILAR_ASIN".equals(moduleType)) {
return similarAsinTaskService.processResultFileJob(job);
}
if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.processResultFileJob(job);
return;
return true;
}
if ("BRAND".equals(moduleType)) {
brandTaskService.processResultFileJob(job);
return;
return true;
}
throw new IllegalArgumentException("unsupported result file job module: " + moduleType);
}
@@ -159,6 +189,10 @@ public class TaskResultFileJobWorker {
appearancePatentTaskService.cleanupResultFileJob(job);
return;
}
if ("SIMILAR_ASIN".equals(moduleType)) {
similarAsinTaskService.cleanupResultFileJob(job);
return;
}
if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.cleanupResultFileJob(job);
}
@@ -1,29 +1,39 @@
package com.nanri.aiimage.modules.task.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.config.TransientStorageProperties;
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Locale;
import java.util.UUID;
@Service
@RequiredArgsConstructor
@Slf4j
public class TransientPayloadStorageService {
private static final String LOCAL_POINTER_PREFIX = "local:";
private static final String RUSTFS_POINTER_PREFIX = "rustfs:";
private static final String OSS_POINTER_PREFIX = "oss:";
private static final String LOCAL_PAYLOAD_DIR = "transient-payload";
private final TransientStorageProperties properties;
private final StorageProperties storageProperties;
private final RustfsObjectStorageService rustfsObjectStorageService;
private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper;
public boolean isWriteEnabled() {
return properties.isEnabled() && rustfsObjectStorageService.isConfigured();
return properties.isEnabled();
}
public String storeScopePayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
@@ -68,6 +78,9 @@ public class TransientPayloadStorageService {
return value;
}
try {
if (pointer.startsWith(LOCAL_POINTER_PREFIX)) {
return readLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length()));
}
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
return rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
}
@@ -85,6 +98,10 @@ public class TransientPayloadStorageService {
if (pointer == null) {
return;
}
if (pointer.startsWith(LOCAL_POINTER_PREFIX)) {
deleteLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length()));
return;
}
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
rustfsObjectStorageService.deleteObject(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
return;
@@ -107,19 +124,28 @@ public class TransientPayloadStorageService {
if (value == null || value.isBlank()) {
return null;
}
if (isPointer(value)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return isPointer(decoded) ? decoded : null;
} catch (Exception ignored) {
return null;
String candidate = value.trim();
for (int i = 0; i < 4; i++) {
if (isPointer(candidate)) {
return candidate;
}
try {
String decoded = objectMapper.readValue(candidate, String.class);
if (decoded == null || decoded.equals(candidate)) {
return null;
}
candidate = decoded.trim();
} catch (Exception ignored) {
return null;
}
}
return null;
}
private boolean isPointer(String value) {
return value != null && (value.startsWith(RUSTFS_POINTER_PREFIX) || value.startsWith(OSS_POINTER_PREFIX));
return value != null && (value.startsWith(LOCAL_POINTER_PREFIX)
|| value.startsWith(RUSTFS_POINTER_PREFIX)
|| value.startsWith(OSS_POINTER_PREFIX));
}
private String store(String category,
@@ -144,7 +170,19 @@ public class TransientPayloadStorageService {
return content;
}
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
String pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content, verifyAfterUpload);
String pointer = storeLocal(objectKey, content);
if (pointer == null && rustfsObjectStorageService.isConfigured()) {
try {
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content, verifyAfterUpload);
} catch (Exception ex) {
log.warn("[transient-payload] rustfs upload failed and local fallback unavailable objectKey={} err={}",
objectKey, ex.getMessage());
throw ex;
}
}
if (pointer == null) {
return content;
}
if (!encodeAsJsonString) {
return pointer;
}
@@ -155,6 +193,61 @@ public class TransientPayloadStorageService {
}
}
private String storeLocal(String objectKey, String content) {
try {
Path root = localPayloadRoot();
Path target = root.resolve(objectKey).normalize();
if (!target.startsWith(root)) {
throw new IllegalArgumentException("invalid local payload key: " + objectKey);
}
Files.createDirectories(target.getParent());
Files.writeString(target,
content == null ? "" : content,
StandardCharsets.UTF_8,
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE);
return LOCAL_POINTER_PREFIX + objectKey;
} catch (Exception ex) {
log.warn("[transient-payload] local store failed objectKey={} err={}", objectKey, ex.getMessage());
return null;
}
}
private String readLocalPayload(String objectKey) {
try {
Path target = resolveLocalPayloadPath(objectKey);
return Files.readString(target, StandardCharsets.UTF_8);
} catch (Exception ex) {
throw new IllegalStateException("failed to read local transient payload", ex);
}
}
private void deleteLocalPayload(String objectKey) {
try {
Files.deleteIfExists(resolveLocalPayloadPath(objectKey));
} catch (Exception ex) {
log.warn("[transient-payload] delete local payload failed objectKey={} err={}", objectKey, ex.getMessage());
}
}
private Path resolveLocalPayloadPath(String objectKey) {
Path root = localPayloadRoot();
Path target = root.resolve(objectKey == null ? "" : objectKey).normalize();
if (!target.startsWith(root)) {
throw new IllegalArgumentException("invalid local payload key: " + objectKey);
}
return target;
}
private Path localPayloadRoot() {
String configuredRoot = storageProperties.getLocalTempDir();
Path tempRoot = configuredRoot == null || configuredRoot.isBlank()
? Path.of(System.getProperty("java.io.tmpdir"))
: Path.of(configuredRoot);
return tempRoot.resolve(LOCAL_PAYLOAD_DIR).normalize();
}
private String buildObjectKey(String category, String moduleType, Long taskId, String scopeHash, String entryKey) {
String normalizedCategory = normalize(category, "unknown");
String normalizedModuleType = normalize(moduleType, "unknown");