异常记录检测修改
This commit is contained in:
+7
@@ -19,6 +19,13 @@ public class TaskScopeStateEntity {
|
||||
private String scopeHash;
|
||||
private String parsedPayloadJson;
|
||||
private String stateJson;
|
||||
private String cozeExecuteId;
|
||||
private String cozeStatus;
|
||||
private LocalDateTime cozeSubmittedAt;
|
||||
private LocalDateTime cozeLastPolledAt;
|
||||
private LocalDateTime cozeCompletedAt;
|
||||
private Integer cozeAttemptCount;
|
||||
private String cozeError;
|
||||
private Integer chunkTotal;
|
||||
private Integer receivedChunkCount;
|
||||
private Integer completed;
|
||||
|
||||
+29
-21
@@ -4,6 +4,7 @@ import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
@@ -18,8 +19,9 @@ public class TaskFileJobLocalDispatcher {
|
||||
private final TaskFileJobMapper taskFileJobMapper;
|
||||
private final ObjectProvider<TaskResultFileJobWorker> taskResultFileJobWorkerProvider;
|
||||
|
||||
@Autowired
|
||||
@Qualifier("taskFileJobDispatchExecutor")
|
||||
private final TaskExecutor taskFileJobDispatchExecutor;
|
||||
private TaskExecutor taskFileJobDispatchExecutor;
|
||||
|
||||
@Value("${aiimage.result-file-job.local-dispatch-enabled:true}")
|
||||
private boolean localDispatchEnabled;
|
||||
@@ -28,26 +30,32 @@ public class TaskFileJobLocalDispatcher {
|
||||
if (!localDispatchEnabled || jobId == null || jobId <= 0) {
|
||||
return false;
|
||||
}
|
||||
taskFileJobDispatchExecutor.execute(() -> {
|
||||
try {
|
||||
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
|
||||
if (job == null) {
|
||||
log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}",
|
||||
jobId, taskId, moduleType);
|
||||
return;
|
||||
try {
|
||||
taskFileJobDispatchExecutor.execute(() -> {
|
||||
try {
|
||||
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
|
||||
if (job == null) {
|
||||
log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}",
|
||||
jobId, taskId, moduleType);
|
||||
return;
|
||||
}
|
||||
TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable();
|
||||
if (taskResultFileJobWorker == null) {
|
||||
log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}",
|
||||
jobId, taskId, moduleType);
|
||||
return;
|
||||
}
|
||||
taskResultFileJobWorker.process(job);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
|
||||
jobId, taskId, moduleType, ex.getMessage(), ex);
|
||||
}
|
||||
TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable();
|
||||
if (taskResultFileJobWorker == null) {
|
||||
log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}",
|
||||
jobId, taskId, moduleType);
|
||||
return;
|
||||
}
|
||||
taskResultFileJobWorker.process(job);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
|
||||
jobId, taskId, moduleType, ex.getMessage(), ex);
|
||||
}
|
||||
});
|
||||
return true;
|
||||
});
|
||||
return true;
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[task-file-job] local dispatch executor rejected jobId={} taskId={} moduleType={} msg={}",
|
||||
jobId, taskId, moduleType, ex.getMessage(), ex);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+18
-1
@@ -53,7 +53,7 @@ public class TaskFileJobService {
|
||||
if (existing == null) {
|
||||
return null;
|
||||
}
|
||||
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus())) {
|
||||
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus()) || "PENDING".equals(existing.getStatus())) {
|
||||
return existing;
|
||||
}
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
@@ -192,6 +192,11 @@ public class TaskFileJobService {
|
||||
return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
|
||||
}
|
||||
|
||||
public boolean hasSuccessfulAssembleJob(Long taskId, String moduleType, Long resultId) {
|
||||
TaskFileJobEntity job = findAssembleJob(taskId, moduleType, resultId);
|
||||
return job != null && "SUCCESS".equals(job.getStatus());
|
||||
}
|
||||
|
||||
public Map<Long, TaskFileJobEntity> findAssembleJobsByResultIds(String moduleType, List<Long> resultIds) {
|
||||
List<Long> normalizedIds = resultIds == null ? List.of() : resultIds.stream()
|
||||
.filter(id -> id != null && id > 0)
|
||||
@@ -225,6 +230,18 @@ public class TaskFileJobService {
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
public long countActiveAssembleJobs(Long taskId, String moduleType) {
|
||||
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
|
||||
return 0L;
|
||||
}
|
||||
Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getTaskId, taskId)
|
||||
.eq(TaskFileJobEntity::getModuleType, moduleType)
|
||||
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
|
||||
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "RUNNING")));
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteTaskJobs(Long taskId, String moduleType) {
|
||||
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
|
||||
|
||||
+45
-11
@@ -8,12 +8,16 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
|
||||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
|
||||
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
|
||||
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.task.TaskExecutor;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@@ -34,8 +38,12 @@ public class TaskResultFileJobWorker {
|
||||
private final QueryAsinTaskService queryAsinTaskService;
|
||||
private final PatrolDeleteTaskService patrolDeleteTaskService;
|
||||
private final AppearancePatentTaskService appearancePatentTaskService;
|
||||
private final SimilarAsinTaskService similarAsinTaskService;
|
||||
private final DeleteBrandRunService deleteBrandRunService;
|
||||
private final BrandTaskService brandTaskService;
|
||||
@Autowired
|
||||
@Qualifier("cozeTaskExecutor")
|
||||
private TaskExecutor cozeTaskExecutor;
|
||||
|
||||
@Value("${aiimage.result-file-job.local-worker-enabled:true}")
|
||||
private boolean localWorkerEnabled;
|
||||
@@ -80,9 +88,29 @@ public class TaskResultFileJobWorker {
|
||||
if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) {
|
||||
return;
|
||||
}
|
||||
if ("APPEARANCE_PATENT".equals(job.getModuleType()) || "SIMILAR_ASIN".equals(job.getModuleType())) {
|
||||
try {
|
||||
cozeTaskExecutor.execute(() -> processInternal(job));
|
||||
return;
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[task-file-job] coze module offload failed, fallback inline jobId={} taskId={} moduleType={} msg={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
processInternal(job);
|
||||
}
|
||||
|
||||
private void processInternal(TaskFileJobEntity job) {
|
||||
long startedAt = System.currentTimeMillis();
|
||||
try {
|
||||
dispatch(job);
|
||||
boolean completed = dispatch(job);
|
||||
if (!completed) {
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
log.info("[task-file-job] process deferred jobId={} taskId={} moduleType={} resultId={} elapsedMs={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
|
||||
System.currentTimeMillis() - startedAt);
|
||||
return;
|
||||
}
|
||||
String resultFileUrl = resolveResultFileUrl(job);
|
||||
taskFileJobService.markSuccess(job, resultFileUrl);
|
||||
cleanupAfterSuccess(job);
|
||||
@@ -108,39 +136,41 @@ public class TaskResultFileJobWorker {
|
||||
return result == null ? null : result.getResultFileUrl();
|
||||
}
|
||||
|
||||
private void dispatch(TaskFileJobEntity job) {
|
||||
private boolean dispatch(TaskFileJobEntity job) {
|
||||
String moduleType = job.getModuleType();
|
||||
if ("SHOP_MATCH".equals(moduleType)) {
|
||||
shopMatchTaskService.processResultFileJob(job);
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
if ("PRICE_TRACK".equals(moduleType)) {
|
||||
priceTrackTaskService.processResultFileJob(job);
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
if ("PRODUCT_RISK_RESOLVE".equals(moduleType)) {
|
||||
productRiskTaskService.processResultFileJob(job);
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
if ("QUERY_ASIN".equals(moduleType)) {
|
||||
queryAsinTaskService.processResultFileJob(job);
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
if ("PATROL_DELETE".equals(moduleType)) {
|
||||
patrolDeleteTaskService.processResultFileJob(job);
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
if ("APPEARANCE_PATENT".equals(moduleType)) {
|
||||
appearancePatentTaskService.processResultFileJob(job);
|
||||
return;
|
||||
return appearancePatentTaskService.processResultFileJob(job);
|
||||
}
|
||||
if ("SIMILAR_ASIN".equals(moduleType)) {
|
||||
return similarAsinTaskService.processResultFileJob(job);
|
||||
}
|
||||
if ("DELETE_BRAND".equals(moduleType)) {
|
||||
deleteBrandRunService.processResultFileJob(job);
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
if ("BRAND".equals(moduleType)) {
|
||||
brandTaskService.processResultFileJob(job);
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
throw new IllegalArgumentException("unsupported result file job module: " + moduleType);
|
||||
}
|
||||
@@ -159,6 +189,10 @@ public class TaskResultFileJobWorker {
|
||||
appearancePatentTaskService.cleanupResultFileJob(job);
|
||||
return;
|
||||
}
|
||||
if ("SIMILAR_ASIN".equals(moduleType)) {
|
||||
similarAsinTaskService.cleanupResultFileJob(job);
|
||||
return;
|
||||
}
|
||||
if ("DELETE_BRAND".equals(moduleType)) {
|
||||
deleteBrandRunService.cleanupResultFileJob(job);
|
||||
}
|
||||
|
||||
+104
-11
@@ -1,29 +1,39 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.Locale;
|
||||
import java.util.UUID;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class TransientPayloadStorageService {
|
||||
|
||||
private static final String LOCAL_POINTER_PREFIX = "local:";
|
||||
private static final String RUSTFS_POINTER_PREFIX = "rustfs:";
|
||||
private static final String OSS_POINTER_PREFIX = "oss:";
|
||||
private static final String LOCAL_PAYLOAD_DIR = "transient-payload";
|
||||
|
||||
private final TransientStorageProperties properties;
|
||||
private final StorageProperties storageProperties;
|
||||
private final RustfsObjectStorageService rustfsObjectStorageService;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public boolean isWriteEnabled() {
|
||||
return properties.isEnabled() && rustfsObjectStorageService.isConfigured();
|
||||
return properties.isEnabled();
|
||||
}
|
||||
|
||||
public String storeScopePayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
|
||||
@@ -68,6 +78,9 @@ public class TransientPayloadStorageService {
|
||||
return value;
|
||||
}
|
||||
try {
|
||||
if (pointer.startsWith(LOCAL_POINTER_PREFIX)) {
|
||||
return readLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length()));
|
||||
}
|
||||
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
||||
return rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
|
||||
}
|
||||
@@ -85,6 +98,10 @@ public class TransientPayloadStorageService {
|
||||
if (pointer == null) {
|
||||
return;
|
||||
}
|
||||
if (pointer.startsWith(LOCAL_POINTER_PREFIX)) {
|
||||
deleteLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length()));
|
||||
return;
|
||||
}
|
||||
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
||||
rustfsObjectStorageService.deleteObject(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
|
||||
return;
|
||||
@@ -107,19 +124,28 @@ public class TransientPayloadStorageService {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
if (isPointer(value)) {
|
||||
return value;
|
||||
}
|
||||
try {
|
||||
String decoded = objectMapper.readValue(value, String.class);
|
||||
return isPointer(decoded) ? decoded : null;
|
||||
} catch (Exception ignored) {
|
||||
return null;
|
||||
String candidate = value.trim();
|
||||
for (int i = 0; i < 4; i++) {
|
||||
if (isPointer(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
try {
|
||||
String decoded = objectMapper.readValue(candidate, String.class);
|
||||
if (decoded == null || decoded.equals(candidate)) {
|
||||
return null;
|
||||
}
|
||||
candidate = decoded.trim();
|
||||
} catch (Exception ignored) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean isPointer(String value) {
|
||||
return value != null && (value.startsWith(RUSTFS_POINTER_PREFIX) || value.startsWith(OSS_POINTER_PREFIX));
|
||||
return value != null && (value.startsWith(LOCAL_POINTER_PREFIX)
|
||||
|| value.startsWith(RUSTFS_POINTER_PREFIX)
|
||||
|| value.startsWith(OSS_POINTER_PREFIX));
|
||||
}
|
||||
|
||||
private String store(String category,
|
||||
@@ -144,7 +170,19 @@ public class TransientPayloadStorageService {
|
||||
return content;
|
||||
}
|
||||
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
|
||||
String pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content, verifyAfterUpload);
|
||||
String pointer = storeLocal(objectKey, content);
|
||||
if (pointer == null && rustfsObjectStorageService.isConfigured()) {
|
||||
try {
|
||||
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content, verifyAfterUpload);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[transient-payload] rustfs upload failed and local fallback unavailable objectKey={} err={}",
|
||||
objectKey, ex.getMessage());
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
if (pointer == null) {
|
||||
return content;
|
||||
}
|
||||
if (!encodeAsJsonString) {
|
||||
return pointer;
|
||||
}
|
||||
@@ -155,6 +193,61 @@ public class TransientPayloadStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
private String storeLocal(String objectKey, String content) {
|
||||
try {
|
||||
Path root = localPayloadRoot();
|
||||
Path target = root.resolve(objectKey).normalize();
|
||||
if (!target.startsWith(root)) {
|
||||
throw new IllegalArgumentException("invalid local payload key: " + objectKey);
|
||||
}
|
||||
Files.createDirectories(target.getParent());
|
||||
Files.writeString(target,
|
||||
content == null ? "" : content,
|
||||
StandardCharsets.UTF_8,
|
||||
StandardOpenOption.CREATE,
|
||||
StandardOpenOption.TRUNCATE_EXISTING,
|
||||
StandardOpenOption.WRITE);
|
||||
return LOCAL_POINTER_PREFIX + objectKey;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[transient-payload] local store failed objectKey={} err={}", objectKey, ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String readLocalPayload(String objectKey) {
|
||||
try {
|
||||
Path target = resolveLocalPayloadPath(objectKey);
|
||||
return Files.readString(target, StandardCharsets.UTF_8);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to read local transient payload", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteLocalPayload(String objectKey) {
|
||||
try {
|
||||
Files.deleteIfExists(resolveLocalPayloadPath(objectKey));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[transient-payload] delete local payload failed objectKey={} err={}", objectKey, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Path resolveLocalPayloadPath(String objectKey) {
|
||||
Path root = localPayloadRoot();
|
||||
Path target = root.resolve(objectKey == null ? "" : objectKey).normalize();
|
||||
if (!target.startsWith(root)) {
|
||||
throw new IllegalArgumentException("invalid local payload key: " + objectKey);
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
private Path localPayloadRoot() {
|
||||
String configuredRoot = storageProperties.getLocalTempDir();
|
||||
Path tempRoot = configuredRoot == null || configuredRoot.isBlank()
|
||||
? Path.of(System.getProperty("java.io.tmpdir"))
|
||||
: Path.of(configuredRoot);
|
||||
return tempRoot.resolve(LOCAL_PAYLOAD_DIR).normalize();
|
||||
}
|
||||
|
||||
private String buildObjectKey(String category, String moduleType, Long taskId, String scopeHash, String entryKey) {
|
||||
String normalizedCategory = normalize(category, "unknown");
|
||||
String normalizedModuleType = normalize(moduleType, "unknown");
|
||||
|
||||
Reference in New Issue
Block a user