完成上架相关优化、oss迁移
This commit is contained in:
@@ -21,7 +21,6 @@
|
||||
<mybatis-plus.version>3.5.7</mybatis-plus.version>
|
||||
<mapstruct.version>1.5.5.Final</mapstruct.version>
|
||||
<easyexcel.version>4.0.3</easyexcel.version>
|
||||
<aliyun.oss.version>3.17.4</aliyun.oss.version>
|
||||
<hutool.version>5.8.36</hutool.version>
|
||||
<rocketmq-spring.version>2.3.5</rocketmq-spring.version>
|
||||
<minio.version>8.5.17</minio.version>
|
||||
@@ -80,11 +79,6 @@
|
||||
<artifactId>poi-ooxml</artifactId>
|
||||
<version>5.2.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.aliyun.oss</groupId>
|
||||
<artifactId>aliyun-sdk-oss</artifactId>
|
||||
<version>${aliyun.oss.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.xiaoymin</groupId>
|
||||
<artifactId>knife4j-openapi3-jakarta-spring-boot-starter</artifactId>
|
||||
|
||||
@@ -11,6 +11,7 @@ public class OssProperties {
|
||||
private String publicEndpoint;
|
||||
private String bucket;
|
||||
private String imageVideoBucket;
|
||||
private String digitalHumanBucket;
|
||||
private String accessKeyId;
|
||||
private String accessKeySecret;
|
||||
}
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ public class AppearancePatentHistoryItemVo {
|
||||
private String sourceFilename;
|
||||
@Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx")
|
||||
private String resultFilename;
|
||||
@Schema(description = "最终结果文件下载地址。后端基于 OSS objectKey 生成的公开直链 URL。", example = "https://bucket.oss-cn-hangzhou.aliyuncs.com/result/appearance_patent/xxx/17-result.xlsx")
|
||||
@Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "http://47.110.241.161:9000/nanri-ai-images/result/appearance_patent/xxx/17-result.xlsx")
|
||||
private String downloadUrl;
|
||||
private Long fileJobId;
|
||||
private String fileStatus;
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ import lombok.Data;
|
||||
public class BrandSourceFileDto {
|
||||
|
||||
@NotBlank(message = "fileUrl 不能为空")
|
||||
@Schema(description = "OSS 文件下载链接,Java 将先下载文件再读取内容。", example = "https://example.oss-cn-hangzhou.aliyuncs.com/source/brand.xlsx")
|
||||
@Schema(description = "对象存储文件下载链接,Java 将先下载文件再读取内容。", example = "http://47.110.241.161:9000/nanri-ai-images/source/brand.xlsx")
|
||||
private String fileUrl;
|
||||
|
||||
@Schema(description = "原始文件名,用于生成任务描述和结果文件名。", example = "品牌样例.xlsx")
|
||||
|
||||
+14
-32
@@ -1,11 +1,8 @@
|
||||
package com.nanri.aiimage.modules.digitalhuman.service;
|
||||
|
||||
import com.aliyun.oss.OSS;
|
||||
import com.aliyun.oss.OSSClientBuilder;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import com.nanri.aiimage.modules.digitalhuman.mapper.DigitalHumanVersionMapper;
|
||||
import com.nanri.aiimage.modules.digitalhuman.model.entity.DigitalHumanVersionEntity;
|
||||
import com.nanri.aiimage.modules.digitalhuman.model.vo.DigitalHumanVersionVo;
|
||||
@@ -33,11 +30,10 @@ public class DigitalHumanVersionService {
|
||||
|
||||
private static final String STATUS_DRAFT = "DRAFT";
|
||||
private static final String STATUS_RELEASED = "RELEASED";
|
||||
private static final String OSS_PATH_PREFIX = "digital-human/versions/";
|
||||
private static final String STORAGE_PATH_PREFIX = "digital-human/versions/";
|
||||
|
||||
private final DigitalHumanVersionMapper versionMapper;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final OssProperties ossProperties;
|
||||
|
||||
@Transactional
|
||||
public DigitalHumanVersionVo uploadVersion(String version, MultipartFile file, String changelog,
|
||||
@@ -67,23 +63,16 @@ public class DigitalHumanVersionService {
|
||||
log.info("[digital-human-version] md5 calculated version={} md5={} elapsedMs={}",
|
||||
version, md5, elapsedMs(startedAt));
|
||||
|
||||
// 构建 OSS 路径
|
||||
String ossObjectKey = OSS_PATH_PREFIX + "v" + version + "/ShuFuDigitalHuman.zip";
|
||||
String ossObjectKey = STORAGE_PATH_PREFIX + "v" + version + "/ShuFuDigitalHuman.zip";
|
||||
|
||||
// 上传到 OSS
|
||||
OSS ossClient = buildOssClient();
|
||||
try {
|
||||
log.info("[digital-human-version] oss upload start version={} objectKey={} bytes={} elapsedMs={}",
|
||||
version, ossObjectKey, tempFile.length(), elapsedMs(startedAt));
|
||||
ossClient.putObject(ossProperties.getBucket(), ossObjectKey, tempFile);
|
||||
if (!ossClient.doesObjectExist(ossProperties.getBucket(), ossObjectKey)) {
|
||||
throw new BusinessException("数字人版本文件上传后在 OSS 中不可见,请重试");
|
||||
}
|
||||
log.info("[digital-human-version] oss uploaded version={} objectKey={} bytes={} elapsedMs={}",
|
||||
version, ossObjectKey, tempFile.length(), elapsedMs(startedAt));
|
||||
} finally {
|
||||
ossClient.shutdown();
|
||||
log.info("[digital-human-version] minio upload start version={} objectKey={} bytes={} elapsedMs={}",
|
||||
version, ossObjectKey, tempFile.length(), elapsedMs(startedAt));
|
||||
ossStorageService.uploadDigitalHumanVersion(tempFile, ossObjectKey);
|
||||
if (!ossStorageService.objectExists(ossObjectKey)) {
|
||||
throw new BusinessException("数字人版本文件上传后在 MinIO 中不可见,请重试");
|
||||
}
|
||||
log.info("[digital-human-version] minio uploaded version={} objectKey={} bytes={} elapsedMs={}",
|
||||
version, ossObjectKey, tempFile.length(), elapsedMs(startedAt));
|
||||
|
||||
// 保存数据库记录
|
||||
DigitalHumanVersionEntity entity = new DigitalHumanVersionEntity();
|
||||
@@ -161,7 +150,7 @@ public class DigitalHumanVersionService {
|
||||
}
|
||||
|
||||
if (!ossStorageService.objectExists(entity.getOssObjectKey())) {
|
||||
throw new BusinessException("数字人版本文件不存在于 OSS,请重新上传该版本:" + entity.getOssObjectKey());
|
||||
throw new BusinessException("数字人版本文件不存在于 MinIO,请重新上传该版本:" + entity.getOssObjectKey());
|
||||
}
|
||||
|
||||
entity.setStatus(STATUS_RELEASED);
|
||||
@@ -184,7 +173,7 @@ public class DigitalHumanVersionService {
|
||||
|
||||
// 清除其他版本的 is_latest 标记
|
||||
if (!ossStorageService.objectExists(entity.getOssObjectKey())) {
|
||||
throw new BusinessException("数字人版本文件不存在于 OSS,请重新上传该版本:" + entity.getOssObjectKey());
|
||||
throw new BusinessException("数字人版本文件不存在于 MinIO,请重新上传该版本:" + entity.getOssObjectKey());
|
||||
}
|
||||
|
||||
versionMapper.update(null, new LambdaUpdateWrapper<DigitalHumanVersionEntity>()
|
||||
@@ -209,11 +198,11 @@ public class DigitalHumanVersionService {
|
||||
throw new BusinessException("最新版本不能删除");
|
||||
}
|
||||
|
||||
// 删除 OSS 文件
|
||||
// 删除 MinIO 文件
|
||||
try {
|
||||
ossStorageService.deleteObject(entity.getOssObjectKey());
|
||||
} catch (Exception e) {
|
||||
log.warn("删除 OSS 文件失败:{}", entity.getOssObjectKey(), e);
|
||||
log.warn("删除 MinIO 文件失败:{}", entity.getOssObjectKey(), e);
|
||||
}
|
||||
|
||||
// 删除数据库记录
|
||||
@@ -227,7 +216,7 @@ public class DigitalHumanVersionService {
|
||||
throw new BusinessException("版本不存在:" + version);
|
||||
}
|
||||
if (!ossStorageService.objectExists(entity.getOssObjectKey())) {
|
||||
throw new BusinessException("数字人版本文件不存在于 OSS,请重新上传该版本:" + entity.getOssObjectKey());
|
||||
throw new BusinessException("数字人版本文件不存在于 MinIO,请重新上传该版本:" + entity.getOssObjectKey());
|
||||
}
|
||||
return ossStorageService.generateDownloadUrl(entity.getOssObjectKey());
|
||||
}
|
||||
@@ -265,11 +254,4 @@ public class DigitalHumanVersionService {
|
||||
}
|
||||
}
|
||||
|
||||
private OSS buildOssClient() {
|
||||
return new OSSClientBuilder().build(
|
||||
"https://" + ossProperties.getEndpoint(),
|
||||
ossProperties.getAccessKeyId(),
|
||||
ossProperties.getAccessKeySecret()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+274
-145
@@ -1,10 +1,14 @@
|
||||
package com.nanri.aiimage.modules.file.service.oss;
|
||||
|
||||
import com.aliyun.oss.OSS;
|
||||
import com.aliyun.oss.OSSClientBuilder;
|
||||
import com.aliyun.oss.model.OSSObject;
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import io.minio.GetObjectArgs;
|
||||
import io.minio.MinioClient;
|
||||
import io.minio.PutObjectArgs;
|
||||
import io.minio.RemoveObjectArgs;
|
||||
import io.minio.StatObjectArgs;
|
||||
import io.minio.UploadObjectArgs;
|
||||
import io.minio.errors.ErrorResponseException;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
@@ -12,232 +16,300 @@ import java.io.File;
|
||||
import java.net.URI;
|
||||
import java.net.URLDecoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class OssStorageService {
|
||||
|
||||
private final OssProperties ossProperties;
|
||||
private static final String IMAGE_VIDEO_MODULE = "IMAGE_VIDEO";
|
||||
private static final String DIGITAL_HUMAN_PREFIX = "digital-human/versions/";
|
||||
|
||||
private final OssProperties ossProperties;
|
||||
private final MinioClient minioClient;
|
||||
|
||||
@Autowired
|
||||
public OssStorageService(OssProperties ossProperties) {
|
||||
this(ossProperties, null);
|
||||
}
|
||||
|
||||
OssStorageService(OssProperties ossProperties, MinioClient minioClient) {
|
||||
this.ossProperties = ossProperties;
|
||||
this.minioClient = minioClient == null ? createClient(ossProperties) : minioClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* 上传结果文件到 OSS,返回 objectKey。
|
||||
* 调用方应存储 objectKey,下载时通过 generateFreshDownloadUrl 生成公开直链。
|
||||
*/
|
||||
public String uploadResultFile(File file, String moduleType) {
|
||||
String objectKey = String.format("result/%s/%s/%s", moduleType.toLowerCase(), UUID.randomUUID(), file.getName());
|
||||
OSS ossClient = buildClient();
|
||||
try {
|
||||
ossClient.putObject(ossProperties.getBucket(), objectKey, file);
|
||||
return objectKey;
|
||||
} finally {
|
||||
ossClient.shutdown();
|
||||
}
|
||||
String objectKey = resultObjectKey(file, moduleType);
|
||||
uploadFile(file, resolveBucket(moduleType), objectKey);
|
||||
return objectKey;
|
||||
}
|
||||
|
||||
public UploadedResult uploadResultFileWithFreshDownloadUrl(File file, String moduleType) {
|
||||
String objectKey = String.format("result/%s/%s/%s", moduleType.toLowerCase(), UUID.randomUUID(), file.getName());
|
||||
String objectKey = resultObjectKey(file, moduleType);
|
||||
String bucket = resolveBucket(moduleType);
|
||||
OSS ossClient = buildClient();
|
||||
try {
|
||||
ossClient.putObject(bucket, objectKey, file);
|
||||
return new UploadedResult(objectKey, getPublicUrl(objectKey, bucket));
|
||||
} finally {
|
||||
ossClient.shutdown();
|
||||
}
|
||||
uploadFile(file, bucket, objectKey);
|
||||
return new UploadedResult(objectKey, getPublicUrl(objectKey, bucket));
|
||||
}
|
||||
|
||||
public UploadedResult uploadPublicFileWithFreshDownloadUrl(File file, String moduleType, String originalFilename) {
|
||||
String normalizedModuleType = moduleType == null || moduleType.isBlank()
|
||||
? "common"
|
||||
: moduleType.trim().toLowerCase();
|
||||
String normalizedModuleType = normalizeModuleType(moduleType, "common");
|
||||
String objectName = sanitizeObjectName(originalFilename);
|
||||
if (objectName.isBlank()) {
|
||||
objectName = file.getName();
|
||||
}
|
||||
String objectKey = String.format("upload/%s/%s/%s", normalizedModuleType, UUID.randomUUID(), objectName);
|
||||
String bucket = resolveBucket(moduleType);
|
||||
OSS ossClient = buildClient();
|
||||
try {
|
||||
ossClient.putObject(bucket, objectKey, file);
|
||||
return new UploadedResult(objectKey, getPublicUrl(objectKey, bucket));
|
||||
} finally {
|
||||
ossClient.shutdown();
|
||||
uploadFile(file, bucket, objectKey);
|
||||
return new UploadedResult(objectKey, getPublicUrl(objectKey, bucket));
|
||||
}
|
||||
|
||||
public String uploadDigitalHumanVersion(File file, String objectKey) {
|
||||
if (objectKey == null || !objectKey.startsWith(DIGITAL_HUMAN_PREFIX)) {
|
||||
throw new IllegalArgumentException("digital human objectKey must start with " + DIGITAL_HUMAN_PREFIX);
|
||||
}
|
||||
uploadFile(file, digitalHumanBucket(), objectKey);
|
||||
return objectKey;
|
||||
}
|
||||
|
||||
public String uploadText(String objectKey, String content) {
|
||||
if (objectKey == null || objectKey.isBlank()) {
|
||||
throw new IllegalArgumentException("objectKey must not be blank");
|
||||
}
|
||||
OSS ossClient = buildClient();
|
||||
try (ByteArrayInputStream stream = new ByteArrayInputStream(
|
||||
Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8))) {
|
||||
ossClient.putObject(ossProperties.getBucket(), objectKey, stream);
|
||||
byte[] bytes = Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8);
|
||||
try (ByteArrayInputStream stream = new ByteArrayInputStream(bytes)) {
|
||||
buildClient().putObject(PutObjectArgs.builder()
|
||||
.bucket(ossProperties.getBucket())
|
||||
.object(objectKey)
|
||||
.stream(stream, bytes.length, -1)
|
||||
.contentType("application/json; charset=utf-8")
|
||||
.build());
|
||||
return objectKey;
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to upload text to oss", ex);
|
||||
} finally {
|
||||
ossClient.shutdown();
|
||||
throw storageFailure("upload text", objectKey, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public String uploadTaskScopePayload(String moduleType, Long taskId, String scopeHash, String content) {
|
||||
String normalizedModuleType = moduleType == null || moduleType.isBlank()
|
||||
? "unknown"
|
||||
: moduleType.trim().toLowerCase();
|
||||
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
|
||||
String objectKey = String.format("task-scope/%s/%s/%s.json", normalizedModuleType, taskId, normalizedScopeHash);
|
||||
String objectKey = String.format("task-scope/%s/%s/%s.json", normalizeModuleType(moduleType), taskId, normalizedScopeHash);
|
||||
return uploadText(objectKey, content);
|
||||
}
|
||||
|
||||
public String uploadTaskParsedPayload(String moduleType, Long taskId, String scopeHash, String content) {
|
||||
String normalizedModuleType = moduleType == null || moduleType.isBlank()
|
||||
? "unknown"
|
||||
: moduleType.trim().toLowerCase();
|
||||
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
|
||||
String objectKey = String.format("task-parsed/%s/%s/%s.json", normalizedModuleType, taskId, normalizedScopeHash);
|
||||
String objectKey = String.format("task-parsed/%s/%s/%s.json", normalizeModuleType(moduleType), taskId, normalizedScopeHash);
|
||||
return uploadText(objectKey, content);
|
||||
}
|
||||
|
||||
public String readObjectAsString(String value) {
|
||||
String objectKey = resolveObjectKey(value);
|
||||
if (objectKey == null || objectKey.isBlank()) {
|
||||
StorageLocation location = resolveStorageLocation(value);
|
||||
if (location == null) {
|
||||
return null;
|
||||
}
|
||||
OSS ossClient = buildClient();
|
||||
try (OSSObject ossObject = ossClient.getObject(ossProperties.getBucket(), objectKey)) {
|
||||
return new String(ossObject.getObjectContent().readAllBytes(), StandardCharsets.UTF_8);
|
||||
try (var stream = buildClient().getObject(GetObjectArgs.builder()
|
||||
.bucket(location.bucket())
|
||||
.object(location.objectKey())
|
||||
.build())) {
|
||||
return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to read object from oss", ex);
|
||||
} finally {
|
||||
ossClient.shutdown();
|
||||
throw storageFailure("read", location.objectKey(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteObject(String value) {
|
||||
String objectKey = resolveObjectKey(value);
|
||||
if (objectKey == null || objectKey.isBlank()) {
|
||||
StorageLocation location = resolveStorageLocation(value);
|
||||
if (location == null) {
|
||||
return;
|
||||
}
|
||||
OSS ossClient = buildClient();
|
||||
try {
|
||||
ossClient.deleteObject(ossProperties.getBucket(), objectKey);
|
||||
} finally {
|
||||
ossClient.shutdown();
|
||||
buildClient().removeObject(RemoveObjectArgs.builder()
|
||||
.bucket(location.bucket())
|
||||
.object(location.objectKey())
|
||||
.build());
|
||||
} catch (Exception ex) {
|
||||
throw storageFailure("delete", location.objectKey(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
public boolean objectExists(String value) {
|
||||
String objectKey = resolveObjectKey(value);
|
||||
if (objectKey == null || objectKey.isBlank()) {
|
||||
StorageLocation location = resolveStorageLocation(value);
|
||||
if (location == null) {
|
||||
return false;
|
||||
}
|
||||
OSS ossClient = buildClient();
|
||||
try {
|
||||
return ossClient.doesObjectExist(ossProperties.getBucket(), objectKey);
|
||||
} finally {
|
||||
ossClient.shutdown();
|
||||
buildClient().statObject(StatObjectArgs.builder()
|
||||
.bucket(location.bucket())
|
||||
.object(location.objectKey())
|
||||
.build());
|
||||
return true;
|
||||
} catch (ErrorResponseException ex) {
|
||||
if (isNotFound(ex)) {
|
||||
return false;
|
||||
}
|
||||
throw storageFailure("stat", location.objectKey(), ex);
|
||||
} catch (Exception ex) {
|
||||
throw storageFailure("stat", location.objectKey(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据 objectKey 生成公开直链下载 URL。
|
||||
*/
|
||||
public String generateDownloadUrl(String objectKey) {
|
||||
return getPublicUrl(objectKey);
|
||||
public String generateDownloadUrl(String value) {
|
||||
return getPublicUrl(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取公开(无签名)URL,格式:https://{bucket}.{endpoint}/{objectKey}
|
||||
*/
|
||||
public String getPublicUrl(String value) {
|
||||
return getPublicUrl(value, ossProperties.getBucket());
|
||||
StorageLocation location = resolveStorageLocation(value);
|
||||
return location == null ? null : getPublicUrl(location.objectKey(), location.bucket());
|
||||
}
|
||||
|
||||
private String getPublicUrl(String value, String bucket) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
String objectKey = resolveObjectKey(value);
|
||||
String normalizedKey = objectKey.startsWith("/") ? objectKey.substring(1) : objectKey;
|
||||
String host = String.format("%s.%s", bucket, publicEndpoint());
|
||||
try {
|
||||
return new URI("https", host, "/" + normalizedKey, null).toASCIIString();
|
||||
} catch (Exception ignored) {
|
||||
return String.format("https://%s/%s", host, normalizedKey);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从存储值中解析出 objectKey,兼容两种格式:
|
||||
* - 旧格式:完整 URL(https://bucket.endpoint/objectKey?Expires=...)
|
||||
* - 新格式:直接是 objectKey(如 result/dedupe/uuid/file.xlsx)
|
||||
*/
|
||||
public String resolveObjectKey(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return value;
|
||||
}
|
||||
try {
|
||||
if (value.startsWith("http://") || value.startsWith("https://")) {
|
||||
String path = URI.create(value).getPath();
|
||||
return path.startsWith("/") ? path.substring(1) : path;
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
int schemeEnd = value.indexOf("://");
|
||||
int pathStart = schemeEnd < 0 ? -1 : value.indexOf('/', schemeEnd + 3);
|
||||
if (pathStart >= 0 && pathStart + 1 < value.length()) {
|
||||
String path = value.substring(pathStart + 1);
|
||||
int queryStart = path.indexOf('?');
|
||||
String objectKey = queryStart >= 0 ? path.substring(0, queryStart) : path;
|
||||
return URLDecoder.decode(objectKey, StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
return value;
|
||||
StorageLocation location = resolveStorageLocation(value);
|
||||
return location == null ? value : location.objectKey();
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据存储值(objectKey 或旧格式完整 URL)生成公开直链下载 URL。
|
||||
* 供各模块 listHistory 和下载接口使用,避免签名过期。
|
||||
*/
|
||||
public String generateFreshDownloadUrl(String value) {
|
||||
return getPublicUrl(value);
|
||||
}
|
||||
|
||||
private void uploadFile(File file, String bucket, String objectKey) {
|
||||
if (file == null || !file.isFile()) {
|
||||
throw new IllegalArgumentException("upload file must exist");
|
||||
}
|
||||
try {
|
||||
buildClient().uploadObject(UploadObjectArgs.builder()
|
||||
.bucket(bucket)
|
||||
.object(objectKey)
|
||||
.filename(file.getAbsolutePath())
|
||||
.build());
|
||||
} catch (Exception ex) {
|
||||
throw storageFailure("upload", objectKey, ex);
|
||||
}
|
||||
}
|
||||
|
||||
private String getPublicUrl(String objectKey, String bucket) {
|
||||
String encodedPath;
|
||||
try {
|
||||
encodedPath = new URI(null, null, "/" + bucket + "/" + trimLeadingSlash(objectKey), null)
|
||||
.toASCIIString();
|
||||
} catch (Exception ex) {
|
||||
encodedPath = "/" + bucket + "/" + trimLeadingSlash(objectKey).replace(" ", "%20");
|
||||
}
|
||||
return trimTrailingSlash(publicEndpoint()) + encodedPath;
|
||||
}
|
||||
|
||||
private StorageLocation resolveStorageLocation(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return generateDownloadUrl(resolveObjectKey(value));
|
||||
String normalizedValue = value.trim();
|
||||
String host = null;
|
||||
String objectKey = normalizedValue;
|
||||
if (normalizedValue.startsWith("http://") || normalizedValue.startsWith("https://")) {
|
||||
try {
|
||||
URI uri = URI.create(normalizedValue);
|
||||
host = uri.getHost();
|
||||
objectKey = decodePath(uri.getRawPath());
|
||||
} catch (Exception ignored) {
|
||||
objectKey = decodePathFromMalformedUrl(normalizedValue);
|
||||
}
|
||||
}
|
||||
|
||||
objectKey = trimLeadingSlash(objectKey);
|
||||
String bucket = bucketFromVirtualHost(host);
|
||||
String pathBucket = bucketPrefix(objectKey);
|
||||
if (pathBucket != null) {
|
||||
bucket = pathBucket;
|
||||
objectKey = objectKey.substring(pathBucket.length() + 1);
|
||||
}
|
||||
if (bucket == null) {
|
||||
bucket = bucketForObjectKey(objectKey);
|
||||
}
|
||||
return objectKey.isBlank() ? null : new StorageLocation(bucket, objectKey);
|
||||
}
|
||||
|
||||
private OSS buildClient() {
|
||||
return new OSSClientBuilder().build(
|
||||
"https://" + ossProperties.getEndpoint(),
|
||||
ossProperties.getAccessKeyId(),
|
||||
ossProperties.getAccessKeySecret()
|
||||
);
|
||||
private String bucketFromVirtualHost(String host) {
|
||||
if (host == null) {
|
||||
return null;
|
||||
}
|
||||
String normalizedHost = host.toLowerCase(Locale.ROOT);
|
||||
for (String bucket : configuredBuckets()) {
|
||||
if (normalizedHost.equals(bucket.toLowerCase(Locale.ROOT))
|
||||
|| normalizedHost.startsWith(bucket.toLowerCase(Locale.ROOT) + ".")) {
|
||||
return bucket;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String resolveBucket(String moduleType) {
|
||||
if (moduleType != null
|
||||
&& "IMAGE_VIDEO".equalsIgnoreCase(moduleType.trim())
|
||||
&& ossProperties.getImageVideoBucket() != null
|
||||
&& !ossProperties.getImageVideoBucket().isBlank()) {
|
||||
return ossProperties.getImageVideoBucket().trim();
|
||||
private String bucketPrefix(String objectKey) {
|
||||
for (String bucket : configuredBuckets()) {
|
||||
if (objectKey.equals(bucket) || objectKey.startsWith(bucket + "/")) {
|
||||
return bucket;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String bucketForObjectKey(String objectKey) {
|
||||
if (objectKey.startsWith(DIGITAL_HUMAN_PREFIX)) {
|
||||
return digitalHumanBucket();
|
||||
}
|
||||
if (objectKey.startsWith("result/image_video/") || objectKey.startsWith("upload/image_video/")) {
|
||||
return imageVideoBucket();
|
||||
}
|
||||
return ossProperties.getBucket();
|
||||
}
|
||||
|
||||
private List<String> configuredBuckets() {
|
||||
return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket())
|
||||
.filter(Objects::nonNull)
|
||||
.map(String::trim)
|
||||
.filter(bucket -> !bucket.isBlank())
|
||||
.distinct()
|
||||
.toList();
|
||||
}
|
||||
|
||||
private MinioClient buildClient() {
|
||||
return minioClient;
|
||||
}
|
||||
|
||||
private static MinioClient createClient(OssProperties properties) {
|
||||
return MinioClient.builder()
|
||||
.endpoint(withScheme(properties.getEndpoint()))
|
||||
.credentials(properties.getAccessKeyId(), properties.getAccessKeySecret())
|
||||
.build();
|
||||
}
|
||||
|
||||
private String resolveBucket(String moduleType) {
|
||||
return IMAGE_VIDEO_MODULE.equalsIgnoreCase(Objects.requireNonNullElse(moduleType, "").trim())
|
||||
? imageVideoBucket()
|
||||
: ossProperties.getBucket();
|
||||
}
|
||||
|
||||
private String imageVideoBucket() {
|
||||
return firstNonBlank(ossProperties.getImageVideoBucket(), ossProperties.getBucket());
|
||||
}
|
||||
|
||||
private String digitalHumanBucket() {
|
||||
return firstNonBlank(ossProperties.getDigitalHumanBucket(), ossProperties.getBucket());
|
||||
}
|
||||
|
||||
private String publicEndpoint() {
|
||||
String endpoint = ossProperties.getPublicEndpoint();
|
||||
if (endpoint == null || endpoint.isBlank()) {
|
||||
endpoint = ossProperties.getEndpoint();
|
||||
}
|
||||
endpoint = endpoint.trim();
|
||||
if (endpoint.startsWith("http://")) {
|
||||
endpoint = endpoint.substring("http://".length());
|
||||
} else if (endpoint.startsWith("https://")) {
|
||||
endpoint = endpoint.substring("https://".length());
|
||||
}
|
||||
return endpoint.endsWith("/") ? endpoint.substring(0, endpoint.length() - 1) : endpoint;
|
||||
return withScheme(firstNonBlank(ossProperties.getPublicEndpoint(), ossProperties.getEndpoint()));
|
||||
}
|
||||
|
||||
private String resultObjectKey(File file, String moduleType) {
|
||||
return String.format("result/%s/%s/%s", normalizeModuleType(moduleType), UUID.randomUUID(), file.getName());
|
||||
}
|
||||
|
||||
private String normalizeModuleType(String moduleType) {
|
||||
return normalizeModuleType(moduleType, "unknown");
|
||||
}
|
||||
|
||||
private String normalizeModuleType(String moduleType, String defaultValue) {
|
||||
return moduleType == null || moduleType.isBlank() ? defaultValue : moduleType.trim().toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String sanitizeObjectName(String filename) {
|
||||
@@ -252,6 +324,63 @@ public class OssStorageService {
|
||||
return normalized.replaceAll("[\\r\\n]", "_");
|
||||
}
|
||||
|
||||
private String decodePath(String rawPath) {
|
||||
if (rawPath == null) {
|
||||
return "";
|
||||
}
|
||||
return URLDecoder.decode(rawPath.replace("+", "%2B"), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private String decodePathFromMalformedUrl(String value) {
|
||||
int schemeEnd = value.indexOf("://");
|
||||
int pathStart = schemeEnd < 0 ? -1 : value.indexOf('/', schemeEnd + 3);
|
||||
if (pathStart < 0 || pathStart + 1 >= value.length()) {
|
||||
return "";
|
||||
}
|
||||
String path = value.substring(pathStart + 1);
|
||||
int queryStart = path.indexOf('?');
|
||||
return URLDecoder.decode(queryStart >= 0 ? path.substring(0, queryStart) : path, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private boolean isNotFound(ErrorResponseException ex) {
|
||||
String code = ex.errorResponse() == null ? null : ex.errorResponse().code();
|
||||
return "NoSuchKey".equals(code) || "NoSuchObject".equals(code) || "NoSuchBucket".equals(code);
|
||||
}
|
||||
|
||||
private IllegalStateException storageFailure(String operation, String objectKey, Exception cause) {
|
||||
return new IllegalStateException("failed to " + operation + " object in MinIO: " + objectKey, cause);
|
||||
}
|
||||
|
||||
private static String withScheme(String endpoint) {
|
||||
String normalized = Objects.requireNonNull(endpoint, "MinIO endpoint must not be null").trim();
|
||||
return normalized.startsWith("http://") || normalized.startsWith("https://")
|
||||
? normalized
|
||||
: "http://" + normalized;
|
||||
}
|
||||
|
||||
private String trimLeadingSlash(String value) {
|
||||
String result = Objects.requireNonNullElse(value, "");
|
||||
while (result.startsWith("/")) {
|
||||
result = result.substring(1);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String trimTrailingSlash(String value) {
|
||||
String result = value;
|
||||
while (result.endsWith("/")) {
|
||||
result = result.substring(0, result.length() - 1);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String firstNonBlank(String preferred, String fallback) {
|
||||
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||
}
|
||||
|
||||
private record StorageLocation(String bucket, String objectKey) {
|
||||
}
|
||||
|
||||
public record UploadedResult(String objectKey, String downloadUrl) {
|
||||
}
|
||||
}
|
||||
|
||||
+18
-6
@@ -3,18 +3,30 @@ package com.nanri.aiimage.modules.imagevideo.mapper;
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.imagevideo.model.entity.ImageVideoAsyncTaskEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
|
||||
@Mapper
|
||||
public interface ImageVideoAsyncTaskMapper extends BaseMapper<ImageVideoAsyncTaskEntity> {
|
||||
|
||||
@Update("UPDATE biz_image_video_async_task "
|
||||
+ "SET status = 'RUNNING', updated_at = NOW(), attempt_count = attempt_count + 1 "
|
||||
+ "WHERE id = #{taskId} AND status = 'PENDING'")
|
||||
int claimPending(Long taskId);
|
||||
+ "SET status = 'RUNNING', owner_instance_id = CASE "
|
||||
+ "WHEN owner_instance_id IS NULL OR owner_instance_id = '' THEN #{ownerInstanceId} "
|
||||
+ "ELSE owner_instance_id END, updated_at = NOW(), attempt_count = attempt_count + 1 "
|
||||
+ "WHERE id = #{taskId} AND status = 'PENDING' "
|
||||
+ "AND (owner_instance_id = #{ownerInstanceId} OR owner_instance_id IS NULL OR owner_instance_id = '')")
|
||||
int claimPending(@Param("taskId") Long taskId, @Param("ownerInstanceId") String ownerInstanceId);
|
||||
|
||||
@Update("UPDATE biz_image_video_async_task "
|
||||
+ "SET status = 'POLLING', updated_at = NOW(), attempt_count = attempt_count + 1 "
|
||||
+ "WHERE id = #{taskId} AND status = 'WAITING'")
|
||||
int claimWaiting(Long taskId);
|
||||
+ "SET status = 'POLLING', owner_instance_id = CASE "
|
||||
+ "WHEN owner_instance_id IS NULL OR owner_instance_id = '' THEN #{ownerInstanceId} "
|
||||
+ "ELSE owner_instance_id END, updated_at = NOW(), attempt_count = attempt_count + 1 "
|
||||
+ "WHERE id = #{taskId} AND status = 'WAITING' "
|
||||
+ "AND (owner_instance_id = #{ownerInstanceId} OR owner_instance_id IS NULL OR owner_instance_id = '')")
|
||||
int claimWaiting(@Param("taskId") Long taskId, @Param("ownerInstanceId") String ownerInstanceId);
|
||||
|
||||
@Update("UPDATE biz_image_video_async_task SET status = 'WAITING', updated_at = NOW() "
|
||||
+ "WHERE status = 'POLLING' "
|
||||
+ "AND (owner_instance_id = #{ownerInstanceId} OR owner_instance_id IS NULL OR owner_instance_id = '')")
|
||||
int requeuePollingTasks(@Param("ownerInstanceId") String ownerInstanceId);
|
||||
}
|
||||
|
||||
+1
@@ -14,6 +14,7 @@ public class ImageVideoAsyncTaskEntity {
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private Long userId;
|
||||
private String ownerInstanceId;
|
||||
private String taskType;
|
||||
private String status;
|
||||
private String requestJson;
|
||||
|
||||
+58
-7
@@ -4,6 +4,8 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.modules.imagevideo.mapper.ImageVideoAsyncTaskMapper;
|
||||
import com.nanri.aiimage.modules.imagevideo.model.dto.DouyinCopyRequest;
|
||||
import com.nanri.aiimage.modules.imagevideo.model.dto.ImageVideoVoiceCloneRequest;
|
||||
@@ -15,7 +17,9 @@ import com.nanri.aiimage.modules.imagevideo.model.dto.ImageVideoWorkflowRunReque
|
||||
import com.nanri.aiimage.modules.imagevideo.model.entity.ImageVideoAsyncTaskEntity;
|
||||
import com.nanri.aiimage.modules.imagevideo.model.vo.ImageVideoAsyncTaskVo;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.core.task.TaskExecutor;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -24,6 +28,7 @@ import java.time.LocalDateTime;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
@Slf4j
|
||||
@@ -51,6 +56,7 @@ public class ImageVideoAsyncTaskService {
|
||||
private final ImageVideoArchiveService archiveService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TaskExecutor cozeTaskExecutor;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
|
||||
public ImageVideoAsyncTaskService(
|
||||
ImageVideoAsyncTaskMapper taskMapper,
|
||||
@@ -58,13 +64,15 @@ public class ImageVideoAsyncTaskService {
|
||||
ImageVideoWorkflowConfigService workflowConfigService,
|
||||
ImageVideoArchiveService archiveService,
|
||||
ObjectMapper objectMapper,
|
||||
@Qualifier("cozeTaskExecutor") TaskExecutor cozeTaskExecutor) {
|
||||
@Qualifier("cozeTaskExecutor") TaskExecutor cozeTaskExecutor,
|
||||
InstanceMetadata instanceMetadata) {
|
||||
this.taskMapper = taskMapper;
|
||||
this.cozeService = cozeService;
|
||||
this.workflowConfigService = workflowConfigService;
|
||||
this.archiveService = archiveService;
|
||||
this.objectMapper = objectMapper;
|
||||
this.cozeTaskExecutor = cozeTaskExecutor;
|
||||
this.instanceMetadata = instanceMetadata;
|
||||
}
|
||||
|
||||
public ImageVideoAsyncTaskVo submitDouyinCopy(DouyinCopyRequest request) {
|
||||
@@ -102,6 +110,7 @@ public class ImageVideoAsyncTaskService {
|
||||
if (task == null) {
|
||||
throw new BusinessException("Image video task not found");
|
||||
}
|
||||
ensureTaskOwnedByCurrentInstance(task, "query async task");
|
||||
task = recoverFalseFailedTask(task);
|
||||
ImageVideoAsyncTaskVo result = toVo(task);
|
||||
if (TaskStatus.FAILED.name().equals(task.getStatus())) {
|
||||
@@ -114,6 +123,9 @@ public class ImageVideoAsyncTaskService {
|
||||
public void dispatchPendingTasks() {
|
||||
List<ImageVideoAsyncTaskEntity> tasks = taskMapper.selectList(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
|
||||
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.PENDING.name())
|
||||
.and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
|
||||
.or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
|
||||
.or().eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, ""))
|
||||
.orderByAsc(ImageVideoAsyncTaskEntity::getId)
|
||||
.last("LIMIT " + DISPATCH_BATCH_SIZE));
|
||||
tasks.forEach(task -> cozeTaskExecutor.execute(() -> executeTask(task.getId())));
|
||||
@@ -123,11 +135,23 @@ public class ImageVideoAsyncTaskService {
|
||||
public void pollWaitingTasks() {
|
||||
List<ImageVideoAsyncTaskEntity> tasks = taskMapper.selectList(new LambdaQueryWrapper<ImageVideoAsyncTaskEntity>()
|
||||
.eq(ImageVideoAsyncTaskEntity::getStatus, TaskStatus.WAITING.name())
|
||||
.and(q -> q.eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, currentInstanceId())
|
||||
.or().isNull(ImageVideoAsyncTaskEntity::getOwnerInstanceId)
|
||||
.or().eq(ImageVideoAsyncTaskEntity::getOwnerInstanceId, ""))
|
||||
.orderByAsc(ImageVideoAsyncTaskEntity::getUpdatedAt)
|
||||
.last("LIMIT " + POLL_BATCH_SIZE));
|
||||
tasks.forEach(task -> cozeTaskExecutor.execute(() -> pollTask(task.getId())));
|
||||
}
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void recoverInterruptedPollingTasks() {
|
||||
int recovered = taskMapper.requeuePollingTasks(currentInstanceId());
|
||||
if (recovered > 0) {
|
||||
log.warn("[image-video] recovered interrupted polling tasks count={} owner={}",
|
||||
recovered, currentInstanceId());
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.image-video.failed-task-cleanup-delay-ms:60000}")
|
||||
public void cleanupFailedTasks() {
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(FAILED_TASK_RETENTION_MINUTES);
|
||||
@@ -146,6 +170,7 @@ public class ImageVideoAsyncTaskService {
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity();
|
||||
task.setUserId(userId);
|
||||
task.setOwnerInstanceId(currentInstanceId());
|
||||
task.setTaskType(type.name());
|
||||
task.setStatus(TaskStatus.PENDING.name());
|
||||
task.setRequestJson(writeJson(payload));
|
||||
@@ -154,12 +179,14 @@ public class ImageVideoAsyncTaskService {
|
||||
task.setCreatedAt(now);
|
||||
task.setUpdatedAt(now);
|
||||
taskMapper.insert(task);
|
||||
log.info("[image-video] async task submitted taskId={} type={} owner={}",
|
||||
task.getId(), task.getTaskType(), task.getOwnerInstanceId());
|
||||
cozeTaskExecutor.execute(() -> executeTask(task.getId()));
|
||||
return toVo(task);
|
||||
}
|
||||
|
||||
private void executeTask(Long taskId) {
|
||||
if (taskMapper.claimPending(taskId) != 1) {
|
||||
if (taskMapper.claimPending(taskId, currentInstanceId()) != 1) {
|
||||
return;
|
||||
}
|
||||
ImageVideoAsyncTaskEntity task = taskMapper.selectById(taskId);
|
||||
@@ -189,17 +216,13 @@ public class ImageVideoAsyncTaskService {
|
||||
}
|
||||
|
||||
private void pollTask(Long taskId) {
|
||||
if (taskMapper.claimWaiting(taskId) != 1) {
|
||||
if (taskMapper.claimWaiting(taskId, currentInstanceId()) != 1) {
|
||||
return;
|
||||
}
|
||||
ImageVideoAsyncTaskEntity task = taskMapper.selectById(taskId);
|
||||
if (task == null) {
|
||||
return;
|
||||
}
|
||||
if (task.getSubmittedAt() != null && task.getSubmittedAt().plusHours(1).isBefore(LocalDateTime.now())) {
|
||||
failTask(task, new BusinessException("Coze task exceeded the one-hour polling limit"));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
TaskType type = TaskType.valueOf(task.getTaskType());
|
||||
String workflowId = workflowIdFor(type);
|
||||
@@ -216,12 +239,20 @@ public class ImageVideoAsyncTaskService {
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (pollingDeadlineExceeded(task)) {
|
||||
failTask(task, new BusinessException("Coze task exceeded the one-hour polling limit"));
|
||||
return;
|
||||
}
|
||||
task.setStatus(TaskStatus.WAITING.name());
|
||||
task.setCozeStatus(cozeStatus);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
taskMapper.updateById(task);
|
||||
} catch (Exception ex) {
|
||||
// Coze history calls are retried by the next polling cycle until the task's overall deadline.
|
||||
if (pollingDeadlineExceeded(task)) {
|
||||
failTask(task, new BusinessException("Coze task exceeded the one-hour polling limit", ex));
|
||||
return;
|
||||
}
|
||||
task.setStatus(TaskStatus.WAITING.name());
|
||||
task.setErrorMessage(truncate(messageOf(ex)));
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
@@ -493,6 +524,26 @@ public class ImageVideoAsyncTaskService {
|
||||
return value.length() <= 1000 ? value : value.substring(0, 1000);
|
||||
}
|
||||
|
||||
private boolean pollingDeadlineExceeded(ImageVideoAsyncTaskEntity task) {
|
||||
return task.getSubmittedAt() != null
|
||||
&& task.getSubmittedAt().plusHours(1).isBefore(LocalDateTime.now());
|
||||
}
|
||||
|
||||
private void ensureTaskOwnedByCurrentInstance(ImageVideoAsyncTaskEntity task, String operation) {
|
||||
String owner = task == null ? null : task.getOwnerInstanceId();
|
||||
if (owner == null || owner.isBlank() || Objects.equals(owner, currentInstanceId())) {
|
||||
return;
|
||||
}
|
||||
log.info("[image-video] route task operation to owner taskId={} operation={} owner={} current={}",
|
||||
task.getId(), operation, owner, currentInstanceId());
|
||||
throw new TaskOwnerMismatchException(task.getId(), operation, owner, currentInstanceId());
|
||||
}
|
||||
|
||||
private String currentInstanceId() {
|
||||
String instanceId = instanceMetadata == null ? null : instanceMetadata.getInstanceId();
|
||||
return instanceId == null || instanceId.isBlank() ? "unknown-instance" : instanceId;
|
||||
}
|
||||
|
||||
private enum TaskStatus {
|
||||
PENDING, RUNNING, WAITING, POLLING, SUCCESS, FAILED
|
||||
}
|
||||
|
||||
+1
-1
@@ -36,7 +36,7 @@ public class PublishController {
|
||||
@PostMapping("/parse")
|
||||
@Operation(
|
||||
summary = "匹配店铺、解析 Excel 并创建多文件批次任务",
|
||||
description = "按文件名(去扩展名)先匹配已管理店铺,再查询紫鸟索引;解析每个非空 Sheet。表头必须依次为 id、ASIN、国家、品牌、价格、状态、同步状态、同步国家。匹配并解析成功的文件为 PENDING,失败文件为 FAILED;只要存在可处理文件,任务为 PENDING,否则任务为 FAILED。")
|
||||
description = "按文件名(去扩展名)先匹配已管理店铺,再查询紫鸟索引;解析每个非空 Sheet。源文件前五列表头必须依次为 id、ASIN、国家、品牌、价格,第六列及以后不解析;状态、同步状态、同步国家由 Python 回传并写入最终结果文件。匹配并解析成功的文件为 PENDING,失败文件为 FAILED;只要存在可处理文件,任务为 PENDING,否则任务为 FAILED。")
|
||||
public ApiResponse<PublishParseVo> parse(@Valid @RequestBody PublishParseRequest request) {
|
||||
return ApiResponse.success(publishTaskService.parseAndCreateTask(request));
|
||||
}
|
||||
|
||||
+1
-1
@@ -40,6 +40,6 @@ public class PublishItemsPageVo {
|
||||
private Long total;
|
||||
@Schema(description = "总页数", example = "14")
|
||||
private Integer totalPages;
|
||||
@Schema(description = "按原 Excel 行序返回的八列数据")
|
||||
@Schema(description = "按原 Excel 行序返回前五列源数据;状态、同步状态、同步国家等待 Python 处理后回传")
|
||||
private List<PublishRowDto> items = new ArrayList<>();
|
||||
}
|
||||
|
||||
+53
-3
@@ -7,6 +7,8 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.publish.mapper.PublishFileMapper;
|
||||
@@ -87,6 +89,7 @@ public class PublishTaskService {
|
||||
private final OssStorageService ossStorageService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TransactionTemplate transactionTemplate;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
|
||||
@Value("${aiimage.publish.stale-timeout-minutes:30}")
|
||||
private int staleTimeoutMinutes;
|
||||
@@ -349,6 +352,12 @@ public class PublishTaskService {
|
||||
List<FileTaskEntity> staleTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.and(owner -> owner
|
||||
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) IS NULL")
|
||||
.or()
|
||||
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = ''")
|
||||
.or()
|
||||
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = {0}", currentInstanceId()))
|
||||
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||
.last("limit 100"));
|
||||
@@ -377,6 +386,7 @@ public class PublishTaskService {
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
throw new BusinessException("task not found");
|
||||
}
|
||||
ensureTaskOwnedByCurrentInstance(task, "process publish result file");
|
||||
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
|
||||
if (result == null || !MODULE_TYPE.equals(result.getModuleType())
|
||||
|| !task.getId().equals(result.getTaskId())) {
|
||||
@@ -522,6 +532,7 @@ public class PublishTaskService {
|
||||
|| task.getUpdatedAt() == null || !task.getUpdatedAt().isBefore(threshold)) {
|
||||
return;
|
||||
}
|
||||
ensureTaskOwnedByCurrentInstance(task, "cleanup stale publish task");
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
|
||||
return;
|
||||
}
|
||||
@@ -551,7 +562,7 @@ public class PublishTaskService {
|
||||
result.setSuccess(0);
|
||||
result.setErrorMessage(null);
|
||||
fileResultMapper.updateById(result);
|
||||
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, result.getId(), "task:" + taskId);
|
||||
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, result.getId(), ownerScopeKey(taskId));
|
||||
}
|
||||
|
||||
private PersistedTask persistTask(PublishParseRequest request, List<PreparedFile> preparedFiles) {
|
||||
@@ -568,7 +579,8 @@ public class PublishTaskService {
|
||||
task.setSuccessFileCount(0);
|
||||
task.setFailedFileCount(failedFiles);
|
||||
task.setRequestJson(writeJson(request, "序列化上架任务失败"));
|
||||
task.setResultJson("{}");
|
||||
task.setResultJson(writeJson(Map.of("ownerInstanceId", currentInstanceId()),
|
||||
"Failed to save publish task instance owner"));
|
||||
task.setErrorMessage(processableFiles > 0 ? null : "全部文件解析或店铺匹配失败");
|
||||
task.setCreatedBy("user:" + request.getUserId());
|
||||
task.setUserId(request.getUserId());
|
||||
@@ -665,7 +677,7 @@ public class PublishTaskService {
|
||||
task.setErrorMessage(null);
|
||||
task.setFinishedAt(null);
|
||||
fileTaskMapper.updateById(task);
|
||||
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, result.getId(), "task:" + taskId);
|
||||
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, result.getId(), ownerScopeKey(taskId));
|
||||
}
|
||||
|
||||
private List<PublishTaskDetailVo> loadTaskDetails(List<FileTaskEntity> tasks) {
|
||||
@@ -801,9 +813,47 @@ public class PublishTaskService {
|
||||
|| (userId != null && !userId.equals(task.getUserId()))) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
ensureTaskOwnedByCurrentInstance(task, "access publish task");
|
||||
return task;
|
||||
}
|
||||
|
||||
public void ensureTaskOwnedByCurrentInstance(FileTaskEntity task, String operation) {
|
||||
String owner = ownerFromTask(task);
|
||||
if (owner == null || owner.isBlank() || Objects.equals(owner, currentInstanceId())) {
|
||||
return;
|
||||
}
|
||||
log.warn("[publish] reject task operation because owner is another instance taskId={} operation={} owner={} current={}",
|
||||
task == null ? null : task.getId(), operation, owner, currentInstanceId());
|
||||
throw new TaskOwnerMismatchException(
|
||||
task == null ? null : task.getId(), operation, owner, currentInstanceId());
|
||||
}
|
||||
|
||||
private String ownerFromTask(FileTaskEntity task) {
|
||||
if (task == null || task.getResultJson() == null || task.getResultJson().isBlank()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(task.getResultJson());
|
||||
if (root == null) {
|
||||
return null;
|
||||
}
|
||||
String owner = root.path("ownerInstanceId").asText("");
|
||||
return owner.isBlank() ? null : owner;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[publish] read task owner failed taskId={} msg={}", task.getId(), safeMessage(ex));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String currentInstanceId() {
|
||||
String instanceId = instanceMetadata == null ? null : instanceMetadata.getInstanceId();
|
||||
return firstNonBlank(instanceId, "unknown-instance");
|
||||
}
|
||||
|
||||
private String ownerScopeKey(Long taskId) {
|
||||
return "task:" + taskId + ":owner:" + currentInstanceId();
|
||||
}
|
||||
|
||||
private PublishFileEntity requireFile(Long taskId, Long fileId) {
|
||||
if (fileId == null || fileId <= 0) {
|
||||
throw new BusinessException("file_id 不合法");
|
||||
|
||||
+9
-15
@@ -30,7 +30,9 @@ import java.util.zip.ZipOutputStream;
|
||||
@Service
|
||||
public class PublishWorkbookService {
|
||||
|
||||
public static final List<String> HEADERS = List.of(
|
||||
public static final List<String> SOURCE_HEADERS = List.of(
|
||||
"id", "ASIN", "国家", "品牌", "价格");
|
||||
public static final List<String> RESULT_HEADERS = List.of(
|
||||
"id", "ASIN", "国家", "品牌", "价格", "状态", "同步状态", "同步国家");
|
||||
public static final String XLSX_CONTENT_TYPE =
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
@@ -66,9 +68,9 @@ public class PublishWorkbookService {
|
||||
if (!validatedSheets.contains(sheetNo)) {
|
||||
throw new BusinessException("工作表 " + sheetName + " 缺少严格的上架表头");
|
||||
}
|
||||
List<String> values = new ArrayList<>(HEADERS.size());
|
||||
List<String> values = new ArrayList<>(SOURCE_HEADERS.size());
|
||||
boolean nonEmpty = false;
|
||||
for (int columnIndex = 0; columnIndex < HEADERS.size(); columnIndex++) {
|
||||
for (int columnIndex = 0; columnIndex < SOURCE_HEADERS.size(); columnIndex++) {
|
||||
String value = normalize(rowMap.get(columnIndex));
|
||||
values.add(value);
|
||||
nonEmpty = nonEmpty || !value.isBlank();
|
||||
@@ -163,19 +165,14 @@ public class PublishWorkbookService {
|
||||
if (headerMap == null || headerMap.isEmpty()) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
for (int index = 0; index < HEADERS.size(); index++) {
|
||||
for (int index = 0; index < SOURCE_HEADERS.size(); index++) {
|
||||
String actual = normalize(headerMap.get(index));
|
||||
String expected = HEADERS.get(index);
|
||||
String expected = SOURCE_HEADERS.get(index);
|
||||
if (!expected.equals(actual)) {
|
||||
throw new BusinessException("Excel 表头不匹配,第 " + (index + 1)
|
||||
+ " 列应为 " + expected + ",实际为 " + actual);
|
||||
}
|
||||
}
|
||||
for (Map.Entry<Integer, String> entry : headerMap.entrySet()) {
|
||||
if (entry.getKey() != null && entry.getKey() >= HEADERS.size() && !normalize(entry.getValue()).isBlank()) {
|
||||
throw new BusinessException("Excel 表头必须严格为: " + String.join("/", HEADERS));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private PublishRowDto toRow(List<String> values) {
|
||||
@@ -185,9 +182,6 @@ public class PublishWorkbookService {
|
||||
row.setCountry(values.get(2));
|
||||
row.setBrand(values.get(3));
|
||||
row.setPrice(values.get(4));
|
||||
row.setStatus(values.get(5));
|
||||
row.setSyncStatus(values.get(6));
|
||||
row.setSyncCountries(values.get(7));
|
||||
return row;
|
||||
}
|
||||
|
||||
@@ -210,9 +204,9 @@ public class PublishWorkbookService {
|
||||
|
||||
private void writeSheet(Sheet sheet, List<PublishRowDto> rows, CellStyle headerStyle) {
|
||||
Row header = sheet.createRow(0);
|
||||
for (int index = 0; index < HEADERS.size(); index++) {
|
||||
for (int index = 0; index < RESULT_HEADERS.size(); index++) {
|
||||
Cell cell = header.createCell(index);
|
||||
cell.setCellValue(HEADERS.get(index));
|
||||
cell.setCellValue(RESULT_HEADERS.get(index));
|
||||
cell.setCellStyle(headerStyle);
|
||||
}
|
||||
int rowIndex = 1;
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ public class SimilarAsinHistoryItemVo {
|
||||
private String sourceFilename;
|
||||
@Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx")
|
||||
private String resultFilename;
|
||||
@Schema(description = "最终结果文件下载地址。后端基于 OSS objectKey 生成的公开直链 URL。", example = "https://bucket.oss-cn-hangzhou.aliyuncs.com/result/similar_asin/xxx/17-result.xlsx")
|
||||
@Schema(description = "最终结果文件下载地址。后端基于 MinIO objectKey 生成的公开直链 URL。", example = "http://47.110.241.161:9000/nanri-ai-images/result/similar_asin/xxx/17-result.xlsx")
|
||||
private String downloadUrl;
|
||||
private Long fileJobId;
|
||||
private String fileStatus;
|
||||
|
||||
+9
-3
@@ -206,6 +206,7 @@ public class SimilarAsinTaskService {
|
||||
"id",
|
||||
"asin",
|
||||
"国家",
|
||||
"价格",
|
||||
"卖家名称",
|
||||
"品牌",
|
||||
"是否有货",
|
||||
@@ -219,9 +220,9 @@ public class SimilarAsinTaskService {
|
||||
"阿里巴巴图片2"
|
||||
);
|
||||
|
||||
private static final int IMG_COL_MAIN = 11;
|
||||
private static final int IMG_COL_PUZZLE1 = 12;
|
||||
private static final int IMG_COL_PUZZLE2 = 13;
|
||||
private static final int IMG_COL_MAIN = 12;
|
||||
private static final int IMG_COL_PUZZLE1 = 13;
|
||||
private static final int IMG_COL_PUZZLE2 = 14;
|
||||
|
||||
private final LocalFileStorageService localFileStorageService;
|
||||
private final OssStorageService ossStorageService;
|
||||
@@ -4533,6 +4534,11 @@ public class SimilarAsinTaskService {
|
||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
|
||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
|
||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
|
||||
row.createCell(col++).setCellValue(resultRow == null
|
||||
? firstNonBlank(parsedRow.getPrice(), readValueByHeader(parsedRow, "价格", "price"))
|
||||
: firstNonBlank(
|
||||
resultRow.getPrice(),
|
||||
firstNonBlank(parsedRow.getPrice(), readValueByHeader(parsedRow, "价格", "price"))));
|
||||
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name", "seller-name", "sellername", "store name", "shop name"));
|
||||
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "品牌", "brand"));
|
||||
String isStock = resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getIsStock());
|
||||
|
||||
+2
-2
@@ -95,7 +95,7 @@ public class TaskFileJobService {
|
||||
List<TaskFileJobEntity> ownerJobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
|
||||
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.in(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN"))
|
||||
.in(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH"))
|
||||
.like(TaskFileJobEntity::getScopeKey, ownerMarker)
|
||||
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
|
||||
.last("limit " + safeLimit));
|
||||
@@ -108,7 +108,7 @@ public class TaskFileJobService {
|
||||
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
|
||||
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.and(wrapper -> wrapper
|
||||
.notIn(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN"))
|
||||
.notIn(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH"))
|
||||
.or()
|
||||
.isNull(TaskFileJobEntity::getScopeKey)
|
||||
.or()
|
||||
|
||||
+3
@@ -103,6 +103,9 @@ public class TaskHeartbeatService {
|
||||
return null;
|
||||
}
|
||||
String moduleType = task.getModuleType() == null ? "" : task.getModuleType();
|
||||
if (PublishTaskService.MODULE_TYPE.equals(moduleType)) {
|
||||
publishTaskService.ensureTaskOwnedByCurrentInstance(task, "publish task heartbeat");
|
||||
}
|
||||
String status = task.getStatus();
|
||||
if (!STATUS_RUNNING.equals(status)) {
|
||||
log.warn("[task-heartbeat] file task is not running taskId={} actualUserId={} moduleType={} status={}",
|
||||
|
||||
+6
-4
@@ -95,7 +95,7 @@ public class TaskResultFileJobWorker {
|
||||
}
|
||||
|
||||
public void process(TaskFileJobEntity job) {
|
||||
if (isOwnerScopedCozeJob(job) && !isOwnedByCurrentInstance(job)) {
|
||||
if (isOwnerScopedJob(job) && !isOwnedByCurrentInstance(job)) {
|
||||
log.debug("[task-file-job] skip owner-scoped job because owner is another instance jobId={} taskId={} moduleType={} owner={} current={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), ownerFromScopeKey(job.getScopeKey()), currentInstanceId());
|
||||
return;
|
||||
@@ -149,7 +149,7 @@ public class TaskResultFileJobWorker {
|
||||
}
|
||||
boolean completed = dispatch(job);
|
||||
if (!completed) {
|
||||
if (isOwnerScopedCozeJob(job)) {
|
||||
if (isOwnerScopedJob(job)) {
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
log.info("[task-file-job] process waiting for async coze result jobId={} taskId={} moduleType={} resultId={} elapsedMs={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
|
||||
@@ -219,11 +219,13 @@ public class TaskResultFileJobWorker {
|
||||
return result == null ? null : result.getResultFileUrl();
|
||||
}
|
||||
|
||||
private boolean isOwnerScopedCozeJob(TaskFileJobEntity job) {
|
||||
private boolean isOwnerScopedJob(TaskFileJobEntity job) {
|
||||
if (job == null || job.getModuleType() == null) {
|
||||
return false;
|
||||
}
|
||||
return "APPEARANCE_PATENT".equals(job.getModuleType()) || "SIMILAR_ASIN".equals(job.getModuleType());
|
||||
return "APPEARANCE_PATENT".equals(job.getModuleType())
|
||||
|| "SIMILAR_ASIN".equals(job.getModuleType())
|
||||
|| "PUBLISH".equals(job.getModuleType());
|
||||
}
|
||||
|
||||
private boolean isOwnedByCurrentInstance(TaskFileJobEntity job) {
|
||||
|
||||
@@ -12,9 +12,12 @@ AIIMAGE_DB_URL=jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEnco
|
||||
AIIMAGE_DB_USERNAME=change-me
|
||||
AIIMAGE_DB_PASSWORD=change-me
|
||||
|
||||
AIIMAGE_OSS_REGION=cn-hangzhou
|
||||
AIIMAGE_OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
|
||||
AIIMAGE_OSS_BUCKET=change-me
|
||||
AIIMAGE_OSS_REGION=us-east-1
|
||||
AIIMAGE_OSS_ENDPOINT=http://47.110.241.161:9000
|
||||
AIIMAGE_OSS_PUBLIC_ENDPOINT=http://47.110.241.161:9000
|
||||
AIIMAGE_OSS_BUCKET=nanri-ai-images
|
||||
AIIMAGE_IMAGE_VIDEO_OSS_BUCKET=shufu-video
|
||||
AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET=nanri-ai-digital-human
|
||||
AIIMAGE_OSS_ACCESS_KEY_ID=change-me
|
||||
AIIMAGE_OSS_ACCESS_KEY_SECRET=change-me
|
||||
|
||||
|
||||
@@ -84,13 +84,14 @@ aiimage:
|
||||
read-timeout-millis: ${AIIMAGE_INSTANCE_ROUTE_READ_TIMEOUT_MILLIS:300000}
|
||||
request-body-cache-limit-bytes: ${AIIMAGE_INSTANCE_ROUTE_REQUEST_BODY_CACHE_LIMIT_BYTES:104857600}
|
||||
oss:
|
||||
region: ${AIIMAGE_OSS_REGION:cn-hangzhou}
|
||||
endpoint: ${AIIMAGE_OSS_ENDPOINT:oss-cn-hangzhou.aliyuncs.com}
|
||||
public-endpoint: ${AIIMAGE_OSS_PUBLIC_ENDPOINT:oss-cn-hangzhou.aliyuncs.com}
|
||||
region: ${AIIMAGE_OSS_REGION:us-east-1}
|
||||
endpoint: ${AIIMAGE_OSS_ENDPOINT:http://47.110.241.161:9000}
|
||||
public-endpoint: ${AIIMAGE_OSS_PUBLIC_ENDPOINT:http://47.110.241.161:9000}
|
||||
bucket: ${AIIMAGE_OSS_BUCKET:nanri-ai-images}
|
||||
image-video-bucket: ${AIIMAGE_IMAGE_VIDEO_OSS_BUCKET:shufu-video}
|
||||
access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:LTAI5tNpyvzMNz9f2dHarsm8}
|
||||
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:bQSZnFH455i8tzyOgeahJmUzwmhynz}
|
||||
digital-human-bucket: ${AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET:nanri-ai-digital-human}
|
||||
access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:appuser}
|
||||
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:AppUser@2024SecureKey}
|
||||
transient-storage:
|
||||
enabled: ${AIIMAGE_TRANSIENT_STORAGE_ENABLED:true}
|
||||
endpoint: ${AIIMAGE_TRANSIENT_STORAGE_ENDPOINT:http://121.196.149.225:9000}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE `biz_image_video_async_task`
|
||||
ADD COLUMN `owner_instance_id` VARCHAR(128) NULL AFTER `user_id`,
|
||||
ADD KEY `idx_image_video_owner_status` (`owner_instance_id`, `status`, `updated_at`);
|
||||
+121
-9
@@ -2,12 +2,16 @@ package com.nanri.aiimage.modules.imagevideo.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.ImageVideoProperties;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.imagevideo.mapper.ImageVideoAsyncTaskMapper;
|
||||
import com.nanri.aiimage.modules.imagevideo.model.dto.DouyinCopyRequest;
|
||||
import com.nanri.aiimage.modules.imagevideo.model.entity.ImageVideoAsyncTaskEntity;
|
||||
import com.nanri.aiimage.modules.imagevideo.model.vo.DouyinCopyVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.core.task.TaskExecutor;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
@@ -15,6 +19,7 @@ import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
@@ -55,7 +60,8 @@ class ImageVideoAsyncTaskServiceTest {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
TaskExecutor directExecutor = Runnable::run;
|
||||
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
|
||||
taskMapper, cozeService, workflowConfigService, mock(ImageVideoArchiveService.class), objectMapper, directExecutor);
|
||||
taskMapper, cozeService, workflowConfigService, mock(ImageVideoArchiveService.class), objectMapper,
|
||||
directExecutor, instanceMetadata("server-121"));
|
||||
|
||||
ImageVideoAsyncTaskEntity task = waitingDouyinTask();
|
||||
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
|
||||
@@ -67,7 +73,7 @@ class ImageVideoAsyncTaskServiceTest {
|
||||
copyResult.setScriptDraft("这条裙裤绝了,遮肉显腿长,快拍!");
|
||||
|
||||
when(taskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(taskMapper.claimWaiting(79L)).thenReturn(1);
|
||||
when(taskMapper.claimWaiting(79L, "server-121")).thenReturn(1);
|
||||
when(taskMapper.selectById(79L)).thenReturn(task);
|
||||
when(workflowConfigService.douyinCopyWorkflowId()).thenReturn("7652941112982798388");
|
||||
when(cozeService.getWorkflowResult(1L, "7652941112982798388", "7662364426301964329"))
|
||||
@@ -90,13 +96,14 @@ class ImageVideoAsyncTaskServiceTest {
|
||||
ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class);
|
||||
ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class);
|
||||
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
|
||||
taskMapper, cozeService, workflowConfigService, mock(ImageVideoArchiveService.class), new ObjectMapper(), Runnable::run);
|
||||
taskMapper, cozeService, workflowConfigService, mock(ImageVideoArchiveService.class), new ObjectMapper(),
|
||||
Runnable::run, instanceMetadata("server-121"));
|
||||
|
||||
ImageVideoAsyncTaskEntity task = waitingDouyinTask();
|
||||
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of("execute_status", "Running")));
|
||||
|
||||
when(taskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(taskMapper.claimWaiting(79L)).thenReturn(1);
|
||||
when(taskMapper.claimWaiting(79L, "server-121")).thenReturn(1);
|
||||
when(taskMapper.selectById(79L)).thenReturn(task);
|
||||
when(workflowConfigService.douyinCopyWorkflowId()).thenReturn("7652941112982798388");
|
||||
when(cozeService.getWorkflowResult(1L, "7652941112982798388", "7662364426301964329"))
|
||||
@@ -117,7 +124,8 @@ class ImageVideoAsyncTaskServiceTest {
|
||||
ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class);
|
||||
ImageVideoArchiveService archiveService = mock(ImageVideoArchiveService.class);
|
||||
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
|
||||
taskMapper, cozeService, workflowConfigService, archiveService, new ObjectMapper(), Runnable::run);
|
||||
taskMapper, cozeService, workflowConfigService, archiveService, new ObjectMapper(), Runnable::run,
|
||||
instanceMetadata("server-121"));
|
||||
ImageVideoAsyncTaskEntity task = waitingWorkflowTask();
|
||||
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
|
||||
"execute_status", "Fail",
|
||||
@@ -125,7 +133,7 @@ class ImageVideoAsyncTaskServiceTest {
|
||||
"error_message", "model unavailable")));
|
||||
|
||||
when(taskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(taskMapper.claimWaiting(98L)).thenReturn(1);
|
||||
when(taskMapper.claimWaiting(98L, "server-121")).thenReturn(1);
|
||||
when(taskMapper.selectById(98L)).thenReturn(task);
|
||||
when(workflowConfigService.imageVideoWorkflowId()).thenReturn("workflow-1");
|
||||
when(cozeService.getWorkflowResult(1L, "workflow-1", "exec-98")).thenReturn(cozeResult);
|
||||
@@ -148,7 +156,8 @@ class ImageVideoAsyncTaskServiceTest {
|
||||
mock(ImageVideoWorkflowConfigService.class),
|
||||
mock(ImageVideoArchiveService.class),
|
||||
new ObjectMapper(),
|
||||
Runnable::run);
|
||||
Runnable::run,
|
||||
instanceMetadata("server-121"));
|
||||
ImageVideoAsyncTaskEntity task = waitingWorkflowTask();
|
||||
task.setStatus("FAILED");
|
||||
task.setCozeStatus("FAIL");
|
||||
@@ -171,7 +180,8 @@ class ImageVideoAsyncTaskServiceTest {
|
||||
mock(ImageVideoWorkflowConfigService.class),
|
||||
mock(ImageVideoArchiveService.class),
|
||||
new ObjectMapper(),
|
||||
Runnable::run);
|
||||
Runnable::run,
|
||||
instanceMetadata("server-121"));
|
||||
ImageVideoAsyncTaskEntity task = waitingWorkflowTask();
|
||||
task.setStatus("FAILED");
|
||||
task.setCozeStatus("RUNNING");
|
||||
@@ -194,7 +204,8 @@ class ImageVideoAsyncTaskServiceTest {
|
||||
mock(ImageVideoWorkflowConfigService.class),
|
||||
mock(ImageVideoArchiveService.class),
|
||||
new ObjectMapper(),
|
||||
Runnable::run);
|
||||
Runnable::run,
|
||||
instanceMetadata("server-121"));
|
||||
when(taskMapper.delete(any())).thenReturn(3);
|
||||
|
||||
service.cleanupFailedTasks();
|
||||
@@ -202,10 +213,107 @@ class ImageVideoAsyncTaskServiceTest {
|
||||
verify(taskMapper).delete(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void submittedTaskIsBoundToCurrentInstance() {
|
||||
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
|
||||
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
|
||||
taskMapper,
|
||||
mock(ImageVideoCozeService.class),
|
||||
mock(ImageVideoWorkflowConfigService.class),
|
||||
mock(ImageVideoArchiveService.class),
|
||||
new ObjectMapper(),
|
||||
Runnable::run,
|
||||
instanceMetadata("server-121"));
|
||||
DouyinCopyRequest request = new DouyinCopyRequest();
|
||||
request.setUserId(1L);
|
||||
request.setUrl("https://example.com/video");
|
||||
|
||||
service.submitDouyinCopy(request);
|
||||
|
||||
ArgumentCaptor<ImageVideoAsyncTaskEntity> taskCaptor = ArgumentCaptor.forClass(ImageVideoAsyncTaskEntity.class);
|
||||
verify(taskMapper).insert(taskCaptor.capture());
|
||||
assertEquals("server-121", taskCaptor.getValue().getOwnerInstanceId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void taskQueryOnAnotherInstanceTriggersOwnerForwarding() {
|
||||
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
|
||||
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
|
||||
taskMapper,
|
||||
mock(ImageVideoCozeService.class),
|
||||
mock(ImageVideoWorkflowConfigService.class),
|
||||
mock(ImageVideoArchiveService.class),
|
||||
new ObjectMapper(),
|
||||
Runnable::run,
|
||||
instanceMetadata("server-121"));
|
||||
ImageVideoAsyncTaskEntity task = waitingWorkflowTask();
|
||||
task.setOwnerInstanceId("server-110");
|
||||
when(taskMapper.selectOne(any())).thenReturn(task);
|
||||
|
||||
TaskOwnerMismatchException exception = assertThrows(
|
||||
TaskOwnerMismatchException.class,
|
||||
() -> service.getTask(98L, 1L));
|
||||
|
||||
assertEquals(98L, exception.getTaskId());
|
||||
assertEquals("server-110", exception.getOwnerInstanceId());
|
||||
assertEquals("server-121", exception.getCurrentInstanceId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void applicationStartRequeuesInterruptedPollingTasksForOwner() {
|
||||
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
|
||||
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
|
||||
taskMapper,
|
||||
mock(ImageVideoCozeService.class),
|
||||
mock(ImageVideoWorkflowConfigService.class),
|
||||
mock(ImageVideoArchiveService.class),
|
||||
new ObjectMapper(),
|
||||
Runnable::run,
|
||||
instanceMetadata("server-121"));
|
||||
when(taskMapper.requeuePollingTasks("server-121")).thenReturn(2);
|
||||
|
||||
service.recoverInterruptedPollingTasks();
|
||||
|
||||
verify(taskMapper).requeuePollingTasks("server-121");
|
||||
}
|
||||
|
||||
@Test
|
||||
void overdueTaskStillConsumesTerminalCozeResultAfterRecovery() {
|
||||
ImageVideoAsyncTaskMapper taskMapper = mock(ImageVideoAsyncTaskMapper.class);
|
||||
ImageVideoCozeService cozeService = mock(ImageVideoCozeService.class);
|
||||
ImageVideoWorkflowConfigService workflowConfigService = mock(ImageVideoWorkflowConfigService.class);
|
||||
ImageVideoAsyncTaskService service = new ImageVideoAsyncTaskService(
|
||||
taskMapper,
|
||||
cozeService,
|
||||
workflowConfigService,
|
||||
mock(ImageVideoArchiveService.class),
|
||||
new ObjectMapper(),
|
||||
Runnable::run,
|
||||
instanceMetadata("server-121"));
|
||||
ImageVideoAsyncTaskEntity task = waitingWorkflowTask();
|
||||
task.setSubmittedAt(LocalDateTime.now().minusHours(2));
|
||||
Map<String, Object> cozeResult = Map.of("data", List.of(Map.of(
|
||||
"execute_status", "Success",
|
||||
"output", "{\"Output\":\"https://example.com/result.mp4\"}")));
|
||||
|
||||
when(taskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(taskMapper.claimWaiting(98L, "server-121")).thenReturn(1);
|
||||
when(taskMapper.selectById(98L)).thenReturn(task);
|
||||
when(workflowConfigService.imageVideoWorkflowId()).thenReturn("workflow-1");
|
||||
when(cozeService.getWorkflowResult(1L, "workflow-1", "exec-98")).thenReturn(cozeResult);
|
||||
|
||||
service.pollWaitingTasks();
|
||||
|
||||
assertEquals("SUCCESS", task.getStatus());
|
||||
assertEquals("SUCCESS", task.getCozeStatus());
|
||||
assertNotNull(task.getCompletedAt());
|
||||
}
|
||||
|
||||
private ImageVideoAsyncTaskEntity waitingDouyinTask() {
|
||||
ImageVideoAsyncTaskEntity task = new ImageVideoAsyncTaskEntity();
|
||||
task.setId(79L);
|
||||
task.setUserId(1L);
|
||||
task.setOwnerInstanceId("server-121");
|
||||
task.setTaskType("DOUYIN_COPY");
|
||||
task.setStatus("WAITING");
|
||||
task.setCozeExecuteId("7662364426301964329");
|
||||
@@ -222,4 +330,8 @@ class ImageVideoAsyncTaskServiceTest {
|
||||
task.setCozeExecuteId("exec-98");
|
||||
return task;
|
||||
}
|
||||
|
||||
private InstanceMetadata instanceMetadata(String instanceId) {
|
||||
return new InstanceMetadata(instanceId);
|
||||
}
|
||||
}
|
||||
|
||||
+77
-2
@@ -4,12 +4,16 @@ import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.publish.mapper.PublishFileMapper;
|
||||
import com.nanri.aiimage.modules.publish.mapper.PublishItemMapper;
|
||||
import com.nanri.aiimage.modules.publish.model.dto.PublishParseRequest;
|
||||
import com.nanri.aiimage.modules.publish.model.dto.PublishResultFileDto;
|
||||
import com.nanri.aiimage.modules.publish.model.dto.PublishRowDto;
|
||||
import com.nanri.aiimage.modules.publish.model.dto.PublishSourceFileDto;
|
||||
import com.nanri.aiimage.modules.publish.model.dto.PublishSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.publish.model.entity.PublishFileEntity;
|
||||
import com.nanri.aiimage.modules.publish.model.entity.PublishItemEntity;
|
||||
@@ -20,6 +24,7 @@ import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -29,9 +34,11 @@ import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.transaction.TransactionStatus;
|
||||
import org.springframework.transaction.support.TransactionCallback;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
@@ -42,6 +49,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -74,11 +82,17 @@ class PublishTaskServiceTest {
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private ObjectMapper objectMapper;
|
||||
@Mock private TransactionTemplate transactionTemplate;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
|
||||
@InjectMocks private PublishTaskService service;
|
||||
|
||||
@BeforeEach
|
||||
void executeTransactionsInline() {
|
||||
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
lenient().when(transactionTemplate.execute(any())).thenAnswer(invocation -> {
|
||||
TransactionCallback<?> callback = invocation.getArgument(0);
|
||||
return callback.doInTransaction(null);
|
||||
});
|
||||
lenient().doAnswer(invocation -> {
|
||||
Consumer<TransactionStatus> callback = invocation.getArgument(0);
|
||||
callback.accept(null);
|
||||
@@ -86,6 +100,65 @@ class PublishTaskServiceTest {
|
||||
}).when(transactionTemplate).executeWithoutResult(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseStoresCurrentInstanceAsTaskOwner() throws Exception {
|
||||
PublishParseRequest request = new PublishParseRequest();
|
||||
request.setUserId(7L);
|
||||
request.setPublishCountry("DE");
|
||||
PublishSourceFileDto source = new PublishSourceFileDto();
|
||||
source.setFileKey("uploads/shop.xlsx");
|
||||
source.setOriginalFilename("shop.xlsx");
|
||||
request.setFiles(List.of(source));
|
||||
|
||||
File localFile = mock(File.class);
|
||||
when(localFile.exists()).thenReturn(true);
|
||||
when(localFileStorageService.findLocalSourceFile(source.getFileKey())).thenReturn(localFile);
|
||||
when(ziniaoShopSwitchService.findIndexedStoreByName("shop", false)).thenReturn(
|
||||
new ZiniaoShopMatchResultVo(true, "shop-1", "shop", "company", "Amazon",
|
||||
7L, null, "MATCHED", null));
|
||||
when(workbookService.parse(localFile)).thenReturn(
|
||||
new PublishWorkbookService.ParsedWorkbook(List.of(row("1"))));
|
||||
ObjectMapper realObjectMapper = new ObjectMapper();
|
||||
when(objectMapper.writeValueAsString(any())).thenAnswer(invocation ->
|
||||
realObjectMapper.writeValueAsString(invocation.getArgument(0)));
|
||||
doAnswer(invocation -> {
|
||||
FileTaskEntity inserted = invocation.getArgument(0);
|
||||
inserted.setId(501L);
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
doAnswer(invocation -> {
|
||||
PublishFileEntity inserted = invocation.getArgument(0);
|
||||
inserted.setId(601L);
|
||||
return 1;
|
||||
}).when(publishFileMapper).insert(any(PublishFileEntity.class));
|
||||
|
||||
service.parseAndCreateTask(request);
|
||||
|
||||
org.mockito.ArgumentCaptor<FileTaskEntity> taskCaptor =
|
||||
org.mockito.ArgumentCaptor.forClass(FileTaskEntity.class);
|
||||
verify(fileTaskMapper).insert(taskCaptor.capture());
|
||||
assertEquals("instance-a", realObjectMapper.readTree(
|
||||
taskCaptor.getValue().getResultJson()).path("ownerInstanceId").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void taskAccessRejectsAnotherInstanceForExistingRouterToForward() throws Exception {
|
||||
long taskId = 109L;
|
||||
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||
task.setResultJson("{\"ownerInstanceId\":\"instance-b\"}");
|
||||
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
||||
when(objectMapper.readTree(task.getResultJson())).thenReturn(
|
||||
new ObjectMapper().readTree(task.getResultJson()));
|
||||
|
||||
TaskOwnerMismatchException error = assertThrows(TaskOwnerMismatchException.class,
|
||||
() -> service.getTaskDetail(taskId, 7L));
|
||||
|
||||
assertEquals(taskId, error.getTaskId());
|
||||
assertEquals("instance-b", error.getOwnerInstanceId());
|
||||
assertEquals("instance-a", error.getCurrentInstanceId());
|
||||
verifyNoInteractions(publishItemMapper);
|
||||
}
|
||||
|
||||
@Test
|
||||
void resultCallbackRejectsAnotherUserBeforeReadingOrReplacingRows() {
|
||||
long taskId = 101L;
|
||||
@@ -180,7 +253,8 @@ class PublishTaskServiceTest {
|
||||
|
||||
assertEquals("SUCCESS", file.getStatus());
|
||||
verify(taskFileJobService).enqueueAssembleResult(
|
||||
taskId, PublishTaskService.MODULE_TYPE, resultId, "task:" + taskId);
|
||||
taskId, PublishTaskService.MODULE_TYPE, resultId,
|
||||
"task:" + taskId + ":owner:instance-a");
|
||||
verify(lock).close();
|
||||
}
|
||||
|
||||
@@ -265,7 +339,8 @@ class PublishTaskServiceTest {
|
||||
verify(taskFileJobService, times(2))
|
||||
.countUnfinishedAssembleJobs(taskId, PublishTaskService.MODULE_TYPE);
|
||||
verify(taskFileJobService).enqueueAssembleResult(
|
||||
taskId, PublishTaskService.MODULE_TYPE, resultId, "task:" + taskId);
|
||||
taskId, PublishTaskService.MODULE_TYPE, resultId,
|
||||
"task:" + taskId + ":owner:instance-a");
|
||||
verify(fileTaskMapper).updateById(task);
|
||||
verify(fileResultMapper).updateById(result);
|
||||
verify(lock).close();
|
||||
|
||||
+20
-10
@@ -22,6 +22,7 @@ import java.util.zip.ZipFile;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@@ -30,13 +31,22 @@ class PublishWorkbookServiceTest {
|
||||
private final PublishWorkbookService service = new PublishWorkbookService();
|
||||
|
||||
@Test
|
||||
void parsesOnlyTheExactEightHeadersAcrossNonEmptySheets() throws Exception {
|
||||
void parsesOnlyFirstFiveColumnsAcrossNonEmptySheets() throws Exception {
|
||||
Path directory = Files.createTempDirectory("publish-parse-");
|
||||
try {
|
||||
File valid = directory.resolve("valid.xlsx").toFile();
|
||||
try (Workbook workbook = new XSSFWorkbook();
|
||||
FileOutputStream output = new FileOutputStream(valid)) {
|
||||
writeSourceSheet(workbook.createSheet("英国数据"), "英国", "B001");
|
||||
Sheet uk = workbook.createSheet("英国数据");
|
||||
writeSourceSheet(uk, "英国", "B001");
|
||||
for (int index = PublishWorkbookService.SOURCE_HEADERS.size();
|
||||
index < PublishWorkbookService.RESULT_HEADERS.size(); index++) {
|
||||
uk.getRow(0).createCell(index)
|
||||
.setCellValue(PublishWorkbookService.RESULT_HEADERS.get(index));
|
||||
}
|
||||
uk.getRow(1).createCell(5).setCellValue("旧状态");
|
||||
uk.getRow(1).createCell(6).setCellValue("旧同步状态");
|
||||
uk.getRow(1).createCell(7).setCellValue("旧同步国家");
|
||||
workbook.createSheet("空白页");
|
||||
writeSourceSheet(workbook.createSheet("德国数据"), "DE", "B002");
|
||||
workbook.write(output);
|
||||
@@ -45,6 +55,9 @@ class PublishWorkbookServiceTest {
|
||||
PublishWorkbookService.ParsedWorkbook parsed = service.parse(valid);
|
||||
assertEquals(2, parsed.rows().size());
|
||||
assertEquals("B001", parsed.rows().get(0).getAsin());
|
||||
assertNull(parsed.rows().get(0).getStatus());
|
||||
assertNull(parsed.rows().get(0).getSyncStatus());
|
||||
assertNull(parsed.rows().get(0).getSyncCountries());
|
||||
assertEquals("DE", parsed.rows().get(1).getCountry());
|
||||
|
||||
File invalid = directory.resolve("invalid.xlsx").toFile();
|
||||
@@ -52,7 +65,7 @@ class PublishWorkbookServiceTest {
|
||||
FileOutputStream output = new FileOutputStream(invalid)) {
|
||||
Sheet sheet = workbook.createSheet("错误表头");
|
||||
Row header = sheet.createRow(0);
|
||||
List<String> headers = new ArrayList<>(PublishWorkbookService.HEADERS);
|
||||
List<String> headers = new ArrayList<>(PublishWorkbookService.SOURCE_HEADERS);
|
||||
headers.set(1, "Asin");
|
||||
for (int index = 0; index < headers.size(); index++) {
|
||||
header.createCell(index).setCellValue(headers.get(index));
|
||||
@@ -82,8 +95,8 @@ class PublishWorkbookServiceTest {
|
||||
Set.of(workbook.getSheetName(0), workbook.getSheetName(1)));
|
||||
Sheet uk = workbook.getSheet("英国");
|
||||
assertNotNull(uk);
|
||||
for (int index = 0; index < PublishWorkbookService.HEADERS.size(); index++) {
|
||||
assertEquals(PublishWorkbookService.HEADERS.get(index),
|
||||
for (int index = 0; index < PublishWorkbookService.RESULT_HEADERS.size(); index++) {
|
||||
assertEquals(PublishWorkbookService.RESULT_HEADERS.get(index),
|
||||
uk.getRow(0).getCell(index).getStringCellValue());
|
||||
}
|
||||
assertEquals(CellType.NUMERIC, uk.getRow(1).getCell(4).getCellType());
|
||||
@@ -124,8 +137,8 @@ class PublishWorkbookServiceTest {
|
||||
|
||||
private void writeSourceSheet(Sheet sheet, String country, String asin) {
|
||||
Row header = sheet.createRow(0);
|
||||
for (int index = 0; index < PublishWorkbookService.HEADERS.size(); index++) {
|
||||
header.createCell(index).setCellValue(PublishWorkbookService.HEADERS.get(index));
|
||||
for (int index = 0; index < PublishWorkbookService.SOURCE_HEADERS.size(); index++) {
|
||||
header.createCell(index).setCellValue(PublishWorkbookService.SOURCE_HEADERS.get(index));
|
||||
}
|
||||
Row data = sheet.createRow(1);
|
||||
data.createCell(0).setCellValue("1");
|
||||
@@ -133,9 +146,6 @@ class PublishWorkbookServiceTest {
|
||||
data.createCell(2).setCellValue(country);
|
||||
data.createCell(3).setCellValue("Brand");
|
||||
data.createCell(4).setCellValue(50);
|
||||
data.createCell(5).setCellValue("成功");
|
||||
data.createCell(6).setCellValue("成功");
|
||||
data.createCell(7).setCellValue("德国,法国");
|
||||
}
|
||||
|
||||
private PublishRowDto row(String id, String asin, String country, String price) {
|
||||
|
||||
+25
@@ -3,10 +3,29 @@ package com.nanri.aiimage.modules.similarasin.service;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class SimilarAsinTaskServiceTest {
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void resultWorkbookRestoresPriceAfterCountryAndShiftsImageColumns() throws Exception {
|
||||
Field headersField = SimilarAsinTaskService.class.getDeclaredField("RESULT_HEADERS");
|
||||
headersField.setAccessible(true);
|
||||
List<String> headers = (List<String>) headersField.get(null);
|
||||
|
||||
assertEquals(List.of(
|
||||
"id", "asin", "国家", "价格", "卖家名称", "品牌", "是否有货", "相似度",
|
||||
"是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"
|
||||
), headers);
|
||||
assertEquals(12, staticIntField("IMG_COL_MAIN"));
|
||||
assertEquals(13, staticIntField("IMG_COL_PUZZLE1"));
|
||||
assertEquals(14, staticIntField("IMG_COL_PUZZLE2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resultStatusUsesReturnedCozeDataAndImages() {
|
||||
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(null));
|
||||
@@ -30,4 +49,10 @@ class SimilarAsinTaskServiceTest {
|
||||
SimilarAsinResultRowDto withVisibleResultData = new SimilarAsinResultRowDto();
|
||||
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withVisibleResultData, "", "80%", "", "", ""));
|
||||
}
|
||||
|
||||
private int staticIntField(String name) throws Exception {
|
||||
Field field = SimilarAsinTaskService.class.getDeclaredField(name);
|
||||
field.setAccessible(true);
|
||||
return field.getInt(null);
|
||||
}
|
||||
}
|
||||
|
||||
+6
-1
@@ -25,6 +25,7 @@ import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.InOrder;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
@@ -33,6 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
@@ -83,7 +85,10 @@ class TaskHeartbeatServiceTest {
|
||||
|
||||
assertTrue(result.isAlive());
|
||||
assertEquals(PublishTaskService.MODULE_TYPE, result.getModuleType());
|
||||
verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
InOrder routingBeforeUpdate = inOrder(publishTaskService, fileTaskMapper);
|
||||
routingBeforeUpdate.verify(publishTaskService)
|
||||
.ensureTaskOwnedByCurrentInstance(task, "publish task heartbeat");
|
||||
routingBeforeUpdate.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
verify(publishTaskService).touchHeartbeat(taskId, request);
|
||||
}
|
||||
}
|
||||
|
||||
+17
-1
@@ -93,13 +93,14 @@ class TaskResultFileJobWorkerTest {
|
||||
job.setTaskId(taskId);
|
||||
job.setResultId(resultId);
|
||||
job.setModuleType(PublishTaskService.MODULE_TYPE);
|
||||
job.setScopeKey("publish:20141");
|
||||
job.setScopeKey("task:20141:owner:instance-a");
|
||||
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setResultFileUrl("result/publish/20141.xlsx");
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
|
||||
when(taskFileJobService.markRunning(jobId)).thenReturn(true);
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
when(taskDistributedLockService.acquire(
|
||||
PublishTaskService.MODULE_TYPE,
|
||||
taskId,
|
||||
@@ -112,4 +113,19 @@ class TaskResultFileJobWorkerTest {
|
||||
verify(taskFileJobService).markSuccess(job, "result/publish/20141.xlsx");
|
||||
verifyNoInteractions(taskResultPayloadService);
|
||||
}
|
||||
|
||||
@Test
|
||||
void publishFileJobOwnedByAnotherInstanceIsNotClaimed() {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(13642L);
|
||||
job.setTaskId(20142L);
|
||||
job.setResultId(22930L);
|
||||
job.setModuleType(PublishTaskService.MODULE_TYPE);
|
||||
job.setScopeKey("task:20142:owner:instance-b");
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
|
||||
worker.process(job);
|
||||
|
||||
verifyNoInteractions(taskFileJobService, taskDistributedLockService, publishTaskService);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,7 +40,6 @@
|
||||
name="publish-country"
|
||||
class="country-check-input"
|
||||
:value="row.code"
|
||||
:disabled="hasQueueWork"
|
||||
/>
|
||||
<span class="country-check-text">{{ row.label }} ({{ row.code }})</span>
|
||||
</label>
|
||||
@@ -54,13 +53,13 @@
|
||||
v-for="row in COUNTRY_OPTIONS"
|
||||
:key="row.code"
|
||||
class="country-check-row"
|
||||
:class="{ disabled: row.code === publishCountry || hasQueueWork }"
|
||||
:class="{ disabled: row.code === publishCountry }"
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
class="country-check-input"
|
||||
:checked="syncCountries.includes(row.code)"
|
||||
:disabled="row.code === publishCountry || hasQueueWork"
|
||||
:disabled="row.code === publishCountry"
|
||||
@change="onSyncCountryChange(row.code, $event)"
|
||||
/>
|
||||
<span class="country-check-text">{{ row.label }} ({{ row.code }})</span>
|
||||
@@ -75,10 +74,10 @@
|
||||
<button
|
||||
type="button"
|
||||
class="btn-run"
|
||||
:disabled="parsing || uploading || hasQueueWork || !uploadedFiles.length"
|
||||
:disabled="parsing || uploading || !uploadedFiles.length"
|
||||
@click="submitRun"
|
||||
>
|
||||
{{ parsing ? '解析中...' : queueWorkerRunning ? '串行上架中...' : '开始上架' }}
|
||||
{{ parsing ? '解析中...' : queueWorkerRunning || hasQueueWork ? '加入等待队列' : '开始上架' }}
|
||||
</button>
|
||||
<span class="loading-msg">{{ operationHint }}</span>
|
||||
</div>
|
||||
@@ -231,7 +230,7 @@ interface PublishDispatchOptions {
|
||||
ziniaoVersion: ZiniaoVersion
|
||||
}
|
||||
|
||||
interface StoredPublishQueue {
|
||||
interface PublishQueueBatch {
|
||||
taskId: number
|
||||
pendingFileIds: number[]
|
||||
activeFileId: number | null
|
||||
@@ -239,6 +238,11 @@ interface StoredPublishQueue {
|
||||
options: PublishDispatchOptions
|
||||
}
|
||||
|
||||
interface StoredPublishQueueState {
|
||||
active: PublishQueueBatch | null
|
||||
pending: PublishQueueBatch[]
|
||||
}
|
||||
|
||||
const selectedPaths = ref<string[]>([])
|
||||
const uploadedFiles = ref<UploadedJavaFile[]>([])
|
||||
const uploading = ref(false)
|
||||
@@ -253,6 +257,7 @@ const currentFiles = ref<PublishFileItem[]>([])
|
||||
const pendingFileIds = ref<number[]>([])
|
||||
const activeFileId = ref<number | null>(null)
|
||||
const dispatchOptions = ref<PublishDispatchOptions | null>(null)
|
||||
const queuedBatches = ref<PublishQueueBatch[]>([])
|
||||
const taskSnapshots = ref<Record<number, PublishTaskDetailVo>>({})
|
||||
const historyItems = ref<PublishTaskDetailVo[]>([])
|
||||
const missingTaskIds = ref<number[]>([])
|
||||
@@ -355,8 +360,9 @@ const progressLoop = useTaskProgressLoop<PublishTaskDetailVo>({
|
||||
})
|
||||
|
||||
const displayPaths = computed(() => selectedPaths.value.slice(0, 8))
|
||||
const hasQueueWork = computed(() => pendingFileIds.value.length > 0 || activeFileId.value != null)
|
||||
const selectionDisabled = computed(() => uploading.value || parsing.value || hasQueueWork.value)
|
||||
const activeBatchHasWork = computed(() => pendingFileIds.value.length > 0 || activeFileId.value != null)
|
||||
const hasQueueWork = computed(() => activeBatchHasWork.value || queuedBatches.value.length > 0)
|
||||
const selectionDisabled = computed(() => uploading.value || parsing.value)
|
||||
|
||||
const currentDetail = computed<PublishTaskDetailVo | null>(() => {
|
||||
const taskId = currentTaskId.value
|
||||
@@ -512,13 +518,12 @@ async function submitRun() {
|
||||
})
|
||||
if (!parsed.taskId) throw new Error('后端未返回有效任务标识')
|
||||
|
||||
currentTaskId.value = parsed.taskId
|
||||
currentFiles.value = parsed.files || []
|
||||
dispatchOptions.value = {
|
||||
const options: PublishDispatchOptions = {
|
||||
publishCountry: publishCountry.value,
|
||||
syncCountries: syncCountries.value.filter((country) => country !== publishCountry.value),
|
||||
ziniaoVersion: ziniaoVersion.value,
|
||||
}
|
||||
const files = parsed.files || []
|
||||
applyTaskSnapshot(parsed.taskId, {
|
||||
task: {
|
||||
id: parsed.taskId,
|
||||
@@ -527,26 +532,44 @@ async function submitRun() {
|
||||
sourceFileCount: parsed.sourceFileCount,
|
||||
totalRows: parsed.totalRows,
|
||||
},
|
||||
files: parsed.files || [],
|
||||
files,
|
||||
result: parsed.result,
|
||||
})
|
||||
|
||||
pendingFileIds.value = currentFiles.value
|
||||
const batch: PublishQueueBatch = {
|
||||
taskId: parsed.taskId,
|
||||
pendingFileIds: files
|
||||
.filter((file) => !isTerminalStatus(file.status) && isMatchedFile(file))
|
||||
.map((file) => file.fileId)
|
||||
.filter((fileId) => Number.isFinite(fileId) && fileId > 0)
|
||||
activeFileId.value = null
|
||||
saveQueueState()
|
||||
.filter((fileId) => Number.isFinite(fileId) && fileId > 0),
|
||||
activeFileId: null,
|
||||
files,
|
||||
options,
|
||||
}
|
||||
progressLoop.add(parsed.taskId)
|
||||
selectedPaths.value = []
|
||||
uploadedFiles.value = []
|
||||
|
||||
await Promise.all([loadDashboard(), loadHistory()])
|
||||
if (!pendingFileIds.value.length) {
|
||||
if (!batch.pendingFileIds.length) {
|
||||
queueMessage.value = '解析完成,当前没有匹配成功且可执行的文件。'
|
||||
ElMessage.warning(queueMessage.value)
|
||||
return
|
||||
}
|
||||
|
||||
queueMessage.value = `批次 ${parsed.taskNo || parsed.taskId} 已创建,开始串行处理 ${pendingFileIds.value.length} 个文件。`
|
||||
const shouldWait = queueWorkerRunning.value || activeBatchHasWork.value || queuedBatches.value.length > 0
|
||||
if (shouldWait) {
|
||||
queuedBatches.value.push(batch)
|
||||
saveQueueState()
|
||||
queueMessage.value = `批次 ${parsed.taskNo || parsed.taskId} 已加入等待队列,前面还有 ${queuedBatches.value.length} 个批次`
|
||||
ElMessage.success(queueMessage.value)
|
||||
if (!queueWorkerRunning.value) void processQueue()
|
||||
return
|
||||
}
|
||||
|
||||
setActiveBatch(batch)
|
||||
saveQueueState()
|
||||
queueMessage.value = `批次 ${parsed.taskNo || parsed.taskId} 已创建,开始串行处理 ${batch.pendingFileIds.length} 个文件。`
|
||||
ElMessage.success(queueMessage.value)
|
||||
void processQueue()
|
||||
} catch (error) {
|
||||
@@ -560,37 +583,85 @@ async function submitRun() {
|
||||
|
||||
function saveQueueState() {
|
||||
if (typeof window === 'undefined') return
|
||||
if (!currentTaskId.value || !dispatchOptions.value || !hasQueueWork.value) {
|
||||
const active: PublishQueueBatch | null = currentTaskId.value && dispatchOptions.value && activeBatchHasWork.value
|
||||
? {
|
||||
taskId: currentTaskId.value,
|
||||
pendingFileIds: pendingFileIds.value,
|
||||
activeFileId: activeFileId.value,
|
||||
files: currentFiles.value,
|
||||
options: dispatchOptions.value,
|
||||
}
|
||||
: null
|
||||
const pending = queuedBatches.value.filter(
|
||||
(batch) => batch.pendingFileIds.length > 0 || batch.activeFileId != null,
|
||||
)
|
||||
if (!active && !pending.length) {
|
||||
window.localStorage.removeItem(queueStorageKey())
|
||||
return
|
||||
}
|
||||
const state: StoredPublishQueue = {
|
||||
taskId: currentTaskId.value,
|
||||
pendingFileIds: pendingFileIds.value,
|
||||
activeFileId: activeFileId.value,
|
||||
files: currentFiles.value,
|
||||
options: dispatchOptions.value,
|
||||
}
|
||||
const state: StoredPublishQueueState = { active, pending }
|
||||
window.localStorage.setItem(queueStorageKey(), JSON.stringify(state))
|
||||
}
|
||||
|
||||
function setActiveBatch(batch: PublishQueueBatch, restoreForm = false) {
|
||||
currentTaskId.value = batch.taskId
|
||||
currentFiles.value = batch.files
|
||||
pendingFileIds.value = [...batch.pendingFileIds]
|
||||
activeFileId.value = batch.activeFileId
|
||||
dispatchOptions.value = batch.options
|
||||
if (restoreForm) {
|
||||
publishCountry.value = batch.options.publishCountry
|
||||
syncCountries.value = [...batch.options.syncCountries]
|
||||
ziniaoVersion.value = batch.options.ziniaoVersion
|
||||
}
|
||||
}
|
||||
|
||||
function takeNextBatch() {
|
||||
while (queuedBatches.value.length) {
|
||||
const next = queuedBatches.value.shift()
|
||||
if (!next) break
|
||||
if (
|
||||
missingTaskIds.value.includes(next.taskId)
|
||||
|| isTerminalStatus(taskSnapshots.value[next.taskId]?.task?.status)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
setActiveBatch(next)
|
||||
saveQueueState()
|
||||
return true
|
||||
}
|
||||
saveQueueState()
|
||||
return false
|
||||
}
|
||||
|
||||
function loadQueueState() {
|
||||
if (typeof window === 'undefined') return
|
||||
try {
|
||||
const raw = window.localStorage.getItem(queueStorageKey())
|
||||
if (!raw) return
|
||||
const state = JSON.parse(raw) as StoredPublishQueue
|
||||
if (!Number.isFinite(state.taskId) || state.taskId <= 0 || !state.options) return
|
||||
currentTaskId.value = state.taskId
|
||||
pendingFileIds.value = (state.pendingFileIds || []).filter((id) => Number.isFinite(id) && id > 0)
|
||||
activeFileId.value = Number.isFinite(state.activeFileId) && Number(state.activeFileId) > 0
|
||||
? Number(state.activeFileId)
|
||||
: null
|
||||
currentFiles.value = Array.isArray(state.files) ? state.files : []
|
||||
dispatchOptions.value = state.options
|
||||
publishCountry.value = state.options.publishCountry
|
||||
syncCountries.value = [...state.options.syncCountries]
|
||||
ziniaoVersion.value = state.options.ziniaoVersion
|
||||
const state = JSON.parse(raw) as Partial<StoredPublishQueueState> & Partial<PublishQueueBatch>
|
||||
const active = state.active && Number.isFinite(state.active.taskId) && state.active.taskId > 0
|
||||
? state.active
|
||||
: !state.active && Number.isFinite(state.taskId) && Number(state.taskId) > 0 && state.options
|
||||
? {
|
||||
taskId: Number(state.taskId),
|
||||
pendingFileIds: state.pendingFileIds || [],
|
||||
activeFileId: state.activeFileId ?? null,
|
||||
files: state.files || [],
|
||||
options: state.options,
|
||||
}
|
||||
: null
|
||||
if (active) setActiveBatch(active, true)
|
||||
queuedBatches.value = (Array.isArray(state.pending) ? state.pending : [])
|
||||
.filter((batch) => Number.isFinite(batch.taskId) && batch.taskId > 0 && batch.options)
|
||||
.map((batch) => ({
|
||||
...batch,
|
||||
pendingFileIds: (batch.pendingFileIds || []).filter((id) => Number.isFinite(id) && id > 0),
|
||||
activeFileId: Number.isFinite(batch.activeFileId) && Number(batch.activeFileId) > 0
|
||||
? Number(batch.activeFileId)
|
||||
: null,
|
||||
files: Array.isArray(batch.files) ? batch.files : [],
|
||||
}))
|
||||
} catch {
|
||||
window.localStorage.removeItem(queueStorageKey())
|
||||
}
|
||||
@@ -711,7 +782,7 @@ async function waitForFileTerminal(taskId: number, fileId: number) {
|
||||
}
|
||||
|
||||
async function processQueue() {
|
||||
if (disposed || queueWorkerRunning.value || !currentTaskId.value || !dispatchOptions.value) return
|
||||
if (disposed || queueWorkerRunning.value || (!activeBatchHasWork.value && !queuedBatches.value.length)) return
|
||||
const api = getPywebviewApi()
|
||||
if (!api?.enqueue_json) {
|
||||
queueMessage.value = '当前客户端未提供 enqueue_json,无法派发上架任务。'
|
||||
@@ -719,76 +790,88 @@ async function processQueue() {
|
||||
return
|
||||
}
|
||||
|
||||
const taskId = currentTaskId.value
|
||||
queueWorkerRunning.value = true
|
||||
progressLoop.add(taskId)
|
||||
try {
|
||||
await activatePublishTask(taskId)
|
||||
while (!disposed && (activeFileId.value != null || pendingFileIds.value.length > 0)) {
|
||||
if (
|
||||
missingTaskIds.value.includes(taskId)
|
||||
|| isTerminalStatus(taskSnapshots.value[taskId]?.task?.status)
|
||||
) {
|
||||
pendingFileIds.value = []
|
||||
activeFileId.value = null
|
||||
saveQueueState()
|
||||
break
|
||||
}
|
||||
if (activeFileId.value != null) {
|
||||
const restoredFileId = activeFileId.value
|
||||
queueMessage.value = `正在等待文件 ${getCurrentFile(restoredFileId)?.sourceFilename || restoredFileId} 完成...`
|
||||
const restoredStatus = await waitForFileTerminal(taskId, restoredFileId)
|
||||
if (restoredStatus === 'STOPPED') return
|
||||
activeFileId.value = null
|
||||
saveQueueState()
|
||||
continue
|
||||
while (!disposed) {
|
||||
if (!currentTaskId.value || !dispatchOptions.value || !activeBatchHasWork.value) {
|
||||
if (!takeNextBatch()) break
|
||||
}
|
||||
|
||||
const nextFileId = pendingFileIds.value.shift()
|
||||
if (!nextFileId) break
|
||||
const file = getCurrentFile(nextFileId)
|
||||
if (!file || isTerminalStatus(file.status)) {
|
||||
saveQueueState()
|
||||
continue
|
||||
}
|
||||
const taskId = currentTaskId.value
|
||||
if (!taskId || !dispatchOptions.value) continue
|
||||
progressLoop.add(taskId)
|
||||
await activatePublishTask(taskId)
|
||||
|
||||
activeFileId.value = nextFileId
|
||||
saveQueueState()
|
||||
try {
|
||||
await activatePublishFile(taskId, nextFileId)
|
||||
updateCurrentFile(nextFileId, { status: 'RUNNING', progressMessage: '已派发到 Python 队列' })
|
||||
const payload = buildQueuePayload(taskId, file)
|
||||
const result = await api.enqueue_json(payload)
|
||||
if (!result?.success) throw new Error(result?.error || 'Python 队列拒绝接收任务')
|
||||
queueMessage.value = pendingFileIds.value.length
|
||||
? `文件 ${file.sourceFilename || nextFileId} 已入队,完成后继续剩余 ${pendingFileIds.value.length} 个文件。`
|
||||
: `文件 ${file.sourceFilename || nextFileId} 已入队,等待执行完成。`
|
||||
const finalStatus = await waitForFileTerminal(taskId, nextFileId)
|
||||
if (finalStatus === 'STOPPED') return
|
||||
queueMessage.value = `文件 ${file.sourceFilename || nextFileId} ${['SUCCESS', 'COMPLETED'].includes(finalStatus) ? '已完成' : '执行失败'},继续下一个文件。`
|
||||
activeFileId.value = null
|
||||
saveQueueState()
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : '文件派发失败'
|
||||
try {
|
||||
await submitDispatchFailure(taskId, file, reason)
|
||||
} catch (compensationError) {
|
||||
const message = compensationError instanceof Error ? compensationError.message : '失败状态提交失败'
|
||||
queueMessage.value = `文件 ${file.sourceFilename || nextFileId} 派发失败,后端未确认失败状态:${message}。串行队列已暂停。`
|
||||
while (!disposed && (activeFileId.value != null || pendingFileIds.value.length > 0)) {
|
||||
if (
|
||||
missingTaskIds.value.includes(taskId)
|
||||
|| isTerminalStatus(taskSnapshots.value[taskId]?.task?.status)
|
||||
) {
|
||||
pendingFileIds.value = []
|
||||
activeFileId.value = null
|
||||
saveQueueState()
|
||||
throw new Error(queueMessage.value)
|
||||
break
|
||||
}
|
||||
if (activeFileId.value != null) {
|
||||
const restoredFileId = activeFileId.value
|
||||
queueMessage.value = `正在等待文件 ${getCurrentFile(restoredFileId)?.sourceFilename || restoredFileId} 完成...`
|
||||
const restoredStatus = await waitForFileTerminal(taskId, restoredFileId)
|
||||
if (restoredStatus === 'STOPPED') return
|
||||
activeFileId.value = null
|
||||
saveQueueState()
|
||||
continue
|
||||
}
|
||||
queueMessage.value = `文件 ${file.sourceFilename || nextFileId} 派发失败,已记录并继续下一个文件。`
|
||||
activeFileId.value = null
|
||||
saveQueueState()
|
||||
}
|
||||
}
|
||||
|
||||
if (!disposed) {
|
||||
queueMessage.value = '本批次文件已全部按顺序派发完成,正在生成结果文件。'
|
||||
const nextFileId = pendingFileIds.value.shift()
|
||||
if (!nextFileId) break
|
||||
const file = getCurrentFile(nextFileId)
|
||||
if (!file || isTerminalStatus(file.status)) {
|
||||
saveQueueState()
|
||||
continue
|
||||
}
|
||||
|
||||
activeFileId.value = nextFileId
|
||||
saveQueueState()
|
||||
try {
|
||||
await activatePublishFile(taskId, nextFileId)
|
||||
updateCurrentFile(nextFileId, { status: 'RUNNING', progressMessage: '已派发到 Python 队列' })
|
||||
const payload = buildQueuePayload(taskId, file)
|
||||
const result = await api.enqueue_json(payload)
|
||||
if (!result?.success) throw new Error(result?.error || 'Python 队列拒绝接收任务')
|
||||
queueMessage.value = pendingFileIds.value.length
|
||||
? `文件 ${file.sourceFilename || nextFileId} 已入队,完成后继续剩余 ${pendingFileIds.value.length} 个文件。`
|
||||
: `文件 ${file.sourceFilename || nextFileId} 已入队,等待执行完成。`
|
||||
const finalStatus = await waitForFileTerminal(taskId, nextFileId)
|
||||
if (finalStatus === 'STOPPED') return
|
||||
queueMessage.value = `文件 ${file.sourceFilename || nextFileId} ${['SUCCESS', 'COMPLETED'].includes(finalStatus) ? '已完成' : '执行失败'},继续下一个文件。`
|
||||
activeFileId.value = null
|
||||
saveQueueState()
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : '文件派发失败'
|
||||
try {
|
||||
await submitDispatchFailure(taskId, file, reason)
|
||||
} catch (compensationError) {
|
||||
const message = compensationError instanceof Error ? compensationError.message : '失败状态提交失败'
|
||||
queueMessage.value = `文件 ${file.sourceFilename || nextFileId} 派发失败,后端未确认失败状态:${message}。串行队列已暂停。`
|
||||
saveQueueState()
|
||||
throw new Error(queueMessage.value)
|
||||
}
|
||||
queueMessage.value = `文件 ${file.sourceFilename || nextFileId} 派发失败,已记录并继续下一个文件。`
|
||||
activeFileId.value = null
|
||||
saveQueueState()
|
||||
}
|
||||
}
|
||||
|
||||
if (disposed) return
|
||||
await progressLoop.refreshOnce()
|
||||
await Promise.all([loadDashboard(), loadHistory()])
|
||||
ElMessage.success('上架批次已完成串行派发')
|
||||
if (queuedBatches.value.length) {
|
||||
queueMessage.value = `当前批次已完成派发,继续处理后续 ${queuedBatches.value.length} 个等待批次。`
|
||||
continue
|
||||
}
|
||||
queueMessage.value = '所有上架批次已按顺序派发完成,正在生成结果文件。'
|
||||
ElMessage.success('上架等待队列已完成派发')
|
||||
break
|
||||
}
|
||||
} catch (error) {
|
||||
if (disposed) return
|
||||
@@ -825,34 +908,57 @@ async function loadHistory() {
|
||||
}
|
||||
|
||||
function reconcileStoredQueue() {
|
||||
function reconcileBatch(batch: PublishQueueBatch) {
|
||||
if (missingTaskIds.value.includes(batch.taskId)) return null
|
||||
const snapshot = taskSnapshots.value[batch.taskId]
|
||||
if (isTerminalStatus(snapshot?.task?.status)) return null
|
||||
const files = mergeFiles(batch.files || [], snapshot?.files || [])
|
||||
let pendingIds = (batch.pendingFileIds || []).filter((fileId) => {
|
||||
const file = files.find((item) => item.fileId === fileId)
|
||||
return file && !isTerminalStatus(file.status)
|
||||
})
|
||||
let activeId = batch.activeFileId
|
||||
if (activeId != null) {
|
||||
const activeStatus = normalizeStatus(files.find((file) => file.fileId === activeId)?.status)
|
||||
if (!activeStatus || activeStatus === 'PENDING') {
|
||||
pendingIds = [activeId, ...pendingIds.filter((fileId) => fileId !== activeId)]
|
||||
activeId = null
|
||||
} else if (isTerminalStatus(activeStatus)) {
|
||||
activeId = null
|
||||
}
|
||||
}
|
||||
if (!pendingIds.length && activeId == null) return null
|
||||
return { ...batch, pendingFileIds: pendingIds, activeFileId: activeId, files }
|
||||
}
|
||||
|
||||
const taskId = currentTaskId.value
|
||||
if (!taskId) return
|
||||
if (missingTaskIds.value.includes(taskId)) {
|
||||
const active = taskId && dispatchOptions.value
|
||||
? reconcileBatch({
|
||||
taskId,
|
||||
pendingFileIds: pendingFileIds.value,
|
||||
activeFileId: activeFileId.value,
|
||||
files: currentFiles.value,
|
||||
options: dispatchOptions.value,
|
||||
})
|
||||
: null
|
||||
if (active) {
|
||||
setActiveBatch(active)
|
||||
} else {
|
||||
currentTaskId.value = null
|
||||
currentFiles.value = []
|
||||
pendingFileIds.value = []
|
||||
activeFileId.value = null
|
||||
saveQueueState()
|
||||
queueMessage.value = `任务 ${taskId} 已不存在,本地队列已清理。`
|
||||
return
|
||||
dispatchOptions.value = null
|
||||
}
|
||||
if (isTerminalStatus(taskSnapshots.value[taskId]?.task?.status)) {
|
||||
pendingFileIds.value = []
|
||||
activeFileId.value = null
|
||||
saveQueueState()
|
||||
return
|
||||
}
|
||||
pendingFileIds.value = pendingFileIds.value.filter((fileId) => {
|
||||
const file = getCurrentFile(fileId)
|
||||
return file && !isTerminalStatus(file.status)
|
||||
})
|
||||
if (activeFileId.value != null) {
|
||||
const activeId = activeFileId.value
|
||||
const activeStatus = normalizeStatus(getCurrentFile(activeId)?.status)
|
||||
queuedBatches.value = queuedBatches.value
|
||||
.map(reconcileBatch)
|
||||
.filter((batch): batch is PublishQueueBatch => batch != null)
|
||||
|
||||
if (active?.activeFileId != null) {
|
||||
const activeId = active.activeFileId
|
||||
const activeStatus = normalizeStatus(active.files.find((file) => file.fileId === activeId)?.status)
|
||||
if (!activeStatus || activeStatus === 'PENDING') {
|
||||
pendingFileIds.value = [activeId, ...pendingFileIds.value.filter((fileId) => fileId !== activeId)]
|
||||
activeFileId.value = null
|
||||
queueMessage.value = `文件 ${activeId} 尚未激活,已恢复到待派发队列。`
|
||||
} else if (isTerminalStatus(activeStatus)) {
|
||||
activeFileId.value = null
|
||||
}
|
||||
}
|
||||
saveQueueState()
|
||||
@@ -935,13 +1041,14 @@ async function downloadResult(detail: PublishTaskDetailVo) {
|
||||
onMounted(async () => {
|
||||
loadQueueState()
|
||||
if (currentTaskId.value) progressLoop.add(currentTaskId.value)
|
||||
for (const batch of queuedBatches.value) progressLoop.add(batch.taskId)
|
||||
await Promise.all([loadDashboard(), loadHistory()])
|
||||
if (currentTaskId.value) await progressLoop.refreshOnce()
|
||||
if (currentTaskId.value || queuedBatches.value.length) await progressLoop.refreshOnce()
|
||||
reconcileStoredQueue()
|
||||
if (hasQueueWork.value) {
|
||||
queueMessage.value = activeFileId.value != null
|
||||
? `检测到未完成文件 ${activeFileId.value},继续等待完成后衔接后续文件。`
|
||||
: `检测到未完成队列,继续处理剩余 ${pendingFileIds.value.length} 个文件。`
|
||||
: `检测到未完成队列,继续处理当前文件及后续 ${queuedBatches.value.length} 个批次。`
|
||||
void processQueue()
|
||||
}
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user