modified:   amazon/detail_spider.py
	modified:   amazon/main.py
This commit is contained in:
铭坤
2026-05-04 19:17:11 +08:00
78 changed files with 7623 additions and 276 deletions

View File

@@ -111,8 +111,29 @@ class ChromeAmzone(ChromeAmzoneBase):
<<<<<<< HEAD
class SpiderTask(TaskBase): class SpiderTask(TaskBase):
task_name = "亚马逊采集" task_name = "亚马逊采集"
=======
class SpiderTask:
mark_name = "亚马逊采集"
def __init__(self, user_info: dict = None):
"""初始化审批任务处理器
Args:
user_info: 用户信息字典,包含 company, username, password
"""
self.user_info = user_info or {}
self.running = True
self.result_api_module = "appearance-patent"
def log(self, message: str, level: str = "INFO"):
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [PriceTask] [{level}] {message}")
>>>>>>> 6d398b66bc24634b13f1de162e8f68836e64441d
@staticmethod @staticmethod
def group_by_id_prefix(data): def group_by_id_prefix(data):
@@ -166,9 +187,16 @@ class SpiderTask(TaskBase):
task_data: 任务数据 task_data: 任务数据
""" """
try: try:
<<<<<<< HEAD
data = task_data.get("data", {}) data = task_data.get("data", {})
task_id = data.get("taskId") task_id = data.get("taskId")
groups = self.normalize_groups(data) groups = self.normalize_groups(data)
=======
data = task_data.get("data", {})
self.result_api_module = "similar-asin" if task_data.get("type") == "similar-asin-run" else "appearance-patent"
task_id = data.get("taskId")
groups = self.normalize_groups(data)
>>>>>>> 6d398b66bc24634b13f1de162e8f68836e64441d
# 用于测试 # 用于测试
limit = data.get("limit", None) limit = data.get("limit", None)
@@ -307,7 +335,7 @@ class SpiderTask(TaskBase):
"""回传处理结果到API """回传处理结果到API
""" """
url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result" url = f"{DELETE_BRAND_API_BASE}/api/{self.result_api_module}/tasks/{task_id}/result"
payload ={ payload ={
"submissionId": f"{task_id}", "submissionId": f"{task_id}",

View File

@@ -65,6 +65,7 @@ class TaskMonitor:
"query-asin-run" : "状态查询", "query-asin-run" : "状态查询",
"patrol-delete-run" : "巡店删除", "patrol-delete-run" : "巡店删除",
"appearance-patent-run" : "亚马逊采集", "appearance-patent-run" : "亚马逊采集",
"similar-asin-run" : "亚马逊采集",
} }
try: try:
while self.running: while self.running:

View File

@@ -0,0 +1,73 @@
package com.nanri.aiimage.common.util;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.http.HttpHeaders;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
public final class DownloadHeaderUtil {
private static final String DEFAULT_FILENAME = "download";
private DownloadHeaderUtil() {
}
public static void setAttachment(HttpServletResponse response, String filename) {
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, contentDisposition(filename));
}
public static String contentDisposition(String filename) {
String normalized = normalizeFilename(filename);
String encodedFilename = URLEncoder.encode(normalized, StandardCharsets.UTF_8).replace("+", "%20");
return "attachment; filename=\"" + asciiFallback(normalized) + "\"; filename*=UTF-8''" + encodedFilename;
}
private static String normalizeFilename(String filename) {
if (filename == null || filename.isBlank()) {
return DEFAULT_FILENAME;
}
return filename.replace("\r", "").replace("\n", "").trim();
}
private static String asciiFallback(String filename) {
String normalized = normalizeFilename(filename);
String extension = extensionOf(normalized);
String base = extension.isEmpty() ? normalized : normalized.substring(0, normalized.length() - extension.length() - 1);
String sanitizedBase = sanitizeAscii(base);
String sanitizedExtension = sanitizeExtension(extension);
if (sanitizedBase.isBlank()) {
sanitizedBase = DEFAULT_FILENAME;
}
String fallback = sanitizedExtension.isBlank() ? sanitizedBase : sanitizedBase + "." + sanitizedExtension;
return fallback.length() <= 120 ? fallback : shorten(fallback, sanitizedExtension);
}
private static String extensionOf(String filename) {
int dotIndex = filename.lastIndexOf('.');
if (dotIndex < 0 || dotIndex == filename.length() - 1) {
return "";
}
return filename.substring(dotIndex + 1);
}
private static String sanitizeAscii(String value) {
return value.replaceAll("[^A-Za-z0-9._-]", "_")
.replaceAll("_+", "_")
.replaceAll("^[_\\.]+|[_\\.]+$", "");
}
private static String sanitizeExtension(String extension) {
return extension.replaceAll("[^A-Za-z0-9]", "");
}
private static String shorten(String fallback, String extension) {
int maxBaseLength = extension.isBlank() ? 120 : Math.max(1, 119 - extension.length());
String base = extension.isBlank() ? fallback : fallback.substring(0, fallback.length() - extension.length() - 1);
base = base.substring(0, Math.min(base.length(), maxBaseLength)).replaceAll("[_\\.]+$", "");
if (base.isBlank()) {
base = DEFAULT_FILENAME;
}
return extension.isBlank() ? base : base + "." + extension;
}
}

View File

@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
@Configuration @Configuration
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class}) @EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class})
public class PropertiesConfig { public class PropertiesConfig {
} }

View File

@@ -0,0 +1,21 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Data
@ConfigurationProperties(prefix = "aiimage.similar-asin")
public class SimilarAsinProperties {
private String cozeBaseUrl = "https://api.coze.cn";
private String cozeWorkflowPath = "/v1/workflow/run";
private String cozeWorkflowHistoryPath = "/v1/workflows/{workflow_id}/run_histories/{execute_id}";
private String cozeWorkflowId = "7632683471312355338";
private String cozeToken = "";
private int cozeBatchSize = 10;
private int cozeConnectTimeoutMillis = 10000;
private int cozeReadTimeoutMillis = 60000;
private int cozePollIntervalMillis = 2000;
private int cozePollTimeoutMillis = 120000;
private int staleTimeoutMinutes = 20;
private String staleFinalizeCron = "0 */2 * * * *";
}

View File

@@ -11,4 +11,5 @@ public class StorageProperties {
private String cleanupCron = "0 0 * * * *"; private String cleanupCron = "0 0 * * * *";
private long sourceRetentionHours = 24; private long sourceRetentionHours = 24;
private long resultRetentionHours = 24; private long resultRetentionHours = 24;
private long transientPayloadRetentionHours = 72;
} }

View File

@@ -4,8 +4,12 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor; import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@Configuration @Configuration
public class TaskFileJobConfig { public class TaskFileJobConfig {
@@ -24,4 +28,16 @@ public class TaskFileJobConfig {
executor.initialize(); executor.initialize();
return executor; return executor;
} }
@Bean(destroyMethod = "shutdown")
public ExecutorService cozeVirtualThreadExecutor() {
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
.name("coze-task-", 0)
.factory());
}
@Bean("cozeTaskExecutor")
public TaskExecutor cozeTaskExecutor(ExecutorService cozeVirtualThreadExecutor) {
return new ConcurrentTaskExecutor(cozeVirtualThreadExecutor);
}
} }

View File

@@ -44,6 +44,42 @@ public class AppearancePatentCozeClient {
} }
} }
public CozeSubmitResponse submitWorkflow(List<AppearancePatentResultRowDto> rows, String prompt) throws Exception {
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt));
ensureSuccess(submitRoot);
return new CozeSubmitResponse(
extractExecuteId(submitRoot),
extractResultDataText(submitRoot),
writeJson(submitRoot)
);
}
public CozePollResponse pollWorkflow(String executeId) throws Exception {
JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId));
ensureSuccess(pollRoot);
String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT);
String dataText = extractResultDataText(pollRoot);
String outputText = dataText.isBlank() ? extractWorkflowOutputText(pollRoot) : "";
String failureMessage = isFailedWorkflowStatus(status)
? firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed")
: "";
return new CozePollResponse(executeId, status, dataText, outputText, failureMessage, writeJson(pollRoot));
}
public List<AppearancePatentResultRowDto> mergeRowsFromDataText(List<AppearancePatentResultRowDto> rows, String dataText) throws Exception {
if (dataText == null || dataText.isBlank()) {
return rows == null ? List.of() : rows.stream().map(this::copy).toList();
}
return mergeRows(rows, parseResults(wrapDataPayload(dataText)));
}
public List<AppearancePatentResultRowDto> markRowsFailed(List<AppearancePatentResultRowDto> rows, String failureMessage) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
private List<AppearancePatentResultRowDto> inspectWithFallback(List<AppearancePatentResultRowDto> rows, String prompt) { private List<AppearancePatentResultRowDto> inspectWithFallback(List<AppearancePatentResultRowDto> rows, String prompt) {
try { try {
if (rows.size() == 1) { if (rows.size() == 1) {
@@ -140,6 +176,7 @@ public class AppearancePatentCozeClient {
long deadline = System.currentTimeMillis() + Math.max(10000, properties.getCozePollTimeoutMillis()); long deadline = System.currentTimeMillis() + Math.max(10000, properties.getCozePollTimeoutMillis());
while (System.currentTimeMillis() < deadline) { while (System.currentTimeMillis() < deadline) {
ensureNotInterrupted();
JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId)); JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId));
ensureSuccess(pollRoot); ensureSuccess(pollRoot);
@@ -149,9 +186,16 @@ public class AppearancePatentCozeClient {
} }
String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT); String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT);
if (status.contains("FAIL") || status.contains("ERROR") || status.contains("CANCEL")) { if (isFailedWorkflowStatus(status)) {
throw new IllegalStateException(firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed")); throw new IllegalStateException(firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed"));
} }
if (isSuccessfulWorkflowStatus(status)) {
String outputText = extractWorkflowOutputText(pollRoot);
if (!outputText.isBlank()) {
return wrapDataPayload(outputText);
}
throw new IllegalStateException("Coze async workflow completed without output");
}
sleepQuietly(Math.max(200, properties.getCozePollIntervalMillis())); sleepQuietly(Math.max(200, properties.getCozePollIntervalMillis()));
} }
@@ -251,7 +295,7 @@ public class AppearancePatentCozeClient {
return List.of(); return List.of();
} }
JsonNode dataRoot = objectMapper.readTree(dataText); JsonNode dataRoot = objectMapper.readTree(dataText);
JsonNode array = dataRoot.path("data"); JsonNode array = dataRoot.isArray() ? dataRoot : dataRoot.path("data");
List<CozeResult> results = new ArrayList<>(); List<CozeResult> results = new ArrayList<>();
if (array.isArray()) { if (array.isArray()) {
for (JsonNode node : array) { for (JsonNode node : array) {
@@ -423,9 +467,21 @@ public class AppearancePatentCozeClient {
} }
private AppearancePatentResultRowDto markFailed(AppearancePatentResultRowDto row, String failureMessage) { private AppearancePatentResultRowDto markFailed(AppearancePatentResultRowDto row, String failureMessage) {
String reviewMessage = failureMessage == null || failureMessage.isBlank()
? "Coze 检测失败,待人工复核"
: "Coze 检测失败,待人工复核:" + failureMessage;
if (row.getError() == null || row.getError().isBlank()) { if (row.getError() == null || row.getError().isBlank()) {
row.setError(failureMessage); row.setError(failureMessage);
} }
if (row.getTitleRisk() == null || row.getTitleRisk().isBlank()) {
row.setTitleRisk(reviewMessage);
}
if (row.getAppearanceRisk() == null || row.getAppearanceRisk().isBlank()) {
row.setAppearanceRisk(reviewMessage);
}
if (row.getPatentRisk() == null || row.getPatentRisk().isBlank()) {
row.setPatentRisk(reviewMessage);
}
if (row.getConclusion() == null || row.getConclusion().isBlank()) { if (row.getConclusion() == null || row.getConclusion().isBlank()) {
row.setConclusion(failureMessage); row.setConclusion(failureMessage);
} }
@@ -502,6 +558,30 @@ public class AppearancePatentCozeClient {
return discoverEmbeddedData(root); return discoverEmbeddedData(root);
} }
private String extractWorkflowOutputText(JsonNode root) {
JsonNode outputNode = findFirstField(root, "output");
if (outputNode == null || outputNode.isNull() || outputNode.isMissingNode()) {
return "";
}
if (!outputNode.isTextual()) {
String discovered = discoverEmbeddedData(outputNode);
return discovered.isBlank() && isResultDataPayload(outputNode) ? outputNode.toString() : discovered;
}
String output = normalize(outputNode.asText(""));
if (output.isBlank()) {
return "";
}
if (looksLikeResultDataPayload(output)) {
return output;
}
JsonNode parsedOutput = parseJsonOrMissing(output);
String nestedOutput = text(firstNonNull(parsedOutput.get("Output"), parsedOutput.get("output")));
if (nestedOutput != null && !nestedOutput.isBlank() && looksLikeResultDataPayload(nestedOutput)) {
return nestedOutput;
}
return discoverEmbeddedData(parsedOutput);
}
private String discoverEmbeddedData(JsonNode node) { private String discoverEmbeddedData(JsonNode node) {
if (node == null || node.isNull() || node.isMissingNode()) { if (node == null || node.isNull() || node.isMissingNode()) {
return ""; return "";
@@ -577,6 +657,22 @@ public class AppearancePatentCozeClient {
|| item.has("patent reason")); || item.has("patent reason"));
} }
private boolean isSuccessfulWorkflowStatus(String status) {
String normalized = normalize(status).toUpperCase(Locale.ROOT);
return normalized.equals("SUCCESS")
|| normalized.equals("SUCCEEDED")
|| normalized.equals("COMPLETED")
|| normalized.equals("COMPLETE")
|| normalized.equals("DONE");
}
private boolean isFailedWorkflowStatus(String status) {
String normalized = normalize(status).toUpperCase(Locale.ROOT);
return normalized.contains("FAIL")
|| normalized.contains("ERROR")
|| normalized.contains("CANCEL");
}
private JsonNode parseJsonOrMissing(String value) { private JsonNode parseJsonOrMissing(String value) {
String normalized = normalize(value); String normalized = normalize(value);
if (!(normalized.startsWith("{") || normalized.startsWith("["))) { if (!(normalized.startsWith("{") || normalized.startsWith("["))) {
@@ -611,6 +707,38 @@ public class AppearancePatentCozeClient {
return findTextByFieldName(root, "execute_id", "executeId"); return findTextByFieldName(root, "execute_id", "executeId");
} }
private JsonNode findFirstField(JsonNode node, String name) {
if (node == null || node.isMissingNode() || node.isNull() || name == null || name.isBlank()) {
return objectMapper.missingNode();
}
if (node.isTextual()) {
return findFirstField(parseJsonOrMissing(node.asText("")), name);
}
if (node.isArray()) {
for (JsonNode child : node) {
JsonNode found = findFirstField(child, name);
if (found != null && !found.isMissingNode() && !found.isNull()) {
return found;
}
}
return objectMapper.missingNode();
}
if (node.isObject()) {
JsonNode direct = node.get(name);
if (direct != null && !direct.isMissingNode() && !direct.isNull()) {
return direct;
}
for (java.util.Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> entry = it.next();
JsonNode found = findFirstField(entry.getValue(), name);
if (found != null && !found.isMissingNode() && !found.isNull()) {
return found;
}
}
}
return objectMapper.missingNode();
}
private String findTextByFieldName(JsonNode node, String... names) { private String findTextByFieldName(JsonNode node, String... names) {
if (node == null || node.isMissingNode() || node.isNull()) { if (node == null || node.isMissingNode() || node.isNull()) {
return ""; return "";
@@ -677,6 +805,7 @@ public class AppearancePatentCozeClient {
private void sleepBeforeRetry(int attemptIndex) { private void sleepBeforeRetry(int attemptIndex) {
long delayMillis = Math.max(1, attemptIndex) * 1500L; long delayMillis = Math.max(1, attemptIndex) * 1500L;
ensureNotInterrupted();
sleepQuietly(delayMillis); sleepQuietly(delayMillis);
} }
@@ -685,6 +814,13 @@ public class AppearancePatentCozeClient {
Thread.sleep(delayMillis); Thread.sleep(delayMillis);
} catch (InterruptedException interruptedException) { } catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
throw new IllegalStateException("Coze workflow interrupted", interruptedException);
}
}
private void ensureNotInterrupted() {
if (Thread.currentThread().isInterrupted()) {
throw new IllegalStateException("Coze workflow interrupted");
} }
} }
@@ -783,6 +919,45 @@ public class AppearancePatentCozeClient {
) { ) {
} }
public record CozeSubmitResponse(
String executeId,
String immediateData,
String rawResponse
) {
}
public record CozePollResponse(
String executeId,
String status,
String dataText,
String outputText,
String failureMessage,
String rawResponse
) {
public boolean hasPayload() {
return dataText != null && !dataText.isBlank() || outputText != null && !outputText.isBlank();
}
public String resolvedPayloadText() {
return dataText != null && !dataText.isBlank() ? dataText : outputText;
}
public boolean isFailed() {
String normalized = status == null ? "" : status.trim().toUpperCase(Locale.ROOT);
return normalized.contains("FAILED") || normalized.contains("ERROR") || normalized.contains("CANCEL");
}
public boolean isFinished() {
String normalized = status == null ? "" : status.trim().toUpperCase(Locale.ROOT);
return isFailed()
|| normalized.contains("SUCCESS")
|| normalized.contains("SUCCEED")
|| normalized.contains("FINISH")
|| normalized.contains("DONE")
|| normalized.contains("COMPLET");
}
}
private static final class PartialCozeResultException extends RuntimeException { private static final class PartialCozeResultException extends RuntimeException {
private final int resolvedCount; private final int resolvedCount;

View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.appearancepatent.controller; package com.nanri.aiimage.modules.appearancepatent.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentTaskBatchRequest; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentTaskBatchRequest;
@@ -14,7 +15,6 @@ import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -28,7 +28,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@RestController @RestController
@@ -129,10 +128,8 @@ public class AppearancePatentController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;

View File

@@ -29,9 +29,17 @@ public class AppearancePatentResultRowDto {
private String country; private String country;
@Schema(description = "商品主图或待检测图片 URL。Java 调用 Coze 时会放入 url_list。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg") @Schema(description = "商品主图或待检测图片 URL。Java 调用 Coze 时会放入 url_list。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg")
@JsonAlias({
"imageUrl", "image_url", "imgUrl", "img_url", "pictureUrl", "picture_url",
"mainImage", "main_image", "mainImageUrl", "main_image_url",
"productImage", "product_image", "productImageUrl", "product_image_url",
"image", "img", "pic", "picture", "link", "imageLink", "image_link",
"图片链接", "商品图片", "商品主图", "主图", "主图链接"
})
private String url; private String url;
@Schema(description = "商品标题。Java 调用 Coze 时会放入 title_list为空时会回退使用 ASIN。", example = "Women Floral Dress Summer Casual") @Schema(description = "商品标题。Java 调用 Coze 时会放入 title_list为空时会回退使用 ASIN。", example = "Women Floral Dress Summer Casual")
@JsonAlias({"productTitle", "product_title", "itemTitle", "item_title", "商品标题", "商品名称", "标题"})
private String title; private String title;
@Schema(description = "单行错误信息。通常用于记录 Python 单行处理异常Coze 失败时后端会尽量保留原始行,不强行写入风险结果。", example = "图片地址为空") @Schema(description = "单行错误信息。通常用于记录 Python 单行处理异常Coze 失败时后端会尽量保留原始行,不强行写入风险结果。", example = "图片地址为空")

View File

@@ -3,9 +3,11 @@ package com.nanri.aiimage.modules.appearancepatent.service;
import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.IdUtil;
import cn.hutool.crypto.digest.DigestUtil; import cn.hutool.crypto.digest.DigestUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.AppearancePatentProperties; import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.config.StorageProperties; import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentCozeClient; import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentCozeClient;
@@ -50,7 +52,10 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory; import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.dao.DuplicateKeyException; import org.springframework.dao.DuplicateKeyException;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.PlatformTransactionManager;
@@ -73,6 +78,7 @@ import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.UUID;
import java.util.function.Supplier; import java.util.function.Supplier;
@Service @Service
@@ -85,7 +91,13 @@ public class AppearancePatentTaskService {
private static final String STATUS_RUNNING = "RUNNING"; private static final String STATUS_RUNNING = "RUNNING";
private static final String STATUS_SUCCESS = "SUCCESS"; private static final String STATUS_SUCCESS = "SUCCESS";
private static final String STATUS_FAILED = "FAILED"; private static final String STATUS_FAILED = "FAILED";
private static final String COZE_STATUS_SUBMITTED = "SUBMITTED";
private static final String COZE_STATUS_RUNNING = "RUNNING";
private static final String COZE_STATUS_DONE = "DONE";
private static final String COZE_STATUS_FAILED = "FAILED";
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
private static final List<String> RESULT_HEADERS = List.of( private static final List<String> RESULT_HEADERS = List.of(
"id", "id",
"asin", "asin",
@@ -112,9 +124,13 @@ public class AppearancePatentTaskService {
private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TransientPayloadStorageService transientPayloadStorageService; private final TransientPayloadStorageService transientPayloadStorageService;
private final PlatformTransactionManager transactionManager; private final PlatformTransactionManager transactionManager;
private final DistributedJobLockService distributedJobLockService;
@Autowired
@Qualifier("cozeTaskExecutor")
private TaskExecutor cozeTaskExecutor;
@Transactional
public AppearancePatentParseVo parseAndCreateTask(AppearancePatentParseRequest request) { public AppearancePatentParseVo parseAndCreateTask(AppearancePatentParseRequest request) {
long startedAt = System.nanoTime();
if (request.getUserId() == null || request.getUserId() <= 0) { if (request.getUserId() == null || request.getUserId() <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
} }
@@ -131,6 +147,7 @@ public class AppearancePatentTaskService {
List<String> mergedHeaders = new ArrayList<>(); List<String> mergedHeaders = new ArrayList<>();
int totalRows = 0; int totalRows = 0;
int droppedRows = 0; int droppedRows = 0;
long parseStartedAt = System.nanoTime();
for (AppearancePatentSourceFileDto source : sourceFiles) { for (AppearancePatentSourceFileDto source : sourceFiles) {
if (source.getFileKey() == null || source.getFileKey().isBlank()) { if (source.getFileKey() == null || source.getFileKey().isBlank()) {
throw new BusinessException("fileKey 不能为空"); throw new BusinessException("fileKey 不能为空");
@@ -149,7 +166,9 @@ public class AppearancePatentTaskService {
if (allRows.isEmpty()) { if (allRows.isEmpty()) {
throw new BusinessException("未解析到有效 ASIN 数据"); throw new BusinessException("未解析到有效 ASIN 数据");
} }
long parsedAt = System.nanoTime();
List<AppearancePatentParsedGroupVo> groups = buildParsedGroups(allRows); List<AppearancePatentParsedGroupVo> groups = buildParsedGroups(allRows);
long groupedAt = System.nanoTime();
FileTaskEntity task = new FileTaskEntity(); FileTaskEntity task = new FileTaskEntity();
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr()); task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
@@ -170,11 +189,14 @@ public class AppearancePatentTaskService {
throw new BusinessException("序列化解析结果失败"); throw new BusinessException("序列化解析结果失败");
} }
fileTaskMapper.insert(task); fileTaskMapper.insert(task);
long taskInsertedAt = System.nanoTime();
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles); String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey); String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), sourceFiles, mergedHeaders, groups, allRows); String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), sourceFiles, mergedHeaders, groups, allRows);
long payloadBuiltAt = System.nanoTime();
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload); String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
long payloadStoredAt = System.nanoTime();
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), sourceFiles, parsedPayloadPointer)); task.setResultJson(buildTaskResultJson(request.getAiPrompt(), sourceFiles, parsedPayloadPointer));
task.setUpdatedAt(LocalDateTime.now()); task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task); fileTaskMapper.updateById(task);
@@ -203,6 +225,7 @@ public class AppearancePatentTaskService {
scope.setCreatedAt(LocalDateTime.now()); scope.setCreatedAt(LocalDateTime.now());
scope.setUpdatedAt(LocalDateTime.now()); scope.setUpdatedAt(LocalDateTime.now());
taskScopeStateMapper.insert(scope); taskScopeStateMapper.insert(scope);
long persistedAt = System.nanoTime();
AppearancePatentParseVo vo = new AppearancePatentParseVo(); AppearancePatentParseVo vo = new AppearancePatentParseVo();
vo.setTaskId(task.getId()); vo.setTaskId(task.getId());
@@ -215,6 +238,20 @@ public class AppearancePatentTaskService {
vo.setAiPrompt(normalize(request.getAiPrompt())); vo.setAiPrompt(normalize(request.getAiPrompt()));
vo.setItems(allRows); vo.setItems(allRows);
vo.setGroups(groups); vo.setGroups(groups);
long finishedAt = System.nanoTime();
log.info("[appearance-patent] parse timing taskId={} files={} rows={} groups={} totalMs={} parseMs={} groupMs={} taskInsertMs={} payloadJsonMs={} payloadStoreMs={} persistMs={} responseMs={}",
task.getId(),
sourceFiles.size(),
allRows.size(),
groups.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(parseStartedAt, parsedAt),
elapsedMs(parsedAt, groupedAt),
elapsedMs(groupedAt, taskInsertedAt),
elapsedMs(taskInsertedAt, payloadBuiltAt),
elapsedMs(payloadBuiltAt, payloadStoredAt),
elapsedMs(payloadStoredAt, persistedAt),
elapsedMs(persistedAt, finishedAt));
return vo; return vo;
} }
@@ -491,6 +528,10 @@ public class AppearancePatentTaskService {
.lt(FileTaskEntity::getUpdatedAt, threshold) .lt(FileTaskEntity::getUpdatedAt, threshold)
.last("limit 50")); .last("limit 50"));
for (FileTaskEntity task : tasks) { for (FileTaskEntity task : tasks) {
if (isJavaSideProcessing(task.getId())) {
touchJavaSideTaskActivity(task.getId());
continue;
}
long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId()); long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId());
if (heartbeatMillis > thresholdMillis) { if (heartbeatMillis > thresholdMillis) {
continue; continue;
@@ -510,6 +551,10 @@ public class AppearancePatentTaskService {
.lt(FileTaskEntity::getUpdatedAt, threshold) .lt(FileTaskEntity::getUpdatedAt, threshold)
.last("limit 50")); .last("limit 50"));
for (FileTaskEntity task : tasks) { for (FileTaskEntity task : tasks) {
if (isJavaSideProcessing(task.getId())) {
touchJavaSideTaskActivity(task.getId());
continue;
}
long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId()); long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId());
if (heartbeatMillis > thresholdMillis) { if (heartbeatMillis > thresholdMillis) {
continue; continue;
@@ -738,12 +783,17 @@ public class AppearancePatentTaskService {
} }
String payloadJson = writeJson(rows == null ? List.of() : rows, "结果序列化失败"); String payloadJson = writeJson(rows == null ? List.of() : rows, "结果序列化失败");
payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "appearance patent chunk payload merge failed"); payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "appearance patent chunk payload merge failed");
String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); String oldPayload = chunk.getPayloadJson();
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(chunk.getPayloadJson(), storedPayload); String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
chunk.setPayloadJson(storedPayload); chunk.setPayloadJson(storedPayload);
chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson)); chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson));
chunk.setUpdatedAt(LocalDateTime.now()); chunk.setUpdatedAt(LocalDateTime.now());
taskChunkMapper.updateById(chunk); int updated = taskChunkMapper.updateById(chunk);
if (updated <= 0) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw new IllegalStateException("appearance patent chunk payload update failed");
}
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
} }
private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives, private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives,
@@ -976,7 +1026,7 @@ public class AppearancePatentTaskService {
assembleResultWorkbook(task, result); assembleResultWorkbook(task, result);
} catch (Exception ex) { } catch (Exception ex) {
log.warn("[appearance-patent] assemble result workbook failed taskId={} err={}", task.getId(), ex.getMessage()); log.warn("[appearance-patent] assemble result workbook failed taskId={} err={}", task.getId(), ex.getMessage());
finalError = firstNonBlank(finalError, "生成外观专利检测结果失败"); finalError = firstNonBlank(finalError, "Generate appearance patent result failed");
} }
} }
} }
@@ -1010,39 +1060,478 @@ public class AppearancePatentTaskService {
return false; return false;
} }
public void processResultFileJob(TaskFileJobEntity job) { public boolean processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null || job.getResultId() == null) { if (job == null || job.getTaskId() == null || job.getResultId() == null) {
throw new BusinessException("结果文件任务参数不完整"); throw new BusinessException("result file job arguments are incomplete");
} }
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId()); FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在"); throw new BusinessException("task not found");
} }
FileResultEntity result = fileResultMapper.selectById(job.getResultId()); FileResultEntity result = fileResultMapper.selectById(job.getResultId());
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) { if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
throw new BusinessException("结果记录不存在"); throw new BusinessException("result record not found");
} }
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>() List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, task.getId()) .eq(TaskChunkEntity::getTaskId, task.getId())
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskChunkEntity::getChunkIndex)); .orderByAsc(TaskChunkEntity::getChunkIndex));
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
int cozeWorkUnits = countCozeWorkUnits(chunks, Math.max(1, properties.getCozeBatchSize())); int cozeWorkUnits = countCozeWorkUnits(chunks, Math.max(1, properties.getCozeBatchSize()));
int totalProgressUnits = Math.max(3, cozeWorkUnits + 3); int totalProgressUnits = Math.max(3, cozeWorkUnits + 3);
int[] completedProgressUnits = {0}; saveFileBuildProgress(task, job, totalProgressUnits, 0, "Submitting Coze");
saveFileBuildProgress(task, job, totalProgressUnits, completedProgressUnits[0], "正在请求 Coze"); boolean pendingCoze = submitCozeBatches(task, result, job, chunks, allRowsByBaseId);
Runnable progressHook = () -> { if (pendingCoze) {
taskFileJobService.touchRunning(job.getId()); taskFileJobService.touchRunning(job.getId());
completedProgressUnits[0] = Math.min(totalProgressUnits - 2, completedProgressUnits[0] + 1); touchJavaSideTaskActivity(task.getId());
saveFileBuildProgress(task, job, totalProgressUnits, completedProgressUnits[0], "正在请求 Coze"); saveFileBuildProgress(task, job, totalProgressUnits, 1, "Coze submitted, waiting for result");
}; return false;
applyCozeToPersistedChunks(task, progressHook); }
completedProgressUnits[0] = Math.max(completedProgressUnits[0], totalProgressUnits - 2); completeCozeFileJob(task, result, job, totalProgressUnits, cozeWorkUnits);
saveFileBuildProgress(task, job, totalProgressUnits, completedProgressUnits[0], "正在组装 xlsx"); return true;
}
@Scheduled(fixedDelayString = "${aiimage.appearance-patent.coze-poll-delay-ms:5000}")
public void pollPendingCozeJobs() {
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("appearance-patent:coze-poll", Duration.ofMinutes(1));
if (lockHandle == null) {
return;
}
try (lockHandle) {
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING))
.isNotNull(TaskScopeStateEntity::getCozeExecuteId)
.orderByAsc(TaskScopeStateEntity::getUpdatedAt)
.last("limit 50"));
if (states == null || states.isEmpty()) {
return;
}
log.info("[appearance-patent] coze poll picked pending states count={}", states.size());
for (TaskScopeStateEntity state : states) {
if (state == null || state.getId() == null) {
continue;
}
cozeTaskExecutor.execute(() -> pollPendingCozeState(state.getId()));
}
}
}
private boolean submitCozeBatches(FileTaskEntity task,
FileResultEntity result,
TaskFileJobEntity job,
List<TaskChunkEntity> chunks,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
if (chunks == null || chunks.isEmpty()) {
return countPendingCozeStates(task.getId()) > 0;
}
if (properties.getCozeToken() == null || properties.getCozeToken().isBlank()) {
log.warn("[appearance-patent] coze token not configured, skip async coze taskId={} jobId={}",
task.getId(), job.getId());
return false;
}
String prompt = readAiPrompt(task);
int batchSize = Math.max(1, properties.getCozeBatchSize());
boolean pending = false;
for (TaskChunkEntity chunk : chunks) {
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
if (persistedRows.isEmpty()) {
continue;
}
List<AppearancePatentResultRowDto> unresolvedRows = pickGroupRepresentativesForCoze(persistedRows.values());
if (unresolvedRows.isEmpty()) {
continue;
}
int batchTotal = Math.max(1, (unresolvedRows.size() + batchSize - 1) / batchSize);
int batchIndex = 1;
for (int i = 0; i < unresolvedRows.size(); i += batchSize) {
List<AppearancePatentResultRowDto> batchRows =
unresolvedRows.subList(i, Math.min(i + batchSize, unresolvedRows.size()));
pending |= submitCozeBatch(task, result, job, chunk, batchRows, batchIndex, batchTotal, prompt, allRowsByBaseId);
batchIndex++;
}
}
return pending || countPendingCozeStates(task.getId()) > 0;
}
private boolean submitCozeBatch(FileTaskEntity task,
FileResultEntity result,
TaskFileJobEntity job,
TaskChunkEntity chunk,
List<AppearancePatentResultRowDto> batchRows,
int batchIndex,
int batchTotal,
String prompt,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
if (batchRows == null || batchRows.isEmpty()) {
return false;
}
String batchScopeKey = buildCozeBatchScopeKey(job.getId(), chunk.getScopeHash(), chunk.getChunkIndex(), batchIndex);
String batchScopeHash = DigestUtil.sha256Hex(batchScopeKey);
TaskScopeStateEntity existing = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, task.getId())
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, batchScopeHash)
.last("limit 1"));
if (existing != null) {
return COZE_STATUS_SUBMITTED.equals(existing.getCozeStatus())
|| COZE_STATUS_RUNNING.equals(existing.getCozeStatus());
}
try {
AppearancePatentCozeClient.CozeSubmitResponse submit = cozeClient.submitWorkflow(batchRows, prompt);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List<AppearancePatentResultRowDto> cozeRows = cozeClient.mergeRowsFromDataText(batchRows, submit.immediateData());
mergeCozeRowsIntoChunk(task, chunk.getScopeHash(), chunk.getChunkIndex(), cozeRows, allRowsByBaseId);
return false;
}
if (submit.executeId() == null || submit.executeId().isBlank()) {
mergeCozeRowsIntoChunk(task,
chunk.getScopeHash(),
chunk.getChunkIndex(),
cozeClient.markRowsFailed(batchRows, "Coze async execute_id missing"),
allRowsByBaseId);
return false;
}
saveCozeBatchState(task, result, job, chunk, batchRows, batchScopeKey, batchScopeHash,
batchIndex, batchTotal, submit.executeId());
log.info("[appearance-patent] coze async submitted taskId={} jobId={} chunk={} batch={}/{} executeId={}",
task.getId(), job.getId(), chunk.getChunkIndex(), batchIndex, batchTotal, submit.executeId());
return true;
} catch (Exception ex) {
String message = firstNonBlank(ex.getMessage(), "Coze submit failed");
log.warn("[appearance-patent] coze async submit failed taskId={} jobId={} chunk={} batch={}/{} err={}",
task.getId(), job.getId(), chunk.getChunkIndex(), batchIndex, batchTotal, message);
mergeCozeRowsIntoChunk(task,
chunk.getScopeHash(),
chunk.getChunkIndex(),
cozeClient.markRowsFailed(batchRows, message),
allRowsByBaseId);
return false;
}
}
private void saveCozeBatchState(FileTaskEntity task,
FileResultEntity result,
TaskFileJobEntity job,
TaskChunkEntity chunk,
List<AppearancePatentResultRowDto> batchRows,
String batchScopeKey,
String batchScopeHash,
int batchIndex,
int batchTotal,
String executeId) {
LocalDateTime now = LocalDateTime.now();
CozeBatchContext context = new CozeBatchContext(
job.getId(),
result.getId(),
chunk.getScopeHash(),
chunk.getChunkIndex(),
batchIndex,
batchTotal
);
String batchPayload = writeJson(batchRows, "serialize coze batch payload failed");
String storedBatchPayload = transientPayloadStorageService.storeParsedPayloadFast(
MODULE_TYPE, task.getId(), batchScopeHash, batchPayload, true);
TaskScopeStateEntity state = new TaskScopeStateEntity();
state.setTaskId(task.getId());
state.setModuleType(MODULE_TYPE);
state.setScopeKey(batchScopeKey);
state.setScopeHash(batchScopeHash);
state.setParsedPayloadJson(storedBatchPayload);
state.setStateJson(writeJson(context, "serialize coze batch context failed"));
state.setCozeExecuteId(executeId);
state.setCozeStatus(COZE_STATUS_SUBMITTED);
state.setCozeSubmittedAt(now);
state.setCozeAttemptCount(0);
state.setChunkTotal(batchTotal);
state.setReceivedChunkCount(batchIndex);
state.setCompleted(0);
state.setCreatedAt(now);
state.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(state);
touchJavaSideTaskActivity(task.getId());
} catch (DuplicateKeyException ex) {
transientPayloadStorageService.deletePayloadIfPresent(storedBatchPayload);
log.info("[appearance-patent] duplicate coze batch state ignored taskId={} scope={}",
task.getId(), batchScopeKey);
}
}
private void pollPendingCozeState(Long stateId) {
TaskScopeStateEntity state = taskScopeStateMapper.selectById(stateId);
if (state == null || state.getCozeExecuteId() == null || state.getCozeExecuteId().isBlank()) {
return;
}
if (!(COZE_STATUS_SUBMITTED.equals(state.getCozeStatus()) || COZE_STATUS_RUNNING.equals(state.getCozeStatus()))) {
return;
}
if (!tryClaimCozeStateForPoll(state)) {
return;
}
CozeBatchContext context = readCozeBatchContext(state);
if (context == null || context.jobId() == null || context.resultId() == null) {
markCozeStateTerminal(state, COZE_STATUS_FAILED, "Coze batch context missing");
return;
}
taskFileJobService.touchRunning(context.jobId());
try {
AppearancePatentCozeClient.CozePollResponse poll = cozeClient.pollWorkflow(state.getCozeExecuteId());
if (!poll.hasPayload() && !poll.isFinished() && !isCozeStateTimedOut(state)) {
updateCozeStateRunning(state, null);
return;
}
String failureMessage = poll.isFailed()
? firstNonBlank(poll.failureMessage(), "Coze async workflow failed")
: "";
if (!poll.hasPayload() && failureMessage.isBlank()) {
failureMessage = isCozeStateTimedOut(state)
? "Coze async workflow poll timeout"
: "Coze async workflow completed without output";
}
List<AppearancePatentResultRowDto> batchRows = readCozeBatchRows(state);
List<AppearancePatentResultRowDto> cozeRows = failureMessage.isBlank()
? cozeClient.mergeRowsFromDataText(batchRows, poll.resolvedPayloadText())
: cozeClient.markRowsFailed(batchRows, failureMessage);
FileTaskEntity task = fileTaskMapper.selectById(state.getTaskId());
if (task != null) {
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
mergeCozeRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), cozeRows, allRowsByBaseId);
}
markCozeStateTerminal(state,
failureMessage.isBlank() ? COZE_STATUS_DONE : COZE_STATUS_FAILED,
failureMessage.isBlank() ? null : failureMessage);
maybeFinalizeCozeJob(state.getTaskId(), context);
} catch (Exception ex) {
String message = firstNonBlank(ex.getMessage(), "Coze poll failed");
if (isCozeStateTimedOut(state)) {
List<AppearancePatentResultRowDto> batchRows = readCozeBatchRows(state);
FileTaskEntity task = fileTaskMapper.selectById(state.getTaskId());
if (task != null) {
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
mergeCozeRowsIntoChunk(task,
context.chunkScopeHash(),
context.chunkIndex(),
cozeClient.markRowsFailed(batchRows, message),
allRowsByBaseId);
}
markCozeStateTerminal(state, COZE_STATUS_FAILED, message);
maybeFinalizeCozeJob(state.getTaskId(), context);
return;
}
log.warn("[appearance-patent] coze poll failed taskId={} stateId={} executeId={} err={}",
state.getTaskId(), state.getId(), state.getCozeExecuteId(), message);
updateCozeStateRunning(state, message);
}
}
private void updateCozeStateRunning(TaskScopeStateEntity state, String error) {
int updated = taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING))
.set(TaskScopeStateEntity::getCozeStatus, COZE_STATUS_RUNNING)
.set(TaskScopeStateEntity::getCozeLastPolledAt, LocalDateTime.now())
.set(TaskScopeStateEntity::getCozeAttemptCount, cozeAttemptCount(state) + 1)
.set(TaskScopeStateEntity::getCozeError, error)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
if (updated > 0) {
touchJavaSideTaskActivity(state.getTaskId());
}
}
private boolean tryClaimCozeStateForPoll(TaskScopeStateEntity state) {
if (state == null || state.getId() == null) {
return false;
}
LocalDateTime now = LocalDateTime.now();
long intervalMillis = Math.max(200L, properties.getCozePollIntervalMillis());
if (state.getCozeLastPolledAt() != null
&& Duration.between(state.getCozeLastPolledAt(), now).toMillis() < intervalMillis) {
return false;
}
return taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING))
.and(wrapper -> wrapper
.isNull(TaskScopeStateEntity::getCozeLastPolledAt)
.or()
.le(TaskScopeStateEntity::getCozeLastPolledAt, now.minus(Duration.ofMillis(intervalMillis))))
.set(TaskScopeStateEntity::getCozeStatus, COZE_STATUS_RUNNING)
.set(TaskScopeStateEntity::getCozeLastPolledAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now)) > 0;
}
private void markCozeStateTerminal(TaskScopeStateEntity state, String status, String error) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING))
.set(TaskScopeStateEntity::getCozeStatus, status)
.set(TaskScopeStateEntity::getCozeCompletedAt, LocalDateTime.now())
.set(TaskScopeStateEntity::getCozeLastPolledAt, LocalDateTime.now())
.set(TaskScopeStateEntity::getCozeAttemptCount, cozeAttemptCount(state) + 1)
.set(TaskScopeStateEntity::getCozeError, error)
.set(TaskScopeStateEntity::getCompleted, 1)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
}
private void maybeFinalizeCozeJob(Long taskId, CozeBatchContext context) {
if (taskId == null || context == null || countPendingCozeStates(taskId) > 0) {
return;
}
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("appearance-patent:coze-finalize:" + taskId, Duration.ofMinutes(5));
if (lockHandle == null) {
return;
}
try (lockHandle) {
if (countPendingCozeStates(taskId) > 0) {
return;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
FileResultEntity result = fileResultMapper.selectById(context.resultId());
TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId());
if (task == null || result == null || job == null || "SUCCESS".equals(job.getStatus())) {
return;
}
int cozeWorkUnits = countCompletedCozeStates(taskId);
int totalProgressUnits = Math.max(3, cozeWorkUnits + 3);
completeCozeFileJob(task, result, job, totalProgressUnits, cozeWorkUnits);
taskFileJobService.markSuccess(job, result.getResultFileUrl());
cleanupResultFileJob(job);
log.info("[appearance-patent] coze async job finalized taskId={} jobId={} resultId={} resultFileUrl={}",
taskId, job.getId(), result.getId(), result.getResultFileUrl());
} catch (Exception ex) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId());
if (job != null) {
taskFileJobService.markFailed(job, firstNonBlank(ex.getMessage(), "appearance patent result file build failed"));
}
log.warn("[appearance-patent] coze async finalize failed taskId={} resultId={} err={}",
taskId, context.resultId(), ex.getMessage(), ex);
}
}
private void completeCozeFileJob(FileTaskEntity task,
FileResultEntity result,
TaskFileJobEntity job,
int totalProgressUnits,
int cozeWorkUnits) {
int assembleProgress = Math.max(1, Math.min(totalProgressUnits - 2, cozeWorkUnits));
saveFileBuildProgress(task, job, totalProgressUnits, assembleProgress, "Assembling xlsx");
assembleResultWorkbook(task, result); assembleResultWorkbook(task, result);
completedProgressUnits[0] = totalProgressUnits - 1; saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits - 1, "Uploading result file");
saveFileBuildProgress(task, job, totalProgressUnits, completedProgressUnits[0], "正在上传结果文件");
fileResultMapper.updateById(result); fileResultMapper.updateById(result);
saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "结果文件已生成"); saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "Result file generated");
}
private void mergeCozeRowsIntoChunk(FileTaskEntity task,
String chunkScopeHash,
Integer chunkIndex,
List<AppearancePatentResultRowDto> cozeRows,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
if (task == null || cozeRows == null || cozeRows.isEmpty()) {
return;
}
Map<String, AppearancePatentResultRowDto> mergedRows = new LinkedHashMap<>();
for (AppearancePatentResultRowDto resultRow : cozeRows) {
for (AppearancePatentResultRowDto expandedRow : expandRows(List.of(resultRow), allRowsByBaseId)) {
mergedRows.put(rowKey(expandedRow), expandedRow);
}
}
mergeChunkPayload(task.getId(), chunkScopeHash, chunkIndex, new ArrayList<>(mergedRows.values()));
}
private List<AppearancePatentResultRowDto> readCozeBatchRows(TaskScopeStateEntity state) {
if (state == null || state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) {
return List.of();
}
try {
String payloadJson = transientPayloadStorageService.resolvePayload(
state.getParsedPayloadJson(), "read appearance patent coze batch failed");
JsonNode array = objectMapper.readTree(payloadJson);
if (!array.isArray()) {
return List.of();
}
List<AppearancePatentResultRowDto> rows = new ArrayList<>();
for (JsonNode node : array) {
rows.add(objectMapper.treeToValue(node, AppearancePatentResultRowDto.class));
}
return rows;
} catch (Exception ex) {
log.warn("[appearance-patent] read coze batch failed taskId={} stateId={} err={}",
state.getTaskId(), state.getId(), ex.getMessage());
return List.of();
}
}
private CozeBatchContext readCozeBatchContext(TaskScopeStateEntity state) {
if (state == null || state.getStateJson() == null || state.getStateJson().isBlank()) {
return null;
}
try {
return objectMapper.readValue(state.getStateJson(), CozeBatchContext.class);
} catch (Exception ex) {
log.warn("[appearance-patent] read coze batch context failed taskId={} stateId={} err={}",
state.getTaskId(), state.getId(), ex.getMessage());
return null;
}
}
private boolean isCozeStateTimedOut(TaskScopeStateEntity state) {
if (state == null || state.getCozeSubmittedAt() == null) {
return false;
}
long timeoutMillis = Math.max(10000L, properties.getCozePollTimeoutMillis());
return Duration.between(state.getCozeSubmittedAt(), LocalDateTime.now()).toMillis() >= timeoutMillis;
}
private int cozeAttemptCount(TaskScopeStateEntity state) {
return state == null || state.getCozeAttemptCount() == null ? 0 : state.getCozeAttemptCount();
}
private int countPendingCozeStates(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING)));
return count == null ? 0 : count.intValue();
}
private int countCompletedCozeStates(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_DONE, COZE_STATUS_FAILED)));
return count == null ? 0 : count.intValue();
}
private boolean isJavaSideProcessing(Long taskId) {
return countPendingCozeStates(taskId) > 0
|| taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE) > 0;
}
private void touchJavaSideTaskActivity(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, taskId)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
}
private String buildCozeBatchScopeKey(Long jobId, String chunkScopeHash, Integer chunkIndex, int batchIndex) {
return "coze:job:" + jobId
+ ":chunk:" + (chunkIndex == null ? 0 : chunkIndex)
+ ":" + firstNonBlank(chunkScopeHash, "unknown")
+ ":batch:" + batchIndex;
} }
private int countCozeWorkUnits(List<TaskChunkEntity> chunks, int batchSize) { private int countCozeWorkUnits(List<TaskChunkEntity> chunks, int batchSize) {
@@ -1101,7 +1590,7 @@ public class AppearancePatentTaskService {
private void assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) { private void assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) {
AppearancePatentParsedPayloadDto parsed = readParsedPayload(task); AppearancePatentParsedPayloadDto parsed = readParsedPayload(task);
Map<String, AppearancePatentResultRowDto> resultMap = loadPersistedResultRows(task.getId()); Map<String, AppearancePatentResultRowDto> resultMap = loadPersistedResultRowsWithRetry(task.getId(), parsed.getAllItems().size());
long resolvedRows = parsed.getAllItems().stream() long resolvedRows = parsed.getAllItems().stream()
.filter(row -> findResultRow(row, resultMap) != null || findResultRowByAsin(row.getAsin(), resultMap) != null) .filter(row -> findResultRow(row, resultMap) != null || findResultRowByAsin(row.getAsin(), resultMap) != null)
.count(); .count();
@@ -1110,12 +1599,20 @@ public class AppearancePatentTaskService {
.count(); .count();
log.info("[appearance-patent] assemble workbook taskId={} parsedRows={} resultRows={} resolvedRows={} reasonRows={}", log.info("[appearance-patent] assemble workbook taskId={} parsedRows={} resultRows={} resolvedRows={} reasonRows={}",
task.getId(), parsed.getAllItems().size(), resultMap.size(), resolvedRows, reasonRows); task.getId(), parsed.getAllItems().size(), resultMap.size(), resolvedRows, reasonRows);
if (!parsed.getAllItems().isEmpty() && resultMap.isEmpty()) {
throw new BusinessException("外观专利检测结果为空,请稍后重试生成结果文件");
}
File outputDir = new File(storageProperties.getLocalTempDir(), "appearance-patent-result"); File outputDir = new File(storageProperties.getLocalTempDir(), "appearance-patent-result");
if (!outputDir.exists() && !outputDir.mkdirs()) { if (!outputDir.exists() && !outputDir.mkdirs()) {
throw new BusinessException("创建结果目录失败"); throw new BusinessException("创建结果目录失败");
} }
String filename = safeFileStem(result.getSourceFilename()) + "-result.xlsx"; String filename = safeFileStem(result.getSourceFilename()) + "-result.xlsx";
File xlsx = new File(outputDir, filename); String tempFilename = safeFileStem(result.getSourceFilename())
+ "-" + task.getId()
+ "-" + result.getId()
+ "-" + UUID.randomUUID()
+ "-result.xlsx";
File xlsx = new File(outputDir, tempFilename);
try { try {
writeResultWorkbook(xlsx, parsed, resultMap); writeResultWorkbook(xlsx, parsed, resultMap);
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE); String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
@@ -1143,6 +1640,33 @@ public class AppearancePatentTaskService {
return result; return result;
} }
private Map<String, AppearancePatentResultRowDto> loadPersistedResultRowsWithRetry(Long taskId, int expectedParsedRows) {
Map<String, AppearancePatentResultRowDto> result = loadPersistedResultRows(taskId);
if (expectedParsedRows <= 0 || !result.isEmpty()) {
return result;
}
for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) {
sleepBeforeResultRowsRetry(attempt);
result = loadPersistedResultRows(taskId);
if (!result.isEmpty()) {
log.info("[appearance-patent] result rows recovered after retry taskId={} attempt={} rows={}",
taskId, attempt, result.size());
return result;
}
log.warn("[appearance-patent] result rows still empty after retry taskId={} attempt={}/{}",
taskId, attempt, RESULT_ROWS_READ_RETRY_LIMIT);
}
return result;
}
private void sleepBeforeResultRowsRetry(int attempt) {
try {
Thread.sleep(RESULT_ROWS_READ_RETRY_DELAY_MS * attempt);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
private void writeResultWorkbook(File xlsx, AppearancePatentParsedPayloadDto parsed, Map<String, AppearancePatentResultRowDto> resultMap) { private void writeResultWorkbook(File xlsx, AppearancePatentParsedPayloadDto parsed, Map<String, AppearancePatentResultRowDto> resultMap) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) { try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
Sheet sheet = workbook.createSheet("外观专利检测结果"); Sheet sheet = workbook.createSheet("外观专利检测结果");
@@ -1615,9 +2139,9 @@ public class AppearancePatentTaskService {
payload.setAiPrompt(normalize(aiPrompt)); payload.setAiPrompt(normalize(aiPrompt));
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles); payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
payload.setHeaders(headers == null ? List.of() : headers); payload.setHeaders(headers == null ? List.of() : headers);
payload.setItems(allRows == null ? List.of() : allRows); payload.setItems(List.of());
payload.setGroups(groups == null ? List.of() : groups); payload.setGroups(groups == null ? List.of() : groups);
payload.setAllItems(allRows == null ? List.of() : allRows); payload.setAllItems(List.of());
return writeJson(payload, "保存解析结果失败"); return writeJson(payload, "保存解析结果失败");
} }
@@ -1633,7 +2157,11 @@ public class AppearancePatentTaskService {
} }
private String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) { private String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) {
return transientPayloadStorageService.storeParsedPayload(MODULE_TYPE, taskId, scopeHash, parsedPayloadJson, true); return transientPayloadStorageService.storeParsedPayloadFast(MODULE_TYPE, taskId, scopeHash, parsedPayloadJson, false);
}
private long elapsedMs(long start, long end) {
return (end - start) / 1_000_000L;
} }
private AppearancePatentParsedPayloadDto readParsedPayload(FileTaskEntity task) { private AppearancePatentParsedPayloadDto readParsedPayload(FileTaskEntity task) {
@@ -1642,15 +2170,41 @@ public class AppearancePatentTaskService {
String pointer = root.path("parsedPayloadRef").asText(""); String pointer = root.path("parsedPayloadRef").asText("");
if (!pointer.isBlank()) { if (!pointer.isBlank()) {
String json = transientPayloadStorageService.resolvePayload(pointer, "read appearance patent parsed payload failed"); String json = transientPayloadStorageService.resolvePayload(pointer, "read appearance patent parsed payload failed");
return objectMapper.readValue(json, AppearancePatentParsedPayloadDto.class); return hydrateParsedPayloadRows(objectMapper.readValue(json, AppearancePatentParsedPayloadDto.class));
} }
if (root.path("allItems").isArray()) { if (root.path("allItems").isArray()) {
return objectMapper.treeToValue(root, AppearancePatentParsedPayloadDto.class); return hydrateParsedPayloadRows(objectMapper.treeToValue(root, AppearancePatentParsedPayloadDto.class));
} }
} catch (Exception ex) { } catch (Exception ex) {
throw new BusinessException("读取解析载荷失败"); throw new BusinessException("读取解析载荷失败");
} }
return new AppearancePatentParsedPayloadDto(); return hydrateParsedPayloadRows(new AppearancePatentParsedPayloadDto());
}
private AppearancePatentParsedPayloadDto hydrateParsedPayloadRows(AppearancePatentParsedPayloadDto payload) {
if (payload == null) {
return new AppearancePatentParsedPayloadDto();
}
List<AppearancePatentParsedRowVo> rows = payload.getAllItems();
if (rows == null || rows.isEmpty()) {
rows = payload.getItems();
}
if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) {
rows = payload.getGroups().stream()
.filter(Objects::nonNull)
.flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream())
.toList();
}
if (rows == null) {
rows = List.of();
}
if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) {
payload.setAllItems(rows);
}
if (payload.getItems() == null || payload.getItems().isEmpty()) {
payload.setItems(rows);
}
return payload;
} }
private Map<String, AppearancePatentResultRowDto> readChunkRows(TaskChunkEntity chunk) { private Map<String, AppearancePatentResultRowDto> readChunkRows(TaskChunkEntity chunk) {
@@ -1831,6 +2385,14 @@ public class AppearancePatentTaskService {
String error) { String error) {
} }
private record CozeBatchContext(Long jobId,
Long resultId,
String chunkScopeHash,
Integer chunkIndex,
Integer batchIndex,
Integer batchTotal) {
}
private record ParsedWorkbook(int totalRows, int droppedRows, List<String> headers, List<AppearancePatentParsedRowVo> allRows) { private record ParsedWorkbook(int totalRows, int droppedRows, List<String> headers, List<AppearancePatentParsedRowVo> allRows) {
} }
} }

View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.brand.controller; package com.nanri.aiimage.modules.brand.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultRequest; import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultRequest;
import com.nanri.aiimage.modules.brand.model.dto.BrandTaskCreateRequest; import com.nanri.aiimage.modules.brand.model.dto.BrandTaskCreateRequest;
import com.nanri.aiimage.modules.brand.model.vo.BrandCrawlPayloadVo; import com.nanri.aiimage.modules.brand.model.vo.BrandCrawlPayloadVo;
@@ -17,7 +18,6 @@ import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import jakarta.servlet.http.HttpServletResponse; import jakarta.servlet.http.HttpServletResponse;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -30,8 +30,6 @@ import org.springframework.web.bind.annotation.RestController;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -213,10 +211,8 @@ public class BrandTaskController {
if (filename.isBlank()) { if (filename.isBlank()) {
filename = "brand_task_" + taskId + ".zip"; filename = "brand_task_" + taskId + ".zip";
} }
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/zip"); response.setContentType("application/zip");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
// 后端代理拉取 OSS 文件并流式写出 // 后端代理拉取 OSS 文件并流式写出
try (InputStream in = URI.create(ossUrl).toURL().openStream()) { try (InputStream in = URI.create(ossUrl).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];

View File

@@ -614,6 +614,9 @@ public class BrandTaskService {
if (STATUS_FAILED.equalsIgnoreCase(freshStatus)) { if (STATUS_FAILED.equalsIgnoreCase(freshStatus)) {
throw new BusinessException("任务已结束,不能继续组装结果"); throw new BusinessException("任务已结束,不能继续组装结果");
} }
for (BrandSourceFileDto sourceFile : sourceFiles) {
brandTaskStorageService.refreshFileAggregate(taskId, sourceFile.getFileUrl());
}
Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskStorageService.getAllFileAggregates(taskId); Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskStorageService.getAllFileAggregates(taskId);
log.info("[brand-finalize] taskId={} aggregateFiles={} expectedFiles={} thread={}", log.info("[brand-finalize] taskId={} aggregateFiles={} expectedFiles={} thread={}",
taskId, taskId,
@@ -621,12 +624,12 @@ public class BrandTaskService {
totalCount, totalCount,
Thread.currentThread().getName()); Thread.currentThread().getName());
if (aggregates.size() < totalCount) { if (aggregates.size() < totalCount) {
return; throw new BusinessException("品牌检测结果分片尚未收齐,请稍后重试");
} }
for (BrandSourceFileDto sourceFile : sourceFiles) { for (BrandSourceFileDto sourceFile : sourceFiles) {
BrandFileAggregateCacheDto aggregate = aggregates.get(sourceFile.getFileUrl()); BrandFileAggregateCacheDto aggregate = aggregates.get(sourceFile.getFileUrl());
if (aggregate == null || !Boolean.TRUE.equals(aggregate.getCompleted())) { if (aggregate == null || !Boolean.TRUE.equals(aggregate.getCompleted())) {
return; throw new BusinessException("品牌检测结果分片尚未收齐,请稍后重试");
} }
} }
@@ -1208,6 +1211,9 @@ public class BrandTaskService {
.eq(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING) .eq(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
.lt(BrandCrawlTaskEntity::getUpdatedAt, threshold)); .lt(BrandCrawlTaskEntity::getUpdatedAt, threshold));
for (BrandCrawlTaskEntity task : runningTasks) { for (BrandCrawlTaskEntity task : runningTasks) {
if (tryRecoverCompletedRunningTask(task)) {
continue;
}
Map<Object, Object> progress = brandTaskProgressCacheService.getProgress(task.getId()); Map<Object, Object> progress = brandTaskProgressCacheService.getProgress(task.getId());
if (progress.isEmpty()) { if (progress.isEmpty()) {
failStaleRunningTask(task.getId(), "任务进度缓存已过期,任务已自动失败"); failStaleRunningTask(task.getId(), "任务进度缓存已过期,任务已自动失败");
@@ -1231,6 +1237,34 @@ public class BrandTaskService {
} }
} }
private boolean tryRecoverCompletedRunningTask(BrandCrawlTaskEntity task) {
if (task == null || task.getId() == null) {
return false;
}
try {
List<BrandSourceFileDto> sourceFiles = parseSourceFiles(task.getFilePaths());
if (sourceFiles.isEmpty()) {
return false;
}
List<BrandParsedFileCacheDto> cachedFiles = brandTaskStorageService.getParsedPayload(task.getId());
Map<String, BrandParsedFileCacheDto> cachedByUrl = indexCachedFiles(cachedFiles);
for (BrandSourceFileDto sourceFile : sourceFiles) {
BrandTaskStorageService.ChunkStoreResult result =
brandTaskStorageService.refreshFileAggregate(task.getId(), sourceFile.getFileUrl());
BrandFileAggregateCacheDto aggregate = result.aggregate();
if (aggregate == null || !Boolean.TRUE.equals(aggregate.getCompleted())) {
return false;
}
}
log.info("[brand-stale-check] recover completed running taskId={} files={}", task.getId(), sourceFiles.size());
finalizeTask(task.getId(), normalizeStrategy(task.getStrategy()), sourceFiles, cachedByUrl, sourceFiles.size());
return true;
} catch (Exception ex) {
log.warn("[brand-stale-check] recover completed running taskId={} failed msg={}", task.getId(), ex.getMessage());
return false;
}
}
private void failStaleRunningTask(Long taskId, String message) { private void failStaleRunningTask(Long taskId, String message) {
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>() brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId) .eq(BrandCrawlTaskEntity::getId, taskId)

View File

@@ -278,20 +278,30 @@ public class BrandTaskStorageService {
if (state == null) { if (state == null) {
throw new BusinessException("Brand task aggregate state missing: " + scopeKey); throw new BusinessException("Brand task aggregate state missing: " + scopeKey);
} }
String storedAggregate = transientPayloadStorageService.storeScopePayload(MODULE_TYPE, taskId, scopeHash, aggregateJson, true); String storedAggregate = transientPayloadStorageService.storeScopePayloadVersioned(MODULE_TYPE, taskId, scopeHash, aggregateJson, true);
int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal(); int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal();
int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount(); int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getStateJson(), storedAggregate); int completed = Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0;
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>() int updated = taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId()) .eq(TaskScopeStateEntity::getId, state.getId())
.and(wrapper -> wrapper
.lt(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.or()
.eq(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.le(TaskScopeStateEntity::getCompleted, completed))
.set(TaskScopeStateEntity::getScopeKey, scopeKey) .set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getStateJson, storedAggregate) .set(TaskScopeStateEntity::getStateJson, storedAggregate)
.set(TaskScopeStateEntity::getChunkTotal, chunkTotal) .set(TaskScopeStateEntity::getChunkTotal, chunkTotal)
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount) .set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.set(TaskScopeStateEntity::getCompleted, Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0) .set(TaskScopeStateEntity::getCompleted, completed)
.set(TaskScopeStateEntity::getLastChunkAt, now) .set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getLastError, null) .set(TaskScopeStateEntity::getLastError, null)
.set(TaskScopeStateEntity::getUpdatedAt, now)); .set(TaskScopeStateEntity::getUpdatedAt, now));
if (updated > 0) {
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getStateJson(), storedAggregate);
} else {
transientPayloadStorageService.deletePayloadIfPresent(storedAggregate);
}
} }
private TaskScopeStateEntity getScopeState(Long taskId, String scopeHash) { private TaskScopeStateEntity getScopeState(Long taskId, String scopeHash) {

View File

@@ -14,10 +14,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -28,9 +25,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import java.io.File; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -100,15 +95,14 @@ public class ConvertRunController {
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "返回结果文件流"), @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "返回结果文件流"),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "结果文件不存在") @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "结果文件不存在")
}) })
public ResponseEntity<Resource> download( public ResponseEntity<Void> download(
@Parameter(description = "格式转换结果记录 ID", required = true) @PathVariable Long resultId, @Parameter(description = "格式转换结果记录 ID", required = true) @PathVariable Long resultId,
@Parameter(name = "user_id", description = "当前登录用户 ID", required = true, in = ParameterIn.QUERY) @Parameter(name = "user_id", description = "当前登录用户 ID", required = true, in = ParameterIn.QUERY)
@RequestParam("user_id") Long userId) { @RequestParam("user_id") Long userId) {
File resultFile = convertRunService.getResultFile(resultId, userId); String downloadUrl = convertRunService.getResultDownloadUrl(resultId, userId);
String encodedFilename = URLEncoder.encode(resultFile.getName(), StandardCharsets.UTF_8).replaceAll("\\+", "%20"); return ResponseEntity.status(302)
return ResponseEntity.ok() .location(URI.create(downloadUrl))
.contentType(MediaType.APPLICATION_OCTET_STREAM) .header(HttpHeaders.CACHE_CONTROL, "no-store")
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=UTF-8''" + encodedFilename) .build();
.body(new FileSystemResource(resultFile));
} }
} }

View File

@@ -29,6 +29,7 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
@@ -43,6 +44,7 @@ public class ConvertRunService {
private static final String MODULE_TYPE = "CONVERT"; private static final String MODULE_TYPE = "CONVERT";
private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer"; private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer";
private static final DateTimeFormatter ZIP_NAME_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
private static final List<String> FIVE_COUNTRY_OUTPUT_FILES = List.of( private static final List<String> FIVE_COUNTRY_OUTPUT_FILES = List.of(
"英国.txt", "英国.txt",
"法国.txt", "法国.txt",
@@ -76,10 +78,12 @@ public class ConvertRunService {
fileTaskMapper.insert(task); fileTaskMapper.insert(task);
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank(); boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
boolean archiveMode = folderMode || request.getFiles().size() > 1;
List<ConvertResultItemVo> items = new ArrayList<>(); List<ConvertResultItemVo> items = new ArrayList<>();
int successCount = 0; int successCount = 0;
int failedCount = 0; int failedCount = 0;
List<ConvertArchiveEntry> archiveEntries = new ArrayList<>(); List<ConvertArchiveEntry> archiveEntries = new ArrayList<>();
List<String> successSourceNames = new ArrayList<>();
for (UploadedSourceFileDto sourceFile : request.getFiles()) { for (UploadedSourceFileDto sourceFile : request.getFiles()) {
try { try {
@@ -92,10 +96,11 @@ public class ConvertRunService {
? inputFile.getName() ? inputFile.getName()
: sourceFile.getOriginalFilename(); : sourceFile.getOriginalFilename();
List<GeneratedConvertFile> generatedFiles = generateOutputFiles(inputFile, template); List<GeneratedConvertFile> generatedFiles = generateOutputFiles(inputFile, template);
if (folderMode) { if (archiveMode) {
for (GeneratedConvertFile generatedFile : generatedFiles) { for (GeneratedConvertFile generatedFile : generatedFiles) {
archiveEntries.add(new ConvertArchiveEntry(sourceFile.getRelativePath(), inputName, generatedFile)); archiveEntries.add(new ConvertArchiveEntry(sourceFile.getRelativePath(), inputName, generatedFile));
} }
successSourceNames.add(inputName);
successCount++; successCount++;
continue; continue;
} }
@@ -153,12 +158,13 @@ public class ConvertRunService {
} }
} }
if (folderMode && !archiveEntries.isEmpty()) { if (archiveMode && !archiveEntries.isEmpty()) {
File zipFile = packageFolderConvertResultsAsZip(request.getArchiveName(), archiveEntries); String archiveName = resolveArchiveName(request.getArchiveName(), successSourceNames);
File zipFile = packageFolderConvertResultsAsZip(archiveName, archiveEntries);
String ossObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE); String ossObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE);
// 只存 objectKey // 只存 objectKey
ConvertResultItemVo item = new ConvertResultItemVo(); ConvertResultItemVo item = new ConvertResultItemVo();
item.setSourceFilename(request.getArchiveName()); item.setSourceFilename(folderMode ? request.getArchiveName() : "Current convert task");
item.setOutputFilename(zipFile.getName()); item.setOutputFilename(zipFile.getName());
item.setSuccess(true); item.setSuccess(true);
item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey)); item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey));
@@ -166,7 +172,7 @@ public class ConvertRunService {
FileResultEntity resultEntity = new FileResultEntity(); FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId()); resultEntity.setTaskId(task.getId());
resultEntity.setModuleType(MODULE_TYPE); resultEntity.setModuleType(MODULE_TYPE);
resultEntity.setSourceFilename(request.getArchiveName()); resultEntity.setSourceFilename(item.getSourceFilename());
resultEntity.setResultFilename(zipFile.getName()); resultEntity.setResultFilename(zipFile.getName());
resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey
resultEntity.setResultFileSize(zipFile.length()); resultEntity.setResultFileSize(zipFile.length());
@@ -205,7 +211,7 @@ public class ConvertRunService {
vo.setResultId(entity.getId()); vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getSourceFilename()); vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename()); vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(null); vo.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage()); vo.setError(entity.getErrorMessage());
return vo; return vo;
@@ -221,17 +227,12 @@ public class ConvertRunService {
fileResultMapper.deleteById(resultId); fileResultMapper.deleteById(resultId);
} }
public File getResultFile(Long resultId, Long userId) { public String getResultDownloadUrl(Long resultId, Long userId) {
FileResultEntity entity = fileResultMapper.selectById(resultId); FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("Convert result file does not exist."); throw new BusinessException("Convert result file does not exist.");
} }
return ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl());
File file = new File(entity.getResultFileUrl());
if (!file.exists()) {
throw new BusinessException("Convert result file does not exist.");
}
return file;
} }
private List<GeneratedConvertFile> generateOutputFiles(File inputFile, ConvertTemplateEntity templateEntity) throws IOException { private List<GeneratedConvertFile> generateOutputFiles(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
@@ -410,6 +411,16 @@ public class ConvertRunService {
return zipFile; return zipFile;
} }
private String resolveArchiveName(String archiveName, List<String> successSourceNames) {
if (archiveName != null && !archiveName.isBlank()) {
return FileUtil.mainName(archiveName.trim());
}
if (successSourceNames.size() == 1) {
return FileUtil.mainName(successSourceNames.getFirst());
}
return "convert-" + LocalDateTime.now().format(ZIP_NAME_DATE_FORMATTER);
}
private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) { private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) {
String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/'); String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/');
String relativeDir = normalizedRelativePath.contains("/") String relativeDir = normalizedRelativePath.contains("/")

View File

@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.deletebrand.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandRunRequest; import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandRunRequest;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandSubmitResultRequest; import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandSubmitResultRequest;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandTaskBatchRequest; import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandTaskBatchRequest;
@@ -19,7 +20,6 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -33,7 +33,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@@ -134,11 +133,8 @@ public class DeleteBrandRunController {
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
String asciiFilename = buildAsciiDownloadFilename(filename, taskId);
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + asciiFilename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;
@@ -152,23 +148,37 @@ public class DeleteBrandRunController {
} }
} }
private String buildAsciiDownloadFilename(String filename, Long taskId) { @GetMapping("/results/{resultId}/download")
String sanitized = filename == null ? "" : filename.replaceAll("[^A-Za-z0-9._-]", "_"); @Operation(summary = "Download one delete-brand result")
sanitized = sanitized.replaceAll("_+", "_"); public void downloadResult(
sanitized = sanitized.replaceAll("^[_\\.]+|[_\\.]+$", ""); @PathVariable Long resultId,
if (!sanitized.isBlank() && sanitized.contains(".")) { @Parameter(name = "user_id", description = "褰撳墠鐧诲綍鐢ㄦ埛 ID", required = true, in = ParameterIn.QUERY)
return sanitized; @RequestParam("user_id") Long userId,
jakarta.servlet.http.HttpServletResponse response) {
String url = deleteBrandRunService.resolveResultDownloadUrl(resultId, userId);
String filename = deleteBrandRunService.resolveResultDownloadFilename(resultId, userId);
if (url == null || url.isBlank()) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "缁撴灉鏃犲彲涓嬭浇");
} }
String ext = "xlsx"; if (filename == null || filename.isBlank()) {
if (filename != null) { String rawPath = URI.create(url).getPath();
int dotIndex = filename.lastIndexOf('.'); filename = rawPath == null ? "delete-brand_" + resultId + ".xlsx" : rawPath.substring(rawPath.lastIndexOf('/') + 1);
if (dotIndex >= 0 && dotIndex < filename.length() - 1) { }
String rawExt = filename.substring(dotIndex + 1).replaceAll("[^A-Za-z0-9]", "");
if (!rawExt.isBlank()) { try {
ext = rawExt; response.setContentType("application/octet-stream");
DownloadHeaderUtil.setAttachment(response, filename);
try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536];
int read;
while ((read = in.read(buffer)) != -1) {
response.getOutputStream().write(buffer, 0, read);
} }
response.getOutputStream().flush();
} }
} catch (Exception ex) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "涓嬭浇澶辫触");
} }
return "delete-brand_" + taskId + "." + ext;
} }
} }

View File

@@ -84,6 +84,20 @@ public class DeleteBrandRunService {
private final TaskPressureProperties taskPressureProperties; private final TaskPressureProperties taskPressureProperties;
private final TaskFileJobService taskFileJobService; private final TaskFileJobService taskFileJobService;
private boolean isUsableIndexedMatch(ZiniaoShopMatchResultVo matchResult) {
if (matchResult == null) {
return false;
}
String shopId = matchResult.getShopId();
if (shopId == null || shopId.isBlank()) {
return false;
}
String status = matchResult.getMatchStatus();
return matchResult.isMatched()
|| ZiniaoShopIndexService.MATCH_STATUS_MATCHED.equals(status)
|| ZiniaoShopIndexService.MATCH_STATUS_STALE.equals(status);
}
public DeleteBrandRunVo run(DeleteBrandRunRequest request) { public DeleteBrandRunVo run(DeleteBrandRunRequest request) {
if (request.getUserId() == null || request.getUserId() <= 0) { if (request.getUserId() == null || request.getUserId() <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
@@ -136,8 +150,8 @@ public class DeleteBrandRunService {
ignored -> ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName(), false)); ignored -> ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName(), false));
item.setMatchStatus(matchResult.getMatchStatus()); item.setMatchStatus(matchResult.getMatchStatus());
item.setMatchMessage(matchResult.getMatchMessage()); item.setMatchMessage(matchResult.getMatchMessage());
item.setMatched(matchResult.isMatched()); item.setMatched(isUsableIndexedMatch(matchResult));
if (matchResult.isMatched()) { if (item.isMatched()) {
item.setShopId(matchResult.getShopId()); item.setShopId(matchResult.getShopId());
item.setCompanyName(matchResult.getCompanyName()); item.setCompanyName(matchResult.getCompanyName());
item.setPlatform(matchResult.getPlatform()); item.setPlatform(matchResult.getPlatform());
@@ -236,35 +250,79 @@ public class DeleteBrandRunService {
} }
public DeleteBrandHistoryVo listHistory(Long userId) { public DeleteBrandHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
DeleteBrandHistoryVo vo = new DeleteBrandHistoryVo(); DeleteBrandHistoryVo vo = new DeleteBrandHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() List<FileResultEntity> createdRows = fileResultMapper.selectList(historyResultQuery()
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId) .eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt) .orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100")); .last("limit 500"));
List<FileTaskEntity> recentTouchedTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getStatus,
FileTaskEntity::getUpdatedAt,
FileTaskEntity::getFinishedAt)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getUserId, userId)
.orderByDesc(FileTaskEntity::getUpdatedAt)
.last("limit 200"));
Map<Long, FileResultEntity> entityById = new LinkedHashMap<>();
for (FileResultEntity entity : createdRows) {
if (entity != null && entity.getId() != null) {
entityById.put(entity.getId(), entity);
}
}
for (FileResultEntity entity : selectHistoryResultRowsByTaskIdsInBatches(userId, recentTouchedTasks.stream()
.map(FileTaskEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList())) {
if (entity != null && entity.getId() != null) {
entityById.putIfAbsent(entity.getId(), entity);
}
}
List<FileResultEntity> entities = new ArrayList<>(entityById.values());
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) {
vo.setItems(List.of());
log.info("[delete-brand] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
// 补充信息 // 补充信息
Map<Long, List<DeleteBrandResultItemVo>> itemsByTaskId = new LinkedHashMap<>(); Map<Long, FileTaskEntity> taskById = new LinkedHashMap<>();
Map<Long, String> statusByTaskId = new LinkedHashMap<>(); Map<Long, String> statusByTaskId = new LinkedHashMap<>();
List<Long> taskIds = entities.stream() List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId) .map(FileResultEntity::getTaskId)
.filter(id -> id != null && id > 0) .filter(id -> id != null && id > 0)
.distinct() .distinct()
.toList(); .toList();
for (FileTaskEntity task : recentTouchedTasks) {
if (task != null) {
taskById.put(task.getId(), task);
statusByTaskId.put(task.getId(), task.getStatus());
}
}
if (!taskIds.isEmpty()) { if (!taskIds.isEmpty()) {
List<FileTaskEntity> tasks = selectTasksByIdsInBatches(taskIds); List<FileTaskEntity> tasks = selectTaskHistoryFieldsByIdsInBatches(taskIds);
for (FileTaskEntity task : tasks) { for (FileTaskEntity task : tasks) {
try { if (task != null) {
taskById.put(task.getId(), task);
statusByTaskId.put(task.getId(), task.getStatus()); statusByTaskId.put(task.getId(), task.getStatus());
if (task.getResultJson() != null && !task.getResultJson().isBlank()) {
List<DeleteBrandResultItemVo> taskItems = objectMapper.readValue(task.getResultJson(), new TypeReference<List<DeleteBrandResultItemVo>>() {
});
itemsByTaskId.put(task.getId(), taskItems);
}
} catch (Exception ignored) {
} }
} }
} }
long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
long jobsLoadedAt = System.nanoTime();
entities = sortDeleteBrandHistoryEntities(entities, taskById, jobMap).stream()
.limit(100)
.toList();
vo.setItems(entities.stream().map(entity -> { vo.setItems(entities.stream().map(entity -> {
DeleteBrandResultItemVo item = new DeleteBrandResultItemVo(); DeleteBrandResultItemVo item = new DeleteBrandResultItemVo();
@@ -275,7 +333,7 @@ public class DeleteBrandRunService {
item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename())); item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename()));
item.setOutputFilename(entity.getResultFilename()); item.setOutputFilename(entity.getResultFilename());
item.setDownloadUrl(null); item.setDownloadUrl(null);
attachFileJobState(item, entity); attachFileJobState(item, entity, jobMap.get(entity.getId()));
item.setTotalRows(entity.getRowCount()); item.setTotalRows(entity.getRowCount());
item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
item.setError(entity.getErrorMessage()); item.setError(entity.getErrorMessage());
@@ -284,7 +342,7 @@ public class DeleteBrandRunService {
// 真实补充 // 真实补充
if (entity.getTaskId() != null) { if (entity.getTaskId() != null) {
item.setTaskStatus(statusByTaskId.get(entity.getTaskId())); item.setTaskStatus(statusByTaskId.get(entity.getTaskId()));
List<DeleteBrandResultItemVo> taskItems = itemsByTaskId.get(entity.getTaskId()); List<DeleteBrandResultItemVo> taskItems = null;
if (taskItems != null && entity.getSourceFilename() != null && !entity.getSourceFilename().isBlank()) { if (taskItems != null && entity.getSourceFilename() != null && !entity.getSourceFilename().isBlank()) {
for (DeleteBrandResultItemVo candidate : taskItems) { for (DeleteBrandResultItemVo candidate : taskItems) {
if (candidate == null) { if (candidate == null) {
@@ -320,12 +378,42 @@ public class DeleteBrandRunService {
} }
return item; return item;
}).toList()); }).toList());
long finishedAt = System.nanoTime();
log.info("[delete-brand] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
statusByTaskId.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo; return vo;
} }
private LambdaQueryWrapper<FileResultEntity> historyResultQuery() {
return new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getRowCount,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt);
}
private void attachFileJobState(DeleteBrandResultItemVo item, FileResultEntity entity) { private void attachFileJobState(DeleteBrandResultItemVo item, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); attachFileJobState(item, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()));
}
private void attachFileJobState(DeleteBrandResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) {
item.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); item.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) { if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null); item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
@@ -336,6 +424,51 @@ public class DeleteBrandRunService {
item.setFileError(job.getErrorMessage()); item.setFileError(job.getErrorMessage());
} }
private List<FileResultEntity> sortDeleteBrandHistoryEntities(
List<FileResultEntity> entities,
Map<Long, FileTaskEntity> taskById,
Map<Long, TaskFileJobEntity> jobByResultId
) {
if (entities == null || entities.isEmpty()) {
return List.of();
}
List<FileResultEntity> sorted = new ArrayList<>(entities);
sorted.sort(Comparator
.comparing((FileResultEntity entity) -> resolveHistorySortTime(entity, taskById, jobByResultId),
Comparator.nullsLast(Comparator.reverseOrder()))
.thenComparing(FileResultEntity::getId, Comparator.nullsLast(Comparator.reverseOrder())));
return sorted;
}
private LocalDateTime resolveHistorySortTime(
FileResultEntity entity,
Map<Long, FileTaskEntity> taskById,
Map<Long, TaskFileJobEntity> jobByResultId
) {
if (entity == null) {
return null;
}
TaskFileJobEntity job = jobByResultId == null ? null : jobByResultId.get(entity.getId());
if (job != null) {
if (job.getFinishedAt() != null) {
return job.getFinishedAt();
}
if (job.getUpdatedAt() != null) {
return job.getUpdatedAt();
}
}
FileTaskEntity task = taskById == null ? null : taskById.get(entity.getTaskId());
if (task != null) {
if (task.getFinishedAt() != null) {
return task.getFinishedAt();
}
if (task.getUpdatedAt() != null) {
return task.getUpdatedAt();
}
}
return entity.getCreatedAt();
}
public DeleteBrandTaskDeletionStatusVo getTaskDeletionStatus(Long taskId, Long userId) { public DeleteBrandTaskDeletionStatusVo getTaskDeletionStatus(Long taskId, Long userId) {
FileTaskEntity task = loadTaskForExecution(taskId); FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
@@ -617,6 +750,13 @@ public class DeleteBrandRunService {
} }
java.util.Map<Long, java.util.Map<Object, Object>> progressByTaskId = deleteBrandTaskCacheService.getProgressBatch(normalizedTaskIds); java.util.Map<Long, java.util.Map<Object, Object>> progressByTaskId = deleteBrandTaskCacheService.getProgressBatch(normalizedTaskIds);
Map<Long, List<FileResultEntity>> resultRowsByTaskId = findProgressResultRowsByTaskIds(normalizedTaskIds);
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, resultRowsByTaskId.values().stream()
.flatMap(List::stream)
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
for (Long taskId : normalizedTaskIds) { for (Long taskId : normalizedTaskIds) {
FileTaskEntity task = taskById.get(taskId); FileTaskEntity task = taskById.get(taskId);
@@ -624,7 +764,8 @@ public class DeleteBrandRunService {
vo.getMissingTaskIds().add(taskId); vo.getMissingTaskIds().add(taskId);
continue; continue;
} }
vo.getItems().add(buildTaskProgressDetail(task, progressByTaskId.get(taskId))); vo.getItems().add(buildTaskProgressDetail(task, progressByTaskId.get(taskId),
resultRowsByTaskId.getOrDefault(taskId, List.of()), jobMap));
} }
return vo; return vo;
} }
@@ -634,6 +775,13 @@ public class DeleteBrandRunService {
} }
private DeleteBrandTaskDetailVo buildTaskProgressDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) { private DeleteBrandTaskDetailVo buildTaskProgressDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) {
return buildTaskProgressDetail(task, progress, List.of(), Map.of());
}
private DeleteBrandTaskDetailVo buildTaskProgressDetail(FileTaskEntity task,
java.util.Map<Object, Object> progress,
List<FileResultEntity> resultRows,
Map<Long, TaskFileJobEntity> jobMap) {
task = refreshIndexedMatchesIfNeeded(task); task = refreshIndexedMatchesIfNeeded(task);
DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo(); DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo();
taskVo.setId(task.getId()); taskVo.setId(task.getId());
@@ -657,10 +805,39 @@ public class DeleteBrandRunService {
DeleteBrandTaskDetailVo detail = new DeleteBrandTaskDetailVo(); DeleteBrandTaskDetailVo detail = new DeleteBrandTaskDetailVo();
detail.setTask(taskVo); detail.setTask(taskVo);
detail.setLine_progress(buildLineProgress(task.getId(), progress)); detail.setLine_progress(buildLineProgress(task.getId(), progress));
detail.setFileProgress(java.util.List.of()); List<DeleteBrandResultItemVo> taskItems = buildProgressResultItems(task, resultRows, jobMap);
detail.setItems(taskItems);
detail.setFileProgress(buildFileProgress(task, taskItems, detail.getLine_progress()));
return detail; return detail;
} }
private List<DeleteBrandResultItemVo> buildProgressResultItems(FileTaskEntity task,
List<FileResultEntity> resultRows,
Map<Long, TaskFileJobEntity> jobMap) {
if (task == null || resultRows == null || resultRows.isEmpty()) {
return List.of();
}
return resultRows.stream()
.filter(entity -> entity != null && entity.getId() != null)
.map(entity -> {
DeleteBrandResultItemVo item = new DeleteBrandResultItemVo();
item.setResultId(entity.getId());
item.setTaskId(entity.getTaskId());
item.setFileKey(entity.getSourceFileUrl());
item.setSourceFilename(entity.getSourceFilename());
item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename()));
item.setOutputFilename(entity.getResultFilename());
item.setDownloadUrl(null);
item.setTotalRows(entity.getRowCount());
item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
item.setError(entity.getErrorMessage());
item.setTaskStatus(task.getStatus());
attachFileJobState(item, entity, jobMap == null ? null : jobMap.get(entity.getId()));
return item;
})
.toList();
}
private DeleteBrandTaskDetailVo buildTaskDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) { private DeleteBrandTaskDetailVo buildTaskDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) {
task = refreshIndexedMatchesIfNeeded(task); task = refreshIndexedMatchesIfNeeded(task);
DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo(); DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo();
@@ -837,7 +1014,7 @@ public class DeleteBrandRunService {
String oldOpenStoreUrl = item.getOpenStoreUrl(); String oldOpenStoreUrl = item.getOpenStoreUrl();
String oldMessage = item.getMatchMessage(); String oldMessage = item.getMatchMessage();
item.setMatched(refreshed.isMatched()); item.setMatched(isUsableIndexedMatch(refreshed));
item.setMatchStatus(refreshed.getMatchStatus()); item.setMatchStatus(refreshed.getMatchStatus());
item.setMatchMessage(refreshed.getMatchMessage()); item.setMatchMessage(refreshed.getMatchMessage());
item.setShopId(refreshed.getShopId()); item.setShopId(refreshed.getShopId());
@@ -1011,6 +1188,65 @@ public class DeleteBrandRunService {
return tasks; return tasks;
} }
private List<FileTaskEntity> selectTaskHistoryFieldsByIdsInBatches(List<Long> taskIds) {
List<FileTaskEntity> tasks = new ArrayList<>();
if (taskIds == null || taskIds.isEmpty()) {
return tasks;
}
int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize());
for (int start = 0; start < taskIds.size(); start += batchSize) {
int end = Math.min(start + batchSize, taskIds.size());
tasks.addAll(fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getStatus,
FileTaskEntity::getUpdatedAt,
FileTaskEntity::getFinishedAt)
.in(FileTaskEntity::getId, taskIds.subList(start, end))));
}
return tasks;
}
private List<FileResultEntity> selectHistoryResultRowsByTaskIdsInBatches(Long userId, List<Long> taskIds) {
List<FileResultEntity> rows = new ArrayList<>();
if (userId == null || taskIds == null || taskIds.isEmpty()) {
return rows;
}
int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize());
for (int start = 0; start < taskIds.size(); start += batchSize) {
int end = Math.min(start + batchSize, taskIds.size());
rows.addAll(fileResultMapper.selectList(historyResultQuery()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId)
.in(FileResultEntity::getTaskId, taskIds.subList(start, end))));
}
return rows;
}
private Map<Long, List<FileResultEntity>> findProgressResultRowsByTaskIds(List<Long> taskIds) {
if (taskIds == null || taskIds.isEmpty()) {
return Map.of();
}
Map<Long, List<FileResultEntity>> rowsByTaskId = new LinkedHashMap<>();
int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize());
for (int start = 0; start < taskIds.size(); start += batchSize) {
int end = Math.min(start + batchSize, taskIds.size());
List<FileResultEntity> rows = fileResultMapper.selectList(historyResultQuery()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.in(FileResultEntity::getTaskId, taskIds.subList(start, end))
.orderByAsc(FileResultEntity::getId));
for (FileResultEntity row : rows) {
if (row != null && row.getTaskId() != null) {
rowsByTaskId.computeIfAbsent(row.getTaskId(), ignored -> new ArrayList<>()).add(row);
}
}
}
return rowsByTaskId;
}
private static long elapsedMs(long startInclusive, long endExclusive) {
return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L);
}
@Transactional @Transactional
public void submitResult(Long taskId, DeleteBrandSubmitResultRequest request) { public void submitResult(Long taskId, DeleteBrandSubmitResultRequest request) {
log.info("[DeleteBrand] Received submitResult for taskId: {}, files count: {}", taskId, request == null || request.getFiles() == null ? 0 : request.getFiles().size()); log.info("[DeleteBrand] Received submitResult for taskId: {}, files count: {}", taskId, request == null || request.getFiles() == null ? 0 : request.getFiles().size());
@@ -1027,6 +1263,16 @@ public class DeleteBrandRunService {
throw new BusinessException("任务不存在"); throw new BusinessException("任务不存在");
} }
if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) { if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) {
log.warn("[DeleteBrand] submitResult rejected because task is terminal -> taskId: {}, status: {}, createdAt: {}, updatedAt: {}, finishedAt: {}, error: {}, sourceFiles: {}, successFiles: {}, failedFiles: {}",
task.getId(),
task.getStatus(),
task.getCreatedAt(),
task.getUpdatedAt(),
task.getFinishedAt(),
task.getErrorMessage(),
task.getSourceFileCount(),
task.getSuccessFileCount(),
task.getFailedFileCount());
throw new BusinessException("任务已结束,拒绝继续提交结果"); throw new BusinessException("任务已结束,拒绝继续提交结果");
} }
@@ -1086,6 +1332,7 @@ public class DeleteBrandRunService {
// 如果该文件已完成,立即更新成品计数的 Redis 提示,让后续 tryFinalizeTask 更准确 // 如果该文件已完成,立即更新成品计数的 Redis 提示,让后续 tryFinalizeTask 更准确
if (isFileCompleted(fileDto)) { if (isFileCompleted(fileDto)) {
log.info("[DeleteBrand] File chunk transmission fully reached via chunkTotal for taskId: {}, file: {}", taskId, fileDto.getSourceFilename()); log.info("[DeleteBrand] File chunk transmission fully reached via chunkTotal for taskId: {}, file: {}", taskId, fileDto.getSourceFilename());
enqueueCompletedFileAssembleJob(task, fileIdentity, parsedFile);
shouldTryFinalize = true; shouldTryFinalize = true;
// 这里暂不直接 increment而是标记需要在 tryFinalizeTask 中重新计算 // 这里暂不直接 increment而是标记需要在 tryFinalizeTask 中重新计算
// 为了保险,直接在 tryFinalizeTask 里重新遍历分片状态 // 为了保险,直接在 tryFinalizeTask 里重新遍历分片状态
@@ -1098,6 +1345,55 @@ public class DeleteBrandRunService {
} }
} }
private void enqueueCompletedFileAssembleJob(FileTaskEntity task,
String fileIdentity,
DeleteBrandParsedFileCacheDto parsedFile) {
if (task == null || task.getId() == null || parsedFile == null) {
return;
}
FileResultEntity resultEntity = findResultEntity(task.getId(), fileIdentity, parsedFile);
if (resultEntity == null) {
log.warn("[DeleteBrand] completed file has no result row, skip early assemble -> taskId: {}, file: {}",
task.getId(), parsedFile.getSourceFilename());
return;
}
if (taskFileJobService.hasSuccessfulAssembleJob(task.getId(), MODULE_TYPE, resultEntity.getId())) {
return;
}
String outputFilename = buildResultFilename(parsedFile);
resultEntity.setResultFilename(outputFilename);
resultEntity.setResultContentType(CONTENT_TYPE_XLSX);
resultEntity.setRowCount(parsedFile.getTotalRows());
resultEntity.setSuccess(1);
resultEntity.setErrorMessage(null);
if (resultEntity.getResultFileUrl() == null || resultEntity.getResultFileUrl().isBlank()) {
resultEntity.setResultFileSize(0L);
}
fileResultMapper.updateById(resultEntity);
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity);
log.info("[DeleteBrand] completed file early assemble job enqueued -> taskId: {}, resultId: {}, file: {}",
task.getId(), resultEntity.getId(), parsedFile.getSourceFilename());
}
private FileResultEntity findResultEntity(Long taskId,
String fileIdentity,
DeleteBrandParsedFileCacheDto parsedFile) {
if (taskId == null || taskId <= 0) {
return null;
}
LambdaQueryWrapper<FileResultEntity> wrapper = new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getTaskId, taskId);
if (fileIdentity != null && !fileIdentity.isBlank()) {
wrapper.eq(FileResultEntity::getSourceFileUrl, fileIdentity);
} else if (parsedFile != null && parsedFile.getSourceFilename() != null && !parsedFile.getSourceFilename().isBlank()) {
wrapper.eq(FileResultEntity::getSourceFilename, parsedFile.getSourceFilename());
} else {
return null;
}
return fileResultMapper.selectOne(wrapper.last("limit 1"));
}
public void tryFinalizeTask(Long taskId, boolean fromCompensation) { public void tryFinalizeTask(Long taskId, boolean fromCompensation) {
if (taskId == null || taskId <= 0) { if (taskId == null || taskId <= 0) {
return; return;
@@ -1291,15 +1587,19 @@ public class DeleteBrandRunService {
throw new BusinessException("结果记录不存在: " + parsedFile.getSourceFilename()); throw new BusinessException("结果记录不存在: " + parsedFile.getSourceFilename());
} }
String outputFilename = buildResultFilename(parsedFile); String outputFilename = buildResultFilename(parsedFile);
boolean alreadyAssembled = taskFileJobService.hasSuccessfulAssembleJob(task.getId(), MODULE_TYPE, resultEntity.getId());
resultEntity.setResultFilename(outputFilename); resultEntity.setResultFilename(outputFilename);
resultEntity.setResultFileUrl(null); if (!alreadyAssembled && (resultEntity.getResultFileUrl() == null || resultEntity.getResultFileUrl().isBlank())) {
resultEntity.setResultFileSize(0L); resultEntity.setResultFileSize(0L);
}
resultEntity.setResultContentType(CONTENT_TYPE_XLSX); resultEntity.setResultContentType(CONTENT_TYPE_XLSX);
resultEntity.setRowCount(parsedFile.getTotalRows()); resultEntity.setRowCount(parsedFile.getTotalRows());
resultEntity.setSuccess(1); resultEntity.setSuccess(1);
resultEntity.setErrorMessage(null); resultEntity.setErrorMessage(null);
fileResultMapper.updateById(resultEntity); fileResultMapper.updateById(resultEntity);
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity); if (!alreadyAssembled) {
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity);
}
DeleteBrandResultItemVo item = new DeleteBrandResultItemVo(); DeleteBrandResultItemVo item = new DeleteBrandResultItemVo();
item.setResultId(resultEntity.getId()); item.setResultId(resultEntity.getId());
@@ -1343,6 +1643,7 @@ public class DeleteBrandRunService {
"finished_files", String.valueOf(successCount), "finished_files", String.valueOf(successCount),
"updated_at", String.valueOf(System.currentTimeMillis()) "updated_at", String.valueOf(System.currentTimeMillis())
)); ));
cleanupCompletedTaskDataIfNoPendingFileJobs(task.getId());
} catch (Exception ex) { } catch (Exception ex) {
task.setStatus("FAILED"); task.setStatus("FAILED");
task.setErrorMessage(ex.getMessage()); task.setErrorMessage(ex.getMessage());
@@ -1419,8 +1720,19 @@ public class DeleteBrandRunService {
if (job == null || job.getTaskId() == null) { if (job == null || job.getTaskId() == null) {
return; return;
} }
if (taskFileJobService.countUnfinishedAssembleJobs(job.getTaskId(), MODULE_TYPE) == 0L) { cleanupCompletedTaskDataIfNoPendingFileJobs(job.getTaskId());
deleteBrandTaskStorageService.deleteTaskData(job.getTaskId()); }
private void cleanupCompletedTaskDataIfNoPendingFileJobs(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || (!"SUCCESS".equals(task.getStatus()) && !"FAILED".equals(task.getStatus()))) {
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) == 0L) {
deleteBrandTaskStorageService.deleteTaskData(taskId);
} }
} }
@@ -1611,6 +1923,32 @@ public class DeleteBrandRunService {
return entity == null ? null : entity.getResultFilename(); return entity == null ? null : entity.getResultFilename();
} }
public String resolveResultDownloadUrl(Long resultId, Long userId) {
FileResultEntity entity = findDownloadableResult(resultId, userId);
return entity == null || entity.getResultFileUrl() == null ? null : ossStorageService.getPublicUrl(entity.getResultFileUrl());
}
public String resolveResultDownloadFilename(Long resultId, Long userId) {
FileResultEntity entity = findDownloadableResult(resultId, userId);
return entity == null ? null : entity.getResultFilename();
}
private FileResultEntity findDownloadableResult(Long resultId, Long userId) {
if (resultId == null || resultId <= 0 || userId == null || userId <= 0) {
return null;
}
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null
|| !MODULE_TYPE.equals(entity.getModuleType())
|| entity.getUserId() == null
|| !entity.getUserId().equals(userId)
|| entity.getResultFileUrl() == null
|| entity.getResultFileUrl().isBlank()) {
return null;
}
return entity;
}
private FileResultEntity findLatestSuccessfulResult(Long taskId) { private FileResultEntity findLatestSuccessfulResult(Long taskId) {
return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)

View File

@@ -134,15 +134,17 @@ public class DeleteBrandStaleTaskService {
} }
} }
boolean hasStartedProgress = lastHeartbeatAt > 0L; boolean hasStartedProgress = lastHeartbeatAt > 0L;
int completedScopeCount = 0;
if (!hasStartedProgress) { if (!hasStartedProgress) {
hasStartedProgress = deleteBrandTaskStorageService.countCompletedScopes(task.getId()) > 0 // Parsed payload is created by Java during the parse step. Only result chunks mean Python has started uploading.
|| deleteBrandTaskStorageService.countParsedPayloadScopes(task.getId()) > 0; completedScopeCount = deleteBrandTaskStorageService.countCompletedScopes(task.getId());
hasStartedProgress = completedScopeCount > 0;
} }
if (lastHeartbeatAt > 0L && nowMillis - lastHeartbeatAt < staleTimeoutMillis) { if (lastHeartbeatAt > 0L && nowMillis - lastHeartbeatAt < staleTimeoutMillis) {
continue; continue;
} }
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
continue; continue;
} }
@@ -151,12 +153,24 @@ public class DeleteBrandStaleTaskService {
FileTaskEntity refreshed = fileTaskMapper.selectById(task.getId()); FileTaskEntity refreshed = fileTaskMapper.selectById(task.getId());
if (refreshed != null && !"RUNNING".equals(refreshed.getStatus())) { if (refreshed != null && !"RUNNING".equals(refreshed.getStatus())) {
deleteBrandTaskCacheService.saveTaskCache(refreshed); deleteBrandTaskCacheService.saveTaskCache(refreshed);
log.info("[stale-check] delete-brand finalized before timeout-fail taskId={} status={} updatedAt={} finishedAt={}",
refreshed.getId(), refreshed.getStatus(), refreshed.getUpdatedAt(), refreshed.getFinishedAt());
continue; continue;
} }
} catch (Exception ex) { } catch (Exception ex) {
log.warn("[stale-check] delete-brand finalize threw taskId={} msg={}", task.getId(), ex.getMessage()); log.warn("[stale-check] delete-brand finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
} }
log.warn("[stale-check] delete-brand failing stale task -> taskId={} updatedAt={} createdAt={} lastHeartbeatAt={} timeoutMinutes={} initialTimeoutMinutes={} completedScopes={} hasStartedProgress={} initialThreshold={}",
task.getId(),
task.getUpdatedAt(),
task.getCreatedAt(),
lastHeartbeatAt,
minutes,
initialMinutes,
completedScopeCount,
hasStartedProgress,
initialThreshold);
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>() int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId()) .eq(FileTaskEntity::getId, task.getId())
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND) .eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
@@ -168,6 +182,7 @@ public class DeleteBrandStaleTaskService {
if (updated > 0) { if (updated > 0) {
deleteBrandTaskCacheService.delete(task.getId()); deleteBrandTaskCacheService.delete(task.getId());
log.warn("[stale-check] delete-brand failed taskId={} reason=timeout", task.getId());
} }
} }
} }
@@ -204,11 +219,11 @@ public class DeleteBrandStaleTaskService {
task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt()); task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt());
continue; continue;
} }
boolean hasStartedProgress = lastPayloadHeartbeatMillis > 0L || hasFinishedRows; boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) { if (!hasStartedProgress) {
hasStartedProgress = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId()); hasStartedProgress = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId());
} }
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
stats.skippedTaskCount++; stats.skippedTaskCount++;
log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}", log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}",
task.getId(), task.getCreatedAt(), initialThreshold); task.getId(), task.getCreatedAt(), initialThreshold);
@@ -271,11 +286,11 @@ public class DeleteBrandStaleTaskService {
stats.skippedTaskCount++; stats.skippedTaskCount++;
continue; continue;
} }
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows; boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) { if (!hasStartedProgress) {
hasStartedProgress = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId()); hasStartedProgress = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId());
} }
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
stats.skippedTaskCount++; stats.skippedTaskCount++;
continue; continue;
} }
@@ -339,11 +354,11 @@ public class DeleteBrandStaleTaskService {
task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt()); task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt());
continue; continue;
} }
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows; boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) { if (!hasStartedProgress) {
hasStartedProgress = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId()); hasStartedProgress = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
} }
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
stats.skippedTaskCount++; stats.skippedTaskCount++;
log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}", log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}",
task.getId(), task.getCreatedAt(), initialThreshold); task.getId(), task.getCreatedAt(), initialThreshold);
@@ -405,11 +420,11 @@ public class DeleteBrandStaleTaskService {
stats.skippedTaskCount++; stats.skippedTaskCount++;
continue; continue;
} }
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows; boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) { if (!hasStartedProgress) {
hasStartedProgress = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId()); hasStartedProgress = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId());
} }
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
stats.skippedTaskCount++; stats.skippedTaskCount++;
continue; continue;
} }
@@ -440,6 +455,14 @@ public class DeleteBrandStaleTaskService {
return stats; return stats;
} }
private boolean isWithinInitialGrace(FileTaskEntity task, LocalDateTime initialThreshold) {
if (task == null || initialThreshold == null) {
return false;
}
LocalDateTime baseline = task.getUpdatedAt() != null ? task.getUpdatedAt() : task.getCreatedAt();
return baseline != null && baseline.isAfter(initialThreshold);
}
@Scheduled(cron = "${aiimage.delete-brand-progress.finalize-check-cron:30 */2 * * * *}") @Scheduled(cron = "${aiimage.delete-brand-progress.finalize-check-cron:30 */2 * * * *}")
public void finalizeCompletedRunningTasks() { public void finalizeCompletedRunningTasks() {
DistributedJobLockService.LockHandle lockHandle = DistributedJobLockService.LockHandle lockHandle =

View File

@@ -19,6 +19,7 @@ import java.util.regex.Pattern;
public class LocalTempCleanupService { public class LocalTempCleanupService {
private static final Pattern ROOT_TEMP_FILE_PATTERN = Pattern.compile("^[a-fA-F0-9]{32}(\\.[^.]+)?$"); private static final Pattern ROOT_TEMP_FILE_PATTERN = Pattern.compile("^[a-fA-F0-9]{32}(\\.[^.]+)?$");
private static final String TRANSIENT_PAYLOAD_DIR_NAME = "transient-payload";
private static final List<String> RESULT_DIR_NAMES = List.of( private static final List<String> RESULT_DIR_NAMES = List.of(
"dedupe-result", "dedupe-result",
"convert-result", "convert-result",
@@ -39,11 +40,14 @@ public class LocalTempCleanupService {
return; return;
} }
Instant sourceExpireBefore = Instant.now().minus(storageProperties.getSourceRetentionHours(), ChronoUnit.HOURS); Instant now = Instant.now();
Instant resultExpireBefore = Instant.now().minus(storageProperties.getResultRetentionHours(), ChronoUnit.HOURS); Instant sourceExpireBefore = now.minus(storageProperties.getSourceRetentionHours(), ChronoUnit.HOURS);
Instant resultExpireBefore = now.minus(storageProperties.getResultRetentionHours(), ChronoUnit.HOURS);
Instant transientPayloadExpireBefore = now.minus(storageProperties.getTransientPayloadRetentionHours(), ChronoUnit.HOURS);
int deletedSourceCount = 0; int deletedSourceCount = 0;
int deletedResultCount = 0; int deletedResultCount = 0;
int deletedTransientPayloadCount = 0;
File[] children = tempDir.listFiles(); File[] children = tempDir.listFiles();
if (children == null || children.length == 0) { if (children == null || children.length == 0) {
return; return;
@@ -60,14 +64,20 @@ public class LocalTempCleanupService {
if (child.isDirectory() && RESULT_DIR_NAMES.contains(child.getName())) { if (child.isDirectory() && RESULT_DIR_NAMES.contains(child.getName())) {
deletedResultCount += deleteExpiredChildrenRecursively(child, resultExpireBefore); deletedResultCount += deleteExpiredChildrenRecursively(child, resultExpireBefore);
deleteEmptyDirectories(child, tempDir); deleteEmptyDirectories(child, tempDir);
continue;
}
if (child.isDirectory() && TRANSIENT_PAYLOAD_DIR_NAME.equals(child.getName())) {
deletedTransientPayloadCount += deleteExpiredChildrenRecursively(child, transientPayloadExpireBefore);
deleteEmptyDirectories(child, tempDir);
} }
} catch (Exception ex) { } catch (Exception ex) {
log.warn("清理本地临时文件失败: path={}", child.getAbsolutePath(), ex); log.warn("local temp cleanup failed: path={}", child.getAbsolutePath(), ex);
} }
} }
if (deletedSourceCount > 0 || deletedResultCount > 0) { if (deletedSourceCount > 0 || deletedResultCount > 0 || deletedTransientPayloadCount > 0) {
log.info("本地临时目录清理完成: sourceDeleted={}, resultDeleted={}, tempDir={}", deletedSourceCount, deletedResultCount, tempDir.getAbsolutePath()); log.info("local temp cleanup completed: sourceDeleted={}, resultDeleted={}, transientPayloadDeleted={}, tempDir={}",
deletedSourceCount, deletedResultCount, deletedTransientPayloadCount, tempDir.getAbsolutePath());
} }
} }

View File

@@ -32,6 +32,10 @@ public class RustfsObjectStorageService {
} }
public String uploadText(String objectKey, String content) { public String uploadText(String objectKey, String content) {
return uploadText(objectKey, content, true);
}
public String uploadText(String objectKey, String content, boolean verifyAfterUpload) {
if (!isConfigured()) { if (!isConfigured()) {
throw new IllegalStateException("transient storage is not configured"); throw new IllegalStateException("transient storage is not configured");
} }
@@ -46,7 +50,9 @@ public class RustfsObjectStorageService {
.stream(stream, bytes.length, -1) .stream(stream, bytes.length, -1)
.contentType("application/json") .contentType("application/json")
.build()); .build());
verifyObjectVisible(objectKey); if (verifyAfterUpload) {
verifyObjectVisible(objectKey);
}
return objectKey; return objectKey;
} catch (Exception ex) { } catch (Exception ex) {
last = ex; last = ex;

View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.patroldelete.controller; package com.nanri.aiimage.modules.patroldelete.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteConditionAddRequest; import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteConditionAddRequest;
import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteCreateTaskRequest; import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteCreateTaskRequest;
import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteSubmitResultRequest; import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteSubmitResultRequest;
@@ -25,7 +26,6 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -39,7 +39,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
@@ -358,10 +357,8 @@ public class PatrolDeleteController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;

View File

@@ -127,6 +127,11 @@ public class PatrolDeleteTaskCacheService {
} }
public void deleteTaskCache(Long taskId) { public void deleteTaskCache(Long taskId) {
evictTaskCacheOnly(taskId);
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
public void evictTaskCacheOnly(Long taskId) {
if (taskId == null || taskId <= 0) { if (taskId == null || taskId <= 0) {
return; return;
} }
@@ -137,7 +142,6 @@ public class PatrolDeleteTaskCacheService {
} catch (Exception ex) { } catch (Exception ex) {
log.warn("[patrol-delete-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage()); log.warn("[patrol-delete-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
} }
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
} }
public void saveTaskCache(FileTaskEntity task) { public void saveTaskCache(FileTaskEntity task) {

View File

@@ -28,6 +28,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -66,6 +67,7 @@ public class PatrolDeleteTaskService {
private final TaskFileJobService taskFileJobService; private final TaskFileJobService taskFileJobService;
private final TaskResultItemService taskResultItemService; private final TaskResultItemService taskResultItemService;
private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private FileTaskEntity loadTaskForExecution(Long taskId) { private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId)); Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -135,32 +137,58 @@ public class PatrolDeleteTaskService {
} }
public PatrolDeleteHistoryVo listHistory(Long userId) { public PatrolDeleteHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
validateUserId(userId); validateUserId(userId);
PatrolDeleteHistoryVo vo = new PatrolDeleteHistoryVo(); PatrolDeleteHistoryVo vo = new PatrolDeleteHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId) .eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt) .orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100")); .last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) { if (entities.isEmpty()) {
vo.setItems(List.of()); vo.setItems(List.of());
log.info("[patrol-delete] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo; return vo;
} }
Map<Long, FileTaskEntity> taskMap = loadTaskMap(entities); Map<Long, FileTaskEntity> taskMap = loadHistoryTaskMap(entities);
Map<Long, Map<Long, PatrolDeleteResultItemVo>> snapshotMap = buildSnapshotMap(taskMap); long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream() Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId) .map(FileResultEntity::getId)
.filter(id -> id != null && id > 0) .filter(id -> id != null && id > 0)
.distinct() .distinct()
.toList()); .toList());
long jobsLoadedAt = System.nanoTime();
List<PatrolDeleteResultItemVo> items = new ArrayList<>(); List<PatrolDeleteResultItemVo> items = new ArrayList<>();
for (FileResultEntity entity : entities) { for (FileResultEntity entity : entities) {
FileTaskEntity task = taskMap.get(entity.getTaskId()); FileTaskEntity task = taskMap.get(entity.getTaskId());
PatrolDeleteResultItemVo snapshot = snapshotMap.getOrDefault(entity.getTaskId(), Map.of()).get(entity.getId()); items.add(toHistoryItem(entity, task, null, jobMap.get(entity.getId())));
items.add(toHistoryItem(entity, task, snapshot, jobMap.get(entity.getId())));
} }
vo.setItems(items); vo.setItems(items);
long finishedAt = System.nanoTime();
log.info("[patrol-delete] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
taskMap.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo; return vo;
} }
@@ -417,17 +445,32 @@ public class PatrolDeleteTaskService {
@Transactional @Transactional
public void deleteTask(Long taskId, Long userId) { public void deleteTask(Long taskId, Long userId) {
long startedAt = System.nanoTime();
validateUserId(userId); validateUserId(userId);
FileTaskEntity task = loadTaskForExecution(taskId); FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在"); throw new BusinessException("任务不存在");
} }
long taskLoadedAt = System.nanoTime();
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>() fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId) .eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)); .eq(FileResultEntity::getModuleType, MODULE_TYPE));
cleanupTaskAuxiliaryData(taskId); long resultsDeletedAt = System.nanoTime();
cleanupTaskAuxiliaryDataFast(taskId);
long auxiliaryDeletedAt = System.nanoTime();
fileTaskMapper.deleteById(taskId); fileTaskMapper.deleteById(taskId);
taskCacheService.deleteTaskCache(taskId); long taskDeletedAt = System.nanoTime();
taskCacheService.evictTaskCacheOnly(taskId);
long finishedAt = System.nanoTime();
log.info("[patrol-delete] delete task timing taskId={} userId={} totalMs={} loadMs={} resultDeleteMs={} auxiliaryDeleteMs={} taskDeleteMs={} cacheMs={}",
taskId,
userId,
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, taskLoadedAt),
elapsedMs(taskLoadedAt, resultsDeletedAt),
elapsedMs(resultsDeletedAt, auxiliaryDeletedAt),
elapsedMs(auxiliaryDeletedAt, taskDeletedAt),
elapsedMs(taskDeletedAt, finishedAt));
} }
@Transactional @Transactional
@@ -439,7 +482,7 @@ public class PatrolDeleteTaskService {
} }
Long taskId = entity.getTaskId(); Long taskId = entity.getTaskId();
fileResultMapper.deleteById(resultId); fileResultMapper.deleteById(resultId);
cleanupResultAuxiliaryData(taskId, resultId); cleanupResultAuxiliaryDataFast(taskId, resultId);
reconcileTaskAfterResultRemoval(taskId); reconcileTaskAfterResultRemoval(taskId);
} }
@@ -453,13 +496,12 @@ public class PatrolDeleteTaskService {
} }
List<FileResultEntity> rows = listTaskRows(taskId); List<FileResultEntity> rows = listTaskRows(taskId);
if (rows.isEmpty()) { if (rows.isEmpty()) {
cleanupTaskAuxiliaryData(taskId); cleanupTaskAuxiliaryDataFast(taskId);
fileTaskMapper.deleteById(taskId); fileTaskMapper.deleteById(taskId);
taskCacheService.deleteTaskCache(taskId); taskCacheService.evictTaskCacheOnly(taskId);
return; return;
} }
updateTaskStatusFromRows(task, rows); updateTaskStatusFromRows(task, rows);
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
fileTaskMapper.updateById(task); fileTaskMapper.updateById(task);
taskCacheService.saveTaskCache(task); taskCacheService.saveTaskCache(task);
} }
@@ -470,11 +512,23 @@ public class PatrolDeleteTaskService {
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
} }
private void cleanupTaskAuxiliaryDataFast(Long taskId) {
taskResultItemService.deleteTaskItemsRowsOnly(taskId, MODULE_TYPE);
taskScopePayloadStorageService.deleteTaskScopePayloadRowsOnly(taskId, MODULE_TYPE);
taskProgressSnapshotService.delete(taskId, MODULE_TYPE);
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
}
private void cleanupResultAuxiliaryData(Long taskId, Long resultId) { private void cleanupResultAuxiliaryData(Long taskId, Long resultId) {
taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId); taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId);
taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId); taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId);
} }
private void cleanupResultAuxiliaryDataFast(Long taskId, Long resultId) {
taskResultItemService.deleteResultItemRowsOnly(taskId, MODULE_TYPE, resultId);
taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId);
}
private long countTasks(Long userId, List<String> statuses) { private long countTasks(Long userId, List<String> statuses) {
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>() Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE) .eq(FileTaskEntity::getModuleType, MODULE_TYPE)
@@ -524,6 +578,35 @@ public class PatrolDeleteTaskService {
return taskMap; return taskMap;
} }
private Map<Long, FileTaskEntity> loadHistoryTaskMap(List<FileResultEntity> entities) {
List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId)
.filter(id -> id != null && id > 0)
.distinct()
.toList();
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
if (taskIds.isEmpty()) {
return taskMap;
}
int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize());
for (int start = 0; start < taskIds.size(); start += batchSize) {
int end = Math.min(start + batchSize, taskIds.size());
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getModuleType,
FileTaskEntity::getStatus,
FileTaskEntity::getFinishedAt)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.in(FileTaskEntity::getId, taskIds.subList(start, end)));
for (FileTaskEntity task : tasks) {
if (task != null && task.getId() != null) {
taskMap.put(task.getId(), task);
}
}
}
return taskMap;
}
private Map<Long, Map<Long, PatrolDeleteResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) { private Map<Long, Map<Long, PatrolDeleteResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
Map<Long, Map<Long, PatrolDeleteResultItemVo>> out = new LinkedHashMap<>(); Map<Long, Map<Long, PatrolDeleteResultItemVo>> out = new LinkedHashMap<>();
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) { for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
@@ -925,6 +1008,10 @@ public class PatrolDeleteTaskService {
return Integer.valueOf(RESULT_SUCCESS).equals(dbValue); return Integer.valueOf(RESULT_SUCCESS).equals(dbValue);
} }
private long elapsedMs(long start, long end) {
return (end - start) / 1_000_000L;
}
private List<PatrolDeleteResultItemVo> parseTaskSnapshots(String json) { private List<PatrolDeleteResultItemVo> parseTaskSnapshots(String json) {
if (blank(json)) { if (blank(json)) {
return new ArrayList<>(); return new ArrayList<>();

View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.pricetrack.controller; package com.nanri.aiimage.modules.pricetrack.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCandidateAddRequest; import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCandidateAddRequest;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCountryPreferenceSaveRequest; import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCreateTaskRequest; import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCreateTaskRequest;
@@ -28,7 +29,6 @@ import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -43,7 +43,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
@@ -244,10 +243,8 @@ public class PriceTrackController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;

View File

@@ -166,14 +166,33 @@ public class PriceTrackTaskService {
} }
public PriceTrackHistoryVo listHistory(Long userId) { public PriceTrackHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
PriceTrackHistoryVo vo = new PriceTrackHistoryVo(); PriceTrackHistoryVo vo = new PriceTrackHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList( List<FileResultEntity> entities = fileResultMapper.selectList(
new LambdaQueryWrapper<FileResultEntity>() new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId) .eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt) .orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100")); .last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) {
vo.setItems(List.of());
log.info("[price-track] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
Map<Long, String> statusByTaskId = new LinkedHashMap<>(); Map<Long, String> statusByTaskId = new LinkedHashMap<>();
List<Long> taskIds = entities.stream() List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId) .map(FileResultEntity::getTaskId)
@@ -186,11 +205,29 @@ public class PriceTrackTaskService {
if (t != null) statusByTaskId.put(t.getId(), t.getStatus()); if (t != null) statusByTaskId.put(t.getId(), t.getStatus());
} }
} }
long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
long jobsLoadedAt = System.nanoTime();
List<PriceTrackResultItemVo> items = new ArrayList<>(); List<PriceTrackResultItemVo> items = new ArrayList<>();
for (FileResultEntity entity : entities) { for (FileResultEntity entity : entities) {
items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()))); items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()), jobMap.get(entity.getId()), false));
} }
vo.setItems(items); vo.setItems(items);
long finishedAt = System.nanoTime();
log.info("[price-track] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
statusByTaskId.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo; return vo;
} }
@@ -952,6 +989,10 @@ public class PriceTrackTaskService {
} }
private PriceTrackResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) { private PriceTrackResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) {
return toHistoryItemVo(entity, taskStatus, null, true);
}
private PriceTrackResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus, TaskFileJobEntity job, boolean mergeRequestFields) {
PriceTrackResultItemVo vo = new PriceTrackResultItemVo(); PriceTrackResultItemVo vo = new PriceTrackResultItemVo();
vo.setResultId(entity.getId()); vo.setResultId(entity.getId());
vo.setTaskId(entity.getTaskId()); vo.setTaskId(entity.getTaskId());
@@ -963,16 +1004,19 @@ public class PriceTrackTaskService {
vo.setError(entity.getErrorMessage()); vo.setError(entity.getErrorMessage());
vo.setOutputFilename(entity.getResultFilename()); vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(null); vo.setDownloadUrl(null);
attachFileJobState(vo, entity); attachFileJobState(vo, entity, job);
vo.setMatched(true); vo.setMatched(true);
if (entity.getTaskId() != null) { if (mergeRequestFields && entity.getTaskId() != null) {
mergeQueueFieldsFromRequest(vo, entity.getTaskId()); mergeQueueFieldsFromRequest(vo, entity.getTaskId());
} }
return vo; return vo;
} }
private void attachFileJobState(PriceTrackResultItemVo vo, FileResultEntity entity) { private void attachFileJobState(PriceTrackResultItemVo vo, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); attachFileJobState(vo, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()));
}
private void attachFileJobState(PriceTrackResultItemVo vo, FileResultEntity entity, TaskFileJobEntity job) {
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) { if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
@@ -1226,6 +1270,10 @@ public class PriceTrackTaskService {
return t == null ? null : t.toString(); return t == null ? null : t.toString();
} }
private static long elapsedMs(long startInclusive, long endExclusive) {
return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L);
}
private void updateTaskStatusFromLatestRows(FileTaskEntity task, List<FileResultEntity> latest) { private void updateTaskStatusFromLatestRows(FileTaskEntity task, List<FileResultEntity> latest) {
String oldStatus = task.getStatus(); String oldStatus = task.getStatus();
int ok = 0; int ok = 0;

View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.productrisk.controller; package com.nanri.aiimage.modules.productrisk.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCreateTaskRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCreateTaskRequest;
@@ -26,7 +27,6 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -41,7 +41,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
@@ -250,10 +249,8 @@ public class ProductRiskResolveController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;

View File

@@ -165,15 +165,34 @@ public class ProductRiskTaskService {
} }
public ProductRiskHistoryVo listHistory(Long userId) { public ProductRiskHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
if (userId == null || userId <= 0) { if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
} }
ProductRiskHistoryVo vo = new ProductRiskHistoryVo(); ProductRiskHistoryVo vo = new ProductRiskHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId) .eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt) .orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100")); .last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) {
vo.setItems(List.of());
log.info("[product-risk] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
Map<Long, String> statusByTaskId = new LinkedHashMap<>(); Map<Long, String> statusByTaskId = new LinkedHashMap<>();
List<Long> taskIds = entities.stream() List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId) .map(FileResultEntity::getTaskId)
@@ -188,11 +207,29 @@ public class ProductRiskTaskService {
} }
} }
} }
long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
long jobsLoadedAt = System.nanoTime();
List<ProductRiskResultItemVo> items = new ArrayList<>(); List<ProductRiskResultItemVo> items = new ArrayList<>();
for (FileResultEntity entity : entities) { for (FileResultEntity entity : entities) {
items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()))); items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()), jobMap.get(entity.getId()), false));
} }
vo.setItems(items); vo.setItems(items);
long finishedAt = System.nanoTime();
log.info("[product-risk] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
statusByTaskId.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo; return vo;
} }
@@ -967,6 +1004,10 @@ public class ProductRiskTaskService {
} }
private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) { private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) {
return toHistoryItemVo(entity, taskStatus, null, true);
}
private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus, TaskFileJobEntity job, boolean mergeRequestFields) {
ProductRiskResultItemVo vo = new ProductRiskResultItemVo(); ProductRiskResultItemVo vo = new ProductRiskResultItemVo();
vo.setResultId(entity.getId()); vo.setResultId(entity.getId());
vo.setTaskId(entity.getTaskId()); vo.setTaskId(entity.getTaskId());
@@ -978,16 +1019,19 @@ public class ProductRiskTaskService {
vo.setError(entity.getErrorMessage()); vo.setError(entity.getErrorMessage());
vo.setOutputFilename(entity.getResultFilename()); vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(null); vo.setDownloadUrl(null);
attachFileJobState(vo, entity); attachFileJobState(vo, entity, job);
vo.setMatched(true); vo.setMatched(true);
if (entity.getTaskId() != null) { if (mergeRequestFields && entity.getTaskId() != null) {
mergeQueueFieldsFromRequest(vo, entity.getTaskId()); mergeQueueFieldsFromRequest(vo, entity.getTaskId());
} }
return vo; return vo;
} }
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) { private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); attachFileJobState(vo, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()));
}
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity, TaskFileJobEntity job) {
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) { if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
@@ -1031,6 +1075,10 @@ public class ProductRiskTaskService {
return t == null ? null : t.toString(); return t == null ? null : t.toString();
} }
private static long elapsedMs(long startInclusive, long endExclusive) {
return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L);
}
private List<ProductRiskShopQueueItemVo> dedupeQueueItems(List<ProductRiskShopQueueItemVo> raw) { private List<ProductRiskShopQueueItemVo> dedupeQueueItems(List<ProductRiskShopQueueItemVo> raw) {
Map<String, ProductRiskShopQueueItemVo> byNorm = new LinkedHashMap<>(); Map<String, ProductRiskShopQueueItemVo> byNorm = new LinkedHashMap<>();
for (ProductRiskShopQueueItemVo item : raw) { for (ProductRiskShopQueueItemVo item : raw) {

View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.queryasin.controller; package com.nanri.aiimage.modules.queryasin.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
@@ -23,7 +24,6 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -37,7 +37,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
@@ -269,10 +268,8 @@ public class QueryAsinTaskController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;

View File

@@ -135,27 +135,58 @@ public class QueryAsinTaskService {
} }
public QueryAsinHistoryVo listHistory(Long userId) { public QueryAsinHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
validateUserId(userId); validateUserId(userId);
QueryAsinHistoryVo vo = new QueryAsinHistoryVo(); QueryAsinHistoryVo vo = new QueryAsinHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId) .eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt) .orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100")); .last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) { if (entities.isEmpty()) {
vo.setItems(List.of()); vo.setItems(List.of());
log.info("[query-asin] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo; return vo;
} }
Map<Long, FileTaskEntity> taskMap = loadTaskMap(entities); Map<Long, FileTaskEntity> taskMap = loadHistoryTaskMap(entities);
Map<Long, Map<Long, QueryAsinResultItemVo>> snapshotMap = buildSnapshotMap(taskMap); long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
long jobsLoadedAt = System.nanoTime();
List<QueryAsinResultItemVo> items = new ArrayList<>(); List<QueryAsinResultItemVo> items = new ArrayList<>();
for (FileResultEntity entity : entities) { for (FileResultEntity entity : entities) {
FileTaskEntity task = taskMap.get(entity.getTaskId()); FileTaskEntity task = taskMap.get(entity.getTaskId());
QueryAsinResultItemVo snapshot = snapshotMap.getOrDefault(entity.getTaskId(), Map.of()).get(entity.getId()); items.add(toHistoryItem(entity, task, null, jobMap.get(entity.getId())));
items.add(toHistoryItem(entity, task, snapshot));
} }
vo.setItems(items); vo.setItems(items);
long finishedAt = System.nanoTime();
log.info("[query-asin] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
taskMap.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo; return vo;
} }
@@ -510,6 +541,31 @@ public class QueryAsinTaskService {
return taskMap; return taskMap;
} }
private Map<Long, FileTaskEntity> loadHistoryTaskMap(List<FileResultEntity> entities) {
List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId)
.filter(id -> id != null && id > 0)
.distinct()
.toList();
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
if (taskIds.isEmpty()) {
return taskMap;
}
int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize());
for (int start = 0; start < taskIds.size(); start += batchSize) {
int end = Math.min(start + batchSize, taskIds.size());
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId, FileTaskEntity::getStatus, FileTaskEntity::getFinishedAt)
.in(FileTaskEntity::getId, taskIds.subList(start, end)));
for (FileTaskEntity task : tasks) {
if (task != null) {
taskMap.put(task.getId(), task);
}
}
}
return taskMap;
}
private Map<Long, Map<Long, QueryAsinResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) { private Map<Long, Map<Long, QueryAsinResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
Map<Long, Map<Long, QueryAsinResultItemVo>> out = new LinkedHashMap<>(); Map<Long, Map<Long, QueryAsinResultItemVo>> out = new LinkedHashMap<>();
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) { for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
@@ -523,6 +579,10 @@ public class QueryAsinTaskService {
} }
private QueryAsinResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, QueryAsinResultItemVo snapshot) { private QueryAsinResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, QueryAsinResultItemVo snapshot) {
return toHistoryItem(entity, task, snapshot, null);
}
private QueryAsinResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, QueryAsinResultItemVo snapshot, TaskFileJobEntity job) {
QueryAsinResultItemVo item = snapshot != null ? snapshot : new QueryAsinResultItemVo(); QueryAsinResultItemVo item = snapshot != null ? snapshot : new QueryAsinResultItemVo();
item.setResultId(entity.getId()); item.setResultId(entity.getId());
item.setTaskId(entity.getTaskId()); item.setTaskId(entity.getTaskId());
@@ -535,7 +595,7 @@ public class QueryAsinTaskService {
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt()); item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename())); item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
item.setDownloadUrl(null); item.setDownloadUrl(null);
attachFileJobState(item, entity); attachFileJobState(item, entity, job);
if (item.getCountryResults() == null) { if (item.getCountryResults() == null) {
item.setCountryResults(new ArrayList<>()); item.setCountryResults(new ArrayList<>());
} }
@@ -546,7 +606,10 @@ public class QueryAsinTaskService {
} }
private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity) { private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); attachFileJobState(item, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()));
}
private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) {
item.setFileReady(!blank(entity.getResultFileUrl())); item.setFileReady(!blank(entity.getResultFileUrl()));
if (job == null) { if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null); item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
@@ -1061,6 +1124,10 @@ public class QueryAsinTaskService {
return value == null || value.isBlank(); return value == null || value.isBlank();
} }
private static long elapsedMs(long startInclusive, long endExclusive) {
return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L);
}
private String blankToNull(String value) { private String blankToNull(String value) {
return blank(value) ? null : value.trim(); return blank(value) ? null : value.trim();
} }

View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.shopmatch.controller; package com.nanri.aiimage.modules.shopmatch.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
@@ -23,7 +24,6 @@ import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -38,7 +38,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
@@ -193,10 +192,8 @@ public class ShopMatchController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No downloadable result"); throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No downloadable result");
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;

View File

@@ -173,15 +173,34 @@ public class ShopMatchTaskService {
} }
public ProductRiskHistoryVo listHistory(Long userId) { public ProductRiskHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
if (userId == null || userId <= 0) { if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
} }
ProductRiskHistoryVo vo = new ProductRiskHistoryVo(); ProductRiskHistoryVo vo = new ProductRiskHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId) .eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt) .orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100")); .last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) {
vo.setItems(List.of());
log.info("[shop-match] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
Map<Long, String> statusByTaskId = new LinkedHashMap<>(); Map<Long, String> statusByTaskId = new LinkedHashMap<>();
List<Long> taskIds = entities.stream() List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId) .map(FileResultEntity::getTaskId)
@@ -195,11 +214,29 @@ public class ShopMatchTaskService {
} }
} }
} }
long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
long jobsLoadedAt = System.nanoTime();
List<ProductRiskResultItemVo> items = new ArrayList<>(); List<ProductRiskResultItemVo> items = new ArrayList<>();
for (FileResultEntity entity : entities) { for (FileResultEntity entity : entities) {
items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()))); items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()), jobMap.get(entity.getId()), false));
} }
vo.setItems(items); vo.setItems(items);
long finishedAt = System.nanoTime();
log.info("[shop-match] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
statusByTaskId.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo; return vo;
} }
@@ -867,6 +904,10 @@ public class ShopMatchTaskService {
} }
private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) { private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) {
return toHistoryItemVo(entity, taskStatus, null, true);
}
private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus, TaskFileJobEntity job, boolean mergeRequestFields) {
ProductRiskResultItemVo vo = new ProductRiskResultItemVo(); ProductRiskResultItemVo vo = new ProductRiskResultItemVo();
vo.setResultId(entity.getId()); vo.setResultId(entity.getId());
vo.setTaskId(entity.getTaskId()); vo.setTaskId(entity.getTaskId());
@@ -878,16 +919,19 @@ public class ShopMatchTaskService {
vo.setError(entity.getErrorMessage()); vo.setError(entity.getErrorMessage());
vo.setOutputFilename(entity.getResultFilename()); vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(null); vo.setDownloadUrl(null);
attachFileJobState(vo, entity); attachFileJobState(vo, entity, job);
vo.setMatched(true); vo.setMatched(true);
if (entity.getTaskId() != null) { if (mergeRequestFields && entity.getTaskId() != null) {
mergeQueueFieldsFromRequest(vo, entity.getTaskId()); mergeQueueFieldsFromRequest(vo, entity.getTaskId());
} }
return vo; return vo;
} }
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) { private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); attachFileJobState(vo, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()));
}
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity, TaskFileJobEntity job) {
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) { if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
@@ -1031,6 +1075,10 @@ public class ShopMatchTaskService {
return time == null ? null : time.toString(); return time == null ? null : time.toString();
} }
private static long elapsedMs(long startInclusive, long endExclusive) {
return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L);
}
private LocalDateTime now() { private LocalDateTime now() {
return LocalDateTime.now(BUSINESS_ZONE); return LocalDateTime.now(BUSINESS_ZONE);
} }

View File

@@ -0,0 +1,990 @@
package com.nanri.aiimage.modules.similarasin.client;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.MediaType;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.stereotype.Component;
import org.springframework.util.StreamUtils;
import org.springframework.web.client.RestClient;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@Component
@RequiredArgsConstructor
@Slf4j
public class SimilarAsinCozeClient {
private final SimilarAsinProperties properties;
private final ObjectMapper objectMapper;
public List<SimilarAsinResultRowDto> inspect(List<SimilarAsinResultRowDto> rows, String prompt) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
if (properties.getCozeToken() == null || properties.getCozeToken().isBlank()) {
log.warn("[similar-asin] coze token not configured, keep raw rows size={}", rows.size());
return rows.stream().map(this::copy).toList();
}
try {
return inspectWithFallback(rows, prompt);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[similar-asin] coze batch failed size={} err={}", rows.size(), failureMessage);
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
}
public CozeSubmitResponse submitWorkflow(List<SimilarAsinResultRowDto> rows, String prompt) throws Exception {
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt));
ensureSuccess(submitRoot);
return new CozeSubmitResponse(
extractExecuteId(submitRoot),
extractResultDataText(submitRoot),
writeJson(submitRoot)
);
}
public CozePollResponse pollWorkflow(String executeId) throws Exception {
JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId));
ensureSuccess(pollRoot);
String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT);
String dataText = extractResultDataText(pollRoot);
String outputText = dataText.isBlank() ? extractWorkflowOutputText(pollRoot) : "";
String failureMessage = isFailedWorkflowStatus(status)
? firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed")
: "";
return new CozePollResponse(executeId, status, dataText, outputText, failureMessage, writeJson(pollRoot));
}
public List<SimilarAsinResultRowDto> mergeRowsFromDataText(List<SimilarAsinResultRowDto> rows, String dataText) throws Exception {
if (dataText == null || dataText.isBlank()) {
return rows == null ? List.of() : rows.stream().map(this::copy).toList();
}
return mergeRows(rows, parseResults(wrapDataPayload(dataText)));
}
public List<SimilarAsinResultRowDto> markRowsFailed(List<SimilarAsinResultRowDto> rows, String failureMessage) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
private List<SimilarAsinResultRowDto> inspectWithFallback(List<SimilarAsinResultRowDto> rows, String prompt) {
try {
if (rows.size() == 1) {
return inspectSingleRowWithRetry(rows, prompt);
}
InspectAttempt attempt = inspectOnce(rows, prompt);
if (attempt.resolvedCount() < rows.size()) {
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
}
return attempt.mergedRows();
} catch (Exception ex) {
if (shouldSplitBatch(rows, ex)) {
int middle = rows.size() / 2;
log.warn("[similar-asin] coze batch fallback split size={} left={} right={} err={}",
rows.size(), middle, rows.size() - middle, failureMessage(ex));
List<SimilarAsinResultRowDto> merged = new ArrayList<>(rows.size());
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt));
return merged;
}
throw propagate(ex);
}
}
private List<SimilarAsinResultRowDto> inspectPartitionWithFailureFallback(List<SimilarAsinResultRowDto> rows, String prompt) {
try {
return inspectWithFallback(rows, prompt);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[similar-asin] coze partition failed size={} err={}", rows.size(), failureMessage);
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
}
private List<SimilarAsinResultRowDto> inspectSingleRowWithRetry(List<SimilarAsinResultRowDto> rows, String prompt) throws Exception {
SimilarAsinResultRowDto row = rows.getFirst();
PartialCozeResultException lastFailure = null;
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
try {
InspectAttempt attempt = inspectOnce(rows, prompt);
if (attempt.resolvedCount() == rows.size()) {
return attempt.mergedRows();
}
log.warn("[similar-asin] coze single unresolved attempt={} rowId={} asin={} country={} title={} url={} raw={}",
attemptIndex,
row.getId(),
row.getAsin(),
row.getCountry(),
abbreviate(row.getTitle(), 120),
abbreviate(row.getUrl(), 120),
abbreviate(attempt.raw(), 500));
lastFailure = new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
} catch (Exception ex) {
if (attemptIndex >= 3 || !isRetryableBatchFailure(ex)) {
throw ex;
}
log.warn("[similar-asin] coze single retryable failure attempt={} rowId={} asin={} country={} err={}",
attemptIndex,
row.getId(),
row.getAsin(),
row.getCountry(),
failureMessage(ex));
}
if (attemptIndex < 3) {
sleepBeforeRetry(attemptIndex);
}
}
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
}
private InspectAttempt inspectOnce(List<SimilarAsinResultRowDto> rows, String prompt) throws Exception {
String raw = runWorkflowAsyncAndWait(rows, prompt);
List<CozeResult> results = parseResults(raw);
if (rows.size() > 1 && !results.isEmpty() && results.stream().noneMatch(this::hasIdentity)) {
throw new PartialCozeResultException(0, rows.size(), results.size());
}
List<SimilarAsinResultRowDto> merged = mergeRows(rows, results);
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
}
private String runWorkflowAsyncAndWait(List<SimilarAsinResultRowDto> rows, String prompt) throws Exception {
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt));
ensureSuccess(submitRoot);
String immediateData = extractResultDataText(submitRoot);
if (!immediateData.isBlank()) {
return wrapDataPayload(immediateData);
}
String executeId = extractExecuteId(submitRoot);
if (executeId == null || executeId.isBlank()) {
throw new IllegalStateException("Coze async execute_id missing");
}
long deadline = System.currentTimeMillis() + Math.max(10000, properties.getCozePollTimeoutMillis());
while (System.currentTimeMillis() < deadline) {
ensureNotInterrupted();
JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId));
ensureSuccess(pollRoot);
String dataText = extractResultDataText(pollRoot);
if (!dataText.isBlank()) {
return wrapDataPayload(dataText);
}
String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT);
if (isFailedWorkflowStatus(status)) {
throw new IllegalStateException(firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed"));
}
if (isSuccessfulWorkflowStatus(status)) {
String outputText = extractWorkflowOutputText(pollRoot);
if (!outputText.isBlank()) {
return wrapDataPayload(outputText);
}
throw new IllegalStateException("Coze async workflow completed without output");
}
sleepQuietly(Math.max(200, properties.getCozePollIntervalMillis()));
}
throw new IllegalStateException("Coze async workflow poll timeout");
}
private String postWorkflow(List<SimilarAsinResultRowDto> rows, String prompt) {
Map<String, Object> parameters = buildParameters(rows, prompt);
Map<String, Object> body = new LinkedHashMap<>();
body.put("workflow_id", properties.getCozeWorkflowId());
body.put("parameters", parameters);
body.put("is_async", Boolean.TRUE);
log.info("[similar-asin] coze request url={} body={}",
joinUrl(properties.getCozeBaseUrl(), properties.getCozeWorkflowPath()),
writeJson(body));
RestClient.RequestBodySpec request = restClient().post()
.uri(joinUrl(properties.getCozeBaseUrl(), properties.getCozeWorkflowPath()))
.headers(headers -> {
headers.setBearerAuth(stripBearer(properties.getCozeToken()));
headers.setContentType(MediaType.APPLICATION_JSON);
});
request.body(body);
return request.exchange((clientRequest, clientResponse) -> {
byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody());
String responseText = responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8);
log.info("[similar-asin] coze submit response status={} body={}",
clientResponse.getStatusCode(),
responseText);
return responseText;
});
}
private String getWorkflowHistory(String executeId) {
String path = properties.getCozeWorkflowHistoryPath()
.replace("{workflow_id}", properties.getCozeWorkflowId())
.replace("{execute_id}", executeId);
return restClient().get()
.uri(joinUrl(properties.getCozeBaseUrl(), path))
.headers(headers -> {
headers.setBearerAuth(stripBearer(properties.getCozeToken()));
headers.setContentType(MediaType.APPLICATION_JSON);
})
.exchange((clientRequest, clientResponse) -> {
byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody());
String responseText = responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8);
log.info("[similar-asin] coze history response executeId={} status={} body={}",
executeId,
clientResponse.getStatusCode(),
responseText);
return responseText;
});
}
private Map<String, Object> buildParameters(List<SimilarAsinResultRowDto> rows, String prompt) {
List<String> groupKeys = rows.stream().map(row -> nonBlank(row.getGroupKey(), rowKey(row))).toList();
List<String> rowIds = rows.stream().map(row -> nonBlank(row.getId(), "")).toList();
List<String> asins = rows.stream().map(row -> nonBlank(row.getAsin(), "")).toList();
List<String> countries = rows.stream().map(row -> nonBlank(row.getCountry(), "")).toList();
List<String> titles = rows.stream().map(row -> nonBlank(row.getTitle(), row.getAsin())).toList();
List<String> urls = rows.stream().map(row -> nonBlank(row.getUrl(), "")).toList();
List<List<String>> urlLists = rows.stream().map(SimilarAsinResultRowDto::getUrls).toList();
Map<String, Object> parameters = new LinkedHashMap<>();
parameters.put("title_list", titles);
parameters.put("url_list", urls);
parameters.put("url_lists", urlLists);
parameters.put("items", buildItemObjects(rows, groupKeys, rowIds, asins, countries, titles, urlLists));
parameters.put("prompt", prompt == null ? "" : prompt);
return parameters;
}
private List<Map<String, Object>> buildItemObjects(List<SimilarAsinResultRowDto> rows,
List<String> groupKeys,
List<String> rowIds,
List<String> asins,
List<String> countries,
List<String> titles,
List<List<String>> urlLists) {
List<Map<String, Object>> items = new ArrayList<>(rows.size());
for (int i = 0; i < rows.size(); i++) {
Map<String, Object> item = new LinkedHashMap<>();
item.put("group_key", groupKeys.get(i));
item.put("row_id", rowIds.get(i));
item.put("asin", asins.get(i));
item.put("country", countries.get(i));
item.put("title", titles.get(i));
item.put("url", urlLists.get(i));
item.put("urls", urlLists.get(i));
items.add(item);
}
return items;
}
private List<CozeResult> parseResults(String raw) throws Exception {
JsonNode root = objectMapper.readTree(raw);
ensureSuccess(root);
String dataText = extractResultDataText(root);
if (dataText.isBlank()) {
return List.of();
}
JsonNode dataRoot = objectMapper.readTree(dataText);
JsonNode array = dataRoot.isArray() ? dataRoot : dataRoot.path("data");
List<CozeResult> results = new ArrayList<>();
if (array.isArray()) {
for (JsonNode node : array) {
JsonNode itemNode = resultItemNode(node);
results.add(new CozeResult(
text(firstNonNull(
firstNonNull(node.get("group_key"), node.get("groupKey")),
firstNonNull(itemNode.get("group_key"), itemNode.get("groupKey")))),
text(firstNonNull(
firstNonNull(node.get("row_id"), firstNonNull(node.get("rowId"), node.get("id"))),
firstNonNull(itemNode.get("row_id"), firstNonNull(itemNode.get("rowId"), itemNode.get("id"))))),
text(firstNonNull(node.get("asin"), itemNode.get("asin"))),
text(firstNonNull(
firstNonNull(node.get("country"), node.get("site")),
firstNonNull(itemNode.get("country"), itemNode.get("site")))),
text(firstNonNull(node.get("title"), firstNonNull(itemNode.get("title"), itemNode.get("title_risk")))),
text(firstNonNull(node.get("appearance"),
firstNonNull(node.get("appearance_risk"),
firstNonNull(itemNode.get("appearance"), itemNode.get("appearance_risk"))))),
text(firstNonNull(firstNonNull(node.get("patent"), node.get("patent ")),
firstNonNull(node.get("patent_risk"),
firstNonNull(firstNonNull(itemNode.get("patent"), itemNode.get("patent ")), itemNode.get("patent_risk"))))),
text(firstNonNull(node.get("result"),
firstNonNull(node.get("conclusion"),
firstNonNull(itemNode.get("result"), itemNode.get("conclusion"))))),
text(firstNonNull(node.get("title_reason"),
firstNonNull(node.get("titleReason"),
firstNonNull(itemNode.get("title_reason"), itemNode.get("titleReason"))))),
text(firstNonNull(node.get("appearance_reason"),
firstNonNull(node.get("appearanceReason"),
firstNonNull(itemNode.get("appearance_reason"), itemNode.get("appearanceReason"))))),
text(firstNonNull(node.get("patent_reason"),
firstNonNull(firstNonNull(node.get("patentReason"), node.get("patent reason")),
firstNonNull(itemNode.get("patent_reason"),
firstNonNull(itemNode.get("patentReason"), itemNode.get("patent reason"))))))
));
}
}
return results;
}
private JsonNode resultItemNode(JsonNode node) {
if (node == null || node.isMissingNode() || node.isNull()) {
return objectMapper.missingNode();
}
JsonNode item = firstNonNull(node.get("item"), node.get("items"));
if (item == null || item.isMissingNode() || item.isNull()) {
return objectMapper.missingNode();
}
if (item.isArray()) {
return item.isEmpty() ? objectMapper.missingNode() : item.get(0);
}
return item;
}
private List<SimilarAsinResultRowDto> mergeRows(List<SimilarAsinResultRowDto> rows, List<CozeResult> results) {
Map<String, CozeResult> resultByGroupKey = new LinkedHashMap<>();
Map<String, CozeResult> resultByCompositeKey = new LinkedHashMap<>();
Map<String, CozeResult> resultByAsinCountry = new LinkedHashMap<>();
Map<String, CozeResult> resultByAsin = new LinkedHashMap<>();
Map<String, CozeResult> resultByRowId = new LinkedHashMap<>();
for (CozeResult result : results) {
String groupKey = normalize(result.groupKey());
if (!groupKey.isBlank()) {
resultByGroupKey.putIfAbsent(groupKey, result);
}
String compositeKey = rowKey(result.rowId(), result.asin(), result.country());
if (!compositeKey.isBlank()) {
resultByCompositeKey.putIfAbsent(compositeKey, result);
}
String asinCountryKey = asinCountryKey(result.asin(), result.country());
if (!asinCountryKey.isBlank()) {
resultByAsinCountry.putIfAbsent(asinCountryKey, result);
}
String asinKey = normalize(result.asin()).toUpperCase(Locale.ROOT);
if (!asinKey.isBlank()) {
resultByAsin.putIfAbsent(asinKey, result);
}
String rowIdKey = normalize(result.rowId());
if (!rowIdKey.isBlank()) {
resultByRowId.putIfAbsent(rowIdKey, result);
}
}
List<SimilarAsinResultRowDto> merged = new ArrayList<>(rows.size());
boolean allowIndexFallback = results.size() == rows.size() && results.stream().noneMatch(this::hasIdentity);
for (int i = 0; i < rows.size(); i++) {
SimilarAsinResultRowDto row = copy(rows.get(i));
CozeResult result = resultByGroupKey.get(normalize(row.getGroupKey()));
if (result == null) {
result = resultByCompositeKey.get(rowKey(row.getId(), row.getAsin(), row.getCountry()));
}
if (result == null) {
result = resultByAsinCountry.get(asinCountryKey(row.getAsin(), row.getCountry()));
}
if (result == null) {
result = resultByAsin.get(normalize(row.getAsin()).toUpperCase(Locale.ROOT));
}
if (result == null) {
result = resultByRowId.get(normalize(row.getId()));
}
if (result == null && allowIndexFallback && i < results.size()) {
result = results.get(i);
}
applyResult(row, result);
merged.add(row);
}
return merged;
}
private String asinCountryKey(String asin, String country) {
String normalizedAsin = normalize(asin).toUpperCase(Locale.ROOT);
if (normalizedAsin.isBlank()) {
return "";
}
return normalizedAsin + "::" + normalize(country);
}
private boolean hasIdentity(CozeResult result) {
if (result == null) {
return false;
}
return !normalize(result.groupKey()).isBlank()
|| !normalize(result.rowId()).isBlank()
|| !normalize(result.asin()).isBlank();
}
private void applyResult(SimilarAsinResultRowDto row, CozeResult result) {
if (row == null || result == null) {
return;
}
row.setTitleRisk(result.title());
row.setAppearanceRisk(result.appearance());
row.setPatentRisk(result.patent());
row.setConclusion(result.result());
row.setTitleReason(result.titleReason());
row.setAppearanceReason(result.appearanceReason());
row.setPatentReason(result.patentReason());
}
private RestClient restClient() {
SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
requestFactory.setConnectTimeout(properties.getCozeConnectTimeoutMillis());
requestFactory.setReadTimeout(properties.getCozeReadTimeoutMillis());
return RestClient.builder().requestFactory(requestFactory).build();
}
private SimilarAsinResultRowDto copy(SimilarAsinResultRowDto source) {
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setSourceFileKey(source.getSourceFileKey());
row.setSourceFilename(source.getSourceFilename());
row.setRowToken(source.getRowToken());
row.setGroupKey(source.getGroupKey());
row.setId(source.getId());
row.setAsin(source.getAsin());
row.setCountry(source.getCountry());
row.setUrls(source.getUrls());
row.setTitle(source.getTitle());
row.setError(source.getError());
row.setDone(source.getDone());
row.setTitleRisk(source.getTitleRisk());
row.setAppearanceRisk(source.getAppearanceRisk());
row.setPatentRisk(source.getPatentRisk());
row.setConclusion(source.getConclusion());
row.setTitleReason(source.getTitleReason());
row.setAppearanceReason(source.getAppearanceReason());
row.setPatentReason(source.getPatentReason());
return row;
}
private SimilarAsinResultRowDto markFailed(SimilarAsinResultRowDto row, String failureMessage) {
String reviewMessage = failureMessage == null || failureMessage.isBlank()
? "Coze 检测失败,待人工复核"
: "Coze 检测失败,待人工复核:" + failureMessage;
if (row.getError() == null || row.getError().isBlank()) {
row.setError(failureMessage);
}
if (row.getTitleRisk() == null || row.getTitleRisk().isBlank()) {
row.setTitleRisk(reviewMessage);
}
if (row.getAppearanceRisk() == null || row.getAppearanceRisk().isBlank()) {
row.setAppearanceRisk(reviewMessage);
}
if (row.getPatentRisk() == null || row.getPatentRisk().isBlank()) {
row.setPatentRisk(reviewMessage);
}
if (row.getConclusion() == null || row.getConclusion().isBlank()) {
row.setConclusion(failureMessage);
}
return row;
}
private boolean shouldSplitBatch(List<SimilarAsinResultRowDto> rows, Exception ex) {
return rows != null && rows.size() > 1 && isRetryableBatchFailure(ex);
}
private boolean isRetryableBatchFailure(Exception ex) {
if (ex instanceof PartialCozeResultException) {
return true;
}
String message = ex == null ? "" : nonBlank(ex.getMessage(), "");
return message.contains("Workflow node execution limit exceeded")
|| message.contains("Read timed out")
|| message.contains("Connection reset")
|| message.contains("I/O error on POST request")
|| message.toLowerCase(Locale.ROOT).contains("timeout");
}
private int resolvedCount(List<SimilarAsinResultRowDto> rows) {
int resolved = 0;
for (SimilarAsinResultRowDto row : rows) {
if (hasResolvedCozeFields(row)) {
resolved++;
}
}
return resolved;
}
private boolean hasResolvedCozeFields(SimilarAsinResultRowDto row) {
if (row == null) {
return false;
}
return !normalize(row.getTitleRisk()).isBlank()
|| !normalize(row.getAppearanceRisk()).isBlank()
|| !normalize(row.getPatentRisk()).isBlank()
|| !normalize(row.getConclusion()).isBlank();
}
private void ensureSuccess(JsonNode root) {
if (root.path("code").asInt(-1) != 0) {
throw new IllegalStateException(root.path("msg").asText("Coze response code is not 0"));
}
}
private String extractResultDataText(JsonNode root) {
if (root == null || root.isMissingNode() || root.isNull()) {
return "";
}
JsonNode dataNode = root.path("data");
if (dataNode.isTextual()) {
String value = dataNode.asText("");
if (looksLikeResultDataPayload(value)) {
return value;
}
return discoverEmbeddedData(parseJsonOrMissing(value));
}
if (dataNode.isObject()) {
String nested = text(firstNonNull(dataNode.get("data"), firstNonNull(dataNode.get("output"), dataNode.get("result"))));
if (nested != null && !nested.isBlank() && looksLikeResultDataPayload(nested)) {
return nested;
}
JsonNode outputs = firstNonNull(dataNode.get("outputs"), dataNode.get("details"));
String discovered = discoverEmbeddedData(outputs);
if (!discovered.isBlank()) {
return discovered;
}
}
return discoverEmbeddedData(root);
}
private String extractWorkflowOutputText(JsonNode root) {
JsonNode outputNode = findFirstField(root, "output");
if (outputNode == null || outputNode.isNull() || outputNode.isMissingNode()) {
return "";
}
if (!outputNode.isTextual()) {
String discovered = discoverEmbeddedData(outputNode);
return discovered.isBlank() && isResultDataPayload(outputNode) ? outputNode.toString() : discovered;
}
String output = normalize(outputNode.asText(""));
if (output.isBlank()) {
return "";
}
if (looksLikeResultDataPayload(output)) {
return output;
}
JsonNode parsedOutput = parseJsonOrMissing(output);
String nestedOutput = text(firstNonNull(parsedOutput.get("Output"), parsedOutput.get("output")));
if (nestedOutput != null && !nestedOutput.isBlank() && looksLikeResultDataPayload(nestedOutput)) {
return nestedOutput;
}
return discoverEmbeddedData(parsedOutput);
}
private String discoverEmbeddedData(JsonNode node) {
if (node == null || node.isNull() || node.isMissingNode()) {
return "";
}
if (node.isTextual()) {
String value = node.asText("");
if (looksLikeResultDataPayload(value)) {
return value;
}
return discoverEmbeddedData(parseJsonOrMissing(value));
}
if (node.isArray()) {
for (JsonNode child : node) {
String discovered = discoverEmbeddedData(child);
if (!discovered.isBlank()) {
return discovered;
}
}
return "";
}
if (node.isObject()) {
if (isResultDataPayload(node)) {
return node.toString();
}
for (java.util.Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> entry = it.next();
String discovered = discoverEmbeddedData(entry.getValue());
if (!discovered.isBlank()) {
return discovered;
}
}
}
return "";
}
private boolean looksLikeResultDataPayload(String value) {
JsonNode parsed = parseJsonOrMissing(value);
return isResultDataPayload(parsed);
}
private boolean isResultDataPayload(JsonNode node) {
if (node == null || node.isMissingNode() || node.isNull()) {
return false;
}
JsonNode array = node.path("data");
if (!array.isArray()) {
return false;
}
for (JsonNode item : array) {
JsonNode itemNode = resultItemNode(item);
if (hasResultFields(item) || hasResultFields(itemNode)) {
return true;
}
}
return false;
}
private boolean hasResultFields(JsonNode item) {
return item != null
&& (item.has("appearance")
|| item.has("appearance_risk")
|| item.has("patent")
|| item.has("patent ")
|| item.has("patent_risk")
|| item.has("result")
|| item.has("conclusion")
|| item.has("title_reason")
|| item.has("titleReason")
|| item.has("appearance_reason")
|| item.has("appearanceReason")
|| item.has("patent_reason")
|| item.has("patentReason")
|| item.has("patent reason"));
}
private boolean isSuccessfulWorkflowStatus(String status) {
String normalized = normalize(status).toUpperCase(Locale.ROOT);
return normalized.equals("SUCCESS")
|| normalized.equals("SUCCEEDED")
|| normalized.equals("COMPLETED")
|| normalized.equals("COMPLETE")
|| normalized.equals("DONE");
}
private boolean isFailedWorkflowStatus(String status) {
String normalized = normalize(status).toUpperCase(Locale.ROOT);
return normalized.contains("FAIL")
|| normalized.contains("ERROR")
|| normalized.contains("CANCEL");
}
private JsonNode parseJsonOrMissing(String value) {
String normalized = normalize(value);
if (!(normalized.startsWith("{") || normalized.startsWith("["))) {
return objectMapper.missingNode();
}
try {
return objectMapper.readTree(normalized);
} catch (Exception ignored) {
return objectMapper.missingNode();
}
}
private String resolveWorkflowStatus(JsonNode root) {
JsonNode dataNode = root.path("data");
JsonNode statusNode = firstNonNull(
firstNonNull(dataNode.get("status"), dataNode.get("execute_status")),
firstNonNull(root.get("status"), root.get("execute_status")));
String status = text(statusNode);
if (status != null && !status.isBlank()) {
return status;
}
return findTextByFieldName(root, "execute_status", "status");
}
private String resolveFailureMessage(JsonNode root) {
JsonNode dataNode = root.path("data");
String message = text(firstNonNull(dataNode.get("error_message"), firstNonNull(dataNode.get("msg"), root.get("msg"))));
return message == null ? "" : message;
}
private String extractExecuteId(JsonNode root) {
return findTextByFieldName(root, "execute_id", "executeId");
}
private JsonNode findFirstField(JsonNode node, String name) {
if (node == null || node.isMissingNode() || node.isNull() || name == null || name.isBlank()) {
return objectMapper.missingNode();
}
if (node.isTextual()) {
return findFirstField(parseJsonOrMissing(node.asText("")), name);
}
if (node.isArray()) {
for (JsonNode child : node) {
JsonNode found = findFirstField(child, name);
if (found != null && !found.isMissingNode() && !found.isNull()) {
return found;
}
}
return objectMapper.missingNode();
}
if (node.isObject()) {
JsonNode direct = node.get(name);
if (direct != null && !direct.isMissingNode() && !direct.isNull()) {
return direct;
}
for (java.util.Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> entry = it.next();
JsonNode found = findFirstField(entry.getValue(), name);
if (found != null && !found.isMissingNode() && !found.isNull()) {
return found;
}
}
}
return objectMapper.missingNode();
}
private String findTextByFieldName(JsonNode node, String... names) {
if (node == null || node.isMissingNode() || node.isNull()) {
return "";
}
if (node.isTextual()) {
return findTextByFieldName(parseJsonOrMissing(node.asText("")), names);
}
if (node.isArray()) {
for (JsonNode child : node) {
String found = findTextByFieldName(child, names);
if (!found.isBlank()) {
return found;
}
}
return "";
}
if (node.isObject()) {
for (String name : names) {
String value = text(node.get(name));
if (value != null && !value.isBlank()) {
return value;
}
}
JsonNode dataNode = node.get("data");
if (dataNode != null && dataNode.isTextual()) {
String found = findTextByFieldName(parseJsonOrMissing(dataNode.asText("")), names);
if (!found.isBlank()) {
return found;
}
}
for (java.util.Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> entry = it.next();
String found = findTextByFieldName(entry.getValue(), names);
if (!found.isBlank()) {
return found;
}
}
}
return "";
}
private String wrapDataPayload(String dataText) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("code", 0);
payload.put("data", dataText);
return writeJson(payload);
}
private String writeJson(Object value) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new IllegalStateException("Failed to serialize Coze payload", ex);
}
}
private String abbreviate(String value, int maxLength) {
String normalized = value == null ? "" : value.trim();
if (normalized.length() <= maxLength) {
return normalized;
}
return normalized.substring(0, Math.max(0, maxLength - 3)) + "...";
}
private void sleepBeforeRetry(int attemptIndex) {
long delayMillis = Math.max(1, attemptIndex) * 1500L;
ensureNotInterrupted();
sleepQuietly(delayMillis);
}
private void sleepQuietly(long delayMillis) {
try {
Thread.sleep(delayMillis);
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Coze workflow interrupted", interruptedException);
}
}
private void ensureNotInterrupted() {
if (Thread.currentThread().isInterrupted()) {
throw new IllegalStateException("Coze workflow interrupted");
}
}
private RuntimeException propagate(Exception ex) {
if (ex instanceof RuntimeException runtimeException) {
return runtimeException;
}
return new IllegalStateException(nonBlank(ex.getMessage(), "Coze call failed"), ex);
}
private JsonNode firstNonNull(JsonNode left, JsonNode right) {
return left == null || left.isNull() ? right : left;
}
private String text(JsonNode node) {
return node == null || node.isNull() ? null : node.asText();
}
private String nonBlank(String value, String fallback) {
return value == null || value.isBlank() ? fallback : value;
}
private String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
private String normalize(String value) {
return value == null ? "" : value.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim();
}
private String rowKey(SimilarAsinResultRowDto row) {
if (row == null) {
return "";
}
return rowKey(row.getId(), row.getAsin(), row.getCountry());
}
private String rowKey(String rowId, String asin, String country) {
return normalize(rowId) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
}
private String failureMessage(Exception ex) {
if (ex instanceof PartialCozeResultException partial) {
return "Coze result incomplete(" + partial.resolvedCount() + "/" + partial.expectedCount() + ")";
}
String message = ex == null ? null : ex.getMessage();
if (message == null || message.isBlank()) {
return "Coze call failed";
}
if (message.contains("Workflow node execution limit exceeded")) {
return "Coze workflow node execution limit exceeded";
}
if (message.contains("Read timed out")) {
return "Coze call timed out";
}
return "Coze call failed: " + message;
}
private String stripBearer(String token) {
String normalized = token == null ? "" : token.trim();
return normalized.regionMatches(true, 0, "Bearer ", 0, 7) ? normalized.substring(7).trim() : normalized;
}
private String joinUrl(String baseUrl, String path) {
String base = baseUrl == null ? "" : baseUrl.trim();
String suffix = path == null ? "" : path.trim();
if (base.endsWith("/") && suffix.startsWith("/")) {
return base + suffix.substring(1);
}
if (!base.endsWith("/") && !suffix.startsWith("/")) {
return base + "/" + suffix;
}
return base + suffix;
}
private record CozeResult(
String groupKey,
String rowId,
String asin,
String country,
String title,
String appearance,
String patent,
String result,
String titleReason,
String appearanceReason,
String patentReason
) {
}
private record InspectAttempt(
String raw,
List<SimilarAsinResultRowDto> mergedRows,
int resolvedCount,
int rawResultCount
) {
}
public record CozeSubmitResponse(
String executeId,
String immediateData,
String rawResponse
) {
}
public record CozePollResponse(
String executeId,
String status,
String dataText,
String outputText,
String failureMessage,
String rawResponse
) {
public boolean hasPayload() {
return dataText != null && !dataText.isBlank() || outputText != null && !outputText.isBlank();
}
public String resolvedPayloadText() {
return dataText != null && !dataText.isBlank() ? dataText : outputText;
}
public boolean isFailed() {
String normalized = status == null ? "" : status.trim().toUpperCase(Locale.ROOT);
return normalized.contains("FAILED") || normalized.contains("ERROR") || normalized.contains("CANCEL");
}
public boolean isFinished() {
String normalized = status == null ? "" : status.trim().toUpperCase(Locale.ROOT);
return isFailed()
|| normalized.contains("SUCCESS")
|| normalized.contains("SUCCEED")
|| normalized.contains("FINISH")
|| normalized.contains("DONE")
|| normalized.contains("COMPLET");
}
}
private static final class PartialCozeResultException extends RuntimeException {
private final int resolvedCount;
private final int expectedCount;
private final int rawResultCount;
private PartialCozeResultException(int resolvedCount, int expectedCount, int rawResultCount) {
super("partial-result resolved=" + resolvedCount + "/" + expectedCount + " raw=" + rawResultCount);
this.resolvedCount = resolvedCount;
this.expectedCount = expectedCount;
this.rawResultCount = rawResultCount;
}
private int resolvedCount() {
return resolvedCount;
}
private int expectedCount() {
return expectedCount;
}
@SuppressWarnings("unused")
private int rawResultCount() {
return rawResultCount;
}
}
}

View File

@@ -0,0 +1,145 @@
package com.nanri.aiimage.modules.similarasin.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/similar-asin")
@Tag(name = "相似ASIN检测", description = "相似ASIN检测任务接口。前端上传 Excel 后由 Java 解析并创建任务Python 回传商品数据Java 负责攒批调用 Coze、补齐子行、生成最终 xlsx 并上传 OSS。")
public class SimilarAsinController {
private final SimilarAsinTaskService service;
@PostMapping("/parse")
@Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件,提取 id、ASIN、国家、URL、标题等字段。返回给前端的数据只包含整数 id 和 n_1 行n_2、n_3 等子行会保存在 OSS 解析载荷中,用于最终结果补齐。创建后的任务状态为 PENDING不会自动推送 Python。")
public ApiResponse<SimilarAsinParseVo> parse(@Valid @RequestBody SimilarAsinParseRequest request) {
return ApiResponse.success(service.parseAndCreateTask(request));
}
@GetMapping("/dashboard")
@Operation(summary = "查询相似ASIN检测总览", description = "查询当前用户的运行中、成功、失败和已结束任务数量。页面进入时请求一次即可,不需要持续轮询。")
public ApiResponse<SimilarAsinDashboardVo> dashboard(
@Parameter(description = "当前用户 ID用于隔离不同用户的任务和历史记录。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(service.dashboard(userId));
}
@GetMapping("/history")
@Operation(summary = "查询相似ASIN检测历史", description = "查询当前用户最近的相似ASIN检测历史记录包含源文件名、任务状态、行数、错误信息和最终 xlsx 下载地址。")
public ApiResponse<SimilarAsinHistoryVo> history(
@Parameter(description = "当前用户 ID用于查询该用户自己的历史记录。", required = true, example = "1")
@RequestParam("user_id") Long userId,
@Parameter(description = "history limit, default 50, max 100", example = "50")
@RequestParam(value = "limit", required = false, defaultValue = "50") Integer limit) {
return ApiResponse.success(service.history(userId, limit));
}
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询任务进度", description = "前端只对活跃任务调用该接口,建议 6 秒一次。接口只返回轻量任务状态,不返回明细结果。")
public ApiResponse<SimilarAsinTaskBatchVo> progress(@Valid @RequestBody SimilarAsinTaskBatchRequest request) {
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
}
@PostMapping("/tasks/{taskId}/activate")
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING并记录后端内部活跃时间。后续活跃时间由 Python 回传结果接口自动刷新,不需要单独心跳接口。")
public ApiResponse<Void> activate(
@Parameter(description = "相似ASIN检测任务 ID即解析接口返回的 taskId。", required = true, example = "3938")
@PathVariable Long taskId,
@Parameter(description = "当前用户 ID必须与创建任务的用户一致。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
service.activateTask(taskId, userId);
return ApiResponse.success(null);
}
@PostMapping("/tasks/{taskId}/result")
@Operation(summary = "提交 Python 回传结果", description = "Python 回传商品数据接口。items 可以是一条或多条Java 先原样保存回传数据,再内部攒够 10 条调用 Coze。done=true 表示 Python 已完成全部回传Java 会强制处理剩余不足 10 条的数据并生成最终 xlsx。")
public ApiResponse<Void> result(
@Parameter(description = "相似ASIN检测任务 ID。任务必须处于 RUNNING 状态。", required = true, example = "3938")
@PathVariable Long taskId,
@Valid @RequestBody SimilarAsinSubmitResultRequest request,
jakarta.servlet.http.HttpServletResponse response) {
response.setCharacterEncoding(StandardCharsets.UTF_8.name());
response.setContentType("application/json;charset=UTF-8");
service.submitResult(taskId, request);
return ApiResponse.success(null);
}
@DeleteMapping("/tasks/{taskId}")
@Operation(summary = "删除任务", description = "删除当前用户的一条相似ASIN检测任务同时清理任务结果、scope 状态和分片记录。")
public ApiResponse<Void> deleteTask(
@Parameter(description = "相似ASIN检测任务 ID。", required = true, example = "3938")
@PathVariable Long taskId,
@Parameter(description = "当前用户 ID必须与创建任务的用户一致。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
service.deleteTask(taskId, userId);
return ApiResponse.success(null);
}
@DeleteMapping("/history/{resultId}")
@Operation(summary = "删除历史记录", description = "删除当前用户的一条相似ASIN检测历史记录。只删除 biz_file_result 记录,不主动删除任务主记录。")
public ApiResponse<Void> deleteHistory(
@Parameter(description = "历史结果记录 ID即 history 接口返回的 resultId。", required = true, example = "1001")
@PathVariable Long resultId,
@Parameter(description = "当前用户 ID。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
service.deleteHistory(resultId, userId);
return ApiResponse.success(null);
}
@GetMapping("/results/{resultId}/download")
@Operation(summary = "下载相似ASIN检测结果文件")
public void downloadResult(
@Parameter(description = "结果记录 ID", required = true, example = "1001")
@PathVariable Long resultId,
@Parameter(description = "当前用户 ID", required = true, example = "1")
@RequestParam("user_id") Long userId,
jakarta.servlet.http.HttpServletResponse response) {
String url = service.resolveResultDownloadUrl(resultId, userId);
String filename = service.resolveResultDownloadFilename(resultId, userId);
if (url == null || url.isBlank()) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
}
try {
response.setContentType("application/octet-stream");
DownloadHeaderUtil.setAttachment(response, filename);
try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536];
int read;
while ((read = in.read(buffer)) != -1) {
response.getOutputStream().write(buffer, 0, read);
}
response.getOutputStream().flush();
}
} catch (Exception ex) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
}
}
}

View File

@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测解析请求")
public class SimilarAsinParseRequest {
@JsonProperty("user_id")
@NotNull
@Schema(description = "当前用户 ID。后端会把创建的任务、历史记录和结果文件归属到该用户。", example = "1", requiredMode = Schema.RequiredMode.REQUIRED)
private Long userId;
@NotEmpty
@Schema(description = "已上传的 Excel 文件列表。支持多文件聚合解析;文件对象来自统一上传接口返回值。", requiredMode = Schema.RequiredMode.REQUIRED)
private List<SimilarAsinSourceFileDto> files;
@JsonProperty("ai_prompt")
@JsonAlias({"aiPrompt", "prompt"})
@Schema(description = "AI 提示词。非必填;为空时前端会使用默认提示词。后端会保存该提示词,并在调用 Coze workflow 时作为 prompt 参数传入。", example = "请排查这些亚马逊商品在英国及欧洲地区是否存在知识产权侵权风险。")
private String aiPrompt;
}

View File

@@ -0,0 +1,31 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测解析载荷")
public class SimilarAsinParsedPayloadDto {
@Schema(description = "AI 提示词")
private String aiPrompt;
@Schema(description = "本次解析的源文件列表")
private List<SimilarAsinSourceFileDto> sourceFiles = new ArrayList<>();
@Schema(description = "Excel 原始表头列表")
private List<String> headers = new ArrayList<>();
@Schema(description = "兼容旧链路的平铺有效行,现为全部有效行")
private List<SimilarAsinParsedRowVo> items = new ArrayList<>();
@Schema(description = "按相邻主 ID 块分组后的完整数据")
private List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
@Schema(description = "完整有效行")
private List<SimilarAsinParsedRowVo> allItems = new ArrayList<>();
}

View File

@@ -0,0 +1,29 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "Python 回传的相似ASIN检测分组结果")
public class SimilarAsinResultGroupDto {
@Schema(description = "来源文件 key")
private String sourceFileKey;
@Schema(description = "来源文件名")
private String sourceFilename;
@Schema(description = "相邻主 ID 块的分组 key")
private String groupKey;
@Schema(description = "主 ID例如 1、2、10")
private String baseId;
@Schema(description = "分组首条展示 ID")
private String displayId;
@Schema(description = "分组内全部抓取结果")
private List<SimilarAsinResultRowDto> items = new ArrayList<>();
}

View File

@@ -0,0 +1,187 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonSetter;
import com.fasterxml.jackson.databind.JsonNode;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
@Data
@Schema(description = "相似ASIN检测单行商品数据")
public class SimilarAsinResultRowDto {
private static final int MAX_IMAGE_URLS = 24;
@Schema(description = "来源文件 key。多文件回传时用于避免行结果串到别的文件。", example = "uploads/20260426/similar_asin_17.xlsx")
private String sourceFileKey;
@Schema(description = "来源文件名。仅用于调试与追踪。", example = "17.xlsx")
private String sourceFilename;
@Schema(description = "行级唯一 token。Python 应从解析结果原样透传。", example = "uploads/20260426/similar_asin_17.xlsx::row::2")
private String rowToken;
@Schema(description = "主数据分组 key。Python 应从解析结果原样透传,用于把同组子行补回。", example = "uploads/20260426/similar_asin_17.xlsx::2@2")
private String groupKey;
@Schema(description = "Excel 中的 id。代表行通常是整数 id 或 n_1例如 2_1最终生成 xlsx 时2_2、2_3 会复用同组 2_1 的 Coze 检测结果。", example = "2_1")
private String id;
@Schema(description = "亚马逊 ASIN。后端会统一按大写处理和匹配。", example = "B0CJ8SNXXV")
private String asin;
@Schema(description = "站点或国家。来自 Excel 的国家列,例如英国、德国、法国。", example = "英国")
private String country;
@Schema(description = "兼容旧链路的首个图片 URL。新链路允许 url 传字符串或数组;数组会同步写入 urls。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg")
private String url;
@Schema(description = "商品图片 URL 列表,最多保留 24 个非空地址。请求中也可以直接把 url 传成数组。", example = "[\"https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg\"]")
private List<String> urls = new ArrayList<>();
@Schema(description = "商品标题。Java 调用 Coze 时会放入 title_list为空时会回退使用 ASIN。", example = "Women Floral Dress Summer Casual")
@JsonAlias({"productTitle", "product_title", "itemTitle", "item_title", "商品标题", "商品名称", "标题"})
private String title;
@Schema(description = "单行错误信息。通常用于记录 Python 单行处理异常Coze 失败时后端会尽量保留原始行,不强行写入风险结果。", example = "图片地址为空")
private String error;
@Schema(description = "单行完成标记。当前主要使用请求体顶层 done 控制任务收尾,该字段仅作兼容。", example = "true")
private Boolean done;
@Schema(description = "Java 调用 Coze 后生成的标题维度检测结果,对应最终 xlsx 的“标题维度商标”列。Python 回传请求中不要传该字段;即使传入,后端也会以 Java/Coze 处理结果为准。", example = "标题未发现明显商标侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
private String titleRisk;
@Schema(description = "Java 调用 Coze 后生成的外观维度检测结果,对应最终 xlsx 的“外观维度外观设计专利”列。Python 回传请求中不要传该字段。", example = "未发现明显外观设计专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
private String appearanceRisk;
@JsonAlias({"patent ", "patent"})
@Schema(description = "Java 调用 Coze 后生成的专利维度检测结果,对应最终 xlsx 的“专利维度(发明/实用新型专利)”列。兼容 Coze 返回字段 patent 和 patent 后带空格的情况Python 回传请求中不要传该字段。", example = "未发现明显发明或实用新型专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
private String patentRisk;
@Schema(description = "Java 调用 Coze 后生成的最终结论,对应最终 xlsx 的“结论”列。Python 回传请求中不要传该字段。", example = "未发现明显侵权风险", accessMode = Schema.AccessMode.READ_ONLY)
private String conclusion;
@JsonAlias({"title_reason", "titleReason"})
@Schema(description = "Coze title reason", accessMode = Schema.AccessMode.READ_ONLY)
private String titleReason;
@JsonAlias({"appearance_reason", "appearanceReason"})
@Schema(description = "Coze appearance reason", accessMode = Schema.AccessMode.READ_ONLY)
private String appearanceReason;
@JsonAlias({"patent_reason", "patentReason", "patent reason"})
@Schema(description = "Coze patent reason", accessMode = Schema.AccessMode.READ_ONLY)
private String patentReason;
public String getUrl() {
if (url != null && !url.isBlank()) {
return url;
}
List<String> normalized = getUrls();
return normalized.isEmpty() ? "" : normalized.get(0);
}
@JsonSetter("url")
@JsonAlias({
"imageUrl", "image_url", "imgUrl", "img_url", "pictureUrl", "picture_url",
"mainImage", "main_image", "mainImageUrl", "main_image_url",
"productImage", "product_image", "productImageUrl", "product_image_url",
"image", "img", "pic", "picture", "link", "imageLink", "image_link"
})
public void setUrl(Object value) {
List<String> normalized = normalizeUrls(value);
this.urls = normalized;
this.url = normalized.isEmpty() ? "" : normalized.get(0);
}
public List<String> getUrls() {
List<String> normalized = normalizeUrls(urls);
if (url != null && !url.isBlank() && normalized.stream().noneMatch(url.trim()::equals)) {
List<String> combined = new ArrayList<>(normalized.size() + 1);
combined.add(url.trim());
combined.addAll(normalized);
return limitUrls(combined);
}
return normalized;
}
@JsonSetter("urls")
@JsonAlias({
"imageUrls", "image_urls", "imgUrls", "img_urls", "pictureUrls", "picture_urls",
"mainImages", "main_images", "mainImageUrls", "main_image_urls",
"productImages", "product_images", "productImageUrls", "product_image_urls",
"images", "imgs", "pics", "pictures", "links", "imageLinks", "image_links",
"图片链接", "商品图片", "商品主图", "主图", "主图链接"
})
public void setUrls(Object urls) {
List<String> normalized = normalizeUrls(urls);
this.urls = normalized;
this.url = normalized.isEmpty() ? "" : normalized.get(0);
}
public boolean hasImageUrl() {
return !getUrls().isEmpty();
}
private static List<String> normalizeUrls(Object value) {
List<String> result = new ArrayList<>();
appendUrls(result, value);
return limitUrls(result);
}
private static void appendUrls(List<String> result, Object value) {
if (value == null) {
return;
}
if (value instanceof JsonNode node) {
if (node.isArray()) {
node.forEach(child -> appendUrls(result, child));
return;
}
if (node.isNull() || node.isMissingNode()) {
return;
}
appendUrls(result, node.asText(""));
return;
}
if (value instanceof Collection<?> collection) {
collection.forEach(item -> appendUrls(result, item));
return;
}
Class<?> valueClass = value.getClass();
if (valueClass.isArray()) {
int length = Array.getLength(value);
for (int i = 0; i < length; i++) {
appendUrls(result, Array.get(value, i));
}
return;
}
String text = String.valueOf(value).trim();
if (!text.isBlank()) {
result.add(text);
}
}
private static List<String> limitUrls(List<String> values) {
Set<String> deduplicated = new LinkedHashSet<>();
if (values != null) {
for (String value : values) {
if (value == null || value.isBlank()) {
continue;
}
deduplicated.add(value.trim());
if (deduplicated.size() >= MAX_IMAGE_URLS) {
break;
}
}
}
return new ArrayList<>(deduplicated);
}
}

View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "相似ASIN检测源文件信息")
public class SimilarAsinSourceFileDto {
@Schema(description = "上传接口返回的临时文件 key。后端根据该 key 查找本地临时 Excel 文件并解析。", example = "uploads/20260426/similar_asin_17.xlsx", requiredMode = Schema.RequiredMode.REQUIRED)
private String fileKey;
@Schema(description = "原始文件名。用于历史记录展示和最终结果文件命名。", example = "17.xlsx")
private String originalFilename;
@Schema(description = "相对目录路径。当前仅记录来源相似ASIN检测不依赖该字段处理。", example = "xlsx/17.xlsx")
private String relativePath;
}

View File

@@ -0,0 +1,32 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "Python 回传相似ASIN检测结果请求")
public class SimilarAsinSubmitResultRequest {
@Schema(description = "本次 Python 回传的提交批次标识", example = "similar-asin-3938")
private String submissionId;
@Schema(description = "当前回传分片序号", example = "1")
private Integer chunkIndex;
@Schema(description = "本任务预计总分片数", example = "36")
private Integer chunkTotal;
@Schema(description = "是否为最后一次回传", example = "false")
private Boolean done;
@Schema(description = "Python 侧任务级错误信息", example = "浏览器执行异常,任务提前结束")
private String error;
@Schema(description = "本次回传的分组结果列表,推荐优先使用")
private List<SimilarAsinResultGroupDto> groups = new ArrayList<>();
@Schema(description = "兼容旧链路的平铺结果列表")
private List<SimilarAsinResultRowDto> items = new ArrayList<>();
}

View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测批量进度查询请求")
public class SimilarAsinTaskBatchRequest {
@NotEmpty
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询,避免每个任务单独请求。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds;
}

View File

@@ -0,0 +1,17 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "相似ASIN检测总览统计")
public class SimilarAsinDashboardVo {
@Schema(description = "运行中任务数量。这里统计 RUNNING 状态任务。", example = "1")
private Long pendingTaskCount;
@Schema(description = "已结束任务数量,等于成功任务数加失败任务数。", example = "12")
private Long processedTaskCount;
@Schema(description = "成功任务数量。", example = "10")
private Long successTaskCount;
@Schema(description = "失败任务数量。", example = "2")
private Long failedTaskCount;
}

View File

@@ -0,0 +1,37 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "相似ASIN检测历史记录项")
public class SimilarAsinHistoryItemVo {
@Schema(description = "结果记录 ID。删除历史、下载结果时使用。", example = "1001")
private Long resultId;
@Schema(description = "任务 ID。", example = "3938")
private Long taskId;
@Schema(description = "源 Excel 文件名。", example = "17.xlsx")
private String sourceFilename;
@Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx")
private String resultFilename;
@Schema(description = "最终结果文件下载地址。后端基于 OSS objectKey 生成的新鲜预签名 URL。", example = "https://bucket.oss-cn-hangzhou.aliyuncs.com/result/similar_asin/xxx/17-result.xlsx?Expires=...")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
private Integer fileProgressPercent;
private Integer fileProgressCurrent;
private Integer fileProgressTotal;
private String fileProgressMessage;
@Schema(description = "任务状态PENDING=已解析待推送RUNNING=执行中SUCCESS=成功FAILED=失败。", example = "SUCCESS")
private String taskStatus;
@Schema(description = "结果是否成功。true 表示任务完成并生成结果文件false 表示失败或未完成。", example = "true")
private Boolean success;
@Schema(description = "错误信息。任务失败时返回,例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final similar ASIN result")
private String error;
@Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716")
private Integer rowCount;
@Schema(description = "历史记录创建时间ISO 本地时间字符串。", example = "2026-04-26T10:30:00")
private String createdAt;
}

View File

@@ -0,0 +1,14 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测历史记录列表")
public class SimilarAsinHistoryVo {
@Schema(description = "历史记录项列表,默认返回最近 100 条。")
private List<SimilarAsinHistoryItemVo> items = new ArrayList<>();
}

View File

@@ -0,0 +1,41 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测解析结果")
public class SimilarAsinParseVo {
@Schema(description = "新创建的任务 ID", example = "3938")
private Long taskId;
@Schema(description = "来源 Excel 文件名,多个文件时为聚合展示文案", example = "17.xlsx 等 2 个文件")
private String sourceFilename;
@Schema(description = "参与本次解析的源文件数量", example = "2")
private Integer sourceFileCount;
@Schema(description = "Excel 中检测到的非空数据总行数", example = "716")
private Integer totalRows;
@Schema(description = "解析出的有效行数", example = "458")
private Integer acceptedRows;
@Schema(description = "因缺少必要字段而被丢弃的行数", example = "12")
private Integer droppedRows;
@Schema(description = "按相邻主 ID 块生成的分组数", example = "36")
private Integer groupCount;
@Schema(description = "本任务最终使用的 AI 提示词")
private String aiPrompt;
@Schema(description = "兼容旧前端的平铺有效行列表,现为全部有效行")
private List<SimilarAsinParsedRowVo> items = new ArrayList<>();
@Schema(description = "返回前端和推送 Python 的分组列表")
private List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
}

View File

@@ -0,0 +1,32 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测解析后的分组数据")
public class SimilarAsinParsedGroupVo {
@Schema(description = "来源文件 key")
private String sourceFileKey;
@Schema(description = "来源文件名")
private String sourceFilename;
@Schema(description = "相邻主 ID 块的分组 key")
private String groupKey;
@Schema(description = "主 ID例如 1、2、10")
private String baseId;
@Schema(description = "分组首条展示 ID例如 1 或 2_1")
private String displayId;
@Schema(description = "分组内行数")
private Integer itemCount;
@Schema(description = "分组内全部行,顺序与原 Excel 保持一致")
private List<SimilarAsinParsedRowVo> items = new ArrayList<>();
}

View File

@@ -0,0 +1,47 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.LinkedHashMap;
import java.util.Map;
@Data
@Schema(description = "相似ASIN检测解析出的单行数据")
public class SimilarAsinParsedRowVo {
@Schema(description = "来源文件 key。用于多文件场景下区分同名/同 ID 行。", example = "uploads/20260426/similar_asin_17.xlsx")
private String sourceFileKey;
@Schema(description = "来源文件名。用于调试和结果追踪。", example = "17.xlsx")
private String sourceFilename;
@Schema(description = "Excel 原始行号,从 1 开始。", example = "2")
private Integer rowIndex;
@Schema(description = "Excel 中原始 id 值。", example = "2_1")
private String sourceId;
@Schema(description = "前端展示和 Python 回传使用的 id。整数 id 原样保留,子数据第一条如 2_1 原样保留。", example = "2_1")
private String displayId;
@Schema(description = "行级唯一 token。多文件、重复主数据时用于精确匹配回传结果。", example = "uploads/20260426/similar_asin_17.xlsx::row::2")
private String rowToken;
@Schema(description = "同一主数据块的分组 key。用于把 2_1、2_2、2_3 等子行重新补回同一组结果。", example = "uploads/20260426/similar_asin_17.xlsx::2@2")
private String groupKey;
@Schema(description = "亚马逊 ASIN。", example = "B0CJ8SNXXV")
private String asin;
@Schema(description = "国家或站点。", example = "英国")
private String country;
@Schema(description = "商品图片 URL 或商品 URL供 Coze 检测使用。", example = "https://webstatic.aiproxy.vip/output/demo.jpg")
private String url;
@Schema(description = "商品标题。", example = "Women Floral Dress Summer Casual")
private String title;
@Schema(description = "该 Excel 行的原始列值映射。最终生成 xlsx 时可以从这里读取价格等字段。", example = "{\"id\":\"2_1\",\"asin\":\"B0CJ8SNXXV\",\"国家\":\"英国\",\"价格\":\"12.99\"}")
private Map<String, String> values = new LinkedHashMap<>();
}

View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测批量进度响应")
public class SimilarAsinTaskBatchVo {
@Schema(description = "查询到的任务详情列表。顺序按请求 taskIds 处理。")
private List<SimilarAsinTaskDetailVo> items = new ArrayList<>();
@Schema(description = "未找到或不属于相似ASIN检测模块的任务 ID 列表。", example = "[99999]")
private List<Long> missingTaskIds = new ArrayList<>();
}

View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测任务进度详情")
public class SimilarAsinTaskDetailVo {
@Schema(description = "任务主记录轻量信息。")
private SimilarAsinTaskItemVo task;
@Schema(description = "预留的任务明细列表。当前进度接口主要返回任务轻量状态,不返回完整结果明细。")
private List<SimilarAsinHistoryItemVo> items = new ArrayList<>();
}

View File

@@ -0,0 +1,23 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "相似ASIN检测任务轻量信息")
public class SimilarAsinTaskItemVo {
@Schema(description = "任务 ID。", example = "3938")
private Long id;
@Schema(description = "任务编号,后端自动生成。", example = "SIMILAR_ASIN-1780000000000000000")
private String taskNo;
@Schema(description = "任务状态PENDING=已解析待推送RUNNING=执行中SUCCESS=成功FAILED=失败。", example = "RUNNING")
private String status;
@Schema(description = "任务级错误信息。失败时返回。", example = "生成相似ASIN检测结果失败")
private String errorMessage;
@Schema(description = "创建时间ISO 本地时间字符串。", example = "2026-04-26T10:00:00")
private String createdAt;
@Schema(description = "最后更新时间,通常由 Python 回传结果或任务收尾更新。", example = "2026-04-26T10:05:00")
private String updatedAt;
@Schema(description = "完成时间。任务未结束时为空。", example = "2026-04-26T10:10:00")
private String finishedAt;
}

View File

@@ -0,0 +1,144 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
@Service
@RequiredArgsConstructor
@Slf4j
public class SimilarAsinTaskCacheService {
private static final long TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
public void appendPendingRow(Long taskId, String scopeHash, Integer chunkIndex, SimilarAsinResultRowDto row) {
if (taskId == null || taskId <= 0 || scopeHash == null || scopeHash.isBlank() || chunkIndex == null || row == null) {
return;
}
try {
stringRedisTemplate.opsForList().rightPush(
pendingRowsKey(taskId),
objectMapper.writeValueAsString(new PendingRow(scopeHash, chunkIndex, row))
);
stringRedisTemplate.expire(pendingRowsKey(taskId), Duration.ofHours(TTL_HOURS));
touchTaskHeartbeat(taskId);
} catch (Exception ignored) {
}
}
public long pendingRowCount(Long taskId) {
Long size;
try {
size = stringRedisTemplate.opsForList().size(pendingRowsKey(taskId));
} catch (Exception ex) {
log.warn("[similar-asin-cache] pending row count degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
return size == null ? 0L : size;
}
public List<PendingRow> drainPendingRows(Long taskId, int limit) {
if (taskId == null || taskId <= 0 || limit <= 0) {
return List.of();
}
String key = pendingRowsKey(taskId);
List<String> values;
try {
values = stringRedisTemplate.opsForList().range(key, 0, limit - 1L);
} catch (Exception ex) {
log.warn("[similar-asin-cache] drain range degraded taskId={} msg={}", taskId, ex.getMessage());
return List.of();
}
if (values == null || values.isEmpty()) {
return List.of();
}
try {
stringRedisTemplate.opsForList().trim(key, values.size(), -1);
} catch (Exception ex) {
log.warn("[similar-asin-cache] drain trim degraded taskId={} msg={}", taskId, ex.getMessage());
return List.of();
}
List<PendingRow> rows = new ArrayList<>();
for (String value : values) {
if (value == null || value.isBlank()) {
continue;
}
try {
rows.add(objectMapper.readValue(value, PendingRow.class));
} catch (Exception ignored) {
}
}
touchTaskHeartbeat(taskId);
return rows;
}
public void touchTaskHeartbeat(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
try {
stringRedisTemplate.opsForValue().set(
heartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(TTL_HOURS)
);
} catch (Exception ex) {
log.warn("[similar-asin-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public long getTaskHeartbeatMillis(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(heartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[similar-asin-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
try {
return Long.parseLong(raw);
} catch (NumberFormatException ignored) {
return 0L;
}
}
public void deleteTaskCache(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
try {
stringRedisTemplate.delete(pendingRowsKey(taskId));
stringRedisTemplate.delete(heartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[similar-asin-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
private String pendingRowsKey(Long taskId) {
return "similar-asin:task:pending-rows:" + taskId;
}
private String heartbeatKey(Long taskId) {
return "similar-asin:task:heartbeat:" + taskId;
}
public record PendingRow(String scopeHash, Integer chunkIndex, SimilarAsinResultRowDto row) {
}
}

View File

@@ -14,9 +14,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -27,6 +25,8 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import java.net.URI;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@RequestMapping("/api/split") @RequestMapping("/api/split")
@@ -93,14 +93,14 @@ public class SplitRunController {
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "返回结果文件流"), @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "返回结果文件流"),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "结果文件不存在") @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "结果文件不存在")
}) })
public ResponseEntity<Resource> download( public ResponseEntity<Void> download(
@Parameter(description = "数据拆分结果记录 ID", required = true) @PathVariable Long resultId, @Parameter(description = "数据拆分结果记录 ID", required = true) @PathVariable Long resultId,
@Parameter(name = "user_id", description = "当前登录用户 ID", required = true, in = ParameterIn.QUERY) @Parameter(name = "user_id", description = "当前登录用户 ID", required = true, in = ParameterIn.QUERY)
@RequestParam("user_id") Long userId) { @RequestParam("user_id") Long userId) {
Resource resource = splitRunService.getResultFile(resultId, userId); String downloadUrl = splitRunService.getResultDownloadUrl(resultId, userId);
return ResponseEntity.ok() return ResponseEntity.status(302)
.contentType(MediaType.APPLICATION_OCTET_STREAM) .location(URI.create(downloadUrl))
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment") .header(HttpHeaders.CACHE_CONTROL, "no-store")
.body(resource); .build();
} }
} }

View File

@@ -19,8 +19,6 @@ import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.io.File; import java.io.File;
@@ -173,7 +171,7 @@ public class SplitRunService {
vo.setResultId(entity.getId()); vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getSourceFilename()); vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename()); vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(null); vo.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
vo.setRowCount(entity.getRowCount()); vo.setRowCount(entity.getRowCount());
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage()); vo.setError(entity.getErrorMessage());
@@ -192,17 +190,12 @@ public class SplitRunService {
fileResultMapper.deleteById(resultId); fileResultMapper.deleteById(resultId);
} }
public Resource getResultFile(Long resultId, Long userId) { public String getResultDownloadUrl(Long resultId, Long userId) {
FileResultEntity entity = fileResultMapper.selectById(resultId); FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("Split result file does not exist."); throw new BusinessException("Split result file does not exist.");
} }
return ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl());
File file = new File(entity.getResultFileUrl());
if (!file.exists()) {
throw new BusinessException("Split result file does not exist.");
}
return new FileSystemResource(file);
} }
private List<SplitChunkResult> splitExcelByLegacyRules( private List<SplitChunkResult> splitExcelByLegacyRules(

View File

@@ -19,6 +19,13 @@ public class TaskScopeStateEntity {
private String scopeHash; private String scopeHash;
private String parsedPayloadJson; private String parsedPayloadJson;
private String stateJson; private String stateJson;
private String cozeExecuteId;
private String cozeStatus;
private LocalDateTime cozeSubmittedAt;
private LocalDateTime cozeLastPolledAt;
private LocalDateTime cozeCompletedAt;
private Integer cozeAttemptCount;
private String cozeError;
private Integer chunkTotal; private Integer chunkTotal;
private Integer receivedChunkCount; private Integer receivedChunkCount;
private Integer completed; private Integer completed;

View File

@@ -4,6 +4,7 @@ import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@@ -18,8 +19,9 @@ public class TaskFileJobLocalDispatcher {
private final TaskFileJobMapper taskFileJobMapper; private final TaskFileJobMapper taskFileJobMapper;
private final ObjectProvider<TaskResultFileJobWorker> taskResultFileJobWorkerProvider; private final ObjectProvider<TaskResultFileJobWorker> taskResultFileJobWorkerProvider;
@Autowired
@Qualifier("taskFileJobDispatchExecutor") @Qualifier("taskFileJobDispatchExecutor")
private final TaskExecutor taskFileJobDispatchExecutor; private TaskExecutor taskFileJobDispatchExecutor;
@Value("${aiimage.result-file-job.local-dispatch-enabled:true}") @Value("${aiimage.result-file-job.local-dispatch-enabled:true}")
private boolean localDispatchEnabled; private boolean localDispatchEnabled;
@@ -28,26 +30,32 @@ public class TaskFileJobLocalDispatcher {
if (!localDispatchEnabled || jobId == null || jobId <= 0) { if (!localDispatchEnabled || jobId == null || jobId <= 0) {
return false; return false;
} }
taskFileJobDispatchExecutor.execute(() -> { try {
try { taskFileJobDispatchExecutor.execute(() -> {
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId); try {
if (job == null) { TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}", if (job == null) {
jobId, taskId, moduleType); log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}",
return; jobId, taskId, moduleType);
return;
}
TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable();
if (taskResultFileJobWorker == null) {
log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}",
jobId, taskId, moduleType);
return;
}
taskResultFileJobWorker.process(job);
} catch (Exception ex) {
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
jobId, taskId, moduleType, ex.getMessage(), ex);
} }
TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable(); });
if (taskResultFileJobWorker == null) { return true;
log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}", } catch (RuntimeException ex) {
jobId, taskId, moduleType); log.warn("[task-file-job] local dispatch executor rejected jobId={} taskId={} moduleType={} msg={}",
return; jobId, taskId, moduleType, ex.getMessage(), ex);
} return false;
taskResultFileJobWorker.process(job); }
} catch (Exception ex) {
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
jobId, taskId, moduleType, ex.getMessage(), ex);
}
});
return true;
} }
} }

View File

@@ -53,7 +53,7 @@ public class TaskFileJobService {
if (existing == null) { if (existing == null) {
return null; return null;
} }
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus())) { if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus()) || "PENDING".equals(existing.getStatus())) {
return existing; return existing;
} }
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>() taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
@@ -192,6 +192,11 @@ public class TaskFileJobService {
return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT); return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
} }
public boolean hasSuccessfulAssembleJob(Long taskId, String moduleType, Long resultId) {
TaskFileJobEntity job = findAssembleJob(taskId, moduleType, resultId);
return job != null && "SUCCESS".equals(job.getStatus());
}
public Map<Long, TaskFileJobEntity> findAssembleJobsByResultIds(String moduleType, List<Long> resultIds) { public Map<Long, TaskFileJobEntity> findAssembleJobsByResultIds(String moduleType, List<Long> resultIds) {
List<Long> normalizedIds = resultIds == null ? List.of() : resultIds.stream() List<Long> normalizedIds = resultIds == null ? List.of() : resultIds.stream()
.filter(id -> id != null && id > 0) .filter(id -> id != null && id > 0)
@@ -225,6 +230,18 @@ public class TaskFileJobService {
return count == null ? 0L : count; return count == null ? 0L : count;
} }
public long countActiveAssembleJobs(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
return 0L;
}
Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getTaskId, taskId)
.eq(TaskFileJobEntity::getModuleType, moduleType)
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "RUNNING")));
return count == null ? 0L : count;
}
@Transactional @Transactional
public void deleteTaskJobs(Long taskId, String moduleType) { public void deleteTaskJobs(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) { if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {

View File

@@ -8,12 +8,16 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService; import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService; import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService; import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -34,8 +38,12 @@ public class TaskResultFileJobWorker {
private final QueryAsinTaskService queryAsinTaskService; private final QueryAsinTaskService queryAsinTaskService;
private final PatrolDeleteTaskService patrolDeleteTaskService; private final PatrolDeleteTaskService patrolDeleteTaskService;
private final AppearancePatentTaskService appearancePatentTaskService; private final AppearancePatentTaskService appearancePatentTaskService;
private final SimilarAsinTaskService similarAsinTaskService;
private final DeleteBrandRunService deleteBrandRunService; private final DeleteBrandRunService deleteBrandRunService;
private final BrandTaskService brandTaskService; private final BrandTaskService brandTaskService;
@Autowired
@Qualifier("cozeTaskExecutor")
private TaskExecutor cozeTaskExecutor;
@Value("${aiimage.result-file-job.local-worker-enabled:true}") @Value("${aiimage.result-file-job.local-worker-enabled:true}")
private boolean localWorkerEnabled; private boolean localWorkerEnabled;
@@ -80,9 +88,29 @@ public class TaskResultFileJobWorker {
if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) { if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) {
return; return;
} }
if ("APPEARANCE_PATENT".equals(job.getModuleType()) || "SIMILAR_ASIN".equals(job.getModuleType())) {
try {
cozeTaskExecutor.execute(() -> processInternal(job));
return;
} catch (RuntimeException ex) {
log.warn("[task-file-job] coze module offload failed, fallback inline jobId={} taskId={} moduleType={} msg={}",
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex);
}
}
processInternal(job);
}
private void processInternal(TaskFileJobEntity job) {
long startedAt = System.currentTimeMillis(); long startedAt = System.currentTimeMillis();
try { try {
dispatch(job); boolean completed = dispatch(job);
if (!completed) {
taskFileJobService.touchRunning(job.getId());
log.info("[task-file-job] process deferred jobId={} taskId={} moduleType={} resultId={} elapsedMs={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
System.currentTimeMillis() - startedAt);
return;
}
String resultFileUrl = resolveResultFileUrl(job); String resultFileUrl = resolveResultFileUrl(job);
taskFileJobService.markSuccess(job, resultFileUrl); taskFileJobService.markSuccess(job, resultFileUrl);
cleanupAfterSuccess(job); cleanupAfterSuccess(job);
@@ -108,39 +136,41 @@ public class TaskResultFileJobWorker {
return result == null ? null : result.getResultFileUrl(); return result == null ? null : result.getResultFileUrl();
} }
private void dispatch(TaskFileJobEntity job) { private boolean dispatch(TaskFileJobEntity job) {
String moduleType = job.getModuleType(); String moduleType = job.getModuleType();
if ("SHOP_MATCH".equals(moduleType)) { if ("SHOP_MATCH".equals(moduleType)) {
shopMatchTaskService.processResultFileJob(job); shopMatchTaskService.processResultFileJob(job);
return; return true;
} }
if ("PRICE_TRACK".equals(moduleType)) { if ("PRICE_TRACK".equals(moduleType)) {
priceTrackTaskService.processResultFileJob(job); priceTrackTaskService.processResultFileJob(job);
return; return true;
} }
if ("PRODUCT_RISK_RESOLVE".equals(moduleType)) { if ("PRODUCT_RISK_RESOLVE".equals(moduleType)) {
productRiskTaskService.processResultFileJob(job); productRiskTaskService.processResultFileJob(job);
return; return true;
} }
if ("QUERY_ASIN".equals(moduleType)) { if ("QUERY_ASIN".equals(moduleType)) {
queryAsinTaskService.processResultFileJob(job); queryAsinTaskService.processResultFileJob(job);
return; return true;
} }
if ("PATROL_DELETE".equals(moduleType)) { if ("PATROL_DELETE".equals(moduleType)) {
patrolDeleteTaskService.processResultFileJob(job); patrolDeleteTaskService.processResultFileJob(job);
return; return true;
} }
if ("APPEARANCE_PATENT".equals(moduleType)) { if ("APPEARANCE_PATENT".equals(moduleType)) {
appearancePatentTaskService.processResultFileJob(job); return appearancePatentTaskService.processResultFileJob(job);
return; }
if ("SIMILAR_ASIN".equals(moduleType)) {
return similarAsinTaskService.processResultFileJob(job);
} }
if ("DELETE_BRAND".equals(moduleType)) { if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.processResultFileJob(job); deleteBrandRunService.processResultFileJob(job);
return; return true;
} }
if ("BRAND".equals(moduleType)) { if ("BRAND".equals(moduleType)) {
brandTaskService.processResultFileJob(job); brandTaskService.processResultFileJob(job);
return; return true;
} }
throw new IllegalArgumentException("unsupported result file job module: " + moduleType); throw new IllegalArgumentException("unsupported result file job module: " + moduleType);
} }
@@ -159,6 +189,10 @@ public class TaskResultFileJobWorker {
appearancePatentTaskService.cleanupResultFileJob(job); appearancePatentTaskService.cleanupResultFileJob(job);
return; return;
} }
if ("SIMILAR_ASIN".equals(moduleType)) {
similarAsinTaskService.cleanupResultFileJob(job);
return;
}
if ("DELETE_BRAND".equals(moduleType)) { if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.cleanupResultFileJob(job); deleteBrandRunService.cleanupResultFileJob(job);
} }

View File

@@ -115,6 +115,17 @@ public class TaskResultItemService {
.eq(TaskResultItemEntity::getResultId, resultId)); .eq(TaskResultItemEntity::getResultId, resultId));
} }
@Transactional
public void deleteResultItemRowsOnly(Long taskId, String moduleType, Long resultId) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0) {
return;
}
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.eq(TaskResultItemEntity::getResultId, resultId));
}
@Transactional @Transactional
public void deleteTaskItems(Long taskId, String moduleType) { public void deleteTaskItems(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) { if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
@@ -134,6 +145,16 @@ public class TaskResultItemService {
.eq(TaskResultItemEntity::getModuleType, moduleType)); .eq(TaskResultItemEntity::getModuleType, moduleType));
} }
@Transactional
public void deleteTaskItemsRowsOnly(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return;
}
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType));
}
private void upsert(Long taskId, private void upsert(Long taskId,
String moduleType, String moduleType,
Long resultId, Long resultId,

View File

@@ -188,6 +188,16 @@ public class TaskScopePayloadStorageService {
} }
} }
@Transactional
public void deleteTaskScopePayloadRowsOnly(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return;
}
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType));
}
@Transactional @Transactional
public <T> void saveParsedPayloadMap(Long taskId, String moduleType, Map<String, T> payloadByScope) { public <T> void saveParsedPayloadMap(Long taskId, String moduleType, Map<String, T> payloadByScope) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || payloadByScope == null || payloadByScope.isEmpty()) { if (taskId == null || taskId <= 0 || isBlank(moduleType) || payloadByScope == null || payloadByScope.isEmpty()) {

View File

@@ -1,39 +1,57 @@
package com.nanri.aiimage.modules.task.service; package com.nanri.aiimage.modules.task.service;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.config.TransientStorageProperties; import com.nanri.aiimage.config.TransientStorageProperties;
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService; import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Locale; import java.util.Locale;
import java.util.UUID; import java.util.UUID;
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
@Slf4j
public class TransientPayloadStorageService { public class TransientPayloadStorageService {
private static final String LOCAL_POINTER_PREFIX = "local:";
private static final String RUSTFS_POINTER_PREFIX = "rustfs:"; private static final String RUSTFS_POINTER_PREFIX = "rustfs:";
private static final String OSS_POINTER_PREFIX = "oss:"; private static final String OSS_POINTER_PREFIX = "oss:";
private static final String LOCAL_PAYLOAD_DIR = "transient-payload";
private final TransientStorageProperties properties; private final TransientStorageProperties properties;
private final StorageProperties storageProperties;
private final RustfsObjectStorageService rustfsObjectStorageService; private final RustfsObjectStorageService rustfsObjectStorageService;
private final OssStorageService ossStorageService; private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
public boolean isWriteEnabled() { public boolean isWriteEnabled() {
return properties.isEnabled() && rustfsObjectStorageService.isConfigured(); return properties.isEnabled();
} }
public String storeScopePayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) { public String storeScopePayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
return store("task-scope", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString); return store("task-scope", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString);
} }
public String storeScopePayloadVersioned(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
return store("task-scope", moduleType, taskId, scopeHash, UUID.randomUUID().toString(), content, encodeAsJsonString);
}
public String storeParsedPayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) { public String storeParsedPayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
return store("task-parsed", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString); return store("task-parsed", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString);
} }
public String storeParsedPayloadFast(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
return store("task-parsed", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString, false);
}
public String storeResultPayload(String moduleType, Long taskId, String scopeHash, String submissionId, String content) { public String storeResultPayload(String moduleType, Long taskId, String scopeHash, String submissionId, String content) {
return store("task-result-payload", moduleType, taskId, scopeHash, submissionId, content, true); return store("task-result-payload", moduleType, taskId, scopeHash, submissionId, content, true);
} }
@@ -47,12 +65,22 @@ public class TransientPayloadStorageService {
return store("task-chunk", moduleType, taskId, scopeHash, entryKey, content, true); return store("task-chunk", moduleType, taskId, scopeHash, entryKey, content, true);
} }
public String storeChunkPayloadVersioned(String moduleType, Long taskId, String scopeHash, Integer chunkIndex, String content) {
String entryKey = chunkIndex == null
? UUID.randomUUID().toString()
: "chunk-" + chunkIndex + "-" + UUID.randomUUID();
return store("task-chunk", moduleType, taskId, scopeHash, entryKey, content, true);
}
public String resolvePayload(String value, String errorMessage) { public String resolvePayload(String value, String errorMessage) {
String pointer = extractPointer(value); String pointer = extractPointer(value);
if (pointer == null) { if (pointer == null) {
return value; return value;
} }
try { try {
if (pointer.startsWith(LOCAL_POINTER_PREFIX)) {
return readLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length()));
}
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) { if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
return rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length())); return rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
} }
@@ -70,6 +98,10 @@ public class TransientPayloadStorageService {
if (pointer == null) { if (pointer == null) {
return; return;
} }
if (pointer.startsWith(LOCAL_POINTER_PREFIX)) {
deleteLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length()));
return;
}
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) { if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
rustfsObjectStorageService.deleteObject(pointer.substring(RUSTFS_POINTER_PREFIX.length())); rustfsObjectStorageService.deleteObject(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
return; return;
@@ -92,19 +124,28 @@ public class TransientPayloadStorageService {
if (value == null || value.isBlank()) { if (value == null || value.isBlank()) {
return null; return null;
} }
if (isPointer(value)) { String candidate = value.trim();
return value; for (int i = 0; i < 4; i++) {
} if (isPointer(candidate)) {
try { return candidate;
String decoded = objectMapper.readValue(value, String.class); }
return isPointer(decoded) ? decoded : null; try {
} catch (Exception ignored) { String decoded = objectMapper.readValue(candidate, String.class);
return null; if (decoded == null || decoded.equals(candidate)) {
return null;
}
candidate = decoded.trim();
} catch (Exception ignored) {
return null;
}
} }
return null;
} }
private boolean isPointer(String value) { private boolean isPointer(String value) {
return value != null && (value.startsWith(RUSTFS_POINTER_PREFIX) || value.startsWith(OSS_POINTER_PREFIX)); return value != null && (value.startsWith(LOCAL_POINTER_PREFIX)
|| value.startsWith(RUSTFS_POINTER_PREFIX)
|| value.startsWith(OSS_POINTER_PREFIX));
} }
private String store(String category, private String store(String category,
@@ -114,11 +155,34 @@ public class TransientPayloadStorageService {
String entryKey, String entryKey,
String content, String content,
boolean encodeAsJsonString) { boolean encodeAsJsonString) {
return store(category, moduleType, taskId, scopeHash, entryKey, content, encodeAsJsonString, true);
}
private String store(String category,
String moduleType,
Long taskId,
String scopeHash,
String entryKey,
String content,
boolean encodeAsJsonString,
boolean verifyAfterUpload) {
if (!isWriteEnabled()) { if (!isWriteEnabled()) {
return content; return content;
} }
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey); String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
String pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content); String pointer = storeLocal(objectKey, content);
if (pointer == null && rustfsObjectStorageService.isConfigured()) {
try {
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content, verifyAfterUpload);
} catch (Exception ex) {
log.warn("[transient-payload] rustfs upload failed and local fallback unavailable objectKey={} err={}",
objectKey, ex.getMessage());
throw ex;
}
}
if (pointer == null) {
return content;
}
if (!encodeAsJsonString) { if (!encodeAsJsonString) {
return pointer; return pointer;
} }
@@ -129,6 +193,61 @@ public class TransientPayloadStorageService {
} }
} }
private String storeLocal(String objectKey, String content) {
try {
Path root = localPayloadRoot();
Path target = root.resolve(objectKey).normalize();
if (!target.startsWith(root)) {
throw new IllegalArgumentException("invalid local payload key: " + objectKey);
}
Files.createDirectories(target.getParent());
Files.writeString(target,
content == null ? "" : content,
StandardCharsets.UTF_8,
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE);
return LOCAL_POINTER_PREFIX + objectKey;
} catch (Exception ex) {
log.warn("[transient-payload] local store failed objectKey={} err={}", objectKey, ex.getMessage());
return null;
}
}
private String readLocalPayload(String objectKey) {
try {
Path target = resolveLocalPayloadPath(objectKey);
return Files.readString(target, StandardCharsets.UTF_8);
} catch (Exception ex) {
throw new IllegalStateException("failed to read local transient payload", ex);
}
}
private void deleteLocalPayload(String objectKey) {
try {
Files.deleteIfExists(resolveLocalPayloadPath(objectKey));
} catch (Exception ex) {
log.warn("[transient-payload] delete local payload failed objectKey={} err={}", objectKey, ex.getMessage());
}
}
private Path resolveLocalPayloadPath(String objectKey) {
Path root = localPayloadRoot();
Path target = root.resolve(objectKey == null ? "" : objectKey).normalize();
if (!target.startsWith(root)) {
throw new IllegalArgumentException("invalid local payload key: " + objectKey);
}
return target;
}
private Path localPayloadRoot() {
String configuredRoot = storageProperties.getLocalTempDir();
Path tempRoot = configuredRoot == null || configuredRoot.isBlank()
? Path.of(System.getProperty("java.io.tmpdir"))
: Path.of(configuredRoot);
return tempRoot.resolve(LOCAL_PAYLOAD_DIR).normalize();
}
private String buildObjectKey(String category, String moduleType, Long taskId, String scopeHash, String entryKey) { private String buildObjectKey(String category, String moduleType, Long taskId, String scopeHash, String entryKey) {
String normalizedCategory = normalize(category, "unknown"); String normalizedCategory = normalize(category, "unknown");
String normalizedModuleType = normalize(moduleType, "unknown"); String normalizedModuleType = normalize(moduleType, "unknown");

View File

@@ -22,4 +22,6 @@ public class ZiniaoShopIndexEntryDto {
private List<Long> sampleUserIds = new ArrayList<>(); private List<Long> sampleUserIds = new ArrayList<>();
private Long lastSeenAt; private Long lastSeenAt;
private Long lastRefreshedAt; private Long lastRefreshedAt;
private Long lastRefreshBlockedAt;
private String refreshBlockedReason;
} }

View File

@@ -52,6 +52,7 @@ public class ZiniaoShopIndexService {
private static final Duration DEFAULT_CURSOR_TTL = Duration.ofHours(24); private static final Duration DEFAULT_CURSOR_TTL = Duration.ofHours(24);
private static final int DEFAULT_REFRESH_BATCH_SIZE = 100; private static final int DEFAULT_REFRESH_BATCH_SIZE = 100;
private static final int SHOP_INDEX_LIST_LIMIT = 10000; private static final int SHOP_INDEX_LIST_LIMIT = 10000;
private static final String REFRESH_BLOCKED_REASON_IP_WHITELIST = "IP_WHITELIST";
private final ZiniaoMemoryStoreService ziniaoMemoryStoreService; private final ZiniaoMemoryStoreService ziniaoMemoryStoreService;
private final ZiniaoTransientCacheService ziniaoTransientCacheService; private final ZiniaoTransientCacheService ziniaoTransientCacheService;
@@ -339,6 +340,9 @@ public class ZiniaoShopIndexService {
cursor.setStatus("FAILED"); cursor.setStatus("FAILED");
cursor.setMessage(ex.getMessage()); cursor.setMessage(ex.getMessage());
cursor.setLastFinishedAt(now); cursor.setLastFinishedAt(now);
if (isIpWhitelistRefreshFailure(ex)) {
markExistingEntriesRefreshBlocked(now, REFRESH_BLOCKED_REASON_IP_WHITELIST, ex.getMessage());
}
ziniaoTransientCacheService.put(CACHE_TYPE_SHOP_INDEX_REFRESH_CURSOR, "global", cursor, DEFAULT_CURSOR_TTL); ziniaoTransientCacheService.put(CACHE_TYPE_SHOP_INDEX_REFRESH_CURSOR, "global", cursor, DEFAULT_CURSOR_TTL);
log.warn("[ziniao-index] refresh failed status=FAILED msg={}", ex.getMessage()); log.warn("[ziniao-index] refresh failed status=FAILED msg={}", ex.getMessage());
if (ex instanceof BusinessException businessException) { if (ex instanceof BusinessException businessException) {
@@ -559,6 +563,11 @@ public class ZiniaoShopIndexService {
if (entry.getLastRefreshedAt() == null || entry.getLastRefreshedAt() <= 0) { if (entry.getLastRefreshedAt() == null || entry.getLastRefreshedAt() <= 0) {
return false; return false;
} }
if (REFRESH_BLOCKED_REASON_IP_WHITELIST.equals(entry.getRefreshBlockedReason())
&& entry.getLastRefreshBlockedAt() != null
&& entry.getLastRefreshBlockedAt() >= entry.getLastRefreshedAt()) {
return true;
}
ZiniaoShopIndexRefreshCursorDto cursor = getRefreshCursor(); ZiniaoShopIndexRefreshCursorDto cursor = getRefreshCursor();
if (cursor == null || !"FAILED".equals(cursor.getStatus())) { if (cursor == null || !"FAILED".equals(cursor.getStatus())) {
return false; return false;
@@ -571,6 +580,49 @@ public class ZiniaoShopIndexService {
return lastFinishedAt != null && lastFinishedAt >= entry.getLastRefreshedAt(); return lastFinishedAt != null && lastFinishedAt >= entry.getLastRefreshedAt();
} }
private boolean isIpWhitelistRefreshFailure(Exception ex) {
if (ex instanceof BusinessException businessException
&& ziniaoAuthService.isIpWhitelistError(businessException)) {
return true;
}
String message = ex == null ? null : ex.getMessage();
return message != null && message.contains("白名单");
}
private void markExistingEntriesRefreshBlocked(long now, String reason, String message) {
List<ZiniaoMemoryStoreEntity> entities = ziniaoMemoryStoreService.listAliveEntitiesByType(
ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY,
SHOP_INDEX_LIST_LIMIT
);
if (entities.isEmpty()) {
return;
}
int updated = 0;
for (ZiniaoMemoryStoreEntity entity : entities) {
String rowKey = entity.getCacheKey();
ZiniaoShopIndexEntryDto existingEntry;
try {
existingEntry = objectMapper.readValue(entity.getPayloadJson(), ZiniaoShopIndexEntryDto.class);
} catch (Exception ex) {
log.warn("[ziniao-index] skip refresh-block mark corrupt shop_index row cacheKey={}", rowKey);
continue;
}
if (existingEntry == null || !STATUS_ACTIVE.equals(existingEntry.getStatus())) {
continue;
}
existingEntry.setLastRefreshBlockedAt(now);
existingEntry.setRefreshBlockedReason(reason);
if (message != null && !message.isBlank()) {
existingEntry.setMessage(message);
}
ziniaoMemoryStoreService.put(ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY, rowKey, existingEntry, resolveEntryTtl());
updated++;
}
if (updated > 0) {
log.warn("[ziniao-index] refresh blocked reason={} existing active rows kept usable count={}", reason, updated);
}
}
private Duration resolveEntryTtl() { private Duration resolveEntryTtl() {
Integer ttlHours = ziniaoProperties.getShopIndexEntryTtlHours(); Integer ttlHours = ziniaoProperties.getShopIndexEntryTtlHours();
if (ttlHours == null || ttlHours <= 0) { if (ttlHours == null || ttlHours <= 0) {

View File

@@ -34,6 +34,14 @@ AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE=10
AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS=60000 AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS=60000
AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES=20 AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES=20
AIIMAGE_SIMILAR_ASIN_COZE_BASE_URL=https://api.coze.cn
AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_PATH=/v1/workflow/run
AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_ID=7632683471312355338
AIIMAGE_SIMILAR_ASIN_COZE_TOKEN=
AIIMAGE_SIMILAR_ASIN_COZE_BATCH_SIZE=10
AIIMAGE_SIMILAR_ASIN_COZE_READ_TIMEOUT_MILLIS=60000
AIIMAGE_SIMILAR_ASIN_STALE_TIMEOUT_MINUTES=20
AIIMAGE_ROCKETMQ_NAME_SERVER=121.196.149.225:9876 AIIMAGE_ROCKETMQ_NAME_SERVER=121.196.149.225:9876
AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED=true AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED=true
AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_ENABLED=true AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_ENABLED=true

View File

@@ -94,6 +94,7 @@ aiimage:
cleanup-cron: ${AIIMAGE_STORAGE_CLEANUP_CRON:0 0 * * * *} cleanup-cron: ${AIIMAGE_STORAGE_CLEANUP_CRON:0 0 * * * *}
source-retention-hours: ${AIIMAGE_STORAGE_SOURCE_RETENTION_HOURS:24} source-retention-hours: ${AIIMAGE_STORAGE_SOURCE_RETENTION_HOURS:24}
result-retention-hours: ${AIIMAGE_STORAGE_RESULT_RETENTION_HOURS:24} result-retention-hours: ${AIIMAGE_STORAGE_RESULT_RETENTION_HOURS:24}
transient-payload-retention-hours: ${AIIMAGE_STORAGE_TRANSIENT_PAYLOAD_RETENTION_HOURS:72}
temp-dir: temp-dir:
retention-hours: ${AIIMAGE_TEMP_DIR_RETENTION_HOURS:24} retention-hours: ${AIIMAGE_TEMP_DIR_RETENTION_HOURS:24}
brand-progress: brand-progress:
@@ -103,6 +104,7 @@ aiimage:
stale-check-cron: ${AIIMAGE_BRAND_PROGRESS_STALE_CHECK_CRON:0 */2 * * * *} stale-check-cron: ${AIIMAGE_BRAND_PROGRESS_STALE_CHECK_CRON:0 */2 * * * *}
delete-brand-progress: delete-brand-progress:
heartbeat-timeout-minutes: ${AIIMAGE_DELETE_BRAND_HEARTBEAT_TIMEOUT_MINUTES:15} heartbeat-timeout-minutes: ${AIIMAGE_DELETE_BRAND_HEARTBEAT_TIMEOUT_MINUTES:15}
delete-brand-initial-timeout-minutes: ${AIIMAGE_DELETE_BRAND_INITIAL_TIMEOUT_MINUTES:20}
stale-check-cron: ${AIIMAGE_DELETE_BRAND_STALE_CHECK_CRON:*/30 * * * * *} stale-check-cron: ${AIIMAGE_DELETE_BRAND_STALE_CHECK_CRON:*/30 * * * * *}
finalize-check-cron: ${AIIMAGE_DELETE_BRAND_FINALIZE_CHECK_CRON:30 */2 * * * *} finalize-check-cron: ${AIIMAGE_DELETE_BRAND_FINALIZE_CHECK_CRON:30 */2 * * * *}
product-risk-stale-timeout-minutes: ${AIIMAGE_PRODUCT_RISK_STALE_TIMEOUT_MINUTES:20} product-risk-stale-timeout-minutes: ${AIIMAGE_PRODUCT_RISK_STALE_TIMEOUT_MINUTES:20}
@@ -148,6 +150,16 @@ aiimage:
coze-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS:60000} coze-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS:60000}
stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:20} stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:20}
stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *} stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *}
similar-asin:
coze-base-url: ${AIIMAGE_SIMILAR_ASIN_COZE_BASE_URL:https://api.coze.cn}
coze-workflow-path: ${AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_PATH:/v1/workflow/run}
coze-workflow-id: ${AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_ID:7632683471312355338}
coze-token: ${AIIMAGE_SIMILAR_ASIN_COZE_TOKEN:Bearer sat_gsInckdsN0qqhnZSK7K4Zevw6neJCHmYHyIbm85oGgJIGEmSdkH5OFctSKLsAHvT}
coze-batch-size: ${AIIMAGE_SIMILAR_ASIN_COZE_BATCH_SIZE:10}
coze-connect-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_CONNECT_TIMEOUT_MILLIS:10000}
coze-read-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_READ_TIMEOUT_MILLIS:60000}
stale-timeout-minutes: ${AIIMAGE_SIMILAR_ASIN_STALE_TIMEOUT_MINUTES:20}
stale-finalize-cron: ${AIIMAGE_SIMILAR_ASIN_STALE_FINALIZE_CRON:0 */2 * * * *}
security: security:
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key} shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
internal-token: ${AIIMAGE_INTERNAL_TOKEN:} internal-token: ${AIIMAGE_INTERNAL_TOKEN:}

View File

@@ -0,0 +1,35 @@
SET @schema_name = DATABASE();
SET @idx_file_result_patrol_history_exists := (
SELECT COUNT(1)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_file_result'
AND INDEX_NAME = 'idx_file_result_patrol_history'
);
SET @sql_add_file_result_patrol_history_idx := IF(
@idx_file_result_patrol_history_exists = 0,
'ALTER TABLE biz_file_result ADD INDEX idx_file_result_patrol_history (module_type, user_id, created_at, id)',
'SELECT 1'
);
PREPARE stmt_add_file_result_patrol_history_idx FROM @sql_add_file_result_patrol_history_idx;
EXECUTE stmt_add_file_result_patrol_history_idx;
DEALLOCATE PREPARE stmt_add_file_result_patrol_history_idx;
SET @idx_file_task_patrol_history_exists := (
SELECT COUNT(1)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_file_task'
AND INDEX_NAME = 'idx_file_task_patrol_history'
);
SET @sql_add_file_task_patrol_history_idx := IF(
@idx_file_task_patrol_history_exists = 0,
'ALTER TABLE biz_file_task ADD INDEX idx_file_task_patrol_history (module_type, id, status, finished_at)',
'SELECT 1'
);
PREPARE stmt_add_file_task_patrol_history_idx FROM @sql_add_file_task_patrol_history_idx;
EXECUTE stmt_add_file_task_patrol_history_idx;
DEALLOCATE PREPARE stmt_add_file_task_patrol_history_idx;

View File

@@ -0,0 +1,127 @@
SET @coze_execute_id_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_execute_id'
);
SET @sql_add_coze_execute_id := IF(
@coze_execute_id_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_execute_id VARCHAR(128) NULL COMMENT ''Coze execute id'' AFTER state_json',
'SELECT 1'
);
PREPARE stmt_add_coze_execute_id FROM @sql_add_coze_execute_id;
EXECUTE stmt_add_coze_execute_id;
DEALLOCATE PREPARE stmt_add_coze_execute_id;
SET @coze_status_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_status'
);
SET @sql_add_coze_status := IF(
@coze_status_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_status VARCHAR(32) NULL COMMENT ''Coze workflow status'' AFTER coze_execute_id',
'SELECT 1'
);
PREPARE stmt_add_coze_status FROM @sql_add_coze_status;
EXECUTE stmt_add_coze_status;
DEALLOCATE PREPARE stmt_add_coze_status;
SET @coze_submitted_at_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_submitted_at'
);
SET @sql_add_coze_submitted_at := IF(
@coze_submitted_at_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_submitted_at DATETIME NULL COMMENT ''Coze submitted time'' AFTER coze_status',
'SELECT 1'
);
PREPARE stmt_add_coze_submitted_at FROM @sql_add_coze_submitted_at;
EXECUTE stmt_add_coze_submitted_at;
DEALLOCATE PREPARE stmt_add_coze_submitted_at;
SET @coze_last_polled_at_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_last_polled_at'
);
SET @sql_add_coze_last_polled_at := IF(
@coze_last_polled_at_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_last_polled_at DATETIME NULL COMMENT ''Coze last polled time'' AFTER coze_submitted_at',
'SELECT 1'
);
PREPARE stmt_add_coze_last_polled_at FROM @sql_add_coze_last_polled_at;
EXECUTE stmt_add_coze_last_polled_at;
DEALLOCATE PREPARE stmt_add_coze_last_polled_at;
SET @coze_completed_at_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_completed_at'
);
SET @sql_add_coze_completed_at := IF(
@coze_completed_at_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_completed_at DATETIME NULL COMMENT ''Coze completed time'' AFTER coze_last_polled_at',
'SELECT 1'
);
PREPARE stmt_add_coze_completed_at FROM @sql_add_coze_completed_at;
EXECUTE stmt_add_coze_completed_at;
DEALLOCATE PREPARE stmt_add_coze_completed_at;
SET @coze_attempt_count_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_attempt_count'
);
SET @sql_add_coze_attempt_count := IF(
@coze_attempt_count_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_attempt_count INT NOT NULL DEFAULT 0 COMMENT ''Coze poll attempt count'' AFTER coze_completed_at',
'SELECT 1'
);
PREPARE stmt_add_coze_attempt_count FROM @sql_add_coze_attempt_count;
EXECUTE stmt_add_coze_attempt_count;
DEALLOCATE PREPARE stmt_add_coze_attempt_count;
SET @coze_error_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND COLUMN_NAME = 'coze_error'
);
SET @sql_add_coze_error := IF(
@coze_error_exists = 0,
'ALTER TABLE biz_task_scope_state ADD COLUMN coze_error VARCHAR(1000) NULL COMMENT ''Coze error message'' AFTER coze_attempt_count',
'SELECT 1'
);
PREPARE stmt_add_coze_error FROM @sql_add_coze_error;
EXECUTE stmt_add_coze_error;
DEALLOCATE PREPARE stmt_add_coze_error;
SET @idx_task_coze_status_exists := (
SELECT COUNT(1)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_task_scope_state'
AND INDEX_NAME = 'idx_task_coze_status'
);
SET @sql_add_idx_task_coze_status := IF(
@idx_task_coze_status_exists = 0,
'ALTER TABLE biz_task_scope_state ADD INDEX idx_task_coze_status (module_type, coze_status, updated_at)',
'SELECT 1'
);
PREPARE stmt_add_idx_task_coze_status FROM @sql_add_idx_task_coze_status;
EXECUTE stmt_add_idx_task_coze_status;
DEALLOCATE PREPARE stmt_add_idx_task_coze_status;

View File

@@ -0,0 +1,15 @@
SET @idx_file_task_delete_brand_history_updated_exists := (
SELECT COUNT(1)
FROM INFORMATION_SCHEMA.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_task'
AND INDEX_NAME = 'idx_file_task_delete_brand_history_updated'
);
SET @sql_add_idx_file_task_delete_brand_history_updated := IF(
@idx_file_task_delete_brand_history_updated_exists = 0,
'ALTER TABLE biz_file_task ADD INDEX idx_file_task_delete_brand_history_updated (module_type, user_id, updated_at)',
'SELECT 1'
);
PREPARE stmt_add_idx_file_task_delete_brand_history_updated FROM @sql_add_idx_file_task_delete_brand_history_updated;
EXECUTE stmt_add_idx_file_task_delete_brand_history_updated;
DEALLOCATE PREPARE stmt_add_idx_file_task_delete_brand_history_updated;

View File

@@ -0,0 +1,14 @@
-- Similar ASIN detection reuses the generic file task/result/chunk tables.
-- Register the frontend app menu column so permission queries can expose the page.
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
SELECT '相似ASIN检测', 'similar_asin', 'app', 'similar-asin', 112
WHERE NOT EXISTS (
SELECT 1 FROM `columns` WHERE `column_key` = 'similar_asin'
);
UPDATE `columns`
SET `name` = '相似ASIN检测',
`menu_type` = 'app',
`route_path` = 'similar-asin',
`sort_order` = 112
WHERE `column_key` = 'similar_asin';

View File

@@ -0,0 +1,12 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>相似ASIN检测</title>
</head>
<body>
<div id="app"></div>
<script type="module" src="/src/similar-asin-main.ts"></script>
</body>
</html>

View File

@@ -331,8 +331,6 @@ async function parseFiles() {
}] }]
: []) : [])
queuePayloadText.value = '' queuePayloadText.value = ''
await loadDashboard()
await loadHistory({ force: true })
ElMessage.success(`解析完成,共 ${res.groupCount || 0} 组 / ${res.acceptedRows}`) ElMessage.success(`解析完成,共 ${res.groupCount || 0} 组 / ${res.acceptedRows}`)
} catch (e) { } catch (e) {
ElMessage.error(e instanceof Error ? e.message : '解析失败') ElMessage.error(e instanceof Error ? e.message : '解析失败')
@@ -465,7 +463,7 @@ function scheduleNextPoll(immediate = false) {
if (disposed) return if (disposed) return
if (!pollingTaskIds.value.length && !pendingFileTaskIds.value.length) return if (!pollingTaskIds.value.length && !pendingFileTaskIds.value.length) return
await refreshTaskProgress() await refreshTaskProgress()
if (!disposed && (pollingTaskIds.value.length || pendingFileTaskIds.value.length)) { if (!disposed && pollTimer.value == null && (pollingTaskIds.value.length || pendingFileTaskIds.value.length)) {
pollTimer.value = timers.setTimeout('task-poll', run, getTaskPollIntervalMs()) pollTimer.value = timers.setTimeout('task-poll', run, getTaskPollIntervalMs())
} }
} }

View File

@@ -215,6 +215,7 @@ import { expandBrandFolderRecursive, type BrandExpandFolderItem } from '@/shared
import { import {
deleteDeleteBrandHistory, deleteDeleteBrandHistory,
getDeleteBrandHistory, getDeleteBrandHistory,
getDeleteBrandResultDownloadUrl,
getDeleteBrandTaskDetails, getDeleteBrandTaskDetails,
getDeleteBrandTaskProgress, getDeleteBrandTaskProgress,
getDeleteBrandTaskDownloadUrl, getDeleteBrandTaskDownloadUrl,
@@ -269,8 +270,9 @@ const timers = createCategorizedTimers('delete-brand')
const displayPaths = computed(() => selectedPaths.value.slice(0, 10)) const displayPaths = computed(() => selectedPaths.value.slice(0, 10))
const currentSectionItems = computed(() => { const currentSectionItems = computed(() => {
const sortedTasks = [...sessionTasks.value].sort((left, right) => (right.createdAt || 0) - (left.createdAt || 0))
const allCurrent: DeleteBrandResultItem[] = []; const allCurrent: DeleteBrandResultItem[] = [];
sessionTasks.value.forEach(task => { sortedTasks.forEach(task => {
allCurrent.push(...task.items); allCurrent.push(...task.items);
}); });
return allCurrent; return allCurrent;
@@ -521,9 +523,10 @@ function shouldShowProgress(item: DeleteBrandResultItem) {
function normalizeDeleteBrandItems(items: DeleteBrandResultItem[]) { function normalizeDeleteBrandItems(items: DeleteBrandResultItem[]) {
return items.map((item) => { return items.map((item) => {
if (item.matched || item.matchStatus === 'MATCHED' || item.matchStatus === 'INDEX_STALE') { if (isUsableMatchedItem(item)) {
return { return {
...item, ...item,
matched: true,
_pushed: false, _pushed: false,
_completed: false, _completed: false,
} as SessionDeleteBrandItem } as SessionDeleteBrandItem
@@ -540,10 +543,10 @@ function normalizeDeleteBrandItems(items: DeleteBrandResultItem[]) {
} }
function formatMatchResult(item: DeleteBrandResultItem) { function formatMatchResult(item: DeleteBrandResultItem) {
if (item.matched && item.matchStatus === 'INDEX_STALE') { if (item.matchStatus === 'INDEX_STALE' && isUsableMatchedItem(item)) {
return item.matchMessage || `已匹配 ${item.shopId || ''}(索引过期)`.trim() return `已匹配 ${item.shopId || ''}(索引过期,可继续推送`.trim()
} }
if (item.matchStatus === 'MATCHED') { if (item.matchStatus === 'MATCHED' || item.matched) {
return `已匹配 ${item.shopId || ''}`.trim() return `已匹配 ${item.shopId || ''}`.trim()
} }
if (item.matchMessage) { if (item.matchMessage) {
@@ -749,6 +752,12 @@ function getQueueStatus(item: DeleteBrandResultItem) {
return '已被全局判定为终态' return '已被全局判定为终态'
} }
// 后端已经把单文件结果组装出来时,以文件终态为准。
// 多店铺同 taskId 场景下任务本身仍是 RUNNING不能因此挡住下一个店铺入队。
if (isFileResultReady(item) || isFileResultReady(sessionItem)) {
return '本文件解析完毕'
}
// 先看前端会话态:一旦该文件已判定完成,优先展示完成,避免被 RUNNING 覆盖导致链式推进卡住。 // 先看前端会话态:一旦该文件已判定完成,优先展示完成,避免被 RUNNING 覆盖导致链式推进卡住。
if (sessionItem?._completed) { if (sessionItem?._completed) {
return '本文件解析完毕' return '本文件解析完毕'
@@ -791,6 +800,21 @@ function canDownloadTaskResult(item: DeleteBrandResultItem) {
return false return false
} }
function isFileResultReady(item?: Partial<DeleteBrandResultItem> | null) {
if (!item) return false
const fileStatus = String(item.fileStatus || '').toUpperCase()
return Boolean(
item.fileReady
|| item.downloadUrl
|| fileStatus === 'SUCCESS'
|| fileStatus === 'COMPLETED',
)
}
function preferLatestValue<T>(latestValue: T | null | undefined, currentValue: T | null | undefined) {
return latestValue ?? currentValue
}
function mergeTaskDetailItemsIntoSession(detail: DeleteBrandTaskDetailVo) { function mergeTaskDetailItemsIntoSession(detail: DeleteBrandTaskDetailVo) {
const taskId = detail?.task?.id const taskId = detail?.task?.id
if (!taskId) return false if (!taskId) return false
@@ -821,10 +845,21 @@ function mergeTaskDetailItemsIntoSession(detail: DeleteBrandTaskDetailVo) {
const merged = { const merged = {
...item, ...item,
...latest, ...latest,
// progress/batch 返回的是轻量文件状态,店铺匹配信息和执行 payload 可能为空;
// 不能覆盖本地解析出的完整店铺数据,否则后续文件会失去入队资格。
shopName: preferLatestValue(latest.shopName, item.shopName),
companyName: preferLatestValue(latest.companyName, item.companyName),
shopId: preferLatestValue(latest.shopId, item.shopId),
platform: preferLatestValue(latest.platform, item.platform),
openStoreUrl: preferLatestValue(latest.openStoreUrl, item.openStoreUrl),
matchStatus: preferLatestValue(latest.matchStatus, item.matchStatus),
matchMessage: preferLatestValue(latest.matchMessage, item.matchMessage),
matched: isUsableMatchedItem(item) || isUsableMatchedItem(latest),
countryCount: preferLatestValue(latest.countryCount, item.countryCount),
countries, countries,
previewRows, previewRows,
_pushed: item._pushed, _pushed: item._pushed,
_completed: item._completed, _completed: item._completed || isFileResultReady(latest),
} as SessionDeleteBrandItem } as SessionDeleteBrandItem
if (JSON.stringify(merged) !== JSON.stringify(item)) { if (JSON.stringify(merged) !== JSON.stringify(item)) {
changed = true changed = true
@@ -837,7 +872,7 @@ function mergeTaskDetailItemsIntoSession(detail: DeleteBrandTaskDetailVo) {
mergedItems.push({ mergedItems.push({
...latest, ...latest,
_pushed: false, _pushed: false,
_completed: false, _completed: isFileResultReady(latest),
} as SessionDeleteBrandItem) } as SessionDeleteBrandItem)
changed = true changed = true
} }
@@ -956,6 +991,8 @@ function ensurePolling(immediate = false) {
} }
function getTaskStatusInfo(item: DeleteBrandResultItem) { function getTaskStatusInfo(item: DeleteBrandResultItem) {
if (item.fileReady || item.downloadUrl) return { className: 'success', text: '已完成' }
// 1. 优先查实时详情中的状态 // 1. 优先查实时详情中的状态
const status = item.taskId ? taskDetails.value[item.taskId]?.task?.status : null const status = item.taskId ? taskDetails.value[item.taskId]?.task?.status : null
if (status === 'SUCCESS') return { className: 'success', text: '已完成' } if (status === 'SUCCESS') return { className: 'success', text: '已完成' }
@@ -1053,13 +1090,9 @@ async function submitRun() {
})), })),
}) })
const normalizedItems = normalizeDeleteBrandItems(result.items || []) const normalizedItems = normalizeDeleteBrandItems(result.items || [])
const hasRunnableItems = normalizedItems.some((item) => const hasRunnableItems = normalizedItems.some((item) => isUsableMatchedItem(item))
item.matchStatus === 'MATCHED' || item.matchStatus === 'INDEX_STALE' || item.matched
)
const hasStaleMatchedItems = normalizedItems.some((item) => item.matchStatus === 'INDEX_STALE' && item.matched) const hasStaleMatchedItems = normalizedItems.some((item) => item.matchStatus === 'INDEX_STALE' && item.matched)
const hasBlockedItems = normalizedItems.some((item) => const hasBlockedItems = normalizedItems.some((item) => !isUsableMatchedItem(item))
!item.matched && item.matchStatus !== 'MATCHED' && item.matchStatus !== 'INDEX_STALE'
)
queuePushResult.value = hasRunnableItems queuePushResult.value = hasRunnableItems
? '解析完成,可在左侧点击“推送到 Python 队列”开始串行处理' ? '解析完成,可在左侧点击“推送到 Python 队列”开始串行处理'
@@ -1097,7 +1130,7 @@ async function downloadTaskResult(item: DeleteBrandResultItem) {
return return
} }
const api = getPywebviewApi() const api = getPywebviewApi()
const url = getDeleteBrandTaskDownloadUrl(taskId) const url = item.resultId ? getDeleteBrandResultDownloadUrl(item.resultId) : getDeleteBrandTaskDownloadUrl(taskId)
const filename = const filename =
item.outputFilename || item.outputFilename ||
item.sourceFilename || item.sourceFilename ||
@@ -1219,8 +1252,24 @@ function getItemKey(item: DeleteBrandResultItem) {
return `task:${item.taskId || 0}:${item.sourceFilename || ''}` return `task:${item.taskId || 0}:${item.sourceFilename || ''}`
} }
function isUsableMatchedItem(item: DeleteBrandResultItem) {
return Boolean(
item.matched
|| item.matchStatus === 'MATCHED'
|| (item.matchStatus === 'INDEX_STALE' && item.shopId),
)
}
function hasRunnableDeleteBrandPayload(item: DeleteBrandResultItem) {
return Boolean(
item.shopName
&& Array.isArray(item.countries)
&& item.countries.length > 0,
)
}
function canPushToQueue(item: DeleteBrandResultItem) { function canPushToQueue(item: DeleteBrandResultItem) {
return Boolean(item.taskId && (item.matchStatus === 'MATCHED' || item.matchStatus === 'INDEX_STALE' || item.matched)) return Boolean(item.taskId && (isUsableMatchedItem(item) || hasRunnableDeleteBrandPayload(item)))
} }
function findNextAutoRunnableItem() { function findNextAutoRunnableItem() {

View File

@@ -0,0 +1,764 @@
<template>
<div class="page-shell module-page">
<BrandTopBar active="similar-asin" />
<div class="main-content">
<aside class="left-panel">
<div class="section-title">上传文件</div>
<div class="upload-zone">
<div class="hint">选择 Excel 后点击解析后端会提取 idASIN国家并保留整数 id 与子数据的第一条</div>
<div class="btns">
<button type="button" class="opt-btn" @click="selectFiles">选择 Excel</button>
<button type="button" class="opt-btn" @click="selectFolder">选择文件夹</button>
</div>
<div class="selected-files">
<span v-if="!selectedFileNames.length">暂未选择文件</span>
<span v-for="name in selectedFileNames" v-else :key="name">{{ name }}</span>
</div>
</div>
<div class="prompt-card">
<div class="section-title">AI 提示词</div>
<textarea v-model="aiPrompt" class="prompt-input" rows="8" placeholder="可选,留空时使用下方默认提示词" />
<div class="prompt-default-label">留空时默认使用以下提示词</div>
<div class="prompt-preview">{{ defaultAiPrompt }}</div>
</div>
<div class="run-row">
<button type="button" class="btn-run" :disabled="parsing || !uploadedFiles.length" @click="parseFiles">
{{ parsing ? '解析中...' : '解析并创建任务' }}
</button>
<button type="button" class="btn-run btn-queue" :disabled="pushing || !parsedRows.length" @click="pushToPythonQueue">
{{ pushing ? '推送中...' : '推送到 Python 队列' }}
</button>
</div>
<p class="loading-msg">Python 回传字段按 asin国家url标题提交后端接收分片后按 10 条一批调用 Coze</p>
<div v-if="parseResult" class="parse-card">
<div>任务 ID{{ parseResult.taskId }}</div>
<div>总行数{{ parseResult.totalRows }}有效{{ parseResult.acceptedRows }}分组{{ parseResult.groupCount || 0 }}过滤{{ parseResult.droppedRows }}</div>
</div>
<pre v-if="queuePayloadText" class="queue-payload">{{ queuePayloadText }}</pre>
</aside>
<section class="right-panel">
<div class="panel-header">相似ASIN检测</div>
<div class="task-list-wrap">
<div class="clean-result-summary">
<div class="summary-card">
<span class="summary-label">运行中任务</span>
<strong>{{ dashboard.pendingTaskCount }}</strong>
</div>
<div class="summary-card">
<span class="summary-label">已结束任务</span>
<strong>{{ dashboard.processedTaskCount }}</strong>
</div>
<div class="summary-card">
<span class="summary-label">成功任务</span>
<strong>{{ dashboard.successTaskCount }}</strong>
</div>
<div class="summary-card">
<span class="summary-label">失败任务</span>
<strong>{{ dashboard.failedTaskCount }}</strong>
</div>
</div>
<div class="subsection-title">匹配任务</div>
<div class="result-list-wrap preview-wrap">
<div class="result-list-header">
<span>解析结果</span>
<span v-if="parsedRows.length" class="muted">显示首组 {{ previewRows.length }} / {{ parseResult?.groupCount || parsedGroups.length }} {{ parsedRows.length }} </span>
</div>
<div v-if="!parsedRows.length" class="empty-tasks">暂无解析结果</div>
<el-table v-else :data="previewRows" height="120" class="result-table">
<el-table-column prop="displayId" label="ID" width="90" />
<el-table-column prop="asin" label="ASIN" width="130" />
<el-table-column prop="country" label="国家" width="100" />
<el-table-column prop="title" label="标题" min-width="160" show-overflow-tooltip />
<el-table-column prop="url" label="URL" min-width="160" show-overflow-tooltip />
</el-table>
</div>
<div class="result-list-wrap">
<div class="result-list-header">
<span>当前任务</span>
</div>
<div v-if="!currentItems.length" class="empty-tasks">暂无当前任务</div>
<ul v-else class="task-list clean-result-list">
<li v-for="item in currentItems" :key="`cur-${item.taskId}`" class="task-item">
<div class="left">
<span class="id">{{ item.sourceFilename || '相似ASIN检测' }}</span>
<div class="files">任务 ID{{ item.taskId }}</div>
<div class="files">行数{{ item.rowCount ?? '-' }}</div>
</div>
<div class="task-right">
<span class="status running">{{ statusText(item) }}</span>
<button type="button" class="btn-delete" @click="deleteTaskRecord(item)">删除</button>
</div>
</li>
</ul>
</div>
<div class="result-list-wrap">
<div class="result-list-header">
<span>历史记录</span>
</div>
<div v-if="!historyOnlyItems.length" class="empty-tasks">暂无历史记录</div>
<ul v-else class="task-list clean-result-list">
<li v-for="item in historyOnlyItems" :key="`his-${item.resultId}-${item.taskId}`" class="task-item">
<div class="left">
<span class="id">{{ item.sourceFilename || '相似ASIN检测' }}</span>
<div class="files">任务 ID{{ item.taskId ?? '-' }}</div>
<div v-if="item.resultFilename" class="files">
{{ item.resultFilename || '下载结果' }}
</div>
<div v-if="pendingResultHint(item)" class="files result-hint">{{ pendingResultHint(item) }}</div>
<div v-if="showFileProgress(item)" class="file-progress">
<div class="file-progress-meta">
<span>{{ item.fileProgressMessage || '结果生成中' }}</span>
<span>{{ fileProgressPercent(item) }}%</span>
</div>
<div class="file-progress-track">
<div class="file-progress-bar" :style="{ width: `${fileProgressPercent(item)}%` }"></div>
</div>
<div class="file-progress-count">
{{ item.fileProgressCurrent || 0 }}/{{ item.fileProgressTotal || 0 }}
</div>
</div>
<div v-if="item.error" class="files">错误{{ item.error }}</div>
</div>
<div class="task-right">
<span class="status" :class="statusClass(item)">{{ statusText(item) }}</span>
<button v-if="canDownload(item)" type="button" class="download" @click="downloadResult(item)">下载</button>
<button v-if="item.resultId" type="button" class="btn-delete" @click="deleteTaskRecord(item)">删除</button>
</div>
</li>
</ul>
</div>
</div>
</section>
</div>
</div>
</template>
<script setup lang="ts">
import { computed, onMounted, onUnmounted, ref } from 'vue'
import { ElMessage } from 'element-plus'
import BrandTopBar from '@/pages/brand/components/BrandTopBar.vue'
import {
activateSimilarAsinTask,
deleteSimilarAsinHistory,
deleteSimilarAsinTask,
getSimilarAsinDashboard,
getSimilarAsinHistory,
getSimilarAsinResultDownloadUrl,
getSimilarAsinTaskProgressBatch,
parseSimilarAsin,
type SimilarAsinParsedGroup,
type SimilarAsinDashboardVo,
type SimilarAsinHistoryItem,
type SimilarAsinParsedRow,
type SimilarAsinParseVo,
type UploadedFileRef,
type UploadFileVo,
} from '@/shared/api/java-modules'
import { expandBrandFolderRecursive, type BrandExpandFolderItem } from '@/shared/api/brand'
import { getPywebviewApi } from '@/shared/bridges/pywebview'
import { getTaskPollIntervalMs } from '@/shared/task-progress-config'
import { createCategorizedTimers } from '@/shared/utils/categorized-timers'
const selectedFileNames = ref<string[]>([])
const uploadedFiles = ref<UploadFileVo[]>([])
const parseResult = ref<SimilarAsinParseVo | null>(null)
const parsedGroups = ref<SimilarAsinParsedGroup[]>([])
const defaultAiPrompt = `请帮我排查以下亚马逊商品在英国及欧洲地区是否存在知识产权侵权风险。
请直接给出明确结论(有侵权风险 或 未发现明显侵权风险),并严格按照以下三个维度提供精简的排查理由:
外观维度: 评估产品外形、图案设计是否与欧洲/英国常见外观专利雷同。
专利维度: 评估产品的核心技术或物理结构是否存在侵权可能。
标题维度: 排查标题中是否包含大牌商标、敏感词或版权保护词汇。
结论XX`
const aiPrompt = ref('')
const parsing = ref(false)
const pushing = ref(false)
const queuePayloadText = ref('')
const pollingTaskIds = ref<number[]>([])
const pendingFileTaskIds = ref<number[]>([])
const pollTimer = ref<number | null>(null)
const pollingInFlight = ref(false)
const HISTORY_CACHE_TTL_MS = 3000
let historyInFlight: Promise<void> | null = null
let lastHistoryLoadedAt = 0
let disposed = false
const timers = createCategorizedTimers('similar-asin')
const dashboard = ref<SimilarAsinDashboardVo>({
pendingTaskCount: 0,
processedTaskCount: 0,
successTaskCount: 0,
failedTaskCount: 0,
})
const historyItems = ref<SimilarAsinHistoryItem[]>([])
const parsedRows = computed<SimilarAsinParsedRow[]>(() =>
parsedGroups.value.flatMap((group) => group.items || []),
)
const previewRows = computed(() => parsedGroups.value[0]?.items || [])
const currentItems = computed(() => historyItems.value.filter((i) => i.taskId != null && pollingTaskIds.value.includes(i.taskId)))
const historyOnlyItems = computed(() =>
historyItems.value.filter((i) => {
const status = normalizeTaskStatus(i)
return status !== 'RUNNING' && status !== 'PENDING'
}),
)
function effectiveAiPrompt() {
return aiPrompt.value.trim() || defaultAiPrompt
}
function uidForStorage() {
return typeof window !== 'undefined' ? window.localStorage.getItem('uid') || '0' : '0'
}
function pollingKey() {
return `similar-asin:tasks:${uidForStorage()}`
}
function savePollingIds() {
if (typeof window === 'undefined') return
const ids = Array.from(new Set([...pollingTaskIds.value, ...pendingFileTaskIds.value]))
if (!ids.length) window.localStorage.removeItem(pollingKey())
else window.localStorage.setItem(pollingKey(), JSON.stringify(ids))
}
function loadPollingIds() {
if (typeof window === 'undefined') return
try {
const raw = window.localStorage.getItem(pollingKey())
const ids = raw ? JSON.parse(raw) : []
pollingTaskIds.value = Array.isArray(ids)
? ids.filter((id): id is number => typeof id === 'number' && id > 0)
: []
} catch {
pollingTaskIds.value = []
}
}
async function uploadAppearancePathsToJava(paths: Array<string | BrandExpandFolderItem>) {
const api = getPywebviewApi()
if (!api?.upload_file_to_java) {
throw new Error('当前桌面端未提供文件上传能力')
}
const files: UploadFileVo[] = []
for (const item of paths) {
const filePath = typeof item === 'string' ? item : item.absolutePath
const relativePath = typeof item === 'string' ? undefined : item.relativePath
const uploaded = await api.upload_file_to_java(filePath, relativePath)
if (!uploaded?.success || !uploaded.data) {
throw new Error(uploaded?.error || uploaded?.message || `上传失败:${filePath}`)
}
files.push(uploaded.data)
}
return files
}
async function selectFiles() {
const api = getPywebviewApi()
if (!api?.select_brand_xlsx_files || !api.upload_file_to_java) {
ElMessage.warning('当前环境不支持文件选择或上传')
return
}
const paths = await api.select_brand_xlsx_files()
if (!paths?.length) return
const files = await uploadAppearancePathsToJava(paths)
uploadedFiles.value = files
selectedFileNames.value = files.map((f) => f.relativePath || f.originalFilename || f.fileKey)
parseResult.value = null
parsedGroups.value = []
queuePayloadText.value = ''
}
async function selectFolder() {
const api = getPywebviewApi()
if (!api?.select_brand_folder) {
ElMessage.warning('当前环境不支持文件夹选择,请在本地客户端中打开')
return
}
try {
const folder = await api.select_brand_folder()
if (!folder) return
const result = await expandBrandFolderRecursive(folder)
if (!result.success || !result.items?.length) {
ElMessage.warning(result.error || '该文件夹下没有可用的 xlsx 文件')
return
}
const files = await uploadAppearancePathsToJava(result.items)
uploadedFiles.value = files
selectedFileNames.value = result.items.map((item) => item.relativePath || item.absolutePath)
parseResult.value = null
parsedGroups.value = []
queuePayloadText.value = ''
ElMessage.success(`已选择文件夹内 ${result.items.length} 个 Excel 文件`)
} catch (error) {
ElMessage.error(error instanceof Error ? error.message : '选择失败')
}
}
async function parseFiles() {
if (!uploadedFiles.value.length) {
ElMessage.warning('请先选择 Excel')
return
}
parsing.value = true
try {
const files: UploadedFileRef[] = uploadedFiles.value.map((f) => ({
fileKey: f.fileKey,
originalFilename: f.originalFilename,
relativePath: f.relativePath,
}))
const res = await parseSimilarAsin(files, effectiveAiPrompt())
parseResult.value = res
parsedGroups.value = res.groups?.length
? res.groups
: (res.items?.length
? [{
sourceFileKey: res.items[0]?.sourceFileKey,
sourceFilename: res.items[0]?.sourceFilename,
groupKey: res.items[0]?.groupKey,
baseId: '',
displayId: res.items[0]?.displayId,
itemCount: res.items.length,
items: res.items,
}]
: [])
queuePayloadText.value = ''
ElMessage.success(`解析完成,共 ${res.groupCount || 0} 组 / ${res.acceptedRows}`)
} catch (e) {
ElMessage.error(e instanceof Error ? e.message : '解析失败')
} finally {
parsing.value = false
}
}
async function pushToPythonQueue() {
const api = getPywebviewApi()
const taskId = parseResult.value?.taskId
if (!taskId || !parsedRows.value.length) {
ElMessage.warning('请先解析文件')
return
}
if (!api?.enqueue_json) {
ElMessage.error('当前环境未启用 pywebview enqueue_json')
return
}
pushing.value = true
try {
const payload = {
type: 'similar-asin-run',
ts: Date.now(),
data: {
taskId,
prompt: parseResult.value?.aiPrompt || effectiveAiPrompt(),
groups: parsedGroups.value.map((group) => ({
sourceFileKey: group.sourceFileKey || '',
sourceFilename: group.sourceFilename || '',
groupKey: group.groupKey || '',
baseId: group.baseId || '',
displayId: group.displayId || '',
items: (group.items || []).map((row) => ({
sourceFileKey: row.sourceFileKey || '',
sourceFilename: row.sourceFilename || '',
rowToken: row.rowToken || '',
groupKey: row.groupKey || '',
id: row.displayId,
asin: row.asin,
country: row.country,
url: row.url || '',
title: row.title || '',
})),
})),
rows: parsedRows.value.map((row) => ({
sourceFileKey: row.sourceFileKey || '',
sourceFilename: row.sourceFilename || '',
rowToken: row.rowToken || '',
groupKey: row.groupKey || '',
id: row.displayId,
asin: row.asin,
country: row.country,
url: row.url || '',
title: row.title || '',
})),
},
}
queuePayloadText.value = JSON.stringify(payload, null, 2)
await activateSimilarAsinTask(taskId)
const result = await api.enqueue_json(payload)
if (!result?.success) {
ElMessage.error(result?.error || '推送失败')
return
}
addPollingTask(taskId)
clearParsedTask()
await loadDashboard()
await loadHistory({ force: true })
ElMessage.success('已推送到 Python 队列')
} finally {
pushing.value = false
}
}
function clearParsedTask() {
parseResult.value = null
parsedGroups.value = []
queuePayloadText.value = ''
}
function addPollingTask(taskId: number) {
if (!pollingTaskIds.value.includes(taskId)) {
pollingTaskIds.value = [...pollingTaskIds.value, taskId]
savePollingIds()
}
ensurePolling()
}
function removePollingTask(taskId: number) {
pollingTaskIds.value = pollingTaskIds.value.filter((id) => id !== taskId)
savePollingIds()
}
function addPendingFileTask(taskId: number) {
if (!pendingFileTaskIds.value.includes(taskId)) {
pendingFileTaskIds.value = [...pendingFileTaskIds.value, taskId]
savePollingIds()
}
ensurePolling()
}
function removePendingFileTask(taskId: number) {
pendingFileTaskIds.value = pendingFileTaskIds.value.filter((id) => id !== taskId)
savePollingIds()
}
function ensurePolling() {
if (disposed) return
if (pollTimer.value != null) return
scheduleNextPoll(true)
}
function stopPolling() {
if (pollTimer.value != null) {
timers.clearTimer('task-poll', pollTimer.value)
pollTimer.value = null
}
}
function scheduleNextPoll(immediate = false) {
if (disposed) return
if (pollTimer.value != null) {
if (!immediate) return
timers.clearTimer('task-poll', pollTimer.value)
pollTimer.value = null
}
const run = async () => {
pollTimer.value = null
if (disposed) return
if (!pollingTaskIds.value.length && !pendingFileTaskIds.value.length) return
await refreshTaskProgress()
if (!disposed && pollTimer.value == null && (pollingTaskIds.value.length || pendingFileTaskIds.value.length)) {
pollTimer.value = timers.setTimeout('task-poll', run, getTaskPollIntervalMs())
}
}
if (immediate) void run()
else pollTimer.value = timers.setTimeout('task-poll', run, getTaskPollIntervalMs())
}
async function refreshTaskProgress() {
if (pollingInFlight.value || (!pollingTaskIds.value.length && !pendingFileTaskIds.value.length)) {
if (!pollingTaskIds.value.length && !pendingFileTaskIds.value.length) stopPolling()
return
}
pollingInFlight.value = true
try {
let shouldRefreshDashboard = false
let shouldRefreshHistory = false
const taskIds = Array.from(new Set([...pollingTaskIds.value, ...pendingFileTaskIds.value]))
if (taskIds.length) {
const batch = await getSimilarAsinTaskProgressBatch(taskIds, { force: pendingFileTaskIds.value.length > 0 })
for (const detail of batch.items || []) {
const task = detail.task
if (!task?.id) continue
const item = detail.items?.[0]
if (item) mergeHistoryItem(item)
if (task.status === 'SUCCESS' || task.status === 'FAILED') {
removePollingTask(task.id)
shouldRefreshDashboard = true
if (task.status === 'SUCCESS') addPendingFileTask(task.id)
else removePendingFileTask(task.id)
}
if (item && task.status === 'SUCCESS' && pendingFileTaskIds.value.includes(task.id)) {
const fileStatus = (item.fileStatus || '').toUpperCase()
if (item.fileReady || fileStatus === 'SUCCESS' || fileStatus === 'FAILED') {
removePendingFileTask(task.id)
shouldRefreshDashboard = true
shouldRefreshHistory = true
}
}
}
if (batch.missingTaskIds?.length) {
batch.missingTaskIds.forEach((taskId) => {
removePollingTask(taskId)
removePendingFileTask(taskId)
})
shouldRefreshHistory = true
shouldRefreshDashboard = true
}
}
if (shouldRefreshDashboard) {
await loadDashboard()
}
if (shouldRefreshHistory) {
await loadHistory()
}
settlePendingFileTasks()
} finally {
pollingInFlight.value = false
}
}
function mergeHistoryItem(item: SimilarAsinHistoryItem) {
if (!item.taskId && !item.resultId) return
const index = historyItems.value.findIndex((row) =>
(item.resultId != null && row.resultId === item.resultId)
|| (item.taskId != null && row.taskId === item.taskId),
)
if (index >= 0) {
historyItems.value[index] = { ...historyItems.value[index], ...item }
} else {
historyItems.value = [item, ...historyItems.value]
}
}
function settlePendingFileTasks() {
if (!pendingFileTaskIds.value.length) return
const remaining: number[] = []
for (const taskId of pendingFileTaskIds.value) {
const item = historyItems.value.find((row) => row.taskId === taskId)
if (!item) {
remaining.push(taskId)
continue
}
const taskStatus = (item.taskStatus || '').toUpperCase()
const fileStatus = (item.fileStatus || '').toUpperCase()
if (taskStatus === 'FAILED' || item.fileReady || fileStatus === 'SUCCESS' || fileStatus === 'FAILED') {
continue
}
if (taskStatus === 'SUCCESS') {
remaining.push(taskId)
}
}
pendingFileTaskIds.value = remaining
savePollingIds()
if (pendingFileTaskIds.value.length) ensurePolling()
}
function seedPendingFileTasksFromHistory() {
const ids = historyItems.value
.filter((item) => item.taskId != null && isResultPreparing(item))
.map((item) => item.taskId as number)
pendingFileTaskIds.value = Array.from(new Set(ids))
savePollingIds()
if (pendingFileTaskIds.value.length) ensurePolling()
}
async function loadDashboard() {
dashboard.value = await getSimilarAsinDashboard()
}
async function loadHistory(options: { force?: boolean } = {}) {
const now = Date.now()
if (!options.force && historyItems.value.length && now - lastHistoryLoadedAt < HISTORY_CACHE_TTL_MS) return
if (historyInFlight) return historyInFlight
historyInFlight = getSimilarAsinHistory()
.then((res) => {
historyItems.value = res.items || []
lastHistoryLoadedAt = Date.now()
})
.finally(() => {
historyInFlight = null
})
return historyInFlight
}
function statusText(item: SimilarAsinHistoryItem) {
const status = normalizeTaskStatus(item)
if (status === 'RUNNING') return '执行中'
if (status === 'PENDING') return '已解析待推送'
if (isResultPreparing(item)) return '结果生成中'
if (isResultBuildFailed(item)) return '结果生成失败'
if (status === 'SUCCESS' || item.success) return '已完成'
if (status === 'FAILED') return '失败'
return '等待中'
}
function statusClass(item: SimilarAsinHistoryItem) {
const status = normalizeTaskStatus(item)
if (status === 'RUNNING' || isResultPreparing(item)) return 'running'
if (isResultBuildFailed(item) || status === 'FAILED') return 'failed'
if (status === 'SUCCESS' || item.success) return 'success'
return 'pending'
}
function isResultPreparing(item: SimilarAsinHistoryItem) {
const status = normalizeTaskStatus(item)
const fileStatus = (item.fileStatus || '').toUpperCase()
if (status !== 'SUCCESS') return false
if (canDownload(item)) return false
if (fileStatus === 'FAILED' || fileStatus === 'SUCCESS') return false
return true
}
function normalizeTaskStatus(item: SimilarAsinHistoryItem) {
return (item.taskStatus || '').toUpperCase()
}
function isResultBuildFailed(item: SimilarAsinHistoryItem) {
return (item.fileStatus || '').toUpperCase() === 'FAILED'
}
function pendingResultHint(item: SimilarAsinHistoryItem) {
if (isResultBuildFailed(item)) {
return item.fileError || '结果文件生成失败,请稍后重试或检查后端日志'
}
if (!isResultPreparing(item)) return ''
const fileStatus = (item.fileStatus || '').toUpperCase()
if (fileStatus === 'RUNNING') {
return '任务已完成,正在请求 Coze 或组装结果文件,下载按钮稍后出现'
}
if (fileStatus === 'PENDING') {
return '任务已完成,结果文件已入队,正在等待后端生成'
}
return '任务已完成,正在生成结果文件,下载按钮稍后出现'
}
function canDownload(item: SimilarAsinHistoryItem) {
return Boolean(item.resultId && (item.fileReady || item.downloadUrl))
}
function fileProgressPercent(item: SimilarAsinHistoryItem) {
const percent = Number(item.fileProgressPercent || 0)
if (!Number.isFinite(percent)) return 0
return Math.max(0, Math.min(100, Math.round(percent)))
}
function showFileProgress(item: SimilarAsinHistoryItem) {
return isResultPreparing(item) && (item.fileProgressTotal || 0) > 0
}
async function downloadResult(item: SimilarAsinHistoryItem) {
if (!item.resultId) return
const api = getPywebviewApi()
if (!api?.save_file_from_url_new) {
ElMessage.error('当前客户端未提供下载能力')
return
}
const url = item.downloadUrl || getSimilarAsinResultDownloadUrl(item.resultId)
const filename = item.resultFilename || `${item.sourceFilename || 'similar-asin'}.xlsx`
const result = await api.save_file_from_url_new(url, filename)
if (result.success) {
ElMessage.success(`已保存: ${result.path || filename}`)
} else if (result.error && result.error !== '用户取消') {
ElMessage.error(result.error)
}
}
async function deleteTaskRecord(item: SimilarAsinHistoryItem) {
try {
if (item.taskStatus === 'RUNNING' && item.taskId) {
await deleteSimilarAsinTask(item.taskId)
removePollingTask(item.taskId)
} else if (item.resultId) {
await deleteSimilarAsinHistory(item.resultId)
}
await loadDashboard()
await loadHistory({ force: true })
ElMessage.success('已删除')
} catch (e) {
ElMessage.error(e instanceof Error ? e.message : '删除失败')
}
}
onMounted(async () => {
loadPollingIds()
await Promise.all([
loadDashboard().catch(() => undefined),
loadHistory().catch(() => undefined),
])
seedPendingFileTasksFromHistory()
if (pollingTaskIds.value.length || pendingFileTaskIds.value.length) ensurePolling()
})
onUnmounted(() => {
disposed = true
stopPolling()
timers.clearScope()
})
</script>
<style scoped>
.module-page { min-height: 100vh; background: #1a1a1a; }
.main-content { display: flex; height: calc(100vh - 56px); min-height: calc(100vh - 56px); }
.left-panel { width: 400px; background: #1e1e1e; padding: 20px; overflow-y: auto; border-right: 1px solid #2a2a2a; }
.right-panel { flex: 1; min-width: 0; background: #1a1a1a; display: flex; flex-direction: column; }
.section-title, .subsection-title { font-size: 13px; color: #bbb; margin-bottom: 10px; }
.upload-zone { border: 1px dashed #3a3a3a; border-radius: 10px; padding: 18px; background: #252525; margin-bottom: 18px; }
.hint, .loading-msg, .files, .muted { color: #888; font-size: 12px; line-height: 1.5; }
.link { color: #6ea8fe; text-decoration: none; }
.link:hover { color: #9fc5ff; }
.btns, .run-row { display: flex; gap: 10px; flex-wrap: wrap; }
.opt-btn, .btn-run, .btn-delete, .download { border: none; cursor: pointer; border-radius: 7px; }
.opt-btn { padding: 8px 14px; color: #ccc; background: #2a2a2a; border: 1px solid #3a3a3a; }
.btn-run { padding: 10px 18px; color: #fff; background: #3498db; font-weight: 600; }
.btn-queue { background: #27ae60; }
.btn-run:disabled { opacity: .55; cursor: not-allowed; }
.selected-files { margin-top: 14px; color: #999; font-size: 12px; word-break: break-all; }
.selected-files span { display: block; margin: 4px 0; }
.prompt-card { margin-bottom: 18px; }
.prompt-input { width: 100%; box-sizing: border-box; resize: vertical; min-height: 180px; padding: 10px 12px; border: 1px solid #333; border-radius: 8px; background: #202020; color: #d8d8d8; font-size: 12px; line-height: 1.6; outline: none; }
.prompt-input:focus { border-color: #3498db; }
.prompt-default-label { margin-top: 10px; color: #8d8d8d; font-size: 12px; }
.prompt-preview { margin-top: 10px; padding: 12px; border: 1px solid #2a2a2a; border-radius: 8px; background: #202020; color: #9ea7b3; font-size: 12px; line-height: 1.6; white-space: pre-wrap; }
.parse-card, .queue-payload { margin-top: 14px; padding: 12px; border: 1px solid #2a2a2a; border-radius: 8px; background: #202020; color: #b8c1cc; font-size: 12px; }
.queue-payload { max-height: 220px; overflow: auto; color: #8fd3ff; white-space: pre-wrap; }
.panel-header { padding: 16px 20px; border-bottom: 1px solid #2a2a2a; font-size: 15px; font-weight: 600; color: #ddd; }
.task-list-wrap { flex: 1; padding: 16px 20px; overflow: auto; }
.clean-result-summary { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; margin-bottom: 16px; }
.summary-card { padding: 14px 16px; border: 1px solid #2a2a2a; border-radius: 8px; background: #1e1e1e; }
.summary-card strong { display: block; margin-top: 8px; color: #eaf4ff; font-size: 22px; }
.summary-label { color: #8d8d8d; font-size: 12px; }
.result-list-wrap { border: 1px solid #2a2a2a; border-radius: 8px; background: #1e1e1e; min-height: 180px; margin: 0 0 16px; }
.result-list-header { display: flex; justify-content: space-between; padding: 12px 16px; border-bottom: 1px solid #2a2a2a; color: #ddd; font-size: 14px; }
.empty-tasks { color: #666; font-size: 13px; padding: 18px; text-align: center; }
.result-table { --el-table-bg-color: #222; --el-table-tr-bg-color: #222; --el-table-header-bg-color: #2a2a2a; --el-table-text-color: #ccc; --el-table-border-color: #333; }
.task-list { list-style: none; margin: 0; padding: 12px; }
.task-item { display: flex; align-items: flex-start; justify-content: space-between; gap: 12px; padding: 12px 14px; border: 1px solid #2a2a2a; border-radius: 8px; margin-bottom: 8px; background: #222; }
.left { flex: 1; min-width: 0; }
.id { color: #e0e0e0; font-size: 13px; font-weight: 600; }
.task-right { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; }
.status { padding: 4px 10px; border-radius: 6px; font-size: 12px; }
.status.success { background: rgba(46, 204, 113, .18); color: #2ecc71; }
.status.failed { background: rgba(231, 76, 60, .18); color: #ff6b6b; }
.status.running { background: rgba(52, 152, 219, .18); color: #3498db; }
.status.pending { background: rgba(149, 165, 166, .18); color: #bdc3c7; }
.result-hint { margin-top: 6px; color: #e0b96d; }
.file-progress { margin-top: 8px; max-width: 520px; }
.file-progress-meta { display: flex; justify-content: space-between; gap: 12px; color: #d8c278; font-size: 12px; }
.file-progress-track { margin-top: 5px; height: 8px; border-radius: 999px; overflow: hidden; background: #303030; border: 1px solid #3b3b3b; }
.file-progress-bar { height: 100%; border-radius: inherit; background: linear-gradient(90deg, #4aa3ff, #f0c75e); transition: width .25s ease; }
.file-progress-count { margin-top: 4px; color: #858585; font-size: 11px; }
.download { padding: 6px 10px; color: #d6ecff; background: rgba(52, 152, 219, .18); }
.btn-delete { padding: 6px 10px; color: #ff8f8f; background: rgba(231, 76, 60, .12); }
@media (max-width: 1100px) {
.main-content { flex-direction: column; height: auto; }
.left-panel { width: 100%; border-right: none; border-bottom: 1px solid #2a2a2a; }
.clean-result-summary { grid-template-columns: repeat(2, minmax(0, 1fr)); }
}
</style>

View File

@@ -46,6 +46,7 @@ import { getCurrentUserAppColumnKeys } from '@/shared/api/permission'
type ActiveNavKey = type ActiveNavKey =
| 'brand' | 'brand'
| 'appearance-patent' | 'appearance-patent'
| 'similar-asin'
| 'dedupe' | 'dedupe'
| 'convert' | 'convert'
| 'split' | 'split'
@@ -84,6 +85,7 @@ const navGroups: ReadonlyArray<NavGroup> = [
{ key: 'variant', label: '变体分析' }, { key: 'variant', label: '变体分析' },
{ key: 'brand', label: '品牌检测', href: '/brand' }, { key: 'brand', label: '品牌检测', href: '/brand' },
{ key: 'appearance-patent', label: '外观专利检测', href: '/new_web_source/appearance-patent.html' }, { key: 'appearance-patent', label: '外观专利检测', href: '/new_web_source/appearance-patent.html' },
{ key: 'similar-asin', label: '相似ASIN检测', href: '/new_web_source/similar-asin.html' },
{ key: 'dedupe', label: '数据去重', href: '/new_web_source/dedupe.html' }, { key: 'dedupe', label: '数据去重', href: '/new_web_source/dedupe.html' },
{ key: 'split', label: '数据拆分', href: '/new_web_source/split.html' }, { key: 'split', label: '数据拆分', href: '/new_web_source/split.html' },
{ key: 'convert', label: '格式转换', href: '/new_web_source/convert.html' }, { key: 'convert', label: '格式转换', href: '/new_web_source/convert.html' },

View File

@@ -309,6 +309,7 @@ export interface DeleteBrandResultItem {
fileKey?: string; fileKey?: string;
sourceFilename: string; sourceFilename: string;
shopName?: string; shopName?: string;
companyName?: string;
matched?: boolean; matched?: boolean;
matchStatus?: DeleteBrandMatchStatus; matchStatus?: DeleteBrandMatchStatus;
matchMessage?: string; matchMessage?: string;
@@ -577,6 +578,10 @@ export function getDeleteBrandTaskDownloadUrl(taskId: number) {
return getJavaDownloadUrl(`/delete-brand/tasks/${taskId}/download`); return getJavaDownloadUrl(`/delete-brand/tasks/${taskId}/download`);
} }
export function getDeleteBrandResultDownloadUrl(resultId: number) {
return getJavaDownloadUrl(`/delete-brand/results/${resultId}/download`);
}
export function submitDeleteBrandResult( export function submitDeleteBrandResult(
taskId: number, taskId: number,
request: DeleteBrandSubmitResultRequest, request: DeleteBrandSubmitResultRequest,
@@ -1598,6 +1603,169 @@ export function getAppearancePatentResultDownloadUrl(resultId: number) {
return getJavaDownloadUrl(`/appearance-patent/results/${resultId}/download`); return getJavaDownloadUrl(`/appearance-patent/results/${resultId}/download`);
} }
// ========== 相似ASIN检测 ==========
export interface SimilarAsinParsedRow {
rowIndex: number;
sourceFileKey?: string;
sourceFilename?: string;
sourceId: string;
displayId: string;
rowToken?: string;
groupKey?: string;
asin: string;
country: string;
url?: string;
title?: string;
}
export interface SimilarAsinParsedGroup {
sourceFileKey?: string;
sourceFilename?: string;
groupKey?: string;
baseId?: string;
displayId?: string;
itemCount?: number;
items: SimilarAsinParsedRow[];
}
export interface SimilarAsinParseVo {
taskId: number;
sourceFilename?: string;
sourceFileCount?: number;
totalRows: number;
acceptedRows: number;
droppedRows: number;
groupCount?: number;
aiPrompt?: string;
items: SimilarAsinParsedRow[];
groups?: SimilarAsinParsedGroup[];
}
export interface SimilarAsinDashboardVo {
pendingTaskCount: number;
processedTaskCount: number;
successTaskCount: number;
failedTaskCount: number;
}
export interface SimilarAsinHistoryItem {
resultId?: number;
taskId?: number;
sourceFilename?: string;
resultFilename?: string;
downloadUrl?: string;
fileJobId?: number;
fileStatus?: string;
fileError?: string;
fileReady?: boolean;
fileProgressPercent?: number;
fileProgressCurrent?: number;
fileProgressTotal?: number;
fileProgressMessage?: string;
taskStatus?: string;
success?: boolean;
error?: string;
rowCount?: number;
createdAt?: string;
}
export interface SimilarAsinHistoryVo {
items: SimilarAsinHistoryItem[];
}
export interface SimilarAsinTaskSummary {
id?: number;
taskNo?: string;
status?: string;
errorMessage?: string;
createdAt?: string;
updatedAt?: string;
finishedAt?: string;
}
export interface SimilarAsinTaskDetailVo {
task?: SimilarAsinTaskSummary;
items?: SimilarAsinHistoryItem[];
}
export interface SimilarAsinTaskBatchVo {
items: SimilarAsinTaskDetailVo[];
missingTaskIds?: number[];
}
export function parseSimilarAsin(files: UploadedFileRef[], aiPrompt: string) {
return unwrapJavaResponse(
post<
JavaApiResponse<SimilarAsinParseVo>,
{ user_id: number; files: UploadedFileRef[]; ai_prompt: string }
>(`${JAVA_API_PREFIX}/similar-asin/parse`, {
user_id: getCurrentUserId(),
files,
ai_prompt: aiPrompt,
}),
);
}
export function getSimilarAsinDashboard() {
return unwrapJavaResponse(
get<JavaApiResponse<SimilarAsinDashboardVo>>(
`${JAVA_API_PREFIX}/similar-asin/dashboard`,
{ params: { user_id: getCurrentUserId() } },
),
);
}
export function getSimilarAsinHistory() {
return unwrapJavaResponse(
get<JavaApiResponse<SimilarAsinHistoryVo>>(
`${JAVA_API_PREFIX}/similar-asin/history`,
{ params: { user_id: getCurrentUserId(), limit: 50 } },
),
);
}
export function getSimilarAsinTaskProgressBatch(
taskIds: number[],
options: TaskProgressBatchOptions = {},
) {
return postTaskProgressBatch<SimilarAsinTaskBatchVo>(
`${JAVA_API_PREFIX}/similar-asin/tasks/progress/batch`,
taskIds,
options,
);
}
export function activateSimilarAsinTask(taskId: number) {
return unwrapJavaResponse(
post<JavaApiResponse<null>, undefined>(
`${JAVA_API_PREFIX}/similar-asin/tasks/${taskId}/activate?user_id=${encodeURIComponent(String(getCurrentUserId()))}`,
undefined,
),
);
}
export function deleteSimilarAsinTask(taskId: number) {
return unwrapJavaResponse(
del<JavaApiResponse<null>>(`${JAVA_API_PREFIX}/similar-asin/tasks/${taskId}`, {
params: { user_id: getCurrentUserId() },
}),
);
}
export function deleteSimilarAsinHistory(resultId: number) {
return unwrapJavaResponse(
del<JavaApiResponse<null>>(
`${JAVA_API_PREFIX}/similar-asin/history/${resultId}`,
{ params: { user_id: getCurrentUserId() } },
),
);
}
export function getSimilarAsinResultDownloadUrl(resultId: number) {
return getJavaDownloadUrl(`/similar-asin/results/${resultId}/download`);
}
// ========== 跟价 ========== // ========== 跟价 ==========
export interface PriceTrackCandidateVo { export interface PriceTrackCandidateVo {

View File

@@ -0,0 +1,7 @@
import { createApp } from 'vue'
import ElementPlus from 'element-plus'
import 'element-plus/dist/index.css'
import '@/styles/main.css'
import BrandSimilarAsinTab from '@/pages/brand/components/BrandSimilarAsinTab.vue'
createApp(BrandSimilarAsinTab).use(ElementPlus).mount('#app')

View File

@@ -47,6 +47,7 @@ export default defineConfig({
split: resolve(__dirname, 'split.html'), split: resolve(__dirname, 'split.html'),
'delete-brand': resolve(__dirname, 'delete-brand.html'), 'delete-brand': resolve(__dirname, 'delete-brand.html'),
'appearance-patent': resolve(__dirname, 'appearance-patent.html'), 'appearance-patent': resolve(__dirname, 'appearance-patent.html'),
'similar-asin': resolve(__dirname, 'similar-asin.html'),
'product-risk': resolve(__dirname, 'product-risk.html'), 'product-risk': resolve(__dirname, 'product-risk.html'),
'shop-match': resolve(__dirname, 'shop-match.html'), 'shop-match': resolve(__dirname, 'shop-match.html'),
'price-track': resolve(__dirname, 'price-track.html'), 'price-track': resolve(__dirname, 'price-track.html'),

View File

@@ -1 +0,0 @@
{"version": "1.0.56"}