super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin

Browse Source 
modified:   amazon/detail_spider.py
	modified:   amazon/main.py
This commit is contained in:
铭坤
2026-05-04 19:17:11 +08:00
parent c315bb350c 6d398b66bc
commit 45a45dc0fe
119 changed files with 7781 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
app/amazon/detail_spider.py
View File

@@ -111,8 +111,29 @@ class ChromeAmzone(ChromeAmzoneBase):
<<<<<<< HEAD
class SpiderTask(TaskBase): class SpiderTask(TaskBase):
task_name = "亚马逊采集" task_name = "亚马逊采集"
=======
class SpiderTask:
mark_name = "亚马逊采集"
def __init__(self, user_info: dict = None):
"""初始化审批任务处理器
Args:
user_info: 用户信息字典，包含 company, username, password
"""
self.user_info = user_info or {}
self.running = True
self.result_api_module = "appearance-patent"
def log(self, message: str, level: str = "INFO"):
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [PriceTask] [{level}] {message}")
>>>>>>> 6d398b66bc24634b13f1de162e8f68836e64441d
@staticmethod @staticmethod
def group_by_id_prefix(data): def group_by_id_prefix(data):
@@ -166,9 +187,16 @@ class SpiderTask(TaskBase):
task_data: 任务数据 task_data: 任务数据
""" """
try: try:
<<<<<<< HEAD
data = task_data.get("data", {}) data = task_data.get("data", {})
task_id = data.get("taskId") task_id = data.get("taskId")
groups = self.normalize_groups(data) groups = self.normalize_groups(data)
=======
data = task_data.get("data", {})
self.result_api_module = "similar-asin" if task_data.get("type") == "similar-asin-run" else "appearance-patent"
task_id = data.get("taskId")
groups = self.normalize_groups(data)
>>>>>>> 6d398b66bc24634b13f1de162e8f68836e64441d
# 用于测试 # 用于测试
limit = data.get("limit", None) limit = data.get("limit", None)
@@ -307,7 +335,7 @@ class SpiderTask(TaskBase):
"""回传处理结果到API """回传处理结果到API
""" """
url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result" url = f"{DELETE_BRAND_API_BASE}/api/{self.result_api_module}/tasks/{task_id}/result"
payload ={ payload ={
"submissionId": f"{task_id}", "submissionId": f"{task_id}",

1
app/amazon/main.py
View File

@@ -65,6 +65,7 @@ class TaskMonitor:
"query-asin-run" : "状态查询", "query-asin-run" : "状态查询",
"patrol-delete-run" : "巡店删除", "patrol-delete-run" : "巡店删除",
"appearance-patent-run" : "亚马逊采集", "appearance-patent-run" : "亚马逊采集",
"similar-asin-run" : "亚马逊采集",
} }
try: try:
while self.running: while self.running:

1
app/assets/appearance-patent-CYKeZX2a.css Normal file
View File

File diff suppressed because one or more lines are too long

1
app/assets/appearance-patent-TI-osB39.css
View File

File diff suppressed because one or more lines are too long

4
app/assets/appearance-patent.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/brand-CABpTGPu.js → app/assets/brand-Cs6TxmtW.js
View File

@@ -1 +1 @@
import{aZ as r}from"./pywebview-BWNiqXug.js";const n="";function a(e){return r(`${n}/api/brand/expand-folder-recursive`,{folder:e})}export{a as e}; import{a_ as r}from"./pywebview-Cz_PRNOK.js";const n="";function a(e){return r(`${n}/api/brand/expand-folder-recursive`,{folder:e})}export{a as e};

1
app/assets/brand-CyNKOr-G.js Normal file
View File

@@ -0,0 +1 @@
import{b6 as r}from"./pywebview-BcagetkZ.js";const n="";function s(e){return r(`${n}/api/brand/expand-folder-recursive`,{folder:e})}export{s as e};

1
app/assets/convert-BKSNvX8i.css Normal file
View File

File diff suppressed because one or more lines are too long

1
app/assets/convert-r7MSXq7I.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/convert.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/dedupe-DMeaPUMZ.css
View File

File diff suppressed because one or more lines are too long

1
app/assets/dedupe-DNlVfFj-.css Normal file
View File

File diff suppressed because one or more lines are too long

2
app/assets/dedupe.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/delete-brand-B5ChOXZY.css
View File

File diff suppressed because one or more lines are too long

1
app/assets/delete-brand-BJod9K65.css Normal file
View File

File diff suppressed because one or more lines are too long

2
app/assets/delete-brand.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/patrol-delete-CBBdonF4.css Normal file
View File

File diff suppressed because one or more lines are too long

1
app/assets/patrol-delete-DyumfGaZ.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/patrol-delete.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/price-track-DMhhVz4u.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/price-track.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/product-risk-CI6qY5U2.css
View File

File diff suppressed because one or more lines are too long

1
app/assets/product-risk-DV5AzjVc.css Normal file
View File

File diff suppressed because one or more lines are too long

2
app/assets/product-risk.js
View File

File diff suppressed because one or more lines are too long

55
app/assets/pywebview-BcagetkZ.js Normal file
View File

File diff suppressed because one or more lines are too long

2
app/assets/pywebview-BFZO8_3q.css → app/assets/pywebview-C-qhaDJK.css
View File

File diff suppressed because one or more lines are too long

26
app/assets/pywebview-BWNiqXug.js → app/assets/pywebview-Cz_PRNOK.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/pywebview-Dp5dN8OO.css Normal file
View File

File diff suppressed because one or more lines are too long

1
app/assets/query-asin-Bw6BQsRN.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/query-asin.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/shop-match-CxCKUsqS.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/shop-match.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/similar-asin-B-_UTsa7.css Normal file
View File

File diff suppressed because one or more lines are too long

6
app/assets/similar-asin.js Normal file
View File

File diff suppressed because one or more lines are too long

1
app/assets/split-BoBVrLdC.css Normal file
View File

File diff suppressed because one or more lines are too long

1
app/assets/split-Cw6sXhAg.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/split.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/zh-cn-BbuLS91u.js → app/assets/zh-cn-BEpMBrLL.js
View File

File diff suppressed because one or more lines are too long

8
app/new_web_source/convert.html
View File

@@ -5,10 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>格式转换 - 数富AI</title> <title>格式转换 - 数富AI</title>
<script type="module" crossorigin src="/assets/convert.js"></script> <script type="module" crossorigin src="/assets/convert.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-BWNiqXug.js"> <link rel="modulepreload" crossorigin href="/assets/pywebview-Cz_PRNOK.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CABpTGPu.js"> <link rel="modulepreload" crossorigin href="/assets/brand-Cs6TxmtW.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-BFZO8_3q.css"> <link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/convert-r7MSXq7I.css"> <link rel="stylesheet" crossorigin href="/assets/convert-BKSNvX8i.css">
</head> </head>
<body> <body>
<div id="app"></div> <div id="app"></div>

8
app/new_web_source/dedupe.html
View File

@@ -5,10 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据去重 - 数富AI</title> <title>数据去重 - 数富AI</title>
<script type="module" crossorigin src="/assets/dedupe.js"></script> <script type="module" crossorigin src="/assets/dedupe.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-BWNiqXug.js"> <link rel="modulepreload" crossorigin href="/assets/pywebview-Cz_PRNOK.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CABpTGPu.js"> <link rel="modulepreload" crossorigin href="/assets/brand-Cs6TxmtW.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-BFZO8_3q.css"> <link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-DMeaPUMZ.css"> <link rel="stylesheet" crossorigin href="/assets/dedupe-DNlVfFj-.css">
</head> </head>
<body> <body>
<div id="app"></div> <div id="app"></div>

8
app/new_web_source/delete-brand.html
View File

@@ -5,11 +5,11 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>删除品牌 - 数富AI</title> <title>删除品牌 - 数富AI</title>
<script type="module" crossorigin src="/assets/delete-brand.js"></script> <script type="module" crossorigin src="/assets/delete-brand.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-BWNiqXug.js"> <link rel="modulepreload" crossorigin href="/assets/pywebview-Cz_PRNOK.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CABpTGPu.js"> <link rel="modulepreload" crossorigin href="/assets/brand-Cs6TxmtW.js">
<link rel="modulepreload" crossorigin href="/assets/categorized-timers-JPA-olTr.js"> <link rel="modulepreload" crossorigin href="/assets/categorized-timers-JPA-olTr.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-BFZO8_3q.css"> <link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/delete-brand-B5ChOXZY.css"> <link rel="stylesheet" crossorigin href="/assets/delete-brand-BJod9K65.css">
</head> </head>
<body> <body>
<div id="app"></div> <div id="app"></div>

8
app/new_web_source/split.html
View File

@@ -5,10 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据拆分 - 数富AI</title> <title>数据拆分 - 数富AI</title>
<script type="module" crossorigin src="/assets/split.js"></script> <script type="module" crossorigin src="/assets/split.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-BWNiqXug.js"> <link rel="modulepreload" crossorigin href="/assets/pywebview-Cz_PRNOK.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CABpTGPu.js"> <link rel="modulepreload" crossorigin href="/assets/brand-Cs6TxmtW.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-BFZO8_3q.css"> <link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/split-Cw6sXhAg.css"> <link rel="stylesheet" crossorigin href="/assets/split-BoBVrLdC.css">
</head> </head>
<body> <body>
<div id="app"></div> <div id="app"></div>

73
backend-java/src/main/java/com/nanri/aiimage/common/util/DownloadHeaderUtil.java Normal file
View File

@@ -0,0 +1,73 @@
package com.nanri.aiimage.common.util;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.http.HttpHeaders;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
public final class DownloadHeaderUtil {
private static final String DEFAULT_FILENAME = "download";
private DownloadHeaderUtil() {
}
public static void setAttachment(HttpServletResponse response, String filename) {
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, contentDisposition(filename));
}
public static String contentDisposition(String filename) {
String normalized = normalizeFilename(filename);
String encodedFilename = URLEncoder.encode(normalized, StandardCharsets.UTF_8).replace("+", "%20");
return "attachment; filename=\"" + asciiFallback(normalized) + "\"; filename*=UTF-8''" + encodedFilename;
}
private static String normalizeFilename(String filename) {
if (filename == null || filename.isBlank()) {
return DEFAULT_FILENAME;
}
return filename.replace("\r", "").replace("\n", "").trim();
}
private static String asciiFallback(String filename) {
String normalized = normalizeFilename(filename);
String extension = extensionOf(normalized);
String base = extension.isEmpty() ? normalized : normalized.substring(0, normalized.length() - extension.length() - 1);
String sanitizedBase = sanitizeAscii(base);
String sanitizedExtension = sanitizeExtension(extension);
if (sanitizedBase.isBlank()) {
sanitizedBase = DEFAULT_FILENAME;
}
String fallback = sanitizedExtension.isBlank() ? sanitizedBase : sanitizedBase + "." + sanitizedExtension;
return fallback.length() <= 120 ? fallback : shorten(fallback, sanitizedExtension);
}
private static String extensionOf(String filename) {
int dotIndex = filename.lastIndexOf('.');
if (dotIndex < 0 || dotIndex == filename.length() - 1) {
return "";
}
return filename.substring(dotIndex + 1);
}
private static String sanitizeAscii(String value) {
return value.replaceAll("[^A-Za-z0-9._-]", "_")
.replaceAll("_+", "_")
.replaceAll("^[_\\.]+|[_\\.]+$", "");
}
private static String sanitizeExtension(String extension) {
return extension.replaceAll("[^A-Za-z0-9]", "");
}
private static String shorten(String fallback, String extension) {
int maxBaseLength = extension.isBlank() ? 120 : Math.max(1, 119 - extension.length());
String base = extension.isBlank() ? fallback : fallback.substring(0, fallback.length() - extension.length() - 1);
base = base.substring(0, Math.min(base.length(), maxBaseLength)).replaceAll("[_\\.]+$", "");
if (base.isBlank()) {
base = DEFAULT_FILENAME;
}
return extension.isBlank() ? base : base + "." + extension;
}
}

2
backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java
View File

@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
@Configuration @Configuration
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class}) @EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class})
public class PropertiesConfig { public class PropertiesConfig {
} }

21
backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java Normal file
View File

@@ -0,0 +1,21 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Data
@ConfigurationProperties(prefix = "aiimage.similar-asin")
public class SimilarAsinProperties {
private String cozeBaseUrl = "https://api.coze.cn";
private String cozeWorkflowPath = "/v1/workflow/run";
private String cozeWorkflowHistoryPath = "/v1/workflows/{workflow_id}/run_histories/{execute_id}";
private String cozeWorkflowId = "7632683471312355338";
private String cozeToken = "";
private int cozeBatchSize = 10;
private int cozeConnectTimeoutMillis = 10000;
private int cozeReadTimeoutMillis = 60000;
private int cozePollIntervalMillis = 2000;
private int cozePollTimeoutMillis = 120000;
private int staleTimeoutMinutes = 20;
private String staleFinalizeCron = "0 */2 * * * *";
}

1
backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java
View File

@@ -11,4 +11,5 @@ public class StorageProperties {
private String cleanupCron = "0 0 * * * *"; private String cleanupCron = "0 0 * * * *";
private long sourceRetentionHours = 24; private long sourceRetentionHours = 24;
private long resultRetentionHours = 24; private long resultRetentionHours = 24;
private long transientPayloadRetentionHours = 72;
} }

16
backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java
View File

@@ -4,8 +4,12 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor; import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@Configuration @Configuration
public class TaskFileJobConfig { public class TaskFileJobConfig {
@@ -24,4 +28,16 @@ public class TaskFileJobConfig {
executor.initialize(); executor.initialize();
return executor; return executor;
} }
@Bean(destroyMethod = "shutdown")
public ExecutorService cozeVirtualThreadExecutor() {
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
.name("coze-task-", 0)
.factory());
}
@Bean("cozeTaskExecutor")
public TaskExecutor cozeTaskExecutor(ExecutorService cozeVirtualThreadExecutor) {
return new ConcurrentTaskExecutor(cozeVirtualThreadExecutor);
}
} }

179
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClient.java
View File

@@ -44,6 +44,42 @@ public class AppearancePatentCozeClient {
} }
} }
public CozeSubmitResponse submitWorkflow(List<AppearancePatentResultRowDto> rows, String prompt) throws Exception {
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt));
ensureSuccess(submitRoot);
return new CozeSubmitResponse(
extractExecuteId(submitRoot),
extractResultDataText(submitRoot),
writeJson(submitRoot)
);
}
public CozePollResponse pollWorkflow(String executeId) throws Exception {
JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId));
ensureSuccess(pollRoot);
String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT);
String dataText = extractResultDataText(pollRoot);
String outputText = dataText.isBlank() ? extractWorkflowOutputText(pollRoot) : "";
String failureMessage = isFailedWorkflowStatus(status)
? firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed")
: "";
return new CozePollResponse(executeId, status, dataText, outputText, failureMessage, writeJson(pollRoot));
}
public List<AppearancePatentResultRowDto> mergeRowsFromDataText(List<AppearancePatentResultRowDto> rows, String dataText) throws Exception {
if (dataText == null || dataText.isBlank()) {
return rows == null ? List.of() : rows.stream().map(this::copy).toList();
}
return mergeRows(rows, parseResults(wrapDataPayload(dataText)));
}
public List<AppearancePatentResultRowDto> markRowsFailed(List<AppearancePatentResultRowDto> rows, String failureMessage) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
private List<AppearancePatentResultRowDto> inspectWithFallback(List<AppearancePatentResultRowDto> rows, String prompt) { private List<AppearancePatentResultRowDto> inspectWithFallback(List<AppearancePatentResultRowDto> rows, String prompt) {
try { try {
if (rows.size() == 1) { if (rows.size() == 1) {
@@ -140,6 +176,7 @@ public class AppearancePatentCozeClient {
long deadline = System.currentTimeMillis() + Math.max(10000, properties.getCozePollTimeoutMillis()); long deadline = System.currentTimeMillis() + Math.max(10000, properties.getCozePollTimeoutMillis());
while (System.currentTimeMillis() < deadline) { while (System.currentTimeMillis() < deadline) {
ensureNotInterrupted();
JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId)); JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId));
ensureSuccess(pollRoot); ensureSuccess(pollRoot);
@@ -149,9 +186,16 @@ public class AppearancePatentCozeClient {
} }
String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT); String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT);
if (status.contains("FAIL") || status.contains("ERROR") || status.contains("CANCEL")) { if (isFailedWorkflowStatus(status)) {
throw new IllegalStateException(firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed")); throw new IllegalStateException(firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed"));
} }
if (isSuccessfulWorkflowStatus(status)) {
String outputText = extractWorkflowOutputText(pollRoot);
if (!outputText.isBlank()) {
return wrapDataPayload(outputText);
}
throw new IllegalStateException("Coze async workflow completed without output");
}
sleepQuietly(Math.max(200, properties.getCozePollIntervalMillis())); sleepQuietly(Math.max(200, properties.getCozePollIntervalMillis()));
} }
@@ -251,7 +295,7 @@ public class AppearancePatentCozeClient {
return List.of(); return List.of();
} }
JsonNode dataRoot = objectMapper.readTree(dataText); JsonNode dataRoot = objectMapper.readTree(dataText);
JsonNode array = dataRoot.path("data"); JsonNode array = dataRoot.isArray() ? dataRoot : dataRoot.path("data");
List<CozeResult> results = new ArrayList<>(); List<CozeResult> results = new ArrayList<>();
if (array.isArray()) { if (array.isArray()) {
for (JsonNode node : array) { for (JsonNode node : array) {
@@ -423,9 +467,21 @@ public class AppearancePatentCozeClient {
} }
private AppearancePatentResultRowDto markFailed(AppearancePatentResultRowDto row, String failureMessage) { private AppearancePatentResultRowDto markFailed(AppearancePatentResultRowDto row, String failureMessage) {
String reviewMessage = failureMessage == null || failureMessage.isBlank()
? "Coze 检测失败，待人工复核"
: "Coze 检测失败，待人工复核：" + failureMessage;
if (row.getError() == null || row.getError().isBlank()) { if (row.getError() == null || row.getError().isBlank()) {
row.setError(failureMessage); row.setError(failureMessage);
} }
if (row.getTitleRisk() == null || row.getTitleRisk().isBlank()) {
row.setTitleRisk(reviewMessage);
}
if (row.getAppearanceRisk() == null || row.getAppearanceRisk().isBlank()) {
row.setAppearanceRisk(reviewMessage);
}
if (row.getPatentRisk() == null || row.getPatentRisk().isBlank()) {
row.setPatentRisk(reviewMessage);
}
if (row.getConclusion() == null || row.getConclusion().isBlank()) { if (row.getConclusion() == null || row.getConclusion().isBlank()) {
row.setConclusion(failureMessage); row.setConclusion(failureMessage);
} }
@@ -502,6 +558,30 @@ public class AppearancePatentCozeClient {
return discoverEmbeddedData(root); return discoverEmbeddedData(root);
} }
private String extractWorkflowOutputText(JsonNode root) {
JsonNode outputNode = findFirstField(root, "output");
if (outputNode == null || outputNode.isNull() || outputNode.isMissingNode()) {
return "";
}
if (!outputNode.isTextual()) {
String discovered = discoverEmbeddedData(outputNode);
return discovered.isBlank() && isResultDataPayload(outputNode) ? outputNode.toString() : discovered;
}
String output = normalize(outputNode.asText(""));
if (output.isBlank()) {
return "";
}
if (looksLikeResultDataPayload(output)) {
return output;
}
JsonNode parsedOutput = parseJsonOrMissing(output);
String nestedOutput = text(firstNonNull(parsedOutput.get("Output"), parsedOutput.get("output")));
if (nestedOutput != null && !nestedOutput.isBlank() && looksLikeResultDataPayload(nestedOutput)) {
return nestedOutput;
}
return discoverEmbeddedData(parsedOutput);
}
private String discoverEmbeddedData(JsonNode node) { private String discoverEmbeddedData(JsonNode node) {
if (node == null || node.isNull() || node.isMissingNode()) { if (node == null || node.isNull() || node.isMissingNode()) {
return ""; return "";
@@ -577,6 +657,22 @@ public class AppearancePatentCozeClient {
|| item.has("patent reason")); || item.has("patent reason"));
} }
private boolean isSuccessfulWorkflowStatus(String status) {
String normalized = normalize(status).toUpperCase(Locale.ROOT);
return normalized.equals("SUCCESS")
|| normalized.equals("SUCCEEDED")
|| normalized.equals("COMPLETED")
|| normalized.equals("COMPLETE")
|| normalized.equals("DONE");
}
private boolean isFailedWorkflowStatus(String status) {
String normalized = normalize(status).toUpperCase(Locale.ROOT);
return normalized.contains("FAIL")
|| normalized.contains("ERROR")
|| normalized.contains("CANCEL");
}
private JsonNode parseJsonOrMissing(String value) { private JsonNode parseJsonOrMissing(String value) {
String normalized = normalize(value); String normalized = normalize(value);
if (!(normalized.startsWith("{") || normalized.startsWith("["))) { if (!(normalized.startsWith("{") || normalized.startsWith("["))) {
@@ -611,6 +707,38 @@ public class AppearancePatentCozeClient {
return findTextByFieldName(root, "execute_id", "executeId"); return findTextByFieldName(root, "execute_id", "executeId");
} }
private JsonNode findFirstField(JsonNode node, String name) {
if (node == null || node.isMissingNode() || node.isNull() || name == null || name.isBlank()) {
return objectMapper.missingNode();
}
if (node.isTextual()) {
return findFirstField(parseJsonOrMissing(node.asText("")), name);
}
if (node.isArray()) {
for (JsonNode child : node) {
JsonNode found = findFirstField(child, name);
if (found != null && !found.isMissingNode() && !found.isNull()) {
return found;
}
}
return objectMapper.missingNode();
}
if (node.isObject()) {
JsonNode direct = node.get(name);
if (direct != null && !direct.isMissingNode() && !direct.isNull()) {
return direct;
}
for (java.util.Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> entry = it.next();
JsonNode found = findFirstField(entry.getValue(), name);
if (found != null && !found.isMissingNode() && !found.isNull()) {
return found;
}
}
}
return objectMapper.missingNode();
}
private String findTextByFieldName(JsonNode node, String... names) { private String findTextByFieldName(JsonNode node, String... names) {
if (node == null || node.isMissingNode() || node.isNull()) { if (node == null || node.isMissingNode() || node.isNull()) {
return ""; return "";
@@ -677,6 +805,7 @@ public class AppearancePatentCozeClient {
private void sleepBeforeRetry(int attemptIndex) { private void sleepBeforeRetry(int attemptIndex) {
long delayMillis = Math.max(1, attemptIndex) * 1500L; long delayMillis = Math.max(1, attemptIndex) * 1500L;
ensureNotInterrupted();
sleepQuietly(delayMillis); sleepQuietly(delayMillis);
} }
@@ -685,6 +814,13 @@ public class AppearancePatentCozeClient {
Thread.sleep(delayMillis); Thread.sleep(delayMillis);
} catch (InterruptedException interruptedException) { } catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
throw new IllegalStateException("Coze workflow interrupted", interruptedException);
}
}
private void ensureNotInterrupted() {
if (Thread.currentThread().isInterrupted()) {
throw new IllegalStateException("Coze workflow interrupted");
} }
} }
@@ -783,6 +919,45 @@ public class AppearancePatentCozeClient {
) { ) {
} }
public record CozeSubmitResponse(
String executeId,
String immediateData,
String rawResponse
) {
}
public record CozePollResponse(
String executeId,
String status,
String dataText,
String outputText,
String failureMessage,
String rawResponse
) {
public boolean hasPayload() {
return dataText != null && !dataText.isBlank() || outputText != null && !outputText.isBlank();
}
public String resolvedPayloadText() {
return dataText != null && !dataText.isBlank() ? dataText : outputText;
}
public boolean isFailed() {
String normalized = status == null ? "" : status.trim().toUpperCase(Locale.ROOT);
return normalized.contains("FAILED") || normalized.contains("ERROR") || normalized.contains("CANCEL");
}
public boolean isFinished() {
String normalized = status == null ? "" : status.trim().toUpperCase(Locale.ROOT);
return isFailed()
|| normalized.contains("SUCCESS")
|| normalized.contains("SUCCEED")
|| normalized.contains("FINISH")
|| normalized.contains("DONE")
|| normalized.contains("COMPLET");
}
}
private static final class PartialCozeResultException extends RuntimeException { private static final class PartialCozeResultException extends RuntimeException {
private final int resolvedCount; private final int resolvedCount;

7
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java
View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.appearancepatent.controller; package com.nanri.aiimage.modules.appearancepatent.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentTaskBatchRequest; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentTaskBatchRequest;
@@ -14,7 +15,6 @@ import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -28,7 +28,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@RestController @RestController
@@ -129,10 +128,8 @@ public class AppearancePatentController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;

8
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultRowDto.java
View File

@@ -29,9 +29,17 @@ public class AppearancePatentResultRowDto {
private String country; private String country;
@Schema(description = "商品主图或待检测图片 URL。Java 调用 Coze 时会放入 url_list。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg") @Schema(description = "商品主图或待检测图片 URL。Java 调用 Coze 时会放入 url_list。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg")
@JsonAlias({
"imageUrl", "image_url", "imgUrl", "img_url", "pictureUrl", "picture_url",
"mainImage", "main_image", "mainImageUrl", "main_image_url",
"productImage", "product_image", "productImageUrl", "product_image_url",
"image", "img", "pic", "picture", "link", "imageLink", "image_link",
"图片链接", "商品图片", "商品主图", "主图", "主图链接"
})
private String url; private String url;
@Schema(description = "商品标题。Java 调用 Coze 时会放入 title_list；为空时会回退使用 ASIN。", example = "Women Floral Dress Summer Casual") @Schema(description = "商品标题。Java 调用 Coze 时会放入 title_list；为空时会回退使用 ASIN。", example = "Women Floral Dress Summer Casual")
@JsonAlias({"productTitle", "product_title", "itemTitle", "item_title", "商品标题", "商品名称", "标题"})
private String title; private String title;
@Schema(description = "单行错误信息。通常用于记录 Python 单行处理异常；Coze 失败时后端会尽量保留原始行，不强行写入风险结果。", example = "图片地址为空") @Schema(description = "单行错误信息。通常用于记录 Python 单行处理异常；Coze 失败时后端会尽量保留原始行，不强行写入风险结果。", example = "图片地址为空")

618
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java
View File

@@ -3,9 +3,11 @@ package com.nanri.aiimage.modules.appearancepatent.service;
import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.IdUtil;
import cn.hutool.crypto.digest.DigestUtil; import cn.hutool.crypto.digest.DigestUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.AppearancePatentProperties; import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.config.StorageProperties; import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentCozeClient; import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentCozeClient;
@@ -50,7 +52,10 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory; import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.dao.DuplicateKeyException; import org.springframework.dao.DuplicateKeyException;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.PlatformTransactionManager;
@@ -73,6 +78,7 @@ import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.UUID;
import java.util.function.Supplier; import java.util.function.Supplier;
@Service @Service
@@ -85,7 +91,13 @@ public class AppearancePatentTaskService {
private static final String STATUS_RUNNING = "RUNNING"; private static final String STATUS_RUNNING = "RUNNING";
private static final String STATUS_SUCCESS = "SUCCESS"; private static final String STATUS_SUCCESS = "SUCCESS";
private static final String STATUS_FAILED = "FAILED"; private static final String STATUS_FAILED = "FAILED";
private static final String COZE_STATUS_SUBMITTED = "SUBMITTED";
private static final String COZE_STATUS_RUNNING = "RUNNING";
private static final String COZE_STATUS_DONE = "DONE";
private static final String COZE_STATUS_FAILED = "FAILED";
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
private static final List<String> RESULT_HEADERS = List.of( private static final List<String> RESULT_HEADERS = List.of(
"id", "id",
"asin", "asin",
@@ -112,9 +124,13 @@ public class AppearancePatentTaskService {
private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TransientPayloadStorageService transientPayloadStorageService; private final TransientPayloadStorageService transientPayloadStorageService;
private final PlatformTransactionManager transactionManager; private final PlatformTransactionManager transactionManager;
private final DistributedJobLockService distributedJobLockService;
@Autowired
@Qualifier("cozeTaskExecutor")
private TaskExecutor cozeTaskExecutor;
@Transactional
public AppearancePatentParseVo parseAndCreateTask(AppearancePatentParseRequest request) { public AppearancePatentParseVo parseAndCreateTask(AppearancePatentParseRequest request) {
long startedAt = System.nanoTime();
if (request.getUserId() == null || request.getUserId() <= 0) { if (request.getUserId() == null || request.getUserId() <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
} }
@@ -131,6 +147,7 @@ public class AppearancePatentTaskService {
List<String> mergedHeaders = new ArrayList<>(); List<String> mergedHeaders = new ArrayList<>();
int totalRows = 0; int totalRows = 0;
int droppedRows = 0; int droppedRows = 0;
long parseStartedAt = System.nanoTime();
for (AppearancePatentSourceFileDto source : sourceFiles) { for (AppearancePatentSourceFileDto source : sourceFiles) {
if (source.getFileKey() == null || source.getFileKey().isBlank()) { if (source.getFileKey() == null || source.getFileKey().isBlank()) {
throw new BusinessException("fileKey 不能为空"); throw new BusinessException("fileKey 不能为空");
@@ -149,7 +166,9 @@ public class AppearancePatentTaskService {
if (allRows.isEmpty()) { if (allRows.isEmpty()) {
throw new BusinessException("未解析到有效 ASIN 数据"); throw new BusinessException("未解析到有效 ASIN 数据");
} }
long parsedAt = System.nanoTime();
List<AppearancePatentParsedGroupVo> groups = buildParsedGroups(allRows); List<AppearancePatentParsedGroupVo> groups = buildParsedGroups(allRows);
long groupedAt = System.nanoTime();
FileTaskEntity task = new FileTaskEntity(); FileTaskEntity task = new FileTaskEntity();
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr()); task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
@@ -170,11 +189,14 @@ public class AppearancePatentTaskService {
throw new BusinessException("序列化解析结果失败"); throw new BusinessException("序列化解析结果失败");
} }
fileTaskMapper.insert(task); fileTaskMapper.insert(task);
long taskInsertedAt = System.nanoTime();
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles); String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey); String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), sourceFiles, mergedHeaders, groups, allRows); String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), sourceFiles, mergedHeaders, groups, allRows);
long payloadBuiltAt = System.nanoTime();
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload); String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
long payloadStoredAt = System.nanoTime();
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), sourceFiles, parsedPayloadPointer)); task.setResultJson(buildTaskResultJson(request.getAiPrompt(), sourceFiles, parsedPayloadPointer));
task.setUpdatedAt(LocalDateTime.now()); task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task); fileTaskMapper.updateById(task);
@@ -203,6 +225,7 @@ public class AppearancePatentTaskService {
scope.setCreatedAt(LocalDateTime.now()); scope.setCreatedAt(LocalDateTime.now());
scope.setUpdatedAt(LocalDateTime.now()); scope.setUpdatedAt(LocalDateTime.now());
taskScopeStateMapper.insert(scope); taskScopeStateMapper.insert(scope);
long persistedAt = System.nanoTime();
AppearancePatentParseVo vo = new AppearancePatentParseVo(); AppearancePatentParseVo vo = new AppearancePatentParseVo();
vo.setTaskId(task.getId()); vo.setTaskId(task.getId());
@@ -215,6 +238,20 @@ public class AppearancePatentTaskService {
vo.setAiPrompt(normalize(request.getAiPrompt())); vo.setAiPrompt(normalize(request.getAiPrompt()));
vo.setItems(allRows); vo.setItems(allRows);
vo.setGroups(groups); vo.setGroups(groups);
long finishedAt = System.nanoTime();
log.info("[appearance-patent] parse timing taskId={} files={} rows={} groups={} totalMs={} parseMs={} groupMs={} taskInsertMs={} payloadJsonMs={} payloadStoreMs={} persistMs={} responseMs={}",
task.getId(),
sourceFiles.size(),
allRows.size(),
groups.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(parseStartedAt, parsedAt),
elapsedMs(parsedAt, groupedAt),
elapsedMs(groupedAt, taskInsertedAt),
elapsedMs(taskInsertedAt, payloadBuiltAt),
elapsedMs(payloadBuiltAt, payloadStoredAt),
elapsedMs(payloadStoredAt, persistedAt),
elapsedMs(persistedAt, finishedAt));
return vo; return vo;
} }
@@ -491,6 +528,10 @@ public class AppearancePatentTaskService {
.lt(FileTaskEntity::getUpdatedAt, threshold) .lt(FileTaskEntity::getUpdatedAt, threshold)
.last("limit 50")); .last("limit 50"));
for (FileTaskEntity task : tasks) { for (FileTaskEntity task : tasks) {
if (isJavaSideProcessing(task.getId())) {
touchJavaSideTaskActivity(task.getId());
continue;
}
long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId()); long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId());
if (heartbeatMillis > thresholdMillis) { if (heartbeatMillis > thresholdMillis) {
continue; continue;
@@ -510,6 +551,10 @@ public class AppearancePatentTaskService {
.lt(FileTaskEntity::getUpdatedAt, threshold) .lt(FileTaskEntity::getUpdatedAt, threshold)
.last("limit 50")); .last("limit 50"));
for (FileTaskEntity task : tasks) { for (FileTaskEntity task : tasks) {
if (isJavaSideProcessing(task.getId())) {
touchJavaSideTaskActivity(task.getId());
continue;
}
long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId()); long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId());
if (heartbeatMillis > thresholdMillis) { if (heartbeatMillis > thresholdMillis) {
continue; continue;
@@ -738,12 +783,17 @@ public class AppearancePatentTaskService {
} }
String payloadJson = writeJson(rows == null ? List.of() : rows, "结果序列化失败"); String payloadJson = writeJson(rows == null ? List.of() : rows, "结果序列化失败");
payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "appearance patent chunk payload merge failed"); payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "appearance patent chunk payload merge failed");
String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); String oldPayload = chunk.getPayloadJson();
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(chunk.getPayloadJson(), storedPayload); String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
chunk.setPayloadJson(storedPayload); chunk.setPayloadJson(storedPayload);
chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson)); chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson));
chunk.setUpdatedAt(LocalDateTime.now()); chunk.setUpdatedAt(LocalDateTime.now());
taskChunkMapper.updateById(chunk); int updated = taskChunkMapper.updateById(chunk);
if (updated <= 0) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw new IllegalStateException("appearance patent chunk payload update failed");
}
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
} }
private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives, private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives,
@@ -976,7 +1026,7 @@ public class AppearancePatentTaskService {
assembleResultWorkbook(task, result); assembleResultWorkbook(task, result);
} catch (Exception ex) { } catch (Exception ex) {
log.warn("[appearance-patent] assemble result workbook failed taskId={} err={}", task.getId(), ex.getMessage()); log.warn("[appearance-patent] assemble result workbook failed taskId={} err={}", task.getId(), ex.getMessage());
finalError = firstNonBlank(finalError, "生成外观专利检测结果失败"); finalError = firstNonBlank(finalError, "Generate appearance patent result failed");
} }
} }
} }
@@ -1010,39 +1060,478 @@ public class AppearancePatentTaskService {
return false; return false;
} }
public void processResultFileJob(TaskFileJobEntity job) { public boolean processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null || job.getResultId() == null) { if (job == null || job.getTaskId() == null || job.getResultId() == null) {
throw new BusinessException("结果文件任务参数不完整"); throw new BusinessException("result file job arguments are incomplete");
} }
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId()); FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在"); throw new BusinessException("task not found");
} }
FileResultEntity result = fileResultMapper.selectById(job.getResultId()); FileResultEntity result = fileResultMapper.selectById(job.getResultId());
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) { if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
throw new BusinessException("结果记录不存在"); throw new BusinessException("result record not found");
} }
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>() List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, task.getId()) .eq(TaskChunkEntity::getTaskId, task.getId())
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskChunkEntity::getChunkIndex)); .orderByAsc(TaskChunkEntity::getChunkIndex));
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
int cozeWorkUnits = countCozeWorkUnits(chunks, Math.max(1, properties.getCozeBatchSize())); int cozeWorkUnits = countCozeWorkUnits(chunks, Math.max(1, properties.getCozeBatchSize()));
int totalProgressUnits = Math.max(3, cozeWorkUnits + 3); int totalProgressUnits = Math.max(3, cozeWorkUnits + 3);
int[] completedProgressUnits = {0}; saveFileBuildProgress(task, job, totalProgressUnits, 0, "Submitting Coze");
saveFileBuildProgress(task, job, totalProgressUnits, completedProgressUnits[0], "正在请求 Coze"); boolean pendingCoze = submitCozeBatches(task, result, job, chunks, allRowsByBaseId);
Runnable progressHook = () -> { if (pendingCoze) {
taskFileJobService.touchRunning(job.getId()); taskFileJobService.touchRunning(job.getId());
completedProgressUnits[0] = Math.min(totalProgressUnits - 2, completedProgressUnits[0] + 1); touchJavaSideTaskActivity(task.getId());
saveFileBuildProgress(task, job, totalProgressUnits, completedProgressUnits[0], "正在请求 Coze"); saveFileBuildProgress(task, job, totalProgressUnits, 1, "Coze submitted, waiting for result");
}; return false;
applyCozeToPersistedChunks(task, progressHook); }
completedProgressUnits[0] = Math.max(completedProgressUnits[0], totalProgressUnits - 2); completeCozeFileJob(task, result, job, totalProgressUnits, cozeWorkUnits);
saveFileBuildProgress(task, job, totalProgressUnits, completedProgressUnits[0], "正在组装 xlsx"); return true;
}
@Scheduled(fixedDelayString = "${aiimage.appearance-patent.coze-poll-delay-ms:5000}")
public void pollPendingCozeJobs() {
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("appearance-patent:coze-poll", Duration.ofMinutes(1));
if (lockHandle == null) {
return;
}
try (lockHandle) {
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING))
.isNotNull(TaskScopeStateEntity::getCozeExecuteId)
.orderByAsc(TaskScopeStateEntity::getUpdatedAt)
.last("limit 50"));
if (states == null || states.isEmpty()) {
return;
}
log.info("[appearance-patent] coze poll picked pending states count={}", states.size());
for (TaskScopeStateEntity state : states) {
if (state == null || state.getId() == null) {
continue;
}
cozeTaskExecutor.execute(() -> pollPendingCozeState(state.getId()));
}
}
}
private boolean submitCozeBatches(FileTaskEntity task,
FileResultEntity result,
TaskFileJobEntity job,
List<TaskChunkEntity> chunks,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
if (chunks == null || chunks.isEmpty()) {
return countPendingCozeStates(task.getId()) > 0;
}
if (properties.getCozeToken() == null || properties.getCozeToken().isBlank()) {
log.warn("[appearance-patent] coze token not configured, skip async coze taskId={} jobId={}",
task.getId(), job.getId());
return false;
}
String prompt = readAiPrompt(task);
int batchSize = Math.max(1, properties.getCozeBatchSize());
boolean pending = false;
for (TaskChunkEntity chunk : chunks) {
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
if (persistedRows.isEmpty()) {
continue;
}
List<AppearancePatentResultRowDto> unresolvedRows = pickGroupRepresentativesForCoze(persistedRows.values());
if (unresolvedRows.isEmpty()) {
continue;
}
int batchTotal = Math.max(1, (unresolvedRows.size() + batchSize - 1) / batchSize);
int batchIndex = 1;
for (int i = 0; i < unresolvedRows.size(); i += batchSize) {
List<AppearancePatentResultRowDto> batchRows =
unresolvedRows.subList(i, Math.min(i + batchSize, unresolvedRows.size()));
pending |= submitCozeBatch(task, result, job, chunk, batchRows, batchIndex, batchTotal, prompt, allRowsByBaseId);
batchIndex++;
}
}
return pending || countPendingCozeStates(task.getId()) > 0;
}
private boolean submitCozeBatch(FileTaskEntity task,
FileResultEntity result,
TaskFileJobEntity job,
TaskChunkEntity chunk,
List<AppearancePatentResultRowDto> batchRows,
int batchIndex,
int batchTotal,
String prompt,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
if (batchRows == null || batchRows.isEmpty()) {
return false;
}
String batchScopeKey = buildCozeBatchScopeKey(job.getId(), chunk.getScopeHash(), chunk.getChunkIndex(), batchIndex);
String batchScopeHash = DigestUtil.sha256Hex(batchScopeKey);
TaskScopeStateEntity existing = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, task.getId())
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, batchScopeHash)
.last("limit 1"));
if (existing != null) {
return COZE_STATUS_SUBMITTED.equals(existing.getCozeStatus())
|| COZE_STATUS_RUNNING.equals(existing.getCozeStatus());
}
try {
AppearancePatentCozeClient.CozeSubmitResponse submit = cozeClient.submitWorkflow(batchRows, prompt);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List<AppearancePatentResultRowDto> cozeRows = cozeClient.mergeRowsFromDataText(batchRows, submit.immediateData());
mergeCozeRowsIntoChunk(task, chunk.getScopeHash(), chunk.getChunkIndex(), cozeRows, allRowsByBaseId);
return false;
}
if (submit.executeId() == null || submit.executeId().isBlank()) {
mergeCozeRowsIntoChunk(task,
chunk.getScopeHash(),
chunk.getChunkIndex(),
cozeClient.markRowsFailed(batchRows, "Coze async execute_id missing"),
allRowsByBaseId);
return false;
}
saveCozeBatchState(task, result, job, chunk, batchRows, batchScopeKey, batchScopeHash,
batchIndex, batchTotal, submit.executeId());
log.info("[appearance-patent] coze async submitted taskId={} jobId={} chunk={} batch={}/{} executeId={}",
task.getId(), job.getId(), chunk.getChunkIndex(), batchIndex, batchTotal, submit.executeId());
return true;
} catch (Exception ex) {
String message = firstNonBlank(ex.getMessage(), "Coze submit failed");
log.warn("[appearance-patent] coze async submit failed taskId={} jobId={} chunk={} batch={}/{} err={}",
task.getId(), job.getId(), chunk.getChunkIndex(), batchIndex, batchTotal, message);
mergeCozeRowsIntoChunk(task,
chunk.getScopeHash(),
chunk.getChunkIndex(),
cozeClient.markRowsFailed(batchRows, message),
allRowsByBaseId);
return false;
}
}
private void saveCozeBatchState(FileTaskEntity task,
FileResultEntity result,
TaskFileJobEntity job,
TaskChunkEntity chunk,
List<AppearancePatentResultRowDto> batchRows,
String batchScopeKey,
String batchScopeHash,
int batchIndex,
int batchTotal,
String executeId) {
LocalDateTime now = LocalDateTime.now();
CozeBatchContext context = new CozeBatchContext(
job.getId(),
result.getId(),
chunk.getScopeHash(),
chunk.getChunkIndex(),
batchIndex,
batchTotal
);
String batchPayload = writeJson(batchRows, "serialize coze batch payload failed");
String storedBatchPayload = transientPayloadStorageService.storeParsedPayloadFast(
MODULE_TYPE, task.getId(), batchScopeHash, batchPayload, true);
TaskScopeStateEntity state = new TaskScopeStateEntity();
state.setTaskId(task.getId());
state.setModuleType(MODULE_TYPE);
state.setScopeKey(batchScopeKey);
state.setScopeHash(batchScopeHash);
state.setParsedPayloadJson(storedBatchPayload);
state.setStateJson(writeJson(context, "serialize coze batch context failed"));
state.setCozeExecuteId(executeId);
state.setCozeStatus(COZE_STATUS_SUBMITTED);
state.setCozeSubmittedAt(now);
state.setCozeAttemptCount(0);
state.setChunkTotal(batchTotal);
state.setReceivedChunkCount(batchIndex);
state.setCompleted(0);
state.setCreatedAt(now);
state.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(state);
touchJavaSideTaskActivity(task.getId());
} catch (DuplicateKeyException ex) {
transientPayloadStorageService.deletePayloadIfPresent(storedBatchPayload);
log.info("[appearance-patent] duplicate coze batch state ignored taskId={} scope={}",
task.getId(), batchScopeKey);
}
}
private void pollPendingCozeState(Long stateId) {
TaskScopeStateEntity state = taskScopeStateMapper.selectById(stateId);
if (state == null || state.getCozeExecuteId() == null || state.getCozeExecuteId().isBlank()) {
return;
}
if (!(COZE_STATUS_SUBMITTED.equals(state.getCozeStatus()) || COZE_STATUS_RUNNING.equals(state.getCozeStatus()))) {
return;
}
if (!tryClaimCozeStateForPoll(state)) {
return;
}
CozeBatchContext context = readCozeBatchContext(state);
if (context == null || context.jobId() == null || context.resultId() == null) {
markCozeStateTerminal(state, COZE_STATUS_FAILED, "Coze batch context missing");
return;
}
taskFileJobService.touchRunning(context.jobId());
try {
AppearancePatentCozeClient.CozePollResponse poll = cozeClient.pollWorkflow(state.getCozeExecuteId());
if (!poll.hasPayload() && !poll.isFinished() && !isCozeStateTimedOut(state)) {
updateCozeStateRunning(state, null);
return;
}
String failureMessage = poll.isFailed()
? firstNonBlank(poll.failureMessage(), "Coze async workflow failed")
: "";
if (!poll.hasPayload() && failureMessage.isBlank()) {
failureMessage = isCozeStateTimedOut(state)
? "Coze async workflow poll timeout"
: "Coze async workflow completed without output";
}
List<AppearancePatentResultRowDto> batchRows = readCozeBatchRows(state);
List<AppearancePatentResultRowDto> cozeRows = failureMessage.isBlank()
? cozeClient.mergeRowsFromDataText(batchRows, poll.resolvedPayloadText())
: cozeClient.markRowsFailed(batchRows, failureMessage);
FileTaskEntity task = fileTaskMapper.selectById(state.getTaskId());
if (task != null) {
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
mergeCozeRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), cozeRows, allRowsByBaseId);
}
markCozeStateTerminal(state,
failureMessage.isBlank() ? COZE_STATUS_DONE : COZE_STATUS_FAILED,
failureMessage.isBlank() ? null : failureMessage);
maybeFinalizeCozeJob(state.getTaskId(), context);
} catch (Exception ex) {
String message = firstNonBlank(ex.getMessage(), "Coze poll failed");
if (isCozeStateTimedOut(state)) {
List<AppearancePatentResultRowDto> batchRows = readCozeBatchRows(state);
FileTaskEntity task = fileTaskMapper.selectById(state.getTaskId());
if (task != null) {
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
mergeCozeRowsIntoChunk(task,
context.chunkScopeHash(),
context.chunkIndex(),
cozeClient.markRowsFailed(batchRows, message),
allRowsByBaseId);
}
markCozeStateTerminal(state, COZE_STATUS_FAILED, message);
maybeFinalizeCozeJob(state.getTaskId(), context);
return;
}
log.warn("[appearance-patent] coze poll failed taskId={} stateId={} executeId={} err={}",
state.getTaskId(), state.getId(), state.getCozeExecuteId(), message);
updateCozeStateRunning(state, message);
}
}
private void updateCozeStateRunning(TaskScopeStateEntity state, String error) {
int updated = taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING))
.set(TaskScopeStateEntity::getCozeStatus, COZE_STATUS_RUNNING)
.set(TaskScopeStateEntity::getCozeLastPolledAt, LocalDateTime.now())
.set(TaskScopeStateEntity::getCozeAttemptCount, cozeAttemptCount(state) + 1)
.set(TaskScopeStateEntity::getCozeError, error)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
if (updated > 0) {
touchJavaSideTaskActivity(state.getTaskId());
}
}
private boolean tryClaimCozeStateForPoll(TaskScopeStateEntity state) {
if (state == null || state.getId() == null) {
return false;
}
LocalDateTime now = LocalDateTime.now();
long intervalMillis = Math.max(200L, properties.getCozePollIntervalMillis());
if (state.getCozeLastPolledAt() != null
&& Duration.between(state.getCozeLastPolledAt(), now).toMillis() < intervalMillis) {
return false;
}
return taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING))
.and(wrapper -> wrapper
.isNull(TaskScopeStateEntity::getCozeLastPolledAt)
.or()
.le(TaskScopeStateEntity::getCozeLastPolledAt, now.minus(Duration.ofMillis(intervalMillis))))
.set(TaskScopeStateEntity::getCozeStatus, COZE_STATUS_RUNNING)
.set(TaskScopeStateEntity::getCozeLastPolledAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now)) > 0;
}
private void markCozeStateTerminal(TaskScopeStateEntity state, String status, String error) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING))
.set(TaskScopeStateEntity::getCozeStatus, status)
.set(TaskScopeStateEntity::getCozeCompletedAt, LocalDateTime.now())
.set(TaskScopeStateEntity::getCozeLastPolledAt, LocalDateTime.now())
.set(TaskScopeStateEntity::getCozeAttemptCount, cozeAttemptCount(state) + 1)
.set(TaskScopeStateEntity::getCozeError, error)
.set(TaskScopeStateEntity::getCompleted, 1)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
}
private void maybeFinalizeCozeJob(Long taskId, CozeBatchContext context) {
if (taskId == null || context == null || countPendingCozeStates(taskId) > 0) {
return;
}
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("appearance-patent:coze-finalize:" + taskId, Duration.ofMinutes(5));
if (lockHandle == null) {
return;
}
try (lockHandle) {
if (countPendingCozeStates(taskId) > 0) {
return;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
FileResultEntity result = fileResultMapper.selectById(context.resultId());
TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId());
if (task == null || result == null || job == null || "SUCCESS".equals(job.getStatus())) {
return;
}
int cozeWorkUnits = countCompletedCozeStates(taskId);
int totalProgressUnits = Math.max(3, cozeWorkUnits + 3);
completeCozeFileJob(task, result, job, totalProgressUnits, cozeWorkUnits);
taskFileJobService.markSuccess(job, result.getResultFileUrl());
cleanupResultFileJob(job);
log.info("[appearance-patent] coze async job finalized taskId={} jobId={} resultId={} resultFileUrl={}",
taskId, job.getId(), result.getId(), result.getResultFileUrl());
} catch (Exception ex) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId());
if (job != null) {
taskFileJobService.markFailed(job, firstNonBlank(ex.getMessage(), "appearance patent result file build failed"));
}
log.warn("[appearance-patent] coze async finalize failed taskId={} resultId={} err={}",
taskId, context.resultId(), ex.getMessage(), ex);
}
}
private void completeCozeFileJob(FileTaskEntity task,
FileResultEntity result,
TaskFileJobEntity job,
int totalProgressUnits,
int cozeWorkUnits) {
int assembleProgress = Math.max(1, Math.min(totalProgressUnits - 2, cozeWorkUnits));
saveFileBuildProgress(task, job, totalProgressUnits, assembleProgress, "Assembling xlsx");
assembleResultWorkbook(task, result); assembleResultWorkbook(task, result);
completedProgressUnits[0] = totalProgressUnits - 1; saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits - 1, "Uploading result file");
saveFileBuildProgress(task, job, totalProgressUnits, completedProgressUnits[0], "正在上传结果文件");
fileResultMapper.updateById(result); fileResultMapper.updateById(result);
saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "结果文件已生成"); saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "Result file generated");
}
private void mergeCozeRowsIntoChunk(FileTaskEntity task,
String chunkScopeHash,
Integer chunkIndex,
List<AppearancePatentResultRowDto> cozeRows,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
if (task == null || cozeRows == null || cozeRows.isEmpty()) {
return;
}
Map<String, AppearancePatentResultRowDto> mergedRows = new LinkedHashMap<>();
for (AppearancePatentResultRowDto resultRow : cozeRows) {
for (AppearancePatentResultRowDto expandedRow : expandRows(List.of(resultRow), allRowsByBaseId)) {
mergedRows.put(rowKey(expandedRow), expandedRow);
}
}
mergeChunkPayload(task.getId(), chunkScopeHash, chunkIndex, new ArrayList<>(mergedRows.values()));
}
private List<AppearancePatentResultRowDto> readCozeBatchRows(TaskScopeStateEntity state) {
if (state == null || state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) {
return List.of();
}
try {
String payloadJson = transientPayloadStorageService.resolvePayload(
state.getParsedPayloadJson(), "read appearance patent coze batch failed");
JsonNode array = objectMapper.readTree(payloadJson);
if (!array.isArray()) {
return List.of();
}
List<AppearancePatentResultRowDto> rows = new ArrayList<>();
for (JsonNode node : array) {
rows.add(objectMapper.treeToValue(node, AppearancePatentResultRowDto.class));
}
return rows;
} catch (Exception ex) {
log.warn("[appearance-patent] read coze batch failed taskId={} stateId={} err={}",
state.getTaskId(), state.getId(), ex.getMessage());
return List.of();
}
}
private CozeBatchContext readCozeBatchContext(TaskScopeStateEntity state) {
if (state == null || state.getStateJson() == null || state.getStateJson().isBlank()) {
return null;
}
try {
return objectMapper.readValue(state.getStateJson(), CozeBatchContext.class);
} catch (Exception ex) {
log.warn("[appearance-patent] read coze batch context failed taskId={} stateId={} err={}",
state.getTaskId(), state.getId(), ex.getMessage());
return null;
}
}
private boolean isCozeStateTimedOut(TaskScopeStateEntity state) {
if (state == null || state.getCozeSubmittedAt() == null) {
return false;
}
long timeoutMillis = Math.max(10000L, properties.getCozePollTimeoutMillis());
return Duration.between(state.getCozeSubmittedAt(), LocalDateTime.now()).toMillis() >= timeoutMillis;
}
private int cozeAttemptCount(TaskScopeStateEntity state) {
return state == null || state.getCozeAttemptCount() == null ? 0 : state.getCozeAttemptCount();
}
private int countPendingCozeStates(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING)));
return count == null ? 0 : count.intValue();
}
private int countCompletedCozeStates(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_DONE, COZE_STATUS_FAILED)));
return count == null ? 0 : count.intValue();
}
private boolean isJavaSideProcessing(Long taskId) {
return countPendingCozeStates(taskId) > 0
|| taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE) > 0;
}
private void touchJavaSideTaskActivity(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, taskId)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now()));
}
private String buildCozeBatchScopeKey(Long jobId, String chunkScopeHash, Integer chunkIndex, int batchIndex) {
return "coze:job:" + jobId
+ ":chunk:" + (chunkIndex == null ? 0 : chunkIndex)
+ ":" + firstNonBlank(chunkScopeHash, "unknown")
+ ":batch:" + batchIndex;
} }
private int countCozeWorkUnits(List<TaskChunkEntity> chunks, int batchSize) { private int countCozeWorkUnits(List<TaskChunkEntity> chunks, int batchSize) {
@@ -1101,7 +1590,7 @@ public class AppearancePatentTaskService {
private void assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) { private void assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) {
AppearancePatentParsedPayloadDto parsed = readParsedPayload(task); AppearancePatentParsedPayloadDto parsed = readParsedPayload(task);
Map<String, AppearancePatentResultRowDto> resultMap = loadPersistedResultRows(task.getId()); Map<String, AppearancePatentResultRowDto> resultMap = loadPersistedResultRowsWithRetry(task.getId(), parsed.getAllItems().size());
long resolvedRows = parsed.getAllItems().stream() long resolvedRows = parsed.getAllItems().stream()
.filter(row -> findResultRow(row, resultMap) != null || findResultRowByAsin(row.getAsin(), resultMap) != null) .filter(row -> findResultRow(row, resultMap) != null || findResultRowByAsin(row.getAsin(), resultMap) != null)
.count(); .count();
@@ -1110,12 +1599,20 @@ public class AppearancePatentTaskService {
.count(); .count();
log.info("[appearance-patent] assemble workbook taskId={} parsedRows={} resultRows={} resolvedRows={} reasonRows={}", log.info("[appearance-patent] assemble workbook taskId={} parsedRows={} resultRows={} resolvedRows={} reasonRows={}",
task.getId(), parsed.getAllItems().size(), resultMap.size(), resolvedRows, reasonRows); task.getId(), parsed.getAllItems().size(), resultMap.size(), resolvedRows, reasonRows);
if (!parsed.getAllItems().isEmpty() && resultMap.isEmpty()) {
throw new BusinessException("外观专利检测结果为空，请稍后重试生成结果文件");
}
File outputDir = new File(storageProperties.getLocalTempDir(), "appearance-patent-result"); File outputDir = new File(storageProperties.getLocalTempDir(), "appearance-patent-result");
if (!outputDir.exists() && !outputDir.mkdirs()) { if (!outputDir.exists() && !outputDir.mkdirs()) {
throw new BusinessException("创建结果目录失败"); throw new BusinessException("创建结果目录失败");
} }
String filename = safeFileStem(result.getSourceFilename()) + "-result.xlsx"; String filename = safeFileStem(result.getSourceFilename()) + "-result.xlsx";
File xlsx = new File(outputDir, filename); String tempFilename = safeFileStem(result.getSourceFilename())
+ "-" + task.getId()
+ "-" + result.getId()
+ "-" + UUID.randomUUID()
+ "-result.xlsx";
File xlsx = new File(outputDir, tempFilename);
try { try {
writeResultWorkbook(xlsx, parsed, resultMap); writeResultWorkbook(xlsx, parsed, resultMap);
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE); String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
@@ -1143,6 +1640,33 @@ public class AppearancePatentTaskService {
return result; return result;
} }
private Map<String, AppearancePatentResultRowDto> loadPersistedResultRowsWithRetry(Long taskId, int expectedParsedRows) {
Map<String, AppearancePatentResultRowDto> result = loadPersistedResultRows(taskId);
if (expectedParsedRows <= 0 || !result.isEmpty()) {
return result;
}
for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) {
sleepBeforeResultRowsRetry(attempt);
result = loadPersistedResultRows(taskId);
if (!result.isEmpty()) {
log.info("[appearance-patent] result rows recovered after retry taskId={} attempt={} rows={}",
taskId, attempt, result.size());
return result;
}
log.warn("[appearance-patent] result rows still empty after retry taskId={} attempt={}/{}",
taskId, attempt, RESULT_ROWS_READ_RETRY_LIMIT);
}
return result;
}
private void sleepBeforeResultRowsRetry(int attempt) {
try {
Thread.sleep(RESULT_ROWS_READ_RETRY_DELAY_MS * attempt);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
private void writeResultWorkbook(File xlsx, AppearancePatentParsedPayloadDto parsed, Map<String, AppearancePatentResultRowDto> resultMap) { private void writeResultWorkbook(File xlsx, AppearancePatentParsedPayloadDto parsed, Map<String, AppearancePatentResultRowDto> resultMap) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) { try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
Sheet sheet = workbook.createSheet("外观专利检测结果"); Sheet sheet = workbook.createSheet("外观专利检测结果");
@@ -1615,9 +2139,9 @@ public class AppearancePatentTaskService {
payload.setAiPrompt(normalize(aiPrompt)); payload.setAiPrompt(normalize(aiPrompt));
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles); payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
payload.setHeaders(headers == null ? List.of() : headers); payload.setHeaders(headers == null ? List.of() : headers);
payload.setItems(allRows == null ? List.of() : allRows); payload.setItems(List.of());
payload.setGroups(groups == null ? List.of() : groups); payload.setGroups(groups == null ? List.of() : groups);
payload.setAllItems(allRows == null ? List.of() : allRows); payload.setAllItems(List.of());
return writeJson(payload, "保存解析结果失败"); return writeJson(payload, "保存解析结果失败");
} }
@@ -1633,7 +2157,11 @@ public class AppearancePatentTaskService {
} }
private String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) { private String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) {
return transientPayloadStorageService.storeParsedPayload(MODULE_TYPE, taskId, scopeHash, parsedPayloadJson, true); return transientPayloadStorageService.storeParsedPayloadFast(MODULE_TYPE, taskId, scopeHash, parsedPayloadJson, false);
}
private long elapsedMs(long start, long end) {
return (end - start) / 1_000_000L;
} }
private AppearancePatentParsedPayloadDto readParsedPayload(FileTaskEntity task) { private AppearancePatentParsedPayloadDto readParsedPayload(FileTaskEntity task) {
@@ -1642,16 +2170,42 @@ public class AppearancePatentTaskService {
String pointer = root.path("parsedPayloadRef").asText(""); String pointer = root.path("parsedPayloadRef").asText("");
if (!pointer.isBlank()) { if (!pointer.isBlank()) {
String json = transientPayloadStorageService.resolvePayload(pointer, "read appearance patent parsed payload failed"); String json = transientPayloadStorageService.resolvePayload(pointer, "read appearance patent parsed payload failed");
return objectMapper.readValue(json, AppearancePatentParsedPayloadDto.class); return hydrateParsedPayloadRows(objectMapper.readValue(json, AppearancePatentParsedPayloadDto.class));
} }
if (root.path("allItems").isArray()) { if (root.path("allItems").isArray()) {
return objectMapper.treeToValue(root, AppearancePatentParsedPayloadDto.class); return hydrateParsedPayloadRows(objectMapper.treeToValue(root, AppearancePatentParsedPayloadDto.class));
} }
} catch (Exception ex) { } catch (Exception ex) {
throw new BusinessException("读取解析载荷失败"); throw new BusinessException("读取解析载荷失败");
} }
return hydrateParsedPayloadRows(new AppearancePatentParsedPayloadDto());
}
private AppearancePatentParsedPayloadDto hydrateParsedPayloadRows(AppearancePatentParsedPayloadDto payload) {
if (payload == null) {
return new AppearancePatentParsedPayloadDto(); return new AppearancePatentParsedPayloadDto();
} }
List<AppearancePatentParsedRowVo> rows = payload.getAllItems();
if (rows == null || rows.isEmpty()) {
rows = payload.getItems();
}
if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) {
rows = payload.getGroups().stream()
.filter(Objects::nonNull)
.flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream())
.toList();
}
if (rows == null) {
rows = List.of();
}
if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) {
payload.setAllItems(rows);
}
if (payload.getItems() == null || payload.getItems().isEmpty()) {
payload.setItems(rows);
}
return payload;
}
private Map<String, AppearancePatentResultRowDto> readChunkRows(TaskChunkEntity chunk) { private Map<String, AppearancePatentResultRowDto> readChunkRows(TaskChunkEntity chunk) {
Map<String, AppearancePatentResultRowDto> rows = new LinkedHashMap<>(); Map<String, AppearancePatentResultRowDto> rows = new LinkedHashMap<>();
@@ -1831,6 +2385,14 @@ public class AppearancePatentTaskService {
String error) { String error) {
} }
private record CozeBatchContext(Long jobId,
Long resultId,
String chunkScopeHash,
Integer chunkIndex,
Integer batchIndex,
Integer batchTotal) {
}
private record ParsedWorkbook(int totalRows, int droppedRows, List<String> headers, List<AppearancePatentParsedRowVo> allRows) { private record ParsedWorkbook(int totalRows, int droppedRows, List<String> headers, List<AppearancePatentParsedRowVo> allRows) {
} }
} }

8
backend-java/src/main/java/com/nanri/aiimage/modules/brand/controller/BrandTaskController.java
View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.brand.controller; package com.nanri.aiimage.modules.brand.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultRequest; import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultRequest;
import com.nanri.aiimage.modules.brand.model.dto.BrandTaskCreateRequest; import com.nanri.aiimage.modules.brand.model.dto.BrandTaskCreateRequest;
import com.nanri.aiimage.modules.brand.model.vo.BrandCrawlPayloadVo; import com.nanri.aiimage.modules.brand.model.vo.BrandCrawlPayloadVo;
@@ -17,7 +18,6 @@ import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import jakarta.servlet.http.HttpServletResponse; import jakarta.servlet.http.HttpServletResponse;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -30,8 +30,6 @@ import org.springframework.web.bind.annotation.RestController;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -213,10 +211,8 @@ public class BrandTaskController {
if (filename.isBlank()) { if (filename.isBlank()) {
filename = "brand_task_" + taskId + ".zip"; filename = "brand_task_" + taskId + ".zip";
} }
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/zip"); response.setContentType("application/zip");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
// 后端代理拉取 OSS 文件并流式写出 // 后端代理拉取 OSS 文件并流式写出
try (InputStream in = URI.create(ossUrl).toURL().openStream()) { try (InputStream in = URI.create(ossUrl).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];

38
backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java
View File

@@ -614,6 +614,9 @@ public class BrandTaskService {
if (STATUS_FAILED.equalsIgnoreCase(freshStatus)) { if (STATUS_FAILED.equalsIgnoreCase(freshStatus)) {
throw new BusinessException("任务已结束，不能继续组装结果"); throw new BusinessException("任务已结束，不能继续组装结果");
} }
for (BrandSourceFileDto sourceFile : sourceFiles) {
brandTaskStorageService.refreshFileAggregate(taskId, sourceFile.getFileUrl());
}
Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskStorageService.getAllFileAggregates(taskId); Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskStorageService.getAllFileAggregates(taskId);
log.info("[brand-finalize] taskId={} aggregateFiles={} expectedFiles={} thread={}", log.info("[brand-finalize] taskId={} aggregateFiles={} expectedFiles={} thread={}",
taskId, taskId,
@@ -621,12 +624,12 @@ public class BrandTaskService {
totalCount, totalCount,
Thread.currentThread().getName()); Thread.currentThread().getName());
if (aggregates.size() < totalCount) { if (aggregates.size() < totalCount) {
return; throw new BusinessException("品牌检测结果分片尚未收齐，请稍后重试");
} }
for (BrandSourceFileDto sourceFile : sourceFiles) { for (BrandSourceFileDto sourceFile : sourceFiles) {
BrandFileAggregateCacheDto aggregate = aggregates.get(sourceFile.getFileUrl()); BrandFileAggregateCacheDto aggregate = aggregates.get(sourceFile.getFileUrl());
if (aggregate == null || !Boolean.TRUE.equals(aggregate.getCompleted())) { if (aggregate == null || !Boolean.TRUE.equals(aggregate.getCompleted())) {
return; throw new BusinessException("品牌检测结果分片尚未收齐，请稍后重试");
} }
} }
@@ -1208,6 +1211,9 @@ public class BrandTaskService {
.eq(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING) .eq(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
.lt(BrandCrawlTaskEntity::getUpdatedAt, threshold)); .lt(BrandCrawlTaskEntity::getUpdatedAt, threshold));
for (BrandCrawlTaskEntity task : runningTasks) { for (BrandCrawlTaskEntity task : runningTasks) {
if (tryRecoverCompletedRunningTask(task)) {
continue;
}
Map<Object, Object> progress = brandTaskProgressCacheService.getProgress(task.getId()); Map<Object, Object> progress = brandTaskProgressCacheService.getProgress(task.getId());
if (progress.isEmpty()) { if (progress.isEmpty()) {
failStaleRunningTask(task.getId(), "任务进度缓存已过期，任务已自动失败"); failStaleRunningTask(task.getId(), "任务进度缓存已过期，任务已自动失败");
@@ -1231,6 +1237,34 @@ public class BrandTaskService {
} }
} }
private boolean tryRecoverCompletedRunningTask(BrandCrawlTaskEntity task) {
if (task == null || task.getId() == null) {
return false;
}
try {
List<BrandSourceFileDto> sourceFiles = parseSourceFiles(task.getFilePaths());
if (sourceFiles.isEmpty()) {
return false;
}
List<BrandParsedFileCacheDto> cachedFiles = brandTaskStorageService.getParsedPayload(task.getId());
Map<String, BrandParsedFileCacheDto> cachedByUrl = indexCachedFiles(cachedFiles);
for (BrandSourceFileDto sourceFile : sourceFiles) {
BrandTaskStorageService.ChunkStoreResult result =
brandTaskStorageService.refreshFileAggregate(task.getId(), sourceFile.getFileUrl());
BrandFileAggregateCacheDto aggregate = result.aggregate();
if (aggregate == null || !Boolean.TRUE.equals(aggregate.getCompleted())) {
return false;
}
}
log.info("[brand-stale-check] recover completed running taskId={} files={}", task.getId(), sourceFiles.size());
finalizeTask(task.getId(), normalizeStrategy(task.getStrategy()), sourceFiles, cachedByUrl, sourceFiles.size());
return true;
} catch (Exception ex) {
log.warn("[brand-stale-check] recover completed running taskId={} failed msg={}", task.getId(), ex.getMessage());
return false;
}
}
private void failStaleRunningTask(Long taskId, String message) { private void failStaleRunningTask(Long taskId, String message) {
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>() brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId) .eq(BrandCrawlTaskEntity::getId, taskId)

18
backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java
View File

@@ -278,20 +278,30 @@ public class BrandTaskStorageService {
if (state == null) { if (state == null) {
throw new BusinessException("Brand task aggregate state missing: " + scopeKey); throw new BusinessException("Brand task aggregate state missing: " + scopeKey);
} }
String storedAggregate = transientPayloadStorageService.storeScopePayload(MODULE_TYPE, taskId, scopeHash, aggregateJson, true); String storedAggregate = transientPayloadStorageService.storeScopePayloadVersioned(MODULE_TYPE, taskId, scopeHash, aggregateJson, true);
int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal(); int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal();
int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount(); int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getStateJson(), storedAggregate); int completed = Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0;
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>() int updated = taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId()) .eq(TaskScopeStateEntity::getId, state.getId())
.and(wrapper -> wrapper
.lt(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.or()
.eq(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.le(TaskScopeStateEntity::getCompleted, completed))
.set(TaskScopeStateEntity::getScopeKey, scopeKey) .set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getStateJson, storedAggregate) .set(TaskScopeStateEntity::getStateJson, storedAggregate)
.set(TaskScopeStateEntity::getChunkTotal, chunkTotal) .set(TaskScopeStateEntity::getChunkTotal, chunkTotal)
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount) .set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.set(TaskScopeStateEntity::getCompleted, Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0) .set(TaskScopeStateEntity::getCompleted, completed)
.set(TaskScopeStateEntity::getLastChunkAt, now) .set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getLastError, null) .set(TaskScopeStateEntity::getLastError, null)
.set(TaskScopeStateEntity::getUpdatedAt, now)); .set(TaskScopeStateEntity::getUpdatedAt, now));
if (updated > 0) {
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getStateJson(), storedAggregate);
} else {
transientPayloadStorageService.deletePayloadIfPresent(storedAggregate);
}
} }
private TaskScopeStateEntity getScopeState(Long taskId, String scopeHash) { private TaskScopeStateEntity getScopeState(Long taskId, String scopeHash) {

20
backend-java/src/main/java/com/nanri/aiimage/modules/convert/controller/ConvertRunController.java
View File

@@ -14,10 +14,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -28,9 +25,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import java.io.File; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -100,15 +95,14 @@ public class ConvertRunController {
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "返回结果文件流"), @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "返回结果文件流"),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "结果文件不存在") @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "结果文件不存在")
}) })
public ResponseEntity<Resource> download( public ResponseEntity<Void> download(
@Parameter(description = "格式转换结果记录 ID", required = true) @PathVariable Long resultId, @Parameter(description = "格式转换结果记录 ID", required = true) @PathVariable Long resultId,
@Parameter(name = "user_id", description = "当前登录用户 ID", required = true, in = ParameterIn.QUERY) @Parameter(name = "user_id", description = "当前登录用户 ID", required = true, in = ParameterIn.QUERY)
@RequestParam("user_id") Long userId) { @RequestParam("user_id") Long userId) {
File resultFile = convertRunService.getResultFile(resultId, userId); String downloadUrl = convertRunService.getResultDownloadUrl(resultId, userId);
String encodedFilename = URLEncoder.encode(resultFile.getName(), StandardCharsets.UTF_8).replaceAll("\\+", "%20"); return ResponseEntity.status(302)
return ResponseEntity.ok() .location(URI.create(downloadUrl))
.contentType(MediaType.APPLICATION_OCTET_STREAM) .header(HttpHeaders.CACHE_CONTROL, "no-store")
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=UTF-8''" + encodedFilename) .build();
.body(new FileSystemResource(resultFile));
} }
} }

37
backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java
View File

@@ -29,6 +29,7 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
@@ -43,6 +44,7 @@ public class ConvertRunService {
private static final String MODULE_TYPE = "CONVERT"; private static final String MODULE_TYPE = "CONVERT";
private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer"; private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer";
private static final DateTimeFormatter ZIP_NAME_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
private static final List<String> FIVE_COUNTRY_OUTPUT_FILES = List.of( private static final List<String> FIVE_COUNTRY_OUTPUT_FILES = List.of(
"英国.txt", "英国.txt",
"法国.txt", "法国.txt",
@@ -76,10 +78,12 @@ public class ConvertRunService {
fileTaskMapper.insert(task); fileTaskMapper.insert(task);
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank(); boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
boolean archiveMode = folderMode || request.getFiles().size() > 1;
List<ConvertResultItemVo> items = new ArrayList<>(); List<ConvertResultItemVo> items = new ArrayList<>();
int successCount = 0; int successCount = 0;
int failedCount = 0; int failedCount = 0;
List<ConvertArchiveEntry> archiveEntries = new ArrayList<>(); List<ConvertArchiveEntry> archiveEntries = new ArrayList<>();
List<String> successSourceNames = new ArrayList<>();
for (UploadedSourceFileDto sourceFile : request.getFiles()) { for (UploadedSourceFileDto sourceFile : request.getFiles()) {
try { try {
@@ -92,10 +96,11 @@ public class ConvertRunService {
? inputFile.getName() ? inputFile.getName()
: sourceFile.getOriginalFilename(); : sourceFile.getOriginalFilename();
List<GeneratedConvertFile> generatedFiles = generateOutputFiles(inputFile, template); List<GeneratedConvertFile> generatedFiles = generateOutputFiles(inputFile, template);
if (folderMode) { if (archiveMode) {
for (GeneratedConvertFile generatedFile : generatedFiles) { for (GeneratedConvertFile generatedFile : generatedFiles) {
archiveEntries.add(new ConvertArchiveEntry(sourceFile.getRelativePath(), inputName, generatedFile)); archiveEntries.add(new ConvertArchiveEntry(sourceFile.getRelativePath(), inputName, generatedFile));
} }
successSourceNames.add(inputName);
successCount++; successCount++;
continue; continue;
} }
@@ -153,12 +158,13 @@ public class ConvertRunService {
} }
} }
if (folderMode && !archiveEntries.isEmpty()) { if (archiveMode && !archiveEntries.isEmpty()) {
File zipFile = packageFolderConvertResultsAsZip(request.getArchiveName(), archiveEntries); String archiveName = resolveArchiveName(request.getArchiveName(), successSourceNames);
File zipFile = packageFolderConvertResultsAsZip(archiveName, archiveEntries);
String ossObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE); String ossObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE);
// 只存 objectKey // 只存 objectKey
ConvertResultItemVo item = new ConvertResultItemVo(); ConvertResultItemVo item = new ConvertResultItemVo();
item.setSourceFilename(request.getArchiveName()); item.setSourceFilename(folderMode ? request.getArchiveName() : "Current convert task");
item.setOutputFilename(zipFile.getName()); item.setOutputFilename(zipFile.getName());
item.setSuccess(true); item.setSuccess(true);
item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey)); item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey));
@@ -166,7 +172,7 @@ public class ConvertRunService {
FileResultEntity resultEntity = new FileResultEntity(); FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId()); resultEntity.setTaskId(task.getId());
resultEntity.setModuleType(MODULE_TYPE); resultEntity.setModuleType(MODULE_TYPE);
resultEntity.setSourceFilename(request.getArchiveName()); resultEntity.setSourceFilename(item.getSourceFilename());
resultEntity.setResultFilename(zipFile.getName()); resultEntity.setResultFilename(zipFile.getName());
resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey
resultEntity.setResultFileSize(zipFile.length()); resultEntity.setResultFileSize(zipFile.length());
@@ -205,7 +211,7 @@ public class ConvertRunService {
vo.setResultId(entity.getId()); vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getSourceFilename()); vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename()); vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(null); vo.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage()); vo.setError(entity.getErrorMessage());
return vo; return vo;
@@ -221,17 +227,12 @@ public class ConvertRunService {
fileResultMapper.deleteById(resultId); fileResultMapper.deleteById(resultId);
} }
public File getResultFile(Long resultId, Long userId) { public String getResultDownloadUrl(Long resultId, Long userId) {
FileResultEntity entity = fileResultMapper.selectById(resultId); FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("Convert result file does not exist."); throw new BusinessException("Convert result file does not exist.");
} }
return ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl());
File file = new File(entity.getResultFileUrl());
if (!file.exists()) {
throw new BusinessException("Convert result file does not exist.");
}
return file;
} }
private List<GeneratedConvertFile> generateOutputFiles(File inputFile, ConvertTemplateEntity templateEntity) throws IOException { private List<GeneratedConvertFile> generateOutputFiles(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
@@ -410,6 +411,16 @@ public class ConvertRunService {
return zipFile; return zipFile;
} }
private String resolveArchiveName(String archiveName, List<String> successSourceNames) {
if (archiveName != null && !archiveName.isBlank()) {
return FileUtil.mainName(archiveName.trim());
}
if (successSourceNames.size() == 1) {
return FileUtil.mainName(successSourceNames.getFirst());
}
return "convert-" + LocalDateTime.now().format(ZIP_NAME_DATE_FORMATTER);
}
private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) { private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) {
String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/'); String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/');
String relativeDir = normalizedRelativePath.contains("/") String relativeDir = normalizedRelativePath.contains("/")

54
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/controller/DeleteBrandRunController.java
View File

@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.deletebrand.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandRunRequest; import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandRunRequest;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandSubmitResultRequest; import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandSubmitResultRequest;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandTaskBatchRequest; import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandTaskBatchRequest;
@@ -19,7 +20,6 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -33,7 +33,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@@ -134,11 +133,8 @@ public class DeleteBrandRunController {
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
String asciiFilename = buildAsciiDownloadFilename(filename, taskId);
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + asciiFilename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;
@@ -152,23 +148,37 @@ public class DeleteBrandRunController {
} }
} }
private String buildAsciiDownloadFilename(String filename, Long taskId) { @GetMapping("/results/{resultId}/download")
String sanitized = filename == null ? "" : filename.replaceAll("[^A-Za-z0-9._-]", "_"); @Operation(summary = "Download one delete-brand result")
sanitized = sanitized.replaceAll("_+", "_"); public void downloadResult(
sanitized = sanitized.replaceAll("^[_\\.]+|[_\\.]+$", ""); @PathVariable Long resultId,
if (!sanitized.isBlank() && sanitized.contains(".")) { @Parameter(name = "user_id", description = "褰撳墠鐧诲綍鐢ㄦ埛 ID", required = true, in = ParameterIn.QUERY)
return sanitized; @RequestParam("user_id") Long userId,
jakarta.servlet.http.HttpServletResponse response) {
String url = deleteBrandRunService.resolveResultDownloadUrl(resultId, userId);
String filename = deleteBrandRunService.resolveResultDownloadFilename(resultId, userId);
if (url == null || url.isBlank()) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "缁撴灉鏃犲彲涓嬭浇");
} }
String ext = "xlsx"; if (filename == null || filename.isBlank()) {
if (filename != null) { String rawPath = URI.create(url).getPath();
int dotIndex = filename.lastIndexOf('.'); filename = rawPath == null ? "delete-brand_" + resultId + ".xlsx" : rawPath.substring(rawPath.lastIndexOf('/') + 1);
if (dotIndex >= 0 && dotIndex < filename.length() - 1) { }
String rawExt = filename.substring(dotIndex + 1).replaceAll("[^A-Za-z0-9]", "");
if (!rawExt.isBlank()) { try {
ext = rawExt; response.setContentType("application/octet-stream");
DownloadHeaderUtil.setAttachment(response, filename);
try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536];
int read;
while ((read = in.read(buffer)) != -1) {
response.getOutputStream().write(buffer, 0, read);
}
response.getOutputStream().flush();
}
} catch (Exception ex) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "涓嬭浇澶辫触");
} }
} }
}
return "delete-brand_" + taskId + "." + ext;
}
} }

384
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java
View File

@@ -84,6 +84,20 @@ public class DeleteBrandRunService {
private final TaskPressureProperties taskPressureProperties; private final TaskPressureProperties taskPressureProperties;
private final TaskFileJobService taskFileJobService; private final TaskFileJobService taskFileJobService;
private boolean isUsableIndexedMatch(ZiniaoShopMatchResultVo matchResult) {
if (matchResult == null) {
return false;
}
String shopId = matchResult.getShopId();
if (shopId == null || shopId.isBlank()) {
return false;
}
String status = matchResult.getMatchStatus();
return matchResult.isMatched()
|| ZiniaoShopIndexService.MATCH_STATUS_MATCHED.equals(status)
|| ZiniaoShopIndexService.MATCH_STATUS_STALE.equals(status);
}
public DeleteBrandRunVo run(DeleteBrandRunRequest request) { public DeleteBrandRunVo run(DeleteBrandRunRequest request) {
if (request.getUserId() == null || request.getUserId() <= 0) { if (request.getUserId() == null || request.getUserId() <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
@@ -136,8 +150,8 @@ public class DeleteBrandRunService {
ignored -> ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName(), false)); ignored -> ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName(), false));
item.setMatchStatus(matchResult.getMatchStatus()); item.setMatchStatus(matchResult.getMatchStatus());
item.setMatchMessage(matchResult.getMatchMessage()); item.setMatchMessage(matchResult.getMatchMessage());
item.setMatched(matchResult.isMatched()); item.setMatched(isUsableIndexedMatch(matchResult));
if (matchResult.isMatched()) { if (item.isMatched()) {
item.setShopId(matchResult.getShopId()); item.setShopId(matchResult.getShopId());
item.setCompanyName(matchResult.getCompanyName()); item.setCompanyName(matchResult.getCompanyName());
item.setPlatform(matchResult.getPlatform()); item.setPlatform(matchResult.getPlatform());
@@ -236,35 +250,79 @@ public class DeleteBrandRunService {
} }
public DeleteBrandHistoryVo listHistory(Long userId) { public DeleteBrandHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
DeleteBrandHistoryVo vo = new DeleteBrandHistoryVo(); DeleteBrandHistoryVo vo = new DeleteBrandHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() List<FileResultEntity> createdRows = fileResultMapper.selectList(historyResultQuery()
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId) .eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt) .orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100")); .last("limit 500"));
List<FileTaskEntity> recentTouchedTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getStatus,
FileTaskEntity::getUpdatedAt,
FileTaskEntity::getFinishedAt)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getUserId, userId)
.orderByDesc(FileTaskEntity::getUpdatedAt)
.last("limit 200"));
Map<Long, FileResultEntity> entityById = new LinkedHashMap<>();
for (FileResultEntity entity : createdRows) {
if (entity != null && entity.getId() != null) {
entityById.put(entity.getId(), entity);
}
}
for (FileResultEntity entity : selectHistoryResultRowsByTaskIdsInBatches(userId, recentTouchedTasks.stream()
.map(FileTaskEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList())) {
if (entity != null && entity.getId() != null) {
entityById.putIfAbsent(entity.getId(), entity);
}
}
List<FileResultEntity> entities = new ArrayList<>(entityById.values());
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) {
vo.setItems(List.of());
log.info("[delete-brand] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
// 补充信息 // 补充信息
Map<Long, List<DeleteBrandResultItemVo>> itemsByTaskId = new LinkedHashMap<>(); Map<Long, FileTaskEntity> taskById = new LinkedHashMap<>();
Map<Long, String> statusByTaskId = new LinkedHashMap<>(); Map<Long, String> statusByTaskId = new LinkedHashMap<>();
List<Long> taskIds = entities.stream() List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId) .map(FileResultEntity::getTaskId)
.filter(id -> id != null && id > 0) .filter(id -> id != null && id > 0)
.distinct() .distinct()
.toList(); .toList();
if (!taskIds.isEmpty()) { for (FileTaskEntity task : recentTouchedTasks) {
List<FileTaskEntity> tasks = selectTasksByIdsInBatches(taskIds); if (task != null) {
for (FileTaskEntity task : tasks) { taskById.put(task.getId(), task);
try {
statusByTaskId.put(task.getId(), task.getStatus()); statusByTaskId.put(task.getId(), task.getStatus());
if (task.getResultJson() != null && !task.getResultJson().isBlank()) {
List<DeleteBrandResultItemVo> taskItems = objectMapper.readValue(task.getResultJson(), new TypeReference<List<DeleteBrandResultItemVo>>() {
});
itemsByTaskId.put(task.getId(), taskItems);
} }
} catch (Exception ignored) { }
if (!taskIds.isEmpty()) {
List<FileTaskEntity> tasks = selectTaskHistoryFieldsByIdsInBatches(taskIds);
for (FileTaskEntity task : tasks) {
if (task != null) {
taskById.put(task.getId(), task);
statusByTaskId.put(task.getId(), task.getStatus());
} }
} }
} }
long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
long jobsLoadedAt = System.nanoTime();
entities = sortDeleteBrandHistoryEntities(entities, taskById, jobMap).stream()
.limit(100)
.toList();
vo.setItems(entities.stream().map(entity -> { vo.setItems(entities.stream().map(entity -> {
DeleteBrandResultItemVo item = new DeleteBrandResultItemVo(); DeleteBrandResultItemVo item = new DeleteBrandResultItemVo();
@@ -275,7 +333,7 @@ public class DeleteBrandRunService {
item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename())); item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename()));
item.setOutputFilename(entity.getResultFilename()); item.setOutputFilename(entity.getResultFilename());
item.setDownloadUrl(null); item.setDownloadUrl(null);
attachFileJobState(item, entity); attachFileJobState(item, entity, jobMap.get(entity.getId()));
item.setTotalRows(entity.getRowCount()); item.setTotalRows(entity.getRowCount());
item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
item.setError(entity.getErrorMessage()); item.setError(entity.getErrorMessage());
@@ -284,7 +342,7 @@ public class DeleteBrandRunService {
// 真实补充 // 真实补充
if (entity.getTaskId() != null) { if (entity.getTaskId() != null) {
item.setTaskStatus(statusByTaskId.get(entity.getTaskId())); item.setTaskStatus(statusByTaskId.get(entity.getTaskId()));
List<DeleteBrandResultItemVo> taskItems = itemsByTaskId.get(entity.getTaskId()); List<DeleteBrandResultItemVo> taskItems = null;
if (taskItems != null && entity.getSourceFilename() != null && !entity.getSourceFilename().isBlank()) { if (taskItems != null && entity.getSourceFilename() != null && !entity.getSourceFilename().isBlank()) {
for (DeleteBrandResultItemVo candidate : taskItems) { for (DeleteBrandResultItemVo candidate : taskItems) {
if (candidate == null) { if (candidate == null) {
@@ -320,12 +378,42 @@ public class DeleteBrandRunService {
} }
return item; return item;
}).toList()); }).toList());
long finishedAt = System.nanoTime();
log.info("[delete-brand] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
statusByTaskId.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo; return vo;
} }
private LambdaQueryWrapper<FileResultEntity> historyResultQuery() {
return new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getRowCount,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt);
}
private void attachFileJobState(DeleteBrandResultItemVo item, FileResultEntity entity) { private void attachFileJobState(DeleteBrandResultItemVo item, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); attachFileJobState(item, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()));
}
private void attachFileJobState(DeleteBrandResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) {
item.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); item.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) { if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null); item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
@@ -336,6 +424,51 @@ public class DeleteBrandRunService {
item.setFileError(job.getErrorMessage()); item.setFileError(job.getErrorMessage());
} }
private List<FileResultEntity> sortDeleteBrandHistoryEntities(
List<FileResultEntity> entities,
Map<Long, FileTaskEntity> taskById,
Map<Long, TaskFileJobEntity> jobByResultId
) {
if (entities == null || entities.isEmpty()) {
return List.of();
}
List<FileResultEntity> sorted = new ArrayList<>(entities);
sorted.sort(Comparator
.comparing((FileResultEntity entity) -> resolveHistorySortTime(entity, taskById, jobByResultId),
Comparator.nullsLast(Comparator.reverseOrder()))
.thenComparing(FileResultEntity::getId, Comparator.nullsLast(Comparator.reverseOrder())));
return sorted;
}
private LocalDateTime resolveHistorySortTime(
FileResultEntity entity,
Map<Long, FileTaskEntity> taskById,
Map<Long, TaskFileJobEntity> jobByResultId
) {
if (entity == null) {
return null;
}
TaskFileJobEntity job = jobByResultId == null ? null : jobByResultId.get(entity.getId());
if (job != null) {
if (job.getFinishedAt() != null) {
return job.getFinishedAt();
}
if (job.getUpdatedAt() != null) {
return job.getUpdatedAt();
}
}
FileTaskEntity task = taskById == null ? null : taskById.get(entity.getTaskId());
if (task != null) {
if (task.getFinishedAt() != null) {
return task.getFinishedAt();
}
if (task.getUpdatedAt() != null) {
return task.getUpdatedAt();
}
}
return entity.getCreatedAt();
}
public DeleteBrandTaskDeletionStatusVo getTaskDeletionStatus(Long taskId, Long userId) { public DeleteBrandTaskDeletionStatusVo getTaskDeletionStatus(Long taskId, Long userId) {
FileTaskEntity task = loadTaskForExecution(taskId); FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
@@ -617,6 +750,13 @@ public class DeleteBrandRunService {
} }
java.util.Map<Long, java.util.Map<Object, Object>> progressByTaskId = deleteBrandTaskCacheService.getProgressBatch(normalizedTaskIds); java.util.Map<Long, java.util.Map<Object, Object>> progressByTaskId = deleteBrandTaskCacheService.getProgressBatch(normalizedTaskIds);
Map<Long, List<FileResultEntity>> resultRowsByTaskId = findProgressResultRowsByTaskIds(normalizedTaskIds);
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, resultRowsByTaskId.values().stream()
.flatMap(List::stream)
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
for (Long taskId : normalizedTaskIds) { for (Long taskId : normalizedTaskIds) {
FileTaskEntity task = taskById.get(taskId); FileTaskEntity task = taskById.get(taskId);
@@ -624,7 +764,8 @@ public class DeleteBrandRunService {
vo.getMissingTaskIds().add(taskId); vo.getMissingTaskIds().add(taskId);
continue; continue;
} }
vo.getItems().add(buildTaskProgressDetail(task, progressByTaskId.get(taskId))); vo.getItems().add(buildTaskProgressDetail(task, progressByTaskId.get(taskId),
resultRowsByTaskId.getOrDefault(taskId, List.of()), jobMap));
} }
return vo; return vo;
} }
@@ -634,6 +775,13 @@ public class DeleteBrandRunService {
} }
private DeleteBrandTaskDetailVo buildTaskProgressDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) { private DeleteBrandTaskDetailVo buildTaskProgressDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) {
return buildTaskProgressDetail(task, progress, List.of(), Map.of());
}
private DeleteBrandTaskDetailVo buildTaskProgressDetail(FileTaskEntity task,
java.util.Map<Object, Object> progress,
List<FileResultEntity> resultRows,
Map<Long, TaskFileJobEntity> jobMap) {
task = refreshIndexedMatchesIfNeeded(task); task = refreshIndexedMatchesIfNeeded(task);
DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo(); DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo();
taskVo.setId(task.getId()); taskVo.setId(task.getId());
@@ -657,10 +805,39 @@ public class DeleteBrandRunService {
DeleteBrandTaskDetailVo detail = new DeleteBrandTaskDetailVo(); DeleteBrandTaskDetailVo detail = new DeleteBrandTaskDetailVo();
detail.setTask(taskVo); detail.setTask(taskVo);
detail.setLine_progress(buildLineProgress(task.getId(), progress)); detail.setLine_progress(buildLineProgress(task.getId(), progress));
detail.setFileProgress(java.util.List.of()); List<DeleteBrandResultItemVo> taskItems = buildProgressResultItems(task, resultRows, jobMap);
detail.setItems(taskItems);
detail.setFileProgress(buildFileProgress(task, taskItems, detail.getLine_progress()));
return detail; return detail;
} }
private List<DeleteBrandResultItemVo> buildProgressResultItems(FileTaskEntity task,
List<FileResultEntity> resultRows,
Map<Long, TaskFileJobEntity> jobMap) {
if (task == null || resultRows == null || resultRows.isEmpty()) {
return List.of();
}
return resultRows.stream()
.filter(entity -> entity != null && entity.getId() != null)
.map(entity -> {
DeleteBrandResultItemVo item = new DeleteBrandResultItemVo();
item.setResultId(entity.getId());
item.setTaskId(entity.getTaskId());
item.setFileKey(entity.getSourceFileUrl());
item.setSourceFilename(entity.getSourceFilename());
item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename()));
item.setOutputFilename(entity.getResultFilename());
item.setDownloadUrl(null);
item.setTotalRows(entity.getRowCount());
item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
item.setError(entity.getErrorMessage());
item.setTaskStatus(task.getStatus());
attachFileJobState(item, entity, jobMap == null ? null : jobMap.get(entity.getId()));
return item;
})
.toList();
}
private DeleteBrandTaskDetailVo buildTaskDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) { private DeleteBrandTaskDetailVo buildTaskDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) {
task = refreshIndexedMatchesIfNeeded(task); task = refreshIndexedMatchesIfNeeded(task);
DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo(); DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo();
@@ -837,7 +1014,7 @@ public class DeleteBrandRunService {
String oldOpenStoreUrl = item.getOpenStoreUrl(); String oldOpenStoreUrl = item.getOpenStoreUrl();
String oldMessage = item.getMatchMessage(); String oldMessage = item.getMatchMessage();
item.setMatched(refreshed.isMatched()); item.setMatched(isUsableIndexedMatch(refreshed));
item.setMatchStatus(refreshed.getMatchStatus()); item.setMatchStatus(refreshed.getMatchStatus());
item.setMatchMessage(refreshed.getMatchMessage()); item.setMatchMessage(refreshed.getMatchMessage());
item.setShopId(refreshed.getShopId()); item.setShopId(refreshed.getShopId());
@@ -1011,6 +1188,65 @@ public class DeleteBrandRunService {
return tasks; return tasks;
} }
private List<FileTaskEntity> selectTaskHistoryFieldsByIdsInBatches(List<Long> taskIds) {
List<FileTaskEntity> tasks = new ArrayList<>();
if (taskIds == null || taskIds.isEmpty()) {
return tasks;
}
int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize());
for (int start = 0; start < taskIds.size(); start += batchSize) {
int end = Math.min(start + batchSize, taskIds.size());
tasks.addAll(fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getStatus,
FileTaskEntity::getUpdatedAt,
FileTaskEntity::getFinishedAt)
.in(FileTaskEntity::getId, taskIds.subList(start, end))));
}
return tasks;
}
private List<FileResultEntity> selectHistoryResultRowsByTaskIdsInBatches(Long userId, List<Long> taskIds) {
List<FileResultEntity> rows = new ArrayList<>();
if (userId == null || taskIds == null || taskIds.isEmpty()) {
return rows;
}
int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize());
for (int start = 0; start < taskIds.size(); start += batchSize) {
int end = Math.min(start + batchSize, taskIds.size());
rows.addAll(fileResultMapper.selectList(historyResultQuery()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId)
.in(FileResultEntity::getTaskId, taskIds.subList(start, end))));
}
return rows;
}
private Map<Long, List<FileResultEntity>> findProgressResultRowsByTaskIds(List<Long> taskIds) {
if (taskIds == null || taskIds.isEmpty()) {
return Map.of();
}
Map<Long, List<FileResultEntity>> rowsByTaskId = new LinkedHashMap<>();
int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize());
for (int start = 0; start < taskIds.size(); start += batchSize) {
int end = Math.min(start + batchSize, taskIds.size());
List<FileResultEntity> rows = fileResultMapper.selectList(historyResultQuery()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.in(FileResultEntity::getTaskId, taskIds.subList(start, end))
.orderByAsc(FileResultEntity::getId));
for (FileResultEntity row : rows) {
if (row != null && row.getTaskId() != null) {
rowsByTaskId.computeIfAbsent(row.getTaskId(), ignored -> new ArrayList<>()).add(row);
}
}
}
return rowsByTaskId;
}
private static long elapsedMs(long startInclusive, long endExclusive) {
return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L);
}
@Transactional @Transactional
public void submitResult(Long taskId, DeleteBrandSubmitResultRequest request) { public void submitResult(Long taskId, DeleteBrandSubmitResultRequest request) {
log.info("[DeleteBrand] Received submitResult for taskId: {}, files count: {}", taskId, request == null || request.getFiles() == null ? 0 : request.getFiles().size()); log.info("[DeleteBrand] Received submitResult for taskId: {}, files count: {}", taskId, request == null || request.getFiles() == null ? 0 : request.getFiles().size());
@@ -1027,6 +1263,16 @@ public class DeleteBrandRunService {
throw new BusinessException("任务不存在"); throw new BusinessException("任务不存在");
} }
if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) { if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) {
log.warn("[DeleteBrand] submitResult rejected because task is terminal -> taskId: {}, status: {}, createdAt: {}, updatedAt: {}, finishedAt: {}, error: {}, sourceFiles: {}, successFiles: {}, failedFiles: {}",
task.getId(),
task.getStatus(),
task.getCreatedAt(),
task.getUpdatedAt(),
task.getFinishedAt(),
task.getErrorMessage(),
task.getSourceFileCount(),
task.getSuccessFileCount(),
task.getFailedFileCount());
throw new BusinessException("任务已结束，拒绝继续提交结果"); throw new BusinessException("任务已结束，拒绝继续提交结果");
} }
@@ -1086,6 +1332,7 @@ public class DeleteBrandRunService {
// 如果该文件已完成，立即更新成品计数的 Redis 提示，让后续 tryFinalizeTask 更准确 // 如果该文件已完成，立即更新成品计数的 Redis 提示，让后续 tryFinalizeTask 更准确
if (isFileCompleted(fileDto)) { if (isFileCompleted(fileDto)) {
log.info("[DeleteBrand] File chunk transmission fully reached via chunkTotal for taskId: {}, file: {}", taskId, fileDto.getSourceFilename()); log.info("[DeleteBrand] File chunk transmission fully reached via chunkTotal for taskId: {}, file: {}", taskId, fileDto.getSourceFilename());
enqueueCompletedFileAssembleJob(task, fileIdentity, parsedFile);
shouldTryFinalize = true; shouldTryFinalize = true;
// 这里暂不直接 increment，而是标记需要在 tryFinalizeTask 中重新计算 // 这里暂不直接 increment，而是标记需要在 tryFinalizeTask 中重新计算
// 为了保险，直接在 tryFinalizeTask 里重新遍历分片状态 // 为了保险，直接在 tryFinalizeTask 里重新遍历分片状态
@@ -1098,6 +1345,55 @@ public class DeleteBrandRunService {
} }
} }
private void enqueueCompletedFileAssembleJob(FileTaskEntity task,
String fileIdentity,
DeleteBrandParsedFileCacheDto parsedFile) {
if (task == null || task.getId() == null || parsedFile == null) {
return;
}
FileResultEntity resultEntity = findResultEntity(task.getId(), fileIdentity, parsedFile);
if (resultEntity == null) {
log.warn("[DeleteBrand] completed file has no result row, skip early assemble -> taskId: {}, file: {}",
task.getId(), parsedFile.getSourceFilename());
return;
}
if (taskFileJobService.hasSuccessfulAssembleJob(task.getId(), MODULE_TYPE, resultEntity.getId())) {
return;
}
String outputFilename = buildResultFilename(parsedFile);
resultEntity.setResultFilename(outputFilename);
resultEntity.setResultContentType(CONTENT_TYPE_XLSX);
resultEntity.setRowCount(parsedFile.getTotalRows());
resultEntity.setSuccess(1);
resultEntity.setErrorMessage(null);
if (resultEntity.getResultFileUrl() == null || resultEntity.getResultFileUrl().isBlank()) {
resultEntity.setResultFileSize(0L);
}
fileResultMapper.updateById(resultEntity);
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity);
log.info("[DeleteBrand] completed file early assemble job enqueued -> taskId: {}, resultId: {}, file: {}",
task.getId(), resultEntity.getId(), parsedFile.getSourceFilename());
}
private FileResultEntity findResultEntity(Long taskId,
String fileIdentity,
DeleteBrandParsedFileCacheDto parsedFile) {
if (taskId == null || taskId <= 0) {
return null;
}
LambdaQueryWrapper<FileResultEntity> wrapper = new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getTaskId, taskId);
if (fileIdentity != null && !fileIdentity.isBlank()) {
wrapper.eq(FileResultEntity::getSourceFileUrl, fileIdentity);
} else if (parsedFile != null && parsedFile.getSourceFilename() != null && !parsedFile.getSourceFilename().isBlank()) {
wrapper.eq(FileResultEntity::getSourceFilename, parsedFile.getSourceFilename());
} else {
return null;
}
return fileResultMapper.selectOne(wrapper.last("limit 1"));
}
public void tryFinalizeTask(Long taskId, boolean fromCompensation) { public void tryFinalizeTask(Long taskId, boolean fromCompensation) {
if (taskId == null || taskId <= 0) { if (taskId == null || taskId <= 0) {
return; return;
@@ -1291,15 +1587,19 @@ public class DeleteBrandRunService {
throw new BusinessException("结果记录不存在: " + parsedFile.getSourceFilename()); throw new BusinessException("结果记录不存在: " + parsedFile.getSourceFilename());
} }
String outputFilename = buildResultFilename(parsedFile); String outputFilename = buildResultFilename(parsedFile);
boolean alreadyAssembled = taskFileJobService.hasSuccessfulAssembleJob(task.getId(), MODULE_TYPE, resultEntity.getId());
resultEntity.setResultFilename(outputFilename); resultEntity.setResultFilename(outputFilename);
resultEntity.setResultFileUrl(null); if (!alreadyAssembled && (resultEntity.getResultFileUrl() == null || resultEntity.getResultFileUrl().isBlank())) {
resultEntity.setResultFileSize(0L); resultEntity.setResultFileSize(0L);
}
resultEntity.setResultContentType(CONTENT_TYPE_XLSX); resultEntity.setResultContentType(CONTENT_TYPE_XLSX);
resultEntity.setRowCount(parsedFile.getTotalRows()); resultEntity.setRowCount(parsedFile.getTotalRows());
resultEntity.setSuccess(1); resultEntity.setSuccess(1);
resultEntity.setErrorMessage(null); resultEntity.setErrorMessage(null);
fileResultMapper.updateById(resultEntity); fileResultMapper.updateById(resultEntity);
if (!alreadyAssembled) {
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity); taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity);
}
DeleteBrandResultItemVo item = new DeleteBrandResultItemVo(); DeleteBrandResultItemVo item = new DeleteBrandResultItemVo();
item.setResultId(resultEntity.getId()); item.setResultId(resultEntity.getId());
@@ -1343,6 +1643,7 @@ public class DeleteBrandRunService {
"finished_files", String.valueOf(successCount), "finished_files", String.valueOf(successCount),
"updated_at", String.valueOf(System.currentTimeMillis()) "updated_at", String.valueOf(System.currentTimeMillis())
)); ));
cleanupCompletedTaskDataIfNoPendingFileJobs(task.getId());
} catch (Exception ex) { } catch (Exception ex) {
task.setStatus("FAILED"); task.setStatus("FAILED");
task.setErrorMessage(ex.getMessage()); task.setErrorMessage(ex.getMessage());
@@ -1419,8 +1720,19 @@ public class DeleteBrandRunService {
if (job == null || job.getTaskId() == null) { if (job == null || job.getTaskId() == null) {
return; return;
} }
if (taskFileJobService.countUnfinishedAssembleJobs(job.getTaskId(), MODULE_TYPE) == 0L) { cleanupCompletedTaskDataIfNoPendingFileJobs(job.getTaskId());
deleteBrandTaskStorageService.deleteTaskData(job.getTaskId()); }
private void cleanupCompletedTaskDataIfNoPendingFileJobs(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || (!"SUCCESS".equals(task.getStatus()) && !"FAILED".equals(task.getStatus()))) {
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) == 0L) {
deleteBrandTaskStorageService.deleteTaskData(taskId);
} }
} }
@@ -1611,6 +1923,32 @@ public class DeleteBrandRunService {
return entity == null ? null : entity.getResultFilename(); return entity == null ? null : entity.getResultFilename();
} }
public String resolveResultDownloadUrl(Long resultId, Long userId) {
FileResultEntity entity = findDownloadableResult(resultId, userId);
return entity == null || entity.getResultFileUrl() == null ? null : ossStorageService.getPublicUrl(entity.getResultFileUrl());
}
public String resolveResultDownloadFilename(Long resultId, Long userId) {
FileResultEntity entity = findDownloadableResult(resultId, userId);
return entity == null ? null : entity.getResultFilename();
}
private FileResultEntity findDownloadableResult(Long resultId, Long userId) {
if (resultId == null || resultId <= 0 || userId == null || userId <= 0) {
return null;
}
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null
|| !MODULE_TYPE.equals(entity.getModuleType())
|| entity.getUserId() == null
|| !entity.getUserId().equals(userId)
|| entity.getResultFileUrl() == null
|| entity.getResultFileUrl().isBlank()) {
return null;
}
return entity;
}
private FileResultEntity findLatestSuccessfulResult(Long taskId) { private FileResultEntity findLatestSuccessfulResult(Long taskId) {
return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)

45
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java
View File

@@ -134,15 +134,17 @@ public class DeleteBrandStaleTaskService {
} }
} }
boolean hasStartedProgress = lastHeartbeatAt > 0L; boolean hasStartedProgress = lastHeartbeatAt > 0L;
int completedScopeCount = 0;
if (!hasStartedProgress) { if (!hasStartedProgress) {
hasStartedProgress = deleteBrandTaskStorageService.countCompletedScopes(task.getId()) > 0 // Parsed payload is created by Java during the parse step. Only result chunks mean Python has started uploading.
|| deleteBrandTaskStorageService.countParsedPayloadScopes(task.getId()) > 0; completedScopeCount = deleteBrandTaskStorageService.countCompletedScopes(task.getId());
hasStartedProgress = completedScopeCount > 0;
} }
if (lastHeartbeatAt > 0L && nowMillis - lastHeartbeatAt < staleTimeoutMillis) { if (lastHeartbeatAt > 0L && nowMillis - lastHeartbeatAt < staleTimeoutMillis) {
continue; continue;
} }
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
continue; continue;
} }
@@ -151,12 +153,24 @@ public class DeleteBrandStaleTaskService {
FileTaskEntity refreshed = fileTaskMapper.selectById(task.getId()); FileTaskEntity refreshed = fileTaskMapper.selectById(task.getId());
if (refreshed != null && !"RUNNING".equals(refreshed.getStatus())) { if (refreshed != null && !"RUNNING".equals(refreshed.getStatus())) {
deleteBrandTaskCacheService.saveTaskCache(refreshed); deleteBrandTaskCacheService.saveTaskCache(refreshed);
log.info("[stale-check] delete-brand finalized before timeout-fail taskId={} status={} updatedAt={} finishedAt={}",
refreshed.getId(), refreshed.getStatus(), refreshed.getUpdatedAt(), refreshed.getFinishedAt());
continue; continue;
} }
} catch (Exception ex) { } catch (Exception ex) {
log.warn("[stale-check] delete-brand finalize threw taskId={} msg={}", task.getId(), ex.getMessage()); log.warn("[stale-check] delete-brand finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
} }
log.warn("[stale-check] delete-brand failing stale task -> taskId={} updatedAt={} createdAt={} lastHeartbeatAt={} timeoutMinutes={} initialTimeoutMinutes={} completedScopes={} hasStartedProgress={} initialThreshold={}",
task.getId(),
task.getUpdatedAt(),
task.getCreatedAt(),
lastHeartbeatAt,
minutes,
initialMinutes,
completedScopeCount,
hasStartedProgress,
initialThreshold);
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>() int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId()) .eq(FileTaskEntity::getId, task.getId())
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND) .eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
@@ -168,6 +182,7 @@ public class DeleteBrandStaleTaskService {
if (updated > 0) { if (updated > 0) {
deleteBrandTaskCacheService.delete(task.getId()); deleteBrandTaskCacheService.delete(task.getId());
log.warn("[stale-check] delete-brand failed taskId={} reason=timeout", task.getId());
} }
} }
} }
@@ -204,11 +219,11 @@ public class DeleteBrandStaleTaskService {
task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt()); task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt());
continue; continue;
} }
boolean hasStartedProgress = lastPayloadHeartbeatMillis > 0L || hasFinishedRows; boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) { if (!hasStartedProgress) {
hasStartedProgress = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId()); hasStartedProgress = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId());
} }
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
stats.skippedTaskCount++; stats.skippedTaskCount++;
log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}", log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}",
task.getId(), task.getCreatedAt(), initialThreshold); task.getId(), task.getCreatedAt(), initialThreshold);
@@ -271,11 +286,11 @@ public class DeleteBrandStaleTaskService {
stats.skippedTaskCount++; stats.skippedTaskCount++;
continue; continue;
} }
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows; boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) { if (!hasStartedProgress) {
hasStartedProgress = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId()); hasStartedProgress = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId());
} }
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
stats.skippedTaskCount++; stats.skippedTaskCount++;
continue; continue;
} }
@@ -339,11 +354,11 @@ public class DeleteBrandStaleTaskService {
task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt()); task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt());
continue; continue;
} }
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows; boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) { if (!hasStartedProgress) {
hasStartedProgress = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId()); hasStartedProgress = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
} }
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
stats.skippedTaskCount++; stats.skippedTaskCount++;
log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}", log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}",
task.getId(), task.getCreatedAt(), initialThreshold); task.getId(), task.getCreatedAt(), initialThreshold);
@@ -405,11 +420,11 @@ public class DeleteBrandStaleTaskService {
stats.skippedTaskCount++; stats.skippedTaskCount++;
continue; continue;
} }
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows; boolean hasStartedProgress = hasFinishedRows;
if (!hasStartedProgress) { if (!hasStartedProgress) {
hasStartedProgress = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId()); hasStartedProgress = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId());
} }
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
stats.skippedTaskCount++; stats.skippedTaskCount++;
continue; continue;
} }
@@ -440,6 +455,14 @@ public class DeleteBrandStaleTaskService {
return stats; return stats;
} }
private boolean isWithinInitialGrace(FileTaskEntity task, LocalDateTime initialThreshold) {
if (task == null || initialThreshold == null) {
return false;
}
LocalDateTime baseline = task.getUpdatedAt() != null ? task.getUpdatedAt() : task.getCreatedAt();
return baseline != null && baseline.isAfter(initialThreshold);
}
@Scheduled(cron = "${aiimage.delete-brand-progress.finalize-check-cron:30 */2 * * * *}") @Scheduled(cron = "${aiimage.delete-brand-progress.finalize-check-cron:30 */2 * * * *}")
public void finalizeCompletedRunningTasks() { public void finalizeCompletedRunningTasks() {
DistributedJobLockService.LockHandle lockHandle = DistributedJobLockService.LockHandle lockHandle =

20
backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java
View File

@@ -19,6 +19,7 @@ import java.util.regex.Pattern;
public class LocalTempCleanupService { public class LocalTempCleanupService {
private static final Pattern ROOT_TEMP_FILE_PATTERN = Pattern.compile("^[a-fA-F0-9]{32}(\\.[^.]+)?$"); private static final Pattern ROOT_TEMP_FILE_PATTERN = Pattern.compile("^[a-fA-F0-9]{32}(\\.[^.]+)?$");
private static final String TRANSIENT_PAYLOAD_DIR_NAME = "transient-payload";
private static final List<String> RESULT_DIR_NAMES = List.of( private static final List<String> RESULT_DIR_NAMES = List.of(
"dedupe-result", "dedupe-result",
"convert-result", "convert-result",
@@ -39,11 +40,14 @@ public class LocalTempCleanupService {
return; return;
} }
Instant sourceExpireBefore = Instant.now().minus(storageProperties.getSourceRetentionHours(), ChronoUnit.HOURS); Instant now = Instant.now();
Instant resultExpireBefore = Instant.now().minus(storageProperties.getResultRetentionHours(), ChronoUnit.HOURS); Instant sourceExpireBefore = now.minus(storageProperties.getSourceRetentionHours(), ChronoUnit.HOURS);
Instant resultExpireBefore = now.minus(storageProperties.getResultRetentionHours(), ChronoUnit.HOURS);
Instant transientPayloadExpireBefore = now.minus(storageProperties.getTransientPayloadRetentionHours(), ChronoUnit.HOURS);
int deletedSourceCount = 0; int deletedSourceCount = 0;
int deletedResultCount = 0; int deletedResultCount = 0;
int deletedTransientPayloadCount = 0;
File[] children = tempDir.listFiles(); File[] children = tempDir.listFiles();
if (children == null || children.length == 0) { if (children == null || children.length == 0) {
return; return;
@@ -60,14 +64,20 @@ public class LocalTempCleanupService {
if (child.isDirectory() && RESULT_DIR_NAMES.contains(child.getName())) { if (child.isDirectory() && RESULT_DIR_NAMES.contains(child.getName())) {
deletedResultCount += deleteExpiredChildrenRecursively(child, resultExpireBefore); deletedResultCount += deleteExpiredChildrenRecursively(child, resultExpireBefore);
deleteEmptyDirectories(child, tempDir); deleteEmptyDirectories(child, tempDir);
continue;
}
if (child.isDirectory() && TRANSIENT_PAYLOAD_DIR_NAME.equals(child.getName())) {
deletedTransientPayloadCount += deleteExpiredChildrenRecursively(child, transientPayloadExpireBefore);
deleteEmptyDirectories(child, tempDir);
} }
} catch (Exception ex) { } catch (Exception ex) {
log.warn("清理本地临时文件失败: path={}", child.getAbsolutePath(), ex); log.warn("local temp cleanup failed: path={}", child.getAbsolutePath(), ex);
} }
} }
if (deletedSourceCount > 0 || deletedResultCount > 0) { if (deletedSourceCount > 0 || deletedResultCount > 0 || deletedTransientPayloadCount > 0) {
log.info("本地临时目录清理完成: sourceDeleted={}, resultDeleted={}, tempDir={}", deletedSourceCount, deletedResultCount, tempDir.getAbsolutePath()); log.info("local temp cleanup completed: sourceDeleted={}, resultDeleted={}, transientPayloadDeleted={}, tempDir={}",
deletedSourceCount, deletedResultCount, deletedTransientPayloadCount, tempDir.getAbsolutePath());
} }
} }

6
backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsObjectStorageService.java
View File

@@ -32,6 +32,10 @@ public class RustfsObjectStorageService {
} }
public String uploadText(String objectKey, String content) { public String uploadText(String objectKey, String content) {
return uploadText(objectKey, content, true);
}
public String uploadText(String objectKey, String content, boolean verifyAfterUpload) {
if (!isConfigured()) { if (!isConfigured()) {
throw new IllegalStateException("transient storage is not configured"); throw new IllegalStateException("transient storage is not configured");
} }
@@ -46,7 +50,9 @@ public class RustfsObjectStorageService {
.stream(stream, bytes.length, -1) .stream(stream, bytes.length, -1)
.contentType("application/json") .contentType("application/json")
.build()); .build());
if (verifyAfterUpload) {
verifyObjectVisible(objectKey); verifyObjectVisible(objectKey);
}
return objectKey; return objectKey;
} catch (Exception ex) { } catch (Exception ex) {
last = ex; last = ex;

7
backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/controller/PatrolDeleteController.java
View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.patroldelete.controller; package com.nanri.aiimage.modules.patroldelete.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteConditionAddRequest; import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteConditionAddRequest;
import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteCreateTaskRequest; import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteCreateTaskRequest;
import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteSubmitResultRequest; import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteSubmitResultRequest;
@@ -25,7 +26,6 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -39,7 +39,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
@@ -358,10 +357,8 @@ public class PatrolDeleteController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;

6
backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskCacheService.java
View File

@@ -127,6 +127,11 @@ public class PatrolDeleteTaskCacheService {
} }
public void deleteTaskCache(Long taskId) { public void deleteTaskCache(Long taskId) {
evictTaskCacheOnly(taskId);
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
public void evictTaskCacheOnly(Long taskId) {
if (taskId == null || taskId <= 0) { if (taskId == null || taskId <= 0) {
return; return;
} }
@@ -137,7 +142,6 @@ public class PatrolDeleteTaskCacheService {
} catch (Exception ex) { } catch (Exception ex) {
log.warn("[patrol-delete-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage()); log.warn("[patrol-delete-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
} }
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
} }
public void saveTaskCache(FileTaskEntity task) { public void saveTaskCache(FileTaskEntity task) {

107
backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java
View File

@@ -28,6 +28,7 @@ import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -66,6 +67,7 @@ public class PatrolDeleteTaskService {
private final TaskFileJobService taskFileJobService; private final TaskFileJobService taskFileJobService;
private final TaskResultItemService taskResultItemService; private final TaskResultItemService taskResultItemService;
private final TaskProgressSnapshotService taskProgressSnapshotService; private final TaskProgressSnapshotService taskProgressSnapshotService;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private FileTaskEntity loadTaskForExecution(Long taskId) { private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId)); Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -135,32 +137,58 @@ public class PatrolDeleteTaskService {
} }
public PatrolDeleteHistoryVo listHistory(Long userId) { public PatrolDeleteHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
validateUserId(userId); validateUserId(userId);
PatrolDeleteHistoryVo vo = new PatrolDeleteHistoryVo(); PatrolDeleteHistoryVo vo = new PatrolDeleteHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId) .eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt) .orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100")); .last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) { if (entities.isEmpty()) {
vo.setItems(List.of()); vo.setItems(List.of());
log.info("[patrol-delete] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo; return vo;
} }
Map<Long, FileTaskEntity> taskMap = loadTaskMap(entities); Map<Long, FileTaskEntity> taskMap = loadHistoryTaskMap(entities);
Map<Long, Map<Long, PatrolDeleteResultItemVo>> snapshotMap = buildSnapshotMap(taskMap); long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream() Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId) .map(FileResultEntity::getId)
.filter(id -> id != null && id > 0) .filter(id -> id != null && id > 0)
.distinct() .distinct()
.toList()); .toList());
long jobsLoadedAt = System.nanoTime();
List<PatrolDeleteResultItemVo> items = new ArrayList<>(); List<PatrolDeleteResultItemVo> items = new ArrayList<>();
for (FileResultEntity entity : entities) { for (FileResultEntity entity : entities) {
FileTaskEntity task = taskMap.get(entity.getTaskId()); FileTaskEntity task = taskMap.get(entity.getTaskId());
PatrolDeleteResultItemVo snapshot = snapshotMap.getOrDefault(entity.getTaskId(), Map.of()).get(entity.getId()); items.add(toHistoryItem(entity, task, null, jobMap.get(entity.getId())));
items.add(toHistoryItem(entity, task, snapshot, jobMap.get(entity.getId())));
} }
vo.setItems(items); vo.setItems(items);
long finishedAt = System.nanoTime();
log.info("[patrol-delete] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
taskMap.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo; return vo;
} }
@@ -417,17 +445,32 @@ public class PatrolDeleteTaskService {
@Transactional @Transactional
public void deleteTask(Long taskId, Long userId) { public void deleteTask(Long taskId, Long userId) {
long startedAt = System.nanoTime();
validateUserId(userId); validateUserId(userId);
FileTaskEntity task = loadTaskForExecution(taskId); FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在"); throw new BusinessException("任务不存在");
} }
long taskLoadedAt = System.nanoTime();
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>() fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId) .eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)); .eq(FileResultEntity::getModuleType, MODULE_TYPE));
cleanupTaskAuxiliaryData(taskId); long resultsDeletedAt = System.nanoTime();
cleanupTaskAuxiliaryDataFast(taskId);
long auxiliaryDeletedAt = System.nanoTime();
fileTaskMapper.deleteById(taskId); fileTaskMapper.deleteById(taskId);
taskCacheService.deleteTaskCache(taskId); long taskDeletedAt = System.nanoTime();
taskCacheService.evictTaskCacheOnly(taskId);
long finishedAt = System.nanoTime();
log.info("[patrol-delete] delete task timing taskId={} userId={} totalMs={} loadMs={} resultDeleteMs={} auxiliaryDeleteMs={} taskDeleteMs={} cacheMs={}",
taskId,
userId,
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, taskLoadedAt),
elapsedMs(taskLoadedAt, resultsDeletedAt),
elapsedMs(resultsDeletedAt, auxiliaryDeletedAt),
elapsedMs(auxiliaryDeletedAt, taskDeletedAt),
elapsedMs(taskDeletedAt, finishedAt));
} }
@Transactional @Transactional
@@ -439,7 +482,7 @@ public class PatrolDeleteTaskService {
} }
Long taskId = entity.getTaskId(); Long taskId = entity.getTaskId();
fileResultMapper.deleteById(resultId); fileResultMapper.deleteById(resultId);
cleanupResultAuxiliaryData(taskId, resultId); cleanupResultAuxiliaryDataFast(taskId, resultId);
reconcileTaskAfterResultRemoval(taskId); reconcileTaskAfterResultRemoval(taskId);
} }
@@ -453,13 +496,12 @@ public class PatrolDeleteTaskService {
} }
List<FileResultEntity> rows = listTaskRows(taskId); List<FileResultEntity> rows = listTaskRows(taskId);
if (rows.isEmpty()) { if (rows.isEmpty()) {
cleanupTaskAuxiliaryData(taskId); cleanupTaskAuxiliaryDataFast(taskId);
fileTaskMapper.deleteById(taskId); fileTaskMapper.deleteById(taskId);
taskCacheService.deleteTaskCache(taskId); taskCacheService.evictTaskCacheOnly(taskId);
return; return;
} }
updateTaskStatusFromRows(task, rows); updateTaskStatusFromRows(task, rows);
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
fileTaskMapper.updateById(task); fileTaskMapper.updateById(task);
taskCacheService.saveTaskCache(task); taskCacheService.saveTaskCache(task);
} }
@@ -470,11 +512,23 @@ public class PatrolDeleteTaskService {
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE); taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
} }
private void cleanupTaskAuxiliaryDataFast(Long taskId) {
taskResultItemService.deleteTaskItemsRowsOnly(taskId, MODULE_TYPE);
taskScopePayloadStorageService.deleteTaskScopePayloadRowsOnly(taskId, MODULE_TYPE);
taskProgressSnapshotService.delete(taskId, MODULE_TYPE);
taskFileJobService.deleteTaskJobs(taskId, MODULE_TYPE);
}
private void cleanupResultAuxiliaryData(Long taskId, Long resultId) { private void cleanupResultAuxiliaryData(Long taskId, Long resultId) {
taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId); taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId);
taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId); taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId);
} }
private void cleanupResultAuxiliaryDataFast(Long taskId, Long resultId) {
taskResultItemService.deleteResultItemRowsOnly(taskId, MODULE_TYPE, resultId);
taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId);
}
private long countTasks(Long userId, List<String> statuses) { private long countTasks(Long userId, List<String> statuses) {
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>() Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE) .eq(FileTaskEntity::getModuleType, MODULE_TYPE)
@@ -524,6 +578,35 @@ public class PatrolDeleteTaskService {
return taskMap; return taskMap;
} }
private Map<Long, FileTaskEntity> loadHistoryTaskMap(List<FileResultEntity> entities) {
List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId)
.filter(id -> id != null && id > 0)
.distinct()
.toList();
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
if (taskIds.isEmpty()) {
return taskMap;
}
int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize());
for (int start = 0; start < taskIds.size(); start += batchSize) {
int end = Math.min(start + batchSize, taskIds.size());
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getModuleType,
FileTaskEntity::getStatus,
FileTaskEntity::getFinishedAt)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.in(FileTaskEntity::getId, taskIds.subList(start, end)));
for (FileTaskEntity task : tasks) {
if (task != null && task.getId() != null) {
taskMap.put(task.getId(), task);
}
}
}
return taskMap;
}
private Map<Long, Map<Long, PatrolDeleteResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) { private Map<Long, Map<Long, PatrolDeleteResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
Map<Long, Map<Long, PatrolDeleteResultItemVo>> out = new LinkedHashMap<>(); Map<Long, Map<Long, PatrolDeleteResultItemVo>> out = new LinkedHashMap<>();
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) { for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
@@ -925,6 +1008,10 @@ public class PatrolDeleteTaskService {
return Integer.valueOf(RESULT_SUCCESS).equals(dbValue); return Integer.valueOf(RESULT_SUCCESS).equals(dbValue);
} }
private long elapsedMs(long start, long end) {
return (end - start) / 1_000_000L;
}
private List<PatrolDeleteResultItemVo> parseTaskSnapshots(String json) { private List<PatrolDeleteResultItemVo> parseTaskSnapshots(String json) {
if (blank(json)) { if (blank(json)) {
return new ArrayList<>(); return new ArrayList<>();

7
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java
View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.pricetrack.controller; package com.nanri.aiimage.modules.pricetrack.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCandidateAddRequest; import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCandidateAddRequest;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCountryPreferenceSaveRequest; import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCreateTaskRequest; import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCreateTaskRequest;
@@ -28,7 +29,6 @@ import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -43,7 +43,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
@@ -244,10 +243,8 @@ public class PriceTrackController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;

56
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java
View File

@@ -166,14 +166,33 @@ public class PriceTrackTaskService {
} }
public PriceTrackHistoryVo listHistory(Long userId) { public PriceTrackHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
PriceTrackHistoryVo vo = new PriceTrackHistoryVo(); PriceTrackHistoryVo vo = new PriceTrackHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList( List<FileResultEntity> entities = fileResultMapper.selectList(
new LambdaQueryWrapper<FileResultEntity>() new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId) .eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt) .orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100")); .last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) {
vo.setItems(List.of());
log.info("[price-track] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
Map<Long, String> statusByTaskId = new LinkedHashMap<>(); Map<Long, String> statusByTaskId = new LinkedHashMap<>();
List<Long> taskIds = entities.stream() List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId) .map(FileResultEntity::getTaskId)
@@ -186,11 +205,29 @@ public class PriceTrackTaskService {
if (t != null) statusByTaskId.put(t.getId(), t.getStatus()); if (t != null) statusByTaskId.put(t.getId(), t.getStatus());
} }
} }
long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
long jobsLoadedAt = System.nanoTime();
List<PriceTrackResultItemVo> items = new ArrayList<>(); List<PriceTrackResultItemVo> items = new ArrayList<>();
for (FileResultEntity entity : entities) { for (FileResultEntity entity : entities) {
items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()))); items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()), jobMap.get(entity.getId()), false));
} }
vo.setItems(items); vo.setItems(items);
long finishedAt = System.nanoTime();
log.info("[price-track] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
statusByTaskId.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo; return vo;
} }
@@ -952,6 +989,10 @@ public class PriceTrackTaskService {
} }
private PriceTrackResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) { private PriceTrackResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) {
return toHistoryItemVo(entity, taskStatus, null, true);
}
private PriceTrackResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus, TaskFileJobEntity job, boolean mergeRequestFields) {
PriceTrackResultItemVo vo = new PriceTrackResultItemVo(); PriceTrackResultItemVo vo = new PriceTrackResultItemVo();
vo.setResultId(entity.getId()); vo.setResultId(entity.getId());
vo.setTaskId(entity.getTaskId()); vo.setTaskId(entity.getTaskId());
@@ -963,16 +1004,19 @@ public class PriceTrackTaskService {
vo.setError(entity.getErrorMessage()); vo.setError(entity.getErrorMessage());
vo.setOutputFilename(entity.getResultFilename()); vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(null); vo.setDownloadUrl(null);
attachFileJobState(vo, entity); attachFileJobState(vo, entity, job);
vo.setMatched(true); vo.setMatched(true);
if (entity.getTaskId() != null) { if (mergeRequestFields && entity.getTaskId() != null) {
mergeQueueFieldsFromRequest(vo, entity.getTaskId()); mergeQueueFieldsFromRequest(vo, entity.getTaskId());
} }
return vo; return vo;
} }
private void attachFileJobState(PriceTrackResultItemVo vo, FileResultEntity entity) { private void attachFileJobState(PriceTrackResultItemVo vo, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); attachFileJobState(vo, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()));
}
private void attachFileJobState(PriceTrackResultItemVo vo, FileResultEntity entity, TaskFileJobEntity job) {
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) { if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
@@ -1226,6 +1270,10 @@ public class PriceTrackTaskService {
return t == null ? null : t.toString(); return t == null ? null : t.toString();
} }
private static long elapsedMs(long startInclusive, long endExclusive) {
return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L);
}
private void updateTaskStatusFromLatestRows(FileTaskEntity task, List<FileResultEntity> latest) { private void updateTaskStatusFromLatestRows(FileTaskEntity task, List<FileResultEntity> latest) {
String oldStatus = task.getStatus(); String oldStatus = task.getStatus();
int ok = 0; int ok = 0;

7
backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/controller/ProductRiskResolveController.java
View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.productrisk.controller; package com.nanri.aiimage.modules.productrisk.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCreateTaskRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCreateTaskRequest;
@@ -26,7 +27,6 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -41,7 +41,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
@@ -250,10 +249,8 @@ public class ProductRiskResolveController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;

56
backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java
View File

@@ -165,15 +165,34 @@ public class ProductRiskTaskService {
} }
public ProductRiskHistoryVo listHistory(Long userId) { public ProductRiskHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
if (userId == null || userId <= 0) { if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
} }
ProductRiskHistoryVo vo = new ProductRiskHistoryVo(); ProductRiskHistoryVo vo = new ProductRiskHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId) .eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt) .orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100")); .last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) {
vo.setItems(List.of());
log.info("[product-risk] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
Map<Long, String> statusByTaskId = new LinkedHashMap<>(); Map<Long, String> statusByTaskId = new LinkedHashMap<>();
List<Long> taskIds = entities.stream() List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId) .map(FileResultEntity::getTaskId)
@@ -188,11 +207,29 @@ public class ProductRiskTaskService {
} }
} }
} }
long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
long jobsLoadedAt = System.nanoTime();
List<ProductRiskResultItemVo> items = new ArrayList<>(); List<ProductRiskResultItemVo> items = new ArrayList<>();
for (FileResultEntity entity : entities) { for (FileResultEntity entity : entities) {
items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()))); items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()), jobMap.get(entity.getId()), false));
} }
vo.setItems(items); vo.setItems(items);
long finishedAt = System.nanoTime();
log.info("[product-risk] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
statusByTaskId.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo; return vo;
} }
@@ -967,6 +1004,10 @@ public class ProductRiskTaskService {
} }
private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) { private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) {
return toHistoryItemVo(entity, taskStatus, null, true);
}
private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus, TaskFileJobEntity job, boolean mergeRequestFields) {
ProductRiskResultItemVo vo = new ProductRiskResultItemVo(); ProductRiskResultItemVo vo = new ProductRiskResultItemVo();
vo.setResultId(entity.getId()); vo.setResultId(entity.getId());
vo.setTaskId(entity.getTaskId()); vo.setTaskId(entity.getTaskId());
@@ -978,16 +1019,19 @@ public class ProductRiskTaskService {
vo.setError(entity.getErrorMessage()); vo.setError(entity.getErrorMessage());
vo.setOutputFilename(entity.getResultFilename()); vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(null); vo.setDownloadUrl(null);
attachFileJobState(vo, entity); attachFileJobState(vo, entity, job);
vo.setMatched(true); vo.setMatched(true);
if (entity.getTaskId() != null) { if (mergeRequestFields && entity.getTaskId() != null) {
mergeQueueFieldsFromRequest(vo, entity.getTaskId()); mergeQueueFieldsFromRequest(vo, entity.getTaskId());
} }
return vo; return vo;
} }
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) { private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); attachFileJobState(vo, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()));
}
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity, TaskFileJobEntity job) {
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) { if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
@@ -1031,6 +1075,10 @@ public class ProductRiskTaskService {
return t == null ? null : t.toString(); return t == null ? null : t.toString();
} }
private static long elapsedMs(long startInclusive, long endExclusive) {
return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L);
}
private List<ProductRiskShopQueueItemVo> dedupeQueueItems(List<ProductRiskShopQueueItemVo> raw) { private List<ProductRiskShopQueueItemVo> dedupeQueueItems(List<ProductRiskShopQueueItemVo> raw) {
Map<String, ProductRiskShopQueueItemVo> byNorm = new LinkedHashMap<>(); Map<String, ProductRiskShopQueueItemVo> byNorm = new LinkedHashMap<>();
for (ProductRiskShopQueueItemVo item : raw) { for (ProductRiskShopQueueItemVo item : raw) {

7
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java
View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.queryasin.controller; package com.nanri.aiimage.modules.queryasin.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
@@ -23,7 +24,6 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -37,7 +37,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
@@ -269,10 +268,8 @@ public class QueryAsinTaskController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;

79
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java
View File

@@ -135,27 +135,58 @@ public class QueryAsinTaskService {
} }
public QueryAsinHistoryVo listHistory(Long userId) { public QueryAsinHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
validateUserId(userId); validateUserId(userId);
QueryAsinHistoryVo vo = new QueryAsinHistoryVo(); QueryAsinHistoryVo vo = new QueryAsinHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId) .eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt) .orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100")); .last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) { if (entities.isEmpty()) {
vo.setItems(List.of()); vo.setItems(List.of());
log.info("[query-asin] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo; return vo;
} }
Map<Long, FileTaskEntity> taskMap = loadTaskMap(entities); Map<Long, FileTaskEntity> taskMap = loadHistoryTaskMap(entities);
Map<Long, Map<Long, QueryAsinResultItemVo>> snapshotMap = buildSnapshotMap(taskMap); long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
long jobsLoadedAt = System.nanoTime();
List<QueryAsinResultItemVo> items = new ArrayList<>(); List<QueryAsinResultItemVo> items = new ArrayList<>();
for (FileResultEntity entity : entities) { for (FileResultEntity entity : entities) {
FileTaskEntity task = taskMap.get(entity.getTaskId()); FileTaskEntity task = taskMap.get(entity.getTaskId());
QueryAsinResultItemVo snapshot = snapshotMap.getOrDefault(entity.getTaskId(), Map.of()).get(entity.getId()); items.add(toHistoryItem(entity, task, null, jobMap.get(entity.getId())));
items.add(toHistoryItem(entity, task, snapshot));
} }
vo.setItems(items); vo.setItems(items);
long finishedAt = System.nanoTime();
log.info("[query-asin] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
taskMap.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo; return vo;
} }
@@ -510,6 +541,31 @@ public class QueryAsinTaskService {
return taskMap; return taskMap;
} }
private Map<Long, FileTaskEntity> loadHistoryTaskMap(List<FileResultEntity> entities) {
List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId)
.filter(id -> id != null && id > 0)
.distinct()
.toList();
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
if (taskIds.isEmpty()) {
return taskMap;
}
int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize());
for (int start = 0; start < taskIds.size(); start += batchSize) {
int end = Math.min(start + batchSize, taskIds.size());
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId, FileTaskEntity::getStatus, FileTaskEntity::getFinishedAt)
.in(FileTaskEntity::getId, taskIds.subList(start, end)));
for (FileTaskEntity task : tasks) {
if (task != null) {
taskMap.put(task.getId(), task);
}
}
}
return taskMap;
}
private Map<Long, Map<Long, QueryAsinResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) { private Map<Long, Map<Long, QueryAsinResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
Map<Long, Map<Long, QueryAsinResultItemVo>> out = new LinkedHashMap<>(); Map<Long, Map<Long, QueryAsinResultItemVo>> out = new LinkedHashMap<>();
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) { for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
@@ -523,6 +579,10 @@ public class QueryAsinTaskService {
} }
private QueryAsinResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, QueryAsinResultItemVo snapshot) { private QueryAsinResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, QueryAsinResultItemVo snapshot) {
return toHistoryItem(entity, task, snapshot, null);
}
private QueryAsinResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, QueryAsinResultItemVo snapshot, TaskFileJobEntity job) {
QueryAsinResultItemVo item = snapshot != null ? snapshot : new QueryAsinResultItemVo(); QueryAsinResultItemVo item = snapshot != null ? snapshot : new QueryAsinResultItemVo();
item.setResultId(entity.getId()); item.setResultId(entity.getId());
item.setTaskId(entity.getTaskId()); item.setTaskId(entity.getTaskId());
@@ -535,7 +595,7 @@ public class QueryAsinTaskService {
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt()); item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename())); item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
item.setDownloadUrl(null); item.setDownloadUrl(null);
attachFileJobState(item, entity); attachFileJobState(item, entity, job);
if (item.getCountryResults() == null) { if (item.getCountryResults() == null) {
item.setCountryResults(new ArrayList<>()); item.setCountryResults(new ArrayList<>());
} }
@@ -546,7 +606,10 @@ public class QueryAsinTaskService {
} }
private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity) { private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); attachFileJobState(item, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()));
}
private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) {
item.setFileReady(!blank(entity.getResultFileUrl())); item.setFileReady(!blank(entity.getResultFileUrl()));
if (job == null) { if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null); item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
@@ -1061,6 +1124,10 @@ public class QueryAsinTaskService {
return value == null || value.isBlank(); return value == null || value.isBlank();
} }
private static long elapsedMs(long startInclusive, long endExclusive) {
return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L);
}
private String blankToNull(String value) { private String blankToNull(String value) {
return blank(value) ? null : value.trim(); return blank(value) ? null : value.trim();
} }

7
backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/controller/ShopMatchController.java
View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.shopmatch.controller; package com.nanri.aiimage.modules.shopmatch.controller;
import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
@@ -23,7 +24,6 @@ import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -38,7 +38,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
@@ -193,10 +192,8 @@ public class ShopMatchController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No downloadable result"); throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No downloadable result");
} }
try { try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream"); response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.setAttachment(response, filename);
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) { try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536]; byte[] buffer = new byte[65536];
int read; int read;

56
backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java
View File

@@ -173,15 +173,34 @@ public class ShopMatchTaskService {
} }
public ProductRiskHistoryVo listHistory(Long userId) { public ProductRiskHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
if (userId == null || userId <= 0) { if (userId == null || userId <= 0) {
throw new BusinessException("user_id 不合法"); throw new BusinessException("user_id 不合法");
} }
ProductRiskHistoryVo vo = new ProductRiskHistoryVo(); ProductRiskHistoryVo vo = new ProductRiskHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>() List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId) .eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt) .orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100")); .last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) {
vo.setItems(List.of());
log.info("[shop-match] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
Map<Long, String> statusByTaskId = new LinkedHashMap<>(); Map<Long, String> statusByTaskId = new LinkedHashMap<>();
List<Long> taskIds = entities.stream() List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId) .map(FileResultEntity::getTaskId)
@@ -195,11 +214,29 @@ public class ShopMatchTaskService {
} }
} }
} }
long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
long jobsLoadedAt = System.nanoTime();
List<ProductRiskResultItemVo> items = new ArrayList<>(); List<ProductRiskResultItemVo> items = new ArrayList<>();
for (FileResultEntity entity : entities) { for (FileResultEntity entity : entities) {
items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()))); items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()), jobMap.get(entity.getId()), false));
} }
vo.setItems(items); vo.setItems(items);
long finishedAt = System.nanoTime();
log.info("[shop-match] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
statusByTaskId.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo; return vo;
} }
@@ -867,6 +904,10 @@ public class ShopMatchTaskService {
} }
private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) { private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) {
return toHistoryItemVo(entity, taskStatus, null, true);
}
private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus, TaskFileJobEntity job, boolean mergeRequestFields) {
ProductRiskResultItemVo vo = new ProductRiskResultItemVo(); ProductRiskResultItemVo vo = new ProductRiskResultItemVo();
vo.setResultId(entity.getId()); vo.setResultId(entity.getId());
vo.setTaskId(entity.getTaskId()); vo.setTaskId(entity.getTaskId());
@@ -878,16 +919,19 @@ public class ShopMatchTaskService {
vo.setError(entity.getErrorMessage()); vo.setError(entity.getErrorMessage());
vo.setOutputFilename(entity.getResultFilename()); vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(null); vo.setDownloadUrl(null);
attachFileJobState(vo, entity); attachFileJobState(vo, entity, job);
vo.setMatched(true); vo.setMatched(true);
if (entity.getTaskId() != null) { if (mergeRequestFields && entity.getTaskId() != null) {
mergeQueueFieldsFromRequest(vo, entity.getTaskId()); mergeQueueFieldsFromRequest(vo, entity.getTaskId());
} }
return vo; return vo;
} }
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) { private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); attachFileJobState(vo, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()));
}
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity, TaskFileJobEntity job) {
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) { if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
@@ -1031,6 +1075,10 @@ public class ShopMatchTaskService {
return time == null ? null : time.toString(); return time == null ? null : time.toString();
} }
private static long elapsedMs(long startInclusive, long endExclusive) {
return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L);
}
private LocalDateTime now() { private LocalDateTime now() {
return LocalDateTime.now(BUSINESS_ZONE); return LocalDateTime.now(BUSINESS_ZONE);
} }

990
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinCozeClient.java Normal file
View File

@@ -0,0 +1,990 @@
package com.nanri.aiimage.modules.similarasin.client;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.MediaType;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.stereotype.Component;
import org.springframework.util.StreamUtils;
import org.springframework.web.client.RestClient;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@Component
@RequiredArgsConstructor
@Slf4j
public class SimilarAsinCozeClient {
private final SimilarAsinProperties properties;
private final ObjectMapper objectMapper;
public List<SimilarAsinResultRowDto> inspect(List<SimilarAsinResultRowDto> rows, String prompt) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
if (properties.getCozeToken() == null || properties.getCozeToken().isBlank()) {
log.warn("[similar-asin] coze token not configured, keep raw rows size={}", rows.size());
return rows.stream().map(this::copy).toList();
}
try {
return inspectWithFallback(rows, prompt);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[similar-asin] coze batch failed size={} err={}", rows.size(), failureMessage);
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
}
public CozeSubmitResponse submitWorkflow(List<SimilarAsinResultRowDto> rows, String prompt) throws Exception {
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt));
ensureSuccess(submitRoot);
return new CozeSubmitResponse(
extractExecuteId(submitRoot),
extractResultDataText(submitRoot),
writeJson(submitRoot)
);
}
public CozePollResponse pollWorkflow(String executeId) throws Exception {
JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId));
ensureSuccess(pollRoot);
String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT);
String dataText = extractResultDataText(pollRoot);
String outputText = dataText.isBlank() ? extractWorkflowOutputText(pollRoot) : "";
String failureMessage = isFailedWorkflowStatus(status)
? firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed")
: "";
return new CozePollResponse(executeId, status, dataText, outputText, failureMessage, writeJson(pollRoot));
}
public List<SimilarAsinResultRowDto> mergeRowsFromDataText(List<SimilarAsinResultRowDto> rows, String dataText) throws Exception {
if (dataText == null || dataText.isBlank()) {
return rows == null ? List.of() : rows.stream().map(this::copy).toList();
}
return mergeRows(rows, parseResults(wrapDataPayload(dataText)));
}
public List<SimilarAsinResultRowDto> markRowsFailed(List<SimilarAsinResultRowDto> rows, String failureMessage) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
private List<SimilarAsinResultRowDto> inspectWithFallback(List<SimilarAsinResultRowDto> rows, String prompt) {
try {
if (rows.size() == 1) {
return inspectSingleRowWithRetry(rows, prompt);
}
InspectAttempt attempt = inspectOnce(rows, prompt);
if (attempt.resolvedCount() < rows.size()) {
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
}
return attempt.mergedRows();
} catch (Exception ex) {
if (shouldSplitBatch(rows, ex)) {
int middle = rows.size() / 2;
log.warn("[similar-asin] coze batch fallback split size={} left={} right={} err={}",
rows.size(), middle, rows.size() - middle, failureMessage(ex));
List<SimilarAsinResultRowDto> merged = new ArrayList<>(rows.size());
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt));
return merged;
}
throw propagate(ex);
}
}
private List<SimilarAsinResultRowDto> inspectPartitionWithFailureFallback(List<SimilarAsinResultRowDto> rows, String prompt) {
try {
return inspectWithFallback(rows, prompt);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[similar-asin] coze partition failed size={} err={}", rows.size(), failureMessage);
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
}
private List<SimilarAsinResultRowDto> inspectSingleRowWithRetry(List<SimilarAsinResultRowDto> rows, String prompt) throws Exception {
SimilarAsinResultRowDto row = rows.getFirst();
PartialCozeResultException lastFailure = null;
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
try {
InspectAttempt attempt = inspectOnce(rows, prompt);
if (attempt.resolvedCount() == rows.size()) {
return attempt.mergedRows();
}
log.warn("[similar-asin] coze single unresolved attempt={} rowId={} asin={} country={} title={} url={} raw={}",
attemptIndex,
row.getId(),
row.getAsin(),
row.getCountry(),
abbreviate(row.getTitle(), 120),
abbreviate(row.getUrl(), 120),
abbreviate(attempt.raw(), 500));
lastFailure = new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
} catch (Exception ex) {
if (attemptIndex >= 3 || !isRetryableBatchFailure(ex)) {
throw ex;
}
log.warn("[similar-asin] coze single retryable failure attempt={} rowId={} asin={} country={} err={}",
attemptIndex,
row.getId(),
row.getAsin(),
row.getCountry(),
failureMessage(ex));
}
if (attemptIndex < 3) {
sleepBeforeRetry(attemptIndex);
}
}
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
}
private InspectAttempt inspectOnce(List<SimilarAsinResultRowDto> rows, String prompt) throws Exception {
String raw = runWorkflowAsyncAndWait(rows, prompt);
List<CozeResult> results = parseResults(raw);
if (rows.size() > 1 && !results.isEmpty() && results.stream().noneMatch(this::hasIdentity)) {
throw new PartialCozeResultException(0, rows.size(), results.size());
}
List<SimilarAsinResultRowDto> merged = mergeRows(rows, results);
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
}
private String runWorkflowAsyncAndWait(List<SimilarAsinResultRowDto> rows, String prompt) throws Exception {
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt));
ensureSuccess(submitRoot);
String immediateData = extractResultDataText(submitRoot);
if (!immediateData.isBlank()) {
return wrapDataPayload(immediateData);
}
String executeId = extractExecuteId(submitRoot);
if (executeId == null || executeId.isBlank()) {
throw new IllegalStateException("Coze async execute_id missing");
}
long deadline = System.currentTimeMillis() + Math.max(10000, properties.getCozePollTimeoutMillis());
while (System.currentTimeMillis() < deadline) {
ensureNotInterrupted();
JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId));
ensureSuccess(pollRoot);
String dataText = extractResultDataText(pollRoot);
if (!dataText.isBlank()) {
return wrapDataPayload(dataText);
}
String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT);
if (isFailedWorkflowStatus(status)) {
throw new IllegalStateException(firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed"));
}
if (isSuccessfulWorkflowStatus(status)) {
String outputText = extractWorkflowOutputText(pollRoot);
if (!outputText.isBlank()) {
return wrapDataPayload(outputText);
}
throw new IllegalStateException("Coze async workflow completed without output");
}
sleepQuietly(Math.max(200, properties.getCozePollIntervalMillis()));
}
throw new IllegalStateException("Coze async workflow poll timeout");
}
private String postWorkflow(List<SimilarAsinResultRowDto> rows, String prompt) {
Map<String, Object> parameters = buildParameters(rows, prompt);
Map<String, Object> body = new LinkedHashMap<>();
body.put("workflow_id", properties.getCozeWorkflowId());
body.put("parameters", parameters);
body.put("is_async", Boolean.TRUE);
log.info("[similar-asin] coze request url={} body={}",
joinUrl(properties.getCozeBaseUrl(), properties.getCozeWorkflowPath()),
writeJson(body));
RestClient.RequestBodySpec request = restClient().post()
.uri(joinUrl(properties.getCozeBaseUrl(), properties.getCozeWorkflowPath()))
.headers(headers -> {
headers.setBearerAuth(stripBearer(properties.getCozeToken()));
headers.setContentType(MediaType.APPLICATION_JSON);
});
request.body(body);
return request.exchange((clientRequest, clientResponse) -> {
byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody());
String responseText = responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8);
log.info("[similar-asin] coze submit response status={} body={}",
clientResponse.getStatusCode(),
responseText);
return responseText;
});
}
private String getWorkflowHistory(String executeId) {
String path = properties.getCozeWorkflowHistoryPath()
.replace("{workflow_id}", properties.getCozeWorkflowId())
.replace("{execute_id}", executeId);
return restClient().get()
.uri(joinUrl(properties.getCozeBaseUrl(), path))
.headers(headers -> {
headers.setBearerAuth(stripBearer(properties.getCozeToken()));
headers.setContentType(MediaType.APPLICATION_JSON);
})
.exchange((clientRequest, clientResponse) -> {
byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody());
String responseText = responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8);
log.info("[similar-asin] coze history response executeId={} status={} body={}",
executeId,
clientResponse.getStatusCode(),
responseText);
return responseText;
});
}
private Map<String, Object> buildParameters(List<SimilarAsinResultRowDto> rows, String prompt) {
List<String> groupKeys = rows.stream().map(row -> nonBlank(row.getGroupKey(), rowKey(row))).toList();
List<String> rowIds = rows.stream().map(row -> nonBlank(row.getId(), "")).toList();
List<String> asins = rows.stream().map(row -> nonBlank(row.getAsin(), "")).toList();
List<String> countries = rows.stream().map(row -> nonBlank(row.getCountry(), "")).toList();
List<String> titles = rows.stream().map(row -> nonBlank(row.getTitle(), row.getAsin())).toList();
List<String> urls = rows.stream().map(row -> nonBlank(row.getUrl(), "")).toList();
List<List<String>> urlLists = rows.stream().map(SimilarAsinResultRowDto::getUrls).toList();
Map<String, Object> parameters = new LinkedHashMap<>();
parameters.put("title_list", titles);
parameters.put("url_list", urls);
parameters.put("url_lists", urlLists);
parameters.put("items", buildItemObjects(rows, groupKeys, rowIds, asins, countries, titles, urlLists));
parameters.put("prompt", prompt == null ? "" : prompt);
return parameters;
}
private List<Map<String, Object>> buildItemObjects(List<SimilarAsinResultRowDto> rows,
List<String> groupKeys,
List<String> rowIds,
List<String> asins,
List<String> countries,
List<String> titles,
List<List<String>> urlLists) {
List<Map<String, Object>> items = new ArrayList<>(rows.size());
for (int i = 0; i < rows.size(); i++) {
Map<String, Object> item = new LinkedHashMap<>();
item.put("group_key", groupKeys.get(i));
item.put("row_id", rowIds.get(i));
item.put("asin", asins.get(i));
item.put("country", countries.get(i));
item.put("title", titles.get(i));
item.put("url", urlLists.get(i));
item.put("urls", urlLists.get(i));
items.add(item);
}
return items;
}
private List<CozeResult> parseResults(String raw) throws Exception {
JsonNode root = objectMapper.readTree(raw);
ensureSuccess(root);
String dataText = extractResultDataText(root);
if (dataText.isBlank()) {
return List.of();
}
JsonNode dataRoot = objectMapper.readTree(dataText);
JsonNode array = dataRoot.isArray() ? dataRoot : dataRoot.path("data");
List<CozeResult> results = new ArrayList<>();
if (array.isArray()) {
for (JsonNode node : array) {
JsonNode itemNode = resultItemNode(node);
results.add(new CozeResult(
text(firstNonNull(
firstNonNull(node.get("group_key"), node.get("groupKey")),
firstNonNull(itemNode.get("group_key"), itemNode.get("groupKey")))),
text(firstNonNull(
firstNonNull(node.get("row_id"), firstNonNull(node.get("rowId"), node.get("id"))),
firstNonNull(itemNode.get("row_id"), firstNonNull(itemNode.get("rowId"), itemNode.get("id"))))),
text(firstNonNull(node.get("asin"), itemNode.get("asin"))),
text(firstNonNull(
firstNonNull(node.get("country"), node.get("site")),
firstNonNull(itemNode.get("country"), itemNode.get("site")))),
text(firstNonNull(node.get("title"), firstNonNull(itemNode.get("title"), itemNode.get("title_risk")))),
text(firstNonNull(node.get("appearance"),
firstNonNull(node.get("appearance_risk"),
firstNonNull(itemNode.get("appearance"), itemNode.get("appearance_risk"))))),
text(firstNonNull(firstNonNull(node.get("patent"), node.get("patent ")),
firstNonNull(node.get("patent_risk"),
firstNonNull(firstNonNull(itemNode.get("patent"), itemNode.get("patent ")), itemNode.get("patent_risk"))))),
text(firstNonNull(node.get("result"),
firstNonNull(node.get("conclusion"),
firstNonNull(itemNode.get("result"), itemNode.get("conclusion"))))),
text(firstNonNull(node.get("title_reason"),
firstNonNull(node.get("titleReason"),
firstNonNull(itemNode.get("title_reason"), itemNode.get("titleReason"))))),
text(firstNonNull(node.get("appearance_reason"),
firstNonNull(node.get("appearanceReason"),
firstNonNull(itemNode.get("appearance_reason"), itemNode.get("appearanceReason"))))),
text(firstNonNull(node.get("patent_reason"),
firstNonNull(firstNonNull(node.get("patentReason"), node.get("patent reason")),
firstNonNull(itemNode.get("patent_reason"),
firstNonNull(itemNode.get("patentReason"), itemNode.get("patent reason"))))))
));
}
}
return results;
}
private JsonNode resultItemNode(JsonNode node) {
if (node == null || node.isMissingNode() || node.isNull()) {
return objectMapper.missingNode();
}
JsonNode item = firstNonNull(node.get("item"), node.get("items"));
if (item == null || item.isMissingNode() || item.isNull()) {
return objectMapper.missingNode();
}
if (item.isArray()) {
return item.isEmpty() ? objectMapper.missingNode() : item.get(0);
}
return item;
}
private List<SimilarAsinResultRowDto> mergeRows(List<SimilarAsinResultRowDto> rows, List<CozeResult> results) {
Map<String, CozeResult> resultByGroupKey = new LinkedHashMap<>();
Map<String, CozeResult> resultByCompositeKey = new LinkedHashMap<>();
Map<String, CozeResult> resultByAsinCountry = new LinkedHashMap<>();
Map<String, CozeResult> resultByAsin = new LinkedHashMap<>();
Map<String, CozeResult> resultByRowId = new LinkedHashMap<>();
for (CozeResult result : results) {
String groupKey = normalize(result.groupKey());
if (!groupKey.isBlank()) {
resultByGroupKey.putIfAbsent(groupKey, result);
}
String compositeKey = rowKey(result.rowId(), result.asin(), result.country());
if (!compositeKey.isBlank()) {
resultByCompositeKey.putIfAbsent(compositeKey, result);
}
String asinCountryKey = asinCountryKey(result.asin(), result.country());
if (!asinCountryKey.isBlank()) {
resultByAsinCountry.putIfAbsent(asinCountryKey, result);
}
String asinKey = normalize(result.asin()).toUpperCase(Locale.ROOT);
if (!asinKey.isBlank()) {
resultByAsin.putIfAbsent(asinKey, result);
}
String rowIdKey = normalize(result.rowId());
if (!rowIdKey.isBlank()) {
resultByRowId.putIfAbsent(rowIdKey, result);
}
}
List<SimilarAsinResultRowDto> merged = new ArrayList<>(rows.size());
boolean allowIndexFallback = results.size() == rows.size() && results.stream().noneMatch(this::hasIdentity);
for (int i = 0; i < rows.size(); i++) {
SimilarAsinResultRowDto row = copy(rows.get(i));
CozeResult result = resultByGroupKey.get(normalize(row.getGroupKey()));
if (result == null) {
result = resultByCompositeKey.get(rowKey(row.getId(), row.getAsin(), row.getCountry()));
}
if (result == null) {
result = resultByAsinCountry.get(asinCountryKey(row.getAsin(), row.getCountry()));
}
if (result == null) {
result = resultByAsin.get(normalize(row.getAsin()).toUpperCase(Locale.ROOT));
}
if (result == null) {
result = resultByRowId.get(normalize(row.getId()));
}
if (result == null && allowIndexFallback && i < results.size()) {
result = results.get(i);
}
applyResult(row, result);
merged.add(row);
}
return merged;
}
private String asinCountryKey(String asin, String country) {
String normalizedAsin = normalize(asin).toUpperCase(Locale.ROOT);
if (normalizedAsin.isBlank()) {
return "";
}
return normalizedAsin + "::" + normalize(country);
}
private boolean hasIdentity(CozeResult result) {
if (result == null) {
return false;
}
return !normalize(result.groupKey()).isBlank()
|| !normalize(result.rowId()).isBlank()
|| !normalize(result.asin()).isBlank();
}
private void applyResult(SimilarAsinResultRowDto row, CozeResult result) {
if (row == null || result == null) {
return;
}
row.setTitleRisk(result.title());
row.setAppearanceRisk(result.appearance());
row.setPatentRisk(result.patent());
row.setConclusion(result.result());
row.setTitleReason(result.titleReason());
row.setAppearanceReason(result.appearanceReason());
row.setPatentReason(result.patentReason());
}
private RestClient restClient() {
SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
requestFactory.setConnectTimeout(properties.getCozeConnectTimeoutMillis());
requestFactory.setReadTimeout(properties.getCozeReadTimeoutMillis());
return RestClient.builder().requestFactory(requestFactory).build();
}
private SimilarAsinResultRowDto copy(SimilarAsinResultRowDto source) {
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setSourceFileKey(source.getSourceFileKey());
row.setSourceFilename(source.getSourceFilename());
row.setRowToken(source.getRowToken());
row.setGroupKey(source.getGroupKey());
row.setId(source.getId());
row.setAsin(source.getAsin());
row.setCountry(source.getCountry());
row.setUrls(source.getUrls());
row.setTitle(source.getTitle());
row.setError(source.getError());
row.setDone(source.getDone());
row.setTitleRisk(source.getTitleRisk());
row.setAppearanceRisk(source.getAppearanceRisk());
row.setPatentRisk(source.getPatentRisk());
row.setConclusion(source.getConclusion());
row.setTitleReason(source.getTitleReason());
row.setAppearanceReason(source.getAppearanceReason());
row.setPatentReason(source.getPatentReason());
return row;
}
private SimilarAsinResultRowDto markFailed(SimilarAsinResultRowDto row, String failureMessage) {
String reviewMessage = failureMessage == null || failureMessage.isBlank()
? "Coze 检测失败，待人工复核"
: "Coze 检测失败，待人工复核：" + failureMessage;
if (row.getError() == null || row.getError().isBlank()) {
row.setError(failureMessage);
}
if (row.getTitleRisk() == null || row.getTitleRisk().isBlank()) {
row.setTitleRisk(reviewMessage);
}
if (row.getAppearanceRisk() == null || row.getAppearanceRisk().isBlank()) {
row.setAppearanceRisk(reviewMessage);
}
if (row.getPatentRisk() == null || row.getPatentRisk().isBlank()) {
row.setPatentRisk(reviewMessage);
}
if (row.getConclusion() == null || row.getConclusion().isBlank()) {
row.setConclusion(failureMessage);
}
return row;
}
private boolean shouldSplitBatch(List<SimilarAsinResultRowDto> rows, Exception ex) {
return rows != null && rows.size() > 1 && isRetryableBatchFailure(ex);
}
private boolean isRetryableBatchFailure(Exception ex) {
if (ex instanceof PartialCozeResultException) {
return true;
}
String message = ex == null ? "" : nonBlank(ex.getMessage(), "");
return message.contains("Workflow node execution limit exceeded")
|| message.contains("Read timed out")
|| message.contains("Connection reset")
|| message.contains("I/O error on POST request")
|| message.toLowerCase(Locale.ROOT).contains("timeout");
}
private int resolvedCount(List<SimilarAsinResultRowDto> rows) {
int resolved = 0;
for (SimilarAsinResultRowDto row : rows) {
if (hasResolvedCozeFields(row)) {
resolved++;
}
}
return resolved;
}
private boolean hasResolvedCozeFields(SimilarAsinResultRowDto row) {
if (row == null) {
return false;
}
return !normalize(row.getTitleRisk()).isBlank()
|| !normalize(row.getAppearanceRisk()).isBlank()
|| !normalize(row.getPatentRisk()).isBlank()
|| !normalize(row.getConclusion()).isBlank();
}
private void ensureSuccess(JsonNode root) {
if (root.path("code").asInt(-1) != 0) {
throw new IllegalStateException(root.path("msg").asText("Coze response code is not 0"));
}
}
private String extractResultDataText(JsonNode root) {
if (root == null || root.isMissingNode() || root.isNull()) {
return "";
}
JsonNode dataNode = root.path("data");
if (dataNode.isTextual()) {
String value = dataNode.asText("");
if (looksLikeResultDataPayload(value)) {
return value;
}
return discoverEmbeddedData(parseJsonOrMissing(value));
}
if (dataNode.isObject()) {
String nested = text(firstNonNull(dataNode.get("data"), firstNonNull(dataNode.get("output"), dataNode.get("result"))));
if (nested != null && !nested.isBlank() && looksLikeResultDataPayload(nested)) {
return nested;
}
JsonNode outputs = firstNonNull(dataNode.get("outputs"), dataNode.get("details"));
String discovered = discoverEmbeddedData(outputs);
if (!discovered.isBlank()) {
return discovered;
}
}
return discoverEmbeddedData(root);
}
private String extractWorkflowOutputText(JsonNode root) {
JsonNode outputNode = findFirstField(root, "output");
if (outputNode == null || outputNode.isNull() || outputNode.isMissingNode()) {
return "";
}
if (!outputNode.isTextual()) {
String discovered = discoverEmbeddedData(outputNode);
return discovered.isBlank() && isResultDataPayload(outputNode) ? outputNode.toString() : discovered;
}
String output = normalize(outputNode.asText(""));
if (output.isBlank()) {
return "";
}
if (looksLikeResultDataPayload(output)) {
return output;
}
JsonNode parsedOutput = parseJsonOrMissing(output);
String nestedOutput = text(firstNonNull(parsedOutput.get("Output"), parsedOutput.get("output")));
if (nestedOutput != null && !nestedOutput.isBlank() && looksLikeResultDataPayload(nestedOutput)) {
return nestedOutput;
}
return discoverEmbeddedData(parsedOutput);
}
private String discoverEmbeddedData(JsonNode node) {
if (node == null || node.isNull() || node.isMissingNode()) {
return "";
}
if (node.isTextual()) {
String value = node.asText("");
if (looksLikeResultDataPayload(value)) {
return value;
}
return discoverEmbeddedData(parseJsonOrMissing(value));
}
if (node.isArray()) {
for (JsonNode child : node) {
String discovered = discoverEmbeddedData(child);
if (!discovered.isBlank()) {
return discovered;
}
}
return "";
}
if (node.isObject()) {
if (isResultDataPayload(node)) {
return node.toString();
}
for (java.util.Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> entry = it.next();
String discovered = discoverEmbeddedData(entry.getValue());
if (!discovered.isBlank()) {
return discovered;
}
}
}
return "";
}
private boolean looksLikeResultDataPayload(String value) {
JsonNode parsed = parseJsonOrMissing(value);
return isResultDataPayload(parsed);
}
private boolean isResultDataPayload(JsonNode node) {
if (node == null || node.isMissingNode() || node.isNull()) {
return false;
}
JsonNode array = node.path("data");
if (!array.isArray()) {
return false;
}
for (JsonNode item : array) {
JsonNode itemNode = resultItemNode(item);
if (hasResultFields(item) || hasResultFields(itemNode)) {
return true;
}
}
return false;
}
private boolean hasResultFields(JsonNode item) {
return item != null
&& (item.has("appearance")
|| item.has("appearance_risk")
|| item.has("patent")
|| item.has("patent ")
|| item.has("patent_risk")
|| item.has("result")
|| item.has("conclusion")
|| item.has("title_reason")
|| item.has("titleReason")
|| item.has("appearance_reason")
|| item.has("appearanceReason")
|| item.has("patent_reason")
|| item.has("patentReason")
|| item.has("patent reason"));
}
private boolean isSuccessfulWorkflowStatus(String status) {
String normalized = normalize(status).toUpperCase(Locale.ROOT);
return normalized.equals("SUCCESS")
|| normalized.equals("SUCCEEDED")
|| normalized.equals("COMPLETED")
|| normalized.equals("COMPLETE")
|| normalized.equals("DONE");
}
private boolean isFailedWorkflowStatus(String status) {
String normalized = normalize(status).toUpperCase(Locale.ROOT);
return normalized.contains("FAIL")
|| normalized.contains("ERROR")
|| normalized.contains("CANCEL");
}
private JsonNode parseJsonOrMissing(String value) {
String normalized = normalize(value);
if (!(normalized.startsWith("{") || normalized.startsWith("["))) {
return objectMapper.missingNode();
}
try {
return objectMapper.readTree(normalized);
} catch (Exception ignored) {
return objectMapper.missingNode();
}
}
private String resolveWorkflowStatus(JsonNode root) {
JsonNode dataNode = root.path("data");
JsonNode statusNode = firstNonNull(
firstNonNull(dataNode.get("status"), dataNode.get("execute_status")),
firstNonNull(root.get("status"), root.get("execute_status")));
String status = text(statusNode);
if (status != null && !status.isBlank()) {
return status;
}
return findTextByFieldName(root, "execute_status", "status");
}
private String resolveFailureMessage(JsonNode root) {
JsonNode dataNode = root.path("data");
String message = text(firstNonNull(dataNode.get("error_message"), firstNonNull(dataNode.get("msg"), root.get("msg"))));
return message == null ? "" : message;
}
private String extractExecuteId(JsonNode root) {
return findTextByFieldName(root, "execute_id", "executeId");
}
private JsonNode findFirstField(JsonNode node, String name) {
if (node == null || node.isMissingNode() || node.isNull() || name == null || name.isBlank()) {
return objectMapper.missingNode();
}
if (node.isTextual()) {
return findFirstField(parseJsonOrMissing(node.asText("")), name);
}
if (node.isArray()) {
for (JsonNode child : node) {
JsonNode found = findFirstField(child, name);
if (found != null && !found.isMissingNode() && !found.isNull()) {
return found;
}
}
return objectMapper.missingNode();
}
if (node.isObject()) {
JsonNode direct = node.get(name);
if (direct != null && !direct.isMissingNode() && !direct.isNull()) {
return direct;
}
for (java.util.Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> entry = it.next();
JsonNode found = findFirstField(entry.getValue(), name);
if (found != null && !found.isMissingNode() && !found.isNull()) {
return found;
}
}
}
return objectMapper.missingNode();
}
private String findTextByFieldName(JsonNode node, String... names) {
if (node == null || node.isMissingNode() || node.isNull()) {
return "";
}
if (node.isTextual()) {
return findTextByFieldName(parseJsonOrMissing(node.asText("")), names);
}
if (node.isArray()) {
for (JsonNode child : node) {
String found = findTextByFieldName(child, names);
if (!found.isBlank()) {
return found;
}
}
return "";
}
if (node.isObject()) {
for (String name : names) {
String value = text(node.get(name));
if (value != null && !value.isBlank()) {
return value;
}
}
JsonNode dataNode = node.get("data");
if (dataNode != null && dataNode.isTextual()) {
String found = findTextByFieldName(parseJsonOrMissing(dataNode.asText("")), names);
if (!found.isBlank()) {
return found;
}
}
for (java.util.Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> entry = it.next();
String found = findTextByFieldName(entry.getValue(), names);
if (!found.isBlank()) {
return found;
}
}
}
return "";
}
private String wrapDataPayload(String dataText) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("code", 0);
payload.put("data", dataText);
return writeJson(payload);
}
private String writeJson(Object value) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new IllegalStateException("Failed to serialize Coze payload", ex);
}
}
private String abbreviate(String value, int maxLength) {
String normalized = value == null ? "" : value.trim();
if (normalized.length() <= maxLength) {
return normalized;
}
return normalized.substring(0, Math.max(0, maxLength - 3)) + "...";
}
private void sleepBeforeRetry(int attemptIndex) {
long delayMillis = Math.max(1, attemptIndex) * 1500L;
ensureNotInterrupted();
sleepQuietly(delayMillis);
}
private void sleepQuietly(long delayMillis) {
try {
Thread.sleep(delayMillis);
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Coze workflow interrupted", interruptedException);
}
}
private void ensureNotInterrupted() {
if (Thread.currentThread().isInterrupted()) {
throw new IllegalStateException("Coze workflow interrupted");
}
}
private RuntimeException propagate(Exception ex) {
if (ex instanceof RuntimeException runtimeException) {
return runtimeException;
}
return new IllegalStateException(nonBlank(ex.getMessage(), "Coze call failed"), ex);
}
private JsonNode firstNonNull(JsonNode left, JsonNode right) {
return left == null || left.isNull() ? right : left;
}
private String text(JsonNode node) {
return node == null || node.isNull() ? null : node.asText();
}
private String nonBlank(String value, String fallback) {
return value == null || value.isBlank() ? fallback : value;
}
private String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
private String normalize(String value) {
return value == null ? "" : value.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim();
}
private String rowKey(SimilarAsinResultRowDto row) {
if (row == null) {
return "";
}
return rowKey(row.getId(), row.getAsin(), row.getCountry());
}
private String rowKey(String rowId, String asin, String country) {
return normalize(rowId) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
}
private String failureMessage(Exception ex) {
if (ex instanceof PartialCozeResultException partial) {
return "Coze result incomplete(" + partial.resolvedCount() + "/" + partial.expectedCount() + ")";
}
String message = ex == null ? null : ex.getMessage();
if (message == null || message.isBlank()) {
return "Coze call failed";
}
if (message.contains("Workflow node execution limit exceeded")) {
return "Coze workflow node execution limit exceeded";
}
if (message.contains("Read timed out")) {
return "Coze call timed out";
}
return "Coze call failed: " + message;
}
private String stripBearer(String token) {
String normalized = token == null ? "" : token.trim();
return normalized.regionMatches(true, 0, "Bearer ", 0, 7) ? normalized.substring(7).trim() : normalized;
}
private String joinUrl(String baseUrl, String path) {
String base = baseUrl == null ? "" : baseUrl.trim();
String suffix = path == null ? "" : path.trim();
if (base.endsWith("/") && suffix.startsWith("/")) {
return base + suffix.substring(1);
}
if (!base.endsWith("/") && !suffix.startsWith("/")) {
return base + "/" + suffix;
}
return base + suffix;
}
private record CozeResult(
String groupKey,
String rowId,
String asin,
String country,
String title,
String appearance,
String patent,
String result,
String titleReason,
String appearanceReason,
String patentReason
) {
}
private record InspectAttempt(
String raw,
List<SimilarAsinResultRowDto> mergedRows,
int resolvedCount,
int rawResultCount
) {
}
public record CozeSubmitResponse(
String executeId,
String immediateData,
String rawResponse
) {
}
public record CozePollResponse(
String executeId,
String status,
String dataText,
String outputText,
String failureMessage,
String rawResponse
) {
public boolean hasPayload() {
return dataText != null && !dataText.isBlank() || outputText != null && !outputText.isBlank();
}
public String resolvedPayloadText() {
return dataText != null && !dataText.isBlank() ? dataText : outputText;
}
public boolean isFailed() {
String normalized = status == null ? "" : status.trim().toUpperCase(Locale.ROOT);
return normalized.contains("FAILED") || normalized.contains("ERROR") || normalized.contains("CANCEL");
}
public boolean isFinished() {
String normalized = status == null ? "" : status.trim().toUpperCase(Locale.ROOT);
return isFailed()
|| normalized.contains("SUCCESS")
|| normalized.contains("SUCCEED")
|| normalized.contains("FINISH")
|| normalized.contains("DONE")
|| normalized.contains("COMPLET");
}
}
private static final class PartialCozeResultException extends RuntimeException {
private final int resolvedCount;
private final int expectedCount;
private final int rawResultCount;
private PartialCozeResultException(int resolvedCount, int expectedCount, int rawResultCount) {
super("partial-result resolved=" + resolvedCount + "/" + expectedCount + " raw=" + rawResultCount);
this.resolvedCount = resolvedCount;
this.expectedCount = expectedCount;
this.rawResultCount = rawResultCount;
}
private int resolvedCount() {
return resolvedCount;
}
private int expectedCount() {
return expectedCount;
}
@SuppressWarnings("unused")
private int rawResultCount() {
return rawResultCount;
}
}
}

145
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/controller/SimilarAsinController.java Normal file
View File

@@ -0,0 +1,145 @@
package com.nanri.aiimage.modules.similarasin.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/similar-asin")
@Tag(name = "相似ASIN检测", description = "相似ASIN检测任务接口。前端上传 Excel 后由 Java 解析并创建任务；Python 回传商品数据；Java 负责攒批调用 Coze、补齐子行、生成最终 xlsx 并上传 OSS。")
public class SimilarAsinController {
private final SimilarAsinTaskService service;
@PostMapping("/parse")
@Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件，提取 id、ASIN、国家、URL、标题等字段。返回给前端的数据只包含整数 id 和 n_1 行；n_2、n_3 等子行会保存在 OSS 解析载荷中，用于最终结果补齐。创建后的任务状态为 PENDING，不会自动推送 Python。")
public ApiResponse<SimilarAsinParseVo> parse(@Valid @RequestBody SimilarAsinParseRequest request) {
return ApiResponse.success(service.parseAndCreateTask(request));
}
@GetMapping("/dashboard")
@Operation(summary = "查询相似ASIN检测总览", description = "查询当前用户的运行中、成功、失败和已结束任务数量。页面进入时请求一次即可，不需要持续轮询。")
public ApiResponse<SimilarAsinDashboardVo> dashboard(
@Parameter(description = "当前用户 ID，用于隔离不同用户的任务和历史记录。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(service.dashboard(userId));
}
@GetMapping("/history")
@Operation(summary = "查询相似ASIN检测历史", description = "查询当前用户最近的相似ASIN检测历史记录，包含源文件名、任务状态、行数、错误信息和最终 xlsx 下载地址。")
public ApiResponse<SimilarAsinHistoryVo> history(
@Parameter(description = "当前用户 ID，用于查询该用户自己的历史记录。", required = true, example = "1")
@RequestParam("user_id") Long userId,
@Parameter(description = "history limit, default 50, max 100", example = "50")
@RequestParam(value = "limit", required = false, defaultValue = "50") Integer limit) {
return ApiResponse.success(service.history(userId, limit));
}
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询任务进度", description = "前端只对活跃任务调用该接口，建议 6 秒一次。接口只返回轻量任务状态，不返回明细结果。")
public ApiResponse<SimilarAsinTaskBatchVo> progress(@Valid @RequestBody SimilarAsinTaskBatchRequest request) {
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
}
@PostMapping("/tasks/{taskId}/activate")
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用，将任务从 PENDING 改为 RUNNING，并记录后端内部活跃时间。后续活跃时间由 Python 回传结果接口自动刷新，不需要单独心跳接口。")
public ApiResponse<Void> activate(
@Parameter(description = "相似ASIN检测任务 ID，即解析接口返回的 taskId。", required = true, example = "3938")
@PathVariable Long taskId,
@Parameter(description = "当前用户 ID，必须与创建任务的用户一致。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
service.activateTask(taskId, userId);
return ApiResponse.success(null);
}
@PostMapping("/tasks/{taskId}/result")
@Operation(summary = "提交 Python 回传结果", description = "Python 回传商品数据接口。items 可以是一条或多条；Java 先原样保存回传数据，再内部攒够 10 条调用 Coze。done=true 表示 Python 已完成全部回传，Java 会强制处理剩余不足 10 条的数据并生成最终 xlsx。")
public ApiResponse<Void> result(
@Parameter(description = "相似ASIN检测任务 ID。任务必须处于 RUNNING 状态。", required = true, example = "3938")
@PathVariable Long taskId,
@Valid @RequestBody SimilarAsinSubmitResultRequest request,
jakarta.servlet.http.HttpServletResponse response) {
response.setCharacterEncoding(StandardCharsets.UTF_8.name());
response.setContentType("application/json;charset=UTF-8");
service.submitResult(taskId, request);
return ApiResponse.success(null);
}
@DeleteMapping("/tasks/{taskId}")
@Operation(summary = "删除任务", description = "删除当前用户的一条相似ASIN检测任务，同时清理任务结果、scope 状态和分片记录。")
public ApiResponse<Void> deleteTask(
@Parameter(description = "相似ASIN检测任务 ID。", required = true, example = "3938")
@PathVariable Long taskId,
@Parameter(description = "当前用户 ID，必须与创建任务的用户一致。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
service.deleteTask(taskId, userId);
return ApiResponse.success(null);
}
@DeleteMapping("/history/{resultId}")
@Operation(summary = "删除历史记录", description = "删除当前用户的一条相似ASIN检测历史记录。只删除 biz_file_result 记录，不主动删除任务主记录。")
public ApiResponse<Void> deleteHistory(
@Parameter(description = "历史结果记录 ID，即 history 接口返回的 resultId。", required = true, example = "1001")
@PathVariable Long resultId,
@Parameter(description = "当前用户 ID。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
service.deleteHistory(resultId, userId);
return ApiResponse.success(null);
}
@GetMapping("/results/{resultId}/download")
@Operation(summary = "下载相似ASIN检测结果文件")
public void downloadResult(
@Parameter(description = "结果记录 ID", required = true, example = "1001")
@PathVariable Long resultId,
@Parameter(description = "当前用户 ID", required = true, example = "1")
@RequestParam("user_id") Long userId,
jakarta.servlet.http.HttpServletResponse response) {
String url = service.resolveResultDownloadUrl(resultId, userId);
String filename = service.resolveResultDownloadFilename(resultId, userId);
if (url == null || url.isBlank()) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
}
try {
response.setContentType("application/octet-stream");
DownloadHeaderUtil.setAttachment(response, filename);
try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536];
int read;
while ((read = in.read(buffer)) != -1) {
response.getOutputStream().write(buffer, 0, read);
}
response.getOutputStream().flush();
}
} catch (Exception ex) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
}
}
}

28
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParseRequest.java Normal file
View File

@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测解析请求")
public class SimilarAsinParseRequest {
@JsonProperty("user_id")
@NotNull
@Schema(description = "当前用户 ID。后端会把创建的任务、历史记录和结果文件归属到该用户。", example = "1", requiredMode = Schema.RequiredMode.REQUIRED)
private Long userId;
@NotEmpty
@Schema(description = "已上传的 Excel 文件列表。支持多文件聚合解析；文件对象来自统一上传接口返回值。", requiredMode = Schema.RequiredMode.REQUIRED)
private List<SimilarAsinSourceFileDto> files;
@JsonProperty("ai_prompt")
@JsonAlias({"aiPrompt", "prompt"})
@Schema(description = "AI 提示词。非必填；为空时前端会使用默认提示词。后端会保存该提示词，并在调用 Coze workflow 时作为 prompt 参数传入。", example = "请排查这些亚马逊商品在英国及欧洲地区是否存在知识产权侵权风险。")
private String aiPrompt;
}

31
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParsedPayloadDto.java Normal file
View File

@@ -0,0 +1,31 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测解析载荷")
public class SimilarAsinParsedPayloadDto {
@Schema(description = "AI 提示词")
private String aiPrompt;
@Schema(description = "本次解析的源文件列表")
private List<SimilarAsinSourceFileDto> sourceFiles = new ArrayList<>();
@Schema(description = "Excel 原始表头列表")
private List<String> headers = new ArrayList<>();
@Schema(description = "兼容旧链路的平铺有效行，现为全部有效行")
private List<SimilarAsinParsedRowVo> items = new ArrayList<>();
@Schema(description = "按相邻主 ID 块分组后的完整数据")
private List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
@Schema(description = "完整有效行")
private List<SimilarAsinParsedRowVo> allItems = new ArrayList<>();
}

29
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinResultGroupDto.java Normal file
View File

@@ -0,0 +1,29 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "Python 回传的相似ASIN检测分组结果")
public class SimilarAsinResultGroupDto {
@Schema(description = "来源文件 key")
private String sourceFileKey;
@Schema(description = "来源文件名")
private String sourceFilename;
@Schema(description = "相邻主 ID 块的分组 key")
private String groupKey;
@Schema(description = "主 ID，例如 1、2、10")
private String baseId;
@Schema(description = "分组首条展示 ID")
private String displayId;
@Schema(description = "分组内全部抓取结果")
private List<SimilarAsinResultRowDto> items = new ArrayList<>();
}

187
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinResultRowDto.java Normal file
View File

@@ -0,0 +1,187 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonSetter;
import com.fasterxml.jackson.databind.JsonNode;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
@Data
@Schema(description = "相似ASIN检测单行商品数据")
public class SimilarAsinResultRowDto {
private static final int MAX_IMAGE_URLS = 24;
@Schema(description = "来源文件 key。多文件回传时用于避免行结果串到别的文件。", example = "uploads/20260426/similar_asin_17.xlsx")
private String sourceFileKey;
@Schema(description = "来源文件名。仅用于调试与追踪。", example = "17.xlsx")
private String sourceFilename;
@Schema(description = "行级唯一 token。Python 应从解析结果原样透传。", example = "uploads/20260426/similar_asin_17.xlsx::row::2")
private String rowToken;
@Schema(description = "主数据分组 key。Python 应从解析结果原样透传，用于把同组子行补回。", example = "uploads/20260426/similar_asin_17.xlsx::2@2")
private String groupKey;
@Schema(description = "Excel 中的 id。代表行通常是整数 id 或 n_1，例如 2_1；最终生成 xlsx 时，2_2、2_3 会复用同组 2_1 的 Coze 检测结果。", example = "2_1")
private String id;
@Schema(description = "亚马逊 ASIN。后端会统一按大写处理和匹配。", example = "B0CJ8SNXXV")
private String asin;
@Schema(description = "站点或国家。来自 Excel 的国家列，例如英国、德国、法国。", example = "英国")
private String country;
@Schema(description = "兼容旧链路的首个图片 URL。新链路允许 url 传字符串或数组；数组会同步写入 urls。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg")
private String url;
@Schema(description = "商品图片 URL 列表，最多保留 24 个非空地址。请求中也可以直接把 url 传成数组。", example = "[\"https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg\"]")
private List<String> urls = new ArrayList<>();
@Schema(description = "商品标题。Java 调用 Coze 时会放入 title_list；为空时会回退使用 ASIN。", example = "Women Floral Dress Summer Casual")
@JsonAlias({"productTitle", "product_title", "itemTitle", "item_title", "商品标题", "商品名称", "标题"})
private String title;
@Schema(description = "单行错误信息。通常用于记录 Python 单行处理异常；Coze 失败时后端会尽量保留原始行，不强行写入风险结果。", example = "图片地址为空")
private String error;
@Schema(description = "单行完成标记。当前主要使用请求体顶层 done 控制任务收尾，该字段仅作兼容。", example = "true")
private Boolean done;
@Schema(description = "Java 调用 Coze 后生成的标题维度检测结果，对应最终 xlsx 的“标题维度（商标）”列。Python 回传请求中不要传该字段；即使传入，后端也会以 Java/Coze 处理结果为准。", example = "标题未发现明显商标侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
private String titleRisk;
@Schema(description = "Java 调用 Coze 后生成的外观维度检测结果，对应最终 xlsx 的“外观维度（外观设计专利）”列。Python 回传请求中不要传该字段。", example = "未发现明显外观设计专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
private String appearanceRisk;
@JsonAlias({"patent ", "patent"})
@Schema(description = "Java 调用 Coze 后生成的专利维度检测结果，对应最终 xlsx 的“专利维度（发明/实用新型专利）”列。兼容 Coze 返回字段 patent 和 patent 后带空格的情况；Python 回传请求中不要传该字段。", example = "未发现明显发明或实用新型专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
private String patentRisk;
@Schema(description = "Java 调用 Coze 后生成的最终结论，对应最终 xlsx 的“结论”列。Python 回传请求中不要传该字段。", example = "未发现明显侵权风险", accessMode = Schema.AccessMode.READ_ONLY)
private String conclusion;
@JsonAlias({"title_reason", "titleReason"})
@Schema(description = "Coze title reason", accessMode = Schema.AccessMode.READ_ONLY)
private String titleReason;
@JsonAlias({"appearance_reason", "appearanceReason"})
@Schema(description = "Coze appearance reason", accessMode = Schema.AccessMode.READ_ONLY)
private String appearanceReason;
@JsonAlias({"patent_reason", "patentReason", "patent reason"})
@Schema(description = "Coze patent reason", accessMode = Schema.AccessMode.READ_ONLY)
private String patentReason;
public String getUrl() {
if (url != null && !url.isBlank()) {
return url;
}
List<String> normalized = getUrls();
return normalized.isEmpty() ? "" : normalized.get(0);
}
@JsonSetter("url")
@JsonAlias({
"imageUrl", "image_url", "imgUrl", "img_url", "pictureUrl", "picture_url",
"mainImage", "main_image", "mainImageUrl", "main_image_url",
"productImage", "product_image", "productImageUrl", "product_image_url",
"image", "img", "pic", "picture", "link", "imageLink", "image_link"
})
public void setUrl(Object value) {
List<String> normalized = normalizeUrls(value);
this.urls = normalized;
this.url = normalized.isEmpty() ? "" : normalized.get(0);
}
public List<String> getUrls() {
List<String> normalized = normalizeUrls(urls);
if (url != null && !url.isBlank() && normalized.stream().noneMatch(url.trim()::equals)) {
List<String> combined = new ArrayList<>(normalized.size() + 1);
combined.add(url.trim());
combined.addAll(normalized);
return limitUrls(combined);
}
return normalized;
}
@JsonSetter("urls")
@JsonAlias({
"imageUrls", "image_urls", "imgUrls", "img_urls", "pictureUrls", "picture_urls",
"mainImages", "main_images", "mainImageUrls", "main_image_urls",
"productImages", "product_images", "productImageUrls", "product_image_urls",
"images", "imgs", "pics", "pictures", "links", "imageLinks", "image_links",
"图片链接", "商品图片", "商品主图", "主图", "主图链接"
})
public void setUrls(Object urls) {
List<String> normalized = normalizeUrls(urls);
this.urls = normalized;
this.url = normalized.isEmpty() ? "" : normalized.get(0);
}
public boolean hasImageUrl() {
return !getUrls().isEmpty();
}
private static List<String> normalizeUrls(Object value) {
List<String> result = new ArrayList<>();
appendUrls(result, value);
return limitUrls(result);
}
private static void appendUrls(List<String> result, Object value) {
if (value == null) {
return;
}
if (value instanceof JsonNode node) {
if (node.isArray()) {
node.forEach(child -> appendUrls(result, child));
return;
}
if (node.isNull() || node.isMissingNode()) {
return;
}
appendUrls(result, node.asText(""));
return;
}
if (value instanceof Collection<?> collection) {
collection.forEach(item -> appendUrls(result, item));
return;
}
Class<?> valueClass = value.getClass();
if (valueClass.isArray()) {
int length = Array.getLength(value);
for (int i = 0; i < length; i++) {
appendUrls(result, Array.get(value, i));
}
return;
}
String text = String.valueOf(value).trim();
if (!text.isBlank()) {
result.add(text);
}
}
private static List<String> limitUrls(List<String> values) {
Set<String> deduplicated = new LinkedHashSet<>();
if (values != null) {
for (String value : values) {
if (value == null || value.isBlank()) {
continue;
}
deduplicated.add(value.trim());
if (deduplicated.size() >= MAX_IMAGE_URLS) {
break;
}
}
}
return new ArrayList<>(deduplicated);
}
}

15
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinSourceFileDto.java Normal file
View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "相似ASIN检测源文件信息")
public class SimilarAsinSourceFileDto {
@Schema(description = "上传接口返回的临时文件 key。后端根据该 key 查找本地临时 Excel 文件并解析。", example = "uploads/20260426/similar_asin_17.xlsx", requiredMode = Schema.RequiredMode.REQUIRED)
private String fileKey;
@Schema(description = "原始文件名。用于历史记录展示和最终结果文件命名。", example = "17.xlsx")
private String originalFilename;
@Schema(description = "相对目录路径。当前仅记录来源，相似ASIN检测不依赖该字段处理。", example = "xlsx/17.xlsx")
private String relativePath;
}

32
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinSubmitResultRequest.java Normal file
View File

@@ -0,0 +1,32 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "Python 回传相似ASIN检测结果请求")
public class SimilarAsinSubmitResultRequest {
@Schema(description = "本次 Python 回传的提交批次标识", example = "similar-asin-3938")
private String submissionId;
@Schema(description = "当前回传分片序号", example = "1")
private Integer chunkIndex;
@Schema(description = "本任务预计总分片数", example = "36")
private Integer chunkTotal;
@Schema(description = "是否为最后一次回传", example = "false")
private Boolean done;
@Schema(description = "Python 侧任务级错误信息", example = "浏览器执行异常，任务提前结束")
private String error;
@Schema(description = "本次回传的分组结果列表，推荐优先使用")
private List<SimilarAsinResultGroupDto> groups = new ArrayList<>();
@Schema(description = "兼容旧链路的平铺结果列表")
private List<SimilarAsinResultRowDto> items = new ArrayList<>();
}

15
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinTaskBatchRequest.java Normal file
View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测批量进度查询请求")
public class SimilarAsinTaskBatchRequest {
@NotEmpty
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务；后端会批量查询，避免每个任务单独请求。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds;
}

17
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinDashboardVo.java Normal file
View File

@@ -0,0 +1,17 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "相似ASIN检测总览统计")
public class SimilarAsinDashboardVo {
@Schema(description = "运行中任务数量。这里统计 RUNNING 状态任务。", example = "1")
private Long pendingTaskCount;
@Schema(description = "已结束任务数量，等于成功任务数加失败任务数。", example = "12")
private Long processedTaskCount;
@Schema(description = "成功任务数量。", example = "10")
private Long successTaskCount;
@Schema(description = "失败任务数量。", example = "2")
private Long failedTaskCount;
}

37
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java Normal file
View File

@@ -0,0 +1,37 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "相似ASIN检测历史记录项")
public class SimilarAsinHistoryItemVo {
@Schema(description = "结果记录 ID。删除历史、下载结果时使用。", example = "1001")
private Long resultId;
@Schema(description = "任务 ID。", example = "3938")
private Long taskId;
@Schema(description = "源 Excel 文件名。", example = "17.xlsx")
private String sourceFilename;
@Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx")
private String resultFilename;
@Schema(description = "最终结果文件下载地址。后端基于 OSS objectKey 生成的新鲜预签名 URL。", example = "https://bucket.oss-cn-hangzhou.aliyuncs.com/result/similar_asin/xxx/17-result.xlsx?Expires=...")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
private Integer fileProgressPercent;
private Integer fileProgressCurrent;
private Integer fileProgressTotal;
private String fileProgressMessage;
@Schema(description = "任务状态：PENDING=已解析待推送，RUNNING=执行中，SUCCESS=成功，FAILED=失败。", example = "SUCCESS")
private String taskStatus;
@Schema(description = "结果是否成功。true 表示任务完成并生成结果文件；false 表示失败或未完成。", example = "true")
private Boolean success;
@Schema(description = "错误信息。任务失败时返回，例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final similar ASIN result")
private String error;
@Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716")
private Integer rowCount;
@Schema(description = "历史记录创建时间，ISO 本地时间字符串。", example = "2026-04-26T10:30:00")
private String createdAt;
}

14
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryVo.java Normal file
View File

@@ -0,0 +1,14 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测历史记录列表")
public class SimilarAsinHistoryVo {
@Schema(description = "历史记录项列表，默认返回最近 100 条。")
private List<SimilarAsinHistoryItemVo> items = new ArrayList<>();
}

41
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParseVo.java Normal file
View File

@@ -0,0 +1,41 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测解析结果")
public class SimilarAsinParseVo {
@Schema(description = "新创建的任务 ID", example = "3938")
private Long taskId;
@Schema(description = "来源 Excel 文件名，多个文件时为聚合展示文案", example = "17.xlsx 等 2 个文件")
private String sourceFilename;
@Schema(description = "参与本次解析的源文件数量", example = "2")
private Integer sourceFileCount;
@Schema(description = "Excel 中检测到的非空数据总行数", example = "716")
private Integer totalRows;
@Schema(description = "解析出的有效行数", example = "458")
private Integer acceptedRows;
@Schema(description = "因缺少必要字段而被丢弃的行数", example = "12")
private Integer droppedRows;
@Schema(description = "按相邻主 ID 块生成的分组数", example = "36")
private Integer groupCount;
@Schema(description = "本任务最终使用的 AI 提示词")
private String aiPrompt;
@Schema(description = "兼容旧前端的平铺有效行列表，现为全部有效行")
private List<SimilarAsinParsedRowVo> items = new ArrayList<>();
@Schema(description = "返回前端和推送 Python 的分组列表")
private List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
}

32
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParsedGroupVo.java Normal file
View File

@@ -0,0 +1,32 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测解析后的分组数据")
public class SimilarAsinParsedGroupVo {
@Schema(description = "来源文件 key")
private String sourceFileKey;
@Schema(description = "来源文件名")
private String sourceFilename;
@Schema(description = "相邻主 ID 块的分组 key")
private String groupKey;
@Schema(description = "主 ID，例如 1、2、10")
private String baseId;
@Schema(description = "分组首条展示 ID，例如 1 或 2_1")
private String displayId;
@Schema(description = "分组内行数")
private Integer itemCount;
@Schema(description = "分组内全部行，顺序与原 Excel 保持一致")
private List<SimilarAsinParsedRowVo> items = new ArrayList<>();
}

47
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParsedRowVo.java Normal file
View File

@@ -0,0 +1,47 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.LinkedHashMap;
import java.util.Map;
@Data
@Schema(description = "相似ASIN检测解析出的单行数据")
public class SimilarAsinParsedRowVo {
@Schema(description = "来源文件 key。用于多文件场景下区分同名/同 ID 行。", example = "uploads/20260426/similar_asin_17.xlsx")
private String sourceFileKey;
@Schema(description = "来源文件名。用于调试和结果追踪。", example = "17.xlsx")
private String sourceFilename;
@Schema(description = "Excel 原始行号，从 1 开始。", example = "2")
private Integer rowIndex;
@Schema(description = "Excel 中原始 id 值。", example = "2_1")
private String sourceId;
@Schema(description = "前端展示和 Python 回传使用的 id。整数 id 原样保留，子数据第一条如 2_1 原样保留。", example = "2_1")
private String displayId;
@Schema(description = "行级唯一 token。多文件、重复主数据时用于精确匹配回传结果。", example = "uploads/20260426/similar_asin_17.xlsx::row::2")
private String rowToken;
@Schema(description = "同一主数据块的分组 key。用于把 2_1、2_2、2_3 等子行重新补回同一组结果。", example = "uploads/20260426/similar_asin_17.xlsx::2@2")
private String groupKey;
@Schema(description = "亚马逊 ASIN。", example = "B0CJ8SNXXV")
private String asin;
@Schema(description = "国家或站点。", example = "英国")
private String country;
@Schema(description = "商品图片 URL 或商品 URL，供 Coze 检测使用。", example = "https://webstatic.aiproxy.vip/output/demo.jpg")
private String url;
@Schema(description = "商品标题。", example = "Women Floral Dress Summer Casual")
private String title;
@Schema(description = "该 Excel 行的原始列值映射。最终生成 xlsx 时可以从这里读取价格等字段。", example = "{\"id\":\"2_1\",\"asin\":\"B0CJ8SNXXV\",\"国家\":\"英国\",\"价格\":\"12.99\"}")
private Map<String, String> values = new LinkedHashMap<>();
}

16
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskBatchVo.java Normal file
View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测批量进度响应")
public class SimilarAsinTaskBatchVo {
@Schema(description = "查询到的任务详情列表。顺序按请求 taskIds 处理。")
private List<SimilarAsinTaskDetailVo> items = new ArrayList<>();
@Schema(description = "未找到或不属于相似ASIN检测模块的任务 ID 列表。", example = "[99999]")
private List<Long> missingTaskIds = new ArrayList<>();
}

16
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskDetailVo.java Normal file
View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测任务进度详情")
public class SimilarAsinTaskDetailVo {
@Schema(description = "任务主记录轻量信息。")
private SimilarAsinTaskItemVo task;
@Schema(description = "预留的任务明细列表。当前进度接口主要返回任务轻量状态，不返回完整结果明细。")
private List<SimilarAsinHistoryItemVo> items = new ArrayList<>();
}

23
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskItemVo.java Normal file
View File

@@ -0,0 +1,23 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "相似ASIN检测任务轻量信息")
public class SimilarAsinTaskItemVo {
@Schema(description = "任务 ID。", example = "3938")
private Long id;
@Schema(description = "任务编号，后端自动生成。", example = "SIMILAR_ASIN-1780000000000000000")
private String taskNo;
@Schema(description = "任务状态：PENDING=已解析待推送，RUNNING=执行中，SUCCESS=成功，FAILED=失败。", example = "RUNNING")
private String status;
@Schema(description = "任务级错误信息。失败时返回。", example = "生成相似ASIN检测结果失败")
private String errorMessage;
@Schema(description = "创建时间，ISO 本地时间字符串。", example = "2026-04-26T10:00:00")
private String createdAt;
@Schema(description = "最后更新时间，通常由 Python 回传结果或任务收尾更新。", example = "2026-04-26T10:05:00")
private String updatedAt;
@Schema(description = "完成时间。任务未结束时为空。", example = "2026-04-26T10:10:00")
private String finishedAt;
}

144
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskCacheService.java Normal file
View File

@@ -0,0 +1,144 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
@Service
@RequiredArgsConstructor
@Slf4j
public class SimilarAsinTaskCacheService {
private static final long TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
public void appendPendingRow(Long taskId, String scopeHash, Integer chunkIndex, SimilarAsinResultRowDto row) {
if (taskId == null || taskId <= 0 || scopeHash == null || scopeHash.isBlank() || chunkIndex == null || row == null) {
return;
}
try {
stringRedisTemplate.opsForList().rightPush(
pendingRowsKey(taskId),
objectMapper.writeValueAsString(new PendingRow(scopeHash, chunkIndex, row))
);
stringRedisTemplate.expire(pendingRowsKey(taskId), Duration.ofHours(TTL_HOURS));
touchTaskHeartbeat(taskId);
} catch (Exception ignored) {
}
}
public long pendingRowCount(Long taskId) {
Long size;
try {
size = stringRedisTemplate.opsForList().size(pendingRowsKey(taskId));
} catch (Exception ex) {
log.warn("[similar-asin-cache] pending row count degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
return size == null ? 0L : size;
}
public List<PendingRow> drainPendingRows(Long taskId, int limit) {
if (taskId == null || taskId <= 0 || limit <= 0) {
return List.of();
}
String key = pendingRowsKey(taskId);
List<String> values;
try {
values = stringRedisTemplate.opsForList().range(key, 0, limit - 1L);
} catch (Exception ex) {
log.warn("[similar-asin-cache] drain range degraded taskId={} msg={}", taskId, ex.getMessage());
return List.of();
}
if (values == null || values.isEmpty()) {
return List.of();
}
try {
stringRedisTemplate.opsForList().trim(key, values.size(), -1);
} catch (Exception ex) {
log.warn("[similar-asin-cache] drain trim degraded taskId={} msg={}", taskId, ex.getMessage());
return List.of();
}
List<PendingRow> rows = new ArrayList<>();
for (String value : values) {
if (value == null || value.isBlank()) {
continue;
}
try {
rows.add(objectMapper.readValue(value, PendingRow.class));
} catch (Exception ignored) {
}
}
touchTaskHeartbeat(taskId);
return rows;
}
public void touchTaskHeartbeat(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
try {
stringRedisTemplate.opsForValue().set(
heartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(TTL_HOURS)
);
} catch (Exception ex) {
log.warn("[similar-asin-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public long getTaskHeartbeatMillis(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(heartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[similar-asin-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
try {
return Long.parseLong(raw);
} catch (NumberFormatException ignored) {
return 0L;
}
}
public void deleteTaskCache(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
try {
stringRedisTemplate.delete(pendingRowsKey(taskId));
stringRedisTemplate.delete(heartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[similar-asin-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
private String pendingRowsKey(Long taskId) {
return "similar-asin:task:pending-rows:" + taskId;
}
private String heartbeatKey(Long taskId) {
return "similar-asin:task:heartbeat:" + taskId;
}
public record PendingRow(String scopeHash, Integer chunkIndex, SimilarAsinResultRowDto row) {
}
}

2413
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java Normal file
View File

File diff suppressed because it is too large Load Diff

16
backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java
View File

@@ -14,9 +14,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid; import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@@ -27,6 +25,8 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import java.net.URI;
@RestController @RestController
@RequiredArgsConstructor @RequiredArgsConstructor
@RequestMapping("/api/split") @RequestMapping("/api/split")
@@ -93,14 +93,14 @@ public class SplitRunController {
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "返回结果文件流"), @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "返回结果文件流"),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "结果文件不存在") @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "结果文件不存在")
}) })
public ResponseEntity<Resource> download( public ResponseEntity<Void> download(
@Parameter(description = "数据拆分结果记录 ID", required = true) @PathVariable Long resultId, @Parameter(description = "数据拆分结果记录 ID", required = true) @PathVariable Long resultId,
@Parameter(name = "user_id", description = "当前登录用户 ID", required = true, in = ParameterIn.QUERY) @Parameter(name = "user_id", description = "当前登录用户 ID", required = true, in = ParameterIn.QUERY)
@RequestParam("user_id") Long userId) { @RequestParam("user_id") Long userId) {
Resource resource = splitRunService.getResultFile(resultId, userId); String downloadUrl = splitRunService.getResultDownloadUrl(resultId, userId);
return ResponseEntity.ok() return ResponseEntity.status(302)
.contentType(MediaType.APPLICATION_OCTET_STREAM) .location(URI.create(downloadUrl))
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment") .header(HttpHeaders.CACHE_CONTROL, "no-store")
.body(resource); .build();
} }
} }

13
backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java
View File

@@ -19,8 +19,6 @@ import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.io.File; import java.io.File;
@@ -173,7 +171,7 @@ public class SplitRunService {
vo.setResultId(entity.getId()); vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getSourceFilename()); vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename()); vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(null); vo.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
vo.setRowCount(entity.getRowCount()); vo.setRowCount(entity.getRowCount());
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage()); vo.setError(entity.getErrorMessage());
@@ -192,17 +190,12 @@ public class SplitRunService {
fileResultMapper.deleteById(resultId); fileResultMapper.deleteById(resultId);
} }
public Resource getResultFile(Long resultId, Long userId) { public String getResultDownloadUrl(Long resultId, Long userId) {
FileResultEntity entity = fileResultMapper.selectById(resultId); FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("Split result file does not exist."); throw new BusinessException("Split result file does not exist.");
} }
return ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl());
File file = new File(entity.getResultFileUrl());
if (!file.exists()) {
throw new BusinessException("Split result file does not exist.");
}
return new FileSystemResource(file);
} }
private List<SplitChunkResult> splitExcelByLegacyRules( private List<SplitChunkResult> splitExcelByLegacyRules(

7
backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskScopeStateEntity.java
View File

@@ -19,6 +19,13 @@ public class TaskScopeStateEntity {
private String scopeHash; private String scopeHash;
private String parsedPayloadJson; private String parsedPayloadJson;
private String stateJson; private String stateJson;
private String cozeExecuteId;
private String cozeStatus;
private LocalDateTime cozeSubmittedAt;
private LocalDateTime cozeLastPolledAt;
private LocalDateTime cozeCompletedAt;
private Integer cozeAttemptCount;
private String cozeError;
private Integer chunkTotal; private Integer chunkTotal;
private Integer receivedChunkCount; private Integer receivedChunkCount;
private Integer completed; private Integer completed;

10
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobLocalDispatcher.java
View File

@@ -4,6 +4,7 @@ import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@@ -18,8 +19,9 @@ public class TaskFileJobLocalDispatcher {
private final TaskFileJobMapper taskFileJobMapper; private final TaskFileJobMapper taskFileJobMapper;
private final ObjectProvider<TaskResultFileJobWorker> taskResultFileJobWorkerProvider; private final ObjectProvider<TaskResultFileJobWorker> taskResultFileJobWorkerProvider;
@Autowired
@Qualifier("taskFileJobDispatchExecutor") @Qualifier("taskFileJobDispatchExecutor")
private final TaskExecutor taskFileJobDispatchExecutor; private TaskExecutor taskFileJobDispatchExecutor;
@Value("${aiimage.result-file-job.local-dispatch-enabled:true}") @Value("${aiimage.result-file-job.local-dispatch-enabled:true}")
private boolean localDispatchEnabled; private boolean localDispatchEnabled;
@@ -28,6 +30,7 @@ public class TaskFileJobLocalDispatcher {
if (!localDispatchEnabled || jobId == null || jobId <= 0) { if (!localDispatchEnabled || jobId == null || jobId <= 0) {
return false; return false;
} }
try {
taskFileJobDispatchExecutor.execute(() -> { taskFileJobDispatchExecutor.execute(() -> {
try { try {
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId); TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
@@ -49,5 +52,10 @@ public class TaskFileJobLocalDispatcher {
} }
}); });
return true; return true;
} catch (RuntimeException ex) {
log.warn("[task-file-job] local dispatch executor rejected jobId={} taskId={} moduleType={} msg={}",
jobId, taskId, moduleType, ex.getMessage(), ex);
return false;
}
} }
} }

19
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java
View File

@@ -53,7 +53,7 @@ public class TaskFileJobService {
if (existing == null) { if (existing == null) {
return null; return null;
} }
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus())) { if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus()) || "PENDING".equals(existing.getStatus())) {
return existing; return existing;
} }
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>() taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
@@ -192,6 +192,11 @@ public class TaskFileJobService {
return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT); return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
} }
public boolean hasSuccessfulAssembleJob(Long taskId, String moduleType, Long resultId) {
TaskFileJobEntity job = findAssembleJob(taskId, moduleType, resultId);
return job != null && "SUCCESS".equals(job.getStatus());
}
public Map<Long, TaskFileJobEntity> findAssembleJobsByResultIds(String moduleType, List<Long> resultIds) { public Map<Long, TaskFileJobEntity> findAssembleJobsByResultIds(String moduleType, List<Long> resultIds) {
List<Long> normalizedIds = resultIds == null ? List.of() : resultIds.stream() List<Long> normalizedIds = resultIds == null ? List.of() : resultIds.stream()
.filter(id -> id != null && id > 0) .filter(id -> id != null && id > 0)
@@ -225,6 +230,18 @@ public class TaskFileJobService {
return count == null ? 0L : count; return count == null ? 0L : count;
} }
public long countActiveAssembleJobs(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
return 0L;
}
Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getTaskId, taskId)
.eq(TaskFileJobEntity::getModuleType, moduleType)
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "RUNNING")));
return count == null ? 0L : count;
}
@Transactional @Transactional
public void deleteTaskJobs(Long taskId, String moduleType) { public void deleteTaskJobs(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) { if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {

56
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java
View File

@@ -8,12 +8,16 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService; import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService; import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService; import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -34,8 +38,12 @@ public class TaskResultFileJobWorker {
private final QueryAsinTaskService queryAsinTaskService; private final QueryAsinTaskService queryAsinTaskService;
private final PatrolDeleteTaskService patrolDeleteTaskService; private final PatrolDeleteTaskService patrolDeleteTaskService;
private final AppearancePatentTaskService appearancePatentTaskService; private final AppearancePatentTaskService appearancePatentTaskService;
private final SimilarAsinTaskService similarAsinTaskService;
private final DeleteBrandRunService deleteBrandRunService; private final DeleteBrandRunService deleteBrandRunService;
private final BrandTaskService brandTaskService; private final BrandTaskService brandTaskService;
@Autowired
@Qualifier("cozeTaskExecutor")
private TaskExecutor cozeTaskExecutor;
@Value("${aiimage.result-file-job.local-worker-enabled:true}") @Value("${aiimage.result-file-job.local-worker-enabled:true}")
private boolean localWorkerEnabled; private boolean localWorkerEnabled;
@@ -80,9 +88,29 @@ public class TaskResultFileJobWorker {
if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) { if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) {
return; return;
} }
if ("APPEARANCE_PATENT".equals(job.getModuleType()) || "SIMILAR_ASIN".equals(job.getModuleType())) {
try {
cozeTaskExecutor.execute(() -> processInternal(job));
return;
} catch (RuntimeException ex) {
log.warn("[task-file-job] coze module offload failed, fallback inline jobId={} taskId={} moduleType={} msg={}",
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex);
}
}
processInternal(job);
}
private void processInternal(TaskFileJobEntity job) {
long startedAt = System.currentTimeMillis(); long startedAt = System.currentTimeMillis();
try { try {
dispatch(job); boolean completed = dispatch(job);
if (!completed) {
taskFileJobService.touchRunning(job.getId());
log.info("[task-file-job] process deferred jobId={} taskId={} moduleType={} resultId={} elapsedMs={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
System.currentTimeMillis() - startedAt);
return;
}
String resultFileUrl = resolveResultFileUrl(job); String resultFileUrl = resolveResultFileUrl(job);
taskFileJobService.markSuccess(job, resultFileUrl); taskFileJobService.markSuccess(job, resultFileUrl);
cleanupAfterSuccess(job); cleanupAfterSuccess(job);
@@ -108,39 +136,41 @@ public class TaskResultFileJobWorker {
return result == null ? null : result.getResultFileUrl(); return result == null ? null : result.getResultFileUrl();
} }
private void dispatch(TaskFileJobEntity job) { private boolean dispatch(TaskFileJobEntity job) {
String moduleType = job.getModuleType(); String moduleType = job.getModuleType();
if ("SHOP_MATCH".equals(moduleType)) { if ("SHOP_MATCH".equals(moduleType)) {
shopMatchTaskService.processResultFileJob(job); shopMatchTaskService.processResultFileJob(job);
return; return true;
} }
if ("PRICE_TRACK".equals(moduleType)) { if ("PRICE_TRACK".equals(moduleType)) {
priceTrackTaskService.processResultFileJob(job); priceTrackTaskService.processResultFileJob(job);
return; return true;
} }
if ("PRODUCT_RISK_RESOLVE".equals(moduleType)) { if ("PRODUCT_RISK_RESOLVE".equals(moduleType)) {
productRiskTaskService.processResultFileJob(job); productRiskTaskService.processResultFileJob(job);
return; return true;
} }
if ("QUERY_ASIN".equals(moduleType)) { if ("QUERY_ASIN".equals(moduleType)) {
queryAsinTaskService.processResultFileJob(job); queryAsinTaskService.processResultFileJob(job);
return; return true;
} }
if ("PATROL_DELETE".equals(moduleType)) { if ("PATROL_DELETE".equals(moduleType)) {
patrolDeleteTaskService.processResultFileJob(job); patrolDeleteTaskService.processResultFileJob(job);
return; return true;
} }
if ("APPEARANCE_PATENT".equals(moduleType)) { if ("APPEARANCE_PATENT".equals(moduleType)) {
appearancePatentTaskService.processResultFileJob(job); return appearancePatentTaskService.processResultFileJob(job);
return; }
if ("SIMILAR_ASIN".equals(moduleType)) {
return similarAsinTaskService.processResultFileJob(job);
} }
if ("DELETE_BRAND".equals(moduleType)) { if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.processResultFileJob(job); deleteBrandRunService.processResultFileJob(job);
return; return true;
} }
if ("BRAND".equals(moduleType)) { if ("BRAND".equals(moduleType)) {
brandTaskService.processResultFileJob(job); brandTaskService.processResultFileJob(job);
return; return true;
} }
throw new IllegalArgumentException("unsupported result file job module: " + moduleType); throw new IllegalArgumentException("unsupported result file job module: " + moduleType);
} }
@@ -159,6 +189,10 @@ public class TaskResultFileJobWorker {
appearancePatentTaskService.cleanupResultFileJob(job); appearancePatentTaskService.cleanupResultFileJob(job);
return; return;
} }
if ("SIMILAR_ASIN".equals(moduleType)) {
similarAsinTaskService.cleanupResultFileJob(job);
return;
}
if ("DELETE_BRAND".equals(moduleType)) { if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.cleanupResultFileJob(job); deleteBrandRunService.cleanupResultFileJob(job);
} }

21
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java
View File

@@ -115,6 +115,17 @@ public class TaskResultItemService {
.eq(TaskResultItemEntity::getResultId, resultId)); .eq(TaskResultItemEntity::getResultId, resultId));
} }
@Transactional
public void deleteResultItemRowsOnly(Long taskId, String moduleType, Long resultId) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0) {
return;
}
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.eq(TaskResultItemEntity::getResultId, resultId));
}
@Transactional @Transactional
public void deleteTaskItems(Long taskId, String moduleType) { public void deleteTaskItems(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) { if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
@@ -134,6 +145,16 @@ public class TaskResultItemService {
.eq(TaskResultItemEntity::getModuleType, moduleType)); .eq(TaskResultItemEntity::getModuleType, moduleType));
} }
@Transactional
public void deleteTaskItemsRowsOnly(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return;
}
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType));
}
private void upsert(Long taskId, private void upsert(Long taskId,
String moduleType, String moduleType,
Long resultId, Long resultId,

10
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java
View File

@@ -188,6 +188,16 @@ public class TaskScopePayloadStorageService {
} }
} }
@Transactional
public void deleteTaskScopePayloadRowsOnly(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return;
}
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType));
}
@Transactional @Transactional
public <T> void saveParsedPayloadMap(Long taskId, String moduleType, Map<String, T> payloadByScope) { public <T> void saveParsedPayloadMap(Long taskId, String moduleType, Map<String, T> payloadByScope) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || payloadByScope == null || payloadByScope.isEmpty()) { if (taskId == null || taskId <= 0 || isBlank(moduleType) || payloadByScope == null || payloadByScope.isEmpty()) {

Some files were not shown because too many files have changed in this diff Show More