From 111f30d150c51832834116fd9f91257e2a680777 Mon Sep 17 00:00:00 2001 From: super <2903208875@qq.com> Date: Mon, 4 May 2026 19:04:55 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BC=82=E5=B8=B8=E8=AE=B0=E5=BD=95=E6=A3=80?= =?UTF-8?q?=E6=B5=8B=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/.env | 4 +- app/amazon/detail_spider.py | 14 +- app/amazon/main.py | 9 +- .../common/util/DownloadHeaderUtil.java | 73 + .../aiimage/config/PropertiesConfig.java | 2 +- .../aiimage/config/SimilarAsinProperties.java | 21 + .../aiimage/config/StorageProperties.java | 1 + .../aiimage/config/TaskFileJobConfig.java | 16 + .../client/AppearancePatentCozeClient.java | 107 +- .../AppearancePatentController.java | 7 +- .../dto/AppearancePatentResultRowDto.java | 8 + .../service/AppearancePatentTaskService.java | 512 +++- .../brand/controller/BrandTaskController.java | 8 +- .../controller/ConvertRunController.java | 20 +- .../convert/service/ConvertRunService.java | 37 +- .../controller/DeleteBrandRunController.java | 50 +- .../service/DeleteBrandRunService.java | 386 ++- .../service/DeleteBrandStaleTaskService.java | 45 +- .../file/service/LocalTempCleanupService.java | 20 +- .../controller/PatrolDeleteController.java | 7 +- .../controller/PriceTrackController.java | 7 +- .../service/PriceTrackTaskService.java | 56 +- .../ProductRiskResolveController.java | 7 +- .../service/ProductRiskTaskService.java | 56 +- .../controller/QueryAsinTaskController.java | 7 +- .../service/QueryAsinTaskService.java | 79 +- .../controller/ShopMatchController.java | 7 +- .../service/ShopMatchTaskService.java | 56 +- .../client/SimilarAsinCozeClient.java | 990 +++++++ .../controller/SimilarAsinController.java | 145 + .../model/dto/SimilarAsinParseRequest.java | 28 + .../dto/SimilarAsinParsedPayloadDto.java | 31 + .../model/dto/SimilarAsinResultGroupDto.java | 29 + .../model/dto/SimilarAsinResultRowDto.java | 187 ++ .../model/dto/SimilarAsinSourceFileDto.java | 15 + .../dto/SimilarAsinSubmitResultRequest.java | 32 + .../dto/SimilarAsinTaskBatchRequest.java | 15 + .../model/vo/SimilarAsinDashboardVo.java | 17 + .../model/vo/SimilarAsinHistoryItemVo.java | 37 + .../model/vo/SimilarAsinHistoryVo.java | 14 + .../model/vo/SimilarAsinParseVo.java | 41 + .../model/vo/SimilarAsinParsedGroupVo.java | 32 + .../model/vo/SimilarAsinParsedRowVo.java | 47 + .../model/vo/SimilarAsinTaskBatchVo.java | 16 + .../model/vo/SimilarAsinTaskDetailVo.java | 16 + .../model/vo/SimilarAsinTaskItemVo.java | 23 + .../service/SimilarAsinTaskCacheService.java | 144 + .../service/SimilarAsinTaskService.java | 2413 +++++++++++++++++ .../split/controller/SplitRunController.java | 16 +- .../split/service/SplitRunService.java | 13 +- .../model/entity/TaskScopeStateEntity.java | 7 + .../service/TaskFileJobLocalDispatcher.java | 50 +- .../task/service/TaskFileJobService.java | 19 +- .../task/service/TaskResultFileJobWorker.java | 56 +- .../TransientPayloadStorageService.java | 115 +- .../model/cache/ZiniaoShopIndexEntryDto.java | 2 + .../service/ZiniaoShopIndexService.java | 52 + .../resources/application-local.example.yml | 8 + .../src/main/resources/application.yml | 12 + .../db/V43__appearance_patent_coze_async.sql | 127 + ...44__delete_brand_history_order_indexes.sql | 15 + .../main/resources/db/V45__similar_asin.sql | 14 + frontend-vue/similar-asin.html | 12 + .../brand/components/BrandDeleteBrandTab.vue | 79 +- .../brand/components/BrandSimilarAsinTab.vue | 764 ++++++ .../pages/brand/components/BrandTopBar.vue | 2 + frontend-vue/src/shared/api/java-modules.ts | 168 ++ frontend-vue/src/similar-asin-main.ts | 7 + frontend-vue/vite.config.ts | 1 + version.txt | 1 - 70 files changed, 7182 insertions(+), 252 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/common/util/DownloadHeaderUtil.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinCozeClient.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/controller/SimilarAsinController.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParseRequest.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParsedPayloadDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinResultGroupDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinResultRowDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinSourceFileDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinSubmitResultRequest.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinTaskBatchRequest.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinDashboardVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParseVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParsedGroupVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParsedRowVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskBatchVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskDetailVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskItemVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskCacheService.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java create mode 100644 backend-java/src/main/resources/db/V43__appearance_patent_coze_async.sql create mode 100644 backend-java/src/main/resources/db/V44__delete_brand_history_order_indexes.sql create mode 100644 backend-java/src/main/resources/db/V45__similar_asin.sql create mode 100644 frontend-vue/similar-asin.html create mode 100644 frontend-vue/src/pages/brand/components/BrandSimilarAsinTab.vue create mode 100644 frontend-vue/src/similar-asin-main.ts delete mode 100644 version.txt diff --git a/app/.env b/app/.env index ffc438a..20a1f7e 100644 --- a/app/.env +++ b/app/.env @@ -11,8 +11,8 @@ zn_username=%E8%87%AA%E5%8A%A8%E5%8C%96_Robot client_name=ShuFuAI -java_api_base=http://47.111.163.154:18080 -# java_api_base=http://127.0.0.1:18080 +# java_api_base=http://47.111.163.154:18080 +java_api_base=http://127.0.0.1:18080 # java_api_base=http://8.136.19.173:18080 # java_api_base=http://121.196.149.225:18080 diff --git a/app/amazon/detail_spider.py b/app/amazon/detail_spider.py index ab5027e..ce409c7 100644 --- a/app/amazon/detail_spider.py +++ b/app/amazon/detail_spider.py @@ -277,9 +277,10 @@ class SpiderTask: Args: user_info: 用户信息字典，包含 company, username, password - """ - self.user_info = user_info or {} - self.running = True + """ + self.user_info = user_info or {} + self.running = True + self.result_api_module = "appearance-patent" def log(self, message: str, level: str = "INFO"): timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -339,8 +340,9 @@ class SpiderTask: task_data: 任务数据 """ try: - data = task_data.get("data", {}) - task_id = data.get("taskId") + data = task_data.get("data", {}) + self.result_api_module = "similar-asin" if task_data.get("type") == "similar-asin-run" else "appearance-patent" + task_id = data.get("taskId") groups = self.normalize_groups(data) # 用于测试 @@ -485,7 +487,7 @@ class SpiderTask: """回传处理结果到API """ - url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result" + url = f"{DELETE_BRAND_API_BASE}/api/{self.result_api_module}/tasks/{task_id}/result" payload ={ "submissionId": f"{int(time.time())}", diff --git a/app/amazon/main.py b/app/amazon/main.py index fd5864c..10ccf3b 100644 --- a/app/amazon/main.py +++ b/app/amazon/main.py @@ -63,10 +63,11 @@ class TaskMonitor: "product-risk-resolve-run" : "产品风险审批", "shop-match-run" : "匹配价格", "price-track-run" : "跟价", - "query-asin-run" : "状态查询", - "patrol-delete-run" : "巡店删除", - "appearance-patent-run" : "亚马逊采集", - } + "query-asin-run" : "状态查询", + "patrol-delete-run" : "巡店删除", + "appearance-patent-run" : "亚马逊采集", + "similar-asin-run" : "亚马逊采集", + } try: while self.running: try: diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/util/DownloadHeaderUtil.java b/backend-java/src/main/java/com/nanri/aiimage/common/util/DownloadHeaderUtil.java new file mode 100644 index 0000000..457e662 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/util/DownloadHeaderUtil.java @@ -0,0 +1,73 @@ +package com.nanri.aiimage.common.util; + +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.http.HttpHeaders; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; + +public final class DownloadHeaderUtil { + + private static final String DEFAULT_FILENAME = "download"; + + private DownloadHeaderUtil() { + } + + public static void setAttachment(HttpServletResponse response, String filename) { + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, contentDisposition(filename)); + } + + public static String contentDisposition(String filename) { + String normalized = normalizeFilename(filename); + String encodedFilename = URLEncoder.encode(normalized, StandardCharsets.UTF_8).replace("+", "%20"); + return "attachment; filename=\"" + asciiFallback(normalized) + "\"; filename*=UTF-8''" + encodedFilename; + } + + private static String normalizeFilename(String filename) { + if (filename == null || filename.isBlank()) { + return DEFAULT_FILENAME; + } + return filename.replace("\r", "").replace("\n", "").trim(); + } + + private static String asciiFallback(String filename) { + String normalized = normalizeFilename(filename); + String extension = extensionOf(normalized); + String base = extension.isEmpty() ? normalized : normalized.substring(0, normalized.length() - extension.length() - 1); + String sanitizedBase = sanitizeAscii(base); + String sanitizedExtension = sanitizeExtension(extension); + if (sanitizedBase.isBlank()) { + sanitizedBase = DEFAULT_FILENAME; + } + String fallback = sanitizedExtension.isBlank() ? sanitizedBase : sanitizedBase + "." + sanitizedExtension; + return fallback.length() <= 120 ? fallback : shorten(fallback, sanitizedExtension); + } + + private static String extensionOf(String filename) { + int dotIndex = filename.lastIndexOf('.'); + if (dotIndex < 0 || dotIndex == filename.length() - 1) { + return ""; + } + return filename.substring(dotIndex + 1); + } + + private static String sanitizeAscii(String value) { + return value.replaceAll("[^A-Za-z0-9._-]", "_") + .replaceAll("_+", "_") + .replaceAll("^[_\\.]+|[_\\.]+$", ""); + } + + private static String sanitizeExtension(String extension) { + return extension.replaceAll("[^A-Za-z0-9]", ""); + } + + private static String shorten(String fallback, String extension) { + int maxBaseLength = extension.isBlank() ? 120 : Math.max(1, 119 - extension.length()); + String base = extension.isBlank() ? fallback : fallback.substring(0, fallback.length() - extension.length() - 1); + base = base.substring(0, Math.min(base.length(), maxBaseLength)).replaceAll("[_\\.]+$", ""); + if (base.isBlank()) { + base = DEFAULT_FILENAME; + } + return extension.isBlank() ? base : base + "." + extension; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java index 5d1e0d5..52f1837 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java @@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Configuration; @Configuration -@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class}) +@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class}) public class PropertiesConfig { } diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java new file mode 100644 index 0000000..711fca0 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java @@ -0,0 +1,21 @@ +package com.nanri.aiimage.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "aiimage.similar-asin") +public class SimilarAsinProperties { + private String cozeBaseUrl = "https://api.coze.cn"; + private String cozeWorkflowPath = "/v1/workflow/run"; + private String cozeWorkflowHistoryPath = "/v1/workflows/{workflow_id}/run_histories/{execute_id}"; + private String cozeWorkflowId = "7632683471312355338"; + private String cozeToken = ""; + private int cozeBatchSize = 10; + private int cozeConnectTimeoutMillis = 10000; + private int cozeReadTimeoutMillis = 60000; + private int cozePollIntervalMillis = 2000; + private int cozePollTimeoutMillis = 120000; + private int staleTimeoutMinutes = 20; + private String staleFinalizeCron = "0 */2 * * * *"; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java index 2fa7509..df040c3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java @@ -11,4 +11,5 @@ public class StorageProperties { private String cleanupCron = "0 0 * * * *"; private long sourceRetentionHours = 24; private long resultRetentionHours = 24; + private long transientPayloadRetentionHours = 72; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java index 6cee12c..fdbbcfb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java @@ -4,8 +4,12 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.task.TaskExecutor; +import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + @Configuration public class TaskFileJobConfig { @@ -24,4 +28,16 @@ public class TaskFileJobConfig { executor.initialize(); return executor; } + + @Bean(destroyMethod = "shutdown") + public ExecutorService cozeVirtualThreadExecutor() { + return Executors.newThreadPerTaskExecutor(Thread.ofVirtual() + .name("coze-task-", 0) + .factory()); + } + + @Bean("cozeTaskExecutor") + public TaskExecutor cozeTaskExecutor(ExecutorService cozeVirtualThreadExecutor) { + return new ConcurrentTaskExecutor(cozeVirtualThreadExecutor); + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClient.java index cc94432..f14f6a6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClient.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClient.java @@ -44,6 +44,42 @@ public class AppearancePatentCozeClient { } } + public CozeSubmitResponse submitWorkflow(List rows, String prompt) throws Exception { + JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt)); + ensureSuccess(submitRoot); + return new CozeSubmitResponse( + extractExecuteId(submitRoot), + extractResultDataText(submitRoot), + writeJson(submitRoot) + ); + } + + public CozePollResponse pollWorkflow(String executeId) throws Exception { + JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId)); + ensureSuccess(pollRoot); + String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT); + String dataText = extractResultDataText(pollRoot); + String outputText = dataText.isBlank() ? extractWorkflowOutputText(pollRoot) : ""; + String failureMessage = isFailedWorkflowStatus(status) + ? firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed") + : ""; + return new CozePollResponse(executeId, status, dataText, outputText, failureMessage, writeJson(pollRoot)); + } + + public List mergeRowsFromDataText(List rows, String dataText) throws Exception { + if (dataText == null || dataText.isBlank()) { + return rows == null ? List.of() : rows.stream().map(this::copy).toList(); + } + return mergeRows(rows, parseResults(wrapDataPayload(dataText))); + } + + public List markRowsFailed(List rows, String failureMessage) { + if (rows == null || rows.isEmpty()) { + return List.of(); + } + return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList(); + } + private List inspectWithFallback(List rows, String prompt) { try { if (rows.size() == 1) { @@ -140,6 +176,7 @@ public class AppearancePatentCozeClient { long deadline = System.currentTimeMillis() + Math.max(10000, properties.getCozePollTimeoutMillis()); while (System.currentTimeMillis() < deadline) { + ensureNotInterrupted(); JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId)); ensureSuccess(pollRoot); @@ -430,9 +467,21 @@ public class AppearancePatentCozeClient { } private AppearancePatentResultRowDto markFailed(AppearancePatentResultRowDto row, String failureMessage) { + String reviewMessage = failureMessage == null || failureMessage.isBlank() + ? "Coze 检测失败，待人工复核" + : "Coze 检测失败，待人工复核：" + failureMessage; if (row.getError() == null || row.getError().isBlank()) { row.setError(failureMessage); } + if (row.getTitleRisk() == null || row.getTitleRisk().isBlank()) { + row.setTitleRisk(reviewMessage); + } + if (row.getAppearanceRisk() == null || row.getAppearanceRisk().isBlank()) { + row.setAppearanceRisk(reviewMessage); + } + if (row.getPatentRisk() == null || row.getPatentRisk().isBlank()) { + row.setPatentRisk(reviewMessage); + } if (row.getConclusion() == null || row.getConclusion().isBlank()) { row.setConclusion(failureMessage); } @@ -515,15 +564,22 @@ public class AppearancePatentCozeClient { return ""; } if (!outputNode.isTextual()) { - return outputNode.toString(); + String discovered = discoverEmbeddedData(outputNode); + return discovered.isBlank() && isResultDataPayload(outputNode) ? outputNode.toString() : discovered; } String output = normalize(outputNode.asText("")); if (output.isBlank()) { return ""; } + if (looksLikeResultDataPayload(output)) { + return output; + } JsonNode parsedOutput = parseJsonOrMissing(output); String nestedOutput = text(firstNonNull(parsedOutput.get("Output"), parsedOutput.get("output"))); - return nestedOutput == null || nestedOutput.isBlank() ? output : nestedOutput; + if (nestedOutput != null && !nestedOutput.isBlank() && looksLikeResultDataPayload(nestedOutput)) { + return nestedOutput; + } + return discoverEmbeddedData(parsedOutput); } private String discoverEmbeddedData(JsonNode node) { @@ -749,6 +805,7 @@ public class AppearancePatentCozeClient { private void sleepBeforeRetry(int attemptIndex) { long delayMillis = Math.max(1, attemptIndex) * 1500L; + ensureNotInterrupted(); sleepQuietly(delayMillis); } @@ -757,6 +814,13 @@ public class AppearancePatentCozeClient { Thread.sleep(delayMillis); } catch (InterruptedException interruptedException) { Thread.currentThread().interrupt(); + throw new IllegalStateException("Coze workflow interrupted", interruptedException); + } + } + + private void ensureNotInterrupted() { + if (Thread.currentThread().isInterrupted()) { + throw new IllegalStateException("Coze workflow interrupted"); } } @@ -855,6 +919,45 @@ public class AppearancePatentCozeClient { ) { } + public record CozeSubmitResponse( + String executeId, + String immediateData, + String rawResponse + ) { + } + + public record CozePollResponse( + String executeId, + String status, + String dataText, + String outputText, + String failureMessage, + String rawResponse + ) { + public boolean hasPayload() { + return dataText != null && !dataText.isBlank() || outputText != null && !outputText.isBlank(); + } + + public String resolvedPayloadText() { + return dataText != null && !dataText.isBlank() ? dataText : outputText; + } + + public boolean isFailed() { + String normalized = status == null ? "" : status.trim().toUpperCase(Locale.ROOT); + return normalized.contains("FAILED") || normalized.contains("ERROR") || normalized.contains("CANCEL"); + } + + public boolean isFinished() { + String normalized = status == null ? "" : status.trim().toUpperCase(Locale.ROOT); + return isFailed() + || normalized.contains("SUCCESS") + || normalized.contains("SUCCEED") + || normalized.contains("FINISH") + || normalized.contains("DONE") + || normalized.contains("COMPLET"); + } + } + private static final class PartialCozeResultException extends RuntimeException { private final int resolvedCount; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java index b4c705a..34632a3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.appearancepatent.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.util.DownloadHeaderUtil; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest; import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentTaskBatchRequest; @@ -14,7 +15,6 @@ import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -28,7 +28,6 @@ import org.springframework.web.server.ResponseStatusException; import java.io.InputStream; import java.net.URI; -import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @RestController @@ -129,10 +128,8 @@ public class AppearancePatentController { throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); } try { - String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20"); response.setContentType("application/octet-stream"); - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename); + DownloadHeaderUtil.setAttachment(response, filename); try (InputStream in = URI.create(url).toURL().openStream()) { byte[] buffer = new byte[65536]; int read; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultRowDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultRowDto.java index f9f43db..ab77eb2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultRowDto.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultRowDto.java @@ -29,9 +29,17 @@ public class AppearancePatentResultRowDto { private String country; @Schema(description = "商品主图或待检测图片 URL。Java 调用 Coze 时会放入 url_list。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg") + @JsonAlias({ + "imageUrl", "image_url", "imgUrl", "img_url", "pictureUrl", "picture_url", + "mainImage", "main_image", "mainImageUrl", "main_image_url", + "productImage", "product_image", "productImageUrl", "product_image_url", + "image", "img", "pic", "picture", "link", "imageLink", "image_link", + "图片链接", "商品图片", "商品主图", "主图", "主图链接" + }) private String url; @Schema(description = "商品标题。Java 调用 Coze 时会放入 title_list；为空时会回退使用 ASIN。", example = "Women Floral Dress Summer Casual") + @JsonAlias({"productTitle", "product_title", "itemTitle", "item_title", "商品标题", "商品名称", "标题"}) private String title; @Schema(description = "单行错误信息。通常用于记录 Python 单行处理异常；Coze 失败时后端会尽量保留原始行，不强行写入风险结果。", example = "图片地址为空") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index 6bdd3f9..1a8794e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -3,9 +3,11 @@ package com.nanri.aiimage.modules.appearancepatent.service; import cn.hutool.core.util.IdUtil; import cn.hutool.crypto.digest.DigestUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.service.DistributedJobLockService; import com.nanri.aiimage.config.AppearancePatentProperties; import com.nanri.aiimage.config.StorageProperties; import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentCozeClient; @@ -50,7 +52,10 @@ import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.WorkbookFactory; import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DuplicateKeyException; +import org.springframework.core.task.TaskExecutor; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.transaction.PlatformTransactionManager; @@ -73,6 +78,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.UUID; import java.util.function.Supplier; @Service @@ -85,6 +91,10 @@ public class AppearancePatentTaskService { private static final String STATUS_RUNNING = "RUNNING"; private static final String STATUS_SUCCESS = "SUCCESS"; private static final String STATUS_FAILED = "FAILED"; + private static final String COZE_STATUS_SUBMITTED = "SUBMITTED"; + private static final String COZE_STATUS_RUNNING = "RUNNING"; + private static final String COZE_STATUS_DONE = "DONE"; + private static final String COZE_STATUS_FAILED = "FAILED"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3; private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L; @@ -114,6 +124,10 @@ public class AppearancePatentTaskService { private final TaskProgressSnapshotService taskProgressSnapshotService; private final TransientPayloadStorageService transientPayloadStorageService; private final PlatformTransactionManager transactionManager; + private final DistributedJobLockService distributedJobLockService; + @Autowired + @Qualifier("cozeTaskExecutor") + private TaskExecutor cozeTaskExecutor; public AppearancePatentParseVo parseAndCreateTask(AppearancePatentParseRequest request) { long startedAt = System.nanoTime(); @@ -514,6 +528,10 @@ public class AppearancePatentTaskService { .lt(FileTaskEntity::getUpdatedAt, threshold) .last("limit 50")); for (FileTaskEntity task : tasks) { + if (isJavaSideProcessing(task.getId())) { + touchJavaSideTaskActivity(task.getId()); + continue; + } long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId()); if (heartbeatMillis > thresholdMillis) { continue; @@ -533,6 +551,10 @@ public class AppearancePatentTaskService { .lt(FileTaskEntity::getUpdatedAt, threshold) .last("limit 50")); for (FileTaskEntity task : tasks) { + if (isJavaSideProcessing(task.getId())) { + touchJavaSideTaskActivity(task.getId()); + continue; + } long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId()); if (heartbeatMillis > thresholdMillis) { continue; @@ -1004,7 +1026,7 @@ public class AppearancePatentTaskService { assembleResultWorkbook(task, result); } catch (Exception ex) { log.warn("[appearance-patent] assemble result workbook failed taskId={} err={}", task.getId(), ex.getMessage()); - finalError = firstNonBlank(finalError, "生成外观专利检测结果失败"); + finalError = firstNonBlank(finalError, "Generate appearance patent result failed"); } } } @@ -1038,39 +1060,478 @@ public class AppearancePatentTaskService { return false; } - public void processResultFileJob(TaskFileJobEntity job) { + public boolean processResultFileJob(TaskFileJobEntity job) { if (job == null || job.getTaskId() == null || job.getResultId() == null) { - throw new BusinessException("结果文件任务参数不完整"); + throw new BusinessException("result file job arguments are incomplete"); } FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId()); if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { - throw new BusinessException("任务不存在"); + throw new BusinessException("task not found"); } FileResultEntity result = fileResultMapper.selectById(job.getResultId()); if (result == null || !MODULE_TYPE.equals(result.getModuleType())) { - throw new BusinessException("结果记录不存在"); + throw new BusinessException("result record not found"); } List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() .eq(TaskChunkEntity::getTaskId, task.getId()) .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) .orderByAsc(TaskChunkEntity::getChunkIndex)); + Map> allRowsByBaseId = loadAllRowsByBaseId(task); int cozeWorkUnits = countCozeWorkUnits(chunks, Math.max(1, properties.getCozeBatchSize())); int totalProgressUnits = Math.max(3, cozeWorkUnits + 3); - int[] completedProgressUnits = {0}; - saveFileBuildProgress(task, job, totalProgressUnits, completedProgressUnits[0], "正在请求 Coze"); - Runnable progressHook = () -> { + saveFileBuildProgress(task, job, totalProgressUnits, 0, "Submitting Coze"); + boolean pendingCoze = submitCozeBatches(task, result, job, chunks, allRowsByBaseId); + if (pendingCoze) { taskFileJobService.touchRunning(job.getId()); - completedProgressUnits[0] = Math.min(totalProgressUnits - 2, completedProgressUnits[0] + 1); - saveFileBuildProgress(task, job, totalProgressUnits, completedProgressUnits[0], "正在请求 Coze"); - }; - applyCozeToPersistedChunks(task, progressHook); - completedProgressUnits[0] = Math.max(completedProgressUnits[0], totalProgressUnits - 2); - saveFileBuildProgress(task, job, totalProgressUnits, completedProgressUnits[0], "正在组装 xlsx"); + touchJavaSideTaskActivity(task.getId()); + saveFileBuildProgress(task, job, totalProgressUnits, 1, "Coze submitted, waiting for result"); + return false; + } + completeCozeFileJob(task, result, job, totalProgressUnits, cozeWorkUnits); + return true; + } + + @Scheduled(fixedDelayString = "${aiimage.appearance-patent.coze-poll-delay-ms:5000}") + public void pollPendingCozeJobs() { + DistributedJobLockService.LockHandle lockHandle = + distributedJobLockService.tryLock("appearance-patent:coze-poll", Duration.ofMinutes(1)); + if (lockHandle == null) { + return; + } + try (lockHandle) { + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING)) + .isNotNull(TaskScopeStateEntity::getCozeExecuteId) + .orderByAsc(TaskScopeStateEntity::getUpdatedAt) + .last("limit 50")); + if (states == null || states.isEmpty()) { + return; + } + log.info("[appearance-patent] coze poll picked pending states count={}", states.size()); + for (TaskScopeStateEntity state : states) { + if (state == null || state.getId() == null) { + continue; + } + cozeTaskExecutor.execute(() -> pollPendingCozeState(state.getId())); + } + } + } + + private boolean submitCozeBatches(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + List chunks, + Map> allRowsByBaseId) { + if (chunks == null || chunks.isEmpty()) { + return countPendingCozeStates(task.getId()) > 0; + } + if (properties.getCozeToken() == null || properties.getCozeToken().isBlank()) { + log.warn("[appearance-patent] coze token not configured, skip async coze taskId={} jobId={}", + task.getId(), job.getId()); + return false; + } + String prompt = readAiPrompt(task); + int batchSize = Math.max(1, properties.getCozeBatchSize()); + boolean pending = false; + for (TaskChunkEntity chunk : chunks) { + Map persistedRows = readChunkRows(chunk); + if (persistedRows.isEmpty()) { + continue; + } + List unresolvedRows = pickGroupRepresentativesForCoze(persistedRows.values()); + if (unresolvedRows.isEmpty()) { + continue; + } + int batchTotal = Math.max(1, (unresolvedRows.size() + batchSize - 1) / batchSize); + int batchIndex = 1; + for (int i = 0; i < unresolvedRows.size(); i += batchSize) { + List batchRows = + unresolvedRows.subList(i, Math.min(i + batchSize, unresolvedRows.size())); + pending |= submitCozeBatch(task, result, job, chunk, batchRows, batchIndex, batchTotal, prompt, allRowsByBaseId); + batchIndex++; + } + } + return pending || countPendingCozeStates(task.getId()) > 0; + } + + private boolean submitCozeBatch(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + TaskChunkEntity chunk, + List batchRows, + int batchIndex, + int batchTotal, + String prompt, + Map> allRowsByBaseId) { + if (batchRows == null || batchRows.isEmpty()) { + return false; + } + String batchScopeKey = buildCozeBatchScopeKey(job.getId(), chunk.getScopeHash(), chunk.getChunkIndex(), batchIndex); + String batchScopeHash = DigestUtil.sha256Hex(batchScopeKey); + TaskScopeStateEntity existing = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, task.getId()) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, batchScopeHash) + .last("limit 1")); + if (existing != null) { + return COZE_STATUS_SUBMITTED.equals(existing.getCozeStatus()) + || COZE_STATUS_RUNNING.equals(existing.getCozeStatus()); + } + try { + AppearancePatentCozeClient.CozeSubmitResponse submit = cozeClient.submitWorkflow(batchRows, prompt); + if (submit.immediateData() != null && !submit.immediateData().isBlank()) { + List cozeRows = cozeClient.mergeRowsFromDataText(batchRows, submit.immediateData()); + mergeCozeRowsIntoChunk(task, chunk.getScopeHash(), chunk.getChunkIndex(), cozeRows, allRowsByBaseId); + return false; + } + if (submit.executeId() == null || submit.executeId().isBlank()) { + mergeCozeRowsIntoChunk(task, + chunk.getScopeHash(), + chunk.getChunkIndex(), + cozeClient.markRowsFailed(batchRows, "Coze async execute_id missing"), + allRowsByBaseId); + return false; + } + saveCozeBatchState(task, result, job, chunk, batchRows, batchScopeKey, batchScopeHash, + batchIndex, batchTotal, submit.executeId()); + log.info("[appearance-patent] coze async submitted taskId={} jobId={} chunk={} batch={}/{} executeId={}", + task.getId(), job.getId(), chunk.getChunkIndex(), batchIndex, batchTotal, submit.executeId()); + return true; + } catch (Exception ex) { + String message = firstNonBlank(ex.getMessage(), "Coze submit failed"); + log.warn("[appearance-patent] coze async submit failed taskId={} jobId={} chunk={} batch={}/{} err={}", + task.getId(), job.getId(), chunk.getChunkIndex(), batchIndex, batchTotal, message); + mergeCozeRowsIntoChunk(task, + chunk.getScopeHash(), + chunk.getChunkIndex(), + cozeClient.markRowsFailed(batchRows, message), + allRowsByBaseId); + return false; + } + } + + private void saveCozeBatchState(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + TaskChunkEntity chunk, + List batchRows, + String batchScopeKey, + String batchScopeHash, + int batchIndex, + int batchTotal, + String executeId) { + LocalDateTime now = LocalDateTime.now(); + CozeBatchContext context = new CozeBatchContext( + job.getId(), + result.getId(), + chunk.getScopeHash(), + chunk.getChunkIndex(), + batchIndex, + batchTotal + ); + String batchPayload = writeJson(batchRows, "serialize coze batch payload failed"); + String storedBatchPayload = transientPayloadStorageService.storeParsedPayloadFast( + MODULE_TYPE, task.getId(), batchScopeHash, batchPayload, true); + TaskScopeStateEntity state = new TaskScopeStateEntity(); + state.setTaskId(task.getId()); + state.setModuleType(MODULE_TYPE); + state.setScopeKey(batchScopeKey); + state.setScopeHash(batchScopeHash); + state.setParsedPayloadJson(storedBatchPayload); + state.setStateJson(writeJson(context, "serialize coze batch context failed")); + state.setCozeExecuteId(executeId); + state.setCozeStatus(COZE_STATUS_SUBMITTED); + state.setCozeSubmittedAt(now); + state.setCozeAttemptCount(0); + state.setChunkTotal(batchTotal); + state.setReceivedChunkCount(batchIndex); + state.setCompleted(0); + state.setCreatedAt(now); + state.setUpdatedAt(now); + try { + taskScopeStateMapper.insert(state); + touchJavaSideTaskActivity(task.getId()); + } catch (DuplicateKeyException ex) { + transientPayloadStorageService.deletePayloadIfPresent(storedBatchPayload); + log.info("[appearance-patent] duplicate coze batch state ignored taskId={} scope={}", + task.getId(), batchScopeKey); + } + } + + private void pollPendingCozeState(Long stateId) { + TaskScopeStateEntity state = taskScopeStateMapper.selectById(stateId); + if (state == null || state.getCozeExecuteId() == null || state.getCozeExecuteId().isBlank()) { + return; + } + if (!(COZE_STATUS_SUBMITTED.equals(state.getCozeStatus()) || COZE_STATUS_RUNNING.equals(state.getCozeStatus()))) { + return; + } + if (!tryClaimCozeStateForPoll(state)) { + return; + } + CozeBatchContext context = readCozeBatchContext(state); + if (context == null || context.jobId() == null || context.resultId() == null) { + markCozeStateTerminal(state, COZE_STATUS_FAILED, "Coze batch context missing"); + return; + } + taskFileJobService.touchRunning(context.jobId()); + try { + AppearancePatentCozeClient.CozePollResponse poll = cozeClient.pollWorkflow(state.getCozeExecuteId()); + if (!poll.hasPayload() && !poll.isFinished() && !isCozeStateTimedOut(state)) { + updateCozeStateRunning(state, null); + return; + } + String failureMessage = poll.isFailed() + ? firstNonBlank(poll.failureMessage(), "Coze async workflow failed") + : ""; + if (!poll.hasPayload() && failureMessage.isBlank()) { + failureMessage = isCozeStateTimedOut(state) + ? "Coze async workflow poll timeout" + : "Coze async workflow completed without output"; + } + List batchRows = readCozeBatchRows(state); + List cozeRows = failureMessage.isBlank() + ? cozeClient.mergeRowsFromDataText(batchRows, poll.resolvedPayloadText()) + : cozeClient.markRowsFailed(batchRows, failureMessage); + FileTaskEntity task = fileTaskMapper.selectById(state.getTaskId()); + if (task != null) { + Map> allRowsByBaseId = loadAllRowsByBaseId(task); + mergeCozeRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), cozeRows, allRowsByBaseId); + } + markCozeStateTerminal(state, + failureMessage.isBlank() ? COZE_STATUS_DONE : COZE_STATUS_FAILED, + failureMessage.isBlank() ? null : failureMessage); + maybeFinalizeCozeJob(state.getTaskId(), context); + } catch (Exception ex) { + String message = firstNonBlank(ex.getMessage(), "Coze poll failed"); + if (isCozeStateTimedOut(state)) { + List batchRows = readCozeBatchRows(state); + FileTaskEntity task = fileTaskMapper.selectById(state.getTaskId()); + if (task != null) { + Map> allRowsByBaseId = loadAllRowsByBaseId(task); + mergeCozeRowsIntoChunk(task, + context.chunkScopeHash(), + context.chunkIndex(), + cozeClient.markRowsFailed(batchRows, message), + allRowsByBaseId); + } + markCozeStateTerminal(state, COZE_STATUS_FAILED, message); + maybeFinalizeCozeJob(state.getTaskId(), context); + return; + } + log.warn("[appearance-patent] coze poll failed taskId={} stateId={} executeId={} err={}", + state.getTaskId(), state.getId(), state.getCozeExecuteId(), message); + updateCozeStateRunning(state, message); + } + } + + private void updateCozeStateRunning(TaskScopeStateEntity state, String error) { + int updated = taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING)) + .set(TaskScopeStateEntity::getCozeStatus, COZE_STATUS_RUNNING) + .set(TaskScopeStateEntity::getCozeLastPolledAt, LocalDateTime.now()) + .set(TaskScopeStateEntity::getCozeAttemptCount, cozeAttemptCount(state) + 1) + .set(TaskScopeStateEntity::getCozeError, error) + .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); + if (updated > 0) { + touchJavaSideTaskActivity(state.getTaskId()); + } + } + + private boolean tryClaimCozeStateForPoll(TaskScopeStateEntity state) { + if (state == null || state.getId() == null) { + return false; + } + LocalDateTime now = LocalDateTime.now(); + long intervalMillis = Math.max(200L, properties.getCozePollIntervalMillis()); + if (state.getCozeLastPolledAt() != null + && Duration.between(state.getCozeLastPolledAt(), now).toMillis() < intervalMillis) { + return false; + } + return taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING)) + .and(wrapper -> wrapper + .isNull(TaskScopeStateEntity::getCozeLastPolledAt) + .or() + .le(TaskScopeStateEntity::getCozeLastPolledAt, now.minus(Duration.ofMillis(intervalMillis)))) + .set(TaskScopeStateEntity::getCozeStatus, COZE_STATUS_RUNNING) + .set(TaskScopeStateEntity::getCozeLastPolledAt, now) + .set(TaskScopeStateEntity::getUpdatedAt, now)) > 0; + } + + private void markCozeStateTerminal(TaskScopeStateEntity state, String status, String error) { + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING)) + .set(TaskScopeStateEntity::getCozeStatus, status) + .set(TaskScopeStateEntity::getCozeCompletedAt, LocalDateTime.now()) + .set(TaskScopeStateEntity::getCozeLastPolledAt, LocalDateTime.now()) + .set(TaskScopeStateEntity::getCozeAttemptCount, cozeAttemptCount(state) + 1) + .set(TaskScopeStateEntity::getCozeError, error) + .set(TaskScopeStateEntity::getCompleted, 1) + .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); + } + + private void maybeFinalizeCozeJob(Long taskId, CozeBatchContext context) { + if (taskId == null || context == null || countPendingCozeStates(taskId) > 0) { + return; + } + DistributedJobLockService.LockHandle lockHandle = + distributedJobLockService.tryLock("appearance-patent:coze-finalize:" + taskId, Duration.ofMinutes(5)); + if (lockHandle == null) { + return; + } + try (lockHandle) { + if (countPendingCozeStates(taskId) > 0) { + return; + } + FileTaskEntity task = fileTaskMapper.selectById(taskId); + FileResultEntity result = fileResultMapper.selectById(context.resultId()); + TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId()); + if (task == null || result == null || job == null || "SUCCESS".equals(job.getStatus())) { + return; + } + int cozeWorkUnits = countCompletedCozeStates(taskId); + int totalProgressUnits = Math.max(3, cozeWorkUnits + 3); + completeCozeFileJob(task, result, job, totalProgressUnits, cozeWorkUnits); + taskFileJobService.markSuccess(job, result.getResultFileUrl()); + cleanupResultFileJob(job); + log.info("[appearance-patent] coze async job finalized taskId={} jobId={} resultId={} resultFileUrl={}", + taskId, job.getId(), result.getId(), result.getResultFileUrl()); + } catch (Exception ex) { + TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId()); + if (job != null) { + taskFileJobService.markFailed(job, firstNonBlank(ex.getMessage(), "appearance patent result file build failed")); + } + log.warn("[appearance-patent] coze async finalize failed taskId={} resultId={} err={}", + taskId, context.resultId(), ex.getMessage(), ex); + } + } + + private void completeCozeFileJob(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + int totalProgressUnits, + int cozeWorkUnits) { + int assembleProgress = Math.max(1, Math.min(totalProgressUnits - 2, cozeWorkUnits)); + saveFileBuildProgress(task, job, totalProgressUnits, assembleProgress, "Assembling xlsx"); assembleResultWorkbook(task, result); - completedProgressUnits[0] = totalProgressUnits - 1; - saveFileBuildProgress(task, job, totalProgressUnits, completedProgressUnits[0], "正在上传结果文件"); + saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits - 1, "Uploading result file"); fileResultMapper.updateById(result); - saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "结果文件已生成"); + saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "Result file generated"); + } + + private void mergeCozeRowsIntoChunk(FileTaskEntity task, + String chunkScopeHash, + Integer chunkIndex, + List cozeRows, + Map> allRowsByBaseId) { + if (task == null || cozeRows == null || cozeRows.isEmpty()) { + return; + } + Map mergedRows = new LinkedHashMap<>(); + for (AppearancePatentResultRowDto resultRow : cozeRows) { + for (AppearancePatentResultRowDto expandedRow : expandRows(List.of(resultRow), allRowsByBaseId)) { + mergedRows.put(rowKey(expandedRow), expandedRow); + } + } + mergeChunkPayload(task.getId(), chunkScopeHash, chunkIndex, new ArrayList<>(mergedRows.values())); + } + + private List readCozeBatchRows(TaskScopeStateEntity state) { + if (state == null || state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) { + return List.of(); + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload( + state.getParsedPayloadJson(), "read appearance patent coze batch failed"); + JsonNode array = objectMapper.readTree(payloadJson); + if (!array.isArray()) { + return List.of(); + } + List rows = new ArrayList<>(); + for (JsonNode node : array) { + rows.add(objectMapper.treeToValue(node, AppearancePatentResultRowDto.class)); + } + return rows; + } catch (Exception ex) { + log.warn("[appearance-patent] read coze batch failed taskId={} stateId={} err={}", + state.getTaskId(), state.getId(), ex.getMessage()); + return List.of(); + } + } + + private CozeBatchContext readCozeBatchContext(TaskScopeStateEntity state) { + if (state == null || state.getStateJson() == null || state.getStateJson().isBlank()) { + return null; + } + try { + return objectMapper.readValue(state.getStateJson(), CozeBatchContext.class); + } catch (Exception ex) { + log.warn("[appearance-patent] read coze batch context failed taskId={} stateId={} err={}", + state.getTaskId(), state.getId(), ex.getMessage()); + return null; + } + } + + private boolean isCozeStateTimedOut(TaskScopeStateEntity state) { + if (state == null || state.getCozeSubmittedAt() == null) { + return false; + } + long timeoutMillis = Math.max(10000L, properties.getCozePollTimeoutMillis()); + return Duration.between(state.getCozeSubmittedAt(), LocalDateTime.now()).toMillis() >= timeoutMillis; + } + + private int cozeAttemptCount(TaskScopeStateEntity state) { + return state == null || state.getCozeAttemptCount() == null ? 0 : state.getCozeAttemptCount(); + } + + private int countPendingCozeStates(Long taskId) { + if (taskId == null || taskId <= 0) { + return 0; + } + Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING))); + return count == null ? 0 : count.intValue(); + } + + private int countCompletedCozeStates(Long taskId) { + if (taskId == null || taskId <= 0) { + return 0; + } + Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_DONE, COZE_STATUS_FAILED))); + return count == null ? 0 : count.intValue(); + } + + private boolean isJavaSideProcessing(Long taskId) { + return countPendingCozeStates(taskId) > 0 + || taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE) > 0; + } + + private void touchJavaSideTaskActivity(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + fileTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(FileTaskEntity::getId, taskId) + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())); + } + + private String buildCozeBatchScopeKey(Long jobId, String chunkScopeHash, Integer chunkIndex, int batchIndex) { + return "coze:job:" + jobId + + ":chunk:" + (chunkIndex == null ? 0 : chunkIndex) + + ":" + firstNonBlank(chunkScopeHash, "unknown") + + ":batch:" + batchIndex; } private int countCozeWorkUnits(List chunks, int batchSize) { @@ -1146,7 +1607,12 @@ public class AppearancePatentTaskService { throw new BusinessException("创建结果目录失败"); } String filename = safeFileStem(result.getSourceFilename()) + "-result.xlsx"; - File xlsx = new File(outputDir, filename); + String tempFilename = safeFileStem(result.getSourceFilename()) + + "-" + task.getId() + + "-" + result.getId() + + "-" + UUID.randomUUID() + + "-result.xlsx"; + File xlsx = new File(outputDir, tempFilename); try { writeResultWorkbook(xlsx, parsed, resultMap); String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE); @@ -1691,7 +2157,7 @@ public class AppearancePatentTaskService { } private String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) { - return transientPayloadStorageService.storeParsedPayloadFast(MODULE_TYPE, taskId, scopeHash, parsedPayloadJson, true); + return transientPayloadStorageService.storeParsedPayloadFast(MODULE_TYPE, taskId, scopeHash, parsedPayloadJson, false); } private long elapsedMs(long start, long end) { @@ -1919,6 +2385,14 @@ public class AppearancePatentTaskService { String error) { } + private record CozeBatchContext(Long jobId, + Long resultId, + String chunkScopeHash, + Integer chunkIndex, + Integer batchIndex, + Integer batchTotal) { + } + private record ParsedWorkbook(int totalRows, int droppedRows, List headers, List allRows) { } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/controller/BrandTaskController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/controller/BrandTaskController.java index 08dcb2e..46580cd 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/controller/BrandTaskController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/controller/BrandTaskController.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.brand.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.util.DownloadHeaderUtil; import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultRequest; import com.nanri.aiimage.modules.brand.model.dto.BrandTaskCreateRequest; import com.nanri.aiimage.modules.brand.model.vo.BrandCrawlPayloadVo; @@ -17,7 +18,6 @@ import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import jakarta.servlet.http.HttpServletResponse; -import org.springframework.http.HttpHeaders; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -30,8 +30,6 @@ import org.springframework.web.bind.annotation.RestController; import java.io.InputStream; import java.net.URI; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; @RestController @RequiredArgsConstructor @@ -213,10 +211,8 @@ public class BrandTaskController { if (filename.isBlank()) { filename = "brand_task_" + taskId + ".zip"; } - String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20"); response.setContentType("application/zip"); - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename); + DownloadHeaderUtil.setAttachment(response, filename); // 后端代理拉取 OSS 文件并流式写出 try (InputStream in = URI.create(ossUrl).toURL().openStream()) { byte[] buffer = new byte[65536]; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/controller/ConvertRunController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/controller/ConvertRunController.java index 0fff6e1..8ece08c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/controller/ConvertRunController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/controller/ConvertRunController.java @@ -14,10 +14,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; -import org.springframework.core.io.FileSystemResource; -import org.springframework.core.io.Resource; import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -28,9 +25,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.io.File; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; +import java.net.URI; @RestController @RequiredArgsConstructor @@ -100,15 +95,14 @@ public class ConvertRunController { @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "返回结果文件流"), @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "结果文件不存在") }) - public ResponseEntity download( + public ResponseEntity download( @Parameter(description = "格式转换结果记录 ID", required = true) @PathVariable Long resultId, @Parameter(name = "user_id", description = "当前登录用户 ID", required = true, in = ParameterIn.QUERY) @RequestParam("user_id") Long userId) { - File resultFile = convertRunService.getResultFile(resultId, userId); - String encodedFilename = URLEncoder.encode(resultFile.getName(), StandardCharsets.UTF_8).replaceAll("\\+", "%20"); - return ResponseEntity.ok() - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=UTF-8''" + encodedFilename) - .body(new FileSystemResource(resultFile)); + String downloadUrl = convertRunService.getResultDownloadUrl(resultId, userId); + return ResponseEntity.status(302) + .location(URI.create(downloadUrl)) + .header(HttpHeaders.CACHE_CONTROL, "no-store") + .build(); } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java index 6d2bf4a..f950335 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -43,6 +44,7 @@ public class ConvertRunService { private static final String MODULE_TYPE = "CONVERT"; private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer"; + private static final DateTimeFormatter ZIP_NAME_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMddHHmmss"); private static final List FIVE_COUNTRY_OUTPUT_FILES = List.of( "英国.txt", "法国.txt", @@ -76,10 +78,12 @@ public class ConvertRunService { fileTaskMapper.insert(task); boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank(); + boolean archiveMode = folderMode || request.getFiles().size() > 1; List items = new ArrayList<>(); int successCount = 0; int failedCount = 0; List archiveEntries = new ArrayList<>(); + List successSourceNames = new ArrayList<>(); for (UploadedSourceFileDto sourceFile : request.getFiles()) { try { @@ -92,10 +96,11 @@ public class ConvertRunService { ? inputFile.getName() : sourceFile.getOriginalFilename(); List generatedFiles = generateOutputFiles(inputFile, template); - if (folderMode) { + if (archiveMode) { for (GeneratedConvertFile generatedFile : generatedFiles) { archiveEntries.add(new ConvertArchiveEntry(sourceFile.getRelativePath(), inputName, generatedFile)); } + successSourceNames.add(inputName); successCount++; continue; } @@ -153,12 +158,13 @@ public class ConvertRunService { } } - if (folderMode && !archiveEntries.isEmpty()) { - File zipFile = packageFolderConvertResultsAsZip(request.getArchiveName(), archiveEntries); + if (archiveMode && !archiveEntries.isEmpty()) { + String archiveName = resolveArchiveName(request.getArchiveName(), successSourceNames); + File zipFile = packageFolderConvertResultsAsZip(archiveName, archiveEntries); String ossObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE); // 只存 objectKey ConvertResultItemVo item = new ConvertResultItemVo(); - item.setSourceFilename(request.getArchiveName()); + item.setSourceFilename(folderMode ? request.getArchiveName() : "Current convert task"); item.setOutputFilename(zipFile.getName()); item.setSuccess(true); item.setDownloadUrl(ossStorageService.generateFreshDownloadUrl(ossObjectKey)); @@ -166,7 +172,7 @@ public class ConvertRunService { FileResultEntity resultEntity = new FileResultEntity(); resultEntity.setTaskId(task.getId()); resultEntity.setModuleType(MODULE_TYPE); - resultEntity.setSourceFilename(request.getArchiveName()); + resultEntity.setSourceFilename(item.getSourceFilename()); resultEntity.setResultFilename(zipFile.getName()); resultEntity.setResultFileUrl(ossObjectKey); // 存 objectKey resultEntity.setResultFileSize(zipFile.length()); @@ -205,7 +211,7 @@ public class ConvertRunService { vo.setResultId(entity.getId()); vo.setSourceFilename(entity.getSourceFilename()); vo.setOutputFilename(entity.getResultFilename()); - vo.setDownloadUrl(null); + vo.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())); vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); vo.setError(entity.getErrorMessage()); return vo; @@ -221,17 +227,12 @@ public class ConvertRunService { fileResultMapper.deleteById(resultId); } - public File getResultFile(Long resultId, Long userId) { + public String getResultDownloadUrl(Long resultId, Long userId) { FileResultEntity entity = fileResultMapper.selectById(resultId); if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { throw new BusinessException("Convert result file does not exist."); } - - File file = new File(entity.getResultFileUrl()); - if (!file.exists()) { - throw new BusinessException("Convert result file does not exist."); - } - return file; + return ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()); } private List generateOutputFiles(File inputFile, ConvertTemplateEntity templateEntity) throws IOException { @@ -410,6 +411,16 @@ public class ConvertRunService { return zipFile; } + private String resolveArchiveName(String archiveName, List successSourceNames) { + if (archiveName != null && !archiveName.isBlank()) { + return FileUtil.mainName(archiveName.trim()); + } + if (successSourceNames.size() == 1) { + return FileUtil.mainName(successSourceNames.getFirst()); + } + return "convert-" + LocalDateTime.now().format(ZIP_NAME_DATE_FORMATTER); + } + private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) { String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/'); String relativeDir = normalizedRelativePath.contains("/") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/controller/DeleteBrandRunController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/controller/DeleteBrandRunController.java index b259a9b..402fe60 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/controller/DeleteBrandRunController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/controller/DeleteBrandRunController.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.deletebrand.controller; import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.util.DownloadHeaderUtil; import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandRunRequest; import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandSubmitResultRequest; import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandTaskBatchRequest; @@ -19,7 +20,6 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -33,7 +33,6 @@ import org.springframework.web.server.ResponseStatusException; import java.io.InputStream; import java.net.URI; -import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -134,11 +133,8 @@ public class DeleteBrandRunController { } try { - String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20"); - String asciiFilename = buildAsciiDownloadFilename(filename, taskId); response.setContentType("application/octet-stream"); - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + asciiFilename + "\"; filename*=UTF-8''" + encodedFilename); + DownloadHeaderUtil.setAttachment(response, filename); try (InputStream in = URI.create(url).toURL().openStream()) { byte[] buffer = new byte[65536]; int read; @@ -152,23 +148,37 @@ public class DeleteBrandRunController { } } - private String buildAsciiDownloadFilename(String filename, Long taskId) { - String sanitized = filename == null ? "" : filename.replaceAll("[^A-Za-z0-9._-]", "_"); - sanitized = sanitized.replaceAll("_+", "_"); - sanitized = sanitized.replaceAll("^[_\\.]+|[_\\.]+$", ""); - if (!sanitized.isBlank() && sanitized.contains(".")) { - return sanitized; + @GetMapping("/results/{resultId}/download") + @Operation(summary = "Download one delete-brand result") + public void downloadResult( + @PathVariable Long resultId, + @Parameter(name = "user_id", description = "褰撳墠鐧诲綍鐢ㄦ埛 ID", required = true, in = ParameterIn.QUERY) + @RequestParam("user_id") Long userId, + jakarta.servlet.http.HttpServletResponse response) { + String url = deleteBrandRunService.resolveResultDownloadUrl(resultId, userId); + String filename = deleteBrandRunService.resolveResultDownloadFilename(resultId, userId); + if (url == null || url.isBlank()) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "缁撴灉鏃犲彲涓嬭浇"); } - String ext = "xlsx"; - if (filename != null) { - int dotIndex = filename.lastIndexOf('.'); - if (dotIndex >= 0 && dotIndex < filename.length() - 1) { - String rawExt = filename.substring(dotIndex + 1).replaceAll("[^A-Za-z0-9]", ""); - if (!rawExt.isBlank()) { - ext = rawExt; + if (filename == null || filename.isBlank()) { + String rawPath = URI.create(url).getPath(); + filename = rawPath == null ? "delete-brand_" + resultId + ".xlsx" : rawPath.substring(rawPath.lastIndexOf('/') + 1); + } + + try { + response.setContentType("application/octet-stream"); + DownloadHeaderUtil.setAttachment(response, filename); + try (InputStream in = URI.create(url).toURL().openStream()) { + byte[] buffer = new byte[65536]; + int read; + while ((read = in.read(buffer)) != -1) { + response.getOutputStream().write(buffer, 0, read); } + response.getOutputStream().flush(); } + } catch (Exception ex) { + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "涓嬭浇澶辫触"); } - return "delete-brand_" + taskId + "." + ext; } + } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java index 52e2e58..7dc571b 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java @@ -84,6 +84,20 @@ public class DeleteBrandRunService { private final TaskPressureProperties taskPressureProperties; private final TaskFileJobService taskFileJobService; + private boolean isUsableIndexedMatch(ZiniaoShopMatchResultVo matchResult) { + if (matchResult == null) { + return false; + } + String shopId = matchResult.getShopId(); + if (shopId == null || shopId.isBlank()) { + return false; + } + String status = matchResult.getMatchStatus(); + return matchResult.isMatched() + || ZiniaoShopIndexService.MATCH_STATUS_MATCHED.equals(status) + || ZiniaoShopIndexService.MATCH_STATUS_STALE.equals(status); + } + public DeleteBrandRunVo run(DeleteBrandRunRequest request) { if (request.getUserId() == null || request.getUserId() <= 0) { throw new BusinessException("user_id 不合法"); @@ -136,8 +150,8 @@ public class DeleteBrandRunService { ignored -> ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName(), false)); item.setMatchStatus(matchResult.getMatchStatus()); item.setMatchMessage(matchResult.getMatchMessage()); - item.setMatched(matchResult.isMatched()); - if (matchResult.isMatched()) { + item.setMatched(isUsableIndexedMatch(matchResult)); + if (item.isMatched()) { item.setShopId(matchResult.getShopId()); item.setCompanyName(matchResult.getCompanyName()); item.setPlatform(matchResult.getPlatform()); @@ -236,35 +250,79 @@ public class DeleteBrandRunService { } public DeleteBrandHistoryVo listHistory(Long userId) { + long startedAt = System.nanoTime(); DeleteBrandHistoryVo vo = new DeleteBrandHistoryVo(); - List entities = fileResultMapper.selectList(new LambdaQueryWrapper() + List createdRows = fileResultMapper.selectList(historyResultQuery() .eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getUserId, userId) .orderByDesc(FileResultEntity::getCreatedAt) - .last("limit 100")); + .last("limit 500")); + List recentTouchedTasks = fileTaskMapper.selectList(new LambdaQueryWrapper() + .select(FileTaskEntity::getId, + FileTaskEntity::getStatus, + FileTaskEntity::getUpdatedAt, + FileTaskEntity::getFinishedAt) + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getUserId, userId) + .orderByDesc(FileTaskEntity::getUpdatedAt) + .last("limit 200")); + Map entityById = new LinkedHashMap<>(); + for (FileResultEntity entity : createdRows) { + if (entity != null && entity.getId() != null) { + entityById.put(entity.getId(), entity); + } + } + for (FileResultEntity entity : selectHistoryResultRowsByTaskIdsInBatches(userId, recentTouchedTasks.stream() + .map(FileTaskEntity::getId) + .filter(id -> id != null && id > 0) + .distinct() + .toList())) { + if (entity != null && entity.getId() != null) { + entityById.putIfAbsent(entity.getId(), entity); + } + } + List entities = new ArrayList<>(entityById.values()); + long resultRowsLoadedAt = System.nanoTime(); + if (entities.isEmpty()) { + vo.setItems(List.of()); + log.info("[delete-brand] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0", + userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt)); + return vo; + } // 补充信息 - Map> itemsByTaskId = new LinkedHashMap<>(); + Map taskById = new LinkedHashMap<>(); Map statusByTaskId = new LinkedHashMap<>(); List taskIds = entities.stream() .map(FileResultEntity::getTaskId) .filter(id -> id != null && id > 0) .distinct() .toList(); + for (FileTaskEntity task : recentTouchedTasks) { + if (task != null) { + taskById.put(task.getId(), task); + statusByTaskId.put(task.getId(), task.getStatus()); + } + } if (!taskIds.isEmpty()) { - List tasks = selectTasksByIdsInBatches(taskIds); + List tasks = selectTaskHistoryFieldsByIdsInBatches(taskIds); for (FileTaskEntity task : tasks) { - try { + if (task != null) { + taskById.put(task.getId(), task); statusByTaskId.put(task.getId(), task.getStatus()); - if (task.getResultJson() != null && !task.getResultJson().isBlank()) { - List taskItems = objectMapper.readValue(task.getResultJson(), new TypeReference>() { - }); - itemsByTaskId.put(task.getId(), taskItems); - } - } catch (Exception ignored) { } } } + long tasksLoadedAt = System.nanoTime(); + Map jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream() + .map(FileResultEntity::getId) + .filter(id -> id != null && id > 0) + .distinct() + .toList()); + long jobsLoadedAt = System.nanoTime(); + entities = sortDeleteBrandHistoryEntities(entities, taskById, jobMap).stream() + .limit(100) + .toList(); vo.setItems(entities.stream().map(entity -> { DeleteBrandResultItemVo item = new DeleteBrandResultItemVo(); @@ -275,7 +333,7 @@ public class DeleteBrandRunService { item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename())); item.setOutputFilename(entity.getResultFilename()); item.setDownloadUrl(null); - attachFileJobState(item, entity); + attachFileJobState(item, entity, jobMap.get(entity.getId())); item.setTotalRows(entity.getRowCount()); item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); item.setError(entity.getErrorMessage()); @@ -284,7 +342,7 @@ public class DeleteBrandRunService { // 真实补充 if (entity.getTaskId() != null) { item.setTaskStatus(statusByTaskId.get(entity.getTaskId())); - List taskItems = itemsByTaskId.get(entity.getTaskId()); + List taskItems = null; if (taskItems != null && entity.getSourceFilename() != null && !entity.getSourceFilename().isBlank()) { for (DeleteBrandResultItemVo candidate : taskItems) { if (candidate == null) { @@ -320,12 +378,42 @@ public class DeleteBrandRunService { } return item; }).toList()); + long finishedAt = System.nanoTime(); + log.info("[delete-brand] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}", + userId, + entities.size(), + statusByTaskId.size(), + jobMap.size(), + elapsedMs(startedAt, finishedAt), + elapsedMs(startedAt, resultRowsLoadedAt), + elapsedMs(resultRowsLoadedAt, tasksLoadedAt), + elapsedMs(tasksLoadedAt, jobsLoadedAt), + elapsedMs(jobsLoadedAt, finishedAt)); return vo; } + private LambdaQueryWrapper historyResultQuery() { + return new LambdaQueryWrapper() + .select(FileResultEntity::getId, + FileResultEntity::getTaskId, + FileResultEntity::getModuleType, + FileResultEntity::getSourceFilename, + FileResultEntity::getSourceFileUrl, + FileResultEntity::getResultFilename, + FileResultEntity::getResultFileUrl, + FileResultEntity::getRowCount, + FileResultEntity::getSuccess, + FileResultEntity::getErrorMessage, + FileResultEntity::getUserId, + FileResultEntity::getCreatedAt); + } + private void attachFileJobState(DeleteBrandResultItemVo item, FileResultEntity entity) { - TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); + attachFileJobState(item, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId())); + } + + private void attachFileJobState(DeleteBrandResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) { item.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); if (job == null) { item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null); @@ -336,6 +424,51 @@ public class DeleteBrandRunService { item.setFileError(job.getErrorMessage()); } + private List sortDeleteBrandHistoryEntities( + List entities, + Map taskById, + Map jobByResultId + ) { + if (entities == null || entities.isEmpty()) { + return List.of(); + } + List sorted = new ArrayList<>(entities); + sorted.sort(Comparator + .comparing((FileResultEntity entity) -> resolveHistorySortTime(entity, taskById, jobByResultId), + Comparator.nullsLast(Comparator.reverseOrder())) + .thenComparing(FileResultEntity::getId, Comparator.nullsLast(Comparator.reverseOrder()))); + return sorted; + } + + private LocalDateTime resolveHistorySortTime( + FileResultEntity entity, + Map taskById, + Map jobByResultId + ) { + if (entity == null) { + return null; + } + TaskFileJobEntity job = jobByResultId == null ? null : jobByResultId.get(entity.getId()); + if (job != null) { + if (job.getFinishedAt() != null) { + return job.getFinishedAt(); + } + if (job.getUpdatedAt() != null) { + return job.getUpdatedAt(); + } + } + FileTaskEntity task = taskById == null ? null : taskById.get(entity.getTaskId()); + if (task != null) { + if (task.getFinishedAt() != null) { + return task.getFinishedAt(); + } + if (task.getUpdatedAt() != null) { + return task.getUpdatedAt(); + } + } + return entity.getCreatedAt(); + } + public DeleteBrandTaskDeletionStatusVo getTaskDeletionStatus(Long taskId, Long userId) { FileTaskEntity task = loadTaskForExecution(taskId); if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { @@ -617,6 +750,13 @@ public class DeleteBrandRunService { } java.util.Map> progressByTaskId = deleteBrandTaskCacheService.getProgressBatch(normalizedTaskIds); + Map> resultRowsByTaskId = findProgressResultRowsByTaskIds(normalizedTaskIds); + Map jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, resultRowsByTaskId.values().stream() + .flatMap(List::stream) + .map(FileResultEntity::getId) + .filter(id -> id != null && id > 0) + .distinct() + .toList()); for (Long taskId : normalizedTaskIds) { FileTaskEntity task = taskById.get(taskId); @@ -624,7 +764,8 @@ public class DeleteBrandRunService { vo.getMissingTaskIds().add(taskId); continue; } - vo.getItems().add(buildTaskProgressDetail(task, progressByTaskId.get(taskId))); + vo.getItems().add(buildTaskProgressDetail(task, progressByTaskId.get(taskId), + resultRowsByTaskId.getOrDefault(taskId, List.of()), jobMap)); } return vo; } @@ -634,6 +775,13 @@ public class DeleteBrandRunService { } private DeleteBrandTaskDetailVo buildTaskProgressDetail(FileTaskEntity task, java.util.Map progress) { + return buildTaskProgressDetail(task, progress, List.of(), Map.of()); + } + + private DeleteBrandTaskDetailVo buildTaskProgressDetail(FileTaskEntity task, + java.util.Map progress, + List resultRows, + Map jobMap) { task = refreshIndexedMatchesIfNeeded(task); DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo(); taskVo.setId(task.getId()); @@ -657,10 +805,39 @@ public class DeleteBrandRunService { DeleteBrandTaskDetailVo detail = new DeleteBrandTaskDetailVo(); detail.setTask(taskVo); detail.setLine_progress(buildLineProgress(task.getId(), progress)); - detail.setFileProgress(java.util.List.of()); + List taskItems = buildProgressResultItems(task, resultRows, jobMap); + detail.setItems(taskItems); + detail.setFileProgress(buildFileProgress(task, taskItems, detail.getLine_progress())); return detail; } + private List buildProgressResultItems(FileTaskEntity task, + List resultRows, + Map jobMap) { + if (task == null || resultRows == null || resultRows.isEmpty()) { + return List.of(); + } + return resultRows.stream() + .filter(entity -> entity != null && entity.getId() != null) + .map(entity -> { + DeleteBrandResultItemVo item = new DeleteBrandResultItemVo(); + item.setResultId(entity.getId()); + item.setTaskId(entity.getTaskId()); + item.setFileKey(entity.getSourceFileUrl()); + item.setSourceFilename(entity.getSourceFilename()); + item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename())); + item.setOutputFilename(entity.getResultFilename()); + item.setDownloadUrl(null); + item.setTotalRows(entity.getRowCount()); + item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); + item.setError(entity.getErrorMessage()); + item.setTaskStatus(task.getStatus()); + attachFileJobState(item, entity, jobMap == null ? null : jobMap.get(entity.getId())); + return item; + }) + .toList(); + } + private DeleteBrandTaskDetailVo buildTaskDetail(FileTaskEntity task, java.util.Map progress) { task = refreshIndexedMatchesIfNeeded(task); DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo(); @@ -837,7 +1014,7 @@ public class DeleteBrandRunService { String oldOpenStoreUrl = item.getOpenStoreUrl(); String oldMessage = item.getMatchMessage(); - item.setMatched(refreshed.isMatched()); + item.setMatched(isUsableIndexedMatch(refreshed)); item.setMatchStatus(refreshed.getMatchStatus()); item.setMatchMessage(refreshed.getMatchMessage()); item.setShopId(refreshed.getShopId()); @@ -1011,6 +1188,65 @@ public class DeleteBrandRunService { return tasks; } + private List selectTaskHistoryFieldsByIdsInBatches(List taskIds) { + List tasks = new ArrayList<>(); + if (taskIds == null || taskIds.isEmpty()) { + return tasks; + } + int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize()); + for (int start = 0; start < taskIds.size(); start += batchSize) { + int end = Math.min(start + batchSize, taskIds.size()); + tasks.addAll(fileTaskMapper.selectList(new LambdaQueryWrapper() + .select(FileTaskEntity::getId, + FileTaskEntity::getStatus, + FileTaskEntity::getUpdatedAt, + FileTaskEntity::getFinishedAt) + .in(FileTaskEntity::getId, taskIds.subList(start, end)))); + } + return tasks; + } + + private List selectHistoryResultRowsByTaskIdsInBatches(Long userId, List taskIds) { + List rows = new ArrayList<>(); + if (userId == null || taskIds == null || taskIds.isEmpty()) { + return rows; + } + int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize()); + for (int start = 0; start < taskIds.size(); start += batchSize) { + int end = Math.min(start + batchSize, taskIds.size()); + rows.addAll(fileResultMapper.selectList(historyResultQuery() + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .eq(FileResultEntity::getUserId, userId) + .in(FileResultEntity::getTaskId, taskIds.subList(start, end)))); + } + return rows; + } + + private Map> findProgressResultRowsByTaskIds(List taskIds) { + if (taskIds == null || taskIds.isEmpty()) { + return Map.of(); + } + Map> rowsByTaskId = new LinkedHashMap<>(); + int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize()); + for (int start = 0; start < taskIds.size(); start += batchSize) { + int end = Math.min(start + batchSize, taskIds.size()); + List rows = fileResultMapper.selectList(historyResultQuery() + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .in(FileResultEntity::getTaskId, taskIds.subList(start, end)) + .orderByAsc(FileResultEntity::getId)); + for (FileResultEntity row : rows) { + if (row != null && row.getTaskId() != null) { + rowsByTaskId.computeIfAbsent(row.getTaskId(), ignored -> new ArrayList<>()).add(row); + } + } + } + return rowsByTaskId; + } + + private static long elapsedMs(long startInclusive, long endExclusive) { + return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L); + } + @Transactional public void submitResult(Long taskId, DeleteBrandSubmitResultRequest request) { log.info("[DeleteBrand] Received submitResult for taskId: {}, files count: {}", taskId, request == null || request.getFiles() == null ? 0 : request.getFiles().size()); @@ -1027,6 +1263,16 @@ public class DeleteBrandRunService { throw new BusinessException("任务不存在"); } if ("SUCCESS".equals(task.getStatus()) || "FAILED".equals(task.getStatus())) { + log.warn("[DeleteBrand] submitResult rejected because task is terminal -> taskId: {}, status: {}, createdAt: {}, updatedAt: {}, finishedAt: {}, error: {}, sourceFiles: {}, successFiles: {}, failedFiles: {}", + task.getId(), + task.getStatus(), + task.getCreatedAt(), + task.getUpdatedAt(), + task.getFinishedAt(), + task.getErrorMessage(), + task.getSourceFileCount(), + task.getSuccessFileCount(), + task.getFailedFileCount()); throw new BusinessException("任务已结束，拒绝继续提交结果"); } @@ -1086,6 +1332,7 @@ public class DeleteBrandRunService { // 如果该文件已完成，立即更新成品计数的 Redis 提示，让后续 tryFinalizeTask 更准确 if (isFileCompleted(fileDto)) { log.info("[DeleteBrand] File chunk transmission fully reached via chunkTotal for taskId: {}, file: {}", taskId, fileDto.getSourceFilename()); + enqueueCompletedFileAssembleJob(task, fileIdentity, parsedFile); shouldTryFinalize = true; // 这里暂不直接 increment，而是标记需要在 tryFinalizeTask 中重新计算 // 为了保险，直接在 tryFinalizeTask 里重新遍历分片状态 @@ -1098,6 +1345,55 @@ public class DeleteBrandRunService { } } + private void enqueueCompletedFileAssembleJob(FileTaskEntity task, + String fileIdentity, + DeleteBrandParsedFileCacheDto parsedFile) { + if (task == null || task.getId() == null || parsedFile == null) { + return; + } + FileResultEntity resultEntity = findResultEntity(task.getId(), fileIdentity, parsedFile); + if (resultEntity == null) { + log.warn("[DeleteBrand] completed file has no result row, skip early assemble -> taskId: {}, file: {}", + task.getId(), parsedFile.getSourceFilename()); + return; + } + if (taskFileJobService.hasSuccessfulAssembleJob(task.getId(), MODULE_TYPE, resultEntity.getId())) { + return; + } + String outputFilename = buildResultFilename(parsedFile); + resultEntity.setResultFilename(outputFilename); + resultEntity.setResultContentType(CONTENT_TYPE_XLSX); + resultEntity.setRowCount(parsedFile.getTotalRows()); + resultEntity.setSuccess(1); + resultEntity.setErrorMessage(null); + if (resultEntity.getResultFileUrl() == null || resultEntity.getResultFileUrl().isBlank()) { + resultEntity.setResultFileSize(0L); + } + fileResultMapper.updateById(resultEntity); + taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity); + log.info("[DeleteBrand] completed file early assemble job enqueued -> taskId: {}, resultId: {}, file: {}", + task.getId(), resultEntity.getId(), parsedFile.getSourceFilename()); + } + + private FileResultEntity findResultEntity(Long taskId, + String fileIdentity, + DeleteBrandParsedFileCacheDto parsedFile) { + if (taskId == null || taskId <= 0) { + return null; + } + LambdaQueryWrapper wrapper = new LambdaQueryWrapper() + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .eq(FileResultEntity::getTaskId, taskId); + if (fileIdentity != null && !fileIdentity.isBlank()) { + wrapper.eq(FileResultEntity::getSourceFileUrl, fileIdentity); + } else if (parsedFile != null && parsedFile.getSourceFilename() != null && !parsedFile.getSourceFilename().isBlank()) { + wrapper.eq(FileResultEntity::getSourceFilename, parsedFile.getSourceFilename()); + } else { + return null; + } + return fileResultMapper.selectOne(wrapper.last("limit 1")); + } + public void tryFinalizeTask(Long taskId, boolean fromCompensation) { if (taskId == null || taskId <= 0) { return; @@ -1291,15 +1587,19 @@ public class DeleteBrandRunService { throw new BusinessException("结果记录不存在: " + parsedFile.getSourceFilename()); } String outputFilename = buildResultFilename(parsedFile); + boolean alreadyAssembled = taskFileJobService.hasSuccessfulAssembleJob(task.getId(), MODULE_TYPE, resultEntity.getId()); resultEntity.setResultFilename(outputFilename); - resultEntity.setResultFileUrl(null); - resultEntity.setResultFileSize(0L); + if (!alreadyAssembled && (resultEntity.getResultFileUrl() == null || resultEntity.getResultFileUrl().isBlank())) { + resultEntity.setResultFileSize(0L); + } resultEntity.setResultContentType(CONTENT_TYPE_XLSX); resultEntity.setRowCount(parsedFile.getTotalRows()); resultEntity.setSuccess(1); resultEntity.setErrorMessage(null); fileResultMapper.updateById(resultEntity); - taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity); + if (!alreadyAssembled) { + taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity); + } DeleteBrandResultItemVo item = new DeleteBrandResultItemVo(); item.setResultId(resultEntity.getId()); @@ -1343,6 +1643,7 @@ public class DeleteBrandRunService { "finished_files", String.valueOf(successCount), "updated_at", String.valueOf(System.currentTimeMillis()) )); + cleanupCompletedTaskDataIfNoPendingFileJobs(task.getId()); } catch (Exception ex) { task.setStatus("FAILED"); task.setErrorMessage(ex.getMessage()); @@ -1419,8 +1720,19 @@ public class DeleteBrandRunService { if (job == null || job.getTaskId() == null) { return; } - if (taskFileJobService.countUnfinishedAssembleJobs(job.getTaskId(), MODULE_TYPE) == 0L) { - deleteBrandTaskStorageService.deleteTaskData(job.getTaskId()); + cleanupCompletedTaskDataIfNoPendingFileJobs(job.getTaskId()); + } + + private void cleanupCompletedTaskDataIfNoPendingFileJobs(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || (!"SUCCESS".equals(task.getStatus()) && !"FAILED".equals(task.getStatus()))) { + return; + } + if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) == 0L) { + deleteBrandTaskStorageService.deleteTaskData(taskId); } } @@ -1611,6 +1923,32 @@ public class DeleteBrandRunService { return entity == null ? null : entity.getResultFilename(); } + public String resolveResultDownloadUrl(Long resultId, Long userId) { + FileResultEntity entity = findDownloadableResult(resultId, userId); + return entity == null || entity.getResultFileUrl() == null ? null : ossStorageService.getPublicUrl(entity.getResultFileUrl()); + } + + public String resolveResultDownloadFilename(Long resultId, Long userId) { + FileResultEntity entity = findDownloadableResult(resultId, userId); + return entity == null ? null : entity.getResultFilename(); + } + + private FileResultEntity findDownloadableResult(Long resultId, Long userId) { + if (resultId == null || resultId <= 0 || userId == null || userId <= 0) { + return null; + } + FileResultEntity entity = fileResultMapper.selectById(resultId); + if (entity == null + || !MODULE_TYPE.equals(entity.getModuleType()) + || entity.getUserId() == null + || !entity.getUserId().equals(userId) + || entity.getResultFileUrl() == null + || entity.getResultFileUrl().isBlank()) { + return null; + } + return entity; + } + private FileResultEntity findLatestSuccessfulResult(Long taskId) { return fileResultMapper.selectList(new LambdaQueryWrapper() .eq(FileResultEntity::getModuleType, MODULE_TYPE) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java index 50ee53c..e743487 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java @@ -134,15 +134,17 @@ public class DeleteBrandStaleTaskService { } } boolean hasStartedProgress = lastHeartbeatAt > 0L; + int completedScopeCount = 0; if (!hasStartedProgress) { - hasStartedProgress = deleteBrandTaskStorageService.countCompletedScopes(task.getId()) > 0 - || deleteBrandTaskStorageService.countParsedPayloadScopes(task.getId()) > 0; + // Parsed payload is created by Java during the parse step. Only result chunks mean Python has started uploading. + completedScopeCount = deleteBrandTaskStorageService.countCompletedScopes(task.getId()); + hasStartedProgress = completedScopeCount > 0; } if (lastHeartbeatAt > 0L && nowMillis - lastHeartbeatAt < staleTimeoutMillis) { continue; } - if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { + if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) { continue; } @@ -151,12 +153,24 @@ public class DeleteBrandStaleTaskService { FileTaskEntity refreshed = fileTaskMapper.selectById(task.getId()); if (refreshed != null && !"RUNNING".equals(refreshed.getStatus())) { deleteBrandTaskCacheService.saveTaskCache(refreshed); + log.info("[stale-check] delete-brand finalized before timeout-fail taskId={} status={} updatedAt={} finishedAt={}", + refreshed.getId(), refreshed.getStatus(), refreshed.getUpdatedAt(), refreshed.getFinishedAt()); continue; } } catch (Exception ex) { log.warn("[stale-check] delete-brand finalize threw taskId={} msg={}", task.getId(), ex.getMessage()); } + log.warn("[stale-check] delete-brand failing stale task -> taskId={} updatedAt={} createdAt={} lastHeartbeatAt={} timeoutMinutes={} initialTimeoutMinutes={} completedScopes={} hasStartedProgress={} initialThreshold={}", + task.getId(), + task.getUpdatedAt(), + task.getCreatedAt(), + lastHeartbeatAt, + minutes, + initialMinutes, + completedScopeCount, + hasStartedProgress, + initialThreshold); int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper() .eq(FileTaskEntity::getId, task.getId()) .eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND) @@ -168,6 +182,7 @@ public class DeleteBrandStaleTaskService { if (updated > 0) { deleteBrandTaskCacheService.delete(task.getId()); + log.warn("[stale-check] delete-brand failed taskId={} reason=timeout", task.getId()); } } } @@ -204,11 +219,11 @@ public class DeleteBrandStaleTaskService { task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt()); continue; } - boolean hasStartedProgress = lastPayloadHeartbeatMillis > 0L || hasFinishedRows; + boolean hasStartedProgress = hasFinishedRows; if (!hasStartedProgress) { hasStartedProgress = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId()); } - if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { + if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) { stats.skippedTaskCount++; log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}", task.getId(), task.getCreatedAt(), initialThreshold); @@ -271,11 +286,11 @@ public class DeleteBrandStaleTaskService { stats.skippedTaskCount++; continue; } - boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows; + boolean hasStartedProgress = hasFinishedRows; if (!hasStartedProgress) { hasStartedProgress = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId()); } - if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { + if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) { stats.skippedTaskCount++; continue; } @@ -339,11 +354,11 @@ public class DeleteBrandStaleTaskService { task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt()); continue; } - boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows; + boolean hasStartedProgress = hasFinishedRows; if (!hasStartedProgress) { hasStartedProgress = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId()); } - if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { + if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) { stats.skippedTaskCount++; log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}", task.getId(), task.getCreatedAt(), initialThreshold); @@ -405,11 +420,11 @@ public class DeleteBrandStaleTaskService { stats.skippedTaskCount++; continue; } - boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows; + boolean hasStartedProgress = hasFinishedRows; if (!hasStartedProgress) { hasStartedProgress = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId()); } - if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { + if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) { stats.skippedTaskCount++; continue; } @@ -440,6 +455,14 @@ public class DeleteBrandStaleTaskService { return stats; } + private boolean isWithinInitialGrace(FileTaskEntity task, LocalDateTime initialThreshold) { + if (task == null || initialThreshold == null) { + return false; + } + LocalDateTime baseline = task.getUpdatedAt() != null ? task.getUpdatedAt() : task.getCreatedAt(); + return baseline != null && baseline.isAfter(initialThreshold); + } + @Scheduled(cron = "${aiimage.delete-brand-progress.finalize-check-cron:30 */2 * * * *}") public void finalizeCompletedRunningTasks() { DistributedJobLockService.LockHandle lockHandle = diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java index fc70404..a21cbff 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java @@ -19,6 +19,7 @@ import java.util.regex.Pattern; public class LocalTempCleanupService { private static final Pattern ROOT_TEMP_FILE_PATTERN = Pattern.compile("^[a-fA-F0-9]{32}(\\.[^.]+)?$"); + private static final String TRANSIENT_PAYLOAD_DIR_NAME = "transient-payload"; private static final List RESULT_DIR_NAMES = List.of( "dedupe-result", "convert-result", @@ -39,11 +40,14 @@ public class LocalTempCleanupService { return; } - Instant sourceExpireBefore = Instant.now().minus(storageProperties.getSourceRetentionHours(), ChronoUnit.HOURS); - Instant resultExpireBefore = Instant.now().minus(storageProperties.getResultRetentionHours(), ChronoUnit.HOURS); + Instant now = Instant.now(); + Instant sourceExpireBefore = now.minus(storageProperties.getSourceRetentionHours(), ChronoUnit.HOURS); + Instant resultExpireBefore = now.minus(storageProperties.getResultRetentionHours(), ChronoUnit.HOURS); + Instant transientPayloadExpireBefore = now.minus(storageProperties.getTransientPayloadRetentionHours(), ChronoUnit.HOURS); int deletedSourceCount = 0; int deletedResultCount = 0; + int deletedTransientPayloadCount = 0; File[] children = tempDir.listFiles(); if (children == null || children.length == 0) { return; @@ -60,14 +64,20 @@ public class LocalTempCleanupService { if (child.isDirectory() && RESULT_DIR_NAMES.contains(child.getName())) { deletedResultCount += deleteExpiredChildrenRecursively(child, resultExpireBefore); deleteEmptyDirectories(child, tempDir); + continue; + } + if (child.isDirectory() && TRANSIENT_PAYLOAD_DIR_NAME.equals(child.getName())) { + deletedTransientPayloadCount += deleteExpiredChildrenRecursively(child, transientPayloadExpireBefore); + deleteEmptyDirectories(child, tempDir); } } catch (Exception ex) { - log.warn("清理本地临时文件失败: path={}", child.getAbsolutePath(), ex); + log.warn("local temp cleanup failed: path={}", child.getAbsolutePath(), ex); } } - if (deletedSourceCount > 0 || deletedResultCount > 0) { - log.info("本地临时目录清理完成: sourceDeleted={}, resultDeleted={}, tempDir={}", deletedSourceCount, deletedResultCount, tempDir.getAbsolutePath()); + if (deletedSourceCount > 0 || deletedResultCount > 0 || deletedTransientPayloadCount > 0) { + log.info("local temp cleanup completed: sourceDeleted={}, resultDeleted={}, transientPayloadDeleted={}, tempDir={}", + deletedSourceCount, deletedResultCount, deletedTransientPayloadCount, tempDir.getAbsolutePath()); } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/controller/PatrolDeleteController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/controller/PatrolDeleteController.java index fc4f4f4..98855b9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/controller/PatrolDeleteController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/controller/PatrolDeleteController.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.patroldelete.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.util.DownloadHeaderUtil; import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteConditionAddRequest; import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteCreateTaskRequest; import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteSubmitResultRequest; @@ -25,7 +26,6 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -39,7 +39,6 @@ import org.springframework.web.server.ResponseStatusException; import java.io.InputStream; import java.net.URI; -import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; @@ -358,10 +357,8 @@ public class PatrolDeleteController { throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); } try { - String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20"); response.setContentType("application/octet-stream"); - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename); + DownloadHeaderUtil.setAttachment(response, filename); try (InputStream in = URI.create(url).toURL().openStream()) { byte[] buffer = new byte[65536]; int read; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java index 8caa38d..49c179f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.pricetrack.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.util.DownloadHeaderUtil; import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCandidateAddRequest; import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCountryPreferenceSaveRequest; import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCreateTaskRequest; @@ -28,7 +29,6 @@ import io.swagger.v3.oas.annotations.enums.ParameterIn; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -43,7 +43,6 @@ import org.springframework.web.server.ResponseStatusException; import java.io.InputStream; import java.net.URI; -import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; @@ -244,10 +243,8 @@ public class PriceTrackController { throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); } try { - String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20"); response.setContentType("application/octet-stream"); - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename); + DownloadHeaderUtil.setAttachment(response, filename); try (InputStream in = URI.create(url).toURL().openStream()) { byte[] buffer = new byte[65536]; int read; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java index 34043cb..c12f046 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java @@ -166,14 +166,33 @@ public class PriceTrackTaskService { } public PriceTrackHistoryVo listHistory(Long userId) { + long startedAt = System.nanoTime(); if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); PriceTrackHistoryVo vo = new PriceTrackHistoryVo(); List entities = fileResultMapper.selectList( new LambdaQueryWrapper() + .select(FileResultEntity::getId, + FileResultEntity::getTaskId, + FileResultEntity::getModuleType, + FileResultEntity::getSourceFilename, + FileResultEntity::getSourceFileUrl, + FileResultEntity::getResultFilename, + FileResultEntity::getResultFileUrl, + FileResultEntity::getSuccess, + FileResultEntity::getErrorMessage, + FileResultEntity::getUserId, + FileResultEntity::getCreatedAt) .eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getUserId, userId) .orderByDesc(FileResultEntity::getCreatedAt) .last("limit 100")); + long resultRowsLoadedAt = System.nanoTime(); + if (entities.isEmpty()) { + vo.setItems(List.of()); + log.info("[price-track] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0", + userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt)); + return vo; + } Map statusByTaskId = new LinkedHashMap<>(); List taskIds = entities.stream() .map(FileResultEntity::getTaskId) @@ -186,11 +205,29 @@ public class PriceTrackTaskService { if (t != null) statusByTaskId.put(t.getId(), t.getStatus()); } } + long tasksLoadedAt = System.nanoTime(); + Map jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream() + .map(FileResultEntity::getId) + .filter(id -> id != null && id > 0) + .distinct() + .toList()); + long jobsLoadedAt = System.nanoTime(); List items = new ArrayList<>(); for (FileResultEntity entity : entities) { - items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()))); + items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()), jobMap.get(entity.getId()), false)); } vo.setItems(items); + long finishedAt = System.nanoTime(); + log.info("[price-track] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}", + userId, + entities.size(), + statusByTaskId.size(), + jobMap.size(), + elapsedMs(startedAt, finishedAt), + elapsedMs(startedAt, resultRowsLoadedAt), + elapsedMs(resultRowsLoadedAt, tasksLoadedAt), + elapsedMs(tasksLoadedAt, jobsLoadedAt), + elapsedMs(jobsLoadedAt, finishedAt)); return vo; } @@ -952,6 +989,10 @@ public class PriceTrackTaskService { } private PriceTrackResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) { + return toHistoryItemVo(entity, taskStatus, null, true); + } + + private PriceTrackResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus, TaskFileJobEntity job, boolean mergeRequestFields) { PriceTrackResultItemVo vo = new PriceTrackResultItemVo(); vo.setResultId(entity.getId()); vo.setTaskId(entity.getTaskId()); @@ -963,16 +1004,19 @@ public class PriceTrackTaskService { vo.setError(entity.getErrorMessage()); vo.setOutputFilename(entity.getResultFilename()); vo.setDownloadUrl(null); - attachFileJobState(vo, entity); + attachFileJobState(vo, entity, job); vo.setMatched(true); - if (entity.getTaskId() != null) { + if (mergeRequestFields && entity.getTaskId() != null) { mergeQueueFieldsFromRequest(vo, entity.getTaskId()); } return vo; } private void attachFileJobState(PriceTrackResultItemVo vo, FileResultEntity entity) { - TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); + attachFileJobState(vo, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId())); + } + + private void attachFileJobState(PriceTrackResultItemVo vo, FileResultEntity entity, TaskFileJobEntity job) { vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); if (job == null) { vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); @@ -1226,6 +1270,10 @@ public class PriceTrackTaskService { return t == null ? null : t.toString(); } + private static long elapsedMs(long startInclusive, long endExclusive) { + return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L); + } + private void updateTaskStatusFromLatestRows(FileTaskEntity task, List latest) { String oldStatus = task.getStatus(); int ok = 0; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/controller/ProductRiskResolveController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/controller/ProductRiskResolveController.java index 3a9151d..b7cce9e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/controller/ProductRiskResolveController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/controller/ProductRiskResolveController.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.productrisk.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.util.DownloadHeaderUtil; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCreateTaskRequest; @@ -26,7 +27,6 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -41,7 +41,6 @@ import org.springframework.web.server.ResponseStatusException; import java.io.InputStream; import java.net.URI; -import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; @@ -250,10 +249,8 @@ public class ProductRiskResolveController { throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); } try { - String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20"); response.setContentType("application/octet-stream"); - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename); + DownloadHeaderUtil.setAttachment(response, filename); try (InputStream in = URI.create(url).toURL().openStream()) { byte[] buffer = new byte[65536]; int read; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java index d0e55c8..aea8c08 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java @@ -165,15 +165,34 @@ public class ProductRiskTaskService { } public ProductRiskHistoryVo listHistory(Long userId) { + long startedAt = System.nanoTime(); if (userId == null || userId <= 0) { throw new BusinessException("user_id 不合法"); } ProductRiskHistoryVo vo = new ProductRiskHistoryVo(); List entities = fileResultMapper.selectList(new LambdaQueryWrapper() + .select(FileResultEntity::getId, + FileResultEntity::getTaskId, + FileResultEntity::getModuleType, + FileResultEntity::getSourceFilename, + FileResultEntity::getSourceFileUrl, + FileResultEntity::getResultFilename, + FileResultEntity::getResultFileUrl, + FileResultEntity::getSuccess, + FileResultEntity::getErrorMessage, + FileResultEntity::getUserId, + FileResultEntity::getCreatedAt) .eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getUserId, userId) .orderByDesc(FileResultEntity::getCreatedAt) .last("limit 100")); + long resultRowsLoadedAt = System.nanoTime(); + if (entities.isEmpty()) { + vo.setItems(List.of()); + log.info("[product-risk] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0", + userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt)); + return vo; + } Map statusByTaskId = new LinkedHashMap<>(); List taskIds = entities.stream() .map(FileResultEntity::getTaskId) @@ -188,11 +207,29 @@ public class ProductRiskTaskService { } } } + long tasksLoadedAt = System.nanoTime(); + Map jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream() + .map(FileResultEntity::getId) + .filter(id -> id != null && id > 0) + .distinct() + .toList()); + long jobsLoadedAt = System.nanoTime(); List items = new ArrayList<>(); for (FileResultEntity entity : entities) { - items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()))); + items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()), jobMap.get(entity.getId()), false)); } vo.setItems(items); + long finishedAt = System.nanoTime(); + log.info("[product-risk] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}", + userId, + entities.size(), + statusByTaskId.size(), + jobMap.size(), + elapsedMs(startedAt, finishedAt), + elapsedMs(startedAt, resultRowsLoadedAt), + elapsedMs(resultRowsLoadedAt, tasksLoadedAt), + elapsedMs(tasksLoadedAt, jobsLoadedAt), + elapsedMs(jobsLoadedAt, finishedAt)); return vo; } @@ -967,6 +1004,10 @@ public class ProductRiskTaskService { } private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) { + return toHistoryItemVo(entity, taskStatus, null, true); + } + + private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus, TaskFileJobEntity job, boolean mergeRequestFields) { ProductRiskResultItemVo vo = new ProductRiskResultItemVo(); vo.setResultId(entity.getId()); vo.setTaskId(entity.getTaskId()); @@ -978,16 +1019,19 @@ public class ProductRiskTaskService { vo.setError(entity.getErrorMessage()); vo.setOutputFilename(entity.getResultFilename()); vo.setDownloadUrl(null); - attachFileJobState(vo, entity); + attachFileJobState(vo, entity, job); vo.setMatched(true); - if (entity.getTaskId() != null) { + if (mergeRequestFields && entity.getTaskId() != null) { mergeQueueFieldsFromRequest(vo, entity.getTaskId()); } return vo; } private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) { - TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); + attachFileJobState(vo, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId())); + } + + private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity, TaskFileJobEntity job) { vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); if (job == null) { vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); @@ -1031,6 +1075,10 @@ public class ProductRiskTaskService { return t == null ? null : t.toString(); } + private static long elapsedMs(long startInclusive, long endExclusive) { + return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L); + } + private List dedupeQueueItems(List raw) { Map byNorm = new LinkedHashMap<>(); for (ProductRiskShopQueueItemVo item : raw) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java index c0859ee..0a76427 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.queryasin.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.util.DownloadHeaderUtil; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo; @@ -23,7 +24,6 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -37,7 +37,6 @@ import org.springframework.web.server.ResponseStatusException; import java.io.InputStream; import java.net.URI; -import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; @@ -269,10 +268,8 @@ public class QueryAsinTaskController { throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); } try { - String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20"); response.setContentType("application/octet-stream"); - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename); + DownloadHeaderUtil.setAttachment(response, filename); try (InputStream in = URI.create(url).toURL().openStream()) { byte[] buffer = new byte[65536]; int read; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java index 35c74a4..6ef72b0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java @@ -135,27 +135,58 @@ public class QueryAsinTaskService { } public QueryAsinHistoryVo listHistory(Long userId) { + long startedAt = System.nanoTime(); validateUserId(userId); QueryAsinHistoryVo vo = new QueryAsinHistoryVo(); List entities = fileResultMapper.selectList(new LambdaQueryWrapper() + .select(FileResultEntity::getId, + FileResultEntity::getTaskId, + FileResultEntity::getModuleType, + FileResultEntity::getSourceFilename, + FileResultEntity::getSourceFileUrl, + FileResultEntity::getResultFilename, + FileResultEntity::getResultFileUrl, + FileResultEntity::getSuccess, + FileResultEntity::getErrorMessage, + FileResultEntity::getUserId, + FileResultEntity::getCreatedAt) .eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getUserId, userId) .orderByDesc(FileResultEntity::getCreatedAt) .last("limit 100")); + long resultRowsLoadedAt = System.nanoTime(); if (entities.isEmpty()) { vo.setItems(List.of()); + log.info("[query-asin] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0", + userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt)); return vo; } - Map taskMap = loadTaskMap(entities); - Map> snapshotMap = buildSnapshotMap(taskMap); + Map taskMap = loadHistoryTaskMap(entities); + long tasksLoadedAt = System.nanoTime(); + Map jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream() + .map(FileResultEntity::getId) + .filter(id -> id != null && id > 0) + .distinct() + .toList()); + long jobsLoadedAt = System.nanoTime(); List items = new ArrayList<>(); for (FileResultEntity entity : entities) { FileTaskEntity task = taskMap.get(entity.getTaskId()); - QueryAsinResultItemVo snapshot = snapshotMap.getOrDefault(entity.getTaskId(), Map.of()).get(entity.getId()); - items.add(toHistoryItem(entity, task, snapshot)); + items.add(toHistoryItem(entity, task, null, jobMap.get(entity.getId()))); } vo.setItems(items); + long finishedAt = System.nanoTime(); + log.info("[query-asin] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}", + userId, + entities.size(), + taskMap.size(), + jobMap.size(), + elapsedMs(startedAt, finishedAt), + elapsedMs(startedAt, resultRowsLoadedAt), + elapsedMs(resultRowsLoadedAt, tasksLoadedAt), + elapsedMs(tasksLoadedAt, jobsLoadedAt), + elapsedMs(jobsLoadedAt, finishedAt)); return vo; } @@ -510,6 +541,31 @@ public class QueryAsinTaskService { return taskMap; } + private Map loadHistoryTaskMap(List entities) { + List taskIds = entities.stream() + .map(FileResultEntity::getTaskId) + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + Map taskMap = new LinkedHashMap<>(); + if (taskIds.isEmpty()) { + return taskMap; + } + int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize()); + for (int start = 0; start < taskIds.size(); start += batchSize) { + int end = Math.min(start + batchSize, taskIds.size()); + List tasks = fileTaskMapper.selectList(new LambdaQueryWrapper() + .select(FileTaskEntity::getId, FileTaskEntity::getStatus, FileTaskEntity::getFinishedAt) + .in(FileTaskEntity::getId, taskIds.subList(start, end))); + for (FileTaskEntity task : tasks) { + if (task != null) { + taskMap.put(task.getId(), task); + } + } + } + return taskMap; + } + private Map> buildSnapshotMap(Map taskMap) { Map> out = new LinkedHashMap<>(); for (Map.Entry entry : taskMap.entrySet()) { @@ -523,6 +579,10 @@ public class QueryAsinTaskService { } private QueryAsinResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, QueryAsinResultItemVo snapshot) { + return toHistoryItem(entity, task, snapshot, null); + } + + private QueryAsinResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, QueryAsinResultItemVo snapshot, TaskFileJobEntity job) { QueryAsinResultItemVo item = snapshot != null ? snapshot : new QueryAsinResultItemVo(); item.setResultId(entity.getId()); item.setTaskId(entity.getTaskId()); @@ -535,7 +595,7 @@ public class QueryAsinTaskService { item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt()); item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename())); item.setDownloadUrl(null); - attachFileJobState(item, entity); + attachFileJobState(item, entity, job); if (item.getCountryResults() == null) { item.setCountryResults(new ArrayList<>()); } @@ -546,7 +606,10 @@ public class QueryAsinTaskService { } private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity) { - TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); + attachFileJobState(item, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId())); + } + + private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) { item.setFileReady(!blank(entity.getResultFileUrl())); if (job == null) { item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null); @@ -1061,6 +1124,10 @@ public class QueryAsinTaskService { return value == null || value.isBlank(); } + private static long elapsedMs(long startInclusive, long endExclusive) { + return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L); + } + private String blankToNull(String value) { return blank(value) ? null : value.trim(); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/controller/ShopMatchController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/controller/ShopMatchController.java index 3d43e77..362c82a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/controller/ShopMatchController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/controller/ShopMatchController.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.shopmatch.controller; import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.util.DownloadHeaderUtil; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCountryPreferenceSaveRequest; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest; @@ -23,7 +24,6 @@ import io.swagger.v3.oas.annotations.enums.ParameterIn; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -38,7 +38,6 @@ import org.springframework.web.server.ResponseStatusException; import java.io.InputStream; import java.net.URI; -import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; @@ -193,10 +192,8 @@ public class ShopMatchController { throw new ResponseStatusException(HttpStatus.NOT_FOUND, "No downloadable result"); } try { - String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20"); response.setContentType("application/octet-stream"); - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename); + DownloadHeaderUtil.setAttachment(response, filename); try (InputStream in = URI.create(url).toURL().openStream()) { byte[] buffer = new byte[65536]; int read; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java index 17035e6..87fdff0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java @@ -173,15 +173,34 @@ public class ShopMatchTaskService { } public ProductRiskHistoryVo listHistory(Long userId) { + long startedAt = System.nanoTime(); if (userId == null || userId <= 0) { throw new BusinessException("user_id 不合法"); } ProductRiskHistoryVo vo = new ProductRiskHistoryVo(); List entities = fileResultMapper.selectList(new LambdaQueryWrapper() + .select(FileResultEntity::getId, + FileResultEntity::getTaskId, + FileResultEntity::getModuleType, + FileResultEntity::getSourceFilename, + FileResultEntity::getSourceFileUrl, + FileResultEntity::getResultFilename, + FileResultEntity::getResultFileUrl, + FileResultEntity::getSuccess, + FileResultEntity::getErrorMessage, + FileResultEntity::getUserId, + FileResultEntity::getCreatedAt) .eq(FileResultEntity::getModuleType, MODULE_TYPE) .eq(FileResultEntity::getUserId, userId) .orderByDesc(FileResultEntity::getCreatedAt) .last("limit 100")); + long resultRowsLoadedAt = System.nanoTime(); + if (entities.isEmpty()) { + vo.setItems(List.of()); + log.info("[shop-match] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0", + userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt)); + return vo; + } Map statusByTaskId = new LinkedHashMap<>(); List taskIds = entities.stream() .map(FileResultEntity::getTaskId) @@ -195,11 +214,29 @@ public class ShopMatchTaskService { } } } + long tasksLoadedAt = System.nanoTime(); + Map jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream() + .map(FileResultEntity::getId) + .filter(id -> id != null && id > 0) + .distinct() + .toList()); + long jobsLoadedAt = System.nanoTime(); List items = new ArrayList<>(); for (FileResultEntity entity : entities) { - items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()))); + items.add(toHistoryItemVo(entity, statusByTaskId.get(entity.getTaskId()), jobMap.get(entity.getId()), false)); } vo.setItems(items); + long finishedAt = System.nanoTime(); + log.info("[shop-match] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}", + userId, + entities.size(), + statusByTaskId.size(), + jobMap.size(), + elapsedMs(startedAt, finishedAt), + elapsedMs(startedAt, resultRowsLoadedAt), + elapsedMs(resultRowsLoadedAt, tasksLoadedAt), + elapsedMs(tasksLoadedAt, jobsLoadedAt), + elapsedMs(jobsLoadedAt, finishedAt)); return vo; } @@ -867,6 +904,10 @@ public class ShopMatchTaskService { } private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus) { + return toHistoryItemVo(entity, taskStatus, null, true); + } + + private ProductRiskResultItemVo toHistoryItemVo(FileResultEntity entity, String taskStatus, TaskFileJobEntity job, boolean mergeRequestFields) { ProductRiskResultItemVo vo = new ProductRiskResultItemVo(); vo.setResultId(entity.getId()); vo.setTaskId(entity.getTaskId()); @@ -878,16 +919,19 @@ public class ShopMatchTaskService { vo.setError(entity.getErrorMessage()); vo.setOutputFilename(entity.getResultFilename()); vo.setDownloadUrl(null); - attachFileJobState(vo, entity); + attachFileJobState(vo, entity, job); vo.setMatched(true); - if (entity.getTaskId() != null) { + if (mergeRequestFields && entity.getTaskId() != null) { mergeQueueFieldsFromRequest(vo, entity.getTaskId()); } return vo; } private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) { - TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); + attachFileJobState(vo, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId())); + } + + private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity, TaskFileJobEntity job) { vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); if (job == null) { vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); @@ -1031,6 +1075,10 @@ public class ShopMatchTaskService { return time == null ? null : time.toString(); } + private static long elapsedMs(long startInclusive, long endExclusive) { + return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L); + } + private LocalDateTime now() { return LocalDateTime.now(BUSINESS_ZONE); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinCozeClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinCozeClient.java new file mode 100644 index 0000000..0b9bdc3 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinCozeClient.java @@ -0,0 +1,990 @@ +package com.nanri.aiimage.modules.similarasin.client; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.config.SimilarAsinProperties; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.MediaType; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.stereotype.Component; +import org.springframework.util.StreamUtils; +import org.springframework.web.client.RestClient; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +@Component +@RequiredArgsConstructor +@Slf4j +public class SimilarAsinCozeClient { + + private final SimilarAsinProperties properties; + private final ObjectMapper objectMapper; + + public List inspect(List rows, String prompt) { + if (rows == null || rows.isEmpty()) { + return List.of(); + } + if (properties.getCozeToken() == null || properties.getCozeToken().isBlank()) { + log.warn("[similar-asin] coze token not configured, keep raw rows size={}", rows.size()); + return rows.stream().map(this::copy).toList(); + } + try { + return inspectWithFallback(rows, prompt); + } catch (Exception ex) { + String failureMessage = failureMessage(ex); + log.warn("[similar-asin] coze batch failed size={} err={}", rows.size(), failureMessage); + return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList(); + } + } + + public CozeSubmitResponse submitWorkflow(List rows, String prompt) throws Exception { + JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt)); + ensureSuccess(submitRoot); + return new CozeSubmitResponse( + extractExecuteId(submitRoot), + extractResultDataText(submitRoot), + writeJson(submitRoot) + ); + } + + public CozePollResponse pollWorkflow(String executeId) throws Exception { + JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId)); + ensureSuccess(pollRoot); + String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT); + String dataText = extractResultDataText(pollRoot); + String outputText = dataText.isBlank() ? extractWorkflowOutputText(pollRoot) : ""; + String failureMessage = isFailedWorkflowStatus(status) + ? firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed") + : ""; + return new CozePollResponse(executeId, status, dataText, outputText, failureMessage, writeJson(pollRoot)); + } + + public List mergeRowsFromDataText(List rows, String dataText) throws Exception { + if (dataText == null || dataText.isBlank()) { + return rows == null ? List.of() : rows.stream().map(this::copy).toList(); + } + return mergeRows(rows, parseResults(wrapDataPayload(dataText))); + } + + public List markRowsFailed(List rows, String failureMessage) { + if (rows == null || rows.isEmpty()) { + return List.of(); + } + return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList(); + } + + private List inspectWithFallback(List rows, String prompt) { + try { + if (rows.size() == 1) { + return inspectSingleRowWithRetry(rows, prompt); + } + InspectAttempt attempt = inspectOnce(rows, prompt); + if (attempt.resolvedCount() < rows.size()) { + throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount()); + } + return attempt.mergedRows(); + } catch (Exception ex) { + if (shouldSplitBatch(rows, ex)) { + int middle = rows.size() / 2; + log.warn("[similar-asin] coze batch fallback split size={} left={} right={} err={}", + rows.size(), middle, rows.size() - middle, failureMessage(ex)); + List merged = new ArrayList<>(rows.size()); + merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt)); + merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt)); + return merged; + } + throw propagate(ex); + } + } + + private List inspectPartitionWithFailureFallback(List rows, String prompt) { + try { + return inspectWithFallback(rows, prompt); + } catch (Exception ex) { + String failureMessage = failureMessage(ex); + log.warn("[similar-asin] coze partition failed size={} err={}", rows.size(), failureMessage); + return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList(); + } + } + + private List inspectSingleRowWithRetry(List rows, String prompt) throws Exception { + SimilarAsinResultRowDto row = rows.getFirst(); + PartialCozeResultException lastFailure = null; + for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) { + try { + InspectAttempt attempt = inspectOnce(rows, prompt); + if (attempt.resolvedCount() == rows.size()) { + return attempt.mergedRows(); + } + log.warn("[similar-asin] coze single unresolved attempt={} rowId={} asin={} country={} title={} url={} raw={}", + attemptIndex, + row.getId(), + row.getAsin(), + row.getCountry(), + abbreviate(row.getTitle(), 120), + abbreviate(row.getUrl(), 120), + abbreviate(attempt.raw(), 500)); + lastFailure = new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount()); + } catch (Exception ex) { + if (attemptIndex >= 3 || !isRetryableBatchFailure(ex)) { + throw ex; + } + log.warn("[similar-asin] coze single retryable failure attempt={} rowId={} asin={} country={} err={}", + attemptIndex, + row.getId(), + row.getAsin(), + row.getCountry(), + failureMessage(ex)); + } + if (attemptIndex < 3) { + sleepBeforeRetry(attemptIndex); + } + } + throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure; + } + + private InspectAttempt inspectOnce(List rows, String prompt) throws Exception { + String raw = runWorkflowAsyncAndWait(rows, prompt); + List results = parseResults(raw); + if (rows.size() > 1 && !results.isEmpty() && results.stream().noneMatch(this::hasIdentity)) { + throw new PartialCozeResultException(0, rows.size(), results.size()); + } + List merged = mergeRows(rows, results); + return new InspectAttempt(raw, merged, resolvedCount(merged), results.size()); + } + + private String runWorkflowAsyncAndWait(List rows, String prompt) throws Exception { + JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt)); + ensureSuccess(submitRoot); + + String immediateData = extractResultDataText(submitRoot); + if (!immediateData.isBlank()) { + return wrapDataPayload(immediateData); + } + + String executeId = extractExecuteId(submitRoot); + if (executeId == null || executeId.isBlank()) { + throw new IllegalStateException("Coze async execute_id missing"); + } + + long deadline = System.currentTimeMillis() + Math.max(10000, properties.getCozePollTimeoutMillis()); + while (System.currentTimeMillis() < deadline) { + ensureNotInterrupted(); + JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId)); + ensureSuccess(pollRoot); + + String dataText = extractResultDataText(pollRoot); + if (!dataText.isBlank()) { + return wrapDataPayload(dataText); + } + + String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT); + if (isFailedWorkflowStatus(status)) { + throw new IllegalStateException(firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed")); + } + if (isSuccessfulWorkflowStatus(status)) { + String outputText = extractWorkflowOutputText(pollRoot); + if (!outputText.isBlank()) { + return wrapDataPayload(outputText); + } + throw new IllegalStateException("Coze async workflow completed without output"); + } + sleepQuietly(Math.max(200, properties.getCozePollIntervalMillis())); + } + + throw new IllegalStateException("Coze async workflow poll timeout"); + } + + private String postWorkflow(List rows, String prompt) { + Map parameters = buildParameters(rows, prompt); + Map body = new LinkedHashMap<>(); + body.put("workflow_id", properties.getCozeWorkflowId()); + body.put("parameters", parameters); + body.put("is_async", Boolean.TRUE); + log.info("[similar-asin] coze request url={} body={}", + joinUrl(properties.getCozeBaseUrl(), properties.getCozeWorkflowPath()), + writeJson(body)); + + RestClient.RequestBodySpec request = restClient().post() + .uri(joinUrl(properties.getCozeBaseUrl(), properties.getCozeWorkflowPath())) + .headers(headers -> { + headers.setBearerAuth(stripBearer(properties.getCozeToken())); + headers.setContentType(MediaType.APPLICATION_JSON); + }); + request.body(body); + return request.exchange((clientRequest, clientResponse) -> { + byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody()); + String responseText = responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8); + log.info("[similar-asin] coze submit response status={} body={}", + clientResponse.getStatusCode(), + responseText); + return responseText; + }); + } + + private String getWorkflowHistory(String executeId) { + String path = properties.getCozeWorkflowHistoryPath() + .replace("{workflow_id}", properties.getCozeWorkflowId()) + .replace("{execute_id}", executeId); + return restClient().get() + .uri(joinUrl(properties.getCozeBaseUrl(), path)) + .headers(headers -> { + headers.setBearerAuth(stripBearer(properties.getCozeToken())); + headers.setContentType(MediaType.APPLICATION_JSON); + }) + .exchange((clientRequest, clientResponse) -> { + byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody()); + String responseText = responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8); + log.info("[similar-asin] coze history response executeId={} status={} body={}", + executeId, + clientResponse.getStatusCode(), + responseText); + return responseText; + }); + } + + private Map buildParameters(List rows, String prompt) { + List groupKeys = rows.stream().map(row -> nonBlank(row.getGroupKey(), rowKey(row))).toList(); + List rowIds = rows.stream().map(row -> nonBlank(row.getId(), "")).toList(); + List asins = rows.stream().map(row -> nonBlank(row.getAsin(), "")).toList(); + List countries = rows.stream().map(row -> nonBlank(row.getCountry(), "")).toList(); + List titles = rows.stream().map(row -> nonBlank(row.getTitle(), row.getAsin())).toList(); + List urls = rows.stream().map(row -> nonBlank(row.getUrl(), "")).toList(); + List> urlLists = rows.stream().map(SimilarAsinResultRowDto::getUrls).toList(); + + Map parameters = new LinkedHashMap<>(); + parameters.put("title_list", titles); + parameters.put("url_list", urls); + parameters.put("url_lists", urlLists); + parameters.put("items", buildItemObjects(rows, groupKeys, rowIds, asins, countries, titles, urlLists)); + parameters.put("prompt", prompt == null ? "" : prompt); + return parameters; + } + + private List> buildItemObjects(List rows, + List groupKeys, + List rowIds, + List asins, + List countries, + List titles, + List> urlLists) { + List> items = new ArrayList<>(rows.size()); + for (int i = 0; i < rows.size(); i++) { + Map item = new LinkedHashMap<>(); + item.put("group_key", groupKeys.get(i)); + item.put("row_id", rowIds.get(i)); + item.put("asin", asins.get(i)); + item.put("country", countries.get(i)); + item.put("title", titles.get(i)); + item.put("url", urlLists.get(i)); + item.put("urls", urlLists.get(i)); + items.add(item); + } + return items; + } + + private List parseResults(String raw) throws Exception { + JsonNode root = objectMapper.readTree(raw); + ensureSuccess(root); + String dataText = extractResultDataText(root); + if (dataText.isBlank()) { + return List.of(); + } + JsonNode dataRoot = objectMapper.readTree(dataText); + JsonNode array = dataRoot.isArray() ? dataRoot : dataRoot.path("data"); + List results = new ArrayList<>(); + if (array.isArray()) { + for (JsonNode node : array) { + JsonNode itemNode = resultItemNode(node); + results.add(new CozeResult( + text(firstNonNull( + firstNonNull(node.get("group_key"), node.get("groupKey")), + firstNonNull(itemNode.get("group_key"), itemNode.get("groupKey")))), + text(firstNonNull( + firstNonNull(node.get("row_id"), firstNonNull(node.get("rowId"), node.get("id"))), + firstNonNull(itemNode.get("row_id"), firstNonNull(itemNode.get("rowId"), itemNode.get("id"))))), + text(firstNonNull(node.get("asin"), itemNode.get("asin"))), + text(firstNonNull( + firstNonNull(node.get("country"), node.get("site")), + firstNonNull(itemNode.get("country"), itemNode.get("site")))), + text(firstNonNull(node.get("title"), firstNonNull(itemNode.get("title"), itemNode.get("title_risk")))), + text(firstNonNull(node.get("appearance"), + firstNonNull(node.get("appearance_risk"), + firstNonNull(itemNode.get("appearance"), itemNode.get("appearance_risk"))))), + text(firstNonNull(firstNonNull(node.get("patent"), node.get("patent ")), + firstNonNull(node.get("patent_risk"), + firstNonNull(firstNonNull(itemNode.get("patent"), itemNode.get("patent ")), itemNode.get("patent_risk"))))), + text(firstNonNull(node.get("result"), + firstNonNull(node.get("conclusion"), + firstNonNull(itemNode.get("result"), itemNode.get("conclusion"))))), + text(firstNonNull(node.get("title_reason"), + firstNonNull(node.get("titleReason"), + firstNonNull(itemNode.get("title_reason"), itemNode.get("titleReason"))))), + text(firstNonNull(node.get("appearance_reason"), + firstNonNull(node.get("appearanceReason"), + firstNonNull(itemNode.get("appearance_reason"), itemNode.get("appearanceReason"))))), + text(firstNonNull(node.get("patent_reason"), + firstNonNull(firstNonNull(node.get("patentReason"), node.get("patent reason")), + firstNonNull(itemNode.get("patent_reason"), + firstNonNull(itemNode.get("patentReason"), itemNode.get("patent reason")))))) + )); + } + } + return results; + } + + private JsonNode resultItemNode(JsonNode node) { + if (node == null || node.isMissingNode() || node.isNull()) { + return objectMapper.missingNode(); + } + JsonNode item = firstNonNull(node.get("item"), node.get("items")); + if (item == null || item.isMissingNode() || item.isNull()) { + return objectMapper.missingNode(); + } + if (item.isArray()) { + return item.isEmpty() ? objectMapper.missingNode() : item.get(0); + } + return item; + } + + private List mergeRows(List rows, List results) { + Map resultByGroupKey = new LinkedHashMap<>(); + Map resultByCompositeKey = new LinkedHashMap<>(); + Map resultByAsinCountry = new LinkedHashMap<>(); + Map resultByAsin = new LinkedHashMap<>(); + Map resultByRowId = new LinkedHashMap<>(); + for (CozeResult result : results) { + String groupKey = normalize(result.groupKey()); + if (!groupKey.isBlank()) { + resultByGroupKey.putIfAbsent(groupKey, result); + } + String compositeKey = rowKey(result.rowId(), result.asin(), result.country()); + if (!compositeKey.isBlank()) { + resultByCompositeKey.putIfAbsent(compositeKey, result); + } + String asinCountryKey = asinCountryKey(result.asin(), result.country()); + if (!asinCountryKey.isBlank()) { + resultByAsinCountry.putIfAbsent(asinCountryKey, result); + } + String asinKey = normalize(result.asin()).toUpperCase(Locale.ROOT); + if (!asinKey.isBlank()) { + resultByAsin.putIfAbsent(asinKey, result); + } + String rowIdKey = normalize(result.rowId()); + if (!rowIdKey.isBlank()) { + resultByRowId.putIfAbsent(rowIdKey, result); + } + } + + List merged = new ArrayList<>(rows.size()); + boolean allowIndexFallback = results.size() == rows.size() && results.stream().noneMatch(this::hasIdentity); + for (int i = 0; i < rows.size(); i++) { + SimilarAsinResultRowDto row = copy(rows.get(i)); + CozeResult result = resultByGroupKey.get(normalize(row.getGroupKey())); + if (result == null) { + result = resultByCompositeKey.get(rowKey(row.getId(), row.getAsin(), row.getCountry())); + } + if (result == null) { + result = resultByAsinCountry.get(asinCountryKey(row.getAsin(), row.getCountry())); + } + if (result == null) { + result = resultByAsin.get(normalize(row.getAsin()).toUpperCase(Locale.ROOT)); + } + if (result == null) { + result = resultByRowId.get(normalize(row.getId())); + } + if (result == null && allowIndexFallback && i < results.size()) { + result = results.get(i); + } + applyResult(row, result); + merged.add(row); + } + return merged; + } + + private String asinCountryKey(String asin, String country) { + String normalizedAsin = normalize(asin).toUpperCase(Locale.ROOT); + if (normalizedAsin.isBlank()) { + return ""; + } + return normalizedAsin + "::" + normalize(country); + } + + private boolean hasIdentity(CozeResult result) { + if (result == null) { + return false; + } + return !normalize(result.groupKey()).isBlank() + || !normalize(result.rowId()).isBlank() + || !normalize(result.asin()).isBlank(); + } + + private void applyResult(SimilarAsinResultRowDto row, CozeResult result) { + if (row == null || result == null) { + return; + } + row.setTitleRisk(result.title()); + row.setAppearanceRisk(result.appearance()); + row.setPatentRisk(result.patent()); + row.setConclusion(result.result()); + row.setTitleReason(result.titleReason()); + row.setAppearanceReason(result.appearanceReason()); + row.setPatentReason(result.patentReason()); + } + + private RestClient restClient() { + SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); + requestFactory.setConnectTimeout(properties.getCozeConnectTimeoutMillis()); + requestFactory.setReadTimeout(properties.getCozeReadTimeoutMillis()); + return RestClient.builder().requestFactory(requestFactory).build(); + } + + private SimilarAsinResultRowDto copy(SimilarAsinResultRowDto source) { + SimilarAsinResultRowDto row = new SimilarAsinResultRowDto(); + row.setSourceFileKey(source.getSourceFileKey()); + row.setSourceFilename(source.getSourceFilename()); + row.setRowToken(source.getRowToken()); + row.setGroupKey(source.getGroupKey()); + row.setId(source.getId()); + row.setAsin(source.getAsin()); + row.setCountry(source.getCountry()); + row.setUrls(source.getUrls()); + row.setTitle(source.getTitle()); + row.setError(source.getError()); + row.setDone(source.getDone()); + row.setTitleRisk(source.getTitleRisk()); + row.setAppearanceRisk(source.getAppearanceRisk()); + row.setPatentRisk(source.getPatentRisk()); + row.setConclusion(source.getConclusion()); + row.setTitleReason(source.getTitleReason()); + row.setAppearanceReason(source.getAppearanceReason()); + row.setPatentReason(source.getPatentReason()); + return row; + } + + private SimilarAsinResultRowDto markFailed(SimilarAsinResultRowDto row, String failureMessage) { + String reviewMessage = failureMessage == null || failureMessage.isBlank() + ? "Coze 检测失败，待人工复核" + : "Coze 检测失败，待人工复核：" + failureMessage; + if (row.getError() == null || row.getError().isBlank()) { + row.setError(failureMessage); + } + if (row.getTitleRisk() == null || row.getTitleRisk().isBlank()) { + row.setTitleRisk(reviewMessage); + } + if (row.getAppearanceRisk() == null || row.getAppearanceRisk().isBlank()) { + row.setAppearanceRisk(reviewMessage); + } + if (row.getPatentRisk() == null || row.getPatentRisk().isBlank()) { + row.setPatentRisk(reviewMessage); + } + if (row.getConclusion() == null || row.getConclusion().isBlank()) { + row.setConclusion(failureMessage); + } + return row; + } + + private boolean shouldSplitBatch(List rows, Exception ex) { + return rows != null && rows.size() > 1 && isRetryableBatchFailure(ex); + } + + private boolean isRetryableBatchFailure(Exception ex) { + if (ex instanceof PartialCozeResultException) { + return true; + } + String message = ex == null ? "" : nonBlank(ex.getMessage(), ""); + return message.contains("Workflow node execution limit exceeded") + || message.contains("Read timed out") + || message.contains("Connection reset") + || message.contains("I/O error on POST request") + || message.toLowerCase(Locale.ROOT).contains("timeout"); + } + + private int resolvedCount(List rows) { + int resolved = 0; + for (SimilarAsinResultRowDto row : rows) { + if (hasResolvedCozeFields(row)) { + resolved++; + } + } + return resolved; + } + + private boolean hasResolvedCozeFields(SimilarAsinResultRowDto row) { + if (row == null) { + return false; + } + return !normalize(row.getTitleRisk()).isBlank() + || !normalize(row.getAppearanceRisk()).isBlank() + || !normalize(row.getPatentRisk()).isBlank() + || !normalize(row.getConclusion()).isBlank(); + } + + private void ensureSuccess(JsonNode root) { + if (root.path("code").asInt(-1) != 0) { + throw new IllegalStateException(root.path("msg").asText("Coze response code is not 0")); + } + } + + private String extractResultDataText(JsonNode root) { + if (root == null || root.isMissingNode() || root.isNull()) { + return ""; + } + + JsonNode dataNode = root.path("data"); + if (dataNode.isTextual()) { + String value = dataNode.asText(""); + if (looksLikeResultDataPayload(value)) { + return value; + } + return discoverEmbeddedData(parseJsonOrMissing(value)); + } + if (dataNode.isObject()) { + String nested = text(firstNonNull(dataNode.get("data"), firstNonNull(dataNode.get("output"), dataNode.get("result")))); + if (nested != null && !nested.isBlank() && looksLikeResultDataPayload(nested)) { + return nested; + } + JsonNode outputs = firstNonNull(dataNode.get("outputs"), dataNode.get("details")); + String discovered = discoverEmbeddedData(outputs); + if (!discovered.isBlank()) { + return discovered; + } + } + + return discoverEmbeddedData(root); + } + + private String extractWorkflowOutputText(JsonNode root) { + JsonNode outputNode = findFirstField(root, "output"); + if (outputNode == null || outputNode.isNull() || outputNode.isMissingNode()) { + return ""; + } + if (!outputNode.isTextual()) { + String discovered = discoverEmbeddedData(outputNode); + return discovered.isBlank() && isResultDataPayload(outputNode) ? outputNode.toString() : discovered; + } + String output = normalize(outputNode.asText("")); + if (output.isBlank()) { + return ""; + } + if (looksLikeResultDataPayload(output)) { + return output; + } + JsonNode parsedOutput = parseJsonOrMissing(output); + String nestedOutput = text(firstNonNull(parsedOutput.get("Output"), parsedOutput.get("output"))); + if (nestedOutput != null && !nestedOutput.isBlank() && looksLikeResultDataPayload(nestedOutput)) { + return nestedOutput; + } + return discoverEmbeddedData(parsedOutput); + } + + private String discoverEmbeddedData(JsonNode node) { + if (node == null || node.isNull() || node.isMissingNode()) { + return ""; + } + if (node.isTextual()) { + String value = node.asText(""); + if (looksLikeResultDataPayload(value)) { + return value; + } + return discoverEmbeddedData(parseJsonOrMissing(value)); + } + if (node.isArray()) { + for (JsonNode child : node) { + String discovered = discoverEmbeddedData(child); + if (!discovered.isBlank()) { + return discovered; + } + } + return ""; + } + if (node.isObject()) { + if (isResultDataPayload(node)) { + return node.toString(); + } + for (java.util.Iterator> it = node.fields(); it.hasNext(); ) { + Map.Entry entry = it.next(); + String discovered = discoverEmbeddedData(entry.getValue()); + if (!discovered.isBlank()) { + return discovered; + } + } + } + return ""; + } + + private boolean looksLikeResultDataPayload(String value) { + JsonNode parsed = parseJsonOrMissing(value); + return isResultDataPayload(parsed); + } + + private boolean isResultDataPayload(JsonNode node) { + if (node == null || node.isMissingNode() || node.isNull()) { + return false; + } + JsonNode array = node.path("data"); + if (!array.isArray()) { + return false; + } + for (JsonNode item : array) { + JsonNode itemNode = resultItemNode(item); + if (hasResultFields(item) || hasResultFields(itemNode)) { + return true; + } + } + return false; + } + + private boolean hasResultFields(JsonNode item) { + return item != null + && (item.has("appearance") + || item.has("appearance_risk") + || item.has("patent") + || item.has("patent ") + || item.has("patent_risk") + || item.has("result") + || item.has("conclusion") + || item.has("title_reason") + || item.has("titleReason") + || item.has("appearance_reason") + || item.has("appearanceReason") + || item.has("patent_reason") + || item.has("patentReason") + || item.has("patent reason")); + } + + private boolean isSuccessfulWorkflowStatus(String status) { + String normalized = normalize(status).toUpperCase(Locale.ROOT); + return normalized.equals("SUCCESS") + || normalized.equals("SUCCEEDED") + || normalized.equals("COMPLETED") + || normalized.equals("COMPLETE") + || normalized.equals("DONE"); + } + + private boolean isFailedWorkflowStatus(String status) { + String normalized = normalize(status).toUpperCase(Locale.ROOT); + return normalized.contains("FAIL") + || normalized.contains("ERROR") + || normalized.contains("CANCEL"); + } + + private JsonNode parseJsonOrMissing(String value) { + String normalized = normalize(value); + if (!(normalized.startsWith("{") || normalized.startsWith("["))) { + return objectMapper.missingNode(); + } + try { + return objectMapper.readTree(normalized); + } catch (Exception ignored) { + return objectMapper.missingNode(); + } + } + + private String resolveWorkflowStatus(JsonNode root) { + JsonNode dataNode = root.path("data"); + JsonNode statusNode = firstNonNull( + firstNonNull(dataNode.get("status"), dataNode.get("execute_status")), + firstNonNull(root.get("status"), root.get("execute_status"))); + String status = text(statusNode); + if (status != null && !status.isBlank()) { + return status; + } + return findTextByFieldName(root, "execute_status", "status"); + } + + private String resolveFailureMessage(JsonNode root) { + JsonNode dataNode = root.path("data"); + String message = text(firstNonNull(dataNode.get("error_message"), firstNonNull(dataNode.get("msg"), root.get("msg")))); + return message == null ? "" : message; + } + + private String extractExecuteId(JsonNode root) { + return findTextByFieldName(root, "execute_id", "executeId"); + } + + private JsonNode findFirstField(JsonNode node, String name) { + if (node == null || node.isMissingNode() || node.isNull() || name == null || name.isBlank()) { + return objectMapper.missingNode(); + } + if (node.isTextual()) { + return findFirstField(parseJsonOrMissing(node.asText("")), name); + } + if (node.isArray()) { + for (JsonNode child : node) { + JsonNode found = findFirstField(child, name); + if (found != null && !found.isMissingNode() && !found.isNull()) { + return found; + } + } + return objectMapper.missingNode(); + } + if (node.isObject()) { + JsonNode direct = node.get(name); + if (direct != null && !direct.isMissingNode() && !direct.isNull()) { + return direct; + } + for (java.util.Iterator> it = node.fields(); it.hasNext(); ) { + Map.Entry entry = it.next(); + JsonNode found = findFirstField(entry.getValue(), name); + if (found != null && !found.isMissingNode() && !found.isNull()) { + return found; + } + } + } + return objectMapper.missingNode(); + } + + private String findTextByFieldName(JsonNode node, String... names) { + if (node == null || node.isMissingNode() || node.isNull()) { + return ""; + } + if (node.isTextual()) { + return findTextByFieldName(parseJsonOrMissing(node.asText("")), names); + } + if (node.isArray()) { + for (JsonNode child : node) { + String found = findTextByFieldName(child, names); + if (!found.isBlank()) { + return found; + } + } + return ""; + } + if (node.isObject()) { + for (String name : names) { + String value = text(node.get(name)); + if (value != null && !value.isBlank()) { + return value; + } + } + JsonNode dataNode = node.get("data"); + if (dataNode != null && dataNode.isTextual()) { + String found = findTextByFieldName(parseJsonOrMissing(dataNode.asText("")), names); + if (!found.isBlank()) { + return found; + } + } + for (java.util.Iterator> it = node.fields(); it.hasNext(); ) { + Map.Entry entry = it.next(); + String found = findTextByFieldName(entry.getValue(), names); + if (!found.isBlank()) { + return found; + } + } + } + return ""; + } + + private String wrapDataPayload(String dataText) { + Map payload = new LinkedHashMap<>(); + payload.put("code", 0); + payload.put("data", dataText); + return writeJson(payload); + } + + private String writeJson(Object value) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new IllegalStateException("Failed to serialize Coze payload", ex); + } + } + + private String abbreviate(String value, int maxLength) { + String normalized = value == null ? "" : value.trim(); + if (normalized.length() <= maxLength) { + return normalized; + } + return normalized.substring(0, Math.max(0, maxLength - 3)) + "..."; + } + + private void sleepBeforeRetry(int attemptIndex) { + long delayMillis = Math.max(1, attemptIndex) * 1500L; + ensureNotInterrupted(); + sleepQuietly(delayMillis); + } + + private void sleepQuietly(long delayMillis) { + try { + Thread.sleep(delayMillis); + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Coze workflow interrupted", interruptedException); + } + } + + private void ensureNotInterrupted() { + if (Thread.currentThread().isInterrupted()) { + throw new IllegalStateException("Coze workflow interrupted"); + } + } + + private RuntimeException propagate(Exception ex) { + if (ex instanceof RuntimeException runtimeException) { + return runtimeException; + } + return new IllegalStateException(nonBlank(ex.getMessage(), "Coze call failed"), ex); + } + + private JsonNode firstNonNull(JsonNode left, JsonNode right) { + return left == null || left.isNull() ? right : left; + } + + private String text(JsonNode node) { + return node == null || node.isNull() ? null : node.asText(); + } + + private String nonBlank(String value, String fallback) { + return value == null || value.isBlank() ? fallback : value; + } + + private String firstNonBlank(String preferred, String fallback) { + return preferred == null || preferred.isBlank() ? fallback : preferred.trim(); + } + + private String normalize(String value) { + return value == null ? "" : value.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim(); + } + + private String rowKey(SimilarAsinResultRowDto row) { + if (row == null) { + return ""; + } + return rowKey(row.getId(), row.getAsin(), row.getCountry()); + } + + private String rowKey(String rowId, String asin, String country) { + return normalize(rowId) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country); + } + + private String failureMessage(Exception ex) { + if (ex instanceof PartialCozeResultException partial) { + return "Coze result incomplete(" + partial.resolvedCount() + "/" + partial.expectedCount() + ")"; + } + String message = ex == null ? null : ex.getMessage(); + if (message == null || message.isBlank()) { + return "Coze call failed"; + } + if (message.contains("Workflow node execution limit exceeded")) { + return "Coze workflow node execution limit exceeded"; + } + if (message.contains("Read timed out")) { + return "Coze call timed out"; + } + return "Coze call failed: " + message; + } + + private String stripBearer(String token) { + String normalized = token == null ? "" : token.trim(); + return normalized.regionMatches(true, 0, "Bearer ", 0, 7) ? normalized.substring(7).trim() : normalized; + } + + private String joinUrl(String baseUrl, String path) { + String base = baseUrl == null ? "" : baseUrl.trim(); + String suffix = path == null ? "" : path.trim(); + if (base.endsWith("/") && suffix.startsWith("/")) { + return base + suffix.substring(1); + } + if (!base.endsWith("/") && !suffix.startsWith("/")) { + return base + "/" + suffix; + } + return base + suffix; + } + + private record CozeResult( + String groupKey, + String rowId, + String asin, + String country, + String title, + String appearance, + String patent, + String result, + String titleReason, + String appearanceReason, + String patentReason + ) { + } + + private record InspectAttempt( + String raw, + List mergedRows, + int resolvedCount, + int rawResultCount + ) { + } + + public record CozeSubmitResponse( + String executeId, + String immediateData, + String rawResponse + ) { + } + + public record CozePollResponse( + String executeId, + String status, + String dataText, + String outputText, + String failureMessage, + String rawResponse + ) { + public boolean hasPayload() { + return dataText != null && !dataText.isBlank() || outputText != null && !outputText.isBlank(); + } + + public String resolvedPayloadText() { + return dataText != null && !dataText.isBlank() ? dataText : outputText; + } + + public boolean isFailed() { + String normalized = status == null ? "" : status.trim().toUpperCase(Locale.ROOT); + return normalized.contains("FAILED") || normalized.contains("ERROR") || normalized.contains("CANCEL"); + } + + public boolean isFinished() { + String normalized = status == null ? "" : status.trim().toUpperCase(Locale.ROOT); + return isFailed() + || normalized.contains("SUCCESS") + || normalized.contains("SUCCEED") + || normalized.contains("FINISH") + || normalized.contains("DONE") + || normalized.contains("COMPLET"); + } + } + + private static final class PartialCozeResultException extends RuntimeException { + + private final int resolvedCount; + private final int expectedCount; + private final int rawResultCount; + + private PartialCozeResultException(int resolvedCount, int expectedCount, int rawResultCount) { + super("partial-result resolved=" + resolvedCount + "/" + expectedCount + " raw=" + rawResultCount); + this.resolvedCount = resolvedCount; + this.expectedCount = expectedCount; + this.rawResultCount = rawResultCount; + } + + private int resolvedCount() { + return resolvedCount; + } + + private int expectedCount() { + return expectedCount; + } + + @SuppressWarnings("unused") + private int rawResultCount() { + return rawResultCount; + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/controller/SimilarAsinController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/controller/SimilarAsinController.java new file mode 100644 index 0000000..53de381 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/controller/SimilarAsinController.java @@ -0,0 +1,145 @@ +package com.nanri.aiimage.modules.similarasin.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.common.util.DownloadHeaderUtil; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo; +import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/similar-asin") +@Tag(name = "相似ASIN检测", description = "相似ASIN检测任务接口。前端上传 Excel 后由 Java 解析并创建任务；Python 回传商品数据；Java 负责攒批调用 Coze、补齐子行、生成最终 xlsx 并上传 OSS。") +public class SimilarAsinController { + + private final SimilarAsinTaskService service; + + @PostMapping("/parse") + @Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件，提取 id、ASIN、国家、URL、标题等字段。返回给前端的数据只包含整数 id 和 n_1 行；n_2、n_3 等子行会保存在 OSS 解析载荷中，用于最终结果补齐。创建后的任务状态为 PENDING，不会自动推送 Python。") + public ApiResponse parse(@Valid @RequestBody SimilarAsinParseRequest request) { + return ApiResponse.success(service.parseAndCreateTask(request)); + } + + @GetMapping("/dashboard") + @Operation(summary = "查询相似ASIN检测总览", description = "查询当前用户的运行中、成功、失败和已结束任务数量。页面进入时请求一次即可，不需要持续轮询。") + public ApiResponse dashboard( + @Parameter(description = "当前用户 ID，用于隔离不同用户的任务和历史记录。", required = true, example = "1") + @RequestParam("user_id") Long userId) { + return ApiResponse.success(service.dashboard(userId)); + } + + @GetMapping("/history") + @Operation(summary = "查询相似ASIN检测历史", description = "查询当前用户最近的相似ASIN检测历史记录，包含源文件名、任务状态、行数、错误信息和最终 xlsx 下载地址。") + public ApiResponse history( + @Parameter(description = "当前用户 ID，用于查询该用户自己的历史记录。", required = true, example = "1") + @RequestParam("user_id") Long userId, + @Parameter(description = "history limit, default 50, max 100", example = "50") + @RequestParam(value = "limit", required = false, defaultValue = "50") Integer limit) { + return ApiResponse.success(service.history(userId, limit)); + } + + @PostMapping("/tasks/progress/batch") + @Operation(summary = "批量查询任务进度", description = "前端只对活跃任务调用该接口，建议 6 秒一次。接口只返回轻量任务状态，不返回明细结果。") + public ApiResponse progress(@Valid @RequestBody SimilarAsinTaskBatchRequest request) { + return ApiResponse.success(service.progressBatch(request.getTaskIds())); + } + + @PostMapping("/tasks/{taskId}/activate") + @Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用，将任务从 PENDING 改为 RUNNING，并记录后端内部活跃时间。后续活跃时间由 Python 回传结果接口自动刷新，不需要单独心跳接口。") + public ApiResponse activate( + @Parameter(description = "相似ASIN检测任务 ID，即解析接口返回的 taskId。", required = true, example = "3938") + @PathVariable Long taskId, + @Parameter(description = "当前用户 ID，必须与创建任务的用户一致。", required = true, example = "1") + @RequestParam("user_id") Long userId) { + service.activateTask(taskId, userId); + return ApiResponse.success(null); + } + + @PostMapping("/tasks/{taskId}/result") + @Operation(summary = "提交 Python 回传结果", description = "Python 回传商品数据接口。items 可以是一条或多条；Java 先原样保存回传数据，再内部攒够 10 条调用 Coze。done=true 表示 Python 已完成全部回传，Java 会强制处理剩余不足 10 条的数据并生成最终 xlsx。") + public ApiResponse result( + @Parameter(description = "相似ASIN检测任务 ID。任务必须处于 RUNNING 状态。", required = true, example = "3938") + @PathVariable Long taskId, + @Valid @RequestBody SimilarAsinSubmitResultRequest request, + jakarta.servlet.http.HttpServletResponse response) { + response.setCharacterEncoding(StandardCharsets.UTF_8.name()); + response.setContentType("application/json;charset=UTF-8"); + service.submitResult(taskId, request); + return ApiResponse.success(null); + } + + @DeleteMapping("/tasks/{taskId}") + @Operation(summary = "删除任务", description = "删除当前用户的一条相似ASIN检测任务，同时清理任务结果、scope 状态和分片记录。") + public ApiResponse deleteTask( + @Parameter(description = "相似ASIN检测任务 ID。", required = true, example = "3938") + @PathVariable Long taskId, + @Parameter(description = "当前用户 ID，必须与创建任务的用户一致。", required = true, example = "1") + @RequestParam("user_id") Long userId) { + service.deleteTask(taskId, userId); + return ApiResponse.success(null); + } + + @DeleteMapping("/history/{resultId}") + @Operation(summary = "删除历史记录", description = "删除当前用户的一条相似ASIN检测历史记录。只删除 biz_file_result 记录，不主动删除任务主记录。") + public ApiResponse deleteHistory( + @Parameter(description = "历史结果记录 ID，即 history 接口返回的 resultId。", required = true, example = "1001") + @PathVariable Long resultId, + @Parameter(description = "当前用户 ID。", required = true, example = "1") + @RequestParam("user_id") Long userId) { + service.deleteHistory(resultId, userId); + return ApiResponse.success(null); + } + + @GetMapping("/results/{resultId}/download") + @Operation(summary = "下载相似ASIN检测结果文件") + public void downloadResult( + @Parameter(description = "结果记录 ID", required = true, example = "1001") + @PathVariable Long resultId, + @Parameter(description = "当前用户 ID", required = true, example = "1") + @RequestParam("user_id") Long userId, + jakarta.servlet.http.HttpServletResponse response) { + String url = service.resolveResultDownloadUrl(resultId, userId); + String filename = service.resolveResultDownloadFilename(resultId, userId); + if (url == null || url.isBlank()) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); + } + try { + response.setContentType("application/octet-stream"); + DownloadHeaderUtil.setAttachment(response, filename); + try (InputStream in = URI.create(url).toURL().openStream()) { + byte[] buffer = new byte[65536]; + int read; + while ((read = in.read(buffer)) != -1) { + response.getOutputStream().write(buffer, 0, read); + } + response.getOutputStream().flush(); + } + } catch (Exception ex) { + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败"); + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParseRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParseRequest.java new file mode 100644 index 0000000..229e663 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParseRequest.java @@ -0,0 +1,28 @@ +package com.nanri.aiimage.modules.similarasin.model.dto; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "相似ASIN检测解析请求") +public class SimilarAsinParseRequest { + @JsonProperty("user_id") + @NotNull + @Schema(description = "当前用户 ID。后端会把创建的任务、历史记录和结果文件归属到该用户。", example = "1", requiredMode = Schema.RequiredMode.REQUIRED) + private Long userId; + + @NotEmpty + @Schema(description = "已上传的 Excel 文件列表。支持多文件聚合解析；文件对象来自统一上传接口返回值。", requiredMode = Schema.RequiredMode.REQUIRED) + private List files; + + @JsonProperty("ai_prompt") + @JsonAlias({"aiPrompt", "prompt"}) + @Schema(description = "AI 提示词。非必填；为空时前端会使用默认提示词。后端会保存该提示词，并在调用 Coze workflow 时作为 prompt 参数传入。", example = "请排查这些亚马逊商品在英国及欧洲地区是否存在知识产权侵权风险。") + private String aiPrompt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParsedPayloadDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParsedPayloadDto.java new file mode 100644 index 0000000..83216af --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParsedPayloadDto.java @@ -0,0 +1,31 @@ +package com.nanri.aiimage.modules.similarasin.model.dto; + +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "相似ASIN检测解析载荷") +public class SimilarAsinParsedPayloadDto { + @Schema(description = "AI 提示词") + private String aiPrompt; + + @Schema(description = "本次解析的源文件列表") + private List sourceFiles = new ArrayList<>(); + + @Schema(description = "Excel 原始表头列表") + private List headers = new ArrayList<>(); + + @Schema(description = "兼容旧链路的平铺有效行，现为全部有效行") + private List items = new ArrayList<>(); + + @Schema(description = "按相邻主 ID 块分组后的完整数据") + private List groups = new ArrayList<>(); + + @Schema(description = "完整有效行") + private List allItems = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinResultGroupDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinResultGroupDto.java new file mode 100644 index 0000000..21fbffc --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinResultGroupDto.java @@ -0,0 +1,29 @@ +package com.nanri.aiimage.modules.similarasin.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "Python 回传的相似ASIN检测分组结果") +public class SimilarAsinResultGroupDto { + @Schema(description = "来源文件 key") + private String sourceFileKey; + + @Schema(description = "来源文件名") + private String sourceFilename; + + @Schema(description = "相邻主 ID 块的分组 key") + private String groupKey; + + @Schema(description = "主 ID，例如 1、2、10") + private String baseId; + + @Schema(description = "分组首条展示 ID") + private String displayId; + + @Schema(description = "分组内全部抓取结果") + private List items = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinResultRowDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinResultRowDto.java new file mode 100644 index 0000000..e228702 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinResultRowDto.java @@ -0,0 +1,187 @@ +package com.nanri.aiimage.modules.similarasin.model.dto; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.databind.JsonNode; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +@Data +@Schema(description = "相似ASIN检测单行商品数据") +public class SimilarAsinResultRowDto { + private static final int MAX_IMAGE_URLS = 24; + + @Schema(description = "来源文件 key。多文件回传时用于避免行结果串到别的文件。", example = "uploads/20260426/similar_asin_17.xlsx") + private String sourceFileKey; + + @Schema(description = "来源文件名。仅用于调试与追踪。", example = "17.xlsx") + private String sourceFilename; + + @Schema(description = "行级唯一 token。Python 应从解析结果原样透传。", example = "uploads/20260426/similar_asin_17.xlsx::row::2") + private String rowToken; + + @Schema(description = "主数据分组 key。Python 应从解析结果原样透传，用于把同组子行补回。", example = "uploads/20260426/similar_asin_17.xlsx::2@2") + private String groupKey; + + @Schema(description = "Excel 中的 id。代表行通常是整数 id 或 n_1，例如 2_1；最终生成 xlsx 时，2_2、2_3 会复用同组 2_1 的 Coze 检测结果。", example = "2_1") + private String id; + + @Schema(description = "亚马逊 ASIN。后端会统一按大写处理和匹配。", example = "B0CJ8SNXXV") + private String asin; + + @Schema(description = "站点或国家。来自 Excel 的国家列，例如英国、德国、法国。", example = "英国") + private String country; + + @Schema(description = "兼容旧链路的首个图片 URL。新链路允许 url 传字符串或数组；数组会同步写入 urls。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg") + private String url; + + @Schema(description = "商品图片 URL 列表，最多保留 24 个非空地址。请求中也可以直接把 url 传成数组。", example = "[\"https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg\"]") + private List urls = new ArrayList<>(); + + @Schema(description = "商品标题。Java 调用 Coze 时会放入 title_list；为空时会回退使用 ASIN。", example = "Women Floral Dress Summer Casual") + @JsonAlias({"productTitle", "product_title", "itemTitle", "item_title", "商品标题", "商品名称", "标题"}) + private String title; + + @Schema(description = "单行错误信息。通常用于记录 Python 单行处理异常；Coze 失败时后端会尽量保留原始行，不强行写入风险结果。", example = "图片地址为空") + private String error; + + @Schema(description = "单行完成标记。当前主要使用请求体顶层 done 控制任务收尾，该字段仅作兼容。", example = "true") + private Boolean done; + + @Schema(description = "Java 调用 Coze 后生成的标题维度检测结果，对应最终 xlsx 的“标题维度（商标）”列。Python 回传请求中不要传该字段；即使传入，后端也会以 Java/Coze 处理结果为准。", example = "标题未发现明显商标侵权风险。", accessMode = Schema.AccessMode.READ_ONLY) + private String titleRisk; + + @Schema(description = "Java 调用 Coze 后生成的外观维度检测结果，对应最终 xlsx 的“外观维度（外观设计专利）”列。Python 回传请求中不要传该字段。", example = "未发现明显外观设计专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY) + private String appearanceRisk; + + @JsonAlias({"patent ", "patent"}) + @Schema(description = "Java 调用 Coze 后生成的专利维度检测结果，对应最终 xlsx 的“专利维度（发明/实用新型专利）”列。兼容 Coze 返回字段 patent 和 patent 后带空格的情况；Python 回传请求中不要传该字段。", example = "未发现明显发明或实用新型专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY) + private String patentRisk; + + @Schema(description = "Java 调用 Coze 后生成的最终结论，对应最终 xlsx 的“结论”列。Python 回传请求中不要传该字段。", example = "未发现明显侵权风险", accessMode = Schema.AccessMode.READ_ONLY) + private String conclusion; + + @JsonAlias({"title_reason", "titleReason"}) + @Schema(description = "Coze title reason", accessMode = Schema.AccessMode.READ_ONLY) + private String titleReason; + + @JsonAlias({"appearance_reason", "appearanceReason"}) + @Schema(description = "Coze appearance reason", accessMode = Schema.AccessMode.READ_ONLY) + private String appearanceReason; + + @JsonAlias({"patent_reason", "patentReason", "patent reason"}) + @Schema(description = "Coze patent reason", accessMode = Schema.AccessMode.READ_ONLY) + private String patentReason; + + public String getUrl() { + if (url != null && !url.isBlank()) { + return url; + } + List normalized = getUrls(); + return normalized.isEmpty() ? "" : normalized.get(0); + } + + @JsonSetter("url") + @JsonAlias({ + "imageUrl", "image_url", "imgUrl", "img_url", "pictureUrl", "picture_url", + "mainImage", "main_image", "mainImageUrl", "main_image_url", + "productImage", "product_image", "productImageUrl", "product_image_url", + "image", "img", "pic", "picture", "link", "imageLink", "image_link" + }) + public void setUrl(Object value) { + List normalized = normalizeUrls(value); + this.urls = normalized; + this.url = normalized.isEmpty() ? "" : normalized.get(0); + } + + public List getUrls() { + List normalized = normalizeUrls(urls); + if (url != null && !url.isBlank() && normalized.stream().noneMatch(url.trim()::equals)) { + List combined = new ArrayList<>(normalized.size() + 1); + combined.add(url.trim()); + combined.addAll(normalized); + return limitUrls(combined); + } + return normalized; + } + + @JsonSetter("urls") + @JsonAlias({ + "imageUrls", "image_urls", "imgUrls", "img_urls", "pictureUrls", "picture_urls", + "mainImages", "main_images", "mainImageUrls", "main_image_urls", + "productImages", "product_images", "productImageUrls", "product_image_urls", + "images", "imgs", "pics", "pictures", "links", "imageLinks", "image_links", + "图片链接", "商品图片", "商品主图", "主图", "主图链接" + }) + public void setUrls(Object urls) { + List normalized = normalizeUrls(urls); + this.urls = normalized; + this.url = normalized.isEmpty() ? "" : normalized.get(0); + } + + public boolean hasImageUrl() { + return !getUrls().isEmpty(); + } + + private static List normalizeUrls(Object value) { + List result = new ArrayList<>(); + appendUrls(result, value); + return limitUrls(result); + } + + private static void appendUrls(List result, Object value) { + if (value == null) { + return; + } + if (value instanceof JsonNode node) { + if (node.isArray()) { + node.forEach(child -> appendUrls(result, child)); + return; + } + if (node.isNull() || node.isMissingNode()) { + return; + } + appendUrls(result, node.asText("")); + return; + } + if (value instanceof Collection collection) { + collection.forEach(item -> appendUrls(result, item)); + return; + } + Class valueClass = value.getClass(); + if (valueClass.isArray()) { + int length = Array.getLength(value); + for (int i = 0; i < length; i++) { + appendUrls(result, Array.get(value, i)); + } + return; + } + String text = String.valueOf(value).trim(); + if (!text.isBlank()) { + result.add(text); + } + } + + private static List limitUrls(List values) { + Set deduplicated = new LinkedHashSet<>(); + if (values != null) { + for (String value : values) { + if (value == null || value.isBlank()) { + continue; + } + deduplicated.add(value.trim()); + if (deduplicated.size() >= MAX_IMAGE_URLS) { + break; + } + } + } + return new ArrayList<>(deduplicated); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinSourceFileDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinSourceFileDto.java new file mode 100644 index 0000000..0f51d96 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinSourceFileDto.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.similarasin.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "相似ASIN检测源文件信息") +public class SimilarAsinSourceFileDto { + @Schema(description = "上传接口返回的临时文件 key。后端根据该 key 查找本地临时 Excel 文件并解析。", example = "uploads/20260426/similar_asin_17.xlsx", requiredMode = Schema.RequiredMode.REQUIRED) + private String fileKey; + @Schema(description = "原始文件名。用于历史记录展示和最终结果文件命名。", example = "17.xlsx") + private String originalFilename; + @Schema(description = "相对目录路径。当前仅记录来源，相似ASIN检测不依赖该字段处理。", example = "xlsx/17.xlsx") + private String relativePath; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinSubmitResultRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinSubmitResultRequest.java new file mode 100644 index 0000000..c402446 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinSubmitResultRequest.java @@ -0,0 +1,32 @@ +package com.nanri.aiimage.modules.similarasin.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "Python 回传相似ASIN检测结果请求") +public class SimilarAsinSubmitResultRequest { + @Schema(description = "本次 Python 回传的提交批次标识", example = "similar-asin-3938") + private String submissionId; + + @Schema(description = "当前回传分片序号", example = "1") + private Integer chunkIndex; + + @Schema(description = "本任务预计总分片数", example = "36") + private Integer chunkTotal; + + @Schema(description = "是否为最后一次回传", example = "false") + private Boolean done; + + @Schema(description = "Python 侧任务级错误信息", example = "浏览器执行异常，任务提前结束") + private String error; + + @Schema(description = "本次回传的分组结果列表，推荐优先使用") + private List groups = new ArrayList<>(); + + @Schema(description = "兼容旧链路的平铺结果列表") + private List items = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinTaskBatchRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinTaskBatchRequest.java new file mode 100644 index 0000000..98f1cee --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinTaskBatchRequest.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.similarasin.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "相似ASIN检测批量进度查询请求") +public class SimilarAsinTaskBatchRequest { + @NotEmpty + @Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务；后端会批量查询，避免每个任务单独请求。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED) + private List taskIds; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinDashboardVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinDashboardVo.java new file mode 100644 index 0000000..0d51329 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinDashboardVo.java @@ -0,0 +1,17 @@ +package com.nanri.aiimage.modules.similarasin.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "相似ASIN检测总览统计") +public class SimilarAsinDashboardVo { + @Schema(description = "运行中任务数量。这里统计 RUNNING 状态任务。", example = "1") + private Long pendingTaskCount; + @Schema(description = "已结束任务数量，等于成功任务数加失败任务数。", example = "12") + private Long processedTaskCount; + @Schema(description = "成功任务数量。", example = "10") + private Long successTaskCount; + @Schema(description = "失败任务数量。", example = "2") + private Long failedTaskCount; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java new file mode 100644 index 0000000..f310d32 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryItemVo.java @@ -0,0 +1,37 @@ +package com.nanri.aiimage.modules.similarasin.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "相似ASIN检测历史记录项") +public class SimilarAsinHistoryItemVo { + @Schema(description = "结果记录 ID。删除历史、下载结果时使用。", example = "1001") + private Long resultId; + @Schema(description = "任务 ID。", example = "3938") + private Long taskId; + @Schema(description = "源 Excel 文件名。", example = "17.xlsx") + private String sourceFilename; + @Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx") + private String resultFilename; + @Schema(description = "最终结果文件下载地址。后端基于 OSS objectKey 生成的新鲜预签名 URL。", example = "https://bucket.oss-cn-hangzhou.aliyuncs.com/result/similar_asin/xxx/17-result.xlsx?Expires=...") + private String downloadUrl; + private Long fileJobId; + private String fileStatus; + private String fileError; + private Boolean fileReady; + private Integer fileProgressPercent; + private Integer fileProgressCurrent; + private Integer fileProgressTotal; + private String fileProgressMessage; + @Schema(description = "任务状态：PENDING=已解析待推送，RUNNING=执行中，SUCCESS=成功，FAILED=失败。", example = "SUCCESS") + private String taskStatus; + @Schema(description = "结果是否成功。true 表示任务完成并生成结果文件；false 表示失败或未完成。", example = "true") + private Boolean success; + @Schema(description = "错误信息。任务失败时返回，例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final similar ASIN result") + private String error; + @Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716") + private Integer rowCount; + @Schema(description = "历史记录创建时间，ISO 本地时间字符串。", example = "2026-04-26T10:30:00") + private String createdAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryVo.java new file mode 100644 index 0000000..1f74ca5 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinHistoryVo.java @@ -0,0 +1,14 @@ +package com.nanri.aiimage.modules.similarasin.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "相似ASIN检测历史记录列表") +public class SimilarAsinHistoryVo { + @Schema(description = "历史记录项列表，默认返回最近 100 条。") + private List items = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParseVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParseVo.java new file mode 100644 index 0000000..5a126af --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParseVo.java @@ -0,0 +1,41 @@ +package com.nanri.aiimage.modules.similarasin.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "相似ASIN检测解析结果") +public class SimilarAsinParseVo { + @Schema(description = "新创建的任务 ID", example = "3938") + private Long taskId; + + @Schema(description = "来源 Excel 文件名，多个文件时为聚合展示文案", example = "17.xlsx 等 2 个文件") + private String sourceFilename; + + @Schema(description = "参与本次解析的源文件数量", example = "2") + private Integer sourceFileCount; + + @Schema(description = "Excel 中检测到的非空数据总行数", example = "716") + private Integer totalRows; + + @Schema(description = "解析出的有效行数", example = "458") + private Integer acceptedRows; + + @Schema(description = "因缺少必要字段而被丢弃的行数", example = "12") + private Integer droppedRows; + + @Schema(description = "按相邻主 ID 块生成的分组数", example = "36") + private Integer groupCount; + + @Schema(description = "本任务最终使用的 AI 提示词") + private String aiPrompt; + + @Schema(description = "兼容旧前端的平铺有效行列表，现为全部有效行") + private List items = new ArrayList<>(); + + @Schema(description = "返回前端和推送 Python 的分组列表") + private List groups = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParsedGroupVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParsedGroupVo.java new file mode 100644 index 0000000..56073de --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParsedGroupVo.java @@ -0,0 +1,32 @@ +package com.nanri.aiimage.modules.similarasin.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "相似ASIN检测解析后的分组数据") +public class SimilarAsinParsedGroupVo { + @Schema(description = "来源文件 key") + private String sourceFileKey; + + @Schema(description = "来源文件名") + private String sourceFilename; + + @Schema(description = "相邻主 ID 块的分组 key") + private String groupKey; + + @Schema(description = "主 ID，例如 1、2、10") + private String baseId; + + @Schema(description = "分组首条展示 ID，例如 1 或 2_1") + private String displayId; + + @Schema(description = "分组内行数") + private Integer itemCount; + + @Schema(description = "分组内全部行，顺序与原 Excel 保持一致") + private List items = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParsedRowVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParsedRowVo.java new file mode 100644 index 0000000..848485c --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParsedRowVo.java @@ -0,0 +1,47 @@ +package com.nanri.aiimage.modules.similarasin.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.LinkedHashMap; +import java.util.Map; + +@Data +@Schema(description = "相似ASIN检测解析出的单行数据") +public class SimilarAsinParsedRowVo { + @Schema(description = "来源文件 key。用于多文件场景下区分同名/同 ID 行。", example = "uploads/20260426/similar_asin_17.xlsx") + private String sourceFileKey; + + @Schema(description = "来源文件名。用于调试和结果追踪。", example = "17.xlsx") + private String sourceFilename; + + @Schema(description = "Excel 原始行号，从 1 开始。", example = "2") + private Integer rowIndex; + + @Schema(description = "Excel 中原始 id 值。", example = "2_1") + private String sourceId; + + @Schema(description = "前端展示和 Python 回传使用的 id。整数 id 原样保留，子数据第一条如 2_1 原样保留。", example = "2_1") + private String displayId; + + @Schema(description = "行级唯一 token。多文件、重复主数据时用于精确匹配回传结果。", example = "uploads/20260426/similar_asin_17.xlsx::row::2") + private String rowToken; + + @Schema(description = "同一主数据块的分组 key。用于把 2_1、2_2、2_3 等子行重新补回同一组结果。", example = "uploads/20260426/similar_asin_17.xlsx::2@2") + private String groupKey; + + @Schema(description = "亚马逊 ASIN。", example = "B0CJ8SNXXV") + private String asin; + + @Schema(description = "国家或站点。", example = "英国") + private String country; + + @Schema(description = "商品图片 URL 或商品 URL，供 Coze 检测使用。", example = "https://webstatic.aiproxy.vip/output/demo.jpg") + private String url; + + @Schema(description = "商品标题。", example = "Women Floral Dress Summer Casual") + private String title; + + @Schema(description = "该 Excel 行的原始列值映射。最终生成 xlsx 时可以从这里读取价格等字段。", example = "{\"id\":\"2_1\",\"asin\":\"B0CJ8SNXXV\",\"国家\":\"英国\",\"价格\":\"12.99\"}") + private Map values = new LinkedHashMap<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskBatchVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskBatchVo.java new file mode 100644 index 0000000..0f4761e --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskBatchVo.java @@ -0,0 +1,16 @@ +package com.nanri.aiimage.modules.similarasin.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "相似ASIN检测批量进度响应") +public class SimilarAsinTaskBatchVo { + @Schema(description = "查询到的任务详情列表。顺序按请求 taskIds 处理。") + private List items = new ArrayList<>(); + @Schema(description = "未找到或不属于相似ASIN检测模块的任务 ID 列表。", example = "[99999]") + private List missingTaskIds = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskDetailVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskDetailVo.java new file mode 100644 index 0000000..d172e69 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskDetailVo.java @@ -0,0 +1,16 @@ +package com.nanri.aiimage.modules.similarasin.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "相似ASIN检测任务进度详情") +public class SimilarAsinTaskDetailVo { + @Schema(description = "任务主记录轻量信息。") + private SimilarAsinTaskItemVo task; + @Schema(description = "预留的任务明细列表。当前进度接口主要返回任务轻量状态，不返回完整结果明细。") + private List items = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskItemVo.java new file mode 100644 index 0000000..826c22b --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinTaskItemVo.java @@ -0,0 +1,23 @@ +package com.nanri.aiimage.modules.similarasin.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "相似ASIN检测任务轻量信息") +public class SimilarAsinTaskItemVo { + @Schema(description = "任务 ID。", example = "3938") + private Long id; + @Schema(description = "任务编号，后端自动生成。", example = "SIMILAR_ASIN-1780000000000000000") + private String taskNo; + @Schema(description = "任务状态：PENDING=已解析待推送，RUNNING=执行中，SUCCESS=成功，FAILED=失败。", example = "RUNNING") + private String status; + @Schema(description = "任务级错误信息。失败时返回。", example = "生成相似ASIN检测结果失败") + private String errorMessage; + @Schema(description = "创建时间，ISO 本地时间字符串。", example = "2026-04-26T10:00:00") + private String createdAt; + @Schema(description = "最后更新时间，通常由 Python 回传结果或任务收尾更新。", example = "2026-04-26T10:05:00") + private String updatedAt; + @Schema(description = "完成时间。任务未结束时为空。", example = "2026-04-26T10:10:00") + private String finishedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskCacheService.java new file mode 100644 index 0000000..25bb7c7 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskCacheService.java @@ -0,0 +1,144 @@ +package com.nanri.aiimage.modules.similarasin.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +@Service +@RequiredArgsConstructor +@Slf4j +public class SimilarAsinTaskCacheService { + + private static final long TTL_HOURS = 24; + + private final StringRedisTemplate stringRedisTemplate; + private final ObjectMapper objectMapper; + + public void appendPendingRow(Long taskId, String scopeHash, Integer chunkIndex, SimilarAsinResultRowDto row) { + if (taskId == null || taskId <= 0 || scopeHash == null || scopeHash.isBlank() || chunkIndex == null || row == null) { + return; + } + try { + stringRedisTemplate.opsForList().rightPush( + pendingRowsKey(taskId), + objectMapper.writeValueAsString(new PendingRow(scopeHash, chunkIndex, row)) + ); + stringRedisTemplate.expire(pendingRowsKey(taskId), Duration.ofHours(TTL_HOURS)); + touchTaskHeartbeat(taskId); + } catch (Exception ignored) { + } + } + + public long pendingRowCount(Long taskId) { + Long size; + try { + size = stringRedisTemplate.opsForList().size(pendingRowsKey(taskId)); + } catch (Exception ex) { + log.warn("[similar-asin-cache] pending row count degraded taskId={} msg={}", taskId, ex.getMessage()); + return 0L; + } + return size == null ? 0L : size; + } + + public List drainPendingRows(Long taskId, int limit) { + if (taskId == null || taskId <= 0 || limit <= 0) { + return List.of(); + } + String key = pendingRowsKey(taskId); + List values; + try { + values = stringRedisTemplate.opsForList().range(key, 0, limit - 1L); + } catch (Exception ex) { + log.warn("[similar-asin-cache] drain range degraded taskId={} msg={}", taskId, ex.getMessage()); + return List.of(); + } + if (values == null || values.isEmpty()) { + return List.of(); + } + try { + stringRedisTemplate.opsForList().trim(key, values.size(), -1); + } catch (Exception ex) { + log.warn("[similar-asin-cache] drain trim degraded taskId={} msg={}", taskId, ex.getMessage()); + return List.of(); + } + List rows = new ArrayList<>(); + for (String value : values) { + if (value == null || value.isBlank()) { + continue; + } + try { + rows.add(objectMapper.readValue(value, PendingRow.class)); + } catch (Exception ignored) { + } + } + touchTaskHeartbeat(taskId); + return rows; + } + + public void touchTaskHeartbeat(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + try { + stringRedisTemplate.opsForValue().set( + heartbeatKey(taskId), + String.valueOf(Instant.now().toEpochMilli()), + Duration.ofHours(TTL_HOURS) + ); + } catch (Exception ex) { + log.warn("[similar-asin-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + } + } + + public long getTaskHeartbeatMillis(Long taskId) { + if (taskId == null || taskId <= 0) { + return 0L; + } + String raw; + try { + raw = stringRedisTemplate.opsForValue().get(heartbeatKey(taskId)); + } catch (Exception ex) { + log.warn("[similar-asin-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + return 0L; + } + if (raw == null || raw.isBlank()) { + return 0L; + } + try { + return Long.parseLong(raw); + } catch (NumberFormatException ignored) { + return 0L; + } + } + + public void deleteTaskCache(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + try { + stringRedisTemplate.delete(pendingRowsKey(taskId)); + stringRedisTemplate.delete(heartbeatKey(taskId)); + } catch (Exception ex) { + log.warn("[similar-asin-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage()); + } + } + + private String pendingRowsKey(Long taskId) { + return "similar-asin:task:pending-rows:" + taskId; + } + + private String heartbeatKey(Long taskId) { + return "similar-asin:task:heartbeat:" + taskId; + } + + public record PendingRow(String scopeHash, Integer chunkIndex, SimilarAsinResultRowDto row) { + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java new file mode 100644 index 0000000..1705920 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -0,0 +1,2413 @@ +package com.nanri.aiimage.modules.similarasin.service; + +import cn.hutool.core.util.IdUtil; +import cn.hutool.crypto.digest.DigestUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.config.SimilarAsinProperties; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultGroupDto; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSourceFileDto; +import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryItemVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskDetailVo; +import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskItemVo; +import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.core.task.TaskExecutor; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.function.Supplier; + +@Service +@RequiredArgsConstructor +@Slf4j +public class SimilarAsinTaskService { + + public static final String MODULE_TYPE = "SIMILAR_ASIN"; + private static final String STATUS_PENDING = "PENDING"; + private static final String STATUS_RUNNING = "RUNNING"; + private static final String STATUS_SUCCESS = "SUCCESS"; + private static final String STATUS_FAILED = "FAILED"; + private static final String COZE_STATUS_SUBMITTED = "SUBMITTED"; + private static final String COZE_STATUS_RUNNING = "RUNNING"; + private static final String COZE_STATUS_DONE = "DONE"; + private static final String COZE_STATUS_FAILED = "FAILED"; + private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3; + private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L; + private static final List RESULT_HEADERS = List.of( + "id", + "asin", + "国家", + "价格", + "标题维度（商标）", + "外观维度（外观设计专利）", + "专利维度（发明/实用新型专利）", + "结论" + ); + + private final LocalFileStorageService localFileStorageService; + private final OssStorageService ossStorageService; + private final StorageProperties storageProperties; + private final FileTaskMapper fileTaskMapper; + private final FileResultMapper fileResultMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final TaskChunkMapper taskChunkMapper; + private final ObjectMapper objectMapper; + private final SimilarAsinCozeClient cozeClient; + private final SimilarAsinTaskCacheService taskCacheService; + private final SimilarAsinProperties properties; + private final TaskFileJobService taskFileJobService; + private final TaskProgressSnapshotService taskProgressSnapshotService; + private final TransientPayloadStorageService transientPayloadStorageService; + private final PlatformTransactionManager transactionManager; + private final DistributedJobLockService distributedJobLockService; + @Autowired + @Qualifier("cozeTaskExecutor") + private TaskExecutor cozeTaskExecutor; + + public SimilarAsinParseVo parseAndCreateTask(SimilarAsinParseRequest request) { + long startedAt = System.nanoTime(); + if (request.getUserId() == null || request.getUserId() <= 0) { + throw new BusinessException("user_id 不合法"); + } + if (request.getFiles() == null || request.getFiles().isEmpty()) { + throw new BusinessException("请先上传 Excel 文件"); + } + List sourceFiles = request.getFiles().stream() + .filter(Objects::nonNull) + .toList(); + if (sourceFiles.isEmpty()) { + throw new BusinessException("请先上传 Excel 文件"); + } + List allRows = new ArrayList<>(); + List mergedHeaders = new ArrayList<>(); + int totalRows = 0; + int droppedRows = 0; + long parseStartedAt = System.nanoTime(); + for (SimilarAsinSourceFileDto source : sourceFiles) { + if (source.getFileKey() == null || source.getFileKey().isBlank()) { + throw new BusinessException("fileKey 不能为空"); + } + File input = localFileStorageService.findLocalSourceFile(source.getFileKey()); + if (input == null || !input.exists()) { + throw new BusinessException("源文件不存在"); + } + + ParsedWorkbook parsed = parseWorkbook(input, source); + totalRows += parsed.totalRows(); + droppedRows += parsed.droppedRows(); + allRows.addAll(parsed.allRows()); + mergeHeaders(mergedHeaders, parsed.headers()); + } + if (allRows.isEmpty()) { + throw new BusinessException("未解析到有效 ASIN 数据"); + } + long parsedAt = System.nanoTime(); + List groups = buildParsedGroups(allRows); + long groupedAt = System.nanoTime(); + + FileTaskEntity task = new FileTaskEntity(); + task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr()); + task.setModuleType(MODULE_TYPE); + task.setTaskMode("PYTHON_QUEUE"); + task.setStatus(STATUS_PENDING); + task.setSourceFileCount(sourceFiles.size()); + task.setSuccessFileCount(0); + task.setFailedFileCount(0); + task.setCreatedBy("user:" + request.getUserId()); + task.setUserId(request.getUserId()); + task.setCreatedAt(LocalDateTime.now()); + task.setUpdatedAt(LocalDateTime.now()); + try { + task.setRequestJson(objectMapper.writeValueAsString(request)); + task.setResultJson("{}"); + } catch (Exception ex) { + throw new BusinessException("序列化解析结果失败"); + } + fileTaskMapper.insert(task); + long taskInsertedAt = System.nanoTime(); + + String aggregateScopeKey = buildAggregateScopeKey(sourceFiles); + String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey); + String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), sourceFiles, mergedHeaders, groups, allRows); + long payloadBuiltAt = System.nanoTime(); + String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload); + long payloadStoredAt = System.nanoTime(); + task.setResultJson(buildTaskResultJson(request.getAiPrompt(), sourceFiles, parsedPayloadPointer)); + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + + String sourceFilename = buildAggregateSourceFilenameLabel(sourceFiles); + FileResultEntity result = new FileResultEntity(); + result.setTaskId(task.getId()); + result.setModuleType(MODULE_TYPE); + result.setSourceFilename(sourceFilename); + result.setSourceFileUrl(aggregateScopeKey); + result.setRowCount(allRows.size()); + result.setUserId(request.getUserId()); + result.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(result); + + TaskScopeStateEntity scope = new TaskScopeStateEntity(); + scope.setTaskId(task.getId()); + scope.setModuleType(MODULE_TYPE); + scope.setScopeKey(aggregateScopeKey); + scope.setScopeHash(sourceScopeHash); + scope.setParsedPayloadJson(writeJson(parsedPayloadPointer, "保存解析载荷指针失败")); + scope.setStateJson("{\"phase\":\"PARSED\"}"); + scope.setChunkTotal(0); + scope.setReceivedChunkCount(0); + scope.setCompleted(0); + scope.setCreatedAt(LocalDateTime.now()); + scope.setUpdatedAt(LocalDateTime.now()); + taskScopeStateMapper.insert(scope); + long persistedAt = System.nanoTime(); + + SimilarAsinParseVo vo = new SimilarAsinParseVo(); + vo.setTaskId(task.getId()); + vo.setSourceFilename(sourceFilename); + vo.setSourceFileCount(sourceFiles.size()); + vo.setTotalRows(totalRows); + vo.setAcceptedRows(allRows.size()); + vo.setDroppedRows(droppedRows); + vo.setGroupCount(groups.size()); + vo.setAiPrompt(normalize(request.getAiPrompt())); + vo.setItems(allRows); + vo.setGroups(groups); + long finishedAt = System.nanoTime(); + log.info("[similar-asin] parse timing taskId={} files={} rows={} groups={} totalMs={} parseMs={} groupMs={} taskInsertMs={} payloadJsonMs={} payloadStoreMs={} persistMs={} responseMs={}", + task.getId(), + sourceFiles.size(), + allRows.size(), + groups.size(), + elapsedMs(startedAt, finishedAt), + elapsedMs(parseStartedAt, parsedAt), + elapsedMs(parsedAt, groupedAt), + elapsedMs(groupedAt, taskInsertedAt), + elapsedMs(taskInsertedAt, payloadBuiltAt), + elapsedMs(payloadBuiltAt, payloadStoredAt), + elapsedMs(payloadStoredAt, persistedAt), + elapsedMs(persistedAt, finishedAt)); + return vo; + } + + @Transactional + public void activateTask(Long taskId, Long userId) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) { + throw new BusinessException("任务不存在"); + } + if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) { + throw new BusinessException("任务已结束"); + } + task.setStatus(STATUS_RUNNING); + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + taskCacheService.touchTaskHeartbeat(taskId); + } + + public SimilarAsinDashboardVo dashboard(Long userId) { + SimilarAsinDashboardVo vo = new SimilarAsinDashboardVo(); + vo.setPendingTaskCount(countTask(userId, STATUS_RUNNING)); + vo.setSuccessTaskCount(countTask(userId, STATUS_SUCCESS)); + vo.setFailedTaskCount(countTask(userId, STATUS_FAILED)); + vo.setProcessedTaskCount(vo.getSuccessTaskCount() + vo.getFailedTaskCount()); + return vo; + } + + public SimilarAsinHistoryVo history(Long userId, Integer limit) { + SimilarAsinHistoryVo vo = new SimilarAsinHistoryVo(); + int safeLimit = Math.max(1, Math.min(limit == null ? 50 : limit, 100)); + List rows = fileResultMapper.selectList(new LambdaQueryWrapper() + .select(FileResultEntity::getId, + FileResultEntity::getTaskId, + FileResultEntity::getSourceFilename, + FileResultEntity::getResultFilename, + FileResultEntity::getResultFileUrl, + FileResultEntity::getRowCount, + FileResultEntity::getSuccess, + FileResultEntity::getErrorMessage, + FileResultEntity::getCreatedAt) + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .eq(FileResultEntity::getUserId, userId) + .orderByDesc(FileResultEntity::getCreatedAt) + .last("limit " + safeLimit)); + Map statusMap = new LinkedHashMap<>(); + List taskIds = rows.stream().map(FileResultEntity::getTaskId).filter(Objects::nonNull).distinct().toList(); + if (!taskIds.isEmpty()) { + for (FileTaskEntity task : fileTaskMapper.selectList(new LambdaQueryWrapper() + .select(FileTaskEntity::getId, FileTaskEntity::getStatus) + .in(FileTaskEntity::getId, taskIds))) { + statusMap.put(task.getId(), task.getStatus()); + } + } + Map jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, rows.stream() + .map(FileResultEntity::getId) + .filter(Objects::nonNull) + .toList()); + for (FileResultEntity row : rows) { + String taskStatus = statusMap.get(row.getTaskId()); + if (STATUS_PENDING.equals(taskStatus)) { + continue; + } + vo.getItems().add(toHistoryItem(row, taskStatus, jobMap.get(row.getId()))); + } + return vo; + } + + public SimilarAsinTaskBatchVo progressBatch(List taskIds) { + SimilarAsinTaskBatchVo vo = new SimilarAsinTaskBatchVo(); + List normalizedIds = taskIds == null ? List.of() : taskIds.stream() + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + if (normalizedIds.isEmpty()) { + return vo; + } + Map taskMap = new LinkedHashMap<>(); + for (FileTaskEntity task : fileTaskMapper.selectList(new LambdaQueryWrapper() + .select(FileTaskEntity::getId, + FileTaskEntity::getTaskNo, + FileTaskEntity::getModuleType, + FileTaskEntity::getStatus, + FileTaskEntity::getErrorMessage, + FileTaskEntity::getCreatedAt, + FileTaskEntity::getUpdatedAt, + FileTaskEntity::getFinishedAt) + .in(FileTaskEntity::getId, normalizedIds))) { + if (task != null && MODULE_TYPE.equals(task.getModuleType())) { + taskMap.put(task.getId(), task); + } + } + List resultRows = fileResultMapper.selectList(new LambdaQueryWrapper() + .select(FileResultEntity::getId, + FileResultEntity::getTaskId, + FileResultEntity::getSourceFilename, + FileResultEntity::getResultFilename, + FileResultEntity::getResultFileUrl, + FileResultEntity::getRowCount, + FileResultEntity::getSuccess, + FileResultEntity::getErrorMessage, + FileResultEntity::getCreatedAt) + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .in(FileResultEntity::getTaskId, normalizedIds) + .orderByDesc(FileResultEntity::getCreatedAt)); + Map resultByTaskId = new LinkedHashMap<>(); + for (FileResultEntity row : resultRows) { + if (row.getTaskId() != null) { + resultByTaskId.putIfAbsent(row.getTaskId(), row); + } + } + Map jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, resultRows.stream() + .map(FileResultEntity::getId) + .filter(Objects::nonNull) + .toList()); + for (Long taskId : normalizedIds) { + FileTaskEntity task = taskMap.get(taskId); + if (task == null) { + vo.getMissingTaskIds().add(taskId); + continue; + } + SimilarAsinTaskDetailVo detail = new SimilarAsinTaskDetailVo(); + detail.setTask(toTaskItem(task)); + FileResultEntity resultRow = resultByTaskId.get(taskId); + if (resultRow != null) { + detail.getItems().add(toHistoryItem(resultRow, task.getStatus(), jobMap.get(resultRow.getId()))); + } + vo.getItems().add(detail); + } + return vo; + } + + public void submitResult(Long taskId, SimilarAsinSubmitResultRequest request) { + if (transactionManager != null) { + SubmitContext context = inNewTransaction(() -> persistSubmittedChunk(taskId, request)); + inNewTransaction(() -> { + completeSubmittedChunk(context); + return null; + }); + return; + } + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + if (!STATUS_RUNNING.equals(task.getStatus())) { + throw new BusinessException("任务不是运行中状态"); + } + + int chunkIndex = request.getChunkIndex() == null ? 0 : request.getChunkIndex(); + int chunkTotal = request.getChunkTotal() == null ? 1 : request.getChunkTotal(); + String scopeKey = firstNonBlank(request.getSubmissionId(), "task:" + taskId); + String scopeHash = DigestUtil.sha256Hex(scopeKey); + taskCacheService.touchTaskHeartbeat(taskId); + + TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, chunkIndex) + .last("limit 1")); + if (existing == null) { + List rawRows = flattenSubmittedRows(request); + String payloadJson = writeJson(rawRows, "结果序列化失败"); + + TaskChunkEntity chunk = new TaskChunkEntity(); + chunk.setTaskId(taskId); + chunk.setModuleType(MODULE_TYPE); + chunk.setScopeKey(scopeKey); + chunk.setScopeHash(scopeHash); + chunk.setChunkIndex(chunkIndex); + chunk.setChunkTotal(chunkTotal); + String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); + chunk.setPayloadJson(storedPayload); + chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson)); + chunk.setCreatedAt(LocalDateTime.now()); + chunk.setUpdatedAt(LocalDateTime.now()); + try { + taskChunkMapper.insert(chunk); + } catch (DuplicateKeyException ex) { + // storeChunkPayload uses a deterministic key like chunk-{index}. When two + // concurrent callbacks submit the same chunk, deleting the loser payload here + // can also remove the winner's shared object and break later assembly. + log.info("[similar-asin] duplicate chunk inserted concurrently taskId={} scope={} chunk={}", taskId, scopeKey, chunkIndex); + } + } else { + log.info("[similar-asin] duplicate chunk ignored taskId={} scope={} chunk={}", taskId, scopeKey, chunkIndex); + } + + TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + if (scope == null) { + scope = new TaskScopeStateEntity(); + scope.setTaskId(taskId); + scope.setModuleType(MODULE_TYPE); + scope.setScopeKey(scopeKey); + scope.setScopeHash(scopeHash); + scope.setCreatedAt(LocalDateTime.now()); + } + scope.setChunkTotal(chunkTotal); + scope.setReceivedChunkCount(countChunks(taskId, scopeHash)); + scope.setLastChunkAt(LocalDateTime.now()); + scope.setLastError(request.getError()); + scope.setCompleted(Boolean.TRUE.equals(request.getDone()) ? 1 : 0); + scope.setUpdatedAt(LocalDateTime.now()); + scope.setStateJson("{\"phase\":\"RECEIVED\",\"coze\":\"PENDING\"}"); + if (scope.getId() == null) { + taskScopeStateMapper.insert(scope); + } else { + taskScopeStateMapper.updateById(scope); + } + + if (Boolean.TRUE.equals(request.getDone()) || request.getError() != null && !request.getError().isBlank()) { + finalizeTask(task, request.getError(), allRowCount(task), true); + } else { + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + } + } + + @Transactional + public void deleteTask(Long taskId, Long userId) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) { + throw new BusinessException("任务不存在"); + } + fileResultMapper.delete(new LambdaQueryWrapper().eq(FileResultEntity::getTaskId, taskId).eq(FileResultEntity::getModuleType, MODULE_TYPE)); + deleteTransientTaskPayloads(taskId); + taskScopeStateMapper.delete(new LambdaQueryWrapper().eq(TaskScopeStateEntity::getTaskId, taskId).eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + taskChunkMapper.delete(new LambdaQueryWrapper().eq(TaskChunkEntity::getTaskId, taskId).eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + taskCacheService.deleteTaskCache(taskId); + fileTaskMapper.deleteById(taskId); + } + + public void deleteHistory(Long resultId, Long userId) { + FileResultEntity row = fileResultMapper.selectById(resultId); + if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) { + throw new BusinessException("记录不存在"); + } + fileResultMapper.deleteById(resultId); + } + + public String resolveResultDownloadUrl(Long resultId, Long userId) { + FileResultEntity row = fileResultMapper.selectById(resultId); + if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) { + throw new BusinessException("记录不存在"); + } + if (row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) { + throw new BusinessException("暂无可下载文件"); + } + return ossStorageService.generateFreshDownloadUrl(row.getResultFileUrl()); + } + + public String resolveResultDownloadFilename(Long resultId, Long userId) { + FileResultEntity row = fileResultMapper.selectById(resultId); + if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) { + throw new BusinessException("记录不存在"); + } + return row.getResultFilename() == null || row.getResultFilename().isBlank() + ? "similar-asin-" + resultId + ".xlsx" + : row.getResultFilename(); + } + + @Scheduled(cron = "${aiimage.similar-asin.stale-finalize-cron:0 */2 * * * *}") + public void finalizeStaleTasks() { + if (transactionManager != null) { + LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes())); + long thresholdMillis = threshold.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + List tasks = fileTaskMapper.selectList(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .lt(FileTaskEntity::getUpdatedAt, threshold) + .last("limit 50")); + for (FileTaskEntity task : tasks) { + if (isJavaSideProcessing(task.getId())) { + touchJavaSideTaskActivity(task.getId()); + continue; + } + long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId()); + if (heartbeatMillis > thresholdMillis) { + continue; + } + inNewTransaction(() -> { + finalizeStaleTask(task.getId(), "Python interrupted before uploading final similar ASIN result"); + return null; + }); + } + return; + } + LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes())); + long thresholdMillis = threshold.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + List tasks = fileTaskMapper.selectList(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .lt(FileTaskEntity::getUpdatedAt, threshold) + .last("limit 50")); + for (FileTaskEntity task : tasks) { + if (isJavaSideProcessing(task.getId())) { + touchJavaSideTaskActivity(task.getId()); + continue; + } + long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId()); + if (heartbeatMillis > thresholdMillis) { + continue; + } + finalizeTask(task, "Python interrupted before uploading final similar ASIN result", allRowCount(task), true); + } + } + + private SubmitContext persistSubmittedChunk(Long taskId, SimilarAsinSubmitResultRequest request) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + if (!STATUS_RUNNING.equals(task.getStatus())) { + throw new BusinessException("任务不是运行中状态"); + } + + int chunkIndex = request.getChunkIndex() == null ? 0 : request.getChunkIndex(); + int chunkTotal = request.getChunkTotal() == null ? 1 : request.getChunkTotal(); + boolean done = Boolean.TRUE.equals(request.getDone()); + String scopeKey = firstNonBlank(request.getSubmissionId(), "task:" + taskId); + String scopeHash = DigestUtil.sha256Hex(scopeKey); + taskCacheService.touchTaskHeartbeat(taskId); + + TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, chunkIndex) + .last("limit 1")); + if (existing == null) { + List rawRows = flattenSubmittedRows(request); + String payloadJson = writeJson(rawRows, "结果序列化失败"); + + TaskChunkEntity chunk = new TaskChunkEntity(); + chunk.setTaskId(taskId); + chunk.setModuleType(MODULE_TYPE); + chunk.setScopeKey(scopeKey); + chunk.setScopeHash(scopeHash); + chunk.setChunkIndex(chunkIndex); + chunk.setChunkTotal(chunkTotal); + String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); + chunk.setPayloadJson(storedPayload); + chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson)); + chunk.setCreatedAt(LocalDateTime.now()); + chunk.setUpdatedAt(LocalDateTime.now()); + try { + taskChunkMapper.insert(chunk); + } catch (DuplicateKeyException ex) { + log.info("[similar-asin] duplicate chunk inserted concurrently taskId={} scope={} chunk={}", taskId, scopeKey, chunkIndex); + } + } else { + log.info("[similar-asin] duplicate chunk ignored taskId={} scope={} chunk={}", taskId, scopeKey, chunkIndex); + } + + upsertScopeState(taskId, scopeKey, scopeHash, chunkTotal, request.getError(), done, false); + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + return new SubmitContext(task, scopeKey, scopeHash, done, request.getError()); + } + + private void completeSubmittedChunk(SubmitContext context) { + FileTaskEntity task = fileTaskMapper.selectById(context.task().getId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + if (!STATUS_RUNNING.equals(task.getStatus())) { + log.info("[similar-asin] skip completion because task already finalized taskId={} status={}", + task.getId(), task.getStatus()); + return; + } + upsertScopeState(task.getId(), context.scopeKey(), context.scopeHash(), null, context.error(), context.forceFlush(), false); + if (context.forceFlush() || context.error() != null && !context.error().isBlank()) { + finalizeTask(task, context.error(), allRowCount(task), true); + return; + } + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + } + + private void finalizeStaleTask(Long taskId, String error) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) { + return; + } + finalizeTask(task, error, allRowCount(task), true); + } + + private void upsertScopeState(Long taskId, + String scopeKey, + String scopeHash, + Integer chunkTotal, + String error, + boolean completed, + boolean cozeDone) { + TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + LocalDateTime now = LocalDateTime.now(); + if (scope == null) { + scope = new TaskScopeStateEntity(); + scope.setTaskId(taskId); + scope.setModuleType(MODULE_TYPE); + scope.setScopeKey(scopeKey); + scope.setScopeHash(scopeHash); + scope.setCreatedAt(now); + } + if (chunkTotal != null) { + scope.setChunkTotal(chunkTotal); + } + scope.setReceivedChunkCount(countChunks(taskId, scopeHash)); + scope.setLastChunkAt(now); + scope.setLastError(error); + scope.setCompleted(completed ? 1 : 0); + scope.setUpdatedAt(now); + scope.setStateJson(cozeDone + ? "{\"phase\":\"RECEIVED\",\"coze\":\"DONE\"}" + : "{\"phase\":\"RECEIVED\",\"coze\":\"PENDING\"}"); + if (scope.getId() == null) { + try { + taskScopeStateMapper.insert(scope); + return; + } catch (DuplicateKeyException ex) { + log.info("[similar-asin] duplicate scope state inserted concurrently taskId={} scope={}", taskId, scopeKey); + scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + if (scope == null) { + log.info("[similar-asin] scope state winner not committed yet, skip duplicate updater taskId={} scope={}", + taskId, scopeKey); + return; + } + if (chunkTotal != null) { + scope.setChunkTotal(chunkTotal); + } + scope.setReceivedChunkCount(countChunks(taskId, scopeHash)); + scope.setLastChunkAt(now); + scope.setLastError(error); + scope.setCompleted(completed ? 1 : 0); + scope.setUpdatedAt(now); + scope.setStateJson(cozeDone + ? "{\"phase\":\"RECEIVED\",\"coze\":\"DONE\"}" + : "{\"phase\":\"RECEIVED\",\"coze\":\"PENDING\"}"); + } + } + taskScopeStateMapper.updateById(scope); + } + + private T inNewTransaction(Supplier action) { + TransactionTemplate template = new TransactionTemplate(transactionManager); + template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + return template.execute(status -> action.get()); + } + + private List applyCozeInBatches(List items, FileTaskEntity task) { + return applyCozeInBatches(items, task, null); + } + + private List applyCozeInBatches(List items, + FileTaskEntity task, + Runnable progressHook) { + if (items == null || items.isEmpty()) { + return List.of(); + } + String prompt = readAiPrompt(task); + int batchSize = Math.max(1, properties.getCozeBatchSize()); + List result = new ArrayList<>(); + for (int i = 0; i < items.size(); i += batchSize) { + result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt)); + if (progressHook != null) { + progressHook.run(); + } + } + return result; + } + + private void applyCozeToPersistedChunks(FileTaskEntity task, Runnable progressHook) { + if (task == null || task.getId() == null) { + return; + } + Map> allRowsByBaseId = loadAllRowsByBaseId(task); + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, task.getId()) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + log.info("[similar-asin] async coze start taskId={} chunks={}", task.getId(), chunks.size()); + for (TaskChunkEntity chunk : chunks) { + Map persistedRows = readChunkRows(chunk); + if (persistedRows.isEmpty()) { + continue; + } + List unresolvedRows = pickGroupRepresentativesForCoze(persistedRows.values()); + if (unresolvedRows.isEmpty()) { + continue; + } + List cozeRows = applyCozeInBatches(unresolvedRows, task, progressHook); + Map mergedRows = new LinkedHashMap<>(); + for (SimilarAsinResultRowDto resultRow : cozeRows) { + for (SimilarAsinResultRowDto expandedRow : expandRows(List.of(resultRow), allRowsByBaseId)) { + mergedRows.put(rowKey(expandedRow), expandedRow); + } + } + mergeChunkPayload(task.getId(), chunk.getScopeHash(), chunk.getChunkIndex(), new ArrayList<>(mergedRows.values())); + log.info("[similar-asin] async coze chunk merged taskId={} chunk={} unresolved={} merged={}", + task.getId(), chunk.getChunkIndex(), unresolvedRows.size(), mergedRows.size()); + } + } + + private void mergeChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, List rows) { + TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, chunkIndex) + .last("limit 1")); + if (chunk == null || rows == null || rows.isEmpty()) { + return; + } + Map persistedRows = readChunkRows(chunk); + for (SimilarAsinResultRowDto row : rows) { + persistedRows.put(rowKey(row), row); + } + String payloadJson = writeJson(rows == null ? List.of() : rows, "结果序列化失败"); + payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "similar ASIN chunk payload merge failed"); + String oldPayload = chunk.getPayloadJson(); + String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); + chunk.setPayloadJson(storedPayload); + chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson)); + chunk.setUpdatedAt(LocalDateTime.now()); + int updated = taskChunkMapper.updateById(chunk); + if (updated <= 0) { + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); + throw new IllegalStateException("similar ASIN chunk payload update failed"); + } + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload); + } + + private List expandRows(List representatives, + Map> allRowsByBaseId) { + if (representatives == null || representatives.isEmpty()) { + return List.of(); + } + List result = new ArrayList<>(); + Set emittedKeys = new LinkedHashSet<>(); + for (SimilarAsinResultRowDto representative : representatives) { + List siblings = resolveSiblingRows(representative, allRowsByBaseId); + if (siblings == null || siblings.isEmpty()) { + result.add(representative); + continue; + } + for (SimilarAsinParsedRowVo sibling : siblings) { + String key = rowKey(sibling); + if (!emittedKeys.add(key)) { + continue; + } + result.add(copyResultToSibling(representative, sibling)); + } + } + return result; + } + + private List resolveSiblingRows(SimilarAsinResultRowDto representative, + Map> groupedRows) { + if (representative == null || groupedRows == null || groupedRows.isEmpty()) { + return List.of(); + } + String representativeGroupKey = normalize(representative.getGroupKey()); + if (!representativeGroupKey.isBlank()) { + return groupedRows.getOrDefault(representativeGroupKey, List.of()); + } + String representativeKey = rowKey(representative); + String representativeLegacyKey = legacyRowKey(representative); + for (List rows : groupedRows.values()) { + for (SimilarAsinParsedRowVo row : rows) { + if (Objects.equals(representativeKey, rowKey(row)) + || Objects.equals(representativeLegacyKey, legacyRowKey(row))) { + return rows; + } + } + } + return List.of(); + } + + private Map> loadAllRowsByBaseId(FileTaskEntity task) { + try { + SimilarAsinParsedPayloadDto payload = readParsedPayload(task); + List rows = payload.getAllItems(); + if (rows == null || rows.isEmpty()) { + rows = payload.getItems(); + } + if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) { + rows = payload.getGroups().stream() + .filter(Objects::nonNull) + .flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream()) + .toList(); + } + return groupRowsByBaseId(rows); + } catch (Exception ex) { + log.warn("[similar-asin] read all rows failed taskId={} err={}", task.getId(), ex.getMessage()); + return new LinkedHashMap<>(); + } + } + + private List pickGroupRepresentativesForCoze(java.util.Collection rows) { + Map> groupedRows = new LinkedHashMap<>(); + if (rows == null) { + return List.of(); + } + for (SimilarAsinResultRowDto row : rows) { + if (row == null) { + continue; + } + String key = firstNonBlank(normalize(row.getGroupKey()), rowKey(row)); + groupedRows.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row); + } + List representatives = new ArrayList<>(); + for (List siblings : groupedRows.values()) { + boolean alreadyResolved = siblings.stream().anyMatch(this::hasResolvedCozeFields); + if (alreadyResolved) { + continue; + } + SimilarAsinResultRowDto candidate = null; + for (SimilarAsinResultRowDto sibling : siblings) { + if (shouldInspectRow(sibling)) { + candidate = sibling; + break; + } + } + if (candidate != null) { + representatives.add(candidate); + } + } + return representatives; + } + + private List buildParsedGroups(List rows) { + List groups = new ArrayList<>(); + for (List siblings : groupRowsByBaseId(rows).values()) { + if (siblings == null || siblings.isEmpty()) { + continue; + } + SimilarAsinParsedRowVo first = siblings.getFirst(); + SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo(); + group.setSourceFileKey(first.getSourceFileKey()); + group.setSourceFilename(first.getSourceFilename()); + group.setGroupKey(firstNonBlank(first.getGroupKey(), buildGroupKey(first.getSourceFileKey(), baseId(first.getDisplayId()), first.getRowIndex()))); + group.setBaseId(baseId(first.getDisplayId())); + group.setDisplayId(firstNonBlank(first.getDisplayId(), first.getSourceId())); + group.setItemCount(siblings.size()); + group.setItems(new ArrayList<>(siblings)); + groups.add(group); + } + return groups; + } + + private Map> groupRowsByBaseId(List rows) { + Map> result = new LinkedHashMap<>(); + if (rows == null) { + return result; + } + for (SimilarAsinParsedRowVo row : rows) { + String key = firstNonBlank(normalize(row.getGroupKey()), baseId(row.getDisplayId())); + result.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row); + } + return result; + } + + private List flattenSubmittedRows(SimilarAsinSubmitResultRequest request) { + if (request == null) { + return List.of(); + } + List groups = request.getGroups(); + if (groups != null && !groups.isEmpty()) { + List rows = new ArrayList<>(); + for (SimilarAsinResultGroupDto group : groups) { + if (group == null || group.getItems() == null || group.getItems().isEmpty()) { + continue; + } + for (SimilarAsinResultRowDto row : group.getItems()) { + if (row == null) { + continue; + } + if (normalize(row.getGroupKey()).isBlank()) { + row.setGroupKey(group.getGroupKey()); + } + if (normalize(row.getSourceFileKey()).isBlank()) { + row.setSourceFileKey(group.getSourceFileKey()); + } + if (normalize(row.getSourceFilename()).isBlank()) { + row.setSourceFilename(group.getSourceFilename()); + } + rows.add(row); + } + } + return rows; + } + return request.getItems() == null ? List.of() : request.getItems(); + } + + private int allRowCount(FileTaskEntity task) { + try { + SimilarAsinParsedPayloadDto payload = readParsedPayload(task); + if (payload.getAllItems() != null && !payload.getAllItems().isEmpty()) { + return payload.getAllItems().size(); + } + if (payload.getItems() != null && !payload.getItems().isEmpty()) { + return payload.getItems().size(); + } + if (payload.getGroups() != null && !payload.getGroups().isEmpty()) { + return payload.getGroups().stream() + .map(SimilarAsinParsedGroupVo::getItems) + .filter(Objects::nonNull) + .mapToInt(List::size) + .sum(); + } + return 0; + } catch (Exception ex) { + log.warn("[similar-asin] read all row count failed taskId={} err={}", task.getId(), ex.getMessage()); + return 0; + } + } + + private SimilarAsinResultRowDto copyResultToSibling(SimilarAsinResultRowDto representative, SimilarAsinParsedRowVo sibling) { + SimilarAsinResultRowDto row = new SimilarAsinResultRowDto(); + row.setSourceFileKey(sibling.getSourceFileKey()); + row.setSourceFilename(sibling.getSourceFilename()); + row.setRowToken(sibling.getRowToken()); + row.setGroupKey(sibling.getGroupKey()); + row.setId(sibling.getDisplayId()); + row.setAsin(sibling.getAsin()); + row.setCountry(sibling.getCountry()); + if (representative.hasImageUrl()) { + row.setUrls(representative.getUrls()); + } else { + row.setUrl(sibling.getUrl()); + } + row.setTitle(firstNonBlank(representative.getTitle(), sibling.getTitle())); + row.setError(representative.getError()); + row.setTitleRisk(representative.getTitleRisk()); + row.setAppearanceRisk(representative.getAppearanceRisk()); + row.setPatentRisk(representative.getPatentRisk()); + row.setConclusion(representative.getConclusion()); + row.setTitleReason(representative.getTitleReason()); + row.setAppearanceReason(representative.getAppearanceReason()); + row.setPatentReason(representative.getPatentReason()); + row.setDone(representative.getDone()); + return row; + } + + private String readAiPrompt(FileTaskEntity task) { + try { + return readParsedPayload(task).getAiPrompt(); + } catch (Exception ignored) { + return ""; + } + } + + private void finalizeTask(FileTaskEntity task, String error, int rowCount, boolean assembleWorkbook) { + String finalError = error; + FileResultEntity result = null; + List rows = fileResultMapper.selectList(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, task.getId()) + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .last("limit 1")); + if (!rows.isEmpty()) { + result = rows.getFirst(); + if (assembleWorkbook && shouldAssembleSynchronously()) { + try { + assembleResultWorkbook(task, result); + } catch (Exception ex) { + log.warn("[similar-asin] assemble result workbook failed taskId={} err={}", task.getId(), ex.getMessage()); + finalError = firstNonBlank(finalError, "Generate similar ASIN result failed"); + } + } + } + boolean failed = finalError != null && !finalError.isBlank(); + task.setStatus(failed ? STATUS_FAILED : STATUS_SUCCESS); + task.setSuccessFileCount(failed ? 0 : 1); + task.setFailedFileCount(failed ? 1 : 0); + task.setErrorMessage(finalError); + task.setUpdatedAt(LocalDateTime.now()); + task.setFinishedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + if (result != null) { + result.setSuccess(failed ? 0 : 1); + result.setErrorMessage(finalError); + result.setRowCount(rowCount > 0 ? rowCount : result.getRowCount()); + if (!failed && assembleWorkbook) { + result.setResultFilename(safeFileStem(result.getSourceFilename()) + "-result.xlsx"); + result.setResultFileUrl(null); + result.setResultFileSize(0L); + result.setResultContentType(CONTENT_TYPE_XLSX); + } + fileResultMapper.updateById(result); + if (!failed && assembleWorkbook) { + taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(), "task:" + task.getId()); + } + } + taskCacheService.deleteTaskCache(task.getId()); + } + + private boolean shouldAssembleSynchronously() { + return false; + } + + public boolean processResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null || job.getResultId() == null) { + throw new BusinessException("result file job arguments are incomplete"); + } + FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("task not found"); + } + FileResultEntity result = fileResultMapper.selectById(job.getResultId()); + if (result == null || !MODULE_TYPE.equals(result.getModuleType())) { + throw new BusinessException("result record not found"); + } + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, task.getId()) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + Map> allRowsByBaseId = loadAllRowsByBaseId(task); + int cozeWorkUnits = countCozeWorkUnits(chunks, Math.max(1, properties.getCozeBatchSize())); + int totalProgressUnits = Math.max(3, cozeWorkUnits + 3); + saveFileBuildProgress(task, job, totalProgressUnits, 0, "Submitting Coze"); + boolean pendingCoze = submitCozeBatches(task, result, job, chunks, allRowsByBaseId); + if (pendingCoze) { + taskFileJobService.touchRunning(job.getId()); + touchJavaSideTaskActivity(task.getId()); + saveFileBuildProgress(task, job, totalProgressUnits, 1, "Coze submitted, waiting for result"); + return false; + } + completeCozeFileJob(task, result, job, totalProgressUnits, cozeWorkUnits); + return true; + } + + @Scheduled(fixedDelayString = "${aiimage.similar-asin.coze-poll-delay-ms:5000}") + public void pollPendingCozeJobs() { + DistributedJobLockService.LockHandle lockHandle = + distributedJobLockService.tryLock("similar-asin:coze-poll", Duration.ofMinutes(1)); + if (lockHandle == null) { + return; + } + try (lockHandle) { + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING)) + .isNotNull(TaskScopeStateEntity::getCozeExecuteId) + .orderByAsc(TaskScopeStateEntity::getUpdatedAt) + .last("limit 50")); + if (states == null || states.isEmpty()) { + return; + } + log.info("[similar-asin] coze poll picked pending states count={}", states.size()); + for (TaskScopeStateEntity state : states) { + if (state == null || state.getId() == null) { + continue; + } + cozeTaskExecutor.execute(() -> pollPendingCozeState(state.getId())); + } + } + } + + private boolean submitCozeBatches(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + List chunks, + Map> allRowsByBaseId) { + if (chunks == null || chunks.isEmpty()) { + return countPendingCozeStates(task.getId()) > 0; + } + if (properties.getCozeToken() == null || properties.getCozeToken().isBlank()) { + log.warn("[similar-asin] coze token not configured, skip async coze taskId={} jobId={}", + task.getId(), job.getId()); + return false; + } + String prompt = readAiPrompt(task); + int batchSize = Math.max(1, properties.getCozeBatchSize()); + boolean pending = false; + for (TaskChunkEntity chunk : chunks) { + Map persistedRows = readChunkRows(chunk); + if (persistedRows.isEmpty()) { + continue; + } + List unresolvedRows = pickGroupRepresentativesForCoze(persistedRows.values()); + if (unresolvedRows.isEmpty()) { + continue; + } + int batchTotal = Math.max(1, (unresolvedRows.size() + batchSize - 1) / batchSize); + int batchIndex = 1; + for (int i = 0; i < unresolvedRows.size(); i += batchSize) { + List batchRows = + unresolvedRows.subList(i, Math.min(i + batchSize, unresolvedRows.size())); + pending |= submitCozeBatch(task, result, job, chunk, batchRows, batchIndex, batchTotal, prompt, allRowsByBaseId); + batchIndex++; + } + } + return pending || countPendingCozeStates(task.getId()) > 0; + } + + private boolean submitCozeBatch(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + TaskChunkEntity chunk, + List batchRows, + int batchIndex, + int batchTotal, + String prompt, + Map> allRowsByBaseId) { + if (batchRows == null || batchRows.isEmpty()) { + return false; + } + String batchScopeKey = buildCozeBatchScopeKey(job.getId(), chunk.getScopeHash(), chunk.getChunkIndex(), batchIndex); + String batchScopeHash = DigestUtil.sha256Hex(batchScopeKey); + TaskScopeStateEntity existing = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, task.getId()) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, batchScopeHash) + .last("limit 1")); + if (existing != null) { + return COZE_STATUS_SUBMITTED.equals(existing.getCozeStatus()) + || COZE_STATUS_RUNNING.equals(existing.getCozeStatus()); + } + try { + SimilarAsinCozeClient.CozeSubmitResponse submit = cozeClient.submitWorkflow(batchRows, prompt); + if (submit.immediateData() != null && !submit.immediateData().isBlank()) { + List cozeRows = cozeClient.mergeRowsFromDataText(batchRows, submit.immediateData()); + mergeCozeRowsIntoChunk(task, chunk.getScopeHash(), chunk.getChunkIndex(), cozeRows, allRowsByBaseId); + return false; + } + if (submit.executeId() == null || submit.executeId().isBlank()) { + mergeCozeRowsIntoChunk(task, + chunk.getScopeHash(), + chunk.getChunkIndex(), + cozeClient.markRowsFailed(batchRows, "Coze async execute_id missing"), + allRowsByBaseId); + return false; + } + saveCozeBatchState(task, result, job, chunk, batchRows, batchScopeKey, batchScopeHash, + batchIndex, batchTotal, submit.executeId()); + log.info("[similar-asin] coze async submitted taskId={} jobId={} chunk={} batch={}/{} executeId={}", + task.getId(), job.getId(), chunk.getChunkIndex(), batchIndex, batchTotal, submit.executeId()); + return true; + } catch (Exception ex) { + String message = firstNonBlank(ex.getMessage(), "Coze submit failed"); + log.warn("[similar-asin] coze async submit failed taskId={} jobId={} chunk={} batch={}/{} err={}", + task.getId(), job.getId(), chunk.getChunkIndex(), batchIndex, batchTotal, message); + mergeCozeRowsIntoChunk(task, + chunk.getScopeHash(), + chunk.getChunkIndex(), + cozeClient.markRowsFailed(batchRows, message), + allRowsByBaseId); + return false; + } + } + + private void saveCozeBatchState(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + TaskChunkEntity chunk, + List batchRows, + String batchScopeKey, + String batchScopeHash, + int batchIndex, + int batchTotal, + String executeId) { + LocalDateTime now = LocalDateTime.now(); + CozeBatchContext context = new CozeBatchContext( + job.getId(), + result.getId(), + chunk.getScopeHash(), + chunk.getChunkIndex(), + batchIndex, + batchTotal + ); + String batchPayload = writeJson(batchRows, "serialize coze batch payload failed"); + String storedBatchPayload = transientPayloadStorageService.storeParsedPayloadFast( + MODULE_TYPE, task.getId(), batchScopeHash, batchPayload, true); + TaskScopeStateEntity state = new TaskScopeStateEntity(); + state.setTaskId(task.getId()); + state.setModuleType(MODULE_TYPE); + state.setScopeKey(batchScopeKey); + state.setScopeHash(batchScopeHash); + state.setParsedPayloadJson(storedBatchPayload); + state.setStateJson(writeJson(context, "serialize coze batch context failed")); + state.setCozeExecuteId(executeId); + state.setCozeStatus(COZE_STATUS_SUBMITTED); + state.setCozeSubmittedAt(now); + state.setCozeAttemptCount(0); + state.setChunkTotal(batchTotal); + state.setReceivedChunkCount(batchIndex); + state.setCompleted(0); + state.setCreatedAt(now); + state.setUpdatedAt(now); + try { + taskScopeStateMapper.insert(state); + touchJavaSideTaskActivity(task.getId()); + } catch (DuplicateKeyException ex) { + transientPayloadStorageService.deletePayloadIfPresent(storedBatchPayload); + log.info("[similar-asin] duplicate coze batch state ignored taskId={} scope={}", + task.getId(), batchScopeKey); + } + } + + private void pollPendingCozeState(Long stateId) { + TaskScopeStateEntity state = taskScopeStateMapper.selectById(stateId); + if (state == null || state.getCozeExecuteId() == null || state.getCozeExecuteId().isBlank()) { + return; + } + if (!(COZE_STATUS_SUBMITTED.equals(state.getCozeStatus()) || COZE_STATUS_RUNNING.equals(state.getCozeStatus()))) { + return; + } + if (!tryClaimCozeStateForPoll(state)) { + return; + } + CozeBatchContext context = readCozeBatchContext(state); + if (context == null || context.jobId() == null || context.resultId() == null) { + markCozeStateTerminal(state, COZE_STATUS_FAILED, "Coze batch context missing"); + return; + } + taskFileJobService.touchRunning(context.jobId()); + try { + SimilarAsinCozeClient.CozePollResponse poll = cozeClient.pollWorkflow(state.getCozeExecuteId()); + if (!poll.hasPayload() && !poll.isFinished() && !isCozeStateTimedOut(state)) { + updateCozeStateRunning(state, null); + return; + } + String failureMessage = poll.isFailed() + ? firstNonBlank(poll.failureMessage(), "Coze async workflow failed") + : ""; + if (!poll.hasPayload() && failureMessage.isBlank()) { + failureMessage = isCozeStateTimedOut(state) + ? "Coze async workflow poll timeout" + : "Coze async workflow completed without output"; + } + List batchRows = readCozeBatchRows(state); + List cozeRows = failureMessage.isBlank() + ? cozeClient.mergeRowsFromDataText(batchRows, poll.resolvedPayloadText()) + : cozeClient.markRowsFailed(batchRows, failureMessage); + FileTaskEntity task = fileTaskMapper.selectById(state.getTaskId()); + if (task != null) { + Map> allRowsByBaseId = loadAllRowsByBaseId(task); + mergeCozeRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), cozeRows, allRowsByBaseId); + } + markCozeStateTerminal(state, + failureMessage.isBlank() ? COZE_STATUS_DONE : COZE_STATUS_FAILED, + failureMessage.isBlank() ? null : failureMessage); + maybeFinalizeCozeJob(state.getTaskId(), context); + } catch (Exception ex) { + String message = firstNonBlank(ex.getMessage(), "Coze poll failed"); + if (isCozeStateTimedOut(state)) { + List batchRows = readCozeBatchRows(state); + FileTaskEntity task = fileTaskMapper.selectById(state.getTaskId()); + if (task != null) { + Map> allRowsByBaseId = loadAllRowsByBaseId(task); + mergeCozeRowsIntoChunk(task, + context.chunkScopeHash(), + context.chunkIndex(), + cozeClient.markRowsFailed(batchRows, message), + allRowsByBaseId); + } + markCozeStateTerminal(state, COZE_STATUS_FAILED, message); + maybeFinalizeCozeJob(state.getTaskId(), context); + return; + } + log.warn("[similar-asin] coze poll failed taskId={} stateId={} executeId={} err={}", + state.getTaskId(), state.getId(), state.getCozeExecuteId(), message); + updateCozeStateRunning(state, message); + } + } + + private void updateCozeStateRunning(TaskScopeStateEntity state, String error) { + int updated = taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING)) + .set(TaskScopeStateEntity::getCozeStatus, COZE_STATUS_RUNNING) + .set(TaskScopeStateEntity::getCozeLastPolledAt, LocalDateTime.now()) + .set(TaskScopeStateEntity::getCozeAttemptCount, cozeAttemptCount(state) + 1) + .set(TaskScopeStateEntity::getCozeError, error) + .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); + if (updated > 0) { + touchJavaSideTaskActivity(state.getTaskId()); + } + } + + private boolean tryClaimCozeStateForPoll(TaskScopeStateEntity state) { + if (state == null || state.getId() == null) { + return false; + } + LocalDateTime now = LocalDateTime.now(); + long intervalMillis = Math.max(200L, properties.getCozePollIntervalMillis()); + if (state.getCozeLastPolledAt() != null + && Duration.between(state.getCozeLastPolledAt(), now).toMillis() < intervalMillis) { + return false; + } + return taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING)) + .and(wrapper -> wrapper + .isNull(TaskScopeStateEntity::getCozeLastPolledAt) + .or() + .le(TaskScopeStateEntity::getCozeLastPolledAt, now.minus(Duration.ofMillis(intervalMillis)))) + .set(TaskScopeStateEntity::getCozeStatus, COZE_STATUS_RUNNING) + .set(TaskScopeStateEntity::getCozeLastPolledAt, now) + .set(TaskScopeStateEntity::getUpdatedAt, now)) > 0; + } + + private void markCozeStateTerminal(TaskScopeStateEntity state, String status, String error) { + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING)) + .set(TaskScopeStateEntity::getCozeStatus, status) + .set(TaskScopeStateEntity::getCozeCompletedAt, LocalDateTime.now()) + .set(TaskScopeStateEntity::getCozeLastPolledAt, LocalDateTime.now()) + .set(TaskScopeStateEntity::getCozeAttemptCount, cozeAttemptCount(state) + 1) + .set(TaskScopeStateEntity::getCozeError, error) + .set(TaskScopeStateEntity::getCompleted, 1) + .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); + } + + private void maybeFinalizeCozeJob(Long taskId, CozeBatchContext context) { + if (taskId == null || context == null || countPendingCozeStates(taskId) > 0) { + return; + } + DistributedJobLockService.LockHandle lockHandle = + distributedJobLockService.tryLock("similar-asin:coze-finalize:" + taskId, Duration.ofMinutes(5)); + if (lockHandle == null) { + return; + } + try (lockHandle) { + if (countPendingCozeStates(taskId) > 0) { + return; + } + FileTaskEntity task = fileTaskMapper.selectById(taskId); + FileResultEntity result = fileResultMapper.selectById(context.resultId()); + TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId()); + if (task == null || result == null || job == null || "SUCCESS".equals(job.getStatus())) { + return; + } + int cozeWorkUnits = countCompletedCozeStates(taskId); + int totalProgressUnits = Math.max(3, cozeWorkUnits + 3); + completeCozeFileJob(task, result, job, totalProgressUnits, cozeWorkUnits); + taskFileJobService.markSuccess(job, result.getResultFileUrl()); + cleanupResultFileJob(job); + log.info("[similar-asin] coze async job finalized taskId={} jobId={} resultId={} resultFileUrl={}", + taskId, job.getId(), result.getId(), result.getResultFileUrl()); + } catch (Exception ex) { + TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId()); + if (job != null) { + taskFileJobService.markFailed(job, firstNonBlank(ex.getMessage(), "similar ASIN result file build failed")); + } + log.warn("[similar-asin] coze async finalize failed taskId={} resultId={} err={}", + taskId, context.resultId(), ex.getMessage(), ex); + } + } + + private void completeCozeFileJob(FileTaskEntity task, + FileResultEntity result, + TaskFileJobEntity job, + int totalProgressUnits, + int cozeWorkUnits) { + int assembleProgress = Math.max(1, Math.min(totalProgressUnits - 2, cozeWorkUnits)); + saveFileBuildProgress(task, job, totalProgressUnits, assembleProgress, "Assembling xlsx"); + assembleResultWorkbook(task, result); + saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits - 1, "Uploading result file"); + fileResultMapper.updateById(result); + saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits, "Result file generated"); + } + + private void mergeCozeRowsIntoChunk(FileTaskEntity task, + String chunkScopeHash, + Integer chunkIndex, + List cozeRows, + Map> allRowsByBaseId) { + if (task == null || cozeRows == null || cozeRows.isEmpty()) { + return; + } + Map mergedRows = new LinkedHashMap<>(); + for (SimilarAsinResultRowDto resultRow : cozeRows) { + for (SimilarAsinResultRowDto expandedRow : expandRows(List.of(resultRow), allRowsByBaseId)) { + mergedRows.put(rowKey(expandedRow), expandedRow); + } + } + mergeChunkPayload(task.getId(), chunkScopeHash, chunkIndex, new ArrayList<>(mergedRows.values())); + } + + private List readCozeBatchRows(TaskScopeStateEntity state) { + if (state == null || state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) { + return List.of(); + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload( + state.getParsedPayloadJson(), "read similar ASIN coze batch failed"); + JsonNode array = objectMapper.readTree(payloadJson); + if (!array.isArray()) { + return List.of(); + } + List rows = new ArrayList<>(); + for (JsonNode node : array) { + rows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class)); + } + return rows; + } catch (Exception ex) { + log.warn("[similar-asin] read coze batch failed taskId={} stateId={} err={}", + state.getTaskId(), state.getId(), ex.getMessage()); + return List.of(); + } + } + + private CozeBatchContext readCozeBatchContext(TaskScopeStateEntity state) { + if (state == null || state.getStateJson() == null || state.getStateJson().isBlank()) { + return null; + } + try { + return objectMapper.readValue(state.getStateJson(), CozeBatchContext.class); + } catch (Exception ex) { + log.warn("[similar-asin] read coze batch context failed taskId={} stateId={} err={}", + state.getTaskId(), state.getId(), ex.getMessage()); + return null; + } + } + + private boolean isCozeStateTimedOut(TaskScopeStateEntity state) { + if (state == null || state.getCozeSubmittedAt() == null) { + return false; + } + long timeoutMillis = Math.max(10000L, properties.getCozePollTimeoutMillis()); + return Duration.between(state.getCozeSubmittedAt(), LocalDateTime.now()).toMillis() >= timeoutMillis; + } + + private int cozeAttemptCount(TaskScopeStateEntity state) { + return state == null || state.getCozeAttemptCount() == null ? 0 : state.getCozeAttemptCount(); + } + + private int countPendingCozeStates(Long taskId) { + if (taskId == null || taskId <= 0) { + return 0; + } + Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING))); + return count == null ? 0 : count.intValue(); + } + + private int countCompletedCozeStates(Long taskId) { + if (taskId == null || taskId <= 0) { + return 0; + } + Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_DONE, COZE_STATUS_FAILED))); + return count == null ? 0 : count.intValue(); + } + + private boolean isJavaSideProcessing(Long taskId) { + return countPendingCozeStates(taskId) > 0 + || taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE) > 0; + } + + private void touchJavaSideTaskActivity(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + fileTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(FileTaskEntity::getId, taskId) + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())); + } + + private String buildCozeBatchScopeKey(Long jobId, String chunkScopeHash, Integer chunkIndex, int batchIndex) { + return "coze:job:" + jobId + + ":chunk:" + (chunkIndex == null ? 0 : chunkIndex) + + ":" + firstNonBlank(chunkScopeHash, "unknown") + + ":batch:" + batchIndex; + } + + private int countCozeWorkUnits(List chunks, int batchSize) { + if (chunks == null || chunks.isEmpty()) { + return 0; + } + int total = 0; + for (TaskChunkEntity chunk : chunks) { + Map persistedRows = readChunkRows(chunk); + if (persistedRows.isEmpty()) { + continue; + } + int unresolved = pickGroupRepresentativesForCoze(persistedRows.values()).size(); + if (unresolved > 0) { + total += Math.max(1, (unresolved + batchSize - 1) / batchSize); + } + } + return total; + } + + private void saveFileBuildProgress(FileTaskEntity task, + TaskFileJobEntity job, + int total, + int completed, + String message) { + if (task == null || task.getId() == null || job == null || job.getId() == null) { + return; + } + int safeTotal = Math.max(1, total); + int safeCompleted = Math.max(0, Math.min(completed, safeTotal)); + taskProgressSnapshotService.save( + task.getId(), + MODULE_TYPE, + STATUS_RUNNING, + safeTotal, + safeCompleted, + 0, + job.getScopeKey(), + message, + Map.of("phase", "RESULT_FILE", "jobId", job.getId()) + ); + } + + public void cleanupResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null) { + return; + } + deleteTransientTaskPayloads(job.getTaskId()); + taskChunkMapper.delete(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, job.getTaskId()) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + taskScopeStateMapper.delete(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, job.getTaskId()) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + } + + private void assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) { + SimilarAsinParsedPayloadDto parsed = readParsedPayload(task); + Map resultMap = loadPersistedResultRowsWithRetry(task.getId(), parsed.getAllItems().size()); + long resolvedRows = parsed.getAllItems().stream() + .filter(row -> findResultRow(row, resultMap) != null || findResultRowByAsin(row.getAsin(), resultMap) != null) + .count(); + long reasonRows = resultMap.values().stream() + .filter(this::hasReasonFields) + .count(); + log.info("[similar-asin] assemble workbook taskId={} parsedRows={} resultRows={} resolvedRows={} reasonRows={}", + task.getId(), parsed.getAllItems().size(), resultMap.size(), resolvedRows, reasonRows); + if (!parsed.getAllItems().isEmpty() && resultMap.isEmpty()) { + throw new BusinessException("相似ASIN检测结果为空，请稍后重试生成结果文件"); + } + File outputDir = new File(storageProperties.getLocalTempDir(), "similar-asin-result"); + if (!outputDir.exists() && !outputDir.mkdirs()) { + throw new BusinessException("创建结果目录失败"); + } + String filename = safeFileStem(result.getSourceFilename()) + "-result.xlsx"; + String tempFilename = safeFileStem(result.getSourceFilename()) + + "-" + task.getId() + + "-" + result.getId() + + "-" + UUID.randomUUID() + + "-result.xlsx"; + File xlsx = new File(outputDir, tempFilename); + try { + writeResultWorkbook(xlsx, parsed, resultMap); + String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE); + result.setResultFilename(filename); + result.setResultFileUrl(objectKey); + result.setResultFileSize(xlsx.length()); + result.setResultContentType(CONTENT_TYPE_XLSX); + result.setRowCount(parsed.getAllItems().size()); + } finally { + if (xlsx.exists() && !xlsx.delete()) { + log.warn("[similar-asin] delete temp xlsx failed file={}", xlsx); + } + } + } + + private Map loadPersistedResultRows(Long taskId) { + Map result = new LinkedHashMap<>(); + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + for (TaskChunkEntity chunk : chunks) { + result.putAll(readChunkRows(chunk)); + } + return result; + } + + private Map loadPersistedResultRowsWithRetry(Long taskId, int expectedParsedRows) { + Map result = loadPersistedResultRows(taskId); + if (expectedParsedRows <= 0 || !result.isEmpty()) { + return result; + } + for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) { + sleepBeforeResultRowsRetry(attempt); + result = loadPersistedResultRows(taskId); + if (!result.isEmpty()) { + log.info("[similar-asin] result rows recovered after retry taskId={} attempt={} rows={}", + taskId, attempt, result.size()); + return result; + } + log.warn("[similar-asin] result rows still empty after retry taskId={} attempt={}/{}", + taskId, attempt, RESULT_ROWS_READ_RETRY_LIMIT); + } + return result; + } + + private void sleepBeforeResultRowsRetry(int attempt) { + try { + Thread.sleep(RESULT_ROWS_READ_RETRY_DELAY_MS * attempt); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + + private void writeResultWorkbook(File xlsx, SimilarAsinParsedPayloadDto parsed, Map resultMap) { + try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) { + Sheet sheet = workbook.createSheet("相似ASIN检测结果"); + CellStyle headerStyle = workbook.createCellStyle(); + Font font = workbook.createFont(); + font.setBold(true); + headerStyle.setFont(font); + + List resultHeaders = new ArrayList<>(RESULT_HEADERS); + resultHeaders.add(4, "标题"); + resultHeaders.add(5, "图片链接"); + Row header = sheet.createRow(0); + for (int i = 0; i < resultHeaders.size(); i++) { + Cell cell = header.createCell(i); + cell.setCellValue(resultHeaders.get(i)); + cell.setCellStyle(headerStyle); + } + + int rowIndex = 1; + for (SimilarAsinParsedRowVo parsedRow : parsed.getAllItems()) { + SimilarAsinResultRowDto resultRow = findResultRow(parsedRow, resultMap); + if (resultRow == null) { + resultRow = findResultRowByAsin(parsedRow.getAsin(), resultMap); + } + String missingReason = ""; + if (resultRow == null) { + missingReason = hasPromptFields(parsedRow) ? "未匹配到检测结果" : "未送检：缺少标题或图片"; + } + Row row = sheet.createRow(rowIndex++); + int col = 0; + row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId())); + row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), "")); + row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), "")); + row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "价格", "price")); + row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getTitle(), "") : firstNonBlank(resultRow.getTitle(), parsedRow.getTitle())); + row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getUrl(), "") : imageUrlCellValue(resultRow, parsedRow.getUrl())); + row.createCell(col++).setCellValue(resultRow == null ? missingReason : userFacingCozeCellValue(resultRow, resultRow.getTitleRisk())); + row.createCell(col++).setCellValue(resultRow == null ? missingReason : userFacingCozeCellValue(resultRow, resultRow.getAppearanceRisk())); + row.createCell(col++).setCellValue(resultRow == null ? missingReason : userFacingCozeCellValue(resultRow, resultRow.getPatentRisk())); + row.createCell(col).setCellValue(resultRow == null ? "未送检" : userFacingConclusion(resultRow)); + } + writeReasonSheet(workbook, headerStyle, parsed, resultMap); + workbook.write(fos); + workbook.dispose(); + } catch (Exception ex) { + throw new BusinessException("生成相似ASIN检测结果失败"); + } + } + + private void writeReasonSheet(SXSSFWorkbook workbook, + CellStyle headerStyle, + SimilarAsinParsedPayloadDto parsed, + Map resultMap) { + Sheet sheet = workbook.createSheet("原因"); + Row header = sheet.createRow(0); + List headers = List.of("ASIN", "外观原因", "专利原因", "标题原因"); + for (int i = 0; i < headers.size(); i++) { + Cell cell = header.createCell(i); + cell.setCellValue(headers.get(i)); + cell.setCellStyle(headerStyle); + } + + Set writtenAsins = new LinkedHashSet<>(); + int rowIndex = 1; + for (SimilarAsinParsedRowVo parsedRow : parsed.getAllItems()) { + String asin = normalize(parsedRow.getAsin()).toUpperCase(Locale.ROOT); + if (asin.isBlank() || !writtenAsins.add(asin)) { + continue; + } + SimilarAsinResultRowDto resultRow = findResultRow(parsedRow, resultMap); + if (resultRow == null) { + resultRow = findResultRowByAsin(asin, resultMap); + } + Row row = sheet.createRow(rowIndex++); + row.createCell(0).setCellValue(asin); + row.createCell(1).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getAppearanceReason(), "")); + row.createCell(2).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getPatentReason(), "")); + row.createCell(3).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getTitleReason(), "")); + rowIndex++; + } + } + + private SimilarAsinResultRowDto findResultRowByAsin(String asin, Map resultMap) { + String normalizedAsin = normalize(asin).toUpperCase(Locale.ROOT); + if (normalizedAsin.isBlank() || resultMap == null || resultMap.isEmpty()) { + return null; + } + SimilarAsinResultRowDto fallback = null; + for (SimilarAsinResultRowDto row : resultMap.values()) { + if (row == null || !normalizedAsin.equals(normalize(row.getAsin()).toUpperCase(Locale.ROOT))) { + continue; + } + if (hasResolvedCozeFields(row) || hasReasonFields(row)) { + return row; + } + if (fallback == null) { + fallback = row; + } + } + return fallback; + } + + private boolean hasReasonFields(SimilarAsinResultRowDto row) { + if (row == null) { + return false; + } + return !normalize(row.getAppearanceReason()).isBlank() + || !normalize(row.getPatentReason()).isBlank() + || !normalize(row.getTitleReason()).isBlank(); + } + + private String imageUrlCellValue(SimilarAsinResultRowDto resultRow, String fallbackUrl) { + if (resultRow == null) { + return firstNonBlank(fallbackUrl, ""); + } + List urls = resultRow.getUrls(); + if (urls != null && !urls.isEmpty()) { + return String.join("\n", urls); + } + return firstNonBlank(resultRow.getUrl(), fallbackUrl); + } + + private ParsedWorkbook parseWorkbook(File input, SimilarAsinSourceFileDto source) { + DataFormatter formatter = new DataFormatter(); + try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) { + Sheet sheet = workbook.getSheetAt(0); + Row header = sheet.getRow(0); + if (header == null) { + throw new BusinessException("Excel 表头为空"); + } + Map headerMap = buildHeaderMap(header, formatter); + List headers = readHeaders(header, formatter); + int idCol = findRequiredHeader(headerMap, "id"); + int asinCol = findRequiredHeader(headerMap, "asin"); + int countryCol = findRequiredHeader(headerMap, "国家", "country"); + int urlCol = findOptionalHeaderExact(headerMap, + "url", "rul", "link", "image", "img", "pic", "picture", + "链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接"); + int titleCol = findOptionalHeaderExact(headerMap, + "标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称"); + + List allRows = new ArrayList<>(); + int total = 0; + int dropped = 0; + String currentBlockBaseId = ""; + String currentGroupKey = ""; + for (int i = 1; i <= sheet.getLastRowNum(); i++) { + Row row = sheet.getRow(i); + if (row == null) { + continue; + } + String id = cell(row, idCol, formatter); + String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT); + String country = cell(row, countryCol, formatter); + if (id.isBlank() && asin.isBlank() && country.isBlank()) { + continue; + } + total++; + if (id.isBlank() || asin.isBlank() || country.isBlank()) { + dropped++; + continue; + } + SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo(); + vo.setSourceFileKey(source.getFileKey()); + vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName())); + vo.setRowIndex(i + 1); + vo.setSourceId(id); + vo.setDisplayId(normalizeDisplayId(id)); + String rowBaseId = baseId(vo.getDisplayId()); + if (!Objects.equals(currentBlockBaseId, rowBaseId)) { + currentBlockBaseId = rowBaseId; + currentGroupKey = buildGroupKey(source.getFileKey(), rowBaseId, vo.getRowIndex()); + } + vo.setGroupKey(currentGroupKey); + vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex())); + vo.setAsin(asin); + vo.setCountry(country); + vo.setUrl(urlCol >= 0 ? cell(row, urlCol, formatter) : ""); + vo.setTitle(titleCol >= 0 ? cell(row, titleCol, formatter) : ""); + vo.setValues(readRowValues(row, headers, formatter)); + allRows.add(vo); + } + hydratePromptFields(allRows); + if (allRows.isEmpty()) { + throw new BusinessException("no valid similar ASIN rows"); + } + return new ParsedWorkbook(total, dropped, headers, allRows); + } catch (BusinessException ex) { + throw ex; + } catch (Exception ex) { + log.warn("[similar-asin] parse failed file={} err={}", input, ex.getMessage()); + throw new BusinessException("解析 Excel 失败"); + } + } + + private void hydratePromptFields(List rows) { + if (rows == null || rows.isEmpty()) { + return; + } + Map> rowsByBaseId = groupRowsByBaseId(rows); + for (List siblings : rowsByBaseId.values()) { + String title = ""; + String url = ""; + for (SimilarAsinParsedRowVo sibling : siblings) { + title = firstNonBlank(title, sibling.getTitle()); + url = firstNonBlank(url, sibling.getUrl()); + } + for (SimilarAsinParsedRowVo sibling : siblings) { + if (normalize(sibling.getTitle()).isBlank()) { + sibling.setTitle(title); + } + if (normalize(sibling.getUrl()).isBlank()) { + sibling.setUrl(url); + } + } + } + } + + private boolean hasPromptFields(SimilarAsinParsedRowVo row) { + return row != null && (!normalize(row.getTitle()).isBlank() || !normalize(row.getUrl()).isBlank()); + } + + private boolean shouldInspectRow(SimilarAsinResultRowDto row) { + return row != null && (!normalize(row.getTitle()).isBlank() || row.hasImageUrl()); + } + + private boolean hasResolvedCozeFields(SimilarAsinResultRowDto row) { + if (row == null) { + return false; + } + return hasUsableCozeField(row.getTitleRisk()) + || hasUsableCozeField(row.getAppearanceRisk()) + || hasUsableCozeField(row.getPatentRisk()) + || hasUsableCozeField(row.getConclusion()); + } + + private boolean hasUsableCozeField(String value) { + String normalized = normalize(value); + return !normalized.isBlank() && !isTechnicalCozeFailure(normalized); + } + + private Map buildHeaderMap(Row header, DataFormatter formatter) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < header.getLastCellNum(); i++) { + String val = normalize(formatter.formatCellValue(header.getCell(i))); + if (!val.isBlank()) { + map.putIfAbsent(val.toLowerCase(Locale.ROOT), i); + } + } + return map; + } + + private List readHeaders(Row header, DataFormatter formatter) { + List headers = new ArrayList<>(); + for (int i = 0; i < header.getLastCellNum(); i++) { + String val = normalize(formatter.formatCellValue(header.getCell(i))); + headers.add(val.isBlank() ? "列" + (i + 1) : val); + } + return headers; + } + + private Map readRowValues(Row row, List headers, DataFormatter formatter) { + Map values = new LinkedHashMap<>(); + for (int i = 0; i < headers.size(); i++) { + values.put(headers.get(i), cell(row, i, formatter)); + } + return values; + } + + private int findRequiredHeader(Map map, String... names) { + int idx = findOptionalHeader(map, names); + if (idx < 0) { + throw new BusinessException("缺少必要表头: " + String.join("/", names)); + } + return idx; + } + + private int findOptionalHeader(Map map, String... names) { + for (Map.Entry entry : map.entrySet()) { + for (String name : names) { + if (entry.getKey().contains(name.toLowerCase(Locale.ROOT))) { + return entry.getValue(); + } + } + } + return -1; + } + + private int findOptionalHeaderExact(Map map, String... names) { + for (Map.Entry entry : map.entrySet()) { + String normalizedHeader = normalizeHeaderAlias(entry.getKey()); + for (String name : names) { + if (normalizedHeader.equals(normalizeHeaderAlias(name))) { + return entry.getValue(); + } + } + } + return -1; + } + + private String normalizeHeaderAlias(String value) { + String normalized = normalize(value).toLowerCase(Locale.ROOT); + return normalized.replaceAll("[\\s_\\-()（）\\[\\]{}:：/\\\\]+", ""); + } + + private String cell(Row row, int col, DataFormatter formatter) { + return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col))); + } + + private String baseId(String id) { + String s = normalize(id); + int idx = s.indexOf('_'); + return idx > 0 ? s.substring(0, idx) : s; + } + + private String normalizeDisplayId(String id) { + return id == null ? "" : id.trim(); + } + + private void mergeHeaders(List mergedHeaders, List headers) { + if (mergedHeaders == null || headers == null || headers.isEmpty()) { + return; + } + Set existing = new LinkedHashSet<>(mergedHeaders); + for (String header : headers) { + if (existing.add(header)) { + mergedHeaders.add(header); + } + } + } + + private String buildAggregateScopeKey(List sourceFiles) { + List fileKeys = sourceFiles == null ? List.of() : sourceFiles.stream() + .map(SimilarAsinSourceFileDto::getFileKey) + .filter(Objects::nonNull) + .map(String::trim) + .filter(value -> !value.isBlank()) + .toList(); + if (fileKeys.isEmpty()) { + return ""; + } + if (fileKeys.size() == 1) { + return fileKeys.get(0); + } + return "aggregate:" + DigestUtil.sha256Hex(String.join("|", fileKeys)); + } + + private String buildAggregateSourceFilenameLabel(List sourceFiles) { + if (sourceFiles == null || sourceFiles.isEmpty()) { + return "similar-asin"; + } + String firstFilename = sourceFiles.stream() + .map(SimilarAsinSourceFileDto::getOriginalFilename) + .filter(Objects::nonNull) + .map(String::trim) + .filter(value -> !value.isBlank()) + .findFirst() + .orElse("similar-asin"); + if (sourceFiles.size() == 1) { + return firstFilename; + } + return firstFilename + " 等 " + sourceFiles.size() + " 个文件"; + } + + private String buildRowToken(String sourceFileKey, Integer rowIndex) { + return normalize(sourceFileKey) + "::row::" + (rowIndex == null ? 0 : rowIndex); + } + + private String buildGroupKey(String sourceFileKey, String rowBaseId, Integer rowIndex) { + return normalize(sourceFileKey) + "::" + normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex); + } + + private long countTask(Long userId, String status) { + Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getUserId, userId) + .eq(FileTaskEntity::getStatus, status)); + return count == null ? 0L : count; + } + + private int countChunks(Long taskId, String scopeHash) { + Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash)); + return count == null ? 0 : count.intValue(); + } + + private SimilarAsinTaskItemVo toTaskItem(FileTaskEntity task) { + SimilarAsinTaskItemVo vo = new SimilarAsinTaskItemVo(); + vo.setId(task.getId()); + vo.setTaskNo(task.getTaskNo()); + vo.setStatus(task.getStatus()); + vo.setErrorMessage(task.getErrorMessage()); + vo.setCreatedAt(fmt(task.getCreatedAt())); + vo.setUpdatedAt(fmt(task.getUpdatedAt())); + vo.setFinishedAt(fmt(task.getFinishedAt())); + return vo; + } + + private SimilarAsinHistoryItemVo toHistoryItem(FileResultEntity row, String taskStatus, TaskFileJobEntity job) { + SimilarAsinHistoryItemVo vo = new SimilarAsinHistoryItemVo(); + vo.setResultId(row.getId()); + vo.setTaskId(row.getTaskId()); + vo.setSourceFilename(row.getSourceFilename()); + vo.setResultFilename(row.getResultFilename()); + vo.setDownloadUrl(null); + attachFileJobState(vo, row, job); + vo.setTaskStatus(taskStatus); + vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1); + vo.setError(row.getErrorMessage()); + vo.setRowCount(row.getRowCount()); + vo.setCreatedAt(fmt(row.getCreatedAt())); + return vo; + } + + private void attachFileJobState(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job) { + vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank()); + if (job == null) { + vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); + attachFileProgress(vo, row, job); + return; + } + vo.setFileJobId(job.getId()); + vo.setFileStatus(job.getStatus()); + vo.setFileError(job.getErrorMessage()); + attachFileProgress(vo, row, job); + } + + private void attachFileProgress(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job) { + if (row == null || row.getTaskId() == null) { + return; + } + if (Boolean.TRUE.equals(vo.getFileReady())) { + vo.setFileProgressCurrent(1); + vo.setFileProgressTotal(1); + vo.setFileProgressPercent(100); + vo.setFileProgressMessage("结果文件已生成"); + return; + } + TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(row.getTaskId(), MODULE_TYPE); + if (snapshot == null) { + return; + } + int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount(); + int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount(); + if (total <= 0) { + return; + } + current = Math.max(0, Math.min(current, total)); + int percent = Math.max(1, Math.min(99, (int) Math.floor(current * 100.0 / total))); + if (job != null && STATUS_RUNNING.equals(job.getStatus())) { + LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : job.getUpdatedAt(); + long elapsedSeconds = baseTime == null ? 0 : Math.max(0, Duration.between(baseTime, LocalDateTime.now()).getSeconds()); + if (current <= 0) { + percent = Math.max(percent, Math.min(35, 8 + (int) (elapsedSeconds / 6))); + } else if (current < total) { + percent = Math.max(percent, Math.min(92, percent + (int) (elapsedSeconds / 10))); + } + } + vo.setFileProgressCurrent(current); + vo.setFileProgressTotal(total); + vo.setFileProgressPercent(percent); + vo.setFileProgressMessage(snapshot.getMessage()); + } + + private String fmt(LocalDateTime t) { + return t == null ? null : t.toString(); + } + + private String firstNonBlank(String preferred, String fallback) { + return preferred == null || preferred.isBlank() ? fallback : preferred.trim(); + } + + private String normalize(String val) { + return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " "); + } + + private String buildParsedPayloadJson(String aiPrompt, List sourceFiles, List headers, List groups, List allRows) { + SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto(); + payload.setAiPrompt(normalize(aiPrompt)); + payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles); + payload.setHeaders(headers == null ? List.of() : headers); + payload.setItems(List.of()); + payload.setGroups(groups == null ? List.of() : groups); + payload.setAllItems(List.of()); + return writeJson(payload, "保存解析结果失败"); + } + + private String buildTaskResultJson(String aiPrompt, List sourceFiles, String parsedPayloadPointer) { + Map payload = new LinkedHashMap<>(); + payload.put("aiPrompt", normalize(aiPrompt)); + payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream() + .map(SimilarAsinSourceFileDto::getFileKey) + .filter(Objects::nonNull) + .toList()); + payload.put("parsedPayloadRef", parsedPayloadPointer); + return writeJson(payload, "保存任务结果索引失败"); + } + + private String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) { + return transientPayloadStorageService.storeParsedPayloadFast(MODULE_TYPE, taskId, scopeHash, parsedPayloadJson, false); + } + + private long elapsedMs(long start, long end) { + return (end - start) / 1_000_000L; + } + + private SimilarAsinParsedPayloadDto readParsedPayload(FileTaskEntity task) { + try { + JsonNode root = objectMapper.readTree(task.getResultJson()); + String pointer = root.path("parsedPayloadRef").asText(""); + if (!pointer.isBlank()) { + String json = transientPayloadStorageService.resolvePayload(pointer, "read similar ASIN parsed payload failed"); + return hydrateParsedPayloadRows(objectMapper.readValue(json, SimilarAsinParsedPayloadDto.class)); + } + if (root.path("allItems").isArray()) { + return hydrateParsedPayloadRows(objectMapper.treeToValue(root, SimilarAsinParsedPayloadDto.class)); + } + } catch (Exception ex) { + throw new BusinessException("读取解析载荷失败"); + } + return hydrateParsedPayloadRows(new SimilarAsinParsedPayloadDto()); + } + + private SimilarAsinParsedPayloadDto hydrateParsedPayloadRows(SimilarAsinParsedPayloadDto payload) { + if (payload == null) { + return new SimilarAsinParsedPayloadDto(); + } + List rows = payload.getAllItems(); + if (rows == null || rows.isEmpty()) { + rows = payload.getItems(); + } + if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) { + rows = payload.getGroups().stream() + .filter(Objects::nonNull) + .flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream()) + .toList(); + } + if (rows == null) { + rows = List.of(); + } + if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) { + payload.setAllItems(rows); + } + if (payload.getItems() == null || payload.getItems().isEmpty()) { + payload.setItems(rows); + } + return payload; + } + + private Map readChunkRows(TaskChunkEntity chunk) { + Map rows = new LinkedHashMap<>(); + if (chunk == null || chunk.getPayloadJson() == null || chunk.getPayloadJson().isBlank()) { + return rows; + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read similar ASIN chunk failed"); + JsonNode array = objectMapper.readTree(payloadJson); + if (!array.isArray()) { + return rows; + } + for (JsonNode node : array) { + SimilarAsinResultRowDto row = objectMapper.treeToValue(node, SimilarAsinResultRowDto.class); + rows.put(rowKey(row), row); + } + } catch (Exception ex) { + log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} err={}", + chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage()); + } + return rows; + } + + private String rowKey(SimilarAsinParsedRowVo row) { + if (row == null) { + return ""; + } + String rowToken = normalize(row.getRowToken()); + if (!rowToken.isBlank()) { + return rowToken; + } + return legacyRowKey(row); + } + + private String rowKey(SimilarAsinResultRowDto row) { + if (row == null) { + return ""; + } + String rowToken = normalize(row.getRowToken()); + if (!rowToken.isBlank()) { + return rowToken; + } + return legacyRowKey(row); + } + + private SimilarAsinResultRowDto findResultRow(SimilarAsinParsedRowVo parsedRow, + Map resultMap) { + if (parsedRow == null || resultMap == null || resultMap.isEmpty()) { + return null; + } + SimilarAsinResultRowDto resultRow = resultMap.get(rowKey(parsedRow)); + if (resultRow != null) { + return resultRow; + } + String legacyKey = legacyRowKey(parsedRow); + if (legacyKey.isBlank()) { + return null; + } + return resultMap.get(legacyKey); + } + + private String legacyRowKey(SimilarAsinParsedRowVo row) { + if (row == null) { + return ""; + } + return rowKey(row.getDisplayId(), row.getAsin(), row.getCountry()); + } + + private String legacyRowKey(SimilarAsinResultRowDto row) { + if (row == null) { + return ""; + } + return rowKey(row.getId(), row.getAsin(), row.getCountry()); + } + + private String rowKey(String id, String asin, String country) { + return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country); + } + + private String readValueByHeader(SimilarAsinParsedRowVo row, String... candidates) { + if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) { + return ""; + } + for (Map.Entry entry : row.getValues().entrySet()) { + String header = normalize(entry.getKey()).toLowerCase(Locale.ROOT); + for (String candidate : candidates) { + String expected = normalize(candidate).toLowerCase(Locale.ROOT); + if (!expected.isBlank() && header.contains(expected)) { + return entry.getValue() == null ? "" : entry.getValue(); + } + } + } + return ""; + } + + private String userFacingCozeCellValue(SimilarAsinResultRowDto row, String value) { + String normalizedValue = normalize(value); + if (!normalizedValue.isBlank() && !isTechnicalCozeFailure(normalizedValue)) { + return value; + } + if (row != null && isTechnicalCozeFailure(row.getError())) { + return "待人工复核"; + } + return firstNonBlank(value, ""); + } + + private String userFacingConclusion(SimilarAsinResultRowDto row) { + if (row == null) { + return ""; + } + String conclusion = normalize(row.getConclusion()); + if (!conclusion.isBlank() && !isTechnicalCozeFailure(conclusion)) { + return row.getConclusion(); + } + if (isTechnicalCozeFailure(row.getError())) { + return "待人工复核"; + } + return firstNonBlank(row.getConclusion(), ""); + } + + private boolean isTechnicalCozeFailure(String value) { + String normalized = normalize(value).toLowerCase(Locale.ROOT); + return normalized.contains("coze") + || normalized.contains("结果不完整") + || normalized.contains("工作流节点执行超限") + || normalized.contains("调用超时") + || normalized.contains("timeout"); + } + + private String safeFileStem(String filename) { + String name = filename == null || filename.isBlank() ? "similar-asin" : filename; + int idx = name.lastIndexOf('.'); + if (idx > 0) { + name = name.substring(0, idx); + } + String safe = name.replaceAll("[\\\\/:*?\"<>|]", "_").trim(); + return safe.isBlank() ? "similar-asin" : safe; + } + + private String writeJson(Object value, String message) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new BusinessException(message); + } + } + + private void deleteTransientTaskPayloads(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + List scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + if (scopes != null) { + for (TaskScopeStateEntity scope : scopes) { + transientPayloadStorageService.deletePayloadIfPresent(scope.getParsedPayloadJson()); + transientPayloadStorageService.deletePayloadIfPresent(scope.getStateJson()); + } + } + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .select(TaskChunkEntity::getPayloadJson) + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + if (chunks != null) { + for (TaskChunkEntity chunk : chunks) { + transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); + } + } + } + + private record SubmitContext(FileTaskEntity task, + String scopeKey, + String scopeHash, + boolean forceFlush, + String error) { + } + + private record CozeBatchContext(Long jobId, + Long resultId, + String chunkScopeHash, + Integer chunkIndex, + Integer batchIndex, + Integer batchTotal) { + } + + private record ParsedWorkbook(int totalRows, int droppedRows, List headers, List allRows) { + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java index bb0fc3d..2f4be25 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java @@ -14,9 +14,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; -import org.springframework.core.io.Resource; import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -27,6 +25,8 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.net.URI; + @RestController @RequiredArgsConstructor @RequestMapping("/api/split") @@ -93,14 +93,14 @@ public class SplitRunController { @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "返回结果文件流"), @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "结果文件不存在") }) - public ResponseEntity download( + public ResponseEntity download( @Parameter(description = "数据拆分结果记录 ID", required = true) @PathVariable Long resultId, @Parameter(name = "user_id", description = "当前登录用户 ID", required = true, in = ParameterIn.QUERY) @RequestParam("user_id") Long userId) { - Resource resource = splitRunService.getResultFile(resultId, userId); - return ResponseEntity.ok() - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment") - .body(resource); + String downloadUrl = splitRunService.getResultDownloadUrl(resultId, userId); + return ResponseEntity.status(302) + .location(URI.create(downloadUrl)) + .header(HttpHeaders.CACHE_CONTROL, "no-store") + .build(); } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java index 91630f6..aa1cde9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java @@ -19,8 +19,6 @@ import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import lombok.RequiredArgsConstructor; import org.apache.poi.xssf.streaming.SXSSFWorkbook; -import org.springframework.core.io.FileSystemResource; -import org.springframework.core.io.Resource; import org.springframework.stereotype.Service; import java.io.File; @@ -173,7 +171,7 @@ public class SplitRunService { vo.setResultId(entity.getId()); vo.setSourceFilename(entity.getSourceFilename()); vo.setOutputFilename(entity.getResultFilename()); - vo.setDownloadUrl(null); + vo.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())); vo.setRowCount(entity.getRowCount()); vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); vo.setError(entity.getErrorMessage()); @@ -192,17 +190,12 @@ public class SplitRunService { fileResultMapper.deleteById(resultId); } - public Resource getResultFile(Long resultId, Long userId) { + public String getResultDownloadUrl(Long resultId, Long userId) { FileResultEntity entity = fileResultMapper.selectById(resultId); if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { throw new BusinessException("Split result file does not exist."); } - - File file = new File(entity.getResultFileUrl()); - if (!file.exists()) { - throw new BusinessException("Split result file does not exist."); - } - return new FileSystemResource(file); + return ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()); } private List splitExcelByLegacyRules( diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskScopeStateEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskScopeStateEntity.java index 5d31c85..2fdeb00 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskScopeStateEntity.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskScopeStateEntity.java @@ -19,6 +19,13 @@ public class TaskScopeStateEntity { private String scopeHash; private String parsedPayloadJson; private String stateJson; + private String cozeExecuteId; + private String cozeStatus; + private LocalDateTime cozeSubmittedAt; + private LocalDateTime cozeLastPolledAt; + private LocalDateTime cozeCompletedAt; + private Integer cozeAttemptCount; + private String cozeError; private Integer chunkTotal; private Integer receivedChunkCount; private Integer completed; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobLocalDispatcher.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobLocalDispatcher.java index 9cfea22..e6b38b4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobLocalDispatcher.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobLocalDispatcher.java @@ -4,6 +4,7 @@ import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper; import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -18,8 +19,9 @@ public class TaskFileJobLocalDispatcher { private final TaskFileJobMapper taskFileJobMapper; private final ObjectProvider taskResultFileJobWorkerProvider; + @Autowired @Qualifier("taskFileJobDispatchExecutor") - private final TaskExecutor taskFileJobDispatchExecutor; + private TaskExecutor taskFileJobDispatchExecutor; @Value("${aiimage.result-file-job.local-dispatch-enabled:true}") private boolean localDispatchEnabled; @@ -28,26 +30,32 @@ public class TaskFileJobLocalDispatcher { if (!localDispatchEnabled || jobId == null || jobId <= 0) { return false; } - taskFileJobDispatchExecutor.execute(() -> { - try { - TaskFileJobEntity job = taskFileJobMapper.selectById(jobId); - if (job == null) { - log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}", - jobId, taskId, moduleType); - return; + try { + taskFileJobDispatchExecutor.execute(() -> { + try { + TaskFileJobEntity job = taskFileJobMapper.selectById(jobId); + if (job == null) { + log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}", + jobId, taskId, moduleType); + return; + } + TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable(); + if (taskResultFileJobWorker == null) { + log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}", + jobId, taskId, moduleType); + return; + } + taskResultFileJobWorker.process(job); + } catch (Exception ex) { + log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}", + jobId, taskId, moduleType, ex.getMessage(), ex); } - TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable(); - if (taskResultFileJobWorker == null) { - log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}", - jobId, taskId, moduleType); - return; - } - taskResultFileJobWorker.process(job); - } catch (Exception ex) { - log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}", - jobId, taskId, moduleType, ex.getMessage(), ex); - } - }); - return true; + }); + return true; + } catch (RuntimeException ex) { + log.warn("[task-file-job] local dispatch executor rejected jobId={} taskId={} moduleType={} msg={}", + jobId, taskId, moduleType, ex.getMessage(), ex); + return false; + } } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java index 151164c..0b210e3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java @@ -53,7 +53,7 @@ public class TaskFileJobService { if (existing == null) { return null; } - if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus())) { + if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus()) || "PENDING".equals(existing.getStatus())) { return existing; } taskFileJobMapper.update(null, new LambdaUpdateWrapper() @@ -192,6 +192,11 @@ public class TaskFileJobService { return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT); } + public boolean hasSuccessfulAssembleJob(Long taskId, String moduleType, Long resultId) { + TaskFileJobEntity job = findAssembleJob(taskId, moduleType, resultId); + return job != null && "SUCCESS".equals(job.getStatus()); + } + public Map findAssembleJobsByResultIds(String moduleType, List resultIds) { List normalizedIds = resultIds == null ? List.of() : resultIds.stream() .filter(id -> id != null && id > 0) @@ -225,6 +230,18 @@ public class TaskFileJobService { return count == null ? 0L : count; } + public long countActiveAssembleJobs(Long taskId, String moduleType) { + if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) { + return 0L; + } + Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskFileJobEntity::getTaskId, taskId) + .eq(TaskFileJobEntity::getModuleType, moduleType) + .eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT) + .in(TaskFileJobEntity::getStatus, List.of("PENDING", "RUNNING"))); + return count == null ? 0L : count; + } + @Transactional public void deleteTaskJobs(Long taskId, String moduleType) { if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java index 2a83f7f..1ebd668 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java @@ -8,12 +8,16 @@ import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService; import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService; import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService; import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService; +import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.task.TaskExecutor; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @@ -34,8 +38,12 @@ public class TaskResultFileJobWorker { private final QueryAsinTaskService queryAsinTaskService; private final PatrolDeleteTaskService patrolDeleteTaskService; private final AppearancePatentTaskService appearancePatentTaskService; + private final SimilarAsinTaskService similarAsinTaskService; private final DeleteBrandRunService deleteBrandRunService; private final BrandTaskService brandTaskService; + @Autowired + @Qualifier("cozeTaskExecutor") + private TaskExecutor cozeTaskExecutor; @Value("${aiimage.result-file-job.local-worker-enabled:true}") private boolean localWorkerEnabled; @@ -80,9 +88,29 @@ public class TaskResultFileJobWorker { if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) { return; } + if ("APPEARANCE_PATENT".equals(job.getModuleType()) || "SIMILAR_ASIN".equals(job.getModuleType())) { + try { + cozeTaskExecutor.execute(() -> processInternal(job)); + return; + } catch (RuntimeException ex) { + log.warn("[task-file-job] coze module offload failed, fallback inline jobId={} taskId={} moduleType={} msg={}", + job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex); + } + } + processInternal(job); + } + + private void processInternal(TaskFileJobEntity job) { long startedAt = System.currentTimeMillis(); try { - dispatch(job); + boolean completed = dispatch(job); + if (!completed) { + taskFileJobService.touchRunning(job.getId()); + log.info("[task-file-job] process deferred jobId={} taskId={} moduleType={} resultId={} elapsedMs={}", + job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), + System.currentTimeMillis() - startedAt); + return; + } String resultFileUrl = resolveResultFileUrl(job); taskFileJobService.markSuccess(job, resultFileUrl); cleanupAfterSuccess(job); @@ -108,39 +136,41 @@ public class TaskResultFileJobWorker { return result == null ? null : result.getResultFileUrl(); } - private void dispatch(TaskFileJobEntity job) { + private boolean dispatch(TaskFileJobEntity job) { String moduleType = job.getModuleType(); if ("SHOP_MATCH".equals(moduleType)) { shopMatchTaskService.processResultFileJob(job); - return; + return true; } if ("PRICE_TRACK".equals(moduleType)) { priceTrackTaskService.processResultFileJob(job); - return; + return true; } if ("PRODUCT_RISK_RESOLVE".equals(moduleType)) { productRiskTaskService.processResultFileJob(job); - return; + return true; } if ("QUERY_ASIN".equals(moduleType)) { queryAsinTaskService.processResultFileJob(job); - return; + return true; } if ("PATROL_DELETE".equals(moduleType)) { patrolDeleteTaskService.processResultFileJob(job); - return; + return true; } if ("APPEARANCE_PATENT".equals(moduleType)) { - appearancePatentTaskService.processResultFileJob(job); - return; + return appearancePatentTaskService.processResultFileJob(job); + } + if ("SIMILAR_ASIN".equals(moduleType)) { + return similarAsinTaskService.processResultFileJob(job); } if ("DELETE_BRAND".equals(moduleType)) { deleteBrandRunService.processResultFileJob(job); - return; + return true; } if ("BRAND".equals(moduleType)) { brandTaskService.processResultFileJob(job); - return; + return true; } throw new IllegalArgumentException("unsupported result file job module: " + moduleType); } @@ -159,6 +189,10 @@ public class TaskResultFileJobWorker { appearancePatentTaskService.cleanupResultFileJob(job); return; } + if ("SIMILAR_ASIN".equals(moduleType)) { + similarAsinTaskService.cleanupResultFileJob(job); + return; + } if ("DELETE_BRAND".equals(moduleType)) { deleteBrandRunService.cleanupResultFileJob(job); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java index afb3d32..297869b 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java @@ -1,29 +1,39 @@ package com.nanri.aiimage.modules.task.service; import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.config.StorageProperties; import com.nanri.aiimage.config.TransientStorageProperties; import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Locale; import java.util.UUID; @Service @RequiredArgsConstructor +@Slf4j public class TransientPayloadStorageService { + private static final String LOCAL_POINTER_PREFIX = "local:"; private static final String RUSTFS_POINTER_PREFIX = "rustfs:"; private static final String OSS_POINTER_PREFIX = "oss:"; + private static final String LOCAL_PAYLOAD_DIR = "transient-payload"; private final TransientStorageProperties properties; + private final StorageProperties storageProperties; private final RustfsObjectStorageService rustfsObjectStorageService; private final OssStorageService ossStorageService; private final ObjectMapper objectMapper; public boolean isWriteEnabled() { - return properties.isEnabled() && rustfsObjectStorageService.isConfigured(); + return properties.isEnabled(); } public String storeScopePayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) { @@ -68,6 +78,9 @@ public class TransientPayloadStorageService { return value; } try { + if (pointer.startsWith(LOCAL_POINTER_PREFIX)) { + return readLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length())); + } if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) { return rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length())); } @@ -85,6 +98,10 @@ public class TransientPayloadStorageService { if (pointer == null) { return; } + if (pointer.startsWith(LOCAL_POINTER_PREFIX)) { + deleteLocalPayload(pointer.substring(LOCAL_POINTER_PREFIX.length())); + return; + } if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) { rustfsObjectStorageService.deleteObject(pointer.substring(RUSTFS_POINTER_PREFIX.length())); return; @@ -107,19 +124,28 @@ public class TransientPayloadStorageService { if (value == null || value.isBlank()) { return null; } - if (isPointer(value)) { - return value; - } - try { - String decoded = objectMapper.readValue(value, String.class); - return isPointer(decoded) ? decoded : null; - } catch (Exception ignored) { - return null; + String candidate = value.trim(); + for (int i = 0; i < 4; i++) { + if (isPointer(candidate)) { + return candidate; + } + try { + String decoded = objectMapper.readValue(candidate, String.class); + if (decoded == null || decoded.equals(candidate)) { + return null; + } + candidate = decoded.trim(); + } catch (Exception ignored) { + return null; + } } + return null; } private boolean isPointer(String value) { - return value != null && (value.startsWith(RUSTFS_POINTER_PREFIX) || value.startsWith(OSS_POINTER_PREFIX)); + return value != null && (value.startsWith(LOCAL_POINTER_PREFIX) + || value.startsWith(RUSTFS_POINTER_PREFIX) + || value.startsWith(OSS_POINTER_PREFIX)); } private String store(String category, @@ -144,7 +170,19 @@ public class TransientPayloadStorageService { return content; } String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey); - String pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content, verifyAfterUpload); + String pointer = storeLocal(objectKey, content); + if (pointer == null && rustfsObjectStorageService.isConfigured()) { + try { + pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content, verifyAfterUpload); + } catch (Exception ex) { + log.warn("[transient-payload] rustfs upload failed and local fallback unavailable objectKey={} err={}", + objectKey, ex.getMessage()); + throw ex; + } + } + if (pointer == null) { + return content; + } if (!encodeAsJsonString) { return pointer; } @@ -155,6 +193,61 @@ public class TransientPayloadStorageService { } } + private String storeLocal(String objectKey, String content) { + try { + Path root = localPayloadRoot(); + Path target = root.resolve(objectKey).normalize(); + if (!target.startsWith(root)) { + throw new IllegalArgumentException("invalid local payload key: " + objectKey); + } + Files.createDirectories(target.getParent()); + Files.writeString(target, + content == null ? "" : content, + StandardCharsets.UTF_8, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE); + return LOCAL_POINTER_PREFIX + objectKey; + } catch (Exception ex) { + log.warn("[transient-payload] local store failed objectKey={} err={}", objectKey, ex.getMessage()); + return null; + } + } + + private String readLocalPayload(String objectKey) { + try { + Path target = resolveLocalPayloadPath(objectKey); + return Files.readString(target, StandardCharsets.UTF_8); + } catch (Exception ex) { + throw new IllegalStateException("failed to read local transient payload", ex); + } + } + + private void deleteLocalPayload(String objectKey) { + try { + Files.deleteIfExists(resolveLocalPayloadPath(objectKey)); + } catch (Exception ex) { + log.warn("[transient-payload] delete local payload failed objectKey={} err={}", objectKey, ex.getMessage()); + } + } + + private Path resolveLocalPayloadPath(String objectKey) { + Path root = localPayloadRoot(); + Path target = root.resolve(objectKey == null ? "" : objectKey).normalize(); + if (!target.startsWith(root)) { + throw new IllegalArgumentException("invalid local payload key: " + objectKey); + } + return target; + } + + private Path localPayloadRoot() { + String configuredRoot = storageProperties.getLocalTempDir(); + Path tempRoot = configuredRoot == null || configuredRoot.isBlank() + ? Path.of(System.getProperty("java.io.tmpdir")) + : Path.of(configuredRoot); + return tempRoot.resolve(LOCAL_PAYLOAD_DIR).normalize(); + } + private String buildObjectKey(String category, String moduleType, Long taskId, String scopeHash, String entryKey) { String normalizedCategory = normalize(category, "unknown"); String normalizedModuleType = normalize(moduleType, "unknown"); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/model/cache/ZiniaoShopIndexEntryDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/model/cache/ZiniaoShopIndexEntryDto.java index 0ea34cd..15c61f2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/model/cache/ZiniaoShopIndexEntryDto.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/model/cache/ZiniaoShopIndexEntryDto.java @@ -22,4 +22,6 @@ public class ZiniaoShopIndexEntryDto { private List sampleUserIds = new ArrayList<>(); private Long lastSeenAt; private Long lastRefreshedAt; + private Long lastRefreshBlockedAt; + private String refreshBlockedReason; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java index 44959e5..7e15824 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java @@ -52,6 +52,7 @@ public class ZiniaoShopIndexService { private static final Duration DEFAULT_CURSOR_TTL = Duration.ofHours(24); private static final int DEFAULT_REFRESH_BATCH_SIZE = 100; private static final int SHOP_INDEX_LIST_LIMIT = 10000; + private static final String REFRESH_BLOCKED_REASON_IP_WHITELIST = "IP_WHITELIST"; private final ZiniaoMemoryStoreService ziniaoMemoryStoreService; private final ZiniaoTransientCacheService ziniaoTransientCacheService; @@ -339,6 +340,9 @@ public class ZiniaoShopIndexService { cursor.setStatus("FAILED"); cursor.setMessage(ex.getMessage()); cursor.setLastFinishedAt(now); + if (isIpWhitelistRefreshFailure(ex)) { + markExistingEntriesRefreshBlocked(now, REFRESH_BLOCKED_REASON_IP_WHITELIST, ex.getMessage()); + } ziniaoTransientCacheService.put(CACHE_TYPE_SHOP_INDEX_REFRESH_CURSOR, "global", cursor, DEFAULT_CURSOR_TTL); log.warn("[ziniao-index] refresh failed status=FAILED msg={}", ex.getMessage()); if (ex instanceof BusinessException businessException) { @@ -559,6 +563,11 @@ public class ZiniaoShopIndexService { if (entry.getLastRefreshedAt() == null || entry.getLastRefreshedAt() <= 0) { return false; } + if (REFRESH_BLOCKED_REASON_IP_WHITELIST.equals(entry.getRefreshBlockedReason()) + && entry.getLastRefreshBlockedAt() != null + && entry.getLastRefreshBlockedAt() >= entry.getLastRefreshedAt()) { + return true; + } ZiniaoShopIndexRefreshCursorDto cursor = getRefreshCursor(); if (cursor == null || !"FAILED".equals(cursor.getStatus())) { return false; @@ -571,6 +580,49 @@ public class ZiniaoShopIndexService { return lastFinishedAt != null && lastFinishedAt >= entry.getLastRefreshedAt(); } + private boolean isIpWhitelistRefreshFailure(Exception ex) { + if (ex instanceof BusinessException businessException + && ziniaoAuthService.isIpWhitelistError(businessException)) { + return true; + } + String message = ex == null ? null : ex.getMessage(); + return message != null && message.contains("白名单"); + } + + private void markExistingEntriesRefreshBlocked(long now, String reason, String message) { + List entities = ziniaoMemoryStoreService.listAliveEntitiesByType( + ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY, + SHOP_INDEX_LIST_LIMIT + ); + if (entities.isEmpty()) { + return; + } + int updated = 0; + for (ZiniaoMemoryStoreEntity entity : entities) { + String rowKey = entity.getCacheKey(); + ZiniaoShopIndexEntryDto existingEntry; + try { + existingEntry = objectMapper.readValue(entity.getPayloadJson(), ZiniaoShopIndexEntryDto.class); + } catch (Exception ex) { + log.warn("[ziniao-index] skip refresh-block mark corrupt shop_index row cacheKey={}", rowKey); + continue; + } + if (existingEntry == null || !STATUS_ACTIVE.equals(existingEntry.getStatus())) { + continue; + } + existingEntry.setLastRefreshBlockedAt(now); + existingEntry.setRefreshBlockedReason(reason); + if (message != null && !message.isBlank()) { + existingEntry.setMessage(message); + } + ziniaoMemoryStoreService.put(ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY, rowKey, existingEntry, resolveEntryTtl()); + updated++; + } + if (updated > 0) { + log.warn("[ziniao-index] refresh blocked reason={} existing active rows kept usable count={}", reason, updated); + } + } + private Duration resolveEntryTtl() { Integer ttlHours = ziniaoProperties.getShopIndexEntryTtlHours(); if (ttlHours == null || ttlHours <= 0) { diff --git a/backend-java/src/main/resources/application-local.example.yml b/backend-java/src/main/resources/application-local.example.yml index f734238..220a2bf 100644 --- a/backend-java/src/main/resources/application-local.example.yml +++ b/backend-java/src/main/resources/application-local.example.yml @@ -34,6 +34,14 @@ AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE=10 AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS=60000 AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES=20 +AIIMAGE_SIMILAR_ASIN_COZE_BASE_URL=https://api.coze.cn +AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_PATH=/v1/workflow/run +AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_ID=7632683471312355338 +AIIMAGE_SIMILAR_ASIN_COZE_TOKEN= +AIIMAGE_SIMILAR_ASIN_COZE_BATCH_SIZE=10 +AIIMAGE_SIMILAR_ASIN_COZE_READ_TIMEOUT_MILLIS=60000 +AIIMAGE_SIMILAR_ASIN_STALE_TIMEOUT_MINUTES=20 + AIIMAGE_ROCKETMQ_NAME_SERVER=121.196.149.225:9876 AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED=true AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_ENABLED=true diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index 06fe730..b089525 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -94,6 +94,7 @@ aiimage: cleanup-cron: ${AIIMAGE_STORAGE_CLEANUP_CRON:0 0 * * * *} source-retention-hours: ${AIIMAGE_STORAGE_SOURCE_RETENTION_HOURS:24} result-retention-hours: ${AIIMAGE_STORAGE_RESULT_RETENTION_HOURS:24} + transient-payload-retention-hours: ${AIIMAGE_STORAGE_TRANSIENT_PAYLOAD_RETENTION_HOURS:72} temp-dir: retention-hours: ${AIIMAGE_TEMP_DIR_RETENTION_HOURS:24} brand-progress: @@ -103,6 +104,7 @@ aiimage: stale-check-cron: ${AIIMAGE_BRAND_PROGRESS_STALE_CHECK_CRON:0 */2 * * * *} delete-brand-progress: heartbeat-timeout-minutes: ${AIIMAGE_DELETE_BRAND_HEARTBEAT_TIMEOUT_MINUTES:15} + delete-brand-initial-timeout-minutes: ${AIIMAGE_DELETE_BRAND_INITIAL_TIMEOUT_MINUTES:20} stale-check-cron: ${AIIMAGE_DELETE_BRAND_STALE_CHECK_CRON:*/30 * * * * *} finalize-check-cron: ${AIIMAGE_DELETE_BRAND_FINALIZE_CHECK_CRON:30 */2 * * * *} product-risk-stale-timeout-minutes: ${AIIMAGE_PRODUCT_RISK_STALE_TIMEOUT_MINUTES:20} @@ -148,6 +150,16 @@ aiimage: coze-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS:60000} stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:20} stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *} + similar-asin: + coze-base-url: ${AIIMAGE_SIMILAR_ASIN_COZE_BASE_URL:https://api.coze.cn} + coze-workflow-path: ${AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_PATH:/v1/workflow/run} + coze-workflow-id: ${AIIMAGE_SIMILAR_ASIN_COZE_WORKFLOW_ID:7632683471312355338} + coze-token: ${AIIMAGE_SIMILAR_ASIN_COZE_TOKEN:Bearer sat_gsInckdsN0qqhnZSK7K4Zevw6neJCHmYHyIbm85oGgJIGEmSdkH5OFctSKLsAHvT} + coze-batch-size: ${AIIMAGE_SIMILAR_ASIN_COZE_BATCH_SIZE:10} + coze-connect-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_CONNECT_TIMEOUT_MILLIS:10000} + coze-read-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_READ_TIMEOUT_MILLIS:60000} + stale-timeout-minutes: ${AIIMAGE_SIMILAR_ASIN_STALE_TIMEOUT_MINUTES:20} + stale-finalize-cron: ${AIIMAGE_SIMILAR_ASIN_STALE_FINALIZE_CRON:0 */2 * * * *} security: shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key} internal-token: ${AIIMAGE_INTERNAL_TOKEN:} diff --git a/backend-java/src/main/resources/db/V43__appearance_patent_coze_async.sql b/backend-java/src/main/resources/db/V43__appearance_patent_coze_async.sql new file mode 100644 index 0000000..b52435f --- /dev/null +++ b/backend-java/src/main/resources/db/V43__appearance_patent_coze_async.sql @@ -0,0 +1,127 @@ +SET @coze_execute_id_exists := ( + SELECT COUNT(1) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_task_scope_state' + AND COLUMN_NAME = 'coze_execute_id' +); +SET @sql_add_coze_execute_id := IF( + @coze_execute_id_exists = 0, + 'ALTER TABLE biz_task_scope_state ADD COLUMN coze_execute_id VARCHAR(128) NULL COMMENT ''Coze execute id'' AFTER state_json', + 'SELECT 1' +); +PREPARE stmt_add_coze_execute_id FROM @sql_add_coze_execute_id; +EXECUTE stmt_add_coze_execute_id; +DEALLOCATE PREPARE stmt_add_coze_execute_id; + +SET @coze_status_exists := ( + SELECT COUNT(1) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_task_scope_state' + AND COLUMN_NAME = 'coze_status' +); +SET @sql_add_coze_status := IF( + @coze_status_exists = 0, + 'ALTER TABLE biz_task_scope_state ADD COLUMN coze_status VARCHAR(32) NULL COMMENT ''Coze workflow status'' AFTER coze_execute_id', + 'SELECT 1' +); +PREPARE stmt_add_coze_status FROM @sql_add_coze_status; +EXECUTE stmt_add_coze_status; +DEALLOCATE PREPARE stmt_add_coze_status; + +SET @coze_submitted_at_exists := ( + SELECT COUNT(1) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_task_scope_state' + AND COLUMN_NAME = 'coze_submitted_at' +); +SET @sql_add_coze_submitted_at := IF( + @coze_submitted_at_exists = 0, + 'ALTER TABLE biz_task_scope_state ADD COLUMN coze_submitted_at DATETIME NULL COMMENT ''Coze submitted time'' AFTER coze_status', + 'SELECT 1' +); +PREPARE stmt_add_coze_submitted_at FROM @sql_add_coze_submitted_at; +EXECUTE stmt_add_coze_submitted_at; +DEALLOCATE PREPARE stmt_add_coze_submitted_at; + +SET @coze_last_polled_at_exists := ( + SELECT COUNT(1) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_task_scope_state' + AND COLUMN_NAME = 'coze_last_polled_at' +); +SET @sql_add_coze_last_polled_at := IF( + @coze_last_polled_at_exists = 0, + 'ALTER TABLE biz_task_scope_state ADD COLUMN coze_last_polled_at DATETIME NULL COMMENT ''Coze last polled time'' AFTER coze_submitted_at', + 'SELECT 1' +); +PREPARE stmt_add_coze_last_polled_at FROM @sql_add_coze_last_polled_at; +EXECUTE stmt_add_coze_last_polled_at; +DEALLOCATE PREPARE stmt_add_coze_last_polled_at; + +SET @coze_completed_at_exists := ( + SELECT COUNT(1) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_task_scope_state' + AND COLUMN_NAME = 'coze_completed_at' +); +SET @sql_add_coze_completed_at := IF( + @coze_completed_at_exists = 0, + 'ALTER TABLE biz_task_scope_state ADD COLUMN coze_completed_at DATETIME NULL COMMENT ''Coze completed time'' AFTER coze_last_polled_at', + 'SELECT 1' +); +PREPARE stmt_add_coze_completed_at FROM @sql_add_coze_completed_at; +EXECUTE stmt_add_coze_completed_at; +DEALLOCATE PREPARE stmt_add_coze_completed_at; + +SET @coze_attempt_count_exists := ( + SELECT COUNT(1) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_task_scope_state' + AND COLUMN_NAME = 'coze_attempt_count' +); +SET @sql_add_coze_attempt_count := IF( + @coze_attempt_count_exists = 0, + 'ALTER TABLE biz_task_scope_state ADD COLUMN coze_attempt_count INT NOT NULL DEFAULT 0 COMMENT ''Coze poll attempt count'' AFTER coze_completed_at', + 'SELECT 1' +); +PREPARE stmt_add_coze_attempt_count FROM @sql_add_coze_attempt_count; +EXECUTE stmt_add_coze_attempt_count; +DEALLOCATE PREPARE stmt_add_coze_attempt_count; + +SET @coze_error_exists := ( + SELECT COUNT(1) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_task_scope_state' + AND COLUMN_NAME = 'coze_error' +); +SET @sql_add_coze_error := IF( + @coze_error_exists = 0, + 'ALTER TABLE biz_task_scope_state ADD COLUMN coze_error VARCHAR(1000) NULL COMMENT ''Coze error message'' AFTER coze_attempt_count', + 'SELECT 1' +); +PREPARE stmt_add_coze_error FROM @sql_add_coze_error; +EXECUTE stmt_add_coze_error; +DEALLOCATE PREPARE stmt_add_coze_error; + +SET @idx_task_coze_status_exists := ( + SELECT COUNT(1) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_task_scope_state' + AND INDEX_NAME = 'idx_task_coze_status' +); +SET @sql_add_idx_task_coze_status := IF( + @idx_task_coze_status_exists = 0, + 'ALTER TABLE biz_task_scope_state ADD INDEX idx_task_coze_status (module_type, coze_status, updated_at)', + 'SELECT 1' +); +PREPARE stmt_add_idx_task_coze_status FROM @sql_add_idx_task_coze_status; +EXECUTE stmt_add_idx_task_coze_status; +DEALLOCATE PREPARE stmt_add_idx_task_coze_status; diff --git a/backend-java/src/main/resources/db/V44__delete_brand_history_order_indexes.sql b/backend-java/src/main/resources/db/V44__delete_brand_history_order_indexes.sql new file mode 100644 index 0000000..17e99fc --- /dev/null +++ b/backend-java/src/main/resources/db/V44__delete_brand_history_order_indexes.sql @@ -0,0 +1,15 @@ +SET @idx_file_task_delete_brand_history_updated_exists := ( + SELECT COUNT(1) + FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_file_task' + AND INDEX_NAME = 'idx_file_task_delete_brand_history_updated' +); +SET @sql_add_idx_file_task_delete_brand_history_updated := IF( + @idx_file_task_delete_brand_history_updated_exists = 0, + 'ALTER TABLE biz_file_task ADD INDEX idx_file_task_delete_brand_history_updated (module_type, user_id, updated_at)', + 'SELECT 1' +); +PREPARE stmt_add_idx_file_task_delete_brand_history_updated FROM @sql_add_idx_file_task_delete_brand_history_updated; +EXECUTE stmt_add_idx_file_task_delete_brand_history_updated; +DEALLOCATE PREPARE stmt_add_idx_file_task_delete_brand_history_updated; diff --git a/backend-java/src/main/resources/db/V45__similar_asin.sql b/backend-java/src/main/resources/db/V45__similar_asin.sql new file mode 100644 index 0000000..6fbff39 --- /dev/null +++ b/backend-java/src/main/resources/db/V45__similar_asin.sql @@ -0,0 +1,14 @@ +-- Similar ASIN detection reuses the generic file task/result/chunk tables. +-- Register the frontend app menu column so permission queries can expose the page. +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`) +SELECT '相似ASIN检测', 'similar_asin', 'app', 'similar-asin', 112 +WHERE NOT EXISTS ( + SELECT 1 FROM `columns` WHERE `column_key` = 'similar_asin' +); + +UPDATE `columns` +SET `name` = '相似ASIN检测', + `menu_type` = 'app', + `route_path` = 'similar-asin', + `sort_order` = 112 +WHERE `column_key` = 'similar_asin'; diff --git a/frontend-vue/similar-asin.html b/frontend-vue/similar-asin.html new file mode 100644 index 0000000..79028f4 --- /dev/null +++ b/frontend-vue/similar-asin.html @@ -0,0 +1,12 @@ + + + + + + 相似ASIN检测 + + +
+ + + diff --git a/frontend-vue/src/pages/brand/components/BrandDeleteBrandTab.vue b/frontend-vue/src/pages/brand/components/BrandDeleteBrandTab.vue index 564d5ce..988ff70 100644 --- a/frontend-vue/src/pages/brand/components/BrandDeleteBrandTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandDeleteBrandTab.vue @@ -215,6 +215,7 @@ import { expandBrandFolderRecursive, type BrandExpandFolderItem } from '@/shared import { deleteDeleteBrandHistory, getDeleteBrandHistory, + getDeleteBrandResultDownloadUrl, getDeleteBrandTaskDetails, getDeleteBrandTaskProgress, getDeleteBrandTaskDownloadUrl, @@ -269,8 +270,9 @@ const timers = createCategorizedTimers('delete-brand') const displayPaths = computed(() => selectedPaths.value.slice(0, 10)) const currentSectionItems = computed(() => { + const sortedTasks = [...sessionTasks.value].sort((left, right) => (right.createdAt || 0) - (left.createdAt || 0)) const allCurrent: DeleteBrandResultItem[] = []; - sessionTasks.value.forEach(task => { + sortedTasks.forEach(task => { allCurrent.push(...task.items); }); return allCurrent; @@ -521,9 +523,10 @@ function shouldShowProgress(item: DeleteBrandResultItem) { function normalizeDeleteBrandItems(items: DeleteBrandResultItem[]) { return items.map((item) => { - if (item.matched || item.matchStatus === 'MATCHED' || item.matchStatus === 'INDEX_STALE') { + if (isUsableMatchedItem(item)) { return { ...item, + matched: true, _pushed: false, _completed: false, } as SessionDeleteBrandItem @@ -540,10 +543,10 @@ function normalizeDeleteBrandItems(items: DeleteBrandResultItem[]) { } function formatMatchResult(item: DeleteBrandResultItem) { - if (item.matched && item.matchStatus === 'INDEX_STALE') { - return item.matchMessage || `已匹配 ${item.shopId || ''}（索引过期）`.trim() + if (item.matchStatus === 'INDEX_STALE' && isUsableMatchedItem(item)) { + return `已匹配 ${item.shopId || ''}（索引过期，可继续推送）`.trim() } - if (item.matchStatus === 'MATCHED') { + if (item.matchStatus === 'MATCHED' || item.matched) { return `已匹配 ${item.shopId || ''}`.trim() } if (item.matchMessage) { @@ -749,6 +752,12 @@ function getQueueStatus(item: DeleteBrandResultItem) { return '已被全局判定为终态' } + // 后端已经把单文件结果组装出来时，以文件终态为准。 + // 多店铺同 taskId 场景下任务本身仍是 RUNNING，不能因此挡住下一个店铺入队。 + if (isFileResultReady(item) || isFileResultReady(sessionItem)) { + return '本文件解析完毕' + } + // 先看前端会话态：一旦该文件已判定完成，优先展示完成，避免被 RUNNING 覆盖导致链式推进卡住。 if (sessionItem?._completed) { return '本文件解析完毕' @@ -791,6 +800,21 @@ function canDownloadTaskResult(item: DeleteBrandResultItem) { return false } +function isFileResultReady(item?: Partial | null) { + if (!item) return false + const fileStatus = String(item.fileStatus || '').toUpperCase() + return Boolean( + item.fileReady + || item.downloadUrl + || fileStatus === 'SUCCESS' + || fileStatus === 'COMPLETED', + ) +} + +function preferLatestValue(latestValue: T | null | undefined, currentValue: T | null | undefined) { + return latestValue ?? currentValue +} + function mergeTaskDetailItemsIntoSession(detail: DeleteBrandTaskDetailVo) { const taskId = detail?.task?.id if (!taskId) return false @@ -821,10 +845,21 @@ function mergeTaskDetailItemsIntoSession(detail: DeleteBrandTaskDetailVo) { const merged = { ...item, ...latest, + // progress/batch 返回的是轻量文件状态，店铺匹配信息和执行 payload 可能为空； + // 不能覆盖本地解析出的完整店铺数据，否则后续文件会失去入队资格。 + shopName: preferLatestValue(latest.shopName, item.shopName), + companyName: preferLatestValue(latest.companyName, item.companyName), + shopId: preferLatestValue(latest.shopId, item.shopId), + platform: preferLatestValue(latest.platform, item.platform), + openStoreUrl: preferLatestValue(latest.openStoreUrl, item.openStoreUrl), + matchStatus: preferLatestValue(latest.matchStatus, item.matchStatus), + matchMessage: preferLatestValue(latest.matchMessage, item.matchMessage), + matched: isUsableMatchedItem(item) || isUsableMatchedItem(latest), + countryCount: preferLatestValue(latest.countryCount, item.countryCount), countries, previewRows, _pushed: item._pushed, - _completed: item._completed, + _completed: item._completed || isFileResultReady(latest), } as SessionDeleteBrandItem if (JSON.stringify(merged) !== JSON.stringify(item)) { changed = true @@ -837,7 +872,7 @@ function mergeTaskDetailItemsIntoSession(detail: DeleteBrandTaskDetailVo) { mergedItems.push({ ...latest, _pushed: false, - _completed: false, + _completed: isFileResultReady(latest), } as SessionDeleteBrandItem) changed = true } @@ -956,6 +991,8 @@ function ensurePolling(immediate = false) { } function getTaskStatusInfo(item: DeleteBrandResultItem) { + if (item.fileReady || item.downloadUrl) return { className: 'success', text: '已完成' } + // 1. 优先查实时详情中的状态 const status = item.taskId ? taskDetails.value[item.taskId]?.task?.status : null if (status === 'SUCCESS') return { className: 'success', text: '已完成' } @@ -1053,13 +1090,9 @@ async function submitRun() { })), }) const normalizedItems = normalizeDeleteBrandItems(result.items || []) - const hasRunnableItems = normalizedItems.some((item) => - item.matchStatus === 'MATCHED' || item.matchStatus === 'INDEX_STALE' || item.matched - ) + const hasRunnableItems = normalizedItems.some((item) => isUsableMatchedItem(item)) const hasStaleMatchedItems = normalizedItems.some((item) => item.matchStatus === 'INDEX_STALE' && item.matched) - const hasBlockedItems = normalizedItems.some((item) => - !item.matched && item.matchStatus !== 'MATCHED' && item.matchStatus !== 'INDEX_STALE' - ) + const hasBlockedItems = normalizedItems.some((item) => !isUsableMatchedItem(item)) queuePushResult.value = hasRunnableItems ? '解析完成，可在左侧点击“推送到 Python 队列”开始串行处理' @@ -1097,7 +1130,7 @@ async function downloadTaskResult(item: DeleteBrandResultItem) { return } const api = getPywebviewApi() - const url = getDeleteBrandTaskDownloadUrl(taskId) + const url = item.resultId ? getDeleteBrandResultDownloadUrl(item.resultId) : getDeleteBrandTaskDownloadUrl(taskId) const filename = item.outputFilename || item.sourceFilename || @@ -1219,8 +1252,24 @@ function getItemKey(item: DeleteBrandResultItem) { return `task:${item.taskId || 0}:${item.sourceFilename || ''}` } +function isUsableMatchedItem(item: DeleteBrandResultItem) { + return Boolean( + item.matched + || item.matchStatus === 'MATCHED' + || (item.matchStatus === 'INDEX_STALE' && item.shopId), + ) +} + +function hasRunnableDeleteBrandPayload(item: DeleteBrandResultItem) { + return Boolean( + item.shopName + && Array.isArray(item.countries) + && item.countries.length > 0, + ) +} + function canPushToQueue(item: DeleteBrandResultItem) { - return Boolean(item.taskId && (item.matchStatus === 'MATCHED' || item.matchStatus === 'INDEX_STALE' || item.matched)) + return Boolean(item.taskId && (isUsableMatchedItem(item) || hasRunnableDeleteBrandPayload(item))) } function findNextAutoRunnableItem() { diff --git a/frontend-vue/src/pages/brand/components/BrandSimilarAsinTab.vue b/frontend-vue/src/pages/brand/components/BrandSimilarAsinTab.vue new file mode 100644 index 0000000..22bdccf --- /dev/null +++ b/frontend-vue/src/pages/brand/components/BrandSimilarAsinTab.vue @@ -0,0 +1,764 @@ +