diff --git a/backend-java/pom.xml b/backend-java/pom.xml
index 40f05716..0df05606 100644
--- a/backend-java/pom.xml
+++ b/backend-java/pom.xml
@@ -26,6 +26,7 @@
8.5.17
3.28.0-GA
0.12.6
+ 1.78.1
@@ -99,6 +100,11 @@
minio
${minio.version}
+
+ org.bouncycastle
+ bcprov-jdk18on
+ ${bouncycastle.version}
+
io.jsonwebtoken
jjwt-api
diff --git a/backend-java/src/main/java/com/nanri/aiimage/AiImageApplication.java b/backend-java/src/main/java/com/nanri/aiimage/AiImageApplication.java
index e00444b0..dc4a97c5 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/AiImageApplication.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/AiImageApplication.java
@@ -2,13 +2,11 @@ package com.nanri.aiimage;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.scheduling.annotation.EnableScheduling;
import java.time.ZoneId;
import java.util.TimeZone;
@SpringBootApplication
-@EnableScheduling
public class AiImageApplication {
private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai");
diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java
index 6f7e61d3..b6f77eac 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java
@@ -16,7 +16,7 @@ public class AppearancePatentProperties {
private String cozeToken = "";
private List cozeCredentials = new ArrayList<>();
private int cozeCredentialStripeSize = 5;
- private int cozeBatchSize = 50;
+ private int cozeBatchSize = 10;
private int cozeConnectTimeoutMillis = 10000;
private int cozeReadTimeoutMillis = 60000;
private int cozePollIntervalMillis = 30000;
diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java
index 3a40eda5..ebd6e4ce 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/config/ModuleCleanupProperties.java
@@ -12,5 +12,7 @@ public class ModuleCleanupProperties {
private boolean enabled = true;
private String cron = "0 0 0 * * *";
private long retentionDays = 7;
- private List moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "SHOP_DATA_CRAWL", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
+ // SHOP_DATA_CRAWL is governed by per-shop latest-three retention in its
+ // task service and must not be removed by the age-based sweep.
+ private List moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/OpenApiConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/OpenApiConfig.java
index 469b2f58..11be28f3 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/config/OpenApiConfig.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/config/OpenApiConfig.java
@@ -2,9 +2,11 @@ package com.nanri.aiimage.config;
import io.swagger.v3.oas.models.ExternalDocumentation;
import io.swagger.v3.oas.models.OpenAPI;
+import io.swagger.v3.oas.models.Components;
import io.swagger.v3.oas.models.info.Contact;
import io.swagger.v3.oas.models.info.Info;
import io.swagger.v3.oas.models.info.License;
+import io.swagger.v3.oas.models.security.SecurityScheme;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -39,6 +41,11 @@ public class OpenApiConfig {
.version("v0.0.1")
.contact(new Contact().name("Nanri AI"))
.license(new License().name("Internal Use")))
+ .components(new Components().addSecuritySchemes("bearerAuth", new SecurityScheme()
+ .type(SecurityScheme.Type.HTTP)
+ .scheme("bearer")
+ .bearerFormat("JWT")
+ .description("管理员登录 JWT")))
.externalDocs(new ExternalDocumentation()
.description("Knife4j 文档")
.url("/doc.html"));
diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java
index a8af0109..938c6539 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/config/OssProperties.java
@@ -12,6 +12,7 @@ public class OssProperties {
private String bucket;
private String imageVideoBucket;
private String digitalHumanBucket;
+ private String templateBucket;
private String accessKeyId;
private String accessKeySecret;
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/SchedulingConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/SchedulingConfig.java
index daf01163..8ce636e8 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/config/SchedulingConfig.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/config/SchedulingConfig.java
@@ -1,8 +1,10 @@
package com.nanri.aiimage.config;
import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
@@ -10,6 +12,8 @@ import java.time.Clock;
import java.time.ZoneId;
@Configuration
+@EnableScheduling
+@ConditionalOnProperty(prefix = "aiimage.scheduling", name = "enabled", havingValue = "true", matchIfMissing = true)
@Slf4j
public class SchedulingConfig {
diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java
index 2afd9a41..5c0404a9 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/config/SimilarAsinProperties.java
@@ -61,13 +61,8 @@ public class SimilarAsinProperties {
*/
private int cozeSubmitMaxRetryCount = 5;
- /**
- * 图片嵌入下载线程池大小。原 SimilarAsinImageEmbedder.DOWNLOAD_POOL_SIZE = 8。
- * P2-10：1000+ 行 ×3 列图片场景下，pool=16 仍是 assemble 阶段瓶颈（实测下载 244s/918s），
- * 提到 32 配合 retry=2、global deadline 显著拉低尾延迟；
- * 受 2GB 堆约束，单图缩略图维持 300KB 以内，整体内存峰值 ≈ 32 * 300KB ≈ 10MB。
- */
- private int imageDownloadPoolSize = 32;
+ /** 图片下载、解码和缩放共享该池；默认 8，避免批量结果生成占满整机 CPU。 */
+ private int imageDownloadPoolSize = 8;
/**
* 单张图片下载超时（秒）。
@@ -77,6 +72,12 @@ public class SimilarAsinProperties {
*/
private int imageDownloadTimeoutSeconds = 5;
+ /** 单个结果文件整批图片预取预算，耗尽后缺图单元格降级为 URL。 */
+ private int imagePrefetchTimeoutSeconds = 1800;
+
+ /** 多源结果文件组装的单任务硬上限；运行期间由文件任务 heartbeat 保活。 */
+ private int resultFileTimeoutMinutes = 90;
+
/**
* assemble 阶段 taskImageCache 的字节上限。
* 默认 256MB：5000 行 × 3 列 × 平均 100KB = 1.5GB 远超 2GB 堆，
@@ -85,6 +86,7 @@ public class SimilarAsinProperties {
* 出现淘汰过频影响命中率时可上调到 512MB；2GB 堆约束下不建议超过 768MB。
*/
private long imageCacheMaxBytes = 256L * 1024L * 1024L;
+ private String imageLocalCacheDir = "";
private boolean imageDbCacheEnabled = false;
/**
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java
index 9f3afea2..f3386e1c 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java
@@ -1338,10 +1338,9 @@ public class AppearancePatentTaskService {
}
}
}
- boolean failedByConclusion = hasInfringingPersistedConclusion(task.getId());
- boolean failed = finalError != null && !finalError.isBlank() || failedByConclusion;
+ boolean failed = finalError != null && !finalError.isBlank();
boolean waitingForAssemble = !failed && assembleWorkbook && !shouldAssembleSynchronously();
- task.setStatus(waitingForAssemble ? STATUS_RUNNING : (failed ? STATUS_FAILED : STATUS_SUCCESS));
+ task.setStatus(resolveTaskExecutionStatus(waitingForAssemble, failed));
task.setSuccessFileCount(failed ? 0 : 1);
task.setFailedFileCount(failed ? 1 : 0);
task.setErrorMessage(finalError);
@@ -2418,10 +2417,9 @@ public class AppearancePatentTaskService {
saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits - 1, "正在上传结果文件");
fileResultMapper.updateById(result);
boolean taskAlreadyFailed = STATUS_FAILED.equals(task.getStatus())
- || (task.getErrorMessage() != null && !task.getErrorMessage().isBlank())
- || hasInfringingPersistedConclusion(task.getId());
+ || (task.getErrorMessage() != null && !task.getErrorMessage().isBlank());
String existingError = task.getErrorMessage();
- task.setStatus(taskAlreadyFailed ? STATUS_FAILED : STATUS_SUCCESS);
+ task.setStatus(resolveTaskExecutionStatus(false, taskAlreadyFailed));
task.setSuccessFileCount(taskAlreadyFailed ? 0 : 1);
task.setFailedFileCount(taskAlreadyFailed ? 1 : 0);
task.setErrorMessage(taskAlreadyFailed ? existingError : null);
@@ -4199,33 +4197,11 @@ public class AppearancePatentTaskService {
return resultRow == null || resultRow.getPrice() == null ? "" : resultRow.getPrice().trim();
}
- private boolean hasInfringingPersistedConclusion(Long taskId) {
- if (taskId == null) {
- return false;
+ static String resolveTaskExecutionStatus(boolean waitingForAssemble, boolean executionFailed) {
+ if (waitingForAssemble) {
+ return STATUS_RUNNING;
}
- return loadPersistedResultRows(taskId).values().stream()
- .map(AppearancePatentResultRowDto::getConclusion)
- .anyMatch(this::isInfringingConclusion);
- }
-
- private boolean isInfringingConclusion(String value) {
- String normalized = normalize(value);
- return normalized.contains("侵权")
- && !isNoInfringementConclusion(normalized);
- }
-
- private boolean isNoInfringementConclusion(String value) {
- String normalized = normalize(value);
- if (normalized.isBlank()) {
- return false;
- }
- return normalized.contains("无侵权")
- || normalized.contains("未侵权")
- || normalized.contains("不侵权")
- || normalized.contains("未发现") && normalized.contains("侵权")
- || normalized.contains("未见") && normalized.contains("侵权")
- || normalized.contains("不存在") && normalized.contains("侵权")
- || normalized.contains("无明显") && normalized.contains("侵权");
+ return executionFailed ? STATUS_FAILED : STATUS_SUCCESS;
}
private String userFacingCozeCellValue(AppearancePatentResultRowDto row, String value) {
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/auth/util/WerkzeugPasswordEncoder.java b/backend-java/src/main/java/com/nanri/aiimage/modules/auth/util/WerkzeugPasswordEncoder.java
index 0d736b8e..0798e81d 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/auth/util/WerkzeugPasswordEncoder.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/auth/util/WerkzeugPasswordEncoder.java
@@ -1,11 +1,13 @@
package com.nanri.aiimage.modules.auth.util;
import lombok.extern.slf4j.Slf4j;
+import org.bouncycastle.crypto.generators.SCrypt;
import org.springframework.stereotype.Component;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
import java.security.SecureRandom;
import java.util.HexFormat;
@@ -51,21 +53,38 @@ public class WerkzeugPasswordEncoder {
String method = storedHash.substring(0, firstSep);
String salt = storedHash.substring(firstSep + 1, secondSep);
String expectedHex = storedHash.substring(secondSep + 1);
- if (!method.startsWith("pbkdf2:sha256")) {
- log.warn("[auth] unsupported password hash scheme: {}", method);
- return false;
+ if (method.startsWith("scrypt:")) {
+ return matchesScrypt(rawPassword, method, salt, expectedHex);
}
- String[] parts = method.split(":");
- int iterations = parts.length >= 3 ? Integer.parseInt(parts[2]) : 600_000;
- byte[] derived = pbkdf2Sha256(rawPassword.toCharArray(), salt.getBytes(StandardCharsets.UTF_8), iterations, DK_BITS);
- String actualHex = HexFormat.of().formatHex(derived);
- return constantTimeEquals(actualHex, expectedHex);
+ if (method.startsWith("pbkdf2:sha256")) {
+ String[] parts = method.split(":");
+ int iterations = parts.length >= 3 ? Integer.parseInt(parts[2]) : 600_000;
+ byte[] derived = pbkdf2Sha256(rawPassword.toCharArray(), salt.getBytes(StandardCharsets.UTF_8), iterations, DK_BITS);
+ String actualHex = HexFormat.of().formatHex(derived);
+ return constantTimeEquals(actualHex, expectedHex);
+ }
+ log.warn("[auth] unsupported password hash scheme: {}", method);
+ return false;
} catch (Exception e) {
log.warn("[auth] verify password failed: {}", e.getMessage());
return false;
}
}
+ private static boolean matchesScrypt(String rawPassword, String method, String salt, String expectedHex) {
+ String[] parts = method.split(":");
+ if (parts.length != 4 || expectedHex.isEmpty() || (expectedHex.length() & 1) != 0) {
+ return false;
+ }
+ int cost = Integer.parseInt(parts[1]);
+ int blockSize = Integer.parseInt(parts[2]);
+ int parallelization = Integer.parseInt(parts[3]);
+ byte[] expected = HexFormat.of().parseHex(expectedHex);
+ byte[] actual = SCrypt.generate(rawPassword.getBytes(StandardCharsets.UTF_8),
+ salt.getBytes(StandardCharsets.UTF_8), cost, blockSize, parallelization, expected.length);
+ return MessageDigest.isEqual(actual, expected);
+ }
+
private static byte[] pbkdf2Sha256(char[] password, byte[] salt, int iterations, int keyBits) throws Exception {
PBEKeySpec spec = new PBEKeySpec(password, salt, iterations, keyBits);
try {
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/controller/CollectDataController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/controller/CollectDataController.java
index 1bbcbaef..83de3b19 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/controller/CollectDataController.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/controller/CollectDataController.java
@@ -53,6 +53,19 @@ public class CollectDataController {
return ApiResponse.success(null);
}
+ @PostMapping("/tasks/{taskId}/fail")
+ @Operation(summary = "标记采集任务失败", description = "桌面端入队失败或无法继续执行时，将任务收敛为 FAILED。")
+ public ApiResponse fail(
+ @Parameter(description = "采集任务 ID", required = true, example = "9001")
+ @PathVariable Long taskId,
+ @Parameter(description = "当前用户 ID", required = true, example = "1")
+ @RequestParam("user_id") Long userId,
+ @Parameter(description = "失败原因")
+ @RequestParam(value = "error", required = false) String error) {
+ service.failTask(taskId, userId, error);
+ return ApiResponse.success(null);
+ }
+
@GetMapping("/tasks/{taskId}/items")
@Operation(summary = "分页获取任务明细数据", description = "供 Python 端拉取，默认每页 50 条；返回任务关联的筛选条件，便于 Python 端按筛选条件采集。")
public ApiResponse items(
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/dto/CollectDataSummaryRowDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/dto/CollectDataSummaryRowDto.java
index b9b310bd..00aa791e 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/dto/CollectDataSummaryRowDto.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/dto/CollectDataSummaryRowDto.java
@@ -51,4 +51,7 @@ public class CollectDataSummaryRowDto {
@JsonAlias({"page", "totalPage", "页数", "max_page", "maxPage"})
@Schema(description = "该关键词总页数（Python 端取所有命中行的最大页码）", example = "10")
private Integer totalPage;
+
+ @Schema(description = "Python 端 ASIN 过滤数量", example = "3")
+ private Integer asinFilter;
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataHistoryItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataHistoryItemVo.java
index 860ed948..9558d654 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataHistoryItemVo.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataHistoryItemVo.java
@@ -49,6 +49,36 @@ public class CollectDataHistoryItemVo {
@Schema(description = "最终结果行数")
private Integer finalRowCount;
+ @Schema(description = "解析任务总行数")
+ private Integer totalRows;
+
+ @Schema(description = "Python 已回传行数")
+ private Integer receivedRows;
+
+ @Schema(description = "Python 已处理关键词数")
+ private Integer processedRows;
+
+ @Schema(description = "当前采集阶段：search / detail")
+ private String collectStage;
+
+ @Schema(description = "当前关键词")
+ private String currentKeyword;
+
+ @Schema(description = "搜索页当前页码")
+ private Integer searchCurrentPage;
+
+ @Schema(description = "搜索页总页数")
+ private Integer searchTotalPages;
+
+ @Schema(description = "详情页已处理 ASIN 数")
+ private Integer detailProcessedAsins;
+
+ @Schema(description = "详情页 ASIN 总数")
+ private Integer detailTotalAsins;
+
+ @Schema(description = "任务进度百分比，0-100")
+ private Integer progressPercent;
+
@Schema(description = "记录创建时间")
private String createdAt;
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataExcelAssemblyService.java
index 5a42da22..ab4ca4ea 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataExcelAssemblyService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataExcelAssemblyService.java
@@ -24,7 +24,7 @@ public class CollectDataExcelAssemblyService {
private static final String[] SHEET_DETAIL_HEADER = {"品牌", "ASIN", "价格", "卖家名称", "关键词", "配送方式"};
private static final String SHEET_SUMMARY_NAME = "结果文件";
- private static final String[] SHEET_SUMMARY_HEADER = {"关键词", "FBA", "FBM", "AMZ", "无配送方式", "页数"};
+ private static final String[] SHEET_SUMMARY_HEADER = {"关键词", "FBA", "FBM", "AMZ", "无配送方式", "页数", "ASIN过滤"};
/**
* 生成采集结果工作簿。
@@ -115,6 +115,7 @@ public class CollectDataExcelAssemblyService {
} else {
row.createCell(5).setCellValue("");
}
+ row.createCell(6).setCellValue(intOrZero(item.getAsinFilter()));
}
return;
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java
index e7d4a2f7..724a4cc4 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java
@@ -43,6 +43,7 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
+import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
@@ -60,6 +61,8 @@ import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionTemplate;
@@ -105,6 +108,7 @@ public class CollectDataService {
private static final int BRAND_CHECK_BATCH_SIZE = 10;
private static final long TASK_LOCK_WAIT_MILLIS = 5000L;
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
+ private static final String STALE_TASK_ERROR = "长时间未收到 Python 心跳，任务已自动失败";
private static final List KEYWORD_HEADER_ALIASES = List.of("关键词", "keyword", "key word");
private static final List STATUS_HEADER_ALIASES = List.of("状态", "status");
@@ -131,6 +135,9 @@ public class CollectDataService {
private final ObjectMapper objectMapper;
private final TransactionTemplate transactionTemplate;
+ @Value("${aiimage.collect-data.stale-timeout-minutes:30}")
+ private long staleTimeoutMinutes;
+
public CollectDataParseVo parseAndCreateTask(CollectDataParseRequest request) {
long startedAt = System.currentTimeMillis();
if (request == null || request.getUserId() == null || request.getUserId() <= 0) {
@@ -216,7 +223,18 @@ public class CollectDataService {
requestPayload.put("filters", filters);
requestPayload.put("files", sources);
task.setRequestJson(objectMapper.writeValueAsString(requestPayload));
- task.setResultJson("{}");
+ Map initialStats = new LinkedHashMap<>();
+ initialStats.put("totalRows", parsedRows.size());
+ initialStats.put("receivedRows", 0);
+ initialStats.put("processedRows", 0);
+ initialStats.put("currentChunkRows", 0);
+ initialStats.put("dedupeFilteredCount", 0);
+ initialStats.put("invalidFilteredCount", 0);
+ initialStats.put("brandRejectedCount", 0);
+ initialStats.put("brandQueryFailedCount", 0);
+ initialStats.put("finalRowCount", 0);
+ initialStats.put("summaries", List.of());
+ task.setResultJson(objectMapper.writeValueAsString(initialStats));
} catch (Exception ex) {
throw new BusinessException("序列化任务信息失败");
}
@@ -274,9 +292,138 @@ public class CollectDataService {
fileTaskMapper.updateById(task);
}
+ @Transactional
+ public void failTask(Long taskId, Long userId, String error) {
+ FileTaskEntity task = requireTask(taskId, userId);
+ if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
+ return;
+ }
+ String message = firstNonBlank(error, "collect-data task dispatch failed");
+ FileResultEntity result = ensureTaskResult(task);
+ CollectDataStats stats = loadStats(task);
+ result.setSuccess(0);
+ result.setErrorMessage(message);
+ result.setRowCount(stats.finalRowCount);
+ fileResultMapper.updateById(result);
+ task.setStatus(STATUS_FAILED);
+ task.setErrorMessage(message);
+ task.setFailedFileCount(1);
+ task.setUpdatedAt(LocalDateTime.now());
+ task.setFinishedAt(LocalDateTime.now());
+ persistStats(task, stats);
+ fileTaskMapper.updateById(task);
+ }
+
+ @Transactional
+ public void updateProgress(Long taskId, TaskHeartbeatRequest request) {
+ if (taskId == null || taskId <= 0 || request == null) {
+ return;
+ }
+ try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
+ FileTaskEntity task = fileTaskMapper.selectById(taskId);
+ if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
+ return;
+ }
+ CollectDataStats stats = loadStats(task);
+ boolean changed = false;
+ Integer current = request.getCurrent();
+ Integer total = request.getTotal();
+ if (current != null && total != null && total > 0) {
+ int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
+ int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
+ if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
+ stats.totalRows = totalRows;
+ stats.processedRows = processedRows;
+ changed = true;
+ }
+ }
+ if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
+ stats.collectStage = request.getCollectStage();
+ changed = true;
+ }
+ if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
+ stats.currentKeyword = request.getCurrentKeyword();
+ changed = true;
+ }
+ if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
+ stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
+ changed = true;
+ }
+ if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
+ stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
+ changed = true;
+ }
+ if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
+ stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
+ changed = true;
+ }
+ if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
+ stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
+ changed = true;
+ }
+ if (!changed) {
+ return;
+ }
+ persistStats(task, stats);
+ task.setUpdatedAt(LocalDateTime.now());
+ fileTaskMapper.updateById(task);
+ }
+ }
+
+ @Scheduled(cron = "${aiimage.collect-data.stale-check-cron:*/30 * * * * *}")
+ public void finalizeStaleTasks() {
+ long timeoutMinutes = Math.max(1L, staleTimeoutMinutes);
+ LocalDateTime threshold = LocalDateTime.now().minusMinutes(timeoutMinutes);
+ List tasks = fileTaskMapper.selectList(new LambdaQueryWrapper()
+ .eq(FileTaskEntity::getModuleType, MODULE_TYPE)
+ .eq(FileTaskEntity::getStatus, STATUS_RUNNING)
+ .lt(FileTaskEntity::getUpdatedAt, threshold)
+ .orderByAsc(FileTaskEntity::getUpdatedAt)
+ .last("limit 200"));
+ for (FileTaskEntity task : tasks) {
+ finalizeStaleTask(task.getId(), threshold, timeoutMinutes);
+ }
+ }
+
+ private void finalizeStaleTask(Long taskId, LocalDateTime threshold, long timeoutMinutes) {
+ try (TaskDistributedLockService.LockHandle lock =
+ taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) {
+ if (lock == null) {
+ return;
+ }
+ FileTaskEntity task = fileTaskMapper.selectById(taskId);
+ if (task == null
+ || !MODULE_TYPE.equals(task.getModuleType())
+ || !STATUS_RUNNING.equals(task.getStatus())
+ || task.getUpdatedAt() == null
+ || !task.getUpdatedAt().isBefore(threshold)) {
+ return;
+ }
+ if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
+ return;
+ }
+
+ FileResultEntity result = ensureTaskResult(task);
+ CollectDataStats stats = loadStats(task);
+ if (hasReceivedChunks(taskId)) {
+ enqueueFinalWorkbook(task, result, stats);
+ log.warn("[collect-data] stale task enqueued partial workbook taskId={} timeoutMinutes={} finalRows={}",
+ taskId, timeoutMinutes, stats.finalRowCount);
+ return;
+ }
+
+ markTaskFailed(task, result, STALE_TASK_ERROR, stats);
+ log.warn("[collect-data] stale task failed without result chunks taskId={} timeoutMinutes={}",
+ taskId, timeoutMinutes);
+ } catch (Exception ex) {
+ log.warn("[collect-data] stale task finalization failed taskId={} msg={}",
+ taskId, ex.getMessage(), ex);
+ }
+ }
+
public CollectDataDashboardVo dashboard(Long userId) {
CollectDataDashboardVo vo = new CollectDataDashboardVo();
- vo.setPendingTaskCount(countTask(userId, STATUS_RUNNING));
+ vo.setPendingTaskCount(countActiveTasks(userId));
vo.setSuccessTaskCount(countTask(userId, STATUS_SUCCESS));
vo.setFailedTaskCount(countTask(userId, STATUS_FAILED));
long processed = (vo.getSuccessTaskCount() == null ? 0 : vo.getSuccessTaskCount())
@@ -967,6 +1114,13 @@ public class CollectDataService {
return count == null ? 0 : count.intValue();
}
+ private boolean hasReceivedChunks(Long taskId) {
+ Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper()
+ .eq(TaskChunkEntity::getTaskId, taskId)
+ .eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
+ return count != null && count > 0L;
+ }
+
private CollectDataSubmitResultVo buildSubmitVo(FileTaskEntity task,
FileResultEntity result,
int chunkIndex,
@@ -999,7 +1153,15 @@ public class CollectDataService {
}
try {
JsonNode root = objectMapper.readTree(task.getResultJson());
+ stats.totalRows = root.path("totalRows").asInt(0);
stats.receivedRows = root.path("receivedRows").asInt(0);
+ stats.processedRows = root.path("processedRows").asInt(0);
+ stats.collectStage = root.path("collectStage").asText(null);
+ stats.currentKeyword = root.path("currentKeyword").asText(null);
+ stats.searchCurrentPage = root.path("searchCurrentPage").asInt(0);
+ stats.searchTotalPages = root.path("searchTotalPages").asInt(0);
+ stats.detailProcessedAsins = root.path("detailProcessedAsins").asInt(0);
+ stats.detailTotalAsins = root.path("detailTotalAsins").asInt(0);
stats.currentChunkRows = root.path("currentChunkRows").asInt(0);
stats.dedupeFilteredCount = root.path("dedupeFilteredCount").asInt(0);
stats.invalidFilteredCount = root.path("invalidFilteredCount").asInt(0);
@@ -1028,7 +1190,15 @@ public class CollectDataService {
}
try {
Map payload = new LinkedHashMap<>();
+ payload.put("totalRows", stats.totalRows);
payload.put("receivedRows", stats.receivedRows);
+ payload.put("processedRows", stats.processedRows);
+ payload.put("collectStage", stats.collectStage);
+ payload.put("currentKeyword", stats.currentKeyword);
+ payload.put("searchCurrentPage", stats.searchCurrentPage);
+ payload.put("searchTotalPages", stats.searchTotalPages);
+ payload.put("detailProcessedAsins", stats.detailProcessedAsins);
+ payload.put("detailTotalAsins", stats.detailTotalAsins);
payload.put("currentChunkRows", stats.currentChunkRows);
payload.put("dedupeFilteredCount", stats.dedupeFilteredCount);
payload.put("invalidFilteredCount", stats.invalidFilteredCount);
@@ -1142,7 +1312,15 @@ public class CollectDataService {
}
private static class CollectDataStats {
+ private int totalRows;
private int receivedRows;
+ private int processedRows;
+ private String collectStage;
+ private String currentKeyword;
+ private int searchCurrentPage;
+ private int searchTotalPages;
+ private int detailProcessedAsins;
+ private int detailTotalAsins;
private int currentChunkRows;
private int dedupeFilteredCount;
private int invalidFilteredCount;
@@ -1316,6 +1494,17 @@ public class CollectDataService {
return count == null ? 0L : count;
}
+ private long countActiveTasks(Long userId) {
+ if (userId == null || userId <= 0) {
+ return 0L;
+ }
+ Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper()
+ .eq(FileTaskEntity::getModuleType, MODULE_TYPE)
+ .eq(FileTaskEntity::getUserId, userId)
+ .in(FileTaskEntity::getStatus, List.of(STATUS_PENDING, STATUS_RUNNING)));
+ return count == null ? 0L : count;
+ }
+
private Map loadTaskMap(List taskIds) {
Map taskMap = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
@@ -1356,10 +1545,31 @@ public class CollectDataService {
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
vo.setBrandRejectedCount(stats.brandRejectedCount);
vo.setFinalRowCount(stats.finalRowCount);
+ vo.setTotalRows(stats.totalRows);
+ vo.setReceivedRows(stats.receivedRows);
+ vo.setProcessedRows(stats.processedRows);
+ vo.setCollectStage(stats.collectStage);
+ vo.setCurrentKeyword(stats.currentKeyword);
+ vo.setSearchCurrentPage(stats.searchCurrentPage);
+ vo.setSearchTotalPages(stats.searchTotalPages);
+ vo.setDetailProcessedAsins(stats.detailProcessedAsins);
+ vo.setDetailTotalAsins(stats.detailTotalAsins);
+ vo.setProgressPercent(calculateProgressPercent(task.getStatus(), stats.totalRows, stats.processedRows));
}
return vo;
}
+ private int calculateProgressPercent(String status, int totalRows, int receivedRows) {
+ if (STATUS_SUCCESS.equals(status)) {
+ return 100;
+ }
+ if (totalRows <= 0) {
+ return 0;
+ }
+ int received = Math.max(0, Math.min(receivedRows, totalRows));
+ return Math.min(99, received * 100 / totalRows);
+ }
+
private String resolveDisplayResultFilename(FileResultEntity row, FileTaskEntity task) {
if (row == null) {
return null;
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java
index 11eda8ad..e9abc63d 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java
@@ -57,8 +57,13 @@ public class DedupeTotalDataController {
@Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize,
@Parameter(description = "数据值模糊搜索关键字") @RequestParam(required = false) String keyword,
@Parameter(description = "用户名模糊搜索关键字") @RequestParam(required = false) String username,
+ @Parameter(description = "开始日期（包含）")
+ @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
+ @Parameter(description = "结束日期（包含）")
+ @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
@Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) {
- return ApiResponse.success(dedupeTotalDataService.page(page, pageSize, keyword, username, operatorId));
+ return ApiResponse.success(dedupeTotalDataService.page(
+ page, pageSize, keyword, username, startDate, endDate, operatorId));
}
@GetMapping("/export")
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java
index d1b1030f..cf97ef6c 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java
@@ -74,7 +74,11 @@ public class DedupeTotalDataService {
return template;
}
- public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username, Long operatorId) {
+ public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username,
+ LocalDate startDate, LocalDate endDate, Long operatorId) {
+ if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
+ throw new BusinessException("开始日期不能晚于结束日期");
+ }
long safePage = Math.max(page, 1);
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
String safeKeyword = keyword == null ? "" : keyword.trim();
@@ -84,6 +88,10 @@ public class DedupeTotalDataService {
.like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
.in(!scope.allUsers(), DedupeTotalDataEntity::getUploaderUserId, scope.userIds())
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
+ .ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
+ startDate == null ? null : startDate.atStartOfDay())
+ .lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
+ endDate == null ? null : endDate.plusDays(1).atStartOfDay())
.orderByDesc(DedupeTotalDataEntity::getId);
Long total = dedupeTotalDataMapper.selectCount(query);
List items = dedupeTotalDataMapper.selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize))
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java
index a21cbffd..a25c38b5 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java
@@ -25,7 +25,9 @@ public class LocalTempCleanupService {
"convert-result",
"split-result",
"brand-source-download",
- "brand-result"
+ "brand-result",
+ "similar-asin-result",
+ "similar-asin-image-cache"
);
private final StorageProperties storageProperties;
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java
index 04b34dcf..8c87e166 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java
@@ -1,7 +1,9 @@
package com.nanri.aiimage.modules.file.service.oss;
import com.nanri.aiimage.config.OssProperties;
+import io.minio.BucketExistsArgs;
import io.minio.GetObjectArgs;
+import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
import io.minio.PutObjectArgs;
import io.minio.RemoveObjectArgs;
@@ -93,6 +95,77 @@ public class OssStorageService {
}
}
+ public void ensureBucketExists(String bucket) {
+ String normalizedBucket = requireStorageName(bucket, "bucket");
+ try {
+ boolean exists = buildClient().bucketExists(BucketExistsArgs.builder()
+ .bucket(normalizedBucket)
+ .build());
+ if (exists) {
+ return;
+ }
+ try {
+ buildClient().makeBucket(MakeBucketArgs.builder()
+ .bucket(normalizedBucket)
+ .build());
+ } catch (ErrorResponseException ex) {
+ if (!isBucketAlreadyPresent(ex)) {
+ throw ex;
+ }
+ }
+ } catch (Exception ex) {
+ throw storageFailure("ensure bucket", normalizedBucket, ex);
+ }
+ }
+
+ public void uploadBytes(String bucket, String objectKey, byte[] content, String contentType) {
+ String normalizedBucket = requireStorageName(bucket, "bucket");
+ String normalizedObjectKey = requireStorageName(objectKey, "objectKey");
+ byte[] bytes = content == null ? new byte[0] : content;
+ try (ByteArrayInputStream stream = new ByteArrayInputStream(bytes)) {
+ buildClient().putObject(PutObjectArgs.builder()
+ .bucket(normalizedBucket)
+ .object(normalizedObjectKey)
+ .stream(stream, bytes.length, -1)
+ .contentType(firstNonBlank(contentType, "application/octet-stream"))
+ .build());
+ } catch (Exception ex) {
+ throw storageFailure("upload", normalizedBucket + "/" + normalizedObjectKey, ex);
+ }
+ }
+
+ public byte[] readObjectBytes(String bucket, String objectKey) {
+ String normalizedBucket = requireStorageName(bucket, "bucket");
+ String normalizedObjectKey = requireStorageName(objectKey, "objectKey");
+ try (var stream = buildClient().getObject(GetObjectArgs.builder()
+ .bucket(normalizedBucket)
+ .object(normalizedObjectKey)
+ .build())) {
+ return stream.readAllBytes();
+ } catch (Exception ex) {
+ throw storageFailure("read", normalizedBucket + "/" + normalizedObjectKey, ex);
+ }
+ }
+
+ public boolean objectExists(String bucket, String objectKey) {
+ String normalizedBucket = requireStorageName(bucket, "bucket");
+ String normalizedObjectKey = requireStorageName(objectKey, "objectKey");
+ try {
+ buildClient().statObject(StatObjectArgs.builder()
+ .bucket(normalizedBucket)
+ .object(normalizedObjectKey)
+ .build());
+ return true;
+ } catch (ErrorResponseException ex) {
+ if (isNotFound(ex)) {
+ return false;
+ }
+ throw storageFailure("stat", normalizedBucket + "/" + normalizedObjectKey, ex);
+ } catch (Exception ex) {
+ throw storageFailure("stat", normalizedBucket + "/" + normalizedObjectKey, ex);
+ }
+ }
+
public String uploadTaskScopePayload(String moduleType, Long taskId, String scopeHash, String content) {
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
String objectKey = String.format("task-scope/%s/%s/%s.json", normalizeModuleType(moduleType), taskId, normalizedScopeHash);
@@ -310,7 +383,7 @@ public class OssStorageService {
}
private List configuredBuckets() {
- return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket())
+ return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket(), templateBucket())
.filter(Objects::nonNull)
.map(String::trim)
.filter(bucket -> !bucket.isBlank())
@@ -395,6 +468,10 @@ public class OssStorageService {
return firstNonBlank(ossProperties.getDigitalHumanBucket(), ossProperties.getBucket());
}
+ private String templateBucket() {
+ return firstNonBlank(ossProperties.getTemplateBucket(), ossProperties.getBucket());
+ }
+
private String publicEndpoint() {
return withScheme(firstNonBlank(ossProperties.getPublicEndpoint(), ossProperties.getEndpoint()));
}
@@ -446,6 +523,11 @@ public class OssStorageService {
return "NoSuchKey".equals(code) || "NoSuchObject".equals(code) || "NoSuchBucket".equals(code);
}
+ private boolean isBucketAlreadyPresent(ErrorResponseException ex) {
+ String code = ex.errorResponse() == null ? null : ex.errorResponse().code();
+ return "BucketAlreadyOwnedByYou".equals(code) || "BucketAlreadyExists".equals(code);
+ }
+
private IllegalStateException storageFailure(String operation, String objectKey, Exception cause) {
return new IllegalStateException("failed to " + operation + " object in MinIO: " + objectKey, cause);
}
@@ -477,6 +559,13 @@ public class OssStorageService {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
+ private String requireStorageName(String value, String field) {
+ if (value == null || value.isBlank()) {
+ throw new IllegalArgumentException(field + " must not be blank");
+ }
+ return value.trim();
+ }
+
private record StorageLocation(String bucket, String objectKey) {
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/filetemplate/ModuleTemplateController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/filetemplate/ModuleTemplateController.java
new file mode 100644
index 00000000..7daeb98c
--- /dev/null
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/filetemplate/ModuleTemplateController.java
@@ -0,0 +1,50 @@
+package com.nanri.aiimage.modules.filetemplate;
+
+import com.nanri.aiimage.common.util.DownloadHeaderUtil;
+import com.nanri.aiimage.modules.filetemplate.ModuleTemplateService.TemplateDownload;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import lombok.RequiredArgsConstructor;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequiredArgsConstructor
+@RequestMapping("/api/module-templates")
+@Tag(name = "模块输入模板", description = "下载各业务模块的固定 Excel 输入模板")
+public class ModuleTemplateController {
+
+ private final ModuleTemplateService moduleTemplateService;
+
+ @GetMapping("/{moduleCode}/download")
+ @Operation(summary = "下载模块输入模板")
+ @ApiResponses({
+ @io.swagger.v3.oas.annotations.responses.ApiResponse(
+ responseCode = "200",
+ description = "Excel 模板文件",
+ content = @Content(
+ mediaType = ModuleTemplateService.XLSX_CONTENT_TYPE,
+ schema = @Schema(type = "string", format = "binary"))),
+ @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "模块模板不存在"),
+ @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "503", description = "模板存储暂不可用")
+ })
+ public ResponseEntity download(
+ @Parameter(description = "模块编码", required = true, example = "publish")
+ @PathVariable String moduleCode) {
+ TemplateDownload download = moduleTemplateService.download(moduleCode);
+ return ResponseEntity.ok()
+ .header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(download.filename()))
+ .contentType(MediaType.parseMediaType(download.contentType()))
+ .contentLength(download.content().length)
+ .body(download.content());
+ }
+}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/filetemplate/ModuleTemplateRegistry.java b/backend-java/src/main/java/com/nanri/aiimage/modules/filetemplate/ModuleTemplateRegistry.java
new file mode 100644
index 00000000..755ac86c
--- /dev/null
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/filetemplate/ModuleTemplateRegistry.java
@@ -0,0 +1,57 @@
+package com.nanri.aiimage.modules.filetemplate;
+
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+@Component
+public class ModuleTemplateRegistry {
+
+ private static final String RESOURCE_PREFIX = "templates/module-input/";
+ private static final String OBJECT_PREFIX = "input/";
+
+ private static final List TEMPLATES = List.of(
+ template("publish", "publish.xlsx", "上架 文档格式.xlsx"),
+ template("delete-brand", "delete-brand.xlsx", "删除指定待售 文档格式.xlsx"),
+ template("appearance-patent", "appearance-patent.xlsx", "外观专利检测 文档格式.xlsx"),
+ template("price-track", "price-track.xlsx", "指定ASIN跟价 文档格式.xlsx"),
+ template("collect-data", "collect-data.xlsx", "数据采集 文档格式.xlsx"),
+ template("similar-asin", "similar-asin.xlsx", "货源查询 文档格式.xlsx")
+ );
+
+ private final Map templatesByCode = TEMPLATES.stream()
+ .collect(Collectors.toUnmodifiableMap(
+ ModuleTemplate::moduleCode,
+ Function.identity()));
+
+ public Optional find(String moduleCode) {
+ if (moduleCode == null || moduleCode.isBlank()) {
+ return Optional.empty();
+ }
+ return Optional.ofNullable(templatesByCode.get(moduleCode.trim().toLowerCase(Locale.ROOT)));
+ }
+
+ public List templates() {
+ return TEMPLATES;
+ }
+
+ private static ModuleTemplate template(String moduleCode, String resourceFilename, String downloadFilename) {
+ return new ModuleTemplate(
+ moduleCode,
+ RESOURCE_PREFIX + resourceFilename,
+ OBJECT_PREFIX + resourceFilename,
+ downloadFilename);
+ }
+
+ public record ModuleTemplate(
+ String moduleCode,
+ String resourcePath,
+ String objectKey,
+ String downloadFilename) {
+ }
+}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/filetemplate/ModuleTemplateService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/filetemplate/ModuleTemplateService.java
new file mode 100644
index 00000000..2a96abbb
--- /dev/null
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/filetemplate/ModuleTemplateService.java
@@ -0,0 +1,89 @@
+package com.nanri.aiimage.modules.filetemplate;
+
+import com.nanri.aiimage.config.OssProperties;
+import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
+import com.nanri.aiimage.modules.filetemplate.ModuleTemplateRegistry.ModuleTemplate;
+import lombok.RequiredArgsConstructor;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.http.HttpStatus;
+import org.springframework.stereotype.Service;
+import org.springframework.web.server.ResponseStatusException;
+
+import java.io.InputStream;
+
+@Service
+@RequiredArgsConstructor
+public class ModuleTemplateService {
+
+ public static final String XLSX_CONTENT_TYPE =
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
+
+ private final ModuleTemplateRegistry registry;
+ private final OssProperties ossProperties;
+ private final OssStorageService ossStorageService;
+
+ public TemplateDownload download(String moduleCode) {
+ ModuleTemplate template = registry.find(moduleCode)
+ .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "模板不存在"));
+ try {
+ ensureStored(template);
+ byte[] content = ossStorageService.readObjectBytes(templateBucket(), template.objectKey());
+ return new TemplateDownload(template.downloadFilename(), XLSX_CONTENT_TYPE, content);
+ } catch (ResponseStatusException ex) {
+ throw ex;
+ } catch (Exception ex) {
+ throw storageUnavailable(ex);
+ }
+ }
+
+ public int synchronizeAll() {
+ try {
+ String bucket = templateBucket();
+ ossStorageService.ensureBucketExists(bucket);
+ int uploaded = 0;
+ for (ModuleTemplate template : registry.templates()) {
+ if (!ossStorageService.objectExists(bucket, template.objectKey())) {
+ uploadResource(bucket, template);
+ uploaded++;
+ }
+ }
+ return uploaded;
+ } catch (ResponseStatusException ex) {
+ throw ex;
+ } catch (Exception ex) {
+ throw storageUnavailable(ex);
+ }
+ }
+
+ private void ensureStored(ModuleTemplate template) {
+ String bucket = templateBucket();
+ ossStorageService.ensureBucketExists(bucket);
+ if (!ossStorageService.objectExists(bucket, template.objectKey())) {
+ uploadResource(bucket, template);
+ }
+ }
+
+ private void uploadResource(String bucket, ModuleTemplate template) {
+ ClassPathResource resource = new ClassPathResource(template.resourcePath());
+ try (InputStream input = resource.getInputStream()) {
+ ossStorageService.uploadBytes(bucket, template.objectKey(), input.readAllBytes(), XLSX_CONTENT_TYPE);
+ } catch (Exception ex) {
+ throw new IllegalStateException("failed to load module template resource: " + template.resourcePath(), ex);
+ }
+ }
+
+ private String templateBucket() {
+ String bucket = ossProperties.getTemplateBucket();
+ if (bucket == null || bucket.isBlank()) {
+ throw new IllegalStateException("template bucket is not configured");
+ }
+ return bucket.trim();
+ }
+
+ private ResponseStatusException storageUnavailable(Exception cause) {
+ return new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "模板存储暂不可用", cause);
+ }
+
+ public record TemplateDownload(String filename, String contentType, byte[] content) {
+ }
+}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/filetemplate/ModuleTemplateStorageInitializer.java b/backend-java/src/main/java/com/nanri/aiimage/modules/filetemplate/ModuleTemplateStorageInitializer.java
new file mode 100644
index 00000000..e074e31e
--- /dev/null
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/filetemplate/ModuleTemplateStorageInitializer.java
@@ -0,0 +1,26 @@
+package com.nanri.aiimage.modules.filetemplate;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Component;
+
+@Component
+@RequiredArgsConstructor
+@Slf4j
+public class ModuleTemplateStorageInitializer {
+
+ private final ModuleTemplateService moduleTemplateService;
+
+ @EventListener(ApplicationReadyEvent.class)
+ public void synchronize() {
+ try {
+ int uploaded = moduleTemplateService.synchronizeAll();
+ log.info("[module-template] startup synchronization complete uploaded={}", uploaded);
+ } catch (Exception ex) {
+ log.warn("[module-template] startup synchronization failed; downloads will retry lazily: {}",
+ ex.getMessage(), ex);
+ }
+ }
+}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuController.java
index e26cc7dc..0cc0a3f1 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuController.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuController.java
@@ -128,6 +128,23 @@ public class PermissionMenuController {
return ApiResponse.success("视频任务权限已更新", grantedCount);
}
+ @GetMapping("/shop-data-crawl-task-permissions")
+ @Operation(summary = "查询店铺数据任务数据权限用户")
+ public ApiResponse> listShopDataCrawlDataPermissionUsers(
+ HttpServletRequest request) {
+ return ApiResponse.success(permissionMenuService.listShopDataCrawlDataPermissionUsers(requireAdmin(request)));
+ }
+
+ @PutMapping("/shop-data-crawl-task-permissions")
+ @Operation(summary = "更新店铺数据任务数据权限用户")
+ public ApiResponse updateShopDataCrawlDataPermissionUsers(
+ HttpServletRequest request,
+ @RequestBody(required = false) ImageVideoDataPermissionUpdateRequest body) {
+ int grantedCount = permissionMenuService.updateShopDataCrawlDataPermissionUsers(
+ requireAdmin(request), body == null ? List.of() : body.getUserIds());
+ return ApiResponse.success("店铺数据任务权限已更新", grantedCount);
+ }
+
private AdminUserEntity requireAdmin(HttpServletRequest request) {
try {
return adminAuthSupport.requireAdmin(request);
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java
index 3d6ecda3..fee0ddaf 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java
@@ -86,6 +86,7 @@ public class PermissionMenuSchemaInitializer {
executeQuietly("UPDATE columns SET sort_order = id WHERE sort_order IS NULL OR sort_order = 0");
executeQuietly("ALTER TABLE columns ADD UNIQUE KEY uk_menu_type_route_path (menu_type, route_path)");
ensureDefaultAdminMenus();
+ boolean shopDataCrawlDataPermissionCreated = ensureShopDataCrawlAdminDefaults();
ensureDefaultAppMenus();
ensureDefaultAppChildMenus();
ensureInternalDataPermissions();
@@ -98,6 +99,9 @@ public class PermissionMenuSchemaInitializer {
FOREIGN KEY (column_id) REFERENCES columns(id) ON DELETE CASCADE
)
""");
+ if (shopDataCrawlDataPermissionCreated) {
+ grantInitialShopDataCrawlDataPermissions();
+ }
}
private void ensureDefaultAdminMenus() {
@@ -172,6 +176,38 @@ public class PermissionMenuSchemaInitializer {
""");
}
+ private boolean ensureShopDataCrawlAdminDefaults() {
+ executeQuietly("""
+ INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
+ SELECT '店铺数据任务管理', 'admin_shop_data_crawl_tasks', 'admin', 'shop-data-crawl-tasks', 82
+ WHERE NOT EXISTS (
+ SELECT 1 FROM columns WHERE column_key = 'admin_shop_data_crawl_tasks'
+ )
+ """);
+ return executeUpdateQuietly("""
+ INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
+ SELECT '店铺数据任务数据查看', 'admin_shop_data_crawl_task_data', 'internal', 'shop-data-crawl-task-data', 0
+ WHERE NOT EXISTS (
+ SELECT 1 FROM columns WHERE column_key = 'admin_shop_data_crawl_task_data'
+ )
+ """) > 0;
+ }
+
+ private void grantInitialShopDataCrawlDataPermissions() {
+ executeQuietly("""
+ INSERT IGNORE INTO user_column_permission (user_id, column_id)
+ SELECT old_perm.user_id, data_col.id
+ FROM user_column_permission old_perm
+ INNER JOIN columns old_col ON old_col.id = old_perm.column_id
+ INNER JOIN columns data_col ON data_col.column_key = 'admin_shop_data_crawl_task_data'
+ LEFT JOIN user_column_permission existing
+ ON existing.user_id = old_perm.user_id
+ AND existing.column_id = data_col.id
+ WHERE old_col.column_key = 'admin_shop_data_crawl_tasks'
+ AND existing.user_id IS NULL
+ """);
+ }
+
private void executeQuietly(String sql) {
try {
jdbcTemplate.execute(sql);
@@ -180,6 +216,15 @@ public class PermissionMenuSchemaInitializer {
}
}
+ private int executeUpdateQuietly(String sql) {
+ try {
+ return jdbcTemplate.update(sql);
+ } catch (Exception ex) {
+ log.debug("[permission-menu] schema init skipped: {}", ex.getMessage());
+ return 0;
+ }
+ }
+
private record DefaultAdminMenu(String name, String columnKey, String routePath, int sortOrder) {
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java
index 5c25facd..5767c54f 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java
@@ -47,6 +47,7 @@ public class PermissionMenuService {
public static final String MENU_TYPE_APP = "app";
public static final String MENU_TYPE_ADMIN = "admin";
private static final String IMAGE_VIDEO_DATA_PERMISSION_KEY = "admin_image_video_task_data";
+ private static final String SHOP_DATA_CRAWL_DATA_PERMISSION_KEY = "admin_shop_data_crawl_task_data";
private final PermissionMenuMapper permissionMenuMapper;
private final UserColumnPermissionMapper userColumnPermissionMapper;
@@ -243,9 +244,8 @@ public class PermissionMenuService {
.filter(user -> user.getId() != null && !isSuperAdmin(user))
.collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left));
if (requestedIds.stream().anyMatch(id -> !grantableUsers.containsKey(id))) {
- throw new BusinessException("包含不存在或不可授权的用户");
+ throw new BusinessException("contains unknown or non-grantable user");
}
-
userColumnPermissionMapper.deleteByMap(Map.of("column_id", dataPermission.getId()));
for (Long userId : requestedIds) {
UserColumnPermissionEntity grant = new UserColumnPermissionEntity();
@@ -256,6 +256,68 @@ public class PermissionMenuService {
return requestedIds.size();
}
+ public List listShopDataCrawlDataPermissionUsers(AdminUserEntity operator) {
+ ensureSuperAdminOperator(operator, "店铺数据任务");
+ PermissionMenuEntity dataPermission = requireDataPermission(
+ SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据");
+ Set grantedUserIds = userColumnPermissionMapper.selectList(
+ new LambdaQueryWrapper()
+ .eq(UserColumnPermissionEntity::getColumnId, dataPermission.getId()))
+ .stream()
+ .map(UserColumnPermissionEntity::getUserId)
+ .filter(id -> id != null && id > 0)
+ .collect(Collectors.toSet());
+ return adminUserMapper.selectList(new LambdaQueryWrapper()
+ .orderByAsc(AdminUserEntity::getUsername)
+ .orderByAsc(AdminUserEntity::getId))
+ .stream()
+ .filter(user -> !isSuperAdmin(user))
+ .map(user -> toImageVideoDataPermissionUserVo(user, grantedUserIds.contains(user.getId())))
+ .toList();
+ }
+
+ @Transactional
+ public int updateShopDataCrawlDataPermissionUsers(AdminUserEntity operator, List userIds) {
+ ensureSuperAdminOperator(operator, "店铺数据任务");
+ PermissionMenuEntity dataPermission = requireDataPermission(
+ SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据");
+ List requestedIds = normalizeColumnIds(userIds);
+ List users = adminUserMapper.selectList(new LambdaQueryWrapper());
+ Map grantableUsers = users.stream()
+ .filter(user -> user.getId() != null && !isSuperAdmin(user))
+ .collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left));
+ if (requestedIds.stream().anyMatch(id -> !grantableUsers.containsKey(id))) {
+ throw new BusinessException("contains unknown or non-grantable user");
+ }
+ userColumnPermissionMapper.deleteByMap(Map.of("column_id", dataPermission.getId()));
+ for (Long userId : requestedIds) {
+ UserColumnPermissionEntity grant = new UserColumnPermissionEntity();
+ grant.setUserId(userId);
+ grant.setColumnId(dataPermission.getId());
+ userColumnPermissionMapper.insert(grant);
+ }
+ return requestedIds.size();
+ }
+
+ /** Verifies both the visible admin menu and its separately managed data grant. */
+ public void requireShopDataCrawlTaskAccess(AdminUserEntity operator) {
+ ensureAdminOperatorIfPresent(operator);
+ if (operator == null || operator.getId() == null || operator.getId() <= 0) {
+ throw new BusinessException(403, "需要管理员权限");
+ }
+ if (isSuperAdmin(operator)) {
+ return;
+ }
+ PermissionMenuEntity taskMenu = requireDataPermission(
+ "admin_shop_data_crawl_tasks", "店铺数据任务管理菜单");
+ PermissionMenuEntity dataPermission = requireDataPermission(
+ SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据");
+ if (!hasEffectiveColumnPermission(operator.getId(), taskMenu.getId())
+ || !hasEffectiveColumnPermission(operator.getId(), dataPermission.getId())) {
+ throw new BusinessException(403, "无权查看店铺数据任务");
+ }
+ }
+
@Transactional
public void updateUserColumnPermissions(Long userId, UserColumnPermissionUpdateRequest request) {
updateUserColumnPermissions(null, userId, request);
@@ -304,26 +366,36 @@ public class PermissionMenuService {
}
}
- PermissionMenuEntity imageVideoDataPermission = normalizedType == null
- ? findImageVideoDataPermission()
- : null;
- Long protectedId = imageVideoDataPermission == null ? null : imageVideoDataPermission.getId();
+ Set protectedIds = new LinkedHashSet<>();
+ if (normalizedType == null) {
+ PermissionMenuEntity imageVideoDataPermission = findImageVideoDataPermission();
+ PermissionMenuEntity shopDataCrawlDataPermission =
+ findDataPermission(SHOP_DATA_CRAWL_DATA_PERMISSION_KEY);
+ if (imageVideoDataPermission != null && imageVideoDataPermission.getId() != null) {
+ protectedIds.add(imageVideoDataPermission.getId());
+ }
+ if (shopDataCrawlDataPermission != null && shopDataCrawlDataPermission.getId() != null) {
+ protectedIds.add(shopDataCrawlDataPermission.getId());
+ }
+ }
List grantIds = requestedIds;
- if (normalizedType == null && protectedId != null) {
+ if (!protectedIds.isEmpty()) {
grantIds = requestedIds.stream()
- .filter(id -> !protectedId.equals(id))
+ .filter(id -> !protectedIds.contains(id))
.toList();
}
Set operatorEffectiveIds = ensureGrantable(operator, grantIds);
LinkedHashSet finalGrantIds = new LinkedHashSet<>(grantIds);
- if (normalizedType == null && protectedId != null) {
- Long existingCount = userColumnPermissionMapper.selectCount(
- new LambdaQueryWrapper()
- .eq(UserColumnPermissionEntity::getUserId, userId)
- .eq(UserColumnPermissionEntity::getColumnId, protectedId));
- if (existingCount != null && existingCount > 0) {
- finalGrantIds.add(protectedId);
+ if (!protectedIds.isEmpty()) {
+ for (Long protectedId : protectedIds) {
+ Long existingCount = userColumnPermissionMapper.selectCount(
+ new LambdaQueryWrapper()
+ .eq(UserColumnPermissionEntity::getUserId, userId)
+ .eq(UserColumnPermissionEntity::getColumnId, protectedId));
+ if (existingCount != null && existingCount > 0) {
+ finalGrantIds.add(protectedId);
+ }
}
}
if (operatorEffectiveIds != null) {
@@ -526,9 +598,7 @@ public class PermissionMenuService {
}
private PermissionMenuEntity findImageVideoDataPermission() {
- return permissionMenuMapper.selectOne(new LambdaQueryWrapper()
- .eq(PermissionMenuEntity::getColumnKey, IMAGE_VIDEO_DATA_PERMISSION_KEY)
- .last("LIMIT 1"));
+ return findDataPermission(IMAGE_VIDEO_DATA_PERMISSION_KEY);
}
private PermissionMenuEntity requireImageVideoDataPermission() {
@@ -546,6 +616,27 @@ public class PermissionMenuService {
}
}
+ private void ensureSuperAdminOperator(AdminUserEntity operator, String resourceName) {
+ ensureAdminOperatorIfPresent(operator);
+ if (operator == null || !isSuperAdmin(operator)) {
+ throw new BusinessException(403, "仅超级管理员可以配置" + resourceName + "权限");
+ }
+ }
+
+ private PermissionMenuEntity findDataPermission(String columnKey) {
+ return permissionMenuMapper.selectOne(new LambdaQueryWrapper()
+ .eq(PermissionMenuEntity::getColumnKey, columnKey)
+ .last("LIMIT 1"));
+ }
+
+ private PermissionMenuEntity requireDataPermission(String columnKey, String displayName) {
+ PermissionMenuEntity permission = findDataPermission(columnKey);
+ if (permission == null || permission.getId() == null) {
+ throw new BusinessException(displayName + "权限尚未初始化");
+ }
+ return permission;
+ }
+
private ImageVideoDataPermissionUserVo toImageVideoDataPermissionUserVo(AdminUserEntity user,
boolean granted) {
ImageVideoDataPermissionUserVo vo = new ImageVideoDataPermissionUserVo();
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java
index 72be0481..c995240d 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java
@@ -63,6 +63,7 @@ public class PriceTrackTaskService {
private static final String MODULE_TYPE = "PRICE_TRACK";
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String ASIN_ROWS_PAYLOAD_SCOPE = "price-track-asin-rows";
+ private static final String NO_USABLE_ROWS_ERROR = "未收到有效跟价数据，未生成结果文件";
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
@@ -1907,7 +1908,7 @@ public class PriceTrackTaskService {
if (payload != null && hasText(payload.getError())) {
return payload.getError().trim();
}
- return "no usable price-track rows received";
+ return NO_USABLE_ROWS_ERROR;
}
private String firstNonBlank(String preferred, String fallback) {
if (preferred != null && !preferred.isBlank()) {
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/controller/PublishController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/controller/PublishController.java
index a593d99e..2e0c6219 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/controller/PublishController.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/controller/PublishController.java
@@ -28,7 +28,7 @@ import org.springframework.web.bind.annotation.RestController;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/publish")
-@Tag(name = "上架", description = "上架 Excel 多文件任务接口。前端创建批次并严格串行派发文件；Python 按页拉取数据、发送统一任务心跳并回传当前店铺完整结果。")
+@Tag(name = "上架", description = "上架 Excel 多文件任务接口。前端创建批次并严格串行派发文件；Python 按页拉取数据、通过结果分片回传进度和当前店铺完整结果，任务心跳用于保活。")
public class PublishController {
private final PublishTaskService publishTaskService;
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/model/vo/PublishFileVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/model/vo/PublishFileVo.java
index 51baaf57..816ded61 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/model/vo/PublishFileVo.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/model/vo/PublishFileVo.java
@@ -38,9 +38,9 @@ public class PublishFileVo {
private Integer percent;
@Schema(description = "与 percent 相同，供公共进度组件使用", example = "50")
private Integer progressPercent;
- @Schema(description = "心跳上报的当前处理数量；未上报时使用 processedRows", example = "341")
+ @Schema(description = "结果分片累计接收的当前处理数量；兼容心跳上报", example = "341")
private Integer progressCurrent;
- @Schema(description = "心跳上报的总处理数量；未上报时使用 totalRows", example = "682")
+ @Schema(description = "解析得到的总处理数量；兼容心跳上报", example = "682")
private Integer progressTotal;
@Schema(description = "文件失败原因的进度展示副本；无错误时为空")
private String progressMessage;
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java
index 3a91248f..0f2f4924 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java
@@ -345,7 +345,11 @@ public class PublishTaskService {
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.set(FileTaskEntity::getUpdatedAt, now));
- if (request == null || (request.getCurrent() == null && request.getTotal() == null)) {
+ // Generic desktop heartbeats use 0/0 when they only need to keep the task alive.
+ // Treat that as no progress update so result chunks cannot be reset to zero.
+ if (request == null
+ || ((request.getCurrent() == null || request.getCurrent() <= 0)
+ && (request.getTotal() == null || request.getTotal() <= 0))) {
return;
}
PublishFileEntity active = publishFileMapper.selectOne(new LambdaQueryWrapper()
@@ -360,12 +364,12 @@ public class PublishTaskService {
.eq(PublishFileEntity::getId, active.getId())
.eq(PublishFileEntity::getStatus, STATUS_RUNNING)
.set(PublishFileEntity::getUpdatedAt, now);
- if (request.getTotal() != null && request.getTotal() >= 0) {
+ if (request.getTotal() != null && request.getTotal() > 0) {
update.set(PublishFileEntity::getTotalRows, request.getTotal());
}
- if (request.getCurrent() != null && request.getCurrent() >= 0) {
+ if (request.getCurrent() != null && request.getCurrent() > 0) {
int current = request.getCurrent();
- if (request.getTotal() != null && request.getTotal() >= 0) {
+ if (request.getTotal() != null && request.getTotal() > 0) {
current = Math.min(current, request.getTotal());
}
update.set(PublishFileEntity::getProcessedRows, current);
@@ -432,6 +436,7 @@ public class PublishTaskService {
throw new BusinessException("no successful publish files");
}
+ TaskOptions options = readTaskOptions(task);
List inputs = new ArrayList<>();
int rowCount = 0;
for (PublishFileEntity file : successfulFiles) {
@@ -443,7 +448,7 @@ public class PublishTaskService {
List rows = items.stream().map(this::toRowDto).toList();
rowCount += rows.size();
inputs.add(new PublishWorkbookService.WorkbookInput(
- file.getSourceFilename(), file.getShopName(), rows));
+ file.getSourceFilename(), file.getShopName(), options.publishCountry(), rows));
}
PublishWorkbookService.PackagedResult packaged = workbookService.packageTaskResult(
@@ -737,6 +742,7 @@ public class PublishTaskService {
} else {
ResultChunkReceipt receipt = persistResultChunk(taskId, file, incoming, storedPayloads);
if (!receipt.completed()) {
+ updateReceivedProgress(taskId, file, receipt.receivedRowCount());
file.setStatus(STATUS_RUNNING);
file.setErrorMessage(null);
file.setUpdatedAt(LocalDateTime.now());
@@ -1025,9 +1031,11 @@ public class PublishTaskService {
if (existing != null) {
validateExistingChunk(existing, chunkTotal, payloadHash);
int receivedChunkCount = countResultChunks(taskId, scopeHash);
- persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
+ int receivedRowCount = resolveReceivedRowCount(taskId, scopeHash, scope, 0, false);
+ persistResultScope(taskId, scopeKey, scopeHash, chunkTotal,
+ receivedChunkCount, receivedRowCount);
return new ResultChunkReceipt(scopeHash, chunkTotal,
- receivedChunkCount >= chunkTotal);
+ receivedChunkCount >= chunkTotal, receivedRowCount);
}
ensureRustfsPayloadStorageEnabled();
@@ -1046,9 +1054,11 @@ public class PublishTaskService {
chunk.setPayloadHash(payloadHash);
chunk.setCreatedAt(LocalDateTime.now());
chunk.setUpdatedAt(LocalDateTime.now());
+ boolean inserted = false;
try {
taskChunkMapper.insert(chunk);
storedPayloads.add(storedPayload);
+ inserted = true;
} catch (DuplicateKeyException ex) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
TaskChunkEntity winner = findResultChunk(taskId, scopeHash, chunkIndex);
@@ -1062,11 +1072,13 @@ public class PublishTaskService {
}
int receivedChunkCount = countResultChunks(taskId, scopeHash);
- persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
- log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} received={}",
- taskId, file.getId(), chunkIndex, chunkTotal, receivedChunkCount);
+ int receivedRowCount = resolveReceivedRowCount(taskId, scopeHash, scope, rows.size(), inserted);
+ persistResultScope(taskId, scopeKey, scopeHash, chunkTotal,
+ receivedChunkCount, receivedRowCount);
+ log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} receivedChunks={} receivedRows={}",
+ taskId, file.getId(), chunkIndex, chunkTotal, receivedChunkCount, receivedRowCount);
return new ResultChunkReceipt(scopeHash, chunkTotal,
- receivedChunkCount >= chunkTotal);
+ receivedChunkCount >= chunkTotal, receivedRowCount);
}
private void validateChunkMetadata(int chunkIndex, int chunkTotal) {
@@ -1119,11 +1131,70 @@ public class PublishTaskService {
return count == null ? 0 : count.intValue();
}
+ /**
+ * Resolve the number of result rows received for a file without re-reading
+ * every payload on every callback. New callbacks keep the count in the
+ * existing scope state JSON; scopes created by older versions are repaired
+ * once by counting their stored chunks.
+ */
+ private int resolveReceivedRowCount(Long taskId,
+ String scopeHash,
+ TaskScopeStateEntity scope,
+ int currentChunkRows,
+ boolean inserted) {
+ Integer persisted = readReceivedRowCount(scope);
+ if (persisted != null) {
+ long next = (long) persisted + (inserted ? Math.max(0, currentChunkRows) : 0);
+ return next > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) Math.max(0, next);
+ }
+ return countReceivedResultRows(taskId, scopeHash);
+ }
+
+ private Integer readReceivedRowCount(TaskScopeStateEntity scope) {
+ if (scope == null || scope.getStateJson() == null || scope.getStateJson().isBlank()) {
+ return null;
+ }
+ try {
+ JsonNode root = objectMapper.readTree(scope.getStateJson());
+ JsonNode receivedRows = root == null ? null : root.get("receivedRows");
+ if (receivedRows == null || !receivedRows.isIntegralNumber()) {
+ return null;
+ }
+ return Math.max(0, receivedRows.asInt(0));
+ } catch (Exception ex) {
+ log.warn("[publish] failed to read received row count from result scope taskId={} scopeHash={} msg={}",
+ scope.getTaskId(), scope.getScopeHash(), safeMessage(ex));
+ return null;
+ }
+ }
+
+ private int countReceivedResultRows(Long taskId, String scopeHash) {
+ List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper()
+ .eq(TaskChunkEntity::getTaskId, taskId)
+ .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
+ .eq(TaskChunkEntity::getScopeHash, scopeHash)
+ .orderByAsc(TaskChunkEntity::getChunkIndex));
+ if (chunks == null || chunks.isEmpty()) {
+ return 0;
+ }
+ TypeReference> listType = new TypeReference<>() {
+ };
+ long count = 0;
+ for (TaskChunkEntity chunk : chunks) {
+ count += readResultChunkRows(chunk, listType).size();
+ if (count >= Integer.MAX_VALUE) {
+ return Integer.MAX_VALUE;
+ }
+ }
+ return (int) count;
+ }
+
private void persistResultScope(Long taskId,
String scopeKey,
String scopeHash,
int chunkTotal,
- int receivedChunkCount) {
+ int receivedChunkCount,
+ int receivedRowCount) {
TaskScopeStateEntity scope = findResultScope(taskId, scopeHash);
validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal);
LocalDateTime now = LocalDateTime.now();
@@ -1141,7 +1212,7 @@ public class PublishTaskService {
scope.setCompleted(completed ? 1 : 0);
scope.setLastChunkAt(now);
scope.setLastError(null);
- scope.setStateJson(completed ? "{\"phase\":\"COMPLETE\"}" : "{\"phase\":\"RECEIVING\"}");
+ scope.setStateJson(resultScopeStateJson(completed, receivedRowCount));
scope.setUpdatedAt(now);
if (scope.getId() != null) {
taskScopeStateMapper.updateById(scope);
@@ -1160,12 +1231,43 @@ public class PublishTaskService {
winner.setCompleted(completed ? 1 : 0);
winner.setLastChunkAt(now);
winner.setLastError(null);
- winner.setStateJson(scope.getStateJson());
+ Integer winnerRowCount = readReceivedRowCount(winner);
+ winner.setStateJson(resultScopeStateJson(completed,
+ Math.max(receivedRowCount, winnerRowCount == null ? 0 : winnerRowCount)));
winner.setUpdatedAt(now);
taskScopeStateMapper.updateById(winner);
}
}
+ private String resultScopeStateJson(boolean completed, int receivedRowCount) {
+ Map state = new LinkedHashMap<>();
+ state.put("phase", completed ? "COMPLETE" : "RECEIVING");
+ state.put("receivedRows", Math.max(0, receivedRowCount));
+ return writeJson(state, "保存上架结果分片进度失败");
+ }
+
+ private void updateReceivedProgress(Long taskId, PublishFileEntity file, int receivedRowCount) {
+ if (file == null) {
+ return;
+ }
+ int total = safeInt(file.getTotalRows());
+ if (total <= 0) {
+ long parsedRows = Objects.requireNonNullElse(publishItemMapper.selectCount(
+ new LambdaQueryWrapper()
+ .eq(PublishItemEntity::getTaskId, taskId)
+ .eq(PublishItemEntity::getFileId, file.getId())), 0L);
+ total = parsedRows >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) Math.max(0L, parsedRows);
+ if (total > 0) {
+ file.setTotalRows(total);
+ }
+ }
+ int progress = Math.max(safeInt(file.getProcessedRows()), Math.max(0, receivedRowCount));
+ if (total > 0) {
+ progress = Math.min(total, progress);
+ }
+ file.setProcessedRows(progress);
+ }
+
private List loadCompleteResultRows(Long taskId, ResultChunkReceipt receipt) {
List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper()
.eq(TaskChunkEntity::getTaskId, taskId)
@@ -1637,6 +1739,7 @@ public class PublishTaskService {
private record ResultChunkReceipt(String scopeHash,
int chunkTotal,
- boolean completed) {
+ boolean completed,
+ int receivedRowCount) {
}
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishWorkbookService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishWorkbookService.java
index a2330690..30ec9bb6 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishWorkbookService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishWorkbookService.java
@@ -91,7 +91,7 @@ public class PublishWorkbookService {
}
}
- public File writeWorkbook(File outputFile, List rows) {
+ public File writeWorkbook(File outputFile, List rows, String publishCountry) {
File parent = outputFile.getParentFile();
if (parent != null) {
parent.mkdirs();
@@ -100,13 +100,9 @@ public class PublishWorkbookService {
workbook.setCompressTempFiles(true);
try (FileOutputStream output = new FileOutputStream(outputFile)) {
CellStyle headerStyle = createHeaderStyle(workbook);
- Map> rowsByCountry = groupByCountry(rows);
- Set usedSheetNames = new LinkedHashSet<>();
- for (Map.Entry> entry : rowsByCountry.entrySet()) {
- String sheetName = uniqueSheetName(entry.getKey(), usedSheetNames);
- Sheet sheet = workbook.createSheet(sheetName);
- writeSheet(sheet, entry.getValue(), headerStyle);
- }
+ String country = countrySheetName(publishCountry);
+ Sheet sheet = workbook.createSheet(country);
+ writeSheet(sheet, rows, headerStyle, country);
workbook.write(output);
return outputFile;
} catch (Exception ex) {
@@ -135,7 +131,7 @@ public class PublishWorkbookService {
+ "_上架结果.xlsx";
String filename = uniqueFilename(desired, usedFilenames);
File workbook = new File(workDirectory, filename);
- writeWorkbook(workbook, input.rows());
+ writeWorkbook(workbook, input.rows(), input.publishCountry());
workbooks.add(workbook);
}
@@ -202,7 +198,10 @@ public class PublishWorkbookService {
return grouped;
}
- private void writeSheet(Sheet sheet, List rows, CellStyle headerStyle) {
+ private void writeSheet(Sheet sheet,
+ List rows,
+ CellStyle headerStyle,
+ String publishCountry) {
Row header = sheet.createRow(0);
for (int index = 0; index < RESULT_HEADERS.size(); index++) {
Cell cell = header.createCell(index);
@@ -210,11 +209,14 @@ public class PublishWorkbookService {
cell.setCellStyle(headerStyle);
}
int rowIndex = 1;
- for (PublishRowDto value : rows) {
+ for (PublishRowDto value : rows == null ? List.of() : rows) {
+ if (value == null) {
+ continue;
+ }
Row row = sheet.createRow(rowIndex++);
setText(row, 0, value.getSourceId());
setText(row, 1, value.getAsin());
- setText(row, 2, value.getCountry());
+ setText(row, 2, publishCountry);
setText(row, 3, value.getBrand());
setPrice(row, 4, value.getPrice());
setText(row, 5, value.getStatus());
@@ -337,7 +339,10 @@ public class PublishWorkbookService {
public record ParsedWorkbook(List rows) {
}
- public record WorkbookInput(String sourceFilename, String shopName, List rows) {
+ public record WorkbookInput(String sourceFilename,
+ String shopName,
+ String publishCountry,
+ List rows) {
}
public record PackagedResult(File file, String filename, String contentType) {
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/AdminShopDataCrawlTaskController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/AdminShopDataCrawlTaskController.java
new file mode 100644
index 00000000..85d58a9e
--- /dev/null
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/AdminShopDataCrawlTaskController.java
@@ -0,0 +1,154 @@
+package com.nanri.aiimage.modules.shopdatacrawl.controller;
+
+import com.nanri.aiimage.common.api.ApiResponse;
+import com.nanri.aiimage.common.exception.BusinessException;
+import com.nanri.aiimage.common.util.DownloadHeaderUtil;
+import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
+import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
+import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
+import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
+import io.swagger.v3.oas.annotations.Operation;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+import lombok.RequiredArgsConstructor;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.server.ResponseStatusException;
+
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+
+/**
+ * Internal compatibility endpoints used by the Flask admin task page.
+ * The user-facing shop-data-crawl endpoints remain unchanged for existing clients.
+ */
+@RestController
+@RequiredArgsConstructor
+@RequestMapping("/api/admin/shop-data-crawl")
+public class AdminShopDataCrawlTaskController {
+
+ @Value("${aiimage.security.internal-token:}")
+ private String internalToken;
+
+ @Value("${aiimage.security.internal-token-file:}")
+ private String internalTokenFile;
+
+ private final ShopDataCrawlTaskService taskService;
+ private final AdminAuthSupport adminAuthSupport;
+ private final PermissionMenuService permissionMenuService;
+
+ @GetMapping("/results/{resultId}/download")
+ @Operation(summary = "下载店铺数据任务结果（内部）")
+ public void download(
+ @PathVariable Long resultId,
+ HttpServletRequest request,
+ HttpServletResponse response) {
+ requireShopDataCrawlTaskAccess(request);
+ String url = taskService.resolveAdminResultDownloadUrl(resultId);
+ String filename = taskService.resolveAdminResultDownloadFilename(resultId);
+ try {
+ response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
+ DownloadHeaderUtil.setAttachment(response, filename);
+ try (InputStream input = URI.create(url).toURL().openStream()) {
+ input.transferTo(response.getOutputStream());
+ }
+ } catch (Exception ex) {
+ throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
+ }
+ }
+
+ @DeleteMapping("/history/{resultId}")
+ @Operation(summary = "删除店铺数据任务结果（内部）")
+ public ApiResponse deleteHistory(
+ @PathVariable Long resultId,
+ HttpServletRequest request) {
+ requireShopDataCrawlTaskAccess(request);
+ taskService.deleteAdminHistory(resultId);
+ return ApiResponse.success(null);
+ }
+
+ private void requireShopDataCrawlTaskAccess(HttpServletRequest request) {
+ AdminUserEntity operator;
+ try {
+ operator = adminAuthSupport.requireAdmin(request);
+ } catch (BusinessException authFailure) {
+ operator = resolveInternalOperator(request);
+ if (operator == null) {
+ throw authFailure;
+ }
+ }
+ permissionMenuService.requireShopDataCrawlTaskAccess(operator);
+ }
+
+ private AdminUserEntity resolveInternalOperator(HttpServletRequest request) {
+ String suppliedToken = request.getHeader("X-Internal-Token");
+ if (!isTrustedInternalRequest(suppliedToken)) {
+ return null;
+ }
+ String rawOperatorId = request.getParameter("operatorId");
+ if (rawOperatorId == null || rawOperatorId.isBlank()) {
+ rawOperatorId = request.getParameter("operator_id");
+ }
+ if (rawOperatorId == null || rawOperatorId.isBlank()) {
+ return null;
+ }
+ try {
+ return permissionMenuService.requireAdminOperator(Long.parseLong(rawOperatorId.trim()));
+ } catch (NumberFormatException ex) {
+ return null;
+ }
+ }
+
+ private boolean isTrustedInternalRequest(String suppliedToken) {
+ String expectedToken = resolveExpectedInternalToken();
+ return !expectedToken.isBlank() && suppliedToken != null && !suppliedToken.isBlank()
+ && MessageDigest.isEqual(
+ expectedToken.getBytes(StandardCharsets.UTF_8),
+ suppliedToken.trim().getBytes(StandardCharsets.UTF_8));
+ }
+
+ private String resolveExpectedInternalToken() {
+ if (internalToken != null && !internalToken.isBlank()) {
+ return internalToken.trim();
+ }
+ Path path = resolveInternalTokenFile();
+ if (path == null || !Files.isRegularFile(path)) {
+ return "";
+ }
+ try {
+ return Files.readString(path, StandardCharsets.UTF_8).trim();
+ } catch (Exception ignored) {
+ return "";
+ }
+ }
+
+ private Path resolveInternalTokenFile() {
+ String configuredPath = internalTokenFile == null ? "" : internalTokenFile.trim();
+ if (!configuredPath.isEmpty()) {
+ if (configuredPath.equals("~") || configuredPath.startsWith("~/") || configuredPath.startsWith("~\\")) {
+ String userHome = System.getProperty("user.home", "").trim();
+ if (userHome.isEmpty()) {
+ return null;
+ }
+ configuredPath = configuredPath.length() == 1
+ ? userHome
+ : Path.of(userHome, configuredPath.substring(2)).toString();
+ }
+ Path configuredTokenPath = Path.of(configuredPath);
+ return configuredTokenPath.isAbsolute() ? configuredTokenPath.normalize() : null;
+ }
+ String userHome = System.getProperty("user.home", "").trim();
+ return userHome.isEmpty()
+ ? null
+ : Path.of(userHome, ".aiimage", "internal-token").toAbsolutePath().normalize();
+ }
+}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java
index 6d530edc..57c7e9ac 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java
@@ -190,6 +190,7 @@ public class ShopDataCrawlTaskController {
{
"date": "2026-07-25",
"asin": "B0EXAMPLE1",
+ "commodityImage": "https://m.media-amazon.com/images/I/example.jpg",
"inventorySales": "120",
"salesRank": "35",
"pageViews": "860",
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlRowDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlRowDto.java
index d528ccd9..99a48063 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlRowDto.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/dto/ShopDataCrawlRowDto.java
@@ -5,7 +5,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
-@Schema(description = "店铺数据抓取结果行；生成 Excel 时按日期、ASIN、库存销量、销售排名、页面浏览量、售出件数、价格、推荐报价的固定列顺序写入")
+@Schema(description = "店铺数据抓取结果行；生成 Excel 时按日期、ASIN、商品图片、库存销量、销售排名、页面浏览量、售出件数、价格、推荐报价、品牌的固定列顺序写入")
public class ShopDataCrawlRowDto {
@JsonAlias("日期")
@Schema(description = "日期列，按来源文本原样保留", example = "2026-07-25")
@@ -15,6 +15,14 @@ public class ShopDataCrawlRowDto {
@Schema(description = "亚马逊商品 ASIN", example = "B0CJ8SNXXV")
private String asin;
+ @JsonAlias({"brand", "品牌"})
+ @Schema(description = "商品品牌，来自 Python 回传的 brand 字段", example = "Example Brand")
+ private String brand;
+
+ @JsonAlias({"commodity_image", "商品图片"})
+ @Schema(description = "商品图片 URL；生成 Excel 时下载并嵌入商品图片列", example = "https://m.media-amazon.com/images/I/example.jpg")
+ private String commodityImage;
+
@JsonAlias({"库存销量", "inventory_sales"})
@Schema(description = "库存销量列，按来源文本原样保留", example = "128")
private String inventorySales;
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java
index 5fb69d3f..5d2a45a9 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyService.java
@@ -4,10 +4,16 @@ import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
+import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
+import org.apache.poi.ss.usermodel.ClientAnchor;
+import org.apache.poi.ss.usermodel.Drawing;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
@@ -19,13 +25,24 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
@Service
+@Slf4j
+@RequiredArgsConstructor
public class ShopDataCrawlExcelAssemblyService {
static final List COUNTRIES = List.of("UK", "DE", "FR", "ES", "IT");
static final List SHEETS = List.of("英国", "德国", "法国", "西班牙", "意大利");
- static final List HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
+ static final List LEGACY_HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
+ static final List HEADERS_WITHOUT_BRAND = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
+ static final List HEADERS = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价", "品牌");
private static final String TEMPLATE = "templates/shop-data-crawl/文档格式.xlsx";
+ private static final int IMAGE_COLUMN = 2;
+ private static final int BRAND_COLUMN = HEADERS.size() - 1;
+ private static final int IMAGE_COLUMN_WIDTH = 18 * 256;
+ private static final float IMAGE_ROW_HEIGHT_POINTS = 80f;
+
+ private final SimilarAsinImageEmbedder imageEmbedder;
public void writeWorkbook(File outputXlsx, List items) {
try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream();
@@ -33,8 +50,11 @@ public class ShopDataCrawlExcelAssemblyService {
FileOutputStream output = new FileOutputStream(outputXlsx)) {
validateTemplate(workbook);
Map> rowsByCountry = rowsByCountry(items);
+ Map imageCache = new ConcurrentHashMap<>();
+ imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache);
+ Map pictureIndexes = new LinkedHashMap<>();
for (int i = 0; i < COUNTRIES.size(); i++) {
- writeSheet(workbook.getSheetAt(i), rowsByCountry.get(COUNTRIES.get(i)));
+ writeSheet(workbook, workbook.getSheetAt(i), rowsByCountry.get(COUNTRIES.get(i)), imageCache, pictureIndexes);
}
workbook.write(output);
} catch (BusinessException ex) {
@@ -58,22 +78,45 @@ public class ShopDataCrawlExcelAssemblyService {
throw new BusinessException("店铺数据抓取模板工作表顺序不正确");
}
Row header = sheet.getRow(0);
- for (int column = 0; column < HEADERS.size(); column++) {
+ boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
+ boolean templateHasBrand = "品牌".equals(cellText(header, BRAND_COLUMN));
+ List expectedHeaders = currentTemplate
+ ? (templateHasBrand ? HEADERS : HEADERS_WITHOUT_BRAND)
+ : LEGACY_HEADERS;
+ for (int column = 0; column < expectedHeaders.size(); column++) {
String actual = header == null || header.getCell(column) == null ? "" : header.getCell(column).getStringCellValue().trim();
- if (!HEADERS.get(column).equals(actual)) {
+ if (!expectedHeaders.get(column).equals(actual)) {
throw new BusinessException("店铺数据抓取模板表头不正确: " + sheet.getSheetName());
}
}
}
}
- private void writeSheet(Sheet sheet, List rows) {
+ private void writeSheet(XSSFWorkbook workbook,
+ Sheet sheet,
+ List rows,
+ Map imageCache,
+ Map pictureIndexes) {
+ Row header = sheet.getRow(0);
Row styleRow = sheet.getRow(1);
+ boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
+ boolean templateHasBrand = "品牌".equals(cellText(header, BRAND_COLUMN));
CellStyle[] styles = new CellStyle[HEADERS.size()];
for (int column = 0; column < styles.length; column++) {
- Cell cell = styleRow == null ? null : styleRow.getCell(column);
+ int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
+ Cell cell = styleRow == null ? null : styleRow.getCell(sourceColumn);
styles[column] = cell == null ? null : cell.getCellStyle();
}
+ int[] columnWidths = new int[HEADERS.size()];
+ for (int column = 0; column < columnWidths.length; column++) {
+ int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
+ columnWidths[column] = sheet.getColumnWidth(sourceColumn);
+ }
+ writeHeaders(sheet, currentTemplate, templateHasBrand);
+ for (int column = 0; column < columnWidths.length; column++) {
+ sheet.setColumnWidth(column, columnWidths[column]);
+ }
+ sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
int last = sheet.getLastRowNum();
for (int rowIndex = 1; rowIndex <= last; rowIndex++) {
Row row = sheet.getRow(rowIndex);
@@ -84,16 +127,97 @@ public class ShopDataCrawlExcelAssemblyService {
int rowIndex = 1;
for (ShopDataCrawlRowDto value : rows == null ? List.of() : rows) {
Row row = sheet.createRow(rowIndex++);
- String[] values = {value.getDate(), value.getAsin(), value.getInventorySales(), value.getSalesRank(),
- value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer()};
+ String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
+ value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
for (int column = 0; column < values.length; column++) {
Cell cell = row.createCell(column);
if (styles[column] != null) cell.setCellStyle(styles[column]);
cell.setCellValue(values[column] == null ? "" : values[column]);
}
+ if (!blank(value.getCommodityImage())) {
+ row.setHeightInPoints(IMAGE_ROW_HEIGHT_POINTS);
+ embedImage(workbook, sheet, row, value.getCommodityImage(), imageCache, pictureIndexes);
+ }
}
}
+ private void writeHeaders(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) {
+ Row header = sheet.getRow(0);
+ if (header == null) header = sheet.createRow(0);
+ CellStyle[] styles = new CellStyle[HEADERS.size()];
+ for (int column = 0; column < HEADERS.size(); column++) {
+ int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
+ Cell source = header.getCell(sourceColumn);
+ styles[column] = source == null ? null : source.getCellStyle();
+ }
+ for (int column = 0; column < HEADERS.size(); column++) {
+ Cell cell = header.getCell(column);
+ if (cell == null) cell = header.createCell(column);
+ if (styles[column] != null) cell.setCellStyle(styles[column]);
+ cell.setCellValue(HEADERS.get(column));
+ }
+ }
+
+ private int templateColumnForOutput(int outputColumn, boolean currentTemplate, boolean templateHasBrand) {
+ if (outputColumn == BRAND_COLUMN) {
+ return templateHasBrand ? BRAND_COLUMN : 1;
+ }
+ return currentTemplate || outputColumn < IMAGE_COLUMN ? outputColumn : outputColumn - 1;
+ }
+
+ private void embedImage(XSSFWorkbook workbook,
+ Sheet sheet,
+ Row row,
+ String imageUrl,
+ Map imageCache,
+ Map pictureIndexes) {
+ String normalizedUrl = imageUrl.trim();
+ try {
+ SimilarAsinImageEmbedder.ResizedImage image = imageCache.get(normalizedUrl);
+ if (image == null) image = imageEmbedder.fetchAndResizeForCache(normalizedUrl);
+ if (image == null) {
+ row.getCell(IMAGE_COLUMN).setCellValue(normalizedUrl);
+ return;
+ }
+ Integer pictureIndex = pictureIndexes.get(normalizedUrl);
+ if (pictureIndex == null) {
+ pictureIndex = workbook.addPicture(image.bytes(), Workbook.PICTURE_TYPE_JPEG);
+ pictureIndexes.put(normalizedUrl, pictureIndex);
+ }
+ Drawing drawing = sheet.createDrawingPatriarch();
+ ClientAnchor anchor = workbook.getCreationHelper().createClientAnchor();
+ anchor.setCol1(IMAGE_COLUMN);
+ anchor.setRow1(row.getRowNum());
+ anchor.setCol2(IMAGE_COLUMN + 1);
+ anchor.setRow2(row.getRowNum() + 1);
+ anchor.setAnchorType(ClientAnchor.AnchorType.MOVE_AND_RESIZE);
+ drawing.createPicture(anchor, pictureIndex);
+ } catch (RuntimeException ex) {
+ log.warn("[shop-data-crawl] embed commodity image failed sheet={} row={} url={} msg={}",
+ sheet.getSheetName(), row.getRowNum() + 1, normalizedUrl, ex.getMessage());
+ row.getCell(IMAGE_COLUMN).setCellValue(normalizedUrl);
+ }
+ }
+
+ private List imageUrls(Map> rowsByCountry) {
+ return rowsByCountry.values().stream()
+ .flatMap(List::stream)
+ .map(ShopDataCrawlRowDto::getCommodityImage)
+ .filter(url -> !blank(url))
+ .map(String::trim)
+ .distinct()
+ .toList();
+ }
+
+ private String cellText(Row row, int column) {
+ Cell cell = row == null ? null : row.getCell(column);
+ return cell == null ? "" : cell.getStringCellValue().trim();
+ }
+
+ private boolean blank(String value) {
+ return value == null || value.isBlank();
+ }
+
private Map> rowsByCountry(List items) {
Map> result = new LinkedHashMap<>();
COUNTRIES.forEach(country -> result.put(country, new ArrayList<>()));
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java
index 24734653..6a71f4a8 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java
@@ -50,6 +50,7 @@ import java.io.File;
import java.time.LocalDateTime;
import java.time.Duration;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -65,6 +66,7 @@ public class ShopDataCrawlTaskService {
private static final int RESULT_PENDING = -1;
private static final int RESULT_FAILED = 0;
private static final int RESULT_SUCCESS = 1;
+ private static final int SHOP_HISTORY_RETENTION_LIMIT = 3;
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
private static final String PARTIAL_RESULT_MESSAGE = "Python 中断，已保留已回传的部分数据";
private static final String RESULT_CHUNK_SCOPE_PREFIX = "result-chunks:";
@@ -290,21 +292,54 @@ public class ShopDataCrawlTaskService {
}
public String resolveResultDownloadUrl(Long resultId, Long userId) {
- FileResultEntity entity = fileResultMapper.selectById(resultId);
- if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
+ validateUserId(userId);
+ FileResultEntity entity = requireResultEntity(resultId);
+ ensureResultOwner(entity, userId);
+ return resolveDownloadUrl(entity);
+ }
+
+ public String resolveResultDownloadFilename(Long resultId, Long userId) {
+ validateUserId(userId);
+ FileResultEntity entity = requireResultEntity(resultId);
+ ensureResultOwner(entity, userId);
+ return resolveDownloadFilename(entity);
+ }
+
+ /**
+ * Used only by the internally authenticated admin task-management endpoint.
+ * The public endpoint must continue to validate the requesting user ID.
+ */
+ public String resolveAdminResultDownloadUrl(Long resultId) {
+ return resolveDownloadUrl(requireResultEntity(resultId));
+ }
+
+ /** See {@link #resolveAdminResultDownloadUrl(Long)}. */
+ public String resolveAdminResultDownloadFilename(Long resultId) {
+ return resolveDownloadFilename(requireResultEntity(resultId));
+ }
+
+ private FileResultEntity requireResultEntity(Long resultId) {
+ FileResultEntity entity = resultId == null || resultId <= 0 ? null : fileResultMapper.selectById(resultId);
+ if (entity == null || !MODULE_TYPE.equals(entity.getModuleType())) {
throw new BusinessException("记录不存在");
}
+ return entity;
+ }
+
+ private void ensureResultOwner(FileResultEntity entity, Long userId) {
+ if (entity == null || !userId.equals(entity.getUserId())) {
+ throw new BusinessException("记录不存在");
+ }
+ }
+
+ private String resolveDownloadUrl(FileResultEntity entity) {
if (blank(entity.getResultFileUrl())) {
throw new BusinessException("暂无可下载文件");
}
return ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl());
}
- public String resolveResultDownloadFilename(Long resultId, Long userId) {
- FileResultEntity entity = fileResultMapper.selectById(resultId);
- if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
- throw new BusinessException("记录不存在");
- }
+ private String resolveDownloadFilename(FileResultEntity entity) {
return !blank(entity.getResultFilename())
? entity.getResultFilename()
: safeFileStem(entity.getSourceFilename()) + ".xlsx";
@@ -584,28 +619,63 @@ public class ShopDataCrawlTaskService {
@Transactional
public void deleteHistory(Long resultId, Long userId) {
validateUserId(userId);
- FileResultEntity entity = fileResultMapper.selectById(resultId);
- if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
- throw new BusinessException("记录不存在");
- }
+ FileResultEntity entity = requireResultEntity(resultId);
+ ensureResultOwner(entity, userId);
Long taskId = entity.getTaskId();
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) throw new BusinessException("任务不存在");
- ensureTaskOwnedByCurrentInstance(task, "delete shop data crawl history");
+ if (!isTerminalTaskStatus(task.getStatus())) {
+ throw new BusinessException(40901, "任务仍在处理中，不能删除");
+ }
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("记录不存在");
}
- taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId);
- taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId);
- String resultFileUrl = latestEntity.getResultFileUrl();
- fileResultMapper.deleteById(resultId);
- reconcileTaskAfterResultRemoval(taskId);
- deleteResultObjectIfUnreferenced(resultFileUrl);
+ deleteResultHistoryRow(latestEntity);
}
}
+ /**
+ * Deletes an administrative result without accepting a caller-controlled owner ID.
+ * The controller that calls this method is restricted to the Flask-to-Java internal channel.
+ */
+ @Transactional
+ public void deleteAdminHistory(Long resultId) {
+ FileResultEntity entity = requireResultEntity(resultId);
+ Long ownerId = entity.getUserId();
+ if (ownerId == null || ownerId <= 0) {
+ throw new BusinessException("记录不存在");
+ }
+ deleteHistory(resultId, ownerId);
+ }
+
+ /**
+ * Removes one result row and all task-owned artifacts that point at it.
+ * The caller must already hold the task lock when the row belongs to a task.
+ */
+ private void deleteResultHistoryRow(FileResultEntity entity) {
+ if (entity == null || entity.getId() == null || entity.getId() <= 0) {
+ return;
+ }
+ Long taskId = entity.getTaskId();
+ Long resultId = entity.getId();
+ String resultFileUrl = entity.getResultFileUrl();
+ taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId);
+ taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId);
+ fileResultMapper.deleteById(resultId);
+ try {
+ reconcileTaskAfterResultRemoval(taskId);
+ } catch (Exception ex) {
+ // The row is already gone. Do not leave its workbook behind merely
+ // because the parent task snapshot could not be rebuilt.
+ log.warn("[shop-data-crawl] reconcile task after result deletion failed taskId={} resultId={} msg={}",
+ taskId, resultId, safeMessage(ex));
+ }
+ // Run this after the row delete so shared object references are counted correctly.
+ deleteResultObjectIfUnreferenced(resultFileUrl);
+ }
+
private void reconcileTaskAfterResultRemoval(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
@@ -630,6 +700,139 @@ public class ShopDataCrawlTaskService {
taskCacheService.saveTaskCache(task);
}
+ private void pruneCompletedHistoryQuietly(FileTaskEntity currentTask, List currentRows) {
+ if (currentRows == null || currentRows.isEmpty()) {
+ return;
+ }
+ for (FileResultEntity row : currentRows) {
+ if (!isRetentionCandidate(row)) {
+ continue;
+ }
+ Long userId = row.getUserId() != null
+ ? row.getUserId()
+ : currentTask == null ? null : currentTask.getUserId();
+ String shopKey = retentionShopKey(row);
+ if (userId == null || shopKey == null) {
+ log.warn("[shop-data-crawl] skip history retention because ownership key is incomplete taskId={} resultId={}",
+ currentTask == null ? null : currentTask.getId(), row.getId());
+ continue;
+ }
+ try {
+ pruneCompletedHistoryForShop(userId, shopKey);
+ } catch (Exception ex) {
+ // Retention is best effort. A cleanup failure must not fail the newly assembled workbook job.
+ log.warn("[shop-data-crawl] history retention failed taskId={} resultId={} msg={}",
+ currentTask == null ? null : currentTask.getId(), row.getId(), safeMessage(ex));
+ }
+ }
+ }
+
+ void pruneCompletedHistoryForShop(Long userId, String shopKey) {
+ if (userId == null || shopKey == null) {
+ return;
+ }
+ List candidates = fileResultMapper.selectList(new LambdaQueryWrapper()
+ .select(FileResultEntity::getId,
+ FileResultEntity::getTaskId,
+ FileResultEntity::getModuleType,
+ FileResultEntity::getSourceFilename,
+ FileResultEntity::getSourceFileUrl,
+ FileResultEntity::getResultFileUrl,
+ FileResultEntity::getSuccess,
+ FileResultEntity::getUserId,
+ FileResultEntity::getCreatedAt)
+ .eq(FileResultEntity::getModuleType, MODULE_TYPE)
+ .eq(FileResultEntity::getUserId, userId)
+ .eq(FileResultEntity::getSuccess, RESULT_SUCCESS)
+ .isNotNull(FileResultEntity::getResultFileUrl)
+ .ne(FileResultEntity::getResultFileUrl, "")
+ .orderByDesc(FileResultEntity::getCreatedAt)
+ .orderByDesc(FileResultEntity::getId));
+ if (candidates == null || candidates.isEmpty()) {
+ return;
+ }
+
+ List shopResults = new ArrayList<>();
+ for (FileResultEntity candidate : candidates) {
+ if (!isRetentionCandidate(candidate)
+ || !Objects.equals(userId, candidate.getUserId())) {
+ continue;
+ }
+ if (!Objects.equals(shopKey, retentionShopKey(candidate))) {
+ continue;
+ }
+ shopResults.add(candidate);
+ }
+
+ Comparator newestFirst = Comparator
+ .comparing(FileResultEntity::getCreatedAt, Comparator.nullsLast(Comparator.reverseOrder()))
+ .thenComparing(FileResultEntity::getId, Comparator.nullsLast(Comparator.reverseOrder()));
+ shopResults.sort(newestFirst);
+ for (int index = SHOP_HISTORY_RETENTION_LIMIT; index < shopResults.size(); index++) {
+ deleteRetentionResultQuietly(shopResults.get(index), userId, shopKey);
+ }
+ }
+
+ private void deleteRetentionResultQuietly(FileResultEntity candidate, Long userId, String shopKey) {
+ if (candidate == null || candidate.getId() == null || candidate.getId() <= 0
+ || candidate.getTaskId() == null || candidate.getTaskId() <= 0) {
+ return;
+ }
+ try {
+ TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(candidate.getTaskId());
+ if (lockHandle == null) {
+ log.info("[shop-data-crawl] skip retained-history deletion because task lock is busy taskId={} resultId={}",
+ candidate.getTaskId(), candidate.getId());
+ return;
+ }
+ try (lockHandle) {
+ FileResultEntity latest = fileResultMapper.selectById(candidate.getId());
+ if (!isRetentionCandidate(latest)
+ || !Objects.equals(userId, latest.getUserId())
+ || !Objects.equals(shopKey, retentionShopKey(latest))) {
+ return;
+ }
+ FileTaskEntity task = fileTaskMapper.selectById(candidate.getTaskId());
+ if (task == null || !MODULE_TYPE.equals(task.getModuleType())
+ || !isTerminalTaskStatus(task.getStatus())) {
+ log.info("[shop-data-crawl] skip retained-history deletion because task is not terminal taskId={} resultId={}",
+ candidate.getTaskId(), candidate.getId());
+ return;
+ }
+ deleteResultHistoryRow(latest);
+ }
+ } catch (Exception ex) {
+ log.warn("[shop-data-crawl] retained-history deletion failed taskId={} resultId={} msg={}",
+ candidate.getTaskId(), candidate.getId(), safeMessage(ex));
+ }
+ }
+
+ private boolean isRetentionCandidate(FileResultEntity row) {
+ return row != null
+ && Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())
+ && !blank(row.getResultFileUrl());
+ }
+
+ private String retentionShopKey(FileResultEntity row) {
+ if (row == null) {
+ return null;
+ }
+ String shopId = trimToNull(row.getSourceFileUrl());
+ if (shopId != null) {
+ return "shop-id:" + shopId;
+ }
+ String shopName = trimToNull(row.getSourceFilename());
+ return shopName == null ? null : "shop-name:" + shopName;
+ }
+
+ private String trimToNull(String value) {
+ if (value == null) {
+ return null;
+ }
+ String normalized = value.trim();
+ return normalized.isEmpty() ? null : normalized;
+ }
+
private long countTasks(Long userId, List statuses) {
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
@@ -1360,6 +1563,7 @@ public class ShopDataCrawlTaskService {
} finally {
FileUtil.del(xlsx);
}
+ pruneCompletedHistoryQuietly(task, rows);
}
public void cleanupResultFileJob(TaskFileJobEntity job) {
@@ -1460,6 +1664,10 @@ public class ShopDataCrawlTaskService {
return Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess()) || Integer.valueOf(RESULT_FAILED).equals(row.getSuccess());
}
+ private boolean isTerminalTaskStatus(String status) {
+ return "SUCCESS".equals(status) || "FAILED".equals(status) || "CANCELLED".equals(status);
+ }
+
private Boolean toSuccessFlag(Integer dbValue, Boolean fallback) {
if (dbValue == null || Integer.valueOf(RESULT_PENDING).equals(dbValue)) {
return fallback;
@@ -1544,6 +1752,8 @@ public class ShopDataCrawlTaskService {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate(trim(source.getDate()));
row.setAsin(trim(source.getAsin()));
+ row.setBrand(trim(source.getBrand()));
+ row.setCommodityImage(trim(source.getCommodityImage()));
row.setInventorySales(trim(source.getInventorySales()));
row.setSalesRank(trim(source.getSalesRank()));
row.setPageViews(trim(source.getPageViews()));
@@ -1581,6 +1791,8 @@ public class ShopDataCrawlTaskService {
return left != null && right != null
&& Objects.equals(trim(left.getDate()), trim(right.getDate()))
&& Objects.equals(trim(left.getAsin()), trim(right.getAsin()))
+ && Objects.equals(trim(left.getBrand()), trim(right.getBrand()))
+ && Objects.equals(trim(left.getCommodityImage()), trim(right.getCommodityImage()))
&& Objects.equals(trim(left.getInventorySales()), trim(right.getInventorySales()))
&& Objects.equals(trim(left.getSalesRank()), trim(right.getSalesRank()))
&& Objects.equals(trim(left.getPageViews()), trim(right.getPageViews()))
@@ -1590,7 +1802,7 @@ public class ShopDataCrawlTaskService {
}
private boolean rowEmpty(ShopDataCrawlRowDto row) {
- return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getInventorySales())
+ return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getBrand()) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
&& blank(row.getSalesRank()) && blank(row.getPageViews()) && blank(row.getUnitsSold())
&& blank(row.getPrice()) && blank(row.getRecommendedOffer()));
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/controller/SkipPriceAsinController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/controller/SkipPriceAsinController.java
index e72dcaa9..59e06be6 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/controller/SkipPriceAsinController.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/controller/SkipPriceAsinController.java
@@ -79,13 +79,13 @@ public class SkipPriceAsinController {
}
@PostMapping
- @Operation(summary = "新增或更新跳过跟价 ASIN", description = "按店铺和国家维度新增或更新跳过跟价 ASIN。")
+ @Operation(summary = "新增跳过跟价 ASIN", description = "新增一条跳过跟价 ASIN 记录。")
public ApiResponse create(
@Parameter(description = "当前操作人用户 ID") @RequestParam(name = "operator_id") Long operatorId,
@Parameter(description = "是否超级管理员") @RequestParam(name = "super_admin", defaultValue = "false") Boolean superAdmin,
@Valid @RequestBody SkipPriceAsinCreateRequest request) {
return ApiResponse.success("保存成功",
- skipPriceAsinService.createOrUpdate(request, operatorId, Boolean.TRUE.equals(superAdmin)));
+ skipPriceAsinService.create(request, operatorId, Boolean.TRUE.equals(superAdmin)));
}
@PostMapping("/import")
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/ShopKeyEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/ShopKeyEntity.java
index 277f52ed..5255b85a 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/ShopKeyEntity.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/ShopKeyEntity.java
@@ -1,6 +1,8 @@
package com.nanri.aiimage.modules.shopkey.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.FieldStrategy;
+import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
@@ -16,6 +18,11 @@ public class ShopKeyEntity {
private String remarkName;
private String ziniaoAccountName;
private String ziniaoToken;
+ private String ipWhitelistStatus;
+ @TableField(updateStrategy = FieldStrategy.ALWAYS)
+ private LocalDateTime ipWhitelistCheckedAt;
+ @TableField(updateStrategy = FieldStrategy.ALWAYS)
+ private String ipWhitelistMessage;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/vo/ShopKeyItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/vo/ShopKeyItemVo.java
index cd8574cc..cf47a167 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/vo/ShopKeyItemVo.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/vo/ShopKeyItemVo.java
@@ -21,6 +21,15 @@ public class ShopKeyItemVo {
@Schema(description = "紫鸟令牌")
private String ziniaoToken;
+ @Schema(description = "IP 白名单状态：UNKNOWN、ALLOWED、BLOCKED")
+ private String ipWhitelistStatus;
+
+ @Schema(description = "最近一次 IP 白名单检测时间")
+ private LocalDateTime ipWhitelistCheckedAt;
+
+ @Schema(description = "最近一次 IP 白名单检测信息")
+ private String ipWhitelistMessage;
+
@Schema(description = "创建时间")
private LocalDateTime createdAt;
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopKeyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopKeyService.java
index 23059954..d10950dc 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopKeyService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopKeyService.java
@@ -21,6 +21,8 @@ import java.util.List;
@Slf4j
public class ShopKeyService {
+ private static final String IP_WHITELIST_STATUS_UNKNOWN = "UNKNOWN";
+
private final ShopKeyMapper shopKeyMapper;
private final ZiniaoShopIndexService ziniaoShopIndexService;
@@ -50,6 +52,7 @@ public class ShopKeyService {
entity.setRemarkName(normalizeOptional(request.getRemarkName()));
entity.setZiniaoAccountName(ziniaoAccountName);
entity.setZiniaoToken(ziniaoToken);
+ entity.setIpWhitelistStatus(IP_WHITELIST_STATUS_UNKNOWN);
shopKeyMapper.insert(entity);
triggerShopIndexRefresh();
return toItemVo(getById(entity.getId()));
@@ -60,9 +63,15 @@ public class ShopKeyService {
ShopKeyEntity entity = getById(id);
String ziniaoAccountName = normalizeRequired(request.getZiniaoAccountName(), "紫鸟账号名称不能为空");
String ziniaoToken = normalizeRequired(request.getZiniaoToken(), "紫鸟令牌不能为空");
+ boolean tokenChanged = !ziniaoToken.equals(entity.getZiniaoToken());
entity.setRemarkName(normalizeOptional(request.getRemarkName()));
entity.setZiniaoAccountName(ziniaoAccountName);
entity.setZiniaoToken(ziniaoToken);
+ if (tokenChanged) {
+ entity.setIpWhitelistStatus(IP_WHITELIST_STATUS_UNKNOWN);
+ entity.setIpWhitelistCheckedAt(null);
+ entity.setIpWhitelistMessage(null);
+ }
shopKeyMapper.updateById(entity);
triggerShopIndexRefresh();
return toItemVo(getById(id));
@@ -101,6 +110,9 @@ public class ShopKeyService {
vo.setRemarkName(entity.getRemarkName());
vo.setZiniaoAccountName(entity.getZiniaoAccountName());
vo.setZiniaoToken(entity.getZiniaoToken());
+ vo.setIpWhitelistStatus(entity.getIpWhitelistStatus());
+ vo.setIpWhitelistCheckedAt(entity.getIpWhitelistCheckedAt());
+ vo.setIpWhitelistMessage(entity.getIpWhitelistMessage());
vo.setCreatedAt(entity.getCreatedAt());
vo.setUpdatedAt(entity.getUpdatedAt());
return vo;
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java
index 9431e3aa..ea636cc7 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java
@@ -210,33 +210,23 @@ public class SkipPriceAsinService {
}
@Transactional
- public SkipPriceAsinItemVo createOrUpdate(SkipPriceAsinCreateRequest request, Long operatorId, boolean superAdmin) {
+ public SkipPriceAsinItemVo create(SkipPriceAsinCreateRequest request, Long operatorId, boolean superAdmin) {
ShopManageGroupEntity group = shopManageGroupService.getAccessibleById(request.getGroupId(), operatorId, superAdmin);
String shopName = normalizeRequired(request.getShopName(), "店铺名不能为空");
Set countries = normalizeCountries(request.getCountries());
Map countryAsinMap = normalizeCountryAsinMap(countries, request);
Map countryMinimumPriceMap = normalizeCountryMinimumPriceMap(countries, request);
- SkipPriceAsinEntity entity = skipPriceAsinMapper.selectOne(new LambdaQueryWrapper()
- .eq(SkipPriceAsinEntity::getGroupId, group.getId())
- .eq(SkipPriceAsinEntity::getShopName, shopName)
- .last("LIMIT 1"));
- if (entity == null) {
- entity = new SkipPriceAsinEntity();
- entity.setGroupId(group.getId());
- entity.setShopName(shopName);
- }
+ SkipPriceAsinEntity entity = new SkipPriceAsinEntity();
+ entity.setGroupId(group.getId());
+ entity.setShopName(shopName);
for (Map.Entry entry : countryAsinMap.entrySet()) {
String country = entry.getKey();
setCountryData(entity, country, entry.getValue(), countryMinimumPriceMap.get(country));
}
- if (entity.getId() == null) {
- skipPriceAsinMapper.insert(entity);
- } else {
- skipPriceAsinMapper.updateById(entity);
- }
+ skipPriceAsinMapper.insert(entity);
SkipPriceAsinEntity saved = getById(entity.getId());
return toItemVo(saved, group.getGroupName());
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinCozeClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinCozeClient.java
index 89cd8c6d..0802cb5b 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinCozeClient.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinCozeClient.java
@@ -47,6 +47,14 @@ public class SimilarAsinCozeClient {
}
public List inspect(List rows, String prompt, String apiKey, boolean imgSwitch) {
+ return inspect(rows, prompt, apiKey, imgSwitch, false);
+ }
+
+ public List inspect(List rows,
+ String prompt,
+ String apiKey,
+ boolean imgSwitch,
+ boolean categorySwitch) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
@@ -55,7 +63,7 @@ public class SimilarAsinCozeClient {
return rows.stream().map(this::copy).toList();
}
try {
- return inspectWithFallback(rows, prompt, apiKey, imgSwitch);
+ return inspectWithFallback(rows, prompt, apiKey, imgSwitch, categorySwitch);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[similar-asin] coze batch failed size={} err={}", rows.size(), failureMessage);
@@ -75,9 +83,10 @@ public class SimilarAsinCozeClient {
String prompt,
String apiKey,
boolean imgSwitch,
+ boolean categorySwitch,
CozeCredentialRef credential) throws Exception {
CozeCredentialRef resolvedCredential = resolveCredential(credential);
- JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, resolvedCredential));
+ JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, resolvedCredential));
ensureSuccess(submitRoot);
return new CozeSubmitResponse(
extractExecuteId(submitRoot),
@@ -87,6 +96,14 @@ public class SimilarAsinCozeClient {
);
}
+ public CozeSubmitResponse submitWorkflow(List rows,
+ String prompt,
+ String apiKey,
+ boolean imgSwitch,
+ CozeCredentialRef credential) throws Exception {
+ return submitWorkflow(rows, prompt, apiKey, imgSwitch, false, credential);
+ }
+
public CozePollResponse pollWorkflow(String executeId) throws Exception {
return pollWorkflow(executeId, null);
}
@@ -119,12 +136,12 @@ public class SimilarAsinCozeClient {
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
- private List inspectWithFallback(List rows, String prompt, String apiKey, boolean imgSwitch) {
+ private List inspectWithFallback(List rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) {
try {
if (rows.size() == 1) {
- return inspectSingleRowWithRetry(rows, prompt, apiKey, imgSwitch);
+ return inspectSingleRowWithRetry(rows, prompt, apiKey, imgSwitch, categorySwitch);
}
- InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch);
+ InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch, categorySwitch);
if (attempt.resolvedCount() < rows.size()) {
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
}
@@ -135,17 +152,17 @@ public class SimilarAsinCozeClient {
log.warn("[similar-asin] coze batch fallback split size={} left={} right={} err={}",
rows.size(), middle, rows.size() - middle, failureMessage(ex));
List merged = new ArrayList<>(rows.size());
- merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, imgSwitch));
- merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, imgSwitch));
+ merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, imgSwitch, categorySwitch));
+ merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, imgSwitch, categorySwitch));
return merged;
}
throw propagate(ex);
}
}
- private List inspectPartitionWithFailureFallback(List rows, String prompt, String apiKey, boolean imgSwitch) {
+ private List inspectPartitionWithFailureFallback(List rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) {
try {
- return inspectWithFallback(rows, prompt, apiKey, imgSwitch);
+ return inspectWithFallback(rows, prompt, apiKey, imgSwitch, categorySwitch);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[similar-asin] coze partition failed size={} err={}", rows.size(), failureMessage);
@@ -153,12 +170,12 @@ public class SimilarAsinCozeClient {
}
}
- private List inspectSingleRowWithRetry(List rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
+ private List inspectSingleRowWithRetry(List rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
SimilarAsinResultRowDto row = rows.getFirst();
PartialCozeResultException lastFailure = null;
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
try {
- InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch);
+ InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch, categorySwitch);
if (attempt.resolvedCount() == rows.size()) {
return attempt.mergedRows();
}
@@ -189,16 +206,16 @@ public class SimilarAsinCozeClient {
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
}
- private InspectAttempt inspectOnce(List rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
- String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, imgSwitch);
+ private InspectAttempt inspectOnce(List rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
+ String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, imgSwitch, categorySwitch);
List results = parseResults(raw);
List merged = mergeRows(rows, results);
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
}
- private String runWorkflowAsyncAndWait(List rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
+ private String runWorkflowAsyncAndWait(List rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
CozeCredentialRef credential = nextCredential();
- JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, credential));
+ JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, credential));
ensureSuccess(submitRoot);
String immediateData = extractResultDataText(submitRoot);
@@ -243,8 +260,9 @@ public class SimilarAsinCozeClient {
String prompt,
String apiKey,
boolean imgSwitch,
+ boolean categorySwitch,
CozeCredentialRef credential) {
- Map parameters = buildParameters(rows, prompt, apiKey, imgSwitch);
+ Map parameters = buildParameters(rows, prompt, apiKey, imgSwitch, categorySwitch);
Map body = new LinkedHashMap<>();
body.put("workflow_id", credential.workflowId());
body.put("parameters", parameters);
@@ -366,6 +384,14 @@ public class SimilarAsinCozeClient {
}
private Map buildParameters(List rows, String prompt, String apiKey, boolean imgSwitch) {
+ return buildParameters(rows, prompt, apiKey, imgSwitch, false);
+ }
+
+ private Map buildParameters(List rows,
+ String prompt,
+ String apiKey,
+ boolean imgSwitch,
+ boolean categorySwitch) {
List asins = rows.stream().map(row -> safeText(row.getAsin())).toList();
List titles = rows.stream().map(row -> safeText(firstNonBlank(row.getTitle(), row.getAsin()))).toList();
List skus = rows.stream().map(row -> safeText(row.getSku())).toList();
@@ -388,6 +414,7 @@ public class SimilarAsinCozeClient {
parameters.put("api_key", apiKey.trim());
}
parameters.put("img_switch", imgSwitch);
+ parameters.put("category_switch", categorySwitch);
return parameters;
}
@@ -827,7 +854,14 @@ public class SimilarAsinCozeClient {
|| !normalize(row.getPatentRisk()).isBlank()
|| !normalize(row.getConclusion()).isBlank()
|| !normalize(row.getIsStock()).isBlank()
- || !normalize(row.getSimilarity()).isBlank();
+ || !normalize(row.getSimilarity()).isBlank()
+ || !normalize(row.getIsConform()).isBlank()
+ || !normalize(row.getReason()).isBlank()
+ || !normalize(row.getCategory()).isBlank()
+ || !normalize(row.getStatus()).isBlank()
+ || !normalize(row.getMainUrl()).isBlank()
+ || !normalize(row.getPuzzleImg1()).isBlank()
+ || !normalize(row.getPuzzleImg2()).isBlank();
}
private void ensureSuccess(JsonNode root) {
@@ -978,7 +1012,21 @@ public class SimilarAsinCozeClient {
|| item.has("appearanceReason")
|| item.has("patent_reason")
|| item.has("patentReason")
- || item.has("patent reason"));
+ || item.has("patent reason")
+ || item.has("main_url")
+ || item.has("mainUrl")
+ || item.has("main_image_url")
+ || item.has("mainImageUrl")
+ || item.has("main_img")
+ || item.has("mainImg")
+ || item.has("puzzle_img1")
+ || item.has("puzzleImg1")
+ || item.has("puzzle_img_1")
+ || item.has("puzzleImg_1")
+ || item.has("puzzle_img2")
+ || item.has("puzzleImg2")
+ || item.has("puzzle_img_2")
+ || item.has("puzzleImg_2"));
}
private boolean isSuccessfulWorkflowStatus(String status) {
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParseRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParseRequest.java
index cc0c81ed..e3582921 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParseRequest.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParseRequest.java
@@ -37,4 +37,9 @@ public class SimilarAsinParseRequest {
@JsonAlias({"imgSwitch"})
@Schema(description = "传递给 Coze workflow parameters.img_switch 的图片检测开关，true 为开启，false 为关闭。")
private Boolean imgSwitch = Boolean.FALSE;
+
+ @JsonProperty("category_switch")
+ @JsonAlias({"categorySwitch"})
+ @Schema(description = "传递给 Coze workflow parameters.category_switch 的类目检测开关，true 为开启，false 为关闭。")
+ private Boolean categorySwitch = Boolean.FALSE;
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParsedPayloadDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParsedPayloadDto.java
index 8231d76e..31cae942 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParsedPayloadDto.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParsedPayloadDto.java
@@ -20,6 +20,9 @@ public class SimilarAsinParsedPayloadDto {
@Schema(description = "传递给 Coze workflow parameters.img_switch 的图片检测开关")
private Boolean imgSwitch = Boolean.FALSE;
+ @Schema(description = "传递给 Coze workflow parameters.category_switch 的类目检测开关")
+ private Boolean categorySwitch = Boolean.FALSE;
+
@Schema(description = "本次解析的源文件列表")
private List sourceFiles = new ArrayList<>();
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParseVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParseVo.java
index ccae28d2..7bbba903 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParseVo.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParseVo.java
@@ -36,6 +36,9 @@ public class SimilarAsinParseVo {
@Schema(description = "图片检测开关，true 为开启，false 为关闭")
private Boolean imgSwitch = Boolean.FALSE;
+ @Schema(description = "类目检测开关，true 为开启，false 为关闭")
+ private Boolean categorySwitch = Boolean.FALSE;
+
@Schema(description = "兼容旧前端的平铺有效行列表，现为全部有效行")
private List items = new ArrayList<>();
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
index c615b5bc..797afb59 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
@@ -83,6 +83,8 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
@@ -96,7 +98,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -219,6 +221,23 @@ public class SimilarAsinTaskService {
"阿里巴巴图片1",
"阿里巴巴图片2"
);
+ private static final Set CATEGORY_RESULT_HEADER_ALIASES = Set.of(
+ FailedStatusRowFilter.canonicalizeHeader("是否符合类目"),
+ FailedStatusRowFilter.canonicalizeHeader("is_conform"),
+ FailedStatusRowFilter.canonicalizeHeader("conform")
+ );
+ private static final Set COZE_RESULT_HEADER_ALIASES = Set.of(
+ FailedStatusRowFilter.canonicalizeHeader("是否有货"),
+ FailedStatusRowFilter.canonicalizeHeader("is_stock"),
+ FailedStatusRowFilter.canonicalizeHeader("相似度"),
+ FailedStatusRowFilter.canonicalizeHeader("similarity"),
+ FailedStatusRowFilter.canonicalizeHeader("是否符合类目"),
+ FailedStatusRowFilter.canonicalizeHeader("is_conform"),
+ FailedStatusRowFilter.canonicalizeHeader("不符合理由"),
+ FailedStatusRowFilter.canonicalizeHeader("reason"),
+ FailedStatusRowFilter.canonicalizeHeader("产品类目"),
+ FailedStatusRowFilter.canonicalizeHeader("category")
+ );
private static final int IMG_COL_MAIN = 12;
private static final int IMG_COL_PUZZLE1 = 13;
@@ -359,6 +378,7 @@ public class SimilarAsinTaskService {
List mergedHeaders = new ArrayList<>();
int totalRows = 0;
int droppedRows = 0;
+ boolean categoryRetryRequired = false;
long parseStartedAt = System.nanoTime();
for (SimilarAsinSourceFileDto source : sourceFiles) {
if (source.getFileKey() == null || source.getFileKey().isBlank()) {
@@ -374,10 +394,17 @@ public class SimilarAsinTaskService {
droppedRows += parsed.droppedRows();
allRows.addAll(parsed.allRows());
mergeHeaders(mergedHeaders, parsed.headers());
+ categoryRetryRequired |= parsed.categoryRetryRequired();
}
if (allRows.isEmpty()) {
throw new BusinessException("未解析到有效 ASIN 数据");
}
+ boolean requestedCategorySwitch = Boolean.TRUE.equals(request.getCategorySwitch());
+ request.setCategorySwitch(requestedCategorySwitch || categoryRetryRequired);
+ if (!requestedCategorySwitch && categoryRetryRequired) {
+ log.info("[similar-asin] category retry detected from failed result rows, enable category_switch automatically files={} rows={}",
+ sourceFiles.size(), allRows.size());
+ }
long parsedAt = System.nanoTime();
List groups = buildParsedGroups(allRows);
long groupedAt = System.nanoTime();
@@ -405,11 +432,11 @@ public class SimilarAsinTaskService {
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
- String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), sourceFiles, mergedHeaders, groups, allRows);
+ String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, mergedHeaders, groups, allRows);
long payloadBuiltAt = System.nanoTime();
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
long payloadStoredAt = System.nanoTime();
- task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), sourceFiles, parsedPayloadPointer));
+ task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, parsedPayloadPointer));
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
@@ -449,6 +476,7 @@ public class SimilarAsinTaskService {
vo.setGroupCount(groups.size());
vo.setAiPrompt(normalize(request.getAiPrompt()));
vo.setImgSwitch(Boolean.TRUE.equals(request.getImgSwitch()));
+ vo.setCategorySwitch(Boolean.TRUE.equals(request.getCategorySwitch()));
vo.setItems(new ArrayList<>(allRows));
vo.setGroups(groups);
long finishedAt = System.nanoTime();
@@ -1228,10 +1256,11 @@ public class SimilarAsinTaskService {
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
boolean imgSwitch = readImgSwitch(task);
+ boolean categorySwitch = readCategorySwitch(task);
int batchSize = resolveCozeBatchSize(imgSwitch);
List result = new ArrayList<>();
for (int i = 0; i < items.size(); i += batchSize) {
- result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, imgSwitch));
+ result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, imgSwitch, categorySwitch));
if (progressHook != null) {
progressHook.run();
}
@@ -1753,6 +1782,73 @@ public class SimilarAsinTaskService {
return true;
}
+ private boolean readCategorySwitch(FileTaskEntity task) {
+ try {
+ JsonNode root = objectMapper.readTree(task.getResultJson());
+ JsonNode savedSwitch = root == null ? null : root.get("categorySwitch");
+ if (savedSwitch != null && !savedSwitch.isNull()) {
+ return savedSwitch.asBoolean(false);
+ }
+ return Boolean.TRUE.equals(readParsedPayload(task).getCategorySwitch());
+ } catch (Exception ex) {
+ throw new BusinessException("读取相似ASIN类目检测开关失败", ex);
+ }
+ }
+
+ @Transactional
+ public void handleResultFileJobFailure(TaskFileJobEntity job, String message) {
+ if (job == null || job.getTaskId() == null) {
+ return;
+ }
+ try (TaskDistributedLockService.LockHandle ignored =
+ requireTaskLock(job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS)) {
+ Long finalizedTaskId = transactionManager == null
+ ? persistResultFileJobFailure(job, message)
+ : inNewTransaction(() -> persistResultFileJobFailure(job, message));
+ if (finalizedTaskId != null) {
+ taskCacheService.deleteTaskCache(finalizedTaskId);
+ clearPoisonWindow(finalizedTaskId);
+ }
+ }
+ }
+
+ private Long persistResultFileJobFailure(TaskFileJobEntity job, String message) {
+ FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
+ if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) {
+ return null;
+ }
+ String owner = ownerFromTask(task);
+ if (!isOwnerCurrent(owner)) {
+ log.warn("[similar-asin] finalize exhausted result job from another instance taskId={} jobId={} owner={} current={}",
+ task.getId(), job.getId(), owner, currentInstanceId());
+ }
+ String failureMessage = firstNonBlank(message, "相似ASIN结果文件生成失败");
+ LocalDateTime now = LocalDateTime.now();
+ task.setStatus(STATUS_FAILED);
+ task.setSuccessFileCount(0);
+ task.setFailedFileCount(1);
+ task.setErrorMessage(failureMessage);
+ task.setUpdatedAt(now);
+ task.setFinishedAt(now);
+ fileTaskMapper.updateById(task);
+
+ FileResultEntity result = job.getResultId() == null ? null : fileResultMapper.selectById(job.getResultId());
+ if (result != null && MODULE_TYPE.equals(result.getModuleType())) {
+ result.setSuccess(0);
+ result.setErrorMessage(failureMessage);
+ result.setResultFileUrl(null);
+ result.setResultFileSize(0L);
+ fileResultMapper.updateById(result);
+ }
+ saveFileBuildProgress(task, job, 1, 1, failureMessage);
+ return task.getId();
+ }
+
+ private void finalizeExhaustedResultFileJob(TaskFileJobEntity job, String message) {
+ handleResultFileJobFailure(job, message);
+ taskFileJobService.markFailureFinalized(job.getId(), message);
+ }
+
@Scheduled(fixedDelayString = "${aiimage.similar-asin.coze-poll-delay-ms:30000}")
public void pollPendingCozeJobs() {
DistributedJobLockService.LockHandle lockHandle =
@@ -1949,6 +2045,7 @@ public class SimilarAsinTaskService {
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
boolean imgSwitch = readImgSwitch(task);
+ boolean categorySwitch = readCategorySwitch(task);
int batchSize = resolveCozeBatchSize(imgSwitch);
// P1-1：检测到 720712008 风暴时强制把 batch 降到 1，隔离毒行；
// 持续 5+ 次提交命中率 ≥ 40% 才会触发，正常波动不影响吞吐。
@@ -2013,7 +2110,7 @@ public class SimilarAsinTaskService {
int batchIndex = 1;
for (int i = 0; i < submitLimit; i += batchSize) {
List batchCandidates = readyCandidates.subList(i, Math.min(i + batchSize, submitLimit));
- pending |= submitCozeBatch(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, allRowsByBaseId);
+ pending |= submitCozeBatch(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, categorySwitch, allRowsByBaseId);
batchIndex++;
}
return pending || countPendingCozeStates(task.getId()) > 0;
@@ -2129,6 +2226,7 @@ public class SimilarAsinTaskService {
String prompt,
String apiKey,
boolean imgSwitch,
+ boolean categorySwitch,
Map> allRowsByBaseId) {
if (batchCandidates == null || batchCandidates.isEmpty()) {
return false;
@@ -2155,7 +2253,7 @@ public class SimilarAsinTaskService {
SimilarAsinCozeClient.CozeCredentialRef credential = cozeClient.nextCredential();
try {
SimilarAsinCozeClient.CozeSubmitResponse submit = submitCozeWorkflowThrottled(
- batchRows, prompt, apiKey, imgSwitch, credential, true);
+ batchRows, prompt, apiKey, imgSwitch, categorySwitch, credential, true);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List cozeRows = cozeClient.mergeRowsFromDataText(batchRows, submit.immediateData());
String emptyResultMessage = emptyCozeResultMessage(cozeRows, batchRows.size());
@@ -2520,7 +2618,7 @@ public class SimilarAsinTaskService {
try {
SimilarAsinCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
- cozeClient.credentialByName(context.credentialName()), false);
+ readCategorySwitch(task), cozeClient.credentialByName(context.credentialName()), false);
Map> allRowsByBaseId = allRowsByBaseIdForPoll(task);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List cozeRows =
@@ -2582,6 +2680,7 @@ public class SimilarAsinTaskService {
String prompt,
String apiKey,
boolean imgSwitch,
+ boolean categorySwitch,
SimilarAsinCozeClient.CozeCredentialRef credential,
boolean allowCredentialFallback) throws Exception {
int attempts = allowCredentialFallback ? Math.max(1, cozeClient.configuredCredentialCount()) : 1;
@@ -2615,7 +2714,7 @@ public class SimilarAsinTaskService {
}
try (lockHandle; borrowedCredential) {
SimilarAsinCozeClient.CozeSubmitResponse response =
- cozeClient.submitWorkflow(rows, prompt, apiKey, imgSwitch, currentCredential);
+ cozeClient.submitWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, currentCredential);
// 提交完成立即记录时间戳，用于下次进入循环时计算节流等待。
lastCozeSubmitAtByCredential.put(credentialKey, System.currentTimeMillis());
return response;
@@ -2807,7 +2906,7 @@ public class SimilarAsinTaskService {
int partIndex = 1;
for (List partRows : partitions) {
SimilarAsinCozeClient.CozeSubmitResponse submit =
- submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
+ submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task), readCategorySwitch(task),
cozeClient.credentialByName(context.credentialName()), false);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List cozeRows =
@@ -2941,7 +3040,7 @@ public class SimilarAsinTaskService {
try {
taskFileJobService.touchRunningIfStale(context.jobId(), properties.getDbJobTouchIntervalMillis());
SimilarAsinCozeClient.CozeSubmitResponse submit =
- submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
+ submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task), readCategorySwitch(task),
cozeClient.credentialByName(context.credentialName()), false);
Map> allRowsByBaseId = allRowsByBaseIdForPoll(task);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
@@ -3280,6 +3379,9 @@ public class SimilarAsinTaskService {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId());
if (job != null) {
taskFileJobService.markFailed(job, firstNonBlank(ex.getMessage(), "相似ASIN结果文件生成失败"));
+ if (taskFileJobService.isRetryExhausted(job.getId())) {
+ finalizeExhaustedResultFileJob(job, ex.getMessage());
+ }
}
log.warn("[相似ASIN] Coze 异步收尾失败 任务ID={} 结果ID={} 错误={}",
taskId, context.resultId(), ex.getMessage(), ex);
@@ -3303,6 +3405,10 @@ public class SimilarAsinTaskService {
if (job == null || "SUCCESS".equals(job.getStatus())) {
return;
}
+ if (taskFileJobService.isRetryExhausted(job.getId())) {
+ finalizeExhaustedResultFileJob(job, job.getErrorMessage());
+ return;
+ }
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task != null && STATUS_RUNNING.equals(task.getStatus()) && !isResultSubmissionComplete(taskId)) {
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
@@ -3324,6 +3430,8 @@ public class SimilarAsinTaskService {
if (requeued) {
log.info("[相似ASIN] Coze 异步结果已就绪，结果文件任务已重新入队 任务ID={} 文件任务ID={} 结果ID={}",
taskId, job.getId(), context.resultId());
+ } else if (taskFileJobService.isRetryExhausted(job.getId())) {
+ finalizeExhaustedResultFileJob(job, job.getErrorMessage());
}
}
}
@@ -4160,6 +4268,7 @@ public class SimilarAsinTaskService {
}
List sourceRows = splitRowsBySourceFile(parsed, parsed.getAllItems(), result.getSourceFilename());
List workbooks = new ArrayList<>();
+ List workbookFiles = new ArrayList<>(sourceRows.size());
File zip = null;
// 跨 workbook 共享 taskImageCache：多源场景下相同 URL 仅下载一次。
// 配合 embed() 写完即 remove()，cache 仅承载 in-flight 图片；
@@ -4170,70 +4279,57 @@ public class SimilarAsinTaskService {
: BoundedImageCache.DEFAULT_MAX_BYTES;
BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes);
try {
- // P3-1：单源文件场景仍走串行降级路径，避免引入线程切换开销；
- // 多源文件场景把 writeResultWorkbook 投到 assembleExecutor 上并发跑，
- // 1000+ 行 ×N 源文件的 assemble 阶段总耗时直接除以 N（受池大小 4 限制）。
- // assembleExecutor 是固定 4 线程池：sourceRows.size() ≤ 4 时全部并行；
- // > 4 时多余源文件排队，避免 4×5000 行同时打开图片缓存爆堆。
- if (sourceRows.size() <= 1) {
- for (SourceRows item : sourceRows) {
- String filename = safeFileStem(item.sourceFilename()) + "-result.xlsx";
- String tempFilename = safeFileStem(item.sourceFilename())
- + "-" + task.getId()
- + "-" + result.getId()
- + "-" + UUID.randomUUID()
- + "-result.xlsx";
- File xlsx = new File(outputDir, tempFilename);
- writeResultWorkbook(xlsx, item.rows(), resultMap, taskImageCache);
- workbooks.add(new SourceResultWorkbook(xlsx, filename, item.rows().size()));
- }
- } else {
- long assembleStart = System.currentTimeMillis();
- List> futures = new ArrayList<>(sourceRows.size());
- for (SourceRows item : sourceRows) {
- final SourceRows captured = item;
- final String filename = safeFileStem(captured.sourceFilename()) + "-result.xlsx";
- final String tempFilename = safeFileStem(captured.sourceFilename())
- + "-" + task.getId()
- + "-" + result.getId()
- + "-" + UUID.randomUUID()
- + "-result.xlsx";
- futures.add(CompletableFuture.supplyAsync(() -> {
- File xlsx = new File(outputDir, tempFilename);
- writeResultWorkbook(xlsx, captured.rows(), resultMap, taskImageCache);
- return new SourceResultWorkbook(xlsx, filename, captured.rows().size());
- }, assembleExecutor));
- }
- for (CompletableFuture future : futures) {
- try {
- // 单源文件 30 分钟硬上限：超时直接抛错，避免被慢源永久阻塞。
- workbooks.add(future.get(30, TimeUnit.MINUTES));
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- for (CompletableFuture remaining : futures) {
- remaining.cancel(true);
- }
- throw new BusinessException("生成相似ASIN检测结果中断", ie);
- } catch (TimeoutException te) {
- for (CompletableFuture remaining : futures) {
- remaining.cancel(true);
- }
- throw new BusinessException("生成相似ASIN检测结果超时（30 分钟）", te);
- } catch (ExecutionException ee) {
- Throwable cause = ee.getCause();
- for (CompletableFuture remaining : futures) {
- remaining.cancel(true);
- }
- if (cause instanceof BusinessException be) {
- throw be;
- }
- throw new BusinessException("生成相似ASIN检测结果失败",
- cause != null ? cause : ee);
- }
- }
- log.info("[similar-asin] assemble parallel finished taskId={} sources={} costMs={}",
- task.getId(), sourceRows.size(), System.currentTimeMillis() - assembleStart);
+ // 单源和多源统一提交真实 Future，确保超时取消能中断实际 workbook 线程。
+ // 所有源文件共享同一个绝对 deadline，避免逐个 Future 各等待一轮完整超时。
+ int timeoutMinutes = Math.max(1, properties.getResultFileTimeoutMinutes());
+ long deadlineNanos = System.nanoTime() + TimeUnit.MINUTES.toNanos(timeoutMinutes);
+ long assembleStart = System.currentTimeMillis();
+ List> futures = new ArrayList<>(sourceRows.size());
+ for (SourceRows item : sourceRows) {
+ final SourceRows captured = item;
+ final String filename = safeFileStem(captured.sourceFilename()) + "-result.xlsx";
+ final String tempFilename = safeFileStem(captured.sourceFilename())
+ + "-" + task.getId()
+ + "-" + result.getId()
+ + "-" + UUID.randomUUID()
+ + "-result.xlsx";
+ final File xlsx = new File(outputDir, tempFilename);
+ workbookFiles.add(xlsx);
+ futures.add(assembleExecutor.submit(() -> {
+ ensureResultAssemblyNotInterrupted();
+ writeResultWorkbook(xlsx, captured.rows(), resultMap, taskImageCache);
+ ensureResultAssemblyNotInterrupted();
+ return new SourceResultWorkbook(xlsx, filename, captured.rows().size());
+ }));
}
+ try {
+ for (Future future : futures) {
+ long remainingNanos = deadlineNanos - System.nanoTime();
+ if (remainingNanos <= 0L) {
+ throw new TimeoutException("result file deadline reached");
+ }
+ workbooks.add(future.get(remainingNanos, TimeUnit.NANOSECONDS));
+ }
+ } catch (InterruptedException ex) {
+ cancelResultAssemblyFutures(futures);
+ Thread.currentThread().interrupt();
+ throw new BusinessException("生成相似ASIN检测结果中断", ex);
+ } catch (TimeoutException ex) {
+ cancelResultAssemblyFutures(futures);
+ throw new BusinessException("生成相似ASIN检测结果超时（" + timeoutMinutes + " 分钟）", ex);
+ } catch (CancellationException ex) {
+ cancelResultAssemblyFutures(futures);
+ throw new BusinessException("生成相似ASIN检测结果已取消", ex);
+ } catch (ExecutionException ex) {
+ cancelResultAssemblyFutures(futures);
+ Throwable cause = ex.getCause();
+ if (cause instanceof BusinessException businessException) {
+ throw businessException;
+ }
+ throw new BusinessException("生成相似ASIN检测结果失败", cause != null ? cause : ex);
+ }
+ log.info("[similar-asin] assemble workbooks finished taskId={} sources={} timeoutMinutes={} costMs={}",
+ task.getId(), sourceRows.size(), timeoutMinutes, System.currentTimeMillis() - assembleStart);
log.info("[similar-asin] assemble image-cache stats taskId={} sources={} cacheSize={} currentBytes={} evictedCount={} evictedBytes={} maxBytes={}",
task.getId(), sourceRows.size(), taskImageCache.size(),
taskImageCache.currentBytes(), taskImageCache.evictedCount(),
@@ -4272,9 +4368,9 @@ public class SimilarAsinTaskService {
result.setResultContentType(contentType);
result.setRowCount(parsed.getAllItems().size());
} finally {
- for (SourceResultWorkbook workbook : workbooks) {
- if (workbook.file().exists() && !workbook.file().delete()) {
- log.warn("[similar-asin] delete temp xlsx failed file={}", workbook.file());
+ for (File workbookFile : workbookFiles) {
+ if (workbookFile.exists() && !workbookFile.delete()) {
+ log.warn("[similar-asin] delete temp xlsx failed file={}", workbookFile);
}
}
if (zip != null && zip.exists() && !zip.delete()) {
@@ -4482,10 +4578,19 @@ public class SimilarAsinTaskService {
List rowsToWrite,
Map resultMap,
Map taskImageCache) {
+ ensureResultAssemblyNotInterrupted();
// Excel 365 "Place in Cell" 图片：写完 workbook 后 patch richData 单元格图片结构。
// 这不是浮动 Drawing，因此点击图片区域会选中单元格，图片不能被拖到任意位置，也不需要工作表保护。
ExcelCellImageWriter.Session excelCellImageSession = ExcelCellImageWriter.createSession();
- try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
+ SimilarAsinImageEmbedder.ImageSpool imageSpool;
+ try {
+ Path spoolDir = Files.createTempDirectory(xlsx.getParentFile().toPath(), "similar-asin-images-");
+ imageSpool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
+ } catch (IOException ex) {
+ throw new BusinessException("create Similar ASIN image spool failed", ex);
+ }
+ try {
+ try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
Sheet sheet = workbook.createSheet("相似asin检测");
CellStyle headerStyle = workbook.createCellStyle();
Font font = workbook.createFont();
@@ -4506,17 +4611,16 @@ public class SimilarAsinTaskService {
cell.setCellStyle(headerStyle);
}
- // P2-8：第一遍预扫所有图片 URL，并行下载到 taskImageCache，
- // POI 写入仍单线程串行注册 cell image（cache 命中直接 resize + register），下载/写入解耦。
+ // 预扫图片 URL 并行下载到任务级临时目录；POI 仍按行串行注册 cell image。
List prefetchUrls = collectImageUrlsForPrefetch(rowsToWrite, resultMap);
if (!prefetchUrls.isEmpty()) {
- // P2-11：先一次性把 DB cache 命中的字节填入 taskImageCache，避免再次走网络。
- // 命中部分从 prefetchUrls 剔除，剩余的真正未命中的 URL 才走 imageEmbedder.prefetch 网络下载。
+ // DB cache 命中的缩略图也立即落到任务临时目录，避免整批字节停留在堆中。
long dbCacheStart = System.currentTimeMillis();
List remainingUrls = new ArrayList<>(prefetchUrls.size());
int dbHit = 0;
for (String url : prefetchUrls) {
- if (url == null || taskImageCache.containsKey(url)) {
+ ensureResultAssemblyNotInterrupted();
+ if (url == null || imageSpool.get(url) != null || taskImageCache.containsKey(url)) {
continue;
}
byte[] cachedBytes = imagePrefetchService.lookup(url);
@@ -4526,7 +4630,7 @@ public class SimilarAsinTaskService {
}
SimilarAsinImageEmbedder.ResizedImage thumb = imageEmbedder.decodeCachedThumb(cachedBytes);
if (thumb != null) {
- taskImageCache.putIfAbsent(url, thumb);
+ imageSpool.put(url, thumb);
dbHit++;
} else {
remainingUrls.add(url);
@@ -4536,14 +4640,16 @@ public class SimilarAsinTaskService {
prefetchUrls.size(), dbHit, remainingUrls.size(), System.currentTimeMillis() - dbCacheStart);
if (!remainingUrls.isEmpty()) {
long prefetchStart = System.currentTimeMillis();
- imageEmbedder.prefetch(remainingUrls, taskImageCache);
- log.info("[similar-asin] image prefetch finished urls={} cached={} costMs={}",
- remainingUrls.size(), taskImageCache.size(), System.currentTimeMillis() - prefetchStart);
+ imageEmbedder.prefetchToDisk(remainingUrls, imageSpool);
+ ensureResultAssemblyNotInterrupted();
+ log.info("[similar-asin] image disk-prefetch finished urls={} spooled={} costMs={}",
+ remainingUrls.size(), imageSpool.size(), System.currentTimeMillis() - prefetchStart);
}
}
int rowIndex = 1;
for (SimilarAsinParsedRowVo parsedRow : rowsToWrite == null ? List.of() : rowsToWrite) {
+ ensureResultAssemblyNotInterrupted();
SimilarAsinResultRowDto resultRow = findResultRow(parsedRow, resultMap);
Row row = sheet.createRow(rowIndex);
int col = 0;
@@ -4575,21 +4681,21 @@ public class SimilarAsinTaskService {
// 行高固定到 IMAGE_ROW_HEIGHT_POINTS（Excel 上限 409pt）：
row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS);
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_MAIN,
- resultRow.getMainUrl(), row, taskImageCache, excelCellImageSession);
+ resultRow.getMainUrl(), row, taskImageCache, excelCellImageSession, imageSpool);
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE1,
- resultRow.getPuzzleImg1(), row, taskImageCache, excelCellImageSession);
+ resultRow.getPuzzleImg1(), row, taskImageCache, excelCellImageSession, imageSpool);
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE2,
- resultRow.getPuzzleImg2(), row, taskImageCache, excelCellImageSession);
+ resultRow.getPuzzleImg2(), row, taskImageCache, excelCellImageSession, imageSpool);
}
rowIndex++;
}
+ ensureResultAssemblyNotInterrupted();
workbook.write(fos);
workbook.dispose();
- } catch (Exception ex) {
- throw new BusinessException("生成相似ASIN检测结果失败", ex);
}
if (!excelCellImageSession.isEmpty()) {
try {
+ ensureResultAssemblyNotInterrupted();
ExcelCellImageWriter.patchXlsxFile(xlsx, excelCellImageSession);
} catch (IOException ex) {
String cause = ex.toString();
@@ -4598,6 +4704,32 @@ public class SimilarAsinTaskService {
throw new BusinessException("写入相似ASIN结果 Excel 单元格图片失败: " + cause, ex);
}
}
+ } catch (BusinessException ex) {
+ throw ex;
+ } catch (Exception ex) {
+ throw new BusinessException("生成相似ASIN检测结果失败", ex);
+ } finally {
+ try {
+ imageSpool.close();
+ } catch (IOException ex) {
+ log.warn("[similar-asin] delete image spool failed xlsx={} err={}",
+ xlsx.getAbsolutePath(), ex.getMessage());
+ }
+ }
+ }
+
+ private static void cancelResultAssemblyFutures(List> futures) {
+ for (Future future : futures) {
+ if (future != null && !future.isDone()) {
+ future.cancel(true);
+ }
+ }
+ }
+
+ private static void ensureResultAssemblyNotInterrupted() {
+ if (Thread.currentThread().isInterrupted()) {
+ throw new BusinessException("生成相似ASIN检测结果中断");
+ }
}
/**
@@ -4714,24 +4846,29 @@ public class SimilarAsinTaskService {
vo.setValues(readRowValues(row, headers, formatter));
allRows.add(vo);
}
- boolean includeBlankStatusRows = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
+ boolean resultWorkbook = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
+ boolean firstPassResultWorkbook = isFirstPassResultWorkbook(headers, allRows);
+ boolean includeBlankStatusRows = resultWorkbook && !firstPassResultWorkbook;
FailedStatusRowFilter.FilterResult filteredRows =
FailedStatusRowFilter.retainRows(
allRows,
statusCol >= 0,
rowVo -> statusHeader == null || rowVo.getValues() == null ? "" : rowVo.getValues().get(statusHeader),
status -> FailedStatusRowFilter.matchesFailedStatus(status)
+ || firstPassResultWorkbook
|| (includeBlankStatusRows && FailedStatusRowFilter.isBlankStatus(status))
);
dropped += filteredRows.filteredCount();
allRows = new ArrayList<>(filteredRows.rows());
+ boolean categoryRetryRequired = shouldEnableCategorySwitchForRetry(
+ headers, allRows, includeBlankStatusRows);
if (statusCol >= 0 && validRows > 0 && allRows.isEmpty()) {
throw new BusinessException(FailedStatusRowFilter.noMatchedRowsMessage());
}
if (allRows.isEmpty()) {
throw new BusinessException("no valid similar ASIN rows");
}
- return new ParsedWorkbook(total, dropped, headers, allRows);
+ return new ParsedWorkbook(total, dropped, headers, allRows, categoryRetryRequired);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
@@ -4763,7 +4900,10 @@ public class SimilarAsinTaskService {
|| hasUsableCozeField(row.getIsConform())
|| hasUsableCozeField(row.getReason())
|| hasUsableCozeField(row.getCategory())
- || hasUsableCozeField(row.getStatus());
+ || hasUsableCozeField(row.getStatus())
+ || hasText(row.getMainUrl())
+ || hasText(row.getPuzzleImg1())
+ || hasText(row.getPuzzleImg2());
}
static String resolveResultStatus(SimilarAsinResultRowDto row, String... visibleResultValues) {
@@ -4885,7 +5025,16 @@ public class SimilarAsinTaskService {
}
private String cell(Row row, int col, DataFormatter formatter) {
- return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col)));
+ if (col < 0) {
+ return "";
+ }
+ String value = normalize(formatter.formatCellValue(row.getCell(col)));
+ return isSpreadsheetErrorValue(value) ? "" : value;
+ }
+
+ private static boolean isSpreadsheetErrorValue(String value) {
+ String normalized = value == null ? "" : value.trim();
+ return normalized.startsWith("#") && normalized.endsWith("!");
}
private String baseId(String id) {
@@ -5373,11 +5522,12 @@ public class SimilarAsinTaskService {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
- private String buildParsedPayloadJson(String aiPrompt, String apiKey, Boolean imgSwitch, List sourceFiles, List headers, List groups, List allRows) {
+ private String buildParsedPayloadJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, List sourceFiles, List headers, List groups, List allRows) {
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
payload.setAiPrompt(normalize(aiPrompt));
payload.setApiKey(normalize(apiKey));
payload.setImgSwitch(Boolean.TRUE.equals(imgSwitch));
+ payload.setCategorySwitch(Boolean.TRUE.equals(categorySwitch));
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
payload.setHeaders(headers == null ? List.of() : headers);
payload.setItems(allRows == null ? List.of() : new ArrayList<>(allRows));
@@ -5386,11 +5536,12 @@ public class SimilarAsinTaskService {
return writeJson(payload, "保存解析结果失败");
}
- private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, List sourceFiles, String parsedPayloadPointer) {
+ private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, List sourceFiles, String parsedPayloadPointer) {
Map payload = new LinkedHashMap<>();
payload.put("aiPrompt", normalize(aiPrompt));
payload.put("apiKey", normalize(apiKey));
payload.put("imgSwitch", Boolean.TRUE.equals(imgSwitch));
+ payload.put("categorySwitch", Boolean.TRUE.equals(categorySwitch));
payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream()
.map(SimilarAsinSourceFileDto::getFileKey)
.filter(Objects::nonNull)
@@ -5746,6 +5897,83 @@ public class SimilarAsinTaskService {
|| hasHeader(headers, "产品类目"));
}
+ static boolean isFirstPassResultWorkbook(List headers,
+ List rows) {
+ if (headers == null || headers.isEmpty() || rows == null || rows.isEmpty()
+ || FailedStatusRowFilter.findStatusColumnIndex(headers) < 0
+ || (!hasCanonicalHeader(headers, "是否有货")
+ && !hasCanonicalHeader(headers, "相似度")
+ && !hasCanonicalHeader(headers, "是否符合类目")
+ && !hasCanonicalHeader(headers, "不符合理由")
+ && !hasCanonicalHeader(headers, "产品类目"))
+ || (!hasCanonicalHeader(headers, "主图")
+ && !hasCanonicalHeader(headers, "阿里巴巴图片1")
+ && !hasCanonicalHeader(headers, "阿里巴巴图片2"))) {
+ return false;
+ }
+ String statusHeader = headers.get(FailedStatusRowFilter.findStatusColumnIndex(headers));
+ for (SimilarAsinParsedRowVo row : rows) {
+ if (row == null || FailedStatusRowFilter.matchesFailedStatus(valueByHeader(row, statusHeader))) {
+ return false;
+ }
+ for (Map.Entry entry : row.getValues() == null
+ ? Map.of().entrySet() : row.getValues().entrySet()) {
+ if (COZE_RESULT_HEADER_ALIASES.contains(FailedStatusRowFilter.canonicalizeHeader(entry.getKey()))
+ && !isBlankOrSpreadsheetError(entry.getValue())) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ private static boolean hasCanonicalHeader(List headers, String expected) {
+ String canonical = FailedStatusRowFilter.canonicalizeHeader(expected);
+ return headers.stream()
+ .filter(Objects::nonNull)
+ .anyMatch(header -> canonical.equals(FailedStatusRowFilter.canonicalizeHeader(header)));
+ }
+
+ private static String valueByHeader(SimilarAsinParsedRowVo row, String header) {
+ if (row == null || row.getValues() == null) {
+ return "";
+ }
+ String canonical = FailedStatusRowFilter.canonicalizeHeader(header);
+ return row.getValues().entrySet().stream()
+ .filter(entry -> canonical.equals(FailedStatusRowFilter.canonicalizeHeader(entry.getKey())))
+ .map(Map.Entry::getValue)
+ .findFirst()
+ .orElse("");
+ }
+
+ private static boolean isBlankOrSpreadsheetError(String value) {
+ return value == null || value.trim().isBlank() || isSpreadsheetErrorValue(value);
+ }
+
+ static boolean shouldEnableCategorySwitchForRetry(List headers,
+ List rows,
+ boolean resultWorkbook) {
+ if (!resultWorkbook || headers == null || headers.isEmpty() || rows == null || rows.isEmpty()
+ || FailedStatusRowFilter.findStatusColumnIndex(headers) < 0) {
+ return false;
+ }
+ String categoryHeader = headers.stream()
+ .filter(Objects::nonNull)
+ .filter(header -> CATEGORY_RESULT_HEADER_ALIASES.contains(
+ FailedStatusRowFilter.canonicalizeHeader(header)))
+ .findFirst()
+ .orElse(null);
+ if (categoryHeader == null) {
+ return false;
+ }
+ return rows.stream()
+ .filter(Objects::nonNull)
+ .map(SimilarAsinParsedRowVo::getValues)
+ .anyMatch(values -> values == null
+ || values.get(categoryHeader) == null
+ || values.get(categoryHeader).trim().isBlank());
+ }
+
private boolean hasHeader(List headers, String candidate) {
if (headers == null || headers.isEmpty()) {
return false;
@@ -5926,6 +6154,10 @@ public class SimilarAsinTaskService {
long size) {
}
- private record ParsedWorkbook(int totalRows, int droppedRows, List headers, List allRows) {
+ private record ParsedWorkbook(int totalRows,
+ int droppedRows,
+ List headers,
+ List allRows,
+ boolean categoryRetryRequired) {
}
}
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/ExcelCellImageWriter.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/ExcelCellImageWriter.java
index da765dfb..e2d1acc2 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/ExcelCellImageWriter.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/ExcelCellImageWriter.java
@@ -13,8 +13,10 @@ import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Enumeration;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -41,6 +43,9 @@ public final class ExcelCellImageWriter {
private static final String RD_RICH_VALUE_TYPES_PART = "xl/richData/rdRichValueTypes.xml";
private static final String RICH_VALUE_REL_PART = "xl/richData/richValueRel.xml";
private static final String METADATA_PART = "xl/metadata.xml";
+ private static final Pattern SHEET_CELL_PATTERN = Pattern.compile(
+ "]*\\br=\"([A-Z]+[1-9][0-9]*)\")[^>]*(?:/>|>.*?)",
+ Pattern.DOTALL);
private static final String REL_TYPE_IMAGE =
"http://schemas.openxmlformats.org/officeDocument/2006/relationships/image";
@@ -116,7 +121,11 @@ public final class ExcelCellImageWriter {
writeEntry(zout, METADATA_PART, buildMetadata(session).getBytes(StandardCharsets.UTF_8));
int idx = 1;
for (RegisteredCellImage image : session.images) {
- writeEntry(zout, mediaPart(idx), image.bytes);
+ if (image.bytes != null) {
+ writeEntry(zout, mediaPart(idx), image.bytes);
+ } else {
+ writeEntry(zout, mediaPart(idx), image.path);
+ }
idx++;
}
}
@@ -166,26 +175,34 @@ public final class ExcelCellImageWriter {
private static PatchSheetResult patchSheet(byte[] original, Session session) {
String content = new String(original, StandardCharsets.UTF_8);
- int idx = 0;
- int patchedCount = 0;
- for (RegisteredCellImage image : session.images) {
- // [MS-XLSX] §2.2.10: cell @vm 是 1-based valueMetadata 索引，0 表示"无 metadata"。
- // futureMetadata/valueMetadata/rdrichvalue 内部块仍是 0-based，所以这里写 idx+1
- // 与 buildMetadata/buildRdRichValue 中的 i 解耦。Excel 365 在 vm=0 时会把 cell
- // 直接当作无图片的 #VALUE! 错误格渲染（任务 10498 J2 复现），必须改 1-based。
- String replacement = "#VALUE!";
- Pattern pattern = Pattern.compile("]*\\br=\"" + Pattern.quote(image.cellRef)
- + "\")[^>]*(?:/>|>.*?)", Pattern.DOTALL);
- Matcher matcher = pattern.matcher(content);
- if (matcher.find()) {
- content = matcher.replaceFirst(Matcher.quoteReplacement(replacement));
- patchedCount++;
- } else {
- log.warn("[excel-cell-image] cell {} not found in sheet xml, skip image", image.cellRef);
- }
- idx++;
+ Map metadataIndexByCell = new HashMap<>(session.images.size());
+ for (int idx = 0; idx < session.images.size(); idx++) {
+ metadataIndexByCell.put(session.images.get(idx).cellRef, idx + 1);
}
- return new PatchSheetResult(content.getBytes(StandardCharsets.UTF_8), patchedCount);
+
+ Set missingCells = new HashSet<>(metadataIndexByCell.keySet());
+ Matcher matcher = SHEET_CELL_PATTERN.matcher(content);
+ StringBuilder patched = new StringBuilder(content.length());
+ int patchedCount = 0;
+ while (matcher.find()) {
+ String cellRef = matcher.group(1);
+ Integer metadataIndex = metadataIndexByCell.get(cellRef);
+ if (metadataIndex == null) {
+ matcher.appendReplacement(patched, Matcher.quoteReplacement(matcher.group()));
+ continue;
+ }
+ // [MS-XLSX] §2.2.10: cell @vm is a 1-based valueMetadata index.
+ String replacement = "#VALUE!";
+ matcher.appendReplacement(patched, Matcher.quoteReplacement(replacement));
+ missingCells.remove(cellRef);
+ patchedCount++;
+ }
+ matcher.appendTail(patched);
+ for (String cellRef : missingCells) {
+ log.warn("[excel-cell-image] cell {} not found in sheet xml, skip image", cellRef);
+ }
+ return new PatchSheetResult(patched.toString().getBytes(StandardCharsets.UTF_8), patchedCount);
}
private static byte[] patchContentTypes(byte[] original) {
@@ -386,6 +403,16 @@ public final class ExcelCellImageWriter {
zout.closeEntry();
}
+ private static void writeEntry(ZipOutputStream zout, String name, Path path) throws IOException {
+ ZipEntry entry = new ZipEntry(name);
+ zout.putNextEntry(entry);
+ try (InputStream in = Files.newInputStream(path)) {
+ in.transferTo(zout);
+ } finally {
+ zout.closeEntry();
+ }
+ }
+
private static void writeDirectoryEntry(ZipOutputStream zout, String name) throws IOException {
ZipEntry entry = new ZipEntry(name);
zout.putNextEntry(entry);
@@ -399,7 +426,15 @@ public final class ExcelCellImageWriter {
if (jpegBytes == null || jpegBytes.length == 0) {
return;
}
- images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), jpegBytes));
+ images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), jpegBytes, null));
+ }
+
+ public synchronized void registerImage(int rowIdx, int colIdx, Path jpegPath) {
+ if (jpegPath == null || !Files.isRegularFile(jpegPath)) {
+ return;
+ }
+ images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), null,
+ jpegPath.toAbsolutePath().normalize()));
}
public synchronized boolean isEmpty() {
@@ -426,7 +461,7 @@ public final class ExcelCellImageWriter {
}
}
- private record RegisteredCellImage(String cellRef, byte[] bytes) {
+ private record RegisteredCellImage(String cellRef, byte[] bytes, Path path) {
}
private record PatchSheetResult(byte[] bytes, int patchedCount) {
diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java
index c3d8047c..131bffc3 100644
--- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java
+++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java
@@ -27,14 +27,22 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InterruptedIOException;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.FileTime;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
@@ -42,6 +50,8 @@ import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
@@ -49,7 +59,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
/**
@@ -68,8 +77,9 @@ public class SimilarAsinImageEmbedder {
static final int DEFAULT_DOWNLOAD_TIMEOUT_SECONDS = 5;
/** P2-10：retry 由 1 升到 2，配合 5s timeout 单图最坏耗时 ≈ 15s。 */
static final int DOWNLOAD_MAX_RETRY = 2;
- /** P2-10：原 8 → P2-8 16 → P2-10 32，1000+ 行 ×3 列场景下显著拉低 assemble 阶段尾延迟。 */
- static final int DEFAULT_DOWNLOAD_POOL_SIZE = 32;
+ /** 图片下载、解码、缩放都在该池执行；默认限制为 8，避免结果生成占满整机 CPU。 */
+ static final int DEFAULT_DOWNLOAD_POOL_SIZE = 8;
+ static final int DEFAULT_PREFETCH_TIMEOUT_SECONDS = 1800;
// 单元格固定尺寸，图片在其中等比缩放（不拉伸）；resize 仍按长边 1280 px 控制堆体积。
public static final float IMAGE_ROW_HEIGHT_POINTS = 409f;
public static final int IMAGE_COL_WIDTH_CHARS = 80;
@@ -102,16 +112,28 @@ public class SimilarAsinImageEmbedder {
private final int downloadTimeoutSeconds;
private final int downloadPoolSize;
+ private final int prefetchTimeoutSeconds;
private final OkHttpClient httpClient;
private final ExecutorService downloadPool;
private final OssStorageService ossStorageService;
+ private final Path localImageCacheDir;
public SimilarAsinImageEmbedder(SimilarAsinProperties properties, OssStorageService ossStorageService) {
int rawTimeout = properties == null ? DEFAULT_DOWNLOAD_TIMEOUT_SECONDS : properties.getImageDownloadTimeoutSeconds();
int rawPool = properties == null ? DEFAULT_DOWNLOAD_POOL_SIZE : properties.getImageDownloadPoolSize();
+ int rawPrefetchTimeout = properties == null
+ ? DEFAULT_PREFETCH_TIMEOUT_SECONDS
+ : properties.getImagePrefetchTimeoutSeconds();
this.downloadTimeoutSeconds = rawTimeout > 0 ? rawTimeout : DEFAULT_DOWNLOAD_TIMEOUT_SECONDS;
this.downloadPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE;
+ this.prefetchTimeoutSeconds = rawPrefetchTimeout > 0
+ ? rawPrefetchTimeout
+ : DEFAULT_PREFETCH_TIMEOUT_SECONDS;
this.ossStorageService = Objects.requireNonNull(ossStorageService, "ossStorageService must not be null");
+ String configuredCacheDir = properties == null ? null : properties.getImageLocalCacheDir();
+ this.localImageCacheDir = configuredCacheDir == null || configuredCacheDir.isBlank()
+ ? null
+ : Path.of(configuredCacheDir.trim()).toAbsolutePath().normalize();
this.httpClient = new OkHttpClient.Builder()
.connectTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
.readTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
@@ -138,6 +160,10 @@ public class SimilarAsinImageEmbedder {
return downloadPoolSize;
}
+ int prefetchTimeoutSeconds() {
+ return prefetchTimeoutSeconds;
+ }
+
OkHttpClient httpClient() {
return httpClient;
}
@@ -159,13 +185,13 @@ public class SimilarAsinImageEmbedder {
if (urls == null || urls.isEmpty() || taskImageCache == null) {
return;
}
- Set distinctUrls = new HashSet<>();
+ Set distinctUrls = new LinkedHashSet<>();
for (String url : urls) {
if (url == null) {
continue;
}
String trimmed = url.trim();
- if (trimmed.isEmpty() || taskImageCache.containsKey(trimmed)) {
+ if (trimmed.isEmpty() || taskImageCache.containsKey(trimmed) || hasLocalCachedThumb(trimmed)) {
continue;
}
distinctUrls.add(trimmed);
@@ -174,58 +200,169 @@ public class SimilarAsinImageEmbedder {
return;
}
CompletionService