新需求更新 同步更新

This commit is contained in:
supernijia
2026-08-06 01:11:54 +08:00
parent 9048bbb7f8
commit 28e7fce11c
112 changed files with 7739 additions and 637 deletions
+6
View File
@@ -26,6 +26,7 @@
<minio.version>8.5.17</minio.version>
<javassist.version>3.28.0-GA</javassist.version>
<jjwt.version>0.12.6</jjwt.version>
<bouncycastle.version>1.78.1</bouncycastle.version>
</properties>
<dependencyManagement>
@@ -99,6 +100,11 @@
<artifactId>minio</artifactId>
<version>${minio.version}</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk18on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-api</artifactId>
@@ -2,13 +2,11 @@ package com.nanri.aiimage;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
import java.time.ZoneId;
import java.util.TimeZone;
@SpringBootApplication
@EnableScheduling
public class AiImageApplication {
private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai");
@@ -16,7 +16,7 @@ public class AppearancePatentProperties {
private String cozeToken = "";
private List<CozeCredential> cozeCredentials = new ArrayList<>();
private int cozeCredentialStripeSize = 5;
private int cozeBatchSize = 50;
private int cozeBatchSize = 10;
private int cozeConnectTimeoutMillis = 10000;
private int cozeReadTimeoutMillis = 60000;
private int cozePollIntervalMillis = 30000;
@@ -12,5 +12,7 @@ public class ModuleCleanupProperties {
private boolean enabled = true;
private String cron = "0 0 0 * * *";
private long retentionDays = 7;
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "SHOP_DATA_CRAWL", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
// SHOP_DATA_CRAWL is governed by per-shop latest-three retention in its
// task service and must not be removed by the age-based sweep.
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
}
@@ -2,9 +2,11 @@ package com.nanri.aiimage.config;
import io.swagger.v3.oas.models.ExternalDocumentation;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.Components;
import io.swagger.v3.oas.models.info.Contact;
import io.swagger.v3.oas.models.info.Info;
import io.swagger.v3.oas.models.info.License;
import io.swagger.v3.oas.models.security.SecurityScheme;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -39,6 +41,11 @@ public class OpenApiConfig {
.version("v0.0.1")
.contact(new Contact().name("Nanri AI"))
.license(new License().name("Internal Use")))
.components(new Components().addSecuritySchemes("bearerAuth", new SecurityScheme()
.type(SecurityScheme.Type.HTTP)
.scheme("bearer")
.bearerFormat("JWT")
.description("管理员登录 JWT")))
.externalDocs(new ExternalDocumentation()
.description("Knife4j 文档")
.url("/doc.html"));
@@ -12,6 +12,7 @@ public class OssProperties {
private String bucket;
private String imageVideoBucket;
private String digitalHumanBucket;
private String templateBucket;
private String accessKeyId;
private String accessKeySecret;
}
@@ -1,8 +1,10 @@
package com.nanri.aiimage.config;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
@@ -10,6 +12,8 @@ import java.time.Clock;
import java.time.ZoneId;
@Configuration
@EnableScheduling
@ConditionalOnProperty(prefix = "aiimage.scheduling", name = "enabled", havingValue = "true", matchIfMissing = true)
@Slf4j
public class SchedulingConfig {
@@ -61,13 +61,8 @@ public class SimilarAsinProperties {
*/
private int cozeSubmitMaxRetryCount = 5;
/**
* 图片嵌入下载线程池大小。原 SimilarAsinImageEmbedder.DOWNLOAD_POOL_SIZE = 8。
* P2-101000+ 行 ×3 列图片场景下,pool=16 仍是 assemble 阶段瓶颈(实测下载 244s/918s),
* 提到 32 配合 retry=2、global deadline 显著拉低尾延迟;
* 受 2GB 堆约束,单图缩略图维持 300KB 以内,整体内存峰值 ≈ 32 * 300KB ≈ 10MB。
*/
private int imageDownloadPoolSize = 32;
/** 图片下载、解码和缩放共享该池;默认 8,避免批量结果生成占满整机 CPU。 */
private int imageDownloadPoolSize = 8;
/**
* 单张图片下载超时(秒)。
@@ -77,6 +72,12 @@ public class SimilarAsinProperties {
*/
private int imageDownloadTimeoutSeconds = 5;
/** 单个结果文件整批图片预取预算,耗尽后缺图单元格降级为 URL。 */
private int imagePrefetchTimeoutSeconds = 1800;
/** 多源结果文件组装的单任务硬上限;运行期间由文件任务 heartbeat 保活。 */
private int resultFileTimeoutMinutes = 90;
/**
* assemble 阶段 taskImageCache 的字节上限。
* 默认 256MB5000 行 × 3 列 × 平均 100KB = 1.5GB 远超 2GB 堆,
@@ -85,6 +86,7 @@ public class SimilarAsinProperties {
* 出现淘汰过频影响命中率时可上调到 512MB;2GB 堆约束下不建议超过 768MB。
*/
private long imageCacheMaxBytes = 256L * 1024L * 1024L;
private String imageLocalCacheDir = "";
private boolean imageDbCacheEnabled = false;
/**
@@ -1338,10 +1338,9 @@ public class AppearancePatentTaskService {
}
}
}
boolean failedByConclusion = hasInfringingPersistedConclusion(task.getId());
boolean failed = finalError != null && !finalError.isBlank() || failedByConclusion;
boolean failed = finalError != null && !finalError.isBlank();
boolean waitingForAssemble = !failed && assembleWorkbook && !shouldAssembleSynchronously();
task.setStatus(waitingForAssemble ? STATUS_RUNNING : (failed ? STATUS_FAILED : STATUS_SUCCESS));
task.setStatus(resolveTaskExecutionStatus(waitingForAssemble, failed));
task.setSuccessFileCount(failed ? 0 : 1);
task.setFailedFileCount(failed ? 1 : 0);
task.setErrorMessage(finalError);
@@ -2418,10 +2417,9 @@ public class AppearancePatentTaskService {
saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits - 1, "正在上传结果文件");
fileResultMapper.updateById(result);
boolean taskAlreadyFailed = STATUS_FAILED.equals(task.getStatus())
|| (task.getErrorMessage() != null && !task.getErrorMessage().isBlank())
|| hasInfringingPersistedConclusion(task.getId());
|| (task.getErrorMessage() != null && !task.getErrorMessage().isBlank());
String existingError = task.getErrorMessage();
task.setStatus(taskAlreadyFailed ? STATUS_FAILED : STATUS_SUCCESS);
task.setStatus(resolveTaskExecutionStatus(false, taskAlreadyFailed));
task.setSuccessFileCount(taskAlreadyFailed ? 0 : 1);
task.setFailedFileCount(taskAlreadyFailed ? 1 : 0);
task.setErrorMessage(taskAlreadyFailed ? existingError : null);
@@ -4199,33 +4197,11 @@ public class AppearancePatentTaskService {
return resultRow == null || resultRow.getPrice() == null ? "" : resultRow.getPrice().trim();
}
private boolean hasInfringingPersistedConclusion(Long taskId) {
if (taskId == null) {
return false;
static String resolveTaskExecutionStatus(boolean waitingForAssemble, boolean executionFailed) {
if (waitingForAssemble) {
return STATUS_RUNNING;
}
return loadPersistedResultRows(taskId).values().stream()
.map(AppearancePatentResultRowDto::getConclusion)
.anyMatch(this::isInfringingConclusion);
}
private boolean isInfringingConclusion(String value) {
String normalized = normalize(value);
return normalized.contains("侵权")
&& !isNoInfringementConclusion(normalized);
}
private boolean isNoInfringementConclusion(String value) {
String normalized = normalize(value);
if (normalized.isBlank()) {
return false;
}
return normalized.contains("无侵权")
|| normalized.contains("未侵权")
|| normalized.contains("不侵权")
|| normalized.contains("未发现") && normalized.contains("侵权")
|| normalized.contains("未见") && normalized.contains("侵权")
|| normalized.contains("不存在") && normalized.contains("侵权")
|| normalized.contains("无明显") && normalized.contains("侵权");
return executionFailed ? STATUS_FAILED : STATUS_SUCCESS;
}
private String userFacingCozeCellValue(AppearancePatentResultRowDto row, String value) {
@@ -1,11 +1,13 @@
package com.nanri.aiimage.modules.auth.util;
import lombok.extern.slf4j.Slf4j;
import org.bouncycastle.crypto.generators.SCrypt;
import org.springframework.stereotype.Component;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.SecureRandom;
import java.util.HexFormat;
@@ -51,21 +53,38 @@ public class WerkzeugPasswordEncoder {
String method = storedHash.substring(0, firstSep);
String salt = storedHash.substring(firstSep + 1, secondSep);
String expectedHex = storedHash.substring(secondSep + 1);
if (!method.startsWith("pbkdf2:sha256")) {
log.warn("[auth] unsupported password hash scheme: {}", method);
return false;
if (method.startsWith("scrypt:")) {
return matchesScrypt(rawPassword, method, salt, expectedHex);
}
if (method.startsWith("pbkdf2:sha256")) {
String[] parts = method.split(":");
int iterations = parts.length >= 3 ? Integer.parseInt(parts[2]) : 600_000;
byte[] derived = pbkdf2Sha256(rawPassword.toCharArray(), salt.getBytes(StandardCharsets.UTF_8), iterations, DK_BITS);
String actualHex = HexFormat.of().formatHex(derived);
return constantTimeEquals(actualHex, expectedHex);
}
log.warn("[auth] unsupported password hash scheme: {}", method);
return false;
} catch (Exception e) {
log.warn("[auth] verify password failed: {}", e.getMessage());
return false;
}
}
private static boolean matchesScrypt(String rawPassword, String method, String salt, String expectedHex) {
String[] parts = method.split(":");
if (parts.length != 4 || expectedHex.isEmpty() || (expectedHex.length() & 1) != 0) {
return false;
}
int cost = Integer.parseInt(parts[1]);
int blockSize = Integer.parseInt(parts[2]);
int parallelization = Integer.parseInt(parts[3]);
byte[] expected = HexFormat.of().parseHex(expectedHex);
byte[] actual = SCrypt.generate(rawPassword.getBytes(StandardCharsets.UTF_8),
salt.getBytes(StandardCharsets.UTF_8), cost, blockSize, parallelization, expected.length);
return MessageDigest.isEqual(actual, expected);
}
private static byte[] pbkdf2Sha256(char[] password, byte[] salt, int iterations, int keyBits) throws Exception {
PBEKeySpec spec = new PBEKeySpec(password, salt, iterations, keyBits);
try {
@@ -53,6 +53,19 @@ public class CollectDataController {
return ApiResponse.success(null);
}
@PostMapping("/tasks/{taskId}/fail")
@Operation(summary = "标记采集任务失败", description = "桌面端入队失败或无法继续执行时,将任务收敛为 FAILED。")
public ApiResponse<Void> fail(
@Parameter(description = "采集任务 ID", required = true, example = "9001")
@PathVariable Long taskId,
@Parameter(description = "当前用户 ID", required = true, example = "1")
@RequestParam("user_id") Long userId,
@Parameter(description = "失败原因")
@RequestParam(value = "error", required = false) String error) {
service.failTask(taskId, userId, error);
return ApiResponse.success(null);
}
@GetMapping("/tasks/{taskId}/items")
@Operation(summary = "分页获取任务明细数据", description = "供 Python 端拉取,默认每页 50 条;返回任务关联的筛选条件,便于 Python 端按筛选条件采集。")
public ApiResponse<CollectDataItemsPageVo> items(
@@ -51,4 +51,7 @@ public class CollectDataSummaryRowDto {
@JsonAlias({"page", "totalPage", "页数", "max_page", "maxPage"})
@Schema(description = "该关键词总页数(Python 端取所有命中行的最大页码)", example = "10")
private Integer totalPage;
@Schema(description = "Python 端 ASIN 过滤数量", example = "3")
private Integer asinFilter;
}
@@ -49,6 +49,36 @@ public class CollectDataHistoryItemVo {
@Schema(description = "最终结果行数")
private Integer finalRowCount;
@Schema(description = "解析任务总行数")
private Integer totalRows;
@Schema(description = "Python 已回传行数")
private Integer receivedRows;
@Schema(description = "Python 已处理关键词数")
private Integer processedRows;
@Schema(description = "当前采集阶段:search / detail")
private String collectStage;
@Schema(description = "当前关键词")
private String currentKeyword;
@Schema(description = "搜索页当前页码")
private Integer searchCurrentPage;
@Schema(description = "搜索页总页数")
private Integer searchTotalPages;
@Schema(description = "详情页已处理 ASIN 数")
private Integer detailProcessedAsins;
@Schema(description = "详情页 ASIN 总数")
private Integer detailTotalAsins;
@Schema(description = "任务进度百分比,0-100")
private Integer progressPercent;
@Schema(description = "记录创建时间")
private String createdAt;
@@ -24,7 +24,7 @@ public class CollectDataExcelAssemblyService {
private static final String[] SHEET_DETAIL_HEADER = {"品牌", "ASIN", "价格", "卖家名称", "关键词", "配送方式"};
private static final String SHEET_SUMMARY_NAME = "结果文件";
private static final String[] SHEET_SUMMARY_HEADER = {"关键词", "FBA", "FBM", "AMZ", "无配送方式", "页数"};
private static final String[] SHEET_SUMMARY_HEADER = {"关键词", "FBA", "FBM", "AMZ", "无配送方式", "页数", "ASIN过滤"};
/**
* 生成采集结果工作簿。
@@ -115,6 +115,7 @@ public class CollectDataExcelAssemblyService {
} else {
row.createCell(5).setCellValue("");
}
row.createCell(6).setCellValue(intOrZero(item.getAsinFilter()));
}
return;
}
@@ -43,6 +43,7 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
@@ -60,6 +61,8 @@ import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionTemplate;
@@ -105,6 +108,7 @@ public class CollectDataService {
private static final int BRAND_CHECK_BATCH_SIZE = 10;
private static final long TASK_LOCK_WAIT_MILLIS = 5000L;
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String STALE_TASK_ERROR = "长时间未收到 Python 心跳,任务已自动失败";
private static final List<String> KEYWORD_HEADER_ALIASES = List.of("关键词", "keyword", "key word");
private static final List<String> STATUS_HEADER_ALIASES = List.of("状态", "status");
@@ -131,6 +135,9 @@ public class CollectDataService {
private final ObjectMapper objectMapper;
private final TransactionTemplate transactionTemplate;
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
private long staleTimeoutMinutes;
public CollectDataParseVo parseAndCreateTask(CollectDataParseRequest request) {
long startedAt = System.currentTimeMillis();
if (request == null || request.getUserId() == null || request.getUserId() <= 0) {
@@ -216,7 +223,18 @@ public class CollectDataService {
requestPayload.put("filters", filters);
requestPayload.put("files", sources);
task.setRequestJson(objectMapper.writeValueAsString(requestPayload));
task.setResultJson("{}");
Map<String, Object> initialStats = new LinkedHashMap<>();
initialStats.put("totalRows", parsedRows.size());
initialStats.put("receivedRows", 0);
initialStats.put("processedRows", 0);
initialStats.put("currentChunkRows", 0);
initialStats.put("dedupeFilteredCount", 0);
initialStats.put("invalidFilteredCount", 0);
initialStats.put("brandRejectedCount", 0);
initialStats.put("brandQueryFailedCount", 0);
initialStats.put("finalRowCount", 0);
initialStats.put("summaries", List.of());
task.setResultJson(objectMapper.writeValueAsString(initialStats));
} catch (Exception ex) {
throw new BusinessException("序列化任务信息失败");
}
@@ -274,9 +292,138 @@ public class CollectDataService {
fileTaskMapper.updateById(task);
}
@Transactional
public void failTask(Long taskId, Long userId, String error) {
FileTaskEntity task = requireTask(taskId, userId);
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
return;
}
String message = firstNonBlank(error, "collect-data task dispatch failed");
FileResultEntity result = ensureTaskResult(task);
CollectDataStats stats = loadStats(task);
result.setSuccess(0);
result.setErrorMessage(message);
result.setRowCount(stats.finalRowCount);
fileResultMapper.updateById(result);
task.setStatus(STATUS_FAILED);
task.setErrorMessage(message);
task.setFailedFileCount(1);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
persistStats(task, stats);
fileTaskMapper.updateById(task);
}
@Transactional
public void updateProgress(Long taskId, TaskHeartbeatRequest request) {
if (taskId == null || taskId <= 0 || request == null) {
return;
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
return;
}
CollectDataStats stats = loadStats(task);
boolean changed = false;
Integer current = request.getCurrent();
Integer total = request.getTotal();
if (current != null && total != null && total > 0) {
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
stats.totalRows = totalRows;
stats.processedRows = processedRows;
changed = true;
}
}
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
stats.collectStage = request.getCollectStage();
changed = true;
}
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
stats.currentKeyword = request.getCurrentKeyword();
changed = true;
}
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
changed = true;
}
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
changed = true;
}
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
changed = true;
}
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
changed = true;
}
if (!changed) {
return;
}
persistStats(task, stats);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
}
}
@Scheduled(cron = "${aiimage.collect-data.stale-check-cron:*/30 * * * * *}")
public void finalizeStaleTasks() {
long timeoutMinutes = Math.max(1L, staleTimeoutMinutes);
LocalDateTime threshold = LocalDateTime.now().minusMinutes(timeoutMinutes);
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.lt(FileTaskEntity::getUpdatedAt, threshold)
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit 200"));
for (FileTaskEntity task : tasks) {
finalizeStaleTask(task.getId(), threshold, timeoutMinutes);
}
}
private void finalizeStaleTask(Long taskId, LocalDateTime threshold, long timeoutMinutes) {
try (TaskDistributedLockService.LockHandle lock =
taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) {
if (lock == null) {
return;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null
|| !MODULE_TYPE.equals(task.getModuleType())
|| !STATUS_RUNNING.equals(task.getStatus())
|| task.getUpdatedAt() == null
|| !task.getUpdatedAt().isBefore(threshold)) {
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
return;
}
FileResultEntity result = ensureTaskResult(task);
CollectDataStats stats = loadStats(task);
if (hasReceivedChunks(taskId)) {
enqueueFinalWorkbook(task, result, stats);
log.warn("[collect-data] stale task enqueued partial workbook taskId={} timeoutMinutes={} finalRows={}",
taskId, timeoutMinutes, stats.finalRowCount);
return;
}
markTaskFailed(task, result, STALE_TASK_ERROR, stats);
log.warn("[collect-data] stale task failed without result chunks taskId={} timeoutMinutes={}",
taskId, timeoutMinutes);
} catch (Exception ex) {
log.warn("[collect-data] stale task finalization failed taskId={} msg={}",
taskId, ex.getMessage(), ex);
}
}
public CollectDataDashboardVo dashboard(Long userId) {
CollectDataDashboardVo vo = new CollectDataDashboardVo();
vo.setPendingTaskCount(countTask(userId, STATUS_RUNNING));
vo.setPendingTaskCount(countActiveTasks(userId));
vo.setSuccessTaskCount(countTask(userId, STATUS_SUCCESS));
vo.setFailedTaskCount(countTask(userId, STATUS_FAILED));
long processed = (vo.getSuccessTaskCount() == null ? 0 : vo.getSuccessTaskCount())
@@ -967,6 +1114,13 @@ public class CollectDataService {
return count == null ? 0 : count.intValue();
}
private boolean hasReceivedChunks(Long taskId) {
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
return count != null && count > 0L;
}
private CollectDataSubmitResultVo buildSubmitVo(FileTaskEntity task,
FileResultEntity result,
int chunkIndex,
@@ -999,7 +1153,15 @@ public class CollectDataService {
}
try {
JsonNode root = objectMapper.readTree(task.getResultJson());
stats.totalRows = root.path("totalRows").asInt(0);
stats.receivedRows = root.path("receivedRows").asInt(0);
stats.processedRows = root.path("processedRows").asInt(0);
stats.collectStage = root.path("collectStage").asText(null);
stats.currentKeyword = root.path("currentKeyword").asText(null);
stats.searchCurrentPage = root.path("searchCurrentPage").asInt(0);
stats.searchTotalPages = root.path("searchTotalPages").asInt(0);
stats.detailProcessedAsins = root.path("detailProcessedAsins").asInt(0);
stats.detailTotalAsins = root.path("detailTotalAsins").asInt(0);
stats.currentChunkRows = root.path("currentChunkRows").asInt(0);
stats.dedupeFilteredCount = root.path("dedupeFilteredCount").asInt(0);
stats.invalidFilteredCount = root.path("invalidFilteredCount").asInt(0);
@@ -1028,7 +1190,15 @@ public class CollectDataService {
}
try {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("totalRows", stats.totalRows);
payload.put("receivedRows", stats.receivedRows);
payload.put("processedRows", stats.processedRows);
payload.put("collectStage", stats.collectStage);
payload.put("currentKeyword", stats.currentKeyword);
payload.put("searchCurrentPage", stats.searchCurrentPage);
payload.put("searchTotalPages", stats.searchTotalPages);
payload.put("detailProcessedAsins", stats.detailProcessedAsins);
payload.put("detailTotalAsins", stats.detailTotalAsins);
payload.put("currentChunkRows", stats.currentChunkRows);
payload.put("dedupeFilteredCount", stats.dedupeFilteredCount);
payload.put("invalidFilteredCount", stats.invalidFilteredCount);
@@ -1142,7 +1312,15 @@ public class CollectDataService {
}
private static class CollectDataStats {
private int totalRows;
private int receivedRows;
private int processedRows;
private String collectStage;
private String currentKeyword;
private int searchCurrentPage;
private int searchTotalPages;
private int detailProcessedAsins;
private int detailTotalAsins;
private int currentChunkRows;
private int dedupeFilteredCount;
private int invalidFilteredCount;
@@ -1316,6 +1494,17 @@ public class CollectDataService {
return count == null ? 0L : count;
}
private long countActiveTasks(Long userId) {
if (userId == null || userId <= 0) {
return 0L;
}
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getUserId, userId)
.in(FileTaskEntity::getStatus, List.of(STATUS_PENDING, STATUS_RUNNING)));
return count == null ? 0L : count;
}
private Map<Long, FileTaskEntity> loadTaskMap(List<Long> taskIds) {
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
@@ -1356,10 +1545,31 @@ public class CollectDataService {
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
vo.setBrandRejectedCount(stats.brandRejectedCount);
vo.setFinalRowCount(stats.finalRowCount);
vo.setTotalRows(stats.totalRows);
vo.setReceivedRows(stats.receivedRows);
vo.setProcessedRows(stats.processedRows);
vo.setCollectStage(stats.collectStage);
vo.setCurrentKeyword(stats.currentKeyword);
vo.setSearchCurrentPage(stats.searchCurrentPage);
vo.setSearchTotalPages(stats.searchTotalPages);
vo.setDetailProcessedAsins(stats.detailProcessedAsins);
vo.setDetailTotalAsins(stats.detailTotalAsins);
vo.setProgressPercent(calculateProgressPercent(task.getStatus(), stats.totalRows, stats.processedRows));
}
return vo;
}
private int calculateProgressPercent(String status, int totalRows, int receivedRows) {
if (STATUS_SUCCESS.equals(status)) {
return 100;
}
if (totalRows <= 0) {
return 0;
}
int received = Math.max(0, Math.min(receivedRows, totalRows));
return Math.min(99, received * 100 / totalRows);
}
private String resolveDisplayResultFilename(FileResultEntity row, FileTaskEntity task) {
if (row == null) {
return null;
@@ -57,8 +57,13 @@ public class DedupeTotalDataController {
@Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize,
@Parameter(description = "数据值模糊搜索关键字") @RequestParam(required = false) String keyword,
@Parameter(description = "用户名模糊搜索关键字") @RequestParam(required = false) String username,
@Parameter(description = "开始日期(包含)")
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
@Parameter(description = "结束日期(包含)")
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
@Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) {
return ApiResponse.success(dedupeTotalDataService.page(page, pageSize, keyword, username, operatorId));
return ApiResponse.success(dedupeTotalDataService.page(
page, pageSize, keyword, username, startDate, endDate, operatorId));
}
@GetMapping("/export")
@@ -74,7 +74,11 @@ public class DedupeTotalDataService {
return template;
}
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username, Long operatorId) {
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username,
LocalDate startDate, LocalDate endDate, Long operatorId) {
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
throw new BusinessException("开始日期不能晚于结束日期");
}
long safePage = Math.max(page, 1);
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
String safeKeyword = keyword == null ? "" : keyword.trim();
@@ -84,6 +88,10 @@ public class DedupeTotalDataService {
.like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
.in(!scope.allUsers(), DedupeTotalDataEntity::getUploaderUserId, scope.userIds())
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
.ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
startDate == null ? null : startDate.atStartOfDay())
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
endDate == null ? null : endDate.plusDays(1).atStartOfDay())
.orderByDesc(DedupeTotalDataEntity::getId);
Long total = dedupeTotalDataMapper.selectCount(query);
List<DedupeTotalDataItemVo> items = dedupeTotalDataMapper.selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize))
@@ -25,7 +25,9 @@ public class LocalTempCleanupService {
"convert-result",
"split-result",
"brand-source-download",
"brand-result"
"brand-result",
"similar-asin-result",
"similar-asin-image-cache"
);
private final StorageProperties storageProperties;
@@ -1,7 +1,9 @@
package com.nanri.aiimage.modules.file.service.oss;
import com.nanri.aiimage.config.OssProperties;
import io.minio.BucketExistsArgs;
import io.minio.GetObjectArgs;
import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
import io.minio.PutObjectArgs;
import io.minio.RemoveObjectArgs;
@@ -93,6 +95,77 @@ public class OssStorageService {
}
}
public void ensureBucketExists(String bucket) {
String normalizedBucket = requireStorageName(bucket, "bucket");
try {
boolean exists = buildClient().bucketExists(BucketExistsArgs.builder()
.bucket(normalizedBucket)
.build());
if (exists) {
return;
}
try {
buildClient().makeBucket(MakeBucketArgs.builder()
.bucket(normalizedBucket)
.build());
} catch (ErrorResponseException ex) {
if (!isBucketAlreadyPresent(ex)) {
throw ex;
}
}
} catch (Exception ex) {
throw storageFailure("ensure bucket", normalizedBucket, ex);
}
}
public void uploadBytes(String bucket, String objectKey, byte[] content, String contentType) {
String normalizedBucket = requireStorageName(bucket, "bucket");
String normalizedObjectKey = requireStorageName(objectKey, "objectKey");
byte[] bytes = content == null ? new byte[0] : content;
try (ByteArrayInputStream stream = new ByteArrayInputStream(bytes)) {
buildClient().putObject(PutObjectArgs.builder()
.bucket(normalizedBucket)
.object(normalizedObjectKey)
.stream(stream, bytes.length, -1)
.contentType(firstNonBlank(contentType, "application/octet-stream"))
.build());
} catch (Exception ex) {
throw storageFailure("upload", normalizedBucket + "/" + normalizedObjectKey, ex);
}
}
public byte[] readObjectBytes(String bucket, String objectKey) {
String normalizedBucket = requireStorageName(bucket, "bucket");
String normalizedObjectKey = requireStorageName(objectKey, "objectKey");
try (var stream = buildClient().getObject(GetObjectArgs.builder()
.bucket(normalizedBucket)
.object(normalizedObjectKey)
.build())) {
return stream.readAllBytes();
} catch (Exception ex) {
throw storageFailure("read", normalizedBucket + "/" + normalizedObjectKey, ex);
}
}
public boolean objectExists(String bucket, String objectKey) {
String normalizedBucket = requireStorageName(bucket, "bucket");
String normalizedObjectKey = requireStorageName(objectKey, "objectKey");
try {
buildClient().statObject(StatObjectArgs.builder()
.bucket(normalizedBucket)
.object(normalizedObjectKey)
.build());
return true;
} catch (ErrorResponseException ex) {
if (isNotFound(ex)) {
return false;
}
throw storageFailure("stat", normalizedBucket + "/" + normalizedObjectKey, ex);
} catch (Exception ex) {
throw storageFailure("stat", normalizedBucket + "/" + normalizedObjectKey, ex);
}
}
public String uploadTaskScopePayload(String moduleType, Long taskId, String scopeHash, String content) {
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
String objectKey = String.format("task-scope/%s/%s/%s.json", normalizeModuleType(moduleType), taskId, normalizedScopeHash);
@@ -310,7 +383,7 @@ public class OssStorageService {
}
private List<String> configuredBuckets() {
return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket())
return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket(), templateBucket())
.filter(Objects::nonNull)
.map(String::trim)
.filter(bucket -> !bucket.isBlank())
@@ -395,6 +468,10 @@ public class OssStorageService {
return firstNonBlank(ossProperties.getDigitalHumanBucket(), ossProperties.getBucket());
}
private String templateBucket() {
return firstNonBlank(ossProperties.getTemplateBucket(), ossProperties.getBucket());
}
private String publicEndpoint() {
return withScheme(firstNonBlank(ossProperties.getPublicEndpoint(), ossProperties.getEndpoint()));
}
@@ -446,6 +523,11 @@ public class OssStorageService {
return "NoSuchKey".equals(code) || "NoSuchObject".equals(code) || "NoSuchBucket".equals(code);
}
private boolean isBucketAlreadyPresent(ErrorResponseException ex) {
String code = ex.errorResponse() == null ? null : ex.errorResponse().code();
return "BucketAlreadyOwnedByYou".equals(code) || "BucketAlreadyExists".equals(code);
}
private IllegalStateException storageFailure(String operation, String objectKey, Exception cause) {
return new IllegalStateException("failed to " + operation + " object in MinIO: " + objectKey, cause);
}
@@ -477,6 +559,13 @@ public class OssStorageService {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
private String requireStorageName(String value, String field) {
if (value == null || value.isBlank()) {
throw new IllegalArgumentException(field + " must not be blank");
}
return value.trim();
}
private record StorageLocation(String bucket, String objectKey) {
}
@@ -0,0 +1,50 @@
package com.nanri.aiimage.modules.filetemplate;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.filetemplate.ModuleTemplateService.TemplateDownload;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/module-templates")
@Tag(name = "模块输入模板", description = "下载各业务模块的固定 Excel 输入模板")
public class ModuleTemplateController {
private final ModuleTemplateService moduleTemplateService;
@GetMapping("/{moduleCode}/download")
@Operation(summary = "下载模块输入模板")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(
responseCode = "200",
description = "Excel 模板文件",
content = @Content(
mediaType = ModuleTemplateService.XLSX_CONTENT_TYPE,
schema = @Schema(type = "string", format = "binary"))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "模块模板不存在"),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "503", description = "模板存储暂不可用")
})
public ResponseEntity<byte[]> download(
@Parameter(description = "模块编码", required = true, example = "publish")
@PathVariable String moduleCode) {
TemplateDownload download = moduleTemplateService.download(moduleCode);
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(download.filename()))
.contentType(MediaType.parseMediaType(download.contentType()))
.contentLength(download.content().length)
.body(download.content());
}
}
@@ -0,0 +1,57 @@
package com.nanri.aiimage.modules.filetemplate;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
@Component
public class ModuleTemplateRegistry {
private static final String RESOURCE_PREFIX = "templates/module-input/";
private static final String OBJECT_PREFIX = "input/";
private static final List<ModuleTemplate> TEMPLATES = List.of(
template("publish", "publish.xlsx", "上架 文档格式.xlsx"),
template("delete-brand", "delete-brand.xlsx", "删除指定待售 文档格式.xlsx"),
template("appearance-patent", "appearance-patent.xlsx", "外观专利检测 文档格式.xlsx"),
template("price-track", "price-track.xlsx", "指定ASIN跟价 文档格式.xlsx"),
template("collect-data", "collect-data.xlsx", "数据采集 文档格式.xlsx"),
template("similar-asin", "similar-asin.xlsx", "货源查询 文档格式.xlsx")
);
private final Map<String, ModuleTemplate> templatesByCode = TEMPLATES.stream()
.collect(Collectors.toUnmodifiableMap(
ModuleTemplate::moduleCode,
Function.identity()));
public Optional<ModuleTemplate> find(String moduleCode) {
if (moduleCode == null || moduleCode.isBlank()) {
return Optional.empty();
}
return Optional.ofNullable(templatesByCode.get(moduleCode.trim().toLowerCase(Locale.ROOT)));
}
public List<ModuleTemplate> templates() {
return TEMPLATES;
}
private static ModuleTemplate template(String moduleCode, String resourceFilename, String downloadFilename) {
return new ModuleTemplate(
moduleCode,
RESOURCE_PREFIX + resourceFilename,
OBJECT_PREFIX + resourceFilename,
downloadFilename);
}
public record ModuleTemplate(
String moduleCode,
String resourcePath,
String objectKey,
String downloadFilename) {
}
}
@@ -0,0 +1,89 @@
package com.nanri.aiimage.modules.filetemplate;
import com.nanri.aiimage.config.OssProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.filetemplate.ModuleTemplateRegistry.ModuleTemplate;
import lombok.RequiredArgsConstructor;
import org.springframework.core.io.ClassPathResource;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream;
@Service
@RequiredArgsConstructor
public class ModuleTemplateService {
public static final String XLSX_CONTENT_TYPE =
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private final ModuleTemplateRegistry registry;
private final OssProperties ossProperties;
private final OssStorageService ossStorageService;
public TemplateDownload download(String moduleCode) {
ModuleTemplate template = registry.find(moduleCode)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "模板不存在"));
try {
ensureStored(template);
byte[] content = ossStorageService.readObjectBytes(templateBucket(), template.objectKey());
return new TemplateDownload(template.downloadFilename(), XLSX_CONTENT_TYPE, content);
} catch (ResponseStatusException ex) {
throw ex;
} catch (Exception ex) {
throw storageUnavailable(ex);
}
}
public int synchronizeAll() {
try {
String bucket = templateBucket();
ossStorageService.ensureBucketExists(bucket);
int uploaded = 0;
for (ModuleTemplate template : registry.templates()) {
if (!ossStorageService.objectExists(bucket, template.objectKey())) {
uploadResource(bucket, template);
uploaded++;
}
}
return uploaded;
} catch (ResponseStatusException ex) {
throw ex;
} catch (Exception ex) {
throw storageUnavailable(ex);
}
}
private void ensureStored(ModuleTemplate template) {
String bucket = templateBucket();
ossStorageService.ensureBucketExists(bucket);
if (!ossStorageService.objectExists(bucket, template.objectKey())) {
uploadResource(bucket, template);
}
}
private void uploadResource(String bucket, ModuleTemplate template) {
ClassPathResource resource = new ClassPathResource(template.resourcePath());
try (InputStream input = resource.getInputStream()) {
ossStorageService.uploadBytes(bucket, template.objectKey(), input.readAllBytes(), XLSX_CONTENT_TYPE);
} catch (Exception ex) {
throw new IllegalStateException("failed to load module template resource: " + template.resourcePath(), ex);
}
}
private String templateBucket() {
String bucket = ossProperties.getTemplateBucket();
if (bucket == null || bucket.isBlank()) {
throw new IllegalStateException("template bucket is not configured");
}
return bucket.trim();
}
private ResponseStatusException storageUnavailable(Exception cause) {
return new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "模板存储暂不可用", cause);
}
public record TemplateDownload(String filename, String contentType, byte[] content) {
}
}
@@ -0,0 +1,26 @@
package com.nanri.aiimage.modules.filetemplate;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
@Component
@RequiredArgsConstructor
@Slf4j
public class ModuleTemplateStorageInitializer {
private final ModuleTemplateService moduleTemplateService;
@EventListener(ApplicationReadyEvent.class)
public void synchronize() {
try {
int uploaded = moduleTemplateService.synchronizeAll();
log.info("[module-template] startup synchronization complete uploaded={}", uploaded);
} catch (Exception ex) {
log.warn("[module-template] startup synchronization failed; downloads will retry lazily: {}",
ex.getMessage(), ex);
}
}
}
@@ -128,6 +128,23 @@ public class PermissionMenuController {
return ApiResponse.success("视频任务权限已更新", grantedCount);
}
@GetMapping("/shop-data-crawl-task-permissions")
@Operation(summary = "查询店铺数据任务数据权限用户")
public ApiResponse<List<ImageVideoDataPermissionUserVo>> listShopDataCrawlDataPermissionUsers(
HttpServletRequest request) {
return ApiResponse.success(permissionMenuService.listShopDataCrawlDataPermissionUsers(requireAdmin(request)));
}
@PutMapping("/shop-data-crawl-task-permissions")
@Operation(summary = "更新店铺数据任务数据权限用户")
public ApiResponse<Integer> updateShopDataCrawlDataPermissionUsers(
HttpServletRequest request,
@RequestBody(required = false) ImageVideoDataPermissionUpdateRequest body) {
int grantedCount = permissionMenuService.updateShopDataCrawlDataPermissionUsers(
requireAdmin(request), body == null ? List.of() : body.getUserIds());
return ApiResponse.success("店铺数据任务权限已更新", grantedCount);
}
private AdminUserEntity requireAdmin(HttpServletRequest request) {
try {
return adminAuthSupport.requireAdmin(request);
@@ -86,6 +86,7 @@ public class PermissionMenuSchemaInitializer {
executeQuietly("UPDATE columns SET sort_order = id WHERE sort_order IS NULL OR sort_order = 0");
executeQuietly("ALTER TABLE columns ADD UNIQUE KEY uk_menu_type_route_path (menu_type, route_path)");
ensureDefaultAdminMenus();
boolean shopDataCrawlDataPermissionCreated = ensureShopDataCrawlAdminDefaults();
ensureDefaultAppMenus();
ensureDefaultAppChildMenus();
ensureInternalDataPermissions();
@@ -98,6 +99,9 @@ public class PermissionMenuSchemaInitializer {
FOREIGN KEY (column_id) REFERENCES columns(id) ON DELETE CASCADE
)
""");
if (shopDataCrawlDataPermissionCreated) {
grantInitialShopDataCrawlDataPermissions();
}
}
private void ensureDefaultAdminMenus() {
@@ -172,6 +176,38 @@ public class PermissionMenuSchemaInitializer {
""");
}
private boolean ensureShopDataCrawlAdminDefaults() {
executeQuietly("""
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
SELECT '店铺数据任务管理', 'admin_shop_data_crawl_tasks', 'admin', 'shop-data-crawl-tasks', 82
WHERE NOT EXISTS (
SELECT 1 FROM columns WHERE column_key = 'admin_shop_data_crawl_tasks'
)
""");
return executeUpdateQuietly("""
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
SELECT '店铺数据任务数据查看', 'admin_shop_data_crawl_task_data', 'internal', 'shop-data-crawl-task-data', 0
WHERE NOT EXISTS (
SELECT 1 FROM columns WHERE column_key = 'admin_shop_data_crawl_task_data'
)
""") > 0;
}
private void grantInitialShopDataCrawlDataPermissions() {
executeQuietly("""
INSERT IGNORE INTO user_column_permission (user_id, column_id)
SELECT old_perm.user_id, data_col.id
FROM user_column_permission old_perm
INNER JOIN columns old_col ON old_col.id = old_perm.column_id
INNER JOIN columns data_col ON data_col.column_key = 'admin_shop_data_crawl_task_data'
LEFT JOIN user_column_permission existing
ON existing.user_id = old_perm.user_id
AND existing.column_id = data_col.id
WHERE old_col.column_key = 'admin_shop_data_crawl_tasks'
AND existing.user_id IS NULL
""");
}
private void executeQuietly(String sql) {
try {
jdbcTemplate.execute(sql);
@@ -180,6 +216,15 @@ public class PermissionMenuSchemaInitializer {
}
}
private int executeUpdateQuietly(String sql) {
try {
return jdbcTemplate.update(sql);
} catch (Exception ex) {
log.debug("[permission-menu] schema init skipped: {}", ex.getMessage());
return 0;
}
}
private record DefaultAdminMenu(String name, String columnKey, String routePath, int sortOrder) {
}
@@ -47,6 +47,7 @@ public class PermissionMenuService {
public static final String MENU_TYPE_APP = "app";
public static final String MENU_TYPE_ADMIN = "admin";
private static final String IMAGE_VIDEO_DATA_PERMISSION_KEY = "admin_image_video_task_data";
private static final String SHOP_DATA_CRAWL_DATA_PERMISSION_KEY = "admin_shop_data_crawl_task_data";
private final PermissionMenuMapper permissionMenuMapper;
private final UserColumnPermissionMapper userColumnPermissionMapper;
@@ -243,9 +244,8 @@ public class PermissionMenuService {
.filter(user -> user.getId() != null && !isSuperAdmin(user))
.collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left));
if (requestedIds.stream().anyMatch(id -> !grantableUsers.containsKey(id))) {
throw new BusinessException("包含不存在或不可授权的用户");
throw new BusinessException("contains unknown or non-grantable user");
}
userColumnPermissionMapper.deleteByMap(Map.of("column_id", dataPermission.getId()));
for (Long userId : requestedIds) {
UserColumnPermissionEntity grant = new UserColumnPermissionEntity();
@@ -256,6 +256,68 @@ public class PermissionMenuService {
return requestedIds.size();
}
public List<ImageVideoDataPermissionUserVo> listShopDataCrawlDataPermissionUsers(AdminUserEntity operator) {
ensureSuperAdminOperator(operator, "店铺数据任务");
PermissionMenuEntity dataPermission = requireDataPermission(
SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据");
Set<Long> grantedUserIds = userColumnPermissionMapper.selectList(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getColumnId, dataPermission.getId()))
.stream()
.map(UserColumnPermissionEntity::getUserId)
.filter(id -> id != null && id > 0)
.collect(Collectors.toSet());
return adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>()
.orderByAsc(AdminUserEntity::getUsername)
.orderByAsc(AdminUserEntity::getId))
.stream()
.filter(user -> !isSuperAdmin(user))
.map(user -> toImageVideoDataPermissionUserVo(user, grantedUserIds.contains(user.getId())))
.toList();
}
@Transactional
public int updateShopDataCrawlDataPermissionUsers(AdminUserEntity operator, List<Long> userIds) {
ensureSuperAdminOperator(operator, "店铺数据任务");
PermissionMenuEntity dataPermission = requireDataPermission(
SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据");
List<Long> requestedIds = normalizeColumnIds(userIds);
List<AdminUserEntity> users = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>());
Map<Long, AdminUserEntity> grantableUsers = users.stream()
.filter(user -> user.getId() != null && !isSuperAdmin(user))
.collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left));
if (requestedIds.stream().anyMatch(id -> !grantableUsers.containsKey(id))) {
throw new BusinessException("contains unknown or non-grantable user");
}
userColumnPermissionMapper.deleteByMap(Map.of("column_id", dataPermission.getId()));
for (Long userId : requestedIds) {
UserColumnPermissionEntity grant = new UserColumnPermissionEntity();
grant.setUserId(userId);
grant.setColumnId(dataPermission.getId());
userColumnPermissionMapper.insert(grant);
}
return requestedIds.size();
}
/** Verifies both the visible admin menu and its separately managed data grant. */
public void requireShopDataCrawlTaskAccess(AdminUserEntity operator) {
ensureAdminOperatorIfPresent(operator);
if (operator == null || operator.getId() == null || operator.getId() <= 0) {
throw new BusinessException(403, "需要管理员权限");
}
if (isSuperAdmin(operator)) {
return;
}
PermissionMenuEntity taskMenu = requireDataPermission(
"admin_shop_data_crawl_tasks", "店铺数据任务管理菜单");
PermissionMenuEntity dataPermission = requireDataPermission(
SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据");
if (!hasEffectiveColumnPermission(operator.getId(), taskMenu.getId())
|| !hasEffectiveColumnPermission(operator.getId(), dataPermission.getId())) {
throw new BusinessException(403, "无权查看店铺数据任务");
}
}
@Transactional
public void updateUserColumnPermissions(Long userId, UserColumnPermissionUpdateRequest request) {
updateUserColumnPermissions(null, userId, request);
@@ -304,20 +366,29 @@ public class PermissionMenuService {
}
}
PermissionMenuEntity imageVideoDataPermission = normalizedType == null
? findImageVideoDataPermission()
: null;
Long protectedId = imageVideoDataPermission == null ? null : imageVideoDataPermission.getId();
Set<Long> protectedIds = new LinkedHashSet<>();
if (normalizedType == null) {
PermissionMenuEntity imageVideoDataPermission = findImageVideoDataPermission();
PermissionMenuEntity shopDataCrawlDataPermission =
findDataPermission(SHOP_DATA_CRAWL_DATA_PERMISSION_KEY);
if (imageVideoDataPermission != null && imageVideoDataPermission.getId() != null) {
protectedIds.add(imageVideoDataPermission.getId());
}
if (shopDataCrawlDataPermission != null && shopDataCrawlDataPermission.getId() != null) {
protectedIds.add(shopDataCrawlDataPermission.getId());
}
}
List<Long> grantIds = requestedIds;
if (normalizedType == null && protectedId != null) {
if (!protectedIds.isEmpty()) {
grantIds = requestedIds.stream()
.filter(id -> !protectedId.equals(id))
.filter(id -> !protectedIds.contains(id))
.toList();
}
Set<Long> operatorEffectiveIds = ensureGrantable(operator, grantIds);
LinkedHashSet<Long> finalGrantIds = new LinkedHashSet<>(grantIds);
if (normalizedType == null && protectedId != null) {
if (!protectedIds.isEmpty()) {
for (Long protectedId : protectedIds) {
Long existingCount = userColumnPermissionMapper.selectCount(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId)
@@ -326,6 +397,7 @@ public class PermissionMenuService {
finalGrantIds.add(protectedId);
}
}
}
if (operatorEffectiveIds != null) {
loadDirectColumnIds(userId).stream()
.filter(id -> normalizedType == null || scopedMenuIds.contains(id))
@@ -526,9 +598,7 @@ public class PermissionMenuService {
}
private PermissionMenuEntity findImageVideoDataPermission() {
return permissionMenuMapper.selectOne(new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(PermissionMenuEntity::getColumnKey, IMAGE_VIDEO_DATA_PERMISSION_KEY)
.last("LIMIT 1"));
return findDataPermission(IMAGE_VIDEO_DATA_PERMISSION_KEY);
}
private PermissionMenuEntity requireImageVideoDataPermission() {
@@ -546,6 +616,27 @@ public class PermissionMenuService {
}
}
private void ensureSuperAdminOperator(AdminUserEntity operator, String resourceName) {
ensureAdminOperatorIfPresent(operator);
if (operator == null || !isSuperAdmin(operator)) {
throw new BusinessException(403, "仅超级管理员可以配置" + resourceName + "权限");
}
}
private PermissionMenuEntity findDataPermission(String columnKey) {
return permissionMenuMapper.selectOne(new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(PermissionMenuEntity::getColumnKey, columnKey)
.last("LIMIT 1"));
}
private PermissionMenuEntity requireDataPermission(String columnKey, String displayName) {
PermissionMenuEntity permission = findDataPermission(columnKey);
if (permission == null || permission.getId() == null) {
throw new BusinessException(displayName + "权限尚未初始化");
}
return permission;
}
private ImageVideoDataPermissionUserVo toImageVideoDataPermissionUserVo(AdminUserEntity user,
boolean granted) {
ImageVideoDataPermissionUserVo vo = new ImageVideoDataPermissionUserVo();
@@ -63,6 +63,7 @@ public class PriceTrackTaskService {
private static final String MODULE_TYPE = "PRICE_TRACK";
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String ASIN_ROWS_PAYLOAD_SCOPE = "price-track-asin-rows";
private static final String NO_USABLE_ROWS_ERROR = "未收到有效跟价数据,未生成结果文件";
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
@@ -1907,7 +1908,7 @@ public class PriceTrackTaskService {
if (payload != null && hasText(payload.getError())) {
return payload.getError().trim();
}
return "no usable price-track rows received";
return NO_USABLE_ROWS_ERROR;
}
private String firstNonBlank(String preferred, String fallback) {
if (preferred != null && !preferred.isBlank()) {
@@ -28,7 +28,7 @@ import org.springframework.web.bind.annotation.RestController;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/publish")
@Tag(name = "上架", description = "上架 Excel 多文件任务接口。前端创建批次并严格串行派发文件;Python 按页拉取数据、发送统一任务心跳并回传当前店铺完整结果")
@Tag(name = "上架", description = "上架 Excel 多文件任务接口。前端创建批次并严格串行派发文件;Python 按页拉取数据、通过结果分片回传进度和当前店铺完整结果,任务心跳用于保活")
public class PublishController {
private final PublishTaskService publishTaskService;
@@ -38,9 +38,9 @@ public class PublishFileVo {
private Integer percent;
@Schema(description = "与 percent 相同,供公共进度组件使用", example = "50")
private Integer progressPercent;
@Schema(description = "心跳上报的当前处理数量;未上报时使用 processedRows", example = "341")
@Schema(description = "结果分片累计接收的当前处理数量;兼容心跳上报", example = "341")
private Integer progressCurrent;
@Schema(description = "心跳上报的总处理数量;未上报时使用 totalRows", example = "682")
@Schema(description = "解析得到的总处理数量;兼容心跳上报", example = "682")
private Integer progressTotal;
@Schema(description = "文件失败原因的进度展示副本;无错误时为空")
private String progressMessage;
@@ -345,7 +345,11 @@ public class PublishTaskService {
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.set(FileTaskEntity::getUpdatedAt, now));
if (request == null || (request.getCurrent() == null && request.getTotal() == null)) {
// Generic desktop heartbeats use 0/0 when they only need to keep the task alive.
// Treat that as no progress update so result chunks cannot be reset to zero.
if (request == null
|| ((request.getCurrent() == null || request.getCurrent() <= 0)
&& (request.getTotal() == null || request.getTotal() <= 0))) {
return;
}
PublishFileEntity active = publishFileMapper.selectOne(new LambdaQueryWrapper<PublishFileEntity>()
@@ -360,12 +364,12 @@ public class PublishTaskService {
.eq(PublishFileEntity::getId, active.getId())
.eq(PublishFileEntity::getStatus, STATUS_RUNNING)
.set(PublishFileEntity::getUpdatedAt, now);
if (request.getTotal() != null && request.getTotal() >= 0) {
if (request.getTotal() != null && request.getTotal() > 0) {
update.set(PublishFileEntity::getTotalRows, request.getTotal());
}
if (request.getCurrent() != null && request.getCurrent() >= 0) {
if (request.getCurrent() != null && request.getCurrent() > 0) {
int current = request.getCurrent();
if (request.getTotal() != null && request.getTotal() >= 0) {
if (request.getTotal() != null && request.getTotal() > 0) {
current = Math.min(current, request.getTotal());
}
update.set(PublishFileEntity::getProcessedRows, current);
@@ -432,6 +436,7 @@ public class PublishTaskService {
throw new BusinessException("no successful publish files");
}
TaskOptions options = readTaskOptions(task);
List<PublishWorkbookService.WorkbookInput> inputs = new ArrayList<>();
int rowCount = 0;
for (PublishFileEntity file : successfulFiles) {
@@ -443,7 +448,7 @@ public class PublishTaskService {
List<PublishRowDto> rows = items.stream().map(this::toRowDto).toList();
rowCount += rows.size();
inputs.add(new PublishWorkbookService.WorkbookInput(
file.getSourceFilename(), file.getShopName(), rows));
file.getSourceFilename(), file.getShopName(), options.publishCountry(), rows));
}
PublishWorkbookService.PackagedResult packaged = workbookService.packageTaskResult(
@@ -737,6 +742,7 @@ public class PublishTaskService {
} else {
ResultChunkReceipt receipt = persistResultChunk(taskId, file, incoming, storedPayloads);
if (!receipt.completed()) {
updateReceivedProgress(taskId, file, receipt.receivedRowCount());
file.setStatus(STATUS_RUNNING);
file.setErrorMessage(null);
file.setUpdatedAt(LocalDateTime.now());
@@ -1025,9 +1031,11 @@ public class PublishTaskService {
if (existing != null) {
validateExistingChunk(existing, chunkTotal, payloadHash);
int receivedChunkCount = countResultChunks(taskId, scopeHash);
persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
int receivedRowCount = resolveReceivedRowCount(taskId, scopeHash, scope, 0, false);
persistResultScope(taskId, scopeKey, scopeHash, chunkTotal,
receivedChunkCount, receivedRowCount);
return new ResultChunkReceipt(scopeHash, chunkTotal,
receivedChunkCount >= chunkTotal);
receivedChunkCount >= chunkTotal, receivedRowCount);
}
ensureRustfsPayloadStorageEnabled();
@@ -1046,9 +1054,11 @@ public class PublishTaskService {
chunk.setPayloadHash(payloadHash);
chunk.setCreatedAt(LocalDateTime.now());
chunk.setUpdatedAt(LocalDateTime.now());
boolean inserted = false;
try {
taskChunkMapper.insert(chunk);
storedPayloads.add(storedPayload);
inserted = true;
} catch (DuplicateKeyException ex) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
TaskChunkEntity winner = findResultChunk(taskId, scopeHash, chunkIndex);
@@ -1062,11 +1072,13 @@ public class PublishTaskService {
}
int receivedChunkCount = countResultChunks(taskId, scopeHash);
persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} received={}",
taskId, file.getId(), chunkIndex, chunkTotal, receivedChunkCount);
int receivedRowCount = resolveReceivedRowCount(taskId, scopeHash, scope, rows.size(), inserted);
persistResultScope(taskId, scopeKey, scopeHash, chunkTotal,
receivedChunkCount, receivedRowCount);
log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} receivedChunks={} receivedRows={}",
taskId, file.getId(), chunkIndex, chunkTotal, receivedChunkCount, receivedRowCount);
return new ResultChunkReceipt(scopeHash, chunkTotal,
receivedChunkCount >= chunkTotal);
receivedChunkCount >= chunkTotal, receivedRowCount);
}
private void validateChunkMetadata(int chunkIndex, int chunkTotal) {
@@ -1119,11 +1131,70 @@ public class PublishTaskService {
return count == null ? 0 : count.intValue();
}
/**
* Resolve the number of result rows received for a file without re-reading
* every payload on every callback. New callbacks keep the count in the
* existing scope state JSON; scopes created by older versions are repaired
* once by counting their stored chunks.
*/
private int resolveReceivedRowCount(Long taskId,
String scopeHash,
TaskScopeStateEntity scope,
int currentChunkRows,
boolean inserted) {
Integer persisted = readReceivedRowCount(scope);
if (persisted != null) {
long next = (long) persisted + (inserted ? Math.max(0, currentChunkRows) : 0);
return next > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) Math.max(0, next);
}
return countReceivedResultRows(taskId, scopeHash);
}
private Integer readReceivedRowCount(TaskScopeStateEntity scope) {
if (scope == null || scope.getStateJson() == null || scope.getStateJson().isBlank()) {
return null;
}
try {
JsonNode root = objectMapper.readTree(scope.getStateJson());
JsonNode receivedRows = root == null ? null : root.get("receivedRows");
if (receivedRows == null || !receivedRows.isIntegralNumber()) {
return null;
}
return Math.max(0, receivedRows.asInt(0));
} catch (Exception ex) {
log.warn("[publish] failed to read received row count from result scope taskId={} scopeHash={} msg={}",
scope.getTaskId(), scope.getScopeHash(), safeMessage(ex));
return null;
}
}
private int countReceivedResultRows(Long taskId, String scopeHash) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.orderByAsc(TaskChunkEntity::getChunkIndex));
if (chunks == null || chunks.isEmpty()) {
return 0;
}
TypeReference<List<PublishRowDto>> listType = new TypeReference<>() {
};
long count = 0;
for (TaskChunkEntity chunk : chunks) {
count += readResultChunkRows(chunk, listType).size();
if (count >= Integer.MAX_VALUE) {
return Integer.MAX_VALUE;
}
}
return (int) count;
}
private void persistResultScope(Long taskId,
String scopeKey,
String scopeHash,
int chunkTotal,
int receivedChunkCount) {
int receivedChunkCount,
int receivedRowCount) {
TaskScopeStateEntity scope = findResultScope(taskId, scopeHash);
validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal);
LocalDateTime now = LocalDateTime.now();
@@ -1141,7 +1212,7 @@ public class PublishTaskService {
scope.setCompleted(completed ? 1 : 0);
scope.setLastChunkAt(now);
scope.setLastError(null);
scope.setStateJson(completed ? "{\"phase\":\"COMPLETE\"}" : "{\"phase\":\"RECEIVING\"}");
scope.setStateJson(resultScopeStateJson(completed, receivedRowCount));
scope.setUpdatedAt(now);
if (scope.getId() != null) {
taskScopeStateMapper.updateById(scope);
@@ -1160,12 +1231,43 @@ public class PublishTaskService {
winner.setCompleted(completed ? 1 : 0);
winner.setLastChunkAt(now);
winner.setLastError(null);
winner.setStateJson(scope.getStateJson());
Integer winnerRowCount = readReceivedRowCount(winner);
winner.setStateJson(resultScopeStateJson(completed,
Math.max(receivedRowCount, winnerRowCount == null ? 0 : winnerRowCount)));
winner.setUpdatedAt(now);
taskScopeStateMapper.updateById(winner);
}
}
private String resultScopeStateJson(boolean completed, int receivedRowCount) {
Map<String, Object> state = new LinkedHashMap<>();
state.put("phase", completed ? "COMPLETE" : "RECEIVING");
state.put("receivedRows", Math.max(0, receivedRowCount));
return writeJson(state, "保存上架结果分片进度失败");
}
private void updateReceivedProgress(Long taskId, PublishFileEntity file, int receivedRowCount) {
if (file == null) {
return;
}
int total = safeInt(file.getTotalRows());
if (total <= 0) {
long parsedRows = Objects.requireNonNullElse(publishItemMapper.selectCount(
new LambdaQueryWrapper<PublishItemEntity>()
.eq(PublishItemEntity::getTaskId, taskId)
.eq(PublishItemEntity::getFileId, file.getId())), 0L);
total = parsedRows >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) Math.max(0L, parsedRows);
if (total > 0) {
file.setTotalRows(total);
}
}
int progress = Math.max(safeInt(file.getProcessedRows()), Math.max(0, receivedRowCount));
if (total > 0) {
progress = Math.min(total, progress);
}
file.setProcessedRows(progress);
}
private List<PublishRowDto> loadCompleteResultRows(Long taskId, ResultChunkReceipt receipt) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
@@ -1637,6 +1739,7 @@ public class PublishTaskService {
private record ResultChunkReceipt(String scopeHash,
int chunkTotal,
boolean completed) {
boolean completed,
int receivedRowCount) {
}
}
@@ -91,7 +91,7 @@ public class PublishWorkbookService {
}
}
public File writeWorkbook(File outputFile, List<PublishRowDto> rows) {
public File writeWorkbook(File outputFile, List<PublishRowDto> rows, String publishCountry) {
File parent = outputFile.getParentFile();
if (parent != null) {
parent.mkdirs();
@@ -100,13 +100,9 @@ public class PublishWorkbookService {
workbook.setCompressTempFiles(true);
try (FileOutputStream output = new FileOutputStream(outputFile)) {
CellStyle headerStyle = createHeaderStyle(workbook);
Map<String, List<PublishRowDto>> rowsByCountry = groupByCountry(rows);
Set<String> usedSheetNames = new LinkedHashSet<>();
for (Map.Entry<String, List<PublishRowDto>> entry : rowsByCountry.entrySet()) {
String sheetName = uniqueSheetName(entry.getKey(), usedSheetNames);
Sheet sheet = workbook.createSheet(sheetName);
writeSheet(sheet, entry.getValue(), headerStyle);
}
String country = countrySheetName(publishCountry);
Sheet sheet = workbook.createSheet(country);
writeSheet(sheet, rows, headerStyle, country);
workbook.write(output);
return outputFile;
} catch (Exception ex) {
@@ -135,7 +131,7 @@ public class PublishWorkbookService {
+ "_上架结果.xlsx";
String filename = uniqueFilename(desired, usedFilenames);
File workbook = new File(workDirectory, filename);
writeWorkbook(workbook, input.rows());
writeWorkbook(workbook, input.rows(), input.publishCountry());
workbooks.add(workbook);
}
@@ -202,7 +198,10 @@ public class PublishWorkbookService {
return grouped;
}
private void writeSheet(Sheet sheet, List<PublishRowDto> rows, CellStyle headerStyle) {
private void writeSheet(Sheet sheet,
List<PublishRowDto> rows,
CellStyle headerStyle,
String publishCountry) {
Row header = sheet.createRow(0);
for (int index = 0; index < RESULT_HEADERS.size(); index++) {
Cell cell = header.createCell(index);
@@ -210,11 +209,14 @@ public class PublishWorkbookService {
cell.setCellStyle(headerStyle);
}
int rowIndex = 1;
for (PublishRowDto value : rows) {
for (PublishRowDto value : rows == null ? List.<PublishRowDto>of() : rows) {
if (value == null) {
continue;
}
Row row = sheet.createRow(rowIndex++);
setText(row, 0, value.getSourceId());
setText(row, 1, value.getAsin());
setText(row, 2, value.getCountry());
setText(row, 2, publishCountry);
setText(row, 3, value.getBrand());
setPrice(row, 4, value.getPrice());
setText(row, 5, value.getStatus());
@@ -337,7 +339,10 @@ public class PublishWorkbookService {
public record ParsedWorkbook(List<PublishRowDto> rows) {
}
public record WorkbookInput(String sourceFilename, String shopName, List<PublishRowDto> rows) {
public record WorkbookInput(String sourceFilename,
String shopName,
String publishCountry,
List<PublishRowDto> rows) {
}
public record PackagedResult(File file, String filename, String contentType) {
@@ -0,0 +1,154 @@
package com.nanri.aiimage.modules.shopdatacrawl.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import io.swagger.v3.oas.annotations.Operation;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
/**
* Internal compatibility endpoints used by the Flask admin task page.
* The user-facing shop-data-crawl endpoints remain unchanged for existing clients.
*/
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/admin/shop-data-crawl")
public class AdminShopDataCrawlTaskController {
@Value("${aiimage.security.internal-token:}")
private String internalToken;
@Value("${aiimage.security.internal-token-file:}")
private String internalTokenFile;
private final ShopDataCrawlTaskService taskService;
private final AdminAuthSupport adminAuthSupport;
private final PermissionMenuService permissionMenuService;
@GetMapping("/results/{resultId}/download")
@Operation(summary = "下载店铺数据任务结果(内部)")
public void download(
@PathVariable Long resultId,
HttpServletRequest request,
HttpServletResponse response) {
requireShopDataCrawlTaskAccess(request);
String url = taskService.resolveAdminResultDownloadUrl(resultId);
String filename = taskService.resolveAdminResultDownloadFilename(resultId);
try {
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
DownloadHeaderUtil.setAttachment(response, filename);
try (InputStream input = URI.create(url).toURL().openStream()) {
input.transferTo(response.getOutputStream());
}
} catch (Exception ex) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
}
}
@DeleteMapping("/history/{resultId}")
@Operation(summary = "删除店铺数据任务结果(内部)")
public ApiResponse<Void> deleteHistory(
@PathVariable Long resultId,
HttpServletRequest request) {
requireShopDataCrawlTaskAccess(request);
taskService.deleteAdminHistory(resultId);
return ApiResponse.success(null);
}
private void requireShopDataCrawlTaskAccess(HttpServletRequest request) {
AdminUserEntity operator;
try {
operator = adminAuthSupport.requireAdmin(request);
} catch (BusinessException authFailure) {
operator = resolveInternalOperator(request);
if (operator == null) {
throw authFailure;
}
}
permissionMenuService.requireShopDataCrawlTaskAccess(operator);
}
private AdminUserEntity resolveInternalOperator(HttpServletRequest request) {
String suppliedToken = request.getHeader("X-Internal-Token");
if (!isTrustedInternalRequest(suppliedToken)) {
return null;
}
String rawOperatorId = request.getParameter("operatorId");
if (rawOperatorId == null || rawOperatorId.isBlank()) {
rawOperatorId = request.getParameter("operator_id");
}
if (rawOperatorId == null || rawOperatorId.isBlank()) {
return null;
}
try {
return permissionMenuService.requireAdminOperator(Long.parseLong(rawOperatorId.trim()));
} catch (NumberFormatException ex) {
return null;
}
}
private boolean isTrustedInternalRequest(String suppliedToken) {
String expectedToken = resolveExpectedInternalToken();
return !expectedToken.isBlank() && suppliedToken != null && !suppliedToken.isBlank()
&& MessageDigest.isEqual(
expectedToken.getBytes(StandardCharsets.UTF_8),
suppliedToken.trim().getBytes(StandardCharsets.UTF_8));
}
private String resolveExpectedInternalToken() {
if (internalToken != null && !internalToken.isBlank()) {
return internalToken.trim();
}
Path path = resolveInternalTokenFile();
if (path == null || !Files.isRegularFile(path)) {
return "";
}
try {
return Files.readString(path, StandardCharsets.UTF_8).trim();
} catch (Exception ignored) {
return "";
}
}
private Path resolveInternalTokenFile() {
String configuredPath = internalTokenFile == null ? "" : internalTokenFile.trim();
if (!configuredPath.isEmpty()) {
if (configuredPath.equals("~") || configuredPath.startsWith("~/") || configuredPath.startsWith("~\\")) {
String userHome = System.getProperty("user.home", "").trim();
if (userHome.isEmpty()) {
return null;
}
configuredPath = configuredPath.length() == 1
? userHome
: Path.of(userHome, configuredPath.substring(2)).toString();
}
Path configuredTokenPath = Path.of(configuredPath);
return configuredTokenPath.isAbsolute() ? configuredTokenPath.normalize() : null;
}
String userHome = System.getProperty("user.home", "").trim();
return userHome.isEmpty()
? null
: Path.of(userHome, ".aiimage", "internal-token").toAbsolutePath().normalize();
}
}
@@ -190,6 +190,7 @@ public class ShopDataCrawlTaskController {
{
"date": "2026-07-25",
"asin": "B0EXAMPLE1",
"commodityImage": "https://m.media-amazon.com/images/I/example.jpg",
"inventorySales": "120",
"salesRank": "35",
"pageViews": "860",
@@ -5,7 +5,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "店铺数据抓取结果行;生成 Excel 时按日期、ASIN、库存销量、销售排名、页面浏览量、售出件数、价格、推荐报价的固定列顺序写入")
@Schema(description = "店铺数据抓取结果行;生成 Excel 时按日期、ASIN、商品图片、库存销量、销售排名、页面浏览量、售出件数、价格、推荐报价、品牌的固定列顺序写入")
public class ShopDataCrawlRowDto {
@JsonAlias("日期")
@Schema(description = "日期列,按来源文本原样保留", example = "2026-07-25")
@@ -15,6 +15,14 @@ public class ShopDataCrawlRowDto {
@Schema(description = "亚马逊商品 ASIN", example = "B0CJ8SNXXV")
private String asin;
@JsonAlias({"brand", "品牌"})
@Schema(description = "商品品牌,来自 Python 回传的 brand 字段", example = "Example Brand")
private String brand;
@JsonAlias({"commodity_image", "商品图片"})
@Schema(description = "商品图片 URL;生成 Excel 时下载并嵌入商品图片列", example = "https://m.media-amazon.com/images/I/example.jpg")
private String commodityImage;
@JsonAlias({"库存销量", "inventory_sales"})
@Schema(description = "库存销量列,按来源文本原样保留", example = "128")
private String inventorySales;
@@ -4,10 +4,16 @@ import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.ClientAnchor;
import org.apache.poi.ss.usermodel.Drawing;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
@@ -19,13 +25,24 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@Slf4j
@RequiredArgsConstructor
public class ShopDataCrawlExcelAssemblyService {
static final List<String> COUNTRIES = List.of("UK", "DE", "FR", "ES", "IT");
static final List<String> SHEETS = List.of("英国", "德国", "法国", "西班牙", "意大利");
static final List<String> HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
static final List<String> LEGACY_HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
static final List<String> HEADERS_WITHOUT_BRAND = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
static final List<String> HEADERS = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价", "品牌");
private static final String TEMPLATE = "templates/shop-data-crawl/文档格式.xlsx";
private static final int IMAGE_COLUMN = 2;
private static final int BRAND_COLUMN = HEADERS.size() - 1;
private static final int IMAGE_COLUMN_WIDTH = 18 * 256;
private static final float IMAGE_ROW_HEIGHT_POINTS = 80f;
private final SimilarAsinImageEmbedder imageEmbedder;
public void writeWorkbook(File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream();
@@ -33,8 +50,11 @@ public class ShopDataCrawlExcelAssemblyService {
FileOutputStream output = new FileOutputStream(outputXlsx)) {
validateTemplate(workbook);
Map<String, List<ShopDataCrawlRowDto>> rowsByCountry = rowsByCountry(items);
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache = new ConcurrentHashMap<>();
imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache);
Map<String, Integer> pictureIndexes = new LinkedHashMap<>();
for (int i = 0; i < COUNTRIES.size(); i++) {
writeSheet(workbook.getSheetAt(i), rowsByCountry.get(COUNTRIES.get(i)));
writeSheet(workbook, workbook.getSheetAt(i), rowsByCountry.get(COUNTRIES.get(i)), imageCache, pictureIndexes);
}
workbook.write(output);
} catch (BusinessException ex) {
@@ -58,22 +78,45 @@ public class ShopDataCrawlExcelAssemblyService {
throw new BusinessException("店铺数据抓取模板工作表顺序不正确");
}
Row header = sheet.getRow(0);
for (int column = 0; column < HEADERS.size(); column++) {
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
boolean templateHasBrand = "品牌".equals(cellText(header, BRAND_COLUMN));
List<String> expectedHeaders = currentTemplate
? (templateHasBrand ? HEADERS : HEADERS_WITHOUT_BRAND)
: LEGACY_HEADERS;
for (int column = 0; column < expectedHeaders.size(); column++) {
String actual = header == null || header.getCell(column) == null ? "" : header.getCell(column).getStringCellValue().trim();
if (!HEADERS.get(column).equals(actual)) {
if (!expectedHeaders.get(column).equals(actual)) {
throw new BusinessException("店铺数据抓取模板表头不正确: " + sheet.getSheetName());
}
}
}
}
private void writeSheet(Sheet sheet, List<ShopDataCrawlRowDto> rows) {
private void writeSheet(XSSFWorkbook workbook,
Sheet sheet,
List<ShopDataCrawlRowDto> rows,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
Map<String, Integer> pictureIndexes) {
Row header = sheet.getRow(0);
Row styleRow = sheet.getRow(1);
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
boolean templateHasBrand = "品牌".equals(cellText(header, BRAND_COLUMN));
CellStyle[] styles = new CellStyle[HEADERS.size()];
for (int column = 0; column < styles.length; column++) {
Cell cell = styleRow == null ? null : styleRow.getCell(column);
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
Cell cell = styleRow == null ? null : styleRow.getCell(sourceColumn);
styles[column] = cell == null ? null : cell.getCellStyle();
}
int[] columnWidths = new int[HEADERS.size()];
for (int column = 0; column < columnWidths.length; column++) {
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
columnWidths[column] = sheet.getColumnWidth(sourceColumn);
}
writeHeaders(sheet, currentTemplate, templateHasBrand);
for (int column = 0; column < columnWidths.length; column++) {
sheet.setColumnWidth(column, columnWidths[column]);
}
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
int last = sheet.getLastRowNum();
for (int rowIndex = 1; rowIndex <= last; rowIndex++) {
Row row = sheet.getRow(rowIndex);
@@ -84,15 +127,96 @@ public class ShopDataCrawlExcelAssemblyService {
int rowIndex = 1;
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
Row row = sheet.createRow(rowIndex++);
String[] values = {value.getDate(), value.getAsin(), value.getInventorySales(), value.getSalesRank(),
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer()};
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
for (int column = 0; column < values.length; column++) {
Cell cell = row.createCell(column);
if (styles[column] != null) cell.setCellStyle(styles[column]);
cell.setCellValue(values[column] == null ? "" : values[column]);
}
if (!blank(value.getCommodityImage())) {
row.setHeightInPoints(IMAGE_ROW_HEIGHT_POINTS);
embedImage(workbook, sheet, row, value.getCommodityImage(), imageCache, pictureIndexes);
}
}
}
private void writeHeaders(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) {
Row header = sheet.getRow(0);
if (header == null) header = sheet.createRow(0);
CellStyle[] styles = new CellStyle[HEADERS.size()];
for (int column = 0; column < HEADERS.size(); column++) {
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
Cell source = header.getCell(sourceColumn);
styles[column] = source == null ? null : source.getCellStyle();
}
for (int column = 0; column < HEADERS.size(); column++) {
Cell cell = header.getCell(column);
if (cell == null) cell = header.createCell(column);
if (styles[column] != null) cell.setCellStyle(styles[column]);
cell.setCellValue(HEADERS.get(column));
}
}
private int templateColumnForOutput(int outputColumn, boolean currentTemplate, boolean templateHasBrand) {
if (outputColumn == BRAND_COLUMN) {
return templateHasBrand ? BRAND_COLUMN : 1;
}
return currentTemplate || outputColumn < IMAGE_COLUMN ? outputColumn : outputColumn - 1;
}
private void embedImage(XSSFWorkbook workbook,
Sheet sheet,
Row row,
String imageUrl,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
Map<String, Integer> pictureIndexes) {
String normalizedUrl = imageUrl.trim();
try {
SimilarAsinImageEmbedder.ResizedImage image = imageCache.get(normalizedUrl);
if (image == null) image = imageEmbedder.fetchAndResizeForCache(normalizedUrl);
if (image == null) {
row.getCell(IMAGE_COLUMN).setCellValue(normalizedUrl);
return;
}
Integer pictureIndex = pictureIndexes.get(normalizedUrl);
if (pictureIndex == null) {
pictureIndex = workbook.addPicture(image.bytes(), Workbook.PICTURE_TYPE_JPEG);
pictureIndexes.put(normalizedUrl, pictureIndex);
}
Drawing<?> drawing = sheet.createDrawingPatriarch();
ClientAnchor anchor = workbook.getCreationHelper().createClientAnchor();
anchor.setCol1(IMAGE_COLUMN);
anchor.setRow1(row.getRowNum());
anchor.setCol2(IMAGE_COLUMN + 1);
anchor.setRow2(row.getRowNum() + 1);
anchor.setAnchorType(ClientAnchor.AnchorType.MOVE_AND_RESIZE);
drawing.createPicture(anchor, pictureIndex);
} catch (RuntimeException ex) {
log.warn("[shop-data-crawl] embed commodity image failed sheet={} row={} url={} msg={}",
sheet.getSheetName(), row.getRowNum() + 1, normalizedUrl, ex.getMessage());
row.getCell(IMAGE_COLUMN).setCellValue(normalizedUrl);
}
}
private List<String> imageUrls(Map<String, List<ShopDataCrawlRowDto>> rowsByCountry) {
return rowsByCountry.values().stream()
.flatMap(List::stream)
.map(ShopDataCrawlRowDto::getCommodityImage)
.filter(url -> !blank(url))
.map(String::trim)
.distinct()
.toList();
}
private String cellText(Row row, int column) {
Cell cell = row == null ? null : row.getCell(column);
return cell == null ? "" : cell.getStringCellValue().trim();
}
private boolean blank(String value) {
return value == null || value.isBlank();
}
private Map<String, List<ShopDataCrawlRowDto>> rowsByCountry(List<ShopDataCrawlResultItemVo> items) {
Map<String, List<ShopDataCrawlRowDto>> result = new LinkedHashMap<>();
@@ -50,6 +50,7 @@ import java.io.File;
import java.time.LocalDateTime;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -65,6 +66,7 @@ public class ShopDataCrawlTaskService {
private static final int RESULT_PENDING = -1;
private static final int RESULT_FAILED = 0;
private static final int RESULT_SUCCESS = 1;
private static final int SHOP_HISTORY_RETENTION_LIMIT = 3;
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
private static final String PARTIAL_RESULT_MESSAGE = "Python 中断,已保留已回传的部分数据";
private static final String RESULT_CHUNK_SCOPE_PREFIX = "result-chunks:";
@@ -290,21 +292,54 @@ public class ShopDataCrawlTaskService {
}
public String resolveResultDownloadUrl(Long resultId, Long userId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
validateUserId(userId);
FileResultEntity entity = requireResultEntity(resultId);
ensureResultOwner(entity, userId);
return resolveDownloadUrl(entity);
}
public String resolveResultDownloadFilename(Long resultId, Long userId) {
validateUserId(userId);
FileResultEntity entity = requireResultEntity(resultId);
ensureResultOwner(entity, userId);
return resolveDownloadFilename(entity);
}
/**
* Used only by the internally authenticated admin task-management endpoint.
* The public endpoint must continue to validate the requesting user ID.
*/
public String resolveAdminResultDownloadUrl(Long resultId) {
return resolveDownloadUrl(requireResultEntity(resultId));
}
/** See {@link #resolveAdminResultDownloadUrl(Long)}. */
public String resolveAdminResultDownloadFilename(Long resultId) {
return resolveDownloadFilename(requireResultEntity(resultId));
}
private FileResultEntity requireResultEntity(Long resultId) {
FileResultEntity entity = resultId == null || resultId <= 0 ? null : fileResultMapper.selectById(resultId);
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType())) {
throw new BusinessException("记录不存在");
}
return entity;
}
private void ensureResultOwner(FileResultEntity entity, Long userId) {
if (entity == null || !userId.equals(entity.getUserId())) {
throw new BusinessException("记录不存在");
}
}
private String resolveDownloadUrl(FileResultEntity entity) {
if (blank(entity.getResultFileUrl())) {
throw new BusinessException("暂无可下载文件");
}
return ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl());
}
public String resolveResultDownloadFilename(Long resultId, Long userId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("记录不存在");
}
private String resolveDownloadFilename(FileResultEntity entity) {
return !blank(entity.getResultFilename())
? entity.getResultFilename()
: safeFileStem(entity.getSourceFilename()) + ".xlsx";
@@ -584,26 +619,61 @@ public class ShopDataCrawlTaskService {
@Transactional
public void deleteHistory(Long resultId, Long userId) {
validateUserId(userId);
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("记录不存在");
}
FileResultEntity entity = requireResultEntity(resultId);
ensureResultOwner(entity, userId);
Long taskId = entity.getTaskId();
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) throw new BusinessException("任务不存在");
ensureTaskOwnedByCurrentInstance(task, "delete shop data crawl history");
if (!isTerminalTaskStatus(task.getStatus())) {
throw new BusinessException(40901, "任务仍在处理中,不能删除");
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("记录不存在");
}
deleteResultHistoryRow(latestEntity);
}
}
/**
* Deletes an administrative result without accepting a caller-controlled owner ID.
* The controller that calls this method is restricted to the Flask-to-Java internal channel.
*/
@Transactional
public void deleteAdminHistory(Long resultId) {
FileResultEntity entity = requireResultEntity(resultId);
Long ownerId = entity.getUserId();
if (ownerId == null || ownerId <= 0) {
throw new BusinessException("记录不存在");
}
deleteHistory(resultId, ownerId);
}
/**
* Removes one result row and all task-owned artifacts that point at it.
* The caller must already hold the task lock when the row belongs to a task.
*/
private void deleteResultHistoryRow(FileResultEntity entity) {
if (entity == null || entity.getId() == null || entity.getId() <= 0) {
return;
}
Long taskId = entity.getTaskId();
Long resultId = entity.getId();
String resultFileUrl = entity.getResultFileUrl();
taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId);
taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId);
String resultFileUrl = latestEntity.getResultFileUrl();
fileResultMapper.deleteById(resultId);
try {
reconcileTaskAfterResultRemoval(taskId);
deleteResultObjectIfUnreferenced(resultFileUrl);
} catch (Exception ex) {
// The row is already gone. Do not leave its workbook behind merely
// because the parent task snapshot could not be rebuilt.
log.warn("[shop-data-crawl] reconcile task after result deletion failed taskId={} resultId={} msg={}",
taskId, resultId, safeMessage(ex));
}
// Run this after the row delete so shared object references are counted correctly.
deleteResultObjectIfUnreferenced(resultFileUrl);
}
private void reconcileTaskAfterResultRemoval(Long taskId) {
@@ -630,6 +700,139 @@ public class ShopDataCrawlTaskService {
taskCacheService.saveTaskCache(task);
}
private void pruneCompletedHistoryQuietly(FileTaskEntity currentTask, List<FileResultEntity> currentRows) {
if (currentRows == null || currentRows.isEmpty()) {
return;
}
for (FileResultEntity row : currentRows) {
if (!isRetentionCandidate(row)) {
continue;
}
Long userId = row.getUserId() != null
? row.getUserId()
: currentTask == null ? null : currentTask.getUserId();
String shopKey = retentionShopKey(row);
if (userId == null || shopKey == null) {
log.warn("[shop-data-crawl] skip history retention because ownership key is incomplete taskId={} resultId={}",
currentTask == null ? null : currentTask.getId(), row.getId());
continue;
}
try {
pruneCompletedHistoryForShop(userId, shopKey);
} catch (Exception ex) {
// Retention is best effort. A cleanup failure must not fail the newly assembled workbook job.
log.warn("[shop-data-crawl] history retention failed taskId={} resultId={} msg={}",
currentTask == null ? null : currentTask.getId(), row.getId(), safeMessage(ex));
}
}
}
void pruneCompletedHistoryForShop(Long userId, String shopKey) {
if (userId == null || shopKey == null) {
return;
}
List<FileResultEntity> candidates = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId)
.eq(FileResultEntity::getSuccess, RESULT_SUCCESS)
.isNotNull(FileResultEntity::getResultFileUrl)
.ne(FileResultEntity::getResultFileUrl, "")
.orderByDesc(FileResultEntity::getCreatedAt)
.orderByDesc(FileResultEntity::getId));
if (candidates == null || candidates.isEmpty()) {
return;
}
List<FileResultEntity> shopResults = new ArrayList<>();
for (FileResultEntity candidate : candidates) {
if (!isRetentionCandidate(candidate)
|| !Objects.equals(userId, candidate.getUserId())) {
continue;
}
if (!Objects.equals(shopKey, retentionShopKey(candidate))) {
continue;
}
shopResults.add(candidate);
}
Comparator<FileResultEntity> newestFirst = Comparator
.comparing(FileResultEntity::getCreatedAt, Comparator.nullsLast(Comparator.reverseOrder()))
.thenComparing(FileResultEntity::getId, Comparator.nullsLast(Comparator.reverseOrder()));
shopResults.sort(newestFirst);
for (int index = SHOP_HISTORY_RETENTION_LIMIT; index < shopResults.size(); index++) {
deleteRetentionResultQuietly(shopResults.get(index), userId, shopKey);
}
}
private void deleteRetentionResultQuietly(FileResultEntity candidate, Long userId, String shopKey) {
if (candidate == null || candidate.getId() == null || candidate.getId() <= 0
|| candidate.getTaskId() == null || candidate.getTaskId() <= 0) {
return;
}
try {
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(candidate.getTaskId());
if (lockHandle == null) {
log.info("[shop-data-crawl] skip retained-history deletion because task lock is busy taskId={} resultId={}",
candidate.getTaskId(), candidate.getId());
return;
}
try (lockHandle) {
FileResultEntity latest = fileResultMapper.selectById(candidate.getId());
if (!isRetentionCandidate(latest)
|| !Objects.equals(userId, latest.getUserId())
|| !Objects.equals(shopKey, retentionShopKey(latest))) {
return;
}
FileTaskEntity task = fileTaskMapper.selectById(candidate.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())
|| !isTerminalTaskStatus(task.getStatus())) {
log.info("[shop-data-crawl] skip retained-history deletion because task is not terminal taskId={} resultId={}",
candidate.getTaskId(), candidate.getId());
return;
}
deleteResultHistoryRow(latest);
}
} catch (Exception ex) {
log.warn("[shop-data-crawl] retained-history deletion failed taskId={} resultId={} msg={}",
candidate.getTaskId(), candidate.getId(), safeMessage(ex));
}
}
private boolean isRetentionCandidate(FileResultEntity row) {
return row != null
&& Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())
&& !blank(row.getResultFileUrl());
}
private String retentionShopKey(FileResultEntity row) {
if (row == null) {
return null;
}
String shopId = trimToNull(row.getSourceFileUrl());
if (shopId != null) {
return "shop-id:" + shopId;
}
String shopName = trimToNull(row.getSourceFilename());
return shopName == null ? null : "shop-name:" + shopName;
}
private String trimToNull(String value) {
if (value == null) {
return null;
}
String normalized = value.trim();
return normalized.isEmpty() ? null : normalized;
}
private long countTasks(Long userId, List<String> statuses) {
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
@@ -1360,6 +1563,7 @@ public class ShopDataCrawlTaskService {
} finally {
FileUtil.del(xlsx);
}
pruneCompletedHistoryQuietly(task, rows);
}
public void cleanupResultFileJob(TaskFileJobEntity job) {
@@ -1460,6 +1664,10 @@ public class ShopDataCrawlTaskService {
return Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess()) || Integer.valueOf(RESULT_FAILED).equals(row.getSuccess());
}
private boolean isTerminalTaskStatus(String status) {
return "SUCCESS".equals(status) || "FAILED".equals(status) || "CANCELLED".equals(status);
}
private Boolean toSuccessFlag(Integer dbValue, Boolean fallback) {
if (dbValue == null || Integer.valueOf(RESULT_PENDING).equals(dbValue)) {
return fallback;
@@ -1544,6 +1752,8 @@ public class ShopDataCrawlTaskService {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate(trim(source.getDate()));
row.setAsin(trim(source.getAsin()));
row.setBrand(trim(source.getBrand()));
row.setCommodityImage(trim(source.getCommodityImage()));
row.setInventorySales(trim(source.getInventorySales()));
row.setSalesRank(trim(source.getSalesRank()));
row.setPageViews(trim(source.getPageViews()));
@@ -1581,6 +1791,8 @@ public class ShopDataCrawlTaskService {
return left != null && right != null
&& Objects.equals(trim(left.getDate()), trim(right.getDate()))
&& Objects.equals(trim(left.getAsin()), trim(right.getAsin()))
&& Objects.equals(trim(left.getBrand()), trim(right.getBrand()))
&& Objects.equals(trim(left.getCommodityImage()), trim(right.getCommodityImage()))
&& Objects.equals(trim(left.getInventorySales()), trim(right.getInventorySales()))
&& Objects.equals(trim(left.getSalesRank()), trim(right.getSalesRank()))
&& Objects.equals(trim(left.getPageViews()), trim(right.getPageViews()))
@@ -1590,7 +1802,7 @@ public class ShopDataCrawlTaskService {
}
private boolean rowEmpty(ShopDataCrawlRowDto row) {
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getInventorySales())
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getBrand()) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
&& blank(row.getSalesRank()) && blank(row.getPageViews()) && blank(row.getUnitsSold())
&& blank(row.getPrice()) && blank(row.getRecommendedOffer()));
}
@@ -79,13 +79,13 @@ public class SkipPriceAsinController {
}
@PostMapping
@Operation(summary = "新增或更新跳过跟价 ASIN", description = "按店铺和国家维度新增或更新跳过跟价 ASIN。")
@Operation(summary = "新增跳过跟价 ASIN", description = "增一条跳过跟价 ASIN 记录")
public ApiResponse<SkipPriceAsinItemVo> create(
@Parameter(description = "当前操作人用户 ID") @RequestParam(name = "operator_id") Long operatorId,
@Parameter(description = "是否超级管理员") @RequestParam(name = "super_admin", defaultValue = "false") Boolean superAdmin,
@Valid @RequestBody SkipPriceAsinCreateRequest request) {
return ApiResponse.success("保存成功",
skipPriceAsinService.createOrUpdate(request, operatorId, Boolean.TRUE.equals(superAdmin)));
skipPriceAsinService.create(request, operatorId, Boolean.TRUE.equals(superAdmin)));
}
@PostMapping("/import")
@@ -1,6 +1,8 @@
package com.nanri.aiimage.modules.shopkey.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.FieldStrategy;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
@@ -16,6 +18,11 @@ public class ShopKeyEntity {
private String remarkName;
private String ziniaoAccountName;
private String ziniaoToken;
private String ipWhitelistStatus;
@TableField(updateStrategy = FieldStrategy.ALWAYS)
private LocalDateTime ipWhitelistCheckedAt;
@TableField(updateStrategy = FieldStrategy.ALWAYS)
private String ipWhitelistMessage;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -21,6 +21,15 @@ public class ShopKeyItemVo {
@Schema(description = "紫鸟令牌")
private String ziniaoToken;
@Schema(description = "IP 白名单状态:UNKNOWN、ALLOWED、BLOCKED")
private String ipWhitelistStatus;
@Schema(description = "最近一次 IP 白名单检测时间")
private LocalDateTime ipWhitelistCheckedAt;
@Schema(description = "最近一次 IP 白名单检测信息")
private String ipWhitelistMessage;
@Schema(description = "创建时间")
private LocalDateTime createdAt;
@@ -21,6 +21,8 @@ import java.util.List;
@Slf4j
public class ShopKeyService {
private static final String IP_WHITELIST_STATUS_UNKNOWN = "UNKNOWN";
private final ShopKeyMapper shopKeyMapper;
private final ZiniaoShopIndexService ziniaoShopIndexService;
@@ -50,6 +52,7 @@ public class ShopKeyService {
entity.setRemarkName(normalizeOptional(request.getRemarkName()));
entity.setZiniaoAccountName(ziniaoAccountName);
entity.setZiniaoToken(ziniaoToken);
entity.setIpWhitelistStatus(IP_WHITELIST_STATUS_UNKNOWN);
shopKeyMapper.insert(entity);
triggerShopIndexRefresh();
return toItemVo(getById(entity.getId()));
@@ -60,9 +63,15 @@ public class ShopKeyService {
ShopKeyEntity entity = getById(id);
String ziniaoAccountName = normalizeRequired(request.getZiniaoAccountName(), "紫鸟账号名称不能为空");
String ziniaoToken = normalizeRequired(request.getZiniaoToken(), "紫鸟令牌不能为空");
boolean tokenChanged = !ziniaoToken.equals(entity.getZiniaoToken());
entity.setRemarkName(normalizeOptional(request.getRemarkName()));
entity.setZiniaoAccountName(ziniaoAccountName);
entity.setZiniaoToken(ziniaoToken);
if (tokenChanged) {
entity.setIpWhitelistStatus(IP_WHITELIST_STATUS_UNKNOWN);
entity.setIpWhitelistCheckedAt(null);
entity.setIpWhitelistMessage(null);
}
shopKeyMapper.updateById(entity);
triggerShopIndexRefresh();
return toItemVo(getById(id));
@@ -101,6 +110,9 @@ public class ShopKeyService {
vo.setRemarkName(entity.getRemarkName());
vo.setZiniaoAccountName(entity.getZiniaoAccountName());
vo.setZiniaoToken(entity.getZiniaoToken());
vo.setIpWhitelistStatus(entity.getIpWhitelistStatus());
vo.setIpWhitelistCheckedAt(entity.getIpWhitelistCheckedAt());
vo.setIpWhitelistMessage(entity.getIpWhitelistMessage());
vo.setCreatedAt(entity.getCreatedAt());
vo.setUpdatedAt(entity.getUpdatedAt());
return vo;
@@ -210,33 +210,23 @@ public class SkipPriceAsinService {
}
@Transactional
public SkipPriceAsinItemVo createOrUpdate(SkipPriceAsinCreateRequest request, Long operatorId, boolean superAdmin) {
public SkipPriceAsinItemVo create(SkipPriceAsinCreateRequest request, Long operatorId, boolean superAdmin) {
ShopManageGroupEntity group = shopManageGroupService.getAccessibleById(request.getGroupId(), operatorId, superAdmin);
String shopName = normalizeRequired(request.getShopName(), "店铺名不能为空");
Set<String> countries = normalizeCountries(request.getCountries());
Map<String, String> countryAsinMap = normalizeCountryAsinMap(countries, request);
Map<String, BigDecimal> countryMinimumPriceMap = normalizeCountryMinimumPriceMap(countries, request);
SkipPriceAsinEntity entity = skipPriceAsinMapper.selectOne(new LambdaQueryWrapper<SkipPriceAsinEntity>()
.eq(SkipPriceAsinEntity::getGroupId, group.getId())
.eq(SkipPriceAsinEntity::getShopName, shopName)
.last("LIMIT 1"));
if (entity == null) {
entity = new SkipPriceAsinEntity();
SkipPriceAsinEntity entity = new SkipPriceAsinEntity();
entity.setGroupId(group.getId());
entity.setShopName(shopName);
}
for (Map.Entry<String, String> entry : countryAsinMap.entrySet()) {
String country = entry.getKey();
setCountryData(entity, country, entry.getValue(), countryMinimumPriceMap.get(country));
}
if (entity.getId() == null) {
skipPriceAsinMapper.insert(entity);
} else {
skipPriceAsinMapper.updateById(entity);
}
SkipPriceAsinEntity saved = getById(entity.getId());
return toItemVo(saved, group.getGroupName());
}
@@ -47,6 +47,14 @@ public class SimilarAsinCozeClient {
}
public List<SimilarAsinResultRowDto> inspect(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
return inspect(rows, prompt, apiKey, imgSwitch, false);
}
public List<SimilarAsinResultRowDto> inspect(List<SimilarAsinResultRowDto> rows,
String prompt,
String apiKey,
boolean imgSwitch,
boolean categorySwitch) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
@@ -55,7 +63,7 @@ public class SimilarAsinCozeClient {
return rows.stream().map(this::copy).toList();
}
try {
return inspectWithFallback(rows, prompt, apiKey, imgSwitch);
return inspectWithFallback(rows, prompt, apiKey, imgSwitch, categorySwitch);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[similar-asin] coze batch failed size={} err={}", rows.size(), failureMessage);
@@ -75,9 +83,10 @@ public class SimilarAsinCozeClient {
String prompt,
String apiKey,
boolean imgSwitch,
boolean categorySwitch,
CozeCredentialRef credential) throws Exception {
CozeCredentialRef resolvedCredential = resolveCredential(credential);
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, resolvedCredential));
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, resolvedCredential));
ensureSuccess(submitRoot);
return new CozeSubmitResponse(
extractExecuteId(submitRoot),
@@ -87,6 +96,14 @@ public class SimilarAsinCozeClient {
);
}
public CozeSubmitResponse submitWorkflow(List<SimilarAsinResultRowDto> rows,
String prompt,
String apiKey,
boolean imgSwitch,
CozeCredentialRef credential) throws Exception {
return submitWorkflow(rows, prompt, apiKey, imgSwitch, false, credential);
}
public CozePollResponse pollWorkflow(String executeId) throws Exception {
return pollWorkflow(executeId, null);
}
@@ -119,12 +136,12 @@ public class SimilarAsinCozeClient {
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
private List<SimilarAsinResultRowDto> inspectWithFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
private List<SimilarAsinResultRowDto> inspectWithFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) {
try {
if (rows.size() == 1) {
return inspectSingleRowWithRetry(rows, prompt, apiKey, imgSwitch);
return inspectSingleRowWithRetry(rows, prompt, apiKey, imgSwitch, categorySwitch);
}
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch);
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch, categorySwitch);
if (attempt.resolvedCount() < rows.size()) {
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
}
@@ -135,17 +152,17 @@ public class SimilarAsinCozeClient {
log.warn("[similar-asin] coze batch fallback split size={} left={} right={} err={}",
rows.size(), middle, rows.size() - middle, failureMessage(ex));
List<SimilarAsinResultRowDto> merged = new ArrayList<>(rows.size());
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, imgSwitch));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, imgSwitch));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, imgSwitch, categorySwitch));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, imgSwitch, categorySwitch));
return merged;
}
throw propagate(ex);
}
}
private List<SimilarAsinResultRowDto> inspectPartitionWithFailureFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
private List<SimilarAsinResultRowDto> inspectPartitionWithFailureFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) {
try {
return inspectWithFallback(rows, prompt, apiKey, imgSwitch);
return inspectWithFallback(rows, prompt, apiKey, imgSwitch, categorySwitch);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[similar-asin] coze partition failed size={} err={}", rows.size(), failureMessage);
@@ -153,12 +170,12 @@ public class SimilarAsinCozeClient {
}
}
private List<SimilarAsinResultRowDto> inspectSingleRowWithRetry(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
private List<SimilarAsinResultRowDto> inspectSingleRowWithRetry(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
SimilarAsinResultRowDto row = rows.getFirst();
PartialCozeResultException lastFailure = null;
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
try {
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch);
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch, categorySwitch);
if (attempt.resolvedCount() == rows.size()) {
return attempt.mergedRows();
}
@@ -189,16 +206,16 @@ public class SimilarAsinCozeClient {
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
}
private InspectAttempt inspectOnce(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, imgSwitch);
private InspectAttempt inspectOnce(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, imgSwitch, categorySwitch);
List<CozeResult> results = parseResults(raw);
List<SimilarAsinResultRowDto> merged = mergeRows(rows, results);
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
}
private String runWorkflowAsyncAndWait(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
private String runWorkflowAsyncAndWait(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
CozeCredentialRef credential = nextCredential();
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, credential));
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, credential));
ensureSuccess(submitRoot);
String immediateData = extractResultDataText(submitRoot);
@@ -243,8 +260,9 @@ public class SimilarAsinCozeClient {
String prompt,
String apiKey,
boolean imgSwitch,
boolean categorySwitch,
CozeCredentialRef credential) {
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey, imgSwitch);
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey, imgSwitch, categorySwitch);
Map<String, Object> body = new LinkedHashMap<>();
body.put("workflow_id", credential.workflowId());
body.put("parameters", parameters);
@@ -366,6 +384,14 @@ public class SimilarAsinCozeClient {
}
private Map<String, Object> buildParameters(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
return buildParameters(rows, prompt, apiKey, imgSwitch, false);
}
private Map<String, Object> buildParameters(List<SimilarAsinResultRowDto> rows,
String prompt,
String apiKey,
boolean imgSwitch,
boolean categorySwitch) {
List<String> asins = rows.stream().map(row -> safeText(row.getAsin())).toList();
List<String> titles = rows.stream().map(row -> safeText(firstNonBlank(row.getTitle(), row.getAsin()))).toList();
List<String> skus = rows.stream().map(row -> safeText(row.getSku())).toList();
@@ -388,6 +414,7 @@ public class SimilarAsinCozeClient {
parameters.put("api_key", apiKey.trim());
}
parameters.put("img_switch", imgSwitch);
parameters.put("category_switch", categorySwitch);
return parameters;
}
@@ -827,7 +854,14 @@ public class SimilarAsinCozeClient {
|| !normalize(row.getPatentRisk()).isBlank()
|| !normalize(row.getConclusion()).isBlank()
|| !normalize(row.getIsStock()).isBlank()
|| !normalize(row.getSimilarity()).isBlank();
|| !normalize(row.getSimilarity()).isBlank()
|| !normalize(row.getIsConform()).isBlank()
|| !normalize(row.getReason()).isBlank()
|| !normalize(row.getCategory()).isBlank()
|| !normalize(row.getStatus()).isBlank()
|| !normalize(row.getMainUrl()).isBlank()
|| !normalize(row.getPuzzleImg1()).isBlank()
|| !normalize(row.getPuzzleImg2()).isBlank();
}
private void ensureSuccess(JsonNode root) {
@@ -978,7 +1012,21 @@ public class SimilarAsinCozeClient {
|| item.has("appearanceReason")
|| item.has("patent_reason")
|| item.has("patentReason")
|| item.has("patent reason"));
|| item.has("patent reason")
|| item.has("main_url")
|| item.has("mainUrl")
|| item.has("main_image_url")
|| item.has("mainImageUrl")
|| item.has("main_img")
|| item.has("mainImg")
|| item.has("puzzle_img1")
|| item.has("puzzleImg1")
|| item.has("puzzle_img_1")
|| item.has("puzzleImg_1")
|| item.has("puzzle_img2")
|| item.has("puzzleImg2")
|| item.has("puzzle_img_2")
|| item.has("puzzleImg_2"));
}
private boolean isSuccessfulWorkflowStatus(String status) {
@@ -37,4 +37,9 @@ public class SimilarAsinParseRequest {
@JsonAlias({"imgSwitch"})
@Schema(description = "传递给 Coze workflow parameters.img_switch 的图片检测开关,true 为开启,false 为关闭。")
private Boolean imgSwitch = Boolean.FALSE;
@JsonProperty("category_switch")
@JsonAlias({"categorySwitch"})
@Schema(description = "传递给 Coze workflow parameters.category_switch 的类目检测开关,true 为开启,false 为关闭。")
private Boolean categorySwitch = Boolean.FALSE;
}
@@ -20,6 +20,9 @@ public class SimilarAsinParsedPayloadDto {
@Schema(description = "传递给 Coze workflow parameters.img_switch 的图片检测开关")
private Boolean imgSwitch = Boolean.FALSE;
@Schema(description = "传递给 Coze workflow parameters.category_switch 的类目检测开关")
private Boolean categorySwitch = Boolean.FALSE;
@Schema(description = "本次解析的源文件列表")
private List<SimilarAsinSourceFileDto> sourceFiles = new ArrayList<>();
@@ -36,6 +36,9 @@ public class SimilarAsinParseVo {
@Schema(description = "图片检测开关,true 为开启,false 为关闭")
private Boolean imgSwitch = Boolean.FALSE;
@Schema(description = "类目检测开关,true 为开启,false 为关闭")
private Boolean categorySwitch = Boolean.FALSE;
@Schema(description = "兼容旧前端的平铺有效行列表,现为全部有效行")
private List<SimilarAsinParsedRowVo> items = new ArrayList<>();
@@ -83,6 +83,8 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
@@ -96,7 +98,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -219,6 +221,23 @@ public class SimilarAsinTaskService {
"阿里巴巴图片1",
"阿里巴巴图片2"
);
private static final Set<String> CATEGORY_RESULT_HEADER_ALIASES = Set.of(
FailedStatusRowFilter.canonicalizeHeader("是否符合类目"),
FailedStatusRowFilter.canonicalizeHeader("is_conform"),
FailedStatusRowFilter.canonicalizeHeader("conform")
);
private static final Set<String> COZE_RESULT_HEADER_ALIASES = Set.of(
FailedStatusRowFilter.canonicalizeHeader("是否有货"),
FailedStatusRowFilter.canonicalizeHeader("is_stock"),
FailedStatusRowFilter.canonicalizeHeader("相似度"),
FailedStatusRowFilter.canonicalizeHeader("similarity"),
FailedStatusRowFilter.canonicalizeHeader("是否符合类目"),
FailedStatusRowFilter.canonicalizeHeader("is_conform"),
FailedStatusRowFilter.canonicalizeHeader("不符合理由"),
FailedStatusRowFilter.canonicalizeHeader("reason"),
FailedStatusRowFilter.canonicalizeHeader("产品类目"),
FailedStatusRowFilter.canonicalizeHeader("category")
);
private static final int IMG_COL_MAIN = 12;
private static final int IMG_COL_PUZZLE1 = 13;
@@ -359,6 +378,7 @@ public class SimilarAsinTaskService {
List<String> mergedHeaders = new ArrayList<>();
int totalRows = 0;
int droppedRows = 0;
boolean categoryRetryRequired = false;
long parseStartedAt = System.nanoTime();
for (SimilarAsinSourceFileDto source : sourceFiles) {
if (source.getFileKey() == null || source.getFileKey().isBlank()) {
@@ -374,10 +394,17 @@ public class SimilarAsinTaskService {
droppedRows += parsed.droppedRows();
allRows.addAll(parsed.allRows());
mergeHeaders(mergedHeaders, parsed.headers());
categoryRetryRequired |= parsed.categoryRetryRequired();
}
if (allRows.isEmpty()) {
throw new BusinessException("未解析到有效 ASIN 数据");
}
boolean requestedCategorySwitch = Boolean.TRUE.equals(request.getCategorySwitch());
request.setCategorySwitch(requestedCategorySwitch || categoryRetryRequired);
if (!requestedCategorySwitch && categoryRetryRequired) {
log.info("[similar-asin] category retry detected from failed result rows, enable category_switch automatically files={} rows={}",
sourceFiles.size(), allRows.size());
}
long parsedAt = System.nanoTime();
List<SimilarAsinParsedGroupVo> groups = buildParsedGroups(allRows);
long groupedAt = System.nanoTime();
@@ -405,11 +432,11 @@ public class SimilarAsinTaskService {
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), sourceFiles, mergedHeaders, groups, allRows);
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, mergedHeaders, groups, allRows);
long payloadBuiltAt = System.nanoTime();
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
long payloadStoredAt = System.nanoTime();
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), sourceFiles, parsedPayloadPointer));
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, parsedPayloadPointer));
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
@@ -449,6 +476,7 @@ public class SimilarAsinTaskService {
vo.setGroupCount(groups.size());
vo.setAiPrompt(normalize(request.getAiPrompt()));
vo.setImgSwitch(Boolean.TRUE.equals(request.getImgSwitch()));
vo.setCategorySwitch(Boolean.TRUE.equals(request.getCategorySwitch()));
vo.setItems(new ArrayList<>(allRows));
vo.setGroups(groups);
long finishedAt = System.nanoTime();
@@ -1228,10 +1256,11 @@ public class SimilarAsinTaskService {
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
boolean imgSwitch = readImgSwitch(task);
boolean categorySwitch = readCategorySwitch(task);
int batchSize = resolveCozeBatchSize(imgSwitch);
List<SimilarAsinResultRowDto> result = new ArrayList<>();
for (int i = 0; i < items.size(); i += batchSize) {
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, imgSwitch));
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, imgSwitch, categorySwitch));
if (progressHook != null) {
progressHook.run();
}
@@ -1753,6 +1782,73 @@ public class SimilarAsinTaskService {
return true;
}
private boolean readCategorySwitch(FileTaskEntity task) {
try {
JsonNode root = objectMapper.readTree(task.getResultJson());
JsonNode savedSwitch = root == null ? null : root.get("categorySwitch");
if (savedSwitch != null && !savedSwitch.isNull()) {
return savedSwitch.asBoolean(false);
}
return Boolean.TRUE.equals(readParsedPayload(task).getCategorySwitch());
} catch (Exception ex) {
throw new BusinessException("读取相似ASIN类目检测开关失败", ex);
}
}
@Transactional
public void handleResultFileJobFailure(TaskFileJobEntity job, String message) {
if (job == null || job.getTaskId() == null) {
return;
}
try (TaskDistributedLockService.LockHandle ignored =
requireTaskLock(job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS)) {
Long finalizedTaskId = transactionManager == null
? persistResultFileJobFailure(job, message)
: inNewTransaction(() -> persistResultFileJobFailure(job, message));
if (finalizedTaskId != null) {
taskCacheService.deleteTaskCache(finalizedTaskId);
clearPoisonWindow(finalizedTaskId);
}
}
}
private Long persistResultFileJobFailure(TaskFileJobEntity job, String message) {
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) {
return null;
}
String owner = ownerFromTask(task);
if (!isOwnerCurrent(owner)) {
log.warn("[similar-asin] finalize exhausted result job from another instance taskId={} jobId={} owner={} current={}",
task.getId(), job.getId(), owner, currentInstanceId());
}
String failureMessage = firstNonBlank(message, "相似ASIN结果文件生成失败");
LocalDateTime now = LocalDateTime.now();
task.setStatus(STATUS_FAILED);
task.setSuccessFileCount(0);
task.setFailedFileCount(1);
task.setErrorMessage(failureMessage);
task.setUpdatedAt(now);
task.setFinishedAt(now);
fileTaskMapper.updateById(task);
FileResultEntity result = job.getResultId() == null ? null : fileResultMapper.selectById(job.getResultId());
if (result != null && MODULE_TYPE.equals(result.getModuleType())) {
result.setSuccess(0);
result.setErrorMessage(failureMessage);
result.setResultFileUrl(null);
result.setResultFileSize(0L);
fileResultMapper.updateById(result);
}
saveFileBuildProgress(task, job, 1, 1, failureMessage);
return task.getId();
}
private void finalizeExhaustedResultFileJob(TaskFileJobEntity job, String message) {
handleResultFileJobFailure(job, message);
taskFileJobService.markFailureFinalized(job.getId(), message);
}
@Scheduled(fixedDelayString = "${aiimage.similar-asin.coze-poll-delay-ms:30000}")
public void pollPendingCozeJobs() {
DistributedJobLockService.LockHandle lockHandle =
@@ -1949,6 +2045,7 @@ public class SimilarAsinTaskService {
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
boolean imgSwitch = readImgSwitch(task);
boolean categorySwitch = readCategorySwitch(task);
int batchSize = resolveCozeBatchSize(imgSwitch);
// P1-1检测到 720712008 风暴时强制把 batch 降到 1隔离毒行
// 持续 5+ 次提交命中率 40% 才会触发正常波动不影响吞吐
@@ -2013,7 +2110,7 @@ public class SimilarAsinTaskService {
int batchIndex = 1;
for (int i = 0; i < submitLimit; i += batchSize) {
List<CozeCandidate> batchCandidates = readyCandidates.subList(i, Math.min(i + batchSize, submitLimit));
pending |= submitCozeBatch(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, allRowsByBaseId);
pending |= submitCozeBatch(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, categorySwitch, allRowsByBaseId);
batchIndex++;
}
return pending || countPendingCozeStates(task.getId()) > 0;
@@ -2129,6 +2226,7 @@ public class SimilarAsinTaskService {
String prompt,
String apiKey,
boolean imgSwitch,
boolean categorySwitch,
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
if (batchCandidates == null || batchCandidates.isEmpty()) {
return false;
@@ -2155,7 +2253,7 @@ public class SimilarAsinTaskService {
SimilarAsinCozeClient.CozeCredentialRef credential = cozeClient.nextCredential();
try {
SimilarAsinCozeClient.CozeSubmitResponse submit = submitCozeWorkflowThrottled(
batchRows, prompt, apiKey, imgSwitch, credential, true);
batchRows, prompt, apiKey, imgSwitch, categorySwitch, credential, true);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List<SimilarAsinResultRowDto> cozeRows = cozeClient.mergeRowsFromDataText(batchRows, submit.immediateData());
String emptyResultMessage = emptyCozeResultMessage(cozeRows, batchRows.size());
@@ -2520,7 +2618,7 @@ public class SimilarAsinTaskService {
try {
SimilarAsinCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
cozeClient.credentialByName(context.credentialName()), false);
readCategorySwitch(task), cozeClient.credentialByName(context.credentialName()), false);
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List<SimilarAsinResultRowDto> cozeRows =
@@ -2582,6 +2680,7 @@ public class SimilarAsinTaskService {
String prompt,
String apiKey,
boolean imgSwitch,
boolean categorySwitch,
SimilarAsinCozeClient.CozeCredentialRef credential,
boolean allowCredentialFallback) throws Exception {
int attempts = allowCredentialFallback ? Math.max(1, cozeClient.configuredCredentialCount()) : 1;
@@ -2615,7 +2714,7 @@ public class SimilarAsinTaskService {
}
try (lockHandle; borrowedCredential) {
SimilarAsinCozeClient.CozeSubmitResponse response =
cozeClient.submitWorkflow(rows, prompt, apiKey, imgSwitch, currentCredential);
cozeClient.submitWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, currentCredential);
// 提交完成立即记录时间戳用于下次进入循环时计算节流等待
lastCozeSubmitAtByCredential.put(credentialKey, System.currentTimeMillis());
return response;
@@ -2807,7 +2906,7 @@ public class SimilarAsinTaskService {
int partIndex = 1;
for (List<SimilarAsinResultRowDto> partRows : partitions) {
SimilarAsinCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task), readCategorySwitch(task),
cozeClient.credentialByName(context.credentialName()), false);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List<SimilarAsinResultRowDto> cozeRows =
@@ -2941,7 +3040,7 @@ public class SimilarAsinTaskService {
try {
taskFileJobService.touchRunningIfStale(context.jobId(), properties.getDbJobTouchIntervalMillis());
SimilarAsinCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task), readCategorySwitch(task),
cozeClient.credentialByName(context.credentialName()), false);
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
@@ -3280,6 +3379,9 @@ public class SimilarAsinTaskService {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId());
if (job != null) {
taskFileJobService.markFailed(job, firstNonBlank(ex.getMessage(), "相似ASIN结果文件生成失败"));
if (taskFileJobService.isRetryExhausted(job.getId())) {
finalizeExhaustedResultFileJob(job, ex.getMessage());
}
}
log.warn("[相似ASIN] Coze 异步收尾失败 任务ID={} 结果ID={} 错误={}",
taskId, context.resultId(), ex.getMessage(), ex);
@@ -3303,6 +3405,10 @@ public class SimilarAsinTaskService {
if (job == null || "SUCCESS".equals(job.getStatus())) {
return;
}
if (taskFileJobService.isRetryExhausted(job.getId())) {
finalizeExhaustedResultFileJob(job, job.getErrorMessage());
return;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task != null && STATUS_RUNNING.equals(task.getStatus()) && !isResultSubmissionComplete(taskId)) {
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
@@ -3324,6 +3430,8 @@ public class SimilarAsinTaskService {
if (requeued) {
log.info("[相似ASIN] Coze 异步结果已就绪,结果文件任务已重新入队 任务ID={} 文件任务ID={} 结果ID={}",
taskId, job.getId(), context.resultId());
} else if (taskFileJobService.isRetryExhausted(job.getId())) {
finalizeExhaustedResultFileJob(job, job.getErrorMessage());
}
}
}
@@ -4160,6 +4268,7 @@ public class SimilarAsinTaskService {
}
List<SourceRows> sourceRows = splitRowsBySourceFile(parsed, parsed.getAllItems(), result.getSourceFilename());
List<SourceResultWorkbook> workbooks = new ArrayList<>();
List<File> workbookFiles = new ArrayList<>(sourceRows.size());
File zip = null;
// workbook 共享 taskImageCache多源场景下相同 URL 仅下载一次
// 配合 embed() 写完即 remove()cache 仅承载 in-flight 图片
@@ -4170,26 +4279,12 @@ public class SimilarAsinTaskService {
: BoundedImageCache.DEFAULT_MAX_BYTES;
BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes);
try {
// P3-1单源文件场景仍走串行降级路径避免引入线程切换开销
// 源文件场景把 writeResultWorkbook 投到 assembleExecutor 上并发跑
// 1000+ ×N 源文件的 assemble 阶段总耗时直接除以 N受池大小 4 限制
// assembleExecutor 是固定 4 线程池sourceRows.size() 4 时全部并行
// > 4 时多余源文件排队避免 4×5000 行同时打开图片缓存爆堆
if (sourceRows.size() <= 1) {
for (SourceRows item : sourceRows) {
String filename = safeFileStem(item.sourceFilename()) + "-result.xlsx";
String tempFilename = safeFileStem(item.sourceFilename())
+ "-" + task.getId()
+ "-" + result.getId()
+ "-" + UUID.randomUUID()
+ "-result.xlsx";
File xlsx = new File(outputDir, tempFilename);
writeResultWorkbook(xlsx, item.rows(), resultMap, taskImageCache);
workbooks.add(new SourceResultWorkbook(xlsx, filename, item.rows().size()));
}
} else {
// 单源和多源统一提交真实 Future确保超时取消能中断实际 workbook 线程
// 所有源文件共享同一个绝对 deadline避免逐个 Future 各等待一轮完整超时
int timeoutMinutes = Math.max(1, properties.getResultFileTimeoutMinutes());
long deadlineNanos = System.nanoTime() + TimeUnit.MINUTES.toNanos(timeoutMinutes);
long assembleStart = System.currentTimeMillis();
List<CompletableFuture<SourceResultWorkbook>> futures = new ArrayList<>(sourceRows.size());
List<Future<SourceResultWorkbook>> futures = new ArrayList<>(sourceRows.size());
for (SourceRows item : sourceRows) {
final SourceRows captured = item;
final String filename = safeFileStem(captured.sourceFilename()) + "-result.xlsx";
@@ -4198,42 +4293,43 @@ public class SimilarAsinTaskService {
+ "-" + result.getId()
+ "-" + UUID.randomUUID()
+ "-result.xlsx";
futures.add(CompletableFuture.supplyAsync(() -> {
File xlsx = new File(outputDir, tempFilename);
final File xlsx = new File(outputDir, tempFilename);
workbookFiles.add(xlsx);
futures.add(assembleExecutor.submit(() -> {
ensureResultAssemblyNotInterrupted();
writeResultWorkbook(xlsx, captured.rows(), resultMap, taskImageCache);
ensureResultAssemblyNotInterrupted();
return new SourceResultWorkbook(xlsx, filename, captured.rows().size());
}, assembleExecutor));
}));
}
for (CompletableFuture<SourceResultWorkbook> future : futures) {
try {
// 单源文件 30 分钟硬上限超时直接抛错避免被慢源永久阻塞
workbooks.add(future.get(30, TimeUnit.MINUTES));
} catch (InterruptedException ie) {
for (Future<SourceResultWorkbook> future : futures) {
long remainingNanos = deadlineNanos - System.nanoTime();
if (remainingNanos <= 0L) {
throw new TimeoutException("result file deadline reached");
}
workbooks.add(future.get(remainingNanos, TimeUnit.NANOSECONDS));
}
} catch (InterruptedException ex) {
cancelResultAssemblyFutures(futures);
Thread.currentThread().interrupt();
for (CompletableFuture<SourceResultWorkbook> remaining : futures) {
remaining.cancel(true);
throw new BusinessException("生成相似ASIN检测结果中断", ex);
} catch (TimeoutException ex) {
cancelResultAssemblyFutures(futures);
throw new BusinessException("生成相似ASIN检测结果超时(" + timeoutMinutes + " 分钟)", ex);
} catch (CancellationException ex) {
cancelResultAssemblyFutures(futures);
throw new BusinessException("生成相似ASIN检测结果已取消", ex);
} catch (ExecutionException ex) {
cancelResultAssemblyFutures(futures);
Throwable cause = ex.getCause();
if (cause instanceof BusinessException businessException) {
throw businessException;
}
throw new BusinessException("生成相似ASIN检测结果中断", ie);
} catch (TimeoutException te) {
for (CompletableFuture<SourceResultWorkbook> remaining : futures) {
remaining.cancel(true);
}
throw new BusinessException("生成相似ASIN检测结果超时(30 分钟)", te);
} catch (ExecutionException ee) {
Throwable cause = ee.getCause();
for (CompletableFuture<SourceResultWorkbook> remaining : futures) {
remaining.cancel(true);
}
if (cause instanceof BusinessException be) {
throw be;
}
throw new BusinessException("生成相似ASIN检测结果失败",
cause != null ? cause : ee);
}
}
log.info("[similar-asin] assemble parallel finished taskId={} sources={} costMs={}",
task.getId(), sourceRows.size(), System.currentTimeMillis() - assembleStart);
throw new BusinessException("生成相似ASIN检测结果失败", cause != null ? cause : ex);
}
log.info("[similar-asin] assemble workbooks finished taskId={} sources={} timeoutMinutes={} costMs={}",
task.getId(), sourceRows.size(), timeoutMinutes, System.currentTimeMillis() - assembleStart);
log.info("[similar-asin] assemble image-cache stats taskId={} sources={} cacheSize={} currentBytes={} evictedCount={} evictedBytes={} maxBytes={}",
task.getId(), sourceRows.size(), taskImageCache.size(),
taskImageCache.currentBytes(), taskImageCache.evictedCount(),
@@ -4272,9 +4368,9 @@ public class SimilarAsinTaskService {
result.setResultContentType(contentType);
result.setRowCount(parsed.getAllItems().size());
} finally {
for (SourceResultWorkbook workbook : workbooks) {
if (workbook.file().exists() && !workbook.file().delete()) {
log.warn("[similar-asin] delete temp xlsx failed file={}", workbook.file());
for (File workbookFile : workbookFiles) {
if (workbookFile.exists() && !workbookFile.delete()) {
log.warn("[similar-asin] delete temp xlsx failed file={}", workbookFile);
}
}
if (zip != null && zip.exists() && !zip.delete()) {
@@ -4482,9 +4578,18 @@ public class SimilarAsinTaskService {
List<SimilarAsinParsedRowVo> rowsToWrite,
Map<String, SimilarAsinResultRowDto> resultMap,
Map<String, SimilarAsinImageEmbedder.ResizedImage> taskImageCache) {
ensureResultAssemblyNotInterrupted();
// Excel 365 "Place in Cell" 图片写完 workbook patch richData 单元格图片结构
// 这不是浮动 Drawing因此点击图片区域会选中单元格图片不能被拖到任意位置也不需要工作表保护
ExcelCellImageWriter.Session excelCellImageSession = ExcelCellImageWriter.createSession();
SimilarAsinImageEmbedder.ImageSpool imageSpool;
try {
Path spoolDir = Files.createTempDirectory(xlsx.getParentFile().toPath(), "similar-asin-images-");
imageSpool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
} catch (IOException ex) {
throw new BusinessException("create Similar ASIN image spool failed", ex);
}
try {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
Sheet sheet = workbook.createSheet("相似asin检测");
CellStyle headerStyle = workbook.createCellStyle();
@@ -4506,17 +4611,16 @@ public class SimilarAsinTaskService {
cell.setCellStyle(headerStyle);
}
// P2-8第一遍预扫所有图片 URL并行下载到 taskImageCache
// POI 写入仍单线程串行注册 cell imagecache 命中直接 resize + register下载/写入解耦
// 预扫图片 URL 并行下载到任务级临时目录POI 仍按行串行注册 cell image
List<String> prefetchUrls = collectImageUrlsForPrefetch(rowsToWrite, resultMap);
if (!prefetchUrls.isEmpty()) {
// P2-11先一次性把 DB cache 命中的字节填入 taskImageCache避免再次走网络
// 命中部分从 prefetchUrls 剔除剩余的真正未命中的 URL 才走 imageEmbedder.prefetch 网络下载
// DB cache 命中的缩略图也立即落到任务临时目录避免整批字节停留在堆中
long dbCacheStart = System.currentTimeMillis();
List<String> remainingUrls = new ArrayList<>(prefetchUrls.size());
int dbHit = 0;
for (String url : prefetchUrls) {
if (url == null || taskImageCache.containsKey(url)) {
ensureResultAssemblyNotInterrupted();
if (url == null || imageSpool.get(url) != null || taskImageCache.containsKey(url)) {
continue;
}
byte[] cachedBytes = imagePrefetchService.lookup(url);
@@ -4526,7 +4630,7 @@ public class SimilarAsinTaskService {
}
SimilarAsinImageEmbedder.ResizedImage thumb = imageEmbedder.decodeCachedThumb(cachedBytes);
if (thumb != null) {
taskImageCache.putIfAbsent(url, thumb);
imageSpool.put(url, thumb);
dbHit++;
} else {
remainingUrls.add(url);
@@ -4536,14 +4640,16 @@ public class SimilarAsinTaskService {
prefetchUrls.size(), dbHit, remainingUrls.size(), System.currentTimeMillis() - dbCacheStart);
if (!remainingUrls.isEmpty()) {
long prefetchStart = System.currentTimeMillis();
imageEmbedder.prefetch(remainingUrls, taskImageCache);
log.info("[similar-asin] image prefetch finished urls={} cached={} costMs={}",
remainingUrls.size(), taskImageCache.size(), System.currentTimeMillis() - prefetchStart);
imageEmbedder.prefetchToDisk(remainingUrls, imageSpool);
ensureResultAssemblyNotInterrupted();
log.info("[similar-asin] image disk-prefetch finished urls={} spooled={} costMs={}",
remainingUrls.size(), imageSpool.size(), System.currentTimeMillis() - prefetchStart);
}
}
int rowIndex = 1;
for (SimilarAsinParsedRowVo parsedRow : rowsToWrite == null ? List.<SimilarAsinParsedRowVo>of() : rowsToWrite) {
ensureResultAssemblyNotInterrupted();
SimilarAsinResultRowDto resultRow = findResultRow(parsedRow, resultMap);
Row row = sheet.createRow(rowIndex);
int col = 0;
@@ -4575,21 +4681,21 @@ public class SimilarAsinTaskService {
// 行高固定到 IMAGE_ROW_HEIGHT_POINTSExcel 上限 409pt
row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS);
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_MAIN,
resultRow.getMainUrl(), row, taskImageCache, excelCellImageSession);
resultRow.getMainUrl(), row, taskImageCache, excelCellImageSession, imageSpool);
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE1,
resultRow.getPuzzleImg1(), row, taskImageCache, excelCellImageSession);
resultRow.getPuzzleImg1(), row, taskImageCache, excelCellImageSession, imageSpool);
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE2,
resultRow.getPuzzleImg2(), row, taskImageCache, excelCellImageSession);
resultRow.getPuzzleImg2(), row, taskImageCache, excelCellImageSession, imageSpool);
}
rowIndex++;
}
ensureResultAssemblyNotInterrupted();
workbook.write(fos);
workbook.dispose();
} catch (Exception ex) {
throw new BusinessException("生成相似ASIN检测结果失败", ex);
}
if (!excelCellImageSession.isEmpty()) {
try {
ensureResultAssemblyNotInterrupted();
ExcelCellImageWriter.patchXlsxFile(xlsx, excelCellImageSession);
} catch (IOException ex) {
String cause = ex.toString();
@@ -4598,6 +4704,32 @@ public class SimilarAsinTaskService {
throw new BusinessException("写入相似ASIN结果 Excel 单元格图片失败: " + cause, ex);
}
}
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("生成相似ASIN检测结果失败", ex);
} finally {
try {
imageSpool.close();
} catch (IOException ex) {
log.warn("[similar-asin] delete image spool failed xlsx={} err={}",
xlsx.getAbsolutePath(), ex.getMessage());
}
}
}
private static void cancelResultAssemblyFutures(List<? extends Future<?>> futures) {
for (Future<?> future : futures) {
if (future != null && !future.isDone()) {
future.cancel(true);
}
}
}
private static void ensureResultAssemblyNotInterrupted() {
if (Thread.currentThread().isInterrupted()) {
throw new BusinessException("生成相似ASIN检测结果中断");
}
}
/**
@@ -4714,24 +4846,29 @@ public class SimilarAsinTaskService {
vo.setValues(readRowValues(row, headers, formatter));
allRows.add(vo);
}
boolean includeBlankStatusRows = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
boolean resultWorkbook = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
boolean firstPassResultWorkbook = isFirstPassResultWorkbook(headers, allRows);
boolean includeBlankStatusRows = resultWorkbook && !firstPassResultWorkbook;
FailedStatusRowFilter.FilterResult<SimilarAsinParsedRowVo> filteredRows =
FailedStatusRowFilter.retainRows(
allRows,
statusCol >= 0,
rowVo -> statusHeader == null || rowVo.getValues() == null ? "" : rowVo.getValues().get(statusHeader),
status -> FailedStatusRowFilter.matchesFailedStatus(status)
|| firstPassResultWorkbook
|| (includeBlankStatusRows && FailedStatusRowFilter.isBlankStatus(status))
);
dropped += filteredRows.filteredCount();
allRows = new ArrayList<>(filteredRows.rows());
boolean categoryRetryRequired = shouldEnableCategorySwitchForRetry(
headers, allRows, includeBlankStatusRows);
if (statusCol >= 0 && validRows > 0 && allRows.isEmpty()) {
throw new BusinessException(FailedStatusRowFilter.noMatchedRowsMessage());
}
if (allRows.isEmpty()) {
throw new BusinessException("no valid similar ASIN rows");
}
return new ParsedWorkbook(total, dropped, headers, allRows);
return new ParsedWorkbook(total, dropped, headers, allRows, categoryRetryRequired);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
@@ -4763,7 +4900,10 @@ public class SimilarAsinTaskService {
|| hasUsableCozeField(row.getIsConform())
|| hasUsableCozeField(row.getReason())
|| hasUsableCozeField(row.getCategory())
|| hasUsableCozeField(row.getStatus());
|| hasUsableCozeField(row.getStatus())
|| hasText(row.getMainUrl())
|| hasText(row.getPuzzleImg1())
|| hasText(row.getPuzzleImg2());
}
static String resolveResultStatus(SimilarAsinResultRowDto row, String... visibleResultValues) {
@@ -4885,7 +5025,16 @@ public class SimilarAsinTaskService {
}
private String cell(Row row, int col, DataFormatter formatter) {
return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col)));
if (col < 0) {
return "";
}
String value = normalize(formatter.formatCellValue(row.getCell(col)));
return isSpreadsheetErrorValue(value) ? "" : value;
}
private static boolean isSpreadsheetErrorValue(String value) {
String normalized = value == null ? "" : value.trim();
return normalized.startsWith("#") && normalized.endsWith("!");
}
private String baseId(String id) {
@@ -5373,11 +5522,12 @@ public class SimilarAsinTaskService {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
private String buildParsedPayloadJson(String aiPrompt, String apiKey, Boolean imgSwitch, List<SimilarAsinSourceFileDto> sourceFiles, List<String> headers, List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
private String buildParsedPayloadJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, List<SimilarAsinSourceFileDto> sourceFiles, List<String> headers, List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
payload.setAiPrompt(normalize(aiPrompt));
payload.setApiKey(normalize(apiKey));
payload.setImgSwitch(Boolean.TRUE.equals(imgSwitch));
payload.setCategorySwitch(Boolean.TRUE.equals(categorySwitch));
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
payload.setHeaders(headers == null ? List.of() : headers);
payload.setItems(allRows == null ? List.of() : new ArrayList<>(allRows));
@@ -5386,11 +5536,12 @@ public class SimilarAsinTaskService {
return writeJson(payload, "保存解析结果失败");
}
private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, List<SimilarAsinSourceFileDto> sourceFiles, String parsedPayloadPointer) {
private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, List<SimilarAsinSourceFileDto> sourceFiles, String parsedPayloadPointer) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("aiPrompt", normalize(aiPrompt));
payload.put("apiKey", normalize(apiKey));
payload.put("imgSwitch", Boolean.TRUE.equals(imgSwitch));
payload.put("categorySwitch", Boolean.TRUE.equals(categorySwitch));
payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream()
.map(SimilarAsinSourceFileDto::getFileKey)
.filter(Objects::nonNull)
@@ -5746,6 +5897,83 @@ public class SimilarAsinTaskService {
|| hasHeader(headers, "产品类目"));
}
static boolean isFirstPassResultWorkbook(List<String> headers,
List<SimilarAsinParsedRowVo> rows) {
if (headers == null || headers.isEmpty() || rows == null || rows.isEmpty()
|| FailedStatusRowFilter.findStatusColumnIndex(headers) < 0
|| (!hasCanonicalHeader(headers, "是否有货")
&& !hasCanonicalHeader(headers, "相似度")
&& !hasCanonicalHeader(headers, "是否符合类目")
&& !hasCanonicalHeader(headers, "不符合理由")
&& !hasCanonicalHeader(headers, "产品类目"))
|| (!hasCanonicalHeader(headers, "主图")
&& !hasCanonicalHeader(headers, "阿里巴巴图片1")
&& !hasCanonicalHeader(headers, "阿里巴巴图片2"))) {
return false;
}
String statusHeader = headers.get(FailedStatusRowFilter.findStatusColumnIndex(headers));
for (SimilarAsinParsedRowVo row : rows) {
if (row == null || FailedStatusRowFilter.matchesFailedStatus(valueByHeader(row, statusHeader))) {
return false;
}
for (Map.Entry<String, String> entry : row.getValues() == null
? Map.<String, String>of().entrySet() : row.getValues().entrySet()) {
if (COZE_RESULT_HEADER_ALIASES.contains(FailedStatusRowFilter.canonicalizeHeader(entry.getKey()))
&& !isBlankOrSpreadsheetError(entry.getValue())) {
return false;
}
}
}
return true;
}
private static boolean hasCanonicalHeader(List<String> headers, String expected) {
String canonical = FailedStatusRowFilter.canonicalizeHeader(expected);
return headers.stream()
.filter(Objects::nonNull)
.anyMatch(header -> canonical.equals(FailedStatusRowFilter.canonicalizeHeader(header)));
}
private static String valueByHeader(SimilarAsinParsedRowVo row, String header) {
if (row == null || row.getValues() == null) {
return "";
}
String canonical = FailedStatusRowFilter.canonicalizeHeader(header);
return row.getValues().entrySet().stream()
.filter(entry -> canonical.equals(FailedStatusRowFilter.canonicalizeHeader(entry.getKey())))
.map(Map.Entry::getValue)
.findFirst()
.orElse("");
}
private static boolean isBlankOrSpreadsheetError(String value) {
return value == null || value.trim().isBlank() || isSpreadsheetErrorValue(value);
}
static boolean shouldEnableCategorySwitchForRetry(List<String> headers,
List<SimilarAsinParsedRowVo> rows,
boolean resultWorkbook) {
if (!resultWorkbook || headers == null || headers.isEmpty() || rows == null || rows.isEmpty()
|| FailedStatusRowFilter.findStatusColumnIndex(headers) < 0) {
return false;
}
String categoryHeader = headers.stream()
.filter(Objects::nonNull)
.filter(header -> CATEGORY_RESULT_HEADER_ALIASES.contains(
FailedStatusRowFilter.canonicalizeHeader(header)))
.findFirst()
.orElse(null);
if (categoryHeader == null) {
return false;
}
return rows.stream()
.filter(Objects::nonNull)
.map(SimilarAsinParsedRowVo::getValues)
.anyMatch(values -> values == null
|| values.get(categoryHeader) == null
|| values.get(categoryHeader).trim().isBlank());
}
private boolean hasHeader(List<String> headers, String candidate) {
if (headers == null || headers.isEmpty()) {
return false;
@@ -5926,6 +6154,10 @@ public class SimilarAsinTaskService {
long size) {
}
private record ParsedWorkbook(int totalRows, int droppedRows, List<String> headers, List<SimilarAsinParsedRowVo> allRows) {
private record ParsedWorkbook(int totalRows,
int droppedRows,
List<String> headers,
List<SimilarAsinParsedRowVo> allRows,
boolean categoryRetryRequired) {
}
}
@@ -13,8 +13,10 @@ import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -41,6 +43,9 @@ public final class ExcelCellImageWriter {
private static final String RD_RICH_VALUE_TYPES_PART = "xl/richData/rdRichValueTypes.xml";
private static final String RICH_VALUE_REL_PART = "xl/richData/richValueRel.xml";
private static final String METADATA_PART = "xl/metadata.xml";
private static final Pattern SHEET_CELL_PATTERN = Pattern.compile(
"<c\\b(?=[^>]*\\br=\"([A-Z]+[1-9][0-9]*)\")[^>]*(?:/>|>.*?</c>)",
Pattern.DOTALL);
private static final String REL_TYPE_IMAGE =
"http://schemas.openxmlformats.org/officeDocument/2006/relationships/image";
@@ -116,7 +121,11 @@ public final class ExcelCellImageWriter {
writeEntry(zout, METADATA_PART, buildMetadata(session).getBytes(StandardCharsets.UTF_8));
int idx = 1;
for (RegisteredCellImage image : session.images) {
if (image.bytes != null) {
writeEntry(zout, mediaPart(idx), image.bytes);
} else {
writeEntry(zout, mediaPart(idx), image.path);
}
idx++;
}
}
@@ -166,26 +175,34 @@ public final class ExcelCellImageWriter {
private static PatchSheetResult patchSheet(byte[] original, Session session) {
String content = new String(original, StandardCharsets.UTF_8);
int idx = 0;
Map<String, Integer> metadataIndexByCell = new HashMap<>(session.images.size());
for (int idx = 0; idx < session.images.size(); idx++) {
metadataIndexByCell.put(session.images.get(idx).cellRef, idx + 1);
}
Set<String> missingCells = new HashSet<>(metadataIndexByCell.keySet());
Matcher matcher = SHEET_CELL_PATTERN.matcher(content);
StringBuilder patched = new StringBuilder(content.length());
int patchedCount = 0;
for (RegisteredCellImage image : session.images) {
// [MS-XLSX] §2.2.10: cell @vm 1-based valueMetadata 索引0 表示"无 metadata"
// futureMetadata/valueMetadata/rdrichvalue 内部块仍是 0-based所以这里写 idx+1
// buildMetadata/buildRdRichValue 中的 i 解耦Excel 365 vm=0 时会把 cell
// 直接当作无图片的 #VALUE! 错误格渲染任务 10498 J2 复现必须改 1-based
String replacement = "<c r=\"" + image.cellRef + "\" t=\"e\" vm=\"" + (idx + 1) + "\"><v>#VALUE!</v></c>";
Pattern pattern = Pattern.compile("<c\\b(?=[^>]*\\br=\"" + Pattern.quote(image.cellRef)
+ "\")[^>]*(?:/>|>.*?</c>)", Pattern.DOTALL);
Matcher matcher = pattern.matcher(content);
if (matcher.find()) {
content = matcher.replaceFirst(Matcher.quoteReplacement(replacement));
while (matcher.find()) {
String cellRef = matcher.group(1);
Integer metadataIndex = metadataIndexByCell.get(cellRef);
if (metadataIndex == null) {
matcher.appendReplacement(patched, Matcher.quoteReplacement(matcher.group()));
continue;
}
// [MS-XLSX] §2.2.10: cell @vm is a 1-based valueMetadata index.
String replacement = "<c r=\"" + cellRef + "\" t=\"e\" vm=\"" + metadataIndex
+ "\"><v>#VALUE!</v></c>";
matcher.appendReplacement(patched, Matcher.quoteReplacement(replacement));
missingCells.remove(cellRef);
patchedCount++;
} else {
log.warn("[excel-cell-image] cell {} not found in sheet xml, skip image", image.cellRef);
}
idx++;
matcher.appendTail(patched);
for (String cellRef : missingCells) {
log.warn("[excel-cell-image] cell {} not found in sheet xml, skip image", cellRef);
}
return new PatchSheetResult(content.getBytes(StandardCharsets.UTF_8), patchedCount);
return new PatchSheetResult(patched.toString().getBytes(StandardCharsets.UTF_8), patchedCount);
}
private static byte[] patchContentTypes(byte[] original) {
@@ -386,6 +403,16 @@ public final class ExcelCellImageWriter {
zout.closeEntry();
}
private static void writeEntry(ZipOutputStream zout, String name, Path path) throws IOException {
ZipEntry entry = new ZipEntry(name);
zout.putNextEntry(entry);
try (InputStream in = Files.newInputStream(path)) {
in.transferTo(zout);
} finally {
zout.closeEntry();
}
}
private static void writeDirectoryEntry(ZipOutputStream zout, String name) throws IOException {
ZipEntry entry = new ZipEntry(name);
zout.putNextEntry(entry);
@@ -399,7 +426,15 @@ public final class ExcelCellImageWriter {
if (jpegBytes == null || jpegBytes.length == 0) {
return;
}
images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), jpegBytes));
images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), jpegBytes, null));
}
public synchronized void registerImage(int rowIdx, int colIdx, Path jpegPath) {
if (jpegPath == null || !Files.isRegularFile(jpegPath)) {
return;
}
images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), null,
jpegPath.toAbsolutePath().normalize()));
}
public synchronized boolean isEmpty() {
@@ -426,7 +461,7 @@ public final class ExcelCellImageWriter {
}
}
private record RegisteredCellImage(String cellRef, byte[] bytes) {
private record RegisteredCellImage(String cellRef, byte[] bytes, Path path) {
}
private record PatchSheetResult(byte[] bytes, int patchedCount) {
@@ -27,14 +27,22 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.FileTime;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
@@ -42,6 +50,8 @@ import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
@@ -49,7 +59,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
/**
@@ -68,8 +77,9 @@ public class SimilarAsinImageEmbedder {
static final int DEFAULT_DOWNLOAD_TIMEOUT_SECONDS = 5;
/** P2-10retry 由 1 升到 2,配合 5s timeout 单图最坏耗时 ≈ 15s。 */
static final int DOWNLOAD_MAX_RETRY = 2;
/** P2-10:原 8 → P2-8 16 → P2-10 321000+ 行 ×3 列场景下显著拉低 assemble 阶段尾延迟。 */
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 32;
/** 图片下载、解码、缩放都在该池执行;默认限制为 8,避免结果生成占满整机 CPU。 */
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 8;
static final int DEFAULT_PREFETCH_TIMEOUT_SECONDS = 1800;
// 单元格固定尺寸图片在其中等比缩放不拉伸resize 仍按长边 1280 px 控制堆体积
public static final float IMAGE_ROW_HEIGHT_POINTS = 409f;
public static final int IMAGE_COL_WIDTH_CHARS = 80;
@@ -102,16 +112,28 @@ public class SimilarAsinImageEmbedder {
private final int downloadTimeoutSeconds;
private final int downloadPoolSize;
private final int prefetchTimeoutSeconds;
private final OkHttpClient httpClient;
private final ExecutorService downloadPool;
private final OssStorageService ossStorageService;
private final Path localImageCacheDir;
public SimilarAsinImageEmbedder(SimilarAsinProperties properties, OssStorageService ossStorageService) {
int rawTimeout = properties == null ? DEFAULT_DOWNLOAD_TIMEOUT_SECONDS : properties.getImageDownloadTimeoutSeconds();
int rawPool = properties == null ? DEFAULT_DOWNLOAD_POOL_SIZE : properties.getImageDownloadPoolSize();
int rawPrefetchTimeout = properties == null
? DEFAULT_PREFETCH_TIMEOUT_SECONDS
: properties.getImagePrefetchTimeoutSeconds();
this.downloadTimeoutSeconds = rawTimeout > 0 ? rawTimeout : DEFAULT_DOWNLOAD_TIMEOUT_SECONDS;
this.downloadPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE;
this.prefetchTimeoutSeconds = rawPrefetchTimeout > 0
? rawPrefetchTimeout
: DEFAULT_PREFETCH_TIMEOUT_SECONDS;
this.ossStorageService = Objects.requireNonNull(ossStorageService, "ossStorageService must not be null");
String configuredCacheDir = properties == null ? null : properties.getImageLocalCacheDir();
this.localImageCacheDir = configuredCacheDir == null || configuredCacheDir.isBlank()
? null
: Path.of(configuredCacheDir.trim()).toAbsolutePath().normalize();
this.httpClient = new OkHttpClient.Builder()
.connectTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
.readTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
@@ -138,6 +160,10 @@ public class SimilarAsinImageEmbedder {
return downloadPoolSize;
}
int prefetchTimeoutSeconds() {
return prefetchTimeoutSeconds;
}
OkHttpClient httpClient() {
return httpClient;
}
@@ -159,13 +185,13 @@ public class SimilarAsinImageEmbedder {
if (urls == null || urls.isEmpty() || taskImageCache == null) {
return;
}
Set<String> distinctUrls = new HashSet<>();
Set<String> distinctUrls = new LinkedHashSet<>();
for (String url : urls) {
if (url == null) {
continue;
}
String trimmed = url.trim();
if (trimmed.isEmpty() || taskImageCache.containsKey(trimmed)) {
if (trimmed.isEmpty() || taskImageCache.containsKey(trimmed) || hasLocalCachedThumb(trimmed)) {
continue;
}
distinctUrls.add(trimmed);
@@ -174,8 +200,51 @@ public class SimilarAsinImageEmbedder {
return;
}
CompletionService<Object> completion = new ExecutorCompletionService<>(downloadPool);
List<Future<?>> futures = new ArrayList<>(distinctUrls.size());
for (String url : distinctUrls) {
Iterator<String> pending = distinctUrls.iterator();
List<Future<?>> active = new ArrayList<>(Math.min(downloadPoolSize, distinctUrls.size()));
while (pending.hasNext() && active.size() < downloadPoolSize) {
active.add(submitPrefetch(completion, pending.next(), taskImageCache));
}
// 全局 deadline url 200ms 的预算clamp [15s, 120s]
long globalDeadlineMs = Math.min(120_000L, Math.max(15_000L, distinctUrls.size() * 200L));
long deadline = System.currentTimeMillis() + globalDeadlineMs;
int total = distinctUrls.size();
int done = 0;
boolean deadlineReached = false;
while (!active.isEmpty()) {
long left = deadline - System.currentTimeMillis();
if (left <= 0) {
deadlineReached = true;
break;
}
try {
Future<Object> f = completion.poll(left, TimeUnit.MILLISECONDS);
if (f == null) {
deadlineReached = true;
break;
}
active.remove(f);
done++;
if (pending.hasNext()) {
active.add(submitPrefetch(completion, pending.next(), taskImageCache));
}
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
cancelAll(active);
break;
}
}
// 超时未完成的 future 主动 cancel避免 idle 持有 OkHttp 连接
if (deadlineReached) {
cancelAll(active);
log.info("[similar-asin][image] prefetch deadline reached total={} done={} cancelled={}",
total, done, total - done);
}
}
private Future<?> submitPrefetch(CompletionService<Object> completion,
String url,
Map<String, ResizedImage> taskImageCache) {
// 显式作为 Runnable 提交 null result避免与 Callable<Object> 重载产生歧义
Runnable task = () -> {
try {
@@ -191,38 +260,106 @@ public class SimilarAsinImageEmbedder {
log.debug("[similar-asin][image] prefetch-fail url={} err={}", url, errorSummary(ex));
}
};
futures.add(completion.submit(task, null));
return completion.submit(task, null);
}
// 全局 deadline url 200ms 的预算clamp [15s, 120s]
long globalDeadlineMs = Math.min(120_000L, Math.max(15_000L, distinctUrls.size() * 200L));
long deadline = System.currentTimeMillis() + globalDeadlineMs;
int total = distinctUrls.size();
int done = 0;
while (done < total) {
long left = deadline - System.currentTimeMillis();
if (left <= 0) {
/**
* Downloads and resizes image URLs concurrently, but spills completed thumbnails to disk instead of
* retaining the full batch in the JVM heap.
*/
public void prefetchToDisk(Collection<String> urls, ImageSpool imageSpool) {
if (urls == null || urls.isEmpty() || imageSpool == null) {
return;
}
Set<String> distinctUrls = new LinkedHashSet<>();
for (String url : urls) {
if (url == null) {
continue;
}
String trimmed = url.trim();
if (!trimmed.isEmpty() && imageSpool.get(trimmed) == null) {
distinctUrls.add(trimmed);
}
}
if (distinctUrls.isEmpty()) {
return;
}
AtomicInteger localHit = new AtomicInteger();
AtomicInteger downloaded = new AtomicInteger();
AtomicInteger failed = new AtomicInteger();
CompletionService<Object> completion = new ExecutorCompletionService<>(downloadPool);
Iterator<String> pending = distinctUrls.iterator();
List<Future<?>> active = new ArrayList<>(Math.min(downloadPoolSize, distinctUrls.size()));
while (pending.hasNext() && active.size() < downloadPoolSize) {
active.add(submitDiskPrefetch(completion, pending.next(), imageSpool, localHit, downloaded, failed));
}
long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(prefetchTimeoutSeconds);
int completed = 0;
boolean deadlineReached = false;
while (!active.isEmpty()) {
long remainingNanos = deadlineNanos - System.nanoTime();
if (remainingNanos <= 0L) {
deadlineReached = true;
break;
}
try {
Future<Object> f = completion.poll(left, TimeUnit.MILLISECONDS);
if (f == null) {
Future<Object> finished = completion.poll(remainingNanos, TimeUnit.NANOSECONDS);
if (finished == null) {
deadlineReached = true;
break;
}
done++;
} catch (InterruptedException ie) {
active.remove(finished);
completed++;
if (pending.hasNext()) {
active.add(submitDiskPrefetch(
completion, pending.next(), imageSpool, localHit, downloaded, failed));
}
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
break;
cancelAll(active);
throw new IllegalStateException("image prefetch interrupted", ex);
}
}
// 超时未完成的 future 主动 cancel避免 idle 持有 OkHttp 连接
if (done < total) {
for (Future<?> f : futures) {
if (!f.isDone()) {
f.cancel(true);
if (deadlineReached) {
cancelAll(active);
}
int skipped = Math.max(0, distinctUrls.size() - completed);
log.info("[similar-asin][image] disk prefetch finished total={} completed={} spooled={} localHit={} downloaded={} failed={} skipped={} deadlineReached={}",
distinctUrls.size(), completed, imageSpool.size(), localHit.get(), downloaded.get(),
failed.get(), skipped, deadlineReached);
}
private Future<?> submitDiskPrefetch(CompletionService<Object> completion,
String url,
ImageSpool imageSpool,
AtomicInteger localHit,
AtomicInteger downloaded,
AtomicInteger failed) {
return completion.submit(() -> {
try {
ResizedImage thumb = readLocalCachedThumb(url);
if (thumb == null) {
thumb = fetchAndResizeDirect(url);
downloaded.incrementAndGet();
} else {
localHit.incrementAndGet();
}
imageSpool.put(url, thumb);
} catch (Exception ex) {
failed.incrementAndGet();
log.debug("[similar-asin][image] disk-prefetch-fail url={} err={}", url, errorSummary(ex));
}
return null;
});
}
private static void cancelAll(Collection<? extends Future<?>> futures) {
for (Future<?> future : futures) {
if (future != null && !future.isDone()) {
future.cancel(true);
}
log.info("[similar-asin][image] prefetch deadline reached total={} done={} cancelled={}",
total, done, total - done);
}
}
@@ -240,8 +377,8 @@ public class SimilarAsinImageEmbedder {
return null;
}
try {
byte[] raw = downloadWithRetry(trimmed);
return resizeImage(trimmed, raw);
ResizedImage cached = readLocalCachedThumb(trimmed);
return cached != null ? cached : fetchAndResizeDirect(trimmed);
} catch (Exception ex) {
log.debug("[similar-asin][image] prefetch-cache-fail url={} err={}", trimmed, errorSummary(ex));
return null;
@@ -292,8 +429,10 @@ public class SimilarAsinImageEmbedder {
ResizedImage thumb = taskImageCache.get(trimmedUrl);
boolean cacheHit = thumb != null;
if (!cacheHit) {
byte[] raw = downloadWithRetry(trimmedUrl);
thumb = resizeImage(trimmedUrl, raw);
thumb = readLocalCachedThumb(trimmedUrl);
if (thumb == null) {
thumb = fetchAndResizeDirect(trimmedUrl);
}
taskImageCache.put(trimmedUrl, thumb);
log.info("[similar-asin][image] cache-miss url={} bytes={} dims={}x{}",
trimmedUrl, thumb.bytes().length, thumb.width(), thumb.height());
@@ -311,7 +450,7 @@ public class SimilarAsinImageEmbedder {
} catch (ResizeException ex) {
log.warn("[similar-asin][image] resize-fail url={} elapsedMs={} err={}",
trimmedUrl, elapsedMs(startedNanos), errorSummary(ex));
} catch (IOException | TimeoutException ex) {
} catch (IOException ex) {
log.warn("[similar-asin][image] download-fail url={} elapsedMs={} err={}",
trimmedUrl, elapsedMs(startedNanos), errorSummary(ex));
} catch (RuntimeException ex) {
@@ -328,10 +467,31 @@ public class SimilarAsinImageEmbedder {
Row row,
Map<String, ResizedImage> taskImageCache,
ExcelCellImageWriter.Session excelCellImageSession) {
return embedAsExcelCellImage(rowIdx, colIdx, url, row, taskImageCache, excelCellImageSession, null);
}
public ImageDim embedAsExcelCellImage(int rowIdx,
int colIdx,
String url,
Row row,
Map<String, ResizedImage> taskImageCache,
ExcelCellImageWriter.Session excelCellImageSession,
ImageSpool imageSpool) {
if (url == null || url.isBlank() || excelCellImageSession == null) {
return null;
}
String trimmedUrl = url.trim();
SpoolImage spooled = imageSpool == null ? null : imageSpool.get(trimmedUrl);
if (spooled != null) {
Cell cell = row.createCell(colIdx);
cell.setCellValue("#VALUE!");
excelCellImageSession.registerImage(rowIdx, colIdx, spooled.path());
return new ImageDim(spooled.width(), spooled.height());
}
if (imageSpool != null) {
row.createCell(colIdx).setCellValue(trimmedUrl);
return null;
}
ResizedImage thumb = downloadAndResize(trimmedUrl, colIdx, row, taskImageCache);
if (thumb == null) {
return null;
@@ -339,66 +499,139 @@ public class SimilarAsinImageEmbedder {
try {
Cell cell = row.createCell(colIdx);
cell.setCellValue("#VALUE!");
if (imageSpool == null) {
excelCellImageSession.registerImage(rowIdx, colIdx, thumb.bytes());
} else {
spooled = imageSpool.put(trimmedUrl, thumb);
excelCellImageSession.registerImage(rowIdx, colIdx, spooled.path());
}
taskImageCache.remove(trimmedUrl);
return spooled == null
? new ImageDim(thumb.width(), thumb.height())
: new ImageDim(spooled.width(), spooled.height());
} catch (IOException | RuntimeException ex) {
taskImageCache.remove(trimmedUrl);
return new ImageDim(thumb.width(), thumb.height());
} catch (RuntimeException ex) {
log.warn("[similar-asin][image] excel-cell-image-fail url={} err={}", trimmedUrl, ex.getMessage());
row.createCell(colIdx).setCellValue(trimmedUrl);
return null;
}
}
private byte[] downloadWithRetry(String url) throws IOException, TimeoutException {
private boolean hasLocalCachedThumb(String url) {
Path path = localCachePath(url);
return path != null && Files.isRegularFile(path);
}
private ResizedImage readLocalCachedThumb(String url) {
Path path = localCachePath(url);
if (path == null || !Files.isRegularFile(path)) {
return null;
}
try {
byte[] bytes = Files.readAllBytes(path);
if (bytes.length == 0 || bytes.length > MAX_DOWNLOAD_BYTES) {
Files.deleteIfExists(path);
return null;
}
ResizedImage image = decodeCachedThumb(bytes);
if (image == null) {
Files.deleteIfExists(path);
return null;
}
Files.setLastModifiedTime(path, FileTime.fromMillis(System.currentTimeMillis()));
return image;
} catch (IOException ex) {
log.debug("[similar-asin][image] local-cache-read-fail path={} err={}", path, ex.getMessage());
return null;
}
}
private Path localCachePath(String url) {
if (localImageCacheDir == null || url == null || url.isBlank()) {
return null;
}
String normalized = ossStorageService.normalizeManagedPublicUrl(url.trim());
String hash = sha256Hex(normalized == null || normalized.isBlank() ? url.trim() : normalized);
return localImageCacheDir.resolve(hash.substring(0, 2)).resolve(hash + ".jpg");
}
private void writeLocalCachedThumb(String url, ResizedImage image) {
Path target = localCachePath(url);
if (target == null || image == null || image.bytes() == null || image.bytes().length == 0) {
return;
}
Path temp = null;
try {
Files.createDirectories(target.getParent());
if (Files.isRegularFile(target)) {
return;
}
temp = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".tmp");
Files.write(temp, image.bytes());
try {
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE);
} catch (AtomicMoveNotSupportedException ex) {
Files.move(temp, target);
}
temp = null;
} catch (java.nio.file.FileAlreadyExistsException ignored) {
// Another download of the same normalized URL won the cache write race.
} catch (IOException ex) {
log.debug("[similar-asin][image] local-cache-write-fail path={} err={}", target, ex.getMessage());
} finally {
if (temp != null) {
try {
Files.deleteIfExists(temp);
} catch (IOException ignored) {
// Best-effort cleanup of a failed cache write.
}
}
}
}
private static String sha256Hex(String value) {
try {
byte[] digest = MessageDigest.getInstance("SHA-256")
.digest(value.getBytes(StandardCharsets.UTF_8));
StringBuilder result = new StringBuilder(digest.length * 2);
for (byte b : digest) {
result.append(String.format("%02x", b & 0xFF));
}
return result.toString();
} catch (NoSuchAlgorithmException ex) {
throw new IllegalStateException("SHA-256 unavailable", ex);
}
}
private ResizedImage fetchAndResizeDirect(String url) throws IOException {
String downloadUrl = normalizeAndValidateDownloadUrl(url);
List<String> candidates = downloadCandidates(downloadUrl);
IOException last = null;
TimeoutException lastTimeout = null;
long waitSeconds = downloadTimeoutSeconds * 2L;
for (int attempt = 0; attempt <= DOWNLOAD_MAX_RETRY; attempt++) {
ensureImageWorkNotInterrupted();
String attemptUrl = candidates.get(Math.min(attempt, candidates.size() - 1));
Future<byte[]> future = downloadPool.submit(() -> doFetch(attemptUrl));
try {
byte[] bytes = future.get(waitSeconds, TimeUnit.SECONDS);
byte[] raw = doFetch(attemptUrl);
if (!attemptUrl.equals(downloadUrl)) {
log.info("[similar-asin][image] download-fallback-success originalUrl={} usedUrl={} attempt={}/{}",
downloadUrl, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1);
}
return bytes;
} catch (java.util.concurrent.ExecutionException ee) {
Throwable cause = ee.getCause();
if (cause instanceof DownloadOversizeException doe) {
throw doe;
ResizedImage image = resizeImage(downloadUrl, raw);
writeLocalCachedThumb(downloadUrl, image);
return image;
} catch (DownloadOversizeException ex) {
throw ex;
} catch (IOException ex) {
if (Thread.currentThread().isInterrupted()) {
InterruptedIOException interrupted = new InterruptedIOException("image download interrupted");
interrupted.initCause(ex);
throw interrupted;
}
if (cause instanceof IOException io) {
last = io;
lastTimeout = null;
logRetry(attemptUrl, attempt, io);
continue;
}
if (cause instanceof RuntimeException re) {
throw re;
}
throw new IOException("image download failed: " + cause.getMessage(), cause);
} catch (java.util.concurrent.TimeoutException te) {
future.cancel(true);
TimeoutException wrapped = new TimeoutException("image download timeout after attempt "
+ (attempt + 1) + "/" + (DOWNLOAD_MAX_RETRY + 1)
+ ", waitSeconds=" + waitSeconds
+ ", url=" + attemptUrl);
wrapped.initCause(te);
lastTimeout = wrapped;
last = null;
logRetry(attemptUrl, attempt, wrapped);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
throw new IOException("image download interrupted", ie);
last = ex;
logRetry(attemptUrl, attempt, ex);
}
}
if (lastTimeout != null) {
throw lastTimeout;
}
throw last != null ? last : new IOException("image download failed without cause");
throw last == null ? new IOException("image download failed: " + downloadUrl) : last;
}
String normalizeAndValidateDownloadUrl(String url) {
@@ -503,6 +736,7 @@ public class SimilarAsinImageEmbedder {
}
private byte[] doFetch(String url) throws IOException {
ensureImageWorkNotInterrupted();
Request req = buildImageRequest(url);
try (Response resp = httpClient.newCall(req).execute()) {
if (!resp.isSuccessful()) {
@@ -522,6 +756,7 @@ public class SimilarAsinImageEmbedder {
int total = 0;
int n;
while ((n = in.read(buf)) != -1) {
ensureImageWorkNotInterrupted();
total += n;
if (total > MAX_DOWNLOAD_BYTES) {
throw new DownloadOversizeException(url, total);
@@ -590,6 +825,7 @@ public class SimilarAsinImageEmbedder {
* 仍然超限才抛 ResizeOversizeException 触发文本兜底
*/
ResizedImage resizeImage(String sourceUrl, byte[] raw) throws IOException {
ensureImageWorkNotInterrupted();
guardImageDimensions(sourceUrl, raw);
BufferedImage src = ImageIO.read(new ByteArrayInputStream(raw));
if (src == null) {
@@ -602,8 +838,11 @@ public class SimilarAsinImageEmbedder {
ResizedImage candidate = null;
ResizedImage smallest = null;
for (int longEdge : FALLBACK_LONG_EDGES) {
ensureImageWorkNotInterrupted();
BufferedImage scaled = scaleAt(src, srcW, srcH, longEdge);
for (float quality : FALLBACK_QUALITIES) {
ResizedImage tried = encodeAt(src, srcW, srcH, longEdge, quality);
ensureImageWorkNotInterrupted();
ResizedImage tried = encodeJpeg(scaled, quality);
if (smallest == null || tried.bytes().length < smallest.bytes().length) {
smallest = tried;
}
@@ -625,7 +864,13 @@ public class SimilarAsinImageEmbedder {
throw new ResizeOversizeException(sourceUrl, reportedSize);
}
private ResizedImage encodeAt(BufferedImage src, int srcW, int srcH, int longEdgePx, float quality) throws IOException {
private static void ensureImageWorkNotInterrupted() throws InterruptedIOException {
if (Thread.currentThread().isInterrupted()) {
throw new InterruptedIOException("image work interrupted");
}
}
private BufferedImage scaleAt(BufferedImage src, int srcW, int srcH, int longEdgePx) {
double ratio = (double) Math.max(srcW, srcH) / longEdgePx;
int dstW = ratio > 1 ? Math.max(1, (int) Math.round(srcW / ratio)) : srcW;
int dstH = ratio > 1 ? Math.max(1, (int) Math.round(srcH / ratio)) : srcH;
@@ -638,6 +883,10 @@ public class SimilarAsinImageEmbedder {
} finally {
g.dispose();
}
return dst;
}
private ResizedImage encodeJpeg(BufferedImage image, float quality) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Iterator<ImageWriter> writers = ImageIO.getImageWritersByFormatName("jpg");
if (!writers.hasNext()) {
@@ -651,14 +900,14 @@ public class SimilarAsinImageEmbedder {
ImageOutputStream ios = ImageIO.createImageOutputStream(baos);
try {
writer.setOutput(ios);
writer.write(null, new IIOImage(dst, null, null), param);
writer.write(null, new IIOImage(image, null, null), param);
} finally {
ios.close();
}
} finally {
writer.dispose();
}
return new ResizedImage(baos.toByteArray(), dstW, dstH);
return new ResizedImage(baos.toByteArray(), image.getWidth(), image.getHeight());
}
/** 缩略图字节 + 实际像素,A 副本随 taskImageCache 生命周期,B 副本进 picture pool。 */
@@ -667,6 +916,85 @@ public class SimilarAsinImageEmbedder {
/** embed() 成功时返回的缩略图实际像素,调用方据此自适应行高/列宽。 */
public record ImageDim(int width, int height) { }
public record SpoolImage(Path path, int width, int height) { }
/** Task-scoped disk storage for resized thumbnails. */
public static final class ImageSpool implements AutoCloseable {
private final Path directory;
private final ConcurrentMap<String, SpoolImage> images = new ConcurrentHashMap<>();
public ImageSpool(Path directory) throws IOException {
this.directory = Objects.requireNonNull(directory, "directory must not be null")
.toAbsolutePath().normalize();
Files.createDirectories(this.directory);
}
public SpoolImage get(String url) {
if (url == null) {
return null;
}
SpoolImage image = images.get(url.trim());
return image != null && Files.isRegularFile(image.path()) ? image : null;
}
public SpoolImage put(String url, ResizedImage image) throws IOException {
if (url == null || url.isBlank() || image == null || image.bytes() == null || image.bytes().length == 0) {
throw new IOException("invalid image spool entry");
}
String key = url.trim();
SpoolImage existing = get(key);
if (existing != null) {
return existing;
}
Path file = Files.createTempFile(directory, "image-", ".jpeg");
boolean retained = false;
try {
Files.write(file, image.bytes());
SpoolImage candidate = new SpoolImage(file, image.width(), image.height());
existing = images.putIfAbsent(key, candidate);
if (existing != null) {
return existing;
}
retained = true;
return candidate;
} finally {
if (!retained) {
Files.deleteIfExists(file);
}
}
}
public int size() {
return images.size();
}
@Override
public void close() throws IOException {
images.clear();
if (!Files.exists(directory)) {
return;
}
IOException failure = null;
try (var paths = Files.walk(directory)) {
for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
try {
Files.deleteIfExists(path);
} catch (IOException ex) {
if (failure == null) {
failure = ex;
} else {
failure.addSuppressed(ex);
}
}
}
}
if (failure != null) {
throw failure;
}
}
}
/** 在解码整张位图前用 ImageReader 仅读取头部尺寸,避免“图像炸弹”导致 heap OOM。 */
private static void guardImageDimensions(String sourceUrl, byte[] raw) throws IOException {
try (ImageInputStream iis = ImageIO.createImageInputStream(new ByteArrayInputStream(raw))) {
@@ -17,4 +17,22 @@ public class TaskHeartbeatRequest {
@Schema(description = "可选的总进度", example = "1100")
private Integer total;
@Schema(description = "采集数据任务当前阶段", example = "detail")
private String collectStage;
@Schema(description = "采集数据任务当前关键词", example = "women bodysuit")
private String currentKeyword;
@Schema(description = "搜索页当前页码", example = "2")
private Integer searchCurrentPage;
@Schema(description = "搜索页总页数", example = "7")
private Integer searchTotalPages;
@Schema(description = "详情页已处理 ASIN 数", example = "34")
private Integer detailProcessedAsins;
@Schema(description = "详情页 ASIN 总数", example = "120")
private Integer detailTotalAsins;
}
@@ -25,4 +25,6 @@ public class TaskFileJobEntity {
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
private LocalDateTime finishedAt;
/** Parent task/result terminal-failure callback completion marker. */
private LocalDateTime terminalCallbackAt;
}
@@ -71,7 +71,8 @@ public class TaskFileJobService {
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getUpdatedAt, now)
.set(TaskFileJobEntity::getFinishedAt, null));
.set(TaskFileJobEntity::getFinishedAt, null)
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(existing.getId());
publishDispatchEvent(refreshed);
return refreshed;
@@ -148,7 +149,8 @@ public class TaskFileJobService {
.set(TaskFileJobEntity::getRetryCount, 0)
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, null));
.set(TaskFileJobEntity::getFinishedAt, null)
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
if (updated > 0) {
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(jobId);
publishDispatchEvent(refreshed);
@@ -159,37 +161,70 @@ public class TaskFileJobService {
@Transactional
public int resetStuckRunningJobs(int stuckMinutes, int limit) {
return resetStuckRunningJobsDetailed(stuckMinutes, limit).resetCount();
}
@Transactional
public StuckJobResetResult resetStuckRunningJobsDetailed(int stuckMinutes, int limit) {
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes));
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.and(wrapper -> wrapper
.and(running -> running
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.and(age -> age
.lt(TaskFileJobEntity::getUpdatedAt, threshold)
.or()
.isNull(TaskFileJobEntity::getUpdatedAt)))
.or(exhausted -> exhausted
.eq(TaskFileJobEntity::getStatus, "FAILED")
.ge(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
.isNull(TaskFileJobEntity::getTerminalCallbackAt)))
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + Math.max(1, Math.min(limit, 200))));
int reset = 0;
List<TaskFileJobEntity> exhaustedJobs = new ArrayList<>();
for (TaskFileJobEntity job : jobs) {
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
if (retryCount >= MAX_RETRY_COUNT) {
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
if ("FAILED".equals(job.getStatus())
&& job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT) {
exhaustedJobs.add(job);
continue;
}
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1);
LocalDateTime now = LocalDateTime.now();
LambdaUpdateWrapper<TaskFileJobEntity> update = new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
.set(TaskFileJobEntity::getUpdatedAt, now);
if (nextRetryCount >= MAX_RETRY_COUNT) {
int updated = taskFileJobMapper.update(null, update
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已达到最大重试次数")
.set(TaskFileJobEntity::getFinishedAt, now)
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
if (updated > 0) {
TaskFileJobEntity exhausted = taskFileJobMapper.selectById(job.getId());
exhaustedJobs.add(exhausted == null ? job : exhausted);
}
continue;
}
int updated = taskFileJobMapper.update(null, update
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
.set(TaskFileJobEntity::getFinishedAt, null)
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
if (updated > 0) {
reset++;
publishDispatchEvent(taskFileJobMapper.selectById(job.getId()));
}
}
return reset;
return new StuckJobResetResult(reset, List.copyOf(exhaustedJobs));
}
public record StuckJobResetResult(int resetCount, List<TaskFileJobEntity> exhaustedJobs) {
}
public boolean markRunning(Long jobId) {
@@ -199,7 +234,37 @@ public class TaskFileJobService {
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, jobId)
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.and(wrapper -> wrapper.isNull(TaskFileJobEntity::getRetryCount)
.or()
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT))
.set(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getTerminalCallbackAt, null)) > 0;
}
/** Returns the persisted RUNNING row so queued execution can later fence itself with updatedAt. */
public TaskFileJobEntity claimRunning(Long jobId) {
if (!markRunning(jobId)) {
return null;
}
TaskFileJobEntity claim = taskFileJobMapper.selectById(jobId);
return claim != null && "RUNNING".equals(claim.getStatus()) ? claim : null;
}
/**
* Activates a previously queued claim. The updatedAt CAS prevents a timed-out old queue entry from
* starting after the same job has already been reset or claimed again.
*/
public boolean activateRunningClaim(TaskFileJobEntity claim) {
if (claim == null || claim.getId() == null || claim.getUpdatedAt() == null) {
return false;
}
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, claim.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.eq(TaskFileJobEntity::getUpdatedAt, claim.getUpdatedAt())
.eq(claim.getRetryCount() != null, TaskFileJobEntity::getRetryCount, claim.getRetryCount())
.isNull(claim.getRetryCount() == null, TaskFileJobEntity::getRetryCount)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())) > 0;
}
@@ -228,6 +293,23 @@ public class TaskFileJobService {
.set(TaskFileJobEntity::getUpdatedAt, now));
}
public void touchRunningAssembleJobsIfStale(Long taskId, String moduleType, long intervalMillis) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
return;
}
LocalDateTime now = LocalDateTime.now();
LocalDateTime cutoff = now.minus(Duration.ofMillis(Math.max(1_000L, intervalMillis)));
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getTaskId, taskId)
.eq(TaskFileJobEntity::getModuleType, moduleType)
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.and(wrapper -> wrapper.isNull(TaskFileJobEntity::getUpdatedAt)
.or()
.le(TaskFileJobEntity::getUpdatedAt, cutoff))
.set(TaskFileJobEntity::getUpdatedAt, now));
}
public boolean deferRunning(Long jobId, String message) {
if (jobId == null || jobId <= 0) {
return false;
@@ -252,10 +334,14 @@ public class TaskFileJobService {
.eq(TaskFileJobEntity::getId, jobId)
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
.ne(TaskFileJobEntity::getStatus, "PENDING")
.and(wrapper -> wrapper.isNull(TaskFileJobEntity::getRetryCount)
.or()
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT))
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, message)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, null));
.set(TaskFileJobEntity::getFinishedAt, null)
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
if (updated <= 0) {
return false;
}
@@ -265,13 +351,19 @@ public class TaskFileJobService {
}
public void markSuccess(TaskFileJobEntity job, String resultFileUrl) {
if (job == null || job.getId() == null) {
return;
}
LocalDateTime now = LocalDateTime.now();
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getStatus, "SUCCESS")
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getResultFileUrl, resultFileUrl)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
.set(TaskFileJobEntity::getUpdatedAt, now)
.set(TaskFileJobEntity::getFinishedAt, now)
.set(TaskFileJobEntity::getTerminalCallbackAt, now));
}
public TaskFileJobEntity findById(Long jobId) {
@@ -283,19 +375,52 @@ public class TaskFileJobService {
public boolean isRetryExhausted(Long jobId) {
TaskFileJobEntity job = findById(jobId);
return job != null && job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT;
return job != null
&& "FAILED".equals(job.getStatus())
&& job.getRetryCount() != null
&& job.getRetryCount() >= MAX_RETRY_COUNT;
}
public void markFailed(TaskFileJobEntity job, String message) {
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount() + 1;
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
TaskFileJobEntity current = job == null || job.getId() == null
? null
: taskFileJobMapper.selectById(job.getId());
if (current == null || "SUCCESS".equals(current.getStatus())) {
return;
}
int currentRetryCount = current.getRetryCount() == null ? 0 : current.getRetryCount();
int retryCount = Math.min(MAX_RETRY_COUNT, currentRetryCount + 1);
LambdaUpdateWrapper<TaskFileJobEntity> update = new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
.eq(current.getRetryCount() != null, TaskFileJobEntity::getRetryCount, current.getRetryCount())
.isNull(current.getRetryCount() == null, TaskFileJobEntity::getRetryCount)
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getRetryCount, retryCount)
.set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, retryCount >= MAX_RETRY_COUNT ? LocalDateTime.now() : null));
.set(TaskFileJobEntity::getFinishedAt,
retryCount >= MAX_RETRY_COUNT ? LocalDateTime.now() : null)
.set(TaskFileJobEntity::getTerminalCallbackAt, null);
taskFileJobMapper.update(null, update);
}
public boolean markFailureFinalized(Long jobId, String message) {
if (jobId == null || jobId <= 0) {
return false;
}
LocalDateTime now = LocalDateTime.now();
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, jobId)
.eq(TaskFileJobEntity::getStatus, "FAILED")
.ge(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
.isNull(TaskFileJobEntity::getTerminalCallbackAt)
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getErrorMessage,
message == null ? "结果文件生成失败" : message)
.set(TaskFileJobEntity::getUpdatedAt, now)
.set(TaskFileJobEntity::getFinishedAt, now)
.set(TaskFileJobEntity::getTerminalCallbackAt, now)) > 0;
}
/**
@@ -310,7 +435,8 @@ public class TaskFileJobService {
.set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
.set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getTerminalCallbackAt, LocalDateTime.now()));
}
public TaskFileJobEntity findAssembleJob(Long taskId, String moduleType, Long resultId) {
@@ -7,6 +7,7 @@ import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCa
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
import com.nanri.aiimage.modules.brand.service.BrandTaskProgressCacheService;
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService;
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
@@ -49,6 +50,7 @@ public class TaskHeartbeatService {
private static final String MODULE_WITHDRAW = "WITHDRAW";
private static final String MODULE_APPEARANCE_PATENT = "APPEARANCE_PATENT";
private static final String MODULE_SIMILAR_ASIN = "SIMILAR_ASIN";
private static final String MODULE_COLLECT_DATA = "COLLECT_DATA";
private static final String MODULE_BRAND = "BRAND";
private final FileTaskMapper fileTaskMapper;
@@ -65,8 +67,10 @@ public class TaskHeartbeatService {
private final AppearancePatentTaskCacheService appearancePatentTaskCacheService;
private final SimilarAsinTaskCacheService similarAsinTaskCacheService;
private final SimilarAsinProperties similarAsinProperties;
private final TaskFileJobService taskFileJobService;
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
private final CollectDataService collectDataService;
public TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request) {
if (taskId == null || taskId <= 0) {
@@ -151,6 +155,7 @@ public class TaskHeartbeatService {
boolean checkpointDue = task.getUpdatedAt() == null || !task.getUpdatedAt().isAfter(cutoff);
similarAsinTaskCacheService.touchTaskHeartbeat(task.getId());
taskFileJobService.touchRunningAssembleJobsIfStale(task.getId(), MODULE_SIMILAR_ASIN, intervalMillis);
if (!checkpointDue) {
saveFileTaskCache(MODULE_SIMILAR_ASIN, task);
return TaskHeartbeatVo.alive(MODULE_SIMILAR_ASIN, STATUS_RUNNING);
@@ -239,6 +244,7 @@ public class TaskHeartbeatService {
case MODULE_DELETE_BRAND -> {
deleteBrandTaskCacheService.saveProgress(taskId, buildDeleteBrandHeartbeatProgress(request), true);
}
case MODULE_COLLECT_DATA -> collectDataService.updateProgress(taskId, request);
default -> {
}
}
@@ -18,6 +18,7 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@@ -28,6 +29,10 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
@Slf4j
@Service
@@ -66,6 +71,15 @@ public class TaskResultFileJobWorker {
@Value("${aiimage.result-file-job.stuck-timeout-minutes:30}")
private int stuckTimeoutMinutes;
@Value("${aiimage.result-file-job.heartbeat-interval-ms:60000}")
private long heartbeatIntervalMillis = 60000L;
private final ScheduledExecutorService jobHeartbeatExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> {
Thread thread = new Thread(runnable, "task-file-job-heartbeat");
thread.setDaemon(true);
return thread;
});
@Scheduled(fixedDelayString = "${aiimage.result-file-job.local-worker-delay-ms:15000}")
public void runPendingJobs() {
if (!localWorkerEnabled) {
@@ -90,9 +104,17 @@ public class TaskResultFileJobWorker {
@Scheduled(fixedDelayString = "${aiimage.result-file-job.stuck-scan-delay-ms:60000}")
public void resetStuckJobs() {
int reset = taskFileJobService.resetStuckRunningJobs(stuckTimeoutMinutes, batchSize);
if (reset > 0) {
log.warn("[task-file-job] reset stuck running jobs count={} timeoutMinutes={}", reset, stuckTimeoutMinutes);
TaskFileJobService.StuckJobResetResult result =
taskFileJobService.resetStuckRunningJobsDetailed(stuckTimeoutMinutes, batchSize);
if (result.resetCount() > 0 || !result.exhaustedJobs().isEmpty()) {
log.warn("[task-file-job] handled stuck running jobs resetCount={} exhaustedCount={} timeoutMinutes={}",
result.resetCount(), result.exhaustedJobs().size(), stuckTimeoutMinutes);
}
for (TaskFileJobEntity job : result.exhaustedJobs()) {
String message = job.getErrorMessage() == null || job.getErrorMessage().isBlank()
? "文件生成任务运行超时,已达到最大重试次数"
: job.getErrorMessage();
finalizeRetryExhausted(job, message);
}
}
@@ -102,19 +124,49 @@ public class TaskResultFileJobWorker {
job.getId(), job.getTaskId(), job.getModuleType(), ownerFromScopeKey(job.getScopeKey()), currentInstanceId());
return;
}
if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) {
if (job == null || job.getId() == null) {
return;
}
TaskFileJobEntity claim = taskFileJobService.claimRunning(job.getId());
if (claim == null) {
return;
}
if ("APPEARANCE_PATENT".equals(job.getModuleType()) || "SIMILAR_ASIN".equals(job.getModuleType())) {
try {
cozeTaskExecutor.execute(() -> processInternal(job));
cozeTaskExecutor.execute(() -> processClaimedWithHeartbeat(job, claim));
return;
} catch (RuntimeException ex) {
log.warn("[task-file-job] coze module offload failed, fallback inline jobId={} taskId={} moduleType={} msg={}",
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex);
}
}
processClaimedWithHeartbeat(job, claim);
}
private ScheduledFuture<?> startJobHeartbeat(TaskFileJobEntity job) {
long interval = Math.max(1000L, heartbeatIntervalMillis);
return jobHeartbeatExecutor.scheduleWithFixedDelay(() -> {
try {
taskFileJobService.touchRunning(job.getId());
} catch (Exception ex) {
log.warn("[task-file-job] heartbeat failed jobId={} taskId={} moduleType={} msg={}",
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage());
}
}, interval, interval, TimeUnit.MILLISECONDS);
}
private void processClaimedWithHeartbeat(TaskFileJobEntity job, TaskFileJobEntity claim) {
if (!taskFileJobService.activateRunningClaim(claim)) {
log.info("[task-file-job] queued claim expired before execution jobId={} taskId={} moduleType={} claimUpdatedAt={}",
job.getId(), job.getTaskId(), job.getModuleType(), claim.getUpdatedAt());
return;
}
ScheduledFuture<?> heartbeat = startJobHeartbeat(job);
try {
processInternal(job);
} finally {
heartbeat.cancel(false);
}
}
private void processInternal(TaskFileJobEntity job) {
@@ -130,13 +182,6 @@ public class TaskResultFileJobWorker {
TaskDistributedLockService.LockHandle lockHandle =
taskDistributedLockService.acquire(job.getModuleType(), job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS);
if (lockHandle == null) {
TaskFileJobEntity latestOnBusy = taskFileJobService.findById(job.getId());
if (latestOnBusy != null && "RUNNING".equals(latestOnBusy.getStatus())) {
taskFileJobService.touchRunning(job.getId());
log.info("[task-file-job] process skipped because task lock is busy and job is already running jobId={} taskId={} moduleType={} resultId={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
return;
}
taskFileJobService.requeue(job.getId(), "Task is busy, waiting for previous task operation");
log.info("[task-file-job] process requeued because task lock is busy jobId={} taskId={} moduleType={} resultId={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
@@ -194,10 +239,37 @@ public class TaskResultFileJobWorker {
log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), message);
taskFileJobService.markFailed(job, message);
if ("SHOP_DATA_CRAWL".equals(job.getModuleType()) && taskFileJobService.isRetryExhausted(job.getId())) {
if (taskFileJobService.isRetryExhausted(job.getId())) {
finalizeRetryExhausted(job, message);
}
}
}
private void finalizeRetryExhausted(TaskFileJobEntity job, String message) {
try {
notifyRetryExhausted(job, message);
if (!taskFileJobService.markFailureFinalized(job.getId(), message)) {
log.warn("[task-file-job] exhausted job terminal callback was already finalized or claim was lost jobId={} taskId={} moduleType={}",
job.getId(), job.getTaskId(), job.getModuleType());
}
} catch (Exception ex) {
// Keep the terminal-callback marker empty so the scheduled scan retries this idempotent callback.
log.error("[task-file-job] failed to finalize exhausted job jobId={} taskId={} moduleType={} msg={}",
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex);
}
}
private void notifyRetryExhausted(TaskFileJobEntity job, String message) {
if ("SHOP_DATA_CRAWL".equals(job.getModuleType())) {
shopDataCrawlTaskService.handleResultFileJobFailure(job, message);
} else if ("SIMILAR_ASIN".equals(job.getModuleType())) {
similarAsinTaskService.handleResultFileJobFailure(job, message);
}
}
@PreDestroy
void shutdownJobHeartbeatExecutor() {
jobHeartbeatExecutor.shutdownNow();
}
private boolean isOrphanJobFailure(Exception ex, String message) {
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.ziniao.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexRefreshCursorDto;
import com.nanri.aiimage.modules.ziniao.model.dto.ZiniaoOpenShopRequest;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoOpenShopVo;
@@ -8,17 +9,23 @@ import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoSessionVo;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopListVo;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoStaffListVo;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoAuthService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexRefreshService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@@ -31,6 +38,8 @@ public class ZiniaoAuthController {
private final ZiniaoAuthService ziniaoAuthService;
private final ZiniaoShopIndexService ziniaoShopIndexService;
private final ZiniaoShopIndexRefreshService ziniaoShopIndexRefreshService;
private final AdminAuthSupport adminAuthSupport;
@GetMapping("/session")
@Operation(summary = "获取紫鸟会话状态", description = "自动获取或复用 appToken,并返回 companyId、当前员工 userId、脱敏 token 和当前店铺信息。")
@@ -56,6 +65,17 @@ public class ZiniaoAuthController {
return ApiResponse.success(ziniaoShopIndexService.getRefreshCursor());
}
@PostMapping("/index-refresh")
@Operation(summary = "手动刷新紫鸟店铺索引", description = "立即执行一轮完整的紫鸟店铺索引刷新,并返回本轮刷新结果。")
@SecurityRequirement(name = "bearerAuth")
public ApiResponse<ZiniaoShopIndexRefreshCursorDto> refreshShopIndex(
HttpServletRequest request,
@Parameter(description = "管理员登录凭证,格式:Bearer <token>")
@RequestHeader(value = HttpHeaders.AUTHORIZATION, required = false) String authorization) {
adminAuthSupport.requireAdmin(request);
return ApiResponse.success("紫鸟店铺索引刷新完成", ziniaoShopIndexRefreshService.refreshShopIndexManually());
}
@GetMapping("/shops")
@Operation(summary = "获取员工可见店铺列表", description = "先通过 API Key 获取 companyId,再按员工 userId 查询该员工有权限的店铺列表,并缓存到当前 session。")
@ApiResponses({
@@ -1,12 +1,15 @@
package com.nanri.aiimage.modules.ziniao.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopkey.mapper.ShopKeyMapper;
import com.nanri.aiimage.modules.shopkey.model.entity.ShopKeyEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -15,29 +18,28 @@ import java.util.Map;
@RequiredArgsConstructor
public class ZiniaoApiKeyProvider {
public static final String IP_WHITELIST_STATUS_ALLOWED = "ALLOWED";
public static final String IP_WHITELIST_STATUS_BLOCKED = "BLOCKED";
private final ShopKeyMapper shopKeyMapper;
public List<ApiKeyAccount> listApiKeyAccounts() {
List<ShopKeyEntity> entities = shopKeyMapper.selectList(new LambdaQueryWrapper<ShopKeyEntity>()
.orderByDesc(ShopKeyEntity::getId));
Map<String, String> accountByApiKey = new LinkedHashMap<>();
Map<String, List<ShopKeyEntity>> entitiesByApiKey = new LinkedHashMap<>();
for (ShopKeyEntity entity : entities) {
String apiKey = normalizeApiKey(entity == null ? null : entity.getZiniaoToken());
if (apiKey == null) {
continue;
}
String accountName = entity.getZiniaoAccountName() == null ? null : entity.getZiniaoAccountName().trim();
if (!accountByApiKey.containsKey(apiKey)) {
accountByApiKey.put(apiKey, accountName);
continue;
entitiesByApiKey.computeIfAbsent(apiKey, ignored -> new ArrayList<>()).add(entity);
}
String existingName = accountByApiKey.get(apiKey);
if ((existingName == null || existingName.isBlank()) && accountName != null && !accountName.isBlank()) {
accountByApiKey.put(apiKey, accountName);
}
}
return accountByApiKey.entrySet().stream()
.map(entry -> new ApiKeyAccount(entry.getKey(), entry.getValue()))
return entitiesByApiKey.entrySet().stream()
.map(entry -> new ApiKeyAccount(
entry.getKey(),
resolveAccountName(entry.getValue()),
entry.getValue().stream().map(ShopKeyEntity::getId).filter(java.util.Objects::nonNull).toList()
))
.toList();
}
@@ -60,6 +62,42 @@ public class ZiniaoApiKeyProvider {
return total != null && total > 0;
}
public void markIpWhitelistAllowed(ApiKeyAccount account) {
updateIpWhitelistStatus(account, IP_WHITELIST_STATUS_ALLOWED, null);
}
public void markIpWhitelistBlocked(ApiKeyAccount account, String message) {
updateIpWhitelistStatus(account, IP_WHITELIST_STATUS_BLOCKED, message);
}
private void updateIpWhitelistStatus(ApiKeyAccount account, String status, String message) {
if (account == null || account.shopKeyIds().isEmpty()) {
return;
}
shopKeyMapper.update(null, new LambdaUpdateWrapper<ShopKeyEntity>()
.in(ShopKeyEntity::getId, account.shopKeyIds())
.set(ShopKeyEntity::getIpWhitelistStatus, status)
.set(ShopKeyEntity::getIpWhitelistCheckedAt, LocalDateTime.now())
.set(ShopKeyEntity::getIpWhitelistMessage, truncateMessage(message)));
}
private String resolveAccountName(List<ShopKeyEntity> entities) {
return entities.stream()
.map(ShopKeyEntity::getZiniaoAccountName)
.filter(name -> name != null && !name.isBlank())
.map(String::trim)
.findFirst()
.orElse(null);
}
private String truncateMessage(String message) {
if (message == null || message.isBlank()) {
return null;
}
String normalized = message.trim();
return normalized.length() <= 500 ? normalized : normalized.substring(0, 500);
}
private String normalizeApiKey(String token) {
if (token == null || token.isBlank()) {
return null;
@@ -71,6 +109,14 @@ public class ZiniaoApiKeyProvider {
return normalized.isBlank() ? null : normalized;
}
public record ApiKeyAccount(String apiKey, String accountName) {
public record ApiKeyAccount(String apiKey, String accountName, List<Long> shopKeyIds) {
public ApiKeyAccount {
shopKeyIds = shopKeyIds == null ? List.of() : List.copyOf(shopKeyIds);
}
public ApiKeyAccount(String apiKey, String accountName) {
this(apiKey, accountName, List.of());
}
}
}
@@ -1,6 +1,8 @@
package com.nanri.aiimage.modules.ziniao.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexRefreshCursorDto;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
@@ -33,4 +35,15 @@ public class ZiniaoShopIndexRefreshService {
}
}
}
public ZiniaoShopIndexRefreshCursorDto refreshShopIndexManually() {
DistributedJobLockService.LockHandle lockHandle = distributedJobLockService.tryLock("ziniao:shop-index-refresh", REFRESH_LOCK_TTL);
if (lockHandle == null) {
throw new BusinessException("紫鸟店铺索引正在刷新,或分布式锁暂时不可用,请稍后重试");
}
try (lockHandle) {
ziniaoShopIndexService.refreshAllShopIndex();
return ziniaoShopIndexService.getRefreshCursor();
}
}
}
@@ -159,6 +159,14 @@ public class ZiniaoShopIndexService {
}
public void refreshShopIndex() {
refreshShopIndex(resolveRefreshBatchSize());
}
public void refreshAllShopIndex() {
refreshShopIndex(0);
}
private void refreshShopIndex(int refreshBatchSize) {
long now = Instant.now().toEpochMilli();
ZiniaoShopIndexRefreshCursorDto previousCursor = ziniaoTransientCacheService
.get(CACHE_TYPE_SHOP_INDEX_REFRESH_CURSOR, "global", ZiniaoShopIndexRefreshCursorDto.class)
@@ -177,7 +185,10 @@ public class ZiniaoShopIndexService {
Map<String, List<ZiniaoShopIndexEntryDto>> grouped = new LinkedHashMap<>();
Map<String, Long> storesFingerprintToUserId = new LinkedHashMap<>();
Set<Long> allInvalidUserIds = new LinkedHashSet<>();
int refreshBatchSize = resolveRefreshBatchSize();
int completedApiKeyCount = 0;
int skippedApiKeyCount = 0;
int whitelistSkippedApiKeyCount = 0;
boolean completeCoverage = true;
try {
List<ZiniaoApiKeyProvider.ApiKeyAccount> allApiKeyAccounts = ziniaoApiKeyProvider.listApiKeyAccounts();
List<ZiniaoApiKeyProvider.ApiKeyAccount> apiKeyAccounts = selectApiKeyBatchByOffset(
@@ -186,14 +197,24 @@ public class ZiniaoShopIndexService {
refreshBatchSize
);
int nextOffset = computeNextOffset(allApiKeyAccounts.size(), previousOffset, apiKeyAccounts.size(), refreshBatchSize);
apiKeyLoop:
for (ZiniaoApiKeyProvider.ApiKeyAccount apiKeyAccount : apiKeyAccounts) {
String apiKey = apiKeyAccount.apiKey();
String companyName = apiKeyAccount.accountName();
Map<String, List<ZiniaoShopIndexEntryDto>> apiKeyGrouped = new LinkedHashMap<>();
Map<String, Long> apiKeyFingerprints = new LinkedHashMap<>();
Long companyId;
try {
companyId = ziniaoAuthService.resolveCompanyIdForIndex(apiKey);
} catch (BusinessException ex) {
rethrowIfIpWhitelistError(ex);
skippedApiKeyCount++;
completeCoverage = false;
if (ziniaoAuthService.isIpWhitelistError(ex)) {
whitelistSkippedApiKeyCount++;
markIpWhitelistBlockedSafely(apiKeyAccount, ex.getMessage());
log.warn("[ziniao-index] skip apiKey reason=IP_WHITELIST accountName={} msg={}", companyName, ex.getMessage());
continue;
}
log.warn("[ziniao-index] skip apiKey while resolving companyId, msg={}", ex.getMessage());
continue;
}
@@ -203,7 +224,15 @@ public class ZiniaoShopIndexService {
try {
staff = ziniaoAuthService.getOrLoadStaffForIndex(apiKey, companyId);
} catch (BusinessException ex) {
rethrowIfIpWhitelistError(ex);
if (ziniaoAuthService.isIpWhitelistError(ex)) {
skippedApiKeyCount++;
whitelistSkippedApiKeyCount++;
completeCoverage = false;
markIpWhitelistBlockedSafely(apiKeyAccount, ex.getMessage());
log.warn("[ziniao-index] skip apiKey reason=IP_WHITELIST accountName={} companyId={} msg={}",
companyName, companyId, ex.getMessage());
continue;
}
throw ex;
}
for (Long userId : buildUserIds(staff)) {
@@ -214,11 +243,21 @@ public class ZiniaoShopIndexService {
try {
stores = ziniaoAuthService.getOrLoadUserStoresForIndex(apiKey, companyId, userId);
} catch (BusinessException ex) {
rethrowIfIpWhitelistError(ex);
if (ziniaoAuthService.isIpWhitelistError(ex)) {
skippedApiKeyCount++;
whitelistSkippedApiKeyCount++;
completeCoverage = false;
markIpWhitelistBlockedSafely(apiKeyAccount, ex.getMessage());
log.warn("[ziniao-index] skip apiKey reason=IP_WHITELIST accountName={} companyId={} userId={} msg={}",
companyName, companyId, userId, ex.getMessage());
continue apiKeyLoop;
}
if (ziniaoAuthService.isInvalidUserStoresError(ex)) {
invalidUserIds.add(userId);
allInvalidUserIds.add(userId);
ziniaoAuthService.evictInvalidUserForIndex(apiKey, companyId, userId);
} else {
completeCoverage = false;
}
log.warn("[ziniao-index] skip user stores, companyId={}, userId={}, msg={}", companyId, userId, ex.getMessage());
continue;
@@ -231,7 +270,10 @@ public class ZiniaoShopIndexService {
continue;
}
String storesFingerprint = buildStoresFingerprint(stores);
Long duplicatedUserId = storesFingerprintToUserId.putIfAbsent(storesFingerprint, userId);
Long duplicatedUserId = storesFingerprintToUserId.get(storesFingerprint);
if (duplicatedUserId == null) {
duplicatedUserId = apiKeyFingerprints.putIfAbsent(storesFingerprint, userId);
}
if (duplicatedUserId != null) {
ziniaoTransientCacheService.delete(
CACHE_TYPE_SHOP_INDEX_SCOPE_SNAPSHOT,
@@ -264,7 +306,7 @@ public class ZiniaoShopIndexService {
entry.setStatus(STATUS_ACTIVE);
entry.setLastSeenAt(now);
entry.setLastRefreshedAt(now);
List<ZiniaoShopIndexEntryDto> candidates = grouped.computeIfAbsent(normalizedShopName, ignored -> new ArrayList<>());
List<ZiniaoShopIndexEntryDto> candidates = apiKeyGrouped.computeIfAbsent(normalizedShopName, ignored -> new ArrayList<>());
if (candidates.stream().noneMatch(existing -> sameCandidate(existing, entry))) {
candidates.add(entry);
}
@@ -278,6 +320,17 @@ public class ZiniaoShopIndexService {
);
}
}
for (Map.Entry<String, List<ZiniaoShopIndexEntryDto>> apiKeyEntry : apiKeyGrouped.entrySet()) {
List<ZiniaoShopIndexEntryDto> candidates = grouped.computeIfAbsent(apiKeyEntry.getKey(), ignored -> new ArrayList<>());
for (ZiniaoShopIndexEntryDto candidate : apiKeyEntry.getValue()) {
if (candidates.stream().noneMatch(existing -> sameCandidate(existing, candidate))) {
candidates.add(candidate);
}
}
}
storesFingerprintToUserId.putAll(apiKeyFingerprints);
markIpWhitelistAllowedSafely(apiKeyAccount);
completedApiKeyCount++;
}
List<ZiniaoShopIndexEntryDto> roundEntries = new ArrayList<>();
@@ -314,28 +367,35 @@ public class ZiniaoShopIndexService {
}
log.info("[ziniao-index] refresh persist shopIndex uniqueShopId={} nameAlias={} groupedNames={} activeRows={}",
byShopId.size(), byNameAlias.size(), grouped.size(), activeCacheKeys.size());
boolean fullRefresh = apiKeyAccounts.size() >= allApiKeyAccounts.size();
boolean fullRefresh = completeCoverage && apiKeyAccounts.size() >= allApiKeyAccounts.size();
if (fullRefresh) {
markMissingEntriesAsStale(activeCacheKeys, now);
} else {
log.info("[ziniao-index] skip stale marking for partial refresh processedApiKeys={}/{} nextOffset={}",
apiKeyAccounts.size(), allApiKeyAccounts.size(), nextOffset);
log.info("[ziniao-index] skip stale marking for partial refresh completedApiKeys={}/{} skippedApiKeys={} nextOffset={}",
completedApiKeyCount, allApiKeyAccounts.size(), skippedApiKeyCount, nextOffset);
}
cursor.setStatus("SUCCESS");
cursor.setMessage(allInvalidUserIds.isEmpty()
? null
: "本轮刷新已跳过无效 userId 数: " + allInvalidUserIds.size());
List<String> refreshMessages = new ArrayList<>();
if (!allInvalidUserIds.isEmpty()) {
refreshMessages.add("本轮刷新已跳过无效 userId 数: " + allInvalidUserIds.size());
}
if (skippedApiKeyCount > 0) {
refreshMessages.add("本轮刷新已跳过 apiKey 数: " + skippedApiKeyCount
+ "IP 白名单: " + whitelistSkippedApiKeyCount + "");
}
cursor.setMessage(refreshMessages.isEmpty() ? null : String.join("", refreshMessages));
cursor.setApiKeyTotal(allApiKeyAccounts.size());
cursor.setLastProcessedApiKeyCount(apiKeyAccounts.size());
cursor.setLastProcessedApiKeyCount(completedApiKeyCount);
cursor.setNextApiKeyOffset(nextOffset);
cursor.setInvalidUserCount(allInvalidUserIds.size());
cursor.setSampleInvalidUserIds(allInvalidUserIds.stream().limit(20).toList());
cursor.setLastFinishedAt(now);
cursor.setLastSuccessAt(now);
ziniaoTransientCacheService.put(CACHE_TYPE_SHOP_INDEX_REFRESH_CURSOR, "global", cursor, DEFAULT_CURSOR_TTL);
log.info("[ziniao-index] refresh success invalidUsersSkipped={} apiKeyProcessed={}/{} nextOffset={}",
allInvalidUserIds.size(), apiKeyAccounts.size(), allApiKeyAccounts.size(), nextOffset);
log.info("[ziniao-index] refresh success invalidUsersSkipped={} apiKeyCompleted={}/{} apiKeySkipped={} whitelistSkipped={} nextOffset={}",
allInvalidUserIds.size(), completedApiKeyCount, allApiKeyAccounts.size(), skippedApiKeyCount,
whitelistSkippedApiKeyCount, nextOffset);
} catch (Exception ex) {
cursor.setStatus("FAILED");
cursor.setMessage(ex.getMessage());
@@ -358,6 +418,24 @@ public class ZiniaoShopIndexService {
log.info("[ziniao-index] cursor invalidated (transient only; shop rows unchanged)");
}
private void markIpWhitelistAllowedSafely(ZiniaoApiKeyProvider.ApiKeyAccount account) {
try {
ziniaoApiKeyProvider.markIpWhitelistAllowed(account);
} catch (Exception ex) {
log.warn("[ziniao-index] failed to record IP whitelist status accountName={} status=ALLOWED msg={}",
account.accountName(), ex.getMessage());
}
}
private void markIpWhitelistBlockedSafely(ZiniaoApiKeyProvider.ApiKeyAccount account, String message) {
try {
ziniaoApiKeyProvider.markIpWhitelistBlocked(account, message);
} catch (Exception ex) {
log.warn("[ziniao-index] failed to record IP whitelist status accountName={} status=BLOCKED msg={}",
account.accountName(), ex.getMessage());
}
}
public String normalizeShopName(String value) {
if (value == null) {
return "";
@@ -723,9 +801,4 @@ public class ZiniaoShopIndexService {
}
}
private void rethrowIfIpWhitelistError(BusinessException ex) {
if (ziniaoAuthService.isIpWhitelistError(ex)) {
throw new BusinessException("紫鸟刷新店铺索引失败:当前服务器 IP 未加入紫鸟白名单,已停止本轮刷新且不会更新索引数据");
}
}
}
@@ -65,7 +65,7 @@ AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL=https://api.coze.cn
AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH=/v1/workflow/run
AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_ID=7632683471312355338
AIIMAGE_APPEARANCE_PATENT_COZE_TOKEN=
AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE=50
AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE=10
AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS=60000
AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES=30
@@ -90,6 +90,7 @@ aiimage:
bucket: ${AIIMAGE_OSS_BUCKET:nanri-ai-images}
image-video-bucket: ${AIIMAGE_IMAGE_VIDEO_OSS_BUCKET:shufu-video}
digital-human-bucket: ${AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET:nanri-ai-digital-human}
template-bucket: ${AIIMAGE_TEMPLATE_OSS_BUCKET:aiimage-templates}
access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:appuser}
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:AppUser@2024SecureKey}
transient-storage:
@@ -153,7 +154,7 @@ aiimage:
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
retention-days: ${AIIMAGE_MODULE_CLEANUP_RETENTION_DAYS:7}
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,SHOP_DATA_CRAWL,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA}
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA}
permission-schema-init:
enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false}
task-pressure:
@@ -185,6 +186,7 @@ aiimage:
local-worker-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS:10000}
stuck-scan-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_STUCK_SCAN_DELAY_MS:60000}
stuck-timeout-minutes: ${AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES:30}
heartbeat-interval-ms: ${AIIMAGE_RESULT_FILE_JOB_HEARTBEAT_INTERVAL_MS:60000}
batch-size: ${AIIMAGE_RESULT_FILE_JOB_BATCH_SIZE:20}
coze-task:
max-concurrent: ${AIIMAGE_COZE_TASK_MAX_CONCURRENT:12}
@@ -200,7 +202,7 @@ aiimage:
coze-workflow-path: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH:/v1/workflow/run}
coze-workflow-id: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_ID:7639685157562089513}
coze-token: ${AIIMAGE_APPEARANCE_PATENT_COZE_TOKEN:}
coze-batch-size: ${AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE:20}
coze-batch-size: ${AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE:10}
coze-connect-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_CONNECT_TIMEOUT_MILLIS:10000}
coze-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS:60000}
coze-poll-interval-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_POLL_INTERVAL_MILLIS:30000}
@@ -225,9 +227,12 @@ aiimage:
coze-submit-min-interval-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_SUBMIT_MIN_INTERVAL_MILLIS:5000}
coze-flush-pending-minutes: ${AIIMAGE_SIMILAR_ASIN_COZE_FLUSH_PENDING_MINUTES:1}
coze-submit-max-retry-count: ${AIIMAGE_SIMILAR_ASIN_COZE_SUBMIT_MAX_RETRY_COUNT:5}
image-download-pool-size: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_POOL_SIZE:32}
image-download-pool-size: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_POOL_SIZE:8}
image-download-timeout-seconds: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_TIMEOUT_SECONDS:5}
image-prefetch-timeout-seconds: ${AIIMAGE_SIMILAR_ASIN_IMAGE_PREFETCH_TIMEOUT_SECONDS:1800}
result-file-timeout-minutes: ${AIIMAGE_SIMILAR_ASIN_RESULT_FILE_TIMEOUT_MINUTES:90}
image-cache-max-bytes: ${AIIMAGE_SIMILAR_ASIN_IMAGE_CACHE_MAX_BYTES:268435456}
image-local-cache-dir: ${AIIMAGE_SIMILAR_ASIN_IMAGE_LOCAL_CACHE_DIR:${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp}/similar-asin-image-cache}
image-db-cache-enabled: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DB_CACHE_ENABLED:false}
stale-timeout-minutes: ${AIIMAGE_SIMILAR_ASIN_STALE_TIMEOUT_MINUTES:30}
stale-finalize-cron: ${AIIMAGE_SIMILAR_ASIN_STALE_FINALIZE_CRON:0 */2 * * * *}
@@ -0,0 +1,4 @@
ALTER TABLE biz_shop_key
ADD COLUMN ip_whitelist_status VARCHAR(16) NOT NULL DEFAULT 'UNKNOWN' COMMENT 'IP白名单状态:UNKNOWN/ALLOWED/BLOCKED' AFTER ziniao_token,
ADD COLUMN ip_whitelist_checked_at DATETIME NULL COMMENT '最近一次IP白名单检测时间' AFTER ip_whitelist_status,
ADD COLUMN ip_whitelist_message VARCHAR(500) NULL COMMENT '最近一次IP白名单检测信息' AFTER ip_whitelist_checked_at;
@@ -0,0 +1,34 @@
-- Durable marker for retry-exhausted result-file callbacks.
SET @terminal_callback_db_name = DATABASE();
SET @terminal_callback_column_exists = (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @terminal_callback_db_name
AND TABLE_NAME = 'biz_task_file_job'
AND COLUMN_NAME = 'terminal_callback_at'
);
SET @terminal_callback_sql = IF(
@terminal_callback_column_exists = 0,
'ALTER TABLE `biz_task_file_job` ADD COLUMN `terminal_callback_at` DATETIME NULL COMMENT ''parent terminal failure callback completed'' AFTER `finished_at`',
'SELECT 1'
);
PREPARE terminal_callback_stmt FROM @terminal_callback_sql;
EXECUTE terminal_callback_stmt;
DEALLOCATE PREPARE terminal_callback_stmt;
SET @terminal_callback_index_exists = (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @terminal_callback_db_name
AND TABLE_NAME = 'biz_task_file_job'
AND INDEX_NAME = 'idx_file_job_terminal_callback'
);
SET @terminal_callback_sql = IF(
@terminal_callback_index_exists = 0,
'ALTER TABLE `biz_task_file_job` ADD INDEX `idx_file_job_terminal_callback` (`status`, `retry_count`, `terminal_callback_at`, `updated_at`)',
'SELECT 1'
);
PREPARE terminal_callback_stmt FROM @terminal_callback_sql;
EXECUTE terminal_callback_stmt;
DEALLOCATE PREPARE terminal_callback_stmt;
@@ -0,0 +1,25 @@
-- Admin menu and independent data permission for shop-data-crawl task history.
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
SELECT '店铺数据任务管理', 'admin_shop_data_crawl_tasks', 'admin', 'shop-data-crawl-tasks', 82
WHERE NOT EXISTS (
SELECT 1 FROM `columns` WHERE `column_key` = 'admin_shop_data_crawl_tasks'
);
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
SELECT '店铺数据任务数据查看', 'admin_shop_data_crawl_task_data', 'internal', 'shop-data-crawl-task-data', 0
WHERE NOT EXISTS (
SELECT 1 FROM `columns` WHERE `column_key` = 'admin_shop_data_crawl_task_data'
);
-- Preserve the video-task convention: users who already received the visible
-- task menu receive the initial data grant. Later changes use the dedicated UI.
INSERT IGNORE INTO `user_column_permission` (`user_id`, `column_id`)
SELECT old_perm.`user_id`, data_col.`id`
FROM `user_column_permission` old_perm
INNER JOIN `columns` old_col ON old_col.`id` = old_perm.`column_id`
INNER JOIN `columns` data_col ON data_col.`column_key` = 'admin_shop_data_crawl_task_data'
LEFT JOIN `user_column_permission` existing
ON existing.`user_id` = old_perm.`user_id`
AND existing.`column_id` = data_col.`id`
WHERE old_col.`column_key` = 'admin_shop_data_crawl_tasks'
AND existing.`user_id` IS NULL;
@@ -10,6 +10,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
class AppearancePatentTaskServiceTest {
@Test
void taskStatusDependsOnExecutionOutcomeOnly() {
assertEquals("RUNNING", AppearancePatentTaskService.resolveTaskExecutionStatus(true, false));
assertEquals("SUCCESS", AppearancePatentTaskService.resolveTaskExecutionStatus(false, false));
assertEquals("FAILED", AppearancePatentTaskService.resolveTaskExecutionStatus(false, true));
}
@Test
void resultStatusFailsOnlyWhenConclusionIsEmpty() {
assertEquals("\u5931\u8d25", AppearancePatentTaskService.resolveResultStatus(null));
@@ -0,0 +1,27 @@
package com.nanri.aiimage.modules.auth.util;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class WerkzeugPasswordEncoderTest {
private final WerkzeugPasswordEncoder encoder = new WerkzeugPasswordEncoder();
@Test
void matchesWerkzeugScryptHash() {
String hash = "scrypt:32768:8:1$VEWwSnHkAFcK3B3E$2065ba2db25c34072bf4d7ae9bd47b8c483289fafc3f5e98ae49bb232d30bf989b21ae0e9f14adc6e893e34a81543cf58641731431961ed9ab353bcebec2e78a";
assertTrue(encoder.matches("test-password", hash));
assertFalse(encoder.matches("wrong-password", hash));
}
@Test
void matchesGeneratedPbkdf2Hash() {
String hash = encoder.hash("test-password");
assertTrue(encoder.matches("test-password", hash));
assertFalse(encoder.matches("wrong-password", hash));
}
}
@@ -0,0 +1,37 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.nio.file.Path;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
class CollectDataExcelAssemblyServiceTest {
@TempDir
Path tempDir;
@Test
void writesAsinFilterToSummarySheet() throws Exception {
CollectDataSummaryRowDto summary = new ObjectMapper().readValue(
"{\"keyword\":\"phone case\",\"asinFilter\":3}",
CollectDataSummaryRowDto.class);
File output = tempDir.resolve("collect-data-result.xlsx").toFile();
new CollectDataExcelAssemblyService().writeWorkbook(output, List.of(), List.of(summary), List.of());
try (Workbook workbook = WorkbookFactory.create(output)) {
Sheet sheet = workbook.getSheet("结果文件");
assertThat(sheet.getRow(0).getCell(6).getStringCellValue()).isEqualTo("ASIN过滤");
assertThat(sheet.getRow(1).getCell(6).getNumericCellValue()).isEqualTo(3);
}
}
}
@@ -0,0 +1,196 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils;
import java.time.LocalDateTime;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class CollectDataServiceTest {
@Mock
private FileTaskMapper fileTaskMapper;
@Mock
private FileResultMapper fileResultMapper;
@Mock
private TaskChunkMapper taskChunkMapper;
@Mock
private TaskDistributedLockService taskDistributedLockService;
@Mock
private TaskFileJobService taskFileJobService;
@Mock
private TaskDistributedLockService.LockHandle lockHandle;
@Spy
private ObjectMapper objectMapper = new ObjectMapper();
@InjectMocks
private CollectDataService service;
@BeforeEach
void setUp() {
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
}
@Test
void dashboardCountsPendingAndRunningTasksAsActive() {
when(fileTaskMapper.selectCount(any())).thenReturn(5L, 3L, 2L);
CollectDataDashboardVo dashboard = service.dashboard(7L);
assertThat(dashboard.getPendingTaskCount()).isEqualTo(5L);
assertThat(dashboard.getSuccessTaskCount()).isEqualTo(3L);
assertThat(dashboard.getFailedTaskCount()).isEqualTo(2L);
assertThat(dashboard.getProcessedTaskCount()).isEqualTo(5L);
}
@Test
void progressBatchExposesProcessedKeywordProgress() {
FileTaskEntity task = new FileTaskEntity();
task.setId(91L);
task.setTaskNo("COLLECT_DATA-91");
task.setModuleType(CollectDataService.MODULE_TYPE);
task.setStatus("RUNNING");
task.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
task.setResultJson("{\"totalRows\":10,\"receivedRows\":24,\"processedRows\":4}");
FileResultEntity result = new FileResultEntity();
result.setId(101L);
result.setTaskId(task.getId());
result.setModuleType(CollectDataService.MODULE_TYPE);
result.setRowCount(10);
result.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
when(fileResultMapper.selectList(any())).thenReturn(List.of(result));
CollectDataTaskBatchVo progress = service.progressBatch(List.of(task.getId()));
assertThat(progress.getItems()).hasSize(1);
assertThat(progress.getItems().getFirst().getItems()).hasSize(1);
assertThat(progress.getItems().getFirst().getItems().getFirst().getTotalRows()).isEqualTo(10);
assertThat(progress.getItems().getFirst().getItems().getFirst().getReceivedRows()).isEqualTo(24);
assertThat(progress.getItems().getFirst().getItems().getFirst().getProcessedRows()).isEqualTo(4);
assertThat(progress.getItems().getFirst().getItems().getFirst().getProgressPercent()).isEqualTo(40);
}
@Test
void failTaskMarksAnActivatedTaskAsFailed() {
FileTaskEntity task = new FileTaskEntity();
task.setId(92L);
task.setUserId(7L);
task.setModuleType(CollectDataService.MODULE_TYPE);
task.setStatus("RUNNING");
task.setResultJson("{\"totalRows\":10,\"receivedRows\":4}");
FileResultEntity result = new FileResultEntity();
result.setId(102L);
result.setTaskId(task.getId());
result.setModuleType(CollectDataService.MODULE_TYPE);
when(fileTaskMapper.selectById(task.getId())).thenReturn(task);
when(fileResultMapper.selectOne(any())).thenReturn(result);
service.failTask(task.getId(), task.getUserId(), "queue unavailable");
assertThat(task.getStatus()).isEqualTo("FAILED");
assertThat(task.getErrorMessage()).isEqualTo("queue unavailable");
assertThat(result.getSuccess()).isZero();
assertThat(result.getErrorMessage()).isEqualTo("queue unavailable");
verify(fileResultMapper).updateById(result);
verify(fileTaskMapper).updateById(task);
}
@Test
void staleTaskWithResultChunksEnqueuesPartialWorkbook() throws Exception {
FileTaskEntity task = staleTask(93L);
task.setResultJson("{\"finalRowCount\":343}");
FileResultEntity result = taskResult(task, 103L);
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, task.getId(), 0L))
.thenReturn(lockHandle);
when(fileTaskMapper.selectById(task.getId())).thenReturn(task);
when(taskFileJobService.countUnfinishedAssembleJobs(task.getId(), CollectDataService.MODULE_TYPE))
.thenReturn(0L);
when(fileResultMapper.selectOne(any())).thenReturn(result);
when(taskChunkMapper.selectCount(any())).thenReturn(170L);
service.finalizeStaleTasks();
verify(taskFileJobService).enqueueAssembleResult(
task.getId(), CollectDataService.MODULE_TYPE, result.getId(), "task:" + task.getId());
assertThat(task.getStatus()).isEqualTo("RUNNING");
assertThat(result.getRowCount()).isEqualTo(343);
verify(lockHandle).close();
}
@Test
void staleTaskWithoutResultChunksFails() throws Exception {
FileTaskEntity task = staleTask(94L);
task.setResultJson("{\"finalRowCount\":0}");
FileResultEntity result = taskResult(task, 104L);
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, task.getId(), 0L))
.thenReturn(lockHandle);
when(fileTaskMapper.selectById(task.getId())).thenReturn(task);
when(taskFileJobService.countUnfinishedAssembleJobs(task.getId(), CollectDataService.MODULE_TYPE))
.thenReturn(0L);
when(fileResultMapper.selectOne(any())).thenReturn(result);
when(taskChunkMapper.selectCount(any())).thenReturn(0L);
service.finalizeStaleTasks();
assertThat(task.getStatus()).isEqualTo("FAILED");
assertThat(task.getErrorMessage()).contains("Python 心跳");
verify(taskFileJobService, never()).enqueueAssembleResult(any(), any(), any(), any());
verify(lockHandle).close();
}
private static FileTaskEntity staleTask(long taskId) {
FileTaskEntity task = new FileTaskEntity();
task.setId(taskId);
task.setModuleType(CollectDataService.MODULE_TYPE);
task.setStatus("RUNNING");
task.setUpdatedAt(LocalDateTime.now().minusMinutes(31));
return task;
}
private static FileResultEntity taskResult(FileTaskEntity task, long resultId) {
FileResultEntity result = new FileResultEntity();
result.setId(resultId);
result.setTaskId(task.getId());
result.setModuleType(CollectDataService.MODULE_TYPE);
result.setSourceFilename("collect.xlsx");
return result;
}
}
@@ -1,5 +1,8 @@
package com.nanri.aiimage.modules.dedupe.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
@@ -16,6 +19,7 @@ import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.test.util.ReflectionTestUtils;
@@ -159,7 +163,7 @@ class DedupeTotalDataServiceTest {
when(dedupeTotalDataMapper.selectCount(any())).thenReturn(0L);
when(dedupeTotalDataMapper.selectList(any())).thenReturn(List.of());
DedupeTotalDataPageVo page = service.page(1, 15, "", "other", 10L);
DedupeTotalDataPageVo page = service.page(1, 15, "", "other", null, null, 10L);
assertEquals(0L, page.getTotal());
assertTrue(page.getItems().isEmpty());
@@ -168,6 +172,47 @@ class DedupeTotalDataServiceTest {
verify(dedupeTotalDataMapper).selectList(any());
}
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
void pageUsesInclusiveDateRange() {
when(adminUserMapper.selectById(1L)).thenReturn(user(1L, "super_admin", "root"));
when(dedupeTotalDataMapper.selectCount(any())).thenReturn(0L);
when(dedupeTotalDataMapper.selectList(any())).thenReturn(List.of());
service.page(
1,
15,
"",
"",
LocalDate.of(2026, 7, 31),
LocalDate.of(2026, 7, 31),
1L);
ArgumentCaptor<LambdaQueryWrapper<DedupeTotalDataEntity>> queryCaptor =
ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
verify(dedupeTotalDataMapper).selectCount(queryCaptor.capture());
LambdaQueryWrapper<DedupeTotalDataEntity> query = queryCaptor.getValue();
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
DedupeTotalDataEntity.class);
query.getSqlSegment();
assertTrue(query.getParamNameValuePairs().containsValue(LocalDate.of(2026, 7, 31).atStartOfDay()));
assertTrue(query.getParamNameValuePairs().containsValue(LocalDate.of(2026, 8, 1).atStartOfDay()));
}
@Test
void pageRejectsReversedDateRange() {
assertThrows(BusinessException.class, () -> service.page(
1,
15,
"",
"",
LocalDate.of(2026, 8, 1),
LocalDate.of(2026, 7, 31),
1L));
verify(dedupeTotalDataMapper, never()).selectCount(any());
}
@Test
void comparableValueLookupRemainsGlobal() {
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678")))
@@ -21,6 +21,7 @@ class OssStorageServiceTest {
properties.setBucket("nanri-ai-images");
properties.setImageVideoBucket("shufu-video");
properties.setDigitalHumanBucket("nanri-ai-digital-human");
properties.setTemplateBucket("aiimage-templates");
properties.setAccessKeyId("test-access-key");
properties.setAccessKeySecret("test-secret-key");
storageService = new OssStorageService(properties);
@@ -54,6 +55,10 @@ class OssStorageServiceTest {
"https://oss.aishufu.top/nanri-ai-digital-human/digital-human/versions/demo.mp4",
storageService.normalizeManagedPublicUrl(
"https://nanri-ai-digital-human.oss.aishufu.top/digital-human/versions/demo.mp4"));
assertEquals(
"https://oss.aishufu.top/aiimage-templates/input/publish.xlsx",
storageService.normalizeManagedPublicUrl(
"http://47.110.241.161:9000/aiimage-templates/input/publish.xlsx"));
}
@Test
@@ -0,0 +1,53 @@
package com.nanri.aiimage.modules.filetemplate;
import org.junit.jupiter.api.Test;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.server.ResponseStatusException;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
class ModuleTemplateControllerTest {
@Test
void returnsXlsxWithUtf8AttachmentFilename() throws Exception {
ModuleTemplateService service = mock(ModuleTemplateService.class);
byte[] bytes = {1, 2, 3, 4};
when(service.download("publish")).thenReturn(new ModuleTemplateService.TemplateDownload(
"上架 文档格式.xlsx",
ModuleTemplateService.XLSX_CONTENT_TYPE,
bytes));
MockMvc mockMvc = MockMvcBuilders.standaloneSetup(new ModuleTemplateController(service)).build();
mockMvc.perform(get("/api/module-templates/publish/download").param("user_id", "1"))
.andExpect(status().isOk())
.andExpect(content().bytes(bytes))
.andExpect(content().contentType(ModuleTemplateService.XLSX_CONTENT_TYPE))
.andExpect(header().longValue(HttpHeaders.CONTENT_LENGTH, bytes.length))
.andExpect(header().string(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"download.xlsx\"; "
+ "filename*=UTF-8''%E4%B8%8A%E6%9E%B6%20%E6%96%87%E6%A1%A3%E6%A0%BC%E5%BC%8F.xlsx"));
}
@Test
void propagatesUnknownModuleAndStorageStatusCodes() throws Exception {
ModuleTemplateService service = mock(ModuleTemplateService.class);
when(service.download("missing"))
.thenThrow(new ResponseStatusException(HttpStatus.NOT_FOUND, "模板不存在"));
when(service.download("publish"))
.thenThrow(new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "模板存储暂不可用"));
MockMvc mockMvc = MockMvcBuilders.standaloneSetup(new ModuleTemplateController(service)).build();
mockMvc.perform(get("/api/module-templates/missing/download"))
.andExpect(status().isNotFound());
mockMvc.perform(get("/api/module-templates/publish/download"))
.andExpect(status().isServiceUnavailable());
}
}
@@ -0,0 +1,51 @@
package com.nanri.aiimage.modules.filetemplate;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.ClassPathResource;
import java.io.InputStream;
import java.util.Set;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
class ModuleTemplateRegistryTest {
@Test
void exposesSixFixedModulesWithPackagedXlsxResources() throws Exception {
ModuleTemplateRegistry registry = new ModuleTemplateRegistry();
assertEquals(Set.of(
"publish",
"delete-brand",
"appearance-patent",
"price-track",
"collect-data",
"similar-asin"),
registry.templates().stream()
.map(ModuleTemplateRegistry.ModuleTemplate::moduleCode)
.collect(Collectors.toSet()));
for (ModuleTemplateRegistry.ModuleTemplate template : registry.templates()) {
assertTrue(template.resourcePath().matches("templates/module-input/[a-z-]+\\.xlsx"));
assertEquals("input/" + template.resourcePath().substring(template.resourcePath().lastIndexOf('/') + 1),
template.objectKey());
ClassPathResource resource = new ClassPathResource(template.resourcePath());
assertTrue(resource.exists(), template.resourcePath());
try (InputStream input = resource.getInputStream()) {
assertArrayEquals(new byte[]{'P', 'K'}, input.readNBytes(2), template.resourcePath());
}
}
}
@Test
void lookupNormalizesCaseAndWhitespaceWithoutAcceptingUnknownCodes() {
ModuleTemplateRegistry registry = new ModuleTemplateRegistry();
assertEquals("上架 文档格式.xlsx", registry.find(" PUBLISH ").orElseThrow().downloadFilename());
assertTrue(registry.find("../../publish").isEmpty());
assertTrue(registry.find(null).isEmpty());
}
}
@@ -0,0 +1,120 @@
package com.nanri.aiimage.modules.filetemplate;
import com.nanri.aiimage.config.OssProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.http.HttpStatus;
import org.springframework.web.server.ResponseStatusException;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
class ModuleTemplateServiceTest {
private static final String BUCKET = "aiimage-templates";
private OssStorageService ossStorageService;
private ModuleTemplateService service;
@BeforeEach
void setUp() {
OssProperties properties = new OssProperties();
properties.setTemplateBucket(BUCKET);
ossStorageService = mock(OssStorageService.class);
service = new ModuleTemplateService(new ModuleTemplateRegistry(), properties, ossStorageService);
}
@Test
void downloadsExistingObjectWithoutUploadingItAgain() {
byte[] stored = {1, 2, 3};
when(ossStorageService.objectExists(BUCKET, "input/publish.xlsx")).thenReturn(true);
when(ossStorageService.readObjectBytes(BUCKET, "input/publish.xlsx")).thenReturn(stored);
ModuleTemplateService.TemplateDownload download = service.download("publish");
assertEquals("上架 文档格式.xlsx", download.filename());
assertEquals(ModuleTemplateService.XLSX_CONTENT_TYPE, download.contentType());
assertArrayEquals(stored, download.content());
verify(ossStorageService).ensureBucketExists(BUCKET);
verify(ossStorageService, never()).uploadBytes(
eq(BUCKET), eq("input/publish.xlsx"), org.mockito.ArgumentMatchers.any(), anyString());
}
@Test
void lazilyUploadsMissingObjectBeforeReadingItFromStorage() {
byte[] stored = {4, 5, 6};
when(ossStorageService.objectExists(BUCKET, "input/publish.xlsx")).thenReturn(false);
when(ossStorageService.readObjectBytes(BUCKET, "input/publish.xlsx")).thenReturn(stored);
ModuleTemplateService.TemplateDownload download = service.download("publish");
assertArrayEquals(stored, download.content());
ArgumentCaptor<byte[]> uploaded = ArgumentCaptor.forClass(byte[].class);
verify(ossStorageService).uploadBytes(
eq(BUCKET), eq("input/publish.xlsx"), uploaded.capture(),
eq(ModuleTemplateService.XLSX_CONTENT_TYPE));
assertTrue(uploaded.getValue().length > 2);
assertArrayEquals(new byte[]{'P', 'K'}, new byte[]{uploaded.getValue()[0], uploaded.getValue()[1]});
verify(ossStorageService).readObjectBytes(BUCKET, "input/publish.xlsx");
}
@Test
void startupSynchronizationUploadsOnlyMissingTemplates() {
when(ossStorageService.objectExists(BUCKET, "input/publish.xlsx")).thenReturn(true);
assertEquals(5, service.synchronizeAll());
verify(ossStorageService).ensureBucketExists(BUCKET);
verify(ossStorageService, times(6)).objectExists(eq(BUCKET), anyString());
verify(ossStorageService, never()).uploadBytes(
eq(BUCKET), eq("input/publish.xlsx"), org.mockito.ArgumentMatchers.any(), anyString());
verify(ossStorageService, times(5)).uploadBytes(
eq(BUCKET), anyString(), org.mockito.ArgumentMatchers.any(),
eq(ModuleTemplateService.XLSX_CONTENT_TYPE));
}
@Test
void unknownModuleReturnsNotFoundWithoutTouchingStorage() {
ResponseStatusException ex = assertThrows(ResponseStatusException.class,
() -> service.download("not-a-module"));
assertEquals(HttpStatus.NOT_FOUND, ex.getStatusCode());
verifyNoInteractions(ossStorageService);
}
@Test
void storageFailureReturnsServiceUnavailable() {
doThrow(new IllegalStateException("MinIO unavailable"))
.when(ossStorageService).ensureBucketExists(BUCKET);
ResponseStatusException ex = assertThrows(ResponseStatusException.class,
() -> service.download("publish"));
assertEquals(HttpStatus.SERVICE_UNAVAILABLE, ex.getStatusCode());
}
@Test
void initializerLeavesApplicationRunningWhenStartupStorageIsUnavailable() {
ModuleTemplateService unavailableService = mock(ModuleTemplateService.class);
doThrow(new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "unavailable"))
.when(unavailableService).synchronizeAll();
ModuleTemplateStorageInitializer initializer = new ModuleTemplateStorageInitializer(unavailableService);
assertDoesNotThrow(initializer::synchronize);
verify(unavailableService).synchronizeAll();
}
}
@@ -248,4 +248,32 @@ class PermissionMenuControllerTest {
verify(service).listImageVideoDataPermissionUsers(operator);
verify(service).updateImageVideoDataPermissionUsers(operator, List.of(20L));
}
@Test
void shopDataCrawlPermissionEndpointsDelegateAuthenticatedOperator() {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService service = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
AdminUserEntity operator = new AdminUserEntity();
operator.setId(1L);
operator.setRole("super_admin");
ImageVideoDataPermissionUserVo permissionUser = new ImageVideoDataPermissionUserVo();
permissionUser.setId(20L);
ImageVideoDataPermissionUpdateRequest body = new ImageVideoDataPermissionUpdateRequest();
body.setUserIds(List.of(20L));
when(authSupport.requireAdmin(request)).thenReturn(operator);
when(service.listShopDataCrawlDataPermissionUsers(operator)).thenReturn(List.of(permissionUser));
when(service.updateShopDataCrawlDataPermissionUsers(operator, List.of(20L))).thenReturn(1);
var listResponse = controller.listShopDataCrawlDataPermissionUsers(request);
var updateResponse = controller.updateShopDataCrawlDataPermissionUsers(request, body);
assertThat(listResponse.getData()).containsExactly(permissionUser);
assertThat(updateResponse.getData()).isEqualTo(1);
verify(authSupport, times(2)).requireAdmin(request);
verify(service).listShopDataCrawlDataPermissionUsers(operator);
verify(service).updateShopDataCrawlDataPermissionUsers(operator, List.of(20L));
}
}
@@ -74,6 +74,28 @@ class PermissionMenuServiceTest {
verify(permissionMapper, times(0)).insert(any(UserColumnPermissionEntity.class));
}
@Test
void preservesBothIndependentDataPermissionsDuringGenericReplacement() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
when(userMapper.selectById(7L)).thenReturn(new AdminUserEntity());
when(menuMapper.selectCount(any())).thenReturn(1L);
when(menuMapper.selectOne(any())).thenReturn(imageVideoDataPermission(), shopDataCrawlDataPermission());
when(permissionMapper.selectCount(any())).thenReturn(1L);
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
request.setColumnIds(List.of(2L));
service.updateUserColumnPermissions(7L, request);
ArgumentCaptor<UserColumnPermissionEntity> captor = ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper, times(3)).insert(captor.capture());
assertThat(captor.getAllValues())
.extracting(UserColumnPermissionEntity::getColumnId)
.containsExactly(2L, 75L, 76L);
}
@Test
void expandsDirectParentGrantToDescendantsWithoutPersistingChildren() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
@@ -526,6 +548,55 @@ class PermissionMenuServiceTest {
assertThat(inserted.getValue().getColumnId()).isEqualTo(75L);
}
@Test
void shopDataCrawlPermissionCanOnlyBeManagedBySuperAdmin() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity explicitAdmin = user(1L, "admin", 1);
assertThatThrownBy(() -> service.listShopDataCrawlDataPermissionUsers(explicitAdmin))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("超级管理员");
assertThatThrownBy(() -> service.updateShopDataCrawlDataPermissionUsers(explicitAdmin, List.of(2L)))
.isInstanceOf(BusinessException.class)
.hasMessageContaining("超级管理员");
verify(menuMapper, never()).selectOne(any());
verify(permissionMapper, never()).delete(any());
}
@Test
void superAdminCanListAndReplaceShopDataCrawlPermissions() {
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
AdminUserMapper userMapper = mock(AdminUserMapper.class);
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
AdminUserEntity operator = user(99L, "super_admin", 1);
AdminUserEntity admin = user(1L, "admin", 1);
admin.setUsername("admin");
AdminUserEntity normal = user(2L, "normal", 0);
normal.setUsername("normal");
when(menuMapper.selectOne(any())).thenReturn(shopDataCrawlDataPermission());
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(1L, 76L)));
when(userMapper.selectList(any())).thenReturn(List.of(operator, admin, normal));
List<ImageVideoDataPermissionUserVo> users = service.listShopDataCrawlDataPermissionUsers(operator);
int grantedCount = service.updateShopDataCrawlDataPermissionUsers(operator, List.of(2L));
assertThat(users).extracting(ImageVideoDataPermissionUserVo::getId).containsExactly(1L, 2L);
assertThat(users.get(0).isGranted()).isTrue();
assertThat(users.get(1).isGranted()).isFalse();
assertThat(grantedCount).isEqualTo(1);
verify(permissionMapper).deleteByMap(Map.of("column_id", 76L));
ArgumentCaptor<UserColumnPermissionEntity> inserted =
ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
verify(permissionMapper).insert(inserted.capture());
assertThat(inserted.getValue().getUserId()).isEqualTo(2L);
assertThat(inserted.getValue().getColumnId()).isEqualTo(76L);
}
private PermissionMenuCreateRequest createRequest(Long parentId, String menuType) {
PermissionMenuCreateRequest request = new PermissionMenuCreateRequest();
request.setName("child");
@@ -575,4 +646,11 @@ class PermissionMenuServiceTest {
entity.setColumnKey("admin_image_video_task_data");
return entity;
}
private PermissionMenuEntity shopDataCrawlDataPermission() {
PermissionMenuEntity entity = new PermissionMenuEntity();
entity.setId(76L);
entity.setColumnKey("admin_shop_data_crawl_task_data");
return entity;
}
}
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.pricetrack.mapper.PriceTrackShopCandidateMapper;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequest;
import com.nanri.aiimage.modules.shopkey.service.SkipPriceAsinService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
@@ -24,8 +25,12 @@ import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -86,4 +91,59 @@ class PriceTrackTaskServiceTest {
verify(priceTrackLoopRunService).syncLoopRunAfterChildTerminal(taskId);
verify(lock).close();
}
@Test
void completedPayloadWithoutUsableRowsFailsWithoutCreatingResultFile() throws Exception {
long taskId = 20818L;
String shopName = "蔡建芳";
FileTaskEntity task = new FileTaskEntity();
task.setId(taskId);
task.setUserId(672L);
task.setModuleType("PRICE_TRACK");
task.setStatus("RUNNING");
task.setRequestJson("{}");
FileResultEntity result = new FileResultEntity();
result.setId(92327L);
result.setTaskId(taskId);
result.setModuleType("PRICE_TRACK");
result.setSourceFilename(shopName);
result.setSuccess(0);
PriceTrackSubmitResultRequest.AsinResult blankRow = new PriceTrackSubmitResultRequest.AsinResult();
blankRow.setShopMallName("Cai Jianfang");
blankRow.setAsin("");
PriceTrackSubmitResultRequest.ShopResult shopResult = new PriceTrackSubmitResultRequest.ShopResult();
shopResult.setShopName(shopName);
shopResult.setCountries(Map.of("DE", List.of(blankRow)));
shopResult.setError("");
shopResult.setSuccess(true);
PriceTrackSubmitResultRequest request = new PriceTrackSubmitResultRequest();
request.setShops(List.of(shopResult));
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire("PRICE_TRACK", taskId)).thenReturn(lock);
when(priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId))).thenReturn(Map.of(taskId, task));
when(fileResultMapper.selectList(any())).thenReturn(List.of(result));
when(ziniaoShopSwitchService.normalizeShopName(shopName)).thenReturn(shopName);
when(excelAssemblyService.normalizeCountriesMap(any())).thenAnswer(invocation -> invocation.getArgument(0));
when(objectMapper.writeValueAsString(any())).thenReturn("[]");
service.submitResult(taskId, request);
assertEquals("FAILED", task.getStatus());
assertEquals(shopName + ": 未收到有效跟价数据,未生成结果文件", task.getErrorMessage());
assertNotNull(task.getFinishedAt());
assertEquals(0, result.getSuccess());
assertEquals("未收到有效跟价数据,未生成结果文件", result.getErrorMessage());
assertEquals(0, result.getRowCount());
assertNull(result.getResultFilename());
assertNull(result.getResultFileUrl());
verify(taskFileJobService, never()).enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
verify(priceTrackLoopRunService).syncLoopRunAfterChildTerminal(taskId);
verify(lock).close();
}
}
@@ -22,6 +22,7 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
@@ -164,6 +165,19 @@ class PublishTaskServiceTest {
taskCaptor.getValue().getResultJson()).path("ownerInstanceId").asText());
}
@Test
void zeroOnlyHeartbeatDoesNotResetResultChunkProgress() {
TaskHeartbeatRequest heartbeat = new TaskHeartbeatRequest();
heartbeat.setCurrent(0);
heartbeat.setTotal(0);
service.touchHeartbeat(20998L, heartbeat);
verify(fileTaskMapper).update(isNull(), any());
verify(publishFileMapper, never()).selectOne(any());
verify(publishFileMapper, never()).update(isNull(), any());
}
@Test
void taskAccessRejectsAnotherInstanceForExistingRouterToForward() throws Exception {
long taskId = 109L;
@@ -344,6 +358,7 @@ class PublishTaskServiceTest {
long resultId = 313L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "分片.xlsx");
file.setTotalRows(2);
FileResultEntity result = result(taskId, resultId);
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
@@ -357,15 +372,21 @@ class PublishTaskServiceTest {
service.submitResult(taskId, chunkResultRequest(7L, fileId, 2, 2, List.of(row("2"))));
assertEquals("RUNNING", file.getStatus());
assertEquals(1, file.getProcessedRows());
assertEquals(1, storedChunks.size());
assertEquals(1, storedScopes.getFirst().getReceivedChunkCount());
assertEquals(0, storedScopes.getFirst().getCompleted());
assertTrue(storedScopes.getFirst().getStateJson().contains("\"receivedRows\":1"));
verify(publishItemMapper, never()).delete(any());
verify(taskFileJobService, never()).enqueueAssembleResult(any(), any(), any(), any());
// Simulate a scope created by the pre-progress implementation.
storedScopes.getFirst().setStateJson("{\"phase\":\"RECEIVING\"}");
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
assertEquals("SUCCESS", file.getStatus());
assertEquals(2, file.getProcessedRows());
assertTrue(storedScopes.getFirst().getStateJson().contains("\"receivedRows\":2"));
assertEquals(2, storedChunks.size());
assertEquals(2, storedScopes.getFirst().getReceivedChunkCount());
assertEquals(1, storedScopes.getFirst().getCompleted());
@@ -386,6 +407,7 @@ class PublishTaskServiceTest {
long fileId = 214L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "重试.xlsx");
file.setTotalRows(2);
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
PublishSubmitResultRequest request = chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")));
@@ -399,11 +421,41 @@ class PublishTaskServiceTest {
service.submitResult(taskId, request);
assertEquals(1, storedChunks.size());
assertEquals(1, file.getProcessedRows());
assertEquals(1, rustfsPayloads.size());
verify(taskChunkMapper, times(1)).insert(any(TaskChunkEntity.class));
verify(publishItemMapper, never()).delete(any());
}
@Test
void progressBatchExposesRowsReceivedByResultSubmission() {
long taskId = 122L;
long fileId = 222L;
FileTaskEntity task = task(taskId, 7L, "RUNNING");
PublishFileEntity file = file(taskId, fileId, "RUNNING", "progress.xlsx");
FileResultEntity result = result(taskId, 322L);
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
when(publishFileMapper.selectById(fileId)).thenReturn(file);
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
when(fileResultMapper.selectOne(any())).thenReturn(result);
when(fileResultMapper.selectList(any())).thenReturn(List.of(result));
when(publishItemMapper.selectCount(any())).thenReturn(3L);
when(taskFileJobService.findAssembleJobsByResultIds(any(), any())).thenReturn(Map.of());
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 3, List.of(row("1"))));
var progress = service.getTaskProgress(7L, List.of(taskId));
assertEquals(1, progress.getItems().getFirst().getTask().getProcessedRows());
assertEquals(3, progress.getItems().getFirst().getTask().getTotalRows());
assertEquals(1, progress.getItems().getFirst().getFiles().getFirst().getProcessedRows());
assertEquals(3, progress.getItems().getFirst().getFiles().getFirst().getTotalRows());
assertEquals(33, progress.getItems().getFirst().getFiles().getFirst().getPercent());
}
@Test
void resultChunkRetryWithDifferentContentIsRejected() {
long taskId = 115L;
@@ -17,11 +17,9 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.zip.ZipFile;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -37,33 +35,31 @@ class PublishWorkbookServiceTest {
File valid = directory.resolve("valid.xlsx").toFile();
try (Workbook workbook = new XSSFWorkbook();
FileOutputStream output = new FileOutputStream(valid)) {
Sheet uk = workbook.createSheet("英国数据");
writeSourceSheet(uk, "英国", "B001");
Sheet first = workbook.createSheet("first");
writeSourceSheet(first, "UK", "B001");
for (int index = PublishWorkbookService.SOURCE_HEADERS.size();
index < PublishWorkbookService.RESULT_HEADERS.size(); index++) {
uk.getRow(0).createCell(index)
first.getRow(0).createCell(index)
.setCellValue(PublishWorkbookService.RESULT_HEADERS.get(index));
}
uk.getRow(1).createCell(5).setCellValue("旧状态");
uk.getRow(1).createCell(6).setCellValue("旧同步状态");
uk.getRow(1).createCell(7).setCellValue("旧同步国家");
workbook.createSheet("空白页");
writeSourceSheet(workbook.createSheet("德国数据"), "DE", "B002");
first.getRow(1).createCell(5).setCellValue("old status");
workbook.createSheet("empty");
writeSourceSheet(workbook.createSheet("second"), "DE", "B002");
workbook.write(output);
}
PublishWorkbookService.ParsedWorkbook parsed = service.parse(valid);
assertEquals(2, parsed.rows().size());
assertEquals("B001", parsed.rows().get(0).getAsin());
assertNull(parsed.rows().get(0).getStatus());
assertNull(parsed.rows().get(0).getSyncStatus());
assertNull(parsed.rows().get(0).getSyncCountries());
assertEquals("B001", parsed.rows().getFirst().getAsin());
assertNull(parsed.rows().getFirst().getStatus());
assertNull(parsed.rows().getFirst().getSyncStatus());
assertNull(parsed.rows().getFirst().getSyncCountries());
assertEquals("DE", parsed.rows().get(1).getCountry());
File invalid = directory.resolve("invalid.xlsx").toFile();
try (Workbook workbook = new XSSFWorkbook();
FileOutputStream output = new FileOutputStream(invalid)) {
Sheet sheet = workbook.createSheet("错误表头");
Sheet sheet = workbook.createSheet("invalid");
Row header = sheet.createRow(0);
List<String> headers = new ArrayList<>(PublishWorkbookService.SOURCE_HEADERS);
headers.set(1, "Asin");
@@ -79,30 +75,34 @@ class PublishWorkbookServiceTest {
}
@Test
void writesOneSheetPerNormalizedCountryWithExactHeaders() throws Exception {
void writesOnlyThePublishCountrySheetAndKeepsSyncColumns() throws Exception {
Path directory = Files.createTempDirectory("publish-sheets-");
try {
File output = directory.resolve("result.xlsx").toFile();
service.writeWorkbook(output, List.of(
row("1", "B001", "UK", "19.99"),
row("2", "B002", "德国", "not-a-number"),
row("3", "B003", "GB", "20")));
row("2", "B002", "FR", "not-a-number"),
row("3", "B003", "GB", "20")), "DE");
try (FileInputStream input = new FileInputStream(output);
Workbook workbook = new XSSFWorkbook(input)) {
assertEquals(2, workbook.getNumberOfSheets());
assertEquals(Set.of("英国", "德国"),
Set.of(workbook.getSheetName(0), workbook.getSheetName(1)));
Sheet uk = workbook.getSheet("英国");
assertNotNull(uk);
assertEquals(1, workbook.getNumberOfSheets());
assertEquals("\u5fb7\u56fd", workbook.getSheetName(0));
Sheet germany = workbook.getSheetAt(0);
for (int index = 0; index < PublishWorkbookService.RESULT_HEADERS.size(); index++) {
assertEquals(PublishWorkbookService.RESULT_HEADERS.get(index),
uk.getRow(0).getCell(index).getStringCellValue());
germany.getRow(0).getCell(index).getStringCellValue());
}
assertEquals(CellType.NUMERIC, germany.getRow(1).getCell(4).getCellType());
assertEquals(19.99D, germany.getRow(1).getCell(4).getNumericCellValue(), 0.0001D);
assertEquals("not-a-number", germany.getRow(2).getCell(4).getStringCellValue());
for (int rowIndex = 1; rowIndex <= 3; rowIndex++) {
assertEquals("\u5fb7\u56fd", germany.getRow(rowIndex).getCell(2).getStringCellValue());
assertEquals("\u82f1\u56fd:\u6210\u529f\uff0c\u6cd5\u56fd:\u6210\u529f",
germany.getRow(rowIndex).getCell(6).getStringCellValue());
assertEquals("\u82f1\u56fd,\u6cd5\u56fd",
germany.getRow(rowIndex).getCell(7).getStringCellValue());
}
assertEquals(CellType.NUMERIC, uk.getRow(1).getCell(4).getCellType());
assertEquals(19.99D, uk.getRow(1).getCell(4).getNumericCellValue(), 0.0001D);
assertEquals("not-a-number",
workbook.getSheet("德国").getRow(1).getCell(4).getStringCellValue());
}
} finally {
FileUtil.del(directory.toFile());
@@ -115,7 +115,7 @@ class PublishWorkbookServiceTest {
try {
List<PublishWorkbookService.WorkbookInput> oneSuccess = List.of(
new PublishWorkbookService.WorkbookInput(
"郭亚庆.xlsx", "郭亚庆", List.of(row("1", "B001", "英国", "50"))));
"shop.xlsx", "shop", "DE", List.of(row("1", "B001", "UK", "50"))));
PublishWorkbookService.PackagedResult single = service.packageTaskResult(
directory.resolve("single").toFile(), "PUBLISH-1", 1, oneSuccess);
@@ -155,9 +155,9 @@ class PublishWorkbookServiceTest {
row.setCountry(country);
row.setBrand("Brand");
row.setPrice(price);
row.setStatus("成功");
row.setSyncStatus("成功");
row.setSyncCountries("德国,法国");
row.setStatus("success");
row.setSyncStatus("\u82f1\u56fd:\u6210\u529f\uff0c\u6cd5\u56fd:\u6210\u529f");
row.setSyncCountries("\u82f1\u56fd,\u6cd5\u56fd");
return row;
}
}
@@ -0,0 +1,84 @@
package com.nanri.aiimage.modules.shopdatacrawl.controller;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import jakarta.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class AdminShopDataCrawlTaskControllerTest {
@Test
void deletesResultForAuthenticatedAdminWithTaskPermissions() {
ShopDataCrawlTaskService taskService = mock(ShopDataCrawlTaskService.class);
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService permissionService = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
AdminShopDataCrawlTaskController controller = new AdminShopDataCrawlTaskController(
taskService, authSupport, permissionService);
AdminUserEntity operator = admin(8L);
when(authSupport.requireAdmin(request)).thenReturn(operator);
controller.deleteHistory(101L, request);
verify(permissionService).requireShopDataCrawlTaskAccess(operator);
verify(taskService).deleteAdminHistory(101L);
}
@Test
void acceptsTrustedFlaskOperatorWhenLegacySessionHasNoJavaJwt() {
ShopDataCrawlTaskService taskService = mock(ShopDataCrawlTaskService.class);
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService permissionService = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
AdminShopDataCrawlTaskController controller = new AdminShopDataCrawlTaskController(
taskService, authSupport, permissionService);
ReflectionTestUtils.setField(controller, "internalToken", "shared-token");
AdminUserEntity operator = admin(8L);
when(authSupport.requireAdmin(request)).thenThrow(new BusinessException(401, "not logged in"));
when(request.getHeader("X-Internal-Token")).thenReturn("shared-token");
when(request.getParameter("operatorId")).thenReturn("8");
when(permissionService.requireAdminOperator(8L)).thenReturn(operator);
controller.deleteHistory(101L, request);
verify(permissionService).requireAdminOperator(8L);
verify(permissionService).requireShopDataCrawlTaskAccess(operator);
verify(taskService).deleteAdminHistory(101L);
}
@Test
void rejectsAdminWithoutIndependentTaskDataPermission() {
ShopDataCrawlTaskService taskService = mock(ShopDataCrawlTaskService.class);
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
PermissionMenuService permissionService = mock(PermissionMenuService.class);
HttpServletRequest request = mock(HttpServletRequest.class);
AdminShopDataCrawlTaskController controller = new AdminShopDataCrawlTaskController(
taskService, authSupport, permissionService);
AdminUserEntity operator = admin(8L);
BusinessException denied = new BusinessException(403, "no data permission");
when(authSupport.requireAdmin(request)).thenReturn(operator);
org.mockito.Mockito.doThrow(denied)
.when(permissionService).requireShopDataCrawlTaskAccess(operator);
assertThatThrownBy(() -> controller.deleteHistory(101L, request)).isSameAs(denied);
verify(taskService, never()).deleteAdminHistory(101L);
}
private AdminUserEntity admin(Long id) {
AdminUserEntity operator = new AdminUserEntity();
operator.setId(id);
operator.setRole("admin");
return operator;
}
}
@@ -3,16 +3,22 @@ package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.nio.file.Path;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
class ShopDataCrawlExcelAssemblyServiceTest {
@TempDir Path tempDir;
@@ -22,6 +28,8 @@ class ShopDataCrawlExcelAssemblyServiceTest {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate("2026-07-25");
row.setAsin("B012345678");
row.setBrand("Example Brand");
row.setCommodityImage("https://m.media-amazon.com/images/I/example.jpg");
row.setInventorySales("11");
row.setSalesRank("22");
row.setPageViews("33");
@@ -35,8 +43,11 @@ class ShopDataCrawlExcelAssemblyServiceTest {
item.setSuccess(true);
item.setCountryResults(List.of(country));
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
when(imageEmbedder.fetchAndResizeForCache(row.getCommodityImage()))
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
File output = tempDir.resolve("result.xlsx").toFile();
new ShopDataCrawlExcelAssemblyService().writeWorkbook(output, List.of(item));
new ShopDataCrawlExcelAssemblyService(imageEmbedder).writeWorkbook(output, List.of(item));
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
assertEquals(ShopDataCrawlExcelAssemblyService.SHEETS,
@@ -50,7 +61,21 @@ class ShopDataCrawlExcelAssemblyServiceTest {
}
assertEquals("2026-07-25", workbook.getSheet("英国").getRow(1).getCell(0).getStringCellValue());
assertEquals("B012345678", workbook.getSheet("英国").getRow(1).getCell(1).getStringCellValue());
assertEquals("11", workbook.getSheet("英国").getRow(1).getCell(3).getStringCellValue());
assertEquals("Example Brand", workbook.getSheet("英国").getRow(1)
.getCell(ShopDataCrawlExcelAssemblyService.HEADERS.size() - 1).getStringCellValue());
assertEquals(1, workbook.getAllPictures().size());
assertEquals(1, workbook.getSheet("英国").getDrawingPatriarch().getShapes().size());
assertEquals(80f, workbook.getSheet("英国").getRow(1).getHeightInPoints());
assertEquals(18 * 256, workbook.getSheet("英国").getColumnWidth(2));
assertEquals(0, workbook.getSheet("德国").getLastRowNum());
}
}
private byte[] jpegBytes() throws Exception {
BufferedImage image = new BufferedImage(2, 2, BufferedImage.TYPE_INT_RGB);
ByteArrayOutputStream output = new ByteArrayOutputStream();
ImageIO.write(image, "jpg", output);
return output.toByteArray();
}
}
@@ -183,6 +183,18 @@ class ShopDataCrawlTaskServiceChunkTest {
"task:" + task.getId() + ":owner:instance-a");
}
@Test
void preservesBrandFromAppClientThroughChunkMerge() {
givenRunningTask(112L, 212L);
ShopDataCrawlRowDto item = row("2026-07-25", "B001");
item.setBrand("Example Brand");
service.submitResult(task.getId(), request(chunk(1, 1, "DE", item)));
assertEquals(1, result.getSuccess());
assertTrue(task.getResultJson().contains("\"brand\":\"Example Brand\""));
}
@Test
void identicalResultChunkRetryIsIdempotent() {
givenRunningTask(102L, 202L);
@@ -528,6 +540,7 @@ class ShopDataCrawlTaskServiceChunkTest {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate(date);
row.setAsin(asin);
row.setCommodityImage("https://m.media-amazon.com/images/I/" + asin + ".jpg");
row.setInventorySales("10");
row.setSalesRank("20");
row.setPageViews("30");
@@ -0,0 +1,171 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.apache.ibatis.session.Configuration;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class ShopDataCrawlTaskServiceRetentionTest {
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
private static final Long USER_ID = 7L;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new Configuration(), "");
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
}
@Mock private FileTaskMapper fileTaskMapper;
@Mock private FileResultMapper fileResultMapper;
@Mock private ShopDataCrawlResolveService resolveService;
@Mock private ShopDataCrawlExcelAssemblyService excelAssemblyService;
@Mock private ShopDataCrawlTaskCacheService cacheService;
@Mock private OssStorageService ossStorageService;
@Mock private ZiniaoShopSwitchService ziniaoShopSwitchService;
@Mock private TaskPressureProperties taskPressureProperties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskResultItemService taskResultItemService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskChunkMapper taskChunkMapper;
@Mock private TaskScopeStateMapper taskScopeStateMapper;
@Mock private TransientPayloadStorageService transientPayloadStorageService;
@Mock private InstanceMetadata instanceMetadata;
@Spy private final ObjectMapper objectMapper = new ObjectMapper();
@InjectMocks private ShopDataCrawlTaskService service;
@Test
void keepsNewestThreePerStableShopAndFallsBackToShopName() {
LocalDateTime now = LocalDateTime.of(2026, 8, 5, 12, 0);
FileResultEntity shopIdOldest = result(11L, 111L, USER_ID, "shop-1", "Renamed Shop",
"result/shop-id-oldest.xlsx", 1, now.minusDays(2));
FileResultEntity shopNameOldest = result(21L, 121L, USER_ID, " ", "Fallback Shop",
"result/shop-name-oldest.xlsx", 1, now.minusDays(4));
List<FileResultEntity> rows = List.of(
result(13L, 113L, USER_ID, "shop-1", "Current Name", "result/13.xlsx", 1, now.minusDays(1)),
result(23L, 123L, USER_ID, null, "Fallback Shop", "result/23.xlsx", 1, now.minusDays(2)),
shopIdOldest,
result(14L, 114L, USER_ID, "shop-1", "Current Name", "result/14.xlsx", 1, now),
result(24L, 124L, USER_ID, null, "Fallback Shop", "result/24.xlsx", 1, now),
result(12L, 112L, USER_ID, "shop-1", "Old Name", "result/12.xlsx", 1, now.minusDays(2)),
shopNameOldest,
result(22L, 122L, USER_ID, null, "Fallback Shop", "result/22.xlsx", 1, now.minusDays(3)),
result(1L, 101L, USER_ID, "shop-1", "Current Name", "result/failed.xlsx", 0, now.minusDays(9)),
result(2L, 102L, USER_ID, "shop-1", "Current Name", null, 1, now.minusDays(9)),
result(3L, 103L, 99L, "shop-1", "Current Name", "result/other-user.xlsx", 1, now.minusDays(9)),
result(4L, 104L, USER_ID, "shop-2", "Current Name", "result/other-shop.xlsx", 1, now.minusDays(9)));
when(fileResultMapper.selectList(any())).thenReturn(rows);
when(fileResultMapper.selectById(11L)).thenReturn(shopIdOldest);
when(fileResultMapper.selectById(21L)).thenReturn(shopNameOldest);
when(fileTaskMapper.selectById(111L)).thenReturn(terminalTask(111L));
when(fileTaskMapper.selectById(121L)).thenReturn(terminalTask(121L));
when(taskDistributedLockService.acquire(MODULE_TYPE, 111L))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
when(taskDistributedLockService.acquire(MODULE_TYPE, 121L))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
when(cacheService.getTaskCacheBatch(any())).thenReturn(Map.of());
when(fileResultMapper.selectCount(any())).thenReturn(0L);
service.pruneCompletedHistoryForShop(USER_ID, "shop-id:shop-1");
service.pruneCompletedHistoryForShop(USER_ID, "shop-name:Fallback Shop");
verify(fileResultMapper).deleteById(11L);
verify(fileResultMapper).deleteById(21L);
verify(taskFileJobService).deleteResultJobs(111L, MODULE_TYPE, 11L);
verify(taskFileJobService).deleteResultJobs(121L, MODULE_TYPE, 21L);
verify(taskResultItemService).deleteResultItem(111L, MODULE_TYPE, 11L);
verify(taskResultItemService).deleteResultItem(121L, MODULE_TYPE, 21L);
verify(ossStorageService).deleteObject("result/shop-id-oldest.xlsx");
verify(ossStorageService).deleteObject("result/shop-name-oldest.xlsx");
verify(fileResultMapper, never()).deleteById(1L);
verify(fileResultMapper, never()).deleteById(2L);
verify(fileResultMapper, never()).deleteById(3L);
verify(fileResultMapper, never()).deleteById(4L);
}
@Test
void doesNotDeleteOldFileWhileOwningTaskIsStillRunning() {
LocalDateTime now = LocalDateTime.of(2026, 8, 5, 12, 0);
FileResultEntity oldest = result(31L, 131L, USER_ID, "shop-running", "Running Shop",
"result/running-oldest.xlsx", 1, now.minusDays(3));
when(fileResultMapper.selectList(any())).thenReturn(List.of(
result(34L, 134L, USER_ID, "shop-running", "Running Shop", "result/34.xlsx", 1, now),
result(33L, 133L, USER_ID, "shop-running", "Running Shop", "result/33.xlsx", 1, now.minusDays(1)),
result(32L, 132L, USER_ID, "shop-running", "Running Shop", "result/32.xlsx", 1, now.minusDays(2)),
oldest));
when(fileResultMapper.selectById(31L)).thenReturn(oldest);
when(fileTaskMapper.selectById(131L)).thenReturn(task(131L, "RUNNING"));
when(taskDistributedLockService.acquire(MODULE_TYPE, 131L))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
service.pruneCompletedHistoryForShop(USER_ID, "shop-id:shop-running");
verify(fileResultMapper, never()).deleteById(31L);
verify(taskFileJobService, never()).deleteResultJobs(131L, MODULE_TYPE, 31L);
verify(ossStorageService, never()).deleteObject("result/running-oldest.xlsx");
}
private FileResultEntity result(Long id, Long taskId, Long userId, String shopId, String shopName,
String resultFileUrl, int success, LocalDateTime createdAt) {
FileResultEntity row = new FileResultEntity();
row.setId(id);
row.setTaskId(taskId);
row.setModuleType(MODULE_TYPE);
row.setUserId(userId);
row.setSourceFileUrl(shopId);
row.setSourceFilename(shopName);
row.setResultFileUrl(resultFileUrl);
row.setSuccess(success);
row.setCreatedAt(createdAt);
return row;
}
private FileTaskEntity terminalTask(Long id) {
return task(id, "SUCCESS");
}
private FileTaskEntity task(Long id, String status) {
FileTaskEntity task = new FileTaskEntity();
task.setId(id);
task.setModuleType(MODULE_TYPE);
task.setStatus(status);
return task;
}
}
@@ -0,0 +1,84 @@
package com.nanri.aiimage.modules.shopkey.service;
import com.nanri.aiimage.modules.shopkey.mapper.ShopManageMapper;
import com.nanri.aiimage.modules.shopkey.mapper.SkipPriceAsinMapper;
import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinCreateRequest;
import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity;
import com.nanri.aiimage.modules.shopkey.model.entity.SkipPriceAsinEntity;
import com.nanri.aiimage.modules.shopkey.model.vo.SkipPriceAsinItemVo;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class SkipPriceAsinServiceTest {
@Mock
private SkipPriceAsinMapper skipPriceAsinMapper;
@Mock
private ShopManageMapper shopManageMapper;
@Mock
private ShopManageGroupService shopManageGroupService;
@InjectMocks
private SkipPriceAsinService service;
@Test
void createInsertsNewRowWhenGroupAndShopAlreadyExist() {
ShopManageGroupEntity group = new ShopManageGroupEntity();
group.setId(10L);
group.setGroupName("group-a");
when(shopManageGroupService.getAccessibleById(10L, 7L, true)).thenReturn(group);
SkipPriceAsinEntity existing = new SkipPriceAsinEntity();
existing.setId(100L);
existing.setGroupId(10L);
existing.setShopName("shop-a");
existing.setAsinDe("OLD-ASIN");
lenient().when(skipPriceAsinMapper.selectOne(any())).thenReturn(existing);
AtomicReference<SkipPriceAsinEntity> inserted = new AtomicReference<>();
when(skipPriceAsinMapper.insert(any(SkipPriceAsinEntity.class))).thenAnswer(invocation -> {
SkipPriceAsinEntity entity = invocation.getArgument(0);
entity.setId(101L);
inserted.set(entity);
return 1;
});
when(skipPriceAsinMapper.selectById(101L)).thenAnswer(invocation -> inserted.get());
SkipPriceAsinCreateRequest request = new SkipPriceAsinCreateRequest();
request.setGroupId(10L);
request.setShopName("shop-a");
request.setCountries(List.of("DE"));
request.setAsinMappings(Map.of("DE", "NEW-ASIN"));
request.setMinimumPriceMappings(Map.of("DE", new BigDecimal("19.99")));
SkipPriceAsinItemVo result = service.create(request, 7L, true);
ArgumentCaptor<SkipPriceAsinEntity> captor = ArgumentCaptor.forClass(SkipPriceAsinEntity.class);
verify(skipPriceAsinMapper).insert(captor.capture());
verify(skipPriceAsinMapper, never()).selectOne(any());
verify(skipPriceAsinMapper, never()).updateById(any(SkipPriceAsinEntity.class));
assertNotSame(existing, captor.getValue());
assertEquals("OLD-ASIN", existing.getAsinDe());
assertEquals(101L, result.getId());
assertEquals("NEW-ASIN", result.getAsinDe());
assertEquals(new BigDecimal("19.99"), result.getMinimumPriceDe());
}
}
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.similarasin.client;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
@@ -180,4 +181,60 @@ class SimilarAsinCozeClientTest {
assertTrue(json.contains("\"price\":10"));
assertTrue(json.contains("\"price\":80"));
}
@Test
@SuppressWarnings("unchecked")
void buildParametersIncludesCategorySwitch() throws Exception {
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin("B0CATEGORY1");
row.setTitle("Category test");
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null);
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
"buildParameters", List.class, String.class, String.class, boolean.class, boolean.class);
method.setAccessible(true);
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, List.of(row), "", "", false, true);
assertEquals(Boolean.TRUE, parameters.get("category_switch"));
assertEquals(Boolean.FALSE, parameters.get("img_switch"));
}
@Test
void imageOnlyWorkflowOutputIsExtractedAndMergedByAsin() throws Exception {
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null);
String imageData = """
{"data":[{
"asin":"B0BQNHDP2F",
"main_url":"https://example.com/main.jpg",
"puzzle_img1":"https://example.com/puzzle-1.jpg",
"puzzle_img2":"https://example.com/puzzle-2.jpg"
}]}
""";
String workflowOutput = objectMapper.writeValueAsString(Map.of(
"node_status", "{}",
"Output", imageData));
var historyResponse = objectMapper.createObjectNode();
historyResponse.put("code", 0);
historyResponse.putArray("data")
.addObject()
.put("execute_status", "Success")
.put("output", workflowOutput);
Method extract = SimilarAsinCozeClient.class.getDeclaredMethod("extractResultDataText", JsonNode.class);
extract.setAccessible(true);
String dataText = (String) extract.invoke(client, historyResponse);
SimilarAsinResultRowDto source = new SimilarAsinResultRowDto();
source.setAsin("B0BQNHDP2F");
List<SimilarAsinResultRowDto> merged = client.mergeRowsFromDataText(List.of(source), dataText);
assertFalse(dataText.isBlank());
assertEquals(1, merged.size());
assertEquals("https://example.com/main.jpg", merged.getFirst().getMainUrl());
assertEquals("https://example.com/puzzle-1.jpg", merged.getFirst().getPuzzleImg1());
assertEquals("https://example.com/puzzle-2.jpg", merged.getFirst().getPuzzleImg2());
Method resolvedCount = SimilarAsinCozeClient.class.getDeclaredMethod("resolvedCount", List.class);
resolvedCount.setAccessible(true);
assertEquals(1, resolvedCount.invoke(client, merged));
}
}
@@ -1,9 +1,11 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Field;
import java.util.LinkedHashMap;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -60,6 +62,57 @@ class SimilarAsinTaskServiceTest {
assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 2, 3));
}
@Test
void failedResultRowsWithBlankIsConformEnableCategoryRetry() {
SimilarAsinParsedRowVo blankCategory = new SimilarAsinParsedRowVo();
blankCategory.setValues(new LinkedHashMap<>());
blankCategory.getValues().put("status", "FAILED");
blankCategory.getValues().put("is_conform", "");
assertTrue(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
List.of("id", "asin", "country", "is_conform", "status"),
List.of(blankCategory),
true));
assertFalse(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
List.of("id", "asin", "country", "is_conform", "status"),
List.of(blankCategory),
false));
blankCategory.getValues().put("is_conform", "符合");
assertFalse(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
List.of("id", "asin", "country", "is_conform", "status"),
List.of(blankCategory),
true));
}
@Test
void firstPassImageResultWorkbookKeepsAllRowsForSecondParse() {
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
row.setValues(new LinkedHashMap<>());
row.getValues().put("状态", "成功");
row.getValues().put("是否有货", "");
row.getValues().put("相似度", "");
row.getValues().put("是否符合类目", "");
row.getValues().put("不符合理由", "");
row.getValues().put("产品类目", "");
assertTrue(SimilarAsinTaskService.isFirstPassResultWorkbook(
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
List.of(row)));
row.getValues().put("是否符合类目", "符合");
assertFalse(SimilarAsinTaskService.isFirstPassResultWorkbook(
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
List.of(row)));
row.getValues().put("是否符合类目", "");
row.getValues().put("状态", "失败");
assertFalse(SimilarAsinTaskService.isFirstPassResultWorkbook(
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
List.of(row)));
}
private int staticIntField(String name) throws Exception {
Field field = SimilarAsinTaskService.class.getDeclaredField(name);
field.setAccessible(true);
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.similarasin.util;
import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -11,7 +12,9 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.junit.jupiter.api.Assertions.assertTrue;
class ExcelCellImageWriterTest {
@@ -42,6 +45,48 @@ class ExcelCellImageWriterTest {
}
}
@Test
void streamsRegisteredImagePathIntoWorkbookMedia() throws Exception {
Path dir = Files.createTempDirectory("excel-cell-image-path-test-");
Path xlsx = dir.resolve("result.xlsx");
Path image = dir.resolve("thumb.jpeg");
byte[] imageBytes = new byte[]{7, 8, 9, 10};
writeMinimalWorkbook(xlsx);
Files.write(image, imageBytes);
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
session.registerImage(1, 9, image);
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
assertArrayEquals(imageBytes,
zip.getInputStream(zip.getEntry("xl/media/excelcellimage1.jpeg")).readAllBytes());
}
}
@Test
void patchesTenThousandImageCellsWithinLinearTimeBudget() {
assertTimeoutPreemptively(Duration.ofSeconds(10), () -> {
Path dir = Files.createTempDirectory("excel-cell-image-linear-test-");
Path xlsx = dir.resolve("result.xlsx");
writeWorkbookWithImageCells(xlsx, 10_000);
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
for (int row = 1; row <= 10_000; row++) {
session.registerImage(row, 9, new byte[]{1});
}
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
String sheet = read(zip, "xl/worksheets/sheet1.xml");
assertEquals(10_000, maxVm(sheet));
assertTrue(sheet.contains("<c r=\"J10001\" t=\"e\" vm=\"10000\"><v>#VALUE!</v></c>"));
}
});
}
private static int maxVm(String sheet) {
Matcher matcher = Pattern.compile("vm=\"(\\d+)\"").matcher(sheet);
int max = -1;
@@ -99,6 +144,20 @@ class ExcelCellImageWriterTest {
}
}
private static void writeWorkbookWithImageCells(Path xlsx, int imageCount) throws Exception {
try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(xlsx))) {
write(out, "[Content_Types].xml", "<Types></Types>");
write(out, "xl/_rels/workbook.xml.rels", "<Relationships></Relationships>");
StringBuilder sheet = new StringBuilder("<worksheet><sheetData>");
for (int row = 2; row <= imageCount + 1; row++) {
sheet.append("<row r=\"").append(row).append("\"><c r=\"J").append(row)
.append("\" t=\"inlineStr\"><is><t>image</t></is></c></row>");
}
sheet.append("</sheetData></worksheet>");
write(out, "xl/worksheets/sheet1.xml", sheet.toString());
}
}
private static void write(ZipOutputStream out, String name, String content) throws Exception {
out.putNextEntry(new ZipEntry(name));
out.write(content.getBytes(StandardCharsets.UTF_8));
@@ -1,7 +1,15 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.nanri.aiimage.config.OssProperties;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Protocol;
import okhttp3.Response;
import okhttp3.ResponseBody;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import javax.imageio.ImageIO;
@@ -10,24 +18,39 @@ import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.lang.reflect.Field;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
class SimilarAsinImageEmbedderTest {
// properties=null 时构造函数走 DEFAULT_DOWNLOAD_TIMEOUT_SECONDS / DEFAULT_DOWNLOAD_POOL_SIZE 兜底
private final SimilarAsinImageEmbedder embedder = new SimilarAsinImageEmbedder(null, createOssStorageService());
@AfterEach
void shutDownEmbedder() {
embedder.shutdown();
}
private static OssStorageService createOssStorageService() {
OssProperties properties = new OssProperties();
properties.setEndpoint("https://oss.aishufu.top");
@@ -40,6 +63,133 @@ class SimilarAsinImageEmbedderTest {
return new OssStorageService(properties);
}
@Test
void defaultsImageDownloadPoolToEight() {
assertEquals(8, new SimilarAsinProperties().getImageDownloadPoolSize());
assertEquals(8, embedder.downloadPoolSize());
}
@Test
void writesFetchedThumbnailToPersistentCacheForNextEmbedderInstance() throws Exception {
Path cacheDir = Files.createTempDirectory("similar-asin-image-cache-test-");
String url = "https://images.example.com/product.jpg";
byte[] sourceImage = createJpegBytes();
AtomicInteger firstNetworkCalls = new AtomicInteger();
AtomicInteger secondNetworkCalls = new AtomicInteger();
SimilarAsinImageEmbedder first = new SimilarAsinImageEmbedder(
properties(1, 5, cacheDir), createOssStorageService());
SimilarAsinImageEmbedder second = new SimilarAsinImageEmbedder(
properties(1, 5, cacheDir), createOssStorageService());
try {
replaceHttpClient(first, respondingClient(firstNetworkCalls, sourceImage));
SimilarAsinImageEmbedder.ResizedImage fetched = first.fetchAndResizeForCache(url);
assertNotNull(fetched);
assertEquals(1, firstNetworkCalls.get());
try (var cachedFiles = Files.walk(cacheDir)) {
assertEquals(1L, cachedFiles.filter(Files::isRegularFile).count());
}
replaceHttpClient(second, new OkHttpClient.Builder()
.addInterceptor(chain -> {
secondNetworkCalls.incrementAndGet();
throw new AssertionError("persistent cache miss triggered a second network request");
})
.build());
SimilarAsinImageEmbedder.ResizedImage cached = second.fetchAndResizeForCache(url);
assertNotNull(cached);
assertArrayEquals(fetched.bytes(), cached.bytes());
assertEquals(fetched.width(), cached.width());
assertEquals(fetched.height(), cached.height());
assertEquals(0, secondNetworkCalls.get());
} finally {
first.shutdown();
second.shutdown();
deleteRecursively(cacheDir);
}
}
@Test
void diskPrefetchDeadlineCancelsInFlightDownload() throws Exception {
SimilarAsinImageEmbedder deadlineEmbedder = new SimilarAsinImageEmbedder(
properties(1, 1, null), createOssStorageService());
CountDownLatch downloadStarted = new CountDownLatch(1);
CountDownLatch cancellationObserved = new CountDownLatch(1);
replaceHttpClient(deadlineEmbedder, new OkHttpClient.Builder()
.addInterceptor(chain -> {
downloadStarted.countDown();
try {
new CountDownLatch(1).await();
throw new AssertionError("blocking download unexpectedly completed");
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
cancellationObserved.countDown();
throw new IOException("cancelled", ex);
}
})
.build());
Path spoolDir = Files.createTempDirectory("similar-asin-prefetch-deadline-test-");
try (SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir)) {
assertTimeoutPreemptively(Duration.ofSeconds(3), () -> deadlineEmbedder.prefetchToDisk(
List.of("https://images.example.com/slow.jpg"), spool));
assertTrue(downloadStarted.await(100, TimeUnit.MILLISECONDS));
assertTrue(cancellationObserved.await(1, TimeUnit.SECONDS),
"deadline should interrupt the active image download");
assertEquals(0, spool.size());
} finally {
deadlineEmbedder.shutdown();
}
}
@Test
void failedDiskPrefetchIsNotDownloadedAgainWhileEmbedding() throws Exception {
SimilarAsinImageEmbedder failedEmbedder = new SimilarAsinImageEmbedder(
properties(1, 5, null), createOssStorageService());
AtomicInteger networkCalls = new AtomicInteger();
replaceHttpClient(failedEmbedder, failingClient(networkCalls));
String url = "https://images.example.com/missing.jpg";
Path spoolDir = Files.createTempDirectory("similar-asin-prefetch-failed-test-");
try (SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
XSSFWorkbook workbook = new XSSFWorkbook()) {
failedEmbedder.prefetchToDisk(List.of(url), spool);
int callsAfterPrefetch = networkCalls.get();
var row = workbook.createSheet().createRow(1);
SimilarAsinImageEmbedder.ImageDim result = failedEmbedder.embedAsExcelCellImage(
1, 9, url, row, new HashMap<>(), ExcelCellImageWriter.createSession(), spool);
assertEquals(SimilarAsinImageEmbedder.DOWNLOAD_MAX_RETRY + 1, callsAfterPrefetch);
assertEquals(callsAfterPrefetch, networkCalls.get());
assertNull(result);
assertEquals(url, row.getCell(9).getStringCellValue());
} finally {
failedEmbedder.shutdown();
}
}
@Test
void imageSpoolWritesThumbnailAndDeletesTaskDirectoryOnClose() throws Exception {
Path directory = Files.createTempDirectory("similar-asin-image-spool-test-");
byte[] bytes = new byte[]{1, 3, 5, 7};
SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(directory);
try {
SimilarAsinImageEmbedder.SpoolImage image = spool.put(
"https://example.com/image.jpg",
new SimilarAsinImageEmbedder.ResizedImage(bytes, 120, 80));
assertEquals(1, spool.size());
assertEquals(120, image.width());
assertEquals(80, image.height());
assertArrayEquals(bytes, Files.readAllBytes(image.path()));
} finally {
spool.close();
}
assertFalse(Files.exists(directory));
}
@Test
void normalizesLegacyMinioUrlBeforeHttpsValidation() {
String normalized = embedder.normalizeAndValidateDownloadUrl(
@@ -247,4 +397,75 @@ class SimilarAsinImageEmbedderTest {
assertTrue(summary.contains("IOException: outer"));
assertTrue(summary.contains("cause=TimeoutException: inner"));
}
private static SimilarAsinProperties properties(int poolSize, int prefetchTimeoutSeconds, Path cacheDir) {
SimilarAsinProperties properties = new SimilarAsinProperties();
properties.setImageDownloadPoolSize(poolSize);
properties.setImagePrefetchTimeoutSeconds(prefetchTimeoutSeconds);
properties.setImageLocalCacheDir(cacheDir == null ? "" : cacheDir.toString());
return properties;
}
private static OkHttpClient respondingClient(AtomicInteger calls, byte[] imageBytes) {
return new OkHttpClient.Builder()
.addInterceptor(chain -> {
calls.incrementAndGet();
return response(chain.request(), 200, "OK", imageBytes);
})
.build();
}
private static OkHttpClient failingClient(AtomicInteger calls) {
return new OkHttpClient.Builder()
.addInterceptor(chain -> {
calls.incrementAndGet();
return response(chain.request(), 503, "Unavailable", new byte[0]);
})
.build();
}
private static Response response(okhttp3.Request request,
int status,
String message,
byte[] body) {
return new Response.Builder()
.request(request)
.protocol(Protocol.HTTP_1_1)
.code(status)
.message(message)
.body(ResponseBody.create(body, MediaType.get("image/jpeg")))
.build();
}
private static void replaceHttpClient(SimilarAsinImageEmbedder target,
OkHttpClient httpClient) throws ReflectiveOperationException {
Field field = SimilarAsinImageEmbedder.class.getDeclaredField("httpClient");
field.setAccessible(true);
field.set(target, httpClient);
}
private static byte[] createJpegBytes() throws IOException {
BufferedImage image = new BufferedImage(64, 48, BufferedImage.TYPE_INT_RGB);
Graphics2D graphics = image.createGraphics();
try {
graphics.setColor(new Color(0x24, 0x68, 0xAC));
graphics.fillRect(0, 0, image.getWidth(), image.getHeight());
} finally {
graphics.dispose();
}
ByteArrayOutputStream output = new ByteArrayOutputStream();
ImageIO.write(image, "jpg", output);
return output.toByteArray();
}
private static void deleteRecursively(Path directory) throws IOException {
if (!Files.exists(directory)) {
return;
}
try (var paths = Files.walk(directory)) {
for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
Files.deleteIfExists(path);
}
}
}
}
@@ -0,0 +1,228 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.context.ApplicationEventPublisher;
import java.time.LocalDateTime;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class TaskFileJobServiceTest {
@Mock private TaskFileJobMapper taskFileJobMapper;
@Mock private ApplicationEventPublisher applicationEventPublisher;
@BeforeAll
static void initializeTableInfo() {
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
TaskFileJobEntity.class);
}
@Test
void stuckJobIsRequeuedAndRetryCountIsIncremented() {
TaskFileJobEntity running = runningJob(101L, 3, LocalDateTime.now().minusHours(1));
TaskFileJobEntity pending = runningJob(101L, 4, LocalDateTime.now());
pending.setStatus("PENDING");
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
when(taskFileJobMapper.selectById(101L)).thenReturn(pending);
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
assertEquals(1, result.resetCount());
assertTrue(result.exhaustedJobs().isEmpty());
verify(applicationEventPublisher).publishEvent(any(TaskFileJobDispatchEvent.class));
assertUpdateContains(4, running.getUpdatedAt());
}
@Test
void stuckJobAtLastRetryWaitsForDurableFailureFinalization() {
TaskFileJobEntity running = runningJob(102L, 4, LocalDateTime.now().minusHours(1));
TaskFileJobEntity failed = runningJob(102L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
failed.setStatus("FAILED");
/*
failed.setErrorMessage("文件生成任务运行超时,已达到最大重试次数");
*/
failed.setErrorMessage("result file job timeout");
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
when(taskFileJobMapper.selectById(102L)).thenReturn(failed);
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
assertEquals(0, result.resetCount());
assertEquals(List.of(failed), result.exhaustedJobs());
verify(applicationEventPublisher, never()).publishEvent(any());
assertUpdateContains(TaskFileJobService.MAX_RETRY_COUNT, running.getUpdatedAt());
}
@Test
void pendingFailureFinalizationIsReturnedAgainWithoutRequeueing() {
TaskFileJobEntity pending = runningJob(106L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
pending.setStatus("FAILED");
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(pending));
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
assertEquals(0, result.resetCount());
assertEquals(List.of(pending), result.exhaustedJobs());
verify(taskFileJobMapper, never()).update(any(), any());
verify(applicationEventPublisher, never()).publishEvent(any());
}
@Test
void queuedClaimActivationUsesUpdatedAtAsFencingToken() {
TaskFileJobEntity claim = runningJob(107L, 1, LocalDateTime.now().minusMinutes(1));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
assertTrue(service.activateRunningClaim(claim));
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
verify(taskFileJobMapper).update(isNull(), update.capture());
assertTrue(update.getValue().getSqlSegment().contains("updated_at"));
assertTrue(update.getValue().getParamNameValuePairs().containsValue(claim.getUpdatedAt()));
}
@Test
void heartbeatRacePreventsStuckJobReset() {
TaskFileJobEntity running = runningJob(103L, 4, LocalDateTime.now().minusHours(1));
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
assertEquals(0, result.resetCount());
assertTrue(result.exhaustedJobs().isEmpty());
verify(taskFileJobMapper, never()).selectById(any());
verify(applicationEventPublisher, never()).publishEvent(any());
assertUpdateContains(TaskFileJobService.MAX_RETRY_COUNT, running.getUpdatedAt());
}
@Test
void exhaustedJobCannotBeClaimedOrRequeued() {
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
assertFalse(service.markRunning(104L));
assertFalse(service.requeue(104L, "retry"));
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> updates = updateCaptor();
verify(taskFileJobMapper, org.mockito.Mockito.times(2)).update(isNull(), updates.capture());
for (LambdaUpdateWrapper<TaskFileJobEntity> update : updates.getAllValues()) {
assertTrue(update.getSqlSegment().contains("retry_count"));
assertTrue(update.getParamNameValuePairs().containsValue(TaskFileJobService.MAX_RETRY_COUNT));
}
verify(taskFileJobMapper, never()).selectById(any());
verify(applicationEventPublisher, never()).publishEvent(any());
}
@Test
void markFailedUsesCurrentRetryCountWithCompareAndSet() {
TaskFileJobEntity stale = runningJob(105L, 0, LocalDateTime.now().minusMinutes(5));
TaskFileJobEntity current = runningJob(105L, 4, LocalDateTime.now());
when(taskFileJobMapper.selectById(105L)).thenReturn(current);
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
service.markFailed(stale, "failed");
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
verify(taskFileJobMapper).update(isNull(), update.capture());
assertTrue(update.getValue().getSqlSegment().contains("retry_count"));
assertTrue(update.getValue().getParamNameValuePairs().containsValue(4));
assertTrue(update.getValue().getParamNameValuePairs().containsValue(TaskFileJobService.MAX_RETRY_COUNT));
}
@Test
void successWriteIsFencedToRunningJob() {
TaskFileJobEntity job = runningJob(108L, 1, LocalDateTime.now());
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
service.markSuccess(job, "result.xlsx");
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
verify(taskFileJobMapper).update(isNull(), update.capture());
assertTrue(update.getValue().getSqlSegment().contains("status"));
assertTrue(update.getValue().getParamNameValuePairs().containsValue("RUNNING"));
}
@Test
void similarAsinHeartbeatOnlyTouchesStaleRunningAssembleJobs() {
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
service.touchRunningAssembleJobsIfStale(20553L, "SIMILAR_ASIN", 60000L);
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
verify(taskFileJobMapper).update(isNull(), update.capture());
assertTrue(update.getValue().getSqlSegment().contains("task_id"));
assertTrue(update.getValue().getParamNameValuePairs().containsValue("SIMILAR_ASIN"));
assertTrue(update.getValue().getParamNameValuePairs().containsValue("ASSEMBLE_RESULT"));
assertTrue(update.getValue().getParamNameValuePairs().containsValue("RUNNING"));
}
@Test
void retryExhaustedOnlyMeansFailedTerminalJob() {
TaskFileJobEntity success = runningJob(109L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
success.setStatus("SUCCESS");
TaskFileJobEntity failed = runningJob(110L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
failed.setStatus("FAILED");
when(taskFileJobMapper.selectById(109L)).thenReturn(success);
when(taskFileJobMapper.selectById(110L)).thenReturn(failed);
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
assertFalse(service.isRetryExhausted(109L));
assertTrue(service.isRetryExhausted(110L));
}
private void assertUpdateContains(int nextRetry, LocalDateTime updatedAt) {
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
verify(taskFileJobMapper).update(isNull(), update.capture());
assertTrue(update.getValue().getSqlSegment().contains("updated_at"));
assertTrue(update.getValue().getParamNameValuePairs().containsValue(updatedAt));
assertTrue(update.getValue().getParamNameValuePairs().containsValue(nextRetry));
}
@SuppressWarnings({"rawtypes", "unchecked"})
private static ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> updateCaptor() {
return ArgumentCaptor.forClass((Class) LambdaUpdateWrapper.class);
}
private static TaskFileJobEntity runningJob(Long id, int retryCount, LocalDateTime updatedAt) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(id);
job.setTaskId(20553L);
job.setResultId(23110L);
job.setModuleType("SIMILAR_ASIN");
job.setStatus("RUNNING");
job.setRetryCount(retryCount);
job.setUpdatedAt(updatedAt);
return job;
}
}
@@ -8,6 +8,7 @@ import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
import com.nanri.aiimage.modules.brand.service.BrandTaskProgressCacheService;
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService;
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
@@ -67,8 +68,10 @@ class TaskHeartbeatServiceTest {
@Mock private AppearancePatentTaskCacheService appearancePatentTaskCacheService;
@Mock private SimilarAsinTaskCacheService similarAsinTaskCacheService;
@Mock private SimilarAsinProperties similarAsinProperties;
@Mock private TaskFileJobService taskFileJobService;
@Mock private DeleteBrandTaskCacheService deleteBrandTaskCacheService;
@Mock private BrandTaskProgressCacheService brandTaskProgressCacheService;
@Mock private CollectDataService collectDataService;
@InjectMocks private TaskHeartbeatService service;
@@ -124,6 +127,28 @@ class TaskHeartbeatServiceTest {
verify(shopDataCrawlTaskCacheService).saveTaskCache(task);
}
@Test
@SuppressWarnings("unchecked")
void collectDataHeartbeatForwardsProcessedKeywordProgress() {
long taskId = 21016L;
FileTaskEntity task = new FileTaskEntity();
task.setId(taskId);
task.setModuleType("COLLECT_DATA");
task.setStatus("RUNNING");
TaskHeartbeatRequest request = new TaskHeartbeatRequest();
request.setCurrent(4);
request.setTotal(19);
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task);
when(brandCrawlTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null);
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
TaskHeartbeatVo result = service.heartbeat(taskId, request);
assertTrue(result.isAlive());
verify(collectDataService).updateProgress(taskId, request);
}
@Test
@SuppressWarnings("unchecked")
void similarAsinHeartbeatUsesRedisWithoutRefreshingRecentDatabaseCheckpoint() {
@@ -142,6 +167,7 @@ class TaskHeartbeatServiceTest {
assertTrue(result.isAlive());
assertEquals("SIMILAR_ASIN", result.getModuleType());
verify(similarAsinTaskCacheService).touchTaskHeartbeat(taskId);
verify(taskFileJobService).touchRunningAssembleJobsIfStale(taskId, "SIMILAR_ASIN", 120000L);
verify(fileTaskMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class));
}
}
@@ -17,6 +17,10 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
import java.time.LocalDateTime;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InOrder;
@@ -24,9 +28,13 @@ import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@@ -72,7 +80,7 @@ class TaskResultFileJobWorkerTest {
result.setResultFileUrl("result/withdraw/20140.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskFileJobService.markRunning(jobId)).thenReturn(true);
allowClaim(job);
when(taskDistributedLockService.acquire("WITHDRAW", taskId, TaskDistributedLockService.DEFAULT_WAIT_MILLIS))
.thenReturn(lock);
when(fileResultMapper.selectById(resultId)).thenReturn(result);
@@ -102,7 +110,7 @@ class TaskResultFileJobWorkerTest {
result.setResultFileUrl("result/publish/20141.xlsx");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(taskFileJobService.markRunning(jobId)).thenReturn(true);
allowClaim(job);
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
when(taskDistributedLockService.acquire(
PublishTaskService.MODULE_TYPE,
@@ -150,7 +158,7 @@ class TaskResultFileJobWorkerTest {
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
when(taskFileJobService.markRunning(jobId)).thenReturn(true);
allowClaim(job);
when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", taskId,
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
when(fileResultMapper.selectById(resultId)).thenReturn(result);
@@ -173,7 +181,7 @@ class TaskResultFileJobWorkerTest {
job.setScopeKey("task:20144:owner:instance-a");
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
when(taskFileJobService.markRunning(job.getId())).thenReturn(true);
allowClaim(job);
when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", job.getTaskId(),
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
doThrow(new IllegalStateException("upload failed"))
@@ -184,5 +192,86 @@ class TaskResultFileJobWorkerTest {
verify(taskFileJobService).markFailed(job, "upload failed");
verify(shopDataCrawlTaskService).handleResultFileJobFailure(job, "upload failed");
verify(taskFileJobService).markFailureFinalized(job.getId(), "upload failed");
}
@Test
void stuckSimilarAsinJobAtRetryLimitFailsOwningTask() {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(13858L);
job.setTaskId(20553L);
job.setResultId(23110L);
job.setModuleType("SIMILAR_ASIN");
job.setRetryCount(TaskFileJobService.MAX_RETRY_COUNT);
job.setErrorMessage("文件生成任务运行超时,已达到最大重试次数");
TaskFileJobService.StuckJobResetResult resetResult =
new TaskFileJobService.StuckJobResetResult(0, List.of(job));
when(taskFileJobService.resetStuckRunningJobsDetailed(0, 0)).thenReturn(resetResult);
worker.resetStuckJobs();
verify(similarAsinTaskService).handleResultFileJobFailure(job, job.getErrorMessage());
verify(taskFileJobService).markFailureFinalized(job.getId(), job.getErrorMessage());
}
@Test
void stuckJobFailureCallbackDoesNotBlockRemainingJobs() {
TaskFileJobEntity first = exhaustedSimilarAsinJob(13858L, 20553L);
TaskFileJobEntity second = exhaustedSimilarAsinJob(13859L, 20554L);
TaskFileJobService.StuckJobResetResult resetResult =
new TaskFileJobService.StuckJobResetResult(0, List.of(first, second));
when(taskFileJobService.resetStuckRunningJobsDetailed(0, 0)).thenReturn(resetResult);
doThrow(new IllegalStateException("owner mismatch"))
.doNothing()
.when(similarAsinTaskService)
.handleResultFileJobFailure(any(), anyString());
worker.resetStuckJobs();
verify(similarAsinTaskService).handleResultFileJobFailure(first, first.getErrorMessage());
verify(similarAsinTaskService).handleResultFileJobFailure(second, second.getErrorMessage());
verify(taskFileJobService, never()).markFailureFinalized(first.getId(), first.getErrorMessage());
verify(taskFileJobService).markFailureFinalized(second.getId(), second.getErrorMessage());
}
@Test
void pendingFailureCallbackIsRetriedOnNextScan() {
TaskFileJobEntity job = exhaustedSimilarAsinJob(13860L, 20555L);
TaskFileJobService.StuckJobResetResult resetResult =
new TaskFileJobService.StuckJobResetResult(0, List.of(job));
when(taskFileJobService.resetStuckRunningJobsDetailed(0, 0))
.thenReturn(resetResult, resetResult);
doThrow(new IllegalStateException("temporary database failure"))
.doNothing()
.when(similarAsinTaskService)
.handleResultFileJobFailure(job, job.getErrorMessage());
worker.resetStuckJobs();
worker.resetStuckJobs();
verify(similarAsinTaskService, times(2))
.handleResultFileJobFailure(job, job.getErrorMessage());
verify(taskFileJobService).markFailureFinalized(job.getId(), job.getErrorMessage());
}
private void allowClaim(TaskFileJobEntity job) {
TaskFileJobEntity claim = new TaskFileJobEntity();
claim.setId(job.getId());
claim.setTaskId(job.getTaskId());
claim.setModuleType(job.getModuleType());
claim.setStatus("RUNNING");
claim.setUpdatedAt(LocalDateTime.now());
when(taskFileJobService.claimRunning(job.getId())).thenReturn(claim);
when(taskFileJobService.activateRunningClaim(claim)).thenReturn(true);
}
private static TaskFileJobEntity exhaustedSimilarAsinJob(long jobId, long taskId) {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setId(jobId);
job.setTaskId(taskId);
job.setModuleType("SIMILAR_ASIN");
job.setRetryCount(TaskFileJobService.MAX_RETRY_COUNT);
job.setErrorMessage("文件生成任务运行超时,已达到最大重试次数");
return job;
}
}
@@ -0,0 +1,62 @@
package com.nanri.aiimage.modules.ziniao.controller;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexRefreshCursorDto;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoAuthService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexRefreshService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
import org.junit.jupiter.api.Test;
import jakarta.servlet.http.HttpServletRequest;
import org.springframework.http.HttpHeaders;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
class ZiniaoAuthControllerTest {
@Test
void manualIndexRefreshEndpointReturnsCompletedCursor() throws Exception {
ZiniaoAuthService authService = mock(ZiniaoAuthService.class);
ZiniaoShopIndexService indexService = mock(ZiniaoShopIndexService.class);
ZiniaoShopIndexRefreshService refreshService = mock(ZiniaoShopIndexRefreshService.class);
AdminAuthSupport adminAuthSupport = mock(AdminAuthSupport.class);
ZiniaoShopIndexRefreshCursorDto cursor = new ZiniaoShopIndexRefreshCursorDto();
cursor.setStatus("SUCCESS");
cursor.setLastProcessedApiKeyCount(3);
when(refreshService.refreshShopIndexManually()).thenReturn(cursor);
ZiniaoAuthController controller = new ZiniaoAuthController(authService, indexService, refreshService, adminAuthSupport);
MockMvc mockMvc = MockMvcBuilders.standaloneSetup(controller).build();
mockMvc.perform(post("/api/ziniao/index-refresh")
.header(HttpHeaders.AUTHORIZATION, "Bearer admin-token"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.success").value(true))
.andExpect(jsonPath("$.data.status").value("SUCCESS"))
.andExpect(jsonPath("$.data.lastProcessedApiKeyCount").value(3));
verify(adminAuthSupport).requireAdmin(org.mockito.ArgumentMatchers.any());
}
@Test
void manualIndexRefreshRequiresAdministrator() {
ZiniaoAuthService authService = mock(ZiniaoAuthService.class);
ZiniaoShopIndexService indexService = mock(ZiniaoShopIndexService.class);
ZiniaoShopIndexRefreshService refreshService = mock(ZiniaoShopIndexRefreshService.class);
AdminAuthSupport adminAuthSupport = mock(AdminAuthSupport.class);
HttpServletRequest request = mock(HttpServletRequest.class);
BusinessException authFailure = new BusinessException(403, "需要管理员权限");
when(adminAuthSupport.requireAdmin(request)).thenThrow(authFailure);
ZiniaoAuthController controller = new ZiniaoAuthController(authService, indexService, refreshService, adminAuthSupport);
assertThatThrownBy(() -> controller.refreshShopIndex(request, null)).isSameAs(authFailure);
verifyNoInteractions(refreshService);
}
}
@@ -0,0 +1,70 @@
package com.nanri.aiimage.modules.ziniao.service;
import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.nanri.aiimage.modules.shopkey.mapper.ShopKeyMapper;
import com.nanri.aiimage.modules.shopkey.model.entity.ShopKeyEntity;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class ZiniaoApiKeyProviderTest {
@BeforeAll
static void initializeMybatisMetadata() {
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), "ziniao-api-key-provider-test"),
ShopKeyEntity.class
);
}
@Test
void duplicateNormalizedTokensShareOneRefreshAccountAndAllRecordIds() {
ShopKeyMapper mapper = mock(ShopKeyMapper.class);
ShopKeyEntity latest = shopKey(12L, " Bearer duplicate-key ", "最新账号");
ShopKeyEntity older = shopKey(8L, "duplicate-key", "旧账号");
when(mapper.selectList(any())).thenReturn(List.of(latest, older));
ZiniaoApiKeyProvider provider = new ZiniaoApiKeyProvider(mapper);
List<ZiniaoApiKeyProvider.ApiKeyAccount> accounts = provider.listApiKeyAccounts();
assertEquals(1, accounts.size());
assertEquals("duplicate-key", accounts.getFirst().apiKey());
assertEquals("最新账号", accounts.getFirst().accountName());
assertEquals(List.of(12L, 8L), accounts.getFirst().shopKeyIds());
}
@Test
void whitelistResultUpdatesEveryRecordForTheNormalizedToken() {
ShopKeyMapper mapper = mock(ShopKeyMapper.class);
ZiniaoApiKeyProvider provider = new ZiniaoApiKeyProvider(mapper);
ZiniaoApiKeyProvider.ApiKeyAccount account = new ZiniaoApiKeyProvider.ApiKeyAccount(
"duplicate-key",
"账号",
List.of(12L, 8L)
);
provider.markIpWhitelistBlocked(account, "当前服务器 IP 未加入紫鸟白名单");
verify(mapper).update(isNull(), any(Wrapper.class));
}
private ShopKeyEntity shopKey(long id, String token, String accountName) {
ShopKeyEntity entity = new ShopKeyEntity();
entity.setId(id);
entity.setZiniaoToken(token);
entity.setZiniaoAccountName(accountName);
return entity;
}
}
@@ -0,0 +1,49 @@
package com.nanri.aiimage.modules.ziniao.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexRefreshCursorDto;
import org.junit.jupiter.api.Test;
import java.time.Duration;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class ZiniaoShopIndexRefreshServiceTest {
@Test
void manualRefreshRunsUnderDistributedLockAndReturnsCursor() {
ZiniaoShopIndexService indexService = mock(ZiniaoShopIndexService.class);
DistributedJobLockService lockService = mock(DistributedJobLockService.class);
DistributedJobLockService.LockHandle lockHandle = mock(DistributedJobLockService.LockHandle.class);
ZiniaoShopIndexRefreshCursorDto cursor = new ZiniaoShopIndexRefreshCursorDto();
cursor.setStatus("SUCCESS");
when(lockService.tryLock("ziniao:shop-index-refresh", Duration.ofMinutes(30))).thenReturn(lockHandle);
when(indexService.getRefreshCursor()).thenReturn(cursor);
ZiniaoShopIndexRefreshService service = new ZiniaoShopIndexRefreshService(indexService, lockService);
ZiniaoShopIndexRefreshCursorDto result = service.refreshShopIndexManually();
assertThat(result).isSameAs(cursor);
verify(indexService).refreshAllShopIndex();
verify(lockHandle).close();
}
@Test
void manualRefreshRejectsConcurrentExecution() {
ZiniaoShopIndexService indexService = mock(ZiniaoShopIndexService.class);
DistributedJobLockService lockService = mock(DistributedJobLockService.class);
when(lockService.tryLock("ziniao:shop-index-refresh", Duration.ofMinutes(30))).thenReturn(null);
ZiniaoShopIndexRefreshService service = new ZiniaoShopIndexRefreshService(indexService, lockService);
assertThatThrownBy(service::refreshShopIndexManually)
.isInstanceOf(BusinessException.class)
.hasMessageContaining("正在刷新");
verify(indexService, never()).refreshAllShopIndex();
}
}
@@ -0,0 +1,241 @@
package com.nanri.aiimage.modules.ziniao.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.ZiniaoProperties;
import com.nanri.aiimage.modules.ziniao.memory.service.ZiniaoMemoryStoreService;
import com.nanri.aiimage.modules.ziniao.memory.service.ZiniaoTransientCacheService;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexEntryDto;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexRefreshCursorDto;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopCacheDto;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoStaffItemVo;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.time.Duration;
import java.util.List;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class ZiniaoShopIndexServiceTest {
@Mock
private ZiniaoMemoryStoreService ziniaoMemoryStoreService;
@Mock
private ZiniaoTransientCacheService ziniaoTransientCacheService;
@Mock
private ZiniaoApiKeyProvider ziniaoApiKeyProvider;
@Mock
private ZiniaoAuthService ziniaoAuthService;
private ZiniaoShopIndexService service;
private ZiniaoProperties properties;
@BeforeEach
void setUp() {
properties = new ZiniaoProperties();
properties.setShopIndexEntryTtlHours(12);
properties.setShopIndexRefreshBatchSize(100);
service = new ZiniaoShopIndexService(
ziniaoMemoryStoreService,
ziniaoTransientCacheService,
ziniaoApiKeyProvider,
ziniaoAuthService,
properties,
new ObjectMapper()
);
when(ziniaoTransientCacheService.get(
"SHOP_INDEX_REFRESH_CURSOR",
"global",
ZiniaoShopIndexRefreshCursorDto.class
)).thenReturn(Optional.empty());
}
@Test
void companyWhitelistFailureSkipsCurrentApiKeyAndRefreshesNextApiKey() {
stubIpWhitelistDetection();
ZiniaoApiKeyProvider.ApiKeyAccount blocked = new ZiniaoApiKeyProvider.ApiKeyAccount("blocked-key", "blocked-account");
ZiniaoApiKeyProvider.ApiKeyAccount allowed = new ZiniaoApiKeyProvider.ApiKeyAccount("allowed-key", "allowed-account");
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(blocked, allowed));
when(ziniaoAuthService.resolveCompanyIdForIndex("blocked-key"))
.thenThrow(new BusinessException("当前服务器 IP 未加入紫鸟白名单"));
when(ziniaoAuthService.resolveCompanyIdForIndex("allowed-key")).thenReturn(2L);
when(ziniaoAuthService.getOrLoadStaffForIndex("allowed-key", 2L)).thenReturn(List.of(staff(22L)));
when(ziniaoAuthService.getOrLoadUserStoresForIndex("allowed-key", 2L, 22L))
.thenReturn(List.of(shop("shop-2", "店铺B")));
service.refreshShopIndex();
verify(ziniaoMemoryStoreService).put(
eq(ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY),
eq("s:shop-2"),
argThat(value -> value instanceof ZiniaoShopIndexEntryDto entry
&& "shop-2".equals(entry.getShopId())),
any(Duration.class)
);
verify(ziniaoMemoryStoreService).put(
eq(ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY),
eq("n:店铺B"),
any(ZiniaoShopIndexEntryDto.class),
any(Duration.class)
);
verify(ziniaoMemoryStoreService, never()).listAliveEntitiesByType(any(), anyInt());
verify(ziniaoApiKeyProvider).markIpWhitelistBlocked(
eq(blocked),
eq("当前服务器 IP 未加入紫鸟白名单")
);
verify(ziniaoApiKeyProvider).markIpWhitelistAllowed(allowed);
ZiniaoShopIndexRefreshCursorDto cursor = capturedCursor();
assertEquals("SUCCESS", cursor.getStatus());
assertEquals(Integer.valueOf(1), cursor.getLastProcessedApiKeyCount());
assertTrue(cursor.getMessage().contains("IP 白名单: 1"));
}
@Test
void whitelistFailureAfterPartialApiKeyScanDiscardsPartialEntriesAndContinues() {
stubIpWhitelistDetection();
ZiniaoApiKeyProvider.ApiKeyAccount partiallyBlocked = new ZiniaoApiKeyProvider.ApiKeyAccount("partial-key", "partial-account");
ZiniaoApiKeyProvider.ApiKeyAccount allowed = new ZiniaoApiKeyProvider.ApiKeyAccount("allowed-key", "allowed-account");
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(partiallyBlocked, allowed));
when(ziniaoAuthService.resolveCompanyIdForIndex("partial-key")).thenReturn(1L);
when(ziniaoAuthService.getOrLoadStaffForIndex("partial-key", 1L))
.thenReturn(List.of(staff(11L), staff(12L)));
when(ziniaoAuthService.getOrLoadUserStoresForIndex("partial-key", 1L, 11L))
.thenReturn(List.of(shop("partial-shop", "半成品店铺")));
when(ziniaoAuthService.getOrLoadUserStoresForIndex("partial-key", 1L, 12L))
.thenThrow(new BusinessException("当前服务器 IP 未加入紫鸟白名单"));
when(ziniaoAuthService.resolveCompanyIdForIndex("allowed-key")).thenReturn(2L);
when(ziniaoAuthService.getOrLoadStaffForIndex("allowed-key", 2L)).thenReturn(List.of(staff(22L)));
when(ziniaoAuthService.getOrLoadUserStoresForIndex("allowed-key", 2L, 22L))
.thenReturn(List.of(shop("allowed-shop", "正常店铺")));
service.refreshShopIndex();
verify(ziniaoMemoryStoreService, never()).put(
eq(ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY),
eq("s:partial-shop"),
any(),
any(Duration.class)
);
verify(ziniaoMemoryStoreService, never()).put(
eq(ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY),
eq("n:半成品店铺"),
any(),
any(Duration.class)
);
verify(ziniaoMemoryStoreService).put(
eq(ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY),
eq("s:allowed-shop"),
any(ZiniaoShopIndexEntryDto.class),
any(Duration.class)
);
verify(ziniaoMemoryStoreService, never()).listAliveEntitiesByType(any(), anyInt());
verify(ziniaoApiKeyProvider).markIpWhitelistBlocked(
eq(partiallyBlocked),
eq("当前服务器 IP 未加入紫鸟白名单")
);
verify(ziniaoApiKeyProvider).markIpWhitelistAllowed(allowed);
verify(ziniaoApiKeyProvider, never()).markIpWhitelistAllowed(partiallyBlocked);
}
@Test
void nonWhitelistCompanyFailureDoesNotOverwriteWhitelistStatus() {
ZiniaoApiKeyProvider.ApiKeyAccount failed = new ZiniaoApiKeyProvider.ApiKeyAccount("failed-key", "failed-account");
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(failed));
when(ziniaoAuthService.resolveCompanyIdForIndex("failed-key"))
.thenThrow(new BusinessException("紫鸟接口临时不可用"));
when(ziniaoAuthService.isIpWhitelistError(any(BusinessException.class))).thenReturn(false);
service.refreshShopIndex();
verify(ziniaoApiKeyProvider, never()).markIpWhitelistBlocked(any(), any());
verify(ziniaoApiKeyProvider, never()).markIpWhitelistAllowed(any());
}
@Test
void whitelistStatusWriteFailureDoesNotStopNextApiKey() {
stubIpWhitelistDetection();
ZiniaoApiKeyProvider.ApiKeyAccount blocked = new ZiniaoApiKeyProvider.ApiKeyAccount("blocked-key", "blocked-account");
ZiniaoApiKeyProvider.ApiKeyAccount allowed = new ZiniaoApiKeyProvider.ApiKeyAccount("allowed-key", "allowed-account");
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(blocked, allowed));
when(ziniaoAuthService.resolveCompanyIdForIndex("blocked-key"))
.thenThrow(new BusinessException("当前服务器 IP 未加入紫鸟白名单"));
when(ziniaoAuthService.resolveCompanyIdForIndex("allowed-key")).thenReturn(2L);
when(ziniaoAuthService.getOrLoadStaffForIndex("allowed-key", 2L)).thenReturn(List.of());
doThrow(new IllegalStateException("数据库暂时不可用"))
.when(ziniaoApiKeyProvider)
.markIpWhitelistBlocked(blocked, "当前服务器 IP 未加入紫鸟白名单");
service.refreshShopIndex();
verify(ziniaoAuthService).resolveCompanyIdForIndex("allowed-key");
verify(ziniaoApiKeyProvider).markIpWhitelistAllowed(allowed);
}
@Test
void fullRefreshIgnoresScheduledBatchLimit() {
properties.setShopIndexRefreshBatchSize(1);
ZiniaoApiKeyProvider.ApiKeyAccount first = new ZiniaoApiKeyProvider.ApiKeyAccount("first-key", "first-account");
ZiniaoApiKeyProvider.ApiKeyAccount second = new ZiniaoApiKeyProvider.ApiKeyAccount("second-key", "second-account");
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(first, second));
when(ziniaoAuthService.resolveCompanyIdForIndex("first-key")).thenReturn(1L);
when(ziniaoAuthService.resolveCompanyIdForIndex("second-key")).thenReturn(2L);
when(ziniaoAuthService.getOrLoadStaffForIndex("first-key", 1L)).thenReturn(List.of());
when(ziniaoAuthService.getOrLoadStaffForIndex("second-key", 2L)).thenReturn(List.of());
service.refreshAllShopIndex();
verify(ziniaoAuthService).resolveCompanyIdForIndex("first-key");
verify(ziniaoAuthService).resolveCompanyIdForIndex("second-key");
ZiniaoShopIndexRefreshCursorDto cursor = capturedCursor();
assertEquals(Integer.valueOf(2), cursor.getLastProcessedApiKeyCount());
assertEquals(Integer.valueOf(0), cursor.getNextApiKeyOffset());
}
private ZiniaoShopIndexRefreshCursorDto capturedCursor() {
ArgumentCaptor<Object> captor = ArgumentCaptor.forClass(Object.class);
verify(ziniaoTransientCacheService, times(2)).put(
eq("SHOP_INDEX_REFRESH_CURSOR"),
eq("global"),
captor.capture(),
any(Duration.class)
);
return (ZiniaoShopIndexRefreshCursorDto) captor.getAllValues().getLast();
}
private void stubIpWhitelistDetection() {
when(ziniaoAuthService.isIpWhitelistError(any(BusinessException.class)))
.thenAnswer(invocation -> invocation.<BusinessException>getArgument(0).getMessage().contains("白名单"));
}
private ZiniaoStaffItemVo staff(long userId) {
ZiniaoStaffItemVo staff = new ZiniaoStaffItemVo();
staff.setUserId(userId);
return staff;
}
private ZiniaoShopCacheDto shop(String shopId, String shopName) {
ZiniaoShopCacheDto shop = new ZiniaoShopCacheDto();
shop.setShopId(shopId);
shop.setShopName(shopName);
shop.setPlatform("亚马逊");
return shop;
}
}
+576 -5
View File
@@ -23,6 +23,7 @@ from flask import (
g,
Response,
send_file,
stream_with_context,
has_request_context,
)
@@ -44,6 +45,7 @@ admin_api = Blueprint('admin_api', __name__, url_prefix='/api/admin')
_backend_java_session_local = threading.local()
_internal_token_lock = threading.Lock()
IMAGE_VIDEO_DATA_PERMISSION_KEY = 'admin_image_video_task_data'
SHOP_DATA_CRAWL_DATA_PERMISSION_KEY = 'admin_shop_data_crawl_task_data'
ADMIN_MENU_ACCESS_CONFIG = {
'dedupe-total-data': {
@@ -81,6 +83,11 @@ ADMIN_MENU_ACCESS_CONFIG = {
'route_path': 'image-video-tasks',
'error': '无权访问视频任务管理模块',
},
'shop-data-crawl-tasks': {
'column_key': 'admin_shop_data_crawl_tasks',
'route_path': 'shop-data-crawl-tasks',
'error': '无权访问店铺数据任务管理模块',
},
}
ADMIN_MENU_ACCESS_CONFIG.update({
@@ -147,6 +154,29 @@ def _backend_java_forward_headers():
return headers
def _backend_java_internal_headers():
"""Headers for Java routes that are callable only from the Flask admin service."""
internal_token = _resolve_internal_token()
if not internal_token:
raise ValueError('内部凭据服务未配置')
headers = _backend_java_forward_headers()
headers['X-Internal-Token'] = internal_token
return headers
def _backend_java_internal_request():
headers = _backend_java_internal_headers()
_, current_row = get_current_admin_role()
operator_id = _get_current_admin_id(current_row)
try:
operator_id = int(operator_id)
except (TypeError, ValueError) as exc:
raise ValueError('当前管理员身份无效') from exc
if operator_id <= 0:
raise ValueError('当前管理员身份无效')
return headers, {'operatorId': operator_id}
def _proxy_backend_java(
method,
path,
@@ -751,6 +781,27 @@ def _ensure_image_video_data_access():
return role, current_row, (jsonify({'success': False, 'error': '无权查看视频任务数据'}), 403)
def _ensure_shop_data_crawl_data_access():
role, current_row = get_current_admin_role()
if role == 'super_admin':
return role, current_row, None
if not role or not current_row:
return role, current_row, (jsonify({'success': False, 'error': '需要登录'}), 403)
try:
_, key_set, route_set = _effective_permission_sets(
_get_current_admin_id(current_row),
menu_type=None,
current_row=current_row,
role=role,
)
except _PermissionProxyError as exc:
return role, current_row, (exc.response, exc.status)
if (SHOP_DATA_CRAWL_DATA_PERMISSION_KEY in key_set
or 'shop-data-crawl-task-data' in route_set):
return role, current_row, None
return role, current_row, (jsonify({'success': False, 'error': '无权查看店铺数据任务'}), 403)
def _ensure_product_category_access():
role, current_row, items, denied = _load_current_backend_menu_items()
if denied:
@@ -1283,6 +1334,458 @@ def _parse_admin_datetime_arg(name):
raise ValueError(f'{name} 时间格式无效') from exc
_SHOP_DATA_CRAWL_ADMIN_COLUMNS = """
r.id AS result_id, r.task_id, r.user_id, r.source_filename AS shop_name,
r.source_file_url AS shop_id, r.result_filename, r.result_file_url,
r.result_file_size, r.result_content_type, r.row_count,
r.success AS result_success, r.error_message AS result_error,
r.created_at AS result_created_at,
t.task_no, t.status AS task_status, t.request_json, t.result_json,
t.error_message AS task_error, t.created_at, t.updated_at, t.finished_at,
u.username,
(SELECT j.id FROM biz_task_file_job j
WHERE j.module_type = 'SHOP_DATA_CRAWL' AND j.result_id = r.id
AND j.job_type = 'ASSEMBLE_RESULT'
ORDER BY j.id DESC LIMIT 1) AS file_job_id,
(SELECT j.status FROM biz_task_file_job j
WHERE j.module_type = 'SHOP_DATA_CRAWL' AND j.result_id = r.id
AND j.job_type = 'ASSEMBLE_RESULT'
ORDER BY j.id DESC LIMIT 1) AS file_status,
(SELECT j.error_message FROM biz_task_file_job j
WHERE j.module_type = 'SHOP_DATA_CRAWL' AND j.result_id = r.id
AND j.job_type = 'ASSEMBLE_RESULT'
ORDER BY j.id DESC LIMIT 1) AS file_error
"""
def _shop_data_crawl_country_codes(request_json):
payload = _parse_json_value(request_json, {})
if not isinstance(payload, dict):
return []
raw = payload.get('countryCodes')
if raw is None:
raw = payload.get('country_codes')
if not isinstance(raw, list):
return []
return [str(value).strip().upper() for value in raw if str(value or '').strip()]
def _shop_data_crawl_group_names(cursor, rows):
shop_names = sorted({
_shop_data_crawl_shop_key(row.get('shop_name'))
for row in rows
if _shop_data_crawl_shop_key(row.get('shop_name'))
})
if not shop_names:
return {}
placeholders = ','.join(['%s'] * len(shop_names))
cursor.execute(
"SELECT TRIM(sm.shop_name) AS shop_name, "
"GROUP_CONCAT(DISTINCT COALESCE(NULLIF(g.group_name, ''), NULLIF(sm.group_name, '')) "
"ORDER BY sm.id SEPARATOR '') AS group_name "
"FROM biz_shop_manage sm "
"LEFT JOIN biz_shop_manage_group g ON g.id = sm.group_id "
f"WHERE TRIM(sm.shop_name) IN ({placeholders}) GROUP BY TRIM(sm.shop_name)",
tuple(shop_names),
)
return {
_shop_data_crawl_shop_key(row.get('shop_name')): row.get('group_name') or ''
for row in cursor.fetchall()
}
def _shop_data_crawl_shop_key(value):
"""Normalize a shop name for grouping while preserving the display value."""
return str(value or '').strip().casefold()
def _shop_data_crawl_group_name(group_names, shop_name):
"""Resolve a group label from either normalized or legacy exact-key maps."""
if not group_names:
return ''
normalized = _shop_data_crawl_shop_key(shop_name)
return group_names.get(normalized, group_names.get(str(shop_name or '').strip(), '')) or ''
def _shop_data_crawl_admin_item(row, group_names=None):
result_success = row.get('result_success')
file_ready = bool((row.get('result_file_url') or '').strip())
if result_success is None or int(result_success) < 0:
success = None
else:
success = bool(int(result_success))
group_names = group_names or {}
shop_name = row.get('shop_name') or ''
file_status = row.get('file_status') or ('SUCCESS' if file_ready else '')
return {
'task_id': row.get('task_id'),
'task_no': row.get('task_no') or '',
'result_id': row.get('result_id'),
'user_id': row.get('user_id'),
'username': row.get('username') or '',
'shop_name': shop_name,
'shop_id': row.get('shop_id') or '',
'group_name': _shop_data_crawl_group_name(group_names, shop_name),
'status': row.get('task_status') or '',
'success': success,
'error': row.get('result_error') or row.get('task_error') or row.get('file_error') or '',
'country_codes': _shop_data_crawl_country_codes(row.get('request_json')),
'output_filename': row.get('result_filename') or '',
'result_file_url': row.get('result_file_url') or '',
'file_ready': file_ready,
'file_job_id': row.get('file_job_id'),
'file_status': file_status,
'file_error': row.get('file_error') or '',
'file_size': int(row.get('result_file_size') or 0),
'row_count': int(row.get('row_count') or 0),
'created_at': _format_admin_datetime(row.get('created_at') or row.get('result_created_at')),
'updated_at': _format_admin_datetime(row.get('updated_at')),
'finished_at': _format_admin_datetime(row.get('finished_at')),
}
def _shop_data_crawl_group_item(group_row, result_rows, group_names):
"""Build one shop group and cap its children to the newest three results."""
raw_shop_name = group_row.get('shop_name') or ''
display_shop_name = raw_shop_name or '未命名'
group_key = _shop_data_crawl_shop_key(raw_shop_name)
children = result_rows.get(group_key)
if children is None:
children = result_rows.get(str(raw_shop_name).strip(), [])
children = children[:3]
result_items = [_shop_data_crawl_admin_item(row, group_names) for row in children]
latest_created_at = group_row.get('latest_created_at')
if latest_created_at is None and result_items:
latest_created_at = result_items[0].get('created_at')
return {
'shop_name': display_shop_name,
'shop_id': result_items[0].get('shop_id', '') if result_items else '',
'group_name': _shop_data_crawl_group_name(group_names, raw_shop_name),
'latest_created_at': _format_admin_datetime(latest_created_at),
'results': result_items,
}
@admin_api.route('/shop-data-crawl-task-permissions', methods=['GET', 'PUT'])
@login_required
def manage_shop_data_crawl_task_permissions():
json_data = None
if request.method == 'PUT':
data = request.get_json(silent=True) or {}
raw_user_ids = data.get('user_ids') if 'user_ids' in data else data.get('userIds')
json_data = {'userIds': raw_user_ids}
result, error_response, status = _proxy_permission_java(
request.method,
'/api/admin/shop-data-crawl-task-permissions',
json_data=json_data,
)
if error_response is not None:
return error_response, status
if request.method == 'GET':
return jsonify({'success': True, 'items': _permission_response_items(result)})
return jsonify({
'success': True,
'granted_count': result.get('data'),
'msg': result.get('message') or '店铺数据任务权限已更新',
})
@admin_api.route('/shop-data-crawl-tasks')
@login_required
def list_shop_data_crawl_tasks():
_, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks')
if not denied:
_, _, denied = _ensure_shop_data_crawl_data_access()
if denied:
return denied
try:
page = max(1, int(request.args.get('page', 1)))
page_size = min(100, max(10, int(request.args.get('page_size', 20))))
shop_name = (request.args.get('shop_name') or request.args.get('shop') or '').strip()
group_name = (request.args.get('group_name') or request.args.get('group') or '').strip()
created_from = _parse_admin_datetime_arg('created_from')
created_to = _parse_admin_datetime_arg('created_to')
conditions = [
"r.module_type = 'SHOP_DATA_CRAWL'",
"t.module_type = 'SHOP_DATA_CRAWL'",
"TRIM(COALESCE(r.result_file_url, '')) <> ''",
]
params = []
if shop_name:
conditions.append('r.source_filename LIKE %s')
params.append('%' + shop_name + '%')
if group_name:
conditions.append(
'EXISTS (SELECT 1 FROM biz_shop_manage sm '
'LEFT JOIN biz_shop_manage_group g ON g.id = sm.group_id '
'WHERE TRIM(COALESCE(sm.shop_name, \'\')) = '
'TRIM(COALESCE(r.source_filename, \'\')) '
"AND COALESCE(NULLIF(g.group_name, ''), NULLIF(sm.group_name, '')) LIKE %s)"
)
params.append('%' + group_name + '%')
if created_from:
conditions.append('t.created_at >= %s')
params.append(created_from)
if created_to:
conditions.append('t.created_at <= %s')
params.append(created_to)
where_sql = ' AND '.join(conditions)
offset = (page - 1) * page_size
conn = get_db()
try:
with conn.cursor() as cur:
shop_key_sql = "TRIM(COALESCE(r.source_filename, ''))"
grouped_from_sql = (
' FROM biz_file_result r '
'JOIN biz_file_task t ON t.id = r.task_id '
'LEFT JOIN users u ON u.id = r.user_id '
'WHERE ' + where_sql
)
cur.execute(
'SELECT COUNT(*) AS total FROM ('
'SELECT ' + shop_key_sql + ' AS shop_key' + grouped_from_sql +
' GROUP BY ' + shop_key_sql +
') shop_groups',
tuple(params),
)
total = int((cur.fetchone() or {}).get('total') or 0)
cur.execute(
'SELECT ' + shop_key_sql + ' AS shop_name, MAX(t.created_at) AS latest_created_at'
+ grouped_from_sql +
' GROUP BY ' + shop_key_sql +
' ORDER BY latest_created_at DESC, shop_name ASC LIMIT %s OFFSET %s',
tuple(params + [page_size, offset]),
)
group_rows = cur.fetchall()
group_names = _shop_data_crawl_group_names(cur, group_rows)
result_rows_by_shop = {}
selected_shop_names = [row.get('shop_name') for row in group_rows]
if selected_shop_names:
placeholders = ','.join(['%s'] * len(selected_shop_names))
cur.execute(
'SELECT ranked.* FROM (SELECT ' + _SHOP_DATA_CRAWL_ADMIN_COLUMNS +
', ROW_NUMBER() OVER (PARTITION BY ' + shop_key_sql +
' ORDER BY t.created_at DESC, r.id DESC) AS shop_row_number '
' FROM biz_file_result r '
'JOIN biz_file_task t ON t.id = r.task_id '
'LEFT JOIN users u ON u.id = r.user_id '
'WHERE ' + where_sql +
f' AND {shop_key_sql} IN ({placeholders})' +
') ranked WHERE ranked.shop_row_number <= 3 '
'ORDER BY ranked.created_at DESC, ranked.result_id DESC',
tuple(params + selected_shop_names),
)
for row in cur.fetchall():
shop_key = _shop_data_crawl_shop_key(row.get('shop_name'))
result_rows_by_shop.setdefault(shop_key, []).append(row)
finally:
conn.close()
payload = {
'items': [
_shop_data_crawl_group_item(group, result_rows_by_shop, group_names)
for group in group_rows
],
'total': total,
'page': page,
'page_size': page_size,
}
# Keep the existing admin response shape while exposing the grouped
# payload for clients that use the newer data envelope.
return jsonify({'success': True, **payload, 'data': payload})
except ValueError as exc:
return jsonify({'success': False, 'error': str(exc)}), 400
except Exception as exc:
return jsonify({'success': False, 'error': str(exc)}), 500
def _load_shop_data_crawl_download_rows(result_ids):
normalized = sorted({int(result_id) for result_id in result_ids if int(result_id) > 0})
if not normalized:
return {}
placeholders = ','.join(['%s'] * len(normalized))
conn = get_db()
try:
with conn.cursor() as cur:
cur.execute(
'SELECT r.id, r.task_id, r.user_id, r.result_filename, r.source_filename, '
'r.result_file_url, t.status AS task_status '
'FROM biz_file_result r JOIN biz_file_task t ON t.id = r.task_id '
'WHERE r.module_type = %s AND t.module_type = %s '
f'AND r.id IN ({placeholders})',
tuple(['SHOP_DATA_CRAWL', 'SHOP_DATA_CRAWL'] + normalized),
)
return {int(row['id']): row for row in cur.fetchall()}
finally:
conn.close()
def _open_shop_data_crawl_download(row):
url = f"{backend_java_base_url}/api/admin/shop-data-crawl/results/{int(row['id'])}/download"
headers, params = _backend_java_internal_request()
return _get_backend_java_session().get(
url,
params=params,
headers=headers,
stream=True,
timeout=(10, 180),
)
@admin_api.route('/shop-data-crawl-tasks/<int:result_id>/download')
@login_required
def download_shop_data_crawl_task(result_id):
_, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks')
if not denied:
_, _, denied = _ensure_shop_data_crawl_data_access()
if denied:
return denied
rows = _load_shop_data_crawl_download_rows([result_id])
row = rows.get(result_id)
if not row or not (row.get('result_file_url') or '').strip():
return jsonify({'success': False, 'error': '结果文件不存在或尚未生成'}), 404
try:
remote = _open_shop_data_crawl_download(row)
except ValueError as exc:
return jsonify({'success': False, 'error': str(exc)}), 503
except requests.RequestException as exc:
return jsonify({'success': False, 'error': f'结果文件下载失败: {exc}'}), 502
if remote.status_code != 200:
message = remote.text[:500] if remote.content else ''
remote.close()
return jsonify({'success': False, 'error': message or '结果文件下载失败'}), remote.status_code
def generate():
try:
for chunk in remote.iter_content(chunk_size=1024 * 1024):
if chunk:
yield chunk
finally:
remote.close()
response = Response(
stream_with_context(generate()),
content_type=remote.headers.get('Content-Type') or
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
)
disposition = remote.headers.get('Content-Disposition')
if disposition:
response.headers['Content-Disposition'] = disposition
else:
filename = row.get('result_filename') or f"{row.get('source_filename') or result_id}.xlsx"
response.headers['Content-Disposition'] = "attachment; filename*=UTF-8''" + quote(filename)
response.headers['Cache-Control'] = 'no-store'
return response
@admin_api.route('/shop-data-crawl-tasks/<int:result_id>', methods=['DELETE'])
@login_required
def delete_shop_data_crawl_task(result_id):
_, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks')
if not denied:
_, _, denied = _ensure_shop_data_crawl_data_access()
if denied:
return denied
rows = _load_shop_data_crawl_download_rows([result_id])
row = rows.get(result_id)
if not row:
return jsonify({'success': False, 'error': '店铺数据任务不存在'}), 404
if (row.get('task_status') or '').upper() not in {'SUCCESS', 'FAILED', 'CANCELLED'}:
return jsonify({'success': False, 'error': '任务仍在处理中,不能删除'}), 409
try:
headers, params = _backend_java_internal_request()
except ValueError as exc:
return jsonify({'success': False, 'error': str(exc)}), 503
result, error_response, status = _proxy_backend_java(
'DELETE',
f'/api/admin/shop-data-crawl/history/{result_id}',
params=params,
headers=headers,
)
if error_response is not None:
return error_response, status
return jsonify({'success': True, 'msg': result.get('message') or '删除成功'})
@admin_api.route('/shop-data-crawl-tasks/download-zip', methods=['POST'])
@login_required
def download_shop_data_crawl_tasks_zip():
_, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks')
if not denied:
_, _, denied = _ensure_shop_data_crawl_data_access()
if denied:
return denied
data = request.get_json(silent=True) or {}
raw_ids = data.get('result_ids') if 'result_ids' in data else data.get('resultIds')
if not isinstance(raw_ids, list) or not raw_ids:
return jsonify({'success': False, 'error': '请至少选择一个结果文件'}), 400
if len(raw_ids) > 100:
return jsonify({'success': False, 'error': '单次最多打包 100 个结果文件'}), 400
try:
result_ids = []
for raw_id in raw_ids:
result_id = int(raw_id)
if result_id <= 0:
raise ValueError
if result_id not in result_ids:
result_ids.append(result_id)
except (TypeError, ValueError):
return jsonify({'success': False, 'error': '结果文件参数无效'}), 400
rows = _load_shop_data_crawl_download_rows(result_ids)
archive = tempfile.SpooledTemporaryFile(max_size=64 * 1024 * 1024, mode='w+b')
errors = []
file_count = 0
used_names = set()
try:
with zipfile.ZipFile(archive, mode='w', compression=zipfile.ZIP_STORED, allowZip64=True) as output_zip:
for result_id in result_ids:
row = rows.get(result_id)
if not row or not (row.get('result_file_url') or '').strip():
errors.append(f'result-{result_id}: 结果文件不存在或尚未生成')
continue
filename = row.get('result_filename') or f"{row.get('source_filename') or result_id}.xlsx"
filename = re.sub(r'[\\/:*?"<>|]+', '_', filename).strip() or f'result-{result_id}.xlsx'
if filename in used_names:
stem, extension = os.path.splitext(filename)
filename = f'{stem}-{result_id}{extension or ".xlsx"}'
used_names.add(filename)
remote = None
try:
remote = _open_shop_data_crawl_download(row)
remote.raise_for_status()
with output_zip.open(filename, mode='w', force_zip64=True) as target:
for chunk in remote.iter_content(chunk_size=1024 * 1024):
if chunk:
target.write(chunk)
file_count += 1
except (requests.RequestException, ValueError) as exc:
errors.append(f'{filename}: 下载失败 ({exc})')
finally:
if remote is not None:
remote.close()
if errors:
output_zip.writestr('download-errors.txt', '\n'.join(errors).encode('utf-8'))
archive.seek(0)
response = send_file(
archive,
mimetype='application/zip',
as_attachment=True,
download_name=f"shop-data-tasks-{datetime.now().strftime('%Y%m%d-%H%M%S')}.zip",
max_age=0,
)
response.headers['X-Archive-File-Count'] = str(file_count)
response.headers['X-Archive-Error-Count'] = str(len(errors))
response.call_on_close(archive.close)
return response
except Exception:
archive.close()
raise
@admin_api.route('/image-video-task-permissions', methods=['GET', 'PUT'])
@login_required
def manage_image_video_task_permissions():
@@ -1525,7 +2028,10 @@ def list_columns():
return error_response, status
items = [
item for item in items
if (item.get('column_key') or '').strip() != IMAGE_VIDEO_DATA_PERMISSION_KEY
if (item.get('column_key') or '').strip() not in {
IMAGE_VIDEO_DATA_PERMISSION_KEY,
SHOP_DATA_CRAWL_DATA_PERMISSION_KEY,
}
]
# Keep the legacy `items` field; some Java-aware callers use `data`/`columns`.
return jsonify({'success': True, 'items': items, 'columns': items, 'data': items})
@@ -2250,6 +2756,9 @@ def list_shop_keys():
'remark_name': item.get('remarkName') or '',
'ziniao_account_name': item.get('ziniaoAccountName') or '',
'ziniao_token': item.get('ziniaoToken') or '',
'ip_whitelist_status': item.get('ipWhitelistStatus') or 'UNKNOWN',
'ip_whitelist_checked_at': (item.get('ipWhitelistCheckedAt') or '').replace('T', ' ')[:19],
'ip_whitelist_message': item.get('ipWhitelistMessage') or '',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16],
}
@@ -2362,16 +2871,23 @@ def list_dedupe_total_data():
page_size = min(100, max(1, int(request.args.get('page_size', 15))))
keyword = (request.args.get('keyword') or '').strip()
username = (request.args.get('username') or '').strip()
data, error_response, status = _proxy_backend_java(
'GET',
'/api/admin/dedupe-total-data',
start_date = (request.args.get('start_date') or request.args.get('startDate') or '').strip()
end_date = (request.args.get('end_date') or request.args.get('endDate') or '').strip()
params = {
'page': page,
'pageSize': page_size,
'keyword': keyword,
'username': username,
'operatorId': current_row.get('id'),
},
}
if start_date:
params['startDate'] = start_date
if end_date:
params['endDate'] = end_date
data, error_response, status = _proxy_backend_java(
'GET',
'/api/admin/dedupe-total-data',
params=params,
)
if error_response is not None:
return error_response, status
@@ -2830,6 +3346,61 @@ def list_shop_manages():
})
@admin_api.route('/shop-manage/<int:item_id>/credential')
@login_required
def get_shop_manage_credential(item_id):
role, current_row, denied = _ensure_backend_menu_access('shop-manage')
if denied:
return denied
shop_name = (request.args.get('shop_name') or '').strip()
if not shop_name:
return jsonify({'success': False, 'error': '店铺名不能为空'}), 400
access_params = {
'page': 1,
'pageSize': 100,
'shopName': shop_name,
'superAdmin': 'true' if role == 'super_admin' else 'false',
}
if current_row and current_row.get('id'):
access_params['operatorId'] = current_row.get('id')
access_result, error_response, status = _proxy_backend_java(
'GET',
'/api/admin/shop-manages',
params=access_params,
)
if error_response is not None:
return error_response, status
accessible_items = ((access_result.get('data') or {}).get('items') or [])
accessible_item = next((
item for item in accessible_items
if str(item.get('id')) == str(item_id) and (item.get('shopName') or '') == shop_name
), None)
if accessible_item is None:
return jsonify({'success': False, 'error': '店铺不存在或无权访问'}), 404
internal_token = _resolve_internal_token()
if not internal_token:
return jsonify({'success': False, 'error': '内部凭据服务未配置'}), 503
credential_result, error_response, status = _proxy_backend_java(
'GET',
'/api/admin/shop-manages/credential',
params={'shopName': shop_name},
headers={'X-Internal-Token': internal_token},
)
if error_response is not None:
return error_response, status
credential = credential_result.get('data') or {}
if str(credential.get('id')) != str(item_id):
return jsonify({'success': False, 'error': '店铺凭据不匹配'}), 409
response = jsonify({'success': True, 'password': credential.get('password') or ''})
response.headers['Cache-Control'] = 'no-store'
return response
@admin_api.route('/shop-manage', methods=['POST'])
@login_required
def create_shop_manage():
+637 -13
View File
@@ -108,6 +108,7 @@
'query-asin': 'panel-query-asin',
'product-categories': 'panel-product-categories',
'image-video-tasks': 'panel-image-video-tasks',
'shop-data-crawl-tasks': 'panel-shop-data-crawl-tasks',
'history': 'panel-history',
'version': 'panel-version',
'digital-human-version': 'panel-digital-human-version'
@@ -126,6 +127,7 @@
else if (tabName === 'query-asin') loadQueryAsin(1);
else if (tabName === 'product-categories') loadProductCategories();
else if (tabName === 'image-video-tasks') loadImageVideoTasks(1);
else if (tabName === 'shop-data-crawl-tasks') loadShopDataCrawlTasks(1);
else if (tabName === 'history') loadHistory(1);
else if (tabName === 'version') loadSoftwareVersions();
else if (tabName === 'digital-human-version') loadDigitalHumanVersions();
@@ -438,7 +440,8 @@
.then(function (res) {
if (!res.success) return;
var availableColumns = (res.items || []).filter(function (item) {
if (item.column_key === 'admin_image_video_task_data') return false;
if (item.column_key === 'admin_image_video_task_data' ||
item.column_key === 'admin_shop_data_crawl_task_data') return false;
return true;
});
if (currentUserRole === 'admin') {
@@ -880,6 +883,8 @@
function updateImageVideoPermissionAccess() {
var button = document.getElementById('btnOpenImageVideoPermissions');
if (button) button.style.display = currentUserRole === 'super_admin' ? 'inline-flex' : 'none';
var shopButton = document.getElementById('btnOpenShopDataTaskPermissions');
if (shopButton) shopButton.style.display = currentUserRole === 'super_admin' ? 'inline-flex' : 'none';
}
function imageVideoPermissionUsersForView() {
if (imageVideoPermissionView === 'granted') {
@@ -1386,6 +1391,523 @@
}
};
// ========== 店铺数据任务管理 ==========
var shopDataTaskPage = 1, shopDataTaskPageSize = 20;
var shopDataTaskGroups = [];
var shopDataTasks = [];
var selectedShopDataResultIds = new Set();
var shopDataDownloadInProgress = false;
var shopDataPermissionUsers = [];
var shopDataPermissionInitialUserIds = new Set();
var selectedShopDataPermissionUserIds = new Set();
var shopDataPermissionView = 'granted';
function buildShopDataTaskQuery(page) {
var params = new URLSearchParams();
params.set('page', String(page || 1));
params.set('page_size', String(shopDataTaskPageSize));
var values = {
shop_name: document.getElementById('shopDataTaskFilterShop').value.trim(),
group_name: document.getElementById('shopDataTaskFilterGroup').value.trim(),
created_from: document.getElementById('shopDataTaskFilterFrom').value,
created_to: document.getElementById('shopDataTaskFilterTo').value
};
Object.keys(values).forEach(function (key) {
if (values[key]) params.set(key, values[key]);
});
return params.toString();
}
function shopDataResultId(item) {
if (!item) return 0;
var value = item.result_id != null ? item.result_id : item.resultId;
var id = Number(value);
return isFinite(id) && id > 0 ? id : 0;
}
function shopDataBoolean(value) {
if (typeof value === 'string') {
return ['1', 'true', 'yes', 'y'].indexOf(value.toLowerCase()) >= 0;
}
return !!value;
}
function shopDataDateValue(value) {
if (!value) return 0;
var timestamp = Date.parse(String(value).replace(' ', 'T'));
return isNaN(timestamp) ? 0 : timestamp;
}
function shopDataResultSort(a, b) {
var dateDiff = shopDataDateValue(b.created_at || b.finished_at) - shopDataDateValue(a.created_at || a.finished_at);
if (dateDiff) return dateDiff;
return shopDataResultId(b) - shopDataResultId(a);
}
function shopDataGroupKey(item) {
var name = String((item && (item.shop_name || item.shop || item.source_filename)) || '').trim();
// Keep the same trimmed, case-insensitive key as the Flask grouping query.
return 'name:' + name.toLowerCase();
}
function shopDataNormalizeResult(group, raw) {
var result = {};
Object.keys(group || {}).forEach(function (key) {
if (key !== 'results' && key !== 'group_results') result[key] = group[key];
});
Object.keys(raw || {}).forEach(function (key) {
if (key !== 'results' && key !== 'group_results') result[key] = raw[key];
});
result.shop_name = result.shop_name || result.shop || result.source_filename || '';
result.shop_id = result.shop_id || result.shopId || result.source_file_url || '';
result.group_name = result.group_name || result.group || '';
result.status = result.status || result.task_status || result.file_status || '';
result.error = result.error || result.result_error || result.error_message || result.task_error || result.file_error || '';
result.output_filename = result.output_filename || result.result_filename || result.filename || '';
result.country_codes = result.country_codes || result.countryCodes || [];
if (result.file_size == null) result.file_size = result.result_file_size;
if (result.row_count == null) result.row_count = result.rows;
if (result.finished_at == null) result.finished_at = result.completed_at;
if (result.result_id == null && raw && raw.resultId != null) result.result_id = raw.resultId;
if (result.result_id == null && raw && raw.id != null && !Array.isArray(raw.results) && !Array.isArray(raw.group_results)) result.result_id = raw.id;
if (result.file_ready == null) {
result.file_ready = !!String(result.result_file_url || result.resultFileUrl || '').trim();
} else {
result.file_ready = shopDataBoolean(result.file_ready);
}
return result;
}
// The admin API now returns one group per shop. Keep a flat result list
// for selection/actions while rendering the grouped view.
function normalizeShopDataTaskGroups(items) {
var groups = [];
var byKey = Object.create(null);
(Array.isArray(items) ? items : []).forEach(function (rawGroup) {
if (!rawGroup || typeof rawGroup !== 'object') return;
var rawResults = Array.isArray(rawGroup.results)
? rawGroup.results
: (Array.isArray(rawGroup.group_results) ? rawGroup.group_results : [rawGroup]);
var groupBase = {};
Object.keys(rawGroup).forEach(function (key) {
if (key !== 'results' && key !== 'group_results') groupBase[key] = rawGroup[key];
});
if (!groupBase.shop_name && rawResults.length) {
groupBase.shop_name = rawResults[0].shop_name || rawResults[0].shop || rawResults[0].source_filename || '';
}
if (!groupBase.shop_id && rawResults.length) {
groupBase.shop_id = rawResults[0].shop_id || rawResults[0].shopId || rawResults[0].source_file_url || '';
}
var key = shopDataGroupKey(groupBase);
var group = byKey[key];
if (!group) {
group = {
key: key,
shop_name: groupBase.shop_name || '',
shop_id: groupBase.shop_id || '',
group_name: groupBase.group_name || groupBase.group || '',
latest_created_at: groupBase.latest_created_at || '',
results: []
};
byKey[key] = group;
groups.push(group);
}
rawResults.forEach(function (rawResult) {
if (!rawResult || typeof rawResult !== 'object') return;
var result = shopDataNormalizeResult(groupBase, rawResult);
var resultId = shopDataResultId(result);
if (!resultId || !result.file_ready) return;
if (resultId && group.results.some(function (existing) { return shopDataResultId(existing) === resultId; })) return;
group.results.push(result);
if (!group.shop_name) group.shop_name = result.shop_name || '';
if (!group.shop_id) group.shop_id = result.shop_id || '';
if (!group.group_name) group.group_name = result.group_name || '';
});
});
groups.forEach(function (group) {
group.results.sort(shopDataResultSort);
group.results = group.results.slice(0, 3);
if (!group.latest_created_at && group.results.length) {
group.latest_created_at = group.results[0].created_at || group.results[0].finished_at || '';
}
});
return groups.filter(function (group) { return group.results.length > 0; });
}
function shopDataDeleteIcon() {
return '<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M3 6h18"></path><path d="M8 6V4h8v2"></path><path d="M19 6l-1 15H6L5 6"></path><path d="M10 11v6m4-6v6"></path></svg>';
}
function renderShopDataStatus(item, status) {
var normalized = String(status || '-').toUpperCase();
var errorTitle = item && item.error ? ' title="' + escapeHtml(item.error) + '"' : '';
return '<span class="image-video-status ' + escapeHtml(normalized) + '"' + errorTitle + '>' + escapeHtml(normalized) + '</span>';
}
function renderShopDataTaskResult(item) {
var resultId = shopDataResultId(item);
var selected = resultId > 0 && selectedShopDataResultIds.has(resultId);
var status = String(item.status || item.file_status || '').toUpperCase();
var terminal = ['SUCCESS', 'FAILED', 'CANCELLED'].indexOf(status) >= 0;
var countryCodes = item.country_codes != null ? item.country_codes : item.countryCodes;
var countries = Array.isArray(countryCodes) ? countryCodes.join('、') : (String(countryCodes || '') || '-');
var filename = item.output_filename || '-';
var checkbox = '<input type="checkbox" data-shop-data-select="' + (resultId || '') + '"' +
(selected ? ' checked' : '') + (item.file_ready && resultId > 0 ? '' : ' disabled') + '>';
return '<div class="shop-data-result' + (selected ? ' selected' : '') + '" data-shop-data-card="' + (resultId || '') + '">' +
'<div class="shop-data-result-head">' +
'<label class="shop-data-task-title">' + checkbox +
'<span title="任务 #' + escapeHtml(item.task_id || item.task_no || '-') + '">任务 #' + escapeHtml(item.task_id || item.task_no || '-') + '</span>' +
'</label>' +
renderShopDataStatus(item, status || item.file_status) +
'</div>' +
'<div class="image-video-card-info">' +
'<div class="image-video-info-row"><label>国家</label><span>' + escapeHtml(countries) + '</span></div>' +
'<div class="image-video-info-row"><label>文件</label><span title="' + escapeHtml(filename) + '">' + escapeHtml(filename) + '</span></div>' +
'</div>' +
'<div class="image-video-card-actions">' +
'<button class="image-video-card-action" type="button" data-shop-data-download="' + (resultId || '') + '"' + (item.file_ready && resultId > 0 ? '' : ' disabled') + '>' + imageVideoDownloadIcon() + '下载文件</button>' +
'<button class="image-video-card-action shop-data-delete-action" type="button" data-shop-data-delete="' + (resultId || '') + '"' + (terminal && resultId > 0 ? '' : ' disabled') + '>' + shopDataDeleteIcon() + '删除</button>' +
'</div>' +
'</div>';
}
function renderShopDataTaskCard(group) {
var results = Array.isArray(group.results) ? group.results : [];
var latest = group.latest_created_at || (results[0] && (results[0].created_at || results[0].finished_at)) || '-';
return '<article class="image-video-card shop-data-task-card" data-shop-data-group="' + escapeHtml(group.key || '') + '">' +
'<div class="image-video-card-body">' +
'<div class="image-video-card-head shop-data-group-head">' +
'<div class="shop-data-task-title"><span title="' + escapeHtml(group.shop_name || '-') + '">' + escapeHtml(group.shop_name || '-') + '</span></div>' +
'<span class="shop-data-group-meta">' + results.length + '/3 份结果</span>' +
'</div>' +
'<div class="image-video-card-info">' +
'<div class="image-video-info-row"><label>分组</label><span>' + escapeHtml(group.group_name || '-') + '</span></div>' +
'<div class="image-video-info-row"><label>最新</label><span>' + escapeHtml(latest) + '</span></div>' +
'</div>' +
'<div class="shop-data-result-list">' +
(results.length ? results.map(renderShopDataTaskResult).join('') : '<div class="image-video-empty">暂无结果</div>') +
'</div>' +
'</div>' +
'</article>';
}
function syncShopDataSelectionUi() {
document.querySelectorAll('[data-shop-data-card]').forEach(function (card) {
var resultId = Number(card.dataset.shopDataCard);
var selected = selectedShopDataResultIds.has(resultId);
card.classList.toggle('selected', selected);
var checkbox = card.querySelector('[data-shop-data-select]');
if (checkbox) checkbox.checked = selected && !checkbox.disabled;
});
var selectable = shopDataTasks.filter(function (item) { return !!item.file_ready && shopDataResultId(item) > 0; });
var selectedCount = selectable.filter(function (item) {
return selectedShopDataResultIds.has(shopDataResultId(item));
}).length;
var selectAll = document.getElementById('shopDataTaskSelectAll');
selectAll.checked = selectable.length > 0 && selectedCount === selectable.length;
selectAll.indeterminate = selectedCount > 0 && selectedCount < selectable.length;
selectAll.disabled = shopDataDownloadInProgress || selectable.length === 0;
var batch = document.getElementById('btnBatchDownloadShopDataTasks');
batch.disabled = shopDataDownloadInProgress || selectedCount === 0;
batch.innerHTML = imageVideoDownloadIcon() + (shopDataDownloadInProgress
? '处理中'
: '批量下载' + (selectedCount ? ' (' + selectedCount + ')' : ''));
}
function renderShopDataTasks() {
var grid = document.getElementById('shopDataTaskGrid');
grid.innerHTML = shopDataTaskGroups.length
? shopDataTaskGroups.map(renderShopDataTaskCard).join('')
: '<div class="image-video-empty">暂无符合条件的店铺数据任务</div>';
syncShopDataSelectionUi();
}
function loadShopDataCrawlTasks(page) {
shopDataTaskPage = page || 1;
selectedShopDataResultIds.clear();
var grid = document.getElementById('shopDataTaskGrid');
grid.innerHTML = '<div class="image-video-empty">加载中...</div>';
document.getElementById('shopDataTaskDownloadProgress').textContent = '';
fetch('/api/admin/shop-data-crawl-tasks?' + buildShopDataTaskQuery(shopDataTaskPage))
.then(function (response) { return response.json(); })
.then(function (res) {
if (!res.success) throw new Error(res.error || '加载失败');
var payload = res.data && typeof res.data === 'object' && !Array.isArray(res.data) ? res.data : res;
shopDataTaskGroups = normalizeShopDataTaskGroups(payload.items || []);
shopDataTasks = shopDataTaskGroups.reduce(function (all, group) {
return all.concat(group.results || []);
}, []);
var total = payload.total != null ? Number(payload.total) : shopDataTaskGroups.length;
var responsePage = payload.page || page;
var responsePageSize = payload.page_size || shopDataTaskPageSize;
document.getElementById('shopDataTaskTotal').textContent = '共 ' + (total || 0) + ' 家店铺 · 每家店铺保留最新 3 份任务结果';
renderShopDataTasks();
renderPagination('shopDataTaskPagination', total, responsePage, responsePageSize, loadShopDataCrawlTasks);
})
.catch(function (error) {
shopDataTaskGroups = [];
shopDataTasks = [];
grid.innerHTML = '<div class="image-video-empty">加载失败:' + escapeHtml(error.message || '') + '</div>';
document.getElementById('shopDataTaskTotal').textContent = '';
syncShopDataSelectionUi();
});
}
function downloadShopDataTask(item) {
var resultId = shopDataResultId(item);
if (!item || !item.file_ready || !resultId) return;
var filename = item.output_filename || ('shop-data-task-' + resultId + '.xlsx');
triggerImageVideoLink('/api/admin/shop-data-crawl-tasks/' + resultId + '/download', filename, false);
}
function deleteShopDataTask(item) {
var resultId = shopDataResultId(item);
if (!item || !resultId || ['SUCCESS', 'FAILED', 'CANCELLED'].indexOf(String(item.status || item.file_status || '').toUpperCase()) < 0) return;
if (!window.confirm('确认删除店铺“' + (item.shop_name || '-') + '”的任务 #' + item.task_id + ' 及结果文件?')) return;
var progress = document.getElementById('shopDataTaskDownloadProgress');
progress.textContent = '正在删除任务 #' + item.task_id + '...';
fetch('/api/admin/shop-data-crawl-tasks/' + resultId, { method: 'DELETE' })
.then(function (response) { return response.json().then(function (data) { return { ok: response.ok, data: data }; }); })
.then(function (result) {
if (!result.ok || !result.data.success) throw new Error(result.data.error || '删除失败');
progress.textContent = result.data.msg || '删除成功';
loadShopDataCrawlTasks(shopDataTaskPage);
})
.catch(function (error) {
progress.textContent = error.message || '删除失败';
});
}
function downloadShopDataTasksZip() {
var resultIds = Array.from(selectedShopDataResultIds);
if (shopDataDownloadInProgress || !resultIds.length) return;
shopDataDownloadInProgress = true;
syncShopDataSelectionUi();
var progress = document.getElementById('shopDataTaskDownloadProgress');
progress.textContent = '正在打包 ' + resultIds.length + ' 个文件...';
fetch('/api/admin/shop-data-crawl-tasks/download-zip', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ result_ids: resultIds }),
__skipLoading: true
}).then(function (response) {
if (!response.ok) {
return response.json().catch(function () { return {}; }).then(function (data) {
throw new Error(data.error || '压缩包生成失败');
});
}
var filename = imageVideoZipFilename(response);
var errorCount = Number(response.headers.get('X-Archive-Error-Count') || 0);
return response.blob().then(function (blob) {
return { blob: blob, filename: filename, errorCount: errorCount };
});
}).then(function (result) {
var objectUrl = URL.createObjectURL(result.blob);
triggerImageVideoLink(objectUrl, result.filename, false);
setTimeout(function () { URL.revokeObjectURL(objectUrl); }, 1000);
progress.textContent = result.errorCount
? '压缩包已下载,' + result.errorCount + ' 个文件失败,详见包内清单'
: '压缩包下载已开始';
}).catch(function (error) {
progress.textContent = error.message || '批量下载失败';
}).finally(function () {
shopDataDownloadInProgress = false;
syncShopDataSelectionUi();
});
}
function shopDataPermissionUsersForView() {
return shopDataPermissionView === 'granted'
? shopDataPermissionUsers.filter(function (user) {
return shopDataPermissionInitialUserIds.has(Number(user.id));
})
: shopDataPermissionUsers;
}
function filteredShopDataPermissionUsers() {
var users = shopDataPermissionUsersForView();
var keyword = (document.getElementById('shopDataTaskPermissionSearch').value || '').trim().toLowerCase();
return keyword ? users.filter(function (user) {
return String(user.username || '').toLowerCase().indexOf(keyword) >= 0;
}) : users;
}
function renderShopDataPermissionUsers() {
var visibleUsers = filteredShopDataPermissionUsers();
document.getElementById('shopDataTaskPermissionGrantedCount').textContent = '(' + shopDataPermissionInitialUserIds.size + ')';
document.getElementById('shopDataTaskPermissionAllCount').textContent = '(' + shopDataPermissionUsers.length + ')';
document.querySelectorAll('[data-shop-data-permission-view]').forEach(function (tab) {
var active = tab.dataset.shopDataPermissionView === shopDataPermissionView;
tab.classList.toggle('active', active);
tab.setAttribute('aria-selected', active ? 'true' : 'false');
});
var pendingCount = shopDataPermissionUsers.filter(function (user) {
var userId = Number(user.id);
return shopDataPermissionInitialUserIds.has(userId) !== selectedShopDataPermissionUserIds.has(userId);
}).length;
document.getElementById('shopDataTaskPermissionSummary').textContent =
(shopDataPermissionView === 'granted' ? '当前显示已分配用户,共 ' + visibleUsers.length + ' 人' : '当前显示全部用户,已分配 ' + shopDataPermissionInitialUserIds.size + ' 人') +
(pendingCount ? ' · 待保存变更 ' + pendingCount + ' 项' : '');
document.getElementById('shopDataTaskPermissionList').innerHTML = visibleUsers.length
? visibleUsers.map(function (user) {
var userId = Number(user.id);
var saved = shopDataPermissionInitialUserIds.has(userId);
var selected = selectedShopDataPermissionUserIds.has(userId);
var changed = saved !== selected;
return '<div class="image-video-permission-row">' +
'<input type="checkbox" data-shop-data-permission-user="' + userId + '"' + (selected ? ' checked' : '') + '>' +
'<span class="image-video-permission-user"><span class="image-video-permission-name">' + escapeHtml(user.username || '-') + '</span><span class="image-video-permission-role">' + escapeHtml(roleLabel(user.role)) + '</span></span>' +
'<button class="image-video-permission-state' + (changed ? ' pending' : (saved ? ' granted' : '')) + '" type="button" data-shop-data-permission-toggle="' + userId + '">' +
(changed ? (selected ? '待保存分配' : '待保存取消') : (saved ? '取消分配' : '分配')) +
'</button></div>';
}).join('')
: '<div class="image-video-permission-empty">暂无匹配用户</div>';
var selectedVisibleCount = visibleUsers.filter(function (user) {
return selectedShopDataPermissionUserIds.has(Number(user.id));
}).length;
var selectAll = document.getElementById('shopDataTaskPermissionSelectAll');
selectAll.checked = visibleUsers.length > 0 && selectedVisibleCount === visibleUsers.length;
selectAll.indeterminate = selectedVisibleCount > 0 && selectedVisibleCount < visibleUsers.length;
selectAll.disabled = visibleUsers.length === 0;
}
function openShopDataTaskPermissions() {
if (currentUserRole !== 'super_admin') return;
var modal = document.getElementById('shopDataTaskPermissionModal');
var saveButton = document.getElementById('btnSaveShopDataTaskPermissions');
var permissionLoaded = false;
modal.classList.add('show');
shopDataPermissionView = 'granted';
document.getElementById('shopDataTaskPermissionSearch').value = '';
document.getElementById('shopDataTaskPermissionMessage').textContent = '';
document.getElementById('shopDataTaskPermissionList').innerHTML = '<div class="image-video-permission-empty">加载中...</div>';
saveButton.disabled = true;
fetch('/api/admin/shop-data-crawl-task-permissions')
.then(function (response) { return response.json(); })
.then(function (res) {
if (!res.success) throw new Error(res.error || '权限加载失败');
shopDataPermissionUsers = res.items || [];
shopDataPermissionInitialUserIds = new Set(shopDataPermissionUsers.filter(function (user) { return !!user.granted; }).map(function (user) { return Number(user.id); }));
selectedShopDataPermissionUserIds = new Set(shopDataPermissionInitialUserIds);
permissionLoaded = true;
renderShopDataPermissionUsers();
}).catch(function (error) {
shopDataPermissionUsers = [];
shopDataPermissionInitialUserIds = new Set();
selectedShopDataPermissionUserIds = new Set();
document.getElementById('shopDataTaskPermissionList').innerHTML = '<div class="image-video-permission-empty">' + escapeHtml(error.message || '权限加载失败') + '</div>';
document.getElementById('shopDataTaskPermissionMessage').textContent = '权限加载失败,请关闭后重试';
}).finally(function () {
saveButton.disabled = !permissionLoaded;
});
}
function closeShopDataTaskPermissions() {
document.getElementById('shopDataTaskPermissionModal').classList.remove('show');
}
function saveShopDataTaskPermissions() {
var message = document.getElementById('shopDataTaskPermissionMessage');
message.textContent = '保存中...';
message.className = 'msg';
document.getElementById('btnSaveShopDataTaskPermissions').disabled = true;
fetch('/api/admin/shop-data-crawl-task-permissions', {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ user_ids: Array.from(selectedShopDataPermissionUserIds).sort(function (a, b) { return a - b; }) })
}).then(function (response) { return response.json(); })
.then(function (res) {
if (!res.success) throw new Error(res.error || '权限保存失败');
shopDataPermissionInitialUserIds = new Set(selectedShopDataPermissionUserIds);
renderShopDataPermissionUsers();
message.textContent = res.msg || '保存成功';
message.className = 'msg ok';
}).catch(function (error) {
message.textContent = error.message || '权限保存失败';
message.className = 'msg err';
}).finally(function () {
document.getElementById('btnSaveShopDataTaskPermissions').disabled = false;
});
}
document.getElementById('btnFilterShopDataTasks').onclick = function () { loadShopDataCrawlTasks(1); };
document.getElementById('btnResetShopDataTasks').onclick = function () {
['shopDataTaskFilterShop', 'shopDataTaskFilterGroup', 'shopDataTaskFilterFrom', 'shopDataTaskFilterTo']
.forEach(function (id) { document.getElementById(id).value = ''; });
loadShopDataCrawlTasks(1);
};
document.getElementById('shopDataTaskSelectAll').onchange = function (event) {
shopDataTasks.forEach(function (item) {
var resultId = shopDataResultId(item);
if (!item.file_ready || !resultId) return;
if (event.target.checked) selectedShopDataResultIds.add(resultId);
else selectedShopDataResultIds.delete(resultId);
});
syncShopDataSelectionUi();
};
document.getElementById('btnBatchDownloadShopDataTasks').onclick = downloadShopDataTasksZip;
document.getElementById('shopDataTaskGrid').onchange = function (event) {
var checkbox = event.target.closest('[data-shop-data-select]');
if (!checkbox) return;
var resultId = Number(checkbox.dataset.shopDataSelect);
if (!resultId) return;
if (checkbox.checked) selectedShopDataResultIds.add(resultId);
else selectedShopDataResultIds.delete(resultId);
syncShopDataSelectionUi();
};
document.getElementById('shopDataTaskGrid').onclick = function (event) {
var downloadButton = event.target.closest('[data-shop-data-download]');
if (downloadButton) {
var downloadItem = shopDataTasks.find(function (task) { return shopDataResultId(task) === Number(downloadButton.dataset.shopDataDownload); });
downloadShopDataTask(downloadItem);
return;
}
var deleteButton = event.target.closest('[data-shop-data-delete]');
if (deleteButton) {
var deleteItem = shopDataTasks.find(function (task) { return shopDataResultId(task) === Number(deleteButton.dataset.shopDataDelete); });
deleteShopDataTask(deleteItem);
}
};
document.getElementById('btnOpenShopDataTaskPermissions').onclick = openShopDataTaskPermissions;
document.getElementById('btnCloseShopDataTaskPermissions').onclick = closeShopDataTaskPermissions;
document.getElementById('btnCancelShopDataTaskPermissions').onclick = closeShopDataTaskPermissions;
document.getElementById('btnSaveShopDataTaskPermissions').onclick = saveShopDataTaskPermissions;
document.querySelectorAll('[data-shop-data-permission-view]').forEach(function (tab) {
tab.onclick = function () {
shopDataPermissionView = tab.dataset.shopDataPermissionView || 'granted';
document.getElementById('shopDataTaskPermissionSearch').value = '';
renderShopDataPermissionUsers();
};
});
document.getElementById('shopDataTaskPermissionSearch').oninput = renderShopDataPermissionUsers;
document.getElementById('shopDataTaskPermissionSelectAll').onchange = function (event) {
filteredShopDataPermissionUsers().forEach(function (user) {
if (event.target.checked) selectedShopDataPermissionUserIds.add(Number(user.id));
else selectedShopDataPermissionUserIds.delete(Number(user.id));
});
renderShopDataPermissionUsers();
};
document.getElementById('shopDataTaskPermissionList').onclick = function (event) {
var button = event.target.closest('[data-shop-data-permission-toggle]');
if (!button) return;
var userId = Number(button.dataset.shopDataPermissionToggle);
if (selectedShopDataPermissionUserIds.has(userId)) selectedShopDataPermissionUserIds.delete(userId);
else selectedShopDataPermissionUserIds.add(userId);
renderShopDataPermissionUsers();
};
document.getElementById('shopDataTaskPermissionList').onchange = function (event) {
var checkbox = event.target.closest('[data-shop-data-permission-user]');
if (!checkbox) return;
var userId = Number(checkbox.dataset.shopDataPermissionUser);
if (checkbox.checked) selectedShopDataPermissionUserIds.add(userId);
else selectedShopDataPermissionUserIds.delete(userId);
renderShopDataPermissionUsers();
};
document.getElementById('shopDataTaskPermissionModal').onclick = function (event) {
if (event.target === event.currentTarget) closeShopDataTaskPermissions();
};
var historyPage = 1, historyPageSize = 15;
function toSqlDatetime(val) {
if (!val) return '';
@@ -1636,15 +2158,33 @@
// ========== 数据去重总数据 ==========
var dedupeTotalDataPage = 1, dedupeTotalDataPageSize = 15;
function getDedupeTotalDataDateRange() {
return {
startDate: document.getElementById('exportDedupeTotalDataStartDate').value || '',
endDate: document.getElementById('exportDedupeTotalDataEndDate').value || ''
};
}
function validateDedupeTotalDataDateRange() {
var dateRange = getDedupeTotalDataDateRange();
if (dateRange.startDate && dateRange.endDate && dateRange.startDate > dateRange.endDate) {
alert('开始日期不能晚于结束日期');
return false;
}
return true;
}
function buildDedupeTotalDataQuery(page) {
var q = 'page=' + (page || 1) + '&page_size=' + dedupeTotalDataPageSize;
var keyword = (document.getElementById('searchDedupeTotalData').value || '').trim();
var username = (document.getElementById('searchDedupeTotalDataUsername').value || '').trim();
var dateRange = getDedupeTotalDataDateRange();
if (keyword) q += '&keyword=' + encodeURIComponent(keyword);
if (username) q += '&username=' + encodeURIComponent(username);
if (dateRange.startDate) q += '&start_date=' + encodeURIComponent(dateRange.startDate);
if (dateRange.endDate) q += '&end_date=' + encodeURIComponent(dateRange.endDate);
return q;
}
function loadDedupeTotalData(page) {
if (!validateDedupeTotalDataDateRange()) return;
dedupeTotalDataPage = page || 1;
fetch('/api/admin/dedupe-total-data?' + buildDedupeTotalDataQuery(dedupeTotalDataPage))
.then(function (r) { return r.json(); })
@@ -1698,16 +2238,12 @@
document.getElementById('btnSearchDedupeTotalData').onclick = function () { loadDedupeTotalData(1); };
document.getElementById('btnExportDedupeTotalData').onclick = function () {
var username = (document.getElementById('searchDedupeTotalDataUsername').value || '').trim();
var startDate = document.getElementById('exportDedupeTotalDataStartDate').value || '';
var endDate = document.getElementById('exportDedupeTotalDataEndDate').value || '';
if (startDate && endDate && startDate > endDate) {
alert('开始日期不能晚于结束日期');
return;
}
var dateRange = getDedupeTotalDataDateRange();
if (!validateDedupeTotalDataDateRange()) return;
var params = [];
if (username) params.push('username=' + encodeURIComponent(username));
if (startDate) params.push('start_date=' + encodeURIComponent(startDate));
if (endDate) params.push('end_date=' + encodeURIComponent(endDate));
if (dateRange.startDate) params.push('start_date=' + encodeURIComponent(dateRange.startDate));
if (dateRange.endDate) params.push('end_date=' + encodeURIComponent(dateRange.endDate));
fetch('/api/admin/dedupe-total-data/export' + (params.length ? ('?' + params.join('&')) : ''))
.then(function (response) {
var contentType = response.headers.get('content-type') || '';
@@ -2112,6 +2648,20 @@
function buildShopKeyQuery(page) {
return 'page=' + (page || 1) + '&page_size=' + shopKeyPageSize;
}
function renderShopKeyWhitelistStatus(item) {
var status = String(item.ip_whitelist_status || 'UNKNOWN').toUpperCase();
var statusMeta = {
ALLOWED: { label: '正常', className: 'is-allowed' },
BLOCKED: { label: '未加白名单', className: 'is-blocked' },
UNKNOWN: { label: '未检测', className: 'is-unknown' }
};
var meta = statusMeta[status] || statusMeta.UNKNOWN;
var details = [];
if (item.ip_whitelist_checked_at) details.push('检测时间:' + item.ip_whitelist_checked_at);
if (item.ip_whitelist_message) details.push(item.ip_whitelist_message);
return '<span class="shop-key-whitelist-status ' + meta.className + '" title="' +
escapeHtml(details.join('\n')) + '">' + escapeHtml(meta.label) + '</span>';
}
function loadShopKeys(page) {
shopKeyPage = page || 1;
fetch('/api/admin/shop-keys?' + buildShopKeyQuery(shopKeyPage))
@@ -2124,11 +2674,11 @@
}
var items = res.items || [];
if (items.length === 0) {
tbody.innerHTML = '<tr><td colspan="7" class="empty-tip">暂无店铺密钥</td></tr>';
tbody.innerHTML = '<tr><td colspan="8" class="empty-tip">暂无店铺密钥</td></tr>';
} else {
tbody.innerHTML = items.map(function (item, index) {
var rowNo = (shopKeyPage - 1) * shopKeyPageSize + index + 1;
return '<tr><td>' + rowNo + '</td><td>' + (item.remark_name || '') + '</td><td>' + (item.ziniao_account_name || '') + '</td><td>' + (item.ziniao_token || '') + '</td><td>' + (item.created_at || '') + '</td><td>' + (item.updated_at || '') + '</td><td>' +
return '<tr><td>' + rowNo + '</td><td>' + escapeHtml(item.remark_name || '') + '</td><td>' + escapeHtml(item.ziniao_account_name || '') + '</td><td>' + escapeHtml(item.ziniao_token || '') + '</td><td>' + renderShopKeyWhitelistStatus(item) + '</td><td>' + escapeHtml(item.created_at || '') + '</td><td>' + escapeHtml(item.updated_at || '') + '</td><td>' +
'<button class="btn btn-sm" data-shop-key-edit="' + item.id + '" data-shop-key="' + (JSON.stringify(item).replace(/"/g, '&quot;')) + '">编辑</button> ' +
'<button class="btn btn-sm btn-danger" data-shop-key-delete="' + item.id + '" data-ziniao-account-name="' + (item.ziniao_account_name || '').replace(/"/g, '&quot;') + '">删除</button>' +
'</td></tr>';
@@ -2138,7 +2688,7 @@
bindShopKeyActions();
})
.catch(function () {
document.getElementById('shopKeyListBody').innerHTML = '<tr><td colspan="7" class="empty-tip">请求失败</td></tr>';
document.getElementById('shopKeyListBody').innerHTML = '<tr><td colspan="8" class="empty-tip">请求失败</td></tr>';
});
}
function bindShopKeyActions() {
@@ -2264,6 +2814,21 @@
return query;
}
function shopPasswordIcon(revealed) {
if (revealed) {
return '<svg viewBox="0 0 24 24" aria-hidden="true"><path d="m2 2 20 20"></path><path d="M6.71 6.71C4.7 8.1 3.34 10.08 2 12c2.12 3.04 5.5 6 10 6 1.67 0 3.17-.41 4.47-1.05"></path><path d="M10.73 5.08A9.36 9.36 0 0 1 12 5c4.5 0 7.88 2.96 10 7a15.82 15.82 0 0 1-2.12 2.91"></path><path d="M14.12 14.12A3 3 0 0 1 9.88 9.88"></path></svg>';
}
return '<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M2.06 12.35a1 1 0 0 1 0-.7C3.54 8.04 7.06 5.5 12 5.5s8.46 2.54 9.94 6.15a1 1 0 0 1 0 .7C20.46 15.96 16.94 18.5 12 18.5S3.54 15.96 2.06 12.35"></path><circle cx="12" cy="12" r="3"></circle></svg>';
}
function renderShopPasswordCell(item) {
var maskedPassword = item.password || '******';
return '<span class="shop-password-cell">' +
'<span class="shop-password-value" data-shop-password-value>' + escapeHtml(maskedPassword) + '</span>' +
'<button type="button" class="shop-password-toggle" data-shop-password-toggle="' + escapeHtml(item.id) + '" data-shop-name="' + escapeHtml(item.shop_name || '') + '" data-masked-password="' + escapeHtml(maskedPassword) + '" aria-label="显示密码" aria-pressed="false" title="显示密码">' +
shopPasswordIcon(false) + '</button></span>';
}
function loadShopManage(page) {
shopManagePage = page || 1;
fetch('/api/admin/shop-manages?' + buildShopManageQuery(shopManagePage))
@@ -2280,7 +2845,7 @@
} else {
tbody.innerHTML = items.map(function (item, index) {
var rowNo = (shopManagePage - 1) * shopManagePageSize + index + 1;
return '<tr><td>' + rowNo + '</td><td>' + (item.group_name || '') + '</td><td>' + (item.shop_name || '') + '</td><td>' + (item.mall_name || '') + '</td><td>' + (item.account || '') + '</td><td>' + (item.password || '') + '</td><td>' + (item.created_at || '') + '</td><td>' + (item.updated_at || '') + '</td><td>' +
return '<tr><td>' + rowNo + '</td><td>' + (item.group_name || '') + '</td><td>' + (item.shop_name || '') + '</td><td>' + (item.mall_name || '') + '</td><td>' + (item.account || '') + '</td><td>' + renderShopPasswordCell(item) + '</td><td>' + (item.created_at || '') + '</td><td>' + (item.updated_at || '') + '</td><td>' +
'<button class="btn btn-sm" data-shop-manage-edit="' + item.id + '" data-shop-manage="' + (JSON.stringify(item).replace(/"/g, '&quot;')) + '">编辑</button> ' +
'<button class="btn btn-sm btn-danger" data-shop-manage-delete="' + item.id + '" data-shop-manage-name="' + (item.shop_name || '').replace(/"/g, '&quot;') + '">删除</button>' +
'</td></tr>';
@@ -2295,6 +2860,42 @@
}
function bindShopManageActions() {
document.querySelectorAll('[data-shop-password-toggle]').forEach(function (btn) {
btn.onclick = function () {
var valueEl = btn.parentElement.querySelector('[data-shop-password-value]');
var revealed = btn.dataset.revealed === 'true';
if (revealed) {
valueEl.textContent = btn.dataset.maskedPassword || '******';
btn.dataset.revealed = 'false';
btn.setAttribute('aria-label', '显示密码');
btn.setAttribute('aria-pressed', 'false');
btn.title = '显示密码';
btn.innerHTML = shopPasswordIcon(false);
return;
}
btn.disabled = true;
valueEl.textContent = '读取中...';
fetch('/api/admin/shop-manage/' + encodeURIComponent(btn.dataset.shopPasswordToggle) + '/credential?shop_name=' + encodeURIComponent(btn.dataset.shopName || ''))
.then(function (r) { return r.json(); })
.then(function (res) {
if (!res.success) throw new Error(res.error || '读取密码失败');
valueEl.textContent = res.password || '';
btn.dataset.revealed = 'true';
btn.setAttribute('aria-label', '隐藏密码');
btn.setAttribute('aria-pressed', 'true');
btn.title = '隐藏密码';
btn.innerHTML = shopPasswordIcon(true);
})
.catch(function (error) {
valueEl.textContent = btn.dataset.maskedPassword || '******';
alert(error.message || '读取密码失败');
})
.finally(function () {
btn.disabled = false;
});
};
});
document.querySelectorAll('[data-shop-manage-edit]').forEach(function (btn) {
btn.onclick = function () {
var item = {};
@@ -4472,6 +5073,7 @@
html += '<span class="page-item' + active + '" onclick="loadDigitalHumanVersions(' + i + ')">' + i + '</span>';
}
pagination.innerHTML = html;
appendPaginationQuickJump(pagination, pages, current, loadDigitalHumanVersions);
} else {
pagination.innerHTML = '';
}
@@ -4918,6 +5520,27 @@
if (document.getElementById('editColumnMenuType')) document.getElementById('editColumnMenuType').onchange = populateColumnParentSelects;
// ========== 分页 ==========
function appendPaginationQuickJump(el, totalPages, page, onPage) {
el.insertAdjacentHTML('beforeend',
'<label class="pagination-jump">跳至<input type="number" min="1" max="' + totalPages + '" step="1" inputmode="numeric" aria-label="跳转页码" data-page-jump-input>页</label>' +
'<button type="button" data-page-jump>跳转</button>');
var jumpInput = el.querySelector('[data-page-jump-input]');
var jumpToPage = function () {
var targetPage = parseInt(jumpInput.value, 10);
if (isNaN(targetPage)) {
jumpInput.focus();
return;
}
targetPage = Math.min(Math.max(targetPage, 1), totalPages);
jumpInput.value = targetPage;
if (targetPage !== page) onPage(targetPage);
};
el.querySelector('[data-page-jump]').onclick = jumpToPage;
jumpInput.onkeydown = function (event) {
if (event.key === 'Enter') jumpToPage();
};
}
function renderPagination(elId, total, page, pageSize, onPage) {
var el = document.getElementById(elId);
if (!el) return;
@@ -4929,6 +5552,7 @@
el.querySelectorAll('[data-p]').forEach(function (b) {
if (!b.disabled) b.onclick = function () { onPage(parseInt(b.dataset.p, 10)); };
});
appendPaginationQuickJump(el, totalPages, page, onPage);
}
// 初始化

Some files were not shown because too many files have changed in this diff Show More