Compare commits
2 Commits
a821d306ac
...
f7e26ee266
| Author | SHA1 | Date | |
|---|---|---|---|
| f7e26ee266 | |||
| 28e7fce11c |
@@ -26,6 +26,7 @@
|
||||
<minio.version>8.5.17</minio.version>
|
||||
<javassist.version>3.28.0-GA</javassist.version>
|
||||
<jjwt.version>0.12.6</jjwt.version>
|
||||
<bouncycastle.version>1.78.1</bouncycastle.version>
|
||||
</properties>
|
||||
|
||||
<dependencyManagement>
|
||||
@@ -99,6 +100,11 @@
|
||||
<artifactId>minio</artifactId>
|
||||
<version>${minio.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.bouncycastle</groupId>
|
||||
<artifactId>bcprov-jdk18on</artifactId>
|
||||
<version>${bouncycastle.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.jsonwebtoken</groupId>
|
||||
<artifactId>jjwt-api</artifactId>
|
||||
|
||||
@@ -2,13 +2,11 @@ package com.nanri.aiimage;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
|
||||
import java.time.ZoneId;
|
||||
import java.util.TimeZone;
|
||||
|
||||
@SpringBootApplication
|
||||
@EnableScheduling
|
||||
public class AiImageApplication {
|
||||
|
||||
private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai");
|
||||
|
||||
@@ -16,7 +16,7 @@ public class AppearancePatentProperties {
|
||||
private String cozeToken = "";
|
||||
private List<CozeCredential> cozeCredentials = new ArrayList<>();
|
||||
private int cozeCredentialStripeSize = 5;
|
||||
private int cozeBatchSize = 50;
|
||||
private int cozeBatchSize = 10;
|
||||
private int cozeConnectTimeoutMillis = 10000;
|
||||
private int cozeReadTimeoutMillis = 60000;
|
||||
private int cozePollIntervalMillis = 30000;
|
||||
|
||||
@@ -12,5 +12,7 @@ public class ModuleCleanupProperties {
|
||||
private boolean enabled = true;
|
||||
private String cron = "0 0 0 * * *";
|
||||
private long retentionDays = 7;
|
||||
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "SHOP_DATA_CRAWL", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
|
||||
// SHOP_DATA_CRAWL is governed by per-shop latest-three retention in its
|
||||
// task service and must not be removed by the age-based sweep.
|
||||
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK", "SHOP_MATCH", "PATROL_DELETE", "QUERY_ASIN", "WITHDRAW", "APPEARANCE_PATENT", "SIMILAR_ASIN", "COLLECT_DATA"));
|
||||
}
|
||||
|
||||
@@ -2,9 +2,11 @@ package com.nanri.aiimage.config;
|
||||
|
||||
import io.swagger.v3.oas.models.ExternalDocumentation;
|
||||
import io.swagger.v3.oas.models.OpenAPI;
|
||||
import io.swagger.v3.oas.models.Components;
|
||||
import io.swagger.v3.oas.models.info.Contact;
|
||||
import io.swagger.v3.oas.models.info.Info;
|
||||
import io.swagger.v3.oas.models.info.License;
|
||||
import io.swagger.v3.oas.models.security.SecurityScheme;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@@ -39,6 +41,11 @@ public class OpenApiConfig {
|
||||
.version("v0.0.1")
|
||||
.contact(new Contact().name("Nanri AI"))
|
||||
.license(new License().name("Internal Use")))
|
||||
.components(new Components().addSecuritySchemes("bearerAuth", new SecurityScheme()
|
||||
.type(SecurityScheme.Type.HTTP)
|
||||
.scheme("bearer")
|
||||
.bearerFormat("JWT")
|
||||
.description("管理员登录 JWT")))
|
||||
.externalDocs(new ExternalDocumentation()
|
||||
.description("Knife4j 文档")
|
||||
.url("/doc.html"));
|
||||
|
||||
@@ -12,6 +12,7 @@ public class OssProperties {
|
||||
private String bucket;
|
||||
private String imageVideoBucket;
|
||||
private String digitalHumanBucket;
|
||||
private String templateBucket;
|
||||
private String accessKeyId;
|
||||
private String accessKeySecret;
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.TaskScheduler;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
|
||||
|
||||
@@ -10,6 +12,8 @@ import java.time.Clock;
|
||||
import java.time.ZoneId;
|
||||
|
||||
@Configuration
|
||||
@EnableScheduling
|
||||
@ConditionalOnProperty(prefix = "aiimage.scheduling", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
@Slf4j
|
||||
public class SchedulingConfig {
|
||||
|
||||
|
||||
@@ -61,13 +61,8 @@ public class SimilarAsinProperties {
|
||||
*/
|
||||
private int cozeSubmitMaxRetryCount = 5;
|
||||
|
||||
/**
|
||||
* 图片嵌入下载线程池大小。原 SimilarAsinImageEmbedder.DOWNLOAD_POOL_SIZE = 8。
|
||||
* P2-10:1000+ 行 ×3 列图片场景下,pool=16 仍是 assemble 阶段瓶颈(实测下载 244s/918s),
|
||||
* 提到 32 配合 retry=2、global deadline 显著拉低尾延迟;
|
||||
* 受 2GB 堆约束,单图缩略图维持 300KB 以内,整体内存峰值 ≈ 32 * 300KB ≈ 10MB。
|
||||
*/
|
||||
private int imageDownloadPoolSize = 32;
|
||||
/** 图片下载、解码和缩放共享该池;默认 8,避免批量结果生成占满整机 CPU。 */
|
||||
private int imageDownloadPoolSize = 8;
|
||||
|
||||
/**
|
||||
* 单张图片下载超时(秒)。
|
||||
@@ -77,6 +72,12 @@ public class SimilarAsinProperties {
|
||||
*/
|
||||
private int imageDownloadTimeoutSeconds = 5;
|
||||
|
||||
/** 单个结果文件整批图片预取预算,耗尽后缺图单元格降级为 URL。 */
|
||||
private int imagePrefetchTimeoutSeconds = 1800;
|
||||
|
||||
/** 多源结果文件组装的单任务硬上限;运行期间由文件任务 heartbeat 保活。 */
|
||||
private int resultFileTimeoutMinutes = 90;
|
||||
|
||||
/**
|
||||
* assemble 阶段 taskImageCache 的字节上限。
|
||||
* 默认 256MB:5000 行 × 3 列 × 平均 100KB = 1.5GB 远超 2GB 堆,
|
||||
@@ -85,6 +86,7 @@ public class SimilarAsinProperties {
|
||||
* 出现淘汰过频影响命中率时可上调到 512MB;2GB 堆约束下不建议超过 768MB。
|
||||
*/
|
||||
private long imageCacheMaxBytes = 256L * 1024L * 1024L;
|
||||
private String imageLocalCacheDir = "";
|
||||
private boolean imageDbCacheEnabled = false;
|
||||
|
||||
/**
|
||||
|
||||
+8
-32
@@ -1338,10 +1338,9 @@ public class AppearancePatentTaskService {
|
||||
}
|
||||
}
|
||||
}
|
||||
boolean failedByConclusion = hasInfringingPersistedConclusion(task.getId());
|
||||
boolean failed = finalError != null && !finalError.isBlank() || failedByConclusion;
|
||||
boolean failed = finalError != null && !finalError.isBlank();
|
||||
boolean waitingForAssemble = !failed && assembleWorkbook && !shouldAssembleSynchronously();
|
||||
task.setStatus(waitingForAssemble ? STATUS_RUNNING : (failed ? STATUS_FAILED : STATUS_SUCCESS));
|
||||
task.setStatus(resolveTaskExecutionStatus(waitingForAssemble, failed));
|
||||
task.setSuccessFileCount(failed ? 0 : 1);
|
||||
task.setFailedFileCount(failed ? 1 : 0);
|
||||
task.setErrorMessage(finalError);
|
||||
@@ -2418,10 +2417,9 @@ public class AppearancePatentTaskService {
|
||||
saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits - 1, "正在上传结果文件");
|
||||
fileResultMapper.updateById(result);
|
||||
boolean taskAlreadyFailed = STATUS_FAILED.equals(task.getStatus())
|
||||
|| (task.getErrorMessage() != null && !task.getErrorMessage().isBlank())
|
||||
|| hasInfringingPersistedConclusion(task.getId());
|
||||
|| (task.getErrorMessage() != null && !task.getErrorMessage().isBlank());
|
||||
String existingError = task.getErrorMessage();
|
||||
task.setStatus(taskAlreadyFailed ? STATUS_FAILED : STATUS_SUCCESS);
|
||||
task.setStatus(resolveTaskExecutionStatus(false, taskAlreadyFailed));
|
||||
task.setSuccessFileCount(taskAlreadyFailed ? 0 : 1);
|
||||
task.setFailedFileCount(taskAlreadyFailed ? 1 : 0);
|
||||
task.setErrorMessage(taskAlreadyFailed ? existingError : null);
|
||||
@@ -4199,33 +4197,11 @@ public class AppearancePatentTaskService {
|
||||
return resultRow == null || resultRow.getPrice() == null ? "" : resultRow.getPrice().trim();
|
||||
}
|
||||
|
||||
private boolean hasInfringingPersistedConclusion(Long taskId) {
|
||||
if (taskId == null) {
|
||||
return false;
|
||||
static String resolveTaskExecutionStatus(boolean waitingForAssemble, boolean executionFailed) {
|
||||
if (waitingForAssemble) {
|
||||
return STATUS_RUNNING;
|
||||
}
|
||||
return loadPersistedResultRows(taskId).values().stream()
|
||||
.map(AppearancePatentResultRowDto::getConclusion)
|
||||
.anyMatch(this::isInfringingConclusion);
|
||||
}
|
||||
|
||||
private boolean isInfringingConclusion(String value) {
|
||||
String normalized = normalize(value);
|
||||
return normalized.contains("侵权")
|
||||
&& !isNoInfringementConclusion(normalized);
|
||||
}
|
||||
|
||||
private boolean isNoInfringementConclusion(String value) {
|
||||
String normalized = normalize(value);
|
||||
if (normalized.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
return normalized.contains("无侵权")
|
||||
|| normalized.contains("未侵权")
|
||||
|| normalized.contains("不侵权")
|
||||
|| normalized.contains("未发现") && normalized.contains("侵权")
|
||||
|| normalized.contains("未见") && normalized.contains("侵权")
|
||||
|| normalized.contains("不存在") && normalized.contains("侵权")
|
||||
|| normalized.contains("无明显") && normalized.contains("侵权");
|
||||
return executionFailed ? STATUS_FAILED : STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
private String userFacingCozeCellValue(AppearancePatentResultRowDto row, String value) {
|
||||
|
||||
+22
-3
@@ -1,11 +1,13 @@
|
||||
package com.nanri.aiimage.modules.auth.util;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.bouncycastle.crypto.generators.SCrypt;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.crypto.SecretKeyFactory;
|
||||
import javax.crypto.spec.PBEKeySpec;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.HexFormat;
|
||||
|
||||
@@ -51,21 +53,38 @@ public class WerkzeugPasswordEncoder {
|
||||
String method = storedHash.substring(0, firstSep);
|
||||
String salt = storedHash.substring(firstSep + 1, secondSep);
|
||||
String expectedHex = storedHash.substring(secondSep + 1);
|
||||
if (!method.startsWith("pbkdf2:sha256")) {
|
||||
log.warn("[auth] unsupported password hash scheme: {}", method);
|
||||
return false;
|
||||
if (method.startsWith("scrypt:")) {
|
||||
return matchesScrypt(rawPassword, method, salt, expectedHex);
|
||||
}
|
||||
if (method.startsWith("pbkdf2:sha256")) {
|
||||
String[] parts = method.split(":");
|
||||
int iterations = parts.length >= 3 ? Integer.parseInt(parts[2]) : 600_000;
|
||||
byte[] derived = pbkdf2Sha256(rawPassword.toCharArray(), salt.getBytes(StandardCharsets.UTF_8), iterations, DK_BITS);
|
||||
String actualHex = HexFormat.of().formatHex(derived);
|
||||
return constantTimeEquals(actualHex, expectedHex);
|
||||
}
|
||||
log.warn("[auth] unsupported password hash scheme: {}", method);
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
log.warn("[auth] verify password failed: {}", e.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean matchesScrypt(String rawPassword, String method, String salt, String expectedHex) {
|
||||
String[] parts = method.split(":");
|
||||
if (parts.length != 4 || expectedHex.isEmpty() || (expectedHex.length() & 1) != 0) {
|
||||
return false;
|
||||
}
|
||||
int cost = Integer.parseInt(parts[1]);
|
||||
int blockSize = Integer.parseInt(parts[2]);
|
||||
int parallelization = Integer.parseInt(parts[3]);
|
||||
byte[] expected = HexFormat.of().parseHex(expectedHex);
|
||||
byte[] actual = SCrypt.generate(rawPassword.getBytes(StandardCharsets.UTF_8),
|
||||
salt.getBytes(StandardCharsets.UTF_8), cost, blockSize, parallelization, expected.length);
|
||||
return MessageDigest.isEqual(actual, expected);
|
||||
}
|
||||
|
||||
private static byte[] pbkdf2Sha256(char[] password, byte[] salt, int iterations, int keyBits) throws Exception {
|
||||
PBEKeySpec spec = new PBEKeySpec(password, salt, iterations, keyBits);
|
||||
try {
|
||||
|
||||
+13
@@ -53,6 +53,19 @@ public class CollectDataController {
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/fail")
|
||||
@Operation(summary = "标记采集任务失败", description = "桌面端入队失败或无法继续执行时,将任务收敛为 FAILED。")
|
||||
public ApiResponse<Void> fail(
|
||||
@Parameter(description = "采集任务 ID", required = true, example = "9001")
|
||||
@PathVariable Long taskId,
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId,
|
||||
@Parameter(description = "失败原因")
|
||||
@RequestParam(value = "error", required = false) String error) {
|
||||
service.failTask(taskId, userId, error);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@GetMapping("/tasks/{taskId}/items")
|
||||
@Operation(summary = "分页获取任务明细数据", description = "供 Python 端拉取,默认每页 50 条;返回任务关联的筛选条件,便于 Python 端按筛选条件采集。")
|
||||
public ApiResponse<CollectDataItemsPageVo> items(
|
||||
|
||||
+3
@@ -51,4 +51,7 @@ public class CollectDataSummaryRowDto {
|
||||
@JsonAlias({"page", "totalPage", "页数", "max_page", "maxPage"})
|
||||
@Schema(description = "该关键词总页数(Python 端取所有命中行的最大页码)", example = "10")
|
||||
private Integer totalPage;
|
||||
|
||||
@Schema(description = "Python 端 ASIN 过滤数量", example = "3")
|
||||
private Integer asinFilter;
|
||||
}
|
||||
|
||||
+30
@@ -49,6 +49,36 @@ public class CollectDataHistoryItemVo {
|
||||
@Schema(description = "最终结果行数")
|
||||
private Integer finalRowCount;
|
||||
|
||||
@Schema(description = "解析任务总行数")
|
||||
private Integer totalRows;
|
||||
|
||||
@Schema(description = "Python 已回传行数")
|
||||
private Integer receivedRows;
|
||||
|
||||
@Schema(description = "Python 已处理关键词数")
|
||||
private Integer processedRows;
|
||||
|
||||
@Schema(description = "当前采集阶段:search / detail")
|
||||
private String collectStage;
|
||||
|
||||
@Schema(description = "当前关键词")
|
||||
private String currentKeyword;
|
||||
|
||||
@Schema(description = "搜索页当前页码")
|
||||
private Integer searchCurrentPage;
|
||||
|
||||
@Schema(description = "搜索页总页数")
|
||||
private Integer searchTotalPages;
|
||||
|
||||
@Schema(description = "详情页已处理 ASIN 数")
|
||||
private Integer detailProcessedAsins;
|
||||
|
||||
@Schema(description = "详情页 ASIN 总数")
|
||||
private Integer detailTotalAsins;
|
||||
|
||||
@Schema(description = "任务进度百分比,0-100")
|
||||
private Integer progressPercent;
|
||||
|
||||
@Schema(description = "记录创建时间")
|
||||
private String createdAt;
|
||||
|
||||
|
||||
+2
-1
@@ -24,7 +24,7 @@ public class CollectDataExcelAssemblyService {
|
||||
private static final String[] SHEET_DETAIL_HEADER = {"品牌", "ASIN", "价格", "卖家名称", "关键词", "配送方式"};
|
||||
|
||||
private static final String SHEET_SUMMARY_NAME = "结果文件";
|
||||
private static final String[] SHEET_SUMMARY_HEADER = {"关键词", "FBA", "FBM", "AMZ", "无配送方式", "页数"};
|
||||
private static final String[] SHEET_SUMMARY_HEADER = {"关键词", "FBA", "FBM", "AMZ", "无配送方式", "页数", "ASIN过滤"};
|
||||
|
||||
/**
|
||||
* 生成采集结果工作簿。
|
||||
@@ -115,6 +115,7 @@ public class CollectDataExcelAssemblyService {
|
||||
} else {
|
||||
row.createCell(5).setCellValue("");
|
||||
}
|
||||
row.createCell(6).setCellValue(intOrZero(item.getAsinFilter()));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
+212
-2
@@ -43,6 +43,7 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
@@ -60,6 +61,8 @@ import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
@@ -105,6 +108,7 @@ public class CollectDataService {
|
||||
private static final int BRAND_CHECK_BATCH_SIZE = 10;
|
||||
private static final long TASK_LOCK_WAIT_MILLIS = 5000L;
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final String STALE_TASK_ERROR = "长时间未收到 Python 心跳,任务已自动失败";
|
||||
|
||||
private static final List<String> KEYWORD_HEADER_ALIASES = List.of("关键词", "keyword", "key word");
|
||||
private static final List<String> STATUS_HEADER_ALIASES = List.of("状态", "status");
|
||||
@@ -131,6 +135,9 @@ public class CollectDataService {
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TransactionTemplate transactionTemplate;
|
||||
|
||||
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
|
||||
private long staleTimeoutMinutes;
|
||||
|
||||
public CollectDataParseVo parseAndCreateTask(CollectDataParseRequest request) {
|
||||
long startedAt = System.currentTimeMillis();
|
||||
if (request == null || request.getUserId() == null || request.getUserId() <= 0) {
|
||||
@@ -216,7 +223,18 @@ public class CollectDataService {
|
||||
requestPayload.put("filters", filters);
|
||||
requestPayload.put("files", sources);
|
||||
task.setRequestJson(objectMapper.writeValueAsString(requestPayload));
|
||||
task.setResultJson("{}");
|
||||
Map<String, Object> initialStats = new LinkedHashMap<>();
|
||||
initialStats.put("totalRows", parsedRows.size());
|
||||
initialStats.put("receivedRows", 0);
|
||||
initialStats.put("processedRows", 0);
|
||||
initialStats.put("currentChunkRows", 0);
|
||||
initialStats.put("dedupeFilteredCount", 0);
|
||||
initialStats.put("invalidFilteredCount", 0);
|
||||
initialStats.put("brandRejectedCount", 0);
|
||||
initialStats.put("brandQueryFailedCount", 0);
|
||||
initialStats.put("finalRowCount", 0);
|
||||
initialStats.put("summaries", List.of());
|
||||
task.setResultJson(objectMapper.writeValueAsString(initialStats));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("序列化任务信息失败");
|
||||
}
|
||||
@@ -274,9 +292,138 @@ public class CollectDataService {
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void failTask(Long taskId, Long userId, String error) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
|
||||
return;
|
||||
}
|
||||
String message = firstNonBlank(error, "collect-data task dispatch failed");
|
||||
FileResultEntity result = ensureTaskResult(task);
|
||||
CollectDataStats stats = loadStats(task);
|
||||
result.setSuccess(0);
|
||||
result.setErrorMessage(message);
|
||||
result.setRowCount(stats.finalRowCount);
|
||||
fileResultMapper.updateById(result);
|
||||
task.setStatus(STATUS_FAILED);
|
||||
task.setErrorMessage(message);
|
||||
task.setFailedFileCount(1);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
persistStats(task, stats);
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void updateProgress(Long taskId, TaskHeartbeatRequest request) {
|
||||
if (taskId == null || taskId <= 0 || request == null) {
|
||||
return;
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
|
||||
return;
|
||||
}
|
||||
CollectDataStats stats = loadStats(task);
|
||||
boolean changed = false;
|
||||
Integer current = request.getCurrent();
|
||||
Integer total = request.getTotal();
|
||||
if (current != null && total != null && total > 0) {
|
||||
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
|
||||
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
|
||||
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
|
||||
stats.totalRows = totalRows;
|
||||
stats.processedRows = processedRows;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
|
||||
stats.collectStage = request.getCollectStage();
|
||||
changed = true;
|
||||
}
|
||||
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
|
||||
stats.currentKeyword = request.getCurrentKeyword();
|
||||
changed = true;
|
||||
}
|
||||
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
|
||||
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
|
||||
changed = true;
|
||||
}
|
||||
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
|
||||
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
|
||||
changed = true;
|
||||
}
|
||||
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
|
||||
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
|
||||
changed = true;
|
||||
}
|
||||
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
|
||||
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
|
||||
changed = true;
|
||||
}
|
||||
if (!changed) {
|
||||
return;
|
||||
}
|
||||
persistStats(task, stats);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(cron = "${aiimage.collect-data.stale-check-cron:*/30 * * * * *}")
|
||||
public void finalizeStaleTasks() {
|
||||
long timeoutMinutes = Math.max(1L, staleTimeoutMinutes);
|
||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(timeoutMinutes);
|
||||
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||
.last("limit 200"));
|
||||
for (FileTaskEntity task : tasks) {
|
||||
finalizeStaleTask(task.getId(), threshold, timeoutMinutes);
|
||||
}
|
||||
}
|
||||
|
||||
private void finalizeStaleTask(Long taskId, LocalDateTime threshold, long timeoutMinutes) {
|
||||
try (TaskDistributedLockService.LockHandle lock =
|
||||
taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) {
|
||||
if (lock == null) {
|
||||
return;
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null
|
||||
|| !MODULE_TYPE.equals(task.getModuleType())
|
||||
|| !STATUS_RUNNING.equals(task.getStatus())
|
||||
|| task.getUpdatedAt() == null
|
||||
|| !task.getUpdatedAt().isBefore(threshold)) {
|
||||
return;
|
||||
}
|
||||
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
|
||||
return;
|
||||
}
|
||||
|
||||
FileResultEntity result = ensureTaskResult(task);
|
||||
CollectDataStats stats = loadStats(task);
|
||||
if (hasReceivedChunks(taskId)) {
|
||||
enqueueFinalWorkbook(task, result, stats);
|
||||
log.warn("[collect-data] stale task enqueued partial workbook taskId={} timeoutMinutes={} finalRows={}",
|
||||
taskId, timeoutMinutes, stats.finalRowCount);
|
||||
return;
|
||||
}
|
||||
|
||||
markTaskFailed(task, result, STALE_TASK_ERROR, stats);
|
||||
log.warn("[collect-data] stale task failed without result chunks taskId={} timeoutMinutes={}",
|
||||
taskId, timeoutMinutes);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] stale task finalization failed taskId={} msg={}",
|
||||
taskId, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
public CollectDataDashboardVo dashboard(Long userId) {
|
||||
CollectDataDashboardVo vo = new CollectDataDashboardVo();
|
||||
vo.setPendingTaskCount(countTask(userId, STATUS_RUNNING));
|
||||
vo.setPendingTaskCount(countActiveTasks(userId));
|
||||
vo.setSuccessTaskCount(countTask(userId, STATUS_SUCCESS));
|
||||
vo.setFailedTaskCount(countTask(userId, STATUS_FAILED));
|
||||
long processed = (vo.getSuccessTaskCount() == null ? 0 : vo.getSuccessTaskCount())
|
||||
@@ -967,6 +1114,13 @@ public class CollectDataService {
|
||||
return count == null ? 0 : count.intValue();
|
||||
}
|
||||
|
||||
private boolean hasReceivedChunks(Long taskId) {
|
||||
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
return count != null && count > 0L;
|
||||
}
|
||||
|
||||
private CollectDataSubmitResultVo buildSubmitVo(FileTaskEntity task,
|
||||
FileResultEntity result,
|
||||
int chunkIndex,
|
||||
@@ -999,7 +1153,15 @@ public class CollectDataService {
|
||||
}
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(task.getResultJson());
|
||||
stats.totalRows = root.path("totalRows").asInt(0);
|
||||
stats.receivedRows = root.path("receivedRows").asInt(0);
|
||||
stats.processedRows = root.path("processedRows").asInt(0);
|
||||
stats.collectStage = root.path("collectStage").asText(null);
|
||||
stats.currentKeyword = root.path("currentKeyword").asText(null);
|
||||
stats.searchCurrentPage = root.path("searchCurrentPage").asInt(0);
|
||||
stats.searchTotalPages = root.path("searchTotalPages").asInt(0);
|
||||
stats.detailProcessedAsins = root.path("detailProcessedAsins").asInt(0);
|
||||
stats.detailTotalAsins = root.path("detailTotalAsins").asInt(0);
|
||||
stats.currentChunkRows = root.path("currentChunkRows").asInt(0);
|
||||
stats.dedupeFilteredCount = root.path("dedupeFilteredCount").asInt(0);
|
||||
stats.invalidFilteredCount = root.path("invalidFilteredCount").asInt(0);
|
||||
@@ -1028,7 +1190,15 @@ public class CollectDataService {
|
||||
}
|
||||
try {
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put("totalRows", stats.totalRows);
|
||||
payload.put("receivedRows", stats.receivedRows);
|
||||
payload.put("processedRows", stats.processedRows);
|
||||
payload.put("collectStage", stats.collectStage);
|
||||
payload.put("currentKeyword", stats.currentKeyword);
|
||||
payload.put("searchCurrentPage", stats.searchCurrentPage);
|
||||
payload.put("searchTotalPages", stats.searchTotalPages);
|
||||
payload.put("detailProcessedAsins", stats.detailProcessedAsins);
|
||||
payload.put("detailTotalAsins", stats.detailTotalAsins);
|
||||
payload.put("currentChunkRows", stats.currentChunkRows);
|
||||
payload.put("dedupeFilteredCount", stats.dedupeFilteredCount);
|
||||
payload.put("invalidFilteredCount", stats.invalidFilteredCount);
|
||||
@@ -1142,7 +1312,15 @@ public class CollectDataService {
|
||||
}
|
||||
|
||||
private static class CollectDataStats {
|
||||
private int totalRows;
|
||||
private int receivedRows;
|
||||
private int processedRows;
|
||||
private String collectStage;
|
||||
private String currentKeyword;
|
||||
private int searchCurrentPage;
|
||||
private int searchTotalPages;
|
||||
private int detailProcessedAsins;
|
||||
private int detailTotalAsins;
|
||||
private int currentChunkRows;
|
||||
private int dedupeFilteredCount;
|
||||
private int invalidFilteredCount;
|
||||
@@ -1316,6 +1494,17 @@ public class CollectDataService {
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
private long countActiveTasks(Long userId) {
|
||||
if (userId == null || userId <= 0) {
|
||||
return 0L;
|
||||
}
|
||||
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getUserId, userId)
|
||||
.in(FileTaskEntity::getStatus, List.of(STATUS_PENDING, STATUS_RUNNING)));
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
private Map<Long, FileTaskEntity> loadTaskMap(List<Long> taskIds) {
|
||||
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
@@ -1356,10 +1545,31 @@ public class CollectDataService {
|
||||
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
|
||||
vo.setBrandRejectedCount(stats.brandRejectedCount);
|
||||
vo.setFinalRowCount(stats.finalRowCount);
|
||||
vo.setTotalRows(stats.totalRows);
|
||||
vo.setReceivedRows(stats.receivedRows);
|
||||
vo.setProcessedRows(stats.processedRows);
|
||||
vo.setCollectStage(stats.collectStage);
|
||||
vo.setCurrentKeyword(stats.currentKeyword);
|
||||
vo.setSearchCurrentPage(stats.searchCurrentPage);
|
||||
vo.setSearchTotalPages(stats.searchTotalPages);
|
||||
vo.setDetailProcessedAsins(stats.detailProcessedAsins);
|
||||
vo.setDetailTotalAsins(stats.detailTotalAsins);
|
||||
vo.setProgressPercent(calculateProgressPercent(task.getStatus(), stats.totalRows, stats.processedRows));
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
private int calculateProgressPercent(String status, int totalRows, int receivedRows) {
|
||||
if (STATUS_SUCCESS.equals(status)) {
|
||||
return 100;
|
||||
}
|
||||
if (totalRows <= 0) {
|
||||
return 0;
|
||||
}
|
||||
int received = Math.max(0, Math.min(receivedRows, totalRows));
|
||||
return Math.min(99, received * 100 / totalRows);
|
||||
}
|
||||
|
||||
private String resolveDisplayResultFilename(FileResultEntity row, FileTaskEntity task) {
|
||||
if (row == null) {
|
||||
return null;
|
||||
|
||||
+6
-1
@@ -57,8 +57,13 @@ public class DedupeTotalDataController {
|
||||
@Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize,
|
||||
@Parameter(description = "数据值模糊搜索关键字") @RequestParam(required = false) String keyword,
|
||||
@Parameter(description = "用户名模糊搜索关键字") @RequestParam(required = false) String username,
|
||||
@Parameter(description = "开始日期(包含)")
|
||||
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
|
||||
@Parameter(description = "结束日期(包含)")
|
||||
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
|
||||
@Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId) {
|
||||
return ApiResponse.success(dedupeTotalDataService.page(page, pageSize, keyword, username, operatorId));
|
||||
return ApiResponse.success(dedupeTotalDataService.page(
|
||||
page, pageSize, keyword, username, startDate, endDate, operatorId));
|
||||
}
|
||||
|
||||
@GetMapping("/export")
|
||||
|
||||
+9
-1
@@ -74,7 +74,11 @@ public class DedupeTotalDataService {
|
||||
return template;
|
||||
}
|
||||
|
||||
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username, Long operatorId) {
|
||||
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username,
|
||||
LocalDate startDate, LocalDate endDate, Long operatorId) {
|
||||
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
|
||||
throw new BusinessException("开始日期不能晚于结束日期");
|
||||
}
|
||||
long safePage = Math.max(page, 1);
|
||||
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
|
||||
String safeKeyword = keyword == null ? "" : keyword.trim();
|
||||
@@ -84,6 +88,10 @@ public class DedupeTotalDataService {
|
||||
.like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
|
||||
.in(!scope.allUsers(), DedupeTotalDataEntity::getUploaderUserId, scope.userIds())
|
||||
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
|
||||
.ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
|
||||
startDate == null ? null : startDate.atStartOfDay())
|
||||
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
|
||||
endDate == null ? null : endDate.plusDays(1).atStartOfDay())
|
||||
.orderByDesc(DedupeTotalDataEntity::getId);
|
||||
Long total = dedupeTotalDataMapper.selectCount(query);
|
||||
List<DedupeTotalDataItemVo> items = dedupeTotalDataMapper.selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize))
|
||||
|
||||
+3
-1
@@ -25,7 +25,9 @@ public class LocalTempCleanupService {
|
||||
"convert-result",
|
||||
"split-result",
|
||||
"brand-source-download",
|
||||
"brand-result"
|
||||
"brand-result",
|
||||
"similar-asin-result",
|
||||
"similar-asin-image-cache"
|
||||
);
|
||||
|
||||
private final StorageProperties storageProperties;
|
||||
|
||||
+90
-1
@@ -1,7 +1,9 @@
|
||||
package com.nanri.aiimage.modules.file.service.oss;
|
||||
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import io.minio.BucketExistsArgs;
|
||||
import io.minio.GetObjectArgs;
|
||||
import io.minio.MakeBucketArgs;
|
||||
import io.minio.MinioClient;
|
||||
import io.minio.PutObjectArgs;
|
||||
import io.minio.RemoveObjectArgs;
|
||||
@@ -93,6 +95,77 @@ public class OssStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
public void ensureBucketExists(String bucket) {
|
||||
String normalizedBucket = requireStorageName(bucket, "bucket");
|
||||
try {
|
||||
boolean exists = buildClient().bucketExists(BucketExistsArgs.builder()
|
||||
.bucket(normalizedBucket)
|
||||
.build());
|
||||
if (exists) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
buildClient().makeBucket(MakeBucketArgs.builder()
|
||||
.bucket(normalizedBucket)
|
||||
.build());
|
||||
} catch (ErrorResponseException ex) {
|
||||
if (!isBucketAlreadyPresent(ex)) {
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw storageFailure("ensure bucket", normalizedBucket, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public void uploadBytes(String bucket, String objectKey, byte[] content, String contentType) {
|
||||
String normalizedBucket = requireStorageName(bucket, "bucket");
|
||||
String normalizedObjectKey = requireStorageName(objectKey, "objectKey");
|
||||
byte[] bytes = content == null ? new byte[0] : content;
|
||||
try (ByteArrayInputStream stream = new ByteArrayInputStream(bytes)) {
|
||||
buildClient().putObject(PutObjectArgs.builder()
|
||||
.bucket(normalizedBucket)
|
||||
.object(normalizedObjectKey)
|
||||
.stream(stream, bytes.length, -1)
|
||||
.contentType(firstNonBlank(contentType, "application/octet-stream"))
|
||||
.build());
|
||||
} catch (Exception ex) {
|
||||
throw storageFailure("upload", normalizedBucket + "/" + normalizedObjectKey, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public byte[] readObjectBytes(String bucket, String objectKey) {
|
||||
String normalizedBucket = requireStorageName(bucket, "bucket");
|
||||
String normalizedObjectKey = requireStorageName(objectKey, "objectKey");
|
||||
try (var stream = buildClient().getObject(GetObjectArgs.builder()
|
||||
.bucket(normalizedBucket)
|
||||
.object(normalizedObjectKey)
|
||||
.build())) {
|
||||
return stream.readAllBytes();
|
||||
} catch (Exception ex) {
|
||||
throw storageFailure("read", normalizedBucket + "/" + normalizedObjectKey, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public boolean objectExists(String bucket, String objectKey) {
|
||||
String normalizedBucket = requireStorageName(bucket, "bucket");
|
||||
String normalizedObjectKey = requireStorageName(objectKey, "objectKey");
|
||||
try {
|
||||
buildClient().statObject(StatObjectArgs.builder()
|
||||
.bucket(normalizedBucket)
|
||||
.object(normalizedObjectKey)
|
||||
.build());
|
||||
return true;
|
||||
} catch (ErrorResponseException ex) {
|
||||
if (isNotFound(ex)) {
|
||||
return false;
|
||||
}
|
||||
throw storageFailure("stat", normalizedBucket + "/" + normalizedObjectKey, ex);
|
||||
} catch (Exception ex) {
|
||||
throw storageFailure("stat", normalizedBucket + "/" + normalizedObjectKey, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public String uploadTaskScopePayload(String moduleType, Long taskId, String scopeHash, String content) {
|
||||
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
|
||||
String objectKey = String.format("task-scope/%s/%s/%s.json", normalizeModuleType(moduleType), taskId, normalizedScopeHash);
|
||||
@@ -310,7 +383,7 @@ public class OssStorageService {
|
||||
}
|
||||
|
||||
private List<String> configuredBuckets() {
|
||||
return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket())
|
||||
return Stream.of(ossProperties.getBucket(), imageVideoBucket(), digitalHumanBucket(), templateBucket())
|
||||
.filter(Objects::nonNull)
|
||||
.map(String::trim)
|
||||
.filter(bucket -> !bucket.isBlank())
|
||||
@@ -395,6 +468,10 @@ public class OssStorageService {
|
||||
return firstNonBlank(ossProperties.getDigitalHumanBucket(), ossProperties.getBucket());
|
||||
}
|
||||
|
||||
private String templateBucket() {
|
||||
return firstNonBlank(ossProperties.getTemplateBucket(), ossProperties.getBucket());
|
||||
}
|
||||
|
||||
private String publicEndpoint() {
|
||||
return withScheme(firstNonBlank(ossProperties.getPublicEndpoint(), ossProperties.getEndpoint()));
|
||||
}
|
||||
@@ -446,6 +523,11 @@ public class OssStorageService {
|
||||
return "NoSuchKey".equals(code) || "NoSuchObject".equals(code) || "NoSuchBucket".equals(code);
|
||||
}
|
||||
|
||||
private boolean isBucketAlreadyPresent(ErrorResponseException ex) {
|
||||
String code = ex.errorResponse() == null ? null : ex.errorResponse().code();
|
||||
return "BucketAlreadyOwnedByYou".equals(code) || "BucketAlreadyExists".equals(code);
|
||||
}
|
||||
|
||||
private IllegalStateException storageFailure(String operation, String objectKey, Exception cause) {
|
||||
return new IllegalStateException("failed to " + operation + " object in MinIO: " + objectKey, cause);
|
||||
}
|
||||
@@ -477,6 +559,13 @@ public class OssStorageService {
|
||||
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||
}
|
||||
|
||||
private String requireStorageName(String value, String field) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException(field + " must not be blank");
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private record StorageLocation(String bucket, String objectKey) {
|
||||
}
|
||||
|
||||
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package com.nanri.aiimage.modules.filetemplate;
|
||||
|
||||
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
|
||||
import com.nanri.aiimage.modules.filetemplate.ModuleTemplateService.TemplateDownload;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.media.Content;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponses;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/module-templates")
|
||||
@Tag(name = "模块输入模板", description = "下载各业务模块的固定 Excel 输入模板")
|
||||
public class ModuleTemplateController {
|
||||
|
||||
private final ModuleTemplateService moduleTemplateService;
|
||||
|
||||
@GetMapping("/{moduleCode}/download")
|
||||
@Operation(summary = "下载模块输入模板")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "Excel 模板文件",
|
||||
content = @Content(
|
||||
mediaType = ModuleTemplateService.XLSX_CONTENT_TYPE,
|
||||
schema = @Schema(type = "string", format = "binary"))),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "模块模板不存在"),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "503", description = "模板存储暂不可用")
|
||||
})
|
||||
public ResponseEntity<byte[]> download(
|
||||
@Parameter(description = "模块编码", required = true, example = "publish")
|
||||
@PathVariable String moduleCode) {
|
||||
TemplateDownload download = moduleTemplateService.download(moduleCode);
|
||||
return ResponseEntity.ok()
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(download.filename()))
|
||||
.contentType(MediaType.parseMediaType(download.contentType()))
|
||||
.contentLength(download.content().length)
|
||||
.body(download.content());
|
||||
}
|
||||
}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
package com.nanri.aiimage.modules.filetemplate;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Component
|
||||
public class ModuleTemplateRegistry {
|
||||
|
||||
private static final String RESOURCE_PREFIX = "templates/module-input/";
|
||||
private static final String OBJECT_PREFIX = "input/";
|
||||
|
||||
private static final List<ModuleTemplate> TEMPLATES = List.of(
|
||||
template("publish", "publish.xlsx", "上架 文档格式.xlsx"),
|
||||
template("delete-brand", "delete-brand.xlsx", "删除指定待售 文档格式.xlsx"),
|
||||
template("appearance-patent", "appearance-patent.xlsx", "外观专利检测 文档格式.xlsx"),
|
||||
template("price-track", "price-track.xlsx", "指定ASIN跟价 文档格式.xlsx"),
|
||||
template("collect-data", "collect-data.xlsx", "数据采集 文档格式.xlsx"),
|
||||
template("similar-asin", "similar-asin.xlsx", "货源查询 文档格式.xlsx")
|
||||
);
|
||||
|
||||
private final Map<String, ModuleTemplate> templatesByCode = TEMPLATES.stream()
|
||||
.collect(Collectors.toUnmodifiableMap(
|
||||
ModuleTemplate::moduleCode,
|
||||
Function.identity()));
|
||||
|
||||
public Optional<ModuleTemplate> find(String moduleCode) {
|
||||
if (moduleCode == null || moduleCode.isBlank()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.ofNullable(templatesByCode.get(moduleCode.trim().toLowerCase(Locale.ROOT)));
|
||||
}
|
||||
|
||||
public List<ModuleTemplate> templates() {
|
||||
return TEMPLATES;
|
||||
}
|
||||
|
||||
private static ModuleTemplate template(String moduleCode, String resourceFilename, String downloadFilename) {
|
||||
return new ModuleTemplate(
|
||||
moduleCode,
|
||||
RESOURCE_PREFIX + resourceFilename,
|
||||
OBJECT_PREFIX + resourceFilename,
|
||||
downloadFilename);
|
||||
}
|
||||
|
||||
public record ModuleTemplate(
|
||||
String moduleCode,
|
||||
String resourcePath,
|
||||
String objectKey,
|
||||
String downloadFilename) {
|
||||
}
|
||||
}
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
package com.nanri.aiimage.modules.filetemplate;
|
||||
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.filetemplate.ModuleTemplateRegistry.ModuleTemplate;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import java.io.InputStream;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ModuleTemplateService {
|
||||
|
||||
public static final String XLSX_CONTENT_TYPE =
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
|
||||
private final ModuleTemplateRegistry registry;
|
||||
private final OssProperties ossProperties;
|
||||
private final OssStorageService ossStorageService;
|
||||
|
||||
public TemplateDownload download(String moduleCode) {
|
||||
ModuleTemplate template = registry.find(moduleCode)
|
||||
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "模板不存在"));
|
||||
try {
|
||||
ensureStored(template);
|
||||
byte[] content = ossStorageService.readObjectBytes(templateBucket(), template.objectKey());
|
||||
return new TemplateDownload(template.downloadFilename(), XLSX_CONTENT_TYPE, content);
|
||||
} catch (ResponseStatusException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw storageUnavailable(ex);
|
||||
}
|
||||
}
|
||||
|
||||
public int synchronizeAll() {
|
||||
try {
|
||||
String bucket = templateBucket();
|
||||
ossStorageService.ensureBucketExists(bucket);
|
||||
int uploaded = 0;
|
||||
for (ModuleTemplate template : registry.templates()) {
|
||||
if (!ossStorageService.objectExists(bucket, template.objectKey())) {
|
||||
uploadResource(bucket, template);
|
||||
uploaded++;
|
||||
}
|
||||
}
|
||||
return uploaded;
|
||||
} catch (ResponseStatusException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw storageUnavailable(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureStored(ModuleTemplate template) {
|
||||
String bucket = templateBucket();
|
||||
ossStorageService.ensureBucketExists(bucket);
|
||||
if (!ossStorageService.objectExists(bucket, template.objectKey())) {
|
||||
uploadResource(bucket, template);
|
||||
}
|
||||
}
|
||||
|
||||
private void uploadResource(String bucket, ModuleTemplate template) {
|
||||
ClassPathResource resource = new ClassPathResource(template.resourcePath());
|
||||
try (InputStream input = resource.getInputStream()) {
|
||||
ossStorageService.uploadBytes(bucket, template.objectKey(), input.readAllBytes(), XLSX_CONTENT_TYPE);
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to load module template resource: " + template.resourcePath(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
private String templateBucket() {
|
||||
String bucket = ossProperties.getTemplateBucket();
|
||||
if (bucket == null || bucket.isBlank()) {
|
||||
throw new IllegalStateException("template bucket is not configured");
|
||||
}
|
||||
return bucket.trim();
|
||||
}
|
||||
|
||||
private ResponseStatusException storageUnavailable(Exception cause) {
|
||||
return new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "模板存储暂不可用", cause);
|
||||
}
|
||||
|
||||
public record TemplateDownload(String filename, String contentType, byte[] content) {
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.nanri.aiimage.modules.filetemplate;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class ModuleTemplateStorageInitializer {
|
||||
|
||||
private final ModuleTemplateService moduleTemplateService;
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void synchronize() {
|
||||
try {
|
||||
int uploaded = moduleTemplateService.synchronizeAll();
|
||||
log.info("[module-template] startup synchronization complete uploaded={}", uploaded);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[module-template] startup synchronization failed; downloads will retry lazily: {}",
|
||||
ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
+17
@@ -128,6 +128,23 @@ public class PermissionMenuController {
|
||||
return ApiResponse.success("视频任务权限已更新", grantedCount);
|
||||
}
|
||||
|
||||
@GetMapping("/shop-data-crawl-task-permissions")
|
||||
@Operation(summary = "查询店铺数据任务数据权限用户")
|
||||
public ApiResponse<List<ImageVideoDataPermissionUserVo>> listShopDataCrawlDataPermissionUsers(
|
||||
HttpServletRequest request) {
|
||||
return ApiResponse.success(permissionMenuService.listShopDataCrawlDataPermissionUsers(requireAdmin(request)));
|
||||
}
|
||||
|
||||
@PutMapping("/shop-data-crawl-task-permissions")
|
||||
@Operation(summary = "更新店铺数据任务数据权限用户")
|
||||
public ApiResponse<Integer> updateShopDataCrawlDataPermissionUsers(
|
||||
HttpServletRequest request,
|
||||
@RequestBody(required = false) ImageVideoDataPermissionUpdateRequest body) {
|
||||
int grantedCount = permissionMenuService.updateShopDataCrawlDataPermissionUsers(
|
||||
requireAdmin(request), body == null ? List.of() : body.getUserIds());
|
||||
return ApiResponse.success("店铺数据任务权限已更新", grantedCount);
|
||||
}
|
||||
|
||||
private AdminUserEntity requireAdmin(HttpServletRequest request) {
|
||||
try {
|
||||
return adminAuthSupport.requireAdmin(request);
|
||||
|
||||
+45
@@ -86,6 +86,7 @@ public class PermissionMenuSchemaInitializer {
|
||||
executeQuietly("UPDATE columns SET sort_order = id WHERE sort_order IS NULL OR sort_order = 0");
|
||||
executeQuietly("ALTER TABLE columns ADD UNIQUE KEY uk_menu_type_route_path (menu_type, route_path)");
|
||||
ensureDefaultAdminMenus();
|
||||
boolean shopDataCrawlDataPermissionCreated = ensureShopDataCrawlAdminDefaults();
|
||||
ensureDefaultAppMenus();
|
||||
ensureDefaultAppChildMenus();
|
||||
ensureInternalDataPermissions();
|
||||
@@ -98,6 +99,9 @@ public class PermissionMenuSchemaInitializer {
|
||||
FOREIGN KEY (column_id) REFERENCES columns(id) ON DELETE CASCADE
|
||||
)
|
||||
""");
|
||||
if (shopDataCrawlDataPermissionCreated) {
|
||||
grantInitialShopDataCrawlDataPermissions();
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureDefaultAdminMenus() {
|
||||
@@ -172,6 +176,38 @@ public class PermissionMenuSchemaInitializer {
|
||||
""");
|
||||
}
|
||||
|
||||
private boolean ensureShopDataCrawlAdminDefaults() {
|
||||
executeQuietly("""
|
||||
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
|
||||
SELECT '店铺数据任务管理', 'admin_shop_data_crawl_tasks', 'admin', 'shop-data-crawl-tasks', 82
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM columns WHERE column_key = 'admin_shop_data_crawl_tasks'
|
||||
)
|
||||
""");
|
||||
return executeUpdateQuietly("""
|
||||
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
|
||||
SELECT '店铺数据任务数据查看', 'admin_shop_data_crawl_task_data', 'internal', 'shop-data-crawl-task-data', 0
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM columns WHERE column_key = 'admin_shop_data_crawl_task_data'
|
||||
)
|
||||
""") > 0;
|
||||
}
|
||||
|
||||
private void grantInitialShopDataCrawlDataPermissions() {
|
||||
executeQuietly("""
|
||||
INSERT IGNORE INTO user_column_permission (user_id, column_id)
|
||||
SELECT old_perm.user_id, data_col.id
|
||||
FROM user_column_permission old_perm
|
||||
INNER JOIN columns old_col ON old_col.id = old_perm.column_id
|
||||
INNER JOIN columns data_col ON data_col.column_key = 'admin_shop_data_crawl_task_data'
|
||||
LEFT JOIN user_column_permission existing
|
||||
ON existing.user_id = old_perm.user_id
|
||||
AND existing.column_id = data_col.id
|
||||
WHERE old_col.column_key = 'admin_shop_data_crawl_tasks'
|
||||
AND existing.user_id IS NULL
|
||||
""");
|
||||
}
|
||||
|
||||
private void executeQuietly(String sql) {
|
||||
try {
|
||||
jdbcTemplate.execute(sql);
|
||||
@@ -180,6 +216,15 @@ public class PermissionMenuSchemaInitializer {
|
||||
}
|
||||
}
|
||||
|
||||
private int executeUpdateQuietly(String sql) {
|
||||
try {
|
||||
return jdbcTemplate.update(sql);
|
||||
} catch (Exception ex) {
|
||||
log.debug("[permission-menu] schema init skipped: {}", ex.getMessage());
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private record DefaultAdminMenu(String name, String columnKey, String routePath, int sortOrder) {
|
||||
}
|
||||
|
||||
|
||||
+103
-12
@@ -47,6 +47,7 @@ public class PermissionMenuService {
|
||||
public static final String MENU_TYPE_APP = "app";
|
||||
public static final String MENU_TYPE_ADMIN = "admin";
|
||||
private static final String IMAGE_VIDEO_DATA_PERMISSION_KEY = "admin_image_video_task_data";
|
||||
private static final String SHOP_DATA_CRAWL_DATA_PERMISSION_KEY = "admin_shop_data_crawl_task_data";
|
||||
|
||||
private final PermissionMenuMapper permissionMenuMapper;
|
||||
private final UserColumnPermissionMapper userColumnPermissionMapper;
|
||||
@@ -243,9 +244,8 @@ public class PermissionMenuService {
|
||||
.filter(user -> user.getId() != null && !isSuperAdmin(user))
|
||||
.collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left));
|
||||
if (requestedIds.stream().anyMatch(id -> !grantableUsers.containsKey(id))) {
|
||||
throw new BusinessException("包含不存在或不可授权的用户");
|
||||
throw new BusinessException("contains unknown or non-grantable user");
|
||||
}
|
||||
|
||||
userColumnPermissionMapper.deleteByMap(Map.of("column_id", dataPermission.getId()));
|
||||
for (Long userId : requestedIds) {
|
||||
UserColumnPermissionEntity grant = new UserColumnPermissionEntity();
|
||||
@@ -256,6 +256,68 @@ public class PermissionMenuService {
|
||||
return requestedIds.size();
|
||||
}
|
||||
|
||||
public List<ImageVideoDataPermissionUserVo> listShopDataCrawlDataPermissionUsers(AdminUserEntity operator) {
|
||||
ensureSuperAdminOperator(operator, "店铺数据任务");
|
||||
PermissionMenuEntity dataPermission = requireDataPermission(
|
||||
SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据");
|
||||
Set<Long> grantedUserIds = userColumnPermissionMapper.selectList(
|
||||
new LambdaQueryWrapper<UserColumnPermissionEntity>()
|
||||
.eq(UserColumnPermissionEntity::getColumnId, dataPermission.getId()))
|
||||
.stream()
|
||||
.map(UserColumnPermissionEntity::getUserId)
|
||||
.filter(id -> id != null && id > 0)
|
||||
.collect(Collectors.toSet());
|
||||
return adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>()
|
||||
.orderByAsc(AdminUserEntity::getUsername)
|
||||
.orderByAsc(AdminUserEntity::getId))
|
||||
.stream()
|
||||
.filter(user -> !isSuperAdmin(user))
|
||||
.map(user -> toImageVideoDataPermissionUserVo(user, grantedUserIds.contains(user.getId())))
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public int updateShopDataCrawlDataPermissionUsers(AdminUserEntity operator, List<Long> userIds) {
|
||||
ensureSuperAdminOperator(operator, "店铺数据任务");
|
||||
PermissionMenuEntity dataPermission = requireDataPermission(
|
||||
SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据");
|
||||
List<Long> requestedIds = normalizeColumnIds(userIds);
|
||||
List<AdminUserEntity> users = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>());
|
||||
Map<Long, AdminUserEntity> grantableUsers = users.stream()
|
||||
.filter(user -> user.getId() != null && !isSuperAdmin(user))
|
||||
.collect(Collectors.toMap(AdminUserEntity::getId, Function.identity(), (left, right) -> left));
|
||||
if (requestedIds.stream().anyMatch(id -> !grantableUsers.containsKey(id))) {
|
||||
throw new BusinessException("contains unknown or non-grantable user");
|
||||
}
|
||||
userColumnPermissionMapper.deleteByMap(Map.of("column_id", dataPermission.getId()));
|
||||
for (Long userId : requestedIds) {
|
||||
UserColumnPermissionEntity grant = new UserColumnPermissionEntity();
|
||||
grant.setUserId(userId);
|
||||
grant.setColumnId(dataPermission.getId());
|
||||
userColumnPermissionMapper.insert(grant);
|
||||
}
|
||||
return requestedIds.size();
|
||||
}
|
||||
|
||||
/** Verifies both the visible admin menu and its separately managed data grant. */
|
||||
public void requireShopDataCrawlTaskAccess(AdminUserEntity operator) {
|
||||
ensureAdminOperatorIfPresent(operator);
|
||||
if (operator == null || operator.getId() == null || operator.getId() <= 0) {
|
||||
throw new BusinessException(403, "需要管理员权限");
|
||||
}
|
||||
if (isSuperAdmin(operator)) {
|
||||
return;
|
||||
}
|
||||
PermissionMenuEntity taskMenu = requireDataPermission(
|
||||
"admin_shop_data_crawl_tasks", "店铺数据任务管理菜单");
|
||||
PermissionMenuEntity dataPermission = requireDataPermission(
|
||||
SHOP_DATA_CRAWL_DATA_PERMISSION_KEY, "店铺数据任务数据");
|
||||
if (!hasEffectiveColumnPermission(operator.getId(), taskMenu.getId())
|
||||
|| !hasEffectiveColumnPermission(operator.getId(), dataPermission.getId())) {
|
||||
throw new BusinessException(403, "无权查看店铺数据任务");
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void updateUserColumnPermissions(Long userId, UserColumnPermissionUpdateRequest request) {
|
||||
updateUserColumnPermissions(null, userId, request);
|
||||
@@ -304,20 +366,29 @@ public class PermissionMenuService {
|
||||
}
|
||||
}
|
||||
|
||||
PermissionMenuEntity imageVideoDataPermission = normalizedType == null
|
||||
? findImageVideoDataPermission()
|
||||
: null;
|
||||
Long protectedId = imageVideoDataPermission == null ? null : imageVideoDataPermission.getId();
|
||||
Set<Long> protectedIds = new LinkedHashSet<>();
|
||||
if (normalizedType == null) {
|
||||
PermissionMenuEntity imageVideoDataPermission = findImageVideoDataPermission();
|
||||
PermissionMenuEntity shopDataCrawlDataPermission =
|
||||
findDataPermission(SHOP_DATA_CRAWL_DATA_PERMISSION_KEY);
|
||||
if (imageVideoDataPermission != null && imageVideoDataPermission.getId() != null) {
|
||||
protectedIds.add(imageVideoDataPermission.getId());
|
||||
}
|
||||
if (shopDataCrawlDataPermission != null && shopDataCrawlDataPermission.getId() != null) {
|
||||
protectedIds.add(shopDataCrawlDataPermission.getId());
|
||||
}
|
||||
}
|
||||
List<Long> grantIds = requestedIds;
|
||||
if (normalizedType == null && protectedId != null) {
|
||||
if (!protectedIds.isEmpty()) {
|
||||
grantIds = requestedIds.stream()
|
||||
.filter(id -> !protectedId.equals(id))
|
||||
.filter(id -> !protectedIds.contains(id))
|
||||
.toList();
|
||||
}
|
||||
Set<Long> operatorEffectiveIds = ensureGrantable(operator, grantIds);
|
||||
LinkedHashSet<Long> finalGrantIds = new LinkedHashSet<>(grantIds);
|
||||
|
||||
if (normalizedType == null && protectedId != null) {
|
||||
if (!protectedIds.isEmpty()) {
|
||||
for (Long protectedId : protectedIds) {
|
||||
Long existingCount = userColumnPermissionMapper.selectCount(
|
||||
new LambdaQueryWrapper<UserColumnPermissionEntity>()
|
||||
.eq(UserColumnPermissionEntity::getUserId, userId)
|
||||
@@ -326,6 +397,7 @@ public class PermissionMenuService {
|
||||
finalGrantIds.add(protectedId);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (operatorEffectiveIds != null) {
|
||||
loadDirectColumnIds(userId).stream()
|
||||
.filter(id -> normalizedType == null || scopedMenuIds.contains(id))
|
||||
@@ -526,9 +598,7 @@ public class PermissionMenuService {
|
||||
}
|
||||
|
||||
private PermissionMenuEntity findImageVideoDataPermission() {
|
||||
return permissionMenuMapper.selectOne(new LambdaQueryWrapper<PermissionMenuEntity>()
|
||||
.eq(PermissionMenuEntity::getColumnKey, IMAGE_VIDEO_DATA_PERMISSION_KEY)
|
||||
.last("LIMIT 1"));
|
||||
return findDataPermission(IMAGE_VIDEO_DATA_PERMISSION_KEY);
|
||||
}
|
||||
|
||||
private PermissionMenuEntity requireImageVideoDataPermission() {
|
||||
@@ -546,6 +616,27 @@ public class PermissionMenuService {
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureSuperAdminOperator(AdminUserEntity operator, String resourceName) {
|
||||
ensureAdminOperatorIfPresent(operator);
|
||||
if (operator == null || !isSuperAdmin(operator)) {
|
||||
throw new BusinessException(403, "仅超级管理员可以配置" + resourceName + "权限");
|
||||
}
|
||||
}
|
||||
|
||||
private PermissionMenuEntity findDataPermission(String columnKey) {
|
||||
return permissionMenuMapper.selectOne(new LambdaQueryWrapper<PermissionMenuEntity>()
|
||||
.eq(PermissionMenuEntity::getColumnKey, columnKey)
|
||||
.last("LIMIT 1"));
|
||||
}
|
||||
|
||||
private PermissionMenuEntity requireDataPermission(String columnKey, String displayName) {
|
||||
PermissionMenuEntity permission = findDataPermission(columnKey);
|
||||
if (permission == null || permission.getId() == null) {
|
||||
throw new BusinessException(displayName + "权限尚未初始化");
|
||||
}
|
||||
return permission;
|
||||
}
|
||||
|
||||
private ImageVideoDataPermissionUserVo toImageVideoDataPermissionUserVo(AdminUserEntity user,
|
||||
boolean granted) {
|
||||
ImageVideoDataPermissionUserVo vo = new ImageVideoDataPermissionUserVo();
|
||||
|
||||
+2
-1
@@ -63,6 +63,7 @@ public class PriceTrackTaskService {
|
||||
private static final String MODULE_TYPE = "PRICE_TRACK";
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final String ASIN_ROWS_PAYLOAD_SCOPE = "price-track-asin-rows";
|
||||
private static final String NO_USABLE_ROWS_ERROR = "未收到有效跟价数据,未生成结果文件";
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
@@ -1907,7 +1908,7 @@ public class PriceTrackTaskService {
|
||||
if (payload != null && hasText(payload.getError())) {
|
||||
return payload.getError().trim();
|
||||
}
|
||||
return "no usable price-track rows received";
|
||||
return NO_USABLE_ROWS_ERROR;
|
||||
}
|
||||
private String firstNonBlank(String preferred, String fallback) {
|
||||
if (preferred != null && !preferred.isBlank()) {
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/publish")
|
||||
@Tag(name = "上架", description = "上架 Excel 多文件任务接口。前端创建批次并严格串行派发文件;Python 按页拉取数据、发送统一任务心跳并回传当前店铺完整结果。")
|
||||
@Tag(name = "上架", description = "上架 Excel 多文件任务接口。前端创建批次并严格串行派发文件;Python 按页拉取数据、通过结果分片回传进度和当前店铺完整结果,任务心跳用于保活。")
|
||||
public class PublishController {
|
||||
|
||||
private final PublishTaskService publishTaskService;
|
||||
|
||||
+2
-2
@@ -38,9 +38,9 @@ public class PublishFileVo {
|
||||
private Integer percent;
|
||||
@Schema(description = "与 percent 相同,供公共进度组件使用", example = "50")
|
||||
private Integer progressPercent;
|
||||
@Schema(description = "心跳上报的当前处理数量;未上报时使用 processedRows", example = "341")
|
||||
@Schema(description = "结果分片累计接收的当前处理数量;兼容心跳上报", example = "341")
|
||||
private Integer progressCurrent;
|
||||
@Schema(description = "心跳上报的总处理数量;未上报时使用 totalRows", example = "682")
|
||||
@Schema(description = "解析得到的总处理数量;兼容心跳上报", example = "682")
|
||||
private Integer progressTotal;
|
||||
@Schema(description = "文件失败原因的进度展示副本;无错误时为空")
|
||||
private String progressMessage;
|
||||
|
||||
+118
-15
@@ -345,7 +345,11 @@ public class PublishTaskService {
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getUpdatedAt, now));
|
||||
if (request == null || (request.getCurrent() == null && request.getTotal() == null)) {
|
||||
// Generic desktop heartbeats use 0/0 when they only need to keep the task alive.
|
||||
// Treat that as no progress update so result chunks cannot be reset to zero.
|
||||
if (request == null
|
||||
|| ((request.getCurrent() == null || request.getCurrent() <= 0)
|
||||
&& (request.getTotal() == null || request.getTotal() <= 0))) {
|
||||
return;
|
||||
}
|
||||
PublishFileEntity active = publishFileMapper.selectOne(new LambdaQueryWrapper<PublishFileEntity>()
|
||||
@@ -360,12 +364,12 @@ public class PublishTaskService {
|
||||
.eq(PublishFileEntity::getId, active.getId())
|
||||
.eq(PublishFileEntity::getStatus, STATUS_RUNNING)
|
||||
.set(PublishFileEntity::getUpdatedAt, now);
|
||||
if (request.getTotal() != null && request.getTotal() >= 0) {
|
||||
if (request.getTotal() != null && request.getTotal() > 0) {
|
||||
update.set(PublishFileEntity::getTotalRows, request.getTotal());
|
||||
}
|
||||
if (request.getCurrent() != null && request.getCurrent() >= 0) {
|
||||
if (request.getCurrent() != null && request.getCurrent() > 0) {
|
||||
int current = request.getCurrent();
|
||||
if (request.getTotal() != null && request.getTotal() >= 0) {
|
||||
if (request.getTotal() != null && request.getTotal() > 0) {
|
||||
current = Math.min(current, request.getTotal());
|
||||
}
|
||||
update.set(PublishFileEntity::getProcessedRows, current);
|
||||
@@ -432,6 +436,7 @@ public class PublishTaskService {
|
||||
throw new BusinessException("no successful publish files");
|
||||
}
|
||||
|
||||
TaskOptions options = readTaskOptions(task);
|
||||
List<PublishWorkbookService.WorkbookInput> inputs = new ArrayList<>();
|
||||
int rowCount = 0;
|
||||
for (PublishFileEntity file : successfulFiles) {
|
||||
@@ -443,7 +448,7 @@ public class PublishTaskService {
|
||||
List<PublishRowDto> rows = items.stream().map(this::toRowDto).toList();
|
||||
rowCount += rows.size();
|
||||
inputs.add(new PublishWorkbookService.WorkbookInput(
|
||||
file.getSourceFilename(), file.getShopName(), rows));
|
||||
file.getSourceFilename(), file.getShopName(), options.publishCountry(), rows));
|
||||
}
|
||||
|
||||
PublishWorkbookService.PackagedResult packaged = workbookService.packageTaskResult(
|
||||
@@ -737,6 +742,7 @@ public class PublishTaskService {
|
||||
} else {
|
||||
ResultChunkReceipt receipt = persistResultChunk(taskId, file, incoming, storedPayloads);
|
||||
if (!receipt.completed()) {
|
||||
updateReceivedProgress(taskId, file, receipt.receivedRowCount());
|
||||
file.setStatus(STATUS_RUNNING);
|
||||
file.setErrorMessage(null);
|
||||
file.setUpdatedAt(LocalDateTime.now());
|
||||
@@ -1025,9 +1031,11 @@ public class PublishTaskService {
|
||||
if (existing != null) {
|
||||
validateExistingChunk(existing, chunkTotal, payloadHash);
|
||||
int receivedChunkCount = countResultChunks(taskId, scopeHash);
|
||||
persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
|
||||
int receivedRowCount = resolveReceivedRowCount(taskId, scopeHash, scope, 0, false);
|
||||
persistResultScope(taskId, scopeKey, scopeHash, chunkTotal,
|
||||
receivedChunkCount, receivedRowCount);
|
||||
return new ResultChunkReceipt(scopeHash, chunkTotal,
|
||||
receivedChunkCount >= chunkTotal);
|
||||
receivedChunkCount >= chunkTotal, receivedRowCount);
|
||||
}
|
||||
|
||||
ensureRustfsPayloadStorageEnabled();
|
||||
@@ -1046,9 +1054,11 @@ public class PublishTaskService {
|
||||
chunk.setPayloadHash(payloadHash);
|
||||
chunk.setCreatedAt(LocalDateTime.now());
|
||||
chunk.setUpdatedAt(LocalDateTime.now());
|
||||
boolean inserted = false;
|
||||
try {
|
||||
taskChunkMapper.insert(chunk);
|
||||
storedPayloads.add(storedPayload);
|
||||
inserted = true;
|
||||
} catch (DuplicateKeyException ex) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||
TaskChunkEntity winner = findResultChunk(taskId, scopeHash, chunkIndex);
|
||||
@@ -1062,11 +1072,13 @@ public class PublishTaskService {
|
||||
}
|
||||
|
||||
int receivedChunkCount = countResultChunks(taskId, scopeHash);
|
||||
persistResultScope(taskId, scopeKey, scopeHash, chunkTotal, receivedChunkCount);
|
||||
log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} received={}",
|
||||
taskId, file.getId(), chunkIndex, chunkTotal, receivedChunkCount);
|
||||
int receivedRowCount = resolveReceivedRowCount(taskId, scopeHash, scope, rows.size(), inserted);
|
||||
persistResultScope(taskId, scopeKey, scopeHash, chunkTotal,
|
||||
receivedChunkCount, receivedRowCount);
|
||||
log.info("[publish] result chunk received taskId={} fileId={} chunk={}/{} receivedChunks={} receivedRows={}",
|
||||
taskId, file.getId(), chunkIndex, chunkTotal, receivedChunkCount, receivedRowCount);
|
||||
return new ResultChunkReceipt(scopeHash, chunkTotal,
|
||||
receivedChunkCount >= chunkTotal);
|
||||
receivedChunkCount >= chunkTotal, receivedRowCount);
|
||||
}
|
||||
|
||||
private void validateChunkMetadata(int chunkIndex, int chunkTotal) {
|
||||
@@ -1119,11 +1131,70 @@ public class PublishTaskService {
|
||||
return count == null ? 0 : count.intValue();
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the number of result rows received for a file without re-reading
|
||||
* every payload on every callback. New callbacks keep the count in the
|
||||
* existing scope state JSON; scopes created by older versions are repaired
|
||||
* once by counting their stored chunks.
|
||||
*/
|
||||
private int resolveReceivedRowCount(Long taskId,
|
||||
String scopeHash,
|
||||
TaskScopeStateEntity scope,
|
||||
int currentChunkRows,
|
||||
boolean inserted) {
|
||||
Integer persisted = readReceivedRowCount(scope);
|
||||
if (persisted != null) {
|
||||
long next = (long) persisted + (inserted ? Math.max(0, currentChunkRows) : 0);
|
||||
return next > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) Math.max(0, next);
|
||||
}
|
||||
return countReceivedResultRows(taskId, scopeHash);
|
||||
}
|
||||
|
||||
private Integer readReceivedRowCount(TaskScopeStateEntity scope) {
|
||||
if (scope == null || scope.getStateJson() == null || scope.getStateJson().isBlank()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(scope.getStateJson());
|
||||
JsonNode receivedRows = root == null ? null : root.get("receivedRows");
|
||||
if (receivedRows == null || !receivedRows.isIntegralNumber()) {
|
||||
return null;
|
||||
}
|
||||
return Math.max(0, receivedRows.asInt(0));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[publish] failed to read received row count from result scope taskId={} scopeHash={} msg={}",
|
||||
scope.getTaskId(), scope.getScopeHash(), safeMessage(ex));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private int countReceivedResultRows(Long taskId, String scopeHash) {
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||
.orderByAsc(TaskChunkEntity::getChunkIndex));
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
TypeReference<List<PublishRowDto>> listType = new TypeReference<>() {
|
||||
};
|
||||
long count = 0;
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
count += readResultChunkRows(chunk, listType).size();
|
||||
if (count >= Integer.MAX_VALUE) {
|
||||
return Integer.MAX_VALUE;
|
||||
}
|
||||
}
|
||||
return (int) count;
|
||||
}
|
||||
|
||||
private void persistResultScope(Long taskId,
|
||||
String scopeKey,
|
||||
String scopeHash,
|
||||
int chunkTotal,
|
||||
int receivedChunkCount) {
|
||||
int receivedChunkCount,
|
||||
int receivedRowCount) {
|
||||
TaskScopeStateEntity scope = findResultScope(taskId, scopeHash);
|
||||
validateChunkTotal(scope == null ? null : scope.getChunkTotal(), chunkTotal);
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
@@ -1141,7 +1212,7 @@ public class PublishTaskService {
|
||||
scope.setCompleted(completed ? 1 : 0);
|
||||
scope.setLastChunkAt(now);
|
||||
scope.setLastError(null);
|
||||
scope.setStateJson(completed ? "{\"phase\":\"COMPLETE\"}" : "{\"phase\":\"RECEIVING\"}");
|
||||
scope.setStateJson(resultScopeStateJson(completed, receivedRowCount));
|
||||
scope.setUpdatedAt(now);
|
||||
if (scope.getId() != null) {
|
||||
taskScopeStateMapper.updateById(scope);
|
||||
@@ -1160,12 +1231,43 @@ public class PublishTaskService {
|
||||
winner.setCompleted(completed ? 1 : 0);
|
||||
winner.setLastChunkAt(now);
|
||||
winner.setLastError(null);
|
||||
winner.setStateJson(scope.getStateJson());
|
||||
Integer winnerRowCount = readReceivedRowCount(winner);
|
||||
winner.setStateJson(resultScopeStateJson(completed,
|
||||
Math.max(receivedRowCount, winnerRowCount == null ? 0 : winnerRowCount)));
|
||||
winner.setUpdatedAt(now);
|
||||
taskScopeStateMapper.updateById(winner);
|
||||
}
|
||||
}
|
||||
|
||||
private String resultScopeStateJson(boolean completed, int receivedRowCount) {
|
||||
Map<String, Object> state = new LinkedHashMap<>();
|
||||
state.put("phase", completed ? "COMPLETE" : "RECEIVING");
|
||||
state.put("receivedRows", Math.max(0, receivedRowCount));
|
||||
return writeJson(state, "保存上架结果分片进度失败");
|
||||
}
|
||||
|
||||
private void updateReceivedProgress(Long taskId, PublishFileEntity file, int receivedRowCount) {
|
||||
if (file == null) {
|
||||
return;
|
||||
}
|
||||
int total = safeInt(file.getTotalRows());
|
||||
if (total <= 0) {
|
||||
long parsedRows = Objects.requireNonNullElse(publishItemMapper.selectCount(
|
||||
new LambdaQueryWrapper<PublishItemEntity>()
|
||||
.eq(PublishItemEntity::getTaskId, taskId)
|
||||
.eq(PublishItemEntity::getFileId, file.getId())), 0L);
|
||||
total = parsedRows >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) Math.max(0L, parsedRows);
|
||||
if (total > 0) {
|
||||
file.setTotalRows(total);
|
||||
}
|
||||
}
|
||||
int progress = Math.max(safeInt(file.getProcessedRows()), Math.max(0, receivedRowCount));
|
||||
if (total > 0) {
|
||||
progress = Math.min(total, progress);
|
||||
}
|
||||
file.setProcessedRows(progress);
|
||||
}
|
||||
|
||||
private List<PublishRowDto> loadCompleteResultRows(Long taskId, ResultChunkReceipt receipt) {
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
@@ -1637,6 +1739,7 @@ public class PublishTaskService {
|
||||
|
||||
private record ResultChunkReceipt(String scopeHash,
|
||||
int chunkTotal,
|
||||
boolean completed) {
|
||||
boolean completed,
|
||||
int receivedRowCount) {
|
||||
}
|
||||
}
|
||||
|
||||
+18
-13
@@ -91,7 +91,7 @@ public class PublishWorkbookService {
|
||||
}
|
||||
}
|
||||
|
||||
public File writeWorkbook(File outputFile, List<PublishRowDto> rows) {
|
||||
public File writeWorkbook(File outputFile, List<PublishRowDto> rows, String publishCountry) {
|
||||
File parent = outputFile.getParentFile();
|
||||
if (parent != null) {
|
||||
parent.mkdirs();
|
||||
@@ -100,13 +100,9 @@ public class PublishWorkbookService {
|
||||
workbook.setCompressTempFiles(true);
|
||||
try (FileOutputStream output = new FileOutputStream(outputFile)) {
|
||||
CellStyle headerStyle = createHeaderStyle(workbook);
|
||||
Map<String, List<PublishRowDto>> rowsByCountry = groupByCountry(rows);
|
||||
Set<String> usedSheetNames = new LinkedHashSet<>();
|
||||
for (Map.Entry<String, List<PublishRowDto>> entry : rowsByCountry.entrySet()) {
|
||||
String sheetName = uniqueSheetName(entry.getKey(), usedSheetNames);
|
||||
Sheet sheet = workbook.createSheet(sheetName);
|
||||
writeSheet(sheet, entry.getValue(), headerStyle);
|
||||
}
|
||||
String country = countrySheetName(publishCountry);
|
||||
Sheet sheet = workbook.createSheet(country);
|
||||
writeSheet(sheet, rows, headerStyle, country);
|
||||
workbook.write(output);
|
||||
return outputFile;
|
||||
} catch (Exception ex) {
|
||||
@@ -135,7 +131,7 @@ public class PublishWorkbookService {
|
||||
+ "_上架结果.xlsx";
|
||||
String filename = uniqueFilename(desired, usedFilenames);
|
||||
File workbook = new File(workDirectory, filename);
|
||||
writeWorkbook(workbook, input.rows());
|
||||
writeWorkbook(workbook, input.rows(), input.publishCountry());
|
||||
workbooks.add(workbook);
|
||||
}
|
||||
|
||||
@@ -202,7 +198,10 @@ public class PublishWorkbookService {
|
||||
return grouped;
|
||||
}
|
||||
|
||||
private void writeSheet(Sheet sheet, List<PublishRowDto> rows, CellStyle headerStyle) {
|
||||
private void writeSheet(Sheet sheet,
|
||||
List<PublishRowDto> rows,
|
||||
CellStyle headerStyle,
|
||||
String publishCountry) {
|
||||
Row header = sheet.createRow(0);
|
||||
for (int index = 0; index < RESULT_HEADERS.size(); index++) {
|
||||
Cell cell = header.createCell(index);
|
||||
@@ -210,11 +209,14 @@ public class PublishWorkbookService {
|
||||
cell.setCellStyle(headerStyle);
|
||||
}
|
||||
int rowIndex = 1;
|
||||
for (PublishRowDto value : rows) {
|
||||
for (PublishRowDto value : rows == null ? List.<PublishRowDto>of() : rows) {
|
||||
if (value == null) {
|
||||
continue;
|
||||
}
|
||||
Row row = sheet.createRow(rowIndex++);
|
||||
setText(row, 0, value.getSourceId());
|
||||
setText(row, 1, value.getAsin());
|
||||
setText(row, 2, value.getCountry());
|
||||
setText(row, 2, publishCountry);
|
||||
setText(row, 3, value.getBrand());
|
||||
setPrice(row, 4, value.getPrice());
|
||||
setText(row, 5, value.getStatus());
|
||||
@@ -337,7 +339,10 @@ public class PublishWorkbookService {
|
||||
public record ParsedWorkbook(List<PublishRowDto> rows) {
|
||||
}
|
||||
|
||||
public record WorkbookInput(String sourceFilename, String shopName, List<PublishRowDto> rows) {
|
||||
public record WorkbookInput(String sourceFilename,
|
||||
String shopName,
|
||||
String publishCountry,
|
||||
List<PublishRowDto> rows) {
|
||||
}
|
||||
|
||||
public record PackagedResult(File file, String filename, String contentType) {
|
||||
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
|
||||
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
|
||||
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
|
||||
/**
|
||||
* Internal compatibility endpoints used by the Flask admin task page.
|
||||
* The user-facing shop-data-crawl endpoints remain unchanged for existing clients.
|
||||
*/
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/admin/shop-data-crawl")
|
||||
public class AdminShopDataCrawlTaskController {
|
||||
|
||||
@Value("${aiimage.security.internal-token:}")
|
||||
private String internalToken;
|
||||
|
||||
@Value("${aiimage.security.internal-token-file:}")
|
||||
private String internalTokenFile;
|
||||
|
||||
private final ShopDataCrawlTaskService taskService;
|
||||
private final AdminAuthSupport adminAuthSupport;
|
||||
private final PermissionMenuService permissionMenuService;
|
||||
|
||||
@GetMapping("/results/{resultId}/download")
|
||||
@Operation(summary = "下载店铺数据任务结果(内部)")
|
||||
public void download(
|
||||
@PathVariable Long resultId,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
requireShopDataCrawlTaskAccess(request);
|
||||
String url = taskService.resolveAdminResultDownloadUrl(resultId);
|
||||
String filename = taskService.resolveAdminResultDownloadFilename(resultId);
|
||||
try {
|
||||
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
DownloadHeaderUtil.setAttachment(response, filename);
|
||||
try (InputStream input = URI.create(url).toURL().openStream()) {
|
||||
input.transferTo(response.getOutputStream());
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
|
||||
}
|
||||
}
|
||||
|
||||
@DeleteMapping("/history/{resultId}")
|
||||
@Operation(summary = "删除店铺数据任务结果(内部)")
|
||||
public ApiResponse<Void> deleteHistory(
|
||||
@PathVariable Long resultId,
|
||||
HttpServletRequest request) {
|
||||
requireShopDataCrawlTaskAccess(request);
|
||||
taskService.deleteAdminHistory(resultId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
private void requireShopDataCrawlTaskAccess(HttpServletRequest request) {
|
||||
AdminUserEntity operator;
|
||||
try {
|
||||
operator = adminAuthSupport.requireAdmin(request);
|
||||
} catch (BusinessException authFailure) {
|
||||
operator = resolveInternalOperator(request);
|
||||
if (operator == null) {
|
||||
throw authFailure;
|
||||
}
|
||||
}
|
||||
permissionMenuService.requireShopDataCrawlTaskAccess(operator);
|
||||
}
|
||||
|
||||
private AdminUserEntity resolveInternalOperator(HttpServletRequest request) {
|
||||
String suppliedToken = request.getHeader("X-Internal-Token");
|
||||
if (!isTrustedInternalRequest(suppliedToken)) {
|
||||
return null;
|
||||
}
|
||||
String rawOperatorId = request.getParameter("operatorId");
|
||||
if (rawOperatorId == null || rawOperatorId.isBlank()) {
|
||||
rawOperatorId = request.getParameter("operator_id");
|
||||
}
|
||||
if (rawOperatorId == null || rawOperatorId.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return permissionMenuService.requireAdminOperator(Long.parseLong(rawOperatorId.trim()));
|
||||
} catch (NumberFormatException ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isTrustedInternalRequest(String suppliedToken) {
|
||||
String expectedToken = resolveExpectedInternalToken();
|
||||
return !expectedToken.isBlank() && suppliedToken != null && !suppliedToken.isBlank()
|
||||
&& MessageDigest.isEqual(
|
||||
expectedToken.getBytes(StandardCharsets.UTF_8),
|
||||
suppliedToken.trim().getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
private String resolveExpectedInternalToken() {
|
||||
if (internalToken != null && !internalToken.isBlank()) {
|
||||
return internalToken.trim();
|
||||
}
|
||||
Path path = resolveInternalTokenFile();
|
||||
if (path == null || !Files.isRegularFile(path)) {
|
||||
return "";
|
||||
}
|
||||
try {
|
||||
return Files.readString(path, StandardCharsets.UTF_8).trim();
|
||||
} catch (Exception ignored) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
private Path resolveInternalTokenFile() {
|
||||
String configuredPath = internalTokenFile == null ? "" : internalTokenFile.trim();
|
||||
if (!configuredPath.isEmpty()) {
|
||||
if (configuredPath.equals("~") || configuredPath.startsWith("~/") || configuredPath.startsWith("~\\")) {
|
||||
String userHome = System.getProperty("user.home", "").trim();
|
||||
if (userHome.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
configuredPath = configuredPath.length() == 1
|
||||
? userHome
|
||||
: Path.of(userHome, configuredPath.substring(2)).toString();
|
||||
}
|
||||
Path configuredTokenPath = Path.of(configuredPath);
|
||||
return configuredTokenPath.isAbsolute() ? configuredTokenPath.normalize() : null;
|
||||
}
|
||||
String userHome = System.getProperty("user.home", "").trim();
|
||||
return userHome.isEmpty()
|
||||
? null
|
||||
: Path.of(userHome, ".aiimage", "internal-token").toAbsolutePath().normalize();
|
||||
}
|
||||
}
|
||||
+1
@@ -190,6 +190,7 @@ public class ShopDataCrawlTaskController {
|
||||
{
|
||||
"date": "2026-07-25",
|
||||
"asin": "B0EXAMPLE1",
|
||||
"commodityImage": "https://m.media-amazon.com/images/I/example.jpg",
|
||||
"inventorySales": "120",
|
||||
"salesRank": "35",
|
||||
"pageViews": "860",
|
||||
|
||||
+9
-1
@@ -5,7 +5,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "店铺数据抓取结果行;生成 Excel 时按日期、ASIN、库存销量、销售排名、页面浏览量、售出件数、价格、推荐报价的固定列顺序写入")
|
||||
@Schema(description = "店铺数据抓取结果行;生成 Excel 时按日期、ASIN、商品图片、库存销量、销售排名、页面浏览量、售出件数、价格、推荐报价、品牌的固定列顺序写入")
|
||||
public class ShopDataCrawlRowDto {
|
||||
@JsonAlias("日期")
|
||||
@Schema(description = "日期列,按来源文本原样保留", example = "2026-07-25")
|
||||
@@ -15,6 +15,14 @@ public class ShopDataCrawlRowDto {
|
||||
@Schema(description = "亚马逊商品 ASIN", example = "B0CJ8SNXXV")
|
||||
private String asin;
|
||||
|
||||
@JsonAlias({"brand", "品牌"})
|
||||
@Schema(description = "商品品牌,来自 Python 回传的 brand 字段", example = "Example Brand")
|
||||
private String brand;
|
||||
|
||||
@JsonAlias({"commodity_image", "商品图片"})
|
||||
@Schema(description = "商品图片 URL;生成 Excel 时下载并嵌入商品图片列", example = "https://m.media-amazon.com/images/I/example.jpg")
|
||||
private String commodityImage;
|
||||
|
||||
@JsonAlias({"库存销量", "inventory_sales"})
|
||||
@Schema(description = "库存销量列,按来源文本原样保留", example = "128")
|
||||
private String inventorySales;
|
||||
|
||||
+132
-8
@@ -4,10 +4,16 @@ import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.CellStyle;
|
||||
import org.apache.poi.ss.usermodel.ClientAnchor;
|
||||
import org.apache.poi.ss.usermodel.Drawing;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -19,13 +25,24 @@ import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class ShopDataCrawlExcelAssemblyService {
|
||||
static final List<String> COUNTRIES = List.of("UK", "DE", "FR", "ES", "IT");
|
||||
static final List<String> SHEETS = List.of("英国", "德国", "法国", "西班牙", "意大利");
|
||||
static final List<String> HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
|
||||
static final List<String> LEGACY_HEADERS = List.of("日期", "ASIN", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
|
||||
static final List<String> HEADERS_WITHOUT_BRAND = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价");
|
||||
static final List<String> HEADERS = List.of("日期", "ASIN", "商品图片", "库存销量", "销售排名", "页面浏览量", "售出件数", "价格", "推荐报价", "品牌");
|
||||
private static final String TEMPLATE = "templates/shop-data-crawl/文档格式.xlsx";
|
||||
private static final int IMAGE_COLUMN = 2;
|
||||
private static final int BRAND_COLUMN = HEADERS.size() - 1;
|
||||
private static final int IMAGE_COLUMN_WIDTH = 18 * 256;
|
||||
private static final float IMAGE_ROW_HEIGHT_POINTS = 80f;
|
||||
|
||||
private final SimilarAsinImageEmbedder imageEmbedder;
|
||||
|
||||
public void writeWorkbook(File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
|
||||
try (InputStream input = new ClassPathResource(TEMPLATE).getInputStream();
|
||||
@@ -33,8 +50,11 @@ public class ShopDataCrawlExcelAssemblyService {
|
||||
FileOutputStream output = new FileOutputStream(outputXlsx)) {
|
||||
validateTemplate(workbook);
|
||||
Map<String, List<ShopDataCrawlRowDto>> rowsByCountry = rowsByCountry(items);
|
||||
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache = new ConcurrentHashMap<>();
|
||||
imageEmbedder.prefetch(imageUrls(rowsByCountry), imageCache);
|
||||
Map<String, Integer> pictureIndexes = new LinkedHashMap<>();
|
||||
for (int i = 0; i < COUNTRIES.size(); i++) {
|
||||
writeSheet(workbook.getSheetAt(i), rowsByCountry.get(COUNTRIES.get(i)));
|
||||
writeSheet(workbook, workbook.getSheetAt(i), rowsByCountry.get(COUNTRIES.get(i)), imageCache, pictureIndexes);
|
||||
}
|
||||
workbook.write(output);
|
||||
} catch (BusinessException ex) {
|
||||
@@ -58,22 +78,45 @@ public class ShopDataCrawlExcelAssemblyService {
|
||||
throw new BusinessException("店铺数据抓取模板工作表顺序不正确");
|
||||
}
|
||||
Row header = sheet.getRow(0);
|
||||
for (int column = 0; column < HEADERS.size(); column++) {
|
||||
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
|
||||
boolean templateHasBrand = "品牌".equals(cellText(header, BRAND_COLUMN));
|
||||
List<String> expectedHeaders = currentTemplate
|
||||
? (templateHasBrand ? HEADERS : HEADERS_WITHOUT_BRAND)
|
||||
: LEGACY_HEADERS;
|
||||
for (int column = 0; column < expectedHeaders.size(); column++) {
|
||||
String actual = header == null || header.getCell(column) == null ? "" : header.getCell(column).getStringCellValue().trim();
|
||||
if (!HEADERS.get(column).equals(actual)) {
|
||||
if (!expectedHeaders.get(column).equals(actual)) {
|
||||
throw new BusinessException("店铺数据抓取模板表头不正确: " + sheet.getSheetName());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void writeSheet(Sheet sheet, List<ShopDataCrawlRowDto> rows) {
|
||||
private void writeSheet(XSSFWorkbook workbook,
|
||||
Sheet sheet,
|
||||
List<ShopDataCrawlRowDto> rows,
|
||||
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
|
||||
Map<String, Integer> pictureIndexes) {
|
||||
Row header = sheet.getRow(0);
|
||||
Row styleRow = sheet.getRow(1);
|
||||
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
|
||||
boolean templateHasBrand = "品牌".equals(cellText(header, BRAND_COLUMN));
|
||||
CellStyle[] styles = new CellStyle[HEADERS.size()];
|
||||
for (int column = 0; column < styles.length; column++) {
|
||||
Cell cell = styleRow == null ? null : styleRow.getCell(column);
|
||||
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
|
||||
Cell cell = styleRow == null ? null : styleRow.getCell(sourceColumn);
|
||||
styles[column] = cell == null ? null : cell.getCellStyle();
|
||||
}
|
||||
int[] columnWidths = new int[HEADERS.size()];
|
||||
for (int column = 0; column < columnWidths.length; column++) {
|
||||
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
|
||||
columnWidths[column] = sheet.getColumnWidth(sourceColumn);
|
||||
}
|
||||
writeHeaders(sheet, currentTemplate, templateHasBrand);
|
||||
for (int column = 0; column < columnWidths.length; column++) {
|
||||
sheet.setColumnWidth(column, columnWidths[column]);
|
||||
}
|
||||
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
|
||||
int last = sheet.getLastRowNum();
|
||||
for (int rowIndex = 1; rowIndex <= last; rowIndex++) {
|
||||
Row row = sheet.getRow(rowIndex);
|
||||
@@ -84,15 +127,96 @@ public class ShopDataCrawlExcelAssemblyService {
|
||||
int rowIndex = 1;
|
||||
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
|
||||
Row row = sheet.createRow(rowIndex++);
|
||||
String[] values = {value.getDate(), value.getAsin(), value.getInventorySales(), value.getSalesRank(),
|
||||
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer()};
|
||||
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
|
||||
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
|
||||
for (int column = 0; column < values.length; column++) {
|
||||
Cell cell = row.createCell(column);
|
||||
if (styles[column] != null) cell.setCellStyle(styles[column]);
|
||||
cell.setCellValue(values[column] == null ? "" : values[column]);
|
||||
}
|
||||
if (!blank(value.getCommodityImage())) {
|
||||
row.setHeightInPoints(IMAGE_ROW_HEIGHT_POINTS);
|
||||
embedImage(workbook, sheet, row, value.getCommodityImage(), imageCache, pictureIndexes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void writeHeaders(Sheet sheet, boolean currentTemplate, boolean templateHasBrand) {
|
||||
Row header = sheet.getRow(0);
|
||||
if (header == null) header = sheet.createRow(0);
|
||||
CellStyle[] styles = new CellStyle[HEADERS.size()];
|
||||
for (int column = 0; column < HEADERS.size(); column++) {
|
||||
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
|
||||
Cell source = header.getCell(sourceColumn);
|
||||
styles[column] = source == null ? null : source.getCellStyle();
|
||||
}
|
||||
for (int column = 0; column < HEADERS.size(); column++) {
|
||||
Cell cell = header.getCell(column);
|
||||
if (cell == null) cell = header.createCell(column);
|
||||
if (styles[column] != null) cell.setCellStyle(styles[column]);
|
||||
cell.setCellValue(HEADERS.get(column));
|
||||
}
|
||||
}
|
||||
|
||||
private int templateColumnForOutput(int outputColumn, boolean currentTemplate, boolean templateHasBrand) {
|
||||
if (outputColumn == BRAND_COLUMN) {
|
||||
return templateHasBrand ? BRAND_COLUMN : 1;
|
||||
}
|
||||
return currentTemplate || outputColumn < IMAGE_COLUMN ? outputColumn : outputColumn - 1;
|
||||
}
|
||||
|
||||
private void embedImage(XSSFWorkbook workbook,
|
||||
Sheet sheet,
|
||||
Row row,
|
||||
String imageUrl,
|
||||
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
|
||||
Map<String, Integer> pictureIndexes) {
|
||||
String normalizedUrl = imageUrl.trim();
|
||||
try {
|
||||
SimilarAsinImageEmbedder.ResizedImage image = imageCache.get(normalizedUrl);
|
||||
if (image == null) image = imageEmbedder.fetchAndResizeForCache(normalizedUrl);
|
||||
if (image == null) {
|
||||
row.getCell(IMAGE_COLUMN).setCellValue(normalizedUrl);
|
||||
return;
|
||||
}
|
||||
Integer pictureIndex = pictureIndexes.get(normalizedUrl);
|
||||
if (pictureIndex == null) {
|
||||
pictureIndex = workbook.addPicture(image.bytes(), Workbook.PICTURE_TYPE_JPEG);
|
||||
pictureIndexes.put(normalizedUrl, pictureIndex);
|
||||
}
|
||||
Drawing<?> drawing = sheet.createDrawingPatriarch();
|
||||
ClientAnchor anchor = workbook.getCreationHelper().createClientAnchor();
|
||||
anchor.setCol1(IMAGE_COLUMN);
|
||||
anchor.setRow1(row.getRowNum());
|
||||
anchor.setCol2(IMAGE_COLUMN + 1);
|
||||
anchor.setRow2(row.getRowNum() + 1);
|
||||
anchor.setAnchorType(ClientAnchor.AnchorType.MOVE_AND_RESIZE);
|
||||
drawing.createPicture(anchor, pictureIndex);
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[shop-data-crawl] embed commodity image failed sheet={} row={} url={} msg={}",
|
||||
sheet.getSheetName(), row.getRowNum() + 1, normalizedUrl, ex.getMessage());
|
||||
row.getCell(IMAGE_COLUMN).setCellValue(normalizedUrl);
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> imageUrls(Map<String, List<ShopDataCrawlRowDto>> rowsByCountry) {
|
||||
return rowsByCountry.values().stream()
|
||||
.flatMap(List::stream)
|
||||
.map(ShopDataCrawlRowDto::getCommodityImage)
|
||||
.filter(url -> !blank(url))
|
||||
.map(String::trim)
|
||||
.distinct()
|
||||
.toList();
|
||||
}
|
||||
|
||||
private String cellText(Row row, int column) {
|
||||
Cell cell = row == null ? null : row.getCell(column);
|
||||
return cell == null ? "" : cell.getStringCellValue().trim();
|
||||
}
|
||||
|
||||
private boolean blank(String value) {
|
||||
return value == null || value.isBlank();
|
||||
}
|
||||
|
||||
private Map<String, List<ShopDataCrawlRowDto>> rowsByCountry(List<ShopDataCrawlResultItemVo> items) {
|
||||
Map<String, List<ShopDataCrawlRowDto>> result = new LinkedHashMap<>();
|
||||
|
||||
+227
-15
@@ -50,6 +50,7 @@ import java.io.File;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -65,6 +66,7 @@ public class ShopDataCrawlTaskService {
|
||||
private static final int RESULT_PENDING = -1;
|
||||
private static final int RESULT_FAILED = 0;
|
||||
private static final int RESULT_SUCCESS = 1;
|
||||
private static final int SHOP_HISTORY_RETENTION_LIMIT = 3;
|
||||
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
|
||||
private static final String PARTIAL_RESULT_MESSAGE = "Python 中断,已保留已回传的部分数据";
|
||||
private static final String RESULT_CHUNK_SCOPE_PREFIX = "result-chunks:";
|
||||
@@ -290,21 +292,54 @@ public class ShopDataCrawlTaskService {
|
||||
}
|
||||
|
||||
public String resolveResultDownloadUrl(Long resultId, Long userId) {
|
||||
FileResultEntity entity = fileResultMapper.selectById(resultId);
|
||||
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
|
||||
validateUserId(userId);
|
||||
FileResultEntity entity = requireResultEntity(resultId);
|
||||
ensureResultOwner(entity, userId);
|
||||
return resolveDownloadUrl(entity);
|
||||
}
|
||||
|
||||
public String resolveResultDownloadFilename(Long resultId, Long userId) {
|
||||
validateUserId(userId);
|
||||
FileResultEntity entity = requireResultEntity(resultId);
|
||||
ensureResultOwner(entity, userId);
|
||||
return resolveDownloadFilename(entity);
|
||||
}
|
||||
|
||||
/**
|
||||
* Used only by the internally authenticated admin task-management endpoint.
|
||||
* The public endpoint must continue to validate the requesting user ID.
|
||||
*/
|
||||
public String resolveAdminResultDownloadUrl(Long resultId) {
|
||||
return resolveDownloadUrl(requireResultEntity(resultId));
|
||||
}
|
||||
|
||||
/** See {@link #resolveAdminResultDownloadUrl(Long)}. */
|
||||
public String resolveAdminResultDownloadFilename(Long resultId) {
|
||||
return resolveDownloadFilename(requireResultEntity(resultId));
|
||||
}
|
||||
|
||||
private FileResultEntity requireResultEntity(Long resultId) {
|
||||
FileResultEntity entity = resultId == null || resultId <= 0 ? null : fileResultMapper.selectById(resultId);
|
||||
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
return entity;
|
||||
}
|
||||
|
||||
private void ensureResultOwner(FileResultEntity entity, Long userId) {
|
||||
if (entity == null || !userId.equals(entity.getUserId())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveDownloadUrl(FileResultEntity entity) {
|
||||
if (blank(entity.getResultFileUrl())) {
|
||||
throw new BusinessException("暂无可下载文件");
|
||||
}
|
||||
return ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl());
|
||||
}
|
||||
|
||||
public String resolveResultDownloadFilename(Long resultId, Long userId) {
|
||||
FileResultEntity entity = fileResultMapper.selectById(resultId);
|
||||
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
private String resolveDownloadFilename(FileResultEntity entity) {
|
||||
return !blank(entity.getResultFilename())
|
||||
? entity.getResultFilename()
|
||||
: safeFileStem(entity.getSourceFilename()) + ".xlsx";
|
||||
@@ -584,26 +619,61 @@ public class ShopDataCrawlTaskService {
|
||||
@Transactional
|
||||
public void deleteHistory(Long resultId, Long userId) {
|
||||
validateUserId(userId);
|
||||
FileResultEntity entity = fileResultMapper.selectById(resultId);
|
||||
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
FileResultEntity entity = requireResultEntity(resultId);
|
||||
ensureResultOwner(entity, userId);
|
||||
Long taskId = entity.getTaskId();
|
||||
FileTaskEntity task = loadTaskForExecution(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) throw new BusinessException("任务不存在");
|
||||
ensureTaskOwnedByCurrentInstance(task, "delete shop data crawl history");
|
||||
if (!isTerminalTaskStatus(task.getStatus())) {
|
||||
throw new BusinessException(40901, "任务仍在处理中,不能删除");
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||
FileResultEntity latestEntity = fileResultMapper.selectById(resultId);
|
||||
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
deleteResultHistoryRow(latestEntity);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes an administrative result without accepting a caller-controlled owner ID.
|
||||
* The controller that calls this method is restricted to the Flask-to-Java internal channel.
|
||||
*/
|
||||
@Transactional
|
||||
public void deleteAdminHistory(Long resultId) {
|
||||
FileResultEntity entity = requireResultEntity(resultId);
|
||||
Long ownerId = entity.getUserId();
|
||||
if (ownerId == null || ownerId <= 0) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
deleteHistory(resultId, ownerId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes one result row and all task-owned artifacts that point at it.
|
||||
* The caller must already hold the task lock when the row belongs to a task.
|
||||
*/
|
||||
private void deleteResultHistoryRow(FileResultEntity entity) {
|
||||
if (entity == null || entity.getId() == null || entity.getId() <= 0) {
|
||||
return;
|
||||
}
|
||||
Long taskId = entity.getTaskId();
|
||||
Long resultId = entity.getId();
|
||||
String resultFileUrl = entity.getResultFileUrl();
|
||||
taskFileJobService.deleteResultJobs(taskId, MODULE_TYPE, resultId);
|
||||
taskResultItemService.deleteResultItem(taskId, MODULE_TYPE, resultId);
|
||||
String resultFileUrl = latestEntity.getResultFileUrl();
|
||||
fileResultMapper.deleteById(resultId);
|
||||
try {
|
||||
reconcileTaskAfterResultRemoval(taskId);
|
||||
deleteResultObjectIfUnreferenced(resultFileUrl);
|
||||
} catch (Exception ex) {
|
||||
// The row is already gone. Do not leave its workbook behind merely
|
||||
// because the parent task snapshot could not be rebuilt.
|
||||
log.warn("[shop-data-crawl] reconcile task after result deletion failed taskId={} resultId={} msg={}",
|
||||
taskId, resultId, safeMessage(ex));
|
||||
}
|
||||
// Run this after the row delete so shared object references are counted correctly.
|
||||
deleteResultObjectIfUnreferenced(resultFileUrl);
|
||||
}
|
||||
|
||||
private void reconcileTaskAfterResultRemoval(Long taskId) {
|
||||
@@ -630,6 +700,139 @@ public class ShopDataCrawlTaskService {
|
||||
taskCacheService.saveTaskCache(task);
|
||||
}
|
||||
|
||||
private void pruneCompletedHistoryQuietly(FileTaskEntity currentTask, List<FileResultEntity> currentRows) {
|
||||
if (currentRows == null || currentRows.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (FileResultEntity row : currentRows) {
|
||||
if (!isRetentionCandidate(row)) {
|
||||
continue;
|
||||
}
|
||||
Long userId = row.getUserId() != null
|
||||
? row.getUserId()
|
||||
: currentTask == null ? null : currentTask.getUserId();
|
||||
String shopKey = retentionShopKey(row);
|
||||
if (userId == null || shopKey == null) {
|
||||
log.warn("[shop-data-crawl] skip history retention because ownership key is incomplete taskId={} resultId={}",
|
||||
currentTask == null ? null : currentTask.getId(), row.getId());
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
pruneCompletedHistoryForShop(userId, shopKey);
|
||||
} catch (Exception ex) {
|
||||
// Retention is best effort. A cleanup failure must not fail the newly assembled workbook job.
|
||||
log.warn("[shop-data-crawl] history retention failed taskId={} resultId={} msg={}",
|
||||
currentTask == null ? null : currentTask.getId(), row.getId(), safeMessage(ex));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void pruneCompletedHistoryForShop(Long userId, String shopKey) {
|
||||
if (userId == null || shopKey == null) {
|
||||
return;
|
||||
}
|
||||
List<FileResultEntity> candidates = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.select(FileResultEntity::getId,
|
||||
FileResultEntity::getTaskId,
|
||||
FileResultEntity::getModuleType,
|
||||
FileResultEntity::getSourceFilename,
|
||||
FileResultEntity::getSourceFileUrl,
|
||||
FileResultEntity::getResultFileUrl,
|
||||
FileResultEntity::getSuccess,
|
||||
FileResultEntity::getUserId,
|
||||
FileResultEntity::getCreatedAt)
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileResultEntity::getUserId, userId)
|
||||
.eq(FileResultEntity::getSuccess, RESULT_SUCCESS)
|
||||
.isNotNull(FileResultEntity::getResultFileUrl)
|
||||
.ne(FileResultEntity::getResultFileUrl, "")
|
||||
.orderByDesc(FileResultEntity::getCreatedAt)
|
||||
.orderByDesc(FileResultEntity::getId));
|
||||
if (candidates == null || candidates.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
List<FileResultEntity> shopResults = new ArrayList<>();
|
||||
for (FileResultEntity candidate : candidates) {
|
||||
if (!isRetentionCandidate(candidate)
|
||||
|| !Objects.equals(userId, candidate.getUserId())) {
|
||||
continue;
|
||||
}
|
||||
if (!Objects.equals(shopKey, retentionShopKey(candidate))) {
|
||||
continue;
|
||||
}
|
||||
shopResults.add(candidate);
|
||||
}
|
||||
|
||||
Comparator<FileResultEntity> newestFirst = Comparator
|
||||
.comparing(FileResultEntity::getCreatedAt, Comparator.nullsLast(Comparator.reverseOrder()))
|
||||
.thenComparing(FileResultEntity::getId, Comparator.nullsLast(Comparator.reverseOrder()));
|
||||
shopResults.sort(newestFirst);
|
||||
for (int index = SHOP_HISTORY_RETENTION_LIMIT; index < shopResults.size(); index++) {
|
||||
deleteRetentionResultQuietly(shopResults.get(index), userId, shopKey);
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteRetentionResultQuietly(FileResultEntity candidate, Long userId, String shopKey) {
|
||||
if (candidate == null || candidate.getId() == null || candidate.getId() <= 0
|
||||
|| candidate.getTaskId() == null || candidate.getTaskId() <= 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(candidate.getTaskId());
|
||||
if (lockHandle == null) {
|
||||
log.info("[shop-data-crawl] skip retained-history deletion because task lock is busy taskId={} resultId={}",
|
||||
candidate.getTaskId(), candidate.getId());
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
FileResultEntity latest = fileResultMapper.selectById(candidate.getId());
|
||||
if (!isRetentionCandidate(latest)
|
||||
|| !Objects.equals(userId, latest.getUserId())
|
||||
|| !Objects.equals(shopKey, retentionShopKey(latest))) {
|
||||
return;
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(candidate.getTaskId());
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())
|
||||
|| !isTerminalTaskStatus(task.getStatus())) {
|
||||
log.info("[shop-data-crawl] skip retained-history deletion because task is not terminal taskId={} resultId={}",
|
||||
candidate.getTaskId(), candidate.getId());
|
||||
return;
|
||||
}
|
||||
deleteResultHistoryRow(latest);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[shop-data-crawl] retained-history deletion failed taskId={} resultId={} msg={}",
|
||||
candidate.getTaskId(), candidate.getId(), safeMessage(ex));
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isRetentionCandidate(FileResultEntity row) {
|
||||
return row != null
|
||||
&& Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())
|
||||
&& !blank(row.getResultFileUrl());
|
||||
}
|
||||
|
||||
private String retentionShopKey(FileResultEntity row) {
|
||||
if (row == null) {
|
||||
return null;
|
||||
}
|
||||
String shopId = trimToNull(row.getSourceFileUrl());
|
||||
if (shopId != null) {
|
||||
return "shop-id:" + shopId;
|
||||
}
|
||||
String shopName = trimToNull(row.getSourceFilename());
|
||||
return shopName == null ? null : "shop-name:" + shopName;
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
String normalized = value.trim();
|
||||
return normalized.isEmpty() ? null : normalized;
|
||||
}
|
||||
|
||||
private long countTasks(Long userId, List<String> statuses) {
|
||||
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
@@ -1360,6 +1563,7 @@ public class ShopDataCrawlTaskService {
|
||||
} finally {
|
||||
FileUtil.del(xlsx);
|
||||
}
|
||||
pruneCompletedHistoryQuietly(task, rows);
|
||||
}
|
||||
|
||||
public void cleanupResultFileJob(TaskFileJobEntity job) {
|
||||
@@ -1460,6 +1664,10 @@ public class ShopDataCrawlTaskService {
|
||||
return Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess()) || Integer.valueOf(RESULT_FAILED).equals(row.getSuccess());
|
||||
}
|
||||
|
||||
private boolean isTerminalTaskStatus(String status) {
|
||||
return "SUCCESS".equals(status) || "FAILED".equals(status) || "CANCELLED".equals(status);
|
||||
}
|
||||
|
||||
private Boolean toSuccessFlag(Integer dbValue, Boolean fallback) {
|
||||
if (dbValue == null || Integer.valueOf(RESULT_PENDING).equals(dbValue)) {
|
||||
return fallback;
|
||||
@@ -1544,6 +1752,8 @@ public class ShopDataCrawlTaskService {
|
||||
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||
row.setDate(trim(source.getDate()));
|
||||
row.setAsin(trim(source.getAsin()));
|
||||
row.setBrand(trim(source.getBrand()));
|
||||
row.setCommodityImage(trim(source.getCommodityImage()));
|
||||
row.setInventorySales(trim(source.getInventorySales()));
|
||||
row.setSalesRank(trim(source.getSalesRank()));
|
||||
row.setPageViews(trim(source.getPageViews()));
|
||||
@@ -1581,6 +1791,8 @@ public class ShopDataCrawlTaskService {
|
||||
return left != null && right != null
|
||||
&& Objects.equals(trim(left.getDate()), trim(right.getDate()))
|
||||
&& Objects.equals(trim(left.getAsin()), trim(right.getAsin()))
|
||||
&& Objects.equals(trim(left.getBrand()), trim(right.getBrand()))
|
||||
&& Objects.equals(trim(left.getCommodityImage()), trim(right.getCommodityImage()))
|
||||
&& Objects.equals(trim(left.getInventorySales()), trim(right.getInventorySales()))
|
||||
&& Objects.equals(trim(left.getSalesRank()), trim(right.getSalesRank()))
|
||||
&& Objects.equals(trim(left.getPageViews()), trim(right.getPageViews()))
|
||||
@@ -1590,7 +1802,7 @@ public class ShopDataCrawlTaskService {
|
||||
}
|
||||
|
||||
private boolean rowEmpty(ShopDataCrawlRowDto row) {
|
||||
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getInventorySales())
|
||||
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getBrand()) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
|
||||
&& blank(row.getSalesRank()) && blank(row.getPageViews()) && blank(row.getUnitsSold())
|
||||
&& blank(row.getPrice()) && blank(row.getRecommendedOffer()));
|
||||
}
|
||||
|
||||
+2
-2
@@ -79,13 +79,13 @@ public class SkipPriceAsinController {
|
||||
}
|
||||
|
||||
@PostMapping
|
||||
@Operation(summary = "新增或更新跳过跟价 ASIN", description = "按店铺和国家维度新增或更新跳过跟价 ASIN。")
|
||||
@Operation(summary = "新增跳过跟价 ASIN", description = "新增一条跳过跟价 ASIN 记录。")
|
||||
public ApiResponse<SkipPriceAsinItemVo> create(
|
||||
@Parameter(description = "当前操作人用户 ID") @RequestParam(name = "operator_id") Long operatorId,
|
||||
@Parameter(description = "是否超级管理员") @RequestParam(name = "super_admin", defaultValue = "false") Boolean superAdmin,
|
||||
@Valid @RequestBody SkipPriceAsinCreateRequest request) {
|
||||
return ApiResponse.success("保存成功",
|
||||
skipPriceAsinService.createOrUpdate(request, operatorId, Boolean.TRUE.equals(superAdmin)));
|
||||
skipPriceAsinService.create(request, operatorId, Boolean.TRUE.equals(superAdmin)));
|
||||
}
|
||||
|
||||
@PostMapping("/import")
|
||||
|
||||
+7
@@ -1,6 +1,8 @@
|
||||
package com.nanri.aiimage.modules.shopkey.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.FieldStrategy;
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
@@ -16,6 +18,11 @@ public class ShopKeyEntity {
|
||||
private String remarkName;
|
||||
private String ziniaoAccountName;
|
||||
private String ziniaoToken;
|
||||
private String ipWhitelistStatus;
|
||||
@TableField(updateStrategy = FieldStrategy.ALWAYS)
|
||||
private LocalDateTime ipWhitelistCheckedAt;
|
||||
@TableField(updateStrategy = FieldStrategy.ALWAYS)
|
||||
private String ipWhitelistMessage;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
|
||||
+9
@@ -21,6 +21,15 @@ public class ShopKeyItemVo {
|
||||
@Schema(description = "紫鸟令牌")
|
||||
private String ziniaoToken;
|
||||
|
||||
@Schema(description = "IP 白名单状态:UNKNOWN、ALLOWED、BLOCKED")
|
||||
private String ipWhitelistStatus;
|
||||
|
||||
@Schema(description = "最近一次 IP 白名单检测时间")
|
||||
private LocalDateTime ipWhitelistCheckedAt;
|
||||
|
||||
@Schema(description = "最近一次 IP 白名单检测信息")
|
||||
private String ipWhitelistMessage;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
|
||||
+12
@@ -21,6 +21,8 @@ import java.util.List;
|
||||
@Slf4j
|
||||
public class ShopKeyService {
|
||||
|
||||
private static final String IP_WHITELIST_STATUS_UNKNOWN = "UNKNOWN";
|
||||
|
||||
private final ShopKeyMapper shopKeyMapper;
|
||||
private final ZiniaoShopIndexService ziniaoShopIndexService;
|
||||
|
||||
@@ -50,6 +52,7 @@ public class ShopKeyService {
|
||||
entity.setRemarkName(normalizeOptional(request.getRemarkName()));
|
||||
entity.setZiniaoAccountName(ziniaoAccountName);
|
||||
entity.setZiniaoToken(ziniaoToken);
|
||||
entity.setIpWhitelistStatus(IP_WHITELIST_STATUS_UNKNOWN);
|
||||
shopKeyMapper.insert(entity);
|
||||
triggerShopIndexRefresh();
|
||||
return toItemVo(getById(entity.getId()));
|
||||
@@ -60,9 +63,15 @@ public class ShopKeyService {
|
||||
ShopKeyEntity entity = getById(id);
|
||||
String ziniaoAccountName = normalizeRequired(request.getZiniaoAccountName(), "紫鸟账号名称不能为空");
|
||||
String ziniaoToken = normalizeRequired(request.getZiniaoToken(), "紫鸟令牌不能为空");
|
||||
boolean tokenChanged = !ziniaoToken.equals(entity.getZiniaoToken());
|
||||
entity.setRemarkName(normalizeOptional(request.getRemarkName()));
|
||||
entity.setZiniaoAccountName(ziniaoAccountName);
|
||||
entity.setZiniaoToken(ziniaoToken);
|
||||
if (tokenChanged) {
|
||||
entity.setIpWhitelistStatus(IP_WHITELIST_STATUS_UNKNOWN);
|
||||
entity.setIpWhitelistCheckedAt(null);
|
||||
entity.setIpWhitelistMessage(null);
|
||||
}
|
||||
shopKeyMapper.updateById(entity);
|
||||
triggerShopIndexRefresh();
|
||||
return toItemVo(getById(id));
|
||||
@@ -101,6 +110,9 @@ public class ShopKeyService {
|
||||
vo.setRemarkName(entity.getRemarkName());
|
||||
vo.setZiniaoAccountName(entity.getZiniaoAccountName());
|
||||
vo.setZiniaoToken(entity.getZiniaoToken());
|
||||
vo.setIpWhitelistStatus(entity.getIpWhitelistStatus());
|
||||
vo.setIpWhitelistCheckedAt(entity.getIpWhitelistCheckedAt());
|
||||
vo.setIpWhitelistMessage(entity.getIpWhitelistMessage());
|
||||
vo.setCreatedAt(entity.getCreatedAt());
|
||||
vo.setUpdatedAt(entity.getUpdatedAt());
|
||||
return vo;
|
||||
|
||||
+2
-12
@@ -210,33 +210,23 @@ public class SkipPriceAsinService {
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public SkipPriceAsinItemVo createOrUpdate(SkipPriceAsinCreateRequest request, Long operatorId, boolean superAdmin) {
|
||||
public SkipPriceAsinItemVo create(SkipPriceAsinCreateRequest request, Long operatorId, boolean superAdmin) {
|
||||
ShopManageGroupEntity group = shopManageGroupService.getAccessibleById(request.getGroupId(), operatorId, superAdmin);
|
||||
String shopName = normalizeRequired(request.getShopName(), "店铺名不能为空");
|
||||
Set<String> countries = normalizeCountries(request.getCountries());
|
||||
Map<String, String> countryAsinMap = normalizeCountryAsinMap(countries, request);
|
||||
Map<String, BigDecimal> countryMinimumPriceMap = normalizeCountryMinimumPriceMap(countries, request);
|
||||
|
||||
SkipPriceAsinEntity entity = skipPriceAsinMapper.selectOne(new LambdaQueryWrapper<SkipPriceAsinEntity>()
|
||||
.eq(SkipPriceAsinEntity::getGroupId, group.getId())
|
||||
.eq(SkipPriceAsinEntity::getShopName, shopName)
|
||||
.last("LIMIT 1"));
|
||||
if (entity == null) {
|
||||
entity = new SkipPriceAsinEntity();
|
||||
SkipPriceAsinEntity entity = new SkipPriceAsinEntity();
|
||||
entity.setGroupId(group.getId());
|
||||
entity.setShopName(shopName);
|
||||
}
|
||||
|
||||
for (Map.Entry<String, String> entry : countryAsinMap.entrySet()) {
|
||||
String country = entry.getKey();
|
||||
setCountryData(entity, country, entry.getValue(), countryMinimumPriceMap.get(country));
|
||||
}
|
||||
|
||||
if (entity.getId() == null) {
|
||||
skipPriceAsinMapper.insert(entity);
|
||||
} else {
|
||||
skipPriceAsinMapper.updateById(entity);
|
||||
}
|
||||
SkipPriceAsinEntity saved = getById(entity.getId());
|
||||
return toItemVo(saved, group.getGroupName());
|
||||
}
|
||||
|
||||
+66
-18
@@ -47,6 +47,14 @@ public class SimilarAsinCozeClient {
|
||||
}
|
||||
|
||||
public List<SimilarAsinResultRowDto> inspect(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
|
||||
return inspect(rows, prompt, apiKey, imgSwitch, false);
|
||||
}
|
||||
|
||||
public List<SimilarAsinResultRowDto> inspect(List<SimilarAsinResultRowDto> rows,
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
boolean categorySwitch) {
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
@@ -55,7 +63,7 @@ public class SimilarAsinCozeClient {
|
||||
return rows.stream().map(this::copy).toList();
|
||||
}
|
||||
try {
|
||||
return inspectWithFallback(rows, prompt, apiKey, imgSwitch);
|
||||
return inspectWithFallback(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
} catch (Exception ex) {
|
||||
String failureMessage = failureMessage(ex);
|
||||
log.warn("[similar-asin] coze batch failed size={} err={}", rows.size(), failureMessage);
|
||||
@@ -75,9 +83,10 @@ public class SimilarAsinCozeClient {
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
boolean categorySwitch,
|
||||
CozeCredentialRef credential) throws Exception {
|
||||
CozeCredentialRef resolvedCredential = resolveCredential(credential);
|
||||
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, resolvedCredential));
|
||||
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, resolvedCredential));
|
||||
ensureSuccess(submitRoot);
|
||||
return new CozeSubmitResponse(
|
||||
extractExecuteId(submitRoot),
|
||||
@@ -87,6 +96,14 @@ public class SimilarAsinCozeClient {
|
||||
);
|
||||
}
|
||||
|
||||
public CozeSubmitResponse submitWorkflow(List<SimilarAsinResultRowDto> rows,
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
CozeCredentialRef credential) throws Exception {
|
||||
return submitWorkflow(rows, prompt, apiKey, imgSwitch, false, credential);
|
||||
}
|
||||
|
||||
public CozePollResponse pollWorkflow(String executeId) throws Exception {
|
||||
return pollWorkflow(executeId, null);
|
||||
}
|
||||
@@ -119,12 +136,12 @@ public class SimilarAsinCozeClient {
|
||||
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
|
||||
}
|
||||
|
||||
private List<SimilarAsinResultRowDto> inspectWithFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
|
||||
private List<SimilarAsinResultRowDto> inspectWithFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) {
|
||||
try {
|
||||
if (rows.size() == 1) {
|
||||
return inspectSingleRowWithRetry(rows, prompt, apiKey, imgSwitch);
|
||||
return inspectSingleRowWithRetry(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
}
|
||||
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch);
|
||||
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
if (attempt.resolvedCount() < rows.size()) {
|
||||
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
|
||||
}
|
||||
@@ -135,17 +152,17 @@ public class SimilarAsinCozeClient {
|
||||
log.warn("[similar-asin] coze batch fallback split size={} left={} right={} err={}",
|
||||
rows.size(), middle, rows.size() - middle, failureMessage(ex));
|
||||
List<SimilarAsinResultRowDto> merged = new ArrayList<>(rows.size());
|
||||
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, imgSwitch));
|
||||
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, imgSwitch));
|
||||
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, imgSwitch, categorySwitch));
|
||||
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, imgSwitch, categorySwitch));
|
||||
return merged;
|
||||
}
|
||||
throw propagate(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private List<SimilarAsinResultRowDto> inspectPartitionWithFailureFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
|
||||
private List<SimilarAsinResultRowDto> inspectPartitionWithFailureFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) {
|
||||
try {
|
||||
return inspectWithFallback(rows, prompt, apiKey, imgSwitch);
|
||||
return inspectWithFallback(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
} catch (Exception ex) {
|
||||
String failureMessage = failureMessage(ex);
|
||||
log.warn("[similar-asin] coze partition failed size={} err={}", rows.size(), failureMessage);
|
||||
@@ -153,12 +170,12 @@ public class SimilarAsinCozeClient {
|
||||
}
|
||||
}
|
||||
|
||||
private List<SimilarAsinResultRowDto> inspectSingleRowWithRetry(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
|
||||
private List<SimilarAsinResultRowDto> inspectSingleRowWithRetry(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
|
||||
SimilarAsinResultRowDto row = rows.getFirst();
|
||||
PartialCozeResultException lastFailure = null;
|
||||
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
|
||||
try {
|
||||
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch);
|
||||
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
if (attempt.resolvedCount() == rows.size()) {
|
||||
return attempt.mergedRows();
|
||||
}
|
||||
@@ -189,16 +206,16 @@ public class SimilarAsinCozeClient {
|
||||
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
|
||||
}
|
||||
|
||||
private InspectAttempt inspectOnce(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
|
||||
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, imgSwitch);
|
||||
private InspectAttempt inspectOnce(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
|
||||
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
List<CozeResult> results = parseResults(raw);
|
||||
List<SimilarAsinResultRowDto> merged = mergeRows(rows, results);
|
||||
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
|
||||
}
|
||||
|
||||
private String runWorkflowAsyncAndWait(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
|
||||
private String runWorkflowAsyncAndWait(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
|
||||
CozeCredentialRef credential = nextCredential();
|
||||
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, credential));
|
||||
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, credential));
|
||||
ensureSuccess(submitRoot);
|
||||
|
||||
String immediateData = extractResultDataText(submitRoot);
|
||||
@@ -243,8 +260,9 @@ public class SimilarAsinCozeClient {
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
boolean categorySwitch,
|
||||
CozeCredentialRef credential) {
|
||||
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey, imgSwitch);
|
||||
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
Map<String, Object> body = new LinkedHashMap<>();
|
||||
body.put("workflow_id", credential.workflowId());
|
||||
body.put("parameters", parameters);
|
||||
@@ -366,6 +384,14 @@ public class SimilarAsinCozeClient {
|
||||
}
|
||||
|
||||
private Map<String, Object> buildParameters(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
|
||||
return buildParameters(rows, prompt, apiKey, imgSwitch, false);
|
||||
}
|
||||
|
||||
private Map<String, Object> buildParameters(List<SimilarAsinResultRowDto> rows,
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
boolean categorySwitch) {
|
||||
List<String> asins = rows.stream().map(row -> safeText(row.getAsin())).toList();
|
||||
List<String> titles = rows.stream().map(row -> safeText(firstNonBlank(row.getTitle(), row.getAsin()))).toList();
|
||||
List<String> skus = rows.stream().map(row -> safeText(row.getSku())).toList();
|
||||
@@ -388,6 +414,7 @@ public class SimilarAsinCozeClient {
|
||||
parameters.put("api_key", apiKey.trim());
|
||||
}
|
||||
parameters.put("img_switch", imgSwitch);
|
||||
parameters.put("category_switch", categorySwitch);
|
||||
return parameters;
|
||||
}
|
||||
|
||||
@@ -827,7 +854,14 @@ public class SimilarAsinCozeClient {
|
||||
|| !normalize(row.getPatentRisk()).isBlank()
|
||||
|| !normalize(row.getConclusion()).isBlank()
|
||||
|| !normalize(row.getIsStock()).isBlank()
|
||||
|| !normalize(row.getSimilarity()).isBlank();
|
||||
|| !normalize(row.getSimilarity()).isBlank()
|
||||
|| !normalize(row.getIsConform()).isBlank()
|
||||
|| !normalize(row.getReason()).isBlank()
|
||||
|| !normalize(row.getCategory()).isBlank()
|
||||
|| !normalize(row.getStatus()).isBlank()
|
||||
|| !normalize(row.getMainUrl()).isBlank()
|
||||
|| !normalize(row.getPuzzleImg1()).isBlank()
|
||||
|| !normalize(row.getPuzzleImg2()).isBlank();
|
||||
}
|
||||
|
||||
private void ensureSuccess(JsonNode root) {
|
||||
@@ -978,7 +1012,21 @@ public class SimilarAsinCozeClient {
|
||||
|| item.has("appearanceReason")
|
||||
|| item.has("patent_reason")
|
||||
|| item.has("patentReason")
|
||||
|| item.has("patent reason"));
|
||||
|| item.has("patent reason")
|
||||
|| item.has("main_url")
|
||||
|| item.has("mainUrl")
|
||||
|| item.has("main_image_url")
|
||||
|| item.has("mainImageUrl")
|
||||
|| item.has("main_img")
|
||||
|| item.has("mainImg")
|
||||
|| item.has("puzzle_img1")
|
||||
|| item.has("puzzleImg1")
|
||||
|| item.has("puzzle_img_1")
|
||||
|| item.has("puzzleImg_1")
|
||||
|| item.has("puzzle_img2")
|
||||
|| item.has("puzzleImg2")
|
||||
|| item.has("puzzle_img_2")
|
||||
|| item.has("puzzleImg_2"));
|
||||
}
|
||||
|
||||
private boolean isSuccessfulWorkflowStatus(String status) {
|
||||
|
||||
+5
@@ -37,4 +37,9 @@ public class SimilarAsinParseRequest {
|
||||
@JsonAlias({"imgSwitch"})
|
||||
@Schema(description = "传递给 Coze workflow parameters.img_switch 的图片检测开关,true 为开启,false 为关闭。")
|
||||
private Boolean imgSwitch = Boolean.FALSE;
|
||||
|
||||
@JsonProperty("category_switch")
|
||||
@JsonAlias({"categorySwitch"})
|
||||
@Schema(description = "传递给 Coze workflow parameters.category_switch 的类目检测开关,true 为开启,false 为关闭。")
|
||||
private Boolean categorySwitch = Boolean.FALSE;
|
||||
}
|
||||
|
||||
+3
@@ -20,6 +20,9 @@ public class SimilarAsinParsedPayloadDto {
|
||||
@Schema(description = "传递给 Coze workflow parameters.img_switch 的图片检测开关")
|
||||
private Boolean imgSwitch = Boolean.FALSE;
|
||||
|
||||
@Schema(description = "传递给 Coze workflow parameters.category_switch 的类目检测开关")
|
||||
private Boolean categorySwitch = Boolean.FALSE;
|
||||
|
||||
@Schema(description = "本次解析的源文件列表")
|
||||
private List<SimilarAsinSourceFileDto> sourceFiles = new ArrayList<>();
|
||||
|
||||
|
||||
+3
@@ -36,6 +36,9 @@ public class SimilarAsinParseVo {
|
||||
@Schema(description = "图片检测开关,true 为开启,false 为关闭")
|
||||
private Boolean imgSwitch = Boolean.FALSE;
|
||||
|
||||
@Schema(description = "类目检测开关,true 为开启,false 为关闭")
|
||||
private Boolean categorySwitch = Boolean.FALSE;
|
||||
|
||||
@Schema(description = "兼容旧前端的平铺有效行列表,现为全部有效行")
|
||||
private List<SimilarAsinParsedRowVo> items = new ArrayList<>();
|
||||
|
||||
|
||||
+314
-82
@@ -83,6 +83,8 @@ import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
@@ -96,7 +98,7 @@ import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CancellationException;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
@@ -219,6 +221,23 @@ public class SimilarAsinTaskService {
|
||||
"阿里巴巴图片1",
|
||||
"阿里巴巴图片2"
|
||||
);
|
||||
private static final Set<String> CATEGORY_RESULT_HEADER_ALIASES = Set.of(
|
||||
FailedStatusRowFilter.canonicalizeHeader("是否符合类目"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("is_conform"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("conform")
|
||||
);
|
||||
private static final Set<String> COZE_RESULT_HEADER_ALIASES = Set.of(
|
||||
FailedStatusRowFilter.canonicalizeHeader("是否有货"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("is_stock"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("相似度"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("similarity"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("是否符合类目"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("is_conform"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("不符合理由"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("reason"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("产品类目"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("category")
|
||||
);
|
||||
|
||||
private static final int IMG_COL_MAIN = 12;
|
||||
private static final int IMG_COL_PUZZLE1 = 13;
|
||||
@@ -359,6 +378,7 @@ public class SimilarAsinTaskService {
|
||||
List<String> mergedHeaders = new ArrayList<>();
|
||||
int totalRows = 0;
|
||||
int droppedRows = 0;
|
||||
boolean categoryRetryRequired = false;
|
||||
long parseStartedAt = System.nanoTime();
|
||||
for (SimilarAsinSourceFileDto source : sourceFiles) {
|
||||
if (source.getFileKey() == null || source.getFileKey().isBlank()) {
|
||||
@@ -374,10 +394,17 @@ public class SimilarAsinTaskService {
|
||||
droppedRows += parsed.droppedRows();
|
||||
allRows.addAll(parsed.allRows());
|
||||
mergeHeaders(mergedHeaders, parsed.headers());
|
||||
categoryRetryRequired |= parsed.categoryRetryRequired();
|
||||
}
|
||||
if (allRows.isEmpty()) {
|
||||
throw new BusinessException("未解析到有效 ASIN 数据");
|
||||
}
|
||||
boolean requestedCategorySwitch = Boolean.TRUE.equals(request.getCategorySwitch());
|
||||
request.setCategorySwitch(requestedCategorySwitch || categoryRetryRequired);
|
||||
if (!requestedCategorySwitch && categoryRetryRequired) {
|
||||
log.info("[similar-asin] category retry detected from failed result rows, enable category_switch automatically files={} rows={}",
|
||||
sourceFiles.size(), allRows.size());
|
||||
}
|
||||
long parsedAt = System.nanoTime();
|
||||
List<SimilarAsinParsedGroupVo> groups = buildParsedGroups(allRows);
|
||||
long groupedAt = System.nanoTime();
|
||||
@@ -405,11 +432,11 @@ public class SimilarAsinTaskService {
|
||||
|
||||
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
|
||||
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
|
||||
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), sourceFiles, mergedHeaders, groups, allRows);
|
||||
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, mergedHeaders, groups, allRows);
|
||||
long payloadBuiltAt = System.nanoTime();
|
||||
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
|
||||
long payloadStoredAt = System.nanoTime();
|
||||
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), sourceFiles, parsedPayloadPointer));
|
||||
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, parsedPayloadPointer));
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
|
||||
@@ -449,6 +476,7 @@ public class SimilarAsinTaskService {
|
||||
vo.setGroupCount(groups.size());
|
||||
vo.setAiPrompt(normalize(request.getAiPrompt()));
|
||||
vo.setImgSwitch(Boolean.TRUE.equals(request.getImgSwitch()));
|
||||
vo.setCategorySwitch(Boolean.TRUE.equals(request.getCategorySwitch()));
|
||||
vo.setItems(new ArrayList<>(allRows));
|
||||
vo.setGroups(groups);
|
||||
long finishedAt = System.nanoTime();
|
||||
@@ -1228,10 +1256,11 @@ public class SimilarAsinTaskService {
|
||||
String prompt = readAiPrompt(task);
|
||||
String apiKey = readApiKey(task);
|
||||
boolean imgSwitch = readImgSwitch(task);
|
||||
boolean categorySwitch = readCategorySwitch(task);
|
||||
int batchSize = resolveCozeBatchSize(imgSwitch);
|
||||
List<SimilarAsinResultRowDto> result = new ArrayList<>();
|
||||
for (int i = 0; i < items.size(); i += batchSize) {
|
||||
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, imgSwitch));
|
||||
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, imgSwitch, categorySwitch));
|
||||
if (progressHook != null) {
|
||||
progressHook.run();
|
||||
}
|
||||
@@ -1753,6 +1782,73 @@ public class SimilarAsinTaskService {
|
||||
return true;
|
||||
}
|
||||
|
||||
private boolean readCategorySwitch(FileTaskEntity task) {
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(task.getResultJson());
|
||||
JsonNode savedSwitch = root == null ? null : root.get("categorySwitch");
|
||||
if (savedSwitch != null && !savedSwitch.isNull()) {
|
||||
return savedSwitch.asBoolean(false);
|
||||
}
|
||||
return Boolean.TRUE.equals(readParsedPayload(task).getCategorySwitch());
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取相似ASIN类目检测开关失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void handleResultFileJobFailure(TaskFileJobEntity job, String message) {
|
||||
if (job == null || job.getTaskId() == null) {
|
||||
return;
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle ignored =
|
||||
requireTaskLock(job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS)) {
|
||||
Long finalizedTaskId = transactionManager == null
|
||||
? persistResultFileJobFailure(job, message)
|
||||
: inNewTransaction(() -> persistResultFileJobFailure(job, message));
|
||||
if (finalizedTaskId != null) {
|
||||
taskCacheService.deleteTaskCache(finalizedTaskId);
|
||||
clearPoisonWindow(finalizedTaskId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Long persistResultFileJobFailure(TaskFileJobEntity job, String message) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) {
|
||||
return null;
|
||||
}
|
||||
String owner = ownerFromTask(task);
|
||||
if (!isOwnerCurrent(owner)) {
|
||||
log.warn("[similar-asin] finalize exhausted result job from another instance taskId={} jobId={} owner={} current={}",
|
||||
task.getId(), job.getId(), owner, currentInstanceId());
|
||||
}
|
||||
String failureMessage = firstNonBlank(message, "相似ASIN结果文件生成失败");
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
task.setStatus(STATUS_FAILED);
|
||||
task.setSuccessFileCount(0);
|
||||
task.setFailedFileCount(1);
|
||||
task.setErrorMessage(failureMessage);
|
||||
task.setUpdatedAt(now);
|
||||
task.setFinishedAt(now);
|
||||
fileTaskMapper.updateById(task);
|
||||
|
||||
FileResultEntity result = job.getResultId() == null ? null : fileResultMapper.selectById(job.getResultId());
|
||||
if (result != null && MODULE_TYPE.equals(result.getModuleType())) {
|
||||
result.setSuccess(0);
|
||||
result.setErrorMessage(failureMessage);
|
||||
result.setResultFileUrl(null);
|
||||
result.setResultFileSize(0L);
|
||||
fileResultMapper.updateById(result);
|
||||
}
|
||||
saveFileBuildProgress(task, job, 1, 1, failureMessage);
|
||||
return task.getId();
|
||||
}
|
||||
|
||||
private void finalizeExhaustedResultFileJob(TaskFileJobEntity job, String message) {
|
||||
handleResultFileJobFailure(job, message);
|
||||
taskFileJobService.markFailureFinalized(job.getId(), message);
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.similar-asin.coze-poll-delay-ms:30000}")
|
||||
public void pollPendingCozeJobs() {
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
@@ -1949,6 +2045,7 @@ public class SimilarAsinTaskService {
|
||||
String prompt = readAiPrompt(task);
|
||||
String apiKey = readApiKey(task);
|
||||
boolean imgSwitch = readImgSwitch(task);
|
||||
boolean categorySwitch = readCategorySwitch(task);
|
||||
int batchSize = resolveCozeBatchSize(imgSwitch);
|
||||
// P1-1:检测到 720712008 风暴时强制把 batch 降到 1,隔离毒行;
|
||||
// 持续 5+ 次提交命中率 ≥ 40% 才会触发,正常波动不影响吞吐。
|
||||
@@ -2013,7 +2110,7 @@ public class SimilarAsinTaskService {
|
||||
int batchIndex = 1;
|
||||
for (int i = 0; i < submitLimit; i += batchSize) {
|
||||
List<CozeCandidate> batchCandidates = readyCandidates.subList(i, Math.min(i + batchSize, submitLimit));
|
||||
pending |= submitCozeBatch(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, allRowsByBaseId);
|
||||
pending |= submitCozeBatch(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, categorySwitch, allRowsByBaseId);
|
||||
batchIndex++;
|
||||
}
|
||||
return pending || countPendingCozeStates(task.getId()) > 0;
|
||||
@@ -2129,6 +2226,7 @@ public class SimilarAsinTaskService {
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
boolean categorySwitch,
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
|
||||
if (batchCandidates == null || batchCandidates.isEmpty()) {
|
||||
return false;
|
||||
@@ -2155,7 +2253,7 @@ public class SimilarAsinTaskService {
|
||||
SimilarAsinCozeClient.CozeCredentialRef credential = cozeClient.nextCredential();
|
||||
try {
|
||||
SimilarAsinCozeClient.CozeSubmitResponse submit = submitCozeWorkflowThrottled(
|
||||
batchRows, prompt, apiKey, imgSwitch, credential, true);
|
||||
batchRows, prompt, apiKey, imgSwitch, categorySwitch, credential, true);
|
||||
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
||||
List<SimilarAsinResultRowDto> cozeRows = cozeClient.mergeRowsFromDataText(batchRows, submit.immediateData());
|
||||
String emptyResultMessage = emptyCozeResultMessage(cozeRows, batchRows.size());
|
||||
@@ -2520,7 +2618,7 @@ public class SimilarAsinTaskService {
|
||||
try {
|
||||
SimilarAsinCozeClient.CozeSubmitResponse submit =
|
||||
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
|
||||
cozeClient.credentialByName(context.credentialName()), false);
|
||||
readCategorySwitch(task), cozeClient.credentialByName(context.credentialName()), false);
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
||||
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
||||
List<SimilarAsinResultRowDto> cozeRows =
|
||||
@@ -2582,6 +2680,7 @@ public class SimilarAsinTaskService {
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
boolean categorySwitch,
|
||||
SimilarAsinCozeClient.CozeCredentialRef credential,
|
||||
boolean allowCredentialFallback) throws Exception {
|
||||
int attempts = allowCredentialFallback ? Math.max(1, cozeClient.configuredCredentialCount()) : 1;
|
||||
@@ -2615,7 +2714,7 @@ public class SimilarAsinTaskService {
|
||||
}
|
||||
try (lockHandle; borrowedCredential) {
|
||||
SimilarAsinCozeClient.CozeSubmitResponse response =
|
||||
cozeClient.submitWorkflow(rows, prompt, apiKey, imgSwitch, currentCredential);
|
||||
cozeClient.submitWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, currentCredential);
|
||||
// 提交完成立即记录时间戳,用于下次进入循环时计算节流等待。
|
||||
lastCozeSubmitAtByCredential.put(credentialKey, System.currentTimeMillis());
|
||||
return response;
|
||||
@@ -2807,7 +2906,7 @@ public class SimilarAsinTaskService {
|
||||
int partIndex = 1;
|
||||
for (List<SimilarAsinResultRowDto> partRows : partitions) {
|
||||
SimilarAsinCozeClient.CozeSubmitResponse submit =
|
||||
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
|
||||
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task), readCategorySwitch(task),
|
||||
cozeClient.credentialByName(context.credentialName()), false);
|
||||
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
||||
List<SimilarAsinResultRowDto> cozeRows =
|
||||
@@ -2941,7 +3040,7 @@ public class SimilarAsinTaskService {
|
||||
try {
|
||||
taskFileJobService.touchRunningIfStale(context.jobId(), properties.getDbJobTouchIntervalMillis());
|
||||
SimilarAsinCozeClient.CozeSubmitResponse submit =
|
||||
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
|
||||
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task), readCategorySwitch(task),
|
||||
cozeClient.credentialByName(context.credentialName()), false);
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
||||
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
||||
@@ -3280,6 +3379,9 @@ public class SimilarAsinTaskService {
|
||||
TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId());
|
||||
if (job != null) {
|
||||
taskFileJobService.markFailed(job, firstNonBlank(ex.getMessage(), "相似ASIN结果文件生成失败"));
|
||||
if (taskFileJobService.isRetryExhausted(job.getId())) {
|
||||
finalizeExhaustedResultFileJob(job, ex.getMessage());
|
||||
}
|
||||
}
|
||||
log.warn("[相似ASIN] Coze 异步收尾失败 任务ID={} 结果ID={} 错误={}",
|
||||
taskId, context.resultId(), ex.getMessage(), ex);
|
||||
@@ -3303,6 +3405,10 @@ public class SimilarAsinTaskService {
|
||||
if (job == null || "SUCCESS".equals(job.getStatus())) {
|
||||
return;
|
||||
}
|
||||
if (taskFileJobService.isRetryExhausted(job.getId())) {
|
||||
finalizeExhaustedResultFileJob(job, job.getErrorMessage());
|
||||
return;
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task != null && STATUS_RUNNING.equals(task.getStatus()) && !isResultSubmissionComplete(taskId)) {
|
||||
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
|
||||
@@ -3324,6 +3430,8 @@ public class SimilarAsinTaskService {
|
||||
if (requeued) {
|
||||
log.info("[相似ASIN] Coze 异步结果已就绪,结果文件任务已重新入队 任务ID={} 文件任务ID={} 结果ID={}",
|
||||
taskId, job.getId(), context.resultId());
|
||||
} else if (taskFileJobService.isRetryExhausted(job.getId())) {
|
||||
finalizeExhaustedResultFileJob(job, job.getErrorMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4160,6 +4268,7 @@ public class SimilarAsinTaskService {
|
||||
}
|
||||
List<SourceRows> sourceRows = splitRowsBySourceFile(parsed, parsed.getAllItems(), result.getSourceFilename());
|
||||
List<SourceResultWorkbook> workbooks = new ArrayList<>();
|
||||
List<File> workbookFiles = new ArrayList<>(sourceRows.size());
|
||||
File zip = null;
|
||||
// 跨 workbook 共享 taskImageCache:多源场景下相同 URL 仅下载一次。
|
||||
// 配合 embed() 写完即 remove(),cache 仅承载 in-flight 图片;
|
||||
@@ -4170,26 +4279,12 @@ public class SimilarAsinTaskService {
|
||||
: BoundedImageCache.DEFAULT_MAX_BYTES;
|
||||
BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes);
|
||||
try {
|
||||
// P3-1:单源文件场景仍走串行降级路径,避免引入线程切换开销;
|
||||
// 多源文件场景把 writeResultWorkbook 投到 assembleExecutor 上并发跑,
|
||||
// 1000+ 行 ×N 源文件的 assemble 阶段总耗时直接除以 N(受池大小 4 限制)。
|
||||
// assembleExecutor 是固定 4 线程池:sourceRows.size() ≤ 4 时全部并行;
|
||||
// > 4 时多余源文件排队,避免 4×5000 行同时打开图片缓存爆堆。
|
||||
if (sourceRows.size() <= 1) {
|
||||
for (SourceRows item : sourceRows) {
|
||||
String filename = safeFileStem(item.sourceFilename()) + "-result.xlsx";
|
||||
String tempFilename = safeFileStem(item.sourceFilename())
|
||||
+ "-" + task.getId()
|
||||
+ "-" + result.getId()
|
||||
+ "-" + UUID.randomUUID()
|
||||
+ "-result.xlsx";
|
||||
File xlsx = new File(outputDir, tempFilename);
|
||||
writeResultWorkbook(xlsx, item.rows(), resultMap, taskImageCache);
|
||||
workbooks.add(new SourceResultWorkbook(xlsx, filename, item.rows().size()));
|
||||
}
|
||||
} else {
|
||||
// 单源和多源统一提交真实 Future,确保超时取消能中断实际 workbook 线程。
|
||||
// 所有源文件共享同一个绝对 deadline,避免逐个 Future 各等待一轮完整超时。
|
||||
int timeoutMinutes = Math.max(1, properties.getResultFileTimeoutMinutes());
|
||||
long deadlineNanos = System.nanoTime() + TimeUnit.MINUTES.toNanos(timeoutMinutes);
|
||||
long assembleStart = System.currentTimeMillis();
|
||||
List<CompletableFuture<SourceResultWorkbook>> futures = new ArrayList<>(sourceRows.size());
|
||||
List<Future<SourceResultWorkbook>> futures = new ArrayList<>(sourceRows.size());
|
||||
for (SourceRows item : sourceRows) {
|
||||
final SourceRows captured = item;
|
||||
final String filename = safeFileStem(captured.sourceFilename()) + "-result.xlsx";
|
||||
@@ -4198,42 +4293,43 @@ public class SimilarAsinTaskService {
|
||||
+ "-" + result.getId()
|
||||
+ "-" + UUID.randomUUID()
|
||||
+ "-result.xlsx";
|
||||
futures.add(CompletableFuture.supplyAsync(() -> {
|
||||
File xlsx = new File(outputDir, tempFilename);
|
||||
final File xlsx = new File(outputDir, tempFilename);
|
||||
workbookFiles.add(xlsx);
|
||||
futures.add(assembleExecutor.submit(() -> {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
writeResultWorkbook(xlsx, captured.rows(), resultMap, taskImageCache);
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
return new SourceResultWorkbook(xlsx, filename, captured.rows().size());
|
||||
}, assembleExecutor));
|
||||
}));
|
||||
}
|
||||
for (CompletableFuture<SourceResultWorkbook> future : futures) {
|
||||
try {
|
||||
// 单源文件 30 分钟硬上限:超时直接抛错,避免被慢源永久阻塞。
|
||||
workbooks.add(future.get(30, TimeUnit.MINUTES));
|
||||
} catch (InterruptedException ie) {
|
||||
for (Future<SourceResultWorkbook> future : futures) {
|
||||
long remainingNanos = deadlineNanos - System.nanoTime();
|
||||
if (remainingNanos <= 0L) {
|
||||
throw new TimeoutException("result file deadline reached");
|
||||
}
|
||||
workbooks.add(future.get(remainingNanos, TimeUnit.NANOSECONDS));
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
cancelResultAssemblyFutures(futures);
|
||||
Thread.currentThread().interrupt();
|
||||
for (CompletableFuture<SourceResultWorkbook> remaining : futures) {
|
||||
remaining.cancel(true);
|
||||
throw new BusinessException("生成相似ASIN检测结果中断", ex);
|
||||
} catch (TimeoutException ex) {
|
||||
cancelResultAssemblyFutures(futures);
|
||||
throw new BusinessException("生成相似ASIN检测结果超时(" + timeoutMinutes + " 分钟)", ex);
|
||||
} catch (CancellationException ex) {
|
||||
cancelResultAssemblyFutures(futures);
|
||||
throw new BusinessException("生成相似ASIN检测结果已取消", ex);
|
||||
} catch (ExecutionException ex) {
|
||||
cancelResultAssemblyFutures(futures);
|
||||
Throwable cause = ex.getCause();
|
||||
if (cause instanceof BusinessException businessException) {
|
||||
throw businessException;
|
||||
}
|
||||
throw new BusinessException("生成相似ASIN检测结果中断", ie);
|
||||
} catch (TimeoutException te) {
|
||||
for (CompletableFuture<SourceResultWorkbook> remaining : futures) {
|
||||
remaining.cancel(true);
|
||||
}
|
||||
throw new BusinessException("生成相似ASIN检测结果超时(30 分钟)", te);
|
||||
} catch (ExecutionException ee) {
|
||||
Throwable cause = ee.getCause();
|
||||
for (CompletableFuture<SourceResultWorkbook> remaining : futures) {
|
||||
remaining.cancel(true);
|
||||
}
|
||||
if (cause instanceof BusinessException be) {
|
||||
throw be;
|
||||
}
|
||||
throw new BusinessException("生成相似ASIN检测结果失败",
|
||||
cause != null ? cause : ee);
|
||||
}
|
||||
}
|
||||
log.info("[similar-asin] assemble parallel finished taskId={} sources={} costMs={}",
|
||||
task.getId(), sourceRows.size(), System.currentTimeMillis() - assembleStart);
|
||||
throw new BusinessException("生成相似ASIN检测结果失败", cause != null ? cause : ex);
|
||||
}
|
||||
log.info("[similar-asin] assemble workbooks finished taskId={} sources={} timeoutMinutes={} costMs={}",
|
||||
task.getId(), sourceRows.size(), timeoutMinutes, System.currentTimeMillis() - assembleStart);
|
||||
log.info("[similar-asin] assemble image-cache stats taskId={} sources={} cacheSize={} currentBytes={} evictedCount={} evictedBytes={} maxBytes={}",
|
||||
task.getId(), sourceRows.size(), taskImageCache.size(),
|
||||
taskImageCache.currentBytes(), taskImageCache.evictedCount(),
|
||||
@@ -4272,9 +4368,9 @@ public class SimilarAsinTaskService {
|
||||
result.setResultContentType(contentType);
|
||||
result.setRowCount(parsed.getAllItems().size());
|
||||
} finally {
|
||||
for (SourceResultWorkbook workbook : workbooks) {
|
||||
if (workbook.file().exists() && !workbook.file().delete()) {
|
||||
log.warn("[similar-asin] delete temp xlsx failed file={}", workbook.file());
|
||||
for (File workbookFile : workbookFiles) {
|
||||
if (workbookFile.exists() && !workbookFile.delete()) {
|
||||
log.warn("[similar-asin] delete temp xlsx failed file={}", workbookFile);
|
||||
}
|
||||
}
|
||||
if (zip != null && zip.exists() && !zip.delete()) {
|
||||
@@ -4482,9 +4578,18 @@ public class SimilarAsinTaskService {
|
||||
List<SimilarAsinParsedRowVo> rowsToWrite,
|
||||
Map<String, SimilarAsinResultRowDto> resultMap,
|
||||
Map<String, SimilarAsinImageEmbedder.ResizedImage> taskImageCache) {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
// Excel 365 "Place in Cell" 图片:写完 workbook 后 patch richData 单元格图片结构。
|
||||
// 这不是浮动 Drawing,因此点击图片区域会选中单元格,图片不能被拖到任意位置,也不需要工作表保护。
|
||||
ExcelCellImageWriter.Session excelCellImageSession = ExcelCellImageWriter.createSession();
|
||||
SimilarAsinImageEmbedder.ImageSpool imageSpool;
|
||||
try {
|
||||
Path spoolDir = Files.createTempDirectory(xlsx.getParentFile().toPath(), "similar-asin-images-");
|
||||
imageSpool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
|
||||
} catch (IOException ex) {
|
||||
throw new BusinessException("create Similar ASIN image spool failed", ex);
|
||||
}
|
||||
try {
|
||||
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
|
||||
Sheet sheet = workbook.createSheet("相似asin检测");
|
||||
CellStyle headerStyle = workbook.createCellStyle();
|
||||
@@ -4506,17 +4611,16 @@ public class SimilarAsinTaskService {
|
||||
cell.setCellStyle(headerStyle);
|
||||
}
|
||||
|
||||
// P2-8:第一遍预扫所有图片 URL,并行下载到 taskImageCache,
|
||||
// POI 写入仍单线程串行注册 cell image(cache 命中直接 resize + register),下载/写入解耦。
|
||||
// 预扫图片 URL 并行下载到任务级临时目录;POI 仍按行串行注册 cell image。
|
||||
List<String> prefetchUrls = collectImageUrlsForPrefetch(rowsToWrite, resultMap);
|
||||
if (!prefetchUrls.isEmpty()) {
|
||||
// P2-11:先一次性把 DB cache 命中的字节填入 taskImageCache,避免再次走网络。
|
||||
// 命中部分从 prefetchUrls 剔除,剩余的真正未命中的 URL 才走 imageEmbedder.prefetch 网络下载。
|
||||
// DB cache 命中的缩略图也立即落到任务临时目录,避免整批字节停留在堆中。
|
||||
long dbCacheStart = System.currentTimeMillis();
|
||||
List<String> remainingUrls = new ArrayList<>(prefetchUrls.size());
|
||||
int dbHit = 0;
|
||||
for (String url : prefetchUrls) {
|
||||
if (url == null || taskImageCache.containsKey(url)) {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
if (url == null || imageSpool.get(url) != null || taskImageCache.containsKey(url)) {
|
||||
continue;
|
||||
}
|
||||
byte[] cachedBytes = imagePrefetchService.lookup(url);
|
||||
@@ -4526,7 +4630,7 @@ public class SimilarAsinTaskService {
|
||||
}
|
||||
SimilarAsinImageEmbedder.ResizedImage thumb = imageEmbedder.decodeCachedThumb(cachedBytes);
|
||||
if (thumb != null) {
|
||||
taskImageCache.putIfAbsent(url, thumb);
|
||||
imageSpool.put(url, thumb);
|
||||
dbHit++;
|
||||
} else {
|
||||
remainingUrls.add(url);
|
||||
@@ -4536,14 +4640,16 @@ public class SimilarAsinTaskService {
|
||||
prefetchUrls.size(), dbHit, remainingUrls.size(), System.currentTimeMillis() - dbCacheStart);
|
||||
if (!remainingUrls.isEmpty()) {
|
||||
long prefetchStart = System.currentTimeMillis();
|
||||
imageEmbedder.prefetch(remainingUrls, taskImageCache);
|
||||
log.info("[similar-asin] image prefetch finished urls={} cached={} costMs={}",
|
||||
remainingUrls.size(), taskImageCache.size(), System.currentTimeMillis() - prefetchStart);
|
||||
imageEmbedder.prefetchToDisk(remainingUrls, imageSpool);
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
log.info("[similar-asin] image disk-prefetch finished urls={} spooled={} costMs={}",
|
||||
remainingUrls.size(), imageSpool.size(), System.currentTimeMillis() - prefetchStart);
|
||||
}
|
||||
}
|
||||
|
||||
int rowIndex = 1;
|
||||
for (SimilarAsinParsedRowVo parsedRow : rowsToWrite == null ? List.<SimilarAsinParsedRowVo>of() : rowsToWrite) {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
SimilarAsinResultRowDto resultRow = findResultRow(parsedRow, resultMap);
|
||||
Row row = sheet.createRow(rowIndex);
|
||||
int col = 0;
|
||||
@@ -4575,21 +4681,21 @@ public class SimilarAsinTaskService {
|
||||
// 行高固定到 IMAGE_ROW_HEIGHT_POINTS(Excel 上限 409pt):
|
||||
row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS);
|
||||
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_MAIN,
|
||||
resultRow.getMainUrl(), row, taskImageCache, excelCellImageSession);
|
||||
resultRow.getMainUrl(), row, taskImageCache, excelCellImageSession, imageSpool);
|
||||
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE1,
|
||||
resultRow.getPuzzleImg1(), row, taskImageCache, excelCellImageSession);
|
||||
resultRow.getPuzzleImg1(), row, taskImageCache, excelCellImageSession, imageSpool);
|
||||
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE2,
|
||||
resultRow.getPuzzleImg2(), row, taskImageCache, excelCellImageSession);
|
||||
resultRow.getPuzzleImg2(), row, taskImageCache, excelCellImageSession, imageSpool);
|
||||
}
|
||||
rowIndex++;
|
||||
}
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
workbook.write(fos);
|
||||
workbook.dispose();
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("生成相似ASIN检测结果失败", ex);
|
||||
}
|
||||
if (!excelCellImageSession.isEmpty()) {
|
||||
try {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
ExcelCellImageWriter.patchXlsxFile(xlsx, excelCellImageSession);
|
||||
} catch (IOException ex) {
|
||||
String cause = ex.toString();
|
||||
@@ -4598,6 +4704,32 @@ public class SimilarAsinTaskService {
|
||||
throw new BusinessException("写入相似ASIN结果 Excel 单元格图片失败: " + cause, ex);
|
||||
}
|
||||
}
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("生成相似ASIN检测结果失败", ex);
|
||||
} finally {
|
||||
try {
|
||||
imageSpool.close();
|
||||
} catch (IOException ex) {
|
||||
log.warn("[similar-asin] delete image spool failed xlsx={} err={}",
|
||||
xlsx.getAbsolutePath(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void cancelResultAssemblyFutures(List<? extends Future<?>> futures) {
|
||||
for (Future<?> future : futures) {
|
||||
if (future != null && !future.isDone()) {
|
||||
future.cancel(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ensureResultAssemblyNotInterrupted() {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
throw new BusinessException("生成相似ASIN检测结果中断");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -4714,24 +4846,29 @@ public class SimilarAsinTaskService {
|
||||
vo.setValues(readRowValues(row, headers, formatter));
|
||||
allRows.add(vo);
|
||||
}
|
||||
boolean includeBlankStatusRows = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
|
||||
boolean resultWorkbook = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
|
||||
boolean firstPassResultWorkbook = isFirstPassResultWorkbook(headers, allRows);
|
||||
boolean includeBlankStatusRows = resultWorkbook && !firstPassResultWorkbook;
|
||||
FailedStatusRowFilter.FilterResult<SimilarAsinParsedRowVo> filteredRows =
|
||||
FailedStatusRowFilter.retainRows(
|
||||
allRows,
|
||||
statusCol >= 0,
|
||||
rowVo -> statusHeader == null || rowVo.getValues() == null ? "" : rowVo.getValues().get(statusHeader),
|
||||
status -> FailedStatusRowFilter.matchesFailedStatus(status)
|
||||
|| firstPassResultWorkbook
|
||||
|| (includeBlankStatusRows && FailedStatusRowFilter.isBlankStatus(status))
|
||||
);
|
||||
dropped += filteredRows.filteredCount();
|
||||
allRows = new ArrayList<>(filteredRows.rows());
|
||||
boolean categoryRetryRequired = shouldEnableCategorySwitchForRetry(
|
||||
headers, allRows, includeBlankStatusRows);
|
||||
if (statusCol >= 0 && validRows > 0 && allRows.isEmpty()) {
|
||||
throw new BusinessException(FailedStatusRowFilter.noMatchedRowsMessage());
|
||||
}
|
||||
if (allRows.isEmpty()) {
|
||||
throw new BusinessException("no valid similar ASIN rows");
|
||||
}
|
||||
return new ParsedWorkbook(total, dropped, headers, allRows);
|
||||
return new ParsedWorkbook(total, dropped, headers, allRows, categoryRetryRequired);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
@@ -4763,7 +4900,10 @@ public class SimilarAsinTaskService {
|
||||
|| hasUsableCozeField(row.getIsConform())
|
||||
|| hasUsableCozeField(row.getReason())
|
||||
|| hasUsableCozeField(row.getCategory())
|
||||
|| hasUsableCozeField(row.getStatus());
|
||||
|| hasUsableCozeField(row.getStatus())
|
||||
|| hasText(row.getMainUrl())
|
||||
|| hasText(row.getPuzzleImg1())
|
||||
|| hasText(row.getPuzzleImg2());
|
||||
}
|
||||
|
||||
static String resolveResultStatus(SimilarAsinResultRowDto row, String... visibleResultValues) {
|
||||
@@ -4885,7 +5025,16 @@ public class SimilarAsinTaskService {
|
||||
}
|
||||
|
||||
private String cell(Row row, int col, DataFormatter formatter) {
|
||||
return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col)));
|
||||
if (col < 0) {
|
||||
return "";
|
||||
}
|
||||
String value = normalize(formatter.formatCellValue(row.getCell(col)));
|
||||
return isSpreadsheetErrorValue(value) ? "" : value;
|
||||
}
|
||||
|
||||
private static boolean isSpreadsheetErrorValue(String value) {
|
||||
String normalized = value == null ? "" : value.trim();
|
||||
return normalized.startsWith("#") && normalized.endsWith("!");
|
||||
}
|
||||
|
||||
private String baseId(String id) {
|
||||
@@ -5373,11 +5522,12 @@ public class SimilarAsinTaskService {
|
||||
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
private String buildParsedPayloadJson(String aiPrompt, String apiKey, Boolean imgSwitch, List<SimilarAsinSourceFileDto> sourceFiles, List<String> headers, List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
|
||||
private String buildParsedPayloadJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, List<SimilarAsinSourceFileDto> sourceFiles, List<String> headers, List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
payload.setAiPrompt(normalize(aiPrompt));
|
||||
payload.setApiKey(normalize(apiKey));
|
||||
payload.setImgSwitch(Boolean.TRUE.equals(imgSwitch));
|
||||
payload.setCategorySwitch(Boolean.TRUE.equals(categorySwitch));
|
||||
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
|
||||
payload.setHeaders(headers == null ? List.of() : headers);
|
||||
payload.setItems(allRows == null ? List.of() : new ArrayList<>(allRows));
|
||||
@@ -5386,11 +5536,12 @@ public class SimilarAsinTaskService {
|
||||
return writeJson(payload, "保存解析结果失败");
|
||||
}
|
||||
|
||||
private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, List<SimilarAsinSourceFileDto> sourceFiles, String parsedPayloadPointer) {
|
||||
private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, List<SimilarAsinSourceFileDto> sourceFiles, String parsedPayloadPointer) {
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put("aiPrompt", normalize(aiPrompt));
|
||||
payload.put("apiKey", normalize(apiKey));
|
||||
payload.put("imgSwitch", Boolean.TRUE.equals(imgSwitch));
|
||||
payload.put("categorySwitch", Boolean.TRUE.equals(categorySwitch));
|
||||
payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream()
|
||||
.map(SimilarAsinSourceFileDto::getFileKey)
|
||||
.filter(Objects::nonNull)
|
||||
@@ -5746,6 +5897,83 @@ public class SimilarAsinTaskService {
|
||||
|| hasHeader(headers, "产品类目"));
|
||||
}
|
||||
|
||||
static boolean isFirstPassResultWorkbook(List<String> headers,
|
||||
List<SimilarAsinParsedRowVo> rows) {
|
||||
if (headers == null || headers.isEmpty() || rows == null || rows.isEmpty()
|
||||
|| FailedStatusRowFilter.findStatusColumnIndex(headers) < 0
|
||||
|| (!hasCanonicalHeader(headers, "是否有货")
|
||||
&& !hasCanonicalHeader(headers, "相似度")
|
||||
&& !hasCanonicalHeader(headers, "是否符合类目")
|
||||
&& !hasCanonicalHeader(headers, "不符合理由")
|
||||
&& !hasCanonicalHeader(headers, "产品类目"))
|
||||
|| (!hasCanonicalHeader(headers, "主图")
|
||||
&& !hasCanonicalHeader(headers, "阿里巴巴图片1")
|
||||
&& !hasCanonicalHeader(headers, "阿里巴巴图片2"))) {
|
||||
return false;
|
||||
}
|
||||
String statusHeader = headers.get(FailedStatusRowFilter.findStatusColumnIndex(headers));
|
||||
for (SimilarAsinParsedRowVo row : rows) {
|
||||
if (row == null || FailedStatusRowFilter.matchesFailedStatus(valueByHeader(row, statusHeader))) {
|
||||
return false;
|
||||
}
|
||||
for (Map.Entry<String, String> entry : row.getValues() == null
|
||||
? Map.<String, String>of().entrySet() : row.getValues().entrySet()) {
|
||||
if (COZE_RESULT_HEADER_ALIASES.contains(FailedStatusRowFilter.canonicalizeHeader(entry.getKey()))
|
||||
&& !isBlankOrSpreadsheetError(entry.getValue())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static boolean hasCanonicalHeader(List<String> headers, String expected) {
|
||||
String canonical = FailedStatusRowFilter.canonicalizeHeader(expected);
|
||||
return headers.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.anyMatch(header -> canonical.equals(FailedStatusRowFilter.canonicalizeHeader(header)));
|
||||
}
|
||||
|
||||
private static String valueByHeader(SimilarAsinParsedRowVo row, String header) {
|
||||
if (row == null || row.getValues() == null) {
|
||||
return "";
|
||||
}
|
||||
String canonical = FailedStatusRowFilter.canonicalizeHeader(header);
|
||||
return row.getValues().entrySet().stream()
|
||||
.filter(entry -> canonical.equals(FailedStatusRowFilter.canonicalizeHeader(entry.getKey())))
|
||||
.map(Map.Entry::getValue)
|
||||
.findFirst()
|
||||
.orElse("");
|
||||
}
|
||||
|
||||
private static boolean isBlankOrSpreadsheetError(String value) {
|
||||
return value == null || value.trim().isBlank() || isSpreadsheetErrorValue(value);
|
||||
}
|
||||
|
||||
static boolean shouldEnableCategorySwitchForRetry(List<String> headers,
|
||||
List<SimilarAsinParsedRowVo> rows,
|
||||
boolean resultWorkbook) {
|
||||
if (!resultWorkbook || headers == null || headers.isEmpty() || rows == null || rows.isEmpty()
|
||||
|| FailedStatusRowFilter.findStatusColumnIndex(headers) < 0) {
|
||||
return false;
|
||||
}
|
||||
String categoryHeader = headers.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(header -> CATEGORY_RESULT_HEADER_ALIASES.contains(
|
||||
FailedStatusRowFilter.canonicalizeHeader(header)))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
if (categoryHeader == null) {
|
||||
return false;
|
||||
}
|
||||
return rows.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(SimilarAsinParsedRowVo::getValues)
|
||||
.anyMatch(values -> values == null
|
||||
|| values.get(categoryHeader) == null
|
||||
|| values.get(categoryHeader).trim().isBlank());
|
||||
}
|
||||
|
||||
private boolean hasHeader(List<String> headers, String candidate) {
|
||||
if (headers == null || headers.isEmpty()) {
|
||||
return false;
|
||||
@@ -5926,6 +6154,10 @@ public class SimilarAsinTaskService {
|
||||
long size) {
|
||||
}
|
||||
|
||||
private record ParsedWorkbook(int totalRows, int droppedRows, List<String> headers, List<SimilarAsinParsedRowVo> allRows) {
|
||||
private record ParsedWorkbook(int totalRows,
|
||||
int droppedRows,
|
||||
List<String> headers,
|
||||
List<SimilarAsinParsedRowVo> allRows,
|
||||
boolean categoryRetryRequired) {
|
||||
}
|
||||
}
|
||||
|
||||
+53
-18
@@ -13,8 +13,10 @@ import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Enumeration;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -41,6 +43,9 @@ public final class ExcelCellImageWriter {
|
||||
private static final String RD_RICH_VALUE_TYPES_PART = "xl/richData/rdRichValueTypes.xml";
|
||||
private static final String RICH_VALUE_REL_PART = "xl/richData/richValueRel.xml";
|
||||
private static final String METADATA_PART = "xl/metadata.xml";
|
||||
private static final Pattern SHEET_CELL_PATTERN = Pattern.compile(
|
||||
"<c\\b(?=[^>]*\\br=\"([A-Z]+[1-9][0-9]*)\")[^>]*(?:/>|>.*?</c>)",
|
||||
Pattern.DOTALL);
|
||||
|
||||
private static final String REL_TYPE_IMAGE =
|
||||
"http://schemas.openxmlformats.org/officeDocument/2006/relationships/image";
|
||||
@@ -116,7 +121,11 @@ public final class ExcelCellImageWriter {
|
||||
writeEntry(zout, METADATA_PART, buildMetadata(session).getBytes(StandardCharsets.UTF_8));
|
||||
int idx = 1;
|
||||
for (RegisteredCellImage image : session.images) {
|
||||
if (image.bytes != null) {
|
||||
writeEntry(zout, mediaPart(idx), image.bytes);
|
||||
} else {
|
||||
writeEntry(zout, mediaPart(idx), image.path);
|
||||
}
|
||||
idx++;
|
||||
}
|
||||
}
|
||||
@@ -166,26 +175,34 @@ public final class ExcelCellImageWriter {
|
||||
|
||||
private static PatchSheetResult patchSheet(byte[] original, Session session) {
|
||||
String content = new String(original, StandardCharsets.UTF_8);
|
||||
int idx = 0;
|
||||
Map<String, Integer> metadataIndexByCell = new HashMap<>(session.images.size());
|
||||
for (int idx = 0; idx < session.images.size(); idx++) {
|
||||
metadataIndexByCell.put(session.images.get(idx).cellRef, idx + 1);
|
||||
}
|
||||
|
||||
Set<String> missingCells = new HashSet<>(metadataIndexByCell.keySet());
|
||||
Matcher matcher = SHEET_CELL_PATTERN.matcher(content);
|
||||
StringBuilder patched = new StringBuilder(content.length());
|
||||
int patchedCount = 0;
|
||||
for (RegisteredCellImage image : session.images) {
|
||||
// [MS-XLSX] §2.2.10: cell @vm 是 1-based valueMetadata 索引,0 表示"无 metadata"。
|
||||
// futureMetadata/valueMetadata/rdrichvalue 内部块仍是 0-based,所以这里写 idx+1
|
||||
// 与 buildMetadata/buildRdRichValue 中的 i 解耦。Excel 365 在 vm=0 时会把 cell
|
||||
// 直接当作无图片的 #VALUE! 错误格渲染(任务 10498 J2 复现),必须改 1-based。
|
||||
String replacement = "<c r=\"" + image.cellRef + "\" t=\"e\" vm=\"" + (idx + 1) + "\"><v>#VALUE!</v></c>";
|
||||
Pattern pattern = Pattern.compile("<c\\b(?=[^>]*\\br=\"" + Pattern.quote(image.cellRef)
|
||||
+ "\")[^>]*(?:/>|>.*?</c>)", Pattern.DOTALL);
|
||||
Matcher matcher = pattern.matcher(content);
|
||||
if (matcher.find()) {
|
||||
content = matcher.replaceFirst(Matcher.quoteReplacement(replacement));
|
||||
while (matcher.find()) {
|
||||
String cellRef = matcher.group(1);
|
||||
Integer metadataIndex = metadataIndexByCell.get(cellRef);
|
||||
if (metadataIndex == null) {
|
||||
matcher.appendReplacement(patched, Matcher.quoteReplacement(matcher.group()));
|
||||
continue;
|
||||
}
|
||||
// [MS-XLSX] §2.2.10: cell @vm is a 1-based valueMetadata index.
|
||||
String replacement = "<c r=\"" + cellRef + "\" t=\"e\" vm=\"" + metadataIndex
|
||||
+ "\"><v>#VALUE!</v></c>";
|
||||
matcher.appendReplacement(patched, Matcher.quoteReplacement(replacement));
|
||||
missingCells.remove(cellRef);
|
||||
patchedCount++;
|
||||
} else {
|
||||
log.warn("[excel-cell-image] cell {} not found in sheet xml, skip image", image.cellRef);
|
||||
}
|
||||
idx++;
|
||||
matcher.appendTail(patched);
|
||||
for (String cellRef : missingCells) {
|
||||
log.warn("[excel-cell-image] cell {} not found in sheet xml, skip image", cellRef);
|
||||
}
|
||||
return new PatchSheetResult(content.getBytes(StandardCharsets.UTF_8), patchedCount);
|
||||
return new PatchSheetResult(patched.toString().getBytes(StandardCharsets.UTF_8), patchedCount);
|
||||
}
|
||||
|
||||
private static byte[] patchContentTypes(byte[] original) {
|
||||
@@ -386,6 +403,16 @@ public final class ExcelCellImageWriter {
|
||||
zout.closeEntry();
|
||||
}
|
||||
|
||||
private static void writeEntry(ZipOutputStream zout, String name, Path path) throws IOException {
|
||||
ZipEntry entry = new ZipEntry(name);
|
||||
zout.putNextEntry(entry);
|
||||
try (InputStream in = Files.newInputStream(path)) {
|
||||
in.transferTo(zout);
|
||||
} finally {
|
||||
zout.closeEntry();
|
||||
}
|
||||
}
|
||||
|
||||
private static void writeDirectoryEntry(ZipOutputStream zout, String name) throws IOException {
|
||||
ZipEntry entry = new ZipEntry(name);
|
||||
zout.putNextEntry(entry);
|
||||
@@ -399,7 +426,15 @@ public final class ExcelCellImageWriter {
|
||||
if (jpegBytes == null || jpegBytes.length == 0) {
|
||||
return;
|
||||
}
|
||||
images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), jpegBytes));
|
||||
images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), jpegBytes, null));
|
||||
}
|
||||
|
||||
public synchronized void registerImage(int rowIdx, int colIdx, Path jpegPath) {
|
||||
if (jpegPath == null || !Files.isRegularFile(jpegPath)) {
|
||||
return;
|
||||
}
|
||||
images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), null,
|
||||
jpegPath.toAbsolutePath().normalize()));
|
||||
}
|
||||
|
||||
public synchronized boolean isEmpty() {
|
||||
@@ -426,7 +461,7 @@ public final class ExcelCellImageWriter {
|
||||
}
|
||||
}
|
||||
|
||||
private record RegisteredCellImage(String cellRef, byte[] bytes) {
|
||||
private record RegisteredCellImage(String cellRef, byte[] bytes, Path path) {
|
||||
}
|
||||
|
||||
private record PatchSheetResult(byte[] bytes, int patchedCount) {
|
||||
|
||||
+405
-77
@@ -27,14 +27,22 @@ import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InterruptedIOException;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.AtomicMoveNotSupportedException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.nio.file.attribute.FileTime;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
@@ -42,6 +50,8 @@ import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.CompletionService;
|
||||
import java.util.concurrent.ExecutorCompletionService;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
@@ -49,7 +59,6 @@ import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
/**
|
||||
@@ -68,8 +77,9 @@ public class SimilarAsinImageEmbedder {
|
||||
static final int DEFAULT_DOWNLOAD_TIMEOUT_SECONDS = 5;
|
||||
/** P2-10:retry 由 1 升到 2,配合 5s timeout 单图最坏耗时 ≈ 15s。 */
|
||||
static final int DOWNLOAD_MAX_RETRY = 2;
|
||||
/** P2-10:原 8 → P2-8 16 → P2-10 32,1000+ 行 ×3 列场景下显著拉低 assemble 阶段尾延迟。 */
|
||||
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 32;
|
||||
/** 图片下载、解码、缩放都在该池执行;默认限制为 8,避免结果生成占满整机 CPU。 */
|
||||
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 8;
|
||||
static final int DEFAULT_PREFETCH_TIMEOUT_SECONDS = 1800;
|
||||
// 单元格固定尺寸,图片在其中等比缩放(不拉伸);resize 仍按长边 1280 px 控制堆体积。
|
||||
public static final float IMAGE_ROW_HEIGHT_POINTS = 409f;
|
||||
public static final int IMAGE_COL_WIDTH_CHARS = 80;
|
||||
@@ -102,16 +112,28 @@ public class SimilarAsinImageEmbedder {
|
||||
|
||||
private final int downloadTimeoutSeconds;
|
||||
private final int downloadPoolSize;
|
||||
private final int prefetchTimeoutSeconds;
|
||||
private final OkHttpClient httpClient;
|
||||
private final ExecutorService downloadPool;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final Path localImageCacheDir;
|
||||
|
||||
public SimilarAsinImageEmbedder(SimilarAsinProperties properties, OssStorageService ossStorageService) {
|
||||
int rawTimeout = properties == null ? DEFAULT_DOWNLOAD_TIMEOUT_SECONDS : properties.getImageDownloadTimeoutSeconds();
|
||||
int rawPool = properties == null ? DEFAULT_DOWNLOAD_POOL_SIZE : properties.getImageDownloadPoolSize();
|
||||
int rawPrefetchTimeout = properties == null
|
||||
? DEFAULT_PREFETCH_TIMEOUT_SECONDS
|
||||
: properties.getImagePrefetchTimeoutSeconds();
|
||||
this.downloadTimeoutSeconds = rawTimeout > 0 ? rawTimeout : DEFAULT_DOWNLOAD_TIMEOUT_SECONDS;
|
||||
this.downloadPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE;
|
||||
this.prefetchTimeoutSeconds = rawPrefetchTimeout > 0
|
||||
? rawPrefetchTimeout
|
||||
: DEFAULT_PREFETCH_TIMEOUT_SECONDS;
|
||||
this.ossStorageService = Objects.requireNonNull(ossStorageService, "ossStorageService must not be null");
|
||||
String configuredCacheDir = properties == null ? null : properties.getImageLocalCacheDir();
|
||||
this.localImageCacheDir = configuredCacheDir == null || configuredCacheDir.isBlank()
|
||||
? null
|
||||
: Path.of(configuredCacheDir.trim()).toAbsolutePath().normalize();
|
||||
this.httpClient = new OkHttpClient.Builder()
|
||||
.connectTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
|
||||
.readTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
|
||||
@@ -138,6 +160,10 @@ public class SimilarAsinImageEmbedder {
|
||||
return downloadPoolSize;
|
||||
}
|
||||
|
||||
int prefetchTimeoutSeconds() {
|
||||
return prefetchTimeoutSeconds;
|
||||
}
|
||||
|
||||
OkHttpClient httpClient() {
|
||||
return httpClient;
|
||||
}
|
||||
@@ -159,13 +185,13 @@ public class SimilarAsinImageEmbedder {
|
||||
if (urls == null || urls.isEmpty() || taskImageCache == null) {
|
||||
return;
|
||||
}
|
||||
Set<String> distinctUrls = new HashSet<>();
|
||||
Set<String> distinctUrls = new LinkedHashSet<>();
|
||||
for (String url : urls) {
|
||||
if (url == null) {
|
||||
continue;
|
||||
}
|
||||
String trimmed = url.trim();
|
||||
if (trimmed.isEmpty() || taskImageCache.containsKey(trimmed)) {
|
||||
if (trimmed.isEmpty() || taskImageCache.containsKey(trimmed) || hasLocalCachedThumb(trimmed)) {
|
||||
continue;
|
||||
}
|
||||
distinctUrls.add(trimmed);
|
||||
@@ -174,8 +200,51 @@ public class SimilarAsinImageEmbedder {
|
||||
return;
|
||||
}
|
||||
CompletionService<Object> completion = new ExecutorCompletionService<>(downloadPool);
|
||||
List<Future<?>> futures = new ArrayList<>(distinctUrls.size());
|
||||
for (String url : distinctUrls) {
|
||||
Iterator<String> pending = distinctUrls.iterator();
|
||||
List<Future<?>> active = new ArrayList<>(Math.min(downloadPoolSize, distinctUrls.size()));
|
||||
while (pending.hasNext() && active.size() < downloadPoolSize) {
|
||||
active.add(submitPrefetch(completion, pending.next(), taskImageCache));
|
||||
}
|
||||
// 全局 deadline:每 url 给 200ms 的预算,clamp 到 [15s, 120s]。
|
||||
long globalDeadlineMs = Math.min(120_000L, Math.max(15_000L, distinctUrls.size() * 200L));
|
||||
long deadline = System.currentTimeMillis() + globalDeadlineMs;
|
||||
int total = distinctUrls.size();
|
||||
int done = 0;
|
||||
boolean deadlineReached = false;
|
||||
while (!active.isEmpty()) {
|
||||
long left = deadline - System.currentTimeMillis();
|
||||
if (left <= 0) {
|
||||
deadlineReached = true;
|
||||
break;
|
||||
}
|
||||
try {
|
||||
Future<Object> f = completion.poll(left, TimeUnit.MILLISECONDS);
|
||||
if (f == null) {
|
||||
deadlineReached = true;
|
||||
break;
|
||||
}
|
||||
active.remove(f);
|
||||
done++;
|
||||
if (pending.hasNext()) {
|
||||
active.add(submitPrefetch(completion, pending.next(), taskImageCache));
|
||||
}
|
||||
} catch (InterruptedException ie) {
|
||||
Thread.currentThread().interrupt();
|
||||
cancelAll(active);
|
||||
break;
|
||||
}
|
||||
}
|
||||
// 超时未完成的 future 主动 cancel,避免 idle 持有 OkHttp 连接。
|
||||
if (deadlineReached) {
|
||||
cancelAll(active);
|
||||
log.info("[similar-asin][image] prefetch deadline reached total={} done={} cancelled={}",
|
||||
total, done, total - done);
|
||||
}
|
||||
}
|
||||
|
||||
private Future<?> submitPrefetch(CompletionService<Object> completion,
|
||||
String url,
|
||||
Map<String, ResizedImage> taskImageCache) {
|
||||
// 显式作为 Runnable 提交(带 null result),避免与 Callable<Object> 重载产生歧义。
|
||||
Runnable task = () -> {
|
||||
try {
|
||||
@@ -191,38 +260,106 @@ public class SimilarAsinImageEmbedder {
|
||||
log.debug("[similar-asin][image] prefetch-fail url={} err={}", url, errorSummary(ex));
|
||||
}
|
||||
};
|
||||
futures.add(completion.submit(task, null));
|
||||
return completion.submit(task, null);
|
||||
}
|
||||
// 全局 deadline:每 url 给 200ms 的预算,clamp 到 [15s, 120s]。
|
||||
long globalDeadlineMs = Math.min(120_000L, Math.max(15_000L, distinctUrls.size() * 200L));
|
||||
long deadline = System.currentTimeMillis() + globalDeadlineMs;
|
||||
int total = distinctUrls.size();
|
||||
int done = 0;
|
||||
while (done < total) {
|
||||
long left = deadline - System.currentTimeMillis();
|
||||
if (left <= 0) {
|
||||
|
||||
/**
|
||||
* Downloads and resizes image URLs concurrently, but spills completed thumbnails to disk instead of
|
||||
* retaining the full batch in the JVM heap.
|
||||
*/
|
||||
public void prefetchToDisk(Collection<String> urls, ImageSpool imageSpool) {
|
||||
if (urls == null || urls.isEmpty() || imageSpool == null) {
|
||||
return;
|
||||
}
|
||||
Set<String> distinctUrls = new LinkedHashSet<>();
|
||||
for (String url : urls) {
|
||||
if (url == null) {
|
||||
continue;
|
||||
}
|
||||
String trimmed = url.trim();
|
||||
if (!trimmed.isEmpty() && imageSpool.get(trimmed) == null) {
|
||||
distinctUrls.add(trimmed);
|
||||
}
|
||||
}
|
||||
if (distinctUrls.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
AtomicInteger localHit = new AtomicInteger();
|
||||
AtomicInteger downloaded = new AtomicInteger();
|
||||
AtomicInteger failed = new AtomicInteger();
|
||||
CompletionService<Object> completion = new ExecutorCompletionService<>(downloadPool);
|
||||
Iterator<String> pending = distinctUrls.iterator();
|
||||
List<Future<?>> active = new ArrayList<>(Math.min(downloadPoolSize, distinctUrls.size()));
|
||||
while (pending.hasNext() && active.size() < downloadPoolSize) {
|
||||
active.add(submitDiskPrefetch(completion, pending.next(), imageSpool, localHit, downloaded, failed));
|
||||
}
|
||||
|
||||
long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(prefetchTimeoutSeconds);
|
||||
int completed = 0;
|
||||
boolean deadlineReached = false;
|
||||
while (!active.isEmpty()) {
|
||||
long remainingNanos = deadlineNanos - System.nanoTime();
|
||||
if (remainingNanos <= 0L) {
|
||||
deadlineReached = true;
|
||||
break;
|
||||
}
|
||||
try {
|
||||
Future<Object> f = completion.poll(left, TimeUnit.MILLISECONDS);
|
||||
if (f == null) {
|
||||
Future<Object> finished = completion.poll(remainingNanos, TimeUnit.NANOSECONDS);
|
||||
if (finished == null) {
|
||||
deadlineReached = true;
|
||||
break;
|
||||
}
|
||||
done++;
|
||||
} catch (InterruptedException ie) {
|
||||
active.remove(finished);
|
||||
completed++;
|
||||
if (pending.hasNext()) {
|
||||
active.add(submitDiskPrefetch(
|
||||
completion, pending.next(), imageSpool, localHit, downloaded, failed));
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
cancelAll(active);
|
||||
throw new IllegalStateException("image prefetch interrupted", ex);
|
||||
}
|
||||
}
|
||||
// 超时未完成的 future 主动 cancel,避免 idle 持有 OkHttp 连接。
|
||||
if (done < total) {
|
||||
for (Future<?> f : futures) {
|
||||
if (!f.isDone()) {
|
||||
f.cancel(true);
|
||||
if (deadlineReached) {
|
||||
cancelAll(active);
|
||||
}
|
||||
int skipped = Math.max(0, distinctUrls.size() - completed);
|
||||
log.info("[similar-asin][image] disk prefetch finished total={} completed={} spooled={} localHit={} downloaded={} failed={} skipped={} deadlineReached={}",
|
||||
distinctUrls.size(), completed, imageSpool.size(), localHit.get(), downloaded.get(),
|
||||
failed.get(), skipped, deadlineReached);
|
||||
}
|
||||
|
||||
private Future<?> submitDiskPrefetch(CompletionService<Object> completion,
|
||||
String url,
|
||||
ImageSpool imageSpool,
|
||||
AtomicInteger localHit,
|
||||
AtomicInteger downloaded,
|
||||
AtomicInteger failed) {
|
||||
return completion.submit(() -> {
|
||||
try {
|
||||
ResizedImage thumb = readLocalCachedThumb(url);
|
||||
if (thumb == null) {
|
||||
thumb = fetchAndResizeDirect(url);
|
||||
downloaded.incrementAndGet();
|
||||
} else {
|
||||
localHit.incrementAndGet();
|
||||
}
|
||||
imageSpool.put(url, thumb);
|
||||
} catch (Exception ex) {
|
||||
failed.incrementAndGet();
|
||||
log.debug("[similar-asin][image] disk-prefetch-fail url={} err={}", url, errorSummary(ex));
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private static void cancelAll(Collection<? extends Future<?>> futures) {
|
||||
for (Future<?> future : futures) {
|
||||
if (future != null && !future.isDone()) {
|
||||
future.cancel(true);
|
||||
}
|
||||
log.info("[similar-asin][image] prefetch deadline reached total={} done={} cancelled={}",
|
||||
total, done, total - done);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,8 +377,8 @@ public class SimilarAsinImageEmbedder {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
byte[] raw = downloadWithRetry(trimmed);
|
||||
return resizeImage(trimmed, raw);
|
||||
ResizedImage cached = readLocalCachedThumb(trimmed);
|
||||
return cached != null ? cached : fetchAndResizeDirect(trimmed);
|
||||
} catch (Exception ex) {
|
||||
log.debug("[similar-asin][image] prefetch-cache-fail url={} err={}", trimmed, errorSummary(ex));
|
||||
return null;
|
||||
@@ -292,8 +429,10 @@ public class SimilarAsinImageEmbedder {
|
||||
ResizedImage thumb = taskImageCache.get(trimmedUrl);
|
||||
boolean cacheHit = thumb != null;
|
||||
if (!cacheHit) {
|
||||
byte[] raw = downloadWithRetry(trimmedUrl);
|
||||
thumb = resizeImage(trimmedUrl, raw);
|
||||
thumb = readLocalCachedThumb(trimmedUrl);
|
||||
if (thumb == null) {
|
||||
thumb = fetchAndResizeDirect(trimmedUrl);
|
||||
}
|
||||
taskImageCache.put(trimmedUrl, thumb);
|
||||
log.info("[similar-asin][image] cache-miss url={} bytes={} dims={}x{}",
|
||||
trimmedUrl, thumb.bytes().length, thumb.width(), thumb.height());
|
||||
@@ -311,7 +450,7 @@ public class SimilarAsinImageEmbedder {
|
||||
} catch (ResizeException ex) {
|
||||
log.warn("[similar-asin][image] resize-fail url={} elapsedMs={} err={}",
|
||||
trimmedUrl, elapsedMs(startedNanos), errorSummary(ex));
|
||||
} catch (IOException | TimeoutException ex) {
|
||||
} catch (IOException ex) {
|
||||
log.warn("[similar-asin][image] download-fail url={} elapsedMs={} err={}",
|
||||
trimmedUrl, elapsedMs(startedNanos), errorSummary(ex));
|
||||
} catch (RuntimeException ex) {
|
||||
@@ -328,10 +467,31 @@ public class SimilarAsinImageEmbedder {
|
||||
Row row,
|
||||
Map<String, ResizedImage> taskImageCache,
|
||||
ExcelCellImageWriter.Session excelCellImageSession) {
|
||||
return embedAsExcelCellImage(rowIdx, colIdx, url, row, taskImageCache, excelCellImageSession, null);
|
||||
}
|
||||
|
||||
public ImageDim embedAsExcelCellImage(int rowIdx,
|
||||
int colIdx,
|
||||
String url,
|
||||
Row row,
|
||||
Map<String, ResizedImage> taskImageCache,
|
||||
ExcelCellImageWriter.Session excelCellImageSession,
|
||||
ImageSpool imageSpool) {
|
||||
if (url == null || url.isBlank() || excelCellImageSession == null) {
|
||||
return null;
|
||||
}
|
||||
String trimmedUrl = url.trim();
|
||||
SpoolImage spooled = imageSpool == null ? null : imageSpool.get(trimmedUrl);
|
||||
if (spooled != null) {
|
||||
Cell cell = row.createCell(colIdx);
|
||||
cell.setCellValue("#VALUE!");
|
||||
excelCellImageSession.registerImage(rowIdx, colIdx, spooled.path());
|
||||
return new ImageDim(spooled.width(), spooled.height());
|
||||
}
|
||||
if (imageSpool != null) {
|
||||
row.createCell(colIdx).setCellValue(trimmedUrl);
|
||||
return null;
|
||||
}
|
||||
ResizedImage thumb = downloadAndResize(trimmedUrl, colIdx, row, taskImageCache);
|
||||
if (thumb == null) {
|
||||
return null;
|
||||
@@ -339,66 +499,139 @@ public class SimilarAsinImageEmbedder {
|
||||
try {
|
||||
Cell cell = row.createCell(colIdx);
|
||||
cell.setCellValue("#VALUE!");
|
||||
if (imageSpool == null) {
|
||||
excelCellImageSession.registerImage(rowIdx, colIdx, thumb.bytes());
|
||||
} else {
|
||||
spooled = imageSpool.put(trimmedUrl, thumb);
|
||||
excelCellImageSession.registerImage(rowIdx, colIdx, spooled.path());
|
||||
}
|
||||
taskImageCache.remove(trimmedUrl);
|
||||
return spooled == null
|
||||
? new ImageDim(thumb.width(), thumb.height())
|
||||
: new ImageDim(spooled.width(), spooled.height());
|
||||
} catch (IOException | RuntimeException ex) {
|
||||
taskImageCache.remove(trimmedUrl);
|
||||
return new ImageDim(thumb.width(), thumb.height());
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[similar-asin][image] excel-cell-image-fail url={} err={}", trimmedUrl, ex.getMessage());
|
||||
row.createCell(colIdx).setCellValue(trimmedUrl);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] downloadWithRetry(String url) throws IOException, TimeoutException {
|
||||
private boolean hasLocalCachedThumb(String url) {
|
||||
Path path = localCachePath(url);
|
||||
return path != null && Files.isRegularFile(path);
|
||||
}
|
||||
|
||||
private ResizedImage readLocalCachedThumb(String url) {
|
||||
Path path = localCachePath(url);
|
||||
if (path == null || !Files.isRegularFile(path)) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
byte[] bytes = Files.readAllBytes(path);
|
||||
if (bytes.length == 0 || bytes.length > MAX_DOWNLOAD_BYTES) {
|
||||
Files.deleteIfExists(path);
|
||||
return null;
|
||||
}
|
||||
ResizedImage image = decodeCachedThumb(bytes);
|
||||
if (image == null) {
|
||||
Files.deleteIfExists(path);
|
||||
return null;
|
||||
}
|
||||
Files.setLastModifiedTime(path, FileTime.fromMillis(System.currentTimeMillis()));
|
||||
return image;
|
||||
} catch (IOException ex) {
|
||||
log.debug("[similar-asin][image] local-cache-read-fail path={} err={}", path, ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private Path localCachePath(String url) {
|
||||
if (localImageCacheDir == null || url == null || url.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
String normalized = ossStorageService.normalizeManagedPublicUrl(url.trim());
|
||||
String hash = sha256Hex(normalized == null || normalized.isBlank() ? url.trim() : normalized);
|
||||
return localImageCacheDir.resolve(hash.substring(0, 2)).resolve(hash + ".jpg");
|
||||
}
|
||||
|
||||
private void writeLocalCachedThumb(String url, ResizedImage image) {
|
||||
Path target = localCachePath(url);
|
||||
if (target == null || image == null || image.bytes() == null || image.bytes().length == 0) {
|
||||
return;
|
||||
}
|
||||
Path temp = null;
|
||||
try {
|
||||
Files.createDirectories(target.getParent());
|
||||
if (Files.isRegularFile(target)) {
|
||||
return;
|
||||
}
|
||||
temp = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".tmp");
|
||||
Files.write(temp, image.bytes());
|
||||
try {
|
||||
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE);
|
||||
} catch (AtomicMoveNotSupportedException ex) {
|
||||
Files.move(temp, target);
|
||||
}
|
||||
temp = null;
|
||||
} catch (java.nio.file.FileAlreadyExistsException ignored) {
|
||||
// Another download of the same normalized URL won the cache write race.
|
||||
} catch (IOException ex) {
|
||||
log.debug("[similar-asin][image] local-cache-write-fail path={} err={}", target, ex.getMessage());
|
||||
} finally {
|
||||
if (temp != null) {
|
||||
try {
|
||||
Files.deleteIfExists(temp);
|
||||
} catch (IOException ignored) {
|
||||
// Best-effort cleanup of a failed cache write.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static String sha256Hex(String value) {
|
||||
try {
|
||||
byte[] digest = MessageDigest.getInstance("SHA-256")
|
||||
.digest(value.getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder result = new StringBuilder(digest.length * 2);
|
||||
for (byte b : digest) {
|
||||
result.append(String.format("%02x", b & 0xFF));
|
||||
}
|
||||
return result.toString();
|
||||
} catch (NoSuchAlgorithmException ex) {
|
||||
throw new IllegalStateException("SHA-256 unavailable", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private ResizedImage fetchAndResizeDirect(String url) throws IOException {
|
||||
String downloadUrl = normalizeAndValidateDownloadUrl(url);
|
||||
List<String> candidates = downloadCandidates(downloadUrl);
|
||||
IOException last = null;
|
||||
TimeoutException lastTimeout = null;
|
||||
long waitSeconds = downloadTimeoutSeconds * 2L;
|
||||
for (int attempt = 0; attempt <= DOWNLOAD_MAX_RETRY; attempt++) {
|
||||
ensureImageWorkNotInterrupted();
|
||||
String attemptUrl = candidates.get(Math.min(attempt, candidates.size() - 1));
|
||||
Future<byte[]> future = downloadPool.submit(() -> doFetch(attemptUrl));
|
||||
try {
|
||||
byte[] bytes = future.get(waitSeconds, TimeUnit.SECONDS);
|
||||
byte[] raw = doFetch(attemptUrl);
|
||||
if (!attemptUrl.equals(downloadUrl)) {
|
||||
log.info("[similar-asin][image] download-fallback-success originalUrl={} usedUrl={} attempt={}/{}",
|
||||
downloadUrl, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1);
|
||||
}
|
||||
return bytes;
|
||||
} catch (java.util.concurrent.ExecutionException ee) {
|
||||
Throwable cause = ee.getCause();
|
||||
if (cause instanceof DownloadOversizeException doe) {
|
||||
throw doe;
|
||||
ResizedImage image = resizeImage(downloadUrl, raw);
|
||||
writeLocalCachedThumb(downloadUrl, image);
|
||||
return image;
|
||||
} catch (DownloadOversizeException ex) {
|
||||
throw ex;
|
||||
} catch (IOException ex) {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
InterruptedIOException interrupted = new InterruptedIOException("image download interrupted");
|
||||
interrupted.initCause(ex);
|
||||
throw interrupted;
|
||||
}
|
||||
if (cause instanceof IOException io) {
|
||||
last = io;
|
||||
lastTimeout = null;
|
||||
logRetry(attemptUrl, attempt, io);
|
||||
continue;
|
||||
}
|
||||
if (cause instanceof RuntimeException re) {
|
||||
throw re;
|
||||
}
|
||||
throw new IOException("image download failed: " + cause.getMessage(), cause);
|
||||
} catch (java.util.concurrent.TimeoutException te) {
|
||||
future.cancel(true);
|
||||
TimeoutException wrapped = new TimeoutException("image download timeout after attempt "
|
||||
+ (attempt + 1) + "/" + (DOWNLOAD_MAX_RETRY + 1)
|
||||
+ ", waitSeconds=" + waitSeconds
|
||||
+ ", url=" + attemptUrl);
|
||||
wrapped.initCause(te);
|
||||
lastTimeout = wrapped;
|
||||
last = null;
|
||||
logRetry(attemptUrl, attempt, wrapped);
|
||||
} catch (InterruptedException ie) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IOException("image download interrupted", ie);
|
||||
last = ex;
|
||||
logRetry(attemptUrl, attempt, ex);
|
||||
}
|
||||
}
|
||||
if (lastTimeout != null) {
|
||||
throw lastTimeout;
|
||||
}
|
||||
throw last != null ? last : new IOException("image download failed without cause");
|
||||
throw last == null ? new IOException("image download failed: " + downloadUrl) : last;
|
||||
}
|
||||
|
||||
String normalizeAndValidateDownloadUrl(String url) {
|
||||
@@ -503,6 +736,7 @@ public class SimilarAsinImageEmbedder {
|
||||
}
|
||||
|
||||
private byte[] doFetch(String url) throws IOException {
|
||||
ensureImageWorkNotInterrupted();
|
||||
Request req = buildImageRequest(url);
|
||||
try (Response resp = httpClient.newCall(req).execute()) {
|
||||
if (!resp.isSuccessful()) {
|
||||
@@ -522,6 +756,7 @@ public class SimilarAsinImageEmbedder {
|
||||
int total = 0;
|
||||
int n;
|
||||
while ((n = in.read(buf)) != -1) {
|
||||
ensureImageWorkNotInterrupted();
|
||||
total += n;
|
||||
if (total > MAX_DOWNLOAD_BYTES) {
|
||||
throw new DownloadOversizeException(url, total);
|
||||
@@ -590,6 +825,7 @@ public class SimilarAsinImageEmbedder {
|
||||
* 仍然超限才抛 ResizeOversizeException 触发文本兜底。
|
||||
*/
|
||||
ResizedImage resizeImage(String sourceUrl, byte[] raw) throws IOException {
|
||||
ensureImageWorkNotInterrupted();
|
||||
guardImageDimensions(sourceUrl, raw);
|
||||
BufferedImage src = ImageIO.read(new ByteArrayInputStream(raw));
|
||||
if (src == null) {
|
||||
@@ -602,8 +838,11 @@ public class SimilarAsinImageEmbedder {
|
||||
ResizedImage candidate = null;
|
||||
ResizedImage smallest = null;
|
||||
for (int longEdge : FALLBACK_LONG_EDGES) {
|
||||
ensureImageWorkNotInterrupted();
|
||||
BufferedImage scaled = scaleAt(src, srcW, srcH, longEdge);
|
||||
for (float quality : FALLBACK_QUALITIES) {
|
||||
ResizedImage tried = encodeAt(src, srcW, srcH, longEdge, quality);
|
||||
ensureImageWorkNotInterrupted();
|
||||
ResizedImage tried = encodeJpeg(scaled, quality);
|
||||
if (smallest == null || tried.bytes().length < smallest.bytes().length) {
|
||||
smallest = tried;
|
||||
}
|
||||
@@ -625,7 +864,13 @@ public class SimilarAsinImageEmbedder {
|
||||
throw new ResizeOversizeException(sourceUrl, reportedSize);
|
||||
}
|
||||
|
||||
private ResizedImage encodeAt(BufferedImage src, int srcW, int srcH, int longEdgePx, float quality) throws IOException {
|
||||
private static void ensureImageWorkNotInterrupted() throws InterruptedIOException {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
throw new InterruptedIOException("image work interrupted");
|
||||
}
|
||||
}
|
||||
|
||||
private BufferedImage scaleAt(BufferedImage src, int srcW, int srcH, int longEdgePx) {
|
||||
double ratio = (double) Math.max(srcW, srcH) / longEdgePx;
|
||||
int dstW = ratio > 1 ? Math.max(1, (int) Math.round(srcW / ratio)) : srcW;
|
||||
int dstH = ratio > 1 ? Math.max(1, (int) Math.round(srcH / ratio)) : srcH;
|
||||
@@ -638,6 +883,10 @@ public class SimilarAsinImageEmbedder {
|
||||
} finally {
|
||||
g.dispose();
|
||||
}
|
||||
return dst;
|
||||
}
|
||||
|
||||
private ResizedImage encodeJpeg(BufferedImage image, float quality) throws IOException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
Iterator<ImageWriter> writers = ImageIO.getImageWritersByFormatName("jpg");
|
||||
if (!writers.hasNext()) {
|
||||
@@ -651,14 +900,14 @@ public class SimilarAsinImageEmbedder {
|
||||
ImageOutputStream ios = ImageIO.createImageOutputStream(baos);
|
||||
try {
|
||||
writer.setOutput(ios);
|
||||
writer.write(null, new IIOImage(dst, null, null), param);
|
||||
writer.write(null, new IIOImage(image, null, null), param);
|
||||
} finally {
|
||||
ios.close();
|
||||
}
|
||||
} finally {
|
||||
writer.dispose();
|
||||
}
|
||||
return new ResizedImage(baos.toByteArray(), dstW, dstH);
|
||||
return new ResizedImage(baos.toByteArray(), image.getWidth(), image.getHeight());
|
||||
}
|
||||
|
||||
/** 缩略图字节 + 实际像素,A 副本随 taskImageCache 生命周期,B 副本进 picture pool。 */
|
||||
@@ -667,6 +916,85 @@ public class SimilarAsinImageEmbedder {
|
||||
/** embed() 成功时返回的缩略图实际像素,调用方据此自适应行高/列宽。 */
|
||||
public record ImageDim(int width, int height) { }
|
||||
|
||||
public record SpoolImage(Path path, int width, int height) { }
|
||||
|
||||
/** Task-scoped disk storage for resized thumbnails. */
|
||||
public static final class ImageSpool implements AutoCloseable {
|
||||
|
||||
private final Path directory;
|
||||
private final ConcurrentMap<String, SpoolImage> images = new ConcurrentHashMap<>();
|
||||
|
||||
public ImageSpool(Path directory) throws IOException {
|
||||
this.directory = Objects.requireNonNull(directory, "directory must not be null")
|
||||
.toAbsolutePath().normalize();
|
||||
Files.createDirectories(this.directory);
|
||||
}
|
||||
|
||||
public SpoolImage get(String url) {
|
||||
if (url == null) {
|
||||
return null;
|
||||
}
|
||||
SpoolImage image = images.get(url.trim());
|
||||
return image != null && Files.isRegularFile(image.path()) ? image : null;
|
||||
}
|
||||
|
||||
public SpoolImage put(String url, ResizedImage image) throws IOException {
|
||||
if (url == null || url.isBlank() || image == null || image.bytes() == null || image.bytes().length == 0) {
|
||||
throw new IOException("invalid image spool entry");
|
||||
}
|
||||
String key = url.trim();
|
||||
SpoolImage existing = get(key);
|
||||
if (existing != null) {
|
||||
return existing;
|
||||
}
|
||||
Path file = Files.createTempFile(directory, "image-", ".jpeg");
|
||||
boolean retained = false;
|
||||
try {
|
||||
Files.write(file, image.bytes());
|
||||
SpoolImage candidate = new SpoolImage(file, image.width(), image.height());
|
||||
existing = images.putIfAbsent(key, candidate);
|
||||
if (existing != null) {
|
||||
return existing;
|
||||
}
|
||||
retained = true;
|
||||
return candidate;
|
||||
} finally {
|
||||
if (!retained) {
|
||||
Files.deleteIfExists(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return images.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
images.clear();
|
||||
if (!Files.exists(directory)) {
|
||||
return;
|
||||
}
|
||||
IOException failure = null;
|
||||
try (var paths = Files.walk(directory)) {
|
||||
for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
|
||||
try {
|
||||
Files.deleteIfExists(path);
|
||||
} catch (IOException ex) {
|
||||
if (failure == null) {
|
||||
failure = ex;
|
||||
} else {
|
||||
failure.addSuppressed(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (failure != null) {
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 在解码整张位图前用 ImageReader 仅读取头部尺寸,避免“图像炸弹”导致 heap OOM。 */
|
||||
private static void guardImageDimensions(String sourceUrl, byte[] raw) throws IOException {
|
||||
try (ImageInputStream iis = ImageIO.createImageInputStream(new ByteArrayInputStream(raw))) {
|
||||
|
||||
+18
@@ -17,4 +17,22 @@ public class TaskHeartbeatRequest {
|
||||
|
||||
@Schema(description = "可选的总进度", example = "1100")
|
||||
private Integer total;
|
||||
|
||||
@Schema(description = "采集数据任务当前阶段", example = "detail")
|
||||
private String collectStage;
|
||||
|
||||
@Schema(description = "采集数据任务当前关键词", example = "women bodysuit")
|
||||
private String currentKeyword;
|
||||
|
||||
@Schema(description = "搜索页当前页码", example = "2")
|
||||
private Integer searchCurrentPage;
|
||||
|
||||
@Schema(description = "搜索页总页数", example = "7")
|
||||
private Integer searchTotalPages;
|
||||
|
||||
@Schema(description = "详情页已处理 ASIN 数", example = "34")
|
||||
private Integer detailProcessedAsins;
|
||||
|
||||
@Schema(description = "详情页 ASIN 总数", example = "120")
|
||||
private Integer detailTotalAsins;
|
||||
}
|
||||
|
||||
+2
@@ -25,4 +25,6 @@ public class TaskFileJobEntity {
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
private LocalDateTime finishedAt;
|
||||
/** Parent task/result terminal-failure callback completion marker. */
|
||||
private LocalDateTime terminalCallbackAt;
|
||||
}
|
||||
|
||||
+148
-22
@@ -71,7 +71,8 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, null)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, null));
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(existing.getId());
|
||||
publishDispatchEvent(refreshed);
|
||||
return refreshed;
|
||||
@@ -148,7 +149,8 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getRetryCount, 0)
|
||||
.set(TaskFileJobEntity::getErrorMessage, null)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, null));
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (updated > 0) {
|
||||
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(jobId);
|
||||
publishDispatchEvent(refreshed);
|
||||
@@ -159,37 +161,70 @@ public class TaskFileJobService {
|
||||
|
||||
@Transactional
|
||||
public int resetStuckRunningJobs(int stuckMinutes, int limit) {
|
||||
return resetStuckRunningJobsDetailed(stuckMinutes, limit).resetCount();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public StuckJobResetResult resetStuckRunningJobsDetailed(int stuckMinutes, int limit) {
|
||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes));
|
||||
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||
.and(wrapper -> wrapper
|
||||
.and(running -> running
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.and(age -> age
|
||||
.lt(TaskFileJobEntity::getUpdatedAt, threshold)
|
||||
.or()
|
||||
.isNull(TaskFileJobEntity::getUpdatedAt)))
|
||||
.or(exhausted -> exhausted
|
||||
.eq(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.ge(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.isNull(TaskFileJobEntity::getTerminalCallbackAt)))
|
||||
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
|
||||
.last("limit " + Math.max(1, Math.min(limit, 200))));
|
||||
int reset = 0;
|
||||
List<TaskFileJobEntity> exhaustedJobs = new ArrayList<>();
|
||||
for (TaskFileJobEntity job : jobs) {
|
||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
||||
if (retryCount >= MAX_RETRY_COUNT) {
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if ("FAILED".equals(job.getStatus())
|
||||
&& job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT) {
|
||||
exhaustedJobs.add(job);
|
||||
continue;
|
||||
}
|
||||
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
||||
int nextRetryCount = Math.min(MAX_RETRY_COUNT, retryCount + 1);
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LambdaUpdateWrapper<TaskFileJobEntity> update = new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(job.getUpdatedAt() != null, TaskFileJobEntity::getUpdatedAt, job.getUpdatedAt())
|
||||
.isNull(job.getUpdatedAt() == null, TaskFileJobEntity::getUpdatedAt)
|
||||
.set(TaskFileJobEntity::getRetryCount, nextRetryCount)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now);
|
||||
if (nextRetryCount >= MAX_RETRY_COUNT) {
|
||||
int updated = taskFileJobMapper.update(null, update
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已达到最大重试次数")
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (updated > 0) {
|
||||
TaskFileJobEntity exhausted = taskFileJobMapper.selectById(job.getId());
|
||||
exhaustedJobs.add(exhausted == null ? job : exhausted);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
int updated = taskFileJobMapper.update(null, update
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (updated > 0) {
|
||||
reset++;
|
||||
publishDispatchEvent(taskFileJobMapper.selectById(job.getId()));
|
||||
}
|
||||
}
|
||||
return reset;
|
||||
return new StuckJobResetResult(reset, List.copyOf(exhaustedJobs));
|
||||
}
|
||||
|
||||
public record StuckJobResetResult(int resetCount, List<TaskFileJobEntity> exhaustedJobs) {
|
||||
}
|
||||
|
||||
public boolean markRunning(Long jobId) {
|
||||
@@ -199,7 +234,37 @@ public class TaskFileJobService {
|
||||
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, jobId)
|
||||
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
|
||||
.and(wrapper -> wrapper.isNull(TaskFileJobEntity::getRetryCount)
|
||||
.or()
|
||||
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT))
|
||||
.set(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null)) > 0;
|
||||
}
|
||||
|
||||
/** Returns the persisted RUNNING row so queued execution can later fence itself with updatedAt. */
|
||||
public TaskFileJobEntity claimRunning(Long jobId) {
|
||||
if (!markRunning(jobId)) {
|
||||
return null;
|
||||
}
|
||||
TaskFileJobEntity claim = taskFileJobMapper.selectById(jobId);
|
||||
return claim != null && "RUNNING".equals(claim.getStatus()) ? claim : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Activates a previously queued claim. The updatedAt CAS prevents a timed-out old queue entry from
|
||||
* starting after the same job has already been reset or claimed again.
|
||||
*/
|
||||
public boolean activateRunningClaim(TaskFileJobEntity claim) {
|
||||
if (claim == null || claim.getId() == null || claim.getUpdatedAt() == null) {
|
||||
return false;
|
||||
}
|
||||
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, claim.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.eq(TaskFileJobEntity::getUpdatedAt, claim.getUpdatedAt())
|
||||
.eq(claim.getRetryCount() != null, TaskFileJobEntity::getRetryCount, claim.getRetryCount())
|
||||
.isNull(claim.getRetryCount() == null, TaskFileJobEntity::getRetryCount)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())) > 0;
|
||||
}
|
||||
|
||||
@@ -228,6 +293,23 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now));
|
||||
}
|
||||
|
||||
public void touchRunningAssembleJobsIfStale(Long taskId, String moduleType, long intervalMillis) {
|
||||
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LocalDateTime cutoff = now.minus(Duration.ofMillis(Math.max(1_000L, intervalMillis)));
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getTaskId, taskId)
|
||||
.eq(TaskFileJobEntity::getModuleType, moduleType)
|
||||
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.and(wrapper -> wrapper.isNull(TaskFileJobEntity::getUpdatedAt)
|
||||
.or()
|
||||
.le(TaskFileJobEntity::getUpdatedAt, cutoff))
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now));
|
||||
}
|
||||
|
||||
public boolean deferRunning(Long jobId, String message) {
|
||||
if (jobId == null || jobId <= 0) {
|
||||
return false;
|
||||
@@ -252,10 +334,14 @@ public class TaskFileJobService {
|
||||
.eq(TaskFileJobEntity::getId, jobId)
|
||||
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
|
||||
.ne(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.and(wrapper -> wrapper.isNull(TaskFileJobEntity::getRetryCount)
|
||||
.or()
|
||||
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT))
|
||||
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||
.set(TaskFileJobEntity::getErrorMessage, message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, null));
|
||||
.set(TaskFileJobEntity::getFinishedAt, null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null));
|
||||
if (updated <= 0) {
|
||||
return false;
|
||||
}
|
||||
@@ -265,13 +351,19 @@ public class TaskFileJobService {
|
||||
}
|
||||
|
||||
public void markSuccess(TaskFileJobEntity job, String resultFileUrl) {
|
||||
if (job == null || job.getId() == null) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||
.set(TaskFileJobEntity::getStatus, "SUCCESS")
|
||||
.set(TaskFileJobEntity::getErrorMessage, null)
|
||||
.set(TaskFileJobEntity::getResultFileUrl, resultFileUrl)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, now));
|
||||
}
|
||||
|
||||
public TaskFileJobEntity findById(Long jobId) {
|
||||
@@ -283,19 +375,52 @@ public class TaskFileJobService {
|
||||
|
||||
public boolean isRetryExhausted(Long jobId) {
|
||||
TaskFileJobEntity job = findById(jobId);
|
||||
return job != null && job.getRetryCount() != null && job.getRetryCount() >= MAX_RETRY_COUNT;
|
||||
return job != null
|
||||
&& "FAILED".equals(job.getStatus())
|
||||
&& job.getRetryCount() != null
|
||||
&& job.getRetryCount() >= MAX_RETRY_COUNT;
|
||||
}
|
||||
|
||||
public void markFailed(TaskFileJobEntity job, String message) {
|
||||
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount() + 1;
|
||||
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
TaskFileJobEntity current = job == null || job.getId() == null
|
||||
? null
|
||||
: taskFileJobMapper.selectById(job.getId());
|
||||
if (current == null || "SUCCESS".equals(current.getStatus())) {
|
||||
return;
|
||||
}
|
||||
int currentRetryCount = current.getRetryCount() == null ? 0 : current.getRetryCount();
|
||||
int retryCount = Math.min(MAX_RETRY_COUNT, currentRetryCount + 1);
|
||||
LambdaUpdateWrapper<TaskFileJobEntity> update = new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, job.getId())
|
||||
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
|
||||
.eq(current.getRetryCount() != null, TaskFileJobEntity::getRetryCount, current.getRetryCount())
|
||||
.isNull(current.getRetryCount() == null, TaskFileJobEntity::getRetryCount)
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getRetryCount, retryCount)
|
||||
.set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, retryCount >= MAX_RETRY_COUNT ? LocalDateTime.now() : null));
|
||||
.set(TaskFileJobEntity::getFinishedAt,
|
||||
retryCount >= MAX_RETRY_COUNT ? LocalDateTime.now() : null)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, null);
|
||||
taskFileJobMapper.update(null, update);
|
||||
}
|
||||
|
||||
public boolean markFailureFinalized(Long jobId, String message) {
|
||||
if (jobId == null || jobId <= 0) {
|
||||
return false;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||
.eq(TaskFileJobEntity::getId, jobId)
|
||||
.eq(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.ge(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.isNull(TaskFileJobEntity::getTerminalCallbackAt)
|
||||
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||
.set(TaskFileJobEntity::getErrorMessage,
|
||||
message == null ? "结果文件生成失败" : message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||
.set(TaskFileJobEntity::getFinishedAt, now)
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, now)) > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -310,7 +435,8 @@ public class TaskFileJobService {
|
||||
.set(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||
.set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message)
|
||||
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
|
||||
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now())
|
||||
.set(TaskFileJobEntity::getTerminalCallbackAt, LocalDateTime.now()));
|
||||
}
|
||||
|
||||
public TaskFileJobEntity findAssembleJob(Long taskId, String moduleType, Long resultId) {
|
||||
|
||||
+6
@@ -7,6 +7,7 @@ import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCa
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||
import com.nanri.aiimage.modules.brand.service.BrandTaskProgressCacheService;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
|
||||
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService;
|
||||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
|
||||
@@ -49,6 +50,7 @@ public class TaskHeartbeatService {
|
||||
private static final String MODULE_WITHDRAW = "WITHDRAW";
|
||||
private static final String MODULE_APPEARANCE_PATENT = "APPEARANCE_PATENT";
|
||||
private static final String MODULE_SIMILAR_ASIN = "SIMILAR_ASIN";
|
||||
private static final String MODULE_COLLECT_DATA = "COLLECT_DATA";
|
||||
private static final String MODULE_BRAND = "BRAND";
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
@@ -65,8 +67,10 @@ public class TaskHeartbeatService {
|
||||
private final AppearancePatentTaskCacheService appearancePatentTaskCacheService;
|
||||
private final SimilarAsinTaskCacheService similarAsinTaskCacheService;
|
||||
private final SimilarAsinProperties similarAsinProperties;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
||||
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||
private final CollectDataService collectDataService;
|
||||
|
||||
public TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
@@ -151,6 +155,7 @@ public class TaskHeartbeatService {
|
||||
boolean checkpointDue = task.getUpdatedAt() == null || !task.getUpdatedAt().isAfter(cutoff);
|
||||
|
||||
similarAsinTaskCacheService.touchTaskHeartbeat(task.getId());
|
||||
taskFileJobService.touchRunningAssembleJobsIfStale(task.getId(), MODULE_SIMILAR_ASIN, intervalMillis);
|
||||
if (!checkpointDue) {
|
||||
saveFileTaskCache(MODULE_SIMILAR_ASIN, task);
|
||||
return TaskHeartbeatVo.alive(MODULE_SIMILAR_ASIN, STATUS_RUNNING);
|
||||
@@ -239,6 +244,7 @@ public class TaskHeartbeatService {
|
||||
case MODULE_DELETE_BRAND -> {
|
||||
deleteBrandTaskCacheService.saveProgress(taskId, buildDeleteBrandHeartbeatProgress(request), true);
|
||||
}
|
||||
case MODULE_COLLECT_DATA -> collectDataService.updateProgress(taskId, request);
|
||||
default -> {
|
||||
}
|
||||
}
|
||||
|
||||
+85
-13
@@ -18,6 +18,7 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -28,6 +29,10 @@ import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -66,6 +71,15 @@ public class TaskResultFileJobWorker {
|
||||
@Value("${aiimage.result-file-job.stuck-timeout-minutes:30}")
|
||||
private int stuckTimeoutMinutes;
|
||||
|
||||
@Value("${aiimage.result-file-job.heartbeat-interval-ms:60000}")
|
||||
private long heartbeatIntervalMillis = 60000L;
|
||||
|
||||
private final ScheduledExecutorService jobHeartbeatExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> {
|
||||
Thread thread = new Thread(runnable, "task-file-job-heartbeat");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.result-file-job.local-worker-delay-ms:15000}")
|
||||
public void runPendingJobs() {
|
||||
if (!localWorkerEnabled) {
|
||||
@@ -90,9 +104,17 @@ public class TaskResultFileJobWorker {
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.result-file-job.stuck-scan-delay-ms:60000}")
|
||||
public void resetStuckJobs() {
|
||||
int reset = taskFileJobService.resetStuckRunningJobs(stuckTimeoutMinutes, batchSize);
|
||||
if (reset > 0) {
|
||||
log.warn("[task-file-job] reset stuck running jobs count={} timeoutMinutes={}", reset, stuckTimeoutMinutes);
|
||||
TaskFileJobService.StuckJobResetResult result =
|
||||
taskFileJobService.resetStuckRunningJobsDetailed(stuckTimeoutMinutes, batchSize);
|
||||
if (result.resetCount() > 0 || !result.exhaustedJobs().isEmpty()) {
|
||||
log.warn("[task-file-job] handled stuck running jobs resetCount={} exhaustedCount={} timeoutMinutes={}",
|
||||
result.resetCount(), result.exhaustedJobs().size(), stuckTimeoutMinutes);
|
||||
}
|
||||
for (TaskFileJobEntity job : result.exhaustedJobs()) {
|
||||
String message = job.getErrorMessage() == null || job.getErrorMessage().isBlank()
|
||||
? "文件生成任务运行超时,已达到最大重试次数"
|
||||
: job.getErrorMessage();
|
||||
finalizeRetryExhausted(job, message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,19 +124,49 @@ public class TaskResultFileJobWorker {
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), ownerFromScopeKey(job.getScopeKey()), currentInstanceId());
|
||||
return;
|
||||
}
|
||||
if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) {
|
||||
if (job == null || job.getId() == null) {
|
||||
return;
|
||||
}
|
||||
TaskFileJobEntity claim = taskFileJobService.claimRunning(job.getId());
|
||||
if (claim == null) {
|
||||
return;
|
||||
}
|
||||
if ("APPEARANCE_PATENT".equals(job.getModuleType()) || "SIMILAR_ASIN".equals(job.getModuleType())) {
|
||||
try {
|
||||
cozeTaskExecutor.execute(() -> processInternal(job));
|
||||
cozeTaskExecutor.execute(() -> processClaimedWithHeartbeat(job, claim));
|
||||
return;
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[task-file-job] coze module offload failed, fallback inline jobId={} taskId={} moduleType={} msg={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
processClaimedWithHeartbeat(job, claim);
|
||||
}
|
||||
|
||||
private ScheduledFuture<?> startJobHeartbeat(TaskFileJobEntity job) {
|
||||
long interval = Math.max(1000L, heartbeatIntervalMillis);
|
||||
return jobHeartbeatExecutor.scheduleWithFixedDelay(() -> {
|
||||
try {
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-file-job] heartbeat failed jobId={} taskId={} moduleType={} msg={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage());
|
||||
}
|
||||
}, interval, interval, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
private void processClaimedWithHeartbeat(TaskFileJobEntity job, TaskFileJobEntity claim) {
|
||||
if (!taskFileJobService.activateRunningClaim(claim)) {
|
||||
log.info("[task-file-job] queued claim expired before execution jobId={} taskId={} moduleType={} claimUpdatedAt={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), claim.getUpdatedAt());
|
||||
return;
|
||||
}
|
||||
ScheduledFuture<?> heartbeat = startJobHeartbeat(job);
|
||||
try {
|
||||
processInternal(job);
|
||||
} finally {
|
||||
heartbeat.cancel(false);
|
||||
}
|
||||
}
|
||||
|
||||
private void processInternal(TaskFileJobEntity job) {
|
||||
@@ -130,13 +182,6 @@ public class TaskResultFileJobWorker {
|
||||
TaskDistributedLockService.LockHandle lockHandle =
|
||||
taskDistributedLockService.acquire(job.getModuleType(), job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS);
|
||||
if (lockHandle == null) {
|
||||
TaskFileJobEntity latestOnBusy = taskFileJobService.findById(job.getId());
|
||||
if (latestOnBusy != null && "RUNNING".equals(latestOnBusy.getStatus())) {
|
||||
taskFileJobService.touchRunning(job.getId());
|
||||
log.info("[task-file-job] process skipped because task lock is busy and job is already running jobId={} taskId={} moduleType={} resultId={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
|
||||
return;
|
||||
}
|
||||
taskFileJobService.requeue(job.getId(), "Task is busy, waiting for previous task operation");
|
||||
log.info("[task-file-job] process requeued because task lock is busy jobId={} taskId={} moduleType={} resultId={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
|
||||
@@ -194,10 +239,37 @@ public class TaskResultFileJobWorker {
|
||||
log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), message);
|
||||
taskFileJobService.markFailed(job, message);
|
||||
if ("SHOP_DATA_CRAWL".equals(job.getModuleType()) && taskFileJobService.isRetryExhausted(job.getId())) {
|
||||
if (taskFileJobService.isRetryExhausted(job.getId())) {
|
||||
finalizeRetryExhausted(job, message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void finalizeRetryExhausted(TaskFileJobEntity job, String message) {
|
||||
try {
|
||||
notifyRetryExhausted(job, message);
|
||||
if (!taskFileJobService.markFailureFinalized(job.getId(), message)) {
|
||||
log.warn("[task-file-job] exhausted job terminal callback was already finalized or claim was lost jobId={} taskId={} moduleType={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType());
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
// Keep the terminal-callback marker empty so the scheduled scan retries this idempotent callback.
|
||||
log.error("[task-file-job] failed to finalize exhausted job jobId={} taskId={} moduleType={} msg={}",
|
||||
job.getId(), job.getTaskId(), job.getModuleType(), ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void notifyRetryExhausted(TaskFileJobEntity job, String message) {
|
||||
if ("SHOP_DATA_CRAWL".equals(job.getModuleType())) {
|
||||
shopDataCrawlTaskService.handleResultFileJobFailure(job, message);
|
||||
} else if ("SIMILAR_ASIN".equals(job.getModuleType())) {
|
||||
similarAsinTaskService.handleResultFileJobFailure(job, message);
|
||||
}
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
void shutdownJobHeartbeatExecutor() {
|
||||
jobHeartbeatExecutor.shutdownNow();
|
||||
}
|
||||
|
||||
private boolean isOrphanJobFailure(Exception ex, String message) {
|
||||
|
||||
+20
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.ziniao.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexRefreshCursorDto;
|
||||
import com.nanri.aiimage.modules.ziniao.model.dto.ZiniaoOpenShopRequest;
|
||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoOpenShopVo;
|
||||
@@ -8,17 +9,23 @@ import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoSessionVo;
|
||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopListVo;
|
||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoStaffListVo;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoAuthService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexRefreshService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.media.Content;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponses;
|
||||
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestHeader;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
@@ -31,6 +38,8 @@ public class ZiniaoAuthController {
|
||||
|
||||
private final ZiniaoAuthService ziniaoAuthService;
|
||||
private final ZiniaoShopIndexService ziniaoShopIndexService;
|
||||
private final ZiniaoShopIndexRefreshService ziniaoShopIndexRefreshService;
|
||||
private final AdminAuthSupport adminAuthSupport;
|
||||
|
||||
@GetMapping("/session")
|
||||
@Operation(summary = "获取紫鸟会话状态", description = "自动获取或复用 appToken,并返回 companyId、当前员工 userId、脱敏 token 和当前店铺信息。")
|
||||
@@ -56,6 +65,17 @@ public class ZiniaoAuthController {
|
||||
return ApiResponse.success(ziniaoShopIndexService.getRefreshCursor());
|
||||
}
|
||||
|
||||
@PostMapping("/index-refresh")
|
||||
@Operation(summary = "手动刷新紫鸟店铺索引", description = "立即执行一轮完整的紫鸟店铺索引刷新,并返回本轮刷新结果。")
|
||||
@SecurityRequirement(name = "bearerAuth")
|
||||
public ApiResponse<ZiniaoShopIndexRefreshCursorDto> refreshShopIndex(
|
||||
HttpServletRequest request,
|
||||
@Parameter(description = "管理员登录凭证,格式:Bearer <token>")
|
||||
@RequestHeader(value = HttpHeaders.AUTHORIZATION, required = false) String authorization) {
|
||||
adminAuthSupport.requireAdmin(request);
|
||||
return ApiResponse.success("紫鸟店铺索引刷新完成", ziniaoShopIndexRefreshService.refreshShopIndexManually());
|
||||
}
|
||||
|
||||
@GetMapping("/shops")
|
||||
@Operation(summary = "获取员工可见店铺列表", description = "先通过 API Key 获取 companyId,再按员工 userId 查询该员工有权限的店铺列表,并缓存到当前 session。")
|
||||
@ApiResponses({
|
||||
|
||||
+59
-13
@@ -1,12 +1,15 @@
|
||||
package com.nanri.aiimage.modules.ziniao.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.shopkey.mapper.ShopKeyMapper;
|
||||
import com.nanri.aiimage.modules.shopkey.model.entity.ShopKeyEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -15,29 +18,28 @@ import java.util.Map;
|
||||
@RequiredArgsConstructor
|
||||
public class ZiniaoApiKeyProvider {
|
||||
|
||||
public static final String IP_WHITELIST_STATUS_ALLOWED = "ALLOWED";
|
||||
public static final String IP_WHITELIST_STATUS_BLOCKED = "BLOCKED";
|
||||
|
||||
private final ShopKeyMapper shopKeyMapper;
|
||||
|
||||
public List<ApiKeyAccount> listApiKeyAccounts() {
|
||||
List<ShopKeyEntity> entities = shopKeyMapper.selectList(new LambdaQueryWrapper<ShopKeyEntity>()
|
||||
.orderByDesc(ShopKeyEntity::getId));
|
||||
Map<String, String> accountByApiKey = new LinkedHashMap<>();
|
||||
Map<String, List<ShopKeyEntity>> entitiesByApiKey = new LinkedHashMap<>();
|
||||
for (ShopKeyEntity entity : entities) {
|
||||
String apiKey = normalizeApiKey(entity == null ? null : entity.getZiniaoToken());
|
||||
if (apiKey == null) {
|
||||
continue;
|
||||
}
|
||||
String accountName = entity.getZiniaoAccountName() == null ? null : entity.getZiniaoAccountName().trim();
|
||||
if (!accountByApiKey.containsKey(apiKey)) {
|
||||
accountByApiKey.put(apiKey, accountName);
|
||||
continue;
|
||||
entitiesByApiKey.computeIfAbsent(apiKey, ignored -> new ArrayList<>()).add(entity);
|
||||
}
|
||||
String existingName = accountByApiKey.get(apiKey);
|
||||
if ((existingName == null || existingName.isBlank()) && accountName != null && !accountName.isBlank()) {
|
||||
accountByApiKey.put(apiKey, accountName);
|
||||
}
|
||||
}
|
||||
return accountByApiKey.entrySet().stream()
|
||||
.map(entry -> new ApiKeyAccount(entry.getKey(), entry.getValue()))
|
||||
return entitiesByApiKey.entrySet().stream()
|
||||
.map(entry -> new ApiKeyAccount(
|
||||
entry.getKey(),
|
||||
resolveAccountName(entry.getValue()),
|
||||
entry.getValue().stream().map(ShopKeyEntity::getId).filter(java.util.Objects::nonNull).toList()
|
||||
))
|
||||
.toList();
|
||||
}
|
||||
|
||||
@@ -60,6 +62,42 @@ public class ZiniaoApiKeyProvider {
|
||||
return total != null && total > 0;
|
||||
}
|
||||
|
||||
public void markIpWhitelistAllowed(ApiKeyAccount account) {
|
||||
updateIpWhitelistStatus(account, IP_WHITELIST_STATUS_ALLOWED, null);
|
||||
}
|
||||
|
||||
public void markIpWhitelistBlocked(ApiKeyAccount account, String message) {
|
||||
updateIpWhitelistStatus(account, IP_WHITELIST_STATUS_BLOCKED, message);
|
||||
}
|
||||
|
||||
private void updateIpWhitelistStatus(ApiKeyAccount account, String status, String message) {
|
||||
if (account == null || account.shopKeyIds().isEmpty()) {
|
||||
return;
|
||||
}
|
||||
shopKeyMapper.update(null, new LambdaUpdateWrapper<ShopKeyEntity>()
|
||||
.in(ShopKeyEntity::getId, account.shopKeyIds())
|
||||
.set(ShopKeyEntity::getIpWhitelistStatus, status)
|
||||
.set(ShopKeyEntity::getIpWhitelistCheckedAt, LocalDateTime.now())
|
||||
.set(ShopKeyEntity::getIpWhitelistMessage, truncateMessage(message)));
|
||||
}
|
||||
|
||||
private String resolveAccountName(List<ShopKeyEntity> entities) {
|
||||
return entities.stream()
|
||||
.map(ShopKeyEntity::getZiniaoAccountName)
|
||||
.filter(name -> name != null && !name.isBlank())
|
||||
.map(String::trim)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
private String truncateMessage(String message) {
|
||||
if (message == null || message.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
String normalized = message.trim();
|
||||
return normalized.length() <= 500 ? normalized : normalized.substring(0, 500);
|
||||
}
|
||||
|
||||
private String normalizeApiKey(String token) {
|
||||
if (token == null || token.isBlank()) {
|
||||
return null;
|
||||
@@ -71,6 +109,14 @@ public class ZiniaoApiKeyProvider {
|
||||
return normalized.isBlank() ? null : normalized;
|
||||
}
|
||||
|
||||
public record ApiKeyAccount(String apiKey, String accountName) {
|
||||
public record ApiKeyAccount(String apiKey, String accountName, List<Long> shopKeyIds) {
|
||||
|
||||
public ApiKeyAccount {
|
||||
shopKeyIds = shopKeyIds == null ? List.of() : List.copyOf(shopKeyIds);
|
||||
}
|
||||
|
||||
public ApiKeyAccount(String apiKey, String accountName) {
|
||||
this(apiKey, accountName, List.of());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+13
@@ -1,6 +1,8 @@
|
||||
package com.nanri.aiimage.modules.ziniao.service;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexRefreshCursorDto;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
@@ -33,4 +35,15 @@ public class ZiniaoShopIndexRefreshService {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ZiniaoShopIndexRefreshCursorDto refreshShopIndexManually() {
|
||||
DistributedJobLockService.LockHandle lockHandle = distributedJobLockService.tryLock("ziniao:shop-index-refresh", REFRESH_LOCK_TTL);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException("紫鸟店铺索引正在刷新,或分布式锁暂时不可用,请稍后重试");
|
||||
}
|
||||
try (lockHandle) {
|
||||
ziniaoShopIndexService.refreshAllShopIndex();
|
||||
return ziniaoShopIndexService.getRefreshCursor();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+93
-20
@@ -159,6 +159,14 @@ public class ZiniaoShopIndexService {
|
||||
}
|
||||
|
||||
public void refreshShopIndex() {
|
||||
refreshShopIndex(resolveRefreshBatchSize());
|
||||
}
|
||||
|
||||
public void refreshAllShopIndex() {
|
||||
refreshShopIndex(0);
|
||||
}
|
||||
|
||||
private void refreshShopIndex(int refreshBatchSize) {
|
||||
long now = Instant.now().toEpochMilli();
|
||||
ZiniaoShopIndexRefreshCursorDto previousCursor = ziniaoTransientCacheService
|
||||
.get(CACHE_TYPE_SHOP_INDEX_REFRESH_CURSOR, "global", ZiniaoShopIndexRefreshCursorDto.class)
|
||||
@@ -177,7 +185,10 @@ public class ZiniaoShopIndexService {
|
||||
Map<String, List<ZiniaoShopIndexEntryDto>> grouped = new LinkedHashMap<>();
|
||||
Map<String, Long> storesFingerprintToUserId = new LinkedHashMap<>();
|
||||
Set<Long> allInvalidUserIds = new LinkedHashSet<>();
|
||||
int refreshBatchSize = resolveRefreshBatchSize();
|
||||
int completedApiKeyCount = 0;
|
||||
int skippedApiKeyCount = 0;
|
||||
int whitelistSkippedApiKeyCount = 0;
|
||||
boolean completeCoverage = true;
|
||||
try {
|
||||
List<ZiniaoApiKeyProvider.ApiKeyAccount> allApiKeyAccounts = ziniaoApiKeyProvider.listApiKeyAccounts();
|
||||
List<ZiniaoApiKeyProvider.ApiKeyAccount> apiKeyAccounts = selectApiKeyBatchByOffset(
|
||||
@@ -186,14 +197,24 @@ public class ZiniaoShopIndexService {
|
||||
refreshBatchSize
|
||||
);
|
||||
int nextOffset = computeNextOffset(allApiKeyAccounts.size(), previousOffset, apiKeyAccounts.size(), refreshBatchSize);
|
||||
apiKeyLoop:
|
||||
for (ZiniaoApiKeyProvider.ApiKeyAccount apiKeyAccount : apiKeyAccounts) {
|
||||
String apiKey = apiKeyAccount.apiKey();
|
||||
String companyName = apiKeyAccount.accountName();
|
||||
Map<String, List<ZiniaoShopIndexEntryDto>> apiKeyGrouped = new LinkedHashMap<>();
|
||||
Map<String, Long> apiKeyFingerprints = new LinkedHashMap<>();
|
||||
Long companyId;
|
||||
try {
|
||||
companyId = ziniaoAuthService.resolveCompanyIdForIndex(apiKey);
|
||||
} catch (BusinessException ex) {
|
||||
rethrowIfIpWhitelistError(ex);
|
||||
skippedApiKeyCount++;
|
||||
completeCoverage = false;
|
||||
if (ziniaoAuthService.isIpWhitelistError(ex)) {
|
||||
whitelistSkippedApiKeyCount++;
|
||||
markIpWhitelistBlockedSafely(apiKeyAccount, ex.getMessage());
|
||||
log.warn("[ziniao-index] skip apiKey reason=IP_WHITELIST accountName={} msg={}", companyName, ex.getMessage());
|
||||
continue;
|
||||
}
|
||||
log.warn("[ziniao-index] skip apiKey while resolving companyId, msg={}", ex.getMessage());
|
||||
continue;
|
||||
}
|
||||
@@ -203,7 +224,15 @@ public class ZiniaoShopIndexService {
|
||||
try {
|
||||
staff = ziniaoAuthService.getOrLoadStaffForIndex(apiKey, companyId);
|
||||
} catch (BusinessException ex) {
|
||||
rethrowIfIpWhitelistError(ex);
|
||||
if (ziniaoAuthService.isIpWhitelistError(ex)) {
|
||||
skippedApiKeyCount++;
|
||||
whitelistSkippedApiKeyCount++;
|
||||
completeCoverage = false;
|
||||
markIpWhitelistBlockedSafely(apiKeyAccount, ex.getMessage());
|
||||
log.warn("[ziniao-index] skip apiKey reason=IP_WHITELIST accountName={} companyId={} msg={}",
|
||||
companyName, companyId, ex.getMessage());
|
||||
continue;
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
for (Long userId : buildUserIds(staff)) {
|
||||
@@ -214,11 +243,21 @@ public class ZiniaoShopIndexService {
|
||||
try {
|
||||
stores = ziniaoAuthService.getOrLoadUserStoresForIndex(apiKey, companyId, userId);
|
||||
} catch (BusinessException ex) {
|
||||
rethrowIfIpWhitelistError(ex);
|
||||
if (ziniaoAuthService.isIpWhitelistError(ex)) {
|
||||
skippedApiKeyCount++;
|
||||
whitelistSkippedApiKeyCount++;
|
||||
completeCoverage = false;
|
||||
markIpWhitelistBlockedSafely(apiKeyAccount, ex.getMessage());
|
||||
log.warn("[ziniao-index] skip apiKey reason=IP_WHITELIST accountName={} companyId={} userId={} msg={}",
|
||||
companyName, companyId, userId, ex.getMessage());
|
||||
continue apiKeyLoop;
|
||||
}
|
||||
if (ziniaoAuthService.isInvalidUserStoresError(ex)) {
|
||||
invalidUserIds.add(userId);
|
||||
allInvalidUserIds.add(userId);
|
||||
ziniaoAuthService.evictInvalidUserForIndex(apiKey, companyId, userId);
|
||||
} else {
|
||||
completeCoverage = false;
|
||||
}
|
||||
log.warn("[ziniao-index] skip user stores, companyId={}, userId={}, msg={}", companyId, userId, ex.getMessage());
|
||||
continue;
|
||||
@@ -231,7 +270,10 @@ public class ZiniaoShopIndexService {
|
||||
continue;
|
||||
}
|
||||
String storesFingerprint = buildStoresFingerprint(stores);
|
||||
Long duplicatedUserId = storesFingerprintToUserId.putIfAbsent(storesFingerprint, userId);
|
||||
Long duplicatedUserId = storesFingerprintToUserId.get(storesFingerprint);
|
||||
if (duplicatedUserId == null) {
|
||||
duplicatedUserId = apiKeyFingerprints.putIfAbsent(storesFingerprint, userId);
|
||||
}
|
||||
if (duplicatedUserId != null) {
|
||||
ziniaoTransientCacheService.delete(
|
||||
CACHE_TYPE_SHOP_INDEX_SCOPE_SNAPSHOT,
|
||||
@@ -264,7 +306,7 @@ public class ZiniaoShopIndexService {
|
||||
entry.setStatus(STATUS_ACTIVE);
|
||||
entry.setLastSeenAt(now);
|
||||
entry.setLastRefreshedAt(now);
|
||||
List<ZiniaoShopIndexEntryDto> candidates = grouped.computeIfAbsent(normalizedShopName, ignored -> new ArrayList<>());
|
||||
List<ZiniaoShopIndexEntryDto> candidates = apiKeyGrouped.computeIfAbsent(normalizedShopName, ignored -> new ArrayList<>());
|
||||
if (candidates.stream().noneMatch(existing -> sameCandidate(existing, entry))) {
|
||||
candidates.add(entry);
|
||||
}
|
||||
@@ -278,6 +320,17 @@ public class ZiniaoShopIndexService {
|
||||
);
|
||||
}
|
||||
}
|
||||
for (Map.Entry<String, List<ZiniaoShopIndexEntryDto>> apiKeyEntry : apiKeyGrouped.entrySet()) {
|
||||
List<ZiniaoShopIndexEntryDto> candidates = grouped.computeIfAbsent(apiKeyEntry.getKey(), ignored -> new ArrayList<>());
|
||||
for (ZiniaoShopIndexEntryDto candidate : apiKeyEntry.getValue()) {
|
||||
if (candidates.stream().noneMatch(existing -> sameCandidate(existing, candidate))) {
|
||||
candidates.add(candidate);
|
||||
}
|
||||
}
|
||||
}
|
||||
storesFingerprintToUserId.putAll(apiKeyFingerprints);
|
||||
markIpWhitelistAllowedSafely(apiKeyAccount);
|
||||
completedApiKeyCount++;
|
||||
}
|
||||
|
||||
List<ZiniaoShopIndexEntryDto> roundEntries = new ArrayList<>();
|
||||
@@ -314,28 +367,35 @@ public class ZiniaoShopIndexService {
|
||||
}
|
||||
log.info("[ziniao-index] refresh persist shopIndex uniqueShopId={} nameAlias={} groupedNames={} activeRows={}",
|
||||
byShopId.size(), byNameAlias.size(), grouped.size(), activeCacheKeys.size());
|
||||
boolean fullRefresh = apiKeyAccounts.size() >= allApiKeyAccounts.size();
|
||||
boolean fullRefresh = completeCoverage && apiKeyAccounts.size() >= allApiKeyAccounts.size();
|
||||
if (fullRefresh) {
|
||||
markMissingEntriesAsStale(activeCacheKeys, now);
|
||||
} else {
|
||||
log.info("[ziniao-index] skip stale marking for partial refresh processedApiKeys={}/{} nextOffset={}",
|
||||
apiKeyAccounts.size(), allApiKeyAccounts.size(), nextOffset);
|
||||
log.info("[ziniao-index] skip stale marking for partial refresh completedApiKeys={}/{} skippedApiKeys={} nextOffset={}",
|
||||
completedApiKeyCount, allApiKeyAccounts.size(), skippedApiKeyCount, nextOffset);
|
||||
}
|
||||
|
||||
cursor.setStatus("SUCCESS");
|
||||
cursor.setMessage(allInvalidUserIds.isEmpty()
|
||||
? null
|
||||
: "本轮刷新已跳过无效 userId 数: " + allInvalidUserIds.size());
|
||||
List<String> refreshMessages = new ArrayList<>();
|
||||
if (!allInvalidUserIds.isEmpty()) {
|
||||
refreshMessages.add("本轮刷新已跳过无效 userId 数: " + allInvalidUserIds.size());
|
||||
}
|
||||
if (skippedApiKeyCount > 0) {
|
||||
refreshMessages.add("本轮刷新已跳过 apiKey 数: " + skippedApiKeyCount
|
||||
+ "(IP 白名单: " + whitelistSkippedApiKeyCount + ")");
|
||||
}
|
||||
cursor.setMessage(refreshMessages.isEmpty() ? null : String.join(";", refreshMessages));
|
||||
cursor.setApiKeyTotal(allApiKeyAccounts.size());
|
||||
cursor.setLastProcessedApiKeyCount(apiKeyAccounts.size());
|
||||
cursor.setLastProcessedApiKeyCount(completedApiKeyCount);
|
||||
cursor.setNextApiKeyOffset(nextOffset);
|
||||
cursor.setInvalidUserCount(allInvalidUserIds.size());
|
||||
cursor.setSampleInvalidUserIds(allInvalidUserIds.stream().limit(20).toList());
|
||||
cursor.setLastFinishedAt(now);
|
||||
cursor.setLastSuccessAt(now);
|
||||
ziniaoTransientCacheService.put(CACHE_TYPE_SHOP_INDEX_REFRESH_CURSOR, "global", cursor, DEFAULT_CURSOR_TTL);
|
||||
log.info("[ziniao-index] refresh success invalidUsersSkipped={} apiKeyProcessed={}/{} nextOffset={}",
|
||||
allInvalidUserIds.size(), apiKeyAccounts.size(), allApiKeyAccounts.size(), nextOffset);
|
||||
log.info("[ziniao-index] refresh success invalidUsersSkipped={} apiKeyCompleted={}/{} apiKeySkipped={} whitelistSkipped={} nextOffset={}",
|
||||
allInvalidUserIds.size(), completedApiKeyCount, allApiKeyAccounts.size(), skippedApiKeyCount,
|
||||
whitelistSkippedApiKeyCount, nextOffset);
|
||||
} catch (Exception ex) {
|
||||
cursor.setStatus("FAILED");
|
||||
cursor.setMessage(ex.getMessage());
|
||||
@@ -358,6 +418,24 @@ public class ZiniaoShopIndexService {
|
||||
log.info("[ziniao-index] cursor invalidated (transient only; shop rows unchanged)");
|
||||
}
|
||||
|
||||
private void markIpWhitelistAllowedSafely(ZiniaoApiKeyProvider.ApiKeyAccount account) {
|
||||
try {
|
||||
ziniaoApiKeyProvider.markIpWhitelistAllowed(account);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[ziniao-index] failed to record IP whitelist status accountName={} status=ALLOWED msg={}",
|
||||
account.accountName(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void markIpWhitelistBlockedSafely(ZiniaoApiKeyProvider.ApiKeyAccount account, String message) {
|
||||
try {
|
||||
ziniaoApiKeyProvider.markIpWhitelistBlocked(account, message);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[ziniao-index] failed to record IP whitelist status accountName={} status=BLOCKED msg={}",
|
||||
account.accountName(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public String normalizeShopName(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
@@ -723,9 +801,4 @@ public class ZiniaoShopIndexService {
|
||||
}
|
||||
}
|
||||
|
||||
private void rethrowIfIpWhitelistError(BusinessException ex) {
|
||||
if (ziniaoAuthService.isIpWhitelistError(ex)) {
|
||||
throw new BusinessException("紫鸟刷新店铺索引失败:当前服务器 IP 未加入紫鸟白名单,已停止本轮刷新且不会更新索引数据");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL=https://api.coze.cn
|
||||
AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH=/v1/workflow/run
|
||||
AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_ID=7632683471312355338
|
||||
AIIMAGE_APPEARANCE_PATENT_COZE_TOKEN=
|
||||
AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE=50
|
||||
AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE=10
|
||||
AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS=60000
|
||||
AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES=30
|
||||
|
||||
|
||||
@@ -90,6 +90,7 @@ aiimage:
|
||||
bucket: ${AIIMAGE_OSS_BUCKET:nanri-ai-images}
|
||||
image-video-bucket: ${AIIMAGE_IMAGE_VIDEO_OSS_BUCKET:shufu-video}
|
||||
digital-human-bucket: ${AIIMAGE_DIGITAL_HUMAN_OSS_BUCKET:nanri-ai-digital-human}
|
||||
template-bucket: ${AIIMAGE_TEMPLATE_OSS_BUCKET:aiimage-templates}
|
||||
access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:appuser}
|
||||
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:AppUser@2024SecureKey}
|
||||
transient-storage:
|
||||
@@ -153,7 +154,7 @@ aiimage:
|
||||
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
|
||||
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
|
||||
retention-days: ${AIIMAGE_MODULE_CLEANUP_RETENTION_DAYS:7}
|
||||
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,SHOP_DATA_CRAWL,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA}
|
||||
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA}
|
||||
permission-schema-init:
|
||||
enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false}
|
||||
task-pressure:
|
||||
@@ -185,6 +186,7 @@ aiimage:
|
||||
local-worker-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS:10000}
|
||||
stuck-scan-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_STUCK_SCAN_DELAY_MS:60000}
|
||||
stuck-timeout-minutes: ${AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES:30}
|
||||
heartbeat-interval-ms: ${AIIMAGE_RESULT_FILE_JOB_HEARTBEAT_INTERVAL_MS:60000}
|
||||
batch-size: ${AIIMAGE_RESULT_FILE_JOB_BATCH_SIZE:20}
|
||||
coze-task:
|
||||
max-concurrent: ${AIIMAGE_COZE_TASK_MAX_CONCURRENT:12}
|
||||
@@ -200,7 +202,7 @@ aiimage:
|
||||
coze-workflow-path: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH:/v1/workflow/run}
|
||||
coze-workflow-id: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_ID:7639685157562089513}
|
||||
coze-token: ${AIIMAGE_APPEARANCE_PATENT_COZE_TOKEN:}
|
||||
coze-batch-size: ${AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE:20}
|
||||
coze-batch-size: ${AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE:10}
|
||||
coze-connect-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_CONNECT_TIMEOUT_MILLIS:10000}
|
||||
coze-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS:60000}
|
||||
coze-poll-interval-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_POLL_INTERVAL_MILLIS:30000}
|
||||
@@ -225,9 +227,12 @@ aiimage:
|
||||
coze-submit-min-interval-millis: ${AIIMAGE_SIMILAR_ASIN_COZE_SUBMIT_MIN_INTERVAL_MILLIS:5000}
|
||||
coze-flush-pending-minutes: ${AIIMAGE_SIMILAR_ASIN_COZE_FLUSH_PENDING_MINUTES:1}
|
||||
coze-submit-max-retry-count: ${AIIMAGE_SIMILAR_ASIN_COZE_SUBMIT_MAX_RETRY_COUNT:5}
|
||||
image-download-pool-size: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_POOL_SIZE:32}
|
||||
image-download-pool-size: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_POOL_SIZE:8}
|
||||
image-download-timeout-seconds: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DOWNLOAD_TIMEOUT_SECONDS:5}
|
||||
image-prefetch-timeout-seconds: ${AIIMAGE_SIMILAR_ASIN_IMAGE_PREFETCH_TIMEOUT_SECONDS:1800}
|
||||
result-file-timeout-minutes: ${AIIMAGE_SIMILAR_ASIN_RESULT_FILE_TIMEOUT_MINUTES:90}
|
||||
image-cache-max-bytes: ${AIIMAGE_SIMILAR_ASIN_IMAGE_CACHE_MAX_BYTES:268435456}
|
||||
image-local-cache-dir: ${AIIMAGE_SIMILAR_ASIN_IMAGE_LOCAL_CACHE_DIR:${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp}/similar-asin-image-cache}
|
||||
image-db-cache-enabled: ${AIIMAGE_SIMILAR_ASIN_IMAGE_DB_CACHE_ENABLED:false}
|
||||
stale-timeout-minutes: ${AIIMAGE_SIMILAR_ASIN_STALE_TIMEOUT_MINUTES:30}
|
||||
stale-finalize-cron: ${AIIMAGE_SIMILAR_ASIN_STALE_FINALIZE_CRON:0 */2 * * * *}
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE biz_shop_key
|
||||
ADD COLUMN ip_whitelist_status VARCHAR(16) NOT NULL DEFAULT 'UNKNOWN' COMMENT 'IP白名单状态:UNKNOWN/ALLOWED/BLOCKED' AFTER ziniao_token,
|
||||
ADD COLUMN ip_whitelist_checked_at DATETIME NULL COMMENT '最近一次IP白名单检测时间' AFTER ip_whitelist_status,
|
||||
ADD COLUMN ip_whitelist_message VARCHAR(500) NULL COMMENT '最近一次IP白名单检测信息' AFTER ip_whitelist_checked_at;
|
||||
@@ -0,0 +1,34 @@
|
||||
-- Durable marker for retry-exhausted result-file callbacks.
|
||||
SET @terminal_callback_db_name = DATABASE();
|
||||
|
||||
SET @terminal_callback_column_exists = (
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA = @terminal_callback_db_name
|
||||
AND TABLE_NAME = 'biz_task_file_job'
|
||||
AND COLUMN_NAME = 'terminal_callback_at'
|
||||
);
|
||||
SET @terminal_callback_sql = IF(
|
||||
@terminal_callback_column_exists = 0,
|
||||
'ALTER TABLE `biz_task_file_job` ADD COLUMN `terminal_callback_at` DATETIME NULL COMMENT ''parent terminal failure callback completed'' AFTER `finished_at`',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE terminal_callback_stmt FROM @terminal_callback_sql;
|
||||
EXECUTE terminal_callback_stmt;
|
||||
DEALLOCATE PREPARE terminal_callback_stmt;
|
||||
|
||||
SET @terminal_callback_index_exists = (
|
||||
SELECT COUNT(*)
|
||||
FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = @terminal_callback_db_name
|
||||
AND TABLE_NAME = 'biz_task_file_job'
|
||||
AND INDEX_NAME = 'idx_file_job_terminal_callback'
|
||||
);
|
||||
SET @terminal_callback_sql = IF(
|
||||
@terminal_callback_index_exists = 0,
|
||||
'ALTER TABLE `biz_task_file_job` ADD INDEX `idx_file_job_terminal_callback` (`status`, `retry_count`, `terminal_callback_at`, `updated_at`)',
|
||||
'SELECT 1'
|
||||
);
|
||||
PREPARE terminal_callback_stmt FROM @terminal_callback_sql;
|
||||
EXECUTE terminal_callback_stmt;
|
||||
DEALLOCATE PREPARE terminal_callback_stmt;
|
||||
@@ -0,0 +1,25 @@
|
||||
-- Admin menu and independent data permission for shop-data-crawl task history.
|
||||
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
|
||||
SELECT '店铺数据任务管理', 'admin_shop_data_crawl_tasks', 'admin', 'shop-data-crawl-tasks', 82
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM `columns` WHERE `column_key` = 'admin_shop_data_crawl_tasks'
|
||||
);
|
||||
|
||||
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
|
||||
SELECT '店铺数据任务数据查看', 'admin_shop_data_crawl_task_data', 'internal', 'shop-data-crawl-task-data', 0
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM `columns` WHERE `column_key` = 'admin_shop_data_crawl_task_data'
|
||||
);
|
||||
|
||||
-- Preserve the video-task convention: users who already received the visible
|
||||
-- task menu receive the initial data grant. Later changes use the dedicated UI.
|
||||
INSERT IGNORE INTO `user_column_permission` (`user_id`, `column_id`)
|
||||
SELECT old_perm.`user_id`, data_col.`id`
|
||||
FROM `user_column_permission` old_perm
|
||||
INNER JOIN `columns` old_col ON old_col.`id` = old_perm.`column_id`
|
||||
INNER JOIN `columns` data_col ON data_col.`column_key` = 'admin_shop_data_crawl_task_data'
|
||||
LEFT JOIN `user_column_permission` existing
|
||||
ON existing.`user_id` = old_perm.`user_id`
|
||||
AND existing.`column_id` = data_col.`id`
|
||||
WHERE old_col.`column_key` = 'admin_shop_data_crawl_tasks'
|
||||
AND existing.`user_id` IS NULL;
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+7
@@ -10,6 +10,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class AppearancePatentTaskServiceTest {
|
||||
|
||||
@Test
|
||||
void taskStatusDependsOnExecutionOutcomeOnly() {
|
||||
assertEquals("RUNNING", AppearancePatentTaskService.resolveTaskExecutionStatus(true, false));
|
||||
assertEquals("SUCCESS", AppearancePatentTaskService.resolveTaskExecutionStatus(false, false));
|
||||
assertEquals("FAILED", AppearancePatentTaskService.resolveTaskExecutionStatus(false, true));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resultStatusFailsOnlyWhenConclusionIsEmpty() {
|
||||
assertEquals("\u5931\u8d25", AppearancePatentTaskService.resolveResultStatus(null));
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.nanri.aiimage.modules.auth.util;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class WerkzeugPasswordEncoderTest {
|
||||
|
||||
private final WerkzeugPasswordEncoder encoder = new WerkzeugPasswordEncoder();
|
||||
|
||||
@Test
|
||||
void matchesWerkzeugScryptHash() {
|
||||
String hash = "scrypt:32768:8:1$VEWwSnHkAFcK3B3E$2065ba2db25c34072bf4d7ae9bd47b8c483289fafc3f5e98ae49bb232d30bf989b21ae0e9f14adc6e893e34a81543cf58641731431961ed9ab353bcebec2e78a";
|
||||
|
||||
assertTrue(encoder.matches("test-password", hash));
|
||||
assertFalse(encoder.matches("wrong-password", hash));
|
||||
}
|
||||
|
||||
@Test
|
||||
void matchesGeneratedPbkdf2Hash() {
|
||||
String hash = encoder.hash("test-password");
|
||||
|
||||
assertTrue(encoder.matches("test-password", hash));
|
||||
assertFalse(encoder.matches("wrong-password", hash));
|
||||
}
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSummaryRowDto;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class CollectDataExcelAssemblyServiceTest {
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
@Test
|
||||
void writesAsinFilterToSummarySheet() throws Exception {
|
||||
CollectDataSummaryRowDto summary = new ObjectMapper().readValue(
|
||||
"{\"keyword\":\"phone case\",\"asinFilter\":3}",
|
||||
CollectDataSummaryRowDto.class);
|
||||
File output = tempDir.resolve("collect-data-result.xlsx").toFile();
|
||||
|
||||
new CollectDataExcelAssemblyService().writeWorkbook(output, List.of(), List.of(summary), List.of());
|
||||
|
||||
try (Workbook workbook = WorkbookFactory.create(output)) {
|
||||
Sheet sheet = workbook.getSheet("结果文件");
|
||||
assertThat(sheet.getRow(0).getCell(6).getStringCellValue()).isEqualTo("ASIN过滤");
|
||||
assertThat(sheet.getRow(1).getCell(6).getNumericCellValue()).isEqualTo(3);
|
||||
}
|
||||
}
|
||||
}
|
||||
+196
@@ -0,0 +1,196 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class CollectDataServiceTest {
|
||||
|
||||
@Mock
|
||||
private FileTaskMapper fileTaskMapper;
|
||||
|
||||
@Mock
|
||||
private FileResultMapper fileResultMapper;
|
||||
|
||||
@Mock
|
||||
private TaskChunkMapper taskChunkMapper;
|
||||
|
||||
@Mock
|
||||
private TaskDistributedLockService taskDistributedLockService;
|
||||
|
||||
@Mock
|
||||
private TaskFileJobService taskFileJobService;
|
||||
|
||||
@Mock
|
||||
private TaskDistributedLockService.LockHandle lockHandle;
|
||||
|
||||
@Spy
|
||||
private ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@InjectMocks
|
||||
private CollectDataService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
ReflectionTestUtils.setField(service, "staleTimeoutMinutes", 30L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dashboardCountsPendingAndRunningTasksAsActive() {
|
||||
when(fileTaskMapper.selectCount(any())).thenReturn(5L, 3L, 2L);
|
||||
|
||||
CollectDataDashboardVo dashboard = service.dashboard(7L);
|
||||
|
||||
assertThat(dashboard.getPendingTaskCount()).isEqualTo(5L);
|
||||
assertThat(dashboard.getSuccessTaskCount()).isEqualTo(3L);
|
||||
assertThat(dashboard.getFailedTaskCount()).isEqualTo(2L);
|
||||
assertThat(dashboard.getProcessedTaskCount()).isEqualTo(5L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void progressBatchExposesProcessedKeywordProgress() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(91L);
|
||||
task.setTaskNo("COLLECT_DATA-91");
|
||||
task.setModuleType(CollectDataService.MODULE_TYPE);
|
||||
task.setStatus("RUNNING");
|
||||
task.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||
task.setResultJson("{\"totalRows\":10,\"receivedRows\":24,\"processedRows\":4}");
|
||||
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(101L);
|
||||
result.setTaskId(task.getId());
|
||||
result.setModuleType(CollectDataService.MODULE_TYPE);
|
||||
result.setRowCount(10);
|
||||
result.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0));
|
||||
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(fileResultMapper.selectList(any())).thenReturn(List.of(result));
|
||||
|
||||
CollectDataTaskBatchVo progress = service.progressBatch(List.of(task.getId()));
|
||||
|
||||
assertThat(progress.getItems()).hasSize(1);
|
||||
assertThat(progress.getItems().getFirst().getItems()).hasSize(1);
|
||||
assertThat(progress.getItems().getFirst().getItems().getFirst().getTotalRows()).isEqualTo(10);
|
||||
assertThat(progress.getItems().getFirst().getItems().getFirst().getReceivedRows()).isEqualTo(24);
|
||||
assertThat(progress.getItems().getFirst().getItems().getFirst().getProcessedRows()).isEqualTo(4);
|
||||
assertThat(progress.getItems().getFirst().getItems().getFirst().getProgressPercent()).isEqualTo(40);
|
||||
}
|
||||
|
||||
@Test
|
||||
void failTaskMarksAnActivatedTaskAsFailed() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(92L);
|
||||
task.setUserId(7L);
|
||||
task.setModuleType(CollectDataService.MODULE_TYPE);
|
||||
task.setStatus("RUNNING");
|
||||
task.setResultJson("{\"totalRows\":10,\"receivedRows\":4}");
|
||||
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(102L);
|
||||
result.setTaskId(task.getId());
|
||||
result.setModuleType(CollectDataService.MODULE_TYPE);
|
||||
|
||||
when(fileTaskMapper.selectById(task.getId())).thenReturn(task);
|
||||
when(fileResultMapper.selectOne(any())).thenReturn(result);
|
||||
|
||||
service.failTask(task.getId(), task.getUserId(), "queue unavailable");
|
||||
|
||||
assertThat(task.getStatus()).isEqualTo("FAILED");
|
||||
assertThat(task.getErrorMessage()).isEqualTo("queue unavailable");
|
||||
assertThat(result.getSuccess()).isZero();
|
||||
assertThat(result.getErrorMessage()).isEqualTo("queue unavailable");
|
||||
verify(fileResultMapper).updateById(result);
|
||||
verify(fileTaskMapper).updateById(task);
|
||||
}
|
||||
|
||||
@Test
|
||||
void staleTaskWithResultChunksEnqueuesPartialWorkbook() throws Exception {
|
||||
FileTaskEntity task = staleTask(93L);
|
||||
task.setResultJson("{\"finalRowCount\":343}");
|
||||
FileResultEntity result = taskResult(task, 103L);
|
||||
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, task.getId(), 0L))
|
||||
.thenReturn(lockHandle);
|
||||
when(fileTaskMapper.selectById(task.getId())).thenReturn(task);
|
||||
when(taskFileJobService.countUnfinishedAssembleJobs(task.getId(), CollectDataService.MODULE_TYPE))
|
||||
.thenReturn(0L);
|
||||
when(fileResultMapper.selectOne(any())).thenReturn(result);
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(170L);
|
||||
|
||||
service.finalizeStaleTasks();
|
||||
|
||||
verify(taskFileJobService).enqueueAssembleResult(
|
||||
task.getId(), CollectDataService.MODULE_TYPE, result.getId(), "task:" + task.getId());
|
||||
assertThat(task.getStatus()).isEqualTo("RUNNING");
|
||||
assertThat(result.getRowCount()).isEqualTo(343);
|
||||
verify(lockHandle).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void staleTaskWithoutResultChunksFails() throws Exception {
|
||||
FileTaskEntity task = staleTask(94L);
|
||||
task.setResultJson("{\"finalRowCount\":0}");
|
||||
FileResultEntity result = taskResult(task, 104L);
|
||||
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(taskDistributedLockService.acquire(CollectDataService.MODULE_TYPE, task.getId(), 0L))
|
||||
.thenReturn(lockHandle);
|
||||
when(fileTaskMapper.selectById(task.getId())).thenReturn(task);
|
||||
when(taskFileJobService.countUnfinishedAssembleJobs(task.getId(), CollectDataService.MODULE_TYPE))
|
||||
.thenReturn(0L);
|
||||
when(fileResultMapper.selectOne(any())).thenReturn(result);
|
||||
when(taskChunkMapper.selectCount(any())).thenReturn(0L);
|
||||
|
||||
service.finalizeStaleTasks();
|
||||
|
||||
assertThat(task.getStatus()).isEqualTo("FAILED");
|
||||
assertThat(task.getErrorMessage()).contains("Python 心跳");
|
||||
verify(taskFileJobService, never()).enqueueAssembleResult(any(), any(), any(), any());
|
||||
verify(lockHandle).close();
|
||||
}
|
||||
|
||||
private static FileTaskEntity staleTask(long taskId) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(taskId);
|
||||
task.setModuleType(CollectDataService.MODULE_TYPE);
|
||||
task.setStatus("RUNNING");
|
||||
task.setUpdatedAt(LocalDateTime.now().minusMinutes(31));
|
||||
return task;
|
||||
}
|
||||
|
||||
private static FileResultEntity taskResult(FileTaskEntity task, long resultId) {
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(resultId);
|
||||
result.setTaskId(task.getId());
|
||||
result.setModuleType(CollectDataService.MODULE_TYPE);
|
||||
result.setSourceFilename("collect.xlsx");
|
||||
return result;
|
||||
}
|
||||
}
|
||||
+46
-1
@@ -1,5 +1,8 @@
|
||||
package com.nanri.aiimage.modules.dedupe.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
|
||||
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
|
||||
@@ -16,6 +19,7 @@ import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
@@ -159,7 +163,7 @@ class DedupeTotalDataServiceTest {
|
||||
when(dedupeTotalDataMapper.selectCount(any())).thenReturn(0L);
|
||||
when(dedupeTotalDataMapper.selectList(any())).thenReturn(List.of());
|
||||
|
||||
DedupeTotalDataPageVo page = service.page(1, 15, "", "other", 10L);
|
||||
DedupeTotalDataPageVo page = service.page(1, 15, "", "other", null, null, 10L);
|
||||
|
||||
assertEquals(0L, page.getTotal());
|
||||
assertTrue(page.getItems().isEmpty());
|
||||
@@ -168,6 +172,47 @@ class DedupeTotalDataServiceTest {
|
||||
verify(dedupeTotalDataMapper).selectList(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
void pageUsesInclusiveDateRange() {
|
||||
when(adminUserMapper.selectById(1L)).thenReturn(user(1L, "super_admin", "root"));
|
||||
when(dedupeTotalDataMapper.selectCount(any())).thenReturn(0L);
|
||||
when(dedupeTotalDataMapper.selectList(any())).thenReturn(List.of());
|
||||
|
||||
service.page(
|
||||
1,
|
||||
15,
|
||||
"",
|
||||
"",
|
||||
LocalDate.of(2026, 7, 31),
|
||||
LocalDate.of(2026, 7, 31),
|
||||
1L);
|
||||
|
||||
ArgumentCaptor<LambdaQueryWrapper<DedupeTotalDataEntity>> queryCaptor =
|
||||
ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
|
||||
verify(dedupeTotalDataMapper).selectCount(queryCaptor.capture());
|
||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = queryCaptor.getValue();
|
||||
TableInfoHelper.initTableInfo(
|
||||
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
DedupeTotalDataEntity.class);
|
||||
query.getSqlSegment();
|
||||
assertTrue(query.getParamNameValuePairs().containsValue(LocalDate.of(2026, 7, 31).atStartOfDay()));
|
||||
assertTrue(query.getParamNameValuePairs().containsValue(LocalDate.of(2026, 8, 1).atStartOfDay()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void pageRejectsReversedDateRange() {
|
||||
assertThrows(BusinessException.class, () -> service.page(
|
||||
1,
|
||||
15,
|
||||
"",
|
||||
"",
|
||||
LocalDate.of(2026, 8, 1),
|
||||
LocalDate.of(2026, 7, 31),
|
||||
1L));
|
||||
verify(dedupeTotalDataMapper, never()).selectCount(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void comparableValueLookupRemainsGlobal() {
|
||||
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678")))
|
||||
|
||||
+5
@@ -21,6 +21,7 @@ class OssStorageServiceTest {
|
||||
properties.setBucket("nanri-ai-images");
|
||||
properties.setImageVideoBucket("shufu-video");
|
||||
properties.setDigitalHumanBucket("nanri-ai-digital-human");
|
||||
properties.setTemplateBucket("aiimage-templates");
|
||||
properties.setAccessKeyId("test-access-key");
|
||||
properties.setAccessKeySecret("test-secret-key");
|
||||
storageService = new OssStorageService(properties);
|
||||
@@ -54,6 +55,10 @@ class OssStorageServiceTest {
|
||||
"https://oss.aishufu.top/nanri-ai-digital-human/digital-human/versions/demo.mp4",
|
||||
storageService.normalizeManagedPublicUrl(
|
||||
"https://nanri-ai-digital-human.oss.aishufu.top/digital-human/versions/demo.mp4"));
|
||||
assertEquals(
|
||||
"https://oss.aishufu.top/aiimage-templates/input/publish.xlsx",
|
||||
storageService.normalizeManagedPublicUrl(
|
||||
"http://47.110.241.161:9000/aiimage-templates/input/publish.xlsx"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package com.nanri.aiimage.modules.filetemplate;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
|
||||
class ModuleTemplateControllerTest {
|
||||
|
||||
@Test
|
||||
void returnsXlsxWithUtf8AttachmentFilename() throws Exception {
|
||||
ModuleTemplateService service = mock(ModuleTemplateService.class);
|
||||
byte[] bytes = {1, 2, 3, 4};
|
||||
when(service.download("publish")).thenReturn(new ModuleTemplateService.TemplateDownload(
|
||||
"上架 文档格式.xlsx",
|
||||
ModuleTemplateService.XLSX_CONTENT_TYPE,
|
||||
bytes));
|
||||
MockMvc mockMvc = MockMvcBuilders.standaloneSetup(new ModuleTemplateController(service)).build();
|
||||
|
||||
mockMvc.perform(get("/api/module-templates/publish/download").param("user_id", "1"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(content().bytes(bytes))
|
||||
.andExpect(content().contentType(ModuleTemplateService.XLSX_CONTENT_TYPE))
|
||||
.andExpect(header().longValue(HttpHeaders.CONTENT_LENGTH, bytes.length))
|
||||
.andExpect(header().string(HttpHeaders.CONTENT_DISPOSITION,
|
||||
"attachment; filename=\"download.xlsx\"; "
|
||||
+ "filename*=UTF-8''%E4%B8%8A%E6%9E%B6%20%E6%96%87%E6%A1%A3%E6%A0%BC%E5%BC%8F.xlsx"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void propagatesUnknownModuleAndStorageStatusCodes() throws Exception {
|
||||
ModuleTemplateService service = mock(ModuleTemplateService.class);
|
||||
when(service.download("missing"))
|
||||
.thenThrow(new ResponseStatusException(HttpStatus.NOT_FOUND, "模板不存在"));
|
||||
when(service.download("publish"))
|
||||
.thenThrow(new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "模板存储暂不可用"));
|
||||
MockMvc mockMvc = MockMvcBuilders.standaloneSetup(new ModuleTemplateController(service)).build();
|
||||
|
||||
mockMvc.perform(get("/api/module-templates/missing/download"))
|
||||
.andExpect(status().isNotFound());
|
||||
mockMvc.perform(get("/api/module-templates/publish/download"))
|
||||
.andExpect(status().isServiceUnavailable());
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package com.nanri.aiimage.modules.filetemplate;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class ModuleTemplateRegistryTest {
|
||||
|
||||
@Test
|
||||
void exposesSixFixedModulesWithPackagedXlsxResources() throws Exception {
|
||||
ModuleTemplateRegistry registry = new ModuleTemplateRegistry();
|
||||
|
||||
assertEquals(Set.of(
|
||||
"publish",
|
||||
"delete-brand",
|
||||
"appearance-patent",
|
||||
"price-track",
|
||||
"collect-data",
|
||||
"similar-asin"),
|
||||
registry.templates().stream()
|
||||
.map(ModuleTemplateRegistry.ModuleTemplate::moduleCode)
|
||||
.collect(Collectors.toSet()));
|
||||
|
||||
for (ModuleTemplateRegistry.ModuleTemplate template : registry.templates()) {
|
||||
assertTrue(template.resourcePath().matches("templates/module-input/[a-z-]+\\.xlsx"));
|
||||
assertEquals("input/" + template.resourcePath().substring(template.resourcePath().lastIndexOf('/') + 1),
|
||||
template.objectKey());
|
||||
ClassPathResource resource = new ClassPathResource(template.resourcePath());
|
||||
assertTrue(resource.exists(), template.resourcePath());
|
||||
try (InputStream input = resource.getInputStream()) {
|
||||
assertArrayEquals(new byte[]{'P', 'K'}, input.readNBytes(2), template.resourcePath());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void lookupNormalizesCaseAndWhitespaceWithoutAcceptingUnknownCodes() {
|
||||
ModuleTemplateRegistry registry = new ModuleTemplateRegistry();
|
||||
|
||||
assertEquals("上架 文档格式.xlsx", registry.find(" PUBLISH ").orElseThrow().downloadFilename());
|
||||
assertTrue(registry.find("../../publish").isEmpty());
|
||||
assertTrue(registry.find(null).isEmpty());
|
||||
}
|
||||
}
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
package com.nanri.aiimage.modules.filetemplate;
|
||||
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ModuleTemplateServiceTest {
|
||||
|
||||
private static final String BUCKET = "aiimage-templates";
|
||||
|
||||
private OssStorageService ossStorageService;
|
||||
private ModuleTemplateService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
OssProperties properties = new OssProperties();
|
||||
properties.setTemplateBucket(BUCKET);
|
||||
ossStorageService = mock(OssStorageService.class);
|
||||
service = new ModuleTemplateService(new ModuleTemplateRegistry(), properties, ossStorageService);
|
||||
}
|
||||
|
||||
@Test
|
||||
void downloadsExistingObjectWithoutUploadingItAgain() {
|
||||
byte[] stored = {1, 2, 3};
|
||||
when(ossStorageService.objectExists(BUCKET, "input/publish.xlsx")).thenReturn(true);
|
||||
when(ossStorageService.readObjectBytes(BUCKET, "input/publish.xlsx")).thenReturn(stored);
|
||||
|
||||
ModuleTemplateService.TemplateDownload download = service.download("publish");
|
||||
|
||||
assertEquals("上架 文档格式.xlsx", download.filename());
|
||||
assertEquals(ModuleTemplateService.XLSX_CONTENT_TYPE, download.contentType());
|
||||
assertArrayEquals(stored, download.content());
|
||||
verify(ossStorageService).ensureBucketExists(BUCKET);
|
||||
verify(ossStorageService, never()).uploadBytes(
|
||||
eq(BUCKET), eq("input/publish.xlsx"), org.mockito.ArgumentMatchers.any(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void lazilyUploadsMissingObjectBeforeReadingItFromStorage() {
|
||||
byte[] stored = {4, 5, 6};
|
||||
when(ossStorageService.objectExists(BUCKET, "input/publish.xlsx")).thenReturn(false);
|
||||
when(ossStorageService.readObjectBytes(BUCKET, "input/publish.xlsx")).thenReturn(stored);
|
||||
|
||||
ModuleTemplateService.TemplateDownload download = service.download("publish");
|
||||
|
||||
assertArrayEquals(stored, download.content());
|
||||
ArgumentCaptor<byte[]> uploaded = ArgumentCaptor.forClass(byte[].class);
|
||||
verify(ossStorageService).uploadBytes(
|
||||
eq(BUCKET), eq("input/publish.xlsx"), uploaded.capture(),
|
||||
eq(ModuleTemplateService.XLSX_CONTENT_TYPE));
|
||||
assertTrue(uploaded.getValue().length > 2);
|
||||
assertArrayEquals(new byte[]{'P', 'K'}, new byte[]{uploaded.getValue()[0], uploaded.getValue()[1]});
|
||||
verify(ossStorageService).readObjectBytes(BUCKET, "input/publish.xlsx");
|
||||
}
|
||||
|
||||
@Test
|
||||
void startupSynchronizationUploadsOnlyMissingTemplates() {
|
||||
when(ossStorageService.objectExists(BUCKET, "input/publish.xlsx")).thenReturn(true);
|
||||
|
||||
assertEquals(5, service.synchronizeAll());
|
||||
|
||||
verify(ossStorageService).ensureBucketExists(BUCKET);
|
||||
verify(ossStorageService, times(6)).objectExists(eq(BUCKET), anyString());
|
||||
verify(ossStorageService, never()).uploadBytes(
|
||||
eq(BUCKET), eq("input/publish.xlsx"), org.mockito.ArgumentMatchers.any(), anyString());
|
||||
verify(ossStorageService, times(5)).uploadBytes(
|
||||
eq(BUCKET), anyString(), org.mockito.ArgumentMatchers.any(),
|
||||
eq(ModuleTemplateService.XLSX_CONTENT_TYPE));
|
||||
}
|
||||
|
||||
@Test
|
||||
void unknownModuleReturnsNotFoundWithoutTouchingStorage() {
|
||||
ResponseStatusException ex = assertThrows(ResponseStatusException.class,
|
||||
() -> service.download("not-a-module"));
|
||||
|
||||
assertEquals(HttpStatus.NOT_FOUND, ex.getStatusCode());
|
||||
verifyNoInteractions(ossStorageService);
|
||||
}
|
||||
|
||||
@Test
|
||||
void storageFailureReturnsServiceUnavailable() {
|
||||
doThrow(new IllegalStateException("MinIO unavailable"))
|
||||
.when(ossStorageService).ensureBucketExists(BUCKET);
|
||||
|
||||
ResponseStatusException ex = assertThrows(ResponseStatusException.class,
|
||||
() -> service.download("publish"));
|
||||
|
||||
assertEquals(HttpStatus.SERVICE_UNAVAILABLE, ex.getStatusCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void initializerLeavesApplicationRunningWhenStartupStorageIsUnavailable() {
|
||||
ModuleTemplateService unavailableService = mock(ModuleTemplateService.class);
|
||||
doThrow(new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "unavailable"))
|
||||
.when(unavailableService).synchronizeAll();
|
||||
ModuleTemplateStorageInitializer initializer = new ModuleTemplateStorageInitializer(unavailableService);
|
||||
|
||||
assertDoesNotThrow(initializer::synchronize);
|
||||
verify(unavailableService).synchronizeAll();
|
||||
}
|
||||
}
|
||||
+28
@@ -248,4 +248,32 @@ class PermissionMenuControllerTest {
|
||||
verify(service).listImageVideoDataPermissionUsers(operator);
|
||||
verify(service).updateImageVideoDataPermissionUsers(operator, List.of(20L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shopDataCrawlPermissionEndpointsDelegateAuthenticatedOperator() {
|
||||
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
|
||||
PermissionMenuService service = mock(PermissionMenuService.class);
|
||||
HttpServletRequest request = mock(HttpServletRequest.class);
|
||||
PermissionMenuController controller = new PermissionMenuController(authSupport, service);
|
||||
AdminUserEntity operator = new AdminUserEntity();
|
||||
operator.setId(1L);
|
||||
operator.setRole("super_admin");
|
||||
ImageVideoDataPermissionUserVo permissionUser = new ImageVideoDataPermissionUserVo();
|
||||
permissionUser.setId(20L);
|
||||
ImageVideoDataPermissionUpdateRequest body = new ImageVideoDataPermissionUpdateRequest();
|
||||
body.setUserIds(List.of(20L));
|
||||
|
||||
when(authSupport.requireAdmin(request)).thenReturn(operator);
|
||||
when(service.listShopDataCrawlDataPermissionUsers(operator)).thenReturn(List.of(permissionUser));
|
||||
when(service.updateShopDataCrawlDataPermissionUsers(operator, List.of(20L))).thenReturn(1);
|
||||
|
||||
var listResponse = controller.listShopDataCrawlDataPermissionUsers(request);
|
||||
var updateResponse = controller.updateShopDataCrawlDataPermissionUsers(request, body);
|
||||
|
||||
assertThat(listResponse.getData()).containsExactly(permissionUser);
|
||||
assertThat(updateResponse.getData()).isEqualTo(1);
|
||||
verify(authSupport, times(2)).requireAdmin(request);
|
||||
verify(service).listShopDataCrawlDataPermissionUsers(operator);
|
||||
verify(service).updateShopDataCrawlDataPermissionUsers(operator, List.of(20L));
|
||||
}
|
||||
}
|
||||
|
||||
+78
@@ -74,6 +74,28 @@ class PermissionMenuServiceTest {
|
||||
verify(permissionMapper, times(0)).insert(any(UserColumnPermissionEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void preservesBothIndependentDataPermissionsDuringGenericReplacement() {
|
||||
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
|
||||
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
|
||||
AdminUserMapper userMapper = mock(AdminUserMapper.class);
|
||||
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
|
||||
when(userMapper.selectById(7L)).thenReturn(new AdminUserEntity());
|
||||
when(menuMapper.selectCount(any())).thenReturn(1L);
|
||||
when(menuMapper.selectOne(any())).thenReturn(imageVideoDataPermission(), shopDataCrawlDataPermission());
|
||||
when(permissionMapper.selectCount(any())).thenReturn(1L);
|
||||
|
||||
UserColumnPermissionUpdateRequest request = new UserColumnPermissionUpdateRequest();
|
||||
request.setColumnIds(List.of(2L));
|
||||
service.updateUserColumnPermissions(7L, request);
|
||||
|
||||
ArgumentCaptor<UserColumnPermissionEntity> captor = ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
|
||||
verify(permissionMapper, times(3)).insert(captor.capture());
|
||||
assertThat(captor.getAllValues())
|
||||
.extracting(UserColumnPermissionEntity::getColumnId)
|
||||
.containsExactly(2L, 75L, 76L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void expandsDirectParentGrantToDescendantsWithoutPersistingChildren() {
|
||||
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
|
||||
@@ -526,6 +548,55 @@ class PermissionMenuServiceTest {
|
||||
assertThat(inserted.getValue().getColumnId()).isEqualTo(75L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shopDataCrawlPermissionCanOnlyBeManagedBySuperAdmin() {
|
||||
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
|
||||
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
|
||||
AdminUserMapper userMapper = mock(AdminUserMapper.class);
|
||||
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
|
||||
AdminUserEntity explicitAdmin = user(1L, "admin", 1);
|
||||
|
||||
assertThatThrownBy(() -> service.listShopDataCrawlDataPermissionUsers(explicitAdmin))
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessageContaining("超级管理员");
|
||||
assertThatThrownBy(() -> service.updateShopDataCrawlDataPermissionUsers(explicitAdmin, List.of(2L)))
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessageContaining("超级管理员");
|
||||
verify(menuMapper, never()).selectOne(any());
|
||||
verify(permissionMapper, never()).delete(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void superAdminCanListAndReplaceShopDataCrawlPermissions() {
|
||||
PermissionMenuMapper menuMapper = mock(PermissionMenuMapper.class);
|
||||
UserColumnPermissionMapper permissionMapper = mock(UserColumnPermissionMapper.class);
|
||||
AdminUserMapper userMapper = mock(AdminUserMapper.class);
|
||||
PermissionMenuService service = new PermissionMenuService(menuMapper, permissionMapper, userMapper);
|
||||
AdminUserEntity operator = user(99L, "super_admin", 1);
|
||||
AdminUserEntity admin = user(1L, "admin", 1);
|
||||
admin.setUsername("admin");
|
||||
AdminUserEntity normal = user(2L, "normal", 0);
|
||||
normal.setUsername("normal");
|
||||
|
||||
when(menuMapper.selectOne(any())).thenReturn(shopDataCrawlDataPermission());
|
||||
when(permissionMapper.selectList(any())).thenReturn(List.of(grant(1L, 76L)));
|
||||
when(userMapper.selectList(any())).thenReturn(List.of(operator, admin, normal));
|
||||
|
||||
List<ImageVideoDataPermissionUserVo> users = service.listShopDataCrawlDataPermissionUsers(operator);
|
||||
int grantedCount = service.updateShopDataCrawlDataPermissionUsers(operator, List.of(2L));
|
||||
|
||||
assertThat(users).extracting(ImageVideoDataPermissionUserVo::getId).containsExactly(1L, 2L);
|
||||
assertThat(users.get(0).isGranted()).isTrue();
|
||||
assertThat(users.get(1).isGranted()).isFalse();
|
||||
assertThat(grantedCount).isEqualTo(1);
|
||||
verify(permissionMapper).deleteByMap(Map.of("column_id", 76L));
|
||||
ArgumentCaptor<UserColumnPermissionEntity> inserted =
|
||||
ArgumentCaptor.forClass(UserColumnPermissionEntity.class);
|
||||
verify(permissionMapper).insert(inserted.capture());
|
||||
assertThat(inserted.getValue().getUserId()).isEqualTo(2L);
|
||||
assertThat(inserted.getValue().getColumnId()).isEqualTo(76L);
|
||||
}
|
||||
|
||||
private PermissionMenuCreateRequest createRequest(Long parentId, String menuType) {
|
||||
PermissionMenuCreateRequest request = new PermissionMenuCreateRequest();
|
||||
request.setName("child");
|
||||
@@ -575,4 +646,11 @@ class PermissionMenuServiceTest {
|
||||
entity.setColumnKey("admin_image_video_task_data");
|
||||
return entity;
|
||||
}
|
||||
|
||||
private PermissionMenuEntity shopDataCrawlDataPermission() {
|
||||
PermissionMenuEntity entity = new PermissionMenuEntity();
|
||||
entity.setId(76L);
|
||||
entity.setColumnKey("admin_shop_data_crawl_task_data");
|
||||
return entity;
|
||||
}
|
||||
}
|
||||
|
||||
+60
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.pricetrack.mapper.PriceTrackShopCandidateMapper;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopkey.service.SkipPriceAsinService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
@@ -24,8 +25,12 @@ import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@@ -86,4 +91,59 @@ class PriceTrackTaskServiceTest {
|
||||
verify(priceTrackLoopRunService).syncLoopRunAfterChildTerminal(taskId);
|
||||
verify(lock).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void completedPayloadWithoutUsableRowsFailsWithoutCreatingResultFile() throws Exception {
|
||||
long taskId = 20818L;
|
||||
String shopName = "蔡建芳";
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(taskId);
|
||||
task.setUserId(672L);
|
||||
task.setModuleType("PRICE_TRACK");
|
||||
task.setStatus("RUNNING");
|
||||
task.setRequestJson("{}");
|
||||
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setId(92327L);
|
||||
result.setTaskId(taskId);
|
||||
result.setModuleType("PRICE_TRACK");
|
||||
result.setSourceFilename(shopName);
|
||||
result.setSuccess(0);
|
||||
|
||||
PriceTrackSubmitResultRequest.AsinResult blankRow = new PriceTrackSubmitResultRequest.AsinResult();
|
||||
blankRow.setShopMallName("Cai Jianfang");
|
||||
blankRow.setAsin("");
|
||||
|
||||
PriceTrackSubmitResultRequest.ShopResult shopResult = new PriceTrackSubmitResultRequest.ShopResult();
|
||||
shopResult.setShopName(shopName);
|
||||
shopResult.setCountries(Map.of("DE", List.of(blankRow)));
|
||||
shopResult.setError("");
|
||||
shopResult.setSuccess(true);
|
||||
|
||||
PriceTrackSubmitResultRequest request = new PriceTrackSubmitResultRequest();
|
||||
request.setShops(List.of(shopResult));
|
||||
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
when(taskDistributedLockService.acquire("PRICE_TRACK", taskId)).thenReturn(lock);
|
||||
when(priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId))).thenReturn(Map.of(taskId, task));
|
||||
when(fileResultMapper.selectList(any())).thenReturn(List.of(result));
|
||||
when(ziniaoShopSwitchService.normalizeShopName(shopName)).thenReturn(shopName);
|
||||
when(excelAssemblyService.normalizeCountriesMap(any())).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
when(objectMapper.writeValueAsString(any())).thenReturn("[]");
|
||||
|
||||
service.submitResult(taskId, request);
|
||||
|
||||
assertEquals("FAILED", task.getStatus());
|
||||
assertEquals(shopName + ": 未收到有效跟价数据,未生成结果文件", task.getErrorMessage());
|
||||
assertNotNull(task.getFinishedAt());
|
||||
assertEquals(0, result.getSuccess());
|
||||
assertEquals("未收到有效跟价数据,未生成结果文件", result.getErrorMessage());
|
||||
assertEquals(0, result.getRowCount());
|
||||
assertNull(result.getResultFilename());
|
||||
assertNull(result.getResultFileUrl());
|
||||
verify(taskFileJobService, never()).enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
|
||||
verify(priceTrackLoopRunService).syncLoopRunAfterChildTerminal(taskId);
|
||||
verify(lock).close();
|
||||
}
|
||||
}
|
||||
|
||||
+52
@@ -22,6 +22,7 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
@@ -164,6 +165,19 @@ class PublishTaskServiceTest {
|
||||
taskCaptor.getValue().getResultJson()).path("ownerInstanceId").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void zeroOnlyHeartbeatDoesNotResetResultChunkProgress() {
|
||||
TaskHeartbeatRequest heartbeat = new TaskHeartbeatRequest();
|
||||
heartbeat.setCurrent(0);
|
||||
heartbeat.setTotal(0);
|
||||
|
||||
service.touchHeartbeat(20998L, heartbeat);
|
||||
|
||||
verify(fileTaskMapper).update(isNull(), any());
|
||||
verify(publishFileMapper, never()).selectOne(any());
|
||||
verify(publishFileMapper, never()).update(isNull(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void taskAccessRejectsAnotherInstanceForExistingRouterToForward() throws Exception {
|
||||
long taskId = 109L;
|
||||
@@ -344,6 +358,7 @@ class PublishTaskServiceTest {
|
||||
long resultId = 313L;
|
||||
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||
PublishFileEntity file = file(taskId, fileId, "RUNNING", "分片.xlsx");
|
||||
file.setTotalRows(2);
|
||||
FileResultEntity result = result(taskId, resultId);
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
|
||||
@@ -357,15 +372,21 @@ class PublishTaskServiceTest {
|
||||
service.submitResult(taskId, chunkResultRequest(7L, fileId, 2, 2, List.of(row("2"))));
|
||||
|
||||
assertEquals("RUNNING", file.getStatus());
|
||||
assertEquals(1, file.getProcessedRows());
|
||||
assertEquals(1, storedChunks.size());
|
||||
assertEquals(1, storedScopes.getFirst().getReceivedChunkCount());
|
||||
assertEquals(0, storedScopes.getFirst().getCompleted());
|
||||
assertTrue(storedScopes.getFirst().getStateJson().contains("\"receivedRows\":1"));
|
||||
verify(publishItemMapper, never()).delete(any());
|
||||
verify(taskFileJobService, never()).enqueueAssembleResult(any(), any(), any(), any());
|
||||
|
||||
// Simulate a scope created by the pre-progress implementation.
|
||||
storedScopes.getFirst().setStateJson("{\"phase\":\"RECEIVING\"}");
|
||||
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 2, List.of(row("1"))));
|
||||
|
||||
assertEquals("SUCCESS", file.getStatus());
|
||||
assertEquals(2, file.getProcessedRows());
|
||||
assertTrue(storedScopes.getFirst().getStateJson().contains("\"receivedRows\":2"));
|
||||
assertEquals(2, storedChunks.size());
|
||||
assertEquals(2, storedScopes.getFirst().getReceivedChunkCount());
|
||||
assertEquals(1, storedScopes.getFirst().getCompleted());
|
||||
@@ -386,6 +407,7 @@ class PublishTaskServiceTest {
|
||||
long fileId = 214L;
|
||||
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||
PublishFileEntity file = file(taskId, fileId, "RUNNING", "重试.xlsx");
|
||||
file.setTotalRows(2);
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
PublishSubmitResultRequest request = chunkResultRequest(7L, fileId, 1, 2, List.of(row("1")));
|
||||
|
||||
@@ -399,11 +421,41 @@ class PublishTaskServiceTest {
|
||||
service.submitResult(taskId, request);
|
||||
|
||||
assertEquals(1, storedChunks.size());
|
||||
assertEquals(1, file.getProcessedRows());
|
||||
assertEquals(1, rustfsPayloads.size());
|
||||
verify(taskChunkMapper, times(1)).insert(any(TaskChunkEntity.class));
|
||||
verify(publishItemMapper, never()).delete(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void progressBatchExposesRowsReceivedByResultSubmission() {
|
||||
long taskId = 122L;
|
||||
long fileId = 222L;
|
||||
FileTaskEntity task = task(taskId, 7L, "RUNNING");
|
||||
PublishFileEntity file = file(taskId, fileId, "RUNNING", "progress.xlsx");
|
||||
FileResultEntity result = result(taskId, 322L);
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
|
||||
when(taskDistributedLockService.acquire(PublishTaskService.MODULE_TYPE, taskId)).thenReturn(lock);
|
||||
when(fileTaskMapper.selectById(taskId)).thenReturn(task);
|
||||
when(fileTaskMapper.selectList(any())).thenReturn(List.of(task));
|
||||
when(publishFileMapper.selectById(fileId)).thenReturn(file);
|
||||
when(publishFileMapper.selectList(any())).thenReturn(List.of(file));
|
||||
when(fileResultMapper.selectOne(any())).thenReturn(result);
|
||||
when(fileResultMapper.selectList(any())).thenReturn(List.of(result));
|
||||
when(publishItemMapper.selectCount(any())).thenReturn(3L);
|
||||
when(taskFileJobService.findAssembleJobsByResultIds(any(), any())).thenReturn(Map.of());
|
||||
|
||||
service.submitResult(taskId, chunkResultRequest(7L, fileId, 1, 3, List.of(row("1"))));
|
||||
|
||||
var progress = service.getTaskProgress(7L, List.of(taskId));
|
||||
assertEquals(1, progress.getItems().getFirst().getTask().getProcessedRows());
|
||||
assertEquals(3, progress.getItems().getFirst().getTask().getTotalRows());
|
||||
assertEquals(1, progress.getItems().getFirst().getFiles().getFirst().getProcessedRows());
|
||||
assertEquals(3, progress.getItems().getFirst().getFiles().getFirst().getTotalRows());
|
||||
assertEquals(33, progress.getItems().getFirst().getFiles().getFirst().getPercent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void resultChunkRetryWithDifferentContentIsRejected() {
|
||||
long taskId = 115L;
|
||||
|
||||
+32
-32
@@ -17,11 +17,9 @@ import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.zip.ZipFile;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
@@ -37,33 +35,31 @@ class PublishWorkbookServiceTest {
|
||||
File valid = directory.resolve("valid.xlsx").toFile();
|
||||
try (Workbook workbook = new XSSFWorkbook();
|
||||
FileOutputStream output = new FileOutputStream(valid)) {
|
||||
Sheet uk = workbook.createSheet("英国数据");
|
||||
writeSourceSheet(uk, "英国", "B001");
|
||||
Sheet first = workbook.createSheet("first");
|
||||
writeSourceSheet(first, "UK", "B001");
|
||||
for (int index = PublishWorkbookService.SOURCE_HEADERS.size();
|
||||
index < PublishWorkbookService.RESULT_HEADERS.size(); index++) {
|
||||
uk.getRow(0).createCell(index)
|
||||
first.getRow(0).createCell(index)
|
||||
.setCellValue(PublishWorkbookService.RESULT_HEADERS.get(index));
|
||||
}
|
||||
uk.getRow(1).createCell(5).setCellValue("旧状态");
|
||||
uk.getRow(1).createCell(6).setCellValue("旧同步状态");
|
||||
uk.getRow(1).createCell(7).setCellValue("旧同步国家");
|
||||
workbook.createSheet("空白页");
|
||||
writeSourceSheet(workbook.createSheet("德国数据"), "DE", "B002");
|
||||
first.getRow(1).createCell(5).setCellValue("old status");
|
||||
workbook.createSheet("empty");
|
||||
writeSourceSheet(workbook.createSheet("second"), "DE", "B002");
|
||||
workbook.write(output);
|
||||
}
|
||||
|
||||
PublishWorkbookService.ParsedWorkbook parsed = service.parse(valid);
|
||||
assertEquals(2, parsed.rows().size());
|
||||
assertEquals("B001", parsed.rows().get(0).getAsin());
|
||||
assertNull(parsed.rows().get(0).getStatus());
|
||||
assertNull(parsed.rows().get(0).getSyncStatus());
|
||||
assertNull(parsed.rows().get(0).getSyncCountries());
|
||||
assertEquals("B001", parsed.rows().getFirst().getAsin());
|
||||
assertNull(parsed.rows().getFirst().getStatus());
|
||||
assertNull(parsed.rows().getFirst().getSyncStatus());
|
||||
assertNull(parsed.rows().getFirst().getSyncCountries());
|
||||
assertEquals("DE", parsed.rows().get(1).getCountry());
|
||||
|
||||
File invalid = directory.resolve("invalid.xlsx").toFile();
|
||||
try (Workbook workbook = new XSSFWorkbook();
|
||||
FileOutputStream output = new FileOutputStream(invalid)) {
|
||||
Sheet sheet = workbook.createSheet("错误表头");
|
||||
Sheet sheet = workbook.createSheet("invalid");
|
||||
Row header = sheet.createRow(0);
|
||||
List<String> headers = new ArrayList<>(PublishWorkbookService.SOURCE_HEADERS);
|
||||
headers.set(1, "Asin");
|
||||
@@ -79,30 +75,34 @@ class PublishWorkbookServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void writesOneSheetPerNormalizedCountryWithExactHeaders() throws Exception {
|
||||
void writesOnlyThePublishCountrySheetAndKeepsSyncColumns() throws Exception {
|
||||
Path directory = Files.createTempDirectory("publish-sheets-");
|
||||
try {
|
||||
File output = directory.resolve("result.xlsx").toFile();
|
||||
service.writeWorkbook(output, List.of(
|
||||
row("1", "B001", "UK", "19.99"),
|
||||
row("2", "B002", "德国", "not-a-number"),
|
||||
row("3", "B003", "GB", "20")));
|
||||
row("2", "B002", "FR", "not-a-number"),
|
||||
row("3", "B003", "GB", "20")), "DE");
|
||||
|
||||
try (FileInputStream input = new FileInputStream(output);
|
||||
Workbook workbook = new XSSFWorkbook(input)) {
|
||||
assertEquals(2, workbook.getNumberOfSheets());
|
||||
assertEquals(Set.of("英国", "德国"),
|
||||
Set.of(workbook.getSheetName(0), workbook.getSheetName(1)));
|
||||
Sheet uk = workbook.getSheet("英国");
|
||||
assertNotNull(uk);
|
||||
assertEquals(1, workbook.getNumberOfSheets());
|
||||
assertEquals("\u5fb7\u56fd", workbook.getSheetName(0));
|
||||
Sheet germany = workbook.getSheetAt(0);
|
||||
for (int index = 0; index < PublishWorkbookService.RESULT_HEADERS.size(); index++) {
|
||||
assertEquals(PublishWorkbookService.RESULT_HEADERS.get(index),
|
||||
uk.getRow(0).getCell(index).getStringCellValue());
|
||||
germany.getRow(0).getCell(index).getStringCellValue());
|
||||
}
|
||||
assertEquals(CellType.NUMERIC, germany.getRow(1).getCell(4).getCellType());
|
||||
assertEquals(19.99D, germany.getRow(1).getCell(4).getNumericCellValue(), 0.0001D);
|
||||
assertEquals("not-a-number", germany.getRow(2).getCell(4).getStringCellValue());
|
||||
for (int rowIndex = 1; rowIndex <= 3; rowIndex++) {
|
||||
assertEquals("\u5fb7\u56fd", germany.getRow(rowIndex).getCell(2).getStringCellValue());
|
||||
assertEquals("\u82f1\u56fd:\u6210\u529f\uff0c\u6cd5\u56fd:\u6210\u529f",
|
||||
germany.getRow(rowIndex).getCell(6).getStringCellValue());
|
||||
assertEquals("\u82f1\u56fd,\u6cd5\u56fd",
|
||||
germany.getRow(rowIndex).getCell(7).getStringCellValue());
|
||||
}
|
||||
assertEquals(CellType.NUMERIC, uk.getRow(1).getCell(4).getCellType());
|
||||
assertEquals(19.99D, uk.getRow(1).getCell(4).getNumericCellValue(), 0.0001D);
|
||||
assertEquals("not-a-number",
|
||||
workbook.getSheet("德国").getRow(1).getCell(4).getStringCellValue());
|
||||
}
|
||||
} finally {
|
||||
FileUtil.del(directory.toFile());
|
||||
@@ -115,7 +115,7 @@ class PublishWorkbookServiceTest {
|
||||
try {
|
||||
List<PublishWorkbookService.WorkbookInput> oneSuccess = List.of(
|
||||
new PublishWorkbookService.WorkbookInput(
|
||||
"郭亚庆.xlsx", "郭亚庆", List.of(row("1", "B001", "英国", "50"))));
|
||||
"shop.xlsx", "shop", "DE", List.of(row("1", "B001", "UK", "50"))));
|
||||
|
||||
PublishWorkbookService.PackagedResult single = service.packageTaskResult(
|
||||
directory.resolve("single").toFile(), "PUBLISH-1", 1, oneSuccess);
|
||||
@@ -155,9 +155,9 @@ class PublishWorkbookServiceTest {
|
||||
row.setCountry(country);
|
||||
row.setBrand("Brand");
|
||||
row.setPrice(price);
|
||||
row.setStatus("成功");
|
||||
row.setSyncStatus("成功");
|
||||
row.setSyncCountries("德国,法国");
|
||||
row.setStatus("success");
|
||||
row.setSyncStatus("\u82f1\u56fd:\u6210\u529f\uff0c\u6cd5\u56fd:\u6210\u529f");
|
||||
row.setSyncCountries("\u82f1\u56fd,\u6cd5\u56fd");
|
||||
return row;
|
||||
}
|
||||
}
|
||||
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.controller;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
|
||||
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class AdminShopDataCrawlTaskControllerTest {
|
||||
|
||||
@Test
|
||||
void deletesResultForAuthenticatedAdminWithTaskPermissions() {
|
||||
ShopDataCrawlTaskService taskService = mock(ShopDataCrawlTaskService.class);
|
||||
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
|
||||
PermissionMenuService permissionService = mock(PermissionMenuService.class);
|
||||
HttpServletRequest request = mock(HttpServletRequest.class);
|
||||
AdminShopDataCrawlTaskController controller = new AdminShopDataCrawlTaskController(
|
||||
taskService, authSupport, permissionService);
|
||||
AdminUserEntity operator = admin(8L);
|
||||
when(authSupport.requireAdmin(request)).thenReturn(operator);
|
||||
|
||||
controller.deleteHistory(101L, request);
|
||||
|
||||
verify(permissionService).requireShopDataCrawlTaskAccess(operator);
|
||||
verify(taskService).deleteAdminHistory(101L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void acceptsTrustedFlaskOperatorWhenLegacySessionHasNoJavaJwt() {
|
||||
ShopDataCrawlTaskService taskService = mock(ShopDataCrawlTaskService.class);
|
||||
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
|
||||
PermissionMenuService permissionService = mock(PermissionMenuService.class);
|
||||
HttpServletRequest request = mock(HttpServletRequest.class);
|
||||
AdminShopDataCrawlTaskController controller = new AdminShopDataCrawlTaskController(
|
||||
taskService, authSupport, permissionService);
|
||||
ReflectionTestUtils.setField(controller, "internalToken", "shared-token");
|
||||
AdminUserEntity operator = admin(8L);
|
||||
when(authSupport.requireAdmin(request)).thenThrow(new BusinessException(401, "not logged in"));
|
||||
when(request.getHeader("X-Internal-Token")).thenReturn("shared-token");
|
||||
when(request.getParameter("operatorId")).thenReturn("8");
|
||||
when(permissionService.requireAdminOperator(8L)).thenReturn(operator);
|
||||
|
||||
controller.deleteHistory(101L, request);
|
||||
|
||||
verify(permissionService).requireAdminOperator(8L);
|
||||
verify(permissionService).requireShopDataCrawlTaskAccess(operator);
|
||||
verify(taskService).deleteAdminHistory(101L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsAdminWithoutIndependentTaskDataPermission() {
|
||||
ShopDataCrawlTaskService taskService = mock(ShopDataCrawlTaskService.class);
|
||||
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
|
||||
PermissionMenuService permissionService = mock(PermissionMenuService.class);
|
||||
HttpServletRequest request = mock(HttpServletRequest.class);
|
||||
AdminShopDataCrawlTaskController controller = new AdminShopDataCrawlTaskController(
|
||||
taskService, authSupport, permissionService);
|
||||
AdminUserEntity operator = admin(8L);
|
||||
BusinessException denied = new BusinessException(403, "no data permission");
|
||||
when(authSupport.requireAdmin(request)).thenReturn(operator);
|
||||
org.mockito.Mockito.doThrow(denied)
|
||||
.when(permissionService).requireShopDataCrawlTaskAccess(operator);
|
||||
|
||||
assertThatThrownBy(() -> controller.deleteHistory(101L, request)).isSameAs(denied);
|
||||
|
||||
verify(taskService, never()).deleteAdminHistory(101L);
|
||||
}
|
||||
|
||||
private AdminUserEntity admin(Long id) {
|
||||
AdminUserEntity operator = new AdminUserEntity();
|
||||
operator.setId(id);
|
||||
operator.setRole("admin");
|
||||
return operator;
|
||||
}
|
||||
}
|
||||
+26
-1
@@ -3,16 +3,22 @@ package com.nanri.aiimage.modules.shopdatacrawl.service;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ShopDataCrawlExcelAssemblyServiceTest {
|
||||
@TempDir Path tempDir;
|
||||
@@ -22,6 +28,8 @@ class ShopDataCrawlExcelAssemblyServiceTest {
|
||||
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||
row.setDate("2026-07-25");
|
||||
row.setAsin("B012345678");
|
||||
row.setBrand("Example Brand");
|
||||
row.setCommodityImage("https://m.media-amazon.com/images/I/example.jpg");
|
||||
row.setInventorySales("11");
|
||||
row.setSalesRank("22");
|
||||
row.setPageViews("33");
|
||||
@@ -35,8 +43,11 @@ class ShopDataCrawlExcelAssemblyServiceTest {
|
||||
item.setSuccess(true);
|
||||
item.setCountryResults(List.of(country));
|
||||
|
||||
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
||||
when(imageEmbedder.fetchAndResizeForCache(row.getCommodityImage()))
|
||||
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
|
||||
File output = tempDir.resolve("result.xlsx").toFile();
|
||||
new ShopDataCrawlExcelAssemblyService().writeWorkbook(output, List.of(item));
|
||||
new ShopDataCrawlExcelAssemblyService(imageEmbedder).writeWorkbook(output, List.of(item));
|
||||
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
|
||||
assertEquals(ShopDataCrawlExcelAssemblyService.SHEETS,
|
||||
@@ -50,7 +61,21 @@ class ShopDataCrawlExcelAssemblyServiceTest {
|
||||
}
|
||||
assertEquals("2026-07-25", workbook.getSheet("英国").getRow(1).getCell(0).getStringCellValue());
|
||||
assertEquals("B012345678", workbook.getSheet("英国").getRow(1).getCell(1).getStringCellValue());
|
||||
assertEquals("11", workbook.getSheet("英国").getRow(1).getCell(3).getStringCellValue());
|
||||
assertEquals("Example Brand", workbook.getSheet("英国").getRow(1)
|
||||
.getCell(ShopDataCrawlExcelAssemblyService.HEADERS.size() - 1).getStringCellValue());
|
||||
assertEquals(1, workbook.getAllPictures().size());
|
||||
assertEquals(1, workbook.getSheet("英国").getDrawingPatriarch().getShapes().size());
|
||||
assertEquals(80f, workbook.getSheet("英国").getRow(1).getHeightInPoints());
|
||||
assertEquals(18 * 256, workbook.getSheet("英国").getColumnWidth(2));
|
||||
assertEquals(0, workbook.getSheet("德国").getLastRowNum());
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] jpegBytes() throws Exception {
|
||||
BufferedImage image = new BufferedImage(2, 2, BufferedImage.TYPE_INT_RGB);
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
ImageIO.write(image, "jpg", output);
|
||||
return output.toByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
+13
@@ -183,6 +183,18 @@ class ShopDataCrawlTaskServiceChunkTest {
|
||||
"task:" + task.getId() + ":owner:instance-a");
|
||||
}
|
||||
|
||||
@Test
|
||||
void preservesBrandFromAppClientThroughChunkMerge() {
|
||||
givenRunningTask(112L, 212L);
|
||||
ShopDataCrawlRowDto item = row("2026-07-25", "B001");
|
||||
item.setBrand("Example Brand");
|
||||
|
||||
service.submitResult(task.getId(), request(chunk(1, 1, "DE", item)));
|
||||
|
||||
assertEquals(1, result.getSuccess());
|
||||
assertTrue(task.getResultJson().contains("\"brand\":\"Example Brand\""));
|
||||
}
|
||||
|
||||
@Test
|
||||
void identicalResultChunkRetryIsIdempotent() {
|
||||
givenRunningTask(102L, 202L);
|
||||
@@ -528,6 +540,7 @@ class ShopDataCrawlTaskServiceChunkTest {
|
||||
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||
row.setDate(date);
|
||||
row.setAsin(asin);
|
||||
row.setCommodityImage("https://m.media-amazon.com/images/I/" + asin + ".jpg");
|
||||
row.setInventorySales("10");
|
||||
row.setSalesRank("20");
|
||||
row.setPageViews("30");
|
||||
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.apache.ibatis.session.Configuration;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class ShopDataCrawlTaskServiceRetentionTest {
|
||||
|
||||
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
|
||||
private static final Long USER_ID = 7L;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new Configuration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||
}
|
||||
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private ShopDataCrawlResolveService resolveService;
|
||||
@Mock private ShopDataCrawlExcelAssemblyService excelAssemblyService;
|
||||
@Mock private ShopDataCrawlTaskCacheService cacheService;
|
||||
@Mock private OssStorageService ossStorageService;
|
||||
@Mock private ZiniaoShopSwitchService ziniaoShopSwitchService;
|
||||
@Mock private TaskPressureProperties taskPressureProperties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private TaskResultItemService taskResultItemService;
|
||||
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private InstanceMetadata instanceMetadata;
|
||||
@Spy private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
@InjectMocks private ShopDataCrawlTaskService service;
|
||||
|
||||
@Test
|
||||
void keepsNewestThreePerStableShopAndFallsBackToShopName() {
|
||||
LocalDateTime now = LocalDateTime.of(2026, 8, 5, 12, 0);
|
||||
FileResultEntity shopIdOldest = result(11L, 111L, USER_ID, "shop-1", "Renamed Shop",
|
||||
"result/shop-id-oldest.xlsx", 1, now.minusDays(2));
|
||||
FileResultEntity shopNameOldest = result(21L, 121L, USER_ID, " ", "Fallback Shop",
|
||||
"result/shop-name-oldest.xlsx", 1, now.minusDays(4));
|
||||
|
||||
List<FileResultEntity> rows = List.of(
|
||||
result(13L, 113L, USER_ID, "shop-1", "Current Name", "result/13.xlsx", 1, now.minusDays(1)),
|
||||
result(23L, 123L, USER_ID, null, "Fallback Shop", "result/23.xlsx", 1, now.minusDays(2)),
|
||||
shopIdOldest,
|
||||
result(14L, 114L, USER_ID, "shop-1", "Current Name", "result/14.xlsx", 1, now),
|
||||
result(24L, 124L, USER_ID, null, "Fallback Shop", "result/24.xlsx", 1, now),
|
||||
result(12L, 112L, USER_ID, "shop-1", "Old Name", "result/12.xlsx", 1, now.minusDays(2)),
|
||||
shopNameOldest,
|
||||
result(22L, 122L, USER_ID, null, "Fallback Shop", "result/22.xlsx", 1, now.minusDays(3)),
|
||||
result(1L, 101L, USER_ID, "shop-1", "Current Name", "result/failed.xlsx", 0, now.minusDays(9)),
|
||||
result(2L, 102L, USER_ID, "shop-1", "Current Name", null, 1, now.minusDays(9)),
|
||||
result(3L, 103L, 99L, "shop-1", "Current Name", "result/other-user.xlsx", 1, now.minusDays(9)),
|
||||
result(4L, 104L, USER_ID, "shop-2", "Current Name", "result/other-shop.xlsx", 1, now.minusDays(9)));
|
||||
|
||||
when(fileResultMapper.selectList(any())).thenReturn(rows);
|
||||
when(fileResultMapper.selectById(11L)).thenReturn(shopIdOldest);
|
||||
when(fileResultMapper.selectById(21L)).thenReturn(shopNameOldest);
|
||||
when(fileTaskMapper.selectById(111L)).thenReturn(terminalTask(111L));
|
||||
when(fileTaskMapper.selectById(121L)).thenReturn(terminalTask(121L));
|
||||
when(taskDistributedLockService.acquire(MODULE_TYPE, 111L))
|
||||
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
|
||||
when(taskDistributedLockService.acquire(MODULE_TYPE, 121L))
|
||||
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
|
||||
when(cacheService.getTaskCacheBatch(any())).thenReturn(Map.of());
|
||||
when(fileResultMapper.selectCount(any())).thenReturn(0L);
|
||||
|
||||
service.pruneCompletedHistoryForShop(USER_ID, "shop-id:shop-1");
|
||||
service.pruneCompletedHistoryForShop(USER_ID, "shop-name:Fallback Shop");
|
||||
|
||||
verify(fileResultMapper).deleteById(11L);
|
||||
verify(fileResultMapper).deleteById(21L);
|
||||
verify(taskFileJobService).deleteResultJobs(111L, MODULE_TYPE, 11L);
|
||||
verify(taskFileJobService).deleteResultJobs(121L, MODULE_TYPE, 21L);
|
||||
verify(taskResultItemService).deleteResultItem(111L, MODULE_TYPE, 11L);
|
||||
verify(taskResultItemService).deleteResultItem(121L, MODULE_TYPE, 21L);
|
||||
verify(ossStorageService).deleteObject("result/shop-id-oldest.xlsx");
|
||||
verify(ossStorageService).deleteObject("result/shop-name-oldest.xlsx");
|
||||
verify(fileResultMapper, never()).deleteById(1L);
|
||||
verify(fileResultMapper, never()).deleteById(2L);
|
||||
verify(fileResultMapper, never()).deleteById(3L);
|
||||
verify(fileResultMapper, never()).deleteById(4L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotDeleteOldFileWhileOwningTaskIsStillRunning() {
|
||||
LocalDateTime now = LocalDateTime.of(2026, 8, 5, 12, 0);
|
||||
FileResultEntity oldest = result(31L, 131L, USER_ID, "shop-running", "Running Shop",
|
||||
"result/running-oldest.xlsx", 1, now.minusDays(3));
|
||||
when(fileResultMapper.selectList(any())).thenReturn(List.of(
|
||||
result(34L, 134L, USER_ID, "shop-running", "Running Shop", "result/34.xlsx", 1, now),
|
||||
result(33L, 133L, USER_ID, "shop-running", "Running Shop", "result/33.xlsx", 1, now.minusDays(1)),
|
||||
result(32L, 132L, USER_ID, "shop-running", "Running Shop", "result/32.xlsx", 1, now.minusDays(2)),
|
||||
oldest));
|
||||
when(fileResultMapper.selectById(31L)).thenReturn(oldest);
|
||||
when(fileTaskMapper.selectById(131L)).thenReturn(task(131L, "RUNNING"));
|
||||
when(taskDistributedLockService.acquire(MODULE_TYPE, 131L))
|
||||
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
|
||||
|
||||
service.pruneCompletedHistoryForShop(USER_ID, "shop-id:shop-running");
|
||||
|
||||
verify(fileResultMapper, never()).deleteById(31L);
|
||||
verify(taskFileJobService, never()).deleteResultJobs(131L, MODULE_TYPE, 31L);
|
||||
verify(ossStorageService, never()).deleteObject("result/running-oldest.xlsx");
|
||||
}
|
||||
|
||||
private FileResultEntity result(Long id, Long taskId, Long userId, String shopId, String shopName,
|
||||
String resultFileUrl, int success, LocalDateTime createdAt) {
|
||||
FileResultEntity row = new FileResultEntity();
|
||||
row.setId(id);
|
||||
row.setTaskId(taskId);
|
||||
row.setModuleType(MODULE_TYPE);
|
||||
row.setUserId(userId);
|
||||
row.setSourceFileUrl(shopId);
|
||||
row.setSourceFilename(shopName);
|
||||
row.setResultFileUrl(resultFileUrl);
|
||||
row.setSuccess(success);
|
||||
row.setCreatedAt(createdAt);
|
||||
return row;
|
||||
}
|
||||
|
||||
private FileTaskEntity terminalTask(Long id) {
|
||||
return task(id, "SUCCESS");
|
||||
}
|
||||
|
||||
private FileTaskEntity task(Long id, String status) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(id);
|
||||
task.setModuleType(MODULE_TYPE);
|
||||
task.setStatus(status);
|
||||
return task;
|
||||
}
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package com.nanri.aiimage.modules.shopkey.service;
|
||||
|
||||
import com.nanri.aiimage.modules.shopkey.mapper.ShopManageMapper;
|
||||
import com.nanri.aiimage.modules.shopkey.mapper.SkipPriceAsinMapper;
|
||||
import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinCreateRequest;
|
||||
import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity;
|
||||
import com.nanri.aiimage.modules.shopkey.model.entity.SkipPriceAsinEntity;
|
||||
import com.nanri.aiimage.modules.shopkey.model.vo.SkipPriceAsinItemVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotSame;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SkipPriceAsinServiceTest {
|
||||
|
||||
@Mock
|
||||
private SkipPriceAsinMapper skipPriceAsinMapper;
|
||||
@Mock
|
||||
private ShopManageMapper shopManageMapper;
|
||||
@Mock
|
||||
private ShopManageGroupService shopManageGroupService;
|
||||
|
||||
@InjectMocks
|
||||
private SkipPriceAsinService service;
|
||||
|
||||
@Test
|
||||
void createInsertsNewRowWhenGroupAndShopAlreadyExist() {
|
||||
ShopManageGroupEntity group = new ShopManageGroupEntity();
|
||||
group.setId(10L);
|
||||
group.setGroupName("group-a");
|
||||
when(shopManageGroupService.getAccessibleById(10L, 7L, true)).thenReturn(group);
|
||||
|
||||
SkipPriceAsinEntity existing = new SkipPriceAsinEntity();
|
||||
existing.setId(100L);
|
||||
existing.setGroupId(10L);
|
||||
existing.setShopName("shop-a");
|
||||
existing.setAsinDe("OLD-ASIN");
|
||||
lenient().when(skipPriceAsinMapper.selectOne(any())).thenReturn(existing);
|
||||
|
||||
AtomicReference<SkipPriceAsinEntity> inserted = new AtomicReference<>();
|
||||
when(skipPriceAsinMapper.insert(any(SkipPriceAsinEntity.class))).thenAnswer(invocation -> {
|
||||
SkipPriceAsinEntity entity = invocation.getArgument(0);
|
||||
entity.setId(101L);
|
||||
inserted.set(entity);
|
||||
return 1;
|
||||
});
|
||||
when(skipPriceAsinMapper.selectById(101L)).thenAnswer(invocation -> inserted.get());
|
||||
|
||||
SkipPriceAsinCreateRequest request = new SkipPriceAsinCreateRequest();
|
||||
request.setGroupId(10L);
|
||||
request.setShopName("shop-a");
|
||||
request.setCountries(List.of("DE"));
|
||||
request.setAsinMappings(Map.of("DE", "NEW-ASIN"));
|
||||
request.setMinimumPriceMappings(Map.of("DE", new BigDecimal("19.99")));
|
||||
|
||||
SkipPriceAsinItemVo result = service.create(request, 7L, true);
|
||||
|
||||
ArgumentCaptor<SkipPriceAsinEntity> captor = ArgumentCaptor.forClass(SkipPriceAsinEntity.class);
|
||||
verify(skipPriceAsinMapper).insert(captor.capture());
|
||||
verify(skipPriceAsinMapper, never()).selectOne(any());
|
||||
verify(skipPriceAsinMapper, never()).updateById(any(SkipPriceAsinEntity.class));
|
||||
assertNotSame(existing, captor.getValue());
|
||||
assertEquals("OLD-ASIN", existing.getAsinDe());
|
||||
assertEquals(101L, result.getId());
|
||||
assertEquals("NEW-ASIN", result.getAsinDe());
|
||||
assertEquals(new BigDecimal("19.99"), result.getMinimumPriceDe());
|
||||
}
|
||||
}
|
||||
+57
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.similarasin.client;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
@@ -180,4 +181,60 @@ class SimilarAsinCozeClientTest {
|
||||
assertTrue(json.contains("\"price\":10"));
|
||||
assertTrue(json.contains("\"price\":80"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void buildParametersIncludesCategorySwitch() throws Exception {
|
||||
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
||||
row.setAsin("B0CATEGORY1");
|
||||
row.setTitle("Category test");
|
||||
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null);
|
||||
|
||||
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
|
||||
"buildParameters", List.class, String.class, String.class, boolean.class, boolean.class);
|
||||
method.setAccessible(true);
|
||||
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, List.of(row), "", "", false, true);
|
||||
|
||||
assertEquals(Boolean.TRUE, parameters.get("category_switch"));
|
||||
assertEquals(Boolean.FALSE, parameters.get("img_switch"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void imageOnlyWorkflowOutputIsExtractedAndMergedByAsin() throws Exception {
|
||||
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null);
|
||||
String imageData = """
|
||||
{"data":[{
|
||||
"asin":"B0BQNHDP2F",
|
||||
"main_url":"https://example.com/main.jpg",
|
||||
"puzzle_img1":"https://example.com/puzzle-1.jpg",
|
||||
"puzzle_img2":"https://example.com/puzzle-2.jpg"
|
||||
}]}
|
||||
""";
|
||||
String workflowOutput = objectMapper.writeValueAsString(Map.of(
|
||||
"node_status", "{}",
|
||||
"Output", imageData));
|
||||
var historyResponse = objectMapper.createObjectNode();
|
||||
historyResponse.put("code", 0);
|
||||
historyResponse.putArray("data")
|
||||
.addObject()
|
||||
.put("execute_status", "Success")
|
||||
.put("output", workflowOutput);
|
||||
Method extract = SimilarAsinCozeClient.class.getDeclaredMethod("extractResultDataText", JsonNode.class);
|
||||
extract.setAccessible(true);
|
||||
String dataText = (String) extract.invoke(client, historyResponse);
|
||||
|
||||
SimilarAsinResultRowDto source = new SimilarAsinResultRowDto();
|
||||
source.setAsin("B0BQNHDP2F");
|
||||
List<SimilarAsinResultRowDto> merged = client.mergeRowsFromDataText(List.of(source), dataText);
|
||||
|
||||
assertFalse(dataText.isBlank());
|
||||
assertEquals(1, merged.size());
|
||||
assertEquals("https://example.com/main.jpg", merged.getFirst().getMainUrl());
|
||||
assertEquals("https://example.com/puzzle-1.jpg", merged.getFirst().getPuzzleImg1());
|
||||
assertEquals("https://example.com/puzzle-2.jpg", merged.getFirst().getPuzzleImg2());
|
||||
|
||||
Method resolvedCount = SimilarAsinCozeClient.class.getDeclaredMethod("resolvedCount", List.class);
|
||||
resolvedCount.setAccessible(true);
|
||||
assertEquals(1, resolvedCount.invoke(client, merged));
|
||||
}
|
||||
}
|
||||
|
||||
+53
@@ -1,9 +1,11 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
@@ -60,6 +62,57 @@ class SimilarAsinTaskServiceTest {
|
||||
assertFalse(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 2, 3));
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedResultRowsWithBlankIsConformEnableCategoryRetry() {
|
||||
SimilarAsinParsedRowVo blankCategory = new SimilarAsinParsedRowVo();
|
||||
blankCategory.setValues(new LinkedHashMap<>());
|
||||
blankCategory.getValues().put("status", "FAILED");
|
||||
blankCategory.getValues().put("is_conform", "");
|
||||
|
||||
assertTrue(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
|
||||
List.of("id", "asin", "country", "is_conform", "status"),
|
||||
List.of(blankCategory),
|
||||
true));
|
||||
|
||||
assertFalse(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
|
||||
List.of("id", "asin", "country", "is_conform", "status"),
|
||||
List.of(blankCategory),
|
||||
false));
|
||||
|
||||
blankCategory.getValues().put("is_conform", "符合");
|
||||
assertFalse(SimilarAsinTaskService.shouldEnableCategorySwitchForRetry(
|
||||
List.of("id", "asin", "country", "is_conform", "status"),
|
||||
List.of(blankCategory),
|
||||
true));
|
||||
}
|
||||
|
||||
@Test
|
||||
void firstPassImageResultWorkbookKeepsAllRowsForSecondParse() {
|
||||
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
|
||||
row.setValues(new LinkedHashMap<>());
|
||||
row.getValues().put("状态", "成功");
|
||||
row.getValues().put("是否有货", "");
|
||||
row.getValues().put("相似度", "");
|
||||
row.getValues().put("是否符合类目", "");
|
||||
row.getValues().put("不符合理由", "");
|
||||
row.getValues().put("产品类目", "");
|
||||
|
||||
assertTrue(SimilarAsinTaskService.isFirstPassResultWorkbook(
|
||||
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
|
||||
List.of(row)));
|
||||
|
||||
row.getValues().put("是否符合类目", "符合");
|
||||
assertFalse(SimilarAsinTaskService.isFirstPassResultWorkbook(
|
||||
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
|
||||
List.of(row)));
|
||||
|
||||
row.getValues().put("是否符合类目", "");
|
||||
row.getValues().put("状态", "失败");
|
||||
assertFalse(SimilarAsinTaskService.isFirstPassResultWorkbook(
|
||||
List.of("id", "asin", "国家", "是否有货", "相似度", "是否符合类目", "不符合理由", "产品类目", "状态", "主图", "阿里巴巴图片1", "阿里巴巴图片2"),
|
||||
List.of(row)));
|
||||
}
|
||||
|
||||
private int staticIntField(String name) throws Exception {
|
||||
Field field = SimilarAsinTaskService.class.getDeclaredField(name);
|
||||
field.setAccessible(true);
|
||||
|
||||
+59
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
@@ -11,7 +12,9 @@ import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipFile;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class ExcelCellImageWriterTest {
|
||||
@@ -42,6 +45,48 @@ class ExcelCellImageWriterTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void streamsRegisteredImagePathIntoWorkbookMedia() throws Exception {
|
||||
Path dir = Files.createTempDirectory("excel-cell-image-path-test-");
|
||||
Path xlsx = dir.resolve("result.xlsx");
|
||||
Path image = dir.resolve("thumb.jpeg");
|
||||
byte[] imageBytes = new byte[]{7, 8, 9, 10};
|
||||
writeMinimalWorkbook(xlsx);
|
||||
Files.write(image, imageBytes);
|
||||
|
||||
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
|
||||
session.registerImage(1, 9, image);
|
||||
|
||||
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
|
||||
|
||||
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
|
||||
assertArrayEquals(imageBytes,
|
||||
zip.getInputStream(zip.getEntry("xl/media/excelcellimage1.jpeg")).readAllBytes());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void patchesTenThousandImageCellsWithinLinearTimeBudget() {
|
||||
assertTimeoutPreemptively(Duration.ofSeconds(10), () -> {
|
||||
Path dir = Files.createTempDirectory("excel-cell-image-linear-test-");
|
||||
Path xlsx = dir.resolve("result.xlsx");
|
||||
writeWorkbookWithImageCells(xlsx, 10_000);
|
||||
|
||||
ExcelCellImageWriter.Session session = ExcelCellImageWriter.createSession();
|
||||
for (int row = 1; row <= 10_000; row++) {
|
||||
session.registerImage(row, 9, new byte[]{1});
|
||||
}
|
||||
|
||||
ExcelCellImageWriter.patchXlsxFile(xlsx.toFile(), session);
|
||||
|
||||
try (ZipFile zip = new ZipFile(xlsx.toFile())) {
|
||||
String sheet = read(zip, "xl/worksheets/sheet1.xml");
|
||||
assertEquals(10_000, maxVm(sheet));
|
||||
assertTrue(sheet.contains("<c r=\"J10001\" t=\"e\" vm=\"10000\"><v>#VALUE!</v></c>"));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private static int maxVm(String sheet) {
|
||||
Matcher matcher = Pattern.compile("vm=\"(\\d+)\"").matcher(sheet);
|
||||
int max = -1;
|
||||
@@ -99,6 +144,20 @@ class ExcelCellImageWriterTest {
|
||||
}
|
||||
}
|
||||
|
||||
private static void writeWorkbookWithImageCells(Path xlsx, int imageCount) throws Exception {
|
||||
try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(xlsx))) {
|
||||
write(out, "[Content_Types].xml", "<Types></Types>");
|
||||
write(out, "xl/_rels/workbook.xml.rels", "<Relationships></Relationships>");
|
||||
StringBuilder sheet = new StringBuilder("<worksheet><sheetData>");
|
||||
for (int row = 2; row <= imageCount + 1; row++) {
|
||||
sheet.append("<row r=\"").append(row).append("\"><c r=\"J").append(row)
|
||||
.append("\" t=\"inlineStr\"><is><t>image</t></is></c></row>");
|
||||
}
|
||||
sheet.append("</sheetData></worksheet>");
|
||||
write(out, "xl/worksheets/sheet1.xml", sheet.toString());
|
||||
}
|
||||
}
|
||||
|
||||
private static void write(ZipOutputStream out, String name, String content) throws Exception {
|
||||
out.putNextEntry(new ZipEntry(name));
|
||||
out.write(content.getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
+222
-1
@@ -1,7 +1,15 @@
|
||||
package com.nanri.aiimage.modules.similarasin.util;
|
||||
|
||||
import com.nanri.aiimage.config.OssProperties;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import okhttp3.MediaType;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Protocol;
|
||||
import okhttp3.Response;
|
||||
import okhttp3.ResponseBody;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
@@ -10,24 +18,39 @@ import java.awt.Graphics2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Field;
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
|
||||
class SimilarAsinImageEmbedderTest {
|
||||
|
||||
// properties=null 时构造函数走 DEFAULT_DOWNLOAD_TIMEOUT_SECONDS / DEFAULT_DOWNLOAD_POOL_SIZE 兜底。
|
||||
private final SimilarAsinImageEmbedder embedder = new SimilarAsinImageEmbedder(null, createOssStorageService());
|
||||
|
||||
@AfterEach
|
||||
void shutDownEmbedder() {
|
||||
embedder.shutdown();
|
||||
}
|
||||
|
||||
private static OssStorageService createOssStorageService() {
|
||||
OssProperties properties = new OssProperties();
|
||||
properties.setEndpoint("https://oss.aishufu.top");
|
||||
@@ -40,6 +63,133 @@ class SimilarAsinImageEmbedderTest {
|
||||
return new OssStorageService(properties);
|
||||
}
|
||||
|
||||
@Test
|
||||
void defaultsImageDownloadPoolToEight() {
|
||||
assertEquals(8, new SimilarAsinProperties().getImageDownloadPoolSize());
|
||||
assertEquals(8, embedder.downloadPoolSize());
|
||||
}
|
||||
|
||||
@Test
|
||||
void writesFetchedThumbnailToPersistentCacheForNextEmbedderInstance() throws Exception {
|
||||
Path cacheDir = Files.createTempDirectory("similar-asin-image-cache-test-");
|
||||
String url = "https://images.example.com/product.jpg";
|
||||
byte[] sourceImage = createJpegBytes();
|
||||
AtomicInteger firstNetworkCalls = new AtomicInteger();
|
||||
AtomicInteger secondNetworkCalls = new AtomicInteger();
|
||||
SimilarAsinImageEmbedder first = new SimilarAsinImageEmbedder(
|
||||
properties(1, 5, cacheDir), createOssStorageService());
|
||||
SimilarAsinImageEmbedder second = new SimilarAsinImageEmbedder(
|
||||
properties(1, 5, cacheDir), createOssStorageService());
|
||||
try {
|
||||
replaceHttpClient(first, respondingClient(firstNetworkCalls, sourceImage));
|
||||
SimilarAsinImageEmbedder.ResizedImage fetched = first.fetchAndResizeForCache(url);
|
||||
|
||||
assertNotNull(fetched);
|
||||
assertEquals(1, firstNetworkCalls.get());
|
||||
try (var cachedFiles = Files.walk(cacheDir)) {
|
||||
assertEquals(1L, cachedFiles.filter(Files::isRegularFile).count());
|
||||
}
|
||||
|
||||
replaceHttpClient(second, new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
secondNetworkCalls.incrementAndGet();
|
||||
throw new AssertionError("persistent cache miss triggered a second network request");
|
||||
})
|
||||
.build());
|
||||
SimilarAsinImageEmbedder.ResizedImage cached = second.fetchAndResizeForCache(url);
|
||||
|
||||
assertNotNull(cached);
|
||||
assertArrayEquals(fetched.bytes(), cached.bytes());
|
||||
assertEquals(fetched.width(), cached.width());
|
||||
assertEquals(fetched.height(), cached.height());
|
||||
assertEquals(0, secondNetworkCalls.get());
|
||||
} finally {
|
||||
first.shutdown();
|
||||
second.shutdown();
|
||||
deleteRecursively(cacheDir);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void diskPrefetchDeadlineCancelsInFlightDownload() throws Exception {
|
||||
SimilarAsinImageEmbedder deadlineEmbedder = new SimilarAsinImageEmbedder(
|
||||
properties(1, 1, null), createOssStorageService());
|
||||
CountDownLatch downloadStarted = new CountDownLatch(1);
|
||||
CountDownLatch cancellationObserved = new CountDownLatch(1);
|
||||
replaceHttpClient(deadlineEmbedder, new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
downloadStarted.countDown();
|
||||
try {
|
||||
new CountDownLatch(1).await();
|
||||
throw new AssertionError("blocking download unexpectedly completed");
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
cancellationObserved.countDown();
|
||||
throw new IOException("cancelled", ex);
|
||||
}
|
||||
})
|
||||
.build());
|
||||
Path spoolDir = Files.createTempDirectory("similar-asin-prefetch-deadline-test-");
|
||||
try (SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir)) {
|
||||
assertTimeoutPreemptively(Duration.ofSeconds(3), () -> deadlineEmbedder.prefetchToDisk(
|
||||
List.of("https://images.example.com/slow.jpg"), spool));
|
||||
|
||||
assertTrue(downloadStarted.await(100, TimeUnit.MILLISECONDS));
|
||||
assertTrue(cancellationObserved.await(1, TimeUnit.SECONDS),
|
||||
"deadline should interrupt the active image download");
|
||||
assertEquals(0, spool.size());
|
||||
} finally {
|
||||
deadlineEmbedder.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedDiskPrefetchIsNotDownloadedAgainWhileEmbedding() throws Exception {
|
||||
SimilarAsinImageEmbedder failedEmbedder = new SimilarAsinImageEmbedder(
|
||||
properties(1, 5, null), createOssStorageService());
|
||||
AtomicInteger networkCalls = new AtomicInteger();
|
||||
replaceHttpClient(failedEmbedder, failingClient(networkCalls));
|
||||
String url = "https://images.example.com/missing.jpg";
|
||||
Path spoolDir = Files.createTempDirectory("similar-asin-prefetch-failed-test-");
|
||||
try (SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
|
||||
XSSFWorkbook workbook = new XSSFWorkbook()) {
|
||||
failedEmbedder.prefetchToDisk(List.of(url), spool);
|
||||
int callsAfterPrefetch = networkCalls.get();
|
||||
var row = workbook.createSheet().createRow(1);
|
||||
|
||||
SimilarAsinImageEmbedder.ImageDim result = failedEmbedder.embedAsExcelCellImage(
|
||||
1, 9, url, row, new HashMap<>(), ExcelCellImageWriter.createSession(), spool);
|
||||
|
||||
assertEquals(SimilarAsinImageEmbedder.DOWNLOAD_MAX_RETRY + 1, callsAfterPrefetch);
|
||||
assertEquals(callsAfterPrefetch, networkCalls.get());
|
||||
assertNull(result);
|
||||
assertEquals(url, row.getCell(9).getStringCellValue());
|
||||
} finally {
|
||||
failedEmbedder.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void imageSpoolWritesThumbnailAndDeletesTaskDirectoryOnClose() throws Exception {
|
||||
Path directory = Files.createTempDirectory("similar-asin-image-spool-test-");
|
||||
byte[] bytes = new byte[]{1, 3, 5, 7};
|
||||
SimilarAsinImageEmbedder.ImageSpool spool = new SimilarAsinImageEmbedder.ImageSpool(directory);
|
||||
try {
|
||||
SimilarAsinImageEmbedder.SpoolImage image = spool.put(
|
||||
"https://example.com/image.jpg",
|
||||
new SimilarAsinImageEmbedder.ResizedImage(bytes, 120, 80));
|
||||
|
||||
assertEquals(1, spool.size());
|
||||
assertEquals(120, image.width());
|
||||
assertEquals(80, image.height());
|
||||
assertArrayEquals(bytes, Files.readAllBytes(image.path()));
|
||||
} finally {
|
||||
spool.close();
|
||||
}
|
||||
|
||||
assertFalse(Files.exists(directory));
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizesLegacyMinioUrlBeforeHttpsValidation() {
|
||||
String normalized = embedder.normalizeAndValidateDownloadUrl(
|
||||
@@ -247,4 +397,75 @@ class SimilarAsinImageEmbedderTest {
|
||||
assertTrue(summary.contains("IOException: outer"));
|
||||
assertTrue(summary.contains("cause=TimeoutException: inner"));
|
||||
}
|
||||
|
||||
private static SimilarAsinProperties properties(int poolSize, int prefetchTimeoutSeconds, Path cacheDir) {
|
||||
SimilarAsinProperties properties = new SimilarAsinProperties();
|
||||
properties.setImageDownloadPoolSize(poolSize);
|
||||
properties.setImagePrefetchTimeoutSeconds(prefetchTimeoutSeconds);
|
||||
properties.setImageLocalCacheDir(cacheDir == null ? "" : cacheDir.toString());
|
||||
return properties;
|
||||
}
|
||||
|
||||
private static OkHttpClient respondingClient(AtomicInteger calls, byte[] imageBytes) {
|
||||
return new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
calls.incrementAndGet();
|
||||
return response(chain.request(), 200, "OK", imageBytes);
|
||||
})
|
||||
.build();
|
||||
}
|
||||
|
||||
private static OkHttpClient failingClient(AtomicInteger calls) {
|
||||
return new OkHttpClient.Builder()
|
||||
.addInterceptor(chain -> {
|
||||
calls.incrementAndGet();
|
||||
return response(chain.request(), 503, "Unavailable", new byte[0]);
|
||||
})
|
||||
.build();
|
||||
}
|
||||
|
||||
private static Response response(okhttp3.Request request,
|
||||
int status,
|
||||
String message,
|
||||
byte[] body) {
|
||||
return new Response.Builder()
|
||||
.request(request)
|
||||
.protocol(Protocol.HTTP_1_1)
|
||||
.code(status)
|
||||
.message(message)
|
||||
.body(ResponseBody.create(body, MediaType.get("image/jpeg")))
|
||||
.build();
|
||||
}
|
||||
|
||||
private static void replaceHttpClient(SimilarAsinImageEmbedder target,
|
||||
OkHttpClient httpClient) throws ReflectiveOperationException {
|
||||
Field field = SimilarAsinImageEmbedder.class.getDeclaredField("httpClient");
|
||||
field.setAccessible(true);
|
||||
field.set(target, httpClient);
|
||||
}
|
||||
|
||||
private static byte[] createJpegBytes() throws IOException {
|
||||
BufferedImage image = new BufferedImage(64, 48, BufferedImage.TYPE_INT_RGB);
|
||||
Graphics2D graphics = image.createGraphics();
|
||||
try {
|
||||
graphics.setColor(new Color(0x24, 0x68, 0xAC));
|
||||
graphics.fillRect(0, 0, image.getWidth(), image.getHeight());
|
||||
} finally {
|
||||
graphics.dispose();
|
||||
}
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
ImageIO.write(image, "jpg", output);
|
||||
return output.toByteArray();
|
||||
}
|
||||
|
||||
private static void deleteRecursively(Path directory) throws IOException {
|
||||
if (!Files.exists(directory)) {
|
||||
return;
|
||||
}
|
||||
try (var paths = Files.walk(directory)) {
|
||||
for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
|
||||
Files.deleteIfExists(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class TaskFileJobServiceTest {
|
||||
|
||||
@Mock private TaskFileJobMapper taskFileJobMapper;
|
||||
@Mock private ApplicationEventPublisher applicationEventPublisher;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeTableInfo() {
|
||||
TableInfoHelper.initTableInfo(
|
||||
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||
TaskFileJobEntity.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void stuckJobIsRequeuedAndRetryCountIsIncremented() {
|
||||
TaskFileJobEntity running = runningJob(101L, 3, LocalDateTime.now().minusHours(1));
|
||||
TaskFileJobEntity pending = runningJob(101L, 4, LocalDateTime.now());
|
||||
pending.setStatus("PENDING");
|
||||
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
when(taskFileJobMapper.selectById(101L)).thenReturn(pending);
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
|
||||
|
||||
assertEquals(1, result.resetCount());
|
||||
assertTrue(result.exhaustedJobs().isEmpty());
|
||||
verify(applicationEventPublisher).publishEvent(any(TaskFileJobDispatchEvent.class));
|
||||
assertUpdateContains(4, running.getUpdatedAt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void stuckJobAtLastRetryWaitsForDurableFailureFinalization() {
|
||||
TaskFileJobEntity running = runningJob(102L, 4, LocalDateTime.now().minusHours(1));
|
||||
TaskFileJobEntity failed = runningJob(102L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
|
||||
failed.setStatus("FAILED");
|
||||
/*
|
||||
failed.setErrorMessage("文件生成任务运行超时,已达到最大重试次数");
|
||||
*/
|
||||
failed.setErrorMessage("result file job timeout");
|
||||
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
when(taskFileJobMapper.selectById(102L)).thenReturn(failed);
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
|
||||
|
||||
assertEquals(0, result.resetCount());
|
||||
assertEquals(List.of(failed), result.exhaustedJobs());
|
||||
verify(applicationEventPublisher, never()).publishEvent(any());
|
||||
assertUpdateContains(TaskFileJobService.MAX_RETRY_COUNT, running.getUpdatedAt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void pendingFailureFinalizationIsReturnedAgainWithoutRequeueing() {
|
||||
TaskFileJobEntity pending = runningJob(106L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
|
||||
pending.setStatus("FAILED");
|
||||
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(pending));
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
|
||||
|
||||
assertEquals(0, result.resetCount());
|
||||
assertEquals(List.of(pending), result.exhaustedJobs());
|
||||
verify(taskFileJobMapper, never()).update(any(), any());
|
||||
verify(applicationEventPublisher, never()).publishEvent(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void queuedClaimActivationUsesUpdatedAtAsFencingToken() {
|
||||
TaskFileJobEntity claim = runningJob(107L, 1, LocalDateTime.now().minusMinutes(1));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
assertTrue(service.activateRunningClaim(claim));
|
||||
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||
assertTrue(update.getValue().getSqlSegment().contains("updated_at"));
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue(claim.getUpdatedAt()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void heartbeatRacePreventsStuckJobReset() {
|
||||
TaskFileJobEntity running = runningJob(103L, 4, LocalDateTime.now().minusHours(1));
|
||||
when(taskFileJobMapper.selectList(any())).thenReturn(List.of(running));
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
TaskFileJobService.StuckJobResetResult result = service.resetStuckRunningJobsDetailed(30, 20);
|
||||
|
||||
assertEquals(0, result.resetCount());
|
||||
assertTrue(result.exhaustedJobs().isEmpty());
|
||||
verify(taskFileJobMapper, never()).selectById(any());
|
||||
verify(applicationEventPublisher, never()).publishEvent(any());
|
||||
assertUpdateContains(TaskFileJobService.MAX_RETRY_COUNT, running.getUpdatedAt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void exhaustedJobCannotBeClaimedOrRequeued() {
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(0);
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
assertFalse(service.markRunning(104L));
|
||||
assertFalse(service.requeue(104L, "retry"));
|
||||
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> updates = updateCaptor();
|
||||
verify(taskFileJobMapper, org.mockito.Mockito.times(2)).update(isNull(), updates.capture());
|
||||
for (LambdaUpdateWrapper<TaskFileJobEntity> update : updates.getAllValues()) {
|
||||
assertTrue(update.getSqlSegment().contains("retry_count"));
|
||||
assertTrue(update.getParamNameValuePairs().containsValue(TaskFileJobService.MAX_RETRY_COUNT));
|
||||
}
|
||||
verify(taskFileJobMapper, never()).selectById(any());
|
||||
verify(applicationEventPublisher, never()).publishEvent(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void markFailedUsesCurrentRetryCountWithCompareAndSet() {
|
||||
TaskFileJobEntity stale = runningJob(105L, 0, LocalDateTime.now().minusMinutes(5));
|
||||
TaskFileJobEntity current = runningJob(105L, 4, LocalDateTime.now());
|
||||
when(taskFileJobMapper.selectById(105L)).thenReturn(current);
|
||||
when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
service.markFailed(stale, "failed");
|
||||
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||
assertTrue(update.getValue().getSqlSegment().contains("retry_count"));
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue(4));
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue(TaskFileJobService.MAX_RETRY_COUNT));
|
||||
}
|
||||
|
||||
@Test
|
||||
void successWriteIsFencedToRunningJob() {
|
||||
TaskFileJobEntity job = runningJob(108L, 1, LocalDateTime.now());
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
service.markSuccess(job, "result.xlsx");
|
||||
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||
assertTrue(update.getValue().getSqlSegment().contains("status"));
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue("RUNNING"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void similarAsinHeartbeatOnlyTouchesStaleRunningAssembleJobs() {
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
service.touchRunningAssembleJobsIfStale(20553L, "SIMILAR_ASIN", 60000L);
|
||||
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||
assertTrue(update.getValue().getSqlSegment().contains("task_id"));
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue("SIMILAR_ASIN"));
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue("ASSEMBLE_RESULT"));
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue("RUNNING"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void retryExhaustedOnlyMeansFailedTerminalJob() {
|
||||
TaskFileJobEntity success = runningJob(109L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
|
||||
success.setStatus("SUCCESS");
|
||||
TaskFileJobEntity failed = runningJob(110L, TaskFileJobService.MAX_RETRY_COUNT, LocalDateTime.now());
|
||||
failed.setStatus("FAILED");
|
||||
when(taskFileJobMapper.selectById(109L)).thenReturn(success);
|
||||
when(taskFileJobMapper.selectById(110L)).thenReturn(failed);
|
||||
TaskFileJobService service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher);
|
||||
|
||||
assertFalse(service.isRetryExhausted(109L));
|
||||
assertTrue(service.isRetryExhausted(110L));
|
||||
}
|
||||
|
||||
private void assertUpdateContains(int nextRetry, LocalDateTime updatedAt) {
|
||||
ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> update = updateCaptor();
|
||||
verify(taskFileJobMapper).update(isNull(), update.capture());
|
||||
assertTrue(update.getValue().getSqlSegment().contains("updated_at"));
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue(updatedAt));
|
||||
assertTrue(update.getValue().getParamNameValuePairs().containsValue(nextRetry));
|
||||
}
|
||||
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
private static ArgumentCaptor<LambdaUpdateWrapper<TaskFileJobEntity>> updateCaptor() {
|
||||
return ArgumentCaptor.forClass((Class) LambdaUpdateWrapper.class);
|
||||
}
|
||||
|
||||
private static TaskFileJobEntity runningJob(Long id, int retryCount, LocalDateTime updatedAt) {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(id);
|
||||
job.setTaskId(20553L);
|
||||
job.setResultId(23110L);
|
||||
job.setModuleType("SIMILAR_ASIN");
|
||||
job.setStatus("RUNNING");
|
||||
job.setRetryCount(retryCount);
|
||||
job.setUpdatedAt(updatedAt);
|
||||
return job;
|
||||
}
|
||||
}
|
||||
+26
@@ -8,6 +8,7 @@ import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.service.BrandTaskProgressCacheService;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
|
||||
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService;
|
||||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
|
||||
@@ -67,8 +68,10 @@ class TaskHeartbeatServiceTest {
|
||||
@Mock private AppearancePatentTaskCacheService appearancePatentTaskCacheService;
|
||||
@Mock private SimilarAsinTaskCacheService similarAsinTaskCacheService;
|
||||
@Mock private SimilarAsinProperties similarAsinProperties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
||||
@Mock private BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||
@Mock private CollectDataService collectDataService;
|
||||
|
||||
@InjectMocks private TaskHeartbeatService service;
|
||||
|
||||
@@ -124,6 +127,28 @@ class TaskHeartbeatServiceTest {
|
||||
verify(shopDataCrawlTaskCacheService).saveTaskCache(task);
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void collectDataHeartbeatForwardsProcessedKeywordProgress() {
|
||||
long taskId = 21016L;
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(taskId);
|
||||
task.setModuleType("COLLECT_DATA");
|
||||
task.setStatus("RUNNING");
|
||||
TaskHeartbeatRequest request = new TaskHeartbeatRequest();
|
||||
request.setCurrent(4);
|
||||
request.setTotal(19);
|
||||
|
||||
when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task);
|
||||
when(brandCrawlTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null);
|
||||
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
|
||||
|
||||
TaskHeartbeatVo result = service.heartbeat(taskId, request);
|
||||
|
||||
assertTrue(result.isAlive());
|
||||
verify(collectDataService).updateProgress(taskId, request);
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void similarAsinHeartbeatUsesRedisWithoutRefreshingRecentDatabaseCheckpoint() {
|
||||
@@ -142,6 +167,7 @@ class TaskHeartbeatServiceTest {
|
||||
assertTrue(result.isAlive());
|
||||
assertEquals("SIMILAR_ASIN", result.getModuleType());
|
||||
verify(similarAsinTaskCacheService).touchTaskHeartbeat(taskId);
|
||||
verify(taskFileJobService).touchRunningAssembleJobsIfStale(taskId, "SIMILAR_ASIN", 120000L);
|
||||
verify(fileTaskMapper, never()).update(isNull(), any(LambdaUpdateWrapper.class));
|
||||
}
|
||||
}
|
||||
|
||||
+93
-4
@@ -17,6 +17,10 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.withdraw.service.WithdrawTaskService;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InOrder;
|
||||
@@ -24,9 +28,13 @@ import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -72,7 +80,7 @@ class TaskResultFileJobWorkerTest {
|
||||
result.setResultFileUrl("result/withdraw/20140.xlsx");
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
|
||||
when(taskFileJobService.markRunning(jobId)).thenReturn(true);
|
||||
allowClaim(job);
|
||||
when(taskDistributedLockService.acquire("WITHDRAW", taskId, TaskDistributedLockService.DEFAULT_WAIT_MILLIS))
|
||||
.thenReturn(lock);
|
||||
when(fileResultMapper.selectById(resultId)).thenReturn(result);
|
||||
@@ -102,7 +110,7 @@ class TaskResultFileJobWorkerTest {
|
||||
result.setResultFileUrl("result/publish/20141.xlsx");
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
|
||||
when(taskFileJobService.markRunning(jobId)).thenReturn(true);
|
||||
allowClaim(job);
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
when(taskDistributedLockService.acquire(
|
||||
PublishTaskService.MODULE_TYPE,
|
||||
@@ -150,7 +158,7 @@ class TaskResultFileJobWorkerTest {
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
when(taskFileJobService.markRunning(jobId)).thenReturn(true);
|
||||
allowClaim(job);
|
||||
when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", taskId,
|
||||
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
|
||||
when(fileResultMapper.selectById(resultId)).thenReturn(result);
|
||||
@@ -173,7 +181,7 @@ class TaskResultFileJobWorkerTest {
|
||||
job.setScopeKey("task:20144:owner:instance-a");
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
when(taskFileJobService.markRunning(job.getId())).thenReturn(true);
|
||||
allowClaim(job);
|
||||
when(taskDistributedLockService.acquire("SHOP_DATA_CRAWL", job.getTaskId(),
|
||||
TaskDistributedLockService.DEFAULT_WAIT_MILLIS)).thenReturn(lock);
|
||||
doThrow(new IllegalStateException("upload failed"))
|
||||
@@ -184,5 +192,86 @@ class TaskResultFileJobWorkerTest {
|
||||
|
||||
verify(taskFileJobService).markFailed(job, "upload failed");
|
||||
verify(shopDataCrawlTaskService).handleResultFileJobFailure(job, "upload failed");
|
||||
verify(taskFileJobService).markFailureFinalized(job.getId(), "upload failed");
|
||||
}
|
||||
|
||||
@Test
|
||||
void stuckSimilarAsinJobAtRetryLimitFailsOwningTask() {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(13858L);
|
||||
job.setTaskId(20553L);
|
||||
job.setResultId(23110L);
|
||||
job.setModuleType("SIMILAR_ASIN");
|
||||
job.setRetryCount(TaskFileJobService.MAX_RETRY_COUNT);
|
||||
job.setErrorMessage("文件生成任务运行超时,已达到最大重试次数");
|
||||
TaskFileJobService.StuckJobResetResult resetResult =
|
||||
new TaskFileJobService.StuckJobResetResult(0, List.of(job));
|
||||
when(taskFileJobService.resetStuckRunningJobsDetailed(0, 0)).thenReturn(resetResult);
|
||||
|
||||
worker.resetStuckJobs();
|
||||
|
||||
verify(similarAsinTaskService).handleResultFileJobFailure(job, job.getErrorMessage());
|
||||
verify(taskFileJobService).markFailureFinalized(job.getId(), job.getErrorMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void stuckJobFailureCallbackDoesNotBlockRemainingJobs() {
|
||||
TaskFileJobEntity first = exhaustedSimilarAsinJob(13858L, 20553L);
|
||||
TaskFileJobEntity second = exhaustedSimilarAsinJob(13859L, 20554L);
|
||||
TaskFileJobService.StuckJobResetResult resetResult =
|
||||
new TaskFileJobService.StuckJobResetResult(0, List.of(first, second));
|
||||
when(taskFileJobService.resetStuckRunningJobsDetailed(0, 0)).thenReturn(resetResult);
|
||||
doThrow(new IllegalStateException("owner mismatch"))
|
||||
.doNothing()
|
||||
.when(similarAsinTaskService)
|
||||
.handleResultFileJobFailure(any(), anyString());
|
||||
|
||||
worker.resetStuckJobs();
|
||||
|
||||
verify(similarAsinTaskService).handleResultFileJobFailure(first, first.getErrorMessage());
|
||||
verify(similarAsinTaskService).handleResultFileJobFailure(second, second.getErrorMessage());
|
||||
verify(taskFileJobService, never()).markFailureFinalized(first.getId(), first.getErrorMessage());
|
||||
verify(taskFileJobService).markFailureFinalized(second.getId(), second.getErrorMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void pendingFailureCallbackIsRetriedOnNextScan() {
|
||||
TaskFileJobEntity job = exhaustedSimilarAsinJob(13860L, 20555L);
|
||||
TaskFileJobService.StuckJobResetResult resetResult =
|
||||
new TaskFileJobService.StuckJobResetResult(0, List.of(job));
|
||||
when(taskFileJobService.resetStuckRunningJobsDetailed(0, 0))
|
||||
.thenReturn(resetResult, resetResult);
|
||||
doThrow(new IllegalStateException("temporary database failure"))
|
||||
.doNothing()
|
||||
.when(similarAsinTaskService)
|
||||
.handleResultFileJobFailure(job, job.getErrorMessage());
|
||||
|
||||
worker.resetStuckJobs();
|
||||
worker.resetStuckJobs();
|
||||
|
||||
verify(similarAsinTaskService, times(2))
|
||||
.handleResultFileJobFailure(job, job.getErrorMessage());
|
||||
verify(taskFileJobService).markFailureFinalized(job.getId(), job.getErrorMessage());
|
||||
}
|
||||
|
||||
private void allowClaim(TaskFileJobEntity job) {
|
||||
TaskFileJobEntity claim = new TaskFileJobEntity();
|
||||
claim.setId(job.getId());
|
||||
claim.setTaskId(job.getTaskId());
|
||||
claim.setModuleType(job.getModuleType());
|
||||
claim.setStatus("RUNNING");
|
||||
claim.setUpdatedAt(LocalDateTime.now());
|
||||
when(taskFileJobService.claimRunning(job.getId())).thenReturn(claim);
|
||||
when(taskFileJobService.activateRunningClaim(claim)).thenReturn(true);
|
||||
}
|
||||
|
||||
private static TaskFileJobEntity exhaustedSimilarAsinJob(long jobId, long taskId) {
|
||||
TaskFileJobEntity job = new TaskFileJobEntity();
|
||||
job.setId(jobId);
|
||||
job.setTaskId(taskId);
|
||||
job.setModuleType("SIMILAR_ASIN");
|
||||
job.setRetryCount(TaskFileJobService.MAX_RETRY_COUNT);
|
||||
job.setErrorMessage("文件生成任务运行超时,已达到最大重试次数");
|
||||
return job;
|
||||
}
|
||||
}
|
||||
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
package com.nanri.aiimage.modules.ziniao.controller;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexRefreshCursorDto;
|
||||
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoAuthService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexRefreshService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
|
||||
class ZiniaoAuthControllerTest {
|
||||
|
||||
@Test
|
||||
void manualIndexRefreshEndpointReturnsCompletedCursor() throws Exception {
|
||||
ZiniaoAuthService authService = mock(ZiniaoAuthService.class);
|
||||
ZiniaoShopIndexService indexService = mock(ZiniaoShopIndexService.class);
|
||||
ZiniaoShopIndexRefreshService refreshService = mock(ZiniaoShopIndexRefreshService.class);
|
||||
AdminAuthSupport adminAuthSupport = mock(AdminAuthSupport.class);
|
||||
ZiniaoShopIndexRefreshCursorDto cursor = new ZiniaoShopIndexRefreshCursorDto();
|
||||
cursor.setStatus("SUCCESS");
|
||||
cursor.setLastProcessedApiKeyCount(3);
|
||||
when(refreshService.refreshShopIndexManually()).thenReturn(cursor);
|
||||
ZiniaoAuthController controller = new ZiniaoAuthController(authService, indexService, refreshService, adminAuthSupport);
|
||||
MockMvc mockMvc = MockMvcBuilders.standaloneSetup(controller).build();
|
||||
|
||||
mockMvc.perform(post("/api/ziniao/index-refresh")
|
||||
.header(HttpHeaders.AUTHORIZATION, "Bearer admin-token"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.success").value(true))
|
||||
.andExpect(jsonPath("$.data.status").value("SUCCESS"))
|
||||
.andExpect(jsonPath("$.data.lastProcessedApiKeyCount").value(3));
|
||||
verify(adminAuthSupport).requireAdmin(org.mockito.ArgumentMatchers.any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void manualIndexRefreshRequiresAdministrator() {
|
||||
ZiniaoAuthService authService = mock(ZiniaoAuthService.class);
|
||||
ZiniaoShopIndexService indexService = mock(ZiniaoShopIndexService.class);
|
||||
ZiniaoShopIndexRefreshService refreshService = mock(ZiniaoShopIndexRefreshService.class);
|
||||
AdminAuthSupport adminAuthSupport = mock(AdminAuthSupport.class);
|
||||
HttpServletRequest request = mock(HttpServletRequest.class);
|
||||
BusinessException authFailure = new BusinessException(403, "需要管理员权限");
|
||||
when(adminAuthSupport.requireAdmin(request)).thenThrow(authFailure);
|
||||
ZiniaoAuthController controller = new ZiniaoAuthController(authService, indexService, refreshService, adminAuthSupport);
|
||||
|
||||
assertThatThrownBy(() -> controller.refreshShopIndex(request, null)).isSameAs(authFailure);
|
||||
verifyNoInteractions(refreshService);
|
||||
}
|
||||
}
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
package com.nanri.aiimage.modules.ziniao.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.nanri.aiimage.modules.shopkey.mapper.ShopKeyMapper;
|
||||
import com.nanri.aiimage.modules.shopkey.model.entity.ShopKeyEntity;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ZiniaoApiKeyProviderTest {
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
TableInfoHelper.initTableInfo(
|
||||
new MapperBuilderAssistant(new MybatisConfiguration(), "ziniao-api-key-provider-test"),
|
||||
ShopKeyEntity.class
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void duplicateNormalizedTokensShareOneRefreshAccountAndAllRecordIds() {
|
||||
ShopKeyMapper mapper = mock(ShopKeyMapper.class);
|
||||
ShopKeyEntity latest = shopKey(12L, " Bearer duplicate-key ", "最新账号");
|
||||
ShopKeyEntity older = shopKey(8L, "duplicate-key", "旧账号");
|
||||
when(mapper.selectList(any())).thenReturn(List.of(latest, older));
|
||||
|
||||
ZiniaoApiKeyProvider provider = new ZiniaoApiKeyProvider(mapper);
|
||||
|
||||
List<ZiniaoApiKeyProvider.ApiKeyAccount> accounts = provider.listApiKeyAccounts();
|
||||
|
||||
assertEquals(1, accounts.size());
|
||||
assertEquals("duplicate-key", accounts.getFirst().apiKey());
|
||||
assertEquals("最新账号", accounts.getFirst().accountName());
|
||||
assertEquals(List.of(12L, 8L), accounts.getFirst().shopKeyIds());
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistResultUpdatesEveryRecordForTheNormalizedToken() {
|
||||
ShopKeyMapper mapper = mock(ShopKeyMapper.class);
|
||||
ZiniaoApiKeyProvider provider = new ZiniaoApiKeyProvider(mapper);
|
||||
ZiniaoApiKeyProvider.ApiKeyAccount account = new ZiniaoApiKeyProvider.ApiKeyAccount(
|
||||
"duplicate-key",
|
||||
"账号",
|
||||
List.of(12L, 8L)
|
||||
);
|
||||
|
||||
provider.markIpWhitelistBlocked(account, "当前服务器 IP 未加入紫鸟白名单");
|
||||
|
||||
verify(mapper).update(isNull(), any(Wrapper.class));
|
||||
}
|
||||
|
||||
private ShopKeyEntity shopKey(long id, String token, String accountName) {
|
||||
ShopKeyEntity entity = new ShopKeyEntity();
|
||||
entity.setId(id);
|
||||
entity.setZiniaoToken(token);
|
||||
entity.setZiniaoAccountName(accountName);
|
||||
return entity;
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package com.nanri.aiimage.modules.ziniao.service;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexRefreshCursorDto;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ZiniaoShopIndexRefreshServiceTest {
|
||||
|
||||
@Test
|
||||
void manualRefreshRunsUnderDistributedLockAndReturnsCursor() {
|
||||
ZiniaoShopIndexService indexService = mock(ZiniaoShopIndexService.class);
|
||||
DistributedJobLockService lockService = mock(DistributedJobLockService.class);
|
||||
DistributedJobLockService.LockHandle lockHandle = mock(DistributedJobLockService.LockHandle.class);
|
||||
ZiniaoShopIndexRefreshCursorDto cursor = new ZiniaoShopIndexRefreshCursorDto();
|
||||
cursor.setStatus("SUCCESS");
|
||||
when(lockService.tryLock("ziniao:shop-index-refresh", Duration.ofMinutes(30))).thenReturn(lockHandle);
|
||||
when(indexService.getRefreshCursor()).thenReturn(cursor);
|
||||
ZiniaoShopIndexRefreshService service = new ZiniaoShopIndexRefreshService(indexService, lockService);
|
||||
|
||||
ZiniaoShopIndexRefreshCursorDto result = service.refreshShopIndexManually();
|
||||
|
||||
assertThat(result).isSameAs(cursor);
|
||||
verify(indexService).refreshAllShopIndex();
|
||||
verify(lockHandle).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void manualRefreshRejectsConcurrentExecution() {
|
||||
ZiniaoShopIndexService indexService = mock(ZiniaoShopIndexService.class);
|
||||
DistributedJobLockService lockService = mock(DistributedJobLockService.class);
|
||||
when(lockService.tryLock("ziniao:shop-index-refresh", Duration.ofMinutes(30))).thenReturn(null);
|
||||
ZiniaoShopIndexRefreshService service = new ZiniaoShopIndexRefreshService(indexService, lockService);
|
||||
|
||||
assertThatThrownBy(service::refreshShopIndexManually)
|
||||
.isInstanceOf(BusinessException.class)
|
||||
.hasMessageContaining("正在刷新");
|
||||
verify(indexService, never()).refreshAllShopIndex();
|
||||
}
|
||||
}
|
||||
+241
@@ -0,0 +1,241 @@
|
||||
package com.nanri.aiimage.modules.ziniao.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.ZiniaoProperties;
|
||||
import com.nanri.aiimage.modules.ziniao.memory.service.ZiniaoMemoryStoreService;
|
||||
import com.nanri.aiimage.modules.ziniao.memory.service.ZiniaoTransientCacheService;
|
||||
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexEntryDto;
|
||||
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexRefreshCursorDto;
|
||||
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopCacheDto;
|
||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoStaffItemVo;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.argThat;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class ZiniaoShopIndexServiceTest {
|
||||
|
||||
@Mock
|
||||
private ZiniaoMemoryStoreService ziniaoMemoryStoreService;
|
||||
@Mock
|
||||
private ZiniaoTransientCacheService ziniaoTransientCacheService;
|
||||
@Mock
|
||||
private ZiniaoApiKeyProvider ziniaoApiKeyProvider;
|
||||
@Mock
|
||||
private ZiniaoAuthService ziniaoAuthService;
|
||||
|
||||
private ZiniaoShopIndexService service;
|
||||
private ZiniaoProperties properties;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
properties = new ZiniaoProperties();
|
||||
properties.setShopIndexEntryTtlHours(12);
|
||||
properties.setShopIndexRefreshBatchSize(100);
|
||||
service = new ZiniaoShopIndexService(
|
||||
ziniaoMemoryStoreService,
|
||||
ziniaoTransientCacheService,
|
||||
ziniaoApiKeyProvider,
|
||||
ziniaoAuthService,
|
||||
properties,
|
||||
new ObjectMapper()
|
||||
);
|
||||
when(ziniaoTransientCacheService.get(
|
||||
"SHOP_INDEX_REFRESH_CURSOR",
|
||||
"global",
|
||||
ZiniaoShopIndexRefreshCursorDto.class
|
||||
)).thenReturn(Optional.empty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void companyWhitelistFailureSkipsCurrentApiKeyAndRefreshesNextApiKey() {
|
||||
stubIpWhitelistDetection();
|
||||
ZiniaoApiKeyProvider.ApiKeyAccount blocked = new ZiniaoApiKeyProvider.ApiKeyAccount("blocked-key", "blocked-account");
|
||||
ZiniaoApiKeyProvider.ApiKeyAccount allowed = new ZiniaoApiKeyProvider.ApiKeyAccount("allowed-key", "allowed-account");
|
||||
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(blocked, allowed));
|
||||
when(ziniaoAuthService.resolveCompanyIdForIndex("blocked-key"))
|
||||
.thenThrow(new BusinessException("当前服务器 IP 未加入紫鸟白名单"));
|
||||
when(ziniaoAuthService.resolveCompanyIdForIndex("allowed-key")).thenReturn(2L);
|
||||
when(ziniaoAuthService.getOrLoadStaffForIndex("allowed-key", 2L)).thenReturn(List.of(staff(22L)));
|
||||
when(ziniaoAuthService.getOrLoadUserStoresForIndex("allowed-key", 2L, 22L))
|
||||
.thenReturn(List.of(shop("shop-2", "店铺B")));
|
||||
|
||||
service.refreshShopIndex();
|
||||
|
||||
verify(ziniaoMemoryStoreService).put(
|
||||
eq(ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY),
|
||||
eq("s:shop-2"),
|
||||
argThat(value -> value instanceof ZiniaoShopIndexEntryDto entry
|
||||
&& "shop-2".equals(entry.getShopId())),
|
||||
any(Duration.class)
|
||||
);
|
||||
verify(ziniaoMemoryStoreService).put(
|
||||
eq(ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY),
|
||||
eq("n:店铺B"),
|
||||
any(ZiniaoShopIndexEntryDto.class),
|
||||
any(Duration.class)
|
||||
);
|
||||
verify(ziniaoMemoryStoreService, never()).listAliveEntitiesByType(any(), anyInt());
|
||||
verify(ziniaoApiKeyProvider).markIpWhitelistBlocked(
|
||||
eq(blocked),
|
||||
eq("当前服务器 IP 未加入紫鸟白名单")
|
||||
);
|
||||
verify(ziniaoApiKeyProvider).markIpWhitelistAllowed(allowed);
|
||||
|
||||
ZiniaoShopIndexRefreshCursorDto cursor = capturedCursor();
|
||||
assertEquals("SUCCESS", cursor.getStatus());
|
||||
assertEquals(Integer.valueOf(1), cursor.getLastProcessedApiKeyCount());
|
||||
assertTrue(cursor.getMessage().contains("IP 白名单: 1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistFailureAfterPartialApiKeyScanDiscardsPartialEntriesAndContinues() {
|
||||
stubIpWhitelistDetection();
|
||||
ZiniaoApiKeyProvider.ApiKeyAccount partiallyBlocked = new ZiniaoApiKeyProvider.ApiKeyAccount("partial-key", "partial-account");
|
||||
ZiniaoApiKeyProvider.ApiKeyAccount allowed = new ZiniaoApiKeyProvider.ApiKeyAccount("allowed-key", "allowed-account");
|
||||
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(partiallyBlocked, allowed));
|
||||
when(ziniaoAuthService.resolveCompanyIdForIndex("partial-key")).thenReturn(1L);
|
||||
when(ziniaoAuthService.getOrLoadStaffForIndex("partial-key", 1L))
|
||||
.thenReturn(List.of(staff(11L), staff(12L)));
|
||||
when(ziniaoAuthService.getOrLoadUserStoresForIndex("partial-key", 1L, 11L))
|
||||
.thenReturn(List.of(shop("partial-shop", "半成品店铺")));
|
||||
when(ziniaoAuthService.getOrLoadUserStoresForIndex("partial-key", 1L, 12L))
|
||||
.thenThrow(new BusinessException("当前服务器 IP 未加入紫鸟白名单"));
|
||||
when(ziniaoAuthService.resolveCompanyIdForIndex("allowed-key")).thenReturn(2L);
|
||||
when(ziniaoAuthService.getOrLoadStaffForIndex("allowed-key", 2L)).thenReturn(List.of(staff(22L)));
|
||||
when(ziniaoAuthService.getOrLoadUserStoresForIndex("allowed-key", 2L, 22L))
|
||||
.thenReturn(List.of(shop("allowed-shop", "正常店铺")));
|
||||
|
||||
service.refreshShopIndex();
|
||||
|
||||
verify(ziniaoMemoryStoreService, never()).put(
|
||||
eq(ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY),
|
||||
eq("s:partial-shop"),
|
||||
any(),
|
||||
any(Duration.class)
|
||||
);
|
||||
verify(ziniaoMemoryStoreService, never()).put(
|
||||
eq(ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY),
|
||||
eq("n:半成品店铺"),
|
||||
any(),
|
||||
any(Duration.class)
|
||||
);
|
||||
verify(ziniaoMemoryStoreService).put(
|
||||
eq(ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY),
|
||||
eq("s:allowed-shop"),
|
||||
any(ZiniaoShopIndexEntryDto.class),
|
||||
any(Duration.class)
|
||||
);
|
||||
verify(ziniaoMemoryStoreService, never()).listAliveEntitiesByType(any(), anyInt());
|
||||
verify(ziniaoApiKeyProvider).markIpWhitelistBlocked(
|
||||
eq(partiallyBlocked),
|
||||
eq("当前服务器 IP 未加入紫鸟白名单")
|
||||
);
|
||||
verify(ziniaoApiKeyProvider).markIpWhitelistAllowed(allowed);
|
||||
verify(ziniaoApiKeyProvider, never()).markIpWhitelistAllowed(partiallyBlocked);
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonWhitelistCompanyFailureDoesNotOverwriteWhitelistStatus() {
|
||||
ZiniaoApiKeyProvider.ApiKeyAccount failed = new ZiniaoApiKeyProvider.ApiKeyAccount("failed-key", "failed-account");
|
||||
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(failed));
|
||||
when(ziniaoAuthService.resolveCompanyIdForIndex("failed-key"))
|
||||
.thenThrow(new BusinessException("紫鸟接口临时不可用"));
|
||||
when(ziniaoAuthService.isIpWhitelistError(any(BusinessException.class))).thenReturn(false);
|
||||
|
||||
service.refreshShopIndex();
|
||||
|
||||
verify(ziniaoApiKeyProvider, never()).markIpWhitelistBlocked(any(), any());
|
||||
verify(ziniaoApiKeyProvider, never()).markIpWhitelistAllowed(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void whitelistStatusWriteFailureDoesNotStopNextApiKey() {
|
||||
stubIpWhitelistDetection();
|
||||
ZiniaoApiKeyProvider.ApiKeyAccount blocked = new ZiniaoApiKeyProvider.ApiKeyAccount("blocked-key", "blocked-account");
|
||||
ZiniaoApiKeyProvider.ApiKeyAccount allowed = new ZiniaoApiKeyProvider.ApiKeyAccount("allowed-key", "allowed-account");
|
||||
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(blocked, allowed));
|
||||
when(ziniaoAuthService.resolveCompanyIdForIndex("blocked-key"))
|
||||
.thenThrow(new BusinessException("当前服务器 IP 未加入紫鸟白名单"));
|
||||
when(ziniaoAuthService.resolveCompanyIdForIndex("allowed-key")).thenReturn(2L);
|
||||
when(ziniaoAuthService.getOrLoadStaffForIndex("allowed-key", 2L)).thenReturn(List.of());
|
||||
doThrow(new IllegalStateException("数据库暂时不可用"))
|
||||
.when(ziniaoApiKeyProvider)
|
||||
.markIpWhitelistBlocked(blocked, "当前服务器 IP 未加入紫鸟白名单");
|
||||
|
||||
service.refreshShopIndex();
|
||||
|
||||
verify(ziniaoAuthService).resolveCompanyIdForIndex("allowed-key");
|
||||
verify(ziniaoApiKeyProvider).markIpWhitelistAllowed(allowed);
|
||||
}
|
||||
|
||||
@Test
|
||||
void fullRefreshIgnoresScheduledBatchLimit() {
|
||||
properties.setShopIndexRefreshBatchSize(1);
|
||||
ZiniaoApiKeyProvider.ApiKeyAccount first = new ZiniaoApiKeyProvider.ApiKeyAccount("first-key", "first-account");
|
||||
ZiniaoApiKeyProvider.ApiKeyAccount second = new ZiniaoApiKeyProvider.ApiKeyAccount("second-key", "second-account");
|
||||
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(first, second));
|
||||
when(ziniaoAuthService.resolveCompanyIdForIndex("first-key")).thenReturn(1L);
|
||||
when(ziniaoAuthService.resolveCompanyIdForIndex("second-key")).thenReturn(2L);
|
||||
when(ziniaoAuthService.getOrLoadStaffForIndex("first-key", 1L)).thenReturn(List.of());
|
||||
when(ziniaoAuthService.getOrLoadStaffForIndex("second-key", 2L)).thenReturn(List.of());
|
||||
|
||||
service.refreshAllShopIndex();
|
||||
|
||||
verify(ziniaoAuthService).resolveCompanyIdForIndex("first-key");
|
||||
verify(ziniaoAuthService).resolveCompanyIdForIndex("second-key");
|
||||
ZiniaoShopIndexRefreshCursorDto cursor = capturedCursor();
|
||||
assertEquals(Integer.valueOf(2), cursor.getLastProcessedApiKeyCount());
|
||||
assertEquals(Integer.valueOf(0), cursor.getNextApiKeyOffset());
|
||||
}
|
||||
|
||||
private ZiniaoShopIndexRefreshCursorDto capturedCursor() {
|
||||
ArgumentCaptor<Object> captor = ArgumentCaptor.forClass(Object.class);
|
||||
verify(ziniaoTransientCacheService, times(2)).put(
|
||||
eq("SHOP_INDEX_REFRESH_CURSOR"),
|
||||
eq("global"),
|
||||
captor.capture(),
|
||||
any(Duration.class)
|
||||
);
|
||||
return (ZiniaoShopIndexRefreshCursorDto) captor.getAllValues().getLast();
|
||||
}
|
||||
|
||||
private void stubIpWhitelistDetection() {
|
||||
when(ziniaoAuthService.isIpWhitelistError(any(BusinessException.class)))
|
||||
.thenAnswer(invocation -> invocation.<BusinessException>getArgument(0).getMessage().contains("白名单"));
|
||||
}
|
||||
|
||||
private ZiniaoStaffItemVo staff(long userId) {
|
||||
ZiniaoStaffItemVo staff = new ZiniaoStaffItemVo();
|
||||
staff.setUserId(userId);
|
||||
return staff;
|
||||
}
|
||||
|
||||
private ZiniaoShopCacheDto shop(String shopId, String shopName) {
|
||||
ZiniaoShopCacheDto shop = new ZiniaoShopCacheDto();
|
||||
shop.setShopId(shopId);
|
||||
shop.setShopName(shopName);
|
||||
shop.setPlatform("亚马逊");
|
||||
return shop;
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ from flask import (
|
||||
g,
|
||||
Response,
|
||||
send_file,
|
||||
stream_with_context,
|
||||
has_request_context,
|
||||
)
|
||||
|
||||
@@ -45,6 +46,7 @@ admin_api = Blueprint('admin_api', __name__, url_prefix='/api/admin')
|
||||
_backend_java_session_local = threading.local()
|
||||
_internal_token_lock = threading.Lock()
|
||||
IMAGE_VIDEO_DATA_PERMISSION_KEY = 'admin_image_video_task_data'
|
||||
SHOP_DATA_CRAWL_DATA_PERMISSION_KEY = 'admin_shop_data_crawl_task_data'
|
||||
|
||||
ADMIN_MENU_ACCESS_CONFIG = {
|
||||
'dedupe-total-data': {
|
||||
@@ -82,6 +84,11 @@ ADMIN_MENU_ACCESS_CONFIG = {
|
||||
'route_path': 'image-video-tasks',
|
||||
'error': '无权访问视频任务管理模块',
|
||||
},
|
||||
'shop-data-crawl-tasks': {
|
||||
'column_key': 'admin_shop_data_crawl_tasks',
|
||||
'route_path': 'shop-data-crawl-tasks',
|
||||
'error': '无权访问店铺数据任务管理模块',
|
||||
},
|
||||
}
|
||||
|
||||
ADMIN_MENU_ACCESS_CONFIG.update({
|
||||
@@ -148,6 +155,29 @@ def _backend_java_forward_headers():
|
||||
return headers
|
||||
|
||||
|
||||
def _backend_java_internal_headers():
|
||||
"""Headers for Java routes that are callable only from the Flask admin service."""
|
||||
internal_token = _resolve_internal_token()
|
||||
if not internal_token:
|
||||
raise ValueError('内部凭据服务未配置')
|
||||
headers = _backend_java_forward_headers()
|
||||
headers['X-Internal-Token'] = internal_token
|
||||
return headers
|
||||
|
||||
|
||||
def _backend_java_internal_request():
|
||||
headers = _backend_java_internal_headers()
|
||||
_, current_row = get_current_admin_role()
|
||||
operator_id = _get_current_admin_id(current_row)
|
||||
try:
|
||||
operator_id = int(operator_id)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError('当前管理员身份无效') from exc
|
||||
if operator_id <= 0:
|
||||
raise ValueError('当前管理员身份无效')
|
||||
return headers, {'operatorId': operator_id}
|
||||
|
||||
|
||||
def _proxy_backend_java(
|
||||
method,
|
||||
path,
|
||||
@@ -752,6 +782,27 @@ def _ensure_image_video_data_access():
|
||||
return role, current_row, (jsonify({'success': False, 'error': '无权查看视频任务数据'}), 403)
|
||||
|
||||
|
||||
def _ensure_shop_data_crawl_data_access():
|
||||
role, current_row = get_current_admin_role()
|
||||
if role == 'super_admin':
|
||||
return role, current_row, None
|
||||
if not role or not current_row:
|
||||
return role, current_row, (jsonify({'success': False, 'error': '需要登录'}), 403)
|
||||
try:
|
||||
_, key_set, route_set = _effective_permission_sets(
|
||||
_get_current_admin_id(current_row),
|
||||
menu_type=None,
|
||||
current_row=current_row,
|
||||
role=role,
|
||||
)
|
||||
except _PermissionProxyError as exc:
|
||||
return role, current_row, (exc.response, exc.status)
|
||||
if (SHOP_DATA_CRAWL_DATA_PERMISSION_KEY in key_set
|
||||
or 'shop-data-crawl-task-data' in route_set):
|
||||
return role, current_row, None
|
||||
return role, current_row, (jsonify({'success': False, 'error': '无权查看店铺数据任务'}), 403)
|
||||
|
||||
|
||||
def _ensure_product_category_access():
|
||||
role, current_row, items, denied = _load_current_backend_menu_items()
|
||||
if denied:
|
||||
@@ -1284,6 +1335,458 @@ def _parse_admin_datetime_arg(name):
|
||||
raise ValueError(f'{name} 时间格式无效') from exc
|
||||
|
||||
|
||||
_SHOP_DATA_CRAWL_ADMIN_COLUMNS = """
|
||||
r.id AS result_id, r.task_id, r.user_id, r.source_filename AS shop_name,
|
||||
r.source_file_url AS shop_id, r.result_filename, r.result_file_url,
|
||||
r.result_file_size, r.result_content_type, r.row_count,
|
||||
r.success AS result_success, r.error_message AS result_error,
|
||||
r.created_at AS result_created_at,
|
||||
t.task_no, t.status AS task_status, t.request_json, t.result_json,
|
||||
t.error_message AS task_error, t.created_at, t.updated_at, t.finished_at,
|
||||
u.username,
|
||||
(SELECT j.id FROM biz_task_file_job j
|
||||
WHERE j.module_type = 'SHOP_DATA_CRAWL' AND j.result_id = r.id
|
||||
AND j.job_type = 'ASSEMBLE_RESULT'
|
||||
ORDER BY j.id DESC LIMIT 1) AS file_job_id,
|
||||
(SELECT j.status FROM biz_task_file_job j
|
||||
WHERE j.module_type = 'SHOP_DATA_CRAWL' AND j.result_id = r.id
|
||||
AND j.job_type = 'ASSEMBLE_RESULT'
|
||||
ORDER BY j.id DESC LIMIT 1) AS file_status,
|
||||
(SELECT j.error_message FROM biz_task_file_job j
|
||||
WHERE j.module_type = 'SHOP_DATA_CRAWL' AND j.result_id = r.id
|
||||
AND j.job_type = 'ASSEMBLE_RESULT'
|
||||
ORDER BY j.id DESC LIMIT 1) AS file_error
|
||||
"""
|
||||
|
||||
|
||||
def _shop_data_crawl_country_codes(request_json):
|
||||
payload = _parse_json_value(request_json, {})
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
raw = payload.get('countryCodes')
|
||||
if raw is None:
|
||||
raw = payload.get('country_codes')
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
return [str(value).strip().upper() for value in raw if str(value or '').strip()]
|
||||
|
||||
|
||||
def _shop_data_crawl_group_names(cursor, rows):
|
||||
shop_names = sorted({
|
||||
_shop_data_crawl_shop_key(row.get('shop_name'))
|
||||
for row in rows
|
||||
if _shop_data_crawl_shop_key(row.get('shop_name'))
|
||||
})
|
||||
if not shop_names:
|
||||
return {}
|
||||
placeholders = ','.join(['%s'] * len(shop_names))
|
||||
cursor.execute(
|
||||
"SELECT TRIM(sm.shop_name) AS shop_name, "
|
||||
"GROUP_CONCAT(DISTINCT COALESCE(NULLIF(g.group_name, ''), NULLIF(sm.group_name, '')) "
|
||||
"ORDER BY sm.id SEPARATOR '、') AS group_name "
|
||||
"FROM biz_shop_manage sm "
|
||||
"LEFT JOIN biz_shop_manage_group g ON g.id = sm.group_id "
|
||||
f"WHERE TRIM(sm.shop_name) IN ({placeholders}) GROUP BY TRIM(sm.shop_name)",
|
||||
tuple(shop_names),
|
||||
)
|
||||
return {
|
||||
_shop_data_crawl_shop_key(row.get('shop_name')): row.get('group_name') or ''
|
||||
for row in cursor.fetchall()
|
||||
}
|
||||
|
||||
|
||||
def _shop_data_crawl_shop_key(value):
|
||||
"""Normalize a shop name for grouping while preserving the display value."""
|
||||
return str(value or '').strip().casefold()
|
||||
|
||||
|
||||
def _shop_data_crawl_group_name(group_names, shop_name):
|
||||
"""Resolve a group label from either normalized or legacy exact-key maps."""
|
||||
if not group_names:
|
||||
return ''
|
||||
normalized = _shop_data_crawl_shop_key(shop_name)
|
||||
return group_names.get(normalized, group_names.get(str(shop_name or '').strip(), '')) or ''
|
||||
|
||||
|
||||
def _shop_data_crawl_admin_item(row, group_names=None):
|
||||
result_success = row.get('result_success')
|
||||
file_ready = bool((row.get('result_file_url') or '').strip())
|
||||
if result_success is None or int(result_success) < 0:
|
||||
success = None
|
||||
else:
|
||||
success = bool(int(result_success))
|
||||
group_names = group_names or {}
|
||||
shop_name = row.get('shop_name') or ''
|
||||
file_status = row.get('file_status') or ('SUCCESS' if file_ready else '')
|
||||
return {
|
||||
'task_id': row.get('task_id'),
|
||||
'task_no': row.get('task_no') or '',
|
||||
'result_id': row.get('result_id'),
|
||||
'user_id': row.get('user_id'),
|
||||
'username': row.get('username') or '',
|
||||
'shop_name': shop_name,
|
||||
'shop_id': row.get('shop_id') or '',
|
||||
'group_name': _shop_data_crawl_group_name(group_names, shop_name),
|
||||
'status': row.get('task_status') or '',
|
||||
'success': success,
|
||||
'error': row.get('result_error') or row.get('task_error') or row.get('file_error') or '',
|
||||
'country_codes': _shop_data_crawl_country_codes(row.get('request_json')),
|
||||
'output_filename': row.get('result_filename') or '',
|
||||
'result_file_url': row.get('result_file_url') or '',
|
||||
'file_ready': file_ready,
|
||||
'file_job_id': row.get('file_job_id'),
|
||||
'file_status': file_status,
|
||||
'file_error': row.get('file_error') or '',
|
||||
'file_size': int(row.get('result_file_size') or 0),
|
||||
'row_count': int(row.get('row_count') or 0),
|
||||
'created_at': _format_admin_datetime(row.get('created_at') or row.get('result_created_at')),
|
||||
'updated_at': _format_admin_datetime(row.get('updated_at')),
|
||||
'finished_at': _format_admin_datetime(row.get('finished_at')),
|
||||
}
|
||||
|
||||
|
||||
def _shop_data_crawl_group_item(group_row, result_rows, group_names):
|
||||
"""Build one shop group and cap its children to the newest three results."""
|
||||
raw_shop_name = group_row.get('shop_name') or ''
|
||||
display_shop_name = raw_shop_name or '未命名'
|
||||
group_key = _shop_data_crawl_shop_key(raw_shop_name)
|
||||
children = result_rows.get(group_key)
|
||||
if children is None:
|
||||
children = result_rows.get(str(raw_shop_name).strip(), [])
|
||||
children = children[:3]
|
||||
result_items = [_shop_data_crawl_admin_item(row, group_names) for row in children]
|
||||
latest_created_at = group_row.get('latest_created_at')
|
||||
if latest_created_at is None and result_items:
|
||||
latest_created_at = result_items[0].get('created_at')
|
||||
return {
|
||||
'shop_name': display_shop_name,
|
||||
'shop_id': result_items[0].get('shop_id', '') if result_items else '',
|
||||
'group_name': _shop_data_crawl_group_name(group_names, raw_shop_name),
|
||||
'latest_created_at': _format_admin_datetime(latest_created_at),
|
||||
'results': result_items,
|
||||
}
|
||||
|
||||
|
||||
@admin_api.route('/shop-data-crawl-task-permissions', methods=['GET', 'PUT'])
|
||||
@login_required
|
||||
def manage_shop_data_crawl_task_permissions():
|
||||
json_data = None
|
||||
if request.method == 'PUT':
|
||||
data = request.get_json(silent=True) or {}
|
||||
raw_user_ids = data.get('user_ids') if 'user_ids' in data else data.get('userIds')
|
||||
json_data = {'userIds': raw_user_ids}
|
||||
result, error_response, status = _proxy_permission_java(
|
||||
request.method,
|
||||
'/api/admin/shop-data-crawl-task-permissions',
|
||||
json_data=json_data,
|
||||
)
|
||||
if error_response is not None:
|
||||
return error_response, status
|
||||
if request.method == 'GET':
|
||||
return jsonify({'success': True, 'items': _permission_response_items(result)})
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'granted_count': result.get('data'),
|
||||
'msg': result.get('message') or '店铺数据任务权限已更新',
|
||||
})
|
||||
|
||||
|
||||
@admin_api.route('/shop-data-crawl-tasks')
|
||||
@login_required
|
||||
def list_shop_data_crawl_tasks():
|
||||
_, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks')
|
||||
if not denied:
|
||||
_, _, denied = _ensure_shop_data_crawl_data_access()
|
||||
if denied:
|
||||
return denied
|
||||
try:
|
||||
page = max(1, int(request.args.get('page', 1)))
|
||||
page_size = min(100, max(10, int(request.args.get('page_size', 20))))
|
||||
shop_name = (request.args.get('shop_name') or request.args.get('shop') or '').strip()
|
||||
group_name = (request.args.get('group_name') or request.args.get('group') or '').strip()
|
||||
created_from = _parse_admin_datetime_arg('created_from')
|
||||
created_to = _parse_admin_datetime_arg('created_to')
|
||||
|
||||
conditions = [
|
||||
"r.module_type = 'SHOP_DATA_CRAWL'",
|
||||
"t.module_type = 'SHOP_DATA_CRAWL'",
|
||||
"TRIM(COALESCE(r.result_file_url, '')) <> ''",
|
||||
]
|
||||
params = []
|
||||
if shop_name:
|
||||
conditions.append('r.source_filename LIKE %s')
|
||||
params.append('%' + shop_name + '%')
|
||||
if group_name:
|
||||
conditions.append(
|
||||
'EXISTS (SELECT 1 FROM biz_shop_manage sm '
|
||||
'LEFT JOIN biz_shop_manage_group g ON g.id = sm.group_id '
|
||||
'WHERE TRIM(COALESCE(sm.shop_name, \'\')) = '
|
||||
'TRIM(COALESCE(r.source_filename, \'\')) '
|
||||
"AND COALESCE(NULLIF(g.group_name, ''), NULLIF(sm.group_name, '')) LIKE %s)"
|
||||
)
|
||||
params.append('%' + group_name + '%')
|
||||
if created_from:
|
||||
conditions.append('t.created_at >= %s')
|
||||
params.append(created_from)
|
||||
if created_to:
|
||||
conditions.append('t.created_at <= %s')
|
||||
params.append(created_to)
|
||||
where_sql = ' AND '.join(conditions)
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
conn = get_db()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
shop_key_sql = "TRIM(COALESCE(r.source_filename, ''))"
|
||||
grouped_from_sql = (
|
||||
' FROM biz_file_result r '
|
||||
'JOIN biz_file_task t ON t.id = r.task_id '
|
||||
'LEFT JOIN users u ON u.id = r.user_id '
|
||||
'WHERE ' + where_sql
|
||||
)
|
||||
cur.execute(
|
||||
'SELECT COUNT(*) AS total FROM ('
|
||||
'SELECT ' + shop_key_sql + ' AS shop_key' + grouped_from_sql +
|
||||
' GROUP BY ' + shop_key_sql +
|
||||
') shop_groups',
|
||||
tuple(params),
|
||||
)
|
||||
total = int((cur.fetchone() or {}).get('total') or 0)
|
||||
|
||||
cur.execute(
|
||||
'SELECT ' + shop_key_sql + ' AS shop_name, MAX(t.created_at) AS latest_created_at'
|
||||
+ grouped_from_sql +
|
||||
' GROUP BY ' + shop_key_sql +
|
||||
' ORDER BY latest_created_at DESC, shop_name ASC LIMIT %s OFFSET %s',
|
||||
tuple(params + [page_size, offset]),
|
||||
)
|
||||
group_rows = cur.fetchall()
|
||||
group_names = _shop_data_crawl_group_names(cur, group_rows)
|
||||
|
||||
result_rows_by_shop = {}
|
||||
selected_shop_names = [row.get('shop_name') for row in group_rows]
|
||||
if selected_shop_names:
|
||||
placeholders = ','.join(['%s'] * len(selected_shop_names))
|
||||
cur.execute(
|
||||
'SELECT ranked.* FROM (SELECT ' + _SHOP_DATA_CRAWL_ADMIN_COLUMNS +
|
||||
', ROW_NUMBER() OVER (PARTITION BY ' + shop_key_sql +
|
||||
' ORDER BY t.created_at DESC, r.id DESC) AS shop_row_number '
|
||||
' FROM biz_file_result r '
|
||||
'JOIN biz_file_task t ON t.id = r.task_id '
|
||||
'LEFT JOIN users u ON u.id = r.user_id '
|
||||
'WHERE ' + where_sql +
|
||||
f' AND {shop_key_sql} IN ({placeholders})' +
|
||||
') ranked WHERE ranked.shop_row_number <= 3 '
|
||||
'ORDER BY ranked.created_at DESC, ranked.result_id DESC',
|
||||
tuple(params + selected_shop_names),
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
shop_key = _shop_data_crawl_shop_key(row.get('shop_name'))
|
||||
result_rows_by_shop.setdefault(shop_key, []).append(row)
|
||||
finally:
|
||||
conn.close()
|
||||
payload = {
|
||||
'items': [
|
||||
_shop_data_crawl_group_item(group, result_rows_by_shop, group_names)
|
||||
for group in group_rows
|
||||
],
|
||||
'total': total,
|
||||
'page': page,
|
||||
'page_size': page_size,
|
||||
}
|
||||
# Keep the existing admin response shape while exposing the grouped
|
||||
# payload for clients that use the newer data envelope.
|
||||
return jsonify({'success': True, **payload, 'data': payload})
|
||||
except ValueError as exc:
|
||||
return jsonify({'success': False, 'error': str(exc)}), 400
|
||||
except Exception as exc:
|
||||
return jsonify({'success': False, 'error': str(exc)}), 500
|
||||
|
||||
|
||||
def _load_shop_data_crawl_download_rows(result_ids):
|
||||
normalized = sorted({int(result_id) for result_id in result_ids if int(result_id) > 0})
|
||||
if not normalized:
|
||||
return {}
|
||||
placeholders = ','.join(['%s'] * len(normalized))
|
||||
conn = get_db()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT r.id, r.task_id, r.user_id, r.result_filename, r.source_filename, '
|
||||
'r.result_file_url, t.status AS task_status '
|
||||
'FROM biz_file_result r JOIN biz_file_task t ON t.id = r.task_id '
|
||||
'WHERE r.module_type = %s AND t.module_type = %s '
|
||||
f'AND r.id IN ({placeholders})',
|
||||
tuple(['SHOP_DATA_CRAWL', 'SHOP_DATA_CRAWL'] + normalized),
|
||||
)
|
||||
return {int(row['id']): row for row in cur.fetchall()}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _open_shop_data_crawl_download(row):
|
||||
url = f"{backend_java_base_url}/api/admin/shop-data-crawl/results/{int(row['id'])}/download"
|
||||
headers, params = _backend_java_internal_request()
|
||||
return _get_backend_java_session().get(
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
stream=True,
|
||||
timeout=(10, 180),
|
||||
)
|
||||
|
||||
|
||||
@admin_api.route('/shop-data-crawl-tasks/<int:result_id>/download')
|
||||
@login_required
|
||||
def download_shop_data_crawl_task(result_id):
|
||||
_, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks')
|
||||
if not denied:
|
||||
_, _, denied = _ensure_shop_data_crawl_data_access()
|
||||
if denied:
|
||||
return denied
|
||||
rows = _load_shop_data_crawl_download_rows([result_id])
|
||||
row = rows.get(result_id)
|
||||
if not row or not (row.get('result_file_url') or '').strip():
|
||||
return jsonify({'success': False, 'error': '结果文件不存在或尚未生成'}), 404
|
||||
try:
|
||||
remote = _open_shop_data_crawl_download(row)
|
||||
except ValueError as exc:
|
||||
return jsonify({'success': False, 'error': str(exc)}), 503
|
||||
except requests.RequestException as exc:
|
||||
return jsonify({'success': False, 'error': f'结果文件下载失败: {exc}'}), 502
|
||||
if remote.status_code != 200:
|
||||
message = remote.text[:500] if remote.content else ''
|
||||
remote.close()
|
||||
return jsonify({'success': False, 'error': message or '结果文件下载失败'}), remote.status_code
|
||||
|
||||
def generate():
|
||||
try:
|
||||
for chunk in remote.iter_content(chunk_size=1024 * 1024):
|
||||
if chunk:
|
||||
yield chunk
|
||||
finally:
|
||||
remote.close()
|
||||
|
||||
response = Response(
|
||||
stream_with_context(generate()),
|
||||
content_type=remote.headers.get('Content-Type') or
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
)
|
||||
disposition = remote.headers.get('Content-Disposition')
|
||||
if disposition:
|
||||
response.headers['Content-Disposition'] = disposition
|
||||
else:
|
||||
filename = row.get('result_filename') or f"{row.get('source_filename') or result_id}.xlsx"
|
||||
response.headers['Content-Disposition'] = "attachment; filename*=UTF-8''" + quote(filename)
|
||||
response.headers['Cache-Control'] = 'no-store'
|
||||
return response
|
||||
|
||||
|
||||
@admin_api.route('/shop-data-crawl-tasks/<int:result_id>', methods=['DELETE'])
|
||||
@login_required
|
||||
def delete_shop_data_crawl_task(result_id):
|
||||
_, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks')
|
||||
if not denied:
|
||||
_, _, denied = _ensure_shop_data_crawl_data_access()
|
||||
if denied:
|
||||
return denied
|
||||
rows = _load_shop_data_crawl_download_rows([result_id])
|
||||
row = rows.get(result_id)
|
||||
if not row:
|
||||
return jsonify({'success': False, 'error': '店铺数据任务不存在'}), 404
|
||||
if (row.get('task_status') or '').upper() not in {'SUCCESS', 'FAILED', 'CANCELLED'}:
|
||||
return jsonify({'success': False, 'error': '任务仍在处理中,不能删除'}), 409
|
||||
try:
|
||||
headers, params = _backend_java_internal_request()
|
||||
except ValueError as exc:
|
||||
return jsonify({'success': False, 'error': str(exc)}), 503
|
||||
result, error_response, status = _proxy_backend_java(
|
||||
'DELETE',
|
||||
f'/api/admin/shop-data-crawl/history/{result_id}',
|
||||
params=params,
|
||||
headers=headers,
|
||||
)
|
||||
if error_response is not None:
|
||||
return error_response, status
|
||||
return jsonify({'success': True, 'msg': result.get('message') or '删除成功'})
|
||||
|
||||
|
||||
@admin_api.route('/shop-data-crawl-tasks/download-zip', methods=['POST'])
|
||||
@login_required
|
||||
def download_shop_data_crawl_tasks_zip():
|
||||
_, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks')
|
||||
if not denied:
|
||||
_, _, denied = _ensure_shop_data_crawl_data_access()
|
||||
if denied:
|
||||
return denied
|
||||
data = request.get_json(silent=True) or {}
|
||||
raw_ids = data.get('result_ids') if 'result_ids' in data else data.get('resultIds')
|
||||
if not isinstance(raw_ids, list) or not raw_ids:
|
||||
return jsonify({'success': False, 'error': '请至少选择一个结果文件'}), 400
|
||||
if len(raw_ids) > 100:
|
||||
return jsonify({'success': False, 'error': '单次最多打包 100 个结果文件'}), 400
|
||||
try:
|
||||
result_ids = []
|
||||
for raw_id in raw_ids:
|
||||
result_id = int(raw_id)
|
||||
if result_id <= 0:
|
||||
raise ValueError
|
||||
if result_id not in result_ids:
|
||||
result_ids.append(result_id)
|
||||
except (TypeError, ValueError):
|
||||
return jsonify({'success': False, 'error': '结果文件参数无效'}), 400
|
||||
|
||||
rows = _load_shop_data_crawl_download_rows(result_ids)
|
||||
archive = tempfile.SpooledTemporaryFile(max_size=64 * 1024 * 1024, mode='w+b')
|
||||
errors = []
|
||||
file_count = 0
|
||||
used_names = set()
|
||||
try:
|
||||
with zipfile.ZipFile(archive, mode='w', compression=zipfile.ZIP_STORED, allowZip64=True) as output_zip:
|
||||
for result_id in result_ids:
|
||||
row = rows.get(result_id)
|
||||
if not row or not (row.get('result_file_url') or '').strip():
|
||||
errors.append(f'result-{result_id}: 结果文件不存在或尚未生成')
|
||||
continue
|
||||
filename = row.get('result_filename') or f"{row.get('source_filename') or result_id}.xlsx"
|
||||
filename = re.sub(r'[\\/:*?"<>|]+', '_', filename).strip() or f'result-{result_id}.xlsx'
|
||||
if filename in used_names:
|
||||
stem, extension = os.path.splitext(filename)
|
||||
filename = f'{stem}-{result_id}{extension or ".xlsx"}'
|
||||
used_names.add(filename)
|
||||
remote = None
|
||||
try:
|
||||
remote = _open_shop_data_crawl_download(row)
|
||||
remote.raise_for_status()
|
||||
with output_zip.open(filename, mode='w', force_zip64=True) as target:
|
||||
for chunk in remote.iter_content(chunk_size=1024 * 1024):
|
||||
if chunk:
|
||||
target.write(chunk)
|
||||
file_count += 1
|
||||
except (requests.RequestException, ValueError) as exc:
|
||||
errors.append(f'{filename}: 下载失败 ({exc})')
|
||||
finally:
|
||||
if remote is not None:
|
||||
remote.close()
|
||||
if errors:
|
||||
output_zip.writestr('download-errors.txt', '\n'.join(errors).encode('utf-8'))
|
||||
archive.seek(0)
|
||||
response = send_file(
|
||||
archive,
|
||||
mimetype='application/zip',
|
||||
as_attachment=True,
|
||||
download_name=f"shop-data-tasks-{datetime.now().strftime('%Y%m%d-%H%M%S')}.zip",
|
||||
max_age=0,
|
||||
)
|
||||
response.headers['X-Archive-File-Count'] = str(file_count)
|
||||
response.headers['X-Archive-Error-Count'] = str(len(errors))
|
||||
response.call_on_close(archive.close)
|
||||
return response
|
||||
except Exception:
|
||||
archive.close()
|
||||
raise
|
||||
|
||||
|
||||
@admin_api.route('/image-video-task-permissions', methods=['GET', 'PUT'])
|
||||
@login_required
|
||||
def manage_image_video_task_permissions():
|
||||
@@ -1526,7 +2029,10 @@ def list_columns():
|
||||
return error_response, status
|
||||
items = [
|
||||
item for item in items
|
||||
if (item.get('column_key') or '').strip() != IMAGE_VIDEO_DATA_PERMISSION_KEY
|
||||
if (item.get('column_key') or '').strip() not in {
|
||||
IMAGE_VIDEO_DATA_PERMISSION_KEY,
|
||||
SHOP_DATA_CRAWL_DATA_PERMISSION_KEY,
|
||||
}
|
||||
]
|
||||
# Keep the legacy `items` field; some Java-aware callers use `data`/`columns`.
|
||||
return jsonify({'success': True, 'items': items, 'columns': items, 'data': items})
|
||||
@@ -2252,6 +2758,9 @@ def list_shop_keys():
|
||||
'remark_name': item.get('remarkName') or '',
|
||||
'ziniao_account_name': item.get('ziniaoAccountName') or '',
|
||||
'ziniao_token': item.get('ziniaoToken') or '',
|
||||
'ip_whitelist_status': item.get('ipWhitelistStatus') or 'UNKNOWN',
|
||||
'ip_whitelist_checked_at': (item.get('ipWhitelistCheckedAt') or '').replace('T', ' ')[:19],
|
||||
'ip_whitelist_message': item.get('ipWhitelistMessage') or '',
|
||||
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
|
||||
'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16],
|
||||
}
|
||||
@@ -2364,16 +2873,23 @@ def list_dedupe_total_data():
|
||||
page_size = min(100, max(1, int(request.args.get('page_size', 15))))
|
||||
keyword = (request.args.get('keyword') or '').strip()
|
||||
username = (request.args.get('username') or '').strip()
|
||||
data, error_response, status = _proxy_backend_java(
|
||||
'GET',
|
||||
'/api/admin/dedupe-total-data',
|
||||
params={
|
||||
start_date = (request.args.get('start_date') or request.args.get('startDate') or '').strip()
|
||||
end_date = (request.args.get('end_date') or request.args.get('endDate') or '').strip()
|
||||
params = {
|
||||
'page': page,
|
||||
'pageSize': page_size,
|
||||
'keyword': keyword,
|
||||
'username': username,
|
||||
'operatorId': current_row.get('id'),
|
||||
},
|
||||
}
|
||||
if start_date:
|
||||
params['startDate'] = start_date
|
||||
if end_date:
|
||||
params['endDate'] = end_date
|
||||
data, error_response, status = _proxy_backend_java(
|
||||
'GET',
|
||||
'/api/admin/dedupe-total-data',
|
||||
params=params,
|
||||
)
|
||||
if error_response is not None:
|
||||
return error_response, status
|
||||
@@ -2832,6 +3348,61 @@ def list_shop_manages():
|
||||
})
|
||||
|
||||
|
||||
@admin_api.route('/shop-manage/<int:item_id>/credential')
|
||||
@login_required
|
||||
def get_shop_manage_credential(item_id):
|
||||
role, current_row, denied = _ensure_backend_menu_access('shop-manage')
|
||||
if denied:
|
||||
return denied
|
||||
|
||||
shop_name = (request.args.get('shop_name') or '').strip()
|
||||
if not shop_name:
|
||||
return jsonify({'success': False, 'error': '店铺名不能为空'}), 400
|
||||
|
||||
access_params = {
|
||||
'page': 1,
|
||||
'pageSize': 100,
|
||||
'shopName': shop_name,
|
||||
'superAdmin': 'true' if role == 'super_admin' else 'false',
|
||||
}
|
||||
if current_row and current_row.get('id'):
|
||||
access_params['operatorId'] = current_row.get('id')
|
||||
access_result, error_response, status = _proxy_backend_java(
|
||||
'GET',
|
||||
'/api/admin/shop-manages',
|
||||
params=access_params,
|
||||
)
|
||||
if error_response is not None:
|
||||
return error_response, status
|
||||
|
||||
accessible_items = ((access_result.get('data') or {}).get('items') or [])
|
||||
accessible_item = next((
|
||||
item for item in accessible_items
|
||||
if str(item.get('id')) == str(item_id) and (item.get('shopName') or '') == shop_name
|
||||
), None)
|
||||
if accessible_item is None:
|
||||
return jsonify({'success': False, 'error': '店铺不存在或无权访问'}), 404
|
||||
|
||||
internal_token = _resolve_internal_token()
|
||||
if not internal_token:
|
||||
return jsonify({'success': False, 'error': '内部凭据服务未配置'}), 503
|
||||
credential_result, error_response, status = _proxy_backend_java(
|
||||
'GET',
|
||||
'/api/admin/shop-manages/credential',
|
||||
params={'shopName': shop_name},
|
||||
headers={'X-Internal-Token': internal_token},
|
||||
)
|
||||
if error_response is not None:
|
||||
return error_response, status
|
||||
|
||||
credential = credential_result.get('data') or {}
|
||||
if str(credential.get('id')) != str(item_id):
|
||||
return jsonify({'success': False, 'error': '店铺凭据不匹配'}), 409
|
||||
response = jsonify({'success': True, 'password': credential.get('password') or ''})
|
||||
response.headers['Cache-Control'] = 'no-store'
|
||||
return response
|
||||
|
||||
|
||||
@admin_api.route('/shop-manage', methods=['POST'])
|
||||
@login_required
|
||||
def create_shop_manage():
|
||||
|
||||
+637
-13
@@ -108,6 +108,7 @@
|
||||
'query-asin': 'panel-query-asin',
|
||||
'product-categories': 'panel-product-categories',
|
||||
'image-video-tasks': 'panel-image-video-tasks',
|
||||
'shop-data-crawl-tasks': 'panel-shop-data-crawl-tasks',
|
||||
'history': 'panel-history',
|
||||
'version': 'panel-version',
|
||||
'digital-human-version': 'panel-digital-human-version'
|
||||
@@ -126,6 +127,7 @@
|
||||
else if (tabName === 'query-asin') loadQueryAsin(1);
|
||||
else if (tabName === 'product-categories') loadProductCategories();
|
||||
else if (tabName === 'image-video-tasks') loadImageVideoTasks(1);
|
||||
else if (tabName === 'shop-data-crawl-tasks') loadShopDataCrawlTasks(1);
|
||||
else if (tabName === 'history') loadHistory(1);
|
||||
else if (tabName === 'version') loadSoftwareVersions();
|
||||
else if (tabName === 'digital-human-version') loadDigitalHumanVersions();
|
||||
@@ -438,7 +440,8 @@
|
||||
.then(function (res) {
|
||||
if (!res.success) return;
|
||||
var availableColumns = (res.items || []).filter(function (item) {
|
||||
if (item.column_key === 'admin_image_video_task_data') return false;
|
||||
if (item.column_key === 'admin_image_video_task_data' ||
|
||||
item.column_key === 'admin_shop_data_crawl_task_data') return false;
|
||||
return true;
|
||||
});
|
||||
if (currentUserRole === 'admin') {
|
||||
@@ -880,6 +883,8 @@
|
||||
function updateImageVideoPermissionAccess() {
|
||||
var button = document.getElementById('btnOpenImageVideoPermissions');
|
||||
if (button) button.style.display = currentUserRole === 'super_admin' ? 'inline-flex' : 'none';
|
||||
var shopButton = document.getElementById('btnOpenShopDataTaskPermissions');
|
||||
if (shopButton) shopButton.style.display = currentUserRole === 'super_admin' ? 'inline-flex' : 'none';
|
||||
}
|
||||
function imageVideoPermissionUsersForView() {
|
||||
if (imageVideoPermissionView === 'granted') {
|
||||
@@ -1386,6 +1391,523 @@
|
||||
}
|
||||
};
|
||||
|
||||
// ========== 店铺数据任务管理 ==========
|
||||
var shopDataTaskPage = 1, shopDataTaskPageSize = 20;
|
||||
var shopDataTaskGroups = [];
|
||||
var shopDataTasks = [];
|
||||
var selectedShopDataResultIds = new Set();
|
||||
var shopDataDownloadInProgress = false;
|
||||
var shopDataPermissionUsers = [];
|
||||
var shopDataPermissionInitialUserIds = new Set();
|
||||
var selectedShopDataPermissionUserIds = new Set();
|
||||
var shopDataPermissionView = 'granted';
|
||||
|
||||
function buildShopDataTaskQuery(page) {
|
||||
var params = new URLSearchParams();
|
||||
params.set('page', String(page || 1));
|
||||
params.set('page_size', String(shopDataTaskPageSize));
|
||||
var values = {
|
||||
shop_name: document.getElementById('shopDataTaskFilterShop').value.trim(),
|
||||
group_name: document.getElementById('shopDataTaskFilterGroup').value.trim(),
|
||||
created_from: document.getElementById('shopDataTaskFilterFrom').value,
|
||||
created_to: document.getElementById('shopDataTaskFilterTo').value
|
||||
};
|
||||
Object.keys(values).forEach(function (key) {
|
||||
if (values[key]) params.set(key, values[key]);
|
||||
});
|
||||
return params.toString();
|
||||
}
|
||||
|
||||
function shopDataResultId(item) {
|
||||
if (!item) return 0;
|
||||
var value = item.result_id != null ? item.result_id : item.resultId;
|
||||
var id = Number(value);
|
||||
return isFinite(id) && id > 0 ? id : 0;
|
||||
}
|
||||
|
||||
function shopDataBoolean(value) {
|
||||
if (typeof value === 'string') {
|
||||
return ['1', 'true', 'yes', 'y'].indexOf(value.toLowerCase()) >= 0;
|
||||
}
|
||||
return !!value;
|
||||
}
|
||||
|
||||
function shopDataDateValue(value) {
|
||||
if (!value) return 0;
|
||||
var timestamp = Date.parse(String(value).replace(' ', 'T'));
|
||||
return isNaN(timestamp) ? 0 : timestamp;
|
||||
}
|
||||
|
||||
function shopDataResultSort(a, b) {
|
||||
var dateDiff = shopDataDateValue(b.created_at || b.finished_at) - shopDataDateValue(a.created_at || a.finished_at);
|
||||
if (dateDiff) return dateDiff;
|
||||
return shopDataResultId(b) - shopDataResultId(a);
|
||||
}
|
||||
|
||||
function shopDataGroupKey(item) {
|
||||
var name = String((item && (item.shop_name || item.shop || item.source_filename)) || '').trim();
|
||||
// Keep the same trimmed, case-insensitive key as the Flask grouping query.
|
||||
return 'name:' + name.toLowerCase();
|
||||
}
|
||||
|
||||
function shopDataNormalizeResult(group, raw) {
|
||||
var result = {};
|
||||
Object.keys(group || {}).forEach(function (key) {
|
||||
if (key !== 'results' && key !== 'group_results') result[key] = group[key];
|
||||
});
|
||||
Object.keys(raw || {}).forEach(function (key) {
|
||||
if (key !== 'results' && key !== 'group_results') result[key] = raw[key];
|
||||
});
|
||||
result.shop_name = result.shop_name || result.shop || result.source_filename || '';
|
||||
result.shop_id = result.shop_id || result.shopId || result.source_file_url || '';
|
||||
result.group_name = result.group_name || result.group || '';
|
||||
result.status = result.status || result.task_status || result.file_status || '';
|
||||
result.error = result.error || result.result_error || result.error_message || result.task_error || result.file_error || '';
|
||||
result.output_filename = result.output_filename || result.result_filename || result.filename || '';
|
||||
result.country_codes = result.country_codes || result.countryCodes || [];
|
||||
if (result.file_size == null) result.file_size = result.result_file_size;
|
||||
if (result.row_count == null) result.row_count = result.rows;
|
||||
if (result.finished_at == null) result.finished_at = result.completed_at;
|
||||
if (result.result_id == null && raw && raw.resultId != null) result.result_id = raw.resultId;
|
||||
if (result.result_id == null && raw && raw.id != null && !Array.isArray(raw.results) && !Array.isArray(raw.group_results)) result.result_id = raw.id;
|
||||
if (result.file_ready == null) {
|
||||
result.file_ready = !!String(result.result_file_url || result.resultFileUrl || '').trim();
|
||||
} else {
|
||||
result.file_ready = shopDataBoolean(result.file_ready);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// The admin API now returns one group per shop. Keep a flat result list
|
||||
// for selection/actions while rendering the grouped view.
|
||||
function normalizeShopDataTaskGroups(items) {
|
||||
var groups = [];
|
||||
var byKey = Object.create(null);
|
||||
(Array.isArray(items) ? items : []).forEach(function (rawGroup) {
|
||||
if (!rawGroup || typeof rawGroup !== 'object') return;
|
||||
var rawResults = Array.isArray(rawGroup.results)
|
||||
? rawGroup.results
|
||||
: (Array.isArray(rawGroup.group_results) ? rawGroup.group_results : [rawGroup]);
|
||||
var groupBase = {};
|
||||
Object.keys(rawGroup).forEach(function (key) {
|
||||
if (key !== 'results' && key !== 'group_results') groupBase[key] = rawGroup[key];
|
||||
});
|
||||
if (!groupBase.shop_name && rawResults.length) {
|
||||
groupBase.shop_name = rawResults[0].shop_name || rawResults[0].shop || rawResults[0].source_filename || '';
|
||||
}
|
||||
if (!groupBase.shop_id && rawResults.length) {
|
||||
groupBase.shop_id = rawResults[0].shop_id || rawResults[0].shopId || rawResults[0].source_file_url || '';
|
||||
}
|
||||
var key = shopDataGroupKey(groupBase);
|
||||
var group = byKey[key];
|
||||
if (!group) {
|
||||
group = {
|
||||
key: key,
|
||||
shop_name: groupBase.shop_name || '',
|
||||
shop_id: groupBase.shop_id || '',
|
||||
group_name: groupBase.group_name || groupBase.group || '',
|
||||
latest_created_at: groupBase.latest_created_at || '',
|
||||
results: []
|
||||
};
|
||||
byKey[key] = group;
|
||||
groups.push(group);
|
||||
}
|
||||
rawResults.forEach(function (rawResult) {
|
||||
if (!rawResult || typeof rawResult !== 'object') return;
|
||||
var result = shopDataNormalizeResult(groupBase, rawResult);
|
||||
var resultId = shopDataResultId(result);
|
||||
if (!resultId || !result.file_ready) return;
|
||||
if (resultId && group.results.some(function (existing) { return shopDataResultId(existing) === resultId; })) return;
|
||||
group.results.push(result);
|
||||
if (!group.shop_name) group.shop_name = result.shop_name || '';
|
||||
if (!group.shop_id) group.shop_id = result.shop_id || '';
|
||||
if (!group.group_name) group.group_name = result.group_name || '';
|
||||
});
|
||||
});
|
||||
groups.forEach(function (group) {
|
||||
group.results.sort(shopDataResultSort);
|
||||
group.results = group.results.slice(0, 3);
|
||||
if (!group.latest_created_at && group.results.length) {
|
||||
group.latest_created_at = group.results[0].created_at || group.results[0].finished_at || '';
|
||||
}
|
||||
});
|
||||
return groups.filter(function (group) { return group.results.length > 0; });
|
||||
}
|
||||
|
||||
function shopDataDeleteIcon() {
|
||||
return '<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M3 6h18"></path><path d="M8 6V4h8v2"></path><path d="M19 6l-1 15H6L5 6"></path><path d="M10 11v6m4-6v6"></path></svg>';
|
||||
}
|
||||
|
||||
function renderShopDataStatus(item, status) {
|
||||
var normalized = String(status || '-').toUpperCase();
|
||||
var errorTitle = item && item.error ? ' title="' + escapeHtml(item.error) + '"' : '';
|
||||
return '<span class="image-video-status ' + escapeHtml(normalized) + '"' + errorTitle + '>' + escapeHtml(normalized) + '</span>';
|
||||
}
|
||||
|
||||
function renderShopDataTaskResult(item) {
|
||||
var resultId = shopDataResultId(item);
|
||||
var selected = resultId > 0 && selectedShopDataResultIds.has(resultId);
|
||||
var status = String(item.status || item.file_status || '').toUpperCase();
|
||||
var terminal = ['SUCCESS', 'FAILED', 'CANCELLED'].indexOf(status) >= 0;
|
||||
var countryCodes = item.country_codes != null ? item.country_codes : item.countryCodes;
|
||||
var countries = Array.isArray(countryCodes) ? countryCodes.join('、') : (String(countryCodes || '') || '-');
|
||||
var filename = item.output_filename || '-';
|
||||
var checkbox = '<input type="checkbox" data-shop-data-select="' + (resultId || '') + '"' +
|
||||
(selected ? ' checked' : '') + (item.file_ready && resultId > 0 ? '' : ' disabled') + '>';
|
||||
return '<div class="shop-data-result' + (selected ? ' selected' : '') + '" data-shop-data-card="' + (resultId || '') + '">' +
|
||||
'<div class="shop-data-result-head">' +
|
||||
'<label class="shop-data-task-title">' + checkbox +
|
||||
'<span title="任务 #' + escapeHtml(item.task_id || item.task_no || '-') + '">任务 #' + escapeHtml(item.task_id || item.task_no || '-') + '</span>' +
|
||||
'</label>' +
|
||||
renderShopDataStatus(item, status || item.file_status) +
|
||||
'</div>' +
|
||||
'<div class="image-video-card-info">' +
|
||||
'<div class="image-video-info-row"><label>国家</label><span>' + escapeHtml(countries) + '</span></div>' +
|
||||
'<div class="image-video-info-row"><label>文件</label><span title="' + escapeHtml(filename) + '">' + escapeHtml(filename) + '</span></div>' +
|
||||
'</div>' +
|
||||
'<div class="image-video-card-actions">' +
|
||||
'<button class="image-video-card-action" type="button" data-shop-data-download="' + (resultId || '') + '"' + (item.file_ready && resultId > 0 ? '' : ' disabled') + '>' + imageVideoDownloadIcon() + '下载文件</button>' +
|
||||
'<button class="image-video-card-action shop-data-delete-action" type="button" data-shop-data-delete="' + (resultId || '') + '"' + (terminal && resultId > 0 ? '' : ' disabled') + '>' + shopDataDeleteIcon() + '删除</button>' +
|
||||
'</div>' +
|
||||
'</div>';
|
||||
}
|
||||
|
||||
function renderShopDataTaskCard(group) {
|
||||
var results = Array.isArray(group.results) ? group.results : [];
|
||||
var latest = group.latest_created_at || (results[0] && (results[0].created_at || results[0].finished_at)) || '-';
|
||||
return '<article class="image-video-card shop-data-task-card" data-shop-data-group="' + escapeHtml(group.key || '') + '">' +
|
||||
'<div class="image-video-card-body">' +
|
||||
'<div class="image-video-card-head shop-data-group-head">' +
|
||||
'<div class="shop-data-task-title"><span title="' + escapeHtml(group.shop_name || '-') + '">' + escapeHtml(group.shop_name || '-') + '</span></div>' +
|
||||
'<span class="shop-data-group-meta">' + results.length + '/3 份结果</span>' +
|
||||
'</div>' +
|
||||
'<div class="image-video-card-info">' +
|
||||
'<div class="image-video-info-row"><label>分组</label><span>' + escapeHtml(group.group_name || '-') + '</span></div>' +
|
||||
'<div class="image-video-info-row"><label>最新</label><span>' + escapeHtml(latest) + '</span></div>' +
|
||||
'</div>' +
|
||||
'<div class="shop-data-result-list">' +
|
||||
(results.length ? results.map(renderShopDataTaskResult).join('') : '<div class="image-video-empty">暂无结果</div>') +
|
||||
'</div>' +
|
||||
'</div>' +
|
||||
'</article>';
|
||||
}
|
||||
|
||||
function syncShopDataSelectionUi() {
|
||||
document.querySelectorAll('[data-shop-data-card]').forEach(function (card) {
|
||||
var resultId = Number(card.dataset.shopDataCard);
|
||||
var selected = selectedShopDataResultIds.has(resultId);
|
||||
card.classList.toggle('selected', selected);
|
||||
var checkbox = card.querySelector('[data-shop-data-select]');
|
||||
if (checkbox) checkbox.checked = selected && !checkbox.disabled;
|
||||
});
|
||||
var selectable = shopDataTasks.filter(function (item) { return !!item.file_ready && shopDataResultId(item) > 0; });
|
||||
var selectedCount = selectable.filter(function (item) {
|
||||
return selectedShopDataResultIds.has(shopDataResultId(item));
|
||||
}).length;
|
||||
var selectAll = document.getElementById('shopDataTaskSelectAll');
|
||||
selectAll.checked = selectable.length > 0 && selectedCount === selectable.length;
|
||||
selectAll.indeterminate = selectedCount > 0 && selectedCount < selectable.length;
|
||||
selectAll.disabled = shopDataDownloadInProgress || selectable.length === 0;
|
||||
var batch = document.getElementById('btnBatchDownloadShopDataTasks');
|
||||
batch.disabled = shopDataDownloadInProgress || selectedCount === 0;
|
||||
batch.innerHTML = imageVideoDownloadIcon() + (shopDataDownloadInProgress
|
||||
? '处理中'
|
||||
: '批量下载' + (selectedCount ? ' (' + selectedCount + ')' : ''));
|
||||
}
|
||||
|
||||
function renderShopDataTasks() {
|
||||
var grid = document.getElementById('shopDataTaskGrid');
|
||||
grid.innerHTML = shopDataTaskGroups.length
|
||||
? shopDataTaskGroups.map(renderShopDataTaskCard).join('')
|
||||
: '<div class="image-video-empty">暂无符合条件的店铺数据任务</div>';
|
||||
syncShopDataSelectionUi();
|
||||
}
|
||||
|
||||
function loadShopDataCrawlTasks(page) {
|
||||
shopDataTaskPage = page || 1;
|
||||
selectedShopDataResultIds.clear();
|
||||
var grid = document.getElementById('shopDataTaskGrid');
|
||||
grid.innerHTML = '<div class="image-video-empty">加载中...</div>';
|
||||
document.getElementById('shopDataTaskDownloadProgress').textContent = '';
|
||||
fetch('/api/admin/shop-data-crawl-tasks?' + buildShopDataTaskQuery(shopDataTaskPage))
|
||||
.then(function (response) { return response.json(); })
|
||||
.then(function (res) {
|
||||
if (!res.success) throw new Error(res.error || '加载失败');
|
||||
var payload = res.data && typeof res.data === 'object' && !Array.isArray(res.data) ? res.data : res;
|
||||
shopDataTaskGroups = normalizeShopDataTaskGroups(payload.items || []);
|
||||
shopDataTasks = shopDataTaskGroups.reduce(function (all, group) {
|
||||
return all.concat(group.results || []);
|
||||
}, []);
|
||||
var total = payload.total != null ? Number(payload.total) : shopDataTaskGroups.length;
|
||||
var responsePage = payload.page || page;
|
||||
var responsePageSize = payload.page_size || shopDataTaskPageSize;
|
||||
document.getElementById('shopDataTaskTotal').textContent = '共 ' + (total || 0) + ' 家店铺 · 每家店铺保留最新 3 份任务结果';
|
||||
renderShopDataTasks();
|
||||
renderPagination('shopDataTaskPagination', total, responsePage, responsePageSize, loadShopDataCrawlTasks);
|
||||
})
|
||||
.catch(function (error) {
|
||||
shopDataTaskGroups = [];
|
||||
shopDataTasks = [];
|
||||
grid.innerHTML = '<div class="image-video-empty">加载失败:' + escapeHtml(error.message || '') + '</div>';
|
||||
document.getElementById('shopDataTaskTotal').textContent = '';
|
||||
syncShopDataSelectionUi();
|
||||
});
|
||||
}
|
||||
|
||||
function downloadShopDataTask(item) {
|
||||
var resultId = shopDataResultId(item);
|
||||
if (!item || !item.file_ready || !resultId) return;
|
||||
var filename = item.output_filename || ('shop-data-task-' + resultId + '.xlsx');
|
||||
triggerImageVideoLink('/api/admin/shop-data-crawl-tasks/' + resultId + '/download', filename, false);
|
||||
}
|
||||
|
||||
function deleteShopDataTask(item) {
|
||||
var resultId = shopDataResultId(item);
|
||||
if (!item || !resultId || ['SUCCESS', 'FAILED', 'CANCELLED'].indexOf(String(item.status || item.file_status || '').toUpperCase()) < 0) return;
|
||||
if (!window.confirm('确认删除店铺“' + (item.shop_name || '-') + '”的任务 #' + item.task_id + ' 及结果文件?')) return;
|
||||
var progress = document.getElementById('shopDataTaskDownloadProgress');
|
||||
progress.textContent = '正在删除任务 #' + item.task_id + '...';
|
||||
fetch('/api/admin/shop-data-crawl-tasks/' + resultId, { method: 'DELETE' })
|
||||
.then(function (response) { return response.json().then(function (data) { return { ok: response.ok, data: data }; }); })
|
||||
.then(function (result) {
|
||||
if (!result.ok || !result.data.success) throw new Error(result.data.error || '删除失败');
|
||||
progress.textContent = result.data.msg || '删除成功';
|
||||
loadShopDataCrawlTasks(shopDataTaskPage);
|
||||
})
|
||||
.catch(function (error) {
|
||||
progress.textContent = error.message || '删除失败';
|
||||
});
|
||||
}
|
||||
|
||||
function downloadShopDataTasksZip() {
|
||||
var resultIds = Array.from(selectedShopDataResultIds);
|
||||
if (shopDataDownloadInProgress || !resultIds.length) return;
|
||||
shopDataDownloadInProgress = true;
|
||||
syncShopDataSelectionUi();
|
||||
var progress = document.getElementById('shopDataTaskDownloadProgress');
|
||||
progress.textContent = '正在打包 ' + resultIds.length + ' 个文件...';
|
||||
fetch('/api/admin/shop-data-crawl-tasks/download-zip', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ result_ids: resultIds }),
|
||||
__skipLoading: true
|
||||
}).then(function (response) {
|
||||
if (!response.ok) {
|
||||
return response.json().catch(function () { return {}; }).then(function (data) {
|
||||
throw new Error(data.error || '压缩包生成失败');
|
||||
});
|
||||
}
|
||||
var filename = imageVideoZipFilename(response);
|
||||
var errorCount = Number(response.headers.get('X-Archive-Error-Count') || 0);
|
||||
return response.blob().then(function (blob) {
|
||||
return { blob: blob, filename: filename, errorCount: errorCount };
|
||||
});
|
||||
}).then(function (result) {
|
||||
var objectUrl = URL.createObjectURL(result.blob);
|
||||
triggerImageVideoLink(objectUrl, result.filename, false);
|
||||
setTimeout(function () { URL.revokeObjectURL(objectUrl); }, 1000);
|
||||
progress.textContent = result.errorCount
|
||||
? '压缩包已下载,' + result.errorCount + ' 个文件失败,详见包内清单'
|
||||
: '压缩包下载已开始';
|
||||
}).catch(function (error) {
|
||||
progress.textContent = error.message || '批量下载失败';
|
||||
}).finally(function () {
|
||||
shopDataDownloadInProgress = false;
|
||||
syncShopDataSelectionUi();
|
||||
});
|
||||
}
|
||||
|
||||
function shopDataPermissionUsersForView() {
|
||||
return shopDataPermissionView === 'granted'
|
||||
? shopDataPermissionUsers.filter(function (user) {
|
||||
return shopDataPermissionInitialUserIds.has(Number(user.id));
|
||||
})
|
||||
: shopDataPermissionUsers;
|
||||
}
|
||||
|
||||
function filteredShopDataPermissionUsers() {
|
||||
var users = shopDataPermissionUsersForView();
|
||||
var keyword = (document.getElementById('shopDataTaskPermissionSearch').value || '').trim().toLowerCase();
|
||||
return keyword ? users.filter(function (user) {
|
||||
return String(user.username || '').toLowerCase().indexOf(keyword) >= 0;
|
||||
}) : users;
|
||||
}
|
||||
|
||||
function renderShopDataPermissionUsers() {
|
||||
var visibleUsers = filteredShopDataPermissionUsers();
|
||||
document.getElementById('shopDataTaskPermissionGrantedCount').textContent = '(' + shopDataPermissionInitialUserIds.size + ')';
|
||||
document.getElementById('shopDataTaskPermissionAllCount').textContent = '(' + shopDataPermissionUsers.length + ')';
|
||||
document.querySelectorAll('[data-shop-data-permission-view]').forEach(function (tab) {
|
||||
var active = tab.dataset.shopDataPermissionView === shopDataPermissionView;
|
||||
tab.classList.toggle('active', active);
|
||||
tab.setAttribute('aria-selected', active ? 'true' : 'false');
|
||||
});
|
||||
var pendingCount = shopDataPermissionUsers.filter(function (user) {
|
||||
var userId = Number(user.id);
|
||||
return shopDataPermissionInitialUserIds.has(userId) !== selectedShopDataPermissionUserIds.has(userId);
|
||||
}).length;
|
||||
document.getElementById('shopDataTaskPermissionSummary').textContent =
|
||||
(shopDataPermissionView === 'granted' ? '当前显示已分配用户,共 ' + visibleUsers.length + ' 人' : '当前显示全部用户,已分配 ' + shopDataPermissionInitialUserIds.size + ' 人') +
|
||||
(pendingCount ? ' · 待保存变更 ' + pendingCount + ' 项' : '');
|
||||
document.getElementById('shopDataTaskPermissionList').innerHTML = visibleUsers.length
|
||||
? visibleUsers.map(function (user) {
|
||||
var userId = Number(user.id);
|
||||
var saved = shopDataPermissionInitialUserIds.has(userId);
|
||||
var selected = selectedShopDataPermissionUserIds.has(userId);
|
||||
var changed = saved !== selected;
|
||||
return '<div class="image-video-permission-row">' +
|
||||
'<input type="checkbox" data-shop-data-permission-user="' + userId + '"' + (selected ? ' checked' : '') + '>' +
|
||||
'<span class="image-video-permission-user"><span class="image-video-permission-name">' + escapeHtml(user.username || '-') + '</span><span class="image-video-permission-role">' + escapeHtml(roleLabel(user.role)) + '</span></span>' +
|
||||
'<button class="image-video-permission-state' + (changed ? ' pending' : (saved ? ' granted' : '')) + '" type="button" data-shop-data-permission-toggle="' + userId + '">' +
|
||||
(changed ? (selected ? '待保存分配' : '待保存取消') : (saved ? '取消分配' : '分配')) +
|
||||
'</button></div>';
|
||||
}).join('')
|
||||
: '<div class="image-video-permission-empty">暂无匹配用户</div>';
|
||||
var selectedVisibleCount = visibleUsers.filter(function (user) {
|
||||
return selectedShopDataPermissionUserIds.has(Number(user.id));
|
||||
}).length;
|
||||
var selectAll = document.getElementById('shopDataTaskPermissionSelectAll');
|
||||
selectAll.checked = visibleUsers.length > 0 && selectedVisibleCount === visibleUsers.length;
|
||||
selectAll.indeterminate = selectedVisibleCount > 0 && selectedVisibleCount < visibleUsers.length;
|
||||
selectAll.disabled = visibleUsers.length === 0;
|
||||
}
|
||||
|
||||
function openShopDataTaskPermissions() {
|
||||
if (currentUserRole !== 'super_admin') return;
|
||||
var modal = document.getElementById('shopDataTaskPermissionModal');
|
||||
var saveButton = document.getElementById('btnSaveShopDataTaskPermissions');
|
||||
var permissionLoaded = false;
|
||||
modal.classList.add('show');
|
||||
shopDataPermissionView = 'granted';
|
||||
document.getElementById('shopDataTaskPermissionSearch').value = '';
|
||||
document.getElementById('shopDataTaskPermissionMessage').textContent = '';
|
||||
document.getElementById('shopDataTaskPermissionList').innerHTML = '<div class="image-video-permission-empty">加载中...</div>';
|
||||
saveButton.disabled = true;
|
||||
fetch('/api/admin/shop-data-crawl-task-permissions')
|
||||
.then(function (response) { return response.json(); })
|
||||
.then(function (res) {
|
||||
if (!res.success) throw new Error(res.error || '权限加载失败');
|
||||
shopDataPermissionUsers = res.items || [];
|
||||
shopDataPermissionInitialUserIds = new Set(shopDataPermissionUsers.filter(function (user) { return !!user.granted; }).map(function (user) { return Number(user.id); }));
|
||||
selectedShopDataPermissionUserIds = new Set(shopDataPermissionInitialUserIds);
|
||||
permissionLoaded = true;
|
||||
renderShopDataPermissionUsers();
|
||||
}).catch(function (error) {
|
||||
shopDataPermissionUsers = [];
|
||||
shopDataPermissionInitialUserIds = new Set();
|
||||
selectedShopDataPermissionUserIds = new Set();
|
||||
document.getElementById('shopDataTaskPermissionList').innerHTML = '<div class="image-video-permission-empty">' + escapeHtml(error.message || '权限加载失败') + '</div>';
|
||||
document.getElementById('shopDataTaskPermissionMessage').textContent = '权限加载失败,请关闭后重试';
|
||||
}).finally(function () {
|
||||
saveButton.disabled = !permissionLoaded;
|
||||
});
|
||||
}
|
||||
|
||||
function closeShopDataTaskPermissions() {
|
||||
document.getElementById('shopDataTaskPermissionModal').classList.remove('show');
|
||||
}
|
||||
|
||||
function saveShopDataTaskPermissions() {
|
||||
var message = document.getElementById('shopDataTaskPermissionMessage');
|
||||
message.textContent = '保存中...';
|
||||
message.className = 'msg';
|
||||
document.getElementById('btnSaveShopDataTaskPermissions').disabled = true;
|
||||
fetch('/api/admin/shop-data-crawl-task-permissions', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ user_ids: Array.from(selectedShopDataPermissionUserIds).sort(function (a, b) { return a - b; }) })
|
||||
}).then(function (response) { return response.json(); })
|
||||
.then(function (res) {
|
||||
if (!res.success) throw new Error(res.error || '权限保存失败');
|
||||
shopDataPermissionInitialUserIds = new Set(selectedShopDataPermissionUserIds);
|
||||
renderShopDataPermissionUsers();
|
||||
message.textContent = res.msg || '保存成功';
|
||||
message.className = 'msg ok';
|
||||
}).catch(function (error) {
|
||||
message.textContent = error.message || '权限保存失败';
|
||||
message.className = 'msg err';
|
||||
}).finally(function () {
|
||||
document.getElementById('btnSaveShopDataTaskPermissions').disabled = false;
|
||||
});
|
||||
}
|
||||
|
||||
document.getElementById('btnFilterShopDataTasks').onclick = function () { loadShopDataCrawlTasks(1); };
|
||||
document.getElementById('btnResetShopDataTasks').onclick = function () {
|
||||
['shopDataTaskFilterShop', 'shopDataTaskFilterGroup', 'shopDataTaskFilterFrom', 'shopDataTaskFilterTo']
|
||||
.forEach(function (id) { document.getElementById(id).value = ''; });
|
||||
loadShopDataCrawlTasks(1);
|
||||
};
|
||||
document.getElementById('shopDataTaskSelectAll').onchange = function (event) {
|
||||
shopDataTasks.forEach(function (item) {
|
||||
var resultId = shopDataResultId(item);
|
||||
if (!item.file_ready || !resultId) return;
|
||||
if (event.target.checked) selectedShopDataResultIds.add(resultId);
|
||||
else selectedShopDataResultIds.delete(resultId);
|
||||
});
|
||||
syncShopDataSelectionUi();
|
||||
};
|
||||
document.getElementById('btnBatchDownloadShopDataTasks').onclick = downloadShopDataTasksZip;
|
||||
document.getElementById('shopDataTaskGrid').onchange = function (event) {
|
||||
var checkbox = event.target.closest('[data-shop-data-select]');
|
||||
if (!checkbox) return;
|
||||
var resultId = Number(checkbox.dataset.shopDataSelect);
|
||||
if (!resultId) return;
|
||||
if (checkbox.checked) selectedShopDataResultIds.add(resultId);
|
||||
else selectedShopDataResultIds.delete(resultId);
|
||||
syncShopDataSelectionUi();
|
||||
};
|
||||
document.getElementById('shopDataTaskGrid').onclick = function (event) {
|
||||
var downloadButton = event.target.closest('[data-shop-data-download]');
|
||||
if (downloadButton) {
|
||||
var downloadItem = shopDataTasks.find(function (task) { return shopDataResultId(task) === Number(downloadButton.dataset.shopDataDownload); });
|
||||
downloadShopDataTask(downloadItem);
|
||||
return;
|
||||
}
|
||||
var deleteButton = event.target.closest('[data-shop-data-delete]');
|
||||
if (deleteButton) {
|
||||
var deleteItem = shopDataTasks.find(function (task) { return shopDataResultId(task) === Number(deleteButton.dataset.shopDataDelete); });
|
||||
deleteShopDataTask(deleteItem);
|
||||
}
|
||||
};
|
||||
document.getElementById('btnOpenShopDataTaskPermissions').onclick = openShopDataTaskPermissions;
|
||||
document.getElementById('btnCloseShopDataTaskPermissions').onclick = closeShopDataTaskPermissions;
|
||||
document.getElementById('btnCancelShopDataTaskPermissions').onclick = closeShopDataTaskPermissions;
|
||||
document.getElementById('btnSaveShopDataTaskPermissions').onclick = saveShopDataTaskPermissions;
|
||||
document.querySelectorAll('[data-shop-data-permission-view]').forEach(function (tab) {
|
||||
tab.onclick = function () {
|
||||
shopDataPermissionView = tab.dataset.shopDataPermissionView || 'granted';
|
||||
document.getElementById('shopDataTaskPermissionSearch').value = '';
|
||||
renderShopDataPermissionUsers();
|
||||
};
|
||||
});
|
||||
document.getElementById('shopDataTaskPermissionSearch').oninput = renderShopDataPermissionUsers;
|
||||
document.getElementById('shopDataTaskPermissionSelectAll').onchange = function (event) {
|
||||
filteredShopDataPermissionUsers().forEach(function (user) {
|
||||
if (event.target.checked) selectedShopDataPermissionUserIds.add(Number(user.id));
|
||||
else selectedShopDataPermissionUserIds.delete(Number(user.id));
|
||||
});
|
||||
renderShopDataPermissionUsers();
|
||||
};
|
||||
document.getElementById('shopDataTaskPermissionList').onclick = function (event) {
|
||||
var button = event.target.closest('[data-shop-data-permission-toggle]');
|
||||
if (!button) return;
|
||||
var userId = Number(button.dataset.shopDataPermissionToggle);
|
||||
if (selectedShopDataPermissionUserIds.has(userId)) selectedShopDataPermissionUserIds.delete(userId);
|
||||
else selectedShopDataPermissionUserIds.add(userId);
|
||||
renderShopDataPermissionUsers();
|
||||
};
|
||||
document.getElementById('shopDataTaskPermissionList').onchange = function (event) {
|
||||
var checkbox = event.target.closest('[data-shop-data-permission-user]');
|
||||
if (!checkbox) return;
|
||||
var userId = Number(checkbox.dataset.shopDataPermissionUser);
|
||||
if (checkbox.checked) selectedShopDataPermissionUserIds.add(userId);
|
||||
else selectedShopDataPermissionUserIds.delete(userId);
|
||||
renderShopDataPermissionUsers();
|
||||
};
|
||||
document.getElementById('shopDataTaskPermissionModal').onclick = function (event) {
|
||||
if (event.target === event.currentTarget) closeShopDataTaskPermissions();
|
||||
};
|
||||
|
||||
var historyPage = 1, historyPageSize = 15;
|
||||
function toSqlDatetime(val) {
|
||||
if (!val) return '';
|
||||
@@ -1636,15 +2158,33 @@
|
||||
|
||||
// ========== 数据去重总数据 ==========
|
||||
var dedupeTotalDataPage = 1, dedupeTotalDataPageSize = 15;
|
||||
function getDedupeTotalDataDateRange() {
|
||||
return {
|
||||
startDate: document.getElementById('exportDedupeTotalDataStartDate').value || '',
|
||||
endDate: document.getElementById('exportDedupeTotalDataEndDate').value || ''
|
||||
};
|
||||
}
|
||||
function validateDedupeTotalDataDateRange() {
|
||||
var dateRange = getDedupeTotalDataDateRange();
|
||||
if (dateRange.startDate && dateRange.endDate && dateRange.startDate > dateRange.endDate) {
|
||||
alert('开始日期不能晚于结束日期');
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
function buildDedupeTotalDataQuery(page) {
|
||||
var q = 'page=' + (page || 1) + '&page_size=' + dedupeTotalDataPageSize;
|
||||
var keyword = (document.getElementById('searchDedupeTotalData').value || '').trim();
|
||||
var username = (document.getElementById('searchDedupeTotalDataUsername').value || '').trim();
|
||||
var dateRange = getDedupeTotalDataDateRange();
|
||||
if (keyword) q += '&keyword=' + encodeURIComponent(keyword);
|
||||
if (username) q += '&username=' + encodeURIComponent(username);
|
||||
if (dateRange.startDate) q += '&start_date=' + encodeURIComponent(dateRange.startDate);
|
||||
if (dateRange.endDate) q += '&end_date=' + encodeURIComponent(dateRange.endDate);
|
||||
return q;
|
||||
}
|
||||
function loadDedupeTotalData(page) {
|
||||
if (!validateDedupeTotalDataDateRange()) return;
|
||||
dedupeTotalDataPage = page || 1;
|
||||
fetch('/api/admin/dedupe-total-data?' + buildDedupeTotalDataQuery(dedupeTotalDataPage))
|
||||
.then(function (r) { return r.json(); })
|
||||
@@ -1698,16 +2238,12 @@
|
||||
document.getElementById('btnSearchDedupeTotalData').onclick = function () { loadDedupeTotalData(1); };
|
||||
document.getElementById('btnExportDedupeTotalData').onclick = function () {
|
||||
var username = (document.getElementById('searchDedupeTotalDataUsername').value || '').trim();
|
||||
var startDate = document.getElementById('exportDedupeTotalDataStartDate').value || '';
|
||||
var endDate = document.getElementById('exportDedupeTotalDataEndDate').value || '';
|
||||
if (startDate && endDate && startDate > endDate) {
|
||||
alert('开始日期不能晚于结束日期');
|
||||
return;
|
||||
}
|
||||
var dateRange = getDedupeTotalDataDateRange();
|
||||
if (!validateDedupeTotalDataDateRange()) return;
|
||||
var params = [];
|
||||
if (username) params.push('username=' + encodeURIComponent(username));
|
||||
if (startDate) params.push('start_date=' + encodeURIComponent(startDate));
|
||||
if (endDate) params.push('end_date=' + encodeURIComponent(endDate));
|
||||
if (dateRange.startDate) params.push('start_date=' + encodeURIComponent(dateRange.startDate));
|
||||
if (dateRange.endDate) params.push('end_date=' + encodeURIComponent(dateRange.endDate));
|
||||
fetch('/api/admin/dedupe-total-data/export' + (params.length ? ('?' + params.join('&')) : ''))
|
||||
.then(function (response) {
|
||||
var contentType = response.headers.get('content-type') || '';
|
||||
@@ -2112,6 +2648,20 @@
|
||||
function buildShopKeyQuery(page) {
|
||||
return 'page=' + (page || 1) + '&page_size=' + shopKeyPageSize;
|
||||
}
|
||||
function renderShopKeyWhitelistStatus(item) {
|
||||
var status = String(item.ip_whitelist_status || 'UNKNOWN').toUpperCase();
|
||||
var statusMeta = {
|
||||
ALLOWED: { label: '正常', className: 'is-allowed' },
|
||||
BLOCKED: { label: '未加白名单', className: 'is-blocked' },
|
||||
UNKNOWN: { label: '未检测', className: 'is-unknown' }
|
||||
};
|
||||
var meta = statusMeta[status] || statusMeta.UNKNOWN;
|
||||
var details = [];
|
||||
if (item.ip_whitelist_checked_at) details.push('检测时间:' + item.ip_whitelist_checked_at);
|
||||
if (item.ip_whitelist_message) details.push(item.ip_whitelist_message);
|
||||
return '<span class="shop-key-whitelist-status ' + meta.className + '" title="' +
|
||||
escapeHtml(details.join('\n')) + '">' + escapeHtml(meta.label) + '</span>';
|
||||
}
|
||||
function loadShopKeys(page) {
|
||||
shopKeyPage = page || 1;
|
||||
fetch('/api/admin/shop-keys?' + buildShopKeyQuery(shopKeyPage))
|
||||
@@ -2124,11 +2674,11 @@
|
||||
}
|
||||
var items = res.items || [];
|
||||
if (items.length === 0) {
|
||||
tbody.innerHTML = '<tr><td colspan="7" class="empty-tip">暂无店铺密钥</td></tr>';
|
||||
tbody.innerHTML = '<tr><td colspan="8" class="empty-tip">暂无店铺密钥</td></tr>';
|
||||
} else {
|
||||
tbody.innerHTML = items.map(function (item, index) {
|
||||
var rowNo = (shopKeyPage - 1) * shopKeyPageSize + index + 1;
|
||||
return '<tr><td>' + rowNo + '</td><td>' + (item.remark_name || '') + '</td><td>' + (item.ziniao_account_name || '') + '</td><td>' + (item.ziniao_token || '') + '</td><td>' + (item.created_at || '') + '</td><td>' + (item.updated_at || '') + '</td><td>' +
|
||||
return '<tr><td>' + rowNo + '</td><td>' + escapeHtml(item.remark_name || '') + '</td><td>' + escapeHtml(item.ziniao_account_name || '') + '</td><td>' + escapeHtml(item.ziniao_token || '') + '</td><td>' + renderShopKeyWhitelistStatus(item) + '</td><td>' + escapeHtml(item.created_at || '') + '</td><td>' + escapeHtml(item.updated_at || '') + '</td><td>' +
|
||||
'<button class="btn btn-sm" data-shop-key-edit="' + item.id + '" data-shop-key="' + (JSON.stringify(item).replace(/"/g, '"')) + '">编辑</button> ' +
|
||||
'<button class="btn btn-sm btn-danger" data-shop-key-delete="' + item.id + '" data-ziniao-account-name="' + (item.ziniao_account_name || '').replace(/"/g, '"') + '">删除</button>' +
|
||||
'</td></tr>';
|
||||
@@ -2138,7 +2688,7 @@
|
||||
bindShopKeyActions();
|
||||
})
|
||||
.catch(function () {
|
||||
document.getElementById('shopKeyListBody').innerHTML = '<tr><td colspan="7" class="empty-tip">请求失败</td></tr>';
|
||||
document.getElementById('shopKeyListBody').innerHTML = '<tr><td colspan="8" class="empty-tip">请求失败</td></tr>';
|
||||
});
|
||||
}
|
||||
function bindShopKeyActions() {
|
||||
@@ -2264,6 +2814,21 @@
|
||||
return query;
|
||||
}
|
||||
|
||||
function shopPasswordIcon(revealed) {
|
||||
if (revealed) {
|
||||
return '<svg viewBox="0 0 24 24" aria-hidden="true"><path d="m2 2 20 20"></path><path d="M6.71 6.71C4.7 8.1 3.34 10.08 2 12c2.12 3.04 5.5 6 10 6 1.67 0 3.17-.41 4.47-1.05"></path><path d="M10.73 5.08A9.36 9.36 0 0 1 12 5c4.5 0 7.88 2.96 10 7a15.82 15.82 0 0 1-2.12 2.91"></path><path d="M14.12 14.12A3 3 0 0 1 9.88 9.88"></path></svg>';
|
||||
}
|
||||
return '<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M2.06 12.35a1 1 0 0 1 0-.7C3.54 8.04 7.06 5.5 12 5.5s8.46 2.54 9.94 6.15a1 1 0 0 1 0 .7C20.46 15.96 16.94 18.5 12 18.5S3.54 15.96 2.06 12.35"></path><circle cx="12" cy="12" r="3"></circle></svg>';
|
||||
}
|
||||
|
||||
function renderShopPasswordCell(item) {
|
||||
var maskedPassword = item.password || '******';
|
||||
return '<span class="shop-password-cell">' +
|
||||
'<span class="shop-password-value" data-shop-password-value>' + escapeHtml(maskedPassword) + '</span>' +
|
||||
'<button type="button" class="shop-password-toggle" data-shop-password-toggle="' + escapeHtml(item.id) + '" data-shop-name="' + escapeHtml(item.shop_name || '') + '" data-masked-password="' + escapeHtml(maskedPassword) + '" aria-label="显示密码" aria-pressed="false" title="显示密码">' +
|
||||
shopPasswordIcon(false) + '</button></span>';
|
||||
}
|
||||
|
||||
function loadShopManage(page) {
|
||||
shopManagePage = page || 1;
|
||||
fetch('/api/admin/shop-manages?' + buildShopManageQuery(shopManagePage))
|
||||
@@ -2280,7 +2845,7 @@
|
||||
} else {
|
||||
tbody.innerHTML = items.map(function (item, index) {
|
||||
var rowNo = (shopManagePage - 1) * shopManagePageSize + index + 1;
|
||||
return '<tr><td>' + rowNo + '</td><td>' + (item.group_name || '') + '</td><td>' + (item.shop_name || '') + '</td><td>' + (item.mall_name || '') + '</td><td>' + (item.account || '') + '</td><td>' + (item.password || '') + '</td><td>' + (item.created_at || '') + '</td><td>' + (item.updated_at || '') + '</td><td>' +
|
||||
return '<tr><td>' + rowNo + '</td><td>' + (item.group_name || '') + '</td><td>' + (item.shop_name || '') + '</td><td>' + (item.mall_name || '') + '</td><td>' + (item.account || '') + '</td><td>' + renderShopPasswordCell(item) + '</td><td>' + (item.created_at || '') + '</td><td>' + (item.updated_at || '') + '</td><td>' +
|
||||
'<button class="btn btn-sm" data-shop-manage-edit="' + item.id + '" data-shop-manage="' + (JSON.stringify(item).replace(/"/g, '"')) + '">编辑</button> ' +
|
||||
'<button class="btn btn-sm btn-danger" data-shop-manage-delete="' + item.id + '" data-shop-manage-name="' + (item.shop_name || '').replace(/"/g, '"') + '">删除</button>' +
|
||||
'</td></tr>';
|
||||
@@ -2295,6 +2860,42 @@
|
||||
}
|
||||
|
||||
function bindShopManageActions() {
|
||||
document.querySelectorAll('[data-shop-password-toggle]').forEach(function (btn) {
|
||||
btn.onclick = function () {
|
||||
var valueEl = btn.parentElement.querySelector('[data-shop-password-value]');
|
||||
var revealed = btn.dataset.revealed === 'true';
|
||||
if (revealed) {
|
||||
valueEl.textContent = btn.dataset.maskedPassword || '******';
|
||||
btn.dataset.revealed = 'false';
|
||||
btn.setAttribute('aria-label', '显示密码');
|
||||
btn.setAttribute('aria-pressed', 'false');
|
||||
btn.title = '显示密码';
|
||||
btn.innerHTML = shopPasswordIcon(false);
|
||||
return;
|
||||
}
|
||||
|
||||
btn.disabled = true;
|
||||
valueEl.textContent = '读取中...';
|
||||
fetch('/api/admin/shop-manage/' + encodeURIComponent(btn.dataset.shopPasswordToggle) + '/credential?shop_name=' + encodeURIComponent(btn.dataset.shopName || ''))
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (res) {
|
||||
if (!res.success) throw new Error(res.error || '读取密码失败');
|
||||
valueEl.textContent = res.password || '';
|
||||
btn.dataset.revealed = 'true';
|
||||
btn.setAttribute('aria-label', '隐藏密码');
|
||||
btn.setAttribute('aria-pressed', 'true');
|
||||
btn.title = '隐藏密码';
|
||||
btn.innerHTML = shopPasswordIcon(true);
|
||||
})
|
||||
.catch(function (error) {
|
||||
valueEl.textContent = btn.dataset.maskedPassword || '******';
|
||||
alert(error.message || '读取密码失败');
|
||||
})
|
||||
.finally(function () {
|
||||
btn.disabled = false;
|
||||
});
|
||||
};
|
||||
});
|
||||
document.querySelectorAll('[data-shop-manage-edit]').forEach(function (btn) {
|
||||
btn.onclick = function () {
|
||||
var item = {};
|
||||
@@ -4472,6 +5073,7 @@
|
||||
html += '<span class="page-item' + active + '" onclick="loadDigitalHumanVersions(' + i + ')">' + i + '</span>';
|
||||
}
|
||||
pagination.innerHTML = html;
|
||||
appendPaginationQuickJump(pagination, pages, current, loadDigitalHumanVersions);
|
||||
} else {
|
||||
pagination.innerHTML = '';
|
||||
}
|
||||
@@ -4918,6 +5520,27 @@
|
||||
if (document.getElementById('editColumnMenuType')) document.getElementById('editColumnMenuType').onchange = populateColumnParentSelects;
|
||||
|
||||
// ========== 分页 ==========
|
||||
function appendPaginationQuickJump(el, totalPages, page, onPage) {
|
||||
el.insertAdjacentHTML('beforeend',
|
||||
'<label class="pagination-jump">跳至<input type="number" min="1" max="' + totalPages + '" step="1" inputmode="numeric" aria-label="跳转页码" data-page-jump-input>页</label>' +
|
||||
'<button type="button" data-page-jump>跳转</button>');
|
||||
var jumpInput = el.querySelector('[data-page-jump-input]');
|
||||
var jumpToPage = function () {
|
||||
var targetPage = parseInt(jumpInput.value, 10);
|
||||
if (isNaN(targetPage)) {
|
||||
jumpInput.focus();
|
||||
return;
|
||||
}
|
||||
targetPage = Math.min(Math.max(targetPage, 1), totalPages);
|
||||
jumpInput.value = targetPage;
|
||||
if (targetPage !== page) onPage(targetPage);
|
||||
};
|
||||
el.querySelector('[data-page-jump]').onclick = jumpToPage;
|
||||
jumpInput.onkeydown = function (event) {
|
||||
if (event.key === 'Enter') jumpToPage();
|
||||
};
|
||||
}
|
||||
|
||||
function renderPagination(elId, total, page, pageSize, onPage) {
|
||||
var el = document.getElementById(elId);
|
||||
if (!el) return;
|
||||
@@ -4929,6 +5552,7 @@
|
||||
el.querySelectorAll('[data-p]').forEach(function (b) {
|
||||
if (!b.disabled) b.onclick = function () { onPage(parseInt(b.dataset.p, 10)); };
|
||||
});
|
||||
appendPaginationQuickJump(el, totalPages, page, onPage);
|
||||
}
|
||||
|
||||
// 初始化
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user