新需求更新 同步更新

This commit is contained in:
supernijia
2026-08-06 01:11:54 +08:00
parent 28423e015b
commit 40f4fae272
112 changed files with 7739 additions and 637 deletions
@@ -47,6 +47,14 @@ public class SimilarAsinCozeClient {
}
public List<SimilarAsinResultRowDto> inspect(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
return inspect(rows, prompt, apiKey, imgSwitch, false);
}
public List<SimilarAsinResultRowDto> inspect(List<SimilarAsinResultRowDto> rows,
String prompt,
String apiKey,
boolean imgSwitch,
boolean categorySwitch) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
@@ -55,7 +63,7 @@ public class SimilarAsinCozeClient {
return rows.stream().map(this::copy).toList();
}
try {
return inspectWithFallback(rows, prompt, apiKey, imgSwitch);
return inspectWithFallback(rows, prompt, apiKey, imgSwitch, categorySwitch);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[similar-asin] coze batch failed size={} err={}", rows.size(), failureMessage);
@@ -75,9 +83,10 @@ public class SimilarAsinCozeClient {
String prompt,
String apiKey,
boolean imgSwitch,
boolean categorySwitch,
CozeCredentialRef credential) throws Exception {
CozeCredentialRef resolvedCredential = resolveCredential(credential);
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, resolvedCredential));
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, resolvedCredential));
ensureSuccess(submitRoot);
return new CozeSubmitResponse(
extractExecuteId(submitRoot),
@@ -87,6 +96,14 @@ public class SimilarAsinCozeClient {
);
}
public CozeSubmitResponse submitWorkflow(List<SimilarAsinResultRowDto> rows,
String prompt,
String apiKey,
boolean imgSwitch,
CozeCredentialRef credential) throws Exception {
return submitWorkflow(rows, prompt, apiKey, imgSwitch, false, credential);
}
public CozePollResponse pollWorkflow(String executeId) throws Exception {
return pollWorkflow(executeId, null);
}
@@ -119,12 +136,12 @@ public class SimilarAsinCozeClient {
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
private List<SimilarAsinResultRowDto> inspectWithFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
private List<SimilarAsinResultRowDto> inspectWithFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) {
try {
if (rows.size() == 1) {
return inspectSingleRowWithRetry(rows, prompt, apiKey, imgSwitch);
return inspectSingleRowWithRetry(rows, prompt, apiKey, imgSwitch, categorySwitch);
}
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch);
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch, categorySwitch);
if (attempt.resolvedCount() < rows.size()) {
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
}
@@ -135,17 +152,17 @@ public class SimilarAsinCozeClient {
log.warn("[similar-asin] coze batch fallback split size={} left={} right={} err={}",
rows.size(), middle, rows.size() - middle, failureMessage(ex));
List<SimilarAsinResultRowDto> merged = new ArrayList<>(rows.size());
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, imgSwitch));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, imgSwitch));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, imgSwitch, categorySwitch));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, imgSwitch, categorySwitch));
return merged;
}
throw propagate(ex);
}
}
private List<SimilarAsinResultRowDto> inspectPartitionWithFailureFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
private List<SimilarAsinResultRowDto> inspectPartitionWithFailureFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) {
try {
return inspectWithFallback(rows, prompt, apiKey, imgSwitch);
return inspectWithFallback(rows, prompt, apiKey, imgSwitch, categorySwitch);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[similar-asin] coze partition failed size={} err={}", rows.size(), failureMessage);
@@ -153,12 +170,12 @@ public class SimilarAsinCozeClient {
}
}
private List<SimilarAsinResultRowDto> inspectSingleRowWithRetry(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
private List<SimilarAsinResultRowDto> inspectSingleRowWithRetry(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
SimilarAsinResultRowDto row = rows.getFirst();
PartialCozeResultException lastFailure = null;
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
try {
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch);
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch, categorySwitch);
if (attempt.resolvedCount() == rows.size()) {
return attempt.mergedRows();
}
@@ -189,16 +206,16 @@ public class SimilarAsinCozeClient {
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
}
private InspectAttempt inspectOnce(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, imgSwitch);
private InspectAttempt inspectOnce(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, imgSwitch, categorySwitch);
List<CozeResult> results = parseResults(raw);
List<SimilarAsinResultRowDto> merged = mergeRows(rows, results);
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
}
private String runWorkflowAsyncAndWait(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
private String runWorkflowAsyncAndWait(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
CozeCredentialRef credential = nextCredential();
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, credential));
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, credential));
ensureSuccess(submitRoot);
String immediateData = extractResultDataText(submitRoot);
@@ -243,8 +260,9 @@ public class SimilarAsinCozeClient {
String prompt,
String apiKey,
boolean imgSwitch,
boolean categorySwitch,
CozeCredentialRef credential) {
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey, imgSwitch);
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey, imgSwitch, categorySwitch);
Map<String, Object> body = new LinkedHashMap<>();
body.put("workflow_id", credential.workflowId());
body.put("parameters", parameters);
@@ -366,6 +384,14 @@ public class SimilarAsinCozeClient {
}
private Map<String, Object> buildParameters(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
return buildParameters(rows, prompt, apiKey, imgSwitch, false);
}
private Map<String, Object> buildParameters(List<SimilarAsinResultRowDto> rows,
String prompt,
String apiKey,
boolean imgSwitch,
boolean categorySwitch) {
List<String> asins = rows.stream().map(row -> safeText(row.getAsin())).toList();
List<String> titles = rows.stream().map(row -> safeText(firstNonBlank(row.getTitle(), row.getAsin()))).toList();
List<String> skus = rows.stream().map(row -> safeText(row.getSku())).toList();
@@ -388,6 +414,7 @@ public class SimilarAsinCozeClient {
parameters.put("api_key", apiKey.trim());
}
parameters.put("img_switch", imgSwitch);
parameters.put("category_switch", categorySwitch);
return parameters;
}
@@ -827,7 +854,14 @@ public class SimilarAsinCozeClient {
|| !normalize(row.getPatentRisk()).isBlank()
|| !normalize(row.getConclusion()).isBlank()
|| !normalize(row.getIsStock()).isBlank()
|| !normalize(row.getSimilarity()).isBlank();
|| !normalize(row.getSimilarity()).isBlank()
|| !normalize(row.getIsConform()).isBlank()
|| !normalize(row.getReason()).isBlank()
|| !normalize(row.getCategory()).isBlank()
|| !normalize(row.getStatus()).isBlank()
|| !normalize(row.getMainUrl()).isBlank()
|| !normalize(row.getPuzzleImg1()).isBlank()
|| !normalize(row.getPuzzleImg2()).isBlank();
}
private void ensureSuccess(JsonNode root) {
@@ -978,7 +1012,21 @@ public class SimilarAsinCozeClient {
|| item.has("appearanceReason")
|| item.has("patent_reason")
|| item.has("patentReason")
|| item.has("patent reason"));
|| item.has("patent reason")
|| item.has("main_url")
|| item.has("mainUrl")
|| item.has("main_image_url")
|| item.has("mainImageUrl")
|| item.has("main_img")
|| item.has("mainImg")
|| item.has("puzzle_img1")
|| item.has("puzzleImg1")
|| item.has("puzzle_img_1")
|| item.has("puzzleImg_1")
|| item.has("puzzle_img2")
|| item.has("puzzleImg2")
|| item.has("puzzle_img_2")
|| item.has("puzzleImg_2"));
}
private boolean isSuccessfulWorkflowStatus(String status) {
@@ -37,4 +37,9 @@ public class SimilarAsinParseRequest {
@JsonAlias({"imgSwitch"})
@Schema(description = "传递给 Coze workflow parameters.img_switch 的图片检测开关,true 为开启,false 为关闭。")
private Boolean imgSwitch = Boolean.FALSE;
@JsonProperty("category_switch")
@JsonAlias({"categorySwitch"})
@Schema(description = "传递给 Coze workflow parameters.category_switch 的类目检测开关,true 为开启,false 为关闭。")
private Boolean categorySwitch = Boolean.FALSE;
}
@@ -20,6 +20,9 @@ public class SimilarAsinParsedPayloadDto {
@Schema(description = "传递给 Coze workflow parameters.img_switch 的图片检测开关")
private Boolean imgSwitch = Boolean.FALSE;
@Schema(description = "传递给 Coze workflow parameters.category_switch 的类目检测开关")
private Boolean categorySwitch = Boolean.FALSE;
@Schema(description = "本次解析的源文件列表")
private List<SimilarAsinSourceFileDto> sourceFiles = new ArrayList<>();
@@ -36,6 +36,9 @@ public class SimilarAsinParseVo {
@Schema(description = "图片检测开关,true 为开启,false 为关闭")
private Boolean imgSwitch = Boolean.FALSE;
@Schema(description = "类目检测开关,true 为开启,false 为关闭")
private Boolean categorySwitch = Boolean.FALSE;
@Schema(description = "兼容旧前端的平铺有效行列表,现为全部有效行")
private List<SimilarAsinParsedRowVo> items = new ArrayList<>();
@@ -83,6 +83,8 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
@@ -96,7 +98,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -219,6 +221,23 @@ public class SimilarAsinTaskService {
"阿里巴巴图片1",
"阿里巴巴图片2"
);
private static final Set<String> CATEGORY_RESULT_HEADER_ALIASES = Set.of(
FailedStatusRowFilter.canonicalizeHeader("是否符合类目"),
FailedStatusRowFilter.canonicalizeHeader("is_conform"),
FailedStatusRowFilter.canonicalizeHeader("conform")
);
private static final Set<String> COZE_RESULT_HEADER_ALIASES = Set.of(
FailedStatusRowFilter.canonicalizeHeader("是否有货"),
FailedStatusRowFilter.canonicalizeHeader("is_stock"),
FailedStatusRowFilter.canonicalizeHeader("相似度"),
FailedStatusRowFilter.canonicalizeHeader("similarity"),
FailedStatusRowFilter.canonicalizeHeader("是否符合类目"),
FailedStatusRowFilter.canonicalizeHeader("is_conform"),
FailedStatusRowFilter.canonicalizeHeader("不符合理由"),
FailedStatusRowFilter.canonicalizeHeader("reason"),
FailedStatusRowFilter.canonicalizeHeader("产品类目"),
FailedStatusRowFilter.canonicalizeHeader("category")
);
private static final int IMG_COL_MAIN = 12;
private static final int IMG_COL_PUZZLE1 = 13;
@@ -359,6 +378,7 @@ public class SimilarAsinTaskService {
List<String> mergedHeaders = new ArrayList<>();
int totalRows = 0;
int droppedRows = 0;
boolean categoryRetryRequired = false;
long parseStartedAt = System.nanoTime();
for (SimilarAsinSourceFileDto source : sourceFiles) {
if (source.getFileKey() == null || source.getFileKey().isBlank()) {
@@ -374,10 +394,17 @@ public class SimilarAsinTaskService {
droppedRows += parsed.droppedRows();
allRows.addAll(parsed.allRows());
mergeHeaders(mergedHeaders, parsed.headers());
categoryRetryRequired |= parsed.categoryRetryRequired();
}
if (allRows.isEmpty()) {
throw new BusinessException("未解析到有效 ASIN 数据");
}
boolean requestedCategorySwitch = Boolean.TRUE.equals(request.getCategorySwitch());
request.setCategorySwitch(requestedCategorySwitch || categoryRetryRequired);
if (!requestedCategorySwitch && categoryRetryRequired) {
log.info("[similar-asin] category retry detected from failed result rows, enable category_switch automatically files={} rows={}",
sourceFiles.size(), allRows.size());
}
long parsedAt = System.nanoTime();
List<SimilarAsinParsedGroupVo> groups = buildParsedGroups(allRows);
long groupedAt = System.nanoTime();
@@ -405,11 +432,11 @@ public class SimilarAsinTaskService {
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), sourceFiles, mergedHeaders, groups, allRows);
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, mergedHeaders, groups, allRows);
long payloadBuiltAt = System.nanoTime();
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
long payloadStoredAt = System.nanoTime();
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), sourceFiles, parsedPayloadPointer));
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, parsedPayloadPointer));
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
@@ -449,6 +476,7 @@ public class SimilarAsinTaskService {
vo.setGroupCount(groups.size());
vo.setAiPrompt(normalize(request.getAiPrompt()));
vo.setImgSwitch(Boolean.TRUE.equals(request.getImgSwitch()));
vo.setCategorySwitch(Boolean.TRUE.equals(request.getCategorySwitch()));
vo.setItems(new ArrayList<>(allRows));
vo.setGroups(groups);
long finishedAt = System.nanoTime();
@@ -1228,10 +1256,11 @@ public class SimilarAsinTaskService {
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
boolean imgSwitch = readImgSwitch(task);
boolean categorySwitch = readCategorySwitch(task);
int batchSize = resolveCozeBatchSize(imgSwitch);
List<SimilarAsinResultRowDto> result = new ArrayList<>();
for (int i = 0; i < items.size(); i += batchSize) {
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, imgSwitch));
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, imgSwitch, categorySwitch));
if (progressHook != null) {
progressHook.run();
}
@@ -1753,6 +1782,73 @@ public class SimilarAsinTaskService {
return true;
}
private boolean readCategorySwitch(FileTaskEntity task) {
try {
JsonNode root = objectMapper.readTree(task.getResultJson());
JsonNode savedSwitch = root == null ? null : root.get("categorySwitch");
if (savedSwitch != null && !savedSwitch.isNull()) {
return savedSwitch.asBoolean(false);
}
return Boolean.TRUE.equals(readParsedPayload(task).getCategorySwitch());
} catch (Exception ex) {
throw new BusinessException("读取相似ASIN类目检测开关失败", ex);
}
}
@Transactional
public void handleResultFileJobFailure(TaskFileJobEntity job, String message) {
if (job == null || job.getTaskId() == null) {
return;
}
try (TaskDistributedLockService.LockHandle ignored =
requireTaskLock(job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS)) {
Long finalizedTaskId = transactionManager == null
? persistResultFileJobFailure(job, message)
: inNewTransaction(() -> persistResultFileJobFailure(job, message));
if (finalizedTaskId != null) {
taskCacheService.deleteTaskCache(finalizedTaskId);
clearPoisonWindow(finalizedTaskId);
}
}
}
private Long persistResultFileJobFailure(TaskFileJobEntity job, String message) {
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) {
return null;
}
String owner = ownerFromTask(task);
if (!isOwnerCurrent(owner)) {
log.warn("[similar-asin] finalize exhausted result job from another instance taskId={} jobId={} owner={} current={}",
task.getId(), job.getId(), owner, currentInstanceId());
}
String failureMessage = firstNonBlank(message, "相似ASIN结果文件生成失败");
LocalDateTime now = LocalDateTime.now();
task.setStatus(STATUS_FAILED);
task.setSuccessFileCount(0);
task.setFailedFileCount(1);
task.setErrorMessage(failureMessage);
task.setUpdatedAt(now);
task.setFinishedAt(now);
fileTaskMapper.updateById(task);
FileResultEntity result = job.getResultId() == null ? null : fileResultMapper.selectById(job.getResultId());
if (result != null && MODULE_TYPE.equals(result.getModuleType())) {
result.setSuccess(0);
result.setErrorMessage(failureMessage);
result.setResultFileUrl(null);
result.setResultFileSize(0L);
fileResultMapper.updateById(result);
}
saveFileBuildProgress(task, job, 1, 1, failureMessage);
return task.getId();
}
private void finalizeExhaustedResultFileJob(TaskFileJobEntity job, String message) {
handleResultFileJobFailure(job, message);
taskFileJobService.markFailureFinalized(job.getId(), message);
}
@Scheduled(fixedDelayString = "${aiimage.similar-asin.coze-poll-delay-ms:30000}")
public void pollPendingCozeJobs() {
DistributedJobLockService.LockHandle lockHandle =
@@ -1949,6 +2045,7 @@ public class SimilarAsinTaskService {
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
boolean imgSwitch = readImgSwitch(task);
boolean categorySwitch = readCategorySwitch(task);
int batchSize = resolveCozeBatchSize(imgSwitch);
// P1-1:检测到 720712008 风暴时强制把 batch 降到 1,隔离毒行;
// 持续 5+ 次提交命中率 ≥ 40% 才会触发,正常波动不影响吞吐。
@@ -2013,7 +2110,7 @@ public class SimilarAsinTaskService {
int batchIndex = 1;
for (int i = 0; i < submitLimit; i += batchSize) {
List<CozeCandidate> batchCandidates = readyCandidates.subList(i, Math.min(i + batchSize, submitLimit));
pending |= submitCozeBatch(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, allRowsByBaseId);
pending |= submitCozeBatch(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, categorySwitch, allRowsByBaseId);
batchIndex++;
}
return pending || countPendingCozeStates(task.getId()) > 0;
@@ -2129,6 +2226,7 @@ public class SimilarAsinTaskService {
String prompt,
String apiKey,
boolean imgSwitch,
boolean categorySwitch,
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
if (batchCandidates == null || batchCandidates.isEmpty()) {
return false;
@@ -2155,7 +2253,7 @@ public class SimilarAsinTaskService {
SimilarAsinCozeClient.CozeCredentialRef credential = cozeClient.nextCredential();
try {
SimilarAsinCozeClient.CozeSubmitResponse submit = submitCozeWorkflowThrottled(
batchRows, prompt, apiKey, imgSwitch, credential, true);
batchRows, prompt, apiKey, imgSwitch, categorySwitch, credential, true);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List<SimilarAsinResultRowDto> cozeRows = cozeClient.mergeRowsFromDataText(batchRows, submit.immediateData());
String emptyResultMessage = emptyCozeResultMessage(cozeRows, batchRows.size());
@@ -2520,7 +2618,7 @@ public class SimilarAsinTaskService {
try {
SimilarAsinCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
cozeClient.credentialByName(context.credentialName()), false);
readCategorySwitch(task), cozeClient.credentialByName(context.credentialName()), false);
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List<SimilarAsinResultRowDto> cozeRows =
@@ -2582,6 +2680,7 @@ public class SimilarAsinTaskService {
String prompt,
String apiKey,
boolean imgSwitch,
boolean categorySwitch,
SimilarAsinCozeClient.CozeCredentialRef credential,
boolean allowCredentialFallback) throws Exception {
int attempts = allowCredentialFallback ? Math.max(1, cozeClient.configuredCredentialCount()) : 1;
@@ -2615,7 +2714,7 @@ public class SimilarAsinTaskService {
}
try (lockHandle; borrowedCredential) {
SimilarAsinCozeClient.CozeSubmitResponse response =
cozeClient.submitWorkflow(rows, prompt, apiKey, imgSwitch, currentCredential);
cozeClient.submitWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, currentCredential);
// 提交完成立即记录时间戳,用于下次进入循环时计算节流等待。
lastCozeSubmitAtByCredential.put(credentialKey, System.currentTimeMillis());
return response;
@@ -2807,7 +2906,7 @@ public class SimilarAsinTaskService {
int partIndex = 1;
for (List<SimilarAsinResultRowDto> partRows : partitions) {
SimilarAsinCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task), readCategorySwitch(task),
cozeClient.credentialByName(context.credentialName()), false);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List<SimilarAsinResultRowDto> cozeRows =
@@ -2941,7 +3040,7 @@ public class SimilarAsinTaskService {
try {
taskFileJobService.touchRunningIfStale(context.jobId(), properties.getDbJobTouchIntervalMillis());
SimilarAsinCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task), readCategorySwitch(task),
cozeClient.credentialByName(context.credentialName()), false);
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
@@ -3280,6 +3379,9 @@ public class SimilarAsinTaskService {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId());
if (job != null) {
taskFileJobService.markFailed(job, firstNonBlank(ex.getMessage(), "相似ASIN结果文件生成失败"));
if (taskFileJobService.isRetryExhausted(job.getId())) {
finalizeExhaustedResultFileJob(job, ex.getMessage());
}
}
log.warn("[相似ASIN] Coze 异步收尾失败 任务ID={} 结果ID={} 错误={}",
taskId, context.resultId(), ex.getMessage(), ex);
@@ -3303,6 +3405,10 @@ public class SimilarAsinTaskService {
if (job == null || "SUCCESS".equals(job.getStatus())) {
return;
}
if (taskFileJobService.isRetryExhausted(job.getId())) {
finalizeExhaustedResultFileJob(job, job.getErrorMessage());
return;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task != null && STATUS_RUNNING.equals(task.getStatus()) && !isResultSubmissionComplete(taskId)) {
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
@@ -3324,6 +3430,8 @@ public class SimilarAsinTaskService {
if (requeued) {
log.info("[相似ASIN] Coze 异步结果已就绪,结果文件任务已重新入队 任务ID={} 文件任务ID={} 结果ID={}",
taskId, job.getId(), context.resultId());
} else if (taskFileJobService.isRetryExhausted(job.getId())) {
finalizeExhaustedResultFileJob(job, job.getErrorMessage());
}
}
}
@@ -4160,6 +4268,7 @@ public class SimilarAsinTaskService {
}
List<SourceRows> sourceRows = splitRowsBySourceFile(parsed, parsed.getAllItems(), result.getSourceFilename());
List<SourceResultWorkbook> workbooks = new ArrayList<>();
List<File> workbookFiles = new ArrayList<>(sourceRows.size());
File zip = null;
// 跨 workbook 共享 taskImageCache:多源场景下相同 URL 仅下载一次。
// 配合 embed() 写完即 remove(),cache 仅承载 in-flight 图片;
@@ -4170,70 +4279,57 @@ public class SimilarAsinTaskService {
: BoundedImageCache.DEFAULT_MAX_BYTES;
BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes);
try {
// P3-1:单源文件场景仍走串行降级路径,避免引入线程切换开销;
// 多源文件场景把 writeResultWorkbook 投到 assembleExecutor 上并发跑,
// 1000+ 行 ×N 源文件的 assemble 阶段总耗时直接除以 N(受池大小 4 限制)。
// assembleExecutor 是固定 4 线程池:sourceRows.size() ≤ 4 时全部并行;
// > 4 时多余源文件排队,避免 4×5000 行同时打开图片缓存爆堆。
if (sourceRows.size() <= 1) {
for (SourceRows item : sourceRows) {
String filename = safeFileStem(item.sourceFilename()) + "-result.xlsx";
String tempFilename = safeFileStem(item.sourceFilename())
+ "-" + task.getId()
+ "-" + result.getId()
+ "-" + UUID.randomUUID()
+ "-result.xlsx";
File xlsx = new File(outputDir, tempFilename);
writeResultWorkbook(xlsx, item.rows(), resultMap, taskImageCache);
workbooks.add(new SourceResultWorkbook(xlsx, filename, item.rows().size()));
}
} else {
long assembleStart = System.currentTimeMillis();
List<CompletableFuture<SourceResultWorkbook>> futures = new ArrayList<>(sourceRows.size());
for (SourceRows item : sourceRows) {
final SourceRows captured = item;
final String filename = safeFileStem(captured.sourceFilename()) + "-result.xlsx";
final String tempFilename = safeFileStem(captured.sourceFilename())
+ "-" + task.getId()
+ "-" + result.getId()
+ "-" + UUID.randomUUID()
+ "-result.xlsx";
futures.add(CompletableFuture.supplyAsync(() -> {
File xlsx = new File(outputDir, tempFilename);
writeResultWorkbook(xlsx, captured.rows(), resultMap, taskImageCache);
return new SourceResultWorkbook(xlsx, filename, captured.rows().size());
}, assembleExecutor));
}
for (CompletableFuture<SourceResultWorkbook> future : futures) {
try {
// 单源文件 30 分钟硬上限:超时直接抛错,避免被慢源永久阻塞。
workbooks.add(future.get(30, TimeUnit.MINUTES));
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
for (CompletableFuture<SourceResultWorkbook> remaining : futures) {
remaining.cancel(true);
}
throw new BusinessException("生成相似ASIN检测结果中断", ie);
} catch (TimeoutException te) {
for (CompletableFuture<SourceResultWorkbook> remaining : futures) {
remaining.cancel(true);
}
throw new BusinessException("生成相似ASIN检测结果超时(30 分钟)", te);
} catch (ExecutionException ee) {
Throwable cause = ee.getCause();
for (CompletableFuture<SourceResultWorkbook> remaining : futures) {
remaining.cancel(true);
}
if (cause instanceof BusinessException be) {
throw be;
}
throw new BusinessException("生成相似ASIN检测结果失败",
cause != null ? cause : ee);
}
}
log.info("[similar-asin] assemble parallel finished taskId={} sources={} costMs={}",
task.getId(), sourceRows.size(), System.currentTimeMillis() - assembleStart);
// 单源和多源统一提交真实 Future,确保超时取消能中断实际 workbook 线程。
// 所有源文件共享同一个绝对 deadline,避免逐个 Future 各等待一轮完整超时。
int timeoutMinutes = Math.max(1, properties.getResultFileTimeoutMinutes());
long deadlineNanos = System.nanoTime() + TimeUnit.MINUTES.toNanos(timeoutMinutes);
long assembleStart = System.currentTimeMillis();
List<Future<SourceResultWorkbook>> futures = new ArrayList<>(sourceRows.size());
for (SourceRows item : sourceRows) {
final SourceRows captured = item;
final String filename = safeFileStem(captured.sourceFilename()) + "-result.xlsx";
final String tempFilename = safeFileStem(captured.sourceFilename())
+ "-" + task.getId()
+ "-" + result.getId()
+ "-" + UUID.randomUUID()
+ "-result.xlsx";
final File xlsx = new File(outputDir, tempFilename);
workbookFiles.add(xlsx);
futures.add(assembleExecutor.submit(() -> {
ensureResultAssemblyNotInterrupted();
writeResultWorkbook(xlsx, captured.rows(), resultMap, taskImageCache);
ensureResultAssemblyNotInterrupted();
return new SourceResultWorkbook(xlsx, filename, captured.rows().size());
}));
}
try {
for (Future<SourceResultWorkbook> future : futures) {
long remainingNanos = deadlineNanos - System.nanoTime();
if (remainingNanos <= 0L) {
throw new TimeoutException("result file deadline reached");
}
workbooks.add(future.get(remainingNanos, TimeUnit.NANOSECONDS));
}
} catch (InterruptedException ex) {
cancelResultAssemblyFutures(futures);
Thread.currentThread().interrupt();
throw new BusinessException("生成相似ASIN检测结果中断", ex);
} catch (TimeoutException ex) {
cancelResultAssemblyFutures(futures);
throw new BusinessException("生成相似ASIN检测结果超时(" + timeoutMinutes + " 分钟)", ex);
} catch (CancellationException ex) {
cancelResultAssemblyFutures(futures);
throw new BusinessException("生成相似ASIN检测结果已取消", ex);
} catch (ExecutionException ex) {
cancelResultAssemblyFutures(futures);
Throwable cause = ex.getCause();
if (cause instanceof BusinessException businessException) {
throw businessException;
}
throw new BusinessException("生成相似ASIN检测结果失败", cause != null ? cause : ex);
}
log.info("[similar-asin] assemble workbooks finished taskId={} sources={} timeoutMinutes={} costMs={}",
task.getId(), sourceRows.size(), timeoutMinutes, System.currentTimeMillis() - assembleStart);
log.info("[similar-asin] assemble image-cache stats taskId={} sources={} cacheSize={} currentBytes={} evictedCount={} evictedBytes={} maxBytes={}",
task.getId(), sourceRows.size(), taskImageCache.size(),
taskImageCache.currentBytes(), taskImageCache.evictedCount(),
@@ -4272,9 +4368,9 @@ public class SimilarAsinTaskService {
result.setResultContentType(contentType);
result.setRowCount(parsed.getAllItems().size());
} finally {
for (SourceResultWorkbook workbook : workbooks) {
if (workbook.file().exists() && !workbook.file().delete()) {
log.warn("[similar-asin] delete temp xlsx failed file={}", workbook.file());
for (File workbookFile : workbookFiles) {
if (workbookFile.exists() && !workbookFile.delete()) {
log.warn("[similar-asin] delete temp xlsx failed file={}", workbookFile);
}
}
if (zip != null && zip.exists() && !zip.delete()) {
@@ -4482,10 +4578,19 @@ public class SimilarAsinTaskService {
List<SimilarAsinParsedRowVo> rowsToWrite,
Map<String, SimilarAsinResultRowDto> resultMap,
Map<String, SimilarAsinImageEmbedder.ResizedImage> taskImageCache) {
ensureResultAssemblyNotInterrupted();
// Excel 365 "Place in Cell" 图片:写完 workbook 后 patch richData 单元格图片结构。
// 这不是浮动 Drawing,因此点击图片区域会选中单元格,图片不能被拖到任意位置,也不需要工作表保护。
ExcelCellImageWriter.Session excelCellImageSession = ExcelCellImageWriter.createSession();
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
SimilarAsinImageEmbedder.ImageSpool imageSpool;
try {
Path spoolDir = Files.createTempDirectory(xlsx.getParentFile().toPath(), "similar-asin-images-");
imageSpool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
} catch (IOException ex) {
throw new BusinessException("create Similar ASIN image spool failed", ex);
}
try {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
Sheet sheet = workbook.createSheet("相似asin检测");
CellStyle headerStyle = workbook.createCellStyle();
Font font = workbook.createFont();
@@ -4506,17 +4611,16 @@ public class SimilarAsinTaskService {
cell.setCellStyle(headerStyle);
}
// P2-8:第一遍预扫所有图片 URL,并行下载到 taskImageCache,
// POI 写入仍单线程串行注册 cell image(cache 命中直接 resize + register),下载/写入解耦。
// 预扫图片 URL 并行下载到任务级临时目录;POI 仍按行串行注册 cell image。
List<String> prefetchUrls = collectImageUrlsForPrefetch(rowsToWrite, resultMap);
if (!prefetchUrls.isEmpty()) {
// P2-11:先一次性把 DB cache 命中的字节填入 taskImageCache,避免再次走网络。
// 命中部分从 prefetchUrls 剔除,剩余的真正未命中的 URL 才走 imageEmbedder.prefetch 网络下载。
// DB cache 命中的缩略图也立即落到任务临时目录,避免整批字节停留在堆中。
long dbCacheStart = System.currentTimeMillis();
List<String> remainingUrls = new ArrayList<>(prefetchUrls.size());
int dbHit = 0;
for (String url : prefetchUrls) {
if (url == null || taskImageCache.containsKey(url)) {
ensureResultAssemblyNotInterrupted();
if (url == null || imageSpool.get(url) != null || taskImageCache.containsKey(url)) {
continue;
}
byte[] cachedBytes = imagePrefetchService.lookup(url);
@@ -4526,7 +4630,7 @@ public class SimilarAsinTaskService {
}
SimilarAsinImageEmbedder.ResizedImage thumb = imageEmbedder.decodeCachedThumb(cachedBytes);
if (thumb != null) {
taskImageCache.putIfAbsent(url, thumb);
imageSpool.put(url, thumb);
dbHit++;
} else {
remainingUrls.add(url);
@@ -4536,14 +4640,16 @@ public class SimilarAsinTaskService {
prefetchUrls.size(), dbHit, remainingUrls.size(), System.currentTimeMillis() - dbCacheStart);
if (!remainingUrls.isEmpty()) {
long prefetchStart = System.currentTimeMillis();
imageEmbedder.prefetch(remainingUrls, taskImageCache);
log.info("[similar-asin] image prefetch finished urls={} cached={} costMs={}",
remainingUrls.size(), taskImageCache.size(), System.currentTimeMillis() - prefetchStart);
imageEmbedder.prefetchToDisk(remainingUrls, imageSpool);
ensureResultAssemblyNotInterrupted();
log.info("[similar-asin] image disk-prefetch finished urls={} spooled={} costMs={}",
remainingUrls.size(), imageSpool.size(), System.currentTimeMillis() - prefetchStart);
}
}
int rowIndex = 1;
for (SimilarAsinParsedRowVo parsedRow : rowsToWrite == null ? List.<SimilarAsinParsedRowVo>of() : rowsToWrite) {
ensureResultAssemblyNotInterrupted();
SimilarAsinResultRowDto resultRow = findResultRow(parsedRow, resultMap);
Row row = sheet.createRow(rowIndex);
int col = 0;
@@ -4575,21 +4681,21 @@ public class SimilarAsinTaskService {
// 行高固定到 IMAGE_ROW_HEIGHT_POINTS(Excel 上限 409pt):
row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS);
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_MAIN,
resultRow.getMainUrl(), row, taskImageCache, excelCellImageSession);
resultRow.getMainUrl(), row, taskImageCache, excelCellImageSession, imageSpool);
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE1,
resultRow.getPuzzleImg1(), row, taskImageCache, excelCellImageSession);
resultRow.getPuzzleImg1(), row, taskImageCache, excelCellImageSession, imageSpool);
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE2,
resultRow.getPuzzleImg2(), row, taskImageCache, excelCellImageSession);
resultRow.getPuzzleImg2(), row, taskImageCache, excelCellImageSession, imageSpool);
}
rowIndex++;
}
ensureResultAssemblyNotInterrupted();
workbook.write(fos);
workbook.dispose();
} catch (Exception ex) {
throw new BusinessException("生成相似ASIN检测结果失败", ex);
}
if (!excelCellImageSession.isEmpty()) {
try {
ensureResultAssemblyNotInterrupted();
ExcelCellImageWriter.patchXlsxFile(xlsx, excelCellImageSession);
} catch (IOException ex) {
String cause = ex.toString();
@@ -4598,6 +4704,32 @@ public class SimilarAsinTaskService {
throw new BusinessException("写入相似ASIN结果 Excel 单元格图片失败: " + cause, ex);
}
}
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("生成相似ASIN检测结果失败", ex);
} finally {
try {
imageSpool.close();
} catch (IOException ex) {
log.warn("[similar-asin] delete image spool failed xlsx={} err={}",
xlsx.getAbsolutePath(), ex.getMessage());
}
}
}
private static void cancelResultAssemblyFutures(List<? extends Future<?>> futures) {
for (Future<?> future : futures) {
if (future != null && !future.isDone()) {
future.cancel(true);
}
}
}
private static void ensureResultAssemblyNotInterrupted() {
if (Thread.currentThread().isInterrupted()) {
throw new BusinessException("生成相似ASIN检测结果中断");
}
}
/**
@@ -4714,24 +4846,29 @@ public class SimilarAsinTaskService {
vo.setValues(readRowValues(row, headers, formatter));
allRows.add(vo);
}
boolean includeBlankStatusRows = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
boolean resultWorkbook = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
boolean firstPassResultWorkbook = isFirstPassResultWorkbook(headers, allRows);
boolean includeBlankStatusRows = resultWorkbook && !firstPassResultWorkbook;
FailedStatusRowFilter.FilterResult<SimilarAsinParsedRowVo> filteredRows =
FailedStatusRowFilter.retainRows(
allRows,
statusCol >= 0,
rowVo -> statusHeader == null || rowVo.getValues() == null ? "" : rowVo.getValues().get(statusHeader),
status -> FailedStatusRowFilter.matchesFailedStatus(status)
|| firstPassResultWorkbook
|| (includeBlankStatusRows && FailedStatusRowFilter.isBlankStatus(status))
);
dropped += filteredRows.filteredCount();
allRows = new ArrayList<>(filteredRows.rows());
boolean categoryRetryRequired = shouldEnableCategorySwitchForRetry(
headers, allRows, includeBlankStatusRows);
if (statusCol >= 0 && validRows > 0 && allRows.isEmpty()) {
throw new BusinessException(FailedStatusRowFilter.noMatchedRowsMessage());
}
if (allRows.isEmpty()) {
throw new BusinessException("no valid similar ASIN rows");
}
return new ParsedWorkbook(total, dropped, headers, allRows);
return new ParsedWorkbook(total, dropped, headers, allRows, categoryRetryRequired);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
@@ -4763,7 +4900,10 @@ public class SimilarAsinTaskService {
|| hasUsableCozeField(row.getIsConform())
|| hasUsableCozeField(row.getReason())
|| hasUsableCozeField(row.getCategory())
|| hasUsableCozeField(row.getStatus());
|| hasUsableCozeField(row.getStatus())
|| hasText(row.getMainUrl())
|| hasText(row.getPuzzleImg1())
|| hasText(row.getPuzzleImg2());
}
static String resolveResultStatus(SimilarAsinResultRowDto row, String... visibleResultValues) {
@@ -4885,7 +5025,16 @@ public class SimilarAsinTaskService {
}
private String cell(Row row, int col, DataFormatter formatter) {
return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col)));
if (col < 0) {
return "";
}
String value = normalize(formatter.formatCellValue(row.getCell(col)));
return isSpreadsheetErrorValue(value) ? "" : value;
}
private static boolean isSpreadsheetErrorValue(String value) {
String normalized = value == null ? "" : value.trim();
return normalized.startsWith("#") && normalized.endsWith("!");
}
private String baseId(String id) {
@@ -5373,11 +5522,12 @@ public class SimilarAsinTaskService {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
private String buildParsedPayloadJson(String aiPrompt, String apiKey, Boolean imgSwitch, List<SimilarAsinSourceFileDto> sourceFiles, List<String> headers, List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
private String buildParsedPayloadJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, List<SimilarAsinSourceFileDto> sourceFiles, List<String> headers, List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
payload.setAiPrompt(normalize(aiPrompt));
payload.setApiKey(normalize(apiKey));
payload.setImgSwitch(Boolean.TRUE.equals(imgSwitch));
payload.setCategorySwitch(Boolean.TRUE.equals(categorySwitch));
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
payload.setHeaders(headers == null ? List.of() : headers);
payload.setItems(allRows == null ? List.of() : new ArrayList<>(allRows));
@@ -5386,11 +5536,12 @@ public class SimilarAsinTaskService {
return writeJson(payload, "保存解析结果失败");
}
private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, List<SimilarAsinSourceFileDto> sourceFiles, String parsedPayloadPointer) {
private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, List<SimilarAsinSourceFileDto> sourceFiles, String parsedPayloadPointer) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("aiPrompt", normalize(aiPrompt));
payload.put("apiKey", normalize(apiKey));
payload.put("imgSwitch", Boolean.TRUE.equals(imgSwitch));
payload.put("categorySwitch", Boolean.TRUE.equals(categorySwitch));
payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream()
.map(SimilarAsinSourceFileDto::getFileKey)
.filter(Objects::nonNull)
@@ -5746,6 +5897,83 @@ public class SimilarAsinTaskService {
|| hasHeader(headers, "产品类目"));
}
static boolean isFirstPassResultWorkbook(List<String> headers,
List<SimilarAsinParsedRowVo> rows) {
if (headers == null || headers.isEmpty() || rows == null || rows.isEmpty()
|| FailedStatusRowFilter.findStatusColumnIndex(headers) < 0
|| (!hasCanonicalHeader(headers, "是否有货")
&& !hasCanonicalHeader(headers, "相似度")
&& !hasCanonicalHeader(headers, "是否符合类目")
&& !hasCanonicalHeader(headers, "不符合理由")
&& !hasCanonicalHeader(headers, "产品类目"))
|| (!hasCanonicalHeader(headers, "主图")
&& !hasCanonicalHeader(headers, "阿里巴巴图片1")
&& !hasCanonicalHeader(headers, "阿里巴巴图片2"))) {
return false;
}
String statusHeader = headers.get(FailedStatusRowFilter.findStatusColumnIndex(headers));
for (SimilarAsinParsedRowVo row : rows) {
if (row == null || FailedStatusRowFilter.matchesFailedStatus(valueByHeader(row, statusHeader))) {
return false;
}
for (Map.Entry<String, String> entry : row.getValues() == null
? Map.<String, String>of().entrySet() : row.getValues().entrySet()) {
if (COZE_RESULT_HEADER_ALIASES.contains(FailedStatusRowFilter.canonicalizeHeader(entry.getKey()))
&& !isBlankOrSpreadsheetError(entry.getValue())) {
return false;
}
}
}
return true;
}
private static boolean hasCanonicalHeader(List<String> headers, String expected) {
String canonical = FailedStatusRowFilter.canonicalizeHeader(expected);
return headers.stream()
.filter(Objects::nonNull)
.anyMatch(header -> canonical.equals(FailedStatusRowFilter.canonicalizeHeader(header)));
}
private static String valueByHeader(SimilarAsinParsedRowVo row, String header) {
if (row == null || row.getValues() == null) {
return "";
}
String canonical = FailedStatusRowFilter.canonicalizeHeader(header);
return row.getValues().entrySet().stream()
.filter(entry -> canonical.equals(FailedStatusRowFilter.canonicalizeHeader(entry.getKey())))
.map(Map.Entry::getValue)
.findFirst()
.orElse("");
}
private static boolean isBlankOrSpreadsheetError(String value) {
return value == null || value.trim().isBlank() || isSpreadsheetErrorValue(value);
}
static boolean shouldEnableCategorySwitchForRetry(List<String> headers,
List<SimilarAsinParsedRowVo> rows,
boolean resultWorkbook) {
if (!resultWorkbook || headers == null || headers.isEmpty() || rows == null || rows.isEmpty()
|| FailedStatusRowFilter.findStatusColumnIndex(headers) < 0) {
return false;
}
String categoryHeader = headers.stream()
.filter(Objects::nonNull)
.filter(header -> CATEGORY_RESULT_HEADER_ALIASES.contains(
FailedStatusRowFilter.canonicalizeHeader(header)))
.findFirst()
.orElse(null);
if (categoryHeader == null) {
return false;
}
return rows.stream()
.filter(Objects::nonNull)
.map(SimilarAsinParsedRowVo::getValues)
.anyMatch(values -> values == null
|| values.get(categoryHeader) == null
|| values.get(categoryHeader).trim().isBlank());
}
private boolean hasHeader(List<String> headers, String candidate) {
if (headers == null || headers.isEmpty()) {
return false;
@@ -5926,6 +6154,10 @@ public class SimilarAsinTaskService {
long size) {
}
private record ParsedWorkbook(int totalRows, int droppedRows, List<String> headers, List<SimilarAsinParsedRowVo> allRows) {
private record ParsedWorkbook(int totalRows,
int droppedRows,
List<String> headers,
List<SimilarAsinParsedRowVo> allRows,
boolean categoryRetryRequired) {
}
}
@@ -13,8 +13,10 @@ import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -41,6 +43,9 @@ public final class ExcelCellImageWriter {
private static final String RD_RICH_VALUE_TYPES_PART = "xl/richData/rdRichValueTypes.xml";
private static final String RICH_VALUE_REL_PART = "xl/richData/richValueRel.xml";
private static final String METADATA_PART = "xl/metadata.xml";
private static final Pattern SHEET_CELL_PATTERN = Pattern.compile(
"<c\\b(?=[^>]*\\br=\"([A-Z]+[1-9][0-9]*)\")[^>]*(?:/>|>.*?</c>)",
Pattern.DOTALL);
private static final String REL_TYPE_IMAGE =
"http://schemas.openxmlformats.org/officeDocument/2006/relationships/image";
@@ -116,7 +121,11 @@ public final class ExcelCellImageWriter {
writeEntry(zout, METADATA_PART, buildMetadata(session).getBytes(StandardCharsets.UTF_8));
int idx = 1;
for (RegisteredCellImage image : session.images) {
writeEntry(zout, mediaPart(idx), image.bytes);
if (image.bytes != null) {
writeEntry(zout, mediaPart(idx), image.bytes);
} else {
writeEntry(zout, mediaPart(idx), image.path);
}
idx++;
}
}
@@ -166,26 +175,34 @@ public final class ExcelCellImageWriter {
private static PatchSheetResult patchSheet(byte[] original, Session session) {
String content = new String(original, StandardCharsets.UTF_8);
int idx = 0;
int patchedCount = 0;
for (RegisteredCellImage image : session.images) {
// [MS-XLSX] §2.2.10: cell @vm 是 1-based valueMetadata 索引,0 表示"无 metadata"。
// futureMetadata/valueMetadata/rdrichvalue 内部块仍是 0-based,所以这里写 idx+1
// 与 buildMetadata/buildRdRichValue 中的 i 解耦。Excel 365 在 vm=0 时会把 cell
// 直接当作无图片的 #VALUE! 错误格渲染(任务 10498 J2 复现),必须改 1-based。
String replacement = "<c r=\"" + image.cellRef + "\" t=\"e\" vm=\"" + (idx + 1) + "\"><v>#VALUE!</v></c>";
Pattern pattern = Pattern.compile("<c\\b(?=[^>]*\\br=\"" + Pattern.quote(image.cellRef)
+ "\")[^>]*(?:/>|>.*?</c>)", Pattern.DOTALL);
Matcher matcher = pattern.matcher(content);
if (matcher.find()) {
content = matcher.replaceFirst(Matcher.quoteReplacement(replacement));
patchedCount++;
} else {
log.warn("[excel-cell-image] cell {} not found in sheet xml, skip image", image.cellRef);
}
idx++;
Map<String, Integer> metadataIndexByCell = new HashMap<>(session.images.size());
for (int idx = 0; idx < session.images.size(); idx++) {
metadataIndexByCell.put(session.images.get(idx).cellRef, idx + 1);
}
return new PatchSheetResult(content.getBytes(StandardCharsets.UTF_8), patchedCount);
Set<String> missingCells = new HashSet<>(metadataIndexByCell.keySet());
Matcher matcher = SHEET_CELL_PATTERN.matcher(content);
StringBuilder patched = new StringBuilder(content.length());
int patchedCount = 0;
while (matcher.find()) {
String cellRef = matcher.group(1);
Integer metadataIndex = metadataIndexByCell.get(cellRef);
if (metadataIndex == null) {
matcher.appendReplacement(patched, Matcher.quoteReplacement(matcher.group()));
continue;
}
// [MS-XLSX] §2.2.10: cell @vm is a 1-based valueMetadata index.
String replacement = "<c r=\"" + cellRef + "\" t=\"e\" vm=\"" + metadataIndex
+ "\"><v>#VALUE!</v></c>";
matcher.appendReplacement(patched, Matcher.quoteReplacement(replacement));
missingCells.remove(cellRef);
patchedCount++;
}
matcher.appendTail(patched);
for (String cellRef : missingCells) {
log.warn("[excel-cell-image] cell {} not found in sheet xml, skip image", cellRef);
}
return new PatchSheetResult(patched.toString().getBytes(StandardCharsets.UTF_8), patchedCount);
}
private static byte[] patchContentTypes(byte[] original) {
@@ -386,6 +403,16 @@ public final class ExcelCellImageWriter {
zout.closeEntry();
}
private static void writeEntry(ZipOutputStream zout, String name, Path path) throws IOException {
ZipEntry entry = new ZipEntry(name);
zout.putNextEntry(entry);
try (InputStream in = Files.newInputStream(path)) {
in.transferTo(zout);
} finally {
zout.closeEntry();
}
}
private static void writeDirectoryEntry(ZipOutputStream zout, String name) throws IOException {
ZipEntry entry = new ZipEntry(name);
zout.putNextEntry(entry);
@@ -399,7 +426,15 @@ public final class ExcelCellImageWriter {
if (jpegBytes == null || jpegBytes.length == 0) {
return;
}
images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), jpegBytes));
images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), jpegBytes, null));
}
public synchronized void registerImage(int rowIdx, int colIdx, Path jpegPath) {
if (jpegPath == null || !Files.isRegularFile(jpegPath)) {
return;
}
images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), null,
jpegPath.toAbsolutePath().normalize()));
}
public synchronized boolean isEmpty() {
@@ -426,7 +461,7 @@ public final class ExcelCellImageWriter {
}
}
private record RegisteredCellImage(String cellRef, byte[] bytes) {
private record RegisteredCellImage(String cellRef, byte[] bytes, Path path) {
}
private record PatchSheetResult(byte[] bytes, int patchedCount) {
@@ -27,14 +27,22 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.FileTime;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
@@ -42,6 +50,8 @@ import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
@@ -49,7 +59,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
/**
@@ -68,8 +77,9 @@ public class SimilarAsinImageEmbedder {
static final int DEFAULT_DOWNLOAD_TIMEOUT_SECONDS = 5;
/** P2-10:retry 由 1 升到 2,配合 5s timeout 单图最坏耗时 ≈ 15s。 */
static final int DOWNLOAD_MAX_RETRY = 2;
/** P2-10:原 8 → P2-8 16 → P2-10 32,1000+ 行 ×3 列场景下显著拉低 assemble 阶段尾延迟。 */
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 32;
/** 图片下载、解码、缩放都在该池执行;默认限制为 8,避免结果生成占满整机 CPU。 */
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 8;
static final int DEFAULT_PREFETCH_TIMEOUT_SECONDS = 1800;
// 单元格固定尺寸,图片在其中等比缩放(不拉伸);resize 仍按长边 1280 px 控制堆体积。
public static final float IMAGE_ROW_HEIGHT_POINTS = 409f;
public static final int IMAGE_COL_WIDTH_CHARS = 80;
@@ -102,16 +112,28 @@ public class SimilarAsinImageEmbedder {
private final int downloadTimeoutSeconds;
private final int downloadPoolSize;
private final int prefetchTimeoutSeconds;
private final OkHttpClient httpClient;
private final ExecutorService downloadPool;
private final OssStorageService ossStorageService;
private final Path localImageCacheDir;
public SimilarAsinImageEmbedder(SimilarAsinProperties properties, OssStorageService ossStorageService) {
int rawTimeout = properties == null ? DEFAULT_DOWNLOAD_TIMEOUT_SECONDS : properties.getImageDownloadTimeoutSeconds();
int rawPool = properties == null ? DEFAULT_DOWNLOAD_POOL_SIZE : properties.getImageDownloadPoolSize();
int rawPrefetchTimeout = properties == null
? DEFAULT_PREFETCH_TIMEOUT_SECONDS
: properties.getImagePrefetchTimeoutSeconds();
this.downloadTimeoutSeconds = rawTimeout > 0 ? rawTimeout : DEFAULT_DOWNLOAD_TIMEOUT_SECONDS;
this.downloadPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE;
this.prefetchTimeoutSeconds = rawPrefetchTimeout > 0
? rawPrefetchTimeout
: DEFAULT_PREFETCH_TIMEOUT_SECONDS;
this.ossStorageService = Objects.requireNonNull(ossStorageService, "ossStorageService must not be null");
String configuredCacheDir = properties == null ? null : properties.getImageLocalCacheDir();
this.localImageCacheDir = configuredCacheDir == null || configuredCacheDir.isBlank()
? null
: Path.of(configuredCacheDir.trim()).toAbsolutePath().normalize();
this.httpClient = new OkHttpClient.Builder()
.connectTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
.readTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
@@ -138,6 +160,10 @@ public class SimilarAsinImageEmbedder {
return downloadPoolSize;
}
int prefetchTimeoutSeconds() {
return prefetchTimeoutSeconds;
}
OkHttpClient httpClient() {
return httpClient;
}
@@ -159,13 +185,13 @@ public class SimilarAsinImageEmbedder {
if (urls == null || urls.isEmpty() || taskImageCache == null) {
return;
}
Set<String> distinctUrls = new HashSet<>();
Set<String> distinctUrls = new LinkedHashSet<>();
for (String url : urls) {
if (url == null) {
continue;
}
String trimmed = url.trim();
if (trimmed.isEmpty() || taskImageCache.containsKey(trimmed)) {
if (trimmed.isEmpty() || taskImageCache.containsKey(trimmed) || hasLocalCachedThumb(trimmed)) {
continue;
}
distinctUrls.add(trimmed);
@@ -174,58 +200,169 @@ public class SimilarAsinImageEmbedder {
return;
}
CompletionService<Object> completion = new ExecutorCompletionService<>(downloadPool);
List<Future<?>> futures = new ArrayList<>(distinctUrls.size());
for (String url : distinctUrls) {
// 显式作为 Runnable 提交(带 null result),避免与 Callable<Object> 重载产生歧义。
Runnable task = () -> {
try {
if (taskImageCache.containsKey(url)) {
return;
}
String downloadUrl = normalizeAndValidateDownloadUrl(url);
byte[] raw = doFetch(downloadUrl);
ResizedImage thumb = resizeImage(downloadUrl, raw);
taskImageCache.putIfAbsent(url, thumb);
} catch (Exception ex) {
// 预下载失败不抛出:embed() 时同 url 会再次尝试并走原有兜底链路。
log.debug("[similar-asin][image] prefetch-fail url={} err={}", url, errorSummary(ex));
}
};
futures.add(completion.submit(task, null));
Iterator<String> pending = distinctUrls.iterator();
List<Future<?>> active = new ArrayList<>(Math.min(downloadPoolSize, distinctUrls.size()));
while (pending.hasNext() && active.size() < downloadPoolSize) {
active.add(submitPrefetch(completion, pending.next(), taskImageCache));
}
// 全局 deadline:每 url 给 200ms 的预算,clamp 到 [15s, 120s]。
long globalDeadlineMs = Math.min(120_000L, Math.max(15_000L, distinctUrls.size() * 200L));
long deadline = System.currentTimeMillis() + globalDeadlineMs;
int total = distinctUrls.size();
int done = 0;
while (done < total) {
boolean deadlineReached = false;
while (!active.isEmpty()) {
long left = deadline - System.currentTimeMillis();
if (left <= 0) {
deadlineReached = true;
break;
}
try {
Future<Object> f = completion.poll(left, TimeUnit.MILLISECONDS);
if (f == null) {
deadlineReached = true;
break;
}
active.remove(f);
done++;
if (pending.hasNext()) {
active.add(submitPrefetch(completion, pending.next(), taskImageCache));
}
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
cancelAll(active);
break;
}
}
// 超时未完成的 future 主动 cancel,避免 idle 持有 OkHttp 连接。
if (done < total) {
for (Future<?> f : futures) {
if (!f.isDone()) {
f.cancel(true);
}
}
if (deadlineReached) {
cancelAll(active);
log.info("[similar-asin][image] prefetch deadline reached total={} done={} cancelled={}",
total, done, total - done);
}
}
private Future<?> submitPrefetch(CompletionService<Object> completion,
String url,
Map<String, ResizedImage> taskImageCache) {
// 显式作为 Runnable 提交(带 null result),避免与 Callable<Object> 重载产生歧义。
Runnable task = () -> {
try {
if (taskImageCache.containsKey(url)) {
return;
}
String downloadUrl = normalizeAndValidateDownloadUrl(url);
byte[] raw = doFetch(downloadUrl);
ResizedImage thumb = resizeImage(downloadUrl, raw);
taskImageCache.putIfAbsent(url, thumb);
} catch (Exception ex) {
// 预下载失败不抛出:embed() 时同 url 会再次尝试并走原有兜底链路。
log.debug("[similar-asin][image] prefetch-fail url={} err={}", url, errorSummary(ex));
}
};
return completion.submit(task, null);
}
/**
* Downloads and resizes image URLs concurrently, but spills completed thumbnails to disk instead of
* retaining the full batch in the JVM heap.
*/
public void prefetchToDisk(Collection<String> urls, ImageSpool imageSpool) {
if (urls == null || urls.isEmpty() || imageSpool == null) {
return;
}
Set<String> distinctUrls = new LinkedHashSet<>();
for (String url : urls) {
if (url == null) {
continue;
}
String trimmed = url.trim();
if (!trimmed.isEmpty() && imageSpool.get(trimmed) == null) {
distinctUrls.add(trimmed);
}
}
if (distinctUrls.isEmpty()) {
return;
}
AtomicInteger localHit = new AtomicInteger();
AtomicInteger downloaded = new AtomicInteger();
AtomicInteger failed = new AtomicInteger();
CompletionService<Object> completion = new ExecutorCompletionService<>(downloadPool);
Iterator<String> pending = distinctUrls.iterator();
List<Future<?>> active = new ArrayList<>(Math.min(downloadPoolSize, distinctUrls.size()));
while (pending.hasNext() && active.size() < downloadPoolSize) {
active.add(submitDiskPrefetch(completion, pending.next(), imageSpool, localHit, downloaded, failed));
}
long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(prefetchTimeoutSeconds);
int completed = 0;
boolean deadlineReached = false;
while (!active.isEmpty()) {
long remainingNanos = deadlineNanos - System.nanoTime();
if (remainingNanos <= 0L) {
deadlineReached = true;
break;
}
try {
Future<Object> finished = completion.poll(remainingNanos, TimeUnit.NANOSECONDS);
if (finished == null) {
deadlineReached = true;
break;
}
active.remove(finished);
completed++;
if (pending.hasNext()) {
active.add(submitDiskPrefetch(
completion, pending.next(), imageSpool, localHit, downloaded, failed));
}
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
cancelAll(active);
throw new IllegalStateException("image prefetch interrupted", ex);
}
}
if (deadlineReached) {
cancelAll(active);
}
int skipped = Math.max(0, distinctUrls.size() - completed);
log.info("[similar-asin][image] disk prefetch finished total={} completed={} spooled={} localHit={} downloaded={} failed={} skipped={} deadlineReached={}",
distinctUrls.size(), completed, imageSpool.size(), localHit.get(), downloaded.get(),
failed.get(), skipped, deadlineReached);
}
private Future<?> submitDiskPrefetch(CompletionService<Object> completion,
String url,
ImageSpool imageSpool,
AtomicInteger localHit,
AtomicInteger downloaded,
AtomicInteger failed) {
return completion.submit(() -> {
try {
ResizedImage thumb = readLocalCachedThumb(url);
if (thumb == null) {
thumb = fetchAndResizeDirect(url);
downloaded.incrementAndGet();
} else {
localHit.incrementAndGet();
}
imageSpool.put(url, thumb);
} catch (Exception ex) {
failed.incrementAndGet();
log.debug("[similar-asin][image] disk-prefetch-fail url={} err={}", url, errorSummary(ex));
}
return null;
});
}
private static void cancelAll(Collection<? extends Future<?>> futures) {
for (Future<?> future : futures) {
if (future != null && !future.isDone()) {
future.cancel(true);
}
}
}
/**
* P2-11:给 {@code SimilarAsinImagePrefetchService} 用的对外入口。
* 仅做下载 + resize,不写 taskImageCache(DB cache 由 service 层处理)。
@@ -240,8 +377,8 @@ public class SimilarAsinImageEmbedder {
return null;
}
try {
byte[] raw = downloadWithRetry(trimmed);
return resizeImage(trimmed, raw);
ResizedImage cached = readLocalCachedThumb(trimmed);
return cached != null ? cached : fetchAndResizeDirect(trimmed);
} catch (Exception ex) {
log.debug("[similar-asin][image] prefetch-cache-fail url={} err={}", trimmed, errorSummary(ex));
return null;
@@ -292,8 +429,10 @@ public class SimilarAsinImageEmbedder {
ResizedImage thumb = taskImageCache.get(trimmedUrl);
boolean cacheHit = thumb != null;
if (!cacheHit) {
byte[] raw = downloadWithRetry(trimmedUrl);
thumb = resizeImage(trimmedUrl, raw);
thumb = readLocalCachedThumb(trimmedUrl);
if (thumb == null) {
thumb = fetchAndResizeDirect(trimmedUrl);
}
taskImageCache.put(trimmedUrl, thumb);
log.info("[similar-asin][image] cache-miss url={} bytes={} dims={}x{}",
trimmedUrl, thumb.bytes().length, thumb.width(), thumb.height());
@@ -311,7 +450,7 @@ public class SimilarAsinImageEmbedder {
} catch (ResizeException ex) {
log.warn("[similar-asin][image] resize-fail url={} elapsedMs={} err={}",
trimmedUrl, elapsedMs(startedNanos), errorSummary(ex));
} catch (IOException | TimeoutException ex) {
} catch (IOException ex) {
log.warn("[similar-asin][image] download-fail url={} elapsedMs={} err={}",
trimmedUrl, elapsedMs(startedNanos), errorSummary(ex));
} catch (RuntimeException ex) {
@@ -328,10 +467,31 @@ public class SimilarAsinImageEmbedder {
Row row,
Map<String, ResizedImage> taskImageCache,
ExcelCellImageWriter.Session excelCellImageSession) {
return embedAsExcelCellImage(rowIdx, colIdx, url, row, taskImageCache, excelCellImageSession, null);
}
public ImageDim embedAsExcelCellImage(int rowIdx,
int colIdx,
String url,
Row row,
Map<String, ResizedImage> taskImageCache,
ExcelCellImageWriter.Session excelCellImageSession,
ImageSpool imageSpool) {
if (url == null || url.isBlank() || excelCellImageSession == null) {
return null;
}
String trimmedUrl = url.trim();
SpoolImage spooled = imageSpool == null ? null : imageSpool.get(trimmedUrl);
if (spooled != null) {
Cell cell = row.createCell(colIdx);
cell.setCellValue("#VALUE!");
excelCellImageSession.registerImage(rowIdx, colIdx, spooled.path());
return new ImageDim(spooled.width(), spooled.height());
}
if (imageSpool != null) {
row.createCell(colIdx).setCellValue(trimmedUrl);
return null;
}
ResizedImage thumb = downloadAndResize(trimmedUrl, colIdx, row, taskImageCache);
if (thumb == null) {
return null;
@@ -339,66 +499,139 @@ public class SimilarAsinImageEmbedder {
try {
Cell cell = row.createCell(colIdx);
cell.setCellValue("#VALUE!");
excelCellImageSession.registerImage(rowIdx, colIdx, thumb.bytes());
if (imageSpool == null) {
excelCellImageSession.registerImage(rowIdx, colIdx, thumb.bytes());
} else {
spooled = imageSpool.put(trimmedUrl, thumb);
excelCellImageSession.registerImage(rowIdx, colIdx, spooled.path());
}
taskImageCache.remove(trimmedUrl);
return spooled == null
? new ImageDim(thumb.width(), thumb.height())
: new ImageDim(spooled.width(), spooled.height());
} catch (IOException | RuntimeException ex) {
taskImageCache.remove(trimmedUrl);
return new ImageDim(thumb.width(), thumb.height());
} catch (RuntimeException ex) {
log.warn("[similar-asin][image] excel-cell-image-fail url={} err={}", trimmedUrl, ex.getMessage());
row.createCell(colIdx).setCellValue(trimmedUrl);
return null;
}
}
private byte[] downloadWithRetry(String url) throws IOException, TimeoutException {
private boolean hasLocalCachedThumb(String url) {
Path path = localCachePath(url);
return path != null && Files.isRegularFile(path);
}
private ResizedImage readLocalCachedThumb(String url) {
Path path = localCachePath(url);
if (path == null || !Files.isRegularFile(path)) {
return null;
}
try {
byte[] bytes = Files.readAllBytes(path);
if (bytes.length == 0 || bytes.length > MAX_DOWNLOAD_BYTES) {
Files.deleteIfExists(path);
return null;
}
ResizedImage image = decodeCachedThumb(bytes);
if (image == null) {
Files.deleteIfExists(path);
return null;
}
Files.setLastModifiedTime(path, FileTime.fromMillis(System.currentTimeMillis()));
return image;
} catch (IOException ex) {
log.debug("[similar-asin][image] local-cache-read-fail path={} err={}", path, ex.getMessage());
return null;
}
}
private Path localCachePath(String url) {
if (localImageCacheDir == null || url == null || url.isBlank()) {
return null;
}
String normalized = ossStorageService.normalizeManagedPublicUrl(url.trim());
String hash = sha256Hex(normalized == null || normalized.isBlank() ? url.trim() : normalized);
return localImageCacheDir.resolve(hash.substring(0, 2)).resolve(hash + ".jpg");
}
private void writeLocalCachedThumb(String url, ResizedImage image) {
Path target = localCachePath(url);
if (target == null || image == null || image.bytes() == null || image.bytes().length == 0) {
return;
}
Path temp = null;
try {
Files.createDirectories(target.getParent());
if (Files.isRegularFile(target)) {
return;
}
temp = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".tmp");
Files.write(temp, image.bytes());
try {
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE);
} catch (AtomicMoveNotSupportedException ex) {
Files.move(temp, target);
}
temp = null;
} catch (java.nio.file.FileAlreadyExistsException ignored) {
// Another download of the same normalized URL won the cache write race.
} catch (IOException ex) {
log.debug("[similar-asin][image] local-cache-write-fail path={} err={}", target, ex.getMessage());
} finally {
if (temp != null) {
try {
Files.deleteIfExists(temp);
} catch (IOException ignored) {
// Best-effort cleanup of a failed cache write.
}
}
}
}
private static String sha256Hex(String value) {
try {
byte[] digest = MessageDigest.getInstance("SHA-256")
.digest(value.getBytes(StandardCharsets.UTF_8));
StringBuilder result = new StringBuilder(digest.length * 2);
for (byte b : digest) {
result.append(String.format("%02x", b & 0xFF));
}
return result.toString();
} catch (NoSuchAlgorithmException ex) {
throw new IllegalStateException("SHA-256 unavailable", ex);
}
}
private ResizedImage fetchAndResizeDirect(String url) throws IOException {
String downloadUrl = normalizeAndValidateDownloadUrl(url);
List<String> candidates = downloadCandidates(downloadUrl);
IOException last = null;
TimeoutException lastTimeout = null;
long waitSeconds = downloadTimeoutSeconds * 2L;
for (int attempt = 0; attempt <= DOWNLOAD_MAX_RETRY; attempt++) {
ensureImageWorkNotInterrupted();
String attemptUrl = candidates.get(Math.min(attempt, candidates.size() - 1));
Future<byte[]> future = downloadPool.submit(() -> doFetch(attemptUrl));
try {
byte[] bytes = future.get(waitSeconds, TimeUnit.SECONDS);
byte[] raw = doFetch(attemptUrl);
if (!attemptUrl.equals(downloadUrl)) {
log.info("[similar-asin][image] download-fallback-success originalUrl={} usedUrl={} attempt={}/{}",
downloadUrl, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1);
}
return bytes;
} catch (java.util.concurrent.ExecutionException ee) {
Throwable cause = ee.getCause();
if (cause instanceof DownloadOversizeException doe) {
throw doe;
ResizedImage image = resizeImage(downloadUrl, raw);
writeLocalCachedThumb(downloadUrl, image);
return image;
} catch (DownloadOversizeException ex) {
throw ex;
} catch (IOException ex) {
if (Thread.currentThread().isInterrupted()) {
InterruptedIOException interrupted = new InterruptedIOException("image download interrupted");
interrupted.initCause(ex);
throw interrupted;
}
if (cause instanceof IOException io) {
last = io;
lastTimeout = null;
logRetry(attemptUrl, attempt, io);
continue;
}
if (cause instanceof RuntimeException re) {
throw re;
}
throw new IOException("image download failed: " + cause.getMessage(), cause);
} catch (java.util.concurrent.TimeoutException te) {
future.cancel(true);
TimeoutException wrapped = new TimeoutException("image download timeout after attempt "
+ (attempt + 1) + "/" + (DOWNLOAD_MAX_RETRY + 1)
+ ", waitSeconds=" + waitSeconds
+ ", url=" + attemptUrl);
wrapped.initCause(te);
lastTimeout = wrapped;
last = null;
logRetry(attemptUrl, attempt, wrapped);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
throw new IOException("image download interrupted", ie);
last = ex;
logRetry(attemptUrl, attempt, ex);
}
}
if (lastTimeout != null) {
throw lastTimeout;
}
throw last != null ? last : new IOException("image download failed without cause");
throw last == null ? new IOException("image download failed: " + downloadUrl) : last;
}
String normalizeAndValidateDownloadUrl(String url) {
@@ -503,6 +736,7 @@ public class SimilarAsinImageEmbedder {
}
private byte[] doFetch(String url) throws IOException {
ensureImageWorkNotInterrupted();
Request req = buildImageRequest(url);
try (Response resp = httpClient.newCall(req).execute()) {
if (!resp.isSuccessful()) {
@@ -522,6 +756,7 @@ public class SimilarAsinImageEmbedder {
int total = 0;
int n;
while ((n = in.read(buf)) != -1) {
ensureImageWorkNotInterrupted();
total += n;
if (total > MAX_DOWNLOAD_BYTES) {
throw new DownloadOversizeException(url, total);
@@ -590,6 +825,7 @@ public class SimilarAsinImageEmbedder {
* 仍然超限才抛 ResizeOversizeException 触发文本兜底。
*/
ResizedImage resizeImage(String sourceUrl, byte[] raw) throws IOException {
ensureImageWorkNotInterrupted();
guardImageDimensions(sourceUrl, raw);
BufferedImage src = ImageIO.read(new ByteArrayInputStream(raw));
if (src == null) {
@@ -602,8 +838,11 @@ public class SimilarAsinImageEmbedder {
ResizedImage candidate = null;
ResizedImage smallest = null;
for (int longEdge : FALLBACK_LONG_EDGES) {
ensureImageWorkNotInterrupted();
BufferedImage scaled = scaleAt(src, srcW, srcH, longEdge);
for (float quality : FALLBACK_QUALITIES) {
ResizedImage tried = encodeAt(src, srcW, srcH, longEdge, quality);
ensureImageWorkNotInterrupted();
ResizedImage tried = encodeJpeg(scaled, quality);
if (smallest == null || tried.bytes().length < smallest.bytes().length) {
smallest = tried;
}
@@ -625,7 +864,13 @@ public class SimilarAsinImageEmbedder {
throw new ResizeOversizeException(sourceUrl, reportedSize);
}
private ResizedImage encodeAt(BufferedImage src, int srcW, int srcH, int longEdgePx, float quality) throws IOException {
private static void ensureImageWorkNotInterrupted() throws InterruptedIOException {
if (Thread.currentThread().isInterrupted()) {
throw new InterruptedIOException("image work interrupted");
}
}
private BufferedImage scaleAt(BufferedImage src, int srcW, int srcH, int longEdgePx) {
double ratio = (double) Math.max(srcW, srcH) / longEdgePx;
int dstW = ratio > 1 ? Math.max(1, (int) Math.round(srcW / ratio)) : srcW;
int dstH = ratio > 1 ? Math.max(1, (int) Math.round(srcH / ratio)) : srcH;
@@ -638,6 +883,10 @@ public class SimilarAsinImageEmbedder {
} finally {
g.dispose();
}
return dst;
}
private ResizedImage encodeJpeg(BufferedImage image, float quality) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Iterator<ImageWriter> writers = ImageIO.getImageWritersByFormatName("jpg");
if (!writers.hasNext()) {
@@ -651,14 +900,14 @@ public class SimilarAsinImageEmbedder {
ImageOutputStream ios = ImageIO.createImageOutputStream(baos);
try {
writer.setOutput(ios);
writer.write(null, new IIOImage(dst, null, null), param);
writer.write(null, new IIOImage(image, null, null), param);
} finally {
ios.close();
}
} finally {
writer.dispose();
}
return new ResizedImage(baos.toByteArray(), dstW, dstH);
return new ResizedImage(baos.toByteArray(), image.getWidth(), image.getHeight());
}
/** 缩略图字节 + 实际像素,A 副本随 taskImageCache 生命周期,B 副本进 picture pool。 */
@@ -667,6 +916,85 @@ public class SimilarAsinImageEmbedder {
/** embed() 成功时返回的缩略图实际像素,调用方据此自适应行高/列宽。 */
public record ImageDim(int width, int height) { }
public record SpoolImage(Path path, int width, int height) { }
/** Task-scoped disk storage for resized thumbnails. */
public static final class ImageSpool implements AutoCloseable {
private final Path directory;
private final ConcurrentMap<String, SpoolImage> images = new ConcurrentHashMap<>();
public ImageSpool(Path directory) throws IOException {
this.directory = Objects.requireNonNull(directory, "directory must not be null")
.toAbsolutePath().normalize();
Files.createDirectories(this.directory);
}
public SpoolImage get(String url) {
if (url == null) {
return null;
}
SpoolImage image = images.get(url.trim());
return image != null && Files.isRegularFile(image.path()) ? image : null;
}
public SpoolImage put(String url, ResizedImage image) throws IOException {
if (url == null || url.isBlank() || image == null || image.bytes() == null || image.bytes().length == 0) {
throw new IOException("invalid image spool entry");
}
String key = url.trim();
SpoolImage existing = get(key);
if (existing != null) {
return existing;
}
Path file = Files.createTempFile(directory, "image-", ".jpeg");
boolean retained = false;
try {
Files.write(file, image.bytes());
SpoolImage candidate = new SpoolImage(file, image.width(), image.height());
existing = images.putIfAbsent(key, candidate);
if (existing != null) {
return existing;
}
retained = true;
return candidate;
} finally {
if (!retained) {
Files.deleteIfExists(file);
}
}
}
public int size() {
return images.size();
}
@Override
public void close() throws IOException {
images.clear();
if (!Files.exists(directory)) {
return;
}
IOException failure = null;
try (var paths = Files.walk(directory)) {
for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
try {
Files.deleteIfExists(path);
} catch (IOException ex) {
if (failure == null) {
failure = ex;
} else {
failure.addSuppressed(ex);
}
}
}
}
if (failure != null) {
throw failure;
}
}
}
/** 在解码整张位图前用 ImageReader 仅读取头部尺寸,避免“图像炸弹”导致 heap OOM。 */
private static void guardImageDimensions(String sourceUrl, byte[] raw) throws IOException {
try (ImageInputStream iis = ImageIO.createImageInputStream(new ByteArrayInputStream(raw))) {