新需求更新 同步更新
This commit is contained in:
+66
-18
@@ -47,6 +47,14 @@ public class SimilarAsinCozeClient {
|
||||
}
|
||||
|
||||
public List<SimilarAsinResultRowDto> inspect(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
|
||||
return inspect(rows, prompt, apiKey, imgSwitch, false);
|
||||
}
|
||||
|
||||
public List<SimilarAsinResultRowDto> inspect(List<SimilarAsinResultRowDto> rows,
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
boolean categorySwitch) {
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
@@ -55,7 +63,7 @@ public class SimilarAsinCozeClient {
|
||||
return rows.stream().map(this::copy).toList();
|
||||
}
|
||||
try {
|
||||
return inspectWithFallback(rows, prompt, apiKey, imgSwitch);
|
||||
return inspectWithFallback(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
} catch (Exception ex) {
|
||||
String failureMessage = failureMessage(ex);
|
||||
log.warn("[similar-asin] coze batch failed size={} err={}", rows.size(), failureMessage);
|
||||
@@ -75,9 +83,10 @@ public class SimilarAsinCozeClient {
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
boolean categorySwitch,
|
||||
CozeCredentialRef credential) throws Exception {
|
||||
CozeCredentialRef resolvedCredential = resolveCredential(credential);
|
||||
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, resolvedCredential));
|
||||
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, resolvedCredential));
|
||||
ensureSuccess(submitRoot);
|
||||
return new CozeSubmitResponse(
|
||||
extractExecuteId(submitRoot),
|
||||
@@ -87,6 +96,14 @@ public class SimilarAsinCozeClient {
|
||||
);
|
||||
}
|
||||
|
||||
public CozeSubmitResponse submitWorkflow(List<SimilarAsinResultRowDto> rows,
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
CozeCredentialRef credential) throws Exception {
|
||||
return submitWorkflow(rows, prompt, apiKey, imgSwitch, false, credential);
|
||||
}
|
||||
|
||||
public CozePollResponse pollWorkflow(String executeId) throws Exception {
|
||||
return pollWorkflow(executeId, null);
|
||||
}
|
||||
@@ -119,12 +136,12 @@ public class SimilarAsinCozeClient {
|
||||
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
|
||||
}
|
||||
|
||||
private List<SimilarAsinResultRowDto> inspectWithFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
|
||||
private List<SimilarAsinResultRowDto> inspectWithFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) {
|
||||
try {
|
||||
if (rows.size() == 1) {
|
||||
return inspectSingleRowWithRetry(rows, prompt, apiKey, imgSwitch);
|
||||
return inspectSingleRowWithRetry(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
}
|
||||
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch);
|
||||
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
if (attempt.resolvedCount() < rows.size()) {
|
||||
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
|
||||
}
|
||||
@@ -135,17 +152,17 @@ public class SimilarAsinCozeClient {
|
||||
log.warn("[similar-asin] coze batch fallback split size={} left={} right={} err={}",
|
||||
rows.size(), middle, rows.size() - middle, failureMessage(ex));
|
||||
List<SimilarAsinResultRowDto> merged = new ArrayList<>(rows.size());
|
||||
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, imgSwitch));
|
||||
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, imgSwitch));
|
||||
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, imgSwitch, categorySwitch));
|
||||
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, imgSwitch, categorySwitch));
|
||||
return merged;
|
||||
}
|
||||
throw propagate(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private List<SimilarAsinResultRowDto> inspectPartitionWithFailureFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
|
||||
private List<SimilarAsinResultRowDto> inspectPartitionWithFailureFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) {
|
||||
try {
|
||||
return inspectWithFallback(rows, prompt, apiKey, imgSwitch);
|
||||
return inspectWithFallback(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
} catch (Exception ex) {
|
||||
String failureMessage = failureMessage(ex);
|
||||
log.warn("[similar-asin] coze partition failed size={} err={}", rows.size(), failureMessage);
|
||||
@@ -153,12 +170,12 @@ public class SimilarAsinCozeClient {
|
||||
}
|
||||
}
|
||||
|
||||
private List<SimilarAsinResultRowDto> inspectSingleRowWithRetry(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
|
||||
private List<SimilarAsinResultRowDto> inspectSingleRowWithRetry(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
|
||||
SimilarAsinResultRowDto row = rows.getFirst();
|
||||
PartialCozeResultException lastFailure = null;
|
||||
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
|
||||
try {
|
||||
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch);
|
||||
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
if (attempt.resolvedCount() == rows.size()) {
|
||||
return attempt.mergedRows();
|
||||
}
|
||||
@@ -189,16 +206,16 @@ public class SimilarAsinCozeClient {
|
||||
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
|
||||
}
|
||||
|
||||
private InspectAttempt inspectOnce(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
|
||||
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, imgSwitch);
|
||||
private InspectAttempt inspectOnce(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
|
||||
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
List<CozeResult> results = parseResults(raw);
|
||||
List<SimilarAsinResultRowDto> merged = mergeRows(rows, results);
|
||||
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
|
||||
}
|
||||
|
||||
private String runWorkflowAsyncAndWait(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
|
||||
private String runWorkflowAsyncAndWait(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch, boolean categorySwitch) throws Exception {
|
||||
CozeCredentialRef credential = nextCredential();
|
||||
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, credential));
|
||||
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, credential));
|
||||
ensureSuccess(submitRoot);
|
||||
|
||||
String immediateData = extractResultDataText(submitRoot);
|
||||
@@ -243,8 +260,9 @@ public class SimilarAsinCozeClient {
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
boolean categorySwitch,
|
||||
CozeCredentialRef credential) {
|
||||
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey, imgSwitch);
|
||||
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey, imgSwitch, categorySwitch);
|
||||
Map<String, Object> body = new LinkedHashMap<>();
|
||||
body.put("workflow_id", credential.workflowId());
|
||||
body.put("parameters", parameters);
|
||||
@@ -366,6 +384,14 @@ public class SimilarAsinCozeClient {
|
||||
}
|
||||
|
||||
private Map<String, Object> buildParameters(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
|
||||
return buildParameters(rows, prompt, apiKey, imgSwitch, false);
|
||||
}
|
||||
|
||||
private Map<String, Object> buildParameters(List<SimilarAsinResultRowDto> rows,
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
boolean categorySwitch) {
|
||||
List<String> asins = rows.stream().map(row -> safeText(row.getAsin())).toList();
|
||||
List<String> titles = rows.stream().map(row -> safeText(firstNonBlank(row.getTitle(), row.getAsin()))).toList();
|
||||
List<String> skus = rows.stream().map(row -> safeText(row.getSku())).toList();
|
||||
@@ -388,6 +414,7 @@ public class SimilarAsinCozeClient {
|
||||
parameters.put("api_key", apiKey.trim());
|
||||
}
|
||||
parameters.put("img_switch", imgSwitch);
|
||||
parameters.put("category_switch", categorySwitch);
|
||||
return parameters;
|
||||
}
|
||||
|
||||
@@ -827,7 +854,14 @@ public class SimilarAsinCozeClient {
|
||||
|| !normalize(row.getPatentRisk()).isBlank()
|
||||
|| !normalize(row.getConclusion()).isBlank()
|
||||
|| !normalize(row.getIsStock()).isBlank()
|
||||
|| !normalize(row.getSimilarity()).isBlank();
|
||||
|| !normalize(row.getSimilarity()).isBlank()
|
||||
|| !normalize(row.getIsConform()).isBlank()
|
||||
|| !normalize(row.getReason()).isBlank()
|
||||
|| !normalize(row.getCategory()).isBlank()
|
||||
|| !normalize(row.getStatus()).isBlank()
|
||||
|| !normalize(row.getMainUrl()).isBlank()
|
||||
|| !normalize(row.getPuzzleImg1()).isBlank()
|
||||
|| !normalize(row.getPuzzleImg2()).isBlank();
|
||||
}
|
||||
|
||||
private void ensureSuccess(JsonNode root) {
|
||||
@@ -978,7 +1012,21 @@ public class SimilarAsinCozeClient {
|
||||
|| item.has("appearanceReason")
|
||||
|| item.has("patent_reason")
|
||||
|| item.has("patentReason")
|
||||
|| item.has("patent reason"));
|
||||
|| item.has("patent reason")
|
||||
|| item.has("main_url")
|
||||
|| item.has("mainUrl")
|
||||
|| item.has("main_image_url")
|
||||
|| item.has("mainImageUrl")
|
||||
|| item.has("main_img")
|
||||
|| item.has("mainImg")
|
||||
|| item.has("puzzle_img1")
|
||||
|| item.has("puzzleImg1")
|
||||
|| item.has("puzzle_img_1")
|
||||
|| item.has("puzzleImg_1")
|
||||
|| item.has("puzzle_img2")
|
||||
|| item.has("puzzleImg2")
|
||||
|| item.has("puzzle_img_2")
|
||||
|| item.has("puzzleImg_2"));
|
||||
}
|
||||
|
||||
private boolean isSuccessfulWorkflowStatus(String status) {
|
||||
|
||||
+5
@@ -37,4 +37,9 @@ public class SimilarAsinParseRequest {
|
||||
@JsonAlias({"imgSwitch"})
|
||||
@Schema(description = "传递给 Coze workflow parameters.img_switch 的图片检测开关,true 为开启,false 为关闭。")
|
||||
private Boolean imgSwitch = Boolean.FALSE;
|
||||
|
||||
@JsonProperty("category_switch")
|
||||
@JsonAlias({"categorySwitch"})
|
||||
@Schema(description = "传递给 Coze workflow parameters.category_switch 的类目检测开关,true 为开启,false 为关闭。")
|
||||
private Boolean categorySwitch = Boolean.FALSE;
|
||||
}
|
||||
|
||||
+3
@@ -20,6 +20,9 @@ public class SimilarAsinParsedPayloadDto {
|
||||
@Schema(description = "传递给 Coze workflow parameters.img_switch 的图片检测开关")
|
||||
private Boolean imgSwitch = Boolean.FALSE;
|
||||
|
||||
@Schema(description = "传递给 Coze workflow parameters.category_switch 的类目检测开关")
|
||||
private Boolean categorySwitch = Boolean.FALSE;
|
||||
|
||||
@Schema(description = "本次解析的源文件列表")
|
||||
private List<SimilarAsinSourceFileDto> sourceFiles = new ArrayList<>();
|
||||
|
||||
|
||||
+3
@@ -36,6 +36,9 @@ public class SimilarAsinParseVo {
|
||||
@Schema(description = "图片检测开关,true 为开启,false 为关闭")
|
||||
private Boolean imgSwitch = Boolean.FALSE;
|
||||
|
||||
@Schema(description = "类目检测开关,true 为开启,false 为关闭")
|
||||
private Boolean categorySwitch = Boolean.FALSE;
|
||||
|
||||
@Schema(description = "兼容旧前端的平铺有效行列表,现为全部有效行")
|
||||
private List<SimilarAsinParsedRowVo> items = new ArrayList<>();
|
||||
|
||||
|
||||
+330
-98
@@ -83,6 +83,8 @@ import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
@@ -96,7 +98,7 @@ import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CancellationException;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
@@ -219,6 +221,23 @@ public class SimilarAsinTaskService {
|
||||
"阿里巴巴图片1",
|
||||
"阿里巴巴图片2"
|
||||
);
|
||||
private static final Set<String> CATEGORY_RESULT_HEADER_ALIASES = Set.of(
|
||||
FailedStatusRowFilter.canonicalizeHeader("是否符合类目"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("is_conform"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("conform")
|
||||
);
|
||||
private static final Set<String> COZE_RESULT_HEADER_ALIASES = Set.of(
|
||||
FailedStatusRowFilter.canonicalizeHeader("是否有货"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("is_stock"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("相似度"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("similarity"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("是否符合类目"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("is_conform"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("不符合理由"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("reason"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("产品类目"),
|
||||
FailedStatusRowFilter.canonicalizeHeader("category")
|
||||
);
|
||||
|
||||
private static final int IMG_COL_MAIN = 12;
|
||||
private static final int IMG_COL_PUZZLE1 = 13;
|
||||
@@ -359,6 +378,7 @@ public class SimilarAsinTaskService {
|
||||
List<String> mergedHeaders = new ArrayList<>();
|
||||
int totalRows = 0;
|
||||
int droppedRows = 0;
|
||||
boolean categoryRetryRequired = false;
|
||||
long parseStartedAt = System.nanoTime();
|
||||
for (SimilarAsinSourceFileDto source : sourceFiles) {
|
||||
if (source.getFileKey() == null || source.getFileKey().isBlank()) {
|
||||
@@ -374,10 +394,17 @@ public class SimilarAsinTaskService {
|
||||
droppedRows += parsed.droppedRows();
|
||||
allRows.addAll(parsed.allRows());
|
||||
mergeHeaders(mergedHeaders, parsed.headers());
|
||||
categoryRetryRequired |= parsed.categoryRetryRequired();
|
||||
}
|
||||
if (allRows.isEmpty()) {
|
||||
throw new BusinessException("未解析到有效 ASIN 数据");
|
||||
}
|
||||
boolean requestedCategorySwitch = Boolean.TRUE.equals(request.getCategorySwitch());
|
||||
request.setCategorySwitch(requestedCategorySwitch || categoryRetryRequired);
|
||||
if (!requestedCategorySwitch && categoryRetryRequired) {
|
||||
log.info("[similar-asin] category retry detected from failed result rows, enable category_switch automatically files={} rows={}",
|
||||
sourceFiles.size(), allRows.size());
|
||||
}
|
||||
long parsedAt = System.nanoTime();
|
||||
List<SimilarAsinParsedGroupVo> groups = buildParsedGroups(allRows);
|
||||
long groupedAt = System.nanoTime();
|
||||
@@ -405,11 +432,11 @@ public class SimilarAsinTaskService {
|
||||
|
||||
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
|
||||
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
|
||||
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), sourceFiles, mergedHeaders, groups, allRows);
|
||||
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, mergedHeaders, groups, allRows);
|
||||
long payloadBuiltAt = System.nanoTime();
|
||||
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
|
||||
long payloadStoredAt = System.nanoTime();
|
||||
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), sourceFiles, parsedPayloadPointer));
|
||||
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), request.getCategorySwitch(), sourceFiles, parsedPayloadPointer));
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
|
||||
@@ -449,6 +476,7 @@ public class SimilarAsinTaskService {
|
||||
vo.setGroupCount(groups.size());
|
||||
vo.setAiPrompt(normalize(request.getAiPrompt()));
|
||||
vo.setImgSwitch(Boolean.TRUE.equals(request.getImgSwitch()));
|
||||
vo.setCategorySwitch(Boolean.TRUE.equals(request.getCategorySwitch()));
|
||||
vo.setItems(new ArrayList<>(allRows));
|
||||
vo.setGroups(groups);
|
||||
long finishedAt = System.nanoTime();
|
||||
@@ -1228,10 +1256,11 @@ public class SimilarAsinTaskService {
|
||||
String prompt = readAiPrompt(task);
|
||||
String apiKey = readApiKey(task);
|
||||
boolean imgSwitch = readImgSwitch(task);
|
||||
boolean categorySwitch = readCategorySwitch(task);
|
||||
int batchSize = resolveCozeBatchSize(imgSwitch);
|
||||
List<SimilarAsinResultRowDto> result = new ArrayList<>();
|
||||
for (int i = 0; i < items.size(); i += batchSize) {
|
||||
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, imgSwitch));
|
||||
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, imgSwitch, categorySwitch));
|
||||
if (progressHook != null) {
|
||||
progressHook.run();
|
||||
}
|
||||
@@ -1753,6 +1782,73 @@ public class SimilarAsinTaskService {
|
||||
return true;
|
||||
}
|
||||
|
||||
private boolean readCategorySwitch(FileTaskEntity task) {
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(task.getResultJson());
|
||||
JsonNode savedSwitch = root == null ? null : root.get("categorySwitch");
|
||||
if (savedSwitch != null && !savedSwitch.isNull()) {
|
||||
return savedSwitch.asBoolean(false);
|
||||
}
|
||||
return Boolean.TRUE.equals(readParsedPayload(task).getCategorySwitch());
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取相似ASIN类目检测开关失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void handleResultFileJobFailure(TaskFileJobEntity job, String message) {
|
||||
if (job == null || job.getTaskId() == null) {
|
||||
return;
|
||||
}
|
||||
try (TaskDistributedLockService.LockHandle ignored =
|
||||
requireTaskLock(job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS)) {
|
||||
Long finalizedTaskId = transactionManager == null
|
||||
? persistResultFileJobFailure(job, message)
|
||||
: inNewTransaction(() -> persistResultFileJobFailure(job, message));
|
||||
if (finalizedTaskId != null) {
|
||||
taskCacheService.deleteTaskCache(finalizedTaskId);
|
||||
clearPoisonWindow(finalizedTaskId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Long persistResultFileJobFailure(TaskFileJobEntity job, String message) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || STATUS_SUCCESS.equals(task.getStatus())) {
|
||||
return null;
|
||||
}
|
||||
String owner = ownerFromTask(task);
|
||||
if (!isOwnerCurrent(owner)) {
|
||||
log.warn("[similar-asin] finalize exhausted result job from another instance taskId={} jobId={} owner={} current={}",
|
||||
task.getId(), job.getId(), owner, currentInstanceId());
|
||||
}
|
||||
String failureMessage = firstNonBlank(message, "相似ASIN结果文件生成失败");
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
task.setStatus(STATUS_FAILED);
|
||||
task.setSuccessFileCount(0);
|
||||
task.setFailedFileCount(1);
|
||||
task.setErrorMessage(failureMessage);
|
||||
task.setUpdatedAt(now);
|
||||
task.setFinishedAt(now);
|
||||
fileTaskMapper.updateById(task);
|
||||
|
||||
FileResultEntity result = job.getResultId() == null ? null : fileResultMapper.selectById(job.getResultId());
|
||||
if (result != null && MODULE_TYPE.equals(result.getModuleType())) {
|
||||
result.setSuccess(0);
|
||||
result.setErrorMessage(failureMessage);
|
||||
result.setResultFileUrl(null);
|
||||
result.setResultFileSize(0L);
|
||||
fileResultMapper.updateById(result);
|
||||
}
|
||||
saveFileBuildProgress(task, job, 1, 1, failureMessage);
|
||||
return task.getId();
|
||||
}
|
||||
|
||||
private void finalizeExhaustedResultFileJob(TaskFileJobEntity job, String message) {
|
||||
handleResultFileJobFailure(job, message);
|
||||
taskFileJobService.markFailureFinalized(job.getId(), message);
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelayString = "${aiimage.similar-asin.coze-poll-delay-ms:30000}")
|
||||
public void pollPendingCozeJobs() {
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
@@ -1949,6 +2045,7 @@ public class SimilarAsinTaskService {
|
||||
String prompt = readAiPrompt(task);
|
||||
String apiKey = readApiKey(task);
|
||||
boolean imgSwitch = readImgSwitch(task);
|
||||
boolean categorySwitch = readCategorySwitch(task);
|
||||
int batchSize = resolveCozeBatchSize(imgSwitch);
|
||||
// P1-1:检测到 720712008 风暴时强制把 batch 降到 1,隔离毒行;
|
||||
// 持续 5+ 次提交命中率 ≥ 40% 才会触发,正常波动不影响吞吐。
|
||||
@@ -2013,7 +2110,7 @@ public class SimilarAsinTaskService {
|
||||
int batchIndex = 1;
|
||||
for (int i = 0; i < submitLimit; i += batchSize) {
|
||||
List<CozeCandidate> batchCandidates = readyCandidates.subList(i, Math.min(i + batchSize, submitLimit));
|
||||
pending |= submitCozeBatch(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, allRowsByBaseId);
|
||||
pending |= submitCozeBatch(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, categorySwitch, allRowsByBaseId);
|
||||
batchIndex++;
|
||||
}
|
||||
return pending || countPendingCozeStates(task.getId()) > 0;
|
||||
@@ -2129,6 +2226,7 @@ public class SimilarAsinTaskService {
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
boolean categorySwitch,
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
|
||||
if (batchCandidates == null || batchCandidates.isEmpty()) {
|
||||
return false;
|
||||
@@ -2155,7 +2253,7 @@ public class SimilarAsinTaskService {
|
||||
SimilarAsinCozeClient.CozeCredentialRef credential = cozeClient.nextCredential();
|
||||
try {
|
||||
SimilarAsinCozeClient.CozeSubmitResponse submit = submitCozeWorkflowThrottled(
|
||||
batchRows, prompt, apiKey, imgSwitch, credential, true);
|
||||
batchRows, prompt, apiKey, imgSwitch, categorySwitch, credential, true);
|
||||
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
||||
List<SimilarAsinResultRowDto> cozeRows = cozeClient.mergeRowsFromDataText(batchRows, submit.immediateData());
|
||||
String emptyResultMessage = emptyCozeResultMessage(cozeRows, batchRows.size());
|
||||
@@ -2520,7 +2618,7 @@ public class SimilarAsinTaskService {
|
||||
try {
|
||||
SimilarAsinCozeClient.CozeSubmitResponse submit =
|
||||
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
|
||||
cozeClient.credentialByName(context.credentialName()), false);
|
||||
readCategorySwitch(task), cozeClient.credentialByName(context.credentialName()), false);
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
||||
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
||||
List<SimilarAsinResultRowDto> cozeRows =
|
||||
@@ -2582,6 +2680,7 @@ public class SimilarAsinTaskService {
|
||||
String prompt,
|
||||
String apiKey,
|
||||
boolean imgSwitch,
|
||||
boolean categorySwitch,
|
||||
SimilarAsinCozeClient.CozeCredentialRef credential,
|
||||
boolean allowCredentialFallback) throws Exception {
|
||||
int attempts = allowCredentialFallback ? Math.max(1, cozeClient.configuredCredentialCount()) : 1;
|
||||
@@ -2615,7 +2714,7 @@ public class SimilarAsinTaskService {
|
||||
}
|
||||
try (lockHandle; borrowedCredential) {
|
||||
SimilarAsinCozeClient.CozeSubmitResponse response =
|
||||
cozeClient.submitWorkflow(rows, prompt, apiKey, imgSwitch, currentCredential);
|
||||
cozeClient.submitWorkflow(rows, prompt, apiKey, imgSwitch, categorySwitch, currentCredential);
|
||||
// 提交完成立即记录时间戳,用于下次进入循环时计算节流等待。
|
||||
lastCozeSubmitAtByCredential.put(credentialKey, System.currentTimeMillis());
|
||||
return response;
|
||||
@@ -2807,7 +2906,7 @@ public class SimilarAsinTaskService {
|
||||
int partIndex = 1;
|
||||
for (List<SimilarAsinResultRowDto> partRows : partitions) {
|
||||
SimilarAsinCozeClient.CozeSubmitResponse submit =
|
||||
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
|
||||
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task), readCategorySwitch(task),
|
||||
cozeClient.credentialByName(context.credentialName()), false);
|
||||
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
||||
List<SimilarAsinResultRowDto> cozeRows =
|
||||
@@ -2941,7 +3040,7 @@ public class SimilarAsinTaskService {
|
||||
try {
|
||||
taskFileJobService.touchRunningIfStale(context.jobId(), properties.getDbJobTouchIntervalMillis());
|
||||
SimilarAsinCozeClient.CozeSubmitResponse submit =
|
||||
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
|
||||
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task), readCategorySwitch(task),
|
||||
cozeClient.credentialByName(context.credentialName()), false);
|
||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
||||
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
||||
@@ -3280,6 +3379,9 @@ public class SimilarAsinTaskService {
|
||||
TaskFileJobEntity job = taskFileJobService.findAssembleJob(taskId, MODULE_TYPE, context.resultId());
|
||||
if (job != null) {
|
||||
taskFileJobService.markFailed(job, firstNonBlank(ex.getMessage(), "相似ASIN结果文件生成失败"));
|
||||
if (taskFileJobService.isRetryExhausted(job.getId())) {
|
||||
finalizeExhaustedResultFileJob(job, ex.getMessage());
|
||||
}
|
||||
}
|
||||
log.warn("[相似ASIN] Coze 异步收尾失败 任务ID={} 结果ID={} 错误={}",
|
||||
taskId, context.resultId(), ex.getMessage(), ex);
|
||||
@@ -3303,6 +3405,10 @@ public class SimilarAsinTaskService {
|
||||
if (job == null || "SUCCESS".equals(job.getStatus())) {
|
||||
return;
|
||||
}
|
||||
if (taskFileJobService.isRetryExhausted(job.getId())) {
|
||||
finalizeExhaustedResultFileJob(job, job.getErrorMessage());
|
||||
return;
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task != null && STATUS_RUNNING.equals(task.getStatus()) && !isResultSubmissionComplete(taskId)) {
|
||||
taskFileJobService.touchRunningIfStale(job.getId(), properties.getDbJobTouchIntervalMillis());
|
||||
@@ -3324,6 +3430,8 @@ public class SimilarAsinTaskService {
|
||||
if (requeued) {
|
||||
log.info("[相似ASIN] Coze 异步结果已就绪,结果文件任务已重新入队 任务ID={} 文件任务ID={} 结果ID={}",
|
||||
taskId, job.getId(), context.resultId());
|
||||
} else if (taskFileJobService.isRetryExhausted(job.getId())) {
|
||||
finalizeExhaustedResultFileJob(job, job.getErrorMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4160,6 +4268,7 @@ public class SimilarAsinTaskService {
|
||||
}
|
||||
List<SourceRows> sourceRows = splitRowsBySourceFile(parsed, parsed.getAllItems(), result.getSourceFilename());
|
||||
List<SourceResultWorkbook> workbooks = new ArrayList<>();
|
||||
List<File> workbookFiles = new ArrayList<>(sourceRows.size());
|
||||
File zip = null;
|
||||
// 跨 workbook 共享 taskImageCache:多源场景下相同 URL 仅下载一次。
|
||||
// 配合 embed() 写完即 remove(),cache 仅承载 in-flight 图片;
|
||||
@@ -4170,70 +4279,57 @@ public class SimilarAsinTaskService {
|
||||
: BoundedImageCache.DEFAULT_MAX_BYTES;
|
||||
BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes);
|
||||
try {
|
||||
// P3-1:单源文件场景仍走串行降级路径,避免引入线程切换开销;
|
||||
// 多源文件场景把 writeResultWorkbook 投到 assembleExecutor 上并发跑,
|
||||
// 1000+ 行 ×N 源文件的 assemble 阶段总耗时直接除以 N(受池大小 4 限制)。
|
||||
// assembleExecutor 是固定 4 线程池:sourceRows.size() ≤ 4 时全部并行;
|
||||
// > 4 时多余源文件排队,避免 4×5000 行同时打开图片缓存爆堆。
|
||||
if (sourceRows.size() <= 1) {
|
||||
for (SourceRows item : sourceRows) {
|
||||
String filename = safeFileStem(item.sourceFilename()) + "-result.xlsx";
|
||||
String tempFilename = safeFileStem(item.sourceFilename())
|
||||
+ "-" + task.getId()
|
||||
+ "-" + result.getId()
|
||||
+ "-" + UUID.randomUUID()
|
||||
+ "-result.xlsx";
|
||||
File xlsx = new File(outputDir, tempFilename);
|
||||
writeResultWorkbook(xlsx, item.rows(), resultMap, taskImageCache);
|
||||
workbooks.add(new SourceResultWorkbook(xlsx, filename, item.rows().size()));
|
||||
}
|
||||
} else {
|
||||
long assembleStart = System.currentTimeMillis();
|
||||
List<CompletableFuture<SourceResultWorkbook>> futures = new ArrayList<>(sourceRows.size());
|
||||
for (SourceRows item : sourceRows) {
|
||||
final SourceRows captured = item;
|
||||
final String filename = safeFileStem(captured.sourceFilename()) + "-result.xlsx";
|
||||
final String tempFilename = safeFileStem(captured.sourceFilename())
|
||||
+ "-" + task.getId()
|
||||
+ "-" + result.getId()
|
||||
+ "-" + UUID.randomUUID()
|
||||
+ "-result.xlsx";
|
||||
futures.add(CompletableFuture.supplyAsync(() -> {
|
||||
File xlsx = new File(outputDir, tempFilename);
|
||||
writeResultWorkbook(xlsx, captured.rows(), resultMap, taskImageCache);
|
||||
return new SourceResultWorkbook(xlsx, filename, captured.rows().size());
|
||||
}, assembleExecutor));
|
||||
}
|
||||
for (CompletableFuture<SourceResultWorkbook> future : futures) {
|
||||
try {
|
||||
// 单源文件 30 分钟硬上限:超时直接抛错,避免被慢源永久阻塞。
|
||||
workbooks.add(future.get(30, TimeUnit.MINUTES));
|
||||
} catch (InterruptedException ie) {
|
||||
Thread.currentThread().interrupt();
|
||||
for (CompletableFuture<SourceResultWorkbook> remaining : futures) {
|
||||
remaining.cancel(true);
|
||||
}
|
||||
throw new BusinessException("生成相似ASIN检测结果中断", ie);
|
||||
} catch (TimeoutException te) {
|
||||
for (CompletableFuture<SourceResultWorkbook> remaining : futures) {
|
||||
remaining.cancel(true);
|
||||
}
|
||||
throw new BusinessException("生成相似ASIN检测结果超时(30 分钟)", te);
|
||||
} catch (ExecutionException ee) {
|
||||
Throwable cause = ee.getCause();
|
||||
for (CompletableFuture<SourceResultWorkbook> remaining : futures) {
|
||||
remaining.cancel(true);
|
||||
}
|
||||
if (cause instanceof BusinessException be) {
|
||||
throw be;
|
||||
}
|
||||
throw new BusinessException("生成相似ASIN检测结果失败",
|
||||
cause != null ? cause : ee);
|
||||
}
|
||||
}
|
||||
log.info("[similar-asin] assemble parallel finished taskId={} sources={} costMs={}",
|
||||
task.getId(), sourceRows.size(), System.currentTimeMillis() - assembleStart);
|
||||
// 单源和多源统一提交真实 Future,确保超时取消能中断实际 workbook 线程。
|
||||
// 所有源文件共享同一个绝对 deadline,避免逐个 Future 各等待一轮完整超时。
|
||||
int timeoutMinutes = Math.max(1, properties.getResultFileTimeoutMinutes());
|
||||
long deadlineNanos = System.nanoTime() + TimeUnit.MINUTES.toNanos(timeoutMinutes);
|
||||
long assembleStart = System.currentTimeMillis();
|
||||
List<Future<SourceResultWorkbook>> futures = new ArrayList<>(sourceRows.size());
|
||||
for (SourceRows item : sourceRows) {
|
||||
final SourceRows captured = item;
|
||||
final String filename = safeFileStem(captured.sourceFilename()) + "-result.xlsx";
|
||||
final String tempFilename = safeFileStem(captured.sourceFilename())
|
||||
+ "-" + task.getId()
|
||||
+ "-" + result.getId()
|
||||
+ "-" + UUID.randomUUID()
|
||||
+ "-result.xlsx";
|
||||
final File xlsx = new File(outputDir, tempFilename);
|
||||
workbookFiles.add(xlsx);
|
||||
futures.add(assembleExecutor.submit(() -> {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
writeResultWorkbook(xlsx, captured.rows(), resultMap, taskImageCache);
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
return new SourceResultWorkbook(xlsx, filename, captured.rows().size());
|
||||
}));
|
||||
}
|
||||
try {
|
||||
for (Future<SourceResultWorkbook> future : futures) {
|
||||
long remainingNanos = deadlineNanos - System.nanoTime();
|
||||
if (remainingNanos <= 0L) {
|
||||
throw new TimeoutException("result file deadline reached");
|
||||
}
|
||||
workbooks.add(future.get(remainingNanos, TimeUnit.NANOSECONDS));
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
cancelResultAssemblyFutures(futures);
|
||||
Thread.currentThread().interrupt();
|
||||
throw new BusinessException("生成相似ASIN检测结果中断", ex);
|
||||
} catch (TimeoutException ex) {
|
||||
cancelResultAssemblyFutures(futures);
|
||||
throw new BusinessException("生成相似ASIN检测结果超时(" + timeoutMinutes + " 分钟)", ex);
|
||||
} catch (CancellationException ex) {
|
||||
cancelResultAssemblyFutures(futures);
|
||||
throw new BusinessException("生成相似ASIN检测结果已取消", ex);
|
||||
} catch (ExecutionException ex) {
|
||||
cancelResultAssemblyFutures(futures);
|
||||
Throwable cause = ex.getCause();
|
||||
if (cause instanceof BusinessException businessException) {
|
||||
throw businessException;
|
||||
}
|
||||
throw new BusinessException("生成相似ASIN检测结果失败", cause != null ? cause : ex);
|
||||
}
|
||||
log.info("[similar-asin] assemble workbooks finished taskId={} sources={} timeoutMinutes={} costMs={}",
|
||||
task.getId(), sourceRows.size(), timeoutMinutes, System.currentTimeMillis() - assembleStart);
|
||||
log.info("[similar-asin] assemble image-cache stats taskId={} sources={} cacheSize={} currentBytes={} evictedCount={} evictedBytes={} maxBytes={}",
|
||||
task.getId(), sourceRows.size(), taskImageCache.size(),
|
||||
taskImageCache.currentBytes(), taskImageCache.evictedCount(),
|
||||
@@ -4272,9 +4368,9 @@ public class SimilarAsinTaskService {
|
||||
result.setResultContentType(contentType);
|
||||
result.setRowCount(parsed.getAllItems().size());
|
||||
} finally {
|
||||
for (SourceResultWorkbook workbook : workbooks) {
|
||||
if (workbook.file().exists() && !workbook.file().delete()) {
|
||||
log.warn("[similar-asin] delete temp xlsx failed file={}", workbook.file());
|
||||
for (File workbookFile : workbookFiles) {
|
||||
if (workbookFile.exists() && !workbookFile.delete()) {
|
||||
log.warn("[similar-asin] delete temp xlsx failed file={}", workbookFile);
|
||||
}
|
||||
}
|
||||
if (zip != null && zip.exists() && !zip.delete()) {
|
||||
@@ -4482,10 +4578,19 @@ public class SimilarAsinTaskService {
|
||||
List<SimilarAsinParsedRowVo> rowsToWrite,
|
||||
Map<String, SimilarAsinResultRowDto> resultMap,
|
||||
Map<String, SimilarAsinImageEmbedder.ResizedImage> taskImageCache) {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
// Excel 365 "Place in Cell" 图片:写完 workbook 后 patch richData 单元格图片结构。
|
||||
// 这不是浮动 Drawing,因此点击图片区域会选中单元格,图片不能被拖到任意位置,也不需要工作表保护。
|
||||
ExcelCellImageWriter.Session excelCellImageSession = ExcelCellImageWriter.createSession();
|
||||
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
|
||||
SimilarAsinImageEmbedder.ImageSpool imageSpool;
|
||||
try {
|
||||
Path spoolDir = Files.createTempDirectory(xlsx.getParentFile().toPath(), "similar-asin-images-");
|
||||
imageSpool = new SimilarAsinImageEmbedder.ImageSpool(spoolDir);
|
||||
} catch (IOException ex) {
|
||||
throw new BusinessException("create Similar ASIN image spool failed", ex);
|
||||
}
|
||||
try {
|
||||
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
|
||||
Sheet sheet = workbook.createSheet("相似asin检测");
|
||||
CellStyle headerStyle = workbook.createCellStyle();
|
||||
Font font = workbook.createFont();
|
||||
@@ -4506,17 +4611,16 @@ public class SimilarAsinTaskService {
|
||||
cell.setCellStyle(headerStyle);
|
||||
}
|
||||
|
||||
// P2-8:第一遍预扫所有图片 URL,并行下载到 taskImageCache,
|
||||
// POI 写入仍单线程串行注册 cell image(cache 命中直接 resize + register),下载/写入解耦。
|
||||
// 预扫图片 URL 并行下载到任务级临时目录;POI 仍按行串行注册 cell image。
|
||||
List<String> prefetchUrls = collectImageUrlsForPrefetch(rowsToWrite, resultMap);
|
||||
if (!prefetchUrls.isEmpty()) {
|
||||
// P2-11:先一次性把 DB cache 命中的字节填入 taskImageCache,避免再次走网络。
|
||||
// 命中部分从 prefetchUrls 剔除,剩余的真正未命中的 URL 才走 imageEmbedder.prefetch 网络下载。
|
||||
// DB cache 命中的缩略图也立即落到任务临时目录,避免整批字节停留在堆中。
|
||||
long dbCacheStart = System.currentTimeMillis();
|
||||
List<String> remainingUrls = new ArrayList<>(prefetchUrls.size());
|
||||
int dbHit = 0;
|
||||
for (String url : prefetchUrls) {
|
||||
if (url == null || taskImageCache.containsKey(url)) {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
if (url == null || imageSpool.get(url) != null || taskImageCache.containsKey(url)) {
|
||||
continue;
|
||||
}
|
||||
byte[] cachedBytes = imagePrefetchService.lookup(url);
|
||||
@@ -4526,7 +4630,7 @@ public class SimilarAsinTaskService {
|
||||
}
|
||||
SimilarAsinImageEmbedder.ResizedImage thumb = imageEmbedder.decodeCachedThumb(cachedBytes);
|
||||
if (thumb != null) {
|
||||
taskImageCache.putIfAbsent(url, thumb);
|
||||
imageSpool.put(url, thumb);
|
||||
dbHit++;
|
||||
} else {
|
||||
remainingUrls.add(url);
|
||||
@@ -4536,14 +4640,16 @@ public class SimilarAsinTaskService {
|
||||
prefetchUrls.size(), dbHit, remainingUrls.size(), System.currentTimeMillis() - dbCacheStart);
|
||||
if (!remainingUrls.isEmpty()) {
|
||||
long prefetchStart = System.currentTimeMillis();
|
||||
imageEmbedder.prefetch(remainingUrls, taskImageCache);
|
||||
log.info("[similar-asin] image prefetch finished urls={} cached={} costMs={}",
|
||||
remainingUrls.size(), taskImageCache.size(), System.currentTimeMillis() - prefetchStart);
|
||||
imageEmbedder.prefetchToDisk(remainingUrls, imageSpool);
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
log.info("[similar-asin] image disk-prefetch finished urls={} spooled={} costMs={}",
|
||||
remainingUrls.size(), imageSpool.size(), System.currentTimeMillis() - prefetchStart);
|
||||
}
|
||||
}
|
||||
|
||||
int rowIndex = 1;
|
||||
for (SimilarAsinParsedRowVo parsedRow : rowsToWrite == null ? List.<SimilarAsinParsedRowVo>of() : rowsToWrite) {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
SimilarAsinResultRowDto resultRow = findResultRow(parsedRow, resultMap);
|
||||
Row row = sheet.createRow(rowIndex);
|
||||
int col = 0;
|
||||
@@ -4575,21 +4681,21 @@ public class SimilarAsinTaskService {
|
||||
// 行高固定到 IMAGE_ROW_HEIGHT_POINTS(Excel 上限 409pt):
|
||||
row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS);
|
||||
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_MAIN,
|
||||
resultRow.getMainUrl(), row, taskImageCache, excelCellImageSession);
|
||||
resultRow.getMainUrl(), row, taskImageCache, excelCellImageSession, imageSpool);
|
||||
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE1,
|
||||
resultRow.getPuzzleImg1(), row, taskImageCache, excelCellImageSession);
|
||||
resultRow.getPuzzleImg1(), row, taskImageCache, excelCellImageSession, imageSpool);
|
||||
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE2,
|
||||
resultRow.getPuzzleImg2(), row, taskImageCache, excelCellImageSession);
|
||||
resultRow.getPuzzleImg2(), row, taskImageCache, excelCellImageSession, imageSpool);
|
||||
}
|
||||
rowIndex++;
|
||||
}
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
workbook.write(fos);
|
||||
workbook.dispose();
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("生成相似ASIN检测结果失败", ex);
|
||||
}
|
||||
if (!excelCellImageSession.isEmpty()) {
|
||||
try {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
ExcelCellImageWriter.patchXlsxFile(xlsx, excelCellImageSession);
|
||||
} catch (IOException ex) {
|
||||
String cause = ex.toString();
|
||||
@@ -4598,6 +4704,32 @@ public class SimilarAsinTaskService {
|
||||
throw new BusinessException("写入相似ASIN结果 Excel 单元格图片失败: " + cause, ex);
|
||||
}
|
||||
}
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("生成相似ASIN检测结果失败", ex);
|
||||
} finally {
|
||||
try {
|
||||
imageSpool.close();
|
||||
} catch (IOException ex) {
|
||||
log.warn("[similar-asin] delete image spool failed xlsx={} err={}",
|
||||
xlsx.getAbsolutePath(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void cancelResultAssemblyFutures(List<? extends Future<?>> futures) {
|
||||
for (Future<?> future : futures) {
|
||||
if (future != null && !future.isDone()) {
|
||||
future.cancel(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ensureResultAssemblyNotInterrupted() {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
throw new BusinessException("生成相似ASIN检测结果中断");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -4714,24 +4846,29 @@ public class SimilarAsinTaskService {
|
||||
vo.setValues(readRowValues(row, headers, formatter));
|
||||
allRows.add(vo);
|
||||
}
|
||||
boolean includeBlankStatusRows = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
|
||||
boolean resultWorkbook = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
|
||||
boolean firstPassResultWorkbook = isFirstPassResultWorkbook(headers, allRows);
|
||||
boolean includeBlankStatusRows = resultWorkbook && !firstPassResultWorkbook;
|
||||
FailedStatusRowFilter.FilterResult<SimilarAsinParsedRowVo> filteredRows =
|
||||
FailedStatusRowFilter.retainRows(
|
||||
allRows,
|
||||
statusCol >= 0,
|
||||
rowVo -> statusHeader == null || rowVo.getValues() == null ? "" : rowVo.getValues().get(statusHeader),
|
||||
status -> FailedStatusRowFilter.matchesFailedStatus(status)
|
||||
|| firstPassResultWorkbook
|
||||
|| (includeBlankStatusRows && FailedStatusRowFilter.isBlankStatus(status))
|
||||
);
|
||||
dropped += filteredRows.filteredCount();
|
||||
allRows = new ArrayList<>(filteredRows.rows());
|
||||
boolean categoryRetryRequired = shouldEnableCategorySwitchForRetry(
|
||||
headers, allRows, includeBlankStatusRows);
|
||||
if (statusCol >= 0 && validRows > 0 && allRows.isEmpty()) {
|
||||
throw new BusinessException(FailedStatusRowFilter.noMatchedRowsMessage());
|
||||
}
|
||||
if (allRows.isEmpty()) {
|
||||
throw new BusinessException("no valid similar ASIN rows");
|
||||
}
|
||||
return new ParsedWorkbook(total, dropped, headers, allRows);
|
||||
return new ParsedWorkbook(total, dropped, headers, allRows, categoryRetryRequired);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
@@ -4763,7 +4900,10 @@ public class SimilarAsinTaskService {
|
||||
|| hasUsableCozeField(row.getIsConform())
|
||||
|| hasUsableCozeField(row.getReason())
|
||||
|| hasUsableCozeField(row.getCategory())
|
||||
|| hasUsableCozeField(row.getStatus());
|
||||
|| hasUsableCozeField(row.getStatus())
|
||||
|| hasText(row.getMainUrl())
|
||||
|| hasText(row.getPuzzleImg1())
|
||||
|| hasText(row.getPuzzleImg2());
|
||||
}
|
||||
|
||||
static String resolveResultStatus(SimilarAsinResultRowDto row, String... visibleResultValues) {
|
||||
@@ -4885,7 +5025,16 @@ public class SimilarAsinTaskService {
|
||||
}
|
||||
|
||||
private String cell(Row row, int col, DataFormatter formatter) {
|
||||
return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col)));
|
||||
if (col < 0) {
|
||||
return "";
|
||||
}
|
||||
String value = normalize(formatter.formatCellValue(row.getCell(col)));
|
||||
return isSpreadsheetErrorValue(value) ? "" : value;
|
||||
}
|
||||
|
||||
private static boolean isSpreadsheetErrorValue(String value) {
|
||||
String normalized = value == null ? "" : value.trim();
|
||||
return normalized.startsWith("#") && normalized.endsWith("!");
|
||||
}
|
||||
|
||||
private String baseId(String id) {
|
||||
@@ -5373,11 +5522,12 @@ public class SimilarAsinTaskService {
|
||||
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
private String buildParsedPayloadJson(String aiPrompt, String apiKey, Boolean imgSwitch, List<SimilarAsinSourceFileDto> sourceFiles, List<String> headers, List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
|
||||
private String buildParsedPayloadJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, List<SimilarAsinSourceFileDto> sourceFiles, List<String> headers, List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
|
||||
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
|
||||
payload.setAiPrompt(normalize(aiPrompt));
|
||||
payload.setApiKey(normalize(apiKey));
|
||||
payload.setImgSwitch(Boolean.TRUE.equals(imgSwitch));
|
||||
payload.setCategorySwitch(Boolean.TRUE.equals(categorySwitch));
|
||||
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
|
||||
payload.setHeaders(headers == null ? List.of() : headers);
|
||||
payload.setItems(allRows == null ? List.of() : new ArrayList<>(allRows));
|
||||
@@ -5386,11 +5536,12 @@ public class SimilarAsinTaskService {
|
||||
return writeJson(payload, "保存解析结果失败");
|
||||
}
|
||||
|
||||
private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, List<SimilarAsinSourceFileDto> sourceFiles, String parsedPayloadPointer) {
|
||||
private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, Boolean categorySwitch, List<SimilarAsinSourceFileDto> sourceFiles, String parsedPayloadPointer) {
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put("aiPrompt", normalize(aiPrompt));
|
||||
payload.put("apiKey", normalize(apiKey));
|
||||
payload.put("imgSwitch", Boolean.TRUE.equals(imgSwitch));
|
||||
payload.put("categorySwitch", Boolean.TRUE.equals(categorySwitch));
|
||||
payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream()
|
||||
.map(SimilarAsinSourceFileDto::getFileKey)
|
||||
.filter(Objects::nonNull)
|
||||
@@ -5746,6 +5897,83 @@ public class SimilarAsinTaskService {
|
||||
|| hasHeader(headers, "产品类目"));
|
||||
}
|
||||
|
||||
static boolean isFirstPassResultWorkbook(List<String> headers,
|
||||
List<SimilarAsinParsedRowVo> rows) {
|
||||
if (headers == null || headers.isEmpty() || rows == null || rows.isEmpty()
|
||||
|| FailedStatusRowFilter.findStatusColumnIndex(headers) < 0
|
||||
|| (!hasCanonicalHeader(headers, "是否有货")
|
||||
&& !hasCanonicalHeader(headers, "相似度")
|
||||
&& !hasCanonicalHeader(headers, "是否符合类目")
|
||||
&& !hasCanonicalHeader(headers, "不符合理由")
|
||||
&& !hasCanonicalHeader(headers, "产品类目"))
|
||||
|| (!hasCanonicalHeader(headers, "主图")
|
||||
&& !hasCanonicalHeader(headers, "阿里巴巴图片1")
|
||||
&& !hasCanonicalHeader(headers, "阿里巴巴图片2"))) {
|
||||
return false;
|
||||
}
|
||||
String statusHeader = headers.get(FailedStatusRowFilter.findStatusColumnIndex(headers));
|
||||
for (SimilarAsinParsedRowVo row : rows) {
|
||||
if (row == null || FailedStatusRowFilter.matchesFailedStatus(valueByHeader(row, statusHeader))) {
|
||||
return false;
|
||||
}
|
||||
for (Map.Entry<String, String> entry : row.getValues() == null
|
||||
? Map.<String, String>of().entrySet() : row.getValues().entrySet()) {
|
||||
if (COZE_RESULT_HEADER_ALIASES.contains(FailedStatusRowFilter.canonicalizeHeader(entry.getKey()))
|
||||
&& !isBlankOrSpreadsheetError(entry.getValue())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static boolean hasCanonicalHeader(List<String> headers, String expected) {
|
||||
String canonical = FailedStatusRowFilter.canonicalizeHeader(expected);
|
||||
return headers.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.anyMatch(header -> canonical.equals(FailedStatusRowFilter.canonicalizeHeader(header)));
|
||||
}
|
||||
|
||||
private static String valueByHeader(SimilarAsinParsedRowVo row, String header) {
|
||||
if (row == null || row.getValues() == null) {
|
||||
return "";
|
||||
}
|
||||
String canonical = FailedStatusRowFilter.canonicalizeHeader(header);
|
||||
return row.getValues().entrySet().stream()
|
||||
.filter(entry -> canonical.equals(FailedStatusRowFilter.canonicalizeHeader(entry.getKey())))
|
||||
.map(Map.Entry::getValue)
|
||||
.findFirst()
|
||||
.orElse("");
|
||||
}
|
||||
|
||||
private static boolean isBlankOrSpreadsheetError(String value) {
|
||||
return value == null || value.trim().isBlank() || isSpreadsheetErrorValue(value);
|
||||
}
|
||||
|
||||
static boolean shouldEnableCategorySwitchForRetry(List<String> headers,
|
||||
List<SimilarAsinParsedRowVo> rows,
|
||||
boolean resultWorkbook) {
|
||||
if (!resultWorkbook || headers == null || headers.isEmpty() || rows == null || rows.isEmpty()
|
||||
|| FailedStatusRowFilter.findStatusColumnIndex(headers) < 0) {
|
||||
return false;
|
||||
}
|
||||
String categoryHeader = headers.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(header -> CATEGORY_RESULT_HEADER_ALIASES.contains(
|
||||
FailedStatusRowFilter.canonicalizeHeader(header)))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
if (categoryHeader == null) {
|
||||
return false;
|
||||
}
|
||||
return rows.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(SimilarAsinParsedRowVo::getValues)
|
||||
.anyMatch(values -> values == null
|
||||
|| values.get(categoryHeader) == null
|
||||
|| values.get(categoryHeader).trim().isBlank());
|
||||
}
|
||||
|
||||
private boolean hasHeader(List<String> headers, String candidate) {
|
||||
if (headers == null || headers.isEmpty()) {
|
||||
return false;
|
||||
@@ -5926,6 +6154,10 @@ public class SimilarAsinTaskService {
|
||||
long size) {
|
||||
}
|
||||
|
||||
private record ParsedWorkbook(int totalRows, int droppedRows, List<String> headers, List<SimilarAsinParsedRowVo> allRows) {
|
||||
private record ParsedWorkbook(int totalRows,
|
||||
int droppedRows,
|
||||
List<String> headers,
|
||||
List<SimilarAsinParsedRowVo> allRows,
|
||||
boolean categoryRetryRequired) {
|
||||
}
|
||||
}
|
||||
|
||||
+57
-22
@@ -13,8 +13,10 @@ import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Enumeration;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -41,6 +43,9 @@ public final class ExcelCellImageWriter {
|
||||
private static final String RD_RICH_VALUE_TYPES_PART = "xl/richData/rdRichValueTypes.xml";
|
||||
private static final String RICH_VALUE_REL_PART = "xl/richData/richValueRel.xml";
|
||||
private static final String METADATA_PART = "xl/metadata.xml";
|
||||
private static final Pattern SHEET_CELL_PATTERN = Pattern.compile(
|
||||
"<c\\b(?=[^>]*\\br=\"([A-Z]+[1-9][0-9]*)\")[^>]*(?:/>|>.*?</c>)",
|
||||
Pattern.DOTALL);
|
||||
|
||||
private static final String REL_TYPE_IMAGE =
|
||||
"http://schemas.openxmlformats.org/officeDocument/2006/relationships/image";
|
||||
@@ -116,7 +121,11 @@ public final class ExcelCellImageWriter {
|
||||
writeEntry(zout, METADATA_PART, buildMetadata(session).getBytes(StandardCharsets.UTF_8));
|
||||
int idx = 1;
|
||||
for (RegisteredCellImage image : session.images) {
|
||||
writeEntry(zout, mediaPart(idx), image.bytes);
|
||||
if (image.bytes != null) {
|
||||
writeEntry(zout, mediaPart(idx), image.bytes);
|
||||
} else {
|
||||
writeEntry(zout, mediaPart(idx), image.path);
|
||||
}
|
||||
idx++;
|
||||
}
|
||||
}
|
||||
@@ -166,26 +175,34 @@ public final class ExcelCellImageWriter {
|
||||
|
||||
private static PatchSheetResult patchSheet(byte[] original, Session session) {
|
||||
String content = new String(original, StandardCharsets.UTF_8);
|
||||
int idx = 0;
|
||||
int patchedCount = 0;
|
||||
for (RegisteredCellImage image : session.images) {
|
||||
// [MS-XLSX] §2.2.10: cell @vm 是 1-based valueMetadata 索引,0 表示"无 metadata"。
|
||||
// futureMetadata/valueMetadata/rdrichvalue 内部块仍是 0-based,所以这里写 idx+1
|
||||
// 与 buildMetadata/buildRdRichValue 中的 i 解耦。Excel 365 在 vm=0 时会把 cell
|
||||
// 直接当作无图片的 #VALUE! 错误格渲染(任务 10498 J2 复现),必须改 1-based。
|
||||
String replacement = "<c r=\"" + image.cellRef + "\" t=\"e\" vm=\"" + (idx + 1) + "\"><v>#VALUE!</v></c>";
|
||||
Pattern pattern = Pattern.compile("<c\\b(?=[^>]*\\br=\"" + Pattern.quote(image.cellRef)
|
||||
+ "\")[^>]*(?:/>|>.*?</c>)", Pattern.DOTALL);
|
||||
Matcher matcher = pattern.matcher(content);
|
||||
if (matcher.find()) {
|
||||
content = matcher.replaceFirst(Matcher.quoteReplacement(replacement));
|
||||
patchedCount++;
|
||||
} else {
|
||||
log.warn("[excel-cell-image] cell {} not found in sheet xml, skip image", image.cellRef);
|
||||
}
|
||||
idx++;
|
||||
Map<String, Integer> metadataIndexByCell = new HashMap<>(session.images.size());
|
||||
for (int idx = 0; idx < session.images.size(); idx++) {
|
||||
metadataIndexByCell.put(session.images.get(idx).cellRef, idx + 1);
|
||||
}
|
||||
return new PatchSheetResult(content.getBytes(StandardCharsets.UTF_8), patchedCount);
|
||||
|
||||
Set<String> missingCells = new HashSet<>(metadataIndexByCell.keySet());
|
||||
Matcher matcher = SHEET_CELL_PATTERN.matcher(content);
|
||||
StringBuilder patched = new StringBuilder(content.length());
|
||||
int patchedCount = 0;
|
||||
while (matcher.find()) {
|
||||
String cellRef = matcher.group(1);
|
||||
Integer metadataIndex = metadataIndexByCell.get(cellRef);
|
||||
if (metadataIndex == null) {
|
||||
matcher.appendReplacement(patched, Matcher.quoteReplacement(matcher.group()));
|
||||
continue;
|
||||
}
|
||||
// [MS-XLSX] §2.2.10: cell @vm is a 1-based valueMetadata index.
|
||||
String replacement = "<c r=\"" + cellRef + "\" t=\"e\" vm=\"" + metadataIndex
|
||||
+ "\"><v>#VALUE!</v></c>";
|
||||
matcher.appendReplacement(patched, Matcher.quoteReplacement(replacement));
|
||||
missingCells.remove(cellRef);
|
||||
patchedCount++;
|
||||
}
|
||||
matcher.appendTail(patched);
|
||||
for (String cellRef : missingCells) {
|
||||
log.warn("[excel-cell-image] cell {} not found in sheet xml, skip image", cellRef);
|
||||
}
|
||||
return new PatchSheetResult(patched.toString().getBytes(StandardCharsets.UTF_8), patchedCount);
|
||||
}
|
||||
|
||||
private static byte[] patchContentTypes(byte[] original) {
|
||||
@@ -386,6 +403,16 @@ public final class ExcelCellImageWriter {
|
||||
zout.closeEntry();
|
||||
}
|
||||
|
||||
private static void writeEntry(ZipOutputStream zout, String name, Path path) throws IOException {
|
||||
ZipEntry entry = new ZipEntry(name);
|
||||
zout.putNextEntry(entry);
|
||||
try (InputStream in = Files.newInputStream(path)) {
|
||||
in.transferTo(zout);
|
||||
} finally {
|
||||
zout.closeEntry();
|
||||
}
|
||||
}
|
||||
|
||||
private static void writeDirectoryEntry(ZipOutputStream zout, String name) throws IOException {
|
||||
ZipEntry entry = new ZipEntry(name);
|
||||
zout.putNextEntry(entry);
|
||||
@@ -399,7 +426,15 @@ public final class ExcelCellImageWriter {
|
||||
if (jpegBytes == null || jpegBytes.length == 0) {
|
||||
return;
|
||||
}
|
||||
images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), jpegBytes));
|
||||
images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), jpegBytes, null));
|
||||
}
|
||||
|
||||
public synchronized void registerImage(int rowIdx, int colIdx, Path jpegPath) {
|
||||
if (jpegPath == null || !Files.isRegularFile(jpegPath)) {
|
||||
return;
|
||||
}
|
||||
images.add(new RegisteredCellImage(cellRef(rowIdx, colIdx), null,
|
||||
jpegPath.toAbsolutePath().normalize()));
|
||||
}
|
||||
|
||||
public synchronized boolean isEmpty() {
|
||||
@@ -426,7 +461,7 @@ public final class ExcelCellImageWriter {
|
||||
}
|
||||
}
|
||||
|
||||
private record RegisteredCellImage(String cellRef, byte[] bytes) {
|
||||
private record RegisteredCellImage(String cellRef, byte[] bytes, Path path) {
|
||||
}
|
||||
|
||||
private record PatchSheetResult(byte[] bytes, int patchedCount) {
|
||||
|
||||
+408
-80
@@ -27,14 +27,22 @@ import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InterruptedIOException;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.AtomicMoveNotSupportedException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.nio.file.attribute.FileTime;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
@@ -42,6 +50,8 @@ import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.CompletionService;
|
||||
import java.util.concurrent.ExecutorCompletionService;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
@@ -49,7 +59,6 @@ import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
/**
|
||||
@@ -68,8 +77,9 @@ public class SimilarAsinImageEmbedder {
|
||||
static final int DEFAULT_DOWNLOAD_TIMEOUT_SECONDS = 5;
|
||||
/** P2-10:retry 由 1 升到 2,配合 5s timeout 单图最坏耗时 ≈ 15s。 */
|
||||
static final int DOWNLOAD_MAX_RETRY = 2;
|
||||
/** P2-10:原 8 → P2-8 16 → P2-10 32,1000+ 行 ×3 列场景下显著拉低 assemble 阶段尾延迟。 */
|
||||
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 32;
|
||||
/** 图片下载、解码、缩放都在该池执行;默认限制为 8,避免结果生成占满整机 CPU。 */
|
||||
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 8;
|
||||
static final int DEFAULT_PREFETCH_TIMEOUT_SECONDS = 1800;
|
||||
// 单元格固定尺寸,图片在其中等比缩放(不拉伸);resize 仍按长边 1280 px 控制堆体积。
|
||||
public static final float IMAGE_ROW_HEIGHT_POINTS = 409f;
|
||||
public static final int IMAGE_COL_WIDTH_CHARS = 80;
|
||||
@@ -102,16 +112,28 @@ public class SimilarAsinImageEmbedder {
|
||||
|
||||
private final int downloadTimeoutSeconds;
|
||||
private final int downloadPoolSize;
|
||||
private final int prefetchTimeoutSeconds;
|
||||
private final OkHttpClient httpClient;
|
||||
private final ExecutorService downloadPool;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final Path localImageCacheDir;
|
||||
|
||||
public SimilarAsinImageEmbedder(SimilarAsinProperties properties, OssStorageService ossStorageService) {
|
||||
int rawTimeout = properties == null ? DEFAULT_DOWNLOAD_TIMEOUT_SECONDS : properties.getImageDownloadTimeoutSeconds();
|
||||
int rawPool = properties == null ? DEFAULT_DOWNLOAD_POOL_SIZE : properties.getImageDownloadPoolSize();
|
||||
int rawPrefetchTimeout = properties == null
|
||||
? DEFAULT_PREFETCH_TIMEOUT_SECONDS
|
||||
: properties.getImagePrefetchTimeoutSeconds();
|
||||
this.downloadTimeoutSeconds = rawTimeout > 0 ? rawTimeout : DEFAULT_DOWNLOAD_TIMEOUT_SECONDS;
|
||||
this.downloadPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE;
|
||||
this.prefetchTimeoutSeconds = rawPrefetchTimeout > 0
|
||||
? rawPrefetchTimeout
|
||||
: DEFAULT_PREFETCH_TIMEOUT_SECONDS;
|
||||
this.ossStorageService = Objects.requireNonNull(ossStorageService, "ossStorageService must not be null");
|
||||
String configuredCacheDir = properties == null ? null : properties.getImageLocalCacheDir();
|
||||
this.localImageCacheDir = configuredCacheDir == null || configuredCacheDir.isBlank()
|
||||
? null
|
||||
: Path.of(configuredCacheDir.trim()).toAbsolutePath().normalize();
|
||||
this.httpClient = new OkHttpClient.Builder()
|
||||
.connectTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
|
||||
.readTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
|
||||
@@ -138,6 +160,10 @@ public class SimilarAsinImageEmbedder {
|
||||
return downloadPoolSize;
|
||||
}
|
||||
|
||||
int prefetchTimeoutSeconds() {
|
||||
return prefetchTimeoutSeconds;
|
||||
}
|
||||
|
||||
OkHttpClient httpClient() {
|
||||
return httpClient;
|
||||
}
|
||||
@@ -159,13 +185,13 @@ public class SimilarAsinImageEmbedder {
|
||||
if (urls == null || urls.isEmpty() || taskImageCache == null) {
|
||||
return;
|
||||
}
|
||||
Set<String> distinctUrls = new HashSet<>();
|
||||
Set<String> distinctUrls = new LinkedHashSet<>();
|
||||
for (String url : urls) {
|
||||
if (url == null) {
|
||||
continue;
|
||||
}
|
||||
String trimmed = url.trim();
|
||||
if (trimmed.isEmpty() || taskImageCache.containsKey(trimmed)) {
|
||||
if (trimmed.isEmpty() || taskImageCache.containsKey(trimmed) || hasLocalCachedThumb(trimmed)) {
|
||||
continue;
|
||||
}
|
||||
distinctUrls.add(trimmed);
|
||||
@@ -174,58 +200,169 @@ public class SimilarAsinImageEmbedder {
|
||||
return;
|
||||
}
|
||||
CompletionService<Object> completion = new ExecutorCompletionService<>(downloadPool);
|
||||
List<Future<?>> futures = new ArrayList<>(distinctUrls.size());
|
||||
for (String url : distinctUrls) {
|
||||
// 显式作为 Runnable 提交(带 null result),避免与 Callable<Object> 重载产生歧义。
|
||||
Runnable task = () -> {
|
||||
try {
|
||||
if (taskImageCache.containsKey(url)) {
|
||||
return;
|
||||
}
|
||||
String downloadUrl = normalizeAndValidateDownloadUrl(url);
|
||||
byte[] raw = doFetch(downloadUrl);
|
||||
ResizedImage thumb = resizeImage(downloadUrl, raw);
|
||||
taskImageCache.putIfAbsent(url, thumb);
|
||||
} catch (Exception ex) {
|
||||
// 预下载失败不抛出:embed() 时同 url 会再次尝试并走原有兜底链路。
|
||||
log.debug("[similar-asin][image] prefetch-fail url={} err={}", url, errorSummary(ex));
|
||||
}
|
||||
};
|
||||
futures.add(completion.submit(task, null));
|
||||
Iterator<String> pending = distinctUrls.iterator();
|
||||
List<Future<?>> active = new ArrayList<>(Math.min(downloadPoolSize, distinctUrls.size()));
|
||||
while (pending.hasNext() && active.size() < downloadPoolSize) {
|
||||
active.add(submitPrefetch(completion, pending.next(), taskImageCache));
|
||||
}
|
||||
// 全局 deadline:每 url 给 200ms 的预算,clamp 到 [15s, 120s]。
|
||||
long globalDeadlineMs = Math.min(120_000L, Math.max(15_000L, distinctUrls.size() * 200L));
|
||||
long deadline = System.currentTimeMillis() + globalDeadlineMs;
|
||||
int total = distinctUrls.size();
|
||||
int done = 0;
|
||||
while (done < total) {
|
||||
boolean deadlineReached = false;
|
||||
while (!active.isEmpty()) {
|
||||
long left = deadline - System.currentTimeMillis();
|
||||
if (left <= 0) {
|
||||
deadlineReached = true;
|
||||
break;
|
||||
}
|
||||
try {
|
||||
Future<Object> f = completion.poll(left, TimeUnit.MILLISECONDS);
|
||||
if (f == null) {
|
||||
deadlineReached = true;
|
||||
break;
|
||||
}
|
||||
active.remove(f);
|
||||
done++;
|
||||
if (pending.hasNext()) {
|
||||
active.add(submitPrefetch(completion, pending.next(), taskImageCache));
|
||||
}
|
||||
} catch (InterruptedException ie) {
|
||||
Thread.currentThread().interrupt();
|
||||
cancelAll(active);
|
||||
break;
|
||||
}
|
||||
}
|
||||
// 超时未完成的 future 主动 cancel,避免 idle 持有 OkHttp 连接。
|
||||
if (done < total) {
|
||||
for (Future<?> f : futures) {
|
||||
if (!f.isDone()) {
|
||||
f.cancel(true);
|
||||
}
|
||||
}
|
||||
if (deadlineReached) {
|
||||
cancelAll(active);
|
||||
log.info("[similar-asin][image] prefetch deadline reached total={} done={} cancelled={}",
|
||||
total, done, total - done);
|
||||
}
|
||||
}
|
||||
|
||||
private Future<?> submitPrefetch(CompletionService<Object> completion,
|
||||
String url,
|
||||
Map<String, ResizedImage> taskImageCache) {
|
||||
// 显式作为 Runnable 提交(带 null result),避免与 Callable<Object> 重载产生歧义。
|
||||
Runnable task = () -> {
|
||||
try {
|
||||
if (taskImageCache.containsKey(url)) {
|
||||
return;
|
||||
}
|
||||
String downloadUrl = normalizeAndValidateDownloadUrl(url);
|
||||
byte[] raw = doFetch(downloadUrl);
|
||||
ResizedImage thumb = resizeImage(downloadUrl, raw);
|
||||
taskImageCache.putIfAbsent(url, thumb);
|
||||
} catch (Exception ex) {
|
||||
// 预下载失败不抛出:embed() 时同 url 会再次尝试并走原有兜底链路。
|
||||
log.debug("[similar-asin][image] prefetch-fail url={} err={}", url, errorSummary(ex));
|
||||
}
|
||||
};
|
||||
return completion.submit(task, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads and resizes image URLs concurrently, but spills completed thumbnails to disk instead of
|
||||
* retaining the full batch in the JVM heap.
|
||||
*/
|
||||
public void prefetchToDisk(Collection<String> urls, ImageSpool imageSpool) {
|
||||
if (urls == null || urls.isEmpty() || imageSpool == null) {
|
||||
return;
|
||||
}
|
||||
Set<String> distinctUrls = new LinkedHashSet<>();
|
||||
for (String url : urls) {
|
||||
if (url == null) {
|
||||
continue;
|
||||
}
|
||||
String trimmed = url.trim();
|
||||
if (!trimmed.isEmpty() && imageSpool.get(trimmed) == null) {
|
||||
distinctUrls.add(trimmed);
|
||||
}
|
||||
}
|
||||
if (distinctUrls.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
AtomicInteger localHit = new AtomicInteger();
|
||||
AtomicInteger downloaded = new AtomicInteger();
|
||||
AtomicInteger failed = new AtomicInteger();
|
||||
CompletionService<Object> completion = new ExecutorCompletionService<>(downloadPool);
|
||||
Iterator<String> pending = distinctUrls.iterator();
|
||||
List<Future<?>> active = new ArrayList<>(Math.min(downloadPoolSize, distinctUrls.size()));
|
||||
while (pending.hasNext() && active.size() < downloadPoolSize) {
|
||||
active.add(submitDiskPrefetch(completion, pending.next(), imageSpool, localHit, downloaded, failed));
|
||||
}
|
||||
|
||||
long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(prefetchTimeoutSeconds);
|
||||
int completed = 0;
|
||||
boolean deadlineReached = false;
|
||||
while (!active.isEmpty()) {
|
||||
long remainingNanos = deadlineNanos - System.nanoTime();
|
||||
if (remainingNanos <= 0L) {
|
||||
deadlineReached = true;
|
||||
break;
|
||||
}
|
||||
try {
|
||||
Future<Object> finished = completion.poll(remainingNanos, TimeUnit.NANOSECONDS);
|
||||
if (finished == null) {
|
||||
deadlineReached = true;
|
||||
break;
|
||||
}
|
||||
active.remove(finished);
|
||||
completed++;
|
||||
if (pending.hasNext()) {
|
||||
active.add(submitDiskPrefetch(
|
||||
completion, pending.next(), imageSpool, localHit, downloaded, failed));
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
cancelAll(active);
|
||||
throw new IllegalStateException("image prefetch interrupted", ex);
|
||||
}
|
||||
}
|
||||
if (deadlineReached) {
|
||||
cancelAll(active);
|
||||
}
|
||||
int skipped = Math.max(0, distinctUrls.size() - completed);
|
||||
log.info("[similar-asin][image] disk prefetch finished total={} completed={} spooled={} localHit={} downloaded={} failed={} skipped={} deadlineReached={}",
|
||||
distinctUrls.size(), completed, imageSpool.size(), localHit.get(), downloaded.get(),
|
||||
failed.get(), skipped, deadlineReached);
|
||||
}
|
||||
|
||||
private Future<?> submitDiskPrefetch(CompletionService<Object> completion,
|
||||
String url,
|
||||
ImageSpool imageSpool,
|
||||
AtomicInteger localHit,
|
||||
AtomicInteger downloaded,
|
||||
AtomicInteger failed) {
|
||||
return completion.submit(() -> {
|
||||
try {
|
||||
ResizedImage thumb = readLocalCachedThumb(url);
|
||||
if (thumb == null) {
|
||||
thumb = fetchAndResizeDirect(url);
|
||||
downloaded.incrementAndGet();
|
||||
} else {
|
||||
localHit.incrementAndGet();
|
||||
}
|
||||
imageSpool.put(url, thumb);
|
||||
} catch (Exception ex) {
|
||||
failed.incrementAndGet();
|
||||
log.debug("[similar-asin][image] disk-prefetch-fail url={} err={}", url, errorSummary(ex));
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private static void cancelAll(Collection<? extends Future<?>> futures) {
|
||||
for (Future<?> future : futures) {
|
||||
if (future != null && !future.isDone()) {
|
||||
future.cancel(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* P2-11:给 {@code SimilarAsinImagePrefetchService} 用的对外入口。
|
||||
* 仅做下载 + resize,不写 taskImageCache(DB cache 由 service 层处理)。
|
||||
@@ -240,8 +377,8 @@ public class SimilarAsinImageEmbedder {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
byte[] raw = downloadWithRetry(trimmed);
|
||||
return resizeImage(trimmed, raw);
|
||||
ResizedImage cached = readLocalCachedThumb(trimmed);
|
||||
return cached != null ? cached : fetchAndResizeDirect(trimmed);
|
||||
} catch (Exception ex) {
|
||||
log.debug("[similar-asin][image] prefetch-cache-fail url={} err={}", trimmed, errorSummary(ex));
|
||||
return null;
|
||||
@@ -292,8 +429,10 @@ public class SimilarAsinImageEmbedder {
|
||||
ResizedImage thumb = taskImageCache.get(trimmedUrl);
|
||||
boolean cacheHit = thumb != null;
|
||||
if (!cacheHit) {
|
||||
byte[] raw = downloadWithRetry(trimmedUrl);
|
||||
thumb = resizeImage(trimmedUrl, raw);
|
||||
thumb = readLocalCachedThumb(trimmedUrl);
|
||||
if (thumb == null) {
|
||||
thumb = fetchAndResizeDirect(trimmedUrl);
|
||||
}
|
||||
taskImageCache.put(trimmedUrl, thumb);
|
||||
log.info("[similar-asin][image] cache-miss url={} bytes={} dims={}x{}",
|
||||
trimmedUrl, thumb.bytes().length, thumb.width(), thumb.height());
|
||||
@@ -311,7 +450,7 @@ public class SimilarAsinImageEmbedder {
|
||||
} catch (ResizeException ex) {
|
||||
log.warn("[similar-asin][image] resize-fail url={} elapsedMs={} err={}",
|
||||
trimmedUrl, elapsedMs(startedNanos), errorSummary(ex));
|
||||
} catch (IOException | TimeoutException ex) {
|
||||
} catch (IOException ex) {
|
||||
log.warn("[similar-asin][image] download-fail url={} elapsedMs={} err={}",
|
||||
trimmedUrl, elapsedMs(startedNanos), errorSummary(ex));
|
||||
} catch (RuntimeException ex) {
|
||||
@@ -328,10 +467,31 @@ public class SimilarAsinImageEmbedder {
|
||||
Row row,
|
||||
Map<String, ResizedImage> taskImageCache,
|
||||
ExcelCellImageWriter.Session excelCellImageSession) {
|
||||
return embedAsExcelCellImage(rowIdx, colIdx, url, row, taskImageCache, excelCellImageSession, null);
|
||||
}
|
||||
|
||||
public ImageDim embedAsExcelCellImage(int rowIdx,
|
||||
int colIdx,
|
||||
String url,
|
||||
Row row,
|
||||
Map<String, ResizedImage> taskImageCache,
|
||||
ExcelCellImageWriter.Session excelCellImageSession,
|
||||
ImageSpool imageSpool) {
|
||||
if (url == null || url.isBlank() || excelCellImageSession == null) {
|
||||
return null;
|
||||
}
|
||||
String trimmedUrl = url.trim();
|
||||
SpoolImage spooled = imageSpool == null ? null : imageSpool.get(trimmedUrl);
|
||||
if (spooled != null) {
|
||||
Cell cell = row.createCell(colIdx);
|
||||
cell.setCellValue("#VALUE!");
|
||||
excelCellImageSession.registerImage(rowIdx, colIdx, spooled.path());
|
||||
return new ImageDim(spooled.width(), spooled.height());
|
||||
}
|
||||
if (imageSpool != null) {
|
||||
row.createCell(colIdx).setCellValue(trimmedUrl);
|
||||
return null;
|
||||
}
|
||||
ResizedImage thumb = downloadAndResize(trimmedUrl, colIdx, row, taskImageCache);
|
||||
if (thumb == null) {
|
||||
return null;
|
||||
@@ -339,66 +499,139 @@ public class SimilarAsinImageEmbedder {
|
||||
try {
|
||||
Cell cell = row.createCell(colIdx);
|
||||
cell.setCellValue("#VALUE!");
|
||||
excelCellImageSession.registerImage(rowIdx, colIdx, thumb.bytes());
|
||||
if (imageSpool == null) {
|
||||
excelCellImageSession.registerImage(rowIdx, colIdx, thumb.bytes());
|
||||
} else {
|
||||
spooled = imageSpool.put(trimmedUrl, thumb);
|
||||
excelCellImageSession.registerImage(rowIdx, colIdx, spooled.path());
|
||||
}
|
||||
taskImageCache.remove(trimmedUrl);
|
||||
return spooled == null
|
||||
? new ImageDim(thumb.width(), thumb.height())
|
||||
: new ImageDim(spooled.width(), spooled.height());
|
||||
} catch (IOException | RuntimeException ex) {
|
||||
taskImageCache.remove(trimmedUrl);
|
||||
return new ImageDim(thumb.width(), thumb.height());
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[similar-asin][image] excel-cell-image-fail url={} err={}", trimmedUrl, ex.getMessage());
|
||||
row.createCell(colIdx).setCellValue(trimmedUrl);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] downloadWithRetry(String url) throws IOException, TimeoutException {
|
||||
private boolean hasLocalCachedThumb(String url) {
|
||||
Path path = localCachePath(url);
|
||||
return path != null && Files.isRegularFile(path);
|
||||
}
|
||||
|
||||
private ResizedImage readLocalCachedThumb(String url) {
|
||||
Path path = localCachePath(url);
|
||||
if (path == null || !Files.isRegularFile(path)) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
byte[] bytes = Files.readAllBytes(path);
|
||||
if (bytes.length == 0 || bytes.length > MAX_DOWNLOAD_BYTES) {
|
||||
Files.deleteIfExists(path);
|
||||
return null;
|
||||
}
|
||||
ResizedImage image = decodeCachedThumb(bytes);
|
||||
if (image == null) {
|
||||
Files.deleteIfExists(path);
|
||||
return null;
|
||||
}
|
||||
Files.setLastModifiedTime(path, FileTime.fromMillis(System.currentTimeMillis()));
|
||||
return image;
|
||||
} catch (IOException ex) {
|
||||
log.debug("[similar-asin][image] local-cache-read-fail path={} err={}", path, ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private Path localCachePath(String url) {
|
||||
if (localImageCacheDir == null || url == null || url.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
String normalized = ossStorageService.normalizeManagedPublicUrl(url.trim());
|
||||
String hash = sha256Hex(normalized == null || normalized.isBlank() ? url.trim() : normalized);
|
||||
return localImageCacheDir.resolve(hash.substring(0, 2)).resolve(hash + ".jpg");
|
||||
}
|
||||
|
||||
private void writeLocalCachedThumb(String url, ResizedImage image) {
|
||||
Path target = localCachePath(url);
|
||||
if (target == null || image == null || image.bytes() == null || image.bytes().length == 0) {
|
||||
return;
|
||||
}
|
||||
Path temp = null;
|
||||
try {
|
||||
Files.createDirectories(target.getParent());
|
||||
if (Files.isRegularFile(target)) {
|
||||
return;
|
||||
}
|
||||
temp = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".tmp");
|
||||
Files.write(temp, image.bytes());
|
||||
try {
|
||||
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE);
|
||||
} catch (AtomicMoveNotSupportedException ex) {
|
||||
Files.move(temp, target);
|
||||
}
|
||||
temp = null;
|
||||
} catch (java.nio.file.FileAlreadyExistsException ignored) {
|
||||
// Another download of the same normalized URL won the cache write race.
|
||||
} catch (IOException ex) {
|
||||
log.debug("[similar-asin][image] local-cache-write-fail path={} err={}", target, ex.getMessage());
|
||||
} finally {
|
||||
if (temp != null) {
|
||||
try {
|
||||
Files.deleteIfExists(temp);
|
||||
} catch (IOException ignored) {
|
||||
// Best-effort cleanup of a failed cache write.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static String sha256Hex(String value) {
|
||||
try {
|
||||
byte[] digest = MessageDigest.getInstance("SHA-256")
|
||||
.digest(value.getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder result = new StringBuilder(digest.length * 2);
|
||||
for (byte b : digest) {
|
||||
result.append(String.format("%02x", b & 0xFF));
|
||||
}
|
||||
return result.toString();
|
||||
} catch (NoSuchAlgorithmException ex) {
|
||||
throw new IllegalStateException("SHA-256 unavailable", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private ResizedImage fetchAndResizeDirect(String url) throws IOException {
|
||||
String downloadUrl = normalizeAndValidateDownloadUrl(url);
|
||||
List<String> candidates = downloadCandidates(downloadUrl);
|
||||
IOException last = null;
|
||||
TimeoutException lastTimeout = null;
|
||||
long waitSeconds = downloadTimeoutSeconds * 2L;
|
||||
for (int attempt = 0; attempt <= DOWNLOAD_MAX_RETRY; attempt++) {
|
||||
ensureImageWorkNotInterrupted();
|
||||
String attemptUrl = candidates.get(Math.min(attempt, candidates.size() - 1));
|
||||
Future<byte[]> future = downloadPool.submit(() -> doFetch(attemptUrl));
|
||||
try {
|
||||
byte[] bytes = future.get(waitSeconds, TimeUnit.SECONDS);
|
||||
byte[] raw = doFetch(attemptUrl);
|
||||
if (!attemptUrl.equals(downloadUrl)) {
|
||||
log.info("[similar-asin][image] download-fallback-success originalUrl={} usedUrl={} attempt={}/{}",
|
||||
downloadUrl, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1);
|
||||
}
|
||||
return bytes;
|
||||
} catch (java.util.concurrent.ExecutionException ee) {
|
||||
Throwable cause = ee.getCause();
|
||||
if (cause instanceof DownloadOversizeException doe) {
|
||||
throw doe;
|
||||
ResizedImage image = resizeImage(downloadUrl, raw);
|
||||
writeLocalCachedThumb(downloadUrl, image);
|
||||
return image;
|
||||
} catch (DownloadOversizeException ex) {
|
||||
throw ex;
|
||||
} catch (IOException ex) {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
InterruptedIOException interrupted = new InterruptedIOException("image download interrupted");
|
||||
interrupted.initCause(ex);
|
||||
throw interrupted;
|
||||
}
|
||||
if (cause instanceof IOException io) {
|
||||
last = io;
|
||||
lastTimeout = null;
|
||||
logRetry(attemptUrl, attempt, io);
|
||||
continue;
|
||||
}
|
||||
if (cause instanceof RuntimeException re) {
|
||||
throw re;
|
||||
}
|
||||
throw new IOException("image download failed: " + cause.getMessage(), cause);
|
||||
} catch (java.util.concurrent.TimeoutException te) {
|
||||
future.cancel(true);
|
||||
TimeoutException wrapped = new TimeoutException("image download timeout after attempt "
|
||||
+ (attempt + 1) + "/" + (DOWNLOAD_MAX_RETRY + 1)
|
||||
+ ", waitSeconds=" + waitSeconds
|
||||
+ ", url=" + attemptUrl);
|
||||
wrapped.initCause(te);
|
||||
lastTimeout = wrapped;
|
||||
last = null;
|
||||
logRetry(attemptUrl, attempt, wrapped);
|
||||
} catch (InterruptedException ie) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IOException("image download interrupted", ie);
|
||||
last = ex;
|
||||
logRetry(attemptUrl, attempt, ex);
|
||||
}
|
||||
}
|
||||
if (lastTimeout != null) {
|
||||
throw lastTimeout;
|
||||
}
|
||||
throw last != null ? last : new IOException("image download failed without cause");
|
||||
throw last == null ? new IOException("image download failed: " + downloadUrl) : last;
|
||||
}
|
||||
|
||||
String normalizeAndValidateDownloadUrl(String url) {
|
||||
@@ -503,6 +736,7 @@ public class SimilarAsinImageEmbedder {
|
||||
}
|
||||
|
||||
private byte[] doFetch(String url) throws IOException {
|
||||
ensureImageWorkNotInterrupted();
|
||||
Request req = buildImageRequest(url);
|
||||
try (Response resp = httpClient.newCall(req).execute()) {
|
||||
if (!resp.isSuccessful()) {
|
||||
@@ -522,6 +756,7 @@ public class SimilarAsinImageEmbedder {
|
||||
int total = 0;
|
||||
int n;
|
||||
while ((n = in.read(buf)) != -1) {
|
||||
ensureImageWorkNotInterrupted();
|
||||
total += n;
|
||||
if (total > MAX_DOWNLOAD_BYTES) {
|
||||
throw new DownloadOversizeException(url, total);
|
||||
@@ -590,6 +825,7 @@ public class SimilarAsinImageEmbedder {
|
||||
* 仍然超限才抛 ResizeOversizeException 触发文本兜底。
|
||||
*/
|
||||
ResizedImage resizeImage(String sourceUrl, byte[] raw) throws IOException {
|
||||
ensureImageWorkNotInterrupted();
|
||||
guardImageDimensions(sourceUrl, raw);
|
||||
BufferedImage src = ImageIO.read(new ByteArrayInputStream(raw));
|
||||
if (src == null) {
|
||||
@@ -602,8 +838,11 @@ public class SimilarAsinImageEmbedder {
|
||||
ResizedImage candidate = null;
|
||||
ResizedImage smallest = null;
|
||||
for (int longEdge : FALLBACK_LONG_EDGES) {
|
||||
ensureImageWorkNotInterrupted();
|
||||
BufferedImage scaled = scaleAt(src, srcW, srcH, longEdge);
|
||||
for (float quality : FALLBACK_QUALITIES) {
|
||||
ResizedImage tried = encodeAt(src, srcW, srcH, longEdge, quality);
|
||||
ensureImageWorkNotInterrupted();
|
||||
ResizedImage tried = encodeJpeg(scaled, quality);
|
||||
if (smallest == null || tried.bytes().length < smallest.bytes().length) {
|
||||
smallest = tried;
|
||||
}
|
||||
@@ -625,7 +864,13 @@ public class SimilarAsinImageEmbedder {
|
||||
throw new ResizeOversizeException(sourceUrl, reportedSize);
|
||||
}
|
||||
|
||||
private ResizedImage encodeAt(BufferedImage src, int srcW, int srcH, int longEdgePx, float quality) throws IOException {
|
||||
private static void ensureImageWorkNotInterrupted() throws InterruptedIOException {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
throw new InterruptedIOException("image work interrupted");
|
||||
}
|
||||
}
|
||||
|
||||
private BufferedImage scaleAt(BufferedImage src, int srcW, int srcH, int longEdgePx) {
|
||||
double ratio = (double) Math.max(srcW, srcH) / longEdgePx;
|
||||
int dstW = ratio > 1 ? Math.max(1, (int) Math.round(srcW / ratio)) : srcW;
|
||||
int dstH = ratio > 1 ? Math.max(1, (int) Math.round(srcH / ratio)) : srcH;
|
||||
@@ -638,6 +883,10 @@ public class SimilarAsinImageEmbedder {
|
||||
} finally {
|
||||
g.dispose();
|
||||
}
|
||||
return dst;
|
||||
}
|
||||
|
||||
private ResizedImage encodeJpeg(BufferedImage image, float quality) throws IOException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
Iterator<ImageWriter> writers = ImageIO.getImageWritersByFormatName("jpg");
|
||||
if (!writers.hasNext()) {
|
||||
@@ -651,14 +900,14 @@ public class SimilarAsinImageEmbedder {
|
||||
ImageOutputStream ios = ImageIO.createImageOutputStream(baos);
|
||||
try {
|
||||
writer.setOutput(ios);
|
||||
writer.write(null, new IIOImage(dst, null, null), param);
|
||||
writer.write(null, new IIOImage(image, null, null), param);
|
||||
} finally {
|
||||
ios.close();
|
||||
}
|
||||
} finally {
|
||||
writer.dispose();
|
||||
}
|
||||
return new ResizedImage(baos.toByteArray(), dstW, dstH);
|
||||
return new ResizedImage(baos.toByteArray(), image.getWidth(), image.getHeight());
|
||||
}
|
||||
|
||||
/** 缩略图字节 + 实际像素,A 副本随 taskImageCache 生命周期,B 副本进 picture pool。 */
|
||||
@@ -667,6 +916,85 @@ public class SimilarAsinImageEmbedder {
|
||||
/** embed() 成功时返回的缩略图实际像素,调用方据此自适应行高/列宽。 */
|
||||
public record ImageDim(int width, int height) { }
|
||||
|
||||
public record SpoolImage(Path path, int width, int height) { }
|
||||
|
||||
/** Task-scoped disk storage for resized thumbnails. */
|
||||
public static final class ImageSpool implements AutoCloseable {
|
||||
|
||||
private final Path directory;
|
||||
private final ConcurrentMap<String, SpoolImage> images = new ConcurrentHashMap<>();
|
||||
|
||||
public ImageSpool(Path directory) throws IOException {
|
||||
this.directory = Objects.requireNonNull(directory, "directory must not be null")
|
||||
.toAbsolutePath().normalize();
|
||||
Files.createDirectories(this.directory);
|
||||
}
|
||||
|
||||
public SpoolImage get(String url) {
|
||||
if (url == null) {
|
||||
return null;
|
||||
}
|
||||
SpoolImage image = images.get(url.trim());
|
||||
return image != null && Files.isRegularFile(image.path()) ? image : null;
|
||||
}
|
||||
|
||||
public SpoolImage put(String url, ResizedImage image) throws IOException {
|
||||
if (url == null || url.isBlank() || image == null || image.bytes() == null || image.bytes().length == 0) {
|
||||
throw new IOException("invalid image spool entry");
|
||||
}
|
||||
String key = url.trim();
|
||||
SpoolImage existing = get(key);
|
||||
if (existing != null) {
|
||||
return existing;
|
||||
}
|
||||
Path file = Files.createTempFile(directory, "image-", ".jpeg");
|
||||
boolean retained = false;
|
||||
try {
|
||||
Files.write(file, image.bytes());
|
||||
SpoolImage candidate = new SpoolImage(file, image.width(), image.height());
|
||||
existing = images.putIfAbsent(key, candidate);
|
||||
if (existing != null) {
|
||||
return existing;
|
||||
}
|
||||
retained = true;
|
||||
return candidate;
|
||||
} finally {
|
||||
if (!retained) {
|
||||
Files.deleteIfExists(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return images.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
images.clear();
|
||||
if (!Files.exists(directory)) {
|
||||
return;
|
||||
}
|
||||
IOException failure = null;
|
||||
try (var paths = Files.walk(directory)) {
|
||||
for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
|
||||
try {
|
||||
Files.deleteIfExists(path);
|
||||
} catch (IOException ex) {
|
||||
if (failure == null) {
|
||||
failure = ex;
|
||||
} else {
|
||||
failure.addSuppressed(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (failure != null) {
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 在解码整张位图前用 ImageReader 仅读取头部尺寸,避免“图像炸弹”导致 heap OOM。 */
|
||||
private static void guardImageDimensions(String sourceUrl, byte[] raw) throws IOException {
|
||||
try (ImageInputStream iis = ImageIO.createImageInputStream(new ByteArrayInputStream(raw))) {
|
||||
|
||||
Reference in New Issue
Block a user