后台更新 服务优化

This commit is contained in:
supernijia
2026-08-11 14:50:21 +08:00
parent 641c70d451
commit 91cd02eeb4
24 changed files with 1623 additions and 502 deletions
@@ -47,8 +47,8 @@ import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
public class SimilarAsinImagePrefetchService {
/** P2-11:预热线程池容量。预热不要求高吞吐,4 个线程足够;避免和 embed 阶段抢 IO 资源。 */
private static final int PREFETCH_POOL_SIZE = 4;
/** P2-11:预热线程池容量。与图片处理 CPU 槽位对齐,避免预热路径绕过主池放大并发。 */
private static final int PREFETCH_POOL_SIZE = 2;
/** 排队等待上一个 task future 时的最大等待时间(秒),避免被 hung future 永久卡住。 */
private static final long INFLIGHT_WAIT_SECONDS = 60L;
@@ -104,7 +104,6 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -279,11 +278,32 @@ public class SimilarAsinTaskService {
* 4 线程足以覆盖常见 1-4 源文件;sourceRows.size() == 1 时仍走串行降级路径,
* 避免对单文件场景引入额外线程开销。
*/
private final ExecutorService assembleExecutor = Executors.newFixedThreadPool(4, namedThreadFactory("similar-asin-assemble"));
/**
* Keep at most two source workbooks in POI/image assembly at once. Each
* workbook has its own SXSSF structures and image spool, so four workers
* multiply the peak even though the task itself is a single job.
*/
private final ExecutorService assembleExecutor = Executors.newFixedThreadPool(2, namedThreadFactory("similar-asin-assemble"));
/** Reused poll workers avoid creating/destroying a new executor every 30s. */
private final ExecutorService cozePollPrefetchExecutor = Executors.newFixedThreadPool(
COZE_POLL_PREFETCH_CONCURRENCY, namedThreadFactory("similar-asin-coze-prefetch"));
@PreDestroy
void shutdownAssembleExecutor() {
assembleExecutor.shutdownNow();
cozePollPrefetchExecutor.shutdownNow();
}
/**
* Credential names normally come from configuration, but stale/rotated
* credentials can otherwise leave throttle entries in the process forever.
*/
@Scheduled(fixedDelayString = "${aiimage.similar-asin.coze-submit-throttle-cleanup-delay-ms:600000}")
void cleanupCozeThrottleState() {
long cutoff = System.currentTimeMillis() - Duration.ofHours(1).toMillis();
lastCozeSubmitAtByCredential.entrySet().removeIf(entry ->
entry.getValue() == null || entry.getValue() < cutoff);
}
private static ThreadFactory namedThreadFactory(String prefix) {
@@ -1898,6 +1918,9 @@ public class SimilarAsinTaskService {
if (stateId == null) {
continue;
}
// The task lock is already held here. The outer wrapper
// would perform an extra selectById for every state before
// entering this method.
pollPendingCozeStateLocked(stateId);
}
} finally {
@@ -1932,11 +1955,15 @@ public class SimilarAsinTaskService {
}
// 先批量加载 state,避免每个预取线程各自查 DB;同时筛掉 claim 之前就不可能 poll 的状态。
List<TaskScopeStateEntity> candidates = new ArrayList<>(stateIds.size());
for (Long stateId : stateIds) {
if (stateId == null) {
continue;
}
TaskScopeStateEntity state = taskScopeStateMapper.selectById(stateId);
List<TaskScopeStateEntity> refreshedStates = taskScopeStateMapper.selectList(
new LambdaQueryWrapper<TaskScopeStateEntity>()
.in(TaskScopeStateEntity::getId, stateIds)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.in(TaskScopeStateEntity::getCozeStatus,
List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING)));
for (TaskScopeStateEntity state : refreshedStates == null
? List.<TaskScopeStateEntity>of()
: refreshedStates) {
if (state == null
|| state.getCozeExecuteId() == null
|| state.getCozeExecuteId().isBlank()) {
@@ -1953,56 +1980,42 @@ public class SimilarAsinTaskService {
if (candidates.size() <= 1) {
return;
}
Semaphore concurrency = new Semaphore(Math.max(1, COZE_POLL_PREFETCH_CONCURRENCY));
// 用临时虚拟线程池:调用结束就 close,不影响主调度线程池的容量。
ExecutorService prefetchPool = Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
.name("similar-asin-coze-prefetch-", 0)
.factory());
// 复用固定大小的预取线程池,限制并发并避免每轮调度反复创建线程池。
Map<Long, Future<SimilarAsinCozeClient.CozePollResponse>> futures = new LinkedHashMap<>();
try {
long prefetchStart = System.currentTimeMillis();
for (TaskScopeStateEntity state : candidates) {
final Long stateId = state.getId();
final String executeId = state.getCozeExecuteId();
final CozeBatchContext context = readCozeBatchContext(state);
if (context == null) {
continue;
}
final String credentialName = context.credentialName();
Future<SimilarAsinCozeClient.CozePollResponse> future = prefetchPool.submit(() -> {
concurrency.acquire();
try {
return cozeClient.pollWorkflow(executeId, cozeClient.credentialByName(credentialName));
} finally {
concurrency.release();
}
});
futures.put(stateId, future);
long prefetchStart = System.currentTimeMillis();
for (TaskScopeStateEntity state : candidates) {
final Long stateId = state.getId();
final String executeId = state.getCozeExecuteId();
final CozeBatchContext context = readCozeBatchContext(state);
if (context == null) {
continue;
}
for (Map.Entry<Long, Future<SimilarAsinCozeClient.CozePollResponse>> entry : futures.entrySet()) {
Long stateId = entry.getKey();
Future<SimilarAsinCozeClient.CozePollResponse> future = entry.getValue();
try {
final String credentialName = context.credentialName();
Future<SimilarAsinCozeClient.CozePollResponse> future = cozePollPrefetchExecutor.submit(
() -> cozeClient.pollWorkflow(executeId, cozeClient.credentialByName(credentialName)));
futures.put(stateId, future);
}
for (Map.Entry<Long, Future<SimilarAsinCozeClient.CozePollResponse>> entry : futures.entrySet()) {
Long stateId = entry.getKey();
Future<SimilarAsinCozeClient.CozePollResponse> future = entry.getValue();
try {
SimilarAsinCozeClient.CozePollResponse response =
future.get(COZE_POLL_PREFETCH_WAIT_MILLIS, java.util.concurrent.TimeUnit.MILLISECONDS);
if (response != null) {
cache.cozePollResponseCache.put(stateId, response);
}
} catch (java.util.concurrent.TimeoutException timeoutEx) {
} catch (java.util.concurrent.TimeoutException timeoutEx) {
future.cancel(true);
log.warn("[similar-asin] coze poll prefetch timeout stateId={} waitMs={}",
stateId, COZE_POLL_PREFETCH_WAIT_MILLIS);
} catch (Exception ex) {
} catch (Exception ex) {
log.warn("[similar-asin] coze poll prefetch failed stateId={} err={}",
stateId, firstNonBlank(ex.getMessage(), ex.getClass().getSimpleName()));
}
}
log.info("[similar-asin] coze poll prefetch finished candidates={} cached={} costMs={}",
candidates.size(), cache.cozePollResponseCache.size(),
System.currentTimeMillis() - prefetchStart);
} finally {
prefetchPool.shutdownNow();
}
log.info("[similar-asin] coze poll prefetch finished candidates={} cached={} costMs={}",
candidates.size(), cache.cozePollResponseCache.size(),
System.currentTimeMillis() - prefetchStart);
}
/**
@@ -4228,7 +4241,166 @@ public class SimilarAsinTaskService {
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
}
/**
* Bounded assembly path. Chunk payloads are selected and decoded one at a
* time for the current source file; the result map is released when that
* source workbook finishes. This prevents a multi-source/large-task
* assemble from retaining every chunk JSON and every result DTO together.
*/
private void assembleResultWorkbookBounded(FileTaskEntity task, FileResultEntity result) {
SimilarAsinParsedPayloadDto parsed = readParsedPayload(task);
List<SourceRows> sourceRows = splitRowsBySourceFile(
parsed, parsed.getAllItems(), result.getSourceFilename());
if (sourceRows.isEmpty()) {
throw new BusinessException("Similar ASIN result rows are empty");
}
File outputDir = new File(storageProperties.getLocalTempDir(), "similar-asin-result");
if (!outputDir.exists() && !outputDir.mkdirs()) {
throw new BusinessException("failed to create result directory");
}
List<SourceResultWorkbook> workbooks = new ArrayList<>();
List<File> workbookFiles = new ArrayList<>(sourceRows.size());
File zip = null;
long imageCacheMaxBytes = properties.getImageCacheMaxBytes() > 0
? properties.getImageCacheMaxBytes()
: BoundedImageCache.DEFAULT_MAX_BYTES;
BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes);
try {
int timeoutMinutes = Math.max(1, properties.getResultFileTimeoutMinutes());
long deadlineNanos = System.nanoTime() + TimeUnit.MINUTES.toNanos(timeoutMinutes);
long assembleStart = System.currentTimeMillis();
List<Future<SourceResultWorkbook>> futures = new ArrayList<>(sourceRows.size());
for (SourceRows item : sourceRows) {
final SourceRows captured = item;
final String filename = safeFileStem(captured.sourceFilename()) + "-result.xlsx";
final String tempFilename = safeFileStem(captured.sourceFilename())
+ "-" + task.getId()
+ "-" + result.getId()
+ "-" + UUID.randomUUID()
+ "-result.xlsx";
final File xlsx = new File(outputDir, tempFilename);
workbookFiles.add(xlsx);
futures.add(assembleExecutor.submit(() -> {
ensureResultAssemblyNotInterrupted();
Map<String, SimilarAsinResultRowDto> sourceResultMap =
loadPersistedResultRowsForRowsWithRetry(task.getId(), captured.rows());
int persistedRows = sourceResultMap.size();
sourceResultMap.entrySet().removeIf(entry -> !isExportableResultRow(entry.getValue()));
long resolvedRows = captured.rows().stream()
.filter(row -> findResultRow(row, sourceResultMap) != null)
.count();
int conformPropagated = CozeGroupResultPropagator.propagateByGroup(
captured.rows(),
SimilarAsinParsedRowVo::getDisplayId,
row -> findResultRow(row, sourceResultMap),
SimilarAsinResultRowDto::getIsConform,
SimilarAsinResultRowDto::setIsConform,
List.of("不符合"),
"[similar-asin] taskId=" + task.getId()
);
writeResultWorkbook(xlsx, captured.rows(), sourceResultMap, taskImageCache);
ensureResultAssemblyNotInterrupted();
return new SourceResultWorkbook(
xlsx, filename, captured.rows().size(), persistedRows,
sourceResultMap.size(), resolvedRows, conformPropagated);
}));
}
try {
for (Future<SourceResultWorkbook> future : futures) {
long remainingNanos = deadlineNanos - System.nanoTime();
if (remainingNanos <= 0L) {
throw new TimeoutException("result file deadline reached");
}
workbooks.add(future.get(remainingNanos, TimeUnit.NANOSECONDS));
}
} catch (InterruptedException ex) {
cancelResultAssemblyFutures(futures);
Thread.currentThread().interrupt();
throw new BusinessException("result workbook assembly interrupted", ex);
} catch (TimeoutException ex) {
cancelResultAssemblyFutures(futures);
throw new BusinessException("result workbook assembly timed out", ex);
} catch (CancellationException ex) {
cancelResultAssemblyFutures(futures);
throw new BusinessException("result workbook assembly cancelled", ex);
} catch (ExecutionException ex) {
cancelResultAssemblyFutures(futures);
Throwable cause = ex.getCause();
if (cause instanceof BusinessException businessException) {
throw businessException;
}
throw new BusinessException("result workbook assembly failed", cause == null ? ex : cause);
}
int persistedResultRows = workbooks.stream()
.mapToInt(SourceResultWorkbook::persistedRows)
.sum();
int resultRows = workbooks.stream()
.mapToInt(SourceResultWorkbook::resultRows)
.sum();
long resolvedRows = workbooks.stream()
.mapToLong(SourceResultWorkbook::resolvedRows)
.sum();
int conformPropagated = workbooks.stream()
.mapToInt(SourceResultWorkbook::conformPropagated)
.sum();
log.info("[similar-asin] bounded assemble workbook taskId={} parsedRows={} persistedRows={} resultRows={} resolvedRows={} propagated={} sources={} costMs={}",
task.getId(), parsed.getAllItems().size(), persistedResultRows, resultRows,
resolvedRows, conformPropagated, sourceRows.size(),
System.currentTimeMillis() - assembleStart);
if (resultRows == 0) {
String failureSummary = collectChunkReadFailureSummary(task.getId());
throw new BusinessException(failureSummary.isBlank()
? "Similar ASIN result rows are empty; please retry result generation"
: "Similar ASIN result rows are empty (" + failureSummary + ")");
}
taskImageCache.clear();
File uploadFile;
String filename;
String contentType;
if (workbooks.size() == 1) {
SourceResultWorkbook workbook = workbooks.get(0);
uploadFile = workbook.file();
filename = workbook.filename();
contentType = CONTENT_TYPE_XLSX;
} else {
filename = safeFileStem(result.getSourceFilename()) + "-result.zip";
String tempFilename = safeFileStem(result.getSourceFilename())
+ "-" + task.getId()
+ "-" + result.getId()
+ "-" + UUID.randomUUID()
+ "-result.zip";
zip = new File(outputDir, tempFilename);
packageResultWorkbooksAsZip(zip, workbooks);
uploadFile = zip;
contentType = CONTENT_TYPE_ZIP;
}
String objectKey = ossStorageService.uploadResultFile(uploadFile, MODULE_TYPE);
result.setResultFilename(filename);
result.setResultFileUrl(objectKey);
result.setResultFileSize(uploadFile.length());
result.setResultContentType(contentType);
result.setRowCount(parsed.getAllItems().size());
} finally {
taskImageCache.clear();
for (File workbookFile : workbookFiles) {
if (workbookFile.exists() && !workbookFile.delete()) {
log.warn("[similar-asin] delete temp xlsx failed file={}", workbookFile);
}
}
if (zip != null && zip.exists() && !zip.delete()) {
log.warn("[similar-asin] delete temp zip failed file={}", zip);
}
}
}
private void assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) {
if (properties.isBoundedResultAssemblyEnabled()) {
assembleResultWorkbookBounded(task, result);
return;
}
SimilarAsinParsedPayloadDto parsed = readParsedPayload(task);
Map<String, SimilarAsinResultRowDto> resultMap = loadPersistedResultRowsWithRetry(task.getId(), parsed.getAllItems().size());
int persistedResultRows = resultMap.size();
@@ -4510,6 +4682,152 @@ public class SimilarAsinTaskService {
return normalized.isBlank() ? "similar-asin-result.xlsx" : normalized;
}
private Map<String, SimilarAsinResultRowDto> loadPersistedResultRowsForRowsWithRetry(
Long taskId,
List<SimilarAsinParsedRowVo> requestedRows) {
Map<String, SimilarAsinResultRowDto> result =
loadPersistedResultRowsForRows(taskId, requestedRows);
if (requestedRows == null || requestedRows.isEmpty() || !result.isEmpty()) {
return result;
}
for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) {
sleepBeforeResultRowsRetry(attempt);
result = loadPersistedResultRowsForRows(taskId, requestedRows);
if (!result.isEmpty()) {
log.info("[similar-asin] source result rows recovered after retry taskId={} attempt={} rows={}",
taskId, attempt, result.size());
}
}
return result;
}
/**
* Select and decode one chunk payload at a time. Only rows needed by the
* current source workbook are retained in the returned map.
*/
private Map<String, SimilarAsinResultRowDto> loadPersistedResultRowsForRows(
Long taskId,
List<SimilarAsinParsedRowVo> requestedRows) {
Map<String, SimilarAsinResultRowDto> result = new LinkedHashMap<>();
Set<String> requestedKeys = resultLookupKeys(requestedRows);
if (taskId == null || requestedKeys.isEmpty()) {
return result;
}
Long lastChunkId = null;
while (true) {
LambdaQueryWrapper<TaskChunkEntity> query = new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskChunkEntity::getId)
.last("LIMIT 1");
if (lastChunkId != null) {
query.gt(TaskChunkEntity::getId, lastChunkId);
}
List<TaskChunkEntity> page = taskChunkMapper.selectList(query);
if (page == null || page.isEmpty()) {
break;
}
TaskChunkEntity chunk = page.getFirst();
appendRequestedResultRows(result, readChunkRows(chunk).values(), requestedKeys, false);
lastChunkId = chunk.getId();
if (lastChunkId == null) {
break;
}
}
appendRequestedOrphanRows(taskId, requestedKeys, result);
return result;
}
private Set<String> resultLookupKeys(List<SimilarAsinParsedRowVo> rows) {
Set<String> keys = new LinkedHashSet<>();
for (SimilarAsinParsedRowVo row : rows == null ? List.<SimilarAsinParsedRowVo>of() : rows) {
String primary = rowKey(row);
if (!primary.isBlank()) {
keys.add(primary);
}
String legacy = legacyRowKey(row);
if (!legacy.isBlank()) {
keys.add(legacy);
}
}
return keys;
}
private void appendRequestedResultRows(Map<String, SimilarAsinResultRowDto> sink,
Iterable<SimilarAsinResultRowDto> rows,
Set<String> requestedKeys,
boolean onlyIfAbsent) {
if (rows == null || requestedKeys == null || requestedKeys.isEmpty()) {
return;
}
for (SimilarAsinResultRowDto row : rows) {
if (row == null) {
continue;
}
String primary = rowKey(row);
String legacy = legacyRowKey(row);
String matchedKey = requestedKeys.contains(primary)
? primary
: requestedKeys.contains(legacy) ? legacy : "";
if (matchedKey.isBlank()) {
continue;
}
if (onlyIfAbsent) {
sink.putIfAbsent(matchedKey, row);
} else {
sink.put(matchedKey, row);
}
}
}
private void appendRequestedOrphanRows(Long taskId,
Set<String> requestedKeys,
Map<String, SimilarAsinResultRowDto> sink) {
Long lastStateId = null;
while (true) {
LambdaQueryWrapper<TaskScopeStateEntity> query = new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getId,
TaskScopeStateEntity::getParsedPayloadJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.likeRight(TaskScopeStateEntity::getScopeKey, ORPHAN_SCOPE_KEY_PREFIX)
.orderByAsc(TaskScopeStateEntity::getId)
.last("LIMIT 1");
if (lastStateId != null) {
query.gt(TaskScopeStateEntity::getId, lastStateId);
}
List<TaskScopeStateEntity> page = taskScopeStateMapper.selectList(query);
if (page == null || page.isEmpty()) {
return;
}
TaskScopeStateEntity state = page.getFirst();
lastStateId = state.getId();
if (lastStateId == null) {
return;
}
if (state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) {
continue;
}
try {
String payloadJson = transientPayloadStorageService.resolvePayload(
state.getParsedPayloadJson(), "read similar ASIN orphan rows failed");
JsonNode array = objectMapper.readTree(payloadJson);
if (!array.isArray()) {
continue;
}
List<SimilarAsinResultRowDto> matchingRows = new ArrayList<>();
for (JsonNode node : array) {
matchingRows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class));
}
appendRequestedResultRows(sink, matchingRows, requestedKeys, true);
} catch (Exception ex) {
log.warn("[similar-asin] read requested orphan rows failed taskId={} stateId={} err={}",
taskId, state.getId(), ex.getMessage());
}
}
}
private Map<String, SimilarAsinResultRowDto> loadPersistedResultRows(Long taskId) {
Map<String, SimilarAsinResultRowDto> result = new LinkedHashMap<>();
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
@@ -6145,7 +6463,14 @@ public class SimilarAsinTaskService {
private record SourceResultWorkbook(File file,
String filename,
int rowCount) {
int rowCount,
int persistedRows,
int resultRows,
long resolvedRows,
int conformPropagated) {
private SourceResultWorkbook(File file, String filename, int rowCount) {
this(file, filename, rowCount, 0, 0, 0L, 0);
}
}
public record ResultDownloadInfo(String url,
@@ -15,6 +15,7 @@ import org.springframework.stereotype.Component;
import javax.imageio.IIOImage;
import javax.imageio.ImageIO;
import javax.imageio.ImageReadParam;
import javax.imageio.ImageReader;
import javax.imageio.ImageWriteParam;
import javax.imageio.ImageWriter;
@@ -50,13 +51,16 @@ import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -77,8 +81,8 @@ public class SimilarAsinImageEmbedder {
static final int DEFAULT_DOWNLOAD_TIMEOUT_SECONDS = 5;
/** P2-10:retry 由 1 升到 2,配合 5s timeout 单图最坏耗时 ≈ 15s。 */
static final int DOWNLOAD_MAX_RETRY = 2;
/** 图片下载、解码、缩放都在该池执行;默认限制为 8,避免结果生成占满整机 CPU。 */
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 8;
/** 4 核生产机默认 2;构造时还会按 JVM 可见 CPU 数的一半做硬上限。 */
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 2;
static final int DEFAULT_PREFETCH_TIMEOUT_SECONDS = 1800;
// 单元格固定尺寸,图片在其中等比缩放(不拉伸);resize 仍按长边 1280 px 控制堆体积。
public static final float IMAGE_ROW_HEIGHT_POINTS = 409f;
@@ -115,6 +119,8 @@ public class SimilarAsinImageEmbedder {
private final int prefetchTimeoutSeconds;
private final OkHttpClient httpClient;
private final ExecutorService downloadPool;
private final Semaphore imageProcessingSlots;
private final ConcurrentMap<String, CompletableFuture<ResizedImage>> inFlightResizes = new ConcurrentHashMap<>();
private final OssStorageService ossStorageService;
private final Path localImageCacheDir;
@@ -125,7 +131,16 @@ public class SimilarAsinImageEmbedder {
? DEFAULT_PREFETCH_TIMEOUT_SECONDS
: properties.getImagePrefetchTimeoutSeconds();
this.downloadTimeoutSeconds = rawTimeout > 0 ? rawTimeout : DEFAULT_DOWNLOAD_TIMEOUT_SECONDS;
this.downloadPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE;
int requestedPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE;
int cpuBoundPoolSize = cpuBoundPoolLimit();
this.downloadPoolSize = Math.min(requestedPoolSize, cpuBoundPoolSize);
if (requestedPoolSize > this.downloadPoolSize) {
log.warn("[similar-asin][image] image pool clamped requested={} effective={} visibleProcessors={}",
requestedPoolSize, this.downloadPoolSize, Runtime.getRuntime().availableProcessors());
}
this.imageProcessingSlots = new Semaphore(this.downloadPoolSize, true);
log.info("[similar-asin][image] image pool initialized requested={} effective={} cpuBoundLimit={}",
requestedPoolSize, this.downloadPoolSize, cpuBoundPoolSize);
this.prefetchTimeoutSeconds = rawPrefetchTimeout > 0
? rawPrefetchTimeout
: DEFAULT_PREFETCH_TIMEOUT_SECONDS;
@@ -152,6 +167,11 @@ public class SimilarAsinImageEmbedder {
downloadPool.shutdownNow();
}
static int cpuBoundPoolLimit() {
int visibleProcessors = Math.max(1, Runtime.getRuntime().availableProcessors());
return Math.max(1, (visibleProcessors + 1) / 2);
}
int downloadTimeoutSeconds() {
return downloadTimeoutSeconds;
}
@@ -251,9 +271,7 @@ public class SimilarAsinImageEmbedder {
if (taskImageCache.containsKey(url)) {
return;
}
String downloadUrl = normalizeAndValidateDownloadUrl(url);
byte[] raw = doFetch(downloadUrl);
ResizedImage thumb = resizeImage(downloadUrl, raw);
ResizedImage thumb = fetchAndResizeDirect(url);
taskImageCache.putIfAbsent(url, thumb);
} catch (Exception ex) {
// 预下载失败不抛出:embed() 时同 url 会再次尝试并走原有兜底链路。
@@ -605,6 +623,25 @@ public class SimilarAsinImageEmbedder {
private ResizedImage fetchAndResizeDirect(String url) throws IOException {
String downloadUrl = normalizeAndValidateDownloadUrl(url);
CompletableFuture<ResizedImage> owned = new CompletableFuture<>();
CompletableFuture<ResizedImage> existing = inFlightResizes.putIfAbsent(downloadUrl, owned);
if (existing != null) {
return awaitInFlightResize(downloadUrl, existing);
}
try {
ResizedImage cached = readLocalCachedThumb(downloadUrl);
ResizedImage result = cached != null ? cached : fetchAndResizeUncached(downloadUrl);
owned.complete(result);
return result;
} catch (IOException | RuntimeException | Error ex) {
owned.completeExceptionally(ex);
throw ex;
} finally {
inFlightResizes.remove(downloadUrl, owned);
}
}
private ResizedImage fetchAndResizeUncached(String downloadUrl) throws IOException {
List<String> candidates = downloadCandidates(downloadUrl);
IOException last = null;
for (int attempt = 0; attempt <= DOWNLOAD_MAX_RETRY; attempt++) {
@@ -634,6 +671,30 @@ public class SimilarAsinImageEmbedder {
throw last == null ? new IOException("image download failed: " + downloadUrl) : last;
}
private static ResizedImage awaitInFlightResize(String url,
CompletableFuture<ResizedImage> future) throws IOException {
try {
return future.get();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
InterruptedIOException interrupted = new InterruptedIOException("image resize wait interrupted url=" + url);
interrupted.initCause(ex);
throw interrupted;
} catch (ExecutionException ex) {
Throwable cause = ex.getCause();
if (cause instanceof IOException ioException) {
throw ioException;
}
if (cause instanceof RuntimeException runtimeException) {
throw runtimeException;
}
if (cause instanceof Error error) {
throw error;
}
throw new IOException("image resize failed url=" + url, cause);
}
}
String normalizeAndValidateDownloadUrl(String url) {
String downloadUrl = ossStorageService.normalizeManagedPublicUrl(url);
validateHttpsUrl(downloadUrl);
@@ -825,43 +886,105 @@ public class SimilarAsinImageEmbedder {
* 仍然超限才抛 ResizeOversizeException 触发文本兜底。
*/
ResizedImage resizeImage(String sourceUrl, byte[] raw) throws IOException {
ensureImageWorkNotInterrupted();
guardImageDimensions(sourceUrl, raw);
BufferedImage src = ImageIO.read(new ByteArrayInputStream(raw));
if (src == null) {
throw new IOException("unsupported image format url=" + sourceUrl);
}
int srcW = src.getWidth();
int srcH = src.getHeight();
// B 方案降级顺序:固定 MAX_THUMB_SIZE 上限 → 优先调整长边像素 → 再调质量。
// 同一 src BufferedImage 解码一次,下面 9 种组合复用,避免重复 ImageIO.read。
ResizedImage candidate = null;
ResizedImage smallest = null;
for (int longEdge : FALLBACK_LONG_EDGES) {
acquireImageProcessingSlot();
try {
ensureImageWorkNotInterrupted();
BufferedImage scaled = scaleAt(src, srcW, srcH, longEdge);
for (float quality : FALLBACK_QUALITIES) {
ensureImageWorkNotInterrupted();
ResizedImage tried = encodeJpeg(scaled, quality);
if (smallest == null || tried.bytes().length < smallest.bytes().length) {
smallest = tried;
BufferedImage src = decodeForResize(sourceUrl, raw);
try {
int srcW = src.getWidth();
int srcH = src.getHeight();
ResizedImage candidate = null;
ResizedImage smallest = null;
for (int longEdge : FALLBACK_LONG_EDGES) {
ensureImageWorkNotInterrupted();
BufferedImage scaled = scaleAt(src, srcW, srcH, longEdge);
try {
for (float quality : FALLBACK_QUALITIES) {
ensureImageWorkNotInterrupted();
ResizedImage tried = encodeJpeg(scaled, quality);
if (smallest == null || tried.bytes().length < smallest.bytes().length) {
smallest = tried;
}
if (tried.bytes().length <= MAX_THUMB_SIZE_BYTES) {
candidate = tried;
break;
}
}
} finally {
scaled.flush();
}
if (candidate != null) {
break;
}
}
if (tried.bytes().length <= MAX_THUMB_SIZE_BYTES) {
candidate = tried;
break;
if (candidate != null) {
return candidate;
}
int reportedSize = smallest != null ? smallest.bytes().length : -1;
throw new ResizeOversizeException(sourceUrl, reportedSize);
} finally {
src.flush();
}
if (candidate != null) {
break;
} finally {
imageProcessingSlots.release();
}
}
private void acquireImageProcessingSlot() throws InterruptedIOException {
try {
imageProcessingSlots.acquire();
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
InterruptedIOException interrupted = new InterruptedIOException("image processing interrupted");
interrupted.initCause(ex);
throw interrupted;
}
}
private static BufferedImage decodeForResize(String sourceUrl, byte[] raw) throws IOException {
try (ImageInputStream iis = ImageIO.createImageInputStream(new ByteArrayInputStream(raw))) {
if (iis == null) {
throw new IOException("unable to create ImageInputStream url=" + sourceUrl);
}
Iterator<ImageReader> readers = ImageIO.getImageReaders(iis);
if (!readers.hasNext()) {
throw new IOException("unsupported image format url=" + sourceUrl);
}
ImageReader reader = readers.next();
try {
reader.setInput(iis, true, true);
int sourceWidth = reader.getWidth(0);
int sourceHeight = reader.getHeight(0);
long pixels = (long) sourceWidth * (long) sourceHeight;
if (sourceWidth <= 0 || sourceHeight <= 0 || pixels > MAX_DECODE_PIXELS) {
throw new ResizeException("image too large url=" + sourceUrl + " pixels=" + pixels);
}
ImageReadParam readParam = reader.getDefaultReadParam();
if (isJpegReader(reader)) {
int subsampling = jpegSourceSubsampling(sourceWidth, sourceHeight);
if (subsampling > 1) {
readParam.setSourceSubsampling(subsampling, subsampling, 0, 0);
}
}
BufferedImage decoded = reader.read(0, readParam);
if (decoded == null) {
throw new IOException("unsupported image format url=" + sourceUrl);
}
return decoded;
} finally {
reader.dispose();
}
}
if (candidate != null) {
return candidate;
}
// 9 组合都没压到上限:抛 ResizeOversizeException 走文本兜底,
// 同时报告 smallest 字节让运维直观知道当前压缩极限。
int reportedSize = smallest != null ? smallest.bytes().length : -1;
throw new ResizeOversizeException(sourceUrl, reportedSize);
}
private static boolean isJpegReader(ImageReader reader) throws IOException {
String format = reader.getFormatName();
return "JPEG".equalsIgnoreCase(format) || "JPG".equalsIgnoreCase(format);
}
static int jpegSourceSubsampling(int sourceWidth, int sourceHeight) {
int ratio = Math.max(sourceWidth, sourceHeight) / TARGET_LONG_EDGE_PX;
return ratio > 1 ? Integer.highestOneBit(ratio) : 1;
}
private static void ensureImageWorkNotInterrupted() throws InterruptedIOException {
@@ -995,29 +1118,6 @@ public class SimilarAsinImageEmbedder {
}
}
/** 在解码整张位图前用 ImageReader 仅读取头部尺寸,避免“图像炸弹”导致 heap OOM。 */
private static void guardImageDimensions(String sourceUrl, byte[] raw) throws IOException {
try (ImageInputStream iis = ImageIO.createImageInputStream(new ByteArrayInputStream(raw))) {
if (iis == null) {
throw new IOException("unable to create ImageInputStream url=" + sourceUrl);
}
Iterator<ImageReader> readers = ImageIO.getImageReaders(iis);
if (!readers.hasNext()) {
throw new IOException("unsupported image format url=" + sourceUrl);
}
ImageReader reader = readers.next();
try {
reader.setInput(iis, true, true);
long pixels = (long) reader.getWidth(0) * (long) reader.getHeight(0);
if (pixels > MAX_DECODE_PIXELS) {
throw new ResizeException("image too large url=" + sourceUrl + " pixels=" + pixels);
}
} finally {
reader.dispose();
}
}
}
private static ThreadFactory namedFactory(String prefix) {
AtomicInteger counter = new AtomicInteger();
return r -> {