后台更新 服务优化
This commit is contained in:
+2
-2
@@ -47,8 +47,8 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
@Slf4j
|
||||
public class SimilarAsinImagePrefetchService {
|
||||
|
||||
/** P2-11:预热线程池容量。预热不要求高吞吐,4 个线程足够;避免和 embed 阶段抢 IO 资源。 */
|
||||
private static final int PREFETCH_POOL_SIZE = 4;
|
||||
/** P2-11:预热线程池容量。与图片处理 CPU 槽位对齐,避免预热路径绕过主池放大并发。 */
|
||||
private static final int PREFETCH_POOL_SIZE = 2;
|
||||
|
||||
/** 排队等待上一个 task future 时的最大等待时间(秒),避免被 hung future 永久卡住。 */
|
||||
private static final long INFLIGHT_WAIT_SECONDS = 60L;
|
||||
|
||||
+369
-44
@@ -104,7 +104,6 @@ import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
@@ -279,11 +278,32 @@ public class SimilarAsinTaskService {
|
||||
* 4 线程足以覆盖常见 1-4 源文件;sourceRows.size() == 1 时仍走串行降级路径,
|
||||
* 避免对单文件场景引入额外线程开销。
|
||||
*/
|
||||
private final ExecutorService assembleExecutor = Executors.newFixedThreadPool(4, namedThreadFactory("similar-asin-assemble"));
|
||||
/**
|
||||
* Keep at most two source workbooks in POI/image assembly at once. Each
|
||||
* workbook has its own SXSSF structures and image spool, so four workers
|
||||
* multiply the peak even though the task itself is a single job.
|
||||
*/
|
||||
private final ExecutorService assembleExecutor = Executors.newFixedThreadPool(2, namedThreadFactory("similar-asin-assemble"));
|
||||
|
||||
/** Reused poll workers avoid creating/destroying a new executor every 30s. */
|
||||
private final ExecutorService cozePollPrefetchExecutor = Executors.newFixedThreadPool(
|
||||
COZE_POLL_PREFETCH_CONCURRENCY, namedThreadFactory("similar-asin-coze-prefetch"));
|
||||
|
||||
@PreDestroy
|
||||
void shutdownAssembleExecutor() {
|
||||
assembleExecutor.shutdownNow();
|
||||
cozePollPrefetchExecutor.shutdownNow();
|
||||
}
|
||||
|
||||
/**
|
||||
* Credential names normally come from configuration, but stale/rotated
|
||||
* credentials can otherwise leave throttle entries in the process forever.
|
||||
*/
|
||||
@Scheduled(fixedDelayString = "${aiimage.similar-asin.coze-submit-throttle-cleanup-delay-ms:600000}")
|
||||
void cleanupCozeThrottleState() {
|
||||
long cutoff = System.currentTimeMillis() - Duration.ofHours(1).toMillis();
|
||||
lastCozeSubmitAtByCredential.entrySet().removeIf(entry ->
|
||||
entry.getValue() == null || entry.getValue() < cutoff);
|
||||
}
|
||||
|
||||
private static ThreadFactory namedThreadFactory(String prefix) {
|
||||
@@ -1898,6 +1918,9 @@ public class SimilarAsinTaskService {
|
||||
if (stateId == null) {
|
||||
continue;
|
||||
}
|
||||
// The task lock is already held here. The outer wrapper
|
||||
// would perform an extra selectById for every state before
|
||||
// entering this method.
|
||||
pollPendingCozeStateLocked(stateId);
|
||||
}
|
||||
} finally {
|
||||
@@ -1932,11 +1955,15 @@ public class SimilarAsinTaskService {
|
||||
}
|
||||
// 先批量加载 state,避免每个预取线程各自查 DB;同时筛掉 claim 之前就不可能 poll 的状态。
|
||||
List<TaskScopeStateEntity> candidates = new ArrayList<>(stateIds.size());
|
||||
for (Long stateId : stateIds) {
|
||||
if (stateId == null) {
|
||||
continue;
|
||||
}
|
||||
TaskScopeStateEntity state = taskScopeStateMapper.selectById(stateId);
|
||||
List<TaskScopeStateEntity> refreshedStates = taskScopeStateMapper.selectList(
|
||||
new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.in(TaskScopeStateEntity::getId, stateIds)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.in(TaskScopeStateEntity::getCozeStatus,
|
||||
List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING)));
|
||||
for (TaskScopeStateEntity state : refreshedStates == null
|
||||
? List.<TaskScopeStateEntity>of()
|
||||
: refreshedStates) {
|
||||
if (state == null
|
||||
|| state.getCozeExecuteId() == null
|
||||
|| state.getCozeExecuteId().isBlank()) {
|
||||
@@ -1953,56 +1980,42 @@ public class SimilarAsinTaskService {
|
||||
if (candidates.size() <= 1) {
|
||||
return;
|
||||
}
|
||||
Semaphore concurrency = new Semaphore(Math.max(1, COZE_POLL_PREFETCH_CONCURRENCY));
|
||||
// 用临时虚拟线程池:调用结束就 close,不影响主调度线程池的容量。
|
||||
ExecutorService prefetchPool = Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
|
||||
.name("similar-asin-coze-prefetch-", 0)
|
||||
.factory());
|
||||
// 复用固定大小的预取线程池,限制并发并避免每轮调度反复创建线程池。
|
||||
Map<Long, Future<SimilarAsinCozeClient.CozePollResponse>> futures = new LinkedHashMap<>();
|
||||
try {
|
||||
long prefetchStart = System.currentTimeMillis();
|
||||
for (TaskScopeStateEntity state : candidates) {
|
||||
final Long stateId = state.getId();
|
||||
final String executeId = state.getCozeExecuteId();
|
||||
final CozeBatchContext context = readCozeBatchContext(state);
|
||||
if (context == null) {
|
||||
continue;
|
||||
}
|
||||
final String credentialName = context.credentialName();
|
||||
Future<SimilarAsinCozeClient.CozePollResponse> future = prefetchPool.submit(() -> {
|
||||
concurrency.acquire();
|
||||
try {
|
||||
return cozeClient.pollWorkflow(executeId, cozeClient.credentialByName(credentialName));
|
||||
} finally {
|
||||
concurrency.release();
|
||||
}
|
||||
});
|
||||
futures.put(stateId, future);
|
||||
long prefetchStart = System.currentTimeMillis();
|
||||
for (TaskScopeStateEntity state : candidates) {
|
||||
final Long stateId = state.getId();
|
||||
final String executeId = state.getCozeExecuteId();
|
||||
final CozeBatchContext context = readCozeBatchContext(state);
|
||||
if (context == null) {
|
||||
continue;
|
||||
}
|
||||
for (Map.Entry<Long, Future<SimilarAsinCozeClient.CozePollResponse>> entry : futures.entrySet()) {
|
||||
Long stateId = entry.getKey();
|
||||
Future<SimilarAsinCozeClient.CozePollResponse> future = entry.getValue();
|
||||
try {
|
||||
final String credentialName = context.credentialName();
|
||||
Future<SimilarAsinCozeClient.CozePollResponse> future = cozePollPrefetchExecutor.submit(
|
||||
() -> cozeClient.pollWorkflow(executeId, cozeClient.credentialByName(credentialName)));
|
||||
futures.put(stateId, future);
|
||||
}
|
||||
for (Map.Entry<Long, Future<SimilarAsinCozeClient.CozePollResponse>> entry : futures.entrySet()) {
|
||||
Long stateId = entry.getKey();
|
||||
Future<SimilarAsinCozeClient.CozePollResponse> future = entry.getValue();
|
||||
try {
|
||||
SimilarAsinCozeClient.CozePollResponse response =
|
||||
future.get(COZE_POLL_PREFETCH_WAIT_MILLIS, java.util.concurrent.TimeUnit.MILLISECONDS);
|
||||
if (response != null) {
|
||||
cache.cozePollResponseCache.put(stateId, response);
|
||||
}
|
||||
} catch (java.util.concurrent.TimeoutException timeoutEx) {
|
||||
} catch (java.util.concurrent.TimeoutException timeoutEx) {
|
||||
future.cancel(true);
|
||||
log.warn("[similar-asin] coze poll prefetch timeout stateId={} waitMs={}",
|
||||
stateId, COZE_POLL_PREFETCH_WAIT_MILLIS);
|
||||
} catch (Exception ex) {
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] coze poll prefetch failed stateId={} err={}",
|
||||
stateId, firstNonBlank(ex.getMessage(), ex.getClass().getSimpleName()));
|
||||
}
|
||||
}
|
||||
log.info("[similar-asin] coze poll prefetch finished candidates={} cached={} costMs={}",
|
||||
candidates.size(), cache.cozePollResponseCache.size(),
|
||||
System.currentTimeMillis() - prefetchStart);
|
||||
} finally {
|
||||
prefetchPool.shutdownNow();
|
||||
}
|
||||
log.info("[similar-asin] coze poll prefetch finished candidates={} cached={} costMs={}",
|
||||
candidates.size(), cache.cozePollResponseCache.size(),
|
||||
System.currentTimeMillis() - prefetchStart);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -4228,7 +4241,166 @@ public class SimilarAsinTaskService {
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounded assembly path. Chunk payloads are selected and decoded one at a
|
||||
* time for the current source file; the result map is released when that
|
||||
* source workbook finishes. This prevents a multi-source/large-task
|
||||
* assemble from retaining every chunk JSON and every result DTO together.
|
||||
*/
|
||||
private void assembleResultWorkbookBounded(FileTaskEntity task, FileResultEntity result) {
|
||||
SimilarAsinParsedPayloadDto parsed = readParsedPayload(task);
|
||||
List<SourceRows> sourceRows = splitRowsBySourceFile(
|
||||
parsed, parsed.getAllItems(), result.getSourceFilename());
|
||||
if (sourceRows.isEmpty()) {
|
||||
throw new BusinessException("Similar ASIN result rows are empty");
|
||||
}
|
||||
|
||||
File outputDir = new File(storageProperties.getLocalTempDir(), "similar-asin-result");
|
||||
if (!outputDir.exists() && !outputDir.mkdirs()) {
|
||||
throw new BusinessException("failed to create result directory");
|
||||
}
|
||||
List<SourceResultWorkbook> workbooks = new ArrayList<>();
|
||||
List<File> workbookFiles = new ArrayList<>(sourceRows.size());
|
||||
File zip = null;
|
||||
long imageCacheMaxBytes = properties.getImageCacheMaxBytes() > 0
|
||||
? properties.getImageCacheMaxBytes()
|
||||
: BoundedImageCache.DEFAULT_MAX_BYTES;
|
||||
BoundedImageCache taskImageCache = new BoundedImageCache(imageCacheMaxBytes);
|
||||
try {
|
||||
int timeoutMinutes = Math.max(1, properties.getResultFileTimeoutMinutes());
|
||||
long deadlineNanos = System.nanoTime() + TimeUnit.MINUTES.toNanos(timeoutMinutes);
|
||||
long assembleStart = System.currentTimeMillis();
|
||||
List<Future<SourceResultWorkbook>> futures = new ArrayList<>(sourceRows.size());
|
||||
for (SourceRows item : sourceRows) {
|
||||
final SourceRows captured = item;
|
||||
final String filename = safeFileStem(captured.sourceFilename()) + "-result.xlsx";
|
||||
final String tempFilename = safeFileStem(captured.sourceFilename())
|
||||
+ "-" + task.getId()
|
||||
+ "-" + result.getId()
|
||||
+ "-" + UUID.randomUUID()
|
||||
+ "-result.xlsx";
|
||||
final File xlsx = new File(outputDir, tempFilename);
|
||||
workbookFiles.add(xlsx);
|
||||
futures.add(assembleExecutor.submit(() -> {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
Map<String, SimilarAsinResultRowDto> sourceResultMap =
|
||||
loadPersistedResultRowsForRowsWithRetry(task.getId(), captured.rows());
|
||||
int persistedRows = sourceResultMap.size();
|
||||
sourceResultMap.entrySet().removeIf(entry -> !isExportableResultRow(entry.getValue()));
|
||||
long resolvedRows = captured.rows().stream()
|
||||
.filter(row -> findResultRow(row, sourceResultMap) != null)
|
||||
.count();
|
||||
int conformPropagated = CozeGroupResultPropagator.propagateByGroup(
|
||||
captured.rows(),
|
||||
SimilarAsinParsedRowVo::getDisplayId,
|
||||
row -> findResultRow(row, sourceResultMap),
|
||||
SimilarAsinResultRowDto::getIsConform,
|
||||
SimilarAsinResultRowDto::setIsConform,
|
||||
List.of("不符合"),
|
||||
"[similar-asin] taskId=" + task.getId()
|
||||
);
|
||||
writeResultWorkbook(xlsx, captured.rows(), sourceResultMap, taskImageCache);
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
return new SourceResultWorkbook(
|
||||
xlsx, filename, captured.rows().size(), persistedRows,
|
||||
sourceResultMap.size(), resolvedRows, conformPropagated);
|
||||
}));
|
||||
}
|
||||
try {
|
||||
for (Future<SourceResultWorkbook> future : futures) {
|
||||
long remainingNanos = deadlineNanos - System.nanoTime();
|
||||
if (remainingNanos <= 0L) {
|
||||
throw new TimeoutException("result file deadline reached");
|
||||
}
|
||||
workbooks.add(future.get(remainingNanos, TimeUnit.NANOSECONDS));
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
cancelResultAssemblyFutures(futures);
|
||||
Thread.currentThread().interrupt();
|
||||
throw new BusinessException("result workbook assembly interrupted", ex);
|
||||
} catch (TimeoutException ex) {
|
||||
cancelResultAssemblyFutures(futures);
|
||||
throw new BusinessException("result workbook assembly timed out", ex);
|
||||
} catch (CancellationException ex) {
|
||||
cancelResultAssemblyFutures(futures);
|
||||
throw new BusinessException("result workbook assembly cancelled", ex);
|
||||
} catch (ExecutionException ex) {
|
||||
cancelResultAssemblyFutures(futures);
|
||||
Throwable cause = ex.getCause();
|
||||
if (cause instanceof BusinessException businessException) {
|
||||
throw businessException;
|
||||
}
|
||||
throw new BusinessException("result workbook assembly failed", cause == null ? ex : cause);
|
||||
}
|
||||
|
||||
int persistedResultRows = workbooks.stream()
|
||||
.mapToInt(SourceResultWorkbook::persistedRows)
|
||||
.sum();
|
||||
int resultRows = workbooks.stream()
|
||||
.mapToInt(SourceResultWorkbook::resultRows)
|
||||
.sum();
|
||||
long resolvedRows = workbooks.stream()
|
||||
.mapToLong(SourceResultWorkbook::resolvedRows)
|
||||
.sum();
|
||||
int conformPropagated = workbooks.stream()
|
||||
.mapToInt(SourceResultWorkbook::conformPropagated)
|
||||
.sum();
|
||||
log.info("[similar-asin] bounded assemble workbook taskId={} parsedRows={} persistedRows={} resultRows={} resolvedRows={} propagated={} sources={} costMs={}",
|
||||
task.getId(), parsed.getAllItems().size(), persistedResultRows, resultRows,
|
||||
resolvedRows, conformPropagated, sourceRows.size(),
|
||||
System.currentTimeMillis() - assembleStart);
|
||||
if (resultRows == 0) {
|
||||
String failureSummary = collectChunkReadFailureSummary(task.getId());
|
||||
throw new BusinessException(failureSummary.isBlank()
|
||||
? "Similar ASIN result rows are empty; please retry result generation"
|
||||
: "Similar ASIN result rows are empty (" + failureSummary + ")");
|
||||
}
|
||||
|
||||
taskImageCache.clear();
|
||||
File uploadFile;
|
||||
String filename;
|
||||
String contentType;
|
||||
if (workbooks.size() == 1) {
|
||||
SourceResultWorkbook workbook = workbooks.get(0);
|
||||
uploadFile = workbook.file();
|
||||
filename = workbook.filename();
|
||||
contentType = CONTENT_TYPE_XLSX;
|
||||
} else {
|
||||
filename = safeFileStem(result.getSourceFilename()) + "-result.zip";
|
||||
String tempFilename = safeFileStem(result.getSourceFilename())
|
||||
+ "-" + task.getId()
|
||||
+ "-" + result.getId()
|
||||
+ "-" + UUID.randomUUID()
|
||||
+ "-result.zip";
|
||||
zip = new File(outputDir, tempFilename);
|
||||
packageResultWorkbooksAsZip(zip, workbooks);
|
||||
uploadFile = zip;
|
||||
contentType = CONTENT_TYPE_ZIP;
|
||||
}
|
||||
String objectKey = ossStorageService.uploadResultFile(uploadFile, MODULE_TYPE);
|
||||
result.setResultFilename(filename);
|
||||
result.setResultFileUrl(objectKey);
|
||||
result.setResultFileSize(uploadFile.length());
|
||||
result.setResultContentType(contentType);
|
||||
result.setRowCount(parsed.getAllItems().size());
|
||||
} finally {
|
||||
taskImageCache.clear();
|
||||
for (File workbookFile : workbookFiles) {
|
||||
if (workbookFile.exists() && !workbookFile.delete()) {
|
||||
log.warn("[similar-asin] delete temp xlsx failed file={}", workbookFile);
|
||||
}
|
||||
}
|
||||
if (zip != null && zip.exists() && !zip.delete()) {
|
||||
log.warn("[similar-asin] delete temp zip failed file={}", zip);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) {
|
||||
if (properties.isBoundedResultAssemblyEnabled()) {
|
||||
assembleResultWorkbookBounded(task, result);
|
||||
return;
|
||||
}
|
||||
SimilarAsinParsedPayloadDto parsed = readParsedPayload(task);
|
||||
Map<String, SimilarAsinResultRowDto> resultMap = loadPersistedResultRowsWithRetry(task.getId(), parsed.getAllItems().size());
|
||||
int persistedResultRows = resultMap.size();
|
||||
@@ -4510,6 +4682,152 @@ public class SimilarAsinTaskService {
|
||||
return normalized.isBlank() ? "similar-asin-result.xlsx" : normalized;
|
||||
}
|
||||
|
||||
private Map<String, SimilarAsinResultRowDto> loadPersistedResultRowsForRowsWithRetry(
|
||||
Long taskId,
|
||||
List<SimilarAsinParsedRowVo> requestedRows) {
|
||||
Map<String, SimilarAsinResultRowDto> result =
|
||||
loadPersistedResultRowsForRows(taskId, requestedRows);
|
||||
if (requestedRows == null || requestedRows.isEmpty() || !result.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) {
|
||||
sleepBeforeResultRowsRetry(attempt);
|
||||
result = loadPersistedResultRowsForRows(taskId, requestedRows);
|
||||
if (!result.isEmpty()) {
|
||||
log.info("[similar-asin] source result rows recovered after retry taskId={} attempt={} rows={}",
|
||||
taskId, attempt, result.size());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Select and decode one chunk payload at a time. Only rows needed by the
|
||||
* current source workbook are retained in the returned map.
|
||||
*/
|
||||
private Map<String, SimilarAsinResultRowDto> loadPersistedResultRowsForRows(
|
||||
Long taskId,
|
||||
List<SimilarAsinParsedRowVo> requestedRows) {
|
||||
Map<String, SimilarAsinResultRowDto> result = new LinkedHashMap<>();
|
||||
Set<String> requestedKeys = resultLookupKeys(requestedRows);
|
||||
if (taskId == null || requestedKeys.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
|
||||
Long lastChunkId = null;
|
||||
while (true) {
|
||||
LambdaQueryWrapper<TaskChunkEntity> query = new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.orderByAsc(TaskChunkEntity::getId)
|
||||
.last("LIMIT 1");
|
||||
if (lastChunkId != null) {
|
||||
query.gt(TaskChunkEntity::getId, lastChunkId);
|
||||
}
|
||||
List<TaskChunkEntity> page = taskChunkMapper.selectList(query);
|
||||
if (page == null || page.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
TaskChunkEntity chunk = page.getFirst();
|
||||
appendRequestedResultRows(result, readChunkRows(chunk).values(), requestedKeys, false);
|
||||
lastChunkId = chunk.getId();
|
||||
if (lastChunkId == null) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
appendRequestedOrphanRows(taskId, requestedKeys, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private Set<String> resultLookupKeys(List<SimilarAsinParsedRowVo> rows) {
|
||||
Set<String> keys = new LinkedHashSet<>();
|
||||
for (SimilarAsinParsedRowVo row : rows == null ? List.<SimilarAsinParsedRowVo>of() : rows) {
|
||||
String primary = rowKey(row);
|
||||
if (!primary.isBlank()) {
|
||||
keys.add(primary);
|
||||
}
|
||||
String legacy = legacyRowKey(row);
|
||||
if (!legacy.isBlank()) {
|
||||
keys.add(legacy);
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
private void appendRequestedResultRows(Map<String, SimilarAsinResultRowDto> sink,
|
||||
Iterable<SimilarAsinResultRowDto> rows,
|
||||
Set<String> requestedKeys,
|
||||
boolean onlyIfAbsent) {
|
||||
if (rows == null || requestedKeys == null || requestedKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (SimilarAsinResultRowDto row : rows) {
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
String primary = rowKey(row);
|
||||
String legacy = legacyRowKey(row);
|
||||
String matchedKey = requestedKeys.contains(primary)
|
||||
? primary
|
||||
: requestedKeys.contains(legacy) ? legacy : "";
|
||||
if (matchedKey.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
if (onlyIfAbsent) {
|
||||
sink.putIfAbsent(matchedKey, row);
|
||||
} else {
|
||||
sink.put(matchedKey, row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void appendRequestedOrphanRows(Long taskId,
|
||||
Set<String> requestedKeys,
|
||||
Map<String, SimilarAsinResultRowDto> sink) {
|
||||
Long lastStateId = null;
|
||||
while (true) {
|
||||
LambdaQueryWrapper<TaskScopeStateEntity> query = new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.select(TaskScopeStateEntity::getId,
|
||||
TaskScopeStateEntity::getParsedPayloadJson)
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.likeRight(TaskScopeStateEntity::getScopeKey, ORPHAN_SCOPE_KEY_PREFIX)
|
||||
.orderByAsc(TaskScopeStateEntity::getId)
|
||||
.last("LIMIT 1");
|
||||
if (lastStateId != null) {
|
||||
query.gt(TaskScopeStateEntity::getId, lastStateId);
|
||||
}
|
||||
List<TaskScopeStateEntity> page = taskScopeStateMapper.selectList(query);
|
||||
if (page == null || page.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
TaskScopeStateEntity state = page.getFirst();
|
||||
lastStateId = state.getId();
|
||||
if (lastStateId == null) {
|
||||
return;
|
||||
}
|
||||
if (state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
String payloadJson = transientPayloadStorageService.resolvePayload(
|
||||
state.getParsedPayloadJson(), "read similar ASIN orphan rows failed");
|
||||
JsonNode array = objectMapper.readTree(payloadJson);
|
||||
if (!array.isArray()) {
|
||||
continue;
|
||||
}
|
||||
List<SimilarAsinResultRowDto> matchingRows = new ArrayList<>();
|
||||
for (JsonNode node : array) {
|
||||
matchingRows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class));
|
||||
}
|
||||
appendRequestedResultRows(sink, matchingRows, requestedKeys, true);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] read requested orphan rows failed taskId={} stateId={} err={}",
|
||||
taskId, state.getId(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, SimilarAsinResultRowDto> loadPersistedResultRows(Long taskId) {
|
||||
Map<String, SimilarAsinResultRowDto> result = new LinkedHashMap<>();
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
@@ -6145,7 +6463,14 @@ public class SimilarAsinTaskService {
|
||||
|
||||
private record SourceResultWorkbook(File file,
|
||||
String filename,
|
||||
int rowCount) {
|
||||
int rowCount,
|
||||
int persistedRows,
|
||||
int resultRows,
|
||||
long resolvedRows,
|
||||
int conformPropagated) {
|
||||
private SourceResultWorkbook(File file, String filename, int rowCount) {
|
||||
this(file, filename, rowCount, 0, 0, 0L, 0);
|
||||
}
|
||||
}
|
||||
|
||||
public record ResultDownloadInfo(String url,
|
||||
|
||||
+160
-60
@@ -15,6 +15,7 @@ import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.imageio.IIOImage;
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.imageio.ImageReadParam;
|
||||
import javax.imageio.ImageReader;
|
||||
import javax.imageio.ImageWriteParam;
|
||||
import javax.imageio.ImageWriter;
|
||||
@@ -50,13 +51,16 @@ import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.CompletionService;
|
||||
import java.util.concurrent.ExecutorCompletionService;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
@@ -77,8 +81,8 @@ public class SimilarAsinImageEmbedder {
|
||||
static final int DEFAULT_DOWNLOAD_TIMEOUT_SECONDS = 5;
|
||||
/** P2-10:retry 由 1 升到 2,配合 5s timeout 单图最坏耗时 ≈ 15s。 */
|
||||
static final int DOWNLOAD_MAX_RETRY = 2;
|
||||
/** 图片下载、解码、缩放都在该池执行;默认限制为 8,避免结果生成占满整机 CPU。 */
|
||||
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 8;
|
||||
/** 4 核生产机默认 2;构造时还会按 JVM 可见 CPU 数的一半做硬上限。 */
|
||||
static final int DEFAULT_DOWNLOAD_POOL_SIZE = 2;
|
||||
static final int DEFAULT_PREFETCH_TIMEOUT_SECONDS = 1800;
|
||||
// 单元格固定尺寸,图片在其中等比缩放(不拉伸);resize 仍按长边 1280 px 控制堆体积。
|
||||
public static final float IMAGE_ROW_HEIGHT_POINTS = 409f;
|
||||
@@ -115,6 +119,8 @@ public class SimilarAsinImageEmbedder {
|
||||
private final int prefetchTimeoutSeconds;
|
||||
private final OkHttpClient httpClient;
|
||||
private final ExecutorService downloadPool;
|
||||
private final Semaphore imageProcessingSlots;
|
||||
private final ConcurrentMap<String, CompletableFuture<ResizedImage>> inFlightResizes = new ConcurrentHashMap<>();
|
||||
private final OssStorageService ossStorageService;
|
||||
private final Path localImageCacheDir;
|
||||
|
||||
@@ -125,7 +131,16 @@ public class SimilarAsinImageEmbedder {
|
||||
? DEFAULT_PREFETCH_TIMEOUT_SECONDS
|
||||
: properties.getImagePrefetchTimeoutSeconds();
|
||||
this.downloadTimeoutSeconds = rawTimeout > 0 ? rawTimeout : DEFAULT_DOWNLOAD_TIMEOUT_SECONDS;
|
||||
this.downloadPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE;
|
||||
int requestedPoolSize = rawPool > 0 ? rawPool : DEFAULT_DOWNLOAD_POOL_SIZE;
|
||||
int cpuBoundPoolSize = cpuBoundPoolLimit();
|
||||
this.downloadPoolSize = Math.min(requestedPoolSize, cpuBoundPoolSize);
|
||||
if (requestedPoolSize > this.downloadPoolSize) {
|
||||
log.warn("[similar-asin][image] image pool clamped requested={} effective={} visibleProcessors={}",
|
||||
requestedPoolSize, this.downloadPoolSize, Runtime.getRuntime().availableProcessors());
|
||||
}
|
||||
this.imageProcessingSlots = new Semaphore(this.downloadPoolSize, true);
|
||||
log.info("[similar-asin][image] image pool initialized requested={} effective={} cpuBoundLimit={}",
|
||||
requestedPoolSize, this.downloadPoolSize, cpuBoundPoolSize);
|
||||
this.prefetchTimeoutSeconds = rawPrefetchTimeout > 0
|
||||
? rawPrefetchTimeout
|
||||
: DEFAULT_PREFETCH_TIMEOUT_SECONDS;
|
||||
@@ -152,6 +167,11 @@ public class SimilarAsinImageEmbedder {
|
||||
downloadPool.shutdownNow();
|
||||
}
|
||||
|
||||
static int cpuBoundPoolLimit() {
|
||||
int visibleProcessors = Math.max(1, Runtime.getRuntime().availableProcessors());
|
||||
return Math.max(1, (visibleProcessors + 1) / 2);
|
||||
}
|
||||
|
||||
int downloadTimeoutSeconds() {
|
||||
return downloadTimeoutSeconds;
|
||||
}
|
||||
@@ -251,9 +271,7 @@ public class SimilarAsinImageEmbedder {
|
||||
if (taskImageCache.containsKey(url)) {
|
||||
return;
|
||||
}
|
||||
String downloadUrl = normalizeAndValidateDownloadUrl(url);
|
||||
byte[] raw = doFetch(downloadUrl);
|
||||
ResizedImage thumb = resizeImage(downloadUrl, raw);
|
||||
ResizedImage thumb = fetchAndResizeDirect(url);
|
||||
taskImageCache.putIfAbsent(url, thumb);
|
||||
} catch (Exception ex) {
|
||||
// 预下载失败不抛出:embed() 时同 url 会再次尝试并走原有兜底链路。
|
||||
@@ -605,6 +623,25 @@ public class SimilarAsinImageEmbedder {
|
||||
|
||||
private ResizedImage fetchAndResizeDirect(String url) throws IOException {
|
||||
String downloadUrl = normalizeAndValidateDownloadUrl(url);
|
||||
CompletableFuture<ResizedImage> owned = new CompletableFuture<>();
|
||||
CompletableFuture<ResizedImage> existing = inFlightResizes.putIfAbsent(downloadUrl, owned);
|
||||
if (existing != null) {
|
||||
return awaitInFlightResize(downloadUrl, existing);
|
||||
}
|
||||
try {
|
||||
ResizedImage cached = readLocalCachedThumb(downloadUrl);
|
||||
ResizedImage result = cached != null ? cached : fetchAndResizeUncached(downloadUrl);
|
||||
owned.complete(result);
|
||||
return result;
|
||||
} catch (IOException | RuntimeException | Error ex) {
|
||||
owned.completeExceptionally(ex);
|
||||
throw ex;
|
||||
} finally {
|
||||
inFlightResizes.remove(downloadUrl, owned);
|
||||
}
|
||||
}
|
||||
|
||||
private ResizedImage fetchAndResizeUncached(String downloadUrl) throws IOException {
|
||||
List<String> candidates = downloadCandidates(downloadUrl);
|
||||
IOException last = null;
|
||||
for (int attempt = 0; attempt <= DOWNLOAD_MAX_RETRY; attempt++) {
|
||||
@@ -634,6 +671,30 @@ public class SimilarAsinImageEmbedder {
|
||||
throw last == null ? new IOException("image download failed: " + downloadUrl) : last;
|
||||
}
|
||||
|
||||
private static ResizedImage awaitInFlightResize(String url,
|
||||
CompletableFuture<ResizedImage> future) throws IOException {
|
||||
try {
|
||||
return future.get();
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
InterruptedIOException interrupted = new InterruptedIOException("image resize wait interrupted url=" + url);
|
||||
interrupted.initCause(ex);
|
||||
throw interrupted;
|
||||
} catch (ExecutionException ex) {
|
||||
Throwable cause = ex.getCause();
|
||||
if (cause instanceof IOException ioException) {
|
||||
throw ioException;
|
||||
}
|
||||
if (cause instanceof RuntimeException runtimeException) {
|
||||
throw runtimeException;
|
||||
}
|
||||
if (cause instanceof Error error) {
|
||||
throw error;
|
||||
}
|
||||
throw new IOException("image resize failed url=" + url, cause);
|
||||
}
|
||||
}
|
||||
|
||||
String normalizeAndValidateDownloadUrl(String url) {
|
||||
String downloadUrl = ossStorageService.normalizeManagedPublicUrl(url);
|
||||
validateHttpsUrl(downloadUrl);
|
||||
@@ -825,43 +886,105 @@ public class SimilarAsinImageEmbedder {
|
||||
* 仍然超限才抛 ResizeOversizeException 触发文本兜底。
|
||||
*/
|
||||
ResizedImage resizeImage(String sourceUrl, byte[] raw) throws IOException {
|
||||
ensureImageWorkNotInterrupted();
|
||||
guardImageDimensions(sourceUrl, raw);
|
||||
BufferedImage src = ImageIO.read(new ByteArrayInputStream(raw));
|
||||
if (src == null) {
|
||||
throw new IOException("unsupported image format url=" + sourceUrl);
|
||||
}
|
||||
int srcW = src.getWidth();
|
||||
int srcH = src.getHeight();
|
||||
// B 方案降级顺序:固定 MAX_THUMB_SIZE 上限 → 优先调整长边像素 → 再调质量。
|
||||
// 同一 src BufferedImage 解码一次,下面 9 种组合复用,避免重复 ImageIO.read。
|
||||
ResizedImage candidate = null;
|
||||
ResizedImage smallest = null;
|
||||
for (int longEdge : FALLBACK_LONG_EDGES) {
|
||||
acquireImageProcessingSlot();
|
||||
try {
|
||||
ensureImageWorkNotInterrupted();
|
||||
BufferedImage scaled = scaleAt(src, srcW, srcH, longEdge);
|
||||
for (float quality : FALLBACK_QUALITIES) {
|
||||
ensureImageWorkNotInterrupted();
|
||||
ResizedImage tried = encodeJpeg(scaled, quality);
|
||||
if (smallest == null || tried.bytes().length < smallest.bytes().length) {
|
||||
smallest = tried;
|
||||
BufferedImage src = decodeForResize(sourceUrl, raw);
|
||||
try {
|
||||
int srcW = src.getWidth();
|
||||
int srcH = src.getHeight();
|
||||
ResizedImage candidate = null;
|
||||
ResizedImage smallest = null;
|
||||
for (int longEdge : FALLBACK_LONG_EDGES) {
|
||||
ensureImageWorkNotInterrupted();
|
||||
BufferedImage scaled = scaleAt(src, srcW, srcH, longEdge);
|
||||
try {
|
||||
for (float quality : FALLBACK_QUALITIES) {
|
||||
ensureImageWorkNotInterrupted();
|
||||
ResizedImage tried = encodeJpeg(scaled, quality);
|
||||
if (smallest == null || tried.bytes().length < smallest.bytes().length) {
|
||||
smallest = tried;
|
||||
}
|
||||
if (tried.bytes().length <= MAX_THUMB_SIZE_BYTES) {
|
||||
candidate = tried;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
scaled.flush();
|
||||
}
|
||||
if (candidate != null) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (tried.bytes().length <= MAX_THUMB_SIZE_BYTES) {
|
||||
candidate = tried;
|
||||
break;
|
||||
if (candidate != null) {
|
||||
return candidate;
|
||||
}
|
||||
int reportedSize = smallest != null ? smallest.bytes().length : -1;
|
||||
throw new ResizeOversizeException(sourceUrl, reportedSize);
|
||||
} finally {
|
||||
src.flush();
|
||||
}
|
||||
if (candidate != null) {
|
||||
break;
|
||||
} finally {
|
||||
imageProcessingSlots.release();
|
||||
}
|
||||
}
|
||||
|
||||
private void acquireImageProcessingSlot() throws InterruptedIOException {
|
||||
try {
|
||||
imageProcessingSlots.acquire();
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
InterruptedIOException interrupted = new InterruptedIOException("image processing interrupted");
|
||||
interrupted.initCause(ex);
|
||||
throw interrupted;
|
||||
}
|
||||
}
|
||||
|
||||
private static BufferedImage decodeForResize(String sourceUrl, byte[] raw) throws IOException {
|
||||
try (ImageInputStream iis = ImageIO.createImageInputStream(new ByteArrayInputStream(raw))) {
|
||||
if (iis == null) {
|
||||
throw new IOException("unable to create ImageInputStream url=" + sourceUrl);
|
||||
}
|
||||
Iterator<ImageReader> readers = ImageIO.getImageReaders(iis);
|
||||
if (!readers.hasNext()) {
|
||||
throw new IOException("unsupported image format url=" + sourceUrl);
|
||||
}
|
||||
ImageReader reader = readers.next();
|
||||
try {
|
||||
reader.setInput(iis, true, true);
|
||||
int sourceWidth = reader.getWidth(0);
|
||||
int sourceHeight = reader.getHeight(0);
|
||||
long pixels = (long) sourceWidth * (long) sourceHeight;
|
||||
if (sourceWidth <= 0 || sourceHeight <= 0 || pixels > MAX_DECODE_PIXELS) {
|
||||
throw new ResizeException("image too large url=" + sourceUrl + " pixels=" + pixels);
|
||||
}
|
||||
ImageReadParam readParam = reader.getDefaultReadParam();
|
||||
if (isJpegReader(reader)) {
|
||||
int subsampling = jpegSourceSubsampling(sourceWidth, sourceHeight);
|
||||
if (subsampling > 1) {
|
||||
readParam.setSourceSubsampling(subsampling, subsampling, 0, 0);
|
||||
}
|
||||
}
|
||||
BufferedImage decoded = reader.read(0, readParam);
|
||||
if (decoded == null) {
|
||||
throw new IOException("unsupported image format url=" + sourceUrl);
|
||||
}
|
||||
return decoded;
|
||||
} finally {
|
||||
reader.dispose();
|
||||
}
|
||||
}
|
||||
if (candidate != null) {
|
||||
return candidate;
|
||||
}
|
||||
// 9 组合都没压到上限:抛 ResizeOversizeException 走文本兜底,
|
||||
// 同时报告 smallest 字节让运维直观知道当前压缩极限。
|
||||
int reportedSize = smallest != null ? smallest.bytes().length : -1;
|
||||
throw new ResizeOversizeException(sourceUrl, reportedSize);
|
||||
}
|
||||
|
||||
private static boolean isJpegReader(ImageReader reader) throws IOException {
|
||||
String format = reader.getFormatName();
|
||||
return "JPEG".equalsIgnoreCase(format) || "JPG".equalsIgnoreCase(format);
|
||||
}
|
||||
|
||||
static int jpegSourceSubsampling(int sourceWidth, int sourceHeight) {
|
||||
int ratio = Math.max(sourceWidth, sourceHeight) / TARGET_LONG_EDGE_PX;
|
||||
return ratio > 1 ? Integer.highestOneBit(ratio) : 1;
|
||||
}
|
||||
|
||||
private static void ensureImageWorkNotInterrupted() throws InterruptedIOException {
|
||||
@@ -995,29 +1118,6 @@ public class SimilarAsinImageEmbedder {
|
||||
}
|
||||
}
|
||||
|
||||
/** 在解码整张位图前用 ImageReader 仅读取头部尺寸,避免“图像炸弹”导致 heap OOM。 */
|
||||
private static void guardImageDimensions(String sourceUrl, byte[] raw) throws IOException {
|
||||
try (ImageInputStream iis = ImageIO.createImageInputStream(new ByteArrayInputStream(raw))) {
|
||||
if (iis == null) {
|
||||
throw new IOException("unable to create ImageInputStream url=" + sourceUrl);
|
||||
}
|
||||
Iterator<ImageReader> readers = ImageIO.getImageReaders(iis);
|
||||
if (!readers.hasNext()) {
|
||||
throw new IOException("unsupported image format url=" + sourceUrl);
|
||||
}
|
||||
ImageReader reader = readers.next();
|
||||
try {
|
||||
reader.setInput(iis, true, true);
|
||||
long pixels = (long) reader.getWidth(0) * (long) reader.getHeight(0);
|
||||
if (pixels > MAX_DECODE_PIXELS) {
|
||||
throw new ResizeException("image too large url=" + sourceUrl + " pixels=" + pixels);
|
||||
}
|
||||
} finally {
|
||||
reader.dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static ThreadFactory namedFactory(String prefix) {
|
||||
AtomicInteger counter = new AtomicInteger();
|
||||
return r -> {
|
||||
|
||||
Reference in New Issue
Block a user