fix(backend-java): 收口并发资源与恢复链路

This commit is contained in:
2026-09-05 14:33:07 +08:00
parent 5a00201931
commit 0bf39b04db
35 changed files with 917 additions and 154 deletions
@@ -10,9 +10,11 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.util.StreamUtils;
import org.springframework.web.client.RestClient;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
@@ -30,6 +32,7 @@ import java.util.Map;
public class SimilarAsinLlmClient {
private static final MediaType APPLICATION_JSON_UTF8 = new MediaType("application", "json", StandardCharsets.UTF_8);
private static final int MAX_LLM_RESPONSE_BYTES = 16 * 1024 * 1024;
private final SimilarAsinProperties properties;
private final ObjectMapper objectMapper;
@@ -115,8 +118,7 @@ public class SimilarAsinLlmClient {
});
request.body(body);
String responseText = request.exchange((clientRequest, clientResponse) -> {
byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody());
String responseBody = responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8);
String responseBody = readResponseBodyBounded(clientResponse.getBody());
if (!clientResponse.getStatusCode().is2xxSuccessful()) {
throw new IllegalStateException("LLM http " + clientResponse.getStatusCode().value()
+ ": " + abbreviate(responseBody, 500));
@@ -262,6 +264,28 @@ public class SimilarAsinLlmClient {
}
}
private String readResponseBodyBounded(InputStream inputStream) throws IOException {
if (inputStream == null) {
return "";
}
try (InputStream input = inputStream; ByteArrayOutputStream output = new ByteArrayOutputStream(8192)) {
byte[] buffer = new byte[8192];
int read;
int total = 0;
while ((read = input.read(buffer)) != -1) {
if (read == 0) {
continue;
}
if ((long) total + read > MAX_LLM_RESPONSE_BYTES) {
throw new IOException("LLM response exceeds " + MAX_LLM_RESPONSE_BYTES + " bytes");
}
output.write(buffer, 0, read);
total += read;
}
return output.toString(StandardCharsets.UTF_8);
}
}
private JsonNode parseJsonOrThrow(String value) {
try {
return objectMapper.readTree(value);
@@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
@@ -57,18 +58,19 @@ public class SimilarAsinImagePrefetchService {
/** 批量命中检查/批量 touch 单次 IN 上限,避免超长 SQL 参数列表。 */
private static final int CACHE_LOOKUP_BATCH_SIZE = 500;
/** 排队等待上一个 task future 时的最大等待时间(秒),避免被 hung future 永久卡住。 */
private static final long INFLIGHT_WAIT_SECONDS = 60L;
/** 单个 task 最多缓存的待预热 URL 数,避免持续回传时内存无界增长。 */
private static final int MAX_PENDING_URLS_PER_TASK = 5_000;
/** 单机最多同时维护的预热 task 数。 */
private static final int MAX_INFLIGHT_TASKS = 200;
private final SimilarAsinImageEmbedder imageEmbedder;
private final TaskImageCacheMapper taskImageCacheMapper;
private final SimilarAsinProperties properties;
/**
* 每个 task 当前 in-flight 的预热 future。enqueue 时如果上一个还没完成,会先等它结束,
* 再串行启动当前 batch 的预热,避免高并发 batch 把图片源站打爆。
*/
/** 每个 task 当前唯一的预热 worker;worker 会合并并持续 drain 待处理 URL。 */
private final ConcurrentHashMap<Long, Future<?>> inflight = new ConcurrentHashMap<>();
private final ConcurrentHashMap<Long, Set<String>> pendingUrlsByTask = new ConcurrentHashMap<>();
private final Object inflightMonitor = new Object();
private final ExecutorService prefetchPool = Executors.newFixedThreadPool(PREFETCH_POOL_SIZE,
namedFactory("similar-asin-prefetch"));
@@ -97,7 +99,10 @@ public class SimilarAsinImagePrefetchService {
prefetchPool.shutdownNow();
touchFlushScheduler.shutdownNow();
flushPendingTouches();
inflight.clear();
synchronized (inflightMonitor) {
inflight.clear();
pendingUrlsByTask.clear();
}
}
/**
@@ -129,37 +134,88 @@ public class SimilarAsinImagePrefetchService {
* 同 task 串行入队(用 inflight map 排队),避免多个 batch 同时打爆图片源站。
*/
public void enqueue(Long taskId, List<String> urls) {
if (!properties.isImageDbCacheEnabled()) {
return;
}
if (taskId == null || urls == null || urls.isEmpty()) {
if (!properties.isImageDbCacheEnabled() || taskId == null || urls == null || urls.isEmpty()) {
return;
}
Set<String> dedup = new LinkedHashSet<>();
for (String u : urls) {
if (u == null) {
continue;
}
String trimmed = u.trim();
if (!trimmed.isEmpty()) {
dedup.add(trimmed);
if (u != null && !u.isBlank()) {
dedup.add(u.trim());
}
}
if (dedup.isEmpty()) {
return;
}
List<String> targets = new ArrayList<>(dedup);
inflight.compute(taskId, (k, prev) -> prefetchPool.submit(() -> {
// 串行等待上一个 batch 完成(带上限,避免被 hung future 永久卡住)。
if (prev != null) {
try {
prev.get(INFLIGHT_WAIT_SECONDS, TimeUnit.SECONDS);
} catch (Exception ignored) {
// 上一个 future 异常/超时不阻塞当前预热——继续即可。
FutureTask<Void> worker = null;
synchronized (inflightMonitor) {
Future<?> current = inflight.get(taskId);
Set<String> pending = pendingUrlsByTask.computeIfAbsent(taskId, ignored -> ConcurrentHashMap.newKeySet());
int accepted = 0;
for (String url : dedup) {
if (pending.size() >= MAX_PENDING_URLS_PER_TASK) {
break;
}
if (pending.add(url)) {
accepted++;
}
}
runPrefetch(taskId, targets);
}));
if (accepted < dedup.size()) {
log.warn("[similar-asin][image] prefetch pending limit reached taskId={} accepted={} dropped={} limit={}",
taskId, accepted, dedup.size() - accepted, MAX_PENDING_URLS_PER_TASK);
}
if (current != null && !current.isDone()) {
return;
}
if (inflight.size() >= MAX_INFLIGHT_TASKS) {
log.warn("[similar-asin][image] prefetch task limit reached taskId={} inflight={} limit={}",
taskId, inflight.size(), MAX_INFLIGHT_TASKS);
return;
}
@SuppressWarnings("unchecked")
FutureTask<Void>[] workerRef = new FutureTask[1];
workerRef[0] = new FutureTask<>(() -> {
drainPrefetch(taskId, workerRef[0]);
return null;
});
worker = workerRef[0];
inflight.put(taskId, worker);
}
try {
prefetchPool.execute(worker);
} catch (RuntimeException ex) {
synchronized (inflightMonitor) {
inflight.remove(taskId, worker);
}
log.warn("[similar-asin][image] prefetch worker rejected taskId={} msg={}", taskId, ex.getMessage());
}
}
private void drainPrefetch(Long taskId, Future<?> worker) {
try {
while (true) {
List<String> targets;
synchronized (inflightMonitor) {
Set<String> pending = pendingUrlsByTask.get(taskId);
if (pending == null || pending.isEmpty()) {
pendingUrlsByTask.remove(taskId, pending);
return;
}
targets = new ArrayList<>(pending);
pending.clear();
}
try {
runPrefetch(taskId, targets);
} catch (RuntimeException ex) {
log.warn("[similar-asin][image] prefetch batch failed taskId={} urls={} msg={}",
taskId, targets.size(), ex.getMessage());
}
}
} finally {
synchronized (inflightMonitor) {
inflight.remove(taskId, worker);
}
}
}
/**