fix(backend-java): 收口并发资源与恢复链路
This commit is contained in:
+27
-3
@@ -10,9 +10,11 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.StreamUtils;
|
||||
import org.springframework.web.client.RestClient;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
@@ -30,6 +32,7 @@ import java.util.Map;
|
||||
public class SimilarAsinLlmClient {
|
||||
|
||||
private static final MediaType APPLICATION_JSON_UTF8 = new MediaType("application", "json", StandardCharsets.UTF_8);
|
||||
private static final int MAX_LLM_RESPONSE_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
private final SimilarAsinProperties properties;
|
||||
private final ObjectMapper objectMapper;
|
||||
@@ -115,8 +118,7 @@ public class SimilarAsinLlmClient {
|
||||
});
|
||||
request.body(body);
|
||||
String responseText = request.exchange((clientRequest, clientResponse) -> {
|
||||
byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody());
|
||||
String responseBody = responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8);
|
||||
String responseBody = readResponseBodyBounded(clientResponse.getBody());
|
||||
if (!clientResponse.getStatusCode().is2xxSuccessful()) {
|
||||
throw new IllegalStateException("LLM http " + clientResponse.getStatusCode().value()
|
||||
+ ": " + abbreviate(responseBody, 500));
|
||||
@@ -262,6 +264,28 @@ public class SimilarAsinLlmClient {
|
||||
}
|
||||
}
|
||||
|
||||
private String readResponseBodyBounded(InputStream inputStream) throws IOException {
|
||||
if (inputStream == null) {
|
||||
return "";
|
||||
}
|
||||
try (InputStream input = inputStream; ByteArrayOutputStream output = new ByteArrayOutputStream(8192)) {
|
||||
byte[] buffer = new byte[8192];
|
||||
int read;
|
||||
int total = 0;
|
||||
while ((read = input.read(buffer)) != -1) {
|
||||
if (read == 0) {
|
||||
continue;
|
||||
}
|
||||
if ((long) total + read > MAX_LLM_RESPONSE_BYTES) {
|
||||
throw new IOException("LLM response exceeds " + MAX_LLM_RESPONSE_BYTES + " bytes");
|
||||
}
|
||||
output.write(buffer, 0, read);
|
||||
total += read;
|
||||
}
|
||||
return output.toString(StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
|
||||
private JsonNode parseJsonOrThrow(String value) {
|
||||
try {
|
||||
return objectMapper.readTree(value);
|
||||
|
||||
+83
-27
@@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.FutureTask;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
@@ -57,18 +58,19 @@ public class SimilarAsinImagePrefetchService {
|
||||
/** 批量命中检查/批量 touch 单次 IN 上限,避免超长 SQL 参数列表。 */
|
||||
private static final int CACHE_LOOKUP_BATCH_SIZE = 500;
|
||||
|
||||
/** 排队等待上一个 task future 时的最大等待时间(秒),避免被 hung future 永久卡住。 */
|
||||
private static final long INFLIGHT_WAIT_SECONDS = 60L;
|
||||
/** 单个 task 最多缓存的待预热 URL 数,避免持续回传时内存无界增长。 */
|
||||
private static final int MAX_PENDING_URLS_PER_TASK = 5_000;
|
||||
/** 单机最多同时维护的预热 task 数。 */
|
||||
private static final int MAX_INFLIGHT_TASKS = 200;
|
||||
|
||||
private final SimilarAsinImageEmbedder imageEmbedder;
|
||||
private final TaskImageCacheMapper taskImageCacheMapper;
|
||||
private final SimilarAsinProperties properties;
|
||||
|
||||
/**
|
||||
* 每个 task 当前 in-flight 的预热 future。enqueue 时如果上一个还没完成,会先等它结束,
|
||||
* 再串行启动当前 batch 的预热,避免高并发 batch 把图片源站打爆。
|
||||
*/
|
||||
/** 每个 task 当前唯一的预热 worker;worker 会合并并持续 drain 待处理 URL。 */
|
||||
private final ConcurrentHashMap<Long, Future<?>> inflight = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<Long, Set<String>> pendingUrlsByTask = new ConcurrentHashMap<>();
|
||||
private final Object inflightMonitor = new Object();
|
||||
|
||||
private final ExecutorService prefetchPool = Executors.newFixedThreadPool(PREFETCH_POOL_SIZE,
|
||||
namedFactory("similar-asin-prefetch"));
|
||||
@@ -97,7 +99,10 @@ public class SimilarAsinImagePrefetchService {
|
||||
prefetchPool.shutdownNow();
|
||||
touchFlushScheduler.shutdownNow();
|
||||
flushPendingTouches();
|
||||
inflight.clear();
|
||||
synchronized (inflightMonitor) {
|
||||
inflight.clear();
|
||||
pendingUrlsByTask.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -129,37 +134,88 @@ public class SimilarAsinImagePrefetchService {
|
||||
* 同 task 串行入队(用 inflight map 排队),避免多个 batch 同时打爆图片源站。
|
||||
*/
|
||||
public void enqueue(Long taskId, List<String> urls) {
|
||||
if (!properties.isImageDbCacheEnabled()) {
|
||||
return;
|
||||
}
|
||||
if (taskId == null || urls == null || urls.isEmpty()) {
|
||||
if (!properties.isImageDbCacheEnabled() || taskId == null || urls == null || urls.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Set<String> dedup = new LinkedHashSet<>();
|
||||
for (String u : urls) {
|
||||
if (u == null) {
|
||||
continue;
|
||||
}
|
||||
String trimmed = u.trim();
|
||||
if (!trimmed.isEmpty()) {
|
||||
dedup.add(trimmed);
|
||||
if (u != null && !u.isBlank()) {
|
||||
dedup.add(u.trim());
|
||||
}
|
||||
}
|
||||
if (dedup.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
List<String> targets = new ArrayList<>(dedup);
|
||||
inflight.compute(taskId, (k, prev) -> prefetchPool.submit(() -> {
|
||||
// 串行等待上一个 batch 完成(带上限,避免被 hung future 永久卡住)。
|
||||
if (prev != null) {
|
||||
try {
|
||||
prev.get(INFLIGHT_WAIT_SECONDS, TimeUnit.SECONDS);
|
||||
} catch (Exception ignored) {
|
||||
// 上一个 future 异常/超时不阻塞当前预热——继续即可。
|
||||
|
||||
FutureTask<Void> worker = null;
|
||||
synchronized (inflightMonitor) {
|
||||
Future<?> current = inflight.get(taskId);
|
||||
Set<String> pending = pendingUrlsByTask.computeIfAbsent(taskId, ignored -> ConcurrentHashMap.newKeySet());
|
||||
int accepted = 0;
|
||||
for (String url : dedup) {
|
||||
if (pending.size() >= MAX_PENDING_URLS_PER_TASK) {
|
||||
break;
|
||||
}
|
||||
if (pending.add(url)) {
|
||||
accepted++;
|
||||
}
|
||||
}
|
||||
runPrefetch(taskId, targets);
|
||||
}));
|
||||
if (accepted < dedup.size()) {
|
||||
log.warn("[similar-asin][image] prefetch pending limit reached taskId={} accepted={} dropped={} limit={}",
|
||||
taskId, accepted, dedup.size() - accepted, MAX_PENDING_URLS_PER_TASK);
|
||||
}
|
||||
if (current != null && !current.isDone()) {
|
||||
return;
|
||||
}
|
||||
if (inflight.size() >= MAX_INFLIGHT_TASKS) {
|
||||
log.warn("[similar-asin][image] prefetch task limit reached taskId={} inflight={} limit={}",
|
||||
taskId, inflight.size(), MAX_INFLIGHT_TASKS);
|
||||
return;
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
FutureTask<Void>[] workerRef = new FutureTask[1];
|
||||
workerRef[0] = new FutureTask<>(() -> {
|
||||
drainPrefetch(taskId, workerRef[0]);
|
||||
return null;
|
||||
});
|
||||
worker = workerRef[0];
|
||||
inflight.put(taskId, worker);
|
||||
}
|
||||
try {
|
||||
prefetchPool.execute(worker);
|
||||
} catch (RuntimeException ex) {
|
||||
synchronized (inflightMonitor) {
|
||||
inflight.remove(taskId, worker);
|
||||
}
|
||||
log.warn("[similar-asin][image] prefetch worker rejected taskId={} msg={}", taskId, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void drainPrefetch(Long taskId, Future<?> worker) {
|
||||
try {
|
||||
while (true) {
|
||||
List<String> targets;
|
||||
synchronized (inflightMonitor) {
|
||||
Set<String> pending = pendingUrlsByTask.get(taskId);
|
||||
if (pending == null || pending.isEmpty()) {
|
||||
pendingUrlsByTask.remove(taskId, pending);
|
||||
return;
|
||||
}
|
||||
targets = new ArrayList<>(pending);
|
||||
pending.clear();
|
||||
}
|
||||
try {
|
||||
runPrefetch(taskId, targets);
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("[similar-asin][image] prefetch batch failed taskId={} urls={} msg={}",
|
||||
taskId, targets.size(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
synchronized (inflightMonitor) {
|
||||
inflight.remove(taskId, worker);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user