性能优化:N+1查询改批量、串行远程调用并发化、紫鸟索引巡检批量回写

This commit is contained in:
2026-08-25 19:42:43 +08:00
parent a0d83d9e8f
commit d08280f3e8
14 changed files with 425 additions and 53 deletions
@@ -17,8 +17,10 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
@@ -50,6 +52,9 @@ public class SimilarAsinImagePrefetchService {
/** P2-11:预热线程池容量。与图片处理 CPU 槽位对齐,避免预热路径绕过主池放大并发。 */
private static final int PREFETCH_POOL_SIZE = 2;
/** 批量命中检查/批量 touch 单次 IN 上限,避免超长 SQL 参数列表。 */
private static final int CACHE_LOOKUP_BATCH_SIZE = 500;
/** 排队等待上一个 task future 时的最大等待时间(秒),避免被 hung future 永久卡住。 */
private static final long INFLIGHT_WAIT_SECONDS = 60L;
@@ -118,21 +123,40 @@ public class SimilarAsinImagePrefetchService {
int hit = 0;
int miss = 0;
int fail = 0;
Map<String, String> hashByUrl = new LinkedHashMap<>();
Set<String> cachedHashes = new LinkedHashSet<>();
for (String url : urls) {
try {
String urlHash = sha256Hex(url);
if (urlHash == null) {
fail++;
continue;
}
String urlHash = sha256Hex(url);
if (urlHash == null) {
fail++;
continue;
}
hashByUrl.put(url, urlHash);
}
if (!hashByUrl.isEmpty()) {
List<String> allHashes = new ArrayList<>(hashByUrl.values());
for (int start = 0; start < allHashes.size(); start += CACHE_LOOKUP_BATCH_SIZE) {
List<String> batch = allHashes.subList(start,
Math.min(start + CACHE_LOOKUP_BATCH_SIZE, allHashes.size()));
// 命中 DB cache:bumping last_used_at 即可,不再下载。
Long existing = taskImageCacheMapper.selectCount(new LambdaQueryWrapper<TaskImageCacheEntity>()
.eq(TaskImageCacheEntity::getUrlHash, urlHash));
if (existing != null && existing > 0L) {
taskImageCacheMapper.touchLastUsed(urlHash);
hit++;
continue;
List<String> found = taskImageCacheMapper.selectUrlHashes(batch);
if (found != null) {
cachedHashes.addAll(found);
}
}
for (int start = 0; start < allHashes.size(); start += CACHE_LOOKUP_BATCH_SIZE) {
taskImageCacheMapper.touchLastUsedBatch(allHashes.subList(start,
Math.min(start + CACHE_LOOKUP_BATCH_SIZE, allHashes.size())));
}
hit = cachedHashes.size();
}
for (Map.Entry<String, String> entry : hashByUrl.entrySet()) {
String url = entry.getKey();
String urlHash = entry.getValue();
if (cachedHashes.contains(urlHash)) {
continue;
}
try {
ResizedImage thumb = imageEmbedder.fetchAndResizeForCache(url);
if (thumb == null || thumb.bytes() == null || thumb.bytes().length == 0) {
fail++;
@@ -3751,7 +3751,13 @@ public class SimilarAsinTaskService {
if (taskId == null || taskId <= 0) {
return;
}
// 只取 flush 需要的列:parsed_payload_json/coze_* 等大字段不拉,避免跨库大结果集传输(每任务可达数百行大 JSON)
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getId,
TaskScopeStateEntity::getTaskId,
TaskScopeStateEntity::getModuleType,
TaskScopeStateEntity::getCozeStatus,
TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_DONE, COZE_STATUS_FAILED)));