性能优化:N+1查询改批量、串行远程调用并发化、紫鸟索引巡检批量回写
This commit is contained in:
+36
-12
@@ -17,8 +17,10 @@ import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
@@ -50,6 +52,9 @@ public class SimilarAsinImagePrefetchService {
|
||||
/** P2-11:预热线程池容量。与图片处理 CPU 槽位对齐,避免预热路径绕过主池放大并发。 */
|
||||
private static final int PREFETCH_POOL_SIZE = 2;
|
||||
|
||||
/** 批量命中检查/批量 touch 单次 IN 上限,避免超长 SQL 参数列表。 */
|
||||
private static final int CACHE_LOOKUP_BATCH_SIZE = 500;
|
||||
|
||||
/** 排队等待上一个 task future 时的最大等待时间(秒),避免被 hung future 永久卡住。 */
|
||||
private static final long INFLIGHT_WAIT_SECONDS = 60L;
|
||||
|
||||
@@ -118,21 +123,40 @@ public class SimilarAsinImagePrefetchService {
|
||||
int hit = 0;
|
||||
int miss = 0;
|
||||
int fail = 0;
|
||||
Map<String, String> hashByUrl = new LinkedHashMap<>();
|
||||
Set<String> cachedHashes = new LinkedHashSet<>();
|
||||
for (String url : urls) {
|
||||
try {
|
||||
String urlHash = sha256Hex(url);
|
||||
if (urlHash == null) {
|
||||
fail++;
|
||||
continue;
|
||||
}
|
||||
String urlHash = sha256Hex(url);
|
||||
if (urlHash == null) {
|
||||
fail++;
|
||||
continue;
|
||||
}
|
||||
hashByUrl.put(url, urlHash);
|
||||
}
|
||||
if (!hashByUrl.isEmpty()) {
|
||||
List<String> allHashes = new ArrayList<>(hashByUrl.values());
|
||||
for (int start = 0; start < allHashes.size(); start += CACHE_LOOKUP_BATCH_SIZE) {
|
||||
List<String> batch = allHashes.subList(start,
|
||||
Math.min(start + CACHE_LOOKUP_BATCH_SIZE, allHashes.size()));
|
||||
// 命中 DB cache:bumping last_used_at 即可,不再下载。
|
||||
Long existing = taskImageCacheMapper.selectCount(new LambdaQueryWrapper<TaskImageCacheEntity>()
|
||||
.eq(TaskImageCacheEntity::getUrlHash, urlHash));
|
||||
if (existing != null && existing > 0L) {
|
||||
taskImageCacheMapper.touchLastUsed(urlHash);
|
||||
hit++;
|
||||
continue;
|
||||
List<String> found = taskImageCacheMapper.selectUrlHashes(batch);
|
||||
if (found != null) {
|
||||
cachedHashes.addAll(found);
|
||||
}
|
||||
}
|
||||
for (int start = 0; start < allHashes.size(); start += CACHE_LOOKUP_BATCH_SIZE) {
|
||||
taskImageCacheMapper.touchLastUsedBatch(allHashes.subList(start,
|
||||
Math.min(start + CACHE_LOOKUP_BATCH_SIZE, allHashes.size())));
|
||||
}
|
||||
hit = cachedHashes.size();
|
||||
}
|
||||
for (Map.Entry<String, String> entry : hashByUrl.entrySet()) {
|
||||
String url = entry.getKey();
|
||||
String urlHash = entry.getValue();
|
||||
if (cachedHashes.contains(urlHash)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
ResizedImage thumb = imageEmbedder.fetchAndResizeForCache(url);
|
||||
if (thumb == null || thumb.bytes() == null || thumb.bytes().length == 0) {
|
||||
fail++;
|
||||
|
||||
+6
@@ -3751,7 +3751,13 @@ public class SimilarAsinTaskService {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return;
|
||||
}
|
||||
// 只取 flush 需要的列:parsed_payload_json/coze_* 等大字段不拉,避免跨库大结果集传输(每任务可达数百行大 JSON)
|
||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.select(TaskScopeStateEntity::getId,
|
||||
TaskScopeStateEntity::getTaskId,
|
||||
TaskScopeStateEntity::getModuleType,
|
||||
TaskScopeStateEntity::getCozeStatus,
|
||||
TaskScopeStateEntity::getStateJson)
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_DONE, COZE_STATUS_FAILED)));
|
||||
|
||||
Reference in New Issue
Block a user