task-14: batch read image db cache and touch only actual hits
This commit is contained in:
+77
-2
@@ -144,9 +144,15 @@ public class SimilarAsinImagePrefetchService {
|
||||
cachedHashes.addAll(found);
|
||||
}
|
||||
}
|
||||
// Task 14:只对实际命中的 url_hash 更新 last_used_at,未命中不 touch。
|
||||
for (int start = 0; start < allHashes.size(); start += CACHE_LOOKUP_BATCH_SIZE) {
|
||||
taskImageCacheMapper.touchLastUsedBatch(allHashes.subList(start,
|
||||
Math.min(start + CACHE_LOOKUP_BATCH_SIZE, allHashes.size())));
|
||||
List<String> batch = allHashes.subList(start,
|
||||
Math.min(start + CACHE_LOOKUP_BATCH_SIZE, allHashes.size()));
|
||||
List<String> hitBatch = new ArrayList<>(batch);
|
||||
hitBatch.retainAll(cachedHashes);
|
||||
if (!hitBatch.isEmpty()) {
|
||||
taskImageCacheMapper.touchLastUsedBatch(hitBatch);
|
||||
}
|
||||
}
|
||||
hit = cachedHashes.size();
|
||||
}
|
||||
@@ -189,6 +195,75 @@ public class SimilarAsinImagePrefetchService {
|
||||
taskId, urls.size(), hit, miss, fail);
|
||||
}
|
||||
|
||||
/**
|
||||
* Task 14:批量直读 DB cache 缩略图。一次 IN 查询返回全部命中字节;
|
||||
* last_used_at 只对实际命中的 url_hash 更新(touch 集合 = 命中集合),
|
||||
* 未命中 url 不产生任何 touch。输入按传入顺序返回,未命中为 null。
|
||||
* 失败/开关关闭返回空列表,由调用方走回退路径。
|
||||
*/
|
||||
List<byte[]> lookupBatch(List<String> urls) {
|
||||
List<byte[]> result = new ArrayList<>();
|
||||
if (!properties.isImageDbCacheEnabled()) {
|
||||
return result;
|
||||
}
|
||||
if (urls == null || urls.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
try {
|
||||
List<String> hashes = new ArrayList<>(urls.size());
|
||||
for (String url : urls) {
|
||||
if (url == null) {
|
||||
result.add(null);
|
||||
continue;
|
||||
}
|
||||
String trimmed = url.trim();
|
||||
if (trimmed.isEmpty()) {
|
||||
result.add(null);
|
||||
continue;
|
||||
}
|
||||
hashes.add(sha256Hex(trimmed));
|
||||
result.add(null);
|
||||
}
|
||||
if (hashes.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
Map<String, byte[]> bytesByHash = new LinkedHashMap<>();
|
||||
List<String> hitHashes = new ArrayList<>();
|
||||
for (int start = 0; start < hashes.size(); start += CACHE_LOOKUP_BATCH_SIZE) {
|
||||
List<String> batch = hashes.subList(start,
|
||||
Math.min(start + CACHE_LOOKUP_BATCH_SIZE, hashes.size()));
|
||||
List<TaskImageCacheEntity> rows = taskImageCacheMapper.selectBytesByUrlHashes(batch);
|
||||
if (rows == null) {
|
||||
continue;
|
||||
}
|
||||
for (TaskImageCacheEntity row : rows) {
|
||||
if (row != null && row.getUrlHash() != null && row.getImageBytes() != null
|
||||
&& row.getImageBytes().length > 0) {
|
||||
if (bytesByHash.putIfAbsent(row.getUrlHash(), row.getImageBytes()) == null) {
|
||||
hitHashes.add(row.getUrlHash());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < hashes.size(); i++) {
|
||||
byte[] bytes = bytesByHash.get(hashes.get(i));
|
||||
if (bytes != null) {
|
||||
result.set(i, bytes);
|
||||
}
|
||||
}
|
||||
if (!hitHashes.isEmpty()) {
|
||||
for (int start = 0; start < hitHashes.size(); start += CACHE_LOOKUP_BATCH_SIZE) {
|
||||
taskImageCacheMapper.touchLastUsedBatch(hitHashes.subList(start,
|
||||
Math.min(start + CACHE_LOOKUP_BATCH_SIZE, hitHashes.size())));
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.debug("[similar-asin] prefetch batch lookup failed urls={} err={}", urls.size(), ex.getMessage());
|
||||
return List.of();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* P2-11:DB cache 直读入口。命中时同步 touchLastUsed,便于 LRU 清理。
|
||||
* 失败/未命中返回 null,由调用方走回退路径。
|
||||
|
||||
+8
@@ -45,6 +45,14 @@ public interface TaskImageCacheMapper extends BaseMapper<TaskImageCacheEntity> {
|
||||
@Select("SELECT image_bytes FROM biz_task_image_cache WHERE url_hash = #{urlHash} LIMIT 1")
|
||||
byte[] selectBytesByUrlHash(@Param("urlHash") String urlHash);
|
||||
|
||||
/**
|
||||
* Task 14:批量直读缩略图字节。只 select url_hash + image_bytes 两列,
|
||||
* 避免把整行 entity(含 url 字符串)拉回 JVM;配合 batchSize 分片控制 IN 长度。
|
||||
*/
|
||||
@Select("<script>SELECT url_hash, image_bytes FROM biz_task_image_cache WHERE url_hash IN " +
|
||||
"<foreach collection='urlHashes' item='hash' open='(' separator=',' close=')'>#{hash}</foreach></script>")
|
||||
List<TaskImageCacheEntity> selectBytesByUrlHashes(@Param("urlHashes") List<String> urlHashes);
|
||||
|
||||
@Select("SELECT COALESCE(SUM(byte_size), 0) FROM biz_task_image_cache")
|
||||
Long sumByteSize();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user