perf(代码审查落地): 后端 A1-A8 与前端 C1-C8 加固,retention 清理补索引(V141)

A 组(backend-java):
- A1 载荷引用计数不再无条件回落全表 COUNT
- A2 结果行读取由 LIMIT 1 改整页
- A3 ziniao 索引缓存过期判断下推 SQL
- A4 循环子任务认领改 CAS,失败一律抛
- A5 建单事务里不碰外部 IO
- A6 连接池补网络层超时(connectTimeout/socketTimeout,env 可覆盖)
- A7 日志留现场 + 异步落盘
- A8 其余:货源图片预取、结果工作簿装配等
- V141:三条 retention DELETE 的可用索引,其中 biz_price_track_loop_run
  用 STORED 生成列承载 COALESCE 时间线(该表实测 881 行/17MB,COPY 秒级)

C 组(frontend-vue):URL 取值走 safe-url、行号取值走 row-id、
交付视频任务状态抽成 delivery-video-task-state.ts,各补用例。
This commit is contained in:
2026-10-03 12:24:16 +08:00
parent fd9aa2caa5
commit 37592a3b62
32 changed files with 1065 additions and 292 deletions
@@ -58,6 +58,9 @@ public class SimilarAsinImagePrefetchService {
/** 批量命中检查/批量 touch 单次 IN 上限,避免超长 SQL 参数列表。 */
private static final int CACHE_LOOKUP_BATCH_SIZE = 500;
/** url_hash 十六进制编码用:逐字节 String.format 在按 URL 调用的路径上会放大成几万次格式化。 */
private static final java.util.HexFormat HEX_FORMAT = java.util.HexFormat.of();
/** 单个 task 最多缓存的待预热 URL 数,避免持续回传时内存无界增长。 */
private static final int MAX_PENDING_URLS_PER_TASK = 5_000;
/** 单机最多同时维护的预热 task 数。 */
@@ -316,7 +319,7 @@ public class SimilarAsinImagePrefetchService {
* 未命中 url 不产生任何 touch。输入按传入顺序返回,未命中为 null。
* 失败/开关关闭返回空列表,由调用方走回退路径。
*/
List<byte[]> lookupBatch(List<String> urls) {
public List<byte[]> lookupBatch(List<String> urls) {
List<byte[]> result = new ArrayList<>();
if (!properties.isImageDbCacheEnabled()) {
return result;
@@ -415,12 +418,7 @@ public class SimilarAsinImagePrefetchService {
private static String sha256Hex(String value) {
try {
MessageDigest md = MessageDigest.getInstance("SHA-256");
byte[] digest = md.digest(value.getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(digest.length * 2);
for (byte b : digest) {
sb.append(String.format("%02x", b & 0xFF));
}
return sb.toString();
return HEX_FORMAT.formatHex(md.digest(value.getBytes(StandardCharsets.UTF_8)));
} catch (NoSuchAlgorithmException ex) {
// SHA-256 在 JDK 中是必备算法,正常环境下不会落到这里。
return null;
@@ -132,6 +132,8 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String CONTENT_TYPE_ZIP = "application/zip";
private static final int MAX_LIGHT_TASK_IDS = 200;
/** progressBatch 的 IN 子句上限:看板单次轮询的在册任务量级远小于它,取值留了 5 倍余量。 */
private static final int MAX_BATCH_TASK_IDS = 1000;
/** Task 19:Python 回传逐行日志采样频率(每 N 行记一行)。 */
private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L;
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
@@ -657,10 +659,19 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
public SimilarAsinTaskBatchVo progressBatch(List<Long> taskIds) {
SimilarAsinTaskBatchVo vo = new SimilarAsinTaskBatchVo();
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
List<Long> requestedIds = taskIds == null ? List.of() : taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
// 上限只为挡住超长 IN 子句。被截的尾部不进 missingTaskIds(与 progressLight 同口径:
// 看板会把 missing 当成「这张单不存在」,而它们只是本轮没查),改为记 WARN 留现场。
if (requestedIds.size() > MAX_BATCH_TASK_IDS) {
log.warn("[similar-asin] progressBatch 收到 {} 个 id,超过上限 {},本轮只刷新前 {} 个",
requestedIds.size(), MAX_BATCH_TASK_IDS, MAX_BATCH_TASK_IDS);
}
List<Long> normalizedIds = requestedIds.size() > MAX_BATCH_TASK_IDS
? requestedIds.subList(0, MAX_BATCH_TASK_IDS)
: requestedIds;
if (normalizedIds.isEmpty()) {
return vo;
}
@@ -91,6 +91,9 @@ public class SimilarAsinResultWorkbookAssembler {
private static final int IMG_COL_PUZZLE1 = 13;
private static final int IMG_COL_PUZZLE2 = 14;
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
/** 结果行读取时 chunk / orphan scope_state 的 keyset 页大小,与 loadChunksKeyset 的 500 同口径。 */
private static final int RESULT_ROWS_PAGE_SIZE = 500;
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
private final SimilarAsinProperties properties;
@@ -562,6 +565,11 @@ public class SimilarAsinResultWorkbookAssembler {
/**
* Select and decode one chunk payload at a time. Only rows needed by the
* current source workbook are retained in the returned map.
*
* <p>按 keyset 整页取 chunk(与 {@link SimilarAsinChunkMergeSupport#loadChunksKeyset} 同口径):
* 原先每轮 {@code LIMIT 1} 只取一行,一个源文件要把该任务的 chunk 表逐行翻完,
* 每行一次 SQL + 一次对象读 + 一次整片 JSON 反序列化;请求行全部命中后立即收尾,
* 不再把剩余 chunk 也解一遍。
*/
private Map<String, SimilarAsinResultRowDto> loadPersistedResultRowsForRows(
Long taskId,
@@ -578,7 +586,7 @@ public class SimilarAsinResultWorkbookAssembler {
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskChunkEntity::getId)
.last("LIMIT 1");
.last("LIMIT " + RESULT_ROWS_PAGE_SIZE);
if (lastChunkId != null) {
query.gt(TaskChunkEntity::getId, lastChunkId);
}
@@ -586,10 +594,16 @@ public class SimilarAsinResultWorkbookAssembler {
if (page == null || page.isEmpty()) {
break;
}
TaskChunkEntity chunk = page.getFirst();
appendRequestedResultRows(result, chunkPayloadSupport.readChunkRows(chunk).values(), requestedKeys, false);
lastChunkId = chunk.getId();
if (lastChunkId == null) {
for (TaskChunkEntity chunk : page) {
appendRequestedResultRows(result, chunkPayloadSupport.readChunkRows(chunk).values(),
requestedKeys, false);
}
TaskChunkEntity tail = page.getLast();
lastChunkId = tail == null ? null : tail.getId();
if (lastChunkId == null || page.size() < RESULT_ROWS_PAGE_SIZE) {
break;
}
if (allRequestedRowsResolved(requestedRows, result)) {
break;
}
}
@@ -597,6 +611,21 @@ public class SimilarAsinResultWorkbookAssembler {
return result;
}
/**
* 请求行是否都已取到结果。判据必须与 {@link SimilarAsinChunkMergeSupport#findResultRow}
* 的匹配口径一致(主键或 legacy 键任一命中即算已取到)——直接用 result.size() 与
* requestedKeys.size() 比会永远不成立,因为 requestedKeys 每行都塞了主键 + legacy 两个键。
*/
private boolean allRequestedRowsResolved(List<SimilarAsinParsedRowVo> requestedRows,
Map<String, SimilarAsinResultRowDto> result) {
for (SimilarAsinParsedRowVo row : requestedRows) {
if (SimilarAsinChunkMergeSupport.findResultRow(row, result) == null) {
return false;
}
}
return true;
}
private Set<String> resultLookupKeys(List<SimilarAsinParsedRowVo> rows) {
Set<String> keys = new LinkedHashSet<>();
for (SimilarAsinParsedRowVo row : rows == null ? List.<SimilarAsinParsedRowVo>of() : rows) {
@@ -651,7 +680,7 @@ public class SimilarAsinResultWorkbookAssembler {
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.likeRight(TaskScopeStateEntity::getScopeKey, SimilarAsinChunkPayloadSupport.ORPHAN_SCOPE_KEY_PREFIX)
.orderByAsc(TaskScopeStateEntity::getId)
.last("LIMIT 1");
.last("LIMIT " + RESULT_ROWS_PAGE_SIZE);
if (lastStateId != null) {
query.gt(TaskScopeStateEntity::getId, lastStateId);
}
@@ -659,29 +688,33 @@ public class SimilarAsinResultWorkbookAssembler {
if (page == null || page.isEmpty()) {
return;
}
TaskScopeStateEntity state = page.getFirst();
lastStateId = state.getId();
if (lastStateId == null) {
return;
}
if (state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) {
continue;
}
try {
String payloadJson = transientPayloadStorageService.resolvePayload(
state.getParsedPayloadJson(), "read similar ASIN orphan rows failed");
JsonNode array = objectMapper.readTree(payloadJson);
if (!array.isArray()) {
TaskScopeStateEntity tail = page.getLast();
lastStateId = tail == null ? null : tail.getId();
boolean lastPage = page.size() < RESULT_ROWS_PAGE_SIZE || lastStateId == null;
for (TaskScopeStateEntity state : page) {
if (state == null || state.getParsedPayloadJson() == null
|| state.getParsedPayloadJson().isBlank()) {
continue;
}
List<SimilarAsinResultRowDto> matchingRows = new ArrayList<>();
for (JsonNode node : array) {
matchingRows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class));
try {
String payloadJson = transientPayloadStorageService.resolvePayload(
state.getParsedPayloadJson(), "read similar ASIN orphan rows failed");
JsonNode array = objectMapper.readTree(payloadJson);
if (!array.isArray()) {
continue;
}
List<SimilarAsinResultRowDto> matchingRows = new ArrayList<>();
for (JsonNode node : array) {
matchingRows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class));
}
appendRequestedResultRows(sink, matchingRows, requestedKeys, true);
} catch (Exception ex) {
log.warn("[similar-asin] read requested orphan rows failed taskId={} stateId={} err={}",
taskId, state.getId(), ex.getMessage());
}
appendRequestedResultRows(sink, matchingRows, requestedKeys, true);
} catch (Exception ex) {
log.warn("[similar-asin] read requested orphan rows failed taskId={} stateId={} err={}",
taskId, state.getId(), ex.getMessage());
}
if (lastPage) {
return;
}
}
}
@@ -788,14 +821,23 @@ public class SimilarAsinResultWorkbookAssembler {
if (!prefetchUrls.isEmpty()) {
// DB cache 命中的缩略图也立即落到任务临时目录,避免整批字节停留在堆中。
long dbCacheStart = System.currentTimeMillis();
List<String> remainingUrls = new ArrayList<>(prefetchUrls.size());
int dbHit = 0;
// 一次 IN 批量查(lookupBatch 内部只对命中项 touch last_used_at):
// 原先逐 URL 调 lookup(),按注释里的量级(4 源 × 4000 URL)每次装配要发几千条单行 SQL。
Set<String> pendingUrlSet = new LinkedHashSet<>();
for (String url : prefetchUrls) {
ensureResultAssemblyNotInterrupted();
if (url == null || imageSpool.get(url) != null || taskImageCache.containsKey(url)) {
continue;
}
byte[] cachedBytes = imagePrefetchService.lookup(url);
pendingUrlSet.add(url);
}
List<String> pendingUrls = new ArrayList<>(pendingUrlSet);
List<byte[]> cachedBatch = imagePrefetchService.lookupBatch(pendingUrls);
List<String> remainingUrls = new ArrayList<>();
int dbHit = 0;
for (int i = 0; i < pendingUrls.size(); i++) {
ensureResultAssemblyNotInterrupted();
String url = pendingUrls.get(i);
byte[] cachedBytes = i < cachedBatch.size() ? cachedBatch.get(i) : null;
if (cachedBytes == null) {
remainingUrls.add(url);
continue;