perf(代码审查落地): 后端 A1-A8 与前端 C1-C8 加固,retention 清理补索引(V141)
A 组(backend-java): - A1 载荷引用计数不再无条件回落全表 COUNT - A2 结果行读取由 LIMIT 1 改整页 - A3 ziniao 索引缓存过期判断下推 SQL - A4 循环子任务认领改 CAS,失败一律抛 - A5 建单事务里不碰外部 IO - A6 连接池补网络层超时(connectTimeout/socketTimeout,env 可覆盖) - A7 日志留现场 + 异步落盘 - A8 其余:货源图片预取、结果工作簿装配等 - V141:三条 retention DELETE 的可用索引,其中 biz_price_track_loop_run 用 STORED 生成列承载 COALESCE 时间线(该表实测 881 行/17MB,COPY 秒级) C 组(frontend-vue):URL 取值走 safe-url、行号取值走 row-id、 交付视频任务状态抽成 delivery-video-task-state.ts,各补用例。
This commit is contained in:
+5
-7
@@ -58,6 +58,9 @@ public class SimilarAsinImagePrefetchService {
|
||||
/** 批量命中检查/批量 touch 单次 IN 上限,避免超长 SQL 参数列表。 */
|
||||
private static final int CACHE_LOOKUP_BATCH_SIZE = 500;
|
||||
|
||||
/** url_hash 十六进制编码用:逐字节 String.format 在按 URL 调用的路径上会放大成几万次格式化。 */
|
||||
private static final java.util.HexFormat HEX_FORMAT = java.util.HexFormat.of();
|
||||
|
||||
/** 单个 task 最多缓存的待预热 URL 数,避免持续回传时内存无界增长。 */
|
||||
private static final int MAX_PENDING_URLS_PER_TASK = 5_000;
|
||||
/** 单机最多同时维护的预热 task 数。 */
|
||||
@@ -316,7 +319,7 @@ public class SimilarAsinImagePrefetchService {
|
||||
* 未命中 url 不产生任何 touch。输入按传入顺序返回,未命中为 null。
|
||||
* 失败/开关关闭返回空列表,由调用方走回退路径。
|
||||
*/
|
||||
List<byte[]> lookupBatch(List<String> urls) {
|
||||
public List<byte[]> lookupBatch(List<String> urls) {
|
||||
List<byte[]> result = new ArrayList<>();
|
||||
if (!properties.isImageDbCacheEnabled()) {
|
||||
return result;
|
||||
@@ -415,12 +418,7 @@ public class SimilarAsinImagePrefetchService {
|
||||
private static String sha256Hex(String value) {
|
||||
try {
|
||||
MessageDigest md = MessageDigest.getInstance("SHA-256");
|
||||
byte[] digest = md.digest(value.getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(digest.length * 2);
|
||||
for (byte b : digest) {
|
||||
sb.append(String.format("%02x", b & 0xFF));
|
||||
}
|
||||
return sb.toString();
|
||||
return HEX_FORMAT.formatHex(md.digest(value.getBytes(StandardCharsets.UTF_8)));
|
||||
} catch (NoSuchAlgorithmException ex) {
|
||||
// SHA-256 在 JDK 中是必备算法,正常环境下不会落到这里。
|
||||
return null;
|
||||
|
||||
+12
-1
@@ -132,6 +132,8 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
||||
private static final int MAX_LIGHT_TASK_IDS = 200;
|
||||
/** progressBatch 的 IN 子句上限:看板单次轮询的在册任务量级远小于它,取值留了 5 倍余量。 */
|
||||
private static final int MAX_BATCH_TASK_IDS = 1000;
|
||||
/** Task 19:Python 回传逐行日志采样频率(每 N 行记一行)。 */
|
||||
private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L;
|
||||
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
|
||||
@@ -657,10 +659,19 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
|
||||
public SimilarAsinTaskBatchVo progressBatch(List<Long> taskIds) {
|
||||
SimilarAsinTaskBatchVo vo = new SimilarAsinTaskBatchVo();
|
||||
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
|
||||
List<Long> requestedIds = taskIds == null ? List.of() : taskIds.stream()
|
||||
.filter(id -> id != null && id > 0)
|
||||
.distinct()
|
||||
.toList();
|
||||
// 上限只为挡住超长 IN 子句。被截的尾部不进 missingTaskIds(与 progressLight 同口径:
|
||||
// 看板会把 missing 当成「这张单不存在」,而它们只是本轮没查),改为记 WARN 留现场。
|
||||
if (requestedIds.size() > MAX_BATCH_TASK_IDS) {
|
||||
log.warn("[similar-asin] progressBatch 收到 {} 个 id,超过上限 {},本轮只刷新前 {} 个",
|
||||
requestedIds.size(), MAX_BATCH_TASK_IDS, MAX_BATCH_TASK_IDS);
|
||||
}
|
||||
List<Long> normalizedIds = requestedIds.size() > MAX_BATCH_TASK_IDS
|
||||
? requestedIds.subList(0, MAX_BATCH_TASK_IDS)
|
||||
: requestedIds;
|
||||
if (normalizedIds.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
|
||||
+72
-30
@@ -91,6 +91,9 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
private static final int IMG_COL_PUZZLE1 = 13;
|
||||
private static final int IMG_COL_PUZZLE2 = 14;
|
||||
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
|
||||
|
||||
/** 结果行读取时 chunk / orphan scope_state 的 keyset 页大小,与 loadChunksKeyset 的 500 同口径。 */
|
||||
private static final int RESULT_ROWS_PAGE_SIZE = 500;
|
||||
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
|
||||
|
||||
private final SimilarAsinProperties properties;
|
||||
@@ -562,6 +565,11 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
/**
|
||||
* Select and decode one chunk payload at a time. Only rows needed by the
|
||||
* current source workbook are retained in the returned map.
|
||||
*
|
||||
* <p>按 keyset 整页取 chunk(与 {@link SimilarAsinChunkMergeSupport#loadChunksKeyset} 同口径):
|
||||
* 原先每轮 {@code LIMIT 1} 只取一行,一个源文件要把该任务的 chunk 表逐行翻完,
|
||||
* 每行一次 SQL + 一次对象读 + 一次整片 JSON 反序列化;请求行全部命中后立即收尾,
|
||||
* 不再把剩余 chunk 也解一遍。
|
||||
*/
|
||||
private Map<String, SimilarAsinResultRowDto> loadPersistedResultRowsForRows(
|
||||
Long taskId,
|
||||
@@ -578,7 +586,7 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.orderByAsc(TaskChunkEntity::getId)
|
||||
.last("LIMIT 1");
|
||||
.last("LIMIT " + RESULT_ROWS_PAGE_SIZE);
|
||||
if (lastChunkId != null) {
|
||||
query.gt(TaskChunkEntity::getId, lastChunkId);
|
||||
}
|
||||
@@ -586,10 +594,16 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
if (page == null || page.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
TaskChunkEntity chunk = page.getFirst();
|
||||
appendRequestedResultRows(result, chunkPayloadSupport.readChunkRows(chunk).values(), requestedKeys, false);
|
||||
lastChunkId = chunk.getId();
|
||||
if (lastChunkId == null) {
|
||||
for (TaskChunkEntity chunk : page) {
|
||||
appendRequestedResultRows(result, chunkPayloadSupport.readChunkRows(chunk).values(),
|
||||
requestedKeys, false);
|
||||
}
|
||||
TaskChunkEntity tail = page.getLast();
|
||||
lastChunkId = tail == null ? null : tail.getId();
|
||||
if (lastChunkId == null || page.size() < RESULT_ROWS_PAGE_SIZE) {
|
||||
break;
|
||||
}
|
||||
if (allRequestedRowsResolved(requestedRows, result)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -597,6 +611,21 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 请求行是否都已取到结果。判据必须与 {@link SimilarAsinChunkMergeSupport#findResultRow}
|
||||
* 的匹配口径一致(主键或 legacy 键任一命中即算已取到)——直接用 result.size() 与
|
||||
* requestedKeys.size() 比会永远不成立,因为 requestedKeys 每行都塞了主键 + legacy 两个键。
|
||||
*/
|
||||
private boolean allRequestedRowsResolved(List<SimilarAsinParsedRowVo> requestedRows,
|
||||
Map<String, SimilarAsinResultRowDto> result) {
|
||||
for (SimilarAsinParsedRowVo row : requestedRows) {
|
||||
if (SimilarAsinChunkMergeSupport.findResultRow(row, result) == null) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private Set<String> resultLookupKeys(List<SimilarAsinParsedRowVo> rows) {
|
||||
Set<String> keys = new LinkedHashSet<>();
|
||||
for (SimilarAsinParsedRowVo row : rows == null ? List.<SimilarAsinParsedRowVo>of() : rows) {
|
||||
@@ -651,7 +680,7 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.likeRight(TaskScopeStateEntity::getScopeKey, SimilarAsinChunkPayloadSupport.ORPHAN_SCOPE_KEY_PREFIX)
|
||||
.orderByAsc(TaskScopeStateEntity::getId)
|
||||
.last("LIMIT 1");
|
||||
.last("LIMIT " + RESULT_ROWS_PAGE_SIZE);
|
||||
if (lastStateId != null) {
|
||||
query.gt(TaskScopeStateEntity::getId, lastStateId);
|
||||
}
|
||||
@@ -659,29 +688,33 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
if (page == null || page.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
TaskScopeStateEntity state = page.getFirst();
|
||||
lastStateId = state.getId();
|
||||
if (lastStateId == null) {
|
||||
return;
|
||||
}
|
||||
if (state.getParsedPayloadJson() == null || state.getParsedPayloadJson().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
String payloadJson = transientPayloadStorageService.resolvePayload(
|
||||
state.getParsedPayloadJson(), "read similar ASIN orphan rows failed");
|
||||
JsonNode array = objectMapper.readTree(payloadJson);
|
||||
if (!array.isArray()) {
|
||||
TaskScopeStateEntity tail = page.getLast();
|
||||
lastStateId = tail == null ? null : tail.getId();
|
||||
boolean lastPage = page.size() < RESULT_ROWS_PAGE_SIZE || lastStateId == null;
|
||||
for (TaskScopeStateEntity state : page) {
|
||||
if (state == null || state.getParsedPayloadJson() == null
|
||||
|| state.getParsedPayloadJson().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
List<SimilarAsinResultRowDto> matchingRows = new ArrayList<>();
|
||||
for (JsonNode node : array) {
|
||||
matchingRows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class));
|
||||
try {
|
||||
String payloadJson = transientPayloadStorageService.resolvePayload(
|
||||
state.getParsedPayloadJson(), "read similar ASIN orphan rows failed");
|
||||
JsonNode array = objectMapper.readTree(payloadJson);
|
||||
if (!array.isArray()) {
|
||||
continue;
|
||||
}
|
||||
List<SimilarAsinResultRowDto> matchingRows = new ArrayList<>();
|
||||
for (JsonNode node : array) {
|
||||
matchingRows.add(objectMapper.treeToValue(node, SimilarAsinResultRowDto.class));
|
||||
}
|
||||
appendRequestedResultRows(sink, matchingRows, requestedKeys, true);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] read requested orphan rows failed taskId={} stateId={} err={}",
|
||||
taskId, state.getId(), ex.getMessage());
|
||||
}
|
||||
appendRequestedResultRows(sink, matchingRows, requestedKeys, true);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] read requested orphan rows failed taskId={} stateId={} err={}",
|
||||
taskId, state.getId(), ex.getMessage());
|
||||
}
|
||||
if (lastPage) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -788,14 +821,23 @@ public class SimilarAsinResultWorkbookAssembler {
|
||||
if (!prefetchUrls.isEmpty()) {
|
||||
// DB cache 命中的缩略图也立即落到任务临时目录,避免整批字节停留在堆中。
|
||||
long dbCacheStart = System.currentTimeMillis();
|
||||
List<String> remainingUrls = new ArrayList<>(prefetchUrls.size());
|
||||
int dbHit = 0;
|
||||
// 一次 IN 批量查(lookupBatch 内部只对命中项 touch last_used_at):
|
||||
// 原先逐 URL 调 lookup(),按注释里的量级(4 源 × 4000 URL)每次装配要发几千条单行 SQL。
|
||||
Set<String> pendingUrlSet = new LinkedHashSet<>();
|
||||
for (String url : prefetchUrls) {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
if (url == null || imageSpool.get(url) != null || taskImageCache.containsKey(url)) {
|
||||
continue;
|
||||
}
|
||||
byte[] cachedBytes = imagePrefetchService.lookup(url);
|
||||
pendingUrlSet.add(url);
|
||||
}
|
||||
List<String> pendingUrls = new ArrayList<>(pendingUrlSet);
|
||||
List<byte[]> cachedBatch = imagePrefetchService.lookupBatch(pendingUrls);
|
||||
List<String> remainingUrls = new ArrayList<>();
|
||||
int dbHit = 0;
|
||||
for (int i = 0; i < pendingUrls.size(); i++) {
|
||||
ensureResultAssemblyNotInterrupted();
|
||||
String url = pendingUrls.get(i);
|
||||
byte[] cachedBytes = i < cachedBatch.size() ? cachedBatch.get(i) : null;
|
||||
if (cachedBytes == null) {
|
||||
remainingUrls.add(url);
|
||||
continue;
|
||||
|
||||
Reference in New Issue
Block a user