perf(F5+): 行数据按需拉取(结果行版本信号)+ 修复 progress/light 恒判 missing
行数据按需拉取(审查 F5 后续): - V125 给 biz_file_result 补 updated_at(DEFAULT/ON UPDATE 由数据库维护, 实体标注 insertStrategy/updateStrategy=NEVER —— 否则 selectById→updateById 的 写回会把旧值写回去、ON UPDATE 不触发,版本信号静默冻结) - 装配器回传 rowsVersion=「最后变更时间毫秒#行数」,5 个品牌工具页版本未变即跳过 带行明细的重型 batch;前端变更信号为 rowsVersion + status/fileStatus/fileReady 复合 (任务收尾常见「行早写完、之后才置成功」,只看行版本会把界面卡在旧状态) 修复线上缺陷(同一功能验证时暴露): - TaskProgressLightAssembler 列裁剪漏选 module_type 却用它做模块过滤 → getModuleType() 恒为 null → light 恒把任务判成 missing;第七批把 light 接进 跟价/定时匹配/商品风险的轮询后,消费方会把运行中任务判为 FAILED - 补选中列 + 守卫用例 taskQueryMustSelectModuleType(已反向验证:去掉修复即红) - 前端 lightClaimsAllTasksMissing:整体性 missing 结论用重型端点复核后再采信 契约与文档:light 白名单补 rowsVersion(Java 契约测试 / spec 06 §2 / 12 个端点描述) 测试:mvn test 2901 全绿;前端 npm test 765 全绿
This commit is contained in:
@@ -113,6 +113,8 @@ import { passGuard } from '@/shared/dispatch-guard-ui'
|
||||
import { formatDateTime } from '@/shared/utils/datetime'
|
||||
import { uploadPathsToJava } from '@/shared/utils/upload-to-java'
|
||||
import { runBatchDelete } from '@/shared/utils/batch-delete'
|
||||
import { getModuleProgressLight } from '@/shared/api/progress-light.ts'
|
||||
import { createRowsVersionTracker } from '@/shared/api/task-progress-polling.ts'
|
||||
|
||||
const selectedFileNames = ref<string[]>([])
|
||||
const uploadedFiles = ref<UploadFileVo[]>([])
|
||||
@@ -545,6 +547,9 @@ function scheduleNextPoll(immediate = false) {
|
||||
else pollTimer.value = timers.setTimeout('task-poll', run, getTaskPollIntervalMs())
|
||||
}
|
||||
|
||||
/** 结果行版本跟踪(审查 F5 后续):版本未变的任务跳过行明细拉取 */
|
||||
const rowsVersionTracker = createRowsVersionTracker()
|
||||
|
||||
async function refreshTaskProgress() {
|
||||
if (pollingInFlight.value || (!pollingTaskIds.value.length && !pendingFileTaskIds.value.length)) {
|
||||
if (!pollingTaskIds.value.length && !pendingFileTaskIds.value.length) stopPolling()
|
||||
@@ -556,7 +561,23 @@ async function refreshTaskProgress() {
|
||||
let shouldRefreshHistory = false
|
||||
const taskIds = Array.from(new Set([...pollingTaskIds.value, ...pendingFileTaskIds.value]))
|
||||
if (taskIds.length) {
|
||||
const batch = await getAppearancePatentTaskProgressBatch(taskIds, { force: pendingFileTaskIds.value.length > 0 })
|
||||
// 行数据按需拉取(审查 F5 后续):等结果文件(pending)时必须强刷;其余情况版本未变的任务跳过
|
||||
let heavyIds = taskIds
|
||||
const mustForce = pendingFileTaskIds.value.length > 0
|
||||
if (!mustForce) {
|
||||
try {
|
||||
const light = await getModuleProgressLight('appearancePatent', taskIds)
|
||||
heavyIds = rowsVersionTracker.selectTasksNeedingRows(light.items || [], taskIds)
|
||||
if (!heavyIds.length && !(light.missingTaskIds || []).length) {
|
||||
// 行数据与状态都没变:本轮不拉行明细,直接结束
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
// 轻量端点不可用(老后端/网络)→ 照旧全量拉取,行为与原来一致
|
||||
heavyIds = taskIds
|
||||
}
|
||||
}
|
||||
const batch = await getAppearancePatentTaskProgressBatch(heavyIds, { force: mustForce })
|
||||
for (const detail of batch.items || []) {
|
||||
const task = detail.task
|
||||
if (!task?.id) continue
|
||||
|
||||
@@ -285,6 +285,8 @@ import { useHistoryPolling } from '@/shared/composables/useHistoryPolling'
|
||||
import { runBatchDelete } from '@/shared/utils/batch-delete'
|
||||
import { isRecordMissingError } from '@/shared/utils/task-queue-state.ts'
|
||||
import { useTablePaging } from '@/shared/composables/useTablePaging.ts'
|
||||
import { getModuleProgressLight } from '@/shared/api/progress-light.ts'
|
||||
import { createRowsVersionTracker } from '@/shared/api/task-progress-polling.ts'
|
||||
|
||||
const MAX_TRANSIENT_ERRORS = 30;
|
||||
/** 任务终态后等待结果文件(Java 侧异步生成)的最大轮次,12 × 10s ≈ 2 分钟 */
|
||||
@@ -797,6 +799,9 @@ function upsertHistoryItems(incoming: PatrolDeleteHistoryItem[]) {
|
||||
mergeHistoryProgressItems(incoming);
|
||||
}
|
||||
|
||||
/** 结果行版本跟踪:版本未变的任务不再拉取行明细(F5 后续) */
|
||||
const rowsVersionTracker = createRowsVersionTracker()
|
||||
|
||||
async function refreshActiveTaskProgress(taskIds?: number[]) {
|
||||
const ids = Array.from(
|
||||
new Set(
|
||||
@@ -809,9 +814,27 @@ async function refreshActiveTaskProgress(taskIds?: number[]) {
|
||||
if (!ids.length) {
|
||||
return [];
|
||||
}
|
||||
const batch = await getPatrolDeleteTaskProgressBatch(ids);
|
||||
// 行数据按需拉取(审查 F5 后续):先问轻量端点拿"结果行版本",版本未变的任务跳过重型 batch
|
||||
let heavyIds = ids;
|
||||
let missingIds: number[] = [];
|
||||
try {
|
||||
const light = await getModuleProgressLight('patrolDelete', ids);
|
||||
heavyIds = rowsVersionTracker.selectTasksNeedingRows(light.items || [], ids);
|
||||
missingIds = light.missingTaskIds || [];
|
||||
if (!heavyIds.length) {
|
||||
if (missingIds.length) {
|
||||
await loadHistory();
|
||||
}
|
||||
return [];
|
||||
}
|
||||
} catch {
|
||||
// 轻量端点不可用(老后端/网络)→ 照旧全量拉取,行为与原来一致
|
||||
heavyIds = ids;
|
||||
}
|
||||
const batch = await getPatrolDeleteTaskProgressBatch(heavyIds);
|
||||
missingIds = Array.from(new Set([...missingIds, ...(batch.missingTaskIds || [])]));
|
||||
mergeHistoryProgressItems(batch.items || []);
|
||||
if ((batch.missingTaskIds || []).length) {
|
||||
if (missingIds.length) {
|
||||
await loadHistory();
|
||||
}
|
||||
return batch.missingTaskIds || [];
|
||||
|
||||
@@ -227,6 +227,8 @@ import { useHistoryPolling } from '@/shared/composables/useHistoryPolling'
|
||||
import { runBatchDelete } from '@/shared/utils/batch-delete'
|
||||
import { isRecordMissingError, parseTaskStartTimes, removeTaskStartTime, upsertTaskStartTime } from '@/shared/utils/task-queue-state.ts'
|
||||
import { useTablePaging } from '@/shared/composables/useTablePaging.ts'
|
||||
import { getModuleProgressLight } from '@/shared/api/progress-light.ts'
|
||||
import { createRowsVersionTracker } from '@/shared/api/task-progress-polling.ts'
|
||||
|
||||
const MAX_TRANSIENT_ERRORS = 30;
|
||||
const ziniaoVersion = useZiniaoVersion();
|
||||
@@ -705,6 +707,9 @@ function mergeHistoryProgressItems(incoming: QueryAsinHistoryItem[]) {
|
||||
historyItems.value = merged;
|
||||
}
|
||||
|
||||
/** 结果行版本跟踪:版本未变的任务不再拉取行明细(F5 后续) */
|
||||
const rowsVersionTracker = createRowsVersionTracker()
|
||||
|
||||
async function refreshActiveTaskProgress(taskIds?: number[]) {
|
||||
const ids = Array.from(
|
||||
new Set(
|
||||
@@ -717,9 +722,28 @@ async function refreshActiveTaskProgress(taskIds?: number[]) {
|
||||
if (!ids.length) {
|
||||
return;
|
||||
}
|
||||
const batch = await getQueryAsinTaskProgressBatch(ids);
|
||||
// 行数据按需拉取(审查 F5 后续):先问轻量端点拿"结果行版本",版本未变的任务跳过重型 batch
|
||||
let heavyIds = ids;
|
||||
let missingIds: number[] = [];
|
||||
try {
|
||||
const light = await getModuleProgressLight('queryAsin', ids);
|
||||
heavyIds = rowsVersionTracker.selectTasksNeedingRows(light.items || [], ids);
|
||||
missingIds = light.missingTaskIds || [];
|
||||
if (!heavyIds.length) {
|
||||
if (missingIds.length) {
|
||||
await loadHistory();
|
||||
reconcileActiveQueueTaskWithHistory();
|
||||
}
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// 轻量端点不可用(老后端/网络)→ 照旧全量拉取,行为与原来一致
|
||||
heavyIds = ids;
|
||||
}
|
||||
const batch = await getQueryAsinTaskProgressBatch(heavyIds);
|
||||
missingIds = Array.from(new Set([...missingIds, ...(batch.missingTaskIds || [])]));
|
||||
mergeHistoryProgressItems(batch.items || []);
|
||||
if ((batch.missingTaskIds || []).length) {
|
||||
if (missingIds.length) {
|
||||
await loadHistory();
|
||||
reconcileActiveQueueTaskWithHistory();
|
||||
}
|
||||
|
||||
@@ -162,6 +162,8 @@ import {
|
||||
} from '@/shared/api/java-modules'
|
||||
import { runBatchDelete } from '@/shared/utils/batch-delete'
|
||||
import { useTablePaging } from '@/shared/composables/useTablePaging.ts'
|
||||
import { getModuleProgressLight } from '@/shared/api/progress-light.ts'
|
||||
import { createRowsVersionTracker } from '@/shared/api/task-progress-polling.ts'
|
||||
|
||||
const COUNTRY_OPTIONS = [
|
||||
{ code: 'UK', label: '英国' },
|
||||
@@ -399,7 +401,21 @@ function mergeProgress(detail: ShopDataCrawlTaskDetailVo) {
|
||||
function isTaskDetail(row: ShopDataCrawlTaskDetailVo | ShopDataCrawlHistoryItem): row is ShopDataCrawlTaskDetailVo {
|
||||
return 'task' in row || 'items' in row
|
||||
}
|
||||
/** 结果行版本跟踪(F5 后续):版本未变的任务跳过行明细拉取 */
|
||||
const rowsVersionTracker = createRowsVersionTracker()
|
||||
|
||||
async function refreshTaskProgress(taskId: number) {
|
||||
// 行数据按需拉取(审查 F5 后续):该任务结果行版本未变时跳过重型 batch;任务已被删除则照旧走原路径
|
||||
try {
|
||||
const light = await getModuleProgressLight('shopDataCrawl', [taskId])
|
||||
const missing = light.missingTaskIds || []
|
||||
if (!missing.length && !rowsVersionTracker.selectTasksNeedingRows(light.items || [], [taskId]).length) {
|
||||
// 返回 true = 任务仍存在(调用方以此为「任务消失→FAILED」判据,不能返回 undefined)
|
||||
return true
|
||||
}
|
||||
} catch {
|
||||
/* 轻量端点不可用 → 照旧全量拉取 */
|
||||
}
|
||||
const batch = await getShopDataCrawlTaskProgressBatch([taskId])
|
||||
for (const row of batch.items || []) {
|
||||
if (isTaskDetail(row)) {
|
||||
|
||||
@@ -267,6 +267,8 @@ import { useHistoryPolling } from '@/shared/composables/useHistoryPolling'
|
||||
import { runBatchDelete } from '@/shared/utils/batch-delete'
|
||||
import { isRecordMissingError, parseTaskStartTimes, removeTaskStartTime, upsertTaskStartTime } from '@/shared/utils/task-queue-state.ts'
|
||||
import { useTablePaging } from '@/shared/composables/useTablePaging.ts'
|
||||
import { getModuleProgressLight } from '@/shared/api/progress-light.ts'
|
||||
import { createRowsVersionTracker } from '@/shared/api/task-progress-polling.ts'
|
||||
|
||||
const MAX_TRANSIENT_ERRORS = 30;
|
||||
const ziniaoVersion = useZiniaoVersion();
|
||||
@@ -829,6 +831,9 @@ function mergeHistoryProgressItems(incoming: WithdrawHistoryItem[]) {
|
||||
historyItems.value = merged;
|
||||
}
|
||||
|
||||
/** 结果行版本跟踪:版本未变的任务不再拉取行明细(F5 后续) */
|
||||
const rowsVersionTracker = createRowsVersionTracker()
|
||||
|
||||
async function refreshActiveTaskProgress(taskIds?: number[]) {
|
||||
const ids = Array.from(
|
||||
new Set(
|
||||
@@ -841,9 +846,28 @@ async function refreshActiveTaskProgress(taskIds?: number[]) {
|
||||
if (!ids.length) {
|
||||
return;
|
||||
}
|
||||
const batch = await getWithdrawTaskProgressBatch(ids);
|
||||
// 行数据按需拉取(审查 F5 后续):先问轻量端点拿"结果行版本",版本未变的任务跳过重型 batch
|
||||
let heavyIds = ids;
|
||||
let missingIds: number[] = [];
|
||||
try {
|
||||
const light = await getModuleProgressLight('withdraw', ids);
|
||||
heavyIds = rowsVersionTracker.selectTasksNeedingRows(light.items || [], ids);
|
||||
missingIds = light.missingTaskIds || [];
|
||||
if (!heavyIds.length) {
|
||||
if (missingIds.length) {
|
||||
await loadHistory();
|
||||
reconcileActiveQueueTaskWithHistory();
|
||||
}
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// 轻量端点不可用(老后端/网络)→ 照旧全量拉取,行为与原来一致
|
||||
heavyIds = ids;
|
||||
}
|
||||
const batch = await getWithdrawTaskProgressBatch(heavyIds);
|
||||
missingIds = Array.from(new Set([...missingIds, ...(batch.missingTaskIds || [])]));
|
||||
mergeHistoryProgressItems(batch.items || []);
|
||||
if ((batch.missingTaskIds || []).length) {
|
||||
if (missingIds.length) {
|
||||
await loadHistory();
|
||||
reconcileActiveQueueTaskWithHistory();
|
||||
}
|
||||
|
||||
@@ -42,6 +42,8 @@ export interface TaskProgressLightItem {
|
||||
fileError?: string | null
|
||||
fileReady?: boolean | null
|
||||
updatedAt?: string | null
|
||||
/** 结果行版本(「最后变更时间毫秒#行数」):未变说明行数据没动过,可跳过重型 batch。 */
|
||||
rowsVersion?: string | null
|
||||
}
|
||||
|
||||
export interface GetModuleProgressLightOptions {
|
||||
|
||||
@@ -61,6 +61,77 @@ export interface PollingBatchLike<T> {
|
||||
missingTaskIds?: number[]
|
||||
}
|
||||
|
||||
/**
|
||||
* 行数据按需拉取(审查 F5 后续):
|
||||
* 轻量响应里的 rowsVersion 是该任务**结果行**的版本(最后变更时间毫秒 + 行数,后端一次 GROUP BY 查出)。
|
||||
* 版本没变说明行数据没动过,就不必再拉带行明细的重型 batch。
|
||||
*
|
||||
* 变更信号是**复合**的:行版本之外还看 status / fileStatus / fileReady——任务收尾时常见
|
||||
* 「行已经写完、之后任务才置成功」或「结果文件才就绪」,只看行版本会把界面卡在旧状态。
|
||||
*
|
||||
* 安全兜底(保证零回归):
|
||||
* - 四项信号全空(老后端 / 字段缺失)→ 视为拿不到信号,该任务照旧每轮拉取;
|
||||
* - 轻量请求本身失败时调用方回退为全量拉取(见各页 catch 分支)。
|
||||
*/
|
||||
export interface TaskProgressChangeSignal {
|
||||
taskId?: number
|
||||
rowsVersion?: string | null
|
||||
status?: string | null
|
||||
fileStatus?: string | null
|
||||
fileReady?: boolean | null
|
||||
}
|
||||
|
||||
/** 组装变更信号;taskId 非法或四项信号全空时返回 null(= 拿不到信号,必须拉取)。 */
|
||||
function changeSignatureOf(raw: unknown): { taskId: number; signature: string } | null {
|
||||
const item = raw as TaskProgressChangeSignal | null
|
||||
if (!item || typeof item.taskId !== 'number' || !(item.taskId > 0)) return null
|
||||
const rowsVersion = typeof item.rowsVersion === 'string' ? item.rowsVersion : ''
|
||||
const status = typeof item.status === 'string' ? item.status : ''
|
||||
const fileStatus = typeof item.fileStatus === 'string' ? item.fileStatus : ''
|
||||
const fileReady = item.fileReady === true ? '1' : item.fileReady === false ? '0' : ''
|
||||
if (!rowsVersion && !status && !fileStatus && !fileReady) return null
|
||||
return { taskId: item.taskId, signature: [rowsVersion, status, fileStatus, fileReady].join('|') }
|
||||
}
|
||||
|
||||
export function createRowsVersionTracker() {
|
||||
const seen = new Map<number, string>()
|
||||
return {
|
||||
/**
|
||||
* 返回"需要拉取行明细"的任务 id(信号变化 / 拿不到信号的)。
|
||||
* 同时记录本轮看到的信号(便于下轮比较)。
|
||||
*/
|
||||
selectTasksNeedingRows(items: unknown[] | null | undefined, requestedIds: number[]): number[] {
|
||||
const signatureById = new Map<number, string>()
|
||||
for (const raw of items || []) {
|
||||
const parsed = changeSignatureOf(raw)
|
||||
if (parsed) signatureById.set(parsed.taskId, parsed.signature)
|
||||
}
|
||||
const needRows: number[] = []
|
||||
for (const taskId of requestedIds) {
|
||||
const signature = signatureById.get(taskId)
|
||||
if (signature === undefined) {
|
||||
needRows.push(taskId)
|
||||
continue
|
||||
}
|
||||
if (seen.get(taskId) !== signature) {
|
||||
needRows.push(taskId)
|
||||
}
|
||||
}
|
||||
for (const [taskId, signature] of signatureById) {
|
||||
seen.set(taskId, signature)
|
||||
}
|
||||
return needRows
|
||||
},
|
||||
/** 任务结束/被删除时清掉记录,避免 Map 无界增长。 */
|
||||
forget(taskId: number): void {
|
||||
seen.delete(taskId)
|
||||
},
|
||||
reset(): void {
|
||||
seen.clear()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export interface PollingProgressOptions<TBatch> {
|
||||
/** 重型端点兜底(轻量端点异常或返回空时调用),不传则不兜底 */
|
||||
fallback?: () => Promise<TBatch>
|
||||
@@ -83,6 +154,23 @@ export function pickPollingBatch<T>(
|
||||
return { items: items as unknown as T[], missingTaskIds: missing }
|
||||
}
|
||||
|
||||
/**
|
||||
* light 是否把**全部**请求任务都判成"不存在"。
|
||||
*
|
||||
* <p>整体性结论风险最大:消费方把 missing 当"任务已消失"(跟价页直接返回 FAILED)。
|
||||
* 2026-09-14 线上即出现过 light 因后端漏选过滤字段而恒判 missing,故这种结论要用重型端点复核。
|
||||
*/
|
||||
export function lightClaimsAllTasksMissing(
|
||||
picked: PollingBatchLike<unknown> | null | undefined,
|
||||
requestedIds: number[],
|
||||
): boolean {
|
||||
const ids = (requestedIds || []).filter((id) => typeof id === 'number' && id > 0)
|
||||
if (!ids.length) return false
|
||||
if ((picked?.items || []).length) return false
|
||||
const missing = picked?.missingTaskIds || []
|
||||
return ids.every((id) => missing.includes(id))
|
||||
}
|
||||
|
||||
/**
|
||||
* 取任务进度:轻量端点优先,异常/空结果时回退重型端点。
|
||||
* 轻量端点返回 empty(既没有 items 也没有 missingTaskIds)时也走兜底——
|
||||
@@ -98,6 +186,10 @@ export async function getPollingProgressBatch<TBatch extends PollingBatchLike<un
|
||||
const light = await getModuleProgressLight(module, taskIds)
|
||||
const picked = pickPollingBatch<unknown>(light.items, light.missingTaskIds)
|
||||
if (picked) {
|
||||
if (fallback && lightClaimsAllTasksMissing(picked, taskIds)) {
|
||||
// 复核:以重型端点为准(它若也判不存在,结论一致;若返回了条目,说明 light 错了)
|
||||
return fallback()
|
||||
}
|
||||
return picked as TBatch
|
||||
}
|
||||
} catch {
|
||||
|
||||
Reference in New Issue
Block a user