Files
crawler-plugin/admin-frontend-vue/tests/dup-matrix-window.test.ts
T
huangzd1997 c8cd75cf6c feat(phase-5): B60 console 分页与矩阵预聚合+P1-B5/6/7/8 性能热点收口
接管另一路会话的 Phase 5 在飞件收口(5A/5C 主体由那一路落码,本笔补齐 P1-B7 与两处台账并整量验证)。

- 5A/B60:撞款 console 的行窗口在服务端真分页(响应 dup 行数 ≤ pageSize,total_dup 仍是未分页
  总数)、dupGroup 判据收归服务端单一出处(前端 dupWithinGroup 删掉,两套同名规则不再分叉)、
  排序先于切窗;后管矩阵改预聚合+列窗 40。读数(200 列 × 50 行):每行 3 店 8.4 → 5.3 → 0.7ms
  (12.4×),每行 40 店 83.7 → 18.4 → 1.8ms(45.3×)。服务端不设列窗是判断不是漏做:shops 每行
  只有 5 个标量,而矩阵的列来自已分页的行——给 shops 再加分页会把"全局指标读全集"和"列窗读本页"
  两套语义搅在一起。浏览器首屏/交互耗时本机拿不到,登记为 Phase 6 现网复量。
- P1-B6:claim 循环的父任务终态判定折成一次批量 IN(TaskFileJobBatchQueries+Mapper XML,
  铁律 1 收口在 Mapper),逐行 CAS 认领一字不动。
- P1-B5:V187 只建 idx_biz_file_task_resume_of;审计给的候选集索引**有意不建**(V184 前缀已覆盖,
  OR/IS-NULL 与 COALESCE 不可索引),判据把这条负判定钉住。真库 EXPLAIN 本机无 MySQL,顺延现网。
- P1-B8:结果载荷的对象存储删除移到 afterCommit,事务里不再做 120s 超时的网络调用。
- P1-B7:结果集读取按窗提交+收割(窗口=线程池队列容量的一半)。审计把它记成"无 limit",但截断
  是正确性错误不是性能取舍,真能收的是"整份结果集被读两遍"——一次性提交时每支 Future 把解好的
  载荷扣在自己身上直到全部收割完,峰值=结果集+另一份整副本;分窗后副本只驻留一窗。流式组装
  契约要改 4 个模块的调用方,本 Phase 未做,别读成 P1-B7 已闭环。
- SourceScanTest 台账随之登记:裸名 25 → 26、调用方 18 → 19(P1-B7 的窗口判据只能打在源码上——
  线程池只有 8 个线程,"同时开始读几行"改前改后读数相同,差异只体现在堆上,mock 数不出来)。
2026-11-03 23:00:44 +08:00

184 lines
8.2 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import test from 'node:test'
import assert from 'node:assert/strict'
import { buildMatrixCells, matrixCellOf, storeColumnsOf } from '../src/pages/tasks/duplicate-console-logic.ts'
import type { DuplicateItem, DuplicateOccurrence } from '../src/pages/tasks/duplicate-model.ts'
/**
* B60 撞款矩阵:预聚合的等价性与前后读数。
*
* 审计指的那件事是「每格多次全量 filter」——模板一格要付 cellCount ×2 + cellTitle(内部再
* cellCount + cellCountryDots + 每站点一次 cellCountryCount),每次都是把该行 occurrences 整份
* filter 一遍,而且**每格都新建一条文案字符串**。下面 `legacyCell` 就是页面原来那几行函数的
* 原样搬运,当"前";与新一次算完的 `buildMatrixCells`(空格按列共用一份视图)对撞:
* 先证明结果逐字相同(预聚合不是靠少算蒙过去的),再同尺量时间。
*/
function occ(asin: string, shopName: string, country: string): DuplicateOccurrence {
return {
asin,
date: '2026-09-01',
price: '',
brand: 'B',
shopName,
groupName: '',
countryCodes: [country],
country,
}
}
const shopKey = (name: string): string => (name || '').trim().toLowerCase()
interface CellView {
count: number
dots: string[]
title: string
}
/** 原页面的逐格算法(一格扫整行 occurrences 若干次,并且每格造一条文案)。 */
function legacyCellOccurrences(item: DuplicateItem, shop: string): DuplicateOccurrence[] {
const key = shopKey(shop)
return (item.occurrences || []).filter((entry) => shopKey(entry.shopName) === key)
}
function legacyCell(item: DuplicateItem, shop: string): CellView {
const count = legacyCellOccurrences(item, shop).length
const codes = new Set<string>()
for (const entry of legacyCellOccurrences(item, shop)) if (entry.country) codes.add(entry.country.toUpperCase())
const dots = [...codes]
const sites = dots.map((code) => `${code} ${legacyCellOccurrences(item, shop).filter((entry) => (entry.country || '').toUpperCase() === code).length}条`)
return {
count,
dots,
title: count ? `${shop} 在售 ${count} 条 · ${sites.join('、')}` : `${shop} 无此 ASIN`,
}
}
/** 页面现在的读法:一次取值;空格取该列那一份共用视图(不再造文案)。 */
function fastCell(cells: ReturnType<typeof buildMatrixCells>, item: DuplicateItem, shop: string, blank: CellView): CellView {
const cell = matrixCellOf(cells, item.asin, shop)
if (!cell.count) return blank
const sites = cell.sites.map((site) => `${site.code} ${site.count}条`)
return {
count: cell.count,
dots: cell.sites.map((site) => site.code),
title: sites.length ? `${shop} 在售 ${cell.count} 条 · ${sites.join('、')}` : `${shop} 在售 ${cell.count} 条`,
}
}
function fixtureItem(asin: string, shops: [string, string][]): DuplicateItem {
const occurrences = shops.map(([shop, country]) => occ(asin, shop, country))
return {
asin,
shopCount: new Set(occurrences.map((entry) => shopKey(entry.shopName))).size,
recordCount: occurrences.length,
occurrences,
}
}
function blankViews(columns: string[]): CellView[] {
return columns.map((shop) => ({ count: 0, dots: [] as string[], title: `${shop} 无此 ASIN` }))
}
/**
* 200 列 × 50 行的矩阵夹具。`shopsPerRow` 决定密度:现实中一个撞款 ASIN 落在几家到几十家店,
* 而逐格 filter 的代价正是按「行内 occurrences × 列数」乘出来的——密度越高旧算法越吃亏,
* 所以两种密度都取读数,不挑对自己有利的那一档。
*/
function matrixFixture(columns: string[], shopsPerRow: number): DuplicateItem[] {
return Array.from({ length: 50 }, (_, index) => {
const pairs: [string, string][] = []
for (let offset = 0; offset < shopsPerRow; offset++) {
const shop = columns[(index * 7 + offset * 61) % columns.length]
pairs.push([shop, 'UK'], [shop, 'DE'])
}
return fixtureItem(`B${String(index).padStart(4, '0')}`, pairs)
})
}
test('矩阵预聚合与逐格 filter 结果逐字相同', () => {
const rows = [
fixtureItem('A1', [['Shop a', 'UK'], ['Shop a', 'uk'], ['ShopB', ''], ['ShopB', 'DE']]),
fixtureItem('A2', [['ShopC', 'FR']]),
]
const columns = storeColumnsOf(rows)
const cells = buildMatrixCells(rows, columns)
const blanks = blankViews(columns)
columns.forEach((column, index) => {
for (const item of rows) {
assert.deepEqual(fastCell(cells, item, column, blanks[index]), legacyCell(item, column), `${item.asin}/${column}`)
}
})
// 每行只给「本行真涉及」的列建格:A1 涉及 2 店(大小写/空格归一后同店)+ A2 涉及 1 店。
assert.equal(cells.size, 3)
})
test('列集或行集为空时不建任何格', () => {
const rows = [fixtureItem('A1', [['ShopA', 'UK']])]
assert.equal(buildMatrixCells(rows, []).size, 0)
assert.equal(buildMatrixCells([], ['ShopA']).size, 0)
})
/** 列窗口:只给本窗点名的列建格(120 列的夹具切 40 列窗口,格子数就按 40 乘)。 */
test('列窗口切窗后每行最多铺满窗口宽度的格', () => {
const window = 40
const rows = Array.from({ length: 50 }, (_, index) =>
fixtureItem(`A${index}`, Array.from({ length: window * 3 }, (_, shop) => [`Shop${shop}`, 'UK'])))
const all = storeColumnsOf(rows)
assert.equal(all.length, window * 3, '夹具本身就有 120 列(不设上限时的形状)')
const columns = all.slice(0, window)
const cells = buildMatrixCells(rows, columns)
assert.equal(columns.length, window, '本窗只铺 40 列')
assert.equal(cells.size, rows.length * window, '窗口外的 80 列一格都不建')
})
/**
* B60 判据的读数:200 列 × 50 行,同一把尺前后各跑一遍(5 轮取最好一次——本机同时有别的路在跑,
* 单次读数能被调度噪声放大好几倍)。
*
* 度量对象是「矩阵格集合的 JS 计算」,审计点名的就是这条。**浏览器首屏/交互耗时本机拿不到**
* (没有真实店铺数据、也没有浏览器现场),按准出括注登记为现网复量项,不许拿这条当"首屏已实测"读。
*/
test('200列×50行:预聚合与按列切窗都必须快过逐格 filter', () => {
const columns = Array.from({ length: 200 }, (_, index) => `Shop${index}`)
const blanks = blankViews(columns)
const windowColumns = columns.slice(0, 40)
const windowBlanks = blanks.slice(0, 40)
const best = (run: () => unknown): number => {
let ms = Number.POSITIVE_INFINITY
for (let attempt = 0; attempt < 5; attempt++) {
const start = performance.now()
run()
ms = Math.min(ms, performance.now() - start)
}
return ms
}
const readings: string[] = []
for (const shopsPerRow of [3, 40]) {
const rows = matrixFixture(columns, shopsPerRow)
const legacySweep = () => {
const out: CellView[] = []
for (const item of rows) for (const column of columns) out.push(legacyCell(item, column))
return out
}
const precomputedSweep = (cols: string[], cellBlanks: CellView[]) => {
const cells = buildMatrixCells(rows, cols)
const out: CellView[] = []
for (const item of rows) {
cols.forEach((column, index) => out.push(fastCell(cells, item, column, cellBlanks[index])))
}
return out
}
assert.deepEqual(precomputedSweep(columns, blanks), legacySweep(), '前后必须同一份格子(快不是靠少算)')
const beforeMs = best(legacySweep)
const precomputeMs = best(() => precomputedSweep(columns, blanks))
const shippedMs = best(() => precomputedSweep(windowColumns, windowBlanks))
readings.push(`每行${shopsPerRow}店:逐格filter=${beforeMs.toFixed(1)}ms → 预聚合=${precomputeMs.toFixed(1)}ms(${(beforeMs / precomputeMs).toFixed(1)}×) → 预聚合+切窗40列=${shippedMs.toFixed(1)}ms(${(beforeMs / shippedMs).toFixed(1)}×)`)
assert.ok(precomputeMs < beforeMs, `预聚合没省下来(每行${shopsPerRow}店):${beforeMs.toFixed(1)}ms → ${precomputeMs.toFixed(1)}ms`)
assert.ok(shippedMs * 2 < beforeMs, `切窗后仍不够快(每行${shopsPerRow}店):${beforeMs.toFixed(1)}ms → ${shippedMs.toFixed(1)}ms`)
}
console.log(`[B60 读数] 200列×50行 ${readings.join(' | ')}`)
})