Files
crawler-plugin/admin-frontend-vue/src/pages/tasks/duplicate-console-logic.ts
T
huangzd1997 c8cd75cf6c feat(phase-5): B60 console 分页与矩阵预聚合+P1-B5/6/7/8 性能热点收口
接管另一路会话的 Phase 5 在飞件收口(5A/5C 主体由那一路落码,本笔补齐 P1-B7 与两处台账并整量验证)。

- 5A/B60:撞款 console 的行窗口在服务端真分页(响应 dup 行数 ≤ pageSize,total_dup 仍是未分页
  总数)、dupGroup 判据收归服务端单一出处(前端 dupWithinGroup 删掉,两套同名规则不再分叉)、
  排序先于切窗;后管矩阵改预聚合+列窗 40。读数(200 列 × 50 行):每行 3 店 8.4 → 5.3 → 0.7ms
  (12.4×),每行 40 店 83.7 → 18.4 → 1.8ms(45.3×)。服务端不设列窗是判断不是漏做:shops 每行
  只有 5 个标量,而矩阵的列来自已分页的行——给 shops 再加分页会把"全局指标读全集"和"列窗读本页"
  两套语义搅在一起。浏览器首屏/交互耗时本机拿不到,登记为 Phase 6 现网复量。
- P1-B6:claim 循环的父任务终态判定折成一次批量 IN(TaskFileJobBatchQueries+Mapper XML,
  铁律 1 收口在 Mapper),逐行 CAS 认领一字不动。
- P1-B5:V187 只建 idx_biz_file_task_resume_of;审计给的候选集索引**有意不建**(V184 前缀已覆盖,
  OR/IS-NULL 与 COALESCE 不可索引),判据把这条负判定钉住。真库 EXPLAIN 本机无 MySQL,顺延现网。
- P1-B8:结果载荷的对象存储删除移到 afterCommit,事务里不再做 120s 超时的网络调用。
- P1-B7:结果集读取按窗提交+收割(窗口=线程池队列容量的一半)。审计把它记成"无 limit",但截断
  是正确性错误不是性能取舍,真能收的是"整份结果集被读两遍"——一次性提交时每支 Future 把解好的
  载荷扣在自己身上直到全部收割完,峰值=结果集+另一份整副本;分窗后副本只驻留一窗。流式组装
  契约要改 4 个模块的调用方,本 Phase 未做,别读成 P1-B7 已闭环。
- SourceScanTest 台账随之登记:裸名 25 → 26、调用方 18 → 19(P1-B7 的窗口判据只能打在源码上——
  线程池只有 8 个线程,"同时开始读几行"改前改后读数相同,差异只体现在堆上,mock 数不出来)。
2026-11-03 23:00:44 +08:00

192 lines
7.2 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/** 撞款控制台视图派生(分组索引 / 店铺列并集 / 矩阵格子预聚合):纯逻辑。 */
import type { DuplicateItem, DuplicateOccurrence, DuplicateShop } from './duplicate-model.ts'
export interface ShopGroups {
/** 分组名 → 成员店铺(展示名,保持首次出现顺序)。 */
members: Record<string, string[]>
/** 店铺展示名 → 所属分组名(含多分组拆分的每一组)。 */
shopGroupNames: Record<string, string[]>
}
const GROUP_SEPARATOR = /[、,,/;;]/
function shopKey(name: string): string {
return (name || '').trim().toLowerCase()
}
function splitTokens(value: string): string[] {
const out: string[] = []
for (const token of (value || '').split(GROUP_SEPARATOR)) {
const group = token.trim()
if (group && !out.includes(group)) out.push(group)
}
return out
}
/** 由店铺分布构建 分组→成员 与 店铺→分组 双向索引。 */
export function indexShopGroups(shops: DuplicateShop[]): ShopGroups {
const members: Record<string, string[]> = {}
const shopGroupNames: Record<string, string[]> = {}
for (const shop of shops || []) {
const display = (shop.shopName || '').trim()
if (!display) continue
const groups = splitTokens(shop.groupName || '')
for (const group of groups) {
const list = members[group] || (members[group] = [])
if (!list.includes(display)) list.push(display)
const owned = shopGroupNames[display] || (shopGroupNames[display] = [])
if (!owned.includes(group)) owned.push(group)
}
}
return { members, shopGroupNames }
}
/** 撞款行涉及店铺的并集列(出现顺序去重);memberShops 非空时仅取交集成员的子集顺序。 */
export function storeColumnsOf(items: DuplicateItem[], memberShops: string[] = []): string[] {
const restricted = new Set(memberShops.map((shop) => shopKey(shop)))
const useRestricted = restricted.size > 0
const order: string[] = []
const seen = new Set<string>()
const push = (name: string): void => {
const key = shopKey(name)
if (!name || seen.has(key) || (useRestricted && !restricted.has(key))) return
seen.add(key)
order.push(name)
}
for (const item of items || []) {
for (const occ of item.occurrences || []) push(occ.shopName || '')
}
if (order.length === 0 && memberShops.length > 0) {
// 无撞款行但该组有成员店铺:矩阵列仍需展示组员店。
for (const shop of memberShops) {
const key = shopKey(shop)
if (!seen.has(key)) {
seen.add(key)
order.push(shop)
}
}
}
return order
}
/** 矩阵一格:该店在售此 ASIN 的条数 + 站点(国家)分布。 */
export interface MatrixCell {
count: number
sites: { code: string; count: number }[]
}
/** 空格的单份实例:模板每格都要拿一个,不能每格 new 一个对象。 */
export const EMPTY_MATRIX_CELL: MatrixCell = { count: 0, sites: [] }
function cellKey(asin: string, shop: string): string {
// 用 \u0000 拼键:ASIN 与店铺名都可打印,任何可见分隔符都可能出现在店铺名里(那样就会串格)。
return `${(asin || '').trim()}\u0000${shopKey(shop)}`
}
/**
* 撞款矩阵的格子集合:把 rows×columns 一次算完(B60 的「预聚合」)。
*
* <p>原形状是每格现算:`cellCount`/`cellCountryDots`/`cellTitle` 各自把该行 occurrences 全 filter
* 一遍,一格 4~6 次、50 行 × 200 列就是每帧上万次整行扫描,勾选/翻页都要重付。这里每行只按
* 店铺分桶一次(O(occurrences)),之后一格一次 Map 取值。
*
* 只给 `columns` 里点名的列建格:不在列集内的 occurrence 直接不装桶,列窗口收窄时开销跟着降。
*/
export function buildMatrixCells(items: DuplicateItem[], columns: string[]): Map<string, MatrixCell> {
const cells = new Map<string, MatrixCell>()
const wanted = new Set<string>()
for (const column of columns || []) wanted.add(shopKey(column))
if (!wanted.size) return cells
for (const item of items || []) {
const buckets = new Map<string, DuplicateOccurrence[]>()
const siteCounts = new Map<string, Map<string, number>>()
for (const occ of item.occurrences || []) {
const key = shopKey(occ.shopName)
if (!wanted.has(key)) continue
const bucket = buckets.get(key)
if (bucket) bucket.push(occ)
else buckets.set(key, [occ])
const code = (occ.country || '').trim().toUpperCase()
if (!code) continue
let counts = siteCounts.get(key)
if (!counts) {
counts = new Map<string, number>()
siteCounts.set(key, counts)
}
counts.set(code, (counts.get(code) || 0) + 1)
}
for (const [key, occurrences] of buckets) {
const codes = siteCounts.get(key)
cells.set(cellKey(item.asin, key), {
count: occurrences.length,
sites: codes ? [...codes].map(([code, count]) => ({ code, count })) : [],
})
}
}
return cells
}
/** 取一格;没有格子时返回同一份空对象(模板里 `cellOf(...).count` 要能直接读)。 */
export function matrixCellOf(cells: Map<string, MatrixCell>, asin: string, shop: string): MatrixCell {
return cells.get(cellKey(asin, shop)) || EMPTY_MATRIX_CELL
}
/** 一个撞款 ASIN 的品牌:取第一条带品牌的上架记录,全空就是"未知品牌"(页面自己决定怎么显示)。 */
export function itemBrandOf(item: DuplicateItem): string {
for (const occ of item.occurrences || []) if (occ.brand) return occ.brand
return ''
}
/** 明细抽屉聚合行:店铺×站点聚合(对齐 reference 抽屉的 店铺/分组/站点/上架时间(升序堆叠)/次数)。 */
export interface DrawerAggRow {
shopName: string
groupName: string
country: string
times: string[]
count: number
}
export function aggregateDrawerRows(occurrences: DuplicateOccurrence[]): DrawerAggRow[] {
const rows = new Map<string, DrawerAggRow>()
for (const occ of occurrences || []) {
const key = `${shopKey(occ.shopName)}|${(occ.country || '').trim().toUpperCase()}`
let row = rows.get(key)
if (!row) {
row = { shopName: occ.shopName || '', groupName: occ.groupName || '', country: (occ.country || '').trim(), times: [], count: 0 }
rows.set(key, row)
}
row.times.push(occ.date || '')
row.count += 1
}
const list = [...rows.values()]
for (const row of list) row.times.sort()
return list
}
/** 明细抽屉汇总徽章:N店在售/N家店铺/国家chips/N次上架/最早上架。 */
export interface DrawerSummary {
shopCount: number
countries: string[]
totalRecords: number
earliest: string
}
export function drawerSummaryOf(occurrences: DuplicateOccurrence[]): DrawerSummary {
const shops = new Set<string>()
const countries: string[] = []
const seenCountry = new Set<string>()
let earliest = ''
let totalRecords = 0
for (const occ of occurrences || []) {
if (occ.shopName) shops.add(shopKey(occ.shopName))
const country = (occ.country || '').trim()
if (country && !seenCountry.has(country)) {
seenCountry.add(country)
countries.push(country)
}
if (occ.date && (!earliest || occ.date < earliest)) earliest = occ.date
totalRecords += 1
}
return { shopCount: shops.size, countries, totalRecords, earliest }
}