Files
crawler-plugin/admin-frontend-vue/tests/duplicate-console.test.ts
T
huangzd1997 c8cd75cf6c feat(phase-5): B60 console 分页与矩阵预聚合+P1-B5/6/7/8 性能热点收口
接管另一路会话的 Phase 5 在飞件收口(5A/5C 主体由那一路落码,本笔补齐 P1-B7 与两处台账并整量验证)。

- 5A/B60:撞款 console 的行窗口在服务端真分页(响应 dup 行数 ≤ pageSize,total_dup 仍是未分页
  总数)、dupGroup 判据收归服务端单一出处(前端 dupWithinGroup 删掉,两套同名规则不再分叉)、
  排序先于切窗;后管矩阵改预聚合+列窗 40。读数(200 列 × 50 行):每行 3 店 8.4 → 5.3 → 0.7ms
  (12.4×),每行 40 店 83.7 → 18.4 → 1.8ms(45.3×)。服务端不设列窗是判断不是漏做:shops 每行
  只有 5 个标量,而矩阵的列来自已分页的行——给 shops 再加分页会把"全局指标读全集"和"列窗读本页"
  两套语义搅在一起。浏览器首屏/交互耗时本机拿不到,登记为 Phase 6 现网复量。
- P1-B6:claim 循环的父任务终态判定折成一次批量 IN(TaskFileJobBatchQueries+Mapper XML,
  铁律 1 收口在 Mapper),逐行 CAS 认领一字不动。
- P1-B5:V187 只建 idx_biz_file_task_resume_of;审计给的候选集索引**有意不建**(V184 前缀已覆盖,
  OR/IS-NULL 与 COALESCE 不可索引),判据把这条负判定钉住。真库 EXPLAIN 本机无 MySQL,顺延现网。
- P1-B8:结果载荷的对象存储删除移到 afterCommit,事务里不再做 120s 超时的网络调用。
- P1-B7:结果集读取按窗提交+收割(窗口=线程池队列容量的一半)。审计把它记成"无 limit",但截断
  是正确性错误不是性能取舍,真能收的是"整份结果集被读两遍"——一次性提交时每支 Future 把解好的
  载荷扣在自己身上直到全部收割完,峰值=结果集+另一份整副本;分窗后副本只驻留一窗。流式组装
  契约要改 4 个模块的调用方,本 Phase 未做,别读成 P1-B7 已闭环。
- SourceScanTest 台账随之登记:裸名 25 → 26、调用方 18 → 19(P1-B7 的窗口判据只能打在源码上——
  线程池只有 8 个线程,"同时开始读几行"改前改后读数相同,差异只体现在堆上,mock 数不出来)。
2026-11-03 23:00:44 +08:00

131 lines
6.3 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import test from 'node:test'
import assert from 'node:assert/strict'
import { parseDuplicateConsole, parseLedgerPage, parseConsoleGroup, parseLedgerRow } from '../src/pages/tasks/duplicate-console-model.ts'
import {
buildMatrixCells,
indexShopGroups,
matrixCellOf,
storeColumnsOf,
} from '../src/pages/tasks/duplicate-console-logic.ts'
function occurrence(asin: string, shop: string, group: string, country: string, date: string) {
return { asin, date, price: '', brand: 'B', shop_name: shop, group_name: group, country_codes: [country], country }
}
/** 已解析模型形态(camel)的 occurrence,供纯逻辑派生入参。 */
function camelOcc(asin: string, shop: string, group: string, country: string, date: string) {
return { asin, date, price: '', brand: 'B', shopName: shop, groupName: group, countryCodes: [country], country }
}
test('test_dup_console_model_parses_overview_dup_and_groups', () => {
const consoleData = parseDuplicateConsole({
success: true,
data: {
pending: false,
scanned_at: '2026-09-04 03:10:00',
summary: { shop_count: 4, asin_total: 6, record_total: 10, duplicate_asin_total: 3, duplicate_shop_count: 4, site_count: 3, asin_per_shop: 2.5, source: 'job' },
shops: [
{ shop_name: 'ShopA', group_name: 'GroupA', country_codes: ['UK', 'DE'], asin_count: 3, record_count: 3 },
{ shop_name: 'ShopB', group_name: 'GroupA', country_codes: ['UK'], asin_count: 3, record_count: 3 },
{ shop_name: 'ShopC', group_name: 'GroupC', country_codes: ['FR'], asin_count: 2, record_count: 2 },
],
dup: [
{ asin: 'E0000001', shop_count: 3, record_count: 3, occurrences: [
occurrence('E0000001', 'ShopA', 'GroupA', 'UK', '2026-08-29'),
occurrence('E0000001', 'ShopB', 'GroupA', 'UK', '2026-08-30'),
occurrence('E0000001', 'ShopC', 'GroupC', 'FR', '2026-08-31'),
] },
],
groups: [{ name: 'GroupA', shop_count: 2, asin_unique: 4, record_count: 6, dup_count: 2 }],
total_dup: 1,
},
})
assert.equal(consoleData.overview.pending, false)
assert.equal(consoleData.overview.summary?.asinTotal, 6)
assert.equal(consoleData.overview.shops.length, 3)
assert.equal(consoleData.totalDup, 1)
assert.equal(consoleData.dup[0].asin, 'E0000001')
assert.equal(consoleData.dup[0].occurrences.length, 3)
const group = consoleData.groups[0]
assert.deepEqual(group, { name: 'GroupA', shopCount: 2, asinUnique: 4, recordCount: 6, dupCount: 2 })
})
/** B60:console 多了行窗口三键;缺字段要退得回安全默认(page 至少 1,pageSize 缺省 0=调用方自持)。 */
test('test_dup_console_model_parses_row_window', () => {
const windowed = parseDuplicateConsole({
success: true,
data: { pending: false, dup: [], groups: [], total_dup: 2650, page: 3, page_size: 50 },
})
assert.equal(windowed.totalDup, 2650, 'total_dup 是该范围未分页总数,不是本页行数')
assert.equal(windowed.page, 3)
assert.equal(windowed.pageSize, 50)
const legacy = parseDuplicateConsole({ success: true, data: { pending: false, dup: [], groups: [] } })
assert.equal(legacy.page, 1, '没有 page 字段时读成第一页,别让分页控件显示第 0 页')
assert.equal(legacy.pageSize, 0)
})
test('test_dup_console_parse_group_ignores_missing_name', () => {
assert.equal(parseConsoleGroup({ shop_count: 1 }), null)
assert.ok(parseConsoleGroup({ name: 'G', shop_count: 2 }))
})
test('test_dup_console_group_derivation_restricts_to_members', () => {
const dupA = { asin: 'A', shopCount: 2, recordCount: 2, occurrences: [
camelOcc('A', 'ShopA', 'GroupA', 'UK', '2026-08-01'),
camelOcc('A', 'ShopB', 'GroupA', 'UK', '2026-08-02'),
] }
const dupE = { asin: 'E', shopCount: 3, recordCount: 3, occurrences: [
camelOcc('E', 'ShopA', 'GroupA', 'UK', '2026-08-01'),
camelOcc('E', 'ShopB', 'GroupA', 'UK', '2026-08-02'),
camelOcc('E', 'ShopC', 'GroupC', 'FR', '2026-08-03'),
] }
const groups = indexShopGroups([
{ shopName: 'ShopA', groupName: 'GroupA', countryCodes: [], asinCount: 0, recordCount: 0 },
{ shopName: 'ShopB', groupName: 'GroupA', countryCodes: [], asinCount: 0, recordCount: 0 },
{ shopName: 'ShopC', groupName: 'GroupC', countryCodes: [], asinCount: 0, recordCount: 0 },
])
assert.deepEqual(groups.members.GroupA, ['ShopA', 'ShopB'])
assert.deepEqual(groups.shopGroupNames.ShopC, ['GroupC'])
assert.deepEqual(storeColumnsOf([dupA, dupE], groups.members.GroupA), ['ShopA', 'ShopB'])
// 「组内 ≥2 店才算撞款」的裁行只剩服务端一处正本(console 的 dup_group)——
// 页面原先在客户端用 dupWithinGroup 再裁一遍,那是同一规则的第二个正本,B60 收掉它。
})
test('test_dup_console_store_columns_order_and_restriction', () => {
const dupA = { asin: 'A', shopCount: 2, recordCount: 2, occurrences: [
camelOcc('A', 'ShopB', 'GroupA', 'UK', '2026-08-01'),
camelOcc('A', 'ShopA', 'GroupA', 'UK', '2026-08-02'),
] }
assert.deepEqual(storeColumnsOf([dupA]), ['ShopB', 'ShopA'])
assert.deepEqual(storeColumnsOf([dupA], ['ShopA', 'ShopB', 'ShopC']), ['ShopB', 'ShopA'])
// 空行 + 显式成员列:仍输出成员店铺。
assert.deepEqual(storeColumnsOf([], ['ShopA', 'ShopB']), ['ShopA', 'ShopB'])
})
test('test_dup_console_ledger_parse_rows_and_paging', () => {
const page = parseLedgerPage({
success: true,
data: {
pending: false,
scanned_at: '2026-09-04 03:10:00',
items: [
{ asin: 'E0000001', brand: 'BrandE', store_count: 3, stores: ['ShopA', 'ShopB', 'ShopC'], groups: ['GroupA', 'GroupC'], countries: ['UK', 'FR'], record_count: 3, earliest: '2026-08-29 00:00:00', latest: '2026-08-31 00:00:00', occurrences: [occurrence('E0000001', 'ShopA', 'GroupA', 'UK', '2026-08-29')] },
],
total: 1,
page: 1,
page_size: 200,
},
})
assert.equal(page.pending, false)
assert.equal(page.total, 1)
assert.equal(page.pageSize, 200)
const row = page.items[0]
assert.equal(row.asin, 'E0000001')
assert.equal(row.storeCount, 3)
assert.equal(row.occurrences.length, 1)
const single = parseLedgerRow({ asin: 'X0000001', store_count: 0, stores: [], groups: [], countries: [], record_count: 0, earliest: '', latest: '', occurrences: [] })
assert.equal(single?.storeCount, 0)
assert.equal(parseLedgerRow({}), null)
})