Files
crawler-plugin/frontend-vue/tests/merge-history-items.test.ts
T
huangzd1997 5caca86ad1 task-84: 优化店铺抓取队列状态合并,消除 historyItems 的线性重复查找
新增纯 TS 合并模块 merge-history-items:以 taskId→行列表 二级索引替代
原 mergeProgress 的 [...map.values()].find() 全表扫描(O(n×m)→O(n+m))。
命中语义与原实现一致(taskId 相同且 resultId 相同或 incoming 无
resultId),命中后删除旧 key 写入新 key,shopName 变化不再留下幽灵
重复条目,同批内后到条目可命中先到新增。BrandShopDataCrawlTab 的
mergeProgress 改用该模块。9 个测试覆盖默认、批量、幂等、空、单元素、
maxItems 限流、非法输入、keyOf 故障零变更与幽灵条目场景。
2026-08-30 22:47:41 +08:00

201 lines
7.7 KiB
TypeScript

import { test } from 'node:test'
import assert from 'node:assert/strict'
import { mergeHistoryItems } from '../src/shared/merge-history-items.ts'
interface Item {
taskId?: number
resultId?: number
shopName: string
taskStatus?: string
error?: string
}
const keyOf = (item: Item) => `${item.taskId || 0}:${item.resultId || 0}:${item.shopName}`
const item = (over: Partial<Item>): Item => ({ shopName: 'a', ...over })
test('test_task_084_merge_normal_default_path', () => {
const existing = [item({ taskId: 1, resultId: 1, taskStatus: 'RUNNING' })]
const incoming = [item({ taskId: 1, resultId: 1, taskStatus: 'SUCCESS' })]
const { items, updatedCount, addedCount } = mergeHistoryItems(existing, incoming, { keyOf })
assert.equal(items.length, 1)
assert.equal(items[0].taskStatus, 'SUCCESS')
assert.deepEqual(items[0], { taskId: 1, resultId: 1, shopName: 'a', taskStatus: 'SUCCESS' })
assert.equal(updatedCount, 1)
assert.equal(addedCount, 0)
assert.equal(existing.length, 1, '入参列表不得被修改')
})
test('test_task_084_merge_normal_multiple_items', () => {
const existing = Array.from({ length: 5 }, (_, i) =>
item({ taskId: 1, resultId: i + 1, shopName: `s${i + 1}`, taskStatus: 'RUNNING' }),
)
const incoming = Array.from({ length: 5 }, (_, i) =>
item({ taskId: 2, resultId: i + 1, shopName: `t${i + 1}`, taskStatus: 'SUCCESS' }),
)
const { items, addedCount, updatedCount } = mergeHistoryItems(existing, incoming, { keyOf })
assert.equal(items.length, 10)
assert.equal(addedCount, 5)
assert.equal(updatedCount, 0)
// 已有条目在前,新条目追加在后,顺序稳定
assert.deepEqual(
items.map((r) => r.shopName),
['s1', 's2', 's3', 's4', 's5', 't1', 't2', 't3', 't4', 't5'],
)
// 批量更新多条:命中后替换原位置,不改变顺序
const update = Array.from({ length: 5 }, (_, i) =>
item({ taskId: 1, resultId: i + 1, shopName: `s${i + 1}`, taskStatus: 'SUCCESS' }),
)
const again = mergeHistoryItems(items, update, { keyOf })
assert.equal(again.items.length, 10)
assert.equal(again.updatedCount, 5)
assert.equal(again.addedCount, 0)
assert.deepEqual(
again.items.map((r) => r.taskStatus),
['SUCCESS', 'SUCCESS', 'SUCCESS', 'SUCCESS', 'SUCCESS', 'SUCCESS', 'SUCCESS', 'SUCCESS', 'SUCCESS', 'SUCCESS'],
)
assert.deepEqual(
again.items.slice(0, 5).map((r) => r.shopName),
['s1', 's2', 's3', 's4', 's5'],
)
})
test('test_task_084_merge_normal_repeated_operation_is_idempotent', () => {
const existing = [item({ taskId: 1, resultId: 1, taskStatus: 'RUNNING' })]
const incoming = [item({ taskId: 1, resultId: 1, taskStatus: 'SUCCESS' })]
const once = mergeHistoryItems(existing, incoming, { keyOf })
const twice = mergeHistoryItems(once.items, incoming, { keyOf })
const thrice = mergeHistoryItems(twice.items, incoming, { keyOf })
for (const result of [once, twice, thrice]) {
assert.equal(result.items.length, 1)
assert.equal(result.items[0].taskStatus, 'SUCCESS')
}
assert.equal(twice.updatedCount, 1)
assert.equal(twice.addedCount, 0)
assert.equal(thrice.updatedCount, 1)
assert.equal(thrice.addedCount, 0)
})
test('test_task_084_merge_boundary_empty_input', () => {
const { items, updatedCount, addedCount } = mergeHistoryItems([], [], { keyOf })
assert.deepEqual(items, [])
assert.equal(updatedCount, 0)
assert.equal(addedCount, 0)
const existing = [item({ taskId: 1, resultId: 1 })]
const noop = mergeHistoryItems(existing, [], { keyOf })
assert.deepEqual(noop.items, existing)
assert.equal(noop.updatedCount, 0)
assert.equal(noop.addedCount, 0)
})
test('test_task_084_merge_boundary_single_item', () => {
const { items, addedCount, updatedCount } = mergeHistoryItems(
[],
[item({ taskId: 7, resultId: 9, taskStatus: 'RUNNING' })],
{ keyOf },
)
assert.equal(items.length, 1)
assert.equal(items[0].taskId, 7)
assert.equal(items[0].resultId, 9)
assert.equal(addedCount, 1)
assert.equal(updatedCount, 0)
})
test('test_task_084_merge_boundary_limit_and_overflow', () => {
const incoming = Array.from({ length: 5 }, (_, i) => item({ taskId: 1, resultId: i + 1 }))
const { items } = mergeHistoryItems([], incoming, { keyOf, maxItems: 3 })
assert.equal(items.length, 3)
assert.deepEqual(
items.map((r) => r.resultId),
[3, 4, 5],
)
// 合并已满列表:仍受 maxItems 约束
const full = mergeHistoryItems(items, [item({ taskId: 2, resultId: 1 })], { keyOf, maxItems: 3 })
assert.equal(full.items.length, 3)
assert.equal(full.items[2].taskId, 2)
})
test('test_task_084_merge_invalid_input_rejected', () => {
assert.throws(
() => mergeHistoryItems([], null as never, { keyOf }),
/incoming 必须是数组/,
)
assert.throws(
() => mergeHistoryItems(null as never, [], { keyOf }),
/existing 必须是数组/,
)
assert.throws(
() => mergeHistoryItems([], [], null as never),
/keyOf 必须是函数/,
)
assert.throws(
() => mergeHistoryItems([], [], { keyOf, maxItems: 0 }),
/maxItems 必须为正数/,
)
assert.throws(
() => mergeHistoryItems([], [], { keyOf, maxItems: -1 }),
/maxItems 必须为正数/,
)
// 非法 taskId/resultId 条目:不匹配任何现有行,按新条目追加,不抛错
const { items, addedCount } = mergeHistoryItems(
[item({ taskId: 1, resultId: 1 })],
[item({ taskId: -1, resultId: 1, shopName: 'bad' }), item({ taskId: 0 })],
{ keyOf },
)
assert.equal(items.length, 3)
assert.equal(addedCount, 2)
})
test('test_task_084_merge_dependency_failure_releases_resources', () => {
const existing = [item({ taskId: 1, resultId: 1 })]
const incoming = [item({ taskId: 1, resultId: 1, taskStatus: 'SUCCESS' })]
let broken = true
const faultyKeyOf = (row: Item) => {
if (broken) throw new Error('key down')
return keyOf(row)
}
assert.throws(() => mergeHistoryItems(existing, incoming, { keyOf: faultyKeyOf }), /key down/)
// 失败路径零状态变更:入参未被修改
assert.deepEqual(existing, [item({ taskId: 1, resultId: 1 })])
// 错误可恢复:修复后同一组输入成功
broken = false
const { items, updatedCount } = mergeHistoryItems(existing, incoming, { keyOf: faultyKeyOf })
assert.equal(items.length, 1)
assert.equal(items[0].taskStatus, 'SUCCESS')
assert.equal(updatedCount, 1)
})
test('test_task_084_merge_dependency_failure_removes_ghost_duplicates', () => {
// 线性 find 语义下的幽灵重复行场景:同 taskId/resultId 但不同 shopName 的
// incoming 命中现有行后,旧 key 不得留下重复条目,且再次合并可精确命中
const existing = [item({ taskId: 1, resultId: 1, shopName: 'a', taskStatus: 'RUNNING' })]
const incoming = [item({ taskId: 1, resultId: 1, shopName: 'b', taskStatus: 'SUCCESS' })]
const once = mergeHistoryItems(existing, incoming, { keyOf })
assert.equal(once.items.length, 1)
assert.deepEqual(once.items[0], {
taskId: 1,
resultId: 1,
shopName: 'b',
taskStatus: 'SUCCESS',
})
assert.equal(once.updatedCount, 1)
const twice = mergeHistoryItems(once.items, incoming, { keyOf })
assert.equal(twice.items.length, 1)
assert.equal(twice.updatedCount, 1)
assert.equal(twice.addedCount, 0)
// 同一批次内后到条目命中先到新增条目:不残留重复
const batch = [
item({ taskId: 1, resultId: 1, shopName: 'x', taskStatus: 'RUNNING' }),
item({ taskId: 1, resultId: 1, shopName: 'y', taskStatus: 'SUCCESS' }),
]
const chained = mergeHistoryItems([], batch, { keyOf })
assert.equal(chained.items.length, 1)
assert.deepEqual(chained.items[0], {
taskId: 1,
resultId: 1,
shopName: 'y',
taskStatus: 'SUCCESS',
})
assert.equal(chained.updatedCount, 1)
assert.equal(chained.addedCount, 1)
})