Files
crawler-plugin/frontend-vue/tests/dispatch-guard.test.ts
T
huangzd1997 759f0b15d8
Build Backend JAR / build (push) Has been cancelled
task-101: 上传/解析/入队三道前端拦截,阻止坏数据卡死 Python 自动化
新增 shared/dispatch-guard.ts(纯 TS 校验,可单测)与 dispatch-guard-ui.ts
(ElMessageBox 弹窗层),在数据流的三个位置设闸:

- 选文件:空选择、非 xlsx/xls/csv、空路径直接拦;重复文件弹确认
- 解析结果:taskId 非法、totalRows 为 0、整批行被丢弃、需要分组却无分组直接拦;
  部分行被丢弃弹确认后才允许推送
- 入队前:结构不符、必填字段缺失、items/groups/country_codes 为空数组、
  行数页数为 0、JSON 不安全值(NaN/Infinity/BigInt/循环引用/数组洞)

覆盖 11 个推 Python 队列的 Tab 与 dedupe/split/convert 三个纯 Java Tab。
闸位按 Tab 挑选以免留下孤儿后端状态:collect-data 校验前置到
activateCollectDataTask 之前(任务保持 PENDING 无需回滚)、product-risk 的国家
检查提到建任务之前;已建任务后才拦下的复用各 Tab 原有失败补偿路径。

两处边界:对象属性 undefined 放行(taskNo 这类可选字段是惯用写法),只拦数组
元素 undefined 与数组洞;price-track 的 asin_rows_by_country 仅在 mode=asin
时要求非空,status 模式下 loadAsinRowsForAppClient 本就返回 {}。

顺带修复 similar-asin / appearance-patent 的 selectFiles 缺少 try/catch,
上传失败会残留上一批文件与解析结果。

测试:tests/dispatch-guard.test.ts 新增 37 个 test_task_101_* 用例。
2026-09-01 12:53:12 +08:00

443 lines
17 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { test } from 'node:test'
import assert from 'node:assert/strict'
import {
EXCEL_CSV_EXTENSIONS,
EXCEL_EXTENSIONS,
basenameOf,
checkParseResult,
checkQueuePayload,
checkSelectedFiles,
extensionOf,
findUnsafeJsonPaths,
guardPassed,
type GuardResult,
} from '../src/shared/dispatch-guard.ts'
function codes(result: GuardResult) {
return result.issues.map((issue) => issue.code)
}
// ---------------------------------------------------------------- 路径工具
test('test_task_101_dispatch_guard_normal_path_helpers', () => {
assert.equal(basenameOf('D:\\brand\\鲍丽明.xlsx'), '鲍丽明.xlsx')
assert.equal(basenameOf('/home/u/a/b.csv'), 'b.csv')
assert.equal(basenameOf('plain.xlsx'), 'plain.xlsx')
assert.equal(basenameOf('D:\\brand\\'), 'brand', '结尾斜杠不应产生空名')
assert.equal(extensionOf('D:\\brand\\鲍丽明.XLSX'), '.xlsx', '扩展名统一小写')
assert.equal(extensionOf('archive.tar.gz'), '.gz')
assert.equal(extensionOf('noext'), '')
assert.equal(extensionOf('.gitignore'), '', '隐藏文件不算扩展名')
assert.equal(extensionOf('trailingdot.'), '')
})
// ------------------------------------------------------------ 文件选择校验
test('test_task_101_dispatch_guard_normal_selected_files_pass', () => {
const result = checkSelectedFiles(
['D:\\brand\\a.xlsx', 'D:\\brand\\b.XLS'],
{ allowedExtensions: EXCEL_EXTENSIONS },
)
assert.equal(result.ok, true)
assert.equal(result.needsConfirm, false)
assert.equal(result.message, '')
assert.deepEqual(codes(result), [])
})
test('test_task_101_dispatch_guard_normal_selected_files_folder_items', () => {
// expandBrandFolderRecursive 返回的是 {absolutePath, relativePath} 结构
const result = checkSelectedFiles(
[
{ absolutePath: 'D:\\brand\\x\\a.xlsx', relativePath: 'x/a.xlsx' },
{ relativePath: 'x/b.csv' },
],
{ allowedExtensions: EXCEL_CSV_EXTENSIONS },
)
assert.equal(result.ok, true)
})
test('test_task_101_dispatch_guard_boundary_selected_files_single', () => {
const result = checkSelectedFiles(['a.xlsx'], { allowedExtensions: EXCEL_EXTENSIONS, maxFiles: 1 })
assert.equal(result.ok, true)
const over = checkSelectedFiles(['a.xlsx', 'b.xlsx'], {
allowedExtensions: EXCEL_EXTENSIONS,
maxFiles: 1,
})
assert.equal(over.ok, false)
assert.ok(codes(over).includes('files.too-many'))
assert.match(over.message, /一次最多选择 1 个文件/)
})
test('test_task_101_dispatch_guard_boundary_selected_files_empty', () => {
const empty = checkSelectedFiles([])
assert.equal(empty.ok, false)
assert.deepEqual(codes(empty), ['files.empty'])
assert.match(empty.message, /没有选择任何文件/)
})
test('test_task_101_dispatch_guard_invalid_selected_files_bad_extension', () => {
const result = checkSelectedFiles(
['D:\\brand\\good.xlsx', 'D:\\brand\\报表.txt', 'D:\\brand\\note.pdf'],
{ allowedExtensions: EXCEL_EXTENSIONS },
)
assert.equal(result.ok, false)
assert.deepEqual(codes(result), ['files.bad-extension'])
assert.match(result.message, /报表\.txt/)
assert.match(result.message, /note\.pdf/)
assert.ok(!result.message.includes('good.xlsx'), '合规文件不应出现在错误里')
})
test('test_task_101_dispatch_guard_invalid_selected_files_not_array', () => {
for (const bad of [null, undefined, 'a.xlsx', 42, {}]) {
const result = checkSelectedFiles(bad)
assert.equal(result.ok, false, `${String(bad)} 应被拦截`)
assert.deepEqual(codes(result), ['files.not-array'])
}
})
test('test_task_101_dispatch_guard_invalid_selected_files_blank_path', () => {
const result = checkSelectedFiles(['a.xlsx', '', ' ', { absolutePath: '' }])
assert.equal(result.ok, false)
assert.ok(codes(result).includes('files.blank-path'))
assert.match(result.message, /有 3 个文件路径为空/)
})
test('test_task_101_dispatch_guard_confirm_selected_files_duplicated', () => {
const result = checkSelectedFiles(['D:\\a.xlsx', 'd:\\A.XLSX'], {
allowedExtensions: EXCEL_EXTENSIONS,
})
assert.equal(result.ok, true, '重复不阻塞')
assert.equal(result.needsConfirm, true)
assert.deepEqual(codes(result), ['files.duplicated'])
assert.match(result.message, /重复选择/)
})
test('test_task_101_dispatch_guard_selected_files_block_wins_over_confirm', () => {
// 同时命中重复(confirm)与非法扩展名(block)时,只展示 block 文案
const result = checkSelectedFiles(['a.txt', 'a.txt'], { allowedExtensions: EXCEL_EXTENSIONS })
assert.equal(result.ok, false)
assert.equal(result.needsConfirm, false)
assert.match(result.message, /不支持/)
assert.ok(!result.message.includes('重复选择'))
})
// ------------------------------------------------------------ 解析结果校验
const parseVo = (over: Record<string, unknown> = {}) => ({
taskId: 12,
totalRows: 100,
acceptedRows: 100,
droppedRows: 0,
groupCount: 8,
...over,
})
test('test_task_101_dispatch_guard_normal_parse_result_pass', () => {
const result = checkParseResult(parseVo())
assert.equal(result.ok, true)
assert.equal(result.needsConfirm, false)
assert.deepEqual(codes(result), [])
})
test('test_task_101_dispatch_guard_normal_parse_result_missing_optional_counts', () => {
// collect-data 的 Vo 里 totalRows/droppedRows/groupCount 都是可选的
const result = checkParseResult({ taskId: 3, acceptedRows: 20 })
assert.equal(result.ok, true)
assert.equal(result.needsConfirm, false)
})
test('test_task_101_dispatch_guard_boundary_parse_result_single_row', () => {
const result = checkParseResult(parseVo({ totalRows: 1, acceptedRows: 1, droppedRows: 0 }))
assert.equal(result.ok, true)
})
test('test_task_101_dispatch_guard_boundary_parse_result_zero_rows_blocked', () => {
// 这是最危险的一种:任务建好了但没有明细,推给 Python 会一直停在 RUNNING
const allDropped = checkParseResult(parseVo({ totalRows: 500, acceptedRows: 0, droppedRows: 500 }))
assert.equal(allDropped.ok, false)
assert.deepEqual(codes(allDropped), ['parse.all-dropped'])
assert.match(allDropped.message, /共读取 500 行/)
assert.match(allDropped.message, /卡在执行中/)
const emptyFile = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0, droppedRows: 0 }))
assert.equal(emptyFile.ok, false)
assert.deepEqual(codes(emptyFile), ['parse.empty-file'])
assert.match(emptyFile.message, /没有读到任何数据行/)
})
test('test_task_101_dispatch_guard_parse_result_zero_rows_allowed_when_not_required', () => {
const result = checkParseResult(parseVo({ totalRows: 0, acceptedRows: 0 }), { requireRows: false })
assert.equal(result.ok, true)
})
test('test_task_101_dispatch_guard_confirm_parse_result_partial_dropped', () => {
const result = checkParseResult(parseVo({ totalRows: 100, acceptedRows: 88, droppedRows: 12 }), {
requiredColumnsHint: 'ASIN / 国家',
})
assert.equal(result.ok, true, '还有 88 行可跑,不该硬拦')
assert.equal(result.needsConfirm, true)
assert.deepEqual(codes(result), ['parse.partial-dropped'])
assert.match(result.message, /12 行/)
assert.match(result.message, /只有 88 行会被执行/)
assert.match(result.message, /ASIN \/ 国家/)
assert.equal(result.title, '解析结果需要确认')
})
test('test_task_101_dispatch_guard_parse_result_require_groups', () => {
const noGroup = checkParseResult(parseVo({ groupCount: 0 }), { requireGroups: true })
assert.equal(noGroup.ok, false)
assert.deepEqual(codes(noGroup), ['parse.no-group'])
const ignored = checkParseResult(parseVo({ groupCount: 0 }))
assert.equal(ignored.ok, true, '没要求分组时 groupCount=0 不拦')
})
test('test_task_101_dispatch_guard_invalid_parse_result_bad_task_id', () => {
for (const taskId of [0, -1, null, undefined, NaN, '12', 1.5, Infinity]) {
const result = checkParseResult(parseVo({ taskId }))
assert.equal(result.ok, false, `taskId=${String(taskId)} 应被拦截`)
assert.deepEqual(codes(result), ['parse.bad-task-id'])
}
})
test('test_task_101_dispatch_guard_invalid_parse_result_not_object', () => {
for (const bad of [null, undefined, 'ok', 7, []]) {
const result = checkParseResult(bad)
assert.equal(result.ok, false, `${String(bad)} 应被拦截`)
assert.deepEqual(codes(result), ['parse.not-object'])
}
})
test('test_task_101_dispatch_guard_invalid_parse_result_bad_counts', () => {
const result = checkParseResult(parseVo({ acceptedRows: NaN, totalRows: -3 }))
assert.equal(result.ok, false)
assert.deepEqual(codes(result), ['parse.bad-count', 'parse.bad-count'])
assert.match(result.message, /totalRows/)
assert.match(result.message, /acceptedRows/)
})
test('test_task_101_dispatch_guard_parse_result_does_not_mutate_input', () => {
const input = parseVo({ totalRows: 10, acceptedRows: 6, droppedRows: 4 })
const snapshot = JSON.parse(JSON.stringify(input))
checkParseResult(input, { requiredColumnsHint: 'ASIN' })
assert.deepEqual(input, snapshot)
})
// ------------------------------------------------------------ 入队 payload
const runPayload = (over: Record<string, unknown> = {}) => ({
type: 'similar-asin-run',
ts: 1700000000000,
data: {
taskId: 12,
user_id: 3,
api_key: 'sk-xxx',
acceptedRows: 88,
...over,
},
})
test('test_task_101_dispatch_guard_normal_queue_payload_pass', () => {
const result = checkQueuePayload(runPayload(), {
expectedType: 'similar-asin-run',
requiredDataKeys: ['taskId', 'api_key'],
})
assert.equal(result.ok, true)
assert.deepEqual(codes(result), [])
})
test('test_task_101_dispatch_guard_normal_queue_payload_optional_undefined_is_fine', () => {
// payload 里到处是 `taskNo: vo?.taskNo`undefined 可选字段不能被误拦
const result = checkQueuePayload(runPayload({ taskNo: undefined, note: undefined }), {
requiredDataKeys: ['taskId'],
})
assert.equal(result.ok, true)
assert.deepEqual(codes(result), [])
})
test('test_task_101_dispatch_guard_boundary_queue_payload_empty_collections', () => {
const emptyArray = checkQueuePayload(runPayload({ items: [] }), { nonEmptyArrayKeys: ['items'] })
assert.equal(emptyArray.ok, false)
assert.deepEqual(codes(emptyArray), ['payload.empty-array'])
assert.match(emptyArray.message, /没得可做/)
const oneItem = checkQueuePayload(runPayload({ items: [{ asin: 'B001' }] }), {
nonEmptyArrayKeys: ['items'],
})
assert.equal(oneItem.ok, true, '1 条也算有内容')
const emptyMap = checkQueuePayload(runPayload({ asin_rows_by_country: {} }), {
nonEmptyObjectKeys: ['asin_rows_by_country'],
})
assert.equal(emptyMap.ok, false)
assert.deepEqual(codes(emptyMap), ['payload.empty-object'])
const filledMap = checkQueuePayload(runPayload({ asin_rows_by_country: { DE: [{ asin: 'B1' }] } }), {
nonEmptyObjectKeys: ['asin_rows_by_country'],
})
assert.equal(filledMap.ok, true)
})
test('test_task_101_dispatch_guard_boundary_queue_payload_positive_numbers', () => {
// 0 行 / 0 页不是 nullrequiredDataKeys 拦不住,但 Python 端会翻 0 页后空转
const zero = checkQueuePayload(runPayload({ totalRows: 0, totalPages: 0 }), {
positiveNumberKeys: ['totalRows', 'totalPages'],
})
assert.equal(zero.ok, false)
assert.deepEqual(codes(zero), ['payload.non-positive'])
assert.match(zero.message, /totalRows=0/)
assert.match(zero.message, /totalPages=0/)
const one = checkQueuePayload(runPayload({ totalRows: 1, totalPages: 1 }), {
positiveNumberKeys: ['totalRows', 'totalPages'],
})
assert.equal(one.ok, true, '1 行 1 页是合法下界')
const missing = checkQueuePayload(runPayload(), { positiveNumberKeys: ['totalRows'] })
assert.equal(missing.ok, false, '字段缺失同样算不可执行')
const negative = checkQueuePayload(runPayload({ totalRows: -5 }), {
positiveNumberKeys: ['totalRows'],
})
assert.equal(negative.ok, false)
})
test('test_task_101_dispatch_guard_invalid_queue_payload_missing_required', () => {
const result = checkQueuePayload(runPayload({ api_key: ' ', taskId: null }), {
requiredDataKeys: ['taskId', 'api_key', 'shop_name'],
})
assert.equal(result.ok, false)
assert.deepEqual(codes(result), ['payload.missing-field'])
assert.match(result.message, /taskId、api_key、shop_name/)
})
test('test_task_101_dispatch_guard_invalid_queue_payload_shape', () => {
const notObject = checkQueuePayload('{"type":"x"}')
assert.equal(notObject.ok, false)
assert.deepEqual(codes(notObject), ['payload.not-object'])
const noType = checkQueuePayload({ data: { taskId: 1 } })
assert.equal(noType.ok, false)
assert.ok(codes(noType).includes('payload.bad-type'))
const wrongType = checkQueuePayload(runPayload(), { expectedType: 'publish-run' })
assert.equal(wrongType.ok, false)
assert.ok(codes(wrongType).includes('payload.type-mismatch'))
const noData = checkQueuePayload({ type: 'x-run', data: null })
assert.equal(noData.ok, false)
assert.ok(codes(noData).includes('payload.bad-data'))
})
test('test_task_101_dispatch_guard_invalid_queue_payload_unsafe_json', () => {
// NaN 会被 JSON.stringify 静默改写成 nullPython 侧拿到 None 后空转
const nan = checkQueuePayload(runPayload({ minPrice: NaN }))
assert.equal(nan.ok, false)
assert.deepEqual(codes(nan), ['payload.unsafe-json'])
assert.match(nan.message, /payload\.data\.minPrice/)
assert.match(nan.message, /NaN 会被序列化成 null/)
const infinite = checkQueuePayload(runPayload({ rounds: Infinity }))
assert.equal(infinite.ok, false)
assert.match(infinite.message, /payload\.data\.rounds/)
})
test('test_task_101_dispatch_guard_invalid_queue_payload_circular_reference', () => {
const payload = runPayload() as Record<string, unknown>
const data = payload.data as Record<string, unknown>
data.self = data
const result = checkQueuePayload(payload)
assert.equal(result.ok, false)
assert.deepEqual(codes(result), ['payload.unsafe-json'])
assert.match(result.message, /循环引用/)
// 校验本身不能抛错,也不能改动输入
assert.equal(data.self, data)
})
// ------------------------------------------------- findUnsafeJsonPaths 细节
test('test_task_101_dispatch_guard_unsafe_json_normal_clean_object', () => {
assert.deepEqual(findUnsafeJsonPaths({ a: 1, b: 'x', c: true, d: null, e: [1, 2], f: {} }), [])
assert.deepEqual(findUnsafeJsonPaths({ when: new Date(0) }), [], '有效 Date 可安全序列化')
})
test('test_task_101_dispatch_guard_unsafe_json_boundary_undefined_placement', () => {
// 对象属性 undefined = 可选字段,放行;数组元素 undefined 会变 null,报告
assert.deepEqual(findUnsafeJsonPaths({ optional: undefined }), [])
const inArray = findUnsafeJsonPaths({ items: [1, undefined, 3] })
assert.equal(inArray.length, 1)
assert.equal(inArray[0].path, 'payload.items[1]')
const holes = findUnsafeJsonPaths({ items: [1, , 3] })
assert.equal(holes.length, 1)
assert.match(holes[0].reason, /空洞/)
})
test('test_task_101_dispatch_guard_unsafe_json_invalid_types', () => {
const found = findUnsafeJsonPaths({
fn: () => 1,
sym: Symbol('s'),
big: BigInt(9),
map: new Map([['a', 1]]),
set: new Set([1]),
badDate: new Date('nope'),
})
const byPath = Object.fromEntries(found.map((item) => [item.path, item.reason]))
assert.match(byPath['payload.fn'], /函数/)
assert.match(byPath['payload.sym'], /Symbol/)
assert.match(byPath['payload.big'], /BigInt/)
assert.match(byPath['payload.map'], /Map/)
assert.match(byPath['payload.set'], /Set/)
assert.match(byPath['payload.badDate'], /无效日期/)
})
test('test_task_101_dispatch_guard_unsafe_json_boundary_deep_nesting', () => {
let deep: Record<string, unknown> = { leaf: 1 }
for (let i = 0; i < 70; i += 1) deep = { next: deep }
const found = findUnsafeJsonPaths(deep)
assert.ok(found.length > 0)
assert.match(found[0].reason, /嵌套层级超过/)
})
test('test_task_101_dispatch_guard_unsafe_json_repeated_reference_is_not_circular', () => {
// 同一个对象被两个 key 引用(DAG)不是环,不该误报
const shared = { asin: 'B001' }
assert.deepEqual(findUnsafeJsonPaths({ a: shared, b: shared }), [])
assert.deepEqual(findUnsafeJsonPaths({ list: [shared, shared] }), [])
})
// ----------------------------------------------------------------- 其他
test('test_task_101_dispatch_guard_passed_helper_is_inert', () => {
const result = guardPassed()
assert.equal(result.ok, true)
assert.equal(result.needsConfirm, false)
assert.equal(result.message, '')
assert.deepEqual(result.issues, [])
})
test('test_task_101_dispatch_guard_repeated_calls_are_idempotent', () => {
const files = ['a.xlsx', 'b.xlsx']
assert.deepEqual(checkSelectedFiles(files), checkSelectedFiles(files))
const vo = parseVo({ droppedRows: 5, acceptedRows: 95 })
assert.deepEqual(checkParseResult(vo), checkParseResult(vo))
const payload = runPayload()
assert.deepEqual(checkQueuePayload(payload), checkQueuePayload(payload))
})
test('test_task_101_dispatch_guard_dependency_failure_does_not_throw', () => {
// Proxy getter 故障时校验不应把异常抛给调用方之外的路径;这里确认异常可被捕获
const poisoned = new Proxy(
{ taskId: 1, totalRows: 5, acceptedRows: 5 },
{
get(target, prop, receiver) {
if (prop === 'acceptedRows') throw new Error('getter down')
return Reflect.get(target, prop, receiver)
},
},
)
assert.throws(() => checkParseResult(poisoned), /getter down/)
// 修好之后同一入口可以继续用,说明模块没有内部状态残留
assert.equal(checkParseResult({ taskId: 1, totalRows: 5, acceptedRows: 5 }).ok, true)
})