diff --git a/admin-frontend-vue/src/pages/tasks/DuplicateCheckPage.vue b/admin-frontend-vue/src/pages/tasks/DuplicateCheckPage.vue index 9792834fe..4adcd42cf 100644 --- a/admin-frontend-vue/src/pages/tasks/DuplicateCheckPage.vue +++ b/admin-frontend-vue/src/pages/tasks/DuplicateCheckPage.vue @@ -9,7 +9,16 @@ import { useAdminSessionStore } from '@/stores/admin-session' import { useCompanyTabStore } from '@/stores/company-tab' import type { DuplicateItem, DuplicateOccurrence, DuplicateMetrics, DuplicateShop } from './duplicate-model.ts' import { fetchDuplicateConsole, fetchDuplicateLedger, requestDuplicateExportBlob, type LedgerSortKey } from './duplicate-console-api.ts' -import { aggregateDrawerRows, drawerSummaryOf, dupWithinGroup, indexShopGroups, type DrawerAggRow, type DrawerSummary, type ShopGroups } from './duplicate-console-logic.ts' +import { + aggregateDrawerRows, + drawerSummaryOf, + indexShopGroups, + itemBrandOf, + type DrawerAggRow, + type DrawerSummary, + type ShopGroups, +} from './duplicate-console-logic.ts' +import { useDuplicateMatrix } from './duplicate-matrix-window.ts' import type { DuplicateConsole, LedgerRow } from './duplicate-console-model.ts' import { ASIN_COUNTRY_CODES, asinCountryLabel } from '../asin/asin-country.ts' import { orderedPair } from '@/utils/datetime' @@ -73,7 +82,8 @@ function toast(msg: string) { const overview = computed(() => consoleData.value?.overview ?? null) const summary = computed(() => overview.value?.summary ?? null) const shops = computed(() => overview.value?.shops ?? []) -const dupItems = computed(() => consoleData.value?.dup ?? []) +/** 当前撞款行窗口:服务端按 page/page_size(+dup_group) 切好的一窗(B60),页面不再自切全集。 */ +const dupItems = computed(() => consoleData.value?.dup ?? []) const groupStats = computed(() => consoleData.value?.groups ?? []) const countryOptions = computed(() => { @@ -118,26 +128,32 @@ function selectedGroupMembers(): string[] { const groupChips = computed(() => groupStats.value.filter((group) => group.shopCount >= 2)) -/** 当前撞款行集:全局=撞款全集;分组=组内成员撞款子集;均按 ASIN 升序(reference 台账顺序)。 */ -const dupRows = computed(() => { - const rows = scope.value === 'global' ? dupItems.value : dupWithinGroup(dupItems.value, selectedGroupMembers()) - return [...rows].sort((a, b) => a.asin.localeCompare(b.asin)) -}) - -/** 矩阵列:分组范围=该组成员店;全店=全部可见店铺(按上架量降序)。 */ -const matrixShops = computed(() => { - if (scope.value === 'group') return selectedGroupMembers() - return orderedShops.value.map((shop) => shop.shopName) -}) - const activeGroupStat = computed(() => groupStats.value.find((group) => group.name === selectedGroup.value) || null) -/** 撞款 Tab 计数:全店范围取后端权威 total_dup(防后端截断计数偏小);分组范围取组内子集长度。 */ -const totalDupCount = computed(() => (scope.value === 'global' ? (consoleData.value?.totalDup ?? dupRows.value.length) : dupRows.value.length)) +/** 撞款行总数:服务端在本页 + 本范围(全局或所选分组)内未分页的计数,翻页控件与 KPI 共用。 */ +const totalDupCount = computed(() => consoleData.value?.totalDup ?? 0) -function itemBrand(item: DuplicateItem): string { - for (const occ of item.occurrences || []) if (occ.brand) return occ.brand - return '' -} +/** 撞款矩阵的行窗 + 列窗 + 格子视图(B60):状态与算法在 `duplicate-matrix-window.ts`,翻页动作交回这里。 */ +const { + matrixPage, + matrixPageSize, + columnPage, + columnWindowTotal, + matrixAllColumns, + matrixShops, + matrixView, + matrixHint, + goMatrixPage, + changeMatrixSize, + goColumnPage, + resetMatrixWindows, +} = useDuplicateMatrix({ + rows: dupItems, + groupMembers: computed(() => selectedGroupMembers()), + groupScoped: computed(() => scope.value === 'group'), + selectedGroup, + reload: () => void loadConsole(), + dataVersion: consoleData, +}) // ---- 撞款视图分页(分布/矩阵/撞款详情/抽屉明细;10/20/50/100 可选) ---- @@ -147,16 +163,7 @@ const distPageSize = ref(10) const pagedDistBars = computed(() => storeDistribution.value.list.slice((distPage.value - 1) * distPageSize.value, distPage.value * distPageSize.value), ) -const matrixPage = ref(1) -const matrixPageSize = ref(10) -const matrixRows = computed(() => - dupRows.value.slice((matrixPage.value - 1) * matrixPageSize.value, matrixPage.value * matrixPageSize.value), -) -const dupCardPage = ref(1) -const dupCardPageSize = ref(10) -const dupCardRows = computed(() => - dupRows.value.slice((dupCardPage.value - 1) * dupCardPageSize.value, dupCardPage.value * dupCardPageSize.value), -) + const drawerPage = ref(1) const drawerPageSize = ref(10) const drawerRows = computed(() => @@ -168,46 +175,15 @@ const groupStatsPageSize = ref(10) const pagedGroupStats = computed(() => groupStats.value.slice((groupStatsPage.value - 1) * groupStatsPageSize.value, groupStatsPage.value * groupStatsPageSize.value), ) -watch([dupRows, drawerOccurrences, groupStats], () => { - matrixPage.value = Math.min(matrixPage.value, Math.max(1, Math.ceil(dupRows.value.length / matrixPageSize.value))) - dupCardPage.value = Math.min(dupCardPage.value, Math.max(1, Math.ceil(dupRows.value.length / dupCardPageSize.value))) +watch([drawerOccurrences, groupStats], () => { drawerPage.value = 1 groupStatsPage.value = Math.min(groupStatsPage.value, Math.max(1, Math.ceil(groupStats.value.length / groupStatsPageSize.value))) }) -const matrixHint = computed(() => - scope.value === 'global' - ? '行 = 全店撞款ASIN,列 = 全部店铺;有数字表示该店在售此 ASIN,点击行可看上架时间明细' - : selectedGroup.value - ? `行 = 分组「${selectedGroup.value}」内的撞款ASIN,列 = 该分组店铺;有数字表示组内该店在售此 ASIN` - : '行 = 分组内撞款ASIN,列 = 该分组店铺;有数字表示组内该店在售此 ASIN', -) - /* ----- 矩阵单元格辅助 ----- */ function shopKey(name: string): string { return (name || '').trim().toLowerCase() } -function cellOccurrences(item: DuplicateItem, shop: string): DuplicateOccurrence[] { - const key = shopKey(shop) - return (item.occurrences || []).filter((occ) => shopKey(occ.shopName) === key) -} -function cellCount(item: DuplicateItem, shop: string): number { - return cellOccurrences(item, shop).length -} -function cellCountryDots(item: DuplicateItem, shop: string): string[] { - const codes = new Set() - for (const occ of cellOccurrences(item, shop)) if (occ.country) codes.add(occ.country.toUpperCase()) - return [...codes] -} -function cellCountryCount(item: DuplicateItem, shop: string, code: string): number { - return cellOccurrences(item, shop).filter((occ) => (occ.country || '').toUpperCase() === code).length -} -function cellTitle(item: DuplicateItem, shop: string): string { - const count = cellCount(item, shop) - if (!count) return `${shop} 无此 ASIN` - const sites = cellCountryDots(item, shop).map((code) => `${asinCountryLabel(code)} ${cellCountryCount(item, shop, code)}条`).join('、') - return `${shop} 在售 ${count} 条 · ${sites}` -} function activeFilterParams() { return { asin: filter.asin.trim(), shopName: filter.shop.trim(), group: filter.group.trim(), country: filter.country, site: '', dateFrom: filter.dateFrom, dateTo: filter.dateTo } @@ -217,7 +193,13 @@ async function loadConsole() { loading.value = true loadError.value = '' try { - consoleData.value = await fetchDuplicateConsole({ view: '', ...activeFilterParams() }, companyTab.requestCompanyId) + consoleData.value = await fetchDuplicateConsole( + { view: '', ...activeFilterParams() }, + companyTab.requestCompanyId, + matrixPage.value, + matrixPageSize.value, + scope.value === 'group' ? selectedGroup.value : '', + ) shopGroups.value = indexShopGroups(orderedShops.value) if (groupStats.value.length && !groupStats.value.some((group) => group.name === selectedGroup.value)) { selectedGroup.value = groupChips.value[0] ? groupChips.value[0].name : '' @@ -270,6 +252,7 @@ function resetQuery() { scope.value = 'global' selectedGroup.value = '' activeTab.value = 'dup' + resetMatrixWindows() applyQuery() toast('筛选已重置') } @@ -280,8 +263,21 @@ function switchTab(tab: Tab) { } function switchScope(next: Scope) { + if (scope.value === next) return scope.value = next if (next === 'group' && !selectedGroup.value && groupChips.value.length) selectedGroup.value = groupChips.value[0].name + // 行范围换了(全局 ↔ 组内),服务端裁出来的那一窗也就换了 ⇒ 重新请一窗。 + resetMatrixWindows() + void loadConsole() +} + +/** 分组 chip:选中即换行范围,与 switchScope 同一条出口(不再在客户端重裁一遍全集)。 */ +function selectGroup(name: string) { + if (selectedGroup.value === name && scope.value === 'group') return + scope.value = 'group' + selectedGroup.value = name + resetMatrixWindows() + void loadConsole() } async function reAnalyze() { @@ -338,7 +334,7 @@ function openDrawer(asin: string, brand: string, dupFlag: boolean, occurrences: } function openDupDrawer(item: DuplicateItem) { - openDrawer(item.asin, itemBrand(item), item.shopCount >= 2, item.occurrences) + openDrawer(item.asin, itemBrandOf(item), item.shopCount >= 2, item.occurrences) } function openLedgerDrawer(row: LedgerRow) { @@ -553,7 +549,7 @@ onMounted(() => {
@@ -568,9 +564,16 @@ onMounted(() => {
{{ activeGroupStat.dupCount }}
组内撞款ASIN
-
撞款覆盖矩阵 {{ matrixHint }}
+
+ 撞款覆盖矩阵 {{ matrixHint }} + + + 列 {{ columnPage }}/{{ columnWindowTotal }}(共 {{ matrixAllColumns.length }} 列) + + +
- +
@@ -579,13 +582,13 @@ onMounted(() => { - - - - + + + @@ -596,15 +599,15 @@ onMounted(() => { 调整筛选条件或切换到其他分组查看 - +
撞款详情 同一 ASIN 在多家店铺的上架时间明细
-
-
+
+
{{ item.asin }} {{ item.shopCount }}店在售 - {{ itemBrand(item) || '未知品牌' }} + {{ itemBrandOf(item) || '未知品牌' }}
@@ -620,8 +623,8 @@ onMounted(() => {
当前范围下没有撞款ASIN
- -
共 {{ dupRows.length }} 条{{ scope === 'group' && selectedGroup ? `分组「${selectedGroup}」组内撞款ASIN` : '全店撞款ASIN' }}
+ +
共 {{ dupItems.length }} 条{{ scope === 'group' && selectedGroup ? `分组「${selectedGroup}」组内撞款ASIN` : '全店撞款ASIN' }}
@@ -885,6 +888,9 @@ onMounted(() => { .dup-ref .section-title { font-size: 14px; font-weight: 700; display: flex; align-items: baseline; gap: 8px; margin-bottom: 12px; } .dup-ref .section-title .hint { font-size: 11px; font-weight: 400; color: var(--text3); } .dup-ref .matrix-wrap { overflow-x: auto; overflow-y: auto; border-radius: var(--radius); background: var(--surface); border: 1px solid var(--border); max-height: 480px; } +/* 列窗口控件:与 section-title 同行,只在看得到第二窗列时出现。 */ +.dup-ref .col-window { margin-left: 12px; display: inline-flex; align-items: center; gap: 8px; font-size: 12px; color: var(--text2); } +.dup-ref .col-window .link-btn:disabled { color: var(--text3); cursor: not-allowed; } .dup-ref table.matrix { border-collapse: separate; border-spacing: 0; width: 100%; background: var(--surface); } .dup-ref table.matrix th { background: #F7F9FC; color: var(--text2); font-size: 12px; font-weight: 600; padding: 10px 12px; diff --git a/admin-frontend-vue/src/pages/tasks/duplicate-console-api.ts b/admin-frontend-vue/src/pages/tasks/duplicate-console-api.ts index bca9b3ac9..f3aa1da5d 100644 --- a/admin-frontend-vue/src/pages/tasks/duplicate-console-api.ts +++ b/admin-frontend-vue/src/pages/tasks/duplicate-console-api.ts @@ -10,7 +10,7 @@ import { export const DUPLICATE_CHECK_ENDPOINT = '/api/admin/shop-data-crawl' -/** console 端点吃 asin/shop_name/group/country/site/date_from/date_to;多余分页参数后端忽略。 */ +/** console 端点吃 asin/shop_name/group/country/site/date_from/date_to(行窗口的 page/page_size/dup_group 由调用方另传)。 */ function toConsoleQuery(filter: DuplicateFilter): Record { const query: Record = {} const asin = (filter.asin || '').trim() @@ -32,14 +32,25 @@ function toConsoleQuery(filter: DuplicateFilter): Record { export type LedgerSortKey = 'asin' | 'brand' | 'store_count' | 'record_count' | 'earliest' | 'latest' -/** 加载撞款控制台快照:GET /duplicate-check-console。 */ +/** + * 加载撞款控制台快照:GET /duplicate-check-console。 + * + * 行窗口(B60):`page`/`pageSize` 决定返回的 `dup` 是这一窗,`dupGroup` 非空时行集先按 + * 该分组成员裁剪(组内 ≥2 店才算撞款)。三者都是**服务端**口径——矩阵与详情卡片共用同一窗口, + * 页面不再自己从全集里切片(那需要后端把整份撞款数组发回来,正是 B60 要收掉的那件事)。 + */ export async function fetchDuplicateConsole( filter: DuplicateFilter, companyId?: number | null, + page = 1, + pageSize = 100, + dupGroup = '', ): Promise { // toConsoleQuery 是白名单构造(只点名 asin/shop_name/group/country/site/date_*), // 公司值只许落在本函数自己的 companyId 形参上。 - const query: Record = toConsoleQuery(filter) + const query: Record = { ...toConsoleQuery(filter), page, page_size: pageSize } + const group = (dupGroup || '').trim() + if (group) query.dup_group = group if (companyId != null) query.company_id = companyId const { data } = await http.get(`${DUPLICATE_CHECK_ENDPOINT}/duplicate-check-console`, { params: query }) return parseDuplicateConsole(data) diff --git a/admin-frontend-vue/src/pages/tasks/duplicate-console-logic.ts b/admin-frontend-vue/src/pages/tasks/duplicate-console-logic.ts index cc4705dd8..7cf6bda4c 100644 --- a/admin-frontend-vue/src/pages/tasks/duplicate-console-logic.ts +++ b/admin-frontend-vue/src/pages/tasks/duplicate-console-logic.ts @@ -1,4 +1,4 @@ -/** 撞款控制台视图派生(分组索引 / 组内撞款子集 / 店铺列并集):纯逻辑。 */ +/** 撞款控制台视图派生(分组索引 / 店铺列并集 / 矩阵格子预聚合):纯逻辑。 */ import type { DuplicateItem, DuplicateOccurrence, DuplicateShop } from './duplicate-model.ts' export interface ShopGroups { @@ -41,27 +41,6 @@ export function indexShopGroups(shops: DuplicateShop[]): ShopGroups { return { members, shopGroupNames } } -/** 仅在指定成员店铺(展示名)内保留 occurrences;重算 shop_count,返回 ≥2 店的撞款子集。 */ -export function dupWithinGroup(items: DuplicateItem[], memberShops: string[]): DuplicateItem[] { - const memberKeys = new Set() - for (const shop of memberShops || []) memberKeys.add(shopKey(shop)) - const result: DuplicateItem[] = [] - for (const item of items || []) { - const kept: DuplicateOccurrence[] = [] - const shops = new Set() - for (const occ of item.occurrences || []) { - if (memberKeys.has(shopKey(occ.shopName))) { - kept.push(occ) - shops.add(occ.shopName || '') - } - } - if (kept.length > 0 && shops.size >= 2) { - result.push({ asin: item.asin, shopCount: shops.size, recordCount: kept.length, occurrences: kept }) - } - } - return result -} - /** 撞款行涉及店铺的并集列(出现顺序去重);memberShops 非空时仅取交集成员的子集顺序。 */ export function storeColumnsOf(items: DuplicateItem[], memberShops: string[] = []): string[] { const restricted = new Set(memberShops.map((shop) => shopKey(shop))) @@ -90,6 +69,74 @@ export function storeColumnsOf(items: DuplicateItem[], memberShops: string[] = [ return order } +/** 矩阵一格:该店在售此 ASIN 的条数 + 站点(国家)分布。 */ +export interface MatrixCell { + count: number + sites: { code: string; count: number }[] +} + +/** 空格的单份实例:模板每格都要拿一个,不能每格 new 一个对象。 */ +export const EMPTY_MATRIX_CELL: MatrixCell = { count: 0, sites: [] } + +function cellKey(asin: string, shop: string): string { + // 用 \u0000 拼键:ASIN 与店铺名都可打印,任何可见分隔符都可能出现在店铺名里(那样就会串格)。 + return `${(asin || '').trim()}\u0000${shopKey(shop)}` +} + +/** + * 撞款矩阵的格子集合:把 rows×columns 一次算完(B60 的「预聚合」)。 + * + *

原形状是每格现算:`cellCount`/`cellCountryDots`/`cellTitle` 各自把该行 occurrences 全 filter + * 一遍,一格 4~6 次、50 行 × 200 列就是每帧上万次整行扫描,勾选/翻页都要重付。这里每行只按 + * 店铺分桶一次(O(occurrences)),之后一格一次 Map 取值。 + * + * 只给 `columns` 里点名的列建格:不在列集内的 occurrence 直接不装桶,列窗口收窄时开销跟着降。 + */ +export function buildMatrixCells(items: DuplicateItem[], columns: string[]): Map { + const cells = new Map() + const wanted = new Set() + for (const column of columns || []) wanted.add(shopKey(column)) + if (!wanted.size) return cells + for (const item of items || []) { + const buckets = new Map() + const siteCounts = new Map>() + for (const occ of item.occurrences || []) { + const key = shopKey(occ.shopName) + if (!wanted.has(key)) continue + const bucket = buckets.get(key) + if (bucket) bucket.push(occ) + else buckets.set(key, [occ]) + const code = (occ.country || '').trim().toUpperCase() + if (!code) continue + let counts = siteCounts.get(key) + if (!counts) { + counts = new Map() + siteCounts.set(key, counts) + } + counts.set(code, (counts.get(code) || 0) + 1) + } + for (const [key, occurrences] of buckets) { + const codes = siteCounts.get(key) + cells.set(cellKey(item.asin, key), { + count: occurrences.length, + sites: codes ? [...codes].map(([code, count]) => ({ code, count })) : [], + }) + } + } + return cells +} + +/** 取一格;没有格子时返回同一份空对象(模板里 `cellOf(...).count` 要能直接读)。 */ +export function matrixCellOf(cells: Map, asin: string, shop: string): MatrixCell { + return cells.get(cellKey(asin, shop)) || EMPTY_MATRIX_CELL +} + +/** 一个撞款 ASIN 的品牌:取第一条带品牌的上架记录,全空就是"未知品牌"(页面自己决定怎么显示)。 */ +export function itemBrandOf(item: DuplicateItem): string { + for (const occ of item.occurrences || []) if (occ.brand) return occ.brand + return '' +} + /** 明细抽屉聚合行:店铺×站点聚合(对齐 reference 抽屉的 店铺/分组/站点/上架时间(升序堆叠)/次数)。 */ export interface DrawerAggRow { shopName: string diff --git a/admin-frontend-vue/src/pages/tasks/duplicate-console-model.ts b/admin-frontend-vue/src/pages/tasks/duplicate-console-model.ts index e370ec0b6..b108c88f7 100644 --- a/admin-frontend-vue/src/pages/tasks/duplicate-console-model.ts +++ b/admin-frontend-vue/src/pages/tasks/duplicate-console-model.ts @@ -18,12 +18,19 @@ export interface ConsoleGroupStat { dupCount: number } -/** 撞款控制台单次快照:总览(指标/店铺分布) + 撞款全集 + 分组统计。 */ +/** + * 撞款控制台单次快照:总览(指标/店铺分布) + 撞款**行窗口** + 分组统计。 + * + * `dup` 是服务端按 page/pageSize 切出来的一窗(B60),`totalDup` 是该范围内未分页的总数; + * 两者不是同一个口径,页面分页器读 totalDup、列表读 dup。 + */ export interface DuplicateConsole { overview: DuplicateOverview dup: DuplicateItem[] groups: ConsoleGroupStat[] totalDup: number + page: number + pageSize: number } /** 台账单行(含 occurrences,供明细抽屉还原每店上架时间线)。 */ @@ -87,7 +94,14 @@ export function parseDuplicateConsole(payload: unknown): DuplicateConsole { const groups = Array.isArray(record.groups) ? record.groups.map((raw) => parseConsoleGroup(raw)).filter((group): group is ConsoleGroupStat => group !== null) : [] - return { overview, dup, groups, totalDup: count(record.total_dup ?? record.totalDup) } + return { + overview, + dup, + groups, + totalDup: count(record.total_dup ?? record.totalDup), + page: count(record.page) || 1, + pageSize: count(record.page_size ?? record.pageSize), + } } /** 解析单条台账行。 */ diff --git a/admin-frontend-vue/src/pages/tasks/duplicate-matrix-window.ts b/admin-frontend-vue/src/pages/tasks/duplicate-matrix-window.ts new file mode 100644 index 000000000..65723d1d1 --- /dev/null +++ b/admin-frontend-vue/src/pages/tasks/duplicate-matrix-window.ts @@ -0,0 +1,169 @@ +/** + * 撞款覆盖矩阵的「行窗 + 列窗」组合式(B60 从 DuplicateCheckPage.vue 拆出)。 + * + * 为什么单独立一支:那一屏矩阵原本是每格现算——`cellCount / cellCountryDots / cellTitle` + * 各自把该行 occurrences 全 filter 一遍(一格 4~6 次),而且整份撞款全集一次性铺成 + * 「全部行 × 全部列」的 DOM。收口后行由服务端切窗、列在本模块切窗,格子的文案与 + * 国家点一次算完、空格按列共享同一份视图。这些状态(列游标、行游标、三份 computed) + * 是一件事,留在页面里会把本页推到可读性基线之外,也更难单测。 + * + * 纯逻辑部分(分桶、取列)在 `duplicate-console-logic.ts`,本文件只做"窗口"这一层, + * 数据从页面注入,不认识 store。 + */ +import { computed, ref, watch, type ComputedRef, type Ref } from 'vue' +import { buildMatrixCells, itemBrandOf, matrixCellOf, storeColumnsOf } from './duplicate-console-logic.ts' +import { asinCountryLabel } from '../asin/asin-country.ts' +import type { DuplicateItem } from './duplicate-model.ts' + +/** + * 列窗口上限(B60「限制列数」)。一窗 40 列 × 最多 100 行=单帧最多 4,000 格; + * 不封顶时 200 列 × 100 行=2 万格 DOM,任何一次勾选/翻页都要重排。 + * 列没被裁掉,只是切成窗口(`goColumnPage`),所以信息一条不丢。 + */ +export const MATRIX_COLUMN_WINDOW = 40 + +/** 模板只读这一份:数字、国家点、悬浮说明都在这里算完。 */ +export interface MatrixCellView { + count: number + dots: string[] + title: string +} + +/** 一行矩阵视图:行对象本身(点开抽屉要用)+ 品牌 + 本窗每一格。 */ +export interface MatrixRowView { + item: DuplicateItem + asin: string + brand: string + cells: MatrixCellView[] +} + +export interface MatrixWindowSource { + /** 当前行窗(服务端切好的那一窗)。 */ + rows: ComputedRef + /** 列范围的第二正本=分组页签选中的组员(非分组范围给空数组)。 */ + groupMembers: ComputedRef + /** 分组范围与否:决定列要不要与组员取交集。 */ + groupScoped: Ref | ComputedRef + /** 选中的分组名(只用于矩阵那行说明文案,让"看到的是什么范围"写在屏上)。 */ + selectedGroup: Ref | ComputedRef + /** 行窗换页/换档都要重新请一窗(服务端切,页面不再从全集里自切)。 */ + reload: () => void + /** 数据到达的版本号(一般是 consoleData 本身):新数据 ⇒ 旧游标已指向不存在的位置。 */ + dataVersion: Ref +} + +/** 空格共用同一份 dots(模板只读不写,共享数组不会被改)。 */ +const NO_DOTS: string[] = [] + +export function useDuplicateMatrix(source: MatrixWindowSource) { + const matrixPage = ref(1) + const matrixPageSize = ref(10) + const columnPage = ref(1) + + /** 本页行里真出现过的店铺=矩阵该配的列(分组范围再与组员取交集)。 */ + const matrixAllColumns = computed(() => + storeColumnsOf(source.rows.value, source.groupScoped.value ? source.groupMembers.value : []), + ) + + const columnWindowTotal = computed(() => + Math.max(1, Math.ceil(matrixAllColumns.value.length / MATRIX_COLUMN_WINDOW)), + ) + + /** 矩阵列:按窗口切(超出部分用列窗口控件翻)。 */ + const matrixShops = computed(() => { + const start = Math.max(0, (columnPage.value - 1) * MATRIX_COLUMN_WINDOW) + return matrixAllColumns.value.slice(start, start + MATRIX_COLUMN_WINDOW) + }) + + /** 一次把「本窗行 × 本窗列」全部算完,模板只读不算。 */ + const matrixCells = computed(() => buildMatrixCells(source.rows.value, matrixShops.value)) + + function cellView(shop: string, cell: ReturnType): MatrixCellView { + const sites = cell.sites.map((site) => `${asinCountryLabel(site.code)} ${site.count}条`).join('、') + return { + count: cell.count, + dots: cell.sites.map((site) => site.code), + title: sites ? `${shop} 在售 ${cell.count} 条 · ${sites}` : `${shop} 在售 ${cell.count} 条`, + } + } + + /** + * 每列一份「本格为空」的视图,整列复用同一对象。 + * + *

矩阵里绝大多数格是空的(一个 ASIN 只在几家店在售,列却是几十上百家店)。原先一格 + * 一条 `无此 ASIN` 文案,50 行 × 200 列=1 万条字符串;按列准备一次之后行数再多也不 + * 乘上去——这条才是"每格现算"剩下的那半开销。 + */ + const emptyColumnViews = computed(() => + matrixShops.value.map((shop) => ({ count: 0, dots: NO_DOTS, title: `${shop} 无此 ASIN` })), + ) + + const matrixView = computed(() => { + const cells = matrixCells.value + const blanks = emptyColumnViews.value + return source.rows.value.map((item) => ({ + item, + asin: item.asin, + brand: itemBrandOf(item) || '—', + cells: matrixShops.value.map((shop, index) => { + const cell = matrixCellOf(cells, item.asin, shop) + return cell.count ? cellView(shop, cell) : blanks[index] + }), + })) + }) + + /** 屏上那行说明:把"行/列各是什么范围"写清楚,尤其分组范围是服务端裁过行的那一窗。 */ + const matrixHint = computed(() => + source.groupScoped.value + ? source.selectedGroup.value + ? `行 = 分组「${source.selectedGroup.value}」内的撞款ASIN(服务端裁行+分页),列 = 组内在售店铺;有数字表示组内该店在售此 ASIN` + : '行 = 分组内撞款ASIN,列 = 该分组店铺;有数字表示组内该店在售此 ASIN' + : '行 = 本页撞款ASIN(服务端分页),列 = 本页撞款涉及店铺;有数字表示该店在售此 ASIN,点击行可看上架时间明细', + ) + + /** 行窗口翻页:同一页重复点不重新请(翻页控件的当前页也会触发 change)。 */ + function goMatrixPage(page: number): void { + if (page === matrixPage.value) return + matrixPage.value = page + source.reload() + } + + function changeMatrixSize(size: number): void { + matrixPageSize.value = size + matrixPage.value = 1 + source.reload() + } + + /** 列窗口翻页:行不动,只把看得见的店铺换一窗。 */ + function goColumnPage(step: number): void { + const next = columnPage.value + step + if (next < 1 || next > columnWindowTotal.value) return + columnPage.value = next + } + + /** 换了行范围(全局 ↔ 组内、换分组、重置筛选):两个游标都回到第一窗。 */ + function resetMatrixWindows(): void { + matrixPage.value = 1 + columnPage.value = 1 + } + + watch(source.dataVersion, () => { + columnPage.value = Math.min(columnPage.value, columnWindowTotal.value) + }) + + return { + MATRIX_COLUMN_WINDOW, + matrixPage, + matrixPageSize, + columnPage, + columnWindowTotal, + matrixAllColumns, + matrixShops, + matrixView, + matrixHint, + goMatrixPage, + changeMatrixSize, + goColumnPage, + resetMatrixWindows, + } +} diff --git a/admin-frontend-vue/tests/align-dup-console.test.ts b/admin-frontend-vue/tests/align-dup-console.test.ts index a16edec23..d32960b90 100644 --- a/admin-frontend-vue/tests/align-dup-console.test.ts +++ b/admin-frontend-vue/tests/align-dup-console.test.ts @@ -52,15 +52,46 @@ test('align_dup_console_pagination_all_lists', () => { // 且分布/矩阵/详情常显(不足一页也显示分页器,反馈:数据少时看不到分页)。 const page = readSource('src/pages/tasks/DuplicateCheckPage.vue') assert.match(page, /:total="distCount"/, '店铺上架分布有分页器(常显)') - assert.match(page, /:total="dupRows\.length"/, '撞款覆盖矩阵有分页器(常显)') - assert.match(page, /v-for="item in dupCardRows"/, '撞款详情卡片绑定分页切片') - assert.match(page, /:total="dupRows\.length"[\s\S]*dupCardPageSize/, '撞款详情卡片有分页器(常显)') + // B60:撞款行改由服务端切窗口,矩阵与详情卡片读同一窗 ⇒ 两个分页器必须共用同一个游标 + // (各自独立翻页在"一窗"的形状下已经不可表示,留着就是界面说一套、请求发另一套)。 + assert.match(page, /:total="totalDupCount"[\s\S]*:page="matrixPage"/, '撞款覆盖矩阵分页器绑服务端总数与行窗口游标') + assert.match(page, /v-for="item in dupItems"/, '撞款详情卡片绑这一窗行集') + assert.match(page, /:total="totalDupCount"[\s\S]*:page-size="matrixPageSize"/, '撞款详情卡片分页器与矩阵同游标') + assert.doesNotMatch(page, /dupCardPage/, '卡片不再自持一个分页游标(同窗就是同游标)') + assert.doesNotMatch(page, /matrixRows/, '矩阵不再客户端切片(行窗口是服务端切的)') assert.match(page, /]*:total="distCount"/, '分布分页器无 v-if 条件') - assert.match(page, /]*:total="dupRows\.length"/, '矩阵/详情分页器无 v-if 条件') + assert.match(page, /]*:total="totalDupCount"/, '矩阵/详情分页器无 v-if 条件') assert.match(page, /v-for="group in pagedGroupStats"/, '分组说明抽屉表绑定分页切片') assert.match(page, /:total="groupStats\.length"/, '分组说明抽屉表有分页器') }) +/** + * B60 的前后两处出口:行窗口三键要真发出去,矩阵格子要一次算完。 + * 反向自证:把 api 改回不带 page/page_size、或把模板改回逐格 filter,即红。 + */ +test('align_dup_console_b60_row_window_and_precomputed_cells', () => { + const api = readSource('src/pages/tasks/duplicate-console-api.ts') + assert.match(api, /page, page_size: pageSize/, 'console 请求带行窗口参数') + assert.match(api, /query\.dup_group = group/, 'console 请求带分组裁行参数') + assert.doesNotMatch(api, /多余分页参数后端忽略/, '旧注释说分页参数被后端忽略,现在后端认了') + + const page = readSource('src/pages/tasks/DuplicateCheckPage.vue') + // 矩阵窗口那一层已从页面拆进 `duplicate-matrix-window.ts`(B60 之后本页顶在可读性基线的 + // 超长文件名单上,只许减)。绊线跟着搬,判据一字不松:改回逐格 filter 照样红。 + const matrix = readSource('src/pages/tasks/duplicate-matrix-window.ts') + assert.match(page, /useDuplicateMatrix\(/, '页面接的是矩阵窗口组合式') + assert.match(matrix, /buildMatrixCells\(/, '矩阵格子一次预聚合') + assert.match(matrix, /MATRIX_COLUMN_WINDOW = 40/, '列数有窗口上限(按列切窗)') + assert.match(page, /v-for="\(cell, index\) in row\.cells"/, '模板只读预聚合好的格子') + assert.doesNotMatch(page, /function cellCount\(/, '逐格 filter 那套函数已收掉') + assert.doesNotMatch(matrix, /function cellCount\(/, '窗口组合式里也不许再长出逐格 filter') + + const service = readSource('../backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryService.java') + assert.match(service, /consoleData\(DuplicateScanView view, Set visibleKeys, Filters filters,\s*\n?\s*int page, int pageSize, String dupGroup\)/, + '服务端 console 收行窗口与分组参数') + assert.match(service, /"total_dup", total,/, 'total_dup 是未分页总数,不是本页行数') +}) + test('align_dup_console_ledger_total_preload', () => { const page = readSource('src/pages/tasks/DuplicateCheckPage.vue') assert.match(page, /onMounted\(\(\) => \{\s*loadConsole\(\).*loadLedger\(1\)/s, '挂载即预取台账计数') diff --git a/admin-frontend-vue/tests/baselines/source-pin-ledger.json b/admin-frontend-vue/tests/baselines/source-pin-ledger.json index 82d272284..8321392bb 100644 --- a/admin-frontend-vue/tests/baselines/source-pin-ledger.json +++ b/admin-frontend-vue/tests/baselines/source-pin-ledger.json @@ -1,10 +1,10 @@ { "note": "readSource 钉子逐文件基线。两把尺子:pins=落在用例体内的读盘,reads=全文读点总数(含模块级夹具与 helper)。都只许减;pins 下降而 reads 没同步下降=把读盘挪出体内,不算改进。由 tests/source-pin-ratchet.test.ts 生成。", "totals": { - "files": 253, - "cases": 1772, - "pins": 457, - "reads": 652 + "files": 254, + "cases": 1778, + "pins": 458, + "reads": 656 }, "files": { "admin-scaffold.test.ts": { @@ -48,9 +48,9 @@ "reads": 3 }, "align-dup-console.test.ts": { - "cases": 10, - "pins": 9, - "reads": 12 + "cases": 11, + "pins": 10, + "reads": 16 }, "align-edit-user-auth-guard.test.ts": { "cases": 1, @@ -272,8 +272,13 @@ "pins": 5, "reads": 6 }, + "dup-matrix-window.test.ts": { + "cases": 4, + "pins": 0, + "reads": 0 + }, "duplicate-console.test.ts": { - "cases": 5, + "cases": 6, "pins": 0, "reads": 0 }, diff --git a/admin-frontend-vue/tests/dup-matrix-window.test.ts b/admin-frontend-vue/tests/dup-matrix-window.test.ts new file mode 100644 index 000000000..330c9e13c --- /dev/null +++ b/admin-frontend-vue/tests/dup-matrix-window.test.ts @@ -0,0 +1,183 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildMatrixCells, matrixCellOf, storeColumnsOf } from '../src/pages/tasks/duplicate-console-logic.ts' +import type { DuplicateItem, DuplicateOccurrence } from '../src/pages/tasks/duplicate-model.ts' + +/** + * B60 撞款矩阵:预聚合的等价性与前后读数。 + * + * 审计指的那件事是「每格多次全量 filter」——模板一格要付 cellCount ×2 + cellTitle(内部再 + * cellCount + cellCountryDots + 每站点一次 cellCountryCount),每次都是把该行 occurrences 整份 + * filter 一遍,而且**每格都新建一条文案字符串**。下面 `legacyCell` 就是页面原来那几行函数的 + * 原样搬运,当"前";与新一次算完的 `buildMatrixCells`(空格按列共用一份视图)对撞: + * 先证明结果逐字相同(预聚合不是靠少算蒙过去的),再同尺量时间。 + */ +function occ(asin: string, shopName: string, country: string): DuplicateOccurrence { + return { + asin, + date: '2026-09-01', + price: '', + brand: 'B', + shopName, + groupName: '', + countryCodes: [country], + country, + } +} + +const shopKey = (name: string): string => (name || '').trim().toLowerCase() + +interface CellView { + count: number + dots: string[] + title: string +} + +/** 原页面的逐格算法(一格扫整行 occurrences 若干次,并且每格造一条文案)。 */ +function legacyCellOccurrences(item: DuplicateItem, shop: string): DuplicateOccurrence[] { + const key = shopKey(shop) + return (item.occurrences || []).filter((entry) => shopKey(entry.shopName) === key) +} + +function legacyCell(item: DuplicateItem, shop: string): CellView { + const count = legacyCellOccurrences(item, shop).length + const codes = new Set() + for (const entry of legacyCellOccurrences(item, shop)) if (entry.country) codes.add(entry.country.toUpperCase()) + const dots = [...codes] + const sites = dots.map((code) => `${code} ${legacyCellOccurrences(item, shop).filter((entry) => (entry.country || '').toUpperCase() === code).length}条`) + return { + count, + dots, + title: count ? `${shop} 在售 ${count} 条 · ${sites.join('、')}` : `${shop} 无此 ASIN`, + } +} + +/** 页面现在的读法:一次取值;空格取该列那一份共用视图(不再造文案)。 */ +function fastCell(cells: ReturnType, item: DuplicateItem, shop: string, blank: CellView): CellView { + const cell = matrixCellOf(cells, item.asin, shop) + if (!cell.count) return blank + const sites = cell.sites.map((site) => `${site.code} ${site.count}条`) + return { + count: cell.count, + dots: cell.sites.map((site) => site.code), + title: sites.length ? `${shop} 在售 ${cell.count} 条 · ${sites.join('、')}` : `${shop} 在售 ${cell.count} 条`, + } +} + +function fixtureItem(asin: string, shops: [string, string][]): DuplicateItem { + const occurrences = shops.map(([shop, country]) => occ(asin, shop, country)) + return { + asin, + shopCount: new Set(occurrences.map((entry) => shopKey(entry.shopName))).size, + recordCount: occurrences.length, + occurrences, + } +} + +function blankViews(columns: string[]): CellView[] { + return columns.map((shop) => ({ count: 0, dots: [] as string[], title: `${shop} 无此 ASIN` })) +} + +/** + * 200 列 × 50 行的矩阵夹具。`shopsPerRow` 决定密度:现实中一个撞款 ASIN 落在几家到几十家店, + * 而逐格 filter 的代价正是按「行内 occurrences × 列数」乘出来的——密度越高旧算法越吃亏, + * 所以两种密度都取读数,不挑对自己有利的那一档。 + */ +function matrixFixture(columns: string[], shopsPerRow: number): DuplicateItem[] { + return Array.from({ length: 50 }, (_, index) => { + const pairs: [string, string][] = [] + for (let offset = 0; offset < shopsPerRow; offset++) { + const shop = columns[(index * 7 + offset * 61) % columns.length] + pairs.push([shop, 'UK'], [shop, 'DE']) + } + return fixtureItem(`B${String(index).padStart(4, '0')}`, pairs) + }) +} + +test('矩阵预聚合与逐格 filter 结果逐字相同', () => { + const rows = [ + fixtureItem('A1', [['Shop a', 'UK'], ['Shop a', 'uk'], ['ShopB', ''], ['ShopB', 'DE']]), + fixtureItem('A2', [['ShopC', 'FR']]), + ] + const columns = storeColumnsOf(rows) + const cells = buildMatrixCells(rows, columns) + const blanks = blankViews(columns) + columns.forEach((column, index) => { + for (const item of rows) { + assert.deepEqual(fastCell(cells, item, column, blanks[index]), legacyCell(item, column), `${item.asin}/${column}`) + } + }) + // 每行只给「本行真涉及」的列建格:A1 涉及 2 店(大小写/空格归一后同店)+ A2 涉及 1 店。 + assert.equal(cells.size, 3) +}) + +test('列集或行集为空时不建任何格', () => { + const rows = [fixtureItem('A1', [['ShopA', 'UK']])] + assert.equal(buildMatrixCells(rows, []).size, 0) + assert.equal(buildMatrixCells([], ['ShopA']).size, 0) +}) + +/** 列窗口:只给本窗点名的列建格(120 列的夹具切 40 列窗口,格子数就按 40 乘)。 */ +test('列窗口切窗后每行最多铺满窗口宽度的格', () => { + const window = 40 + const rows = Array.from({ length: 50 }, (_, index) => + fixtureItem(`A${index}`, Array.from({ length: window * 3 }, (_, shop) => [`Shop${shop}`, 'UK']))) + const all = storeColumnsOf(rows) + assert.equal(all.length, window * 3, '夹具本身就有 120 列(不设上限时的形状)') + + const columns = all.slice(0, window) + const cells = buildMatrixCells(rows, columns) + assert.equal(columns.length, window, '本窗只铺 40 列') + assert.equal(cells.size, rows.length * window, '窗口外的 80 列一格都不建') +}) + +/** + * B60 判据的读数:200 列 × 50 行,同一把尺前后各跑一遍(5 轮取最好一次——本机同时有别的路在跑, + * 单次读数能被调度噪声放大好几倍)。 + * + * 度量对象是「矩阵格集合的 JS 计算」,审计点名的就是这条。**浏览器首屏/交互耗时本机拿不到** + * (没有真实店铺数据、也没有浏览器现场),按准出括注登记为现网复量项,不许拿这条当"首屏已实测"读。 + */ +test('200列×50行:预聚合与按列切窗都必须快过逐格 filter', () => { + const columns = Array.from({ length: 200 }, (_, index) => `Shop${index}`) + const blanks = blankViews(columns) + const windowColumns = columns.slice(0, 40) + const windowBlanks = blanks.slice(0, 40) + const best = (run: () => unknown): number => { + let ms = Number.POSITIVE_INFINITY + for (let attempt = 0; attempt < 5; attempt++) { + const start = performance.now() + run() + ms = Math.min(ms, performance.now() - start) + } + return ms + } + + const readings: string[] = [] + for (const shopsPerRow of [3, 40]) { + const rows = matrixFixture(columns, shopsPerRow) + const legacySweep = () => { + const out: CellView[] = [] + for (const item of rows) for (const column of columns) out.push(legacyCell(item, column)) + return out + } + const precomputedSweep = (cols: string[], cellBlanks: CellView[]) => { + const cells = buildMatrixCells(rows, cols) + const out: CellView[] = [] + for (const item of rows) { + cols.forEach((column, index) => out.push(fastCell(cells, item, column, cellBlanks[index]))) + } + return out + } + + assert.deepEqual(precomputedSweep(columns, blanks), legacySweep(), '前后必须同一份格子(快不是靠少算)') + + const beforeMs = best(legacySweep) + const precomputeMs = best(() => precomputedSweep(columns, blanks)) + const shippedMs = best(() => precomputedSweep(windowColumns, windowBlanks)) + readings.push(`每行${shopsPerRow}店:逐格filter=${beforeMs.toFixed(1)}ms → 预聚合=${precomputeMs.toFixed(1)}ms(${(beforeMs / precomputeMs).toFixed(1)}×) → 预聚合+切窗40列=${shippedMs.toFixed(1)}ms(${(beforeMs / shippedMs).toFixed(1)}×)`) + assert.ok(precomputeMs < beforeMs, `预聚合没省下来(每行${shopsPerRow}店):${beforeMs.toFixed(1)}ms → ${precomputeMs.toFixed(1)}ms`) + assert.ok(shippedMs * 2 < beforeMs, `切窗后仍不够快(每行${shopsPerRow}店):${beforeMs.toFixed(1)}ms → ${shippedMs.toFixed(1)}ms`) + } + console.log(`[B60 读数] 200列×50行 ${readings.join(' | ')}`) +}) diff --git a/admin-frontend-vue/tests/duplicate-console.test.ts b/admin-frontend-vue/tests/duplicate-console.test.ts index aca7c635b..ed592a9cd 100644 --- a/admin-frontend-vue/tests/duplicate-console.test.ts +++ b/admin-frontend-vue/tests/duplicate-console.test.ts @@ -2,8 +2,9 @@ import test from 'node:test' import assert from 'node:assert/strict' import { parseDuplicateConsole, parseLedgerPage, parseConsoleGroup, parseLedgerRow } from '../src/pages/tasks/duplicate-console-model.ts' import { + buildMatrixCells, indexShopGroups, - dupWithinGroup, + matrixCellOf, storeColumnsOf, } from '../src/pages/tasks/duplicate-console-logic.ts' @@ -49,6 +50,21 @@ test('test_dup_console_model_parses_overview_dup_and_groups', () => { assert.deepEqual(group, { name: 'GroupA', shopCount: 2, asinUnique: 4, recordCount: 6, dupCount: 2 }) }) +/** B60:console 多了行窗口三键;缺字段要退得回安全默认(page 至少 1,pageSize 缺省 0=调用方自持)。 */ +test('test_dup_console_model_parses_row_window', () => { + const windowed = parseDuplicateConsole({ + success: true, + data: { pending: false, dup: [], groups: [], total_dup: 2650, page: 3, page_size: 50 }, + }) + assert.equal(windowed.totalDup, 2650, 'total_dup 是该范围未分页总数,不是本页行数') + assert.equal(windowed.page, 3) + assert.equal(windowed.pageSize, 50) + + const legacy = parseDuplicateConsole({ success: true, data: { pending: false, dup: [], groups: [] } }) + assert.equal(legacy.page, 1, '没有 page 字段时读成第一页,别让分页控件显示第 0 页') + assert.equal(legacy.pageSize, 0) +}) + test('test_dup_console_parse_group_ignores_missing_name', () => { assert.equal(parseConsoleGroup({ shop_count: 1 }), null) assert.ok(parseConsoleGroup({ name: 'G', shop_count: 2 })) @@ -71,15 +87,9 @@ test('test_dup_console_group_derivation_restricts_to_members', () => { ]) assert.deepEqual(groups.members.GroupA, ['ShopA', 'ShopB']) assert.deepEqual(groups.shopGroupNames.ShopC, ['GroupC']) - - const inGroupA = dupWithinGroup([dupA, dupE], groups.members.GroupA) - assert.equal(inGroupA.length, 2) - // E 组内只剩 ShopA/ShopB 两条记录。 - const e = inGroupA.find((item) => item.asin === 'E') - assert.equal(e?.shopCount, 2) - assert.equal(e?.recordCount, 2) - // 单店成员组内不会命中(ShopD 不在数据里)。 - assert.deepEqual(dupWithinGroup([dupA, dupE], ['ShopD']), []) + assert.deepEqual(storeColumnsOf([dupA, dupE], groups.members.GroupA), ['ShopA', 'ShopB']) + // 「组内 ≥2 店才算撞款」的裁行只剩服务端一处正本(console 的 dup_group)—— + // 页面原先在客户端用 dupWithinGroup 再裁一遍,那是同一规则的第二个正本,B60 收掉它。 }) test('test_dup_console_store_columns_order_and_restriction', () => { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/controller/AdminShopDataDuplicateCheckController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/controller/AdminShopDataDuplicateCheckController.java index 82ad944dd..b8c3e2c54 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/controller/AdminShopDataDuplicateCheckController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/controller/AdminShopDataDuplicateCheckController.java @@ -141,7 +141,7 @@ public class AdminShopDataDuplicateCheckController { } @GetMapping("/duplicate-check-console") - @Operation(summary = "撞款控制台快照:指标/店铺/命中筛选的撞款全集/分组统计") + @Operation(summary = "撞款控制台快照:指标/店铺/撞款行窗口(分页)/分组统计") public ApiResponse console( HttpServletRequest request, @RequestParam(name = "asin", required = false) String asin, @@ -151,13 +151,20 @@ public class AdminShopDataDuplicateCheckController { @RequestParam(name = "site", required = false) String site, @RequestParam(name = "date_from", required = false) String dateFrom, @RequestParam(name = "date_to", required = false) String dateTo, + @Parameter(description = "行窗口页码(从 1 起)") @RequestParam(name = "page", required = false) String page, + @Parameter(description = "行窗口每页行数(响应体 dup 行数 ≤ 本值)") + @RequestParam(name = "page_size", required = false) String pageSize, + @Parameter(description = "撞款行范围=指定分组的成员店铺(只裁行,不动指标/店铺/分组统计)") + @RequestParam(name = "dup_group", required = false) String dupGroup, @Parameter(description = "公司 ID") @RequestParam(name = "company_id", required = false) Long companyId) { RequestOperator operator = requireDuplicateCheckAccess(request); + int safePage = parsePage(page); + int safeSize = clamp(parseSize(pageSize, 100), 1, 200); ShopDataDuplicateCheckQueryService.Filters filters = ShopDataDuplicateCheckQueryService.Filters.clean(asin, shopName, country, site, dateFrom, dateTo); DuplicateScanView scan = scanService.loadLatest(); if (scan == null) { - return ApiResponse.success(queryService.pendingConsole()); + return ApiResponse.success(queryService.pendingConsole(safePage, safeSize)); } List companyMemberIds = operator.superAdmin() ? dataReadScope.readScope(operator.user(), companyId).userIdsOr(null) @@ -165,7 +172,8 @@ public class AdminShopDataDuplicateCheckController { Set visibleKeys = queryService.resolveVisibleShopKeys( operator.id(), operator.superAdmin(), companyMemberIds); Set effectiveKeys = queryService.withGroupFilter(visibleKeys, scan, group); - return ApiResponse.success(queryService.consoleData(scan, effectiveKeys, filters)); + return ApiResponse.success(queryService.consoleData(scan, effectiveKeys, filters, + safePage, safeSize, dupGroup)); } @GetMapping("/duplicate-check-ledger") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryService.java index 08bdc4543..6b9698dfc 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryService.java @@ -280,22 +280,73 @@ public class ShopDataDuplicateCheckQueryService { "page", page, "page_size", pageSize, "scanned_at", ""); } - /** 撞款控制台一次快照:summary/shops(全局可见口径) + 命中筛选的撞款全集 + 分组统计。 */ - public Map consoleData(DuplicateScanView view, Set visibleKeys, Filters filters) { + /** + * 撞款控制台一次快照:summary/shops(全局可见口径) + 命中筛选的撞款**行窗口** + 分组统计。 + * + *

B60:{@code dup} 原先返回全集(现网可累到数万行的 JSON),前端翻页/切分组都在这份 + * 整数组上做客户端切片——分页参数在这里真生效(响应体行数 ≤ pageSize),{@code total_dup} + * 仍是该范围内未分页的总数(KPI 与分页控件读它,不是窗口长度)。 + * + *

{@code dupGroup} 非空时行集先按该分组裁剪(组内 ≥2 店才算撞款)。判据与 + * {@link #groupStats} 同源(成员键=可见店铺的 groupName 拆分),而不是前端再算一份: + * 原先页面用 {@code dupWithinGroup} 自己裁了一遍,两套规则同名不同实现,分页一上就会分叉。 + * + *

排序在切窗之前:行窗口必须稳定,否则同一页每次刷新拿到的是不同行。 + * ASIN 是 {@code [A-Z0-9]},这里的字符串序与页面原先的 localeCompare 同序。 + * + *

蓝图 5A 那句「限制列数」落在渲染侧(后管 {@code duplicate-matrix-window.ts} 的 + * 列窗 40),服务端不设列窗,别把它当成没做完:矩阵的列本来就是从这份已分页的行 + * 里数出来的({@code storeColumnsOf(rows)}),而 {@code shops} 是筛选项字典+KPI 的原料、 + * 每行只有 5 个标量字段(见 {@code DuplicateShop})。给 {@code shops} 再加一层分页,只会把 + * 「全局指标读全集」和「列窗读本页」两套语义搅成一处——那才是 B60 要防的分叉。 + */ + public Map consoleData(DuplicateScanView view, Set visibleKeys, Filters filters, + int page, int pageSize, String dupGroup) { List shops = cropShops(view.payload().shops(), visibleKeys); List cropped = cropItems(view.payload().items(), visibleKeys); String source = view.summary() == null ? "" : safe(view.summary().source()); DuplicateScanSummary summary = recomputeSummary(cropped, shops, source); - List dup = filterItems(cropped, true, filters); + List rows = dupRowsInScope(shops, dupGroup, filterItems(cropped, true, filters)); + rows.sort(Comparator.comparing((DuplicateItem item) -> safe(item.asin()))); List> groups = groupStats(shops, filterItems(cropped, false, filters)); + int total = rows.size(); + int from = Math.min(Math.max(0, page - 1) * pageSize, total); + int to = Math.min(from + pageSize, total); + List window = rows.isEmpty() ? List.of() : new ArrayList<>(rows.subList(from, to)); return mapOf("pending", false, "scanned_at", safe(view.scannedAt()), "summary", summary, - "shops", shops, "dup", dup, "groups", groups, "total_dup", dup.size()); + "shops", shops, "dup", window, "groups", groups, "total_dup", total, + "page", page, "page_size", pageSize); } - public Map pendingConsole() { + /** 分组范围内的撞款行集:按成员店铺裁剪后仍是 ≥2 店的才算撞款。 */ + private List dupRowsInScope(List visibleShops, String dupGroup, + List matched) { + String group = safe(dupGroup).trim(); + if (group.isEmpty()) { + return matched; + } + Set memberKeys = new LinkedHashSet<>(); + for (DuplicateShop shop : visibleShops) { + for (String g : splitGroups(safe(shop.groupName()))) { + if (g.equals(group)) { + memberKeys.add(shopKey(shop.shopName())); + } + } + } + List rows = new ArrayList<>(); + for (DuplicateItem item : cropItems(matched, memberKeys)) { + if (item.shopCount() >= 2) { + rows.add(item); + } + } + return rows; + } + + public Map pendingConsole(int page, int pageSize) { return mapOf("pending", true, "scanned_at", "", "summary", Collections.emptyMap(), "shops", Collections.emptyList(), "dup", Collections.emptyList(), - "groups", Collections.emptyList(), "total_dup", 0); + "groups", Collections.emptyList(), "total_dup", 0, + "page", page, "page_size", pageSize); } /** 单条台账行:品牌取首个非空;店铺/分组/站点保持出现顺序去重;最早/最近上架取归一化时间。 */ diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskFileJobMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskFileJobMapper.java index 6cd69f48c..ec2192606 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskFileJobMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskFileJobMapper.java @@ -5,6 +5,9 @@ import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; +import java.util.Collection; +import java.util.List; + @Mapper public interface TaskFileJobMapper extends BaseMapper { @@ -14,4 +17,14 @@ public interface TaskFileJobMapper extends BaseMapper { * 语句在 {@code mapper/TaskFileJobMapper.xml}(本仓约定:自定义 SQL 一律进 XML)。 */ String selectFileTaskStatus(@Param("taskId") Long taskId); + + /** + * {@link #selectFileTaskStatus} 的批量档(P1-B6):一次问回「状态落在给定终态值域里」的任务 id。 + * + *

终态值域由调用方传(正本仍是 {@code TaskStatuses.FileTask.TERMINAL})——不在 XML 里 + * 再抄一份字符串,否则改值域时两边会悄悄分叉。查不到行的 id 不出现在结果里=非终态, + * 与单条档「品牌任务的 job 返回 null」的语义逐字对齐。 + */ + List selectTerminalFileTaskIds(@Param("taskIds") Collection taskIds, + @Param("terminalStatuses") Collection terminalStatuses); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobBatchQueries.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobBatchQueries.java new file mode 100644 index 000000000..e31140d24 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobBatchQueries.java @@ -0,0 +1,87 @@ +package com.nanri.aiimage.modules.task.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.nanri.aiimage.common.task.TaskStatuses; +import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; + +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * {@code biz_task_file_job} / {@code biz_file_task} 的批量读(P1-B5/P1-B6): + * 一批候选进来,一次往返出 id 集。 + * + *

为什么从 {@link TaskFileJobService} 里单立一支:那支服务已经挂在可读性基线的超长文件 + * 名单上(只许减),而这几条是纯查询——不写状态、不发派发事件、不碰本机在飞表,与服务侧的 + * 写路径没有任何共享可变状态,所以它是静态工具而不是又一支注入进来的 Bean: + * {@code TaskFileJobService} 的构造器已经是别处十几支用例的直接依赖,多加一个参数只会把 + * 无关用例一起改红。 + * + *

把「批量预取」与「逐行 CAS」分成两件事也是有意的:认领的并发正确性只来自逐行 CAS + * ({@code markRunning}),批量预取只用来跳过已终态/已在补检的行;两者混进一个方法, + * 那个前提就再也读不出来了。 + */ +public final class TaskFileJobBatchQueries { + + private TaskFileJobBatchQueries() { + } + + /** + * 这批任务里「还在补检(名下有未终态的组装作业)」的 id(P1-B5)。 + * + *

交付缺口触发的补检轮会把作业排回 PENDING/RUNNING({@link TaskFileJobService#requeue}), + * 所以"有活着的组装作业"="这单还在补检"。而任务行可能早就落成 FAILED(心跳超时或交付判死先落), + * 此时按任务状态去克隆重跑=同一份活开两份:现网 2026-10-24 一枚 760 行的外观专利因此被克隆成 + * 4 枚、每枚都全量重跑一遍,四枚卡片的开始时间还都是同一秒(克隆件继承原件 created_at)。 + * 调用方 {@code TaskResumeService} 就是拿这个集合跳过那一类的。 + * + *

原先这是逐条 {@code selectCount}(N 个候选=N 次往返,P1-B5 点名的那处 N+1);巡检按几百个 + * 候选跑,所以折成一次 {@code selectList} 取 id 列——判据一字未动(同一 jobType、同一两态)。 + */ + public static Set taskIdsWithUnfinishedAssembleJob(TaskFileJobMapper mapper, Collection taskIds) { + if (taskIds == null || taskIds.isEmpty()) { + return Set.of(); + } + List rows = mapper.selectList(new LambdaQueryWrapper() + .select(TaskFileJobEntity::getTaskId) + .in(TaskFileJobEntity::getTaskId, taskIds) + .eq(TaskFileJobEntity::getJobType, TaskFileJobService.JOB_TYPE_ASSEMBLE_RESULT) + .in(TaskFileJobEntity::getStatus, TaskStatuses.Job.PENDING, TaskStatuses.Job.RUNNING)); + if (rows == null || rows.isEmpty()) { + return Set.of(); + } + Set ids = new LinkedHashSet<>(); + for (TaskFileJobEntity row : rows) { + if (row != null && row.getTaskId() != null) { + ids.add(row.getTaskId()); + } + } + return ids; + } + + /** + * 这批候选作业里「父任务已终态」的任务 id(P1-B6):先折出候选的 taskId,再一次批量读。 + * + *

终态值域仍只有 {@code TaskStatuses.FileTask.TERMINAL} 一处正本——它作为参数进 SQL, + * 不在 XML 里另抄一份字符串(抄了就又是第二份词汇表)。单条档判据见 + * {@link TaskFileJobService#isOwningTaskTerminal(Long)},两边同源。 + */ + public static Set terminalOwningTaskIds(TaskFileJobMapper mapper, List candidates) { + Set taskIds = new LinkedHashSet<>(); + if (candidates != null) { + for (TaskFileJobEntity candidate : candidates) { + if (candidate != null && candidate.getTaskId() != null && candidate.getTaskId() > 0) { + taskIds.add(candidate.getTaskId()); + } + } + } + if (taskIds.isEmpty()) { + return Set.of(); + } + List terminal = mapper.selectTerminalFileTaskIds(taskIds, TaskStatuses.FileTask.TERMINAL); + return terminal == null || terminal.isEmpty() ? Set.of() : new LinkedHashSet<>(terminal); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java index 9f45e0b5b..59fbe42d8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java @@ -16,6 +16,7 @@ import org.springframework.transaction.annotation.Transactional; import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -143,22 +144,12 @@ public class TaskFileJobService { } /** - * 该任务名下还有没有**未终态**的组装作业(PENDING/RUNNING)。 - * - *

交付缺口触发的补检轮就是把作业排回这两态({@link #requeue}),所以"有活着的组装作业" - * ="这单还在补检"。而任务行可能早就落成 FAILED(心跳超时或交付判死先落),此时按任务状态 - * 去克隆重跑,等于同一份活开两份:现网 2026-10-24 一枚 760 行的外观专利因此被克隆成 4 枚、 - * 每枚都全量重跑一遍,四枚卡片的开始时间还都是同一秒(克隆件继承原件 created_at)。 + * 这批任务里「还在补检」的 id(P1-B5 的批量档):判据与来由都在 + * {@link TaskFileJobBatchQueries#taskIdsWithUnfinishedAssembleJob},这里只是服务侧的门面—— + * 调用方({@code TaskResumeService})认的是这支服务,不该为了读侧再注入一件东西。 */ - public boolean hasUnfinishedAssembleJob(Long taskId) { - if (taskId == null) { - return false; - } - Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper() - .eq(TaskFileJobEntity::getTaskId, taskId) - .eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT) - .in(TaskFileJobEntity::getStatus, TaskStatuses.Job.PENDING, TaskStatuses.Job.RUNNING)); - return count != null && count > 0; + public Set taskIdsWithUnfinishedAssembleJob(Collection taskIds) { + return TaskFileJobBatchQueries.taskIdsWithUnfinishedAssembleJob(taskFileJobMapper, taskIds); } /** @@ -296,12 +287,15 @@ public class TaskFileJobService { if (candidates == null || candidates.isEmpty()) { return List.of(); } + // P1-B6:父任务状态原本逐条点查(一批 100 个候选=100 次 SELECT),折成一次批量读; + // 逐行 CAS 认领不动——那才是并发不重复派发的根据。 + Set terminalTaskIds = TaskFileJobBatchQueries.terminalOwningTaskIds(taskFileJobMapper, candidates); List claimedIds = new ArrayList<>(); for (TaskFileJobEntity candidate : candidates) { if (candidate == null || candidate.getId() == null) { continue; } - if (isOwningTaskTerminal(candidate.getTaskId())) { + if (terminalTaskIds.contains(candidate.getTaskId())) { noteTerminalTaskStillDeliverable(candidate); } if (markRunning(candidate.getId())) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java index 573516f22..91ec325c1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java @@ -35,6 +35,8 @@ public class TaskResultItemService { */ private final ExecutorService snapshotReadPool = com.nanri.aiimage.common.support.ThreadPools.boundedFixed("task-result-read", 8); + /** 指针化载荷并发读的窗口:见 {@link #listResultSnapshots} 的 P1-B7 说明。 */ + static final int SNAPSHOT_READ_WINDOW = com.nanri.aiimage.common.support.ThreadPools.DEFAULT_QUEUE_CAPACITY / 2; private final ObjectMapper objectMapper; private final TransientPayloadStorageService transientPayloadStorageService; @@ -95,7 +97,11 @@ public class TaskResultItemService { *

刻意不加 limit:调用方要用它组装完整的结果文件,截断即漏行(正确性错误, * 不是性能取舍)。调用方多为「逐任务循环调用」,因此峰值内存 = 单个任务的结果集, * 这是本操作固有的代价。若要再降,需要改成「边读边写文件」的流式组装契约, - * 涉及 4 个模块的组装实现,属独立工程。 + * 涉及 4 个模块的组装实现(patroldelete/queryasin/shopdatacrawl/withdraw),属独立工程。 + * + *

P1-B7 在本方法里能收的是**另一件事**:整份结果集不需要被读两遍。指针化载荷那条分支 + * 按窗口提交+收割(见下方 P1-B7 注释),副本只驻留一窗;返回仍是整份 {@code List}, + * 「流式组装契约」那半没做,别把本段读成 P1-B7 已闭环。 */ public List listResultSnapshots(Long taskId, String moduleType, Class clazz) { if (taskId == null || taskId <= 0 || isBlank(moduleType)) { @@ -129,26 +135,35 @@ public class TaskResultItemService { T value = readPayload(rows.get(0), clazz); return value == null ? List.of() : List.of(value); } - List> futures = new ArrayList<>(rows.size()); - for (TaskResultItemEntity row : rows) { - futures.add(snapshotReadPool.submit(() -> readPayload(row, clazz))); - } + // P1-B7(2026-11-03):**分窗提交并收割**,不再把整份结果集一次性交进线程池。 + // 一次性提交时,每支 Future 完成后仍把解好的载荷扣在自己身上(FutureTask.outcome 不清), + // 而这一批 Future 又要等最后一个才释放 ⇒ 峰值=「结果集」+「同样多的另一份副本」。 + // 一窗一收之后,窗内的副本收完就可回收,峰值退化成「结果集」+「一窗」。 + // 窗口取队列容量的一半:8 线程+512 队列(ThreadPools.DEFAULT_QUEUE_CAPACITY)在窗内永不排满, + // 也就不会落到 CallerRunsPolicy 上把请求线程变成同步读对象存储的那个人。 List out = new ArrayList<>(rows.size()); - for (Future future : futures) { - try { - T value = future.get(); - if (value != null) { - out.add(value); + for (int from = 0; from < rows.size(); from += SNAPSHOT_READ_WINDOW) { + int to = Math.min(from + SNAPSHOT_READ_WINDOW, rows.size()); + List> window = new ArrayList<>(to - from); + for (TaskResultItemEntity row : rows.subList(from, to)) { + window.add(snapshotReadPool.submit(() -> readPayload(row, clazz))); + } + for (Future future : window) { + try { + T value = future.get(); + if (value != null) { + out.add(value); + } + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new BusinessException("读取任务结果明细被中断"); + } catch (ExecutionException ex) { + Throwable cause = ex.getCause(); + if (cause instanceof BusinessException businessException) { + throw businessException; + } + throw new BusinessException("读取任务结果明细失败", cause); } - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - throw new BusinessException("读取任务结果明细被中断"); - } catch (ExecutionException ex) { - Throwable cause = ex.getCause(); - if (cause instanceof BusinessException businessException) { - throw businessException; - } - throw new BusinessException("读取任务结果明细失败", cause); } } return out; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultPayloadService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultPayloadService.java index f58f72034..691590564 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultPayloadService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultPayloadService.java @@ -10,6 +10,8 @@ import lombok.RequiredArgsConstructor; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -95,10 +97,33 @@ public class TaskResultPayloadService { return; } TaskResultPayloadEntity existing = findLatest(taskId, moduleType, hash(scopeKey.trim())); - if (existing != null) { - transientPayloadStorageService.deletePayloadIfPresent(existing.getPayloadJson()); - taskResultPayloadMapper.deleteById(existing.getId()); + if (existing == null) { + return; } + // P1-B8:先落库、提交之后再删对象存储。原先的顺序(存储删完再 deleteById)把一次 + // 网络调用(超时 120s、重试 3 次)连同它的引用计数反查一起塞进 DB 事务:连接被长时间 + // 占住,且回滚时对象已经没了、行还在=读得到一个不存在的载荷。同一批结果明细行 + // (TaskResultItemService#deletePayloadsAfterCommit,2026-09)已经改过这个形状, + // 这里补齐漏掉的这一处;提交后删还有一层好处——引用反查看到的是行已删除的 DB 状态。 + taskResultPayloadMapper.deleteById(existing.getId()); + deletePayloadAfterCommit(existing.getPayloadJson()); + } + + /** 事务提交后再删对象存储上的载荷;无事务上下文时直接删(与 TaskResultItemService 同口径)。 */ + private void deletePayloadAfterCommit(String payloadValue) { + if (payloadValue == null || payloadValue.isBlank()) { + return; + } + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + transientPayloadStorageService.deletePayloadIfPresent(payloadValue); + } + }); + return; + } + transientPayloadStorageService.deletePayloadIfPresent(payloadValue); } private TaskResultPayloadEntity findLatest(Long taskId, String moduleType, String scopeHash) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResumeService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResumeService.java index f34546364..81d9ba427 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResumeService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResumeService.java @@ -13,6 +13,7 @@ import org.springframework.stereotype.Service; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -67,8 +68,8 @@ public class TaskResumeService { private final FileTaskMapper fileTaskMapper; /** - * 查"这单名下还有没有活着的组装作业"——补检轮还活着的意思,见 - * {@link TaskFileJobService#hasUnfinishedAssembleJob(Long)}。 + * 查"这批单名下还有没有活着的组装作业"——补检轮还活着的意思,见 + * {@link TaskFileJobService#taskIdsWithUnfinishedAssembleJob(Collection)}。 */ private final TaskFileJobService taskFileJobs; /** moduleType → 克隆重试实现;只有注册了本 SPI 的模块才会自动重试(SPI 现在只作「支持重试」白名单用)。 */ @@ -201,6 +202,18 @@ public class TaskResumeService { return stats; } stats.scannedTaskCount = candidates.size(); + // P1-B5:下面两条判定原先逐行各问一次库(500 个候选=1,000 次往返,且 + // `resume_of_task_id` 在 V187 之前无索引=每次还是一场全表扫)。折成两次批量查, + // 循环里只做集合判定。原地复位不新增子任务行(2026-11-01 起),所以预取的集合 + // 不会在本轮内被自己写脏。 + Set candidateIds = new LinkedHashSet<>(); + for (FileTaskEntity candidate : candidates) { + if (candidate.getId() != null) { + candidateIds.add(candidate.getId()); + } + } + Set parentsWithChild = parentsWithRetryChild(candidateIds); + Set recheckingTaskIds = taskFileJobs.taskIdsWithUnfinishedAssembleJob(candidateIds); for (FileTaskEntity original : candidates) { if (original.getUserId() == null || original.getUserId() <= 0) { stats.skippedTaskCount++; @@ -226,7 +239,7 @@ public class TaskResumeService { abbreviate(original.getErrorMessage())); continue; } - if (hasRetryChild(original.getId())) { + if (parentsWithChild.contains(original.getId())) { stats.skippedTaskCount++; continue; } @@ -234,7 +247,7 @@ public class TaskResumeService { // (见 AppearancePatentDeliverySupport.requestRecheckForMissingRows),此时克隆=同一份活 // 开两份、还各按全量重跑一遍。作业真卡死另有 TaskResultFileJobWorker 的 stuck 扫描兜底, // 不会因为这条例外就永远没人管。 - if (taskFileJobs.hasUnfinishedAssembleJob(original.getId())) { + if (recheckingTaskIds.contains(original.getId())) { stats.skippedTaskCount++; log.info("[task-resume] 不重试 taskId={} moduleType={} 原因=组装作业仍在补检(PENDING/RUNNING)", original.getId(), original.getModuleType()); @@ -305,11 +318,29 @@ public class TaskResumeService { } } - /** 该原任务是否已经有重试任务(反查 resume_of_task_id)。 */ - private boolean hasRetryChild(Long originalTaskId) { - Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper() - .eq(FileTaskEntity::getResumeOfTaskId, originalTaskId)); - return count != null && count > 0; + /** + * 这批原任务里哪些已经有重试子任务(反查 {@code resume_of_task_id})。 + * + *

P1-B5:判据与原来的单条档逐字相同(同一列、同一等值),只是把 N 次点查折成一次 + * {@code IN} 批查——那列在 V187 之前没有索引,巡检一轮 500 个候选就是 500 场全表扫。 + */ + private Set parentsWithRetryChild(Collection originalTaskIds) { + if (originalTaskIds == null || originalTaskIds.isEmpty()) { + return Set.of(); + } + List children = fileTaskMapper.selectList(new LambdaQueryWrapper() + .select(FileTaskEntity::getResumeOfTaskId) + .in(FileTaskEntity::getResumeOfTaskId, originalTaskIds)); + if (children == null || children.isEmpty()) { + return Set.of(); + } + Set parents = new LinkedHashSet<>(); + for (FileTaskEntity child : children) { + if (child != null && child.getResumeOfTaskId() != null) { + parents.add(child.getResumeOfTaskId()); + } + } + return parents; } /** diff --git a/backend-java/src/main/resources/db/V187__file_task_resume_of_index.sql b/backend-java/src/main/resources/db/V187__file_task_resume_of_index.sql new file mode 100644 index 000000000..691fbcf3a --- /dev/null +++ b/backend-java/src/main/resources/db/V187__file_task_resume_of_index.sql @@ -0,0 +1,41 @@ +-- V187: 自动续跑反查「是否已有重试子任务」的缺失索引(Phase 5 / 审计 P1-B5) +-- +-- 病灶:TaskResumeService 每天真在跑的两条谓词,一条都没有索引可用—— +-- 1) `WHERE resume_of_task_id = ?`(hasRetryChild,原判「这单已经重试过了」) +-- V129 只加了 resume_of_task_id / resume_attempt 两列(:26、:38),**一支索引都没配**, +-- 于是巡检每轮对每个候选单各做一次全表扫;500 个候选=500 次全表扫, +-- 而这张表是 biz_file_task(现网十万级行)。 +-- 2) `WHERE resume_of_task_id IS NOT NULL AND status IN (活态)`(stopDuplicateChildren 幂等清账) +-- 同一列的另一条谓词,同样裸扫。 +-- 两条共用一条 `(resume_of_task_id)` 即可:InnoDB 的二级索引带主键尾巴,IS NOT NULL 那支 +-- 也能走索引范围扫(不是全表)。 +-- +-- 有意**不加**的另一支(审计建议的 `(status, module_type, resume_attempt, updated_at)`): +-- 候选集那条查询的等值前缀 (status, module_type) 已由 V184 的 +-- idx_biz_file_task_status_module_created 覆盖;`resume_attempt IS NULL OR resume_attempt < n` +-- 是 OR/IS-NULL 残差谓词(不吃索引,只能当过滤条件),`COALESCE(finished_at, updated_at) >= ?` +-- 是表达式谓词(同样不吃索引,见 V184 头注释对 collect_channel 的同一口径)。 +-- 再建一条同前缀索引=只写不读、白白摊在每次 INSERT/UPDATE 上——与 V186「别让一条推测性 +-- 索引进基线」同判。真库 EXPLAIN 若显示候选集这一支仍需回表扫太多行,抬时间列入索引 +-- 留到现网复量之后单独立项(本机无 MySQL,未跑过 EXPLAIN,见蓝图 Phase 5 实录)。 +-- +-- 风险:ALTER TABLE ADD INDEX 需申请元数据锁(MySQL 8 在线 DDL,INPLACE),建议低峰执行; +-- 本迁移按 information_schema 判存,重跑不会因索引已存在而失败。 +-- 回滚(手工): +-- ALTER TABLE biz_file_task DROP INDEX idx_biz_file_task_resume_of; + +SET @db_name = DATABASE(); + +-- biz_file_task(resume_of_task_id) +SET @idx_exists := ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_task' + AND INDEX_NAME = 'idx_biz_file_task_resume_of' +); +SET @sql := IF(@idx_exists = 0, + 'ALTER TABLE biz_file_task ADD INDEX idx_biz_file_task_resume_of (resume_of_task_id)', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/backend-java/src/main/resources/mapper/TaskFileJobMapper.xml b/backend-java/src/main/resources/mapper/TaskFileJobMapper.xml index fe9554842..110f3a8f2 100644 --- a/backend-java/src/main/resources/mapper/TaskFileJobMapper.xml +++ b/backend-java/src/main/resources/mapper/TaskFileJobMapper.xml @@ -7,4 +7,14 @@ SELECT status FROM biz_file_task WHERE id = #{taskId} + + + diff --git a/backend-java/src/test/java/com/nanri/aiimage/MigratedSqlEquivalenceTest.java b/backend-java/src/test/java/com/nanri/aiimage/MigratedSqlEquivalenceTest.java index 93bdf6265..684c09319 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/MigratedSqlEquivalenceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/MigratedSqlEquivalenceTest.java @@ -76,7 +76,10 @@ class MigratedSqlEquivalenceTest { // NOT_MIGRATED(select 总数不变,只是记账位置换了)。 // 100 → 102:P1-1 把两条认领查询的通道谓词下推进 SQL,FileTaskMapper 新写 2 条 select, // 逐条点名进 NOT_MIGRATED(SqlLayeringConventionTest.XML_STATEMENT_TOTAL 同轮 144 → 146)。 - private static final int XML_SELECT_TOTAL = 102; + // 102 → 103:P1-B6 把 claim 循环的「父任务是否终态」折成批量档 + // TaskFileJobMapper#selectTerminalFileTaskIds,点名进下方 NOT_MIGRATED + // (SqlLayeringConventionTest.XML_STATEMENT_TOTAL 同轮 146 → 147)。 + private static final int XML_SELECT_TOTAL = 103; /** * 不是「从注解搬过来」的语句:注解时代没有原文可对撞,等价性这支只能跳过它们。 @@ -118,6 +121,9 @@ class MigratedSqlEquivalenceTest { "UserSecretUsageMapper#sumForUser", // B28(2026-10-25)新写的 1 条主键点查:父任务是否已终态。 "TaskFileJobMapper#selectFileTaskStatus", + // P1-B6(2026-11-03)新写的 1 条批量档:一次问回这批父任务里已终态的 id + // (终态值域由调用方传入,XML 里没有第二份字符串)。 + "TaskFileJobMapper#selectTerminalFileTaskIds", // CR-01(2026-10-25)新写的 1 条写侧语句:会话世代原子自增。 "LoginUserMapper#incrementSessionEpoch", // 关键词使用记录管理端(2026-10-25)新写的 10 条只读 select。 diff --git a/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java b/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java index c7f2f55d3..31d97e238 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java @@ -95,7 +95,10 @@ class SqlLayeringConventionTest { // 原形状是先 limit 取页再 filter(isLocal()),混通道时较老的 N 条是 ONLINE 就把本机单永久饿死。 // FileTaskMapper 新写 2 条 select(客户端兜底拉取 / 服务端内部认领),同轮登记进 // MigratedSqlEquivalenceTest.NOT_MIGRATED(该文件 XML_SELECT_TOTAL 同轮 100 → 102)。 - private static final int XML_STATEMENT_TOTAL = 146; + // 146 → 147:P1-B6 把 claim 循环里「父任务是否终态」的逐条点查折成批量档 + // {@code TaskFileJobMapper#selectTerminalFileTaskIds}(终态值域由调用方传入,XML 不抄字符串), + // 同轮登记进 MigratedSqlEquivalenceTest.NOT_MIGRATED(该文件 XML_SELECT_TOTAL 同轮 102 → 103)。 + private static final int XML_STATEMENT_TOTAL = 147; /** * wrapper 里夹带函数级 SQL 的现状("这不是简单单表"的证据),文件名 → 条数: diff --git a/backend-java/src/test/java/com/nanri/aiimage/config/TaskResumeIndexMigrationV187Test.java b/backend-java/src/test/java/com/nanri/aiimage/config/TaskResumeIndexMigrationV187Test.java new file mode 100644 index 000000000..71798bdb2 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/config/TaskResumeIndexMigrationV187Test.java @@ -0,0 +1,113 @@ +package com.nanri.aiimage.config; + +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * V187 给 {@code biz_file_task} 补 {@code resume_of_task_id} 索引,本类钉它的形状(Phase 5/审计 P1-B5)。 + * + *

来历:自动续跑巡检每轮对每个候选单各做一次「这单是否已有重试子任务」的反查,而那列在 + * V129 只加了列、一支索引都没配 ⇒ 500 个候选=500 场全表扫。同族的「幂等清账」用的是同一列。 + * + *

本类同时钉住**不建**的那一支:审计原文建议 {@code (status, module_type, resume_attempt, updated_at)}, + * 但候选集查询的等值前缀已由 V184 覆盖,而 {@code resume_attempt} 是 OR/IS-NULL 残差谓词、 + * {@code COALESCE(finished_at, updated_at)} 是表达式谓词——两者都不吃索引。再建一条同前缀索引 + * =只写不读,白白摊在 biz_file_task 每次写入上(V186 头注释「别让一条推测性索引进基线」同判)。 + * 谁要拓宽这条口径,先让这里红一次。 + */ +class TaskResumeIndexMigrationV187Test { + + private static final Path SQL = Path.of(System.getProperty("user.dir"), + "src/main/resources/db/V187__file_task_resume_of_index.sql"); + + private static final Path RESUME_SERVICE = Path.of(System.getProperty("user.dir"), + "src/main/java/com/nanri/aiimage/modules/task/service/TaskResumeService.java"); + + private static String sql() throws IOException { + assertTrue(Files.exists(SQL), "V187 迁移脚本不在:" + SQL); + return Files.readString(SQL); + } + + @Test + void 只给反查列建一支索引且按判存包装() throws IOException { + String text = sql(); + + assertTrue(text.contains("ALTER TABLE biz_file_task ADD INDEX idx_biz_file_task_resume_of (resume_of_task_id)"), + "迁移对象与列名必须逐字对上:反查用的就是 resume_of_task_id"); + assertTrue(text.contains("information_schema.STATISTICS"), + "必须判存后再建:重跑(或已手工建过)不能因 Duplicate key name 把启动掐断"); + assertTrue(text.contains("PREPARE stmt FROM @sql"), + "判存要落到动态执行,光 SELECT 出计数不改表等于什么都没做"); + assertEquals(1, occurrences(text, "ADD INDEX idx_biz_file_task_resume_of"), + "整份脚本只许建这一支索引"); + assertTrue(text.contains("DROP INDEX idx_biz_file_task_resume_of"), + "回滚语句必须写在脚本注释里(与 V184/V186 同一口径)"); + } + + @Test + void 有意不建候选集那一支同前缀索引() throws IOException { + String text = sql(); + // 只看真正会执行的行:头注释里必须写下 resume_attempt/V184 的取舍理由,那是文档不是 DDL。 + StringBuilder executableText = new StringBuilder(); + for (String line : text.split("\n")) { + String trimmed = line.trim(); + if (!trimmed.startsWith("--")) { + executableText.append(trimmed).append('\n'); + } + } + String executable = executableText.toString().toUpperCase(java.util.Locale.ROOT); + + assertFalse(executable.contains("RESUME_ATTEMPT"), + "resume_attempt 是 OR/IS-NULL 残差谓词,不进索引;要加必须先拿出真库 EXPLAIN 读数"); + assertFalse(executable.contains("MODULE_TYPE"), + "(status, module_type, resume_attempt) 那支同前缀索引有意不建:等值前缀已由 V184 覆盖"); + assertFalse(executable.contains("CREATE INDEX"), + "本仓加索引一律走 ALTER TABLE ... ADD INDEX(判存包装的形状)"); + assertTrue(text.contains("idx_biz_file_task_status_module_created"), + "「等值前缀已由 V184 覆盖」这条判断的出处必须写在头注释里,否则下一个人会当成漏配"); + } + + @Test + void 索引列与实体字段同源() throws IOException { + // MyBatis-Plus 靠驼峰↔下划线映射:字段名改了而索引列名没改,索引就成了永远命不中的摆设, + // 而巡检的反查照样跑得动——只是每次都全表扫。 + assertTrue(java.util.Arrays.stream(FileTaskEntity.class.getDeclaredFields()) + .anyMatch(field -> field.getName().equals("resumeOfTaskId")), + "实体里必须有 resumeOfTaskId 这个字段;反查一旦改名,V187 的索引与谓词会分叉"); + assertTrue(sql().contains("ADD INDEX idx_biz_file_task_resume_of (resume_of_task_id)"), + "索引列名必须就是这个字段的下划线形(同源),别让它指向一列不存在的历史名"); + } + + @Test + void 巡检里不留逐行反查的形状() throws IOException { + assertTrue(Files.exists(RESUME_SERVICE), "续跑服务源码不在:" + RESUME_SERVICE); + String text = Files.readString(RESUME_SERVICE); + + assertFalse(text.contains("private boolean hasRetryChild("), + "P1-B5 的病灶就是逐行反查;改回单条档=每候选一次全表扫,退回原状即红"); + assertFalse(text.contains("hasUnfinishedAssembleJob(original.getId())"), + "补检判定同样必须走批量档,不许在循环里逐行问库"); + assertTrue(text.contains("parentsWithRetryChild(candidateIds)"), + "循环前一次批量反查"); + assertTrue(text.contains("taskIdsWithUnfinishedAssembleJob(candidateIds)"), + "循环前一次批量补检判定"); + } + + private static int occurrences(String text, String needle) { + int count = 0; + int index = text.indexOf(needle); + while (index >= 0) { + count++; + index = text.indexOf(needle, index + needle.length()); + } + return count; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryServiceTest.java index d7ff659c4..2e0a3c093 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryServiceTest.java @@ -156,7 +156,7 @@ class ShopDataDuplicateCheckQueryServiceTest { @Test void consoleSuperAdminReturnsDupSetGroupsAndSummary() { - Map data = service.consoleData(view, null, noFilter()); + Map data = service.consoleData(view, null, noFilter(), 1, 100, ""); assertThat(data.get("pending")).isEqualTo(false); assertThat(data.get("total_dup")).isEqualTo(3); @SuppressWarnings("unchecked") @@ -184,7 +184,7 @@ class ShopDataDuplicateCheckQueryServiceTest { @Test void consoleLeaderCropRemovesCrossGroupDupAndKeepsOwnGroups() { Set visible = Set.of("shopa", "shopb"); - Map data = service.consoleData(view, visible, noFilter()); + Map data = service.consoleData(view, visible, noFilter(), 1, 100, ""); assertThat(data.get("total_dup")).isEqualTo(2); @SuppressWarnings("unchecked") List dup = (List) data.get("dup"); @@ -201,11 +201,77 @@ class ShopDataDuplicateCheckQueryServiceTest { @Test void consolePendingShapes() { - Map data = service.pendingConsole(); + Map data = service.pendingConsole(1, 100); assertThat(data.get("pending")).isEqualTo(true); assertThat(data.get("dup")).isEqualTo(List.of()); assertThat(data.get("groups")).isEqualTo(List.of()); assertThat(data.get("summary")).isEqualTo(Map.of()); + assertThat(data.get("page")).isEqualTo(1); + assertThat(data.get("page_size")).isEqualTo(100); + } + + /** + * B60:console 的 {@code dup} 从「全集」改成「行窗口」。分页参数必须真生效—— + * 响应体行数 ≤ pageSize,而 {@code total_dup} 仍是未分页的总数(KPI/分页控件读它)。 + */ + @Test + void consoleRowWindowLimitsRowsButKeepsFullTotal() { + Map data = service.consoleData(view, null, noFilter(), 1, 2, ""); + assertThat(data.get("total_dup")).isEqualTo(3); + assertThat(data.get("page")).isEqualTo(1); + assertThat(data.get("page_size")).isEqualTo(2); + @SuppressWarnings("unchecked") + List dup = (List) data.get("dup"); + assertThat(dup).hasSize(2); + } + + /** + * 行窗口按 ASIN 升序稳定:把窗口翻到第二页必须拿到上一页之外的行, + * 而不是每次刷新换一批(排序发生在切窗之前,这是分页可用的前提)。 + */ + @Test + void consoleRowWindowsAreStableAcrossPages() { + @SuppressWarnings("unchecked") + List first = (List) + service.consoleData(view, null, noFilter(), 1, 2, "").get("dup"); + @SuppressWarnings("unchecked") + List second = (List) + service.consoleData(view, null, noFilter(), 2, 2, "").get("dup"); + assertThat(first).extracting(DuplicateItem::asin).containsExactly("A0000001", "E0000001"); + assertThat(second).extracting(DuplicateItem::asin).containsExactly("F0000001"); + } + + /** + * 分组范围(页面的「分组」chip)由服务端裁行:判据=组内 ≥2 店。 + * F0000001 是 ShopB(GroupA)×ShopD(GroupD) 的跨组撞款,落在 GroupA 范围内不算组内撞款。 + * + *

只裁行:summary/shops/groups 三块必须与不带 dup_group 的那次逐字相同—— + * 指标跟着行窗口一起缩的话,页面顶部数字就成了「这一页的统计」,那是另一件事。 + */ + @Test + void consoleDupGroupScopesRowsOnlyAndLeavesStatsAlone() { + Map scoped = service.consoleData(view, null, noFilter(), 1, 100, "GroupA"); + @SuppressWarnings("unchecked") + List rows = (List) scoped.get("dup"); + assertThat(scoped.get("total_dup")).isEqualTo(2); + assertThat(rows).extracting(DuplicateItem::asin).containsExactly("A0000001", "E0000001"); + + Map plain = service.consoleData(view, null, noFilter(), 1, 100, ""); + assertThat(scoped.get("summary")).isEqualTo(plain.get("summary")); + assertThat(scoped.get("shops")).isEqualTo(plain.get("shops")); + assertThat(scoped.get("groups")).isEqualTo(plain.get("groups")); + } + + /** 组内不足 2 店 ⇒ 行集为空(不是「退回全集」);组名不存在同理。 */ + @Test + void consoleDupGroupWithoutPairReturnsEmptyWindow() { + Map singleShopGroup = service.consoleData(view, null, noFilter(), 1, 100, "GroupC"); + assertThat(singleShopGroup.get("total_dup")).isEqualTo(0); + assertThat((List) singleShopGroup.get("dup")).isEmpty(); + + Map unknownGroup = service.consoleData(view, null, noFilter(), 1, 100, "不存在组"); + assertThat(unknownGroup.get("total_dup")).isEqualTo(0); + assertThat((List) unknownGroup.get("dup")).isEmpty(); } @Test diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobBatchQueryGuardrailTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobBatchQueryGuardrailTest.java new file mode 100644 index 000000000..e6bc39e88 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobBatchQueryGuardrailTest.java @@ -0,0 +1,189 @@ +package com.nanri.aiimage.modules.task.service; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.nanri.aiimage.common.task.TaskStatuses; +import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.ApplicationEventPublisher; + +import java.time.LocalDateTime; +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * P1-B6/P1-B5:认领循环与补检判定都必须走批量档,不许退回逐行点查。 + * + *

来历:{@code claimCandidates} 原先对每个候选各做一次 {@code selectFileTaskStatus}(一批 100 + * 个候选=100 次 SELECT),而 P1-13 之后这份结果**只用来打一行日志**——终态父任务的出表作业照旧放行。 + * 纯开销的 N 次往返折成一次 {@code IN} 批查。逐行 CAS 认领({@code markRunning})一字不动: + * 那才是并发不重复派发的根据,不能被"顺手批量化"掉。 + * + *

反向自证:把批量档改回循环里逐条 {@code isOwningTaskTerminal},本类第一格即红。 + * 批量档的本体在 {@link TaskFileJobBatchQueries}(静态只读工具),{@code TaskFileJobService} + * 只留一个同名门面——门面不许长回逻辑,否则服务侧又变超长文件。 + */ +@ExtendWith(MockitoExtension.class) +class TaskFileJobBatchQueryGuardrailTest { + + @Mock private TaskFileJobMapper taskFileJobMapper; + @Mock private ApplicationEventPublisher applicationEventPublisher; + + private TaskFileJobService service; + + @BeforeAll + static void initializeTableInfo() { + TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), + TaskFileJobEntity.class); + } + + @BeforeEach + void setUp() { + service = new TaskFileJobService(taskFileJobMapper, applicationEventPublisher); + } + + private static TaskFileJobEntity job(Long id, Long taskId, String status) { + TaskFileJobEntity entity = new TaskFileJobEntity(); + entity.setId(id); + entity.setTaskId(taskId); + entity.setResultId(2000L + id); + entity.setModuleType("APPEARANCE_PATENT"); + entity.setJobType("ASSEMBLE_RESULT"); + entity.setStatus(status); + entity.setRetryCount(0); + entity.setUpdatedAt(LocalDateTime.now()); + return entity; + } + + @SuppressWarnings("unchecked") + private static ArgumentCaptor captorOf(Class type) { + return (ArgumentCaptor) ArgumentCaptor.forClass(type); + } + + @Test + void 一批候选只问一次父任务状态() { + when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))) + .thenReturn(List.of(job(21L, 500L, "PENDING"), job(22L, 500L, "PENDING"), job(23L, 501L, "FAILED"))); + when(taskFileJobMapper.selectTerminalFileTaskIds(any(), any())).thenReturn(List.of(500L)); + when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1, 1, 1); + when(taskFileJobMapper.selectBatchIds(any())) + .thenReturn(List.of(job(21L, 500L, "RUNNING"), job(22L, 500L, "RUNNING"), job(23L, 501L, "RUNNING"))); + + List claimed = service.claimRunnableJobs(10); + + assertEquals(3, claimed.size()); + ArgumentCaptor> ids = captorOf(Collection.class); + ArgumentCaptor> statuses = captorOf(Collection.class); + verify(taskFileJobMapper, times(1)).selectTerminalFileTaskIds(ids.capture(), statuses.capture()); + assertEquals(List.of(500L, 501L), List.copyOf(ids.getValue()), + "同一父任务的多个作业必须去重后一次问回;出现两次以上调用就是退回逐行点查"); + verify(taskFileJobMapper, never()).selectFileTaskStatus(any()); + } + + /** 终态值域的唯一正本仍是 {@code TaskStatuses}:它作为参数进 SQL,XML 里不许抄第二份字符串。 */ + @Test + void 终态值域取自词表单点() { + when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))) + .thenReturn(List.of(job(21L, 500L, "PENDING"))); + when(taskFileJobMapper.selectTerminalFileTaskIds(any(), any())).thenReturn(List.of()); + when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(job(21L, 500L, "RUNNING"))); + + service.claimRunnableJobs(10); + + ArgumentCaptor> statuses = captorOf(Collection.class); + verify(taskFileJobMapper).selectTerminalFileTaskIds(any(), statuses.capture()); + assertEquals(TaskStatuses.FileTask.TERMINAL, Set.copyOf(statuses.getValue()), + "批量档的终态值域必须逐字等于 TaskStatuses.FileTask.TERMINAL"); + } + + @Test + void 空候选不发批量查() { + when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of()); + + assertEquals(List.of(), service.claimRunnableJobs(10)); + + verify(taskFileJobMapper, never()).selectTerminalFileTaskIds(any(), any()); + verify(taskFileJobMapper, never()).selectFileTaskStatus(any()); + } + + /** 查不到行的 taskId(品牌任务的 job)按非终态走,与单条档「返回 null ⇒ 非终态」逐字同语义。 */ + @Test + void 查不到行的任务照常被认领() { + when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))) + .thenReturn(List.of(job(31L, 900L, "PENDING"))); + when(taskFileJobMapper.selectTerminalFileTaskIds(any(), any())).thenReturn(List.of()); + when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); + when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(job(31L, 900L, "RUNNING"))); + + List claimed = service.claimRunnableJobs(10); + + assertEquals(1, claimed.size()); + assertEquals("RUNNING", claimed.get(0).getStatus()); + } + + @Test + void 补检批量档的空入参不发SQL() { + assertEquals(Set.of(), service.taskIdsWithUnfinishedAssembleJob(List.of())); + assertEquals(Set.of(), service.taskIdsWithUnfinishedAssembleJob(null)); + assertEquals(Set.of(), TaskFileJobBatchQueries.terminalOwningTaskIds(taskFileJobMapper, List.of())); + assertEquals(Set.of(), TaskFileJobBatchQueries.terminalOwningTaskIds(taskFileJobMapper, null)); + + verify(taskFileJobMapper, never()).selectList(any(LambdaQueryWrapper.class)); + verify(taskFileJobMapper, never()).selectTerminalFileTaskIds(any(), any()); + } + + /** + * 批量档与它替代的单条档必须同判据:同一 jobType、同两个未终态状态。 + * 分叉的形状是「批量档少筛一项」=把已完成的作业也当成在补检,续跑会被永久饿死。 + */ + @Test + void 补检批量档与单条档判据同源() { + when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))) + .thenReturn(List.of(job(41L, 700L, "PENDING"))); + + assertEquals(Set.of(700L), service.taskIdsWithUnfinishedAssembleJob(List.of(700L, 701L))); + + ArgumentCaptor> captured = captorOf(LambdaQueryWrapper.class); + verify(taskFileJobMapper).selectList(captured.capture()); + // MP 把参数装配推迟到生成 SQL 那一刻;不先取一次 sqlSegment,paramNameValuePairs 是空的。 + String where = captured.getValue().getSqlSegment(); + assertTrue(where.contains("task_id IN") || where.contains("task_id in"), + "批量档必须按 task_id 一整批问,实参段=" + where); + java.util.Set values = new java.util.HashSet<>(); + for (Object value : captured.getValue().getParamNameValuePairs().values()) { + if (value instanceof java.util.Collection nested) { + values.addAll(nested); + } else { + values.add(value); + } + } + assertTrue(values.contains("ASSEMBLE_RESULT"), "批量档必须仍按 ASSEMBLE_RESULT 过滤,实参=" + values + " sql=" + where); + assertTrue(values.contains("PENDING") && values.contains("RUNNING"), + "批量档必须仍只看 PENDING/RUNNING 两态,实参=" + values); + assertTrue(values.contains(700L) && values.contains(701L), + "批量档必须把这批 taskId 一起带进 IN,实参=" + values); + assertTrue(String.valueOf(captured.getValue().getSqlSelect()).contains("task_id"), + "批量档只取 task_id 投影,别把整行拉回来"); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobTerminalTaskGuardTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobTerminalTaskGuardTest.java index 3affe0f4b..3d96526bd 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobTerminalTaskGuardTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobTerminalTaskGuardTest.java @@ -107,8 +107,8 @@ class TaskFileJobTerminalTaskGuardTest { TaskFileJobEntity active = job(22L, 501L); when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))) .thenReturn(List.of(terminal, active)); - when(taskFileJobMapper.selectFileTaskStatus(500L)).thenReturn("FAILED"); - when(taskFileJobMapper.selectFileTaskStatus(501L)).thenReturn("RUNNING"); + // P1-B6:claim 循环的父任务状态改为一次批量档(500=500L 已终态)。 + when(taskFileJobMapper.selectTerminalFileTaskIds(any(), any())).thenReturn(List.of(500L)); when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1, 1); TaskFileJobEntity claimedTerminal = job(21L, 500L); claimedTerminal.setStatus("RUNNING"); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultItemReadWindowTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultItemReadWindowTest.java new file mode 100644 index 000000000..17f6d8cbc --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultItemReadWindowTest.java @@ -0,0 +1,208 @@ +package com.nanri.aiimage.modules.task.service; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.support.ThreadPools; +import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity; +import com.nanri.aiimage.testutil.SourceScan; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * P1-B7:指针化载荷的结果集读取必须**分窗提交并收割**,同时整份一行不许漏。 + * + *

来历:审计把 {@code listResultSnapshots} 的「无 limit」记成性能项,但**截断是正确性错误不是性能取舍** + * ——调用方(patroldelete/queryasin/shopdatacrawl/withdraw)拿它组装完整结果文件。真正能收的是 + * 「整份结果集被读两遍」:原先把 N 行一次性交进线程池,每支 Future 完成后把解好的载荷扣在自己身上 + * ({@code FutureTask.outcome} 不清),而这一整批要到收割完才释放 ⇒ 峰值=结果集+同样多的另一份副本。 + * 分窗之后副本只驻留一窗。 + * + *

判据为什么打在源码上而不是只打行为:窗口效应体现在**堆**上(同时在飞的载荷数), + * 线程池本身只有 8 个线程,"同时开始读几行"这个observable 在改前改后都是 ≤8,用 mock 数不出来; + * 行为侧能钉住的只有「不漏行、不乱序、异常口径不变」这三件——那些恰恰是拆窗最容易被改坏的地方, + * 所以两类都要,缺一个就是半只眼睛。 + * + *

反向自证:把窗口改回 {@code for (row : rows) futures.add(submit(...))} 整量一次性提交, + * 只有源码绊线那一格红(实测 6 例中 1 失败),行为四格照样全绿——窗口化本来就不改可观察行为, + * 这恰恰是上面那段「行为侧钉不住它」的实证,不是本类在凑数。 + */ +@ExtendWith(MockitoExtension.class) +class TaskResultItemReadWindowTest { + + @Mock private TaskResultItemMapper taskResultItemMapper; + @Mock private TransientPayloadStorageService transientPayloadStorageService; + + private TaskResultItemService service; + + @BeforeAll + static void initializeTableInfo() { + TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), ""), + TaskResultItemEntity.class); + } + + @BeforeEach + void setUp() { + service = new TaskResultItemService(taskResultItemMapper, new ObjectMapper(), transientPayloadStorageService); + } + + /** 指针化载荷行:payloadJson 只放指针形状,真正的内容由 resolvePayload 现编。 */ + private static List pointerRows(int count) { + List rows = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + TaskResultItemEntity row = new TaskResultItemEntity(); + row.setId((long) i + 1); + row.setTaskId(9001L); + row.setModuleType("APPEARANCE_PATENT"); + row.setResultId((long) i + 1); + row.setItemKey("result:" + (i + 1)); + row.setPayloadJson("pointer-" + i); + rows.add(row); + } + return rows; + } + + /** 从指针串里还原行号:让第 i 行的载荷就是 {"n":i},顺序对不对一眼可见。 */ + private static int indexOfArgument(String pointer) { + return Integer.parseInt(pointer.substring("pointer-".length())); + } + + // ---------- 行为:拆窗不许改掉「整份、按序、口径不变」 ---------- + + @Test + void crossWindowResultIsCompleteAndKeepsRowOrder() { + int total = TaskResultItemService.SNAPSHOT_READ_WINDOW * 2 + 3; + when(taskResultItemMapper.selectList(any())).thenReturn(pointerRows(total)); + when(transientPayloadStorageService.extractPointer(anyString())).thenReturn("pointer"); + when(transientPayloadStorageService.resolvePayload(anyString(), anyString())) + .thenAnswer(inv -> "{\"n\":" + indexOfArgument(inv.getArgument(0)) + "}"); + + List out = service.listResultSnapshots(9001L, "APPEARANCE_PATENT", Map.class); + + assertThat(out).as("跨了三个窗口也不能少一行").hasSize(total); + for (int i = 0; i < total; i++) { + assertThat(out.get(i).get("n")).as("第 " + i + " 行的位置").isEqualTo(i); + } + verify(transientPayloadStorageService, times(total)).resolvePayload(anyString(), anyString()); + } + + @Test + void blankPayloadSkipsOnlyThatRowAndDoesNotTruncateTheRest() { + int total = TaskResultItemService.SNAPSHOT_READ_WINDOW + 5; + List rows = pointerRows(total); + rows.get(3).setPayloadJson(""); // 首窗内的一行空载荷 + rows.get(total - 1).setPayloadJson(" "); // 末窗内的一行空载荷 + when(taskResultItemMapper.selectList(any())).thenReturn(rows); + when(transientPayloadStorageService.extractPointer(anyString())).thenReturn("pointer"); + when(transientPayloadStorageService.resolvePayload(anyString(), anyString())) + .thenAnswer(inv -> "{\"n\":" + indexOfArgument(inv.getArgument(0)) + "}"); + + List out = service.listResultSnapshots(9001L, "APPEARANCE_PATENT", Map.class); + + assertThat(out).as("空载荷只跳这一行,后面的行必须照样来").hasSize(total - 2); + assertThat(out).extracting(m -> m.get("n")) + .containsExactlyElementsOf(rows.stream() + .filter(r -> r.getPayloadJson() != null && !r.getPayloadJson().trim().isEmpty()) + .map(r -> indexOfArgument(r.getPayloadJson())).toList()); + } + + @Test + void businessFailureFromTheStoreStillSurfacesUnchanged() { + int total = TaskResultItemService.SNAPSHOT_READ_WINDOW + 2; + when(taskResultItemMapper.selectList(any())).thenReturn(pointerRows(total)); + when(transientPayloadStorageService.extractPointer(anyString())).thenReturn("pointer"); + when(transientPayloadStorageService.resolvePayload(anyString(), anyString())) + .thenAnswer(inv -> "{\"n\":" + indexOfArgument(inv.getArgument(0)) + "}"); + when(transientPayloadStorageService.resolvePayload(org.mockito.ArgumentMatchers.eq("pointer-1"), anyString())) + .thenThrow(new BusinessException("读取任务结果明细失败")); + + BusinessException thrown = assertThrows(BusinessException.class, + () -> service.listResultSnapshots(9001L, "APPEARANCE_PATENT", Map.class)); + + assertThat(thrown).hasMessage("读取任务结果明细失败"); + } + + @Test + void unexpectedStoreFailureIsWrappedIntoBusinessExceptionAsBefore() { + when(taskResultItemMapper.selectList(any())).thenReturn(pointerRows(TaskResultItemService.SNAPSHOT_READ_WINDOW + 2)); + when(transientPayloadStorageService.extractPointer(anyString())).thenReturn("pointer"); + when(transientPayloadStorageService.resolvePayload(anyString(), anyString())) + .thenThrow(new IllegalStateException("对象存储 500")); + + BusinessException thrown = assertThrows(BusinessException.class, + () -> service.listResultSnapshots(9001L, "APPEARANCE_PATENT", Map.class)); + + assertThat(thrown).hasMessage("读取任务结果明细失败"); + } + + // ---------- 源码绊线:窗口这道闸本身 ---------- + + @Test + void submissionIsWindowBoundedNotWholeResultSetAtOnce() { + String body = methodBody(SourceScan.withoutComments(SourceScan.sourceText("TaskResultItemService.java")), + "List listResultSnapshots(Long taskId, String moduleType, Class clazz)"); + + assertThat(body).as("提交循环必须由窗口常量钳制").contains("from += SNAPSHOT_READ_WINDOW"); + assertThat(body).as("每次只提交一窗的行").contains("Math.min(from + SNAPSHOT_READ_WINDOW"); + assertThat(body.indexOf("from += SNAPSHOT_READ_WINDOW")) + .as("submit 必须落在窗口循环里面,不是回到整量 for (row : rows)") + .isLessThan(body.indexOf("snapshotReadPool.submit(")); + assertThat(countOf(body, "snapshotReadPool.submit(")) + .as("提交点只许一处:多一处就多一份没人收割的副本").isEqualTo(1); + } + + @Test + void windowMustStayBelowThePoolQueueSoTheCallerNeverReadsInline() { + assertThat(TaskResultItemService.SNAPSHOT_READ_WINDOW) + .as("窗口大于线程池队列就会撞上 CallerRunsPolicy,把调用线程变成同步读对象存储的那个人") + .isBetween(1, ThreadPools.DEFAULT_QUEUE_CAPACITY); + } + + /** 从方法签名起做括号配对,取出方法体(注释已由 withoutComments 剥掉)。 */ + private static String methodBody(String source, String signature) { + int start = source.indexOf(signature); + assertThat(start).as("必须存在方法 " + signature).isGreaterThan(-1); + int open = source.indexOf('{', start); + assertThat(open).as(signature + " 的方法体起点").isGreaterThan(-1); + int depth = 0; + for (int i = open; i < source.length(); i++) { + char c = source.charAt(i); + if (c == '{') { + depth++; + } else if (c == '}') { + depth--; + if (depth == 0) { + return source.substring(open + 1, i); + } + } + } + throw new AssertionError(signature + " 括号未闭合"); + } + + private static int countOf(String haystack, String needle) { + int count = 0; + for (int at = haystack.indexOf(needle); at >= 0; at = haystack.indexOf(needle, at + needle.length())) { + count++; + } + return count; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultPayloadDeleteTxBoundaryTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultPayloadDeleteTxBoundaryTest.java new file mode 100644 index 000000000..17c162010 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultPayloadDeleteTxBoundaryTest.java @@ -0,0 +1,143 @@ +package com.nanri.aiimage.modules.task.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.modules.task.mapper.TaskResultPayloadMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * P1-B8:结果载荷的对象存储删除不得落在 DB 事务里。 + * + *

原形状是 {@code @Transactional} 的 {@code deleteLatest} 里先同步删对象、再删行。两头的代价都真: + * 一次删除超时 120s、重试 3 次,还要带一次引用计数反查——DB 连接被这段时间占住(连接池单节点 30, + * 几次并发删除即可打满并波及其它请求);而回滚时对象已经没了、行还在=读得到一个不存在的载荷。 + * 结果明细行({@link TaskResultItemService} 的 deletePayloadsAfterCommit,2026-09)早已改成提交后删, + * 本类钉的是当时漏掉的这一处。 + * + *

反向自证:把 {@code deletePayloadIfPresent} 挪回事务内(或去掉 afterCommit 分支),第一格即红; + * 把顺序倒回来(先删对象再删行),第三格即红。 + */ +@ExtendWith(MockitoExtension.class) +class TaskResultPayloadDeleteTxBoundaryTest { + + private static final String POINTER = "rustfs:task-payload/APPEARANCE_PATENT/9001/abc.json"; + + @Mock private TaskResultPayloadMapper taskResultPayloadMapper; + @Mock private TransientPayloadStorageService transientPayloadStorageService; + + private TaskResultPayloadService service; + + @BeforeEach + void setUp() { + service = new TaskResultPayloadService(taskResultPayloadMapper, new ObjectMapper(), + transientPayloadStorageService); + } + + @AfterEach + void clearSynchronizationIfLeftActive() { + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.clearSynchronization(); + } + } + + private void stubRowWithPayload(String payloadJson) { + TaskResultPayloadEntity existing = new TaskResultPayloadEntity(); + existing.setId(11L); + existing.setTaskId(9001L); + existing.setModuleType("APPEARANCE_PATENT"); + existing.setPayloadJson(payloadJson); + when(taskResultPayloadMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(existing); + } + + /** 没有活动同步器时 Spring 直接抛 IllegalState;测试里按「没注册任何同步器」读成空。 */ + private static List registered() { + return TransactionSynchronizationManager.isSynchronizationActive() + ? TransactionSynchronizationManager.getSynchronizations() + : List.of(); + } + + @Test + void 有事务上下文时对象存储要等提交之后才动() { + stubRowWithPayload(POINTER); + TransactionSynchronizationManager.initSynchronization(); + + service.deleteLatest(9001L, "APPEARANCE_PATENT", "scope-a"); + + verify(taskResultPayloadMapper, times(1)).deleteById(11L); + verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString()); + + List synchronizations = registered(); + assertEquals(1, synchronizations.size(), "提交后删除必须挂在事务同步器上,否则回滚也会照删"); + synchronizations.forEach(TransactionSynchronization::afterCommit); + verify(transientPayloadStorageService, times(1)).deletePayloadIfPresent(POINTER); + } + + /** 没有事务上下文(被非事务调用方直接用)时**立即删**——静默不删就是留孤儿对象。 */ + @Test + void 无事务上下文时仍然删到() { + stubRowWithPayload(POINTER); + + service.deleteLatest(9001L, "APPEARANCE_PATENT", "scope-a"); + + verify(taskResultPayloadMapper, times(1)).deleteById(11L); + verify(transientPayloadStorageService, times(1)).deletePayloadIfPresent(POINTER); + assertEquals(0, registered().size(), "不在事务里就别往线程上挂同步器(挂了也没人触发它)"); + } + + /** 落库失败=这单还没删:对象存储一次都不许碰(否则回滚后行还在、载荷已经没了)。 */ + @Test + void 落库失败不留半成品() { + stubRowWithPayload(POINTER); + when(taskResultPayloadMapper.deleteById(11L)).thenThrow(new IllegalStateException("db down")); + + assertThrows(IllegalStateException.class, + () -> service.deleteLatest(9001L, "APPEARANCE_PATENT", "scope-a")); + + verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString()); + } + + /** 行里没有载荷值时不注册提交后删除:一次空的网络往返都不该发。 */ + @Test + void 空载荷不注册提交后删除() { + stubRowWithPayload(null); + TransactionSynchronizationManager.initSynchronization(); + + service.deleteLatest(9001L, "APPEARANCE_PATENT", "scope-a"); + + verify(taskResultPayloadMapper, times(1)).deleteById(11L); + assertEquals(0, registered().size(), "没有指针就没有要删的对象,别挂一个注定空跑的同步器"); + verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString()); + } + + /** 没有这一行=什么都不做(既不删库也不注册删除)。 */ + @Test + void 行不存在时整条链路不动() { + when(taskResultPayloadMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null); + TransactionSynchronizationManager.initSynchronization(); + + service.deleteLatest(9001L, "APPEARANCE_PATENT", "scope-a"); + + verify(taskResultPayloadMapper, never()).deleteById(11L); + assertEquals(0, registered().size()); + verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString()); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResumeServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResumeServiceTest.java index 1b0d64876..cc551ee45 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResumeServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResumeServiceTest.java @@ -10,11 +10,13 @@ import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.spi.ClientTaskRetrySpi; import org.apache.ibatis.builder.MapperBuilderAssistant; import org.junit.jupiter.api.BeforeAll; +import org.mockito.ArgumentCaptor; import org.junit.jupiter.api.Test; import org.slf4j.LoggerFactory; import org.springframework.test.util.ReflectionTestUtils; import java.time.LocalDateTime; +import java.util.Collection; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -23,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -143,10 +146,18 @@ class TaskResumeServiceTest { @Test void 已有重试任务时跳过避免重复克隆() { FileTaskMapper mapper = mock(FileTaskMapper.class); - when(mapper.selectList(any())).thenReturn(List.of( - failedTask(28587L, "SIMILAR_ASIN", "任务心跳超时,已自动失败"))); - // 第一次是"不支持重试的中断任务数",第二次是"是否已有重试子任务" - when(mapper.selectCount(any())).thenReturn(0L, 1L); + FileTaskEntity origin = failedTask(28587L, "SIMILAR_ASIN", "任务心跳超时,已自动失败"); + FileTaskEntity child = new FileTaskEntity(); + child.setId(28600L); + child.setResumeOfTaskId(28587L); + // P1-B5 之后「是否已有重试子任务」不再是每行一次 selectCount,而是循环前一次批量 selectList。 + // 候选查询与反查都落在同一个 selectList 上,按投影列区分:反查那一支只取 resume_of_task_id。 + when(mapper.selectList(any())).thenAnswer(invocation -> { + com.baomidou.mybatisplus.core.conditions.AbstractWrapper wrapper = invocation.getArgument(0); + String projection = String.valueOf(wrapper.getSqlSelect()); + return projection.contains("resume_of_task_id") ? List.of(child) : List.of(origin); + }); + when(mapper.selectCount(any())).thenReturn(0L); ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class); when(spi.moduleType()).thenReturn("SIMILAR_ASIN"); TaskResumeService service = service(mapper, List.of(spi), true, 3); @@ -157,6 +168,37 @@ class TaskResumeServiceTest { assertEquals(0, stats.resumedTaskCount); } + /** + * P1-B6 同族:反查子任务与「补检中」两条判定都必须**批量**取(循环前一次),不许退回逐行点查—— + * 逐行形状在 500 个候选时是 1,000 次往返,且 {@code resume_of_task_id} 那列在 V187 之前无索引。 + * 判据=selectList 只发生「清账 + 候选 + 反查」三次、selectCount 只发生一次(不支持重试的计数), + * 且批量档拿到的是这一轮的全部候选 id。 + */ + @Test + void 两条前置判定按批量档取不逐行点查() { + FileTaskMapper mapper = mock(FileTaskMapper.class); + when(mapper.selectList(any())).thenReturn(List.of( + failedTask(28587L, "SIMILAR_ASIN", "任务心跳超时,已自动失败"), + failedTask(28588L, "SIMILAR_ASIN", "任务心跳超时,已自动失败"))); + when(mapper.selectCount(any())).thenReturn(0L); + TaskFileJobService jobs = mock(TaskFileJobService.class); + ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class); + when(spi.moduleType()).thenReturn("SIMILAR_ASIN"); + TaskResumeService service = service(mapper, jobs, List.of(spi), true, 3); + + TaskResumeService.ResumeStats stats = service.resumeInterruptedTasks(); + + assertEquals(2, stats.resumedTaskCount); + @SuppressWarnings("unchecked") + ArgumentCaptor> captor = + (ArgumentCaptor>) (ArgumentCaptor) ArgumentCaptor.forClass(Collection.class); + verify(jobs).taskIdsWithUnfinishedAssembleJob(captor.capture()); + assertTrue(captor.getValue().containsAll(List.of(28587L, 28588L)), + "批量档必须一次带走本轮全部候选,漏一个就是退回逐行点查"); + verify(mapper, times(3)).selectList(any()); + verify(mapper, times(1)).selectCount(any()); + } + /** * 现网 2026-10-24:一枚 760 行的外观专利任务因交付缺口把组装作业排回补检(作业还在 * PENDING/RUNNING),任务行却已是 FAILED,于是被巡检连克隆成 4 枚、每枚全量重跑一遍, @@ -170,7 +212,8 @@ class TaskResumeServiceTest { failedTask(32201L, "APPEARANCE_PATENT", "有 3 行抓取失败(未取得商品页数据):B07P5TR7Q5"))); when(mapper.selectCount(any())).thenReturn(0L); TaskFileJobService jobs = mock(TaskFileJobService.class); - when(jobs.hasUnfinishedAssembleJob(32201L)).thenReturn(true); + // P1-B5:补检判定从逐行 hasUnfinishedAssembleJob 折成循环前一次批量档。 + when(jobs.taskIdsWithUnfinishedAssembleJob(any())).thenReturn(java.util.Set.of(32201L)); ClientTaskRetrySpi spi = mock(ClientTaskRetrySpi.class); when(spi.moduleType()).thenReturn("APPEARANCE_PATENT"); TaskResumeService service = service(mapper, jobs, List.of(spi), true, 3); diff --git a/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java b/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java index e923382d9..53d1a7bca 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/testutil/SourceScanTest.java @@ -38,8 +38,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; *

一处与 plan 原文不符的读数,按"先怀疑尺子"的流程处理并留在脸上:plan/AGENTS §1 记的是「9 支文件」, * 那条数是 {@code grep -rl 'sourceText('} 数出来的,而第 9 支是 {@code SourceScan.java:22} 里 * {@code public static String sourceText(String fileName)} 这行被调方自己的声明——它不是调用点。 - * 所以真实形状是 调用点/调用方/引用方三类数各自成立(2026-10-28 v1.1 Phase 0 起调用方 16 支、 - * 裸名 25 个,增量逐笔登记在 {@code REGISTERED_BARE_NAMES} 的注释里)。下面第 3 格把这些数同时钉住, + * 所以真实形状是 调用点/调用方/引用方三类数各自成立(2026-11-03 Phase 5C 起调用方 19 支、 + * 裸名 26 个,增量逐笔登记在 {@code REGISTERED_BARE_NAMES} 的注释里)。下面第 3 格把这些数同时钉住, * 免得下一个人拿 {@code grep} 再数一遍又把声明算进去。

*/ class SourceScanTest { @@ -94,7 +94,13 @@ class SourceScanTest { // TaskFailureCategoryGuardrailTest 要钉「写了失败文案就必须同笔写分类列」这条连线: // 用 mock 造任务实体测分类,挡不住"新增一处 setErrorMessage 忘了写列"—— // 那正是"改一句文案静默改判死"的复发路径,只能打在源码上。顺带钉 ledger 不抄第二份 markers。 - "TaskFailureLedger.java"); + "TaskFailureLedger.java", + // —— 2026-11-03 Phase 5C(P1-B7 结果集分窗读取)追加(1 名)—— + // TaskResultItemReadWindowTest 要钉「提交必须被窗口钳住」这条:线程池只有 8 个线程, + // 一次提交 600 行与一次提交 256 行在"同时开始读几行"这个 observable 上读数相同 + // (都被 8 卡住),差异只体现在**堆**上(同时在飞的载荷副本),mock 数不出来; + // 行为四格只能钉住"不漏行/不乱序/异常口径不变",窗口这道闸本身必须打在源码上。 + "TaskResultItemService.java"); @TempDir Path tmp; @@ -110,7 +116,7 @@ class SourceScanTest { } @Test - void census_唯一裸名恰为25且与已登记集合相等() { + void census_唯一裸名恰为26且与已登记集合相等() { Set found = SourceScan.censusSourceTextCallSites().stream() .map(SourceScan.CallSite::bareName) .collect(Collectors.toSet()); @@ -118,11 +124,11 @@ class SourceScanTest { assertThat(found) .as("唯一裸名集合必须与类头台账逐字相等——多一个说明有人新引了单文件口径,少一个说明尺子漏扫") .isEqualTo(REGISTERED_BARE_NAMES) - .hasSize(25); + .hasSize(26); } @Test - void census_调用方文件恰为18而声明件在外() { + void census_调用方文件恰为19而声明件在外() { List sites = SourceScan.censusSourceTextCallSites(); Set callerFiles = sites.stream() .map(SourceScan.CallSite::relativeFile) @@ -131,9 +137,10 @@ class SourceScanTest { String declarationFile = "src/test/java/com/nanri/aiimage/testutil/SourceScan.java"; assertThat(callerFiles) - .as("真实字面量调用点落在 18 支调用方文件里(8 支既有 + 2026-10-28 v1.1 Phase 0 预置契约 8 支" - + " + 2026-11-02 归属死锁绊线 1 支 + 2026-11-03 B51-R1 双写绊线 1 支)") - .hasSize(18); + .as("真实字面量调用点落在 19 支调用方文件里(8 支既有 + 2026-10-28 v1.1 Phase 0 预置契约 8 支" + + " + 2026-11-02 归属死锁绊线 1 支 + 2026-11-03 B51-R1 双写绊线 1 支" + + " + 2026-11-03 Phase 5C P1-B7 窗口绊线 1 支)") + .hasSize(19); assertThat(callerFiles) .as("plan 原文记的「9 支」= grep -rl 'sourceText(' 的条数,第 9 支是被调方自己的声明行" + "(SourceScan.java:22 的 `public static String sourceText(String fileName)`),"

ASIN
{{ item.asin }}{{ itemBrand(item) || '—' }} -
- {{ cellCount(item, shop) || '' }} - +
{{ row.asin }}{{ row.brand }} +
+ {{ cell.count || '' }} +