收口(去重库): 五处比对与写入口按 company_id 认公司,不再跨公司去重
读侧(列表/导出)早已按公司收窄,但"去重"这个动作本身一直查全平台: selectExistingDataValues 是 `WHERE data_value IN (...)`,一个租户条件都没有。 库里唯一键自 V145 起就是 uk_company_data_value(company_id, data_value)—— 同值本来就允许归两家,是代码没跟上库的口径。 五处一起收口(第 5 处由编译器抓出,正是 V145 头注释 §1.2 点名的采集侧): 去重执行 ASIN 过滤 / 导入预检"已存在就跳过" / 手工新增与编辑的重名校验 (文案改「本公司已存在该数据」)/ 删除导入(超管不选分组时原先一次删掉全平台 同名行)/ 采集结果过滤(CollectDataBatchQuery 直连 mapper,改为经 dedupe 侧推导)。 超管代导入的归属分裂一并修掉:前端两个导入接口把公司 tab 发下去——原先 group_id 落别家的组、company_id 落超管自己那一家。公司身份一律在请求线程上由 DataReadScope.readCompanyIdOrOwn 解析后传入(导入与去重跑在虚拟线程,那时已无 请求上下文),service 侧 requireScopedCompanyId 只判空不兜底。 删两条零调用者的跨公司通道(selectAllDataValues/listComparableValues 全表读、 deleteFromExcel 两个同步重载)。反转一条反向钉:comparableValueLookupRemainsGlobal 原显式断言"比对不参与公司收窄"(多公司一期的半套遗留),现为 IsScopedToOneCompany。 三把尺子随之改动,均带出处:tsv 基准行搬家(三个标量双双不动)、前端发送口分母 17→18 组/21→22 处、CatchSilenceRatchet 重抛下限 387→386(随死代码离开树那处)。 验证:后端全量 4198 例 0 失败/0 错误/7 跳过;前端 1778 例全绿;变异自证 6/6 全部 KILLED,注入文件按 sha256 还原并复跑绿。 现网只读探针:misfiled=0、同值跨家=0 ⇒ 存量比对结果不变,行为改变只在将来 A 家导入 B 家已有 ASIN 那一刻;两口径差额 1,452 行(0.26%)全属公司 1、 全源于 10-05 账号硬删。
This commit is contained in:
@@ -32,7 +32,9 @@ export const COMPANY_SCOPED_PAGES: readonly CompanyScopedPage[] = [
|
|||||||
// 【这两页已于 2026-10-06 退出注册表】admin_invalid_asin_data(品牌数据库)与 admin_product_categories
|
// 【这两页已于 2026-10-06 退出注册表】admin_invalid_asin_data(品牌数据库)与 admin_product_categories
|
||||||
// (商品类目)按控制方定案退回**全平台共享**,摘的不只是 tab:后端读写两侧的 company_id 谓词一起删
|
// (商品类目)按控制方定案退回**全平台共享**,摘的不只是 tab:后端读写两侧的 company_id 谓词一起删
|
||||||
// (V151 退唯一键)。反向钉在 tests/company-tab-wiring.test.ts——把它们再加回来必须先推翻那条定案。
|
// (V151 退唯一键)。反向钉在 tests/company-tab-wiring.test.ts——把它们再加回来必须先推翻那条定案。
|
||||||
{ menuKey: 'admin_dedupe_total_data', apiFiles: ['src/pages/asin/dedupe-total-api.ts', 'src/pages/asin/dedupe-total-export.ts'], pageFiles: ['src/pages/asin/DedupeRegistryPage.vue'] },
|
// dedupe-import-api.ts 是 10-09 补进来的:导入/删除导入这两支写侧出口原先不发 company_id,
|
||||||
|
// 超管在别家 tab 点「新增导入」,数据落进了超管自己那一家(group_id 却是别家的组)=归属分裂。
|
||||||
|
{ menuKey: 'admin_dedupe_total_data', apiFiles: ['src/pages/asin/dedupe-total-api.ts', 'src/pages/asin/dedupe-total-export.ts', 'src/pages/asin/dedupe-import-api.ts'], pageFiles: ['src/pages/asin/DedupeRegistryPage.vue'] },
|
||||||
{ menuKey: 'admin_query_asin', apiFiles: ['src/pages/asin/query-asin-api.ts', 'src/pages/asin/query-asin-export.ts'], pageFiles: ['src/pages/asin/QueryAsinPage.vue'] },
|
{ menuKey: 'admin_query_asin', apiFiles: ['src/pages/asin/query-asin-api.ts', 'src/pages/asin/query-asin-export.ts'], pageFiles: ['src/pages/asin/QueryAsinPage.vue'] },
|
||||||
{ menuKey: 'admin_skip_price_asin', apiFiles: ['src/pages/asin/skip-price-api.ts', 'src/pages/asin/skip-price-export.ts'], pageFiles: ['src/pages/asin/SkipPricePage.vue'] },
|
{ menuKey: 'admin_skip_price_asin', apiFiles: ['src/pages/asin/skip-price-api.ts', 'src/pages/asin/skip-price-export.ts'], pageFiles: ['src/pages/asin/SkipPricePage.vue'] },
|
||||||
// 批次 3(控制方 2026-10-05 点名「店铺撞款监控 店铺数据也要做隔离」)。这两页的可见集不是按 company_id 列筛的,
|
// 批次 3(控制方 2026-10-05 点名「店铺撞款监控 店铺数据也要做隔离」)。这两页的可见集不是按 company_id 列筛的,
|
||||||
|
|||||||
@@ -250,7 +250,9 @@ async function submitImport(): Promise<void> {
|
|||||||
importProgress.value = mode === 'add' ? '正在上传并解析文件…' : '正在上传删除清单…'
|
importProgress.value = mode === 'add' ? '正在上传并解析文件…' : '正在上传删除清单…'
|
||||||
try {
|
try {
|
||||||
const started =
|
const started =
|
||||||
mode === 'add' ? await startDedupeImport(file, groupId) : await startDedupeDeleteImport(file, groupId)
|
mode === 'add'
|
||||||
|
? await startDedupeImport(file, groupId, companyTab.requestCompanyId)
|
||||||
|
: await startDedupeDeleteImport(file, groupId, companyTab.requestCompanyId)
|
||||||
const poll = mode === 'add' ? () => fetchDedupeImportProgress(started) : () => fetchDedupeDeleteImportProgress(started)
|
const poll = mode === 'add' ? () => fetchDedupeImportProgress(started) : () => fetchDedupeDeleteImportProgress(started)
|
||||||
const finished = await pollImport(poll)
|
const finished = await pollImport(poll)
|
||||||
importVisible.value = false
|
importVisible.value = false
|
||||||
|
|||||||
@@ -1,14 +1,20 @@
|
|||||||
/** 去重汇总导入启动适配(任务 65):multipart POST /import(file+groupId) → importId。 */
|
/**
|
||||||
|
* 去重汇总导入启动适配(任务 65):multipart POST /import(file+groupId+company_id) → importId。
|
||||||
|
*
|
||||||
|
* <p>company_id 决定这批数据写进哪一家:超管认页面上方 tab,非超管 requestCompanyId 为 null
|
||||||
|
* ⇒ 一个字段都不发,后端按会话恒锁本公司。</p>
|
||||||
|
*/
|
||||||
import { http } from '@/api/http'
|
import { http } from '@/api/http'
|
||||||
import { parseDedupeImportProgress, parseDedupeImportStart, type DedupeImportProgress } from './dedupe-import-model'
|
import { parseDedupeImportProgress, parseDedupeImportStart, type DedupeImportProgress } from './dedupe-import-model'
|
||||||
|
|
||||||
export const DEDUPE_IMPORT_ENDPOINT = '/api/admin/dedupe-total-data/import'
|
export const DEDUPE_IMPORT_ENDPOINT = '/api/admin/dedupe-total-data/import'
|
||||||
export const DEDUPE_DELETE_IMPORT_ENDPOINT = '/api/admin/dedupe-total-data/delete-import'
|
export const DEDUPE_DELETE_IMPORT_ENDPOINT = '/api/admin/dedupe-total-data/delete-import'
|
||||||
|
|
||||||
export async function startDedupeImport(file: File, groupId: number): Promise<string> {
|
export async function startDedupeImport(file: File, groupId: number, companyId?: number | null): Promise<string> {
|
||||||
const form = new FormData()
|
const form = new FormData()
|
||||||
form.append('file', file, file.name)
|
form.append('file', file, file.name)
|
||||||
form.append('groupId', String(groupId))
|
form.append('groupId', String(groupId))
|
||||||
|
if (companyId != null) form.append('company_id', String(companyId))
|
||||||
const { data } = await http.post<unknown>(DEDUPE_IMPORT_ENDPOINT, form)
|
const { data } = await http.post<unknown>(DEDUPE_IMPORT_ENDPOINT, form)
|
||||||
return parseDedupeImportStart(data)
|
return parseDedupeImportStart(data)
|
||||||
}
|
}
|
||||||
@@ -20,10 +26,12 @@ export async function fetchDedupeImportProgress(importId: string): Promise<Dedup
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** 删除导入启动:multipart POST /delete-import(file+groupId) → importId。 */
|
/** 删除导入启动:multipart POST /delete-import(file+groupId) → importId。 */
|
||||||
export async function startDedupeDeleteImport(file: File, groupId: number): Promise<string> {
|
export async function startDedupeDeleteImport(file: File, groupId: number,
|
||||||
|
companyId?: number | null): Promise<string> {
|
||||||
const form = new FormData()
|
const form = new FormData()
|
||||||
form.append('file', file, file.name)
|
form.append('file', file, file.name)
|
||||||
form.append('groupId', String(groupId))
|
form.append('groupId', String(groupId))
|
||||||
|
if (companyId != null) form.append('company_id', String(companyId))
|
||||||
const { data } = await http.post<unknown>(DEDUPE_DELETE_IMPORT_ENDPOINT, form)
|
const { data } = await http.post<unknown>(DEDUPE_DELETE_IMPORT_ENDPOINT, form)
|
||||||
return parseDedupeImportStart(data)
|
return parseDedupeImportStart(data)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -98,6 +98,8 @@ const SEND_POINTS: ReadonlyArray<SendPoint> = [
|
|||||||
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-total-api.ts', fn: 'fetchDedupeTotalList', line: 'if (params.companyId != null) query.company_id = params.companyId', window: 400, title: 'GET /dedupe-total-data 列表' },
|
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-total-api.ts', fn: 'fetchDedupeTotalList', line: 'if (params.companyId != null) query.company_id = params.companyId', window: 400, title: 'GET /dedupe-total-data 列表' },
|
||||||
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-total-api.ts', fn: 'fetchDedupeTotalGroups', line: 'http.get<unknown>(DEDUPE_GROUPS_ENDPOINT, { params: companyId != null ? { company_id: companyId } : {} })', window: 400, title: 'GET /shop-manages/groups 分组候选(与店铺管理同源,两页口径要对得上)' },
|
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-total-api.ts', fn: 'fetchDedupeTotalGroups', line: 'http.get<unknown>(DEDUPE_GROUPS_ENDPOINT, { params: companyId != null ? { company_id: companyId } : {} })', window: 400, title: 'GET /shop-manages/groups 分组候选(与店铺管理同源,两页口径要对得上)' },
|
||||||
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-total-export.ts', fn: 'buildDedupeExportParams', line: 'if (companyId != null) params.company_id = String(companyId)', window: 800, title: 'GET /dedupe-total-data/export 导出' },
|
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-total-export.ts', fn: 'buildDedupeExportParams', line: 'if (companyId != null) params.company_id = String(companyId)', window: 800, title: 'GET /dedupe-total-data/export 导出' },
|
||||||
|
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-import-api.ts', fn: 'startDedupeImport', line: "if (companyId != null) form.append('company_id', String(companyId))", window: 400, title: 'POST /dedupe-total-data/import 新增导入(写侧归属,不发就落进操作者自己那一家)' },
|
||||||
|
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-import-api.ts', fn: 'startDedupeDeleteImport', line: "if (companyId != null) form.append('company_id', String(companyId))", window: 900, title: 'POST /dedupe-total-data/delete-import 删除导入(不发就会按别家的值删错行)' },
|
||||||
{ menuKey: 'admin_query_asin', file: 'src/pages/asin/query-asin-api.ts', fn: 'fetchQueryAsinList', line: 'if (params.companyId != null) query.company_id = params.companyId', window: 400, title: 'GET /query-asins 列表' },
|
{ menuKey: 'admin_query_asin', file: 'src/pages/asin/query-asin-api.ts', fn: 'fetchQueryAsinList', line: 'if (params.companyId != null) query.company_id = params.companyId', window: 400, title: 'GET /query-asins 列表' },
|
||||||
{ menuKey: 'admin_query_asin', file: 'src/pages/asin/query-asin-export.ts', fn: 'buildQueryAsinExportParams', line: 'if (companyId != null) params.company_id = String(companyId)', window: 700, title: 'GET /query-asins/export 导出' },
|
{ menuKey: 'admin_query_asin', file: 'src/pages/asin/query-asin-export.ts', fn: 'buildQueryAsinExportParams', line: 'if (companyId != null) params.company_id = String(companyId)', window: 700, title: 'GET /query-asins/export 导出' },
|
||||||
{ menuKey: 'admin_skip_price_asin', file: 'src/pages/asin/skip-price-api.ts', fn: 'fetchSkipPriceList', line: 'if (params.companyId != null) query.company_id = params.companyId', window: 400, title: 'GET /skip-price-asins 列表(最低价 ASIN)' },
|
{ menuKey: 'admin_skip_price_asin', file: 'src/pages/asin/skip-price-api.ts', fn: 'fetchSkipPriceList', line: 'if (params.companyId != null) query.company_id = params.companyId', window: 400, title: 'GET /skip-price-asins 列表(最低价 ASIN)' },
|
||||||
@@ -237,6 +239,13 @@ const COMPANY_ID_SOURCE_PINS: Readonly<Record<string, readonly CompanyIdPin[]>>
|
|||||||
|
|
||||||
/** 严格判据的分母由本轮实扫得出(修完不许恒绿也不许恒红):求值了几条 shape、其中几条通过。 */
|
/** 严格判据的分母由本轮实扫得出(修完不许恒绿也不许恒红):求值了几条 shape、其中几条通过。 */
|
||||||
let strictPinsEvaluated = 0
|
let strictPinsEvaluated = 0
|
||||||
|
/**
|
||||||
|
* 「company_id 真被下传」的形状。原先只认 `company_id:` / `company_id =` 两形——那是按
|
||||||
|
* 「一律走查询参数」写的。去重库的导入与删除导入走 multipart body(form.append),
|
||||||
|
* 同一件事换了载体就被判成没接线;这里把那一形补进来,取值口仍按登记表的 shape 逐条严判。
|
||||||
|
*/
|
||||||
|
const COMPANY_ID_WIRED = /company_id\s*[:=]|form\.append\(\s*['"]company_id['"]/
|
||||||
|
|
||||||
let strictPinsPassed = 0
|
let strictPinsPassed = 0
|
||||||
|
|
||||||
test('正向:声明 companyScoped 的页,其 api 文件必须真的带 company_id,页面组件必须 watch activeId', () => {
|
test('正向:声明 companyScoped 的页,其 api 文件必须真的带 company_id,页面组件必须 watch activeId', () => {
|
||||||
@@ -254,8 +263,8 @@ test('正向:声明 companyScoped 的页,其 api 文件必须真的带 company
|
|||||||
// 旧式 /company_id|companyId/ 接近恒真,Task 8-12 只要写一行 `const companyId = null`
|
// 旧式 /company_id|companyId/ 接近恒真,Task 8-12 只要写一行 `const companyId = null`
|
||||||
// 或一句「// 由 companyId 决定」就能把这页判成已接线,而实际什么都没下传(约束 6:注释不算)。
|
// 或一句「// 由 companyId 决定」就能把这页判成已接线,而实际什么都没下传(约束 6:注释不算)。
|
||||||
// 读之前先 stripComments,与反向那半支同口径——说明注释里的 company_id 不是接线。
|
// 读之前先 stripComments,与反向那半支同口径——说明注释里的 company_id 不是接线。
|
||||||
if (!/company_id\s*[:=]/.test(stripComments(read(file)))) {
|
if (!COMPANY_ID_WIRED.test(stripComments(read(file)))) {
|
||||||
gaps.push(`${file} 没把 company_id 当成查询参数下传`)
|
gaps.push(`${file} 没把 company_id 下传(查询参数或 multipart 字段都算,裸 token 不算)`)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// 取值口登记(复评 I-2):光"把 company_id 塞进查询对象"不够,右半边必须取自**本函数自己的入参**。
|
// 取值口登记(复评 I-2):光"把 company_id 塞进查询对象"不够,右半边必须取自**本函数自己的入参**。
|
||||||
@@ -265,7 +274,7 @@ test('正向:声明 companyScoped 的页,其 api 文件必须真的带 company
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
const src = stripComments(read(pin.file))
|
const src = stripComments(read(pin.file))
|
||||||
const wired = /company_id\s*[:=]/.test(src)
|
const wired = COMPANY_ID_WIRED.test(src)
|
||||||
if (pin.shape === null) {
|
if (pin.shape === null) {
|
||||||
// 没登记取值口却已经下传了参数 = 这一页的取值口径没人看守(洞就是这么留下的)。
|
// 没登记取值口却已经下传了参数 = 这一页的取值口径没人看守(洞就是这么留下的)。
|
||||||
if (wired) gaps.push(`${pin.file} 已经把 company_id 当下传参数了,登记表里 shape 却还是 null —— 把它换成"company_id 右侧 = 本函数入参"的正则`)
|
if (wired) gaps.push(`${pin.file} 已经把 company_id 当下传参数了,登记表里 shape 却还是 null —— 把它换成"company_id 右侧 = 本函数入参"的正则`)
|
||||||
@@ -357,8 +366,8 @@ test('正向:声明 companyScoped 的页,其 api 文件必须真的带 company
|
|||||||
if (hit) hit.pinned += 1
|
if (hit) hit.pinned += 1
|
||||||
else grouped.set(key, { file: p.file, line: p.line, pinned: 1 })
|
else grouped.set(key, { file: p.file, line: p.line, pinned: 1 })
|
||||||
}
|
}
|
||||||
assert.equal(grouped.size, 17,
|
assert.equal(grouped.size, 18,
|
||||||
`发送口表按 (文件, 整行) 应聚成 17 组(21 处发送口),现在却有 ${grouped.size} 组 —— 表被改形状了,对平判据失去分母`)
|
`发送口表按 (文件, 整行) 应聚成 18 组(22 处发送口),现在却有 ${grouped.size} 组 —— 表被改形状了,对平判据失去分母`)
|
||||||
for (const g of grouped.values()) {
|
for (const g of grouped.values()) {
|
||||||
const actual = occurrences(stripComments(read(g.file)), g.line)
|
const actual = occurrences(stripComments(read(g.file)), g.line)
|
||||||
assert.equal(g.pinned, actual,
|
assert.equal(g.pinned, actual,
|
||||||
|
|||||||
+2
-1
@@ -122,7 +122,8 @@ public class CollectDataBrandRecheckInjector {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// 复查去重/无效品牌:补测窗口内该 ASIN 可能已被其他任务采走(first-writer-wins)
|
// 复查去重/无效品牌:补测窗口内该 ASIN 可能已被其他任务采走(first-writer-wins)
|
||||||
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(chunkClean);
|
CollectDataBatchQuery.FilterResult filtered =
|
||||||
|
collectDataBatchQuery.filter(chunkClean, task.getUserId());
|
||||||
if (filtered.kept().isEmpty()) {
|
if (filtered.kept().isEmpty()) {
|
||||||
log.info("[brand-recheck] 补测行全部被去重/无效过滤,不注入 taskId={} rows={}",
|
log.info("[brand-recheck] 补测行全部被去重/无效过滤,不注入 taskId={} rows={}",
|
||||||
task.getId(), chunkClean.size());
|
task.getId(), chunkClean.size());
|
||||||
|
|||||||
+4
-2
@@ -830,7 +830,7 @@ public class CollectDataService {
|
|||||||
|
|
||||||
public CollectDataSubmitResultVo submitResult(Long taskId, CollectDataSubmitResultRequest request) {
|
public CollectDataSubmitResultVo submitResult(Long taskId, CollectDataSubmitResultRequest request) {
|
||||||
if (request == null) {
|
if (request == null) {
|
||||||
throw new BusinessException("request is empty");
|
throw new BusinessException("请求内容为空");
|
||||||
}
|
}
|
||||||
ensureRustfsPayloadStorageEnabled();
|
ensureRustfsPayloadStorageEnabled();
|
||||||
|
|
||||||
@@ -861,7 +861,9 @@ public class CollectDataService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
long prepareStartAt = System.currentTimeMillis();
|
long prepareStartAt = System.currentTimeMillis();
|
||||||
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(rowsForFiltering);
|
// 比对集认任务归属账号那一家:chunk 回传的过滤结果不该受别家导入过什么影响
|
||||||
|
CollectDataBatchQuery.FilterResult filtered =
|
||||||
|
collectDataBatchQuery.filter(rowsForFiltering, probe.getUserId());
|
||||||
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome = brandBatchFilter.filter(filtered.kept());
|
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome = brandBatchFilter.filter(filtered.kept());
|
||||||
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms",
|
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms",
|
||||||
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt);
|
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt);
|
||||||
|
|||||||
+14
-12
@@ -2,7 +2,7 @@ package com.nanri.aiimage.modules.collectdata.util;
|
|||||||
|
|
||||||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||||
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
|
import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService;
|
||||||
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
|
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
|
||||||
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
|
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
@@ -22,7 +22,7 @@ import java.util.regex.Pattern;
|
|||||||
* 多次降为一次,并保证空输入、单元素与超限集合的确定行为。
|
* 多次降为一次,并保证空输入、单元素与超限集合的确定行为。
|
||||||
*
|
*
|
||||||
* 过滤语义(与 CollectDataService 原 filterByExistingAsin 一致):
|
* 过滤语义(与 CollectDataService 原 filterByExistingAsin 一致):
|
||||||
* - ASIN 存在于去重表 → dedupeFiltered(增加计数)
|
* - ASIN 存在于「本任务归属公司」的去重表 → dedupeFiltered(增加计数)
|
||||||
* - 品牌存在于无效品牌表 → invalidFiltered(增加计数)
|
* - 品牌存在于无效品牌表 → invalidFiltered(增加计数)
|
||||||
* - 其余行保留
|
* - 其余行保留
|
||||||
*/
|
*/
|
||||||
@@ -32,21 +32,22 @@ public class CollectDataBatchQuery {
|
|||||||
|
|
||||||
private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
|
private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
|
||||||
|
|
||||||
private final DedupeTotalDataMapper dedupeTotalDataMapper;
|
private final DedupeTotalDataService dedupeTotalDataService;
|
||||||
private final InvalidAsinDataMapper invalidAsinDataMapper;
|
private final InvalidAsinDataMapper invalidAsinDataMapper;
|
||||||
|
|
||||||
/**
|
|
||||||
* 批量集合查询过滤(兼容入口)。返回过滤后的行列表;入参为 null/空时返回空列表。
|
|
||||||
*/
|
|
||||||
public List<CollectDataResultRowVo> filterByExistingAsin(List<CollectDataResultRowVo> rows) {
|
|
||||||
return filter(rows).kept();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 批量集合查询过滤并统计两类命中:去重表命中的 ASIN 行数与无效品牌表
|
* 批量集合查询过滤并统计两类命中:去重表命中的 ASIN 行数与无效品牌表
|
||||||
* 命中的品牌行数(null/空行不计数)。入参为 null/空时返回空结果。
|
* 命中的品牌行数(null/空行不计数)。入参为 null/空时返回空结果。
|
||||||
|
*
|
||||||
|
* <p>去重表那一支的比对集只认 {@code ownerUserId} 那一家:「这个 ASIN 已经有人采过了」不能拿
|
||||||
|
* 别家导入的值来判。此前它查的是全平台,于是 B 公司导入的 ASIN 会让 A 公司的采集结果少行,而
|
||||||
|
* A 公司在自己后台根本看不到那行是谁的——V145 头注释 §1.2 点名的就是这一条。推导发生在
|
||||||
|
* {@code DedupeTotalDataService#findExistingComparableValuesForOwner},不在本文件:</p>
|
||||||
|
*
|
||||||
|
* <p>无效品牌那一支<b>不带</b>公司条件,是定案不是漏改:品牌库 biz_invalid_asin_data 已于
|
||||||
|
* 2026-10-06 退回全平台共享(V151 退唯一键),反向钉在 CompanyIdParamGoesThroughReadScopeTest。</p>
|
||||||
*/
|
*/
|
||||||
public FilterResult filter(List<CollectDataResultRowVo> rows) {
|
public FilterResult filter(List<CollectDataResultRowVo> rows, Long ownerUserId) {
|
||||||
if (rows == null || rows.isEmpty()) {
|
if (rows == null || rows.isEmpty()) {
|
||||||
return new FilterResult(List.of(), 0, 0);
|
return new FilterResult(List.of(), 0, 0);
|
||||||
}
|
}
|
||||||
@@ -55,7 +56,8 @@ public class CollectDataBatchQuery {
|
|||||||
.map(CollectDataResultRowVo::getAsin).toList());
|
.map(CollectDataResultRowVo::getAsin).toList());
|
||||||
Set<String> dedupeValues = new HashSet<>();
|
Set<String> dedupeValues = new HashSet<>();
|
||||||
if (!asins.isEmpty()) {
|
if (!asins.isEmpty()) {
|
||||||
List<String> existing = dedupeTotalDataMapper.selectExistingDataValues(asins);
|
Set<String> existing = dedupeTotalDataService
|
||||||
|
.findExistingComparableValuesForOwner(asins, ownerUserId);
|
||||||
if (existing != null) {
|
if (existing != null) {
|
||||||
for (String value : existing) {
|
for (String value : existing) {
|
||||||
String normalized = normalizeAsin(value);
|
String normalized = normalizeAsin(value);
|
||||||
|
|||||||
+19
-4
@@ -155,7 +155,11 @@ public class DedupeTotalDataController {
|
|||||||
HttpServletRequest httpRequest,
|
HttpServletRequest httpRequest,
|
||||||
@Valid @RequestBody DedupeTotalDataCreateRequest request) {
|
@Valid @RequestBody DedupeTotalDataCreateRequest request) {
|
||||||
RequestOperator operator = requireDedupeTotalDataAccess(httpRequest);
|
RequestOperator operator = requireDedupeTotalDataAccess(httpRequest);
|
||||||
return ApiResponse.success("创建成功", dedupeTotalDataService.create(request, operator.id()));
|
AdminUserEntity user = operator.user();
|
||||||
|
Long requestedCompanyId = request.getCompanyId();
|
||||||
|
Long companyId = dataReadScope.readCompanyIdOrOwn(user, requestedCompanyId);
|
||||||
|
Long operatorId = operator.id();
|
||||||
|
return ApiResponse.success("创建成功", dedupeTotalDataService.create(request, companyId, operatorId));
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/import")
|
@PostMapping("/import")
|
||||||
@@ -167,9 +171,14 @@ public class DedupeTotalDataController {
|
|||||||
public ApiResponse<DedupeTotalDataImportStartVo> importExcel(
|
public ApiResponse<DedupeTotalDataImportStartVo> importExcel(
|
||||||
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file,
|
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file,
|
||||||
@Parameter(description = "分组 ID", required = true) @RequestParam Long groupId,
|
@Parameter(description = "分组 ID", required = true) @RequestParam Long groupId,
|
||||||
|
@Parameter(description = "公司 ID") @RequestParam(name = "company_id", required = false) Long companyId,
|
||||||
HttpServletRequest request) {
|
HttpServletRequest request) {
|
||||||
RequestOperator operator = requireDedupeTotalDataAccess(request);
|
RequestOperator operator = requireDedupeTotalDataAccess(request);
|
||||||
return ApiResponse.success("开始导入", dedupeTotalDataService.startImport(file, groupId, operator.id()));
|
AdminUserEntity user = operator.user();
|
||||||
|
Long scopedCompanyId = dataReadScope.readCompanyIdOrOwn(user, companyId);
|
||||||
|
Long operatorId = operator.id();
|
||||||
|
return ApiResponse.success("开始导入",
|
||||||
|
dedupeTotalDataService.startImport(file, groupId, scopedCompanyId, operatorId));
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/import/{importId}")
|
@GetMapping("/import/{importId}")
|
||||||
@@ -190,9 +199,14 @@ public class DedupeTotalDataController {
|
|||||||
public ApiResponse<DedupeTotalDataImportStartVo> deleteImportExcel(
|
public ApiResponse<DedupeTotalDataImportStartVo> deleteImportExcel(
|
||||||
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file,
|
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file,
|
||||||
@Parameter(description = "分组 ID", required = true) @RequestParam Long groupId,
|
@Parameter(description = "分组 ID", required = true) @RequestParam Long groupId,
|
||||||
|
@Parameter(description = "公司 ID") @RequestParam(name = "company_id", required = false) Long companyId,
|
||||||
HttpServletRequest request) {
|
HttpServletRequest request) {
|
||||||
RequestOperator operator = requireDedupeTotalDataAccess(request);
|
RequestOperator operator = requireDedupeTotalDataAccess(request);
|
||||||
return ApiResponse.success("开始删除", dedupeTotalDataService.startDeleteImport(file, groupId, operator.id()));
|
AdminUserEntity user = operator.user();
|
||||||
|
Long scopedCompanyId = dataReadScope.readCompanyIdOrOwn(user, companyId);
|
||||||
|
Long operatorId = operator.id();
|
||||||
|
return ApiResponse.success("开始删除",
|
||||||
|
dedupeTotalDataService.startDeleteImport(file, groupId, scopedCompanyId, operatorId));
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/delete-import/{importId}")
|
@GetMapping("/delete-import/{importId}")
|
||||||
@@ -285,7 +299,8 @@ public class DedupeTotalDataController {
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
// 公司 tab 的读范围就地算在 handler 里,不留私有收口方法:G2 的断言 A 只在这支 handler 的
|
// 公司 tab 的读范围与写侧归属(create / import / delete-import)都就地算在 handler 里,
|
||||||
|
// 不留私有收口方法:G2 的断言 A 只在这支 handler 的
|
||||||
// 块内找 readScope(,私有方法在它自己的闭括号处收口、不算在内(CompanyIdParamGoesThroughReadScopeTest
|
// 块内找 readScope(,私有方法在它自己的闭括号处收口、不算在内(CompanyIdParamGoesThroughReadScopeTest
|
||||||
// 类注释第 5、7 条),挪进去 = 这一页的 tab 没人看守。收窄只给超管:非超管已由 resolveAccessScope
|
// 类注释第 5、7 条),挪进去 = 这一页的 tab 没人看守。收窄只给超管:非超管已由 resolveAccessScope
|
||||||
// 按会话裁剪,且他的可见集里含「靠分组可见」的行,叠一层账号 IN 会把那批行静默删掉
|
// 按会话裁剪,且他的可见集里含「靠分组可见」的行,叠一层账号 IN 会把那批行静默删掉
|
||||||
|
|||||||
+13
-16
@@ -4,18 +4,25 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
|||||||
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
|
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
|
||||||
import org.apache.ibatis.annotations.Mapper;
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
import org.apache.ibatis.annotations.Param;
|
import org.apache.ibatis.annotations.Param;
|
||||||
import org.apache.ibatis.annotations.SelectProvider;
|
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.StringJoiner;
|
|
||||||
|
|
||||||
@Mapper
|
@Mapper
|
||||||
public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> {
|
public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> {
|
||||||
|
|
||||||
List<String> selectAllDataValues();
|
/**
|
||||||
|
* 比对集按公司取:一次去重/一批导入只能看见 {@code companyId} 名下的值。
|
||||||
@SelectProvider(type = SqlProvider.class, method = "selectExistingDataValues")
|
*
|
||||||
List<String> selectExistingDataValues(@Param("values") List<String> values);
|
* <p>这条曾是 @SelectProvider 手拼 IN 列表、且不带任何租户条件,后果有两样:去重执行会把别家
|
||||||
|
* 导入的 ASIN 当「已存在」从输出文件里删掉;导入会把别家已有的值静默跳过,本公司库里永远补不上
|
||||||
|
* 这一行。本表唯一键自 V145 起就是 uk_company_data_value(company_id, data_value)——同值本来就
|
||||||
|
* 允许归两家,是代码没跟上库的口径。</p>
|
||||||
|
*
|
||||||
|
* <p>{@code companyId} 必须由调用方保证非空:MyBatis 会把它渲染成 {@code company_id = NULL}
|
||||||
|
* 这条永假条件,静默给出空集,比当场炸更难发现。拦空值的是服务层闸门。</p>
|
||||||
|
*/
|
||||||
|
List<String> selectExistingDataValues(@Param("values") List<String> values,
|
||||||
|
@Param("companyId") Long companyId);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Import is deliberately a single SQL round trip per batch. The unique
|
* Import is deliberately a single SQL round trip per batch. The unique
|
||||||
@@ -23,14 +30,4 @@ public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity>
|
|||||||
* the same value between the pre-check and this statement.
|
* the same value between the pre-check and this statement.
|
||||||
*/
|
*/
|
||||||
int insertBatchIgnore(@Param("rows") List<DedupeTotalDataEntity> rows);
|
int insertBatchIgnore(@Param("rows") List<DedupeTotalDataEntity> rows);
|
||||||
|
|
||||||
class SqlProvider {
|
|
||||||
public String selectExistingDataValues(@Param("values") List<String> values) {
|
|
||||||
StringJoiner placeholders = new StringJoiner(", ");
|
|
||||||
for (int i = 0; i < values.size(); i++) {
|
|
||||||
placeholders.add("#{values[" + i + "]}");
|
|
||||||
}
|
|
||||||
return "SELECT data_value FROM biz_dedupe_total_data WHERE data_value IN (" + placeholders + ")";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
+4
@@ -18,4 +18,8 @@ public class DedupeTotalDataCreateRequest {
|
|||||||
@NotNull(message = "请选择分组")
|
@NotNull(message = "请选择分组")
|
||||||
@Schema(description = "分组 ID", requiredMode = Schema.RequiredMode.REQUIRED)
|
@Schema(description = "分组 ID", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
private Long groupId;
|
private Long groupId;
|
||||||
|
|
||||||
|
/** 页面上方公司 tab 选中的归属公司;非超管带什么都不算,超管不带则拒绝写入。 */
|
||||||
|
@Schema(description = "公司 ID")
|
||||||
|
private Long companyId;
|
||||||
}
|
}
|
||||||
|
|||||||
+20
-7
@@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
|||||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.common.service.CompanyScopeSupport;
|
||||||
import com.nanri.aiimage.common.service.NodeSharedStore;
|
import com.nanri.aiimage.common.service.NodeSharedStore;
|
||||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||||
import com.nanri.aiimage.config.InstanceMetadata;
|
import com.nanri.aiimage.config.InstanceMetadata;
|
||||||
@@ -79,6 +80,7 @@ public class DedupeRunService {
|
|||||||
private final StorageProperties storageProperties;
|
private final StorageProperties storageProperties;
|
||||||
private final OssStorageService ossStorageService;
|
private final OssStorageService ossStorageService;
|
||||||
private final DedupeTotalDataService dedupeTotalDataService;
|
private final DedupeTotalDataService dedupeTotalDataService;
|
||||||
|
private final CompanyScopeSupport companyScopeSupport;
|
||||||
private final InstanceMetadata instanceMetadata;
|
private final InstanceMetadata instanceMetadata;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -102,6 +104,7 @@ public class DedupeRunService {
|
|||||||
StorageProperties storageProperties,
|
StorageProperties storageProperties,
|
||||||
OssStorageService ossStorageService,
|
OssStorageService ossStorageService,
|
||||||
DedupeTotalDataService dedupeTotalDataService,
|
DedupeTotalDataService dedupeTotalDataService,
|
||||||
|
CompanyScopeSupport companyScopeSupport,
|
||||||
StringRedisTemplate stringRedisTemplate,
|
StringRedisTemplate stringRedisTemplate,
|
||||||
ObjectMapper objectMapper,
|
ObjectMapper objectMapper,
|
||||||
InstanceMetadata instanceMetadata) {
|
InstanceMetadata instanceMetadata) {
|
||||||
@@ -110,6 +113,7 @@ public class DedupeRunService {
|
|||||||
this.storageProperties = storageProperties;
|
this.storageProperties = storageProperties;
|
||||||
this.ossStorageService = ossStorageService;
|
this.ossStorageService = ossStorageService;
|
||||||
this.dedupeTotalDataService = dedupeTotalDataService;
|
this.dedupeTotalDataService = dedupeTotalDataService;
|
||||||
|
this.companyScopeSupport = companyScopeSupport;
|
||||||
this.instanceMetadata = instanceMetadata;
|
this.instanceMetadata = instanceMetadata;
|
||||||
this.runProgressMap = new NodeSharedStore<>("aiimage:dedupe:run:progress", RUN_STATE_TTL,
|
this.runProgressMap = new NodeSharedStore<>("aiimage:dedupe:run:progress", RUN_STATE_TTL,
|
||||||
DedupeRunProgressVo.class, stringRedisTemplate, objectMapper);
|
DedupeRunProgressVo.class, stringRedisTemplate, objectMapper);
|
||||||
@@ -140,6 +144,11 @@ public class DedupeRunService {
|
|||||||
throw new BusinessException("请至少选择一种 ID 保留规则");
|
throw new BusinessException("请至少选择一种 ID 保留规则");
|
||||||
}
|
}
|
||||||
cleanupExpiredProgress();
|
cleanupExpiredProgress();
|
||||||
|
// 比对集锁在提交人自己那一家:「这个 ASIN 去重库里已经有了」只能拿本公司导入过的值来判,
|
||||||
|
// 否则别家导入的 ASIN 会把本应保留的行从输出文件里静默删掉。
|
||||||
|
// 在请求线程上解析:一是解析失败要当场报错,不能任务跑一半才炸;二是清洗跑在虚拟线程上,
|
||||||
|
// 那时已经没有请求上下文了。
|
||||||
|
Long companyId = companyScopeSupport.requireCompanyId(request.getUserId());
|
||||||
|
|
||||||
if (!tryAcquireRunningTaskSlot(request.getUserId())) {
|
if (!tryAcquireRunningTaskSlot(request.getUserId())) {
|
||||||
throw new BusinessException("已有其他去重任务正在处理中,请等待完成后再试");
|
throw new BusinessException("已有其他去重任务正在处理中,请等待完成后再试");
|
||||||
@@ -182,7 +191,7 @@ public class DedupeRunService {
|
|||||||
runProgressMap.putNow(runId, snapshot(progress));
|
runProgressMap.putNow(runId, snapshot(progress));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress));
|
Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress, companyId));
|
||||||
log.info("dedupe run submitted runId={} taskNo={} userId={} files={} thread={}",
|
log.info("dedupe run submitted runId={} taskNo={} userId={} files={} thread={}",
|
||||||
runId, task.getTaskNo(), request.getUserId(), request.getFiles().size(), thread.threadId());
|
runId, task.getTaskNo(), request.getUserId(), request.getFiles().size(), thread.threadId());
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@@ -292,7 +301,8 @@ public class DedupeRunService {
|
|||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request, DedupeRunProgressVo progress) {
|
private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request,
|
||||||
|
DedupeRunProgressVo progress, Long companyId) {
|
||||||
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
|
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
|
||||||
Map<String, DedupeArchiveEntry> archiveEntries = new ConcurrentHashMap<>();
|
Map<String, DedupeArchiveEntry> archiveEntries = new ConcurrentHashMap<>();
|
||||||
List<DedupeResultItemVo> outcomeItems = new ArrayList<>();
|
List<DedupeResultItemVo> outcomeItems = new ArrayList<>();
|
||||||
@@ -304,7 +314,8 @@ public class DedupeRunService {
|
|||||||
List<Future<?>> workers = new ArrayList<>(request.getFiles().size());
|
List<Future<?>> workers = new ArrayList<>(request.getFiles().size());
|
||||||
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
|
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
|
||||||
workers.add(fileExecutor.submit(() -> {
|
workers.add(fileExecutor.submit(() -> {
|
||||||
DedupeResultItemVo item = processFile(sourceFile, request, folderMode, task, archiveEntries);
|
DedupeResultItemVo item = processFile(sourceFile, request, folderMode, task, archiveEntries,
|
||||||
|
companyId);
|
||||||
synchronized (progress) {
|
synchronized (progress) {
|
||||||
progress.setProcessedCount(processedCount.incrementAndGet());
|
progress.setProcessedCount(processedCount.incrementAndGet());
|
||||||
if (item.isSuccess()) {
|
if (item.isSuccess()) {
|
||||||
@@ -466,7 +477,7 @@ public class DedupeRunService {
|
|||||||
|
|
||||||
private DedupeResultItemVo processFile(DedupeSourceFileDto sourceFile, DedupeRunRequest request,
|
private DedupeResultItemVo processFile(DedupeSourceFileDto sourceFile, DedupeRunRequest request,
|
||||||
boolean folderMode, FileTaskEntity task,
|
boolean folderMode, FileTaskEntity task,
|
||||||
Map<String, DedupeArchiveEntry> archiveEntries) {
|
Map<String, DedupeArchiveEntry> archiveEntries, Long companyId) {
|
||||||
DedupeResultItemVo item = new DedupeResultItemVo();
|
DedupeResultItemVo item = new DedupeResultItemVo();
|
||||||
item.setSourceFilename(sourceFile.getOriginalFilename());
|
item.setSourceFilename(sourceFile.getOriginalFilename());
|
||||||
try {
|
try {
|
||||||
@@ -496,7 +507,8 @@ public class DedupeRunService {
|
|||||||
request.getSelectedColumns(),
|
request.getSelectedColumns(),
|
||||||
request.isKeepIntegerIds(),
|
request.isKeepIntegerIds(),
|
||||||
request.isKeepUnderscoreIds(),
|
request.isKeepUnderscoreIds(),
|
||||||
request.isKeepIntegerMainIdsWhenNoSubIds()
|
request.isKeepIntegerMainIdsWhenNoSubIds(),
|
||||||
|
companyId
|
||||||
);
|
);
|
||||||
long cleanNs = elapsedNs(cleanStartNs);
|
long cleanNs = elapsedNs(cleanStartNs);
|
||||||
|
|
||||||
@@ -679,7 +691,7 @@ public class DedupeRunService {
|
|||||||
*/
|
*/
|
||||||
private void cleanExcelByStreamRules(File inputFile, File outputFile, List<String> selectedColumns,
|
private void cleanExcelByStreamRules(File inputFile, File outputFile, List<String> selectedColumns,
|
||||||
boolean keepIntegerIds, boolean keepUnderscoreIds,
|
boolean keepIntegerIds, boolean keepUnderscoreIds,
|
||||||
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
|
boolean keepIntegerMainIdsWhenNoSubIds, Long companyId) throws Exception {
|
||||||
List<String> orderedSelectedColumns = reorderExportColumns(selectedColumns);
|
List<String> orderedSelectedColumns = reorderExportColumns(selectedColumns);
|
||||||
long readStartNs = System.nanoTime();
|
long readStartNs = System.nanoTime();
|
||||||
DedupeReadResult readResult = new DedupeReadResult();
|
DedupeReadResult readResult = new DedupeReadResult();
|
||||||
@@ -746,7 +758,8 @@ public class DedupeRunService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
long dbStartNs = System.nanoTime();
|
long dbStartNs = System.nanoTime();
|
||||||
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
|
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(
|
||||||
|
candidateAsinValues, companyId);
|
||||||
long dbNs = elapsedNs(dbStartNs);
|
long dbNs = elapsedNs(dbStartNs);
|
||||||
|
|
||||||
long writeStartNs = System.nanoTime();
|
long writeStartNs = System.nanoTime();
|
||||||
|
|||||||
+80
-66
@@ -471,12 +471,12 @@ public class DedupeTotalDataService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request, Long operatorId) {
|
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request, Long companyId, Long operatorId) {
|
||||||
AdminUserEntity uploader = getOperator(operatorId);
|
AdminUserEntity uploader = getOperator(operatorId);
|
||||||
Long companyId = companyScopeSupport.requireCompanyId(uploader);
|
requireScopedCompanyId(companyId);
|
||||||
ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), uploader);
|
ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), uploader);
|
||||||
String dataValue = normalizeComparableValue(request.getDataValue());
|
String dataValue = normalizeComparableValue(request.getDataValue());
|
||||||
ensureUnique(dataValue, null);
|
ensureUnique(dataValue, null, companyId);
|
||||||
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
|
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
|
||||||
entity.setDataValue(dataValue);
|
entity.setDataValue(dataValue);
|
||||||
entity.setGroupId(group.getId());
|
entity.setGroupId(group.getId());
|
||||||
@@ -488,14 +488,27 @@ public class DedupeTotalDataService {
|
|||||||
return toItemVo(getById(entity.getId()), group.getGroupName());
|
return toItemVo(getById(entity.getId()), group.getGroupName());
|
||||||
}
|
}
|
||||||
|
|
||||||
public Set<String> listComparableValues() {
|
/**
|
||||||
return dedupeTotalDataMapper.selectAllDataValues().stream()
|
* 去重比对的取数口:只在 {@code companyId} 这一家名下找「已存在的值」。
|
||||||
.map(this::normalizeComparableValue)
|
*
|
||||||
.filter(value -> !value.isEmpty())
|
* <p>公司身份必须由调用方给死,null 不放行:{@code company_id = NULL} 在 SQL 里是永假条件,
|
||||||
.collect(java.util.stream.Collectors.toSet());
|
* 会静默返回空集,「一家都没比对上」和「比对集是空的」在结果上长得一样——那正是这次要拆的形态。
|
||||||
|
* 同理由删掉了此前的 {@code listComparableValues()}:它把全平台的值整池端出来,且零调用者。</p>
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* 采集侧的比对入口:调用方只交任务归属账号,公司由这里推导。
|
||||||
|
*
|
||||||
|
* <p>不让调用方自己传 companyId:采集过滤器住在 collectdata 模块,同一个文件里还住着品牌库那一支
|
||||||
|
* (2026-10-06 定案全平台共享、不许带公司维度),公司字样落进去会让那条反向钉失去分辨力。</p>
|
||||||
|
*/
|
||||||
|
public Set<String> findExistingComparableValuesForOwner(Collection<String> values, Long ownerUserId) {
|
||||||
|
return findExistingComparableValues(values, companyScopeSupport.requireCompanyId(ownerUserId));
|
||||||
}
|
}
|
||||||
|
|
||||||
public Set<String> findExistingComparableValues(Collection<String> values) {
|
public Set<String> findExistingComparableValues(Collection<String> values, Long companyId) {
|
||||||
|
if (companyId == null) {
|
||||||
|
throw new BusinessException("去重比对缺少归属公司,比对集必须锁在一家公司内");
|
||||||
|
}
|
||||||
if (values == null || values.isEmpty()) {
|
if (values == null || values.isEmpty()) {
|
||||||
return Set.of();
|
return Set.of();
|
||||||
}
|
}
|
||||||
@@ -510,7 +523,8 @@ public class DedupeTotalDataService {
|
|||||||
Set<String> existingValues = new HashSet<>();
|
Set<String> existingValues = new HashSet<>();
|
||||||
for (int start = 0; start < normalizedValues.size(); start += COMPARE_BATCH_SIZE) {
|
for (int start = 0; start < normalizedValues.size(); start += COMPARE_BATCH_SIZE) {
|
||||||
int end = Math.min(start + COMPARE_BATCH_SIZE, normalizedValues.size());
|
int end = Math.min(start + COMPARE_BATCH_SIZE, normalizedValues.size());
|
||||||
List<String> batch = dedupeTotalDataMapper.selectExistingDataValues(normalizedValues.subList(start, end));
|
List<String> batch = dedupeTotalDataMapper.selectExistingDataValues(
|
||||||
|
normalizedValues.subList(start, end), companyId);
|
||||||
if (batch != null) {
|
if (batch != null) {
|
||||||
batch.stream()
|
batch.stream()
|
||||||
.map(this::normalizeComparableValueOrBlank)
|
.map(this::normalizeComparableValueOrBlank)
|
||||||
@@ -521,11 +535,8 @@ public class DedupeTotalDataService {
|
|||||||
return existingValues;
|
return existingValues;
|
||||||
}
|
}
|
||||||
|
|
||||||
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long operatorId) {
|
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long groupId, Long companyId,
|
||||||
return startImport(file, null, operatorId);
|
Long operatorId) {
|
||||||
}
|
|
||||||
|
|
||||||
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long groupId, Long operatorId) {
|
|
||||||
cleanupExpiredProgress();
|
cleanupExpiredProgress();
|
||||||
if (file == null || file.isEmpty()) {
|
if (file == null || file.isEmpty()) {
|
||||||
throw new BusinessException("请上传 xlsx 文件");
|
throw new BusinessException("请上传 xlsx 文件");
|
||||||
@@ -534,7 +545,7 @@ public class DedupeTotalDataService {
|
|||||||
throw new BusinessException("导入文件超过大小上限");
|
throw new BusinessException("导入文件超过大小上限");
|
||||||
}
|
}
|
||||||
AdminUserEntity uploader = getOperator(operatorId);
|
AdminUserEntity uploader = getOperator(operatorId);
|
||||||
Long companyId = companyScopeSupport.requireCompanyId(uploader);
|
requireScopedCompanyId(companyId);
|
||||||
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
|
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
|
||||||
if (!importSlots.tryAcquire()) {
|
if (!importSlots.tryAcquire()) {
|
||||||
throw new BusinessException("导入任务过多,请稍后重试");
|
throw new BusinessException("导入任务过多,请稍后重试");
|
||||||
@@ -582,11 +593,8 @@ public class DedupeTotalDataService {
|
|||||||
return progress;
|
return progress;
|
||||||
}
|
}
|
||||||
|
|
||||||
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long operatorId) {
|
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long groupId, Long companyId,
|
||||||
return startDeleteImport(file, null, operatorId);
|
Long operatorId) {
|
||||||
}
|
|
||||||
|
|
||||||
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long groupId, Long operatorId) {
|
|
||||||
cleanupExpiredProgress();
|
cleanupExpiredProgress();
|
||||||
if (file == null || file.isEmpty()) {
|
if (file == null || file.isEmpty()) {
|
||||||
throw new BusinessException("请上传 xlsx 文件");
|
throw new BusinessException("请上传 xlsx 文件");
|
||||||
@@ -595,6 +603,7 @@ public class DedupeTotalDataService {
|
|||||||
throw new BusinessException("导入文件超过大小上限");
|
throw new BusinessException("导入文件超过大小上限");
|
||||||
}
|
}
|
||||||
AdminUserEntity operator = getOperator(operatorId);
|
AdminUserEntity operator = getOperator(operatorId);
|
||||||
|
requireScopedCompanyId(companyId);
|
||||||
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
|
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
|
||||||
if (!importSlots.tryAcquire()) {
|
if (!importSlots.tryAcquire()) {
|
||||||
throw new BusinessException("导入任务过多,请稍后重试");
|
throw new BusinessException("导入任务过多,请稍后重试");
|
||||||
@@ -615,7 +624,7 @@ public class DedupeTotalDataService {
|
|||||||
File tempFile = saveMultipartToTempFile(file);
|
File tempFile = saveMultipartToTempFile(file);
|
||||||
String filename = file.getOriginalFilename();
|
String filename = file.getOriginalFilename();
|
||||||
Thread.ofVirtual().start(() -> runDeleteImportTask(
|
Thread.ofVirtual().start(() -> runDeleteImportTask(
|
||||||
importId, tempFile, filename, operator.getId(), group.getId()));
|
importId, tempFile, filename, operator.getId(), group.getId(), companyId));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
importSlots.release();
|
importSlots.release();
|
||||||
deleteImportProgressMap.remove(importId);
|
deleteImportProgressMap.remove(importId);
|
||||||
@@ -642,7 +651,7 @@ public class DedupeTotalDataService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void runDeleteImportTask(String importId, File tempFile, String filename,
|
private void runDeleteImportTask(String importId, File tempFile, String filename,
|
||||||
Long operatorId, Long groupId) {
|
Long operatorId, Long groupId, Long companyId) {
|
||||||
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
|
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
|
||||||
if (progress == null) {
|
if (progress == null) {
|
||||||
importSlots.release();
|
importSlots.release();
|
||||||
@@ -652,7 +661,7 @@ public class DedupeTotalDataService {
|
|||||||
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
|
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
|
||||||
AccessScope scope = resolveAccessScope(operatorId);
|
AccessScope scope = resolveAccessScope(operatorId);
|
||||||
DedupeTotalDataImportVo result = deleteFromExcelInternal(
|
DedupeTotalDataImportVo result = deleteFromExcelInternal(
|
||||||
inputStream, filename, progress, scope, groupId);
|
inputStream, filename, progress, scope, groupId, companyId);
|
||||||
progress.setTotalRows(result.getTotalRows());
|
progress.setTotalRows(result.getTotalRows());
|
||||||
progress.setAsinCount(result.getAsinCount());
|
progress.setAsinCount(result.getAsinCount());
|
||||||
progress.setInsertedCount(result.getInsertedCount());
|
progress.setInsertedCount(result.getInsertedCount());
|
||||||
@@ -722,16 +731,13 @@ public class DedupeTotalDataService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long operatorId) {
|
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long groupId, Long companyId,
|
||||||
return importFromExcel(file, null, operatorId);
|
Long operatorId) {
|
||||||
}
|
|
||||||
|
|
||||||
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long groupId, Long operatorId) {
|
|
||||||
if (file == null || file.isEmpty()) {
|
if (file == null || file.isEmpty()) {
|
||||||
throw new BusinessException("请上传 xlsx 文件");
|
throw new BusinessException("请上传 xlsx 文件");
|
||||||
}
|
}
|
||||||
AdminUserEntity uploader = getOperator(operatorId);
|
AdminUserEntity uploader = getOperator(operatorId);
|
||||||
Long companyId = companyScopeSupport.requireCompanyId(uploader);
|
requireScopedCompanyId(companyId);
|
||||||
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
|
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
|
||||||
try (InputStream inputStream = file.getInputStream()) {
|
try (InputStream inputStream = file.getInputStream()) {
|
||||||
return importFromExcelInternal(
|
return importFromExcelInternal(
|
||||||
@@ -744,29 +750,6 @@ public class DedupeTotalDataService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file, Long operatorId) {
|
|
||||||
return deleteFromExcel(file, null, operatorId);
|
|
||||||
}
|
|
||||||
|
|
||||||
public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file, Long groupId, Long operatorId) {
|
|
||||||
if (file == null || file.isEmpty()) {
|
|
||||||
throw new BusinessException("请上传 xlsx 文件");
|
|
||||||
}
|
|
||||||
AdminUserEntity operator = getOperator(operatorId);
|
|
||||||
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
|
|
||||||
AccessScope scope = resolveAccessScope(operatorId);
|
|
||||||
try (InputStream inputStream = file.getInputStream()) {
|
|
||||||
return deleteFromExcelInternal(
|
|
||||||
inputStream, file.getOriginalFilename(), null, scope, group.getId());
|
|
||||||
} catch (BusinessException e) {
|
|
||||||
throw e;
|
|
||||||
} catch (Exception e) {
|
|
||||||
// 此前丢弃原始异常:排查时只能看到一句「删除 Excel 匹配数据失败」
|
|
||||||
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败", e);
|
|
||||||
throw new BusinessException("删除 Excel 匹配数据失败", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private DedupeTotalDataImportVo importFromExcelInternal(InputStream inputStream, String filename,
|
private DedupeTotalDataImportVo importFromExcelInternal(InputStream inputStream, String filename,
|
||||||
DedupeTotalDataImportProgressVo progress,
|
DedupeTotalDataImportProgressVo progress,
|
||||||
Long uploaderUserId, String uploaderUsername,
|
Long uploaderUserId, String uploaderUsername,
|
||||||
@@ -965,7 +948,7 @@ public class DedupeTotalDataService {
|
|||||||
if (values == null || values.isEmpty()) {
|
if (values == null || values.isEmpty()) {
|
||||||
return new ImportBatchResult(0, 0);
|
return new ImportBatchResult(0, 0);
|
||||||
}
|
}
|
||||||
Set<String> existingValues = findExistingComparableValues(values);
|
Set<String> existingValues = findExistingComparableValues(values, companyId);
|
||||||
List<DedupeTotalDataEntity> rows = new ArrayList<>(values.size());
|
List<DedupeTotalDataEntity> rows = new ArrayList<>(values.size());
|
||||||
for (String value : values) {
|
for (String value : values) {
|
||||||
if (value == null || value.isBlank() || existingValues.contains(value)) {
|
if (value == null || value.isBlank() || existingValues.contains(value)) {
|
||||||
@@ -998,7 +981,7 @@ public class DedupeTotalDataService {
|
|||||||
for (DedupeTotalDataEntity row : rows) {
|
for (DedupeTotalDataEntity row : rows) {
|
||||||
try {
|
try {
|
||||||
Boolean oneInserted = executeInNewTransaction(() -> {
|
Boolean oneInserted = executeInNewTransaction(() -> {
|
||||||
if (existsDataValue(row.getDataValue())) {
|
if (existsDataValue(row.getDataValue(), companyId)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return dedupeTotalDataMapper.insert(row) > 0;
|
return dedupeTotalDataMapper.insert(row) > 0;
|
||||||
@@ -1040,14 +1023,14 @@ public class DedupeTotalDataService {
|
|||||||
|
|
||||||
private DedupeTotalDataImportVo deleteFromExcelInternal(InputStream inputStream, String filename,
|
private DedupeTotalDataImportVo deleteFromExcelInternal(InputStream inputStream, String filename,
|
||||||
DedupeTotalDataImportProgressVo progress,
|
DedupeTotalDataImportProgressVo progress,
|
||||||
AccessScope scope, Long groupId) {
|
AccessScope scope, Long groupId, Long companyId) {
|
||||||
String lowerFilename = filename == null ? "" : filename.toLowerCase();
|
String lowerFilename = filename == null ? "" : filename.toLowerCase();
|
||||||
if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) {
|
if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) {
|
||||||
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
|
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2026-09 全维度审查 C5:与导入同样改流式解析,避免 50 万行时 DOM 占 1~2GB 堆
|
// 2026-09 全维度审查 C5:与导入同样改流式解析,避免 50 万行时 DOM 占 1~2GB 堆
|
||||||
DeleteSession session = new DeleteSession(progress, scope, groupId);
|
DeleteSession session = new DeleteSession(progress, scope, groupId, companyId);
|
||||||
try {
|
try {
|
||||||
ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() {
|
ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() {
|
||||||
private boolean headerReady;
|
private boolean headerReady;
|
||||||
@@ -1094,6 +1077,7 @@ public class DedupeTotalDataService {
|
|||||||
private final DedupeTotalDataImportProgressVo progress;
|
private final DedupeTotalDataImportProgressVo progress;
|
||||||
private final AccessScope scope;
|
private final AccessScope scope;
|
||||||
private final Long groupId;
|
private final Long groupId;
|
||||||
|
private final Long companyId;
|
||||||
private final Set<String> seenInFile = new HashSet<>();
|
private final Set<String> seenInFile = new HashSet<>();
|
||||||
private final List<String> pendingDeletes = new ArrayList<>();
|
private final List<String> pendingDeletes = new ArrayList<>();
|
||||||
private Map<String, Integer> headerIndex = Map.of();
|
private Map<String, Integer> headerIndex = Map.of();
|
||||||
@@ -1102,10 +1086,12 @@ public class DedupeTotalDataService {
|
|||||||
private int deletedCount;
|
private int deletedCount;
|
||||||
private int skippedCount;
|
private int skippedCount;
|
||||||
|
|
||||||
private DeleteSession(DedupeTotalDataImportProgressVo progress, AccessScope scope, Long groupId) {
|
private DeleteSession(DedupeTotalDataImportProgressVo progress, AccessScope scope, Long groupId,
|
||||||
|
Long companyId) {
|
||||||
this.progress = progress;
|
this.progress = progress;
|
||||||
this.scope = scope;
|
this.scope = scope;
|
||||||
this.groupId = groupId;
|
this.groupId = groupId;
|
||||||
|
this.companyId = companyId;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void acceptRow(Map<Integer, String> rowMap) {
|
private void acceptRow(Map<Integer, String> rowMap) {
|
||||||
@@ -1124,7 +1110,7 @@ public class DedupeTotalDataService {
|
|||||||
// 批量删除:攒批 + IN 一次删,避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1
|
// 批量删除:攒批 + IN 一次删,避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1
|
||||||
pendingDeletes.add(asin);
|
pendingDeletes.add(asin);
|
||||||
if (pendingDeletes.size() >= DELETE_BATCH_SIZE) {
|
if (pendingDeletes.size() >= DELETE_BATCH_SIZE) {
|
||||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
|
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId, companyId);
|
||||||
pendingDeletes.clear();
|
pendingDeletes.clear();
|
||||||
}
|
}
|
||||||
updateProgress();
|
updateProgress();
|
||||||
@@ -1146,7 +1132,7 @@ public class DedupeTotalDataService {
|
|||||||
|
|
||||||
private DedupeTotalDataImportVo finish() {
|
private DedupeTotalDataImportVo finish() {
|
||||||
if (!pendingDeletes.isEmpty()) {
|
if (!pendingDeletes.isEmpty()) {
|
||||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
|
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId, companyId);
|
||||||
pendingDeletes.clear();
|
pendingDeletes.clear();
|
||||||
}
|
}
|
||||||
updateProgress();
|
updateProgress();
|
||||||
@@ -1167,7 +1153,9 @@ public class DedupeTotalDataService {
|
|||||||
DedupeTotalDataEntity entity = getAccessibleById(id, operatorId);
|
DedupeTotalDataEntity entity = getAccessibleById(id, operatorId);
|
||||||
AdminUserEntity operator = getOperator(operatorId);
|
AdminUserEntity operator = getOperator(operatorId);
|
||||||
String dataValue = normalizeComparableValue(request.getDataValue());
|
String dataValue = normalizeComparableValue(request.getDataValue());
|
||||||
ensureUnique(dataValue, id);
|
// 改值时的重名比对认这一行自己的归属,不认操作者的 tab:超管在别家页签里改别家的行,
|
||||||
|
// 要比的是那一家的库。
|
||||||
|
ensureUnique(dataValue, id, entity.getCompanyId());
|
||||||
entity.setDataValue(dataValue);
|
entity.setDataValue(dataValue);
|
||||||
ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), operator);
|
ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), operator);
|
||||||
entity.setGroupId(group.getId());
|
entity.setGroupId(group.getId());
|
||||||
@@ -1207,32 +1195,58 @@ public class DedupeTotalDataService {
|
|||||||
return normalizeExcelText(value).toUpperCase(Locale.ROOT);
|
return normalizeExcelText(value).toUpperCase(Locale.ROOT);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void ensureUnique(String dataValue, Long excludeId) {
|
/**
|
||||||
|
* 「已存在」只在一家公司内成立:本表唯一键自 V145 起是 uk_company_data_value(company_id,
|
||||||
|
* data_value),同值本来就允许归两家。此前这一刀不带公司,结果是 B 公司导入过的 ASIN
|
||||||
|
* 会让 A 公司的「新增」报"数据已存在"——用户看到的是"我加不进去",而库里根本没有他的行。
|
||||||
|
*/
|
||||||
|
private void ensureUnique(String dataValue, Long excludeId, Long companyId) {
|
||||||
|
requireScopedCompanyId(companyId);
|
||||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||||
|
.eq(DedupeTotalDataEntity::getCompanyId, companyId)
|
||||||
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
|
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
|
||||||
.last("LIMIT 1");
|
.last("LIMIT 1");
|
||||||
DedupeTotalDataEntity exists = dedupeTotalDataMapper.selectOne(query);
|
DedupeTotalDataEntity exists = dedupeTotalDataMapper.selectOne(query);
|
||||||
if (exists != null && (excludeId == null || !exists.getId().equals(excludeId))) {
|
if (exists != null && (excludeId == null || !exists.getId().equals(excludeId))) {
|
||||||
throw new BusinessException("数据已存在");
|
throw new BusinessException("本公司已存在该数据");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean existsDataValue(String dataValue) {
|
private boolean existsDataValue(String dataValue, Long companyId) {
|
||||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||||
|
.eq(DedupeTotalDataEntity::getCompanyId, companyId)
|
||||||
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
|
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
|
||||||
.last("LIMIT 1");
|
.last("LIMIT 1");
|
||||||
return dedupeTotalDataMapper.selectOne(query) != null;
|
return dedupeTotalDataMapper.selectOne(query) != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 公司身份闸门:本表每一条按公司取的判据都从这一句过。
|
||||||
|
*
|
||||||
|
* <p>值由 controller 在请求线程上算好传进来({@code DataReadScope.readCompanyIdOrOwn}:超管认
|
||||||
|
* tab、非超管恒本公司)。这里只判空,不补默认——导入与去重跑在虚拟线程上,那时已经没有请求
|
||||||
|
* 上下文可查了,兜底只能兜出一个"谁都比对不上"的空集。</p>
|
||||||
|
*/
|
||||||
|
static void requireScopedCompanyId(Long companyId) {
|
||||||
|
if (companyId == null) {
|
||||||
|
throw new BusinessException(
|
||||||
|
"缺少归属公司:请先在页面上方选择公司,再操作去重数据库");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/** 批量删除大小:单事务 IN 删除的阈值,兼顾锁窗口与事务日志。 */
|
/** 批量删除大小:单事务 IN 删除的阈值,兼顾锁窗口与事务日志。 */
|
||||||
private static final int DELETE_BATCH_SIZE = 500;
|
private static final int DELETE_BATCH_SIZE = 500;
|
||||||
|
|
||||||
/** 整批一个独立事务按 IN 一次删除;返回实际删除行数(计入 deletedCount,不再额外计 skipped)。 */
|
/** 整批一个独立事务按 IN 一次删除;返回实际删除行数(计入 deletedCount,不再额外计 skipped)。 */
|
||||||
private int deleteBatchByDataValues(List<String> dataValues, AccessScope scope, Long groupId) {
|
/** 包内可见只为让用例抓到真正的删除谓词(同 {@link #requireCompanyAttribution} 的先例)。 */
|
||||||
|
int deleteBatchByDataValues(List<String> dataValues, AccessScope scope, Long groupId,
|
||||||
|
Long companyId) {
|
||||||
|
requireScopedCompanyId(companyId);
|
||||||
List<String> batch = List.copyOf(dataValues);
|
List<String> batch = List.copyOf(dataValues);
|
||||||
Integer deleted = newRequiresNewTemplate().execute(status -> {
|
Integer deleted = newRequiresNewTemplate().execute(status -> {
|
||||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||||
|
.eq(DedupeTotalDataEntity::getCompanyId, companyId)
|
||||||
.in(DedupeTotalDataEntity::getDataValue, batch);
|
.in(DedupeTotalDataEntity::getDataValue, batch);
|
||||||
applyGroupScope(query, scope, groupId);
|
applyGroupScope(query, scope, groupId);
|
||||||
return dedupeTotalDataMapper.delete(query);
|
return dedupeTotalDataMapper.delete(query);
|
||||||
@@ -1512,6 +1526,6 @@ public class DedupeTotalDataService {
|
|||||||
* @param wholeCompany 公司超管:不做分组维度裁剪,userIds = 本公司全体成员,
|
* @param wholeCompany 公司超管:不做分组维度裁剪,userIds = 本公司全体成员,
|
||||||
* 查询条件退化为 uploader_user_id IN (成员)——公司超管的"组"就是整家公司
|
* 查询条件退化为 uploader_user_id IN (成员)——公司超管的"组"就是整家公司
|
||||||
*/
|
*/
|
||||||
private record AccessScope(boolean allUsers, Set<Long> groupIds, Set<Long> userIds, boolean wholeCompany) {
|
record AccessScope(boolean allUsers, Set<Long> groupIds, Set<Long> userIds, boolean wholeCompany) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,11 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "https://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "https://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||||
<mapper namespace="com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper">
|
<mapper namespace="com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper">
|
||||||
<!-- 由注解 SQL 平移而来,SQL 文本未改写;语句 id 与接口方法名逐字一致。
|
<!-- 由注解 SQL 平移而来,语句 id 与接口方法名逐字一致;等价性由 MigratedSqlEquivalenceTest
|
||||||
等价性由 MigratedSqlEquivalenceTest 按搬迁前的 fixture 逐条比对。 -->
|
按基准账本逐条比对。
|
||||||
|
一处例外要说清:selectExistingDataValues 不是平移。它当年是 @SelectProvider、没进过这本账,
|
||||||
|
账本里与它同名的那条基准记的就是它按公司收窄之后的形态;同批摘掉了已零调用者的
|
||||||
|
selectAllDataValues,所以账本行数与 select 总数双双不动。 -->
|
||||||
<insert id="insertBatchIgnore">
|
<insert id="insertBatchIgnore">
|
||||||
INSERT IGNORE INTO biz_dedupe_total_data
|
INSERT IGNORE INTO biz_dedupe_total_data
|
||||||
(data_value, country, group_id, uploader_user_id, uploader_username, company_id)
|
(data_value, country, group_id, uploader_user_id, uploader_username, company_id)
|
||||||
@@ -11,7 +14,10 @@
|
|||||||
(#{row.dataValue}, #{row.country}, #{row.groupId}, #{row.uploaderUserId}, #{row.uploaderUsername}, #{row.companyId})
|
(#{row.dataValue}, #{row.country}, #{row.groupId}, #{row.uploaderUserId}, #{row.uploaderUsername}, #{row.companyId})
|
||||||
</foreach>
|
</foreach>
|
||||||
</insert>
|
</insert>
|
||||||
<select id="selectAllDataValues" resultType="string">
|
<select id="selectExistingDataValues" resultType="string">
|
||||||
SELECT data_value FROM biz_dedupe_total_data
|
SELECT data_value FROM biz_dedupe_total_data
|
||||||
|
WHERE company_id = #{companyId}
|
||||||
|
AND data_value IN
|
||||||
|
<foreach collection="values" item="value" open="(" separator="," close=")">#{value}</foreach>
|
||||||
</select>
|
</select>
|
||||||
</mapper>
|
</mapper>
|
||||||
|
|||||||
@@ -71,7 +71,10 @@ class CatchSilenceRatchetTest {
|
|||||||
// 随死代码一起离开树;实测重抛数 387、不重抛与只 log 两数未涨 ⇒ 没有任何一处"异常会往外走"
|
// 随死代码一起离开树;实测重抛数 387、不重抛与只 log 两数未涨 ⇒ 没有任何一处"异常会往外走"
|
||||||
// 被改回吞掉。这一格的本意是"重抛只许涨不许跌",所以调基线必须带着这样的逐处出处;
|
// 被改回吞掉。这一格的本意是"重抛只许涨不许跌",所以调基线必须带着这样的逐处出处;
|
||||||
// 拿"总数变小了"当理由不算——那种情况下真正要看的是不重抛那一侧有没有涨。
|
// 拿"总数变小了"当理由不算——那种情况下真正要看的是不重抛那一侧有没有涨。
|
||||||
static final int RETHROW_FLOOR = 387;
|
// 10-09 再下调 387 -> 386,出处点名:去重库按公司收口那一笔删掉了 DedupeTotalDataService.deleteFromExcel
|
||||||
|
// 的两个零调用者重载,其中 `throw new BusinessException("删除 Excel 匹配数据失败", e)` 是 RETHROW 形状,
|
||||||
|
// 随死代码一起离开树;实测重抛 386、不重抛与只 log 两数未涨(那两格仍绿)=> 没有任何一处「异常会往外走」被改回吞掉。
|
||||||
|
static final int RETHROW_FLOOR = 386;
|
||||||
static final int STRICT_NOT_RETHROW = 559;
|
static final int STRICT_NOT_RETHROW = 559;
|
||||||
static final int STRICT_LOG_ONLY = 172; // 严格口径:块内每条顶层语句都是 log 调用才算
|
static final int STRICT_LOG_ONLY = 172; // 严格口径:块内每条顶层语句都是 log 调用才算
|
||||||
static final int STRICT_EMPTY = 56;
|
static final int STRICT_EMPTY = 56;
|
||||||
|
|||||||
+2
-2
@@ -119,7 +119,7 @@ class CollectDataBrandRecheckInjectorTest {
|
|||||||
String payload = new ObjectMapper().writeValueAsString(List.of(row("B01", "Nike"), row("B02", "Nike")));
|
String payload = new ObjectMapper().writeValueAsString(List.of(row("B01", "Nike"), row("B02", "Nike")));
|
||||||
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(payload);
|
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(payload);
|
||||||
CollectDataResultRowVo kept = row("B01", "Nike");
|
CollectDataResultRowVo kept = row("B01", "Nike");
|
||||||
when(collectDataBatchQuery.filter(anyList())).thenReturn(
|
when(collectDataBatchQuery.filter(anyList(), any())).thenReturn(
|
||||||
new CollectDataBatchQuery.FilterResult(List.of(kept), 1, 0));
|
new CollectDataBatchQuery.FilterResult(List.of(kept), 1, 0));
|
||||||
when(transientPayloadStorageService.storeResultPayload(eq("COLLECT_DATA"), eq(100L), eq("hash1"),
|
when(transientPayloadStorageService.storeResultPayload(eq("COLLECT_DATA"), eq(100L), eq("hash1"),
|
||||||
eq("recheck-chunk-1"), anyString())).thenReturn("ptr2");
|
eq("recheck-chunk-1"), anyString())).thenReturn("ptr2");
|
||||||
@@ -179,7 +179,7 @@ class CollectDataBrandRecheckInjectorTest {
|
|||||||
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(
|
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(
|
||||||
"[{\"asin\":\"B01\",\"brand\":\"Nike\"}]");
|
"[{\"asin\":\"B01\",\"brand\":\"Nike\"}]");
|
||||||
// 补测窗口内该 ASIN 已被其他任务采走:复查全滤掉
|
// 补测窗口内该 ASIN 已被其他任务采走:复查全滤掉
|
||||||
when(collectDataBatchQuery.filter(anyList())).thenReturn(
|
when(collectDataBatchQuery.filter(anyList(), any())).thenReturn(
|
||||||
new CollectDataBatchQuery.FilterResult(List.of(), 1, 0));
|
new CollectDataBatchQuery.FilterResult(List.of(), 1, 0));
|
||||||
|
|
||||||
GroupOutcome outcome = injector.apply(task(100L), Map.of("nike", RecheckVerdict.CLEAN));
|
GroupOutcome outcome = injector.apply(task(100L), Map.of("nike", RecheckVerdict.CLEAN));
|
||||||
|
|||||||
+1
-1
@@ -145,7 +145,7 @@ class CollectDataServiceTxBoundaryTest {
|
|||||||
}).when(taskChunkMapper).insert(any(TaskChunkEntity.class));
|
}).when(taskChunkMapper).insert(any(TaskChunkEntity.class));
|
||||||
CollectDataResultRowVo row = new CollectDataResultRowVo();
|
CollectDataResultRowVo row = new CollectDataResultRowVo();
|
||||||
row.setAsin("B0COLLECT1");
|
row.setAsin("B0COLLECT1");
|
||||||
lenient().when(collectDataBatchQuery.filter(any())).thenReturn(
|
lenient().when(collectDataBatchQuery.filter(any(), any())).thenReturn(
|
||||||
new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0));
|
new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0));
|
||||||
lenient().when(brandBatchFilter.filter(any())).thenReturn(
|
lenient().when(brandBatchFilter.filter(any())).thenReturn(
|
||||||
new CollectDataBrandBatchFilter.BrandBatchOutcome(List.of(), List.of(), List.of(row)));
|
new CollectDataBrandBatchFilter.BrandBatchOutcome(List.of(), List.of(), List.of(row)));
|
||||||
|
|||||||
+56
-31
@@ -1,7 +1,7 @@
|
|||||||
package com.nanri.aiimage.modules.collectdata.util;
|
package com.nanri.aiimage.modules.collectdata.util;
|
||||||
|
|
||||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||||
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
|
import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService;
|
||||||
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
|
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
|
||||||
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
|
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
@@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test;
|
|||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
@@ -28,25 +29,36 @@ import static org.mockito.Mockito.when;
|
|||||||
*/
|
*/
|
||||||
class CollectDataBatchQueryTest {
|
class CollectDataBatchQueryTest {
|
||||||
|
|
||||||
private DedupeTotalDataMapper dedupeMapper;
|
/** 采集任务归属账号:比对集由它推导,「推不出公司就不比对」那条闸门钉在 dedupe 侧的用例里。 */
|
||||||
|
private static final Long OWNER_USER_ID = 7L;
|
||||||
|
|
||||||
|
private DedupeTotalDataService dedupeService;
|
||||||
private InvalidAsinDataMapper invalidAsinMapper;
|
private InvalidAsinDataMapper invalidAsinMapper;
|
||||||
private CollectDataBatchQuery query;
|
private CollectDataBatchQuery query;
|
||||||
|
|
||||||
@BeforeEach
|
@BeforeEach
|
||||||
void setUp() {
|
void setUp() {
|
||||||
dedupeMapper = mock(DedupeTotalDataMapper.class);
|
dedupeService = mock(DedupeTotalDataService.class);
|
||||||
invalidAsinMapper = mock(InvalidAsinDataMapper.class);
|
invalidAsinMapper = mock(InvalidAsinDataMapper.class);
|
||||||
query = new CollectDataBatchQuery(dedupeMapper, invalidAsinMapper);
|
query = new CollectDataBatchQuery(dedupeService, invalidAsinMapper);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 生产入口已改成 {@code filter(rows, ownerUserId)};本类逐支测的是过滤语义,
|
||||||
|
* 公司归属由这一层统一喂,不逐支重复——归属本身由下面那两支新用例专钉。
|
||||||
|
*/
|
||||||
|
private List<CollectDataResultRowVo> filterByExistingAsin(List<CollectDataResultRowVo> rows) {
|
||||||
|
return query.filter(rows, OWNER_USER_ID).kept();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void test_task_045_asin_brand_normal_default_path() {
|
void test_task_045_asin_brand_normal_default_path() {
|
||||||
// 正常输入:去重表已存在的 ASIN 被过滤,无效品牌表中的品牌被过滤,其余保留。
|
// 正常输入:去重表已存在的 ASIN 被过滤,无效品牌表中的品牌被过滤,其余保留。
|
||||||
when(dedupeMapper.selectExistingDataValues(anyList()))
|
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any()))
|
||||||
.thenReturn(List.of("B000000001"));
|
.thenReturn(Set.of("B000000001"));
|
||||||
when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("Zara")));
|
when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("Zara")));
|
||||||
|
|
||||||
List<CollectDataResultRowVo> out = query.filterByExistingAsin(List.of(
|
List<CollectDataResultRowVo> out = filterByExistingAsin(List.of(
|
||||||
row("B000000001", "Nike"),
|
row("B000000001", "Nike"),
|
||||||
row("B000000002", "Zara"),
|
row("B000000002", "Zara"),
|
||||||
row("B000000003", "H&M")
|
row("B000000003", "H&M")
|
||||||
@@ -68,14 +80,14 @@ class CollectDataBatchQueryTest {
|
|||||||
for (int i = 0; i < 300; i++) {
|
for (int i = 0; i < 300; i++) {
|
||||||
rows.add(row("B" + String.format("%09d", i + 1), "bad"));
|
rows.add(row("B" + String.format("%09d", i + 1), "bad"));
|
||||||
}
|
}
|
||||||
List<String> existing = new ArrayList<>();
|
Set<String> existing = new java.util.HashSet<>();
|
||||||
for (int i = 0; i < 100; i++) {
|
for (int i = 0; i < 100; i++) {
|
||||||
existing.add("D" + String.format("%09d", i + 1));
|
existing.add("D" + String.format("%09d", i + 1));
|
||||||
}
|
}
|
||||||
when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(existing);
|
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())).thenReturn(existing);
|
||||||
when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("bad")));
|
when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("bad")));
|
||||||
|
|
||||||
List<CollectDataResultRowVo> out = query.filterByExistingAsin(rows);
|
List<CollectDataResultRowVo> out = filterByExistingAsin(rows);
|
||||||
|
|
||||||
assertEquals(200, out.size(), "100 行去重 + 300 行无效品牌 = 过滤 400 行");
|
assertEquals(200, out.size(), "100 行去重 + 300 行无效品牌 = 过滤 400 行");
|
||||||
assertEquals("D000000101", out.get(0).getAsin(), "输出首行保持输入顺序");
|
assertEquals("D000000101", out.get(0).getAsin(), "输出首行保持输入顺序");
|
||||||
@@ -87,51 +99,51 @@ class CollectDataBatchQueryTest {
|
|||||||
@Test
|
@Test
|
||||||
void test_task_045_asin_brand_normal_repeated_operation_is_idempotent() {
|
void test_task_045_asin_brand_normal_repeated_operation_is_idempotent() {
|
||||||
// 幂等:同一输入重复执行输出一致,两次都发批量查询。
|
// 幂等:同一输入重复执行输出一致,两次都发批量查询。
|
||||||
when(dedupeMapper.selectExistingDataValues(anyList()))
|
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any()))
|
||||||
.thenReturn(List.of("B000000001"));
|
.thenReturn(Set.of("B000000001"));
|
||||||
when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("bad")));
|
when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("bad")));
|
||||||
List<CollectDataResultRowVo> rows = List.of(
|
List<CollectDataResultRowVo> rows = List.of(
|
||||||
row("B000000001", "bad"),
|
row("B000000001", "bad"),
|
||||||
row("B000000002", "good")
|
row("B000000002", "good")
|
||||||
);
|
);
|
||||||
|
|
||||||
List<CollectDataResultRowVo> first = query.filterByExistingAsin(rows);
|
List<CollectDataResultRowVo> first = filterByExistingAsin(rows);
|
||||||
List<CollectDataResultRowVo> second = query.filterByExistingAsin(rows);
|
List<CollectDataResultRowVo> second = filterByExistingAsin(rows);
|
||||||
|
|
||||||
assertEquals(first, second, "重复执行输出一致");
|
assertEquals(first, second, "重复执行输出一致");
|
||||||
assertEquals(1, first.size());
|
assertEquals(1, first.size());
|
||||||
verify(dedupeMapper, times(2)).selectExistingDataValues(anyList());
|
verify(dedupeService, times(2)).findExistingComparableValuesForOwner(anyList(), any());
|
||||||
verify(invalidAsinMapper, times(2)).selectList(any());
|
verify(invalidAsinMapper, times(2)).selectList(any());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void test_task_045_asin_brand_boundary_empty_input() {
|
void test_task_045_asin_brand_boundary_empty_input() {
|
||||||
// 空输入:空列表/无 ASIN 无品牌的行集合不触发数据库查询,安全跳过。
|
// 空输入:空列表/无 ASIN 无品牌的行集合不触发数据库查询,安全跳过。
|
||||||
List<CollectDataResultRowVo> empty = query.filterByExistingAsin(List.of());
|
List<CollectDataResultRowVo> empty = filterByExistingAsin(List.of());
|
||||||
assertEquals(0, empty.size(), "空列表返回空");
|
assertEquals(0, empty.size(), "空列表返回空");
|
||||||
|
|
||||||
List<CollectDataResultRowVo> blankRows = query.filterByExistingAsin(List.of(
|
List<CollectDataResultRowVo> blankRows = filterByExistingAsin(List.of(
|
||||||
row("", ""), row(null, null)
|
row("", ""), row(null, null)
|
||||||
));
|
));
|
||||||
assertEquals(2, blankRows.size(), "无 ASIN 无品牌行不触发查询且全保留");
|
assertEquals(2, blankRows.size(), "无 ASIN 无品牌行不触发查询且全保留");
|
||||||
verify(dedupeMapper, never()).selectExistingDataValues(anyList());
|
verify(dedupeService, never()).findExistingComparableValuesForOwner(anyList(), any());
|
||||||
verify(invalidAsinMapper, never()).selectList(any());
|
verify(invalidAsinMapper, never()).selectList(any());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void test_task_045_asin_brand_boundary_single_item() {
|
void test_task_045_asin_brand_boundary_single_item() {
|
||||||
// 单元素:单行命中/未命中均正确,不依赖批量路径。
|
// 单元素:单行命中/未命中均正确,不依赖批量路径。
|
||||||
when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(List.of());
|
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())).thenReturn(Set.of());
|
||||||
when(invalidAsinMapper.selectList(any())).thenReturn(List.of());
|
when(invalidAsinMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
|
||||||
List<CollectDataResultRowVo> out = query.filterByExistingAsin(List.of(
|
List<CollectDataResultRowVo> out = filterByExistingAsin(List.of(
|
||||||
row("B000000001", "solo")
|
row("B000000001", "solo")
|
||||||
));
|
));
|
||||||
assertEquals(1, out.size(), "单行未命中保留");
|
assertEquals(1, out.size(), "单行未命中保留");
|
||||||
assertEquals("solo", out.get(0).getBrand());
|
assertEquals("solo", out.get(0).getBrand());
|
||||||
|
|
||||||
when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(List.of("B000000001"));
|
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())).thenReturn(Set.of("B000000001"));
|
||||||
List<CollectDataResultRowVo> hit = query.filterByExistingAsin(List.of(
|
List<CollectDataResultRowVo> hit = filterByExistingAsin(List.of(
|
||||||
row("B000000001", "solo")
|
row("B000000001", "solo")
|
||||||
));
|
));
|
||||||
assertEquals(0, hit.size(), "单行命中去重表被过滤");
|
assertEquals(0, hit.size(), "单行命中去重表被过滤");
|
||||||
@@ -144,26 +156,26 @@ class CollectDataBatchQueryTest {
|
|||||||
for (int i = 0; i < 5000; i++) {
|
for (int i = 0; i < 5000; i++) {
|
||||||
rows.add(row("B" + String.format("%09d", i), "brand-" + i));
|
rows.add(row("B" + String.format("%09d", i), "brand-" + i));
|
||||||
}
|
}
|
||||||
when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(List.of());
|
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())).thenReturn(Set.of());
|
||||||
when(invalidAsinMapper.selectList(any())).thenReturn(List.of());
|
when(invalidAsinMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
|
||||||
List<CollectDataResultRowVo> out = query.filterByExistingAsin(rows);
|
List<CollectDataResultRowVo> out = filterByExistingAsin(rows);
|
||||||
|
|
||||||
assertEquals(5000, out.size(), "超大集合全量保留");
|
assertEquals(5000, out.size(), "超大集合全量保留");
|
||||||
verify(dedupeMapper).selectExistingDataValues(anyList());
|
verify(dedupeService).findExistingComparableValuesForOwner(anyList(), any());
|
||||||
verify(invalidAsinMapper).selectList(any());
|
verify(invalidAsinMapper).selectList(any());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void test_task_045_asin_brand_invalid_input_rejected() {
|
void test_task_045_asin_brand_invalid_input_rejected() {
|
||||||
// 非法参数:null 行安全跳过;数据库返回 null 集合安全处理。
|
// 非法参数:null 行安全跳过;数据库返回 null 集合安全处理。
|
||||||
when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(null);
|
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())).thenReturn(null);
|
||||||
when(invalidAsinMapper.selectList(any())).thenReturn(null);
|
when(invalidAsinMapper.selectList(any())).thenReturn(null);
|
||||||
|
|
||||||
List<CollectDataResultRowVo> rows = new ArrayList<>();
|
List<CollectDataResultRowVo> rows = new ArrayList<>();
|
||||||
rows.add(null);
|
rows.add(null);
|
||||||
rows.add(row("B000000001", " Nike "));
|
rows.add(row("B000000001", " Nike "));
|
||||||
List<CollectDataResultRowVo> out = query.filterByExistingAsin(rows);
|
List<CollectDataResultRowVo> out = filterByExistingAsin(rows);
|
||||||
|
|
||||||
assertEquals(1, out.size(), "null 行不丢弃");
|
assertEquals(1, out.size(), "null 行不丢弃");
|
||||||
assertEquals("B000000001", out.get(0).getAsin(), "正常行保留");
|
assertEquals("B000000001", out.get(0).getAsin(), "正常行保留");
|
||||||
@@ -173,26 +185,39 @@ class CollectDataBatchQueryTest {
|
|||||||
@Test
|
@Test
|
||||||
void test_task_045_asin_brand_dependency_failure_releases_resources() {
|
void test_task_045_asin_brand_dependency_failure_releases_resources() {
|
||||||
// 依赖失败:去重表查询抛错时异常向上传播;恢复后再次执行成功,不残留状态。
|
// 依赖失败:去重表查询抛错时异常向上传播;恢复后再次执行成功,不残留状态。
|
||||||
when(dedupeMapper.selectExistingDataValues(anyList()))
|
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any()))
|
||||||
.thenThrow(new RuntimeException("db down"))
|
.thenThrow(new RuntimeException("db down"))
|
||||||
.thenReturn(List.of("B000000001"));
|
.thenReturn(Set.of("B000000001"));
|
||||||
when(invalidAsinMapper.selectList(any())).thenReturn(List.of());
|
when(invalidAsinMapper.selectList(any())).thenReturn(List.of());
|
||||||
|
|
||||||
boolean threw = false;
|
boolean threw = false;
|
||||||
try {
|
try {
|
||||||
query.filterByExistingAsin(List.of(row("B000000001", "nike")));
|
filterByExistingAsin(List.of(row("B000000001", "nike")));
|
||||||
} catch (RuntimeException ex) {
|
} catch (RuntimeException ex) {
|
||||||
threw = "db down".equals(ex.getMessage());
|
threw = "db down".equals(ex.getMessage());
|
||||||
}
|
}
|
||||||
assertTrue(threw, "去重查询失败向上抛错");
|
assertTrue(threw, "去重查询失败向上抛错");
|
||||||
|
|
||||||
List<CollectDataResultRowVo> out = query.filterByExistingAsin(List.of(
|
List<CollectDataResultRowVo> out = filterByExistingAsin(List.of(
|
||||||
row("B000000001", "nike"), row("B000000002", "adidas")
|
row("B000000001", "nike"), row("B000000002", "adidas")
|
||||||
));
|
));
|
||||||
assertEquals(1, out.size(), "恢复后过滤正确");
|
assertEquals(1, out.size(), "恢复后过滤正确");
|
||||||
assertEquals("B000000002", out.get(0).getAsin(), "恢复后命中逻辑仍生效");
|
assertEquals("B000000002", out.get(0).getAsin(), "恢复后命中逻辑仍生效");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 采集结果按去重库过滤时,比对集必须由「任务归属账号」那一家推导。
|
||||||
|
* 这一支钉的是参数真的传下去了:换成常量或漏传,下面这条 eq 就红。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void comparisonSetIsRequestedForTheTaskOwnerAccount() {
|
||||||
|
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())).thenReturn(Set.of());
|
||||||
|
|
||||||
|
filterByExistingAsin(List.of(row("B000000001", "nike")));
|
||||||
|
|
||||||
|
verify(dedupeService).findExistingComparableValuesForOwner(List.of("B000000001"), OWNER_USER_ID);
|
||||||
|
}
|
||||||
|
|
||||||
private static CollectDataResultRowVo row(String asin, String brand) {
|
private static CollectDataResultRowVo row(String asin, String brand) {
|
||||||
CollectDataResultRowVo row = new CollectDataResultRowVo();
|
CollectDataResultRowVo row = new CollectDataResultRowVo();
|
||||||
row.setAsin(asin);
|
row.setAsin(asin);
|
||||||
|
|||||||
+27
-1
@@ -15,7 +15,8 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
|
|||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@code insertBatchIgnore} 生成 SQL 的审计:去重库写侧必须显式带 {@code company_id}。
|
* 去重库两条自定义语句的 SQL 审计:写侧 {@code insertBatchIgnore} 必须显式落 {@code company_id},
|
||||||
|
* 比对侧 {@code selectExistingDataValues} 必须按 {@code company_id} 收窄。
|
||||||
*
|
*
|
||||||
* <p>为什么必须单独有一支:服务层测试用 mock 捕获的是实体对象,看不到最终 SQL——本表写侧
|
* <p>为什么必须单独有一支:服务层测试用 mock 捕获的是实体对象,看不到最终 SQL——本表写侧
|
||||||
* 靠的是注解里的列清单(不是 MyBatis-Plus 的实体字段推导),把 {@code company_id} 从清单里漏掉
|
* 靠的是注解里的列清单(不是 MyBatis-Plus 的实体字段推导),把 {@code company_id} 从清单里漏掉
|
||||||
@@ -43,6 +44,31 @@ class DedupeTotalDataMapperSqlTest {
|
|||||||
assertFalse(sql.contains("$"), "不得出现 ${} 拼接面: " + sql);
|
assertFalse(sql.contains("$"), "不得出现 ${} 拼接面: " + sql);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 比对集的租户谓词审计。断的是最终 SQL 而不是方法签名:这条从 @SelectProvider 迁到 XML 之后,
|
||||||
|
* 少写一个 {@code AND company_id = #{companyId}} 在 Java 侧照样编译通过、照样返回行,
|
||||||
|
* 而后果是「别家导入的 ASIN 被当成已存在」——去重执行少行、导入静默跳过。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void selectExistingValuesAreScopedToOneCompany() throws Exception {
|
||||||
|
Configuration configuration = MapperXmlSupport.configurationWith(DedupeTotalDataMapper.class);
|
||||||
|
MappedStatement statement = configuration.getMappedStatement(
|
||||||
|
DedupeTotalDataMapper.class.getName() + ".selectExistingDataValues");
|
||||||
|
BoundSql boundSql = statement.getBoundSql(java.util.Map.of(
|
||||||
|
"values", List.of("B000000001", "B000000002"), "companyId", 3L));
|
||||||
|
|
||||||
|
String sql = boundSql.getSql().replaceAll("\\s+", " ");
|
||||||
|
assertTrue(sql.contains("WHERE company_id = ?"),
|
||||||
|
"比对集必须按 company_id 收窄,否则一次去重会看见全平台: " + sql);
|
||||||
|
assertTrue(sql.contains("data_value IN ("),
|
||||||
|
"值集合仍按 IN 批量比对(收窄不该把它拆成逐行查): " + sql);
|
||||||
|
assertEquals(3, boundSql.getParameterMappings().size(),
|
||||||
|
"2 个值 + 1 个公司 = 3 个占位符: " + sql);
|
||||||
|
assertEquals("companyId", boundSql.getParameterMappings().get(0).getProperty(),
|
||||||
|
"第一个占位符就是公司(谓词排在 IN 之前,走 uk_company_data_value 前缀): " + sql);
|
||||||
|
assertFalse(sql.contains("$"), "不得出现 ${} 拼接面: " + sql);
|
||||||
|
}
|
||||||
|
|
||||||
private static BoundSql boundSqlFor(int rowCount) throws Exception {
|
private static BoundSql boundSqlFor(int rowCount) throws Exception {
|
||||||
// SQL 已从注解迁到 mapper XML(档 B 口径),原文与语句都要从 XML 那侧取
|
// SQL 已从注解迁到 mapper XML(档 B 口径),原文与语句都要从 XML 那侧取
|
||||||
String script = MapperXmlSupport.statementBody(DedupeTotalDataMapper.class, "insertBatchIgnore");
|
String script = MapperXmlSupport.statementBody(DedupeTotalDataMapper.class, "insertBatchIgnore");
|
||||||
|
|||||||
+4
-2
@@ -69,6 +69,8 @@ class DedupeRunProgressResumeTest {
|
|||||||
@Mock
|
@Mock
|
||||||
private DedupeTotalDataService dedupeTotalDataService;
|
private DedupeTotalDataService dedupeTotalDataService;
|
||||||
@Mock
|
@Mock
|
||||||
|
private com.nanri.aiimage.common.service.CompanyScopeSupport companyScopeSupport;
|
||||||
|
@Mock
|
||||||
private StringRedisTemplate redisTemplate;
|
private StringRedisTemplate redisTemplate;
|
||||||
@Mock
|
@Mock
|
||||||
private ValueOperations<String, String> valueOperations;
|
private ValueOperations<String, String> valueOperations;
|
||||||
@@ -102,7 +104,7 @@ class DedupeRunProgressResumeTest {
|
|||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
return new DedupeRunService(fileTaskMapper, fileResultMapper, storageProperties,
|
return new DedupeRunService(fileTaskMapper, fileResultMapper, storageProperties,
|
||||||
ossStorageService, dedupeTotalDataService, redisTemplate, objectMapper,
|
ossStorageService, dedupeTotalDataService, companyScopeSupport, redisTemplate, objectMapper,
|
||||||
new InstanceMetadata("server-110"));
|
new InstanceMetadata("server-110"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -425,7 +427,7 @@ class DedupeRunProgressResumeTest {
|
|||||||
// 异步 worker 是否来得及读这个目录取决于线程时序,只做兜底、不参与断言
|
// 异步 worker 是否来得及读这个目录取决于线程时序,只做兜底、不参与断言
|
||||||
lenient().when(storageProperties.getLocalTempDir()).thenReturn("D:/definitely-not-a-real-dedupe-dir/xx");
|
lenient().when(storageProperties.getLocalTempDir()).thenReturn("D:/definitely-not-a-real-dedupe-dir/xx");
|
||||||
DedupeRunService service = new DedupeRunService(fileTaskMapper, fileResultMapper, storageProperties,
|
DedupeRunService service = new DedupeRunService(fileTaskMapper, fileResultMapper, storageProperties,
|
||||||
ossStorageService, dedupeTotalDataService, redisTemplate, objectMapper,
|
ossStorageService, dedupeTotalDataService, companyScopeSupport, redisTemplate, objectMapper,
|
||||||
new InstanceMetadata("server-110"));
|
new InstanceMetadata("server-110"));
|
||||||
|
|
||||||
DedupeRunProgressVo submitted = service.submitRun(request(1));
|
DedupeRunProgressVo submitted = service.submitRun(request(1));
|
||||||
|
|||||||
+19
-3
@@ -25,6 +25,8 @@ import java.util.Set;
|
|||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -144,6 +146,20 @@ class DedupeRunServiceCleanRulesTest {
|
|||||||
return rows.stream().map(DedupeCandidateRow::asinValue).toList();
|
return rows.stream().map(DedupeCandidateRow::asinValue).toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 去重执行拿哪一家的库比对:提交人自己那一家。
|
||||||
|
* 摘掉 DedupeRunService 里往下传的 companyId,这支会红(比对收到的是 null 或别的值)。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void comparisonSetIsTakenForTheSubmittingAccountCompanyOnly(@TempDir Path tempDir) throws Exception {
|
||||||
|
runClean(tempDir, List.<String[]>of(
|
||||||
|
new String[]{"1", "B0AAAAAA01", "英国", "10.00", "FBM", "品牌A", "在售", "2026-09-01"},
|
||||||
|
new String[]{"1_2", "B0AAAAAA02", "英国", "11.00", "FBM", "品牌A", "在售", "2026-09-01"}),
|
||||||
|
List.of("id", "ASIN", "国家", "价格", "品牌"));
|
||||||
|
|
||||||
|
verify(dedupeTotalDataService).findExistingComparableValues(any(), eq(1L));
|
||||||
|
}
|
||||||
|
|
||||||
private File runClean(Path tempDir, List<String[]> rows, List<String> selectedColumns) throws Exception {
|
private File runClean(Path tempDir, List<String[]> rows, List<String> selectedColumns) throws Exception {
|
||||||
File input = tempDir.resolve("input.xlsx").toFile();
|
File input = tempDir.resolve("input.xlsx").toFile();
|
||||||
File output = tempDir.resolve("output.xlsx").toFile();
|
File output = tempDir.resolve("output.xlsx").toFile();
|
||||||
@@ -151,8 +167,8 @@ class DedupeRunServiceCleanRulesTest {
|
|||||||
|
|
||||||
DedupeRunService service = new DedupeRunService(
|
DedupeRunService service = new DedupeRunService(
|
||||||
fileTaskMapper, fileResultMapper, storageProperties, ossStorageService, dedupeTotalDataService,
|
fileTaskMapper, fileResultMapper, storageProperties, ossStorageService, dedupeTotalDataService,
|
||||||
null, null, new com.nanri.aiimage.config.InstanceMetadata("test-instance"));
|
null, null, null, new com.nanri.aiimage.config.InstanceMetadata("test-instance"));
|
||||||
when(dedupeTotalDataService.findExistingComparableValues(any())).thenReturn(Set.of());
|
when(dedupeTotalDataService.findExistingComparableValues(any(), any())).thenReturn(Set.of());
|
||||||
when(dedupeTotalDataService.normalizeComparableValueOrBlank(any()))
|
when(dedupeTotalDataService.normalizeComparableValueOrBlank(any()))
|
||||||
.thenAnswer(invocation -> {
|
.thenAnswer(invocation -> {
|
||||||
String value = invocation.getArgument(0);
|
String value = invocation.getArgument(0);
|
||||||
@@ -160,7 +176,7 @@ class DedupeRunServiceCleanRulesTest {
|
|||||||
});
|
});
|
||||||
|
|
||||||
ReflectionTestUtils.invokeMethod(service, "cleanExcelByStreamRules",
|
ReflectionTestUtils.invokeMethod(service, "cleanExcelByStreamRules",
|
||||||
input, output, selectedColumns, false, true, true);
|
input, output, selectedColumns, false, true, true, 1L);
|
||||||
return output;
|
return output;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+83
-24
@@ -44,6 +44,7 @@ import java.util.Set;
|
|||||||
import java.util.zip.ZipEntry;
|
import java.util.zip.ZipEntry;
|
||||||
import java.util.zip.ZipInputStream;
|
import java.util.zip.ZipInputStream;
|
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
@@ -77,7 +78,6 @@ class DedupeTotalDataServiceTest {
|
|||||||
void createRecordsUploaderIdentity() {
|
void createRecordsUploaderIdentity() {
|
||||||
AdminUserEntity uploader = user(23L, "normal", "member-a");
|
AdminUserEntity uploader = user(23L, "normal", "member-a");
|
||||||
when(adminUserMapper.selectById(23L)).thenReturn(uploader);
|
when(adminUserMapper.selectById(23L)).thenReturn(uploader);
|
||||||
when(companyScopeSupport.requireCompanyId(any(AdminUserEntity.class))).thenReturn(1L);
|
|
||||||
stubWritableGroup(23L, 7L);
|
stubWritableGroup(23L, 7L);
|
||||||
when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null);
|
when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null);
|
||||||
when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class))).thenAnswer(invocation -> {
|
when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class))).thenAnswer(invocation -> {
|
||||||
@@ -98,7 +98,7 @@ class DedupeTotalDataServiceTest {
|
|||||||
DedupeTotalDataCreateRequest request = new DedupeTotalDataCreateRequest();
|
DedupeTotalDataCreateRequest request = new DedupeTotalDataCreateRequest();
|
||||||
request.setDataValue(" b012345678 ");
|
request.setDataValue(" b012345678 ");
|
||||||
request.setGroupId(7L);
|
request.setGroupId(7L);
|
||||||
DedupeTotalDataItemVo item = service.create(request, 23L);
|
DedupeTotalDataItemVo item = service.create(request, 1L, 23L);
|
||||||
|
|
||||||
ArgumentCaptor<DedupeTotalDataEntity> captor = ArgumentCaptor.forClass(DedupeTotalDataEntity.class);
|
ArgumentCaptor<DedupeTotalDataEntity> captor = ArgumentCaptor.forClass(DedupeTotalDataEntity.class);
|
||||||
verify(dedupeTotalDataMapper).insert(captor.capture());
|
verify(dedupeTotalDataMapper).insert(captor.capture());
|
||||||
@@ -184,13 +184,12 @@ class DedupeTotalDataServiceTest {
|
|||||||
@Test
|
@Test
|
||||||
void duplicateExcelValueKeepsOriginalUploader() throws Exception {
|
void duplicateExcelValueKeepsOriginalUploader() throws Exception {
|
||||||
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
|
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
|
||||||
when(companyScopeSupport.requireCompanyId(any(AdminUserEntity.class))).thenReturn(1L);
|
|
||||||
stubWritableGroup(23L, 7L);
|
stubWritableGroup(23L, 7L);
|
||||||
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678")))
|
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"), 1L))
|
||||||
.thenReturn(List.of("B012345678"));
|
.thenReturn(List.of("B012345678"));
|
||||||
MockMultipartFile file = asinWorkbook("B012345678");
|
MockMultipartFile file = asinWorkbook("B012345678");
|
||||||
|
|
||||||
var result = service.importFromExcel(file, 7L, 23L);
|
var result = service.importFromExcel(file, 7L, 1L, 23L);
|
||||||
|
|
||||||
assertEquals(0, result.getInsertedCount());
|
assertEquals(0, result.getInsertedCount());
|
||||||
assertEquals(1, result.getSkippedCount());
|
assertEquals(1, result.getSkippedCount());
|
||||||
@@ -199,17 +198,15 @@ class DedupeTotalDataServiceTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
void importRejectsOperatorWithoutCompany() throws Exception {
|
void importRejectsOperatorWithoutCompany() throws Exception {
|
||||||
AdminUserEntity noCompany = user(23L, "normal", "member-a");
|
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
|
||||||
noCompany.setCompanyId(null);
|
|
||||||
when(adminUserMapper.selectById(23L)).thenReturn(noCompany);
|
|
||||||
when(companyScopeSupport.requireCompanyId(any(AdminUserEntity.class)))
|
|
||||||
.thenThrow(new BusinessException(403, "当前账号没有所属公司,无法按公司读写数据"));
|
|
||||||
|
|
||||||
|
// 公司身份由 controller 在请求线程上算好传进来(超管认 tab、非超管认自己那一家);
|
||||||
|
// service 这道闸门钉的是「没算出来就不许写」,它是最后那道。
|
||||||
BusinessException exception = assertThrows(
|
BusinessException exception = assertThrows(
|
||||||
BusinessException.class,
|
BusinessException.class,
|
||||||
() -> service.importFromExcel(asinWorkbook("B012345678"), 7L, 23L));
|
() -> service.importFromExcel(asinWorkbook("B012345678"), 7L, null, 23L));
|
||||||
|
|
||||||
assertEquals("当前账号没有所属公司,无法按公司读写数据", exception.getMessage());
|
assertEquals("缺少归属公司:请先在页面上方选择公司,再操作去重数据库", exception.getMessage());
|
||||||
verify(dedupeTotalDataMapper, never()).insertBatchIgnore(any());
|
verify(dedupeTotalDataMapper, never()).insertBatchIgnore(any());
|
||||||
verify(dedupeTotalDataMapper, never()).insert(any(DedupeTotalDataEntity.class));
|
verify(dedupeTotalDataMapper, never()).insert(any(DedupeTotalDataEntity.class));
|
||||||
}
|
}
|
||||||
@@ -217,13 +214,12 @@ class DedupeTotalDataServiceTest {
|
|||||||
@Test
|
@Test
|
||||||
void importReadsAndNormalizesCountryColumn() throws Exception {
|
void importReadsAndNormalizesCountryColumn() throws Exception {
|
||||||
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
|
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
|
||||||
when(companyScopeSupport.requireCompanyId(any(AdminUserEntity.class))).thenReturn(1L);
|
|
||||||
stubWritableGroup(23L, 7L);
|
stubWritableGroup(23L, 7L);
|
||||||
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"))).thenReturn(List.of());
|
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"), 1L)).thenReturn(List.of());
|
||||||
when(dedupeTotalDataMapper.insertBatchIgnore(any())).thenReturn(1);
|
when(dedupeTotalDataMapper.insertBatchIgnore(any())).thenReturn(1);
|
||||||
MockMultipartFile file = asinCountryWorkbook("B012345678", "德国, US,United Kingdom");
|
MockMultipartFile file = asinCountryWorkbook("B012345678", "德国, US,United Kingdom");
|
||||||
|
|
||||||
var result = service.importFromExcel(file, 7L, 23L);
|
var result = service.importFromExcel(file, 7L, 1L, 23L);
|
||||||
|
|
||||||
assertEquals(1, result.getInsertedCount());
|
assertEquals(1, result.getInsertedCount());
|
||||||
ArgumentCaptor<List<DedupeTotalDataEntity>> captor = ArgumentCaptor.forClass(List.class);
|
ArgumentCaptor<List<DedupeTotalDataEntity>> captor = ArgumentCaptor.forClass(List.class);
|
||||||
@@ -237,12 +233,11 @@ class DedupeTotalDataServiceTest {
|
|||||||
@Test
|
@Test
|
||||||
void importKeepsCountryBlankWhenHeaderMissing() throws Exception {
|
void importKeepsCountryBlankWhenHeaderMissing() throws Exception {
|
||||||
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
|
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
|
||||||
when(companyScopeSupport.requireCompanyId(any(AdminUserEntity.class))).thenReturn(1L);
|
|
||||||
stubWritableGroup(23L, 7L);
|
stubWritableGroup(23L, 7L);
|
||||||
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"))).thenReturn(List.of());
|
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"), 1L)).thenReturn(List.of());
|
||||||
when(dedupeTotalDataMapper.insertBatchIgnore(any())).thenReturn(1);
|
when(dedupeTotalDataMapper.insertBatchIgnore(any())).thenReturn(1);
|
||||||
|
|
||||||
var result = service.importFromExcel(asinWorkbook("B012345678"), 7L, 23L);
|
var result = service.importFromExcel(asinWorkbook("B012345678"), 7L, 1L, 23L);
|
||||||
|
|
||||||
assertEquals(1, result.getInsertedCount());
|
assertEquals(1, result.getInsertedCount());
|
||||||
ArgumentCaptor<List<DedupeTotalDataEntity>> captor = ArgumentCaptor.forClass(List.class);
|
ArgumentCaptor<List<DedupeTotalDataEntity>> captor = ArgumentCaptor.forClass(List.class);
|
||||||
@@ -274,17 +269,16 @@ class DedupeTotalDataServiceTest {
|
|||||||
@Test
|
@Test
|
||||||
void concurrentDuplicateDuringImportIsSkipped() throws Exception {
|
void concurrentDuplicateDuringImportIsSkipped() throws Exception {
|
||||||
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
|
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
|
||||||
when(companyScopeSupport.requireCompanyId(any(AdminUserEntity.class))).thenReturn(1L);
|
|
||||||
stubWritableGroup(23L, 7L);
|
stubWritableGroup(23L, 7L);
|
||||||
when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null);
|
when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null);
|
||||||
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"))).thenReturn(List.of());
|
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"), 1L)).thenReturn(List.of());
|
||||||
when(transactionManager.getTransaction(any())).thenReturn(transactionStatus);
|
when(transactionManager.getTransaction(any())).thenReturn(transactionStatus);
|
||||||
when(dedupeTotalDataMapper.insertBatchIgnore(any()))
|
when(dedupeTotalDataMapper.insertBatchIgnore(any()))
|
||||||
.thenThrow(new DuplicateKeyException("duplicate batch"));
|
.thenThrow(new DuplicateKeyException("duplicate batch"));
|
||||||
when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class)))
|
when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class)))
|
||||||
.thenThrow(new DuplicateKeyException("duplicate"));
|
.thenThrow(new DuplicateKeyException("duplicate"));
|
||||||
|
|
||||||
var result = service.importFromExcel(asinWorkbook("b012345678"), 7L, 23L);
|
var result = service.importFromExcel(asinWorkbook("b012345678"), 7L, 1L, 23L);
|
||||||
|
|
||||||
assertEquals(0, result.getInsertedCount());
|
assertEquals(0, result.getInsertedCount());
|
||||||
assertEquals(1, result.getSkippedCount());
|
assertEquals(1, result.getSkippedCount());
|
||||||
@@ -360,18 +354,83 @@ class DedupeTotalDataServiceTest {
|
|||||||
verify(dedupeTotalDataMapper, never()).selectCount(any());
|
verify(dedupeTotalDataMapper, never()).selectCount(any());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 比对集只认一家。这条曾经叫 comparableValueLookupRemainsGlobal,显式钉着「不参与公司收窄」
|
||||||
|
* (2026-10 多公司一期)——那是半套发布的遗留:库里 uk_company_data_value 自 V145 起就允许
|
||||||
|
* 同值归两家,代码却仍拿全平台当比对集。2026-10-09 按控制方定案反转。
|
||||||
|
*/
|
||||||
@Test
|
@Test
|
||||||
void comparableValueLookupRemainsGlobal() {
|
void comparableValueLookupIsScopedToOneCompany() {
|
||||||
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678")))
|
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"), 1L))
|
||||||
.thenReturn(List.of("b012345678"));
|
.thenReturn(List.of("b012345678"));
|
||||||
|
|
||||||
Set<String> values = service.findExistingComparableValues(List.of(" b012345678 "));
|
Set<String> values = service.findExistingComparableValues(List.of(" b012345678 "), 1L);
|
||||||
|
|
||||||
assertEquals(Set.of("B012345678"), values);
|
assertEquals(Set.of("B012345678"), values);
|
||||||
verify(adminUserMapper, never()).selectById(any(Long.class));
|
verify(adminUserMapper, never()).selectById(any(Long.class));
|
||||||
verify(shopManageGroupMapper, never()).selectManagedMemberUserIds(any(Long.class));
|
verify(shopManageGroupMapper, never()).selectManagedMemberUserIds(any(Long.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 删除导入的谓词必须带公司,且**对平台超管也一样**:allUsers 那一站原本只按分组裁,
|
||||||
|
* 超管不选分组时删除条件只剩一个 data_value IN——同名 ASIN 在几家库里就删几次。
|
||||||
|
* 这条走真 wrapper(不是 mock 的返回值):MP 的 IN 段要先把 getSqlSegment 求值一次才铺出来。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void deleteImportNeverReachesAnotherCompanyRows() {
|
||||||
|
when(transactionManager.getTransaction(any())).thenReturn(transactionStatus);
|
||||||
|
when(dedupeTotalDataMapper.delete(any())).thenReturn(1);
|
||||||
|
|
||||||
|
service.deleteBatchByDataValues(List.of("B012345678"),
|
||||||
|
new DedupeTotalDataService.AccessScope(true, java.util.Set.of(), java.util.Set.of(), false),
|
||||||
|
null, 3L);
|
||||||
|
|
||||||
|
ArgumentCaptor<LambdaQueryWrapper<DedupeTotalDataEntity>> captor =
|
||||||
|
ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||||
|
verify(dedupeTotalDataMapper).delete(captor.capture());
|
||||||
|
LambdaQueryWrapper<DedupeTotalDataEntity> wrapper = captor.getValue();
|
||||||
|
wrapper.getSqlSegment();
|
||||||
|
assertThat(wrapper.getSqlSegment()).contains("company_id");
|
||||||
|
assertThat(wrapper.getParamNameValuePairs().values()).contains(3L);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 「数据已存在」只在本公司内成立。这条钉的是重名查询的 wrapper:摘掉 company_id 那一刀,
|
||||||
|
* B 家导入过的 ASIN 会让 A 家的「新增」报已存在,而 A 家库里根本没有那行。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void createChecksDuplicatesInsideOneCompanyOnly() {
|
||||||
|
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
|
||||||
|
stubWritableGroup(23L, 7L);
|
||||||
|
when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null);
|
||||||
|
when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class))).thenAnswer(invocation -> {
|
||||||
|
DedupeTotalDataEntity entity = invocation.getArgument(0);
|
||||||
|
entity.setId(91L);
|
||||||
|
return 1;
|
||||||
|
});
|
||||||
|
when(dedupeTotalDataMapper.selectById(91L)).thenReturn(data(91L, 23L));
|
||||||
|
|
||||||
|
DedupeTotalDataCreateRequest request = new DedupeTotalDataCreateRequest();
|
||||||
|
request.setDataValue("B012345678");
|
||||||
|
request.setGroupId(7L);
|
||||||
|
service.create(request, 3L, 23L);
|
||||||
|
|
||||||
|
ArgumentCaptor<LambdaQueryWrapper<DedupeTotalDataEntity>> captor =
|
||||||
|
ArgumentCaptor.forClass(LambdaQueryWrapper.class);
|
||||||
|
verify(dedupeTotalDataMapper).selectOne(captor.capture());
|
||||||
|
captor.getValue().getSqlSegment();
|
||||||
|
assertThat(captor.getValue().getSqlSegment()).contains("company_id");
|
||||||
|
assertThat(captor.getValue().getParamNameValuePairs().values()).contains(3L);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 没有归属公司就不比对:宁可炸,也不给出一条"比对集是空的"与"一家都没命中"同形的路径。 */
|
||||||
|
@Test
|
||||||
|
void comparableValueLookupRefusesACompanylessComparison() {
|
||||||
|
assertThrows(BusinessException.class,
|
||||||
|
() -> service.findExistingComparableValues(List.of("B012345678"), null));
|
||||||
|
verify(dedupeTotalDataMapper, never()).selectExistingDataValues(any(), any());
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void exportUsesScopedUsernameAndProducesWorkbook() throws Exception {
|
void exportUsesScopedUsernameAndProducesWorkbook() throws Exception {
|
||||||
when(adminUserMapper.selectById(10L)).thenReturn(user(10L, "admin", "leader"));
|
when(adminUserMapper.selectById(10L)).thenReturn(user(10L, "admin", "leader"));
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ AdminUserMapper selectCompanySuperAdminEarliestExpiry Select PURE SELECT MIN(exp
|
|||||||
BrandCrawlTaskMapper selectExpiredTerminalTaskIds Select PURE SELECT id FROM brand_crawl_tasks\nWHERE status IN ('success', 'failed', 'cancelled')\n AND updated_at < #{cutoff}\nORDER BY id ASC\nLIMIT #{batchSize}
|
BrandCrawlTaskMapper selectExpiredTerminalTaskIds Select PURE SELECT id FROM brand_crawl_tasks\nWHERE status IN ('success', 'failed', 'cancelled')\n AND updated_at < #{cutoff}\nORDER BY id ASC\nLIMIT #{batchSize}
|
||||||
CollectDataItemMapper insertBatch Insert PURE <script>\nINSERT INTO biz_collect_data_item\n (task_id, row_index, source_file_key, source_filename, keyword, status_value, extra_json, created_at)\nVALUES\n<foreach collection="rows" item="row" separator=",">\n (#{row.taskId}, #{row.rowIndex}, #{row.sourceFileKey}, #{row.sourceFilename},\n #{row.keyword}, #{row.statusValue}, #{row.extraJson}, #{row.createdAt})\n</foreach>\n</script>
|
CollectDataItemMapper insertBatch Insert PURE <script>\nINSERT INTO biz_collect_data_item\n (task_id, row_index, source_file_key, source_filename, keyword, status_value, extra_json, created_at)\nVALUES\n<foreach collection="rows" item="row" separator=",">\n (#{row.taskId}, #{row.rowIndex}, #{row.sourceFileKey}, #{row.sourceFilename},\n #{row.keyword}, #{row.statusValue}, #{row.extraJson}, #{row.createdAt})\n</foreach>\n</script>
|
||||||
DedupeTotalDataMapper insertBatchIgnore Insert PURE <script>\nINSERT IGNORE INTO biz_dedupe_total_data\n (data_value, country, group_id, uploader_user_id, uploader_username, company_id)\nVALUES\n<foreach collection="rows" item="row" separator=",">\n (#{row.dataValue}, #{row.country}, #{row.groupId}, #{row.uploaderUserId}, #{row.uploaderUsername}, #{row.companyId})\n</foreach>\n</script>
|
DedupeTotalDataMapper insertBatchIgnore Insert PURE <script>\nINSERT IGNORE INTO biz_dedupe_total_data\n (data_value, country, group_id, uploader_user_id, uploader_username, company_id)\nVALUES\n<foreach collection="rows" item="row" separator=",">\n (#{row.dataValue}, #{row.country}, #{row.groupId}, #{row.uploaderUserId}, #{row.uploaderUsername}, #{row.companyId})\n</foreach>\n</script>
|
||||||
DedupeTotalDataMapper selectAllDataValues Select PURE SELECT data_value FROM biz_dedupe_total_data
|
DedupeTotalDataMapper selectExistingDataValues Select PURE <script>\nSELECT data_value FROM biz_dedupe_total_data\nWHERE company_id = #{companyId}\n AND data_value IN\n<foreach collection="values" item="value" open="(" separator="," close=")">#{value}</foreach>\n</script>
|
||||||
DeviceLogFileMapper deleteOlderThanBatch Delete PURE DELETE FROM device_log_file\nWHERE log_date < #{cutoff}\nLIMIT #{batchSize}
|
DeviceLogFileMapper deleteOlderThanBatch Delete PURE DELETE FROM device_log_file\nWHERE log_date < #{cutoff}\nLIMIT #{batchSize}
|
||||||
FileResultMapper selectRowsUpdatedAtByTaskIds Select PURE <script>\nSELECT task_id AS taskId, MAX(updated_at) AS latestRowAt, COUNT(*) AS rowCount\nFROM biz_file_result\nWHERE module_type = #{moduleType}\n AND task_id IN\n <foreach item="taskId" collection="taskIds" open="(" separator="," close=")">#{taskId}</foreach>\nGROUP BY task_id\n</script>
|
FileResultMapper selectRowsUpdatedAtByTaskIds Select PURE <script>\nSELECT task_id AS taskId, MAX(updated_at) AS latestRowAt, COUNT(*) AS rowCount\nFROM biz_file_result\nWHERE module_type = #{moduleType}\n AND task_id IN\n <foreach item="taskId" collection="taskIds" open="(" separator="," close=")">#{taskId}</foreach>\nGROUP BY task_id\n</script>
|
||||||
FileTaskMapper selectExpiredTerminalTaskIds Select PURE SELECT id FROM biz_file_task\nWHERE module_type = #{moduleType}\n AND status IN ('SUCCESS', 'FAILED', 'CANCELLED', 'CANCELED')\n AND updated_at < #{cutoff}\nORDER BY id ASC\nLIMIT #{batchSize}
|
FileTaskMapper selectExpiredTerminalTaskIds Select PURE SELECT id FROM biz_file_task\nWHERE module_type = #{moduleType}\n AND status IN ('SUCCESS', 'FAILED', 'CANCELLED', 'CANCELED')\n AND updated_at < #{cutoff}\nORDER BY id ASC\nLIMIT #{batchSize}
|
||||||
|
|||||||
|
Can't render this file because it contains an unexpected character in line 1 and column 222.
|
Reference in New Issue
Block a user