收口(去重库): 五处比对与写入口按 company_id 认公司,不再跨公司去重

读侧(列表/导出)早已按公司收窄,但"去重"这个动作本身一直查全平台:
selectExistingDataValues 是 `WHERE data_value IN (...)`,一个租户条件都没有。
库里唯一键自 V145 起就是 uk_company_data_value(company_id, data_value)——
同值本来就允许归两家,是代码没跟上库的口径。

五处一起收口(第 5 处由编译器抓出,正是 V145 头注释 §1.2 点名的采集侧):
去重执行 ASIN 过滤 / 导入预检"已存在就跳过" / 手工新增与编辑的重名校验
(文案改「本公司已存在该数据」)/ 删除导入(超管不选分组时原先一次删掉全平台
同名行)/ 采集结果过滤(CollectDataBatchQuery 直连 mapper,改为经 dedupe 侧推导)。

超管代导入的归属分裂一并修掉:前端两个导入接口把公司 tab 发下去——原先
group_id 落别家的组、company_id 落超管自己那一家。公司身份一律在请求线程上由
DataReadScope.readCompanyIdOrOwn 解析后传入(导入与去重跑在虚拟线程,那时已无
请求上下文),service 侧 requireScopedCompanyId 只判空不兜底。

删两条零调用者的跨公司通道(selectAllDataValues/listComparableValues 全表读、
deleteFromExcel 两个同步重载)。反转一条反向钉:comparableValueLookupRemainsGlobal
原显式断言"比对不参与公司收窄"(多公司一期的半套遗留),现为 IsScopedToOneCompany。

三把尺子随之改动,均带出处:tsv 基准行搬家(三个标量双双不动)、前端发送口分母
17→18 组/21→22 处、CatchSilenceRatchet 重抛下限 387→386(随死代码离开树那处)。

验证:后端全量 4198 例 0 失败/0 错误/7 跳过;前端 1778 例全绿;变异自证
6/6 全部 KILLED,注入文件按 sha256 还原并复跑绿。
现网只读探针:misfiled=0、同值跨家=0 ⇒ 存量比对结果不变,行为改变只在将来
A 家导入 B 家已有 ASIN 那一刻;两口径差额 1,452 行(0.26%)全属公司 1、
全源于 10-05 账号硬删。
This commit is contained in:
2026-10-09 17:57:37 +08:00
parent 974d0c77d0
commit 5ccf3c896f
22 changed files with 393 additions and 187 deletions
@@ -32,7 +32,9 @@ export const COMPANY_SCOPED_PAGES: readonly CompanyScopedPage[] = [
// 【这两页已于 2026-10-06 退出注册表】admin_invalid_asin_data(品牌数据库)与 admin_product_categories
// (商品类目)按控制方定案退回**全平台共享**,摘的不只是 tab:后端读写两侧的 company_id 谓词一起删
// (V151 退唯一键)。反向钉在 tests/company-tab-wiring.test.ts——把它们再加回来必须先推翻那条定案。
{ menuKey: 'admin_dedupe_total_data', apiFiles: ['src/pages/asin/dedupe-total-api.ts', 'src/pages/asin/dedupe-total-export.ts'], pageFiles: ['src/pages/asin/DedupeRegistryPage.vue'] },
// dedupe-import-api.ts 是 10-09 补进来的:导入/删除导入这两支写侧出口原先不发 company_id,
// 超管在别家 tab 点「新增导入」,数据落进了超管自己那一家(group_id 却是别家的组)=归属分裂。
{ menuKey: 'admin_dedupe_total_data', apiFiles: ['src/pages/asin/dedupe-total-api.ts', 'src/pages/asin/dedupe-total-export.ts', 'src/pages/asin/dedupe-import-api.ts'], pageFiles: ['src/pages/asin/DedupeRegistryPage.vue'] },
{ menuKey: 'admin_query_asin', apiFiles: ['src/pages/asin/query-asin-api.ts', 'src/pages/asin/query-asin-export.ts'], pageFiles: ['src/pages/asin/QueryAsinPage.vue'] },
{ menuKey: 'admin_skip_price_asin', apiFiles: ['src/pages/asin/skip-price-api.ts', 'src/pages/asin/skip-price-export.ts'], pageFiles: ['src/pages/asin/SkipPricePage.vue'] },
// 批次 3(控制方 2026-10-05 点名「店铺撞款监控 店铺数据也要做隔离」)。这两页的可见集不是按 company_id 列筛的,
@@ -250,7 +250,9 @@ async function submitImport(): Promise<void> {
importProgress.value = mode === 'add' ? '正在上传并解析文件…' : '正在上传删除清单…'
try {
const started =
mode === 'add' ? await startDedupeImport(file, groupId) : await startDedupeDeleteImport(file, groupId)
mode === 'add'
? await startDedupeImport(file, groupId, companyTab.requestCompanyId)
: await startDedupeDeleteImport(file, groupId, companyTab.requestCompanyId)
const poll = mode === 'add' ? () => fetchDedupeImportProgress(started) : () => fetchDedupeDeleteImportProgress(started)
const finished = await pollImport(poll)
importVisible.value = false
@@ -1,14 +1,20 @@
/** 去重汇总导入启动适配(任务 65):multipart POST /import(file+groupId) → importId。 */
/**
* 去重汇总导入启动适配(任务 65):multipart POST /import(file+groupId+company_id) → importId。
*
* <p>company_id 决定这批数据写进哪一家:超管认页面上方 tab,非超管 requestCompanyId 为 null
* ⇒ 一个字段都不发,后端按会话恒锁本公司。</p>
*/
import { http } from '@/api/http'
import { parseDedupeImportProgress, parseDedupeImportStart, type DedupeImportProgress } from './dedupe-import-model'
export const DEDUPE_IMPORT_ENDPOINT = '/api/admin/dedupe-total-data/import'
export const DEDUPE_DELETE_IMPORT_ENDPOINT = '/api/admin/dedupe-total-data/delete-import'
export async function startDedupeImport(file: File, groupId: number): Promise<string> {
export async function startDedupeImport(file: File, groupId: number, companyId?: number | null): Promise<string> {
const form = new FormData()
form.append('file', file, file.name)
form.append('groupId', String(groupId))
if (companyId != null) form.append('company_id', String(companyId))
const { data } = await http.post<unknown>(DEDUPE_IMPORT_ENDPOINT, form)
return parseDedupeImportStart(data)
}
@@ -20,10 +26,12 @@ export async function fetchDedupeImportProgress(importId: string): Promise<Dedup
}
/** 删除导入启动:multipart POST /delete-import(file+groupId) → importId。 */
export async function startDedupeDeleteImport(file: File, groupId: number): Promise<string> {
export async function startDedupeDeleteImport(file: File, groupId: number,
companyId?: number | null): Promise<string> {
const form = new FormData()
form.append('file', file, file.name)
form.append('groupId', String(groupId))
if (companyId != null) form.append('company_id', String(companyId))
const { data } = await http.post<unknown>(DEDUPE_DELETE_IMPORT_ENDPOINT, form)
return parseDedupeImportStart(data)
}
@@ -98,6 +98,8 @@ const SEND_POINTS: ReadonlyArray<SendPoint> = [
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-total-api.ts', fn: 'fetchDedupeTotalList', line: 'if (params.companyId != null) query.company_id = params.companyId', window: 400, title: 'GET /dedupe-total-data 列表' },
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-total-api.ts', fn: 'fetchDedupeTotalGroups', line: 'http.get<unknown>(DEDUPE_GROUPS_ENDPOINT, { params: companyId != null ? { company_id: companyId } : {} })', window: 400, title: 'GET /shop-manages/groups 分组候选(与店铺管理同源,两页口径要对得上)' },
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-total-export.ts', fn: 'buildDedupeExportParams', line: 'if (companyId != null) params.company_id = String(companyId)', window: 800, title: 'GET /dedupe-total-data/export 导出' },
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-import-api.ts', fn: 'startDedupeImport', line: "if (companyId != null) form.append('company_id', String(companyId))", window: 400, title: 'POST /dedupe-total-data/import 新增导入(写侧归属,不发就落进操作者自己那一家)' },
{ menuKey: 'admin_dedupe_total_data', file: 'src/pages/asin/dedupe-import-api.ts', fn: 'startDedupeDeleteImport', line: "if (companyId != null) form.append('company_id', String(companyId))", window: 900, title: 'POST /dedupe-total-data/delete-import 删除导入(不发就会按别家的值删错行)' },
{ menuKey: 'admin_query_asin', file: 'src/pages/asin/query-asin-api.ts', fn: 'fetchQueryAsinList', line: 'if (params.companyId != null) query.company_id = params.companyId', window: 400, title: 'GET /query-asins 列表' },
{ menuKey: 'admin_query_asin', file: 'src/pages/asin/query-asin-export.ts', fn: 'buildQueryAsinExportParams', line: 'if (companyId != null) params.company_id = String(companyId)', window: 700, title: 'GET /query-asins/export 导出' },
{ menuKey: 'admin_skip_price_asin', file: 'src/pages/asin/skip-price-api.ts', fn: 'fetchSkipPriceList', line: 'if (params.companyId != null) query.company_id = params.companyId', window: 400, title: 'GET /skip-price-asins 列表(最低价 ASIN)' },
@@ -237,6 +239,13 @@ const COMPANY_ID_SOURCE_PINS: Readonly<Record<string, readonly CompanyIdPin[]>>
/** 严格判据的分母由本轮实扫得出(修完不许恒绿也不许恒红):求值了几条 shape、其中几条通过。 */
let strictPinsEvaluated = 0
/**
* 「company_id 真被下传」的形状。原先只认 `company_id:` / `company_id =` 两形——那是按
* 「一律走查询参数」写的。去重库的导入与删除导入走 multipart body(form.append),
* 同一件事换了载体就被判成没接线;这里把那一形补进来,取值口仍按登记表的 shape 逐条严判。
*/
const COMPANY_ID_WIRED = /company_id\s*[:=]|form\.append\(\s*['"]company_id['"]/
let strictPinsPassed = 0
test('正向:声明 companyScoped 的页,其 api 文件必须真的带 company_id,页面组件必须 watch activeId', () => {
@@ -254,8 +263,8 @@ test('正向:声明 companyScoped 的页,其 api 文件必须真的带 company
// 旧式 /company_id|companyId/ 接近恒真,Task 8-12 只要写一行 `const companyId = null`
// 或一句「// 由 companyId 决定」就能把这页判成已接线,而实际什么都没下传(约束 6:注释不算)。
// 读之前先 stripComments,与反向那半支同口径——说明注释里的 company_id 不是接线。
if (!/company_id\s*[:=]/.test(stripComments(read(file)))) {
gaps.push(`${file} 没把 company_id 当成查询参数下传`)
if (!COMPANY_ID_WIRED.test(stripComments(read(file)))) {
gaps.push(`${file} 没把 company_id 下传(查询参数或 multipart 字段都算,裸 token 不算)`)
}
}
// 取值口登记(复评 I-2):光"把 company_id 塞进查询对象"不够,右半边必须取自**本函数自己的入参**。
@@ -265,7 +274,7 @@ test('正向:声明 companyScoped 的页,其 api 文件必须真的带 company
continue
}
const src = stripComments(read(pin.file))
const wired = /company_id\s*[:=]/.test(src)
const wired = COMPANY_ID_WIRED.test(src)
if (pin.shape === null) {
// 没登记取值口却已经下传了参数 = 这一页的取值口径没人看守(洞就是这么留下的)。
if (wired) gaps.push(`${pin.file} 已经把 company_id 当下传参数了,登记表里 shape 却还是 null —— 把它换成"company_id 右侧 = 本函数入参"的正则`)
@@ -357,8 +366,8 @@ test('正向:声明 companyScoped 的页,其 api 文件必须真的带 company
if (hit) hit.pinned += 1
else grouped.set(key, { file: p.file, line: p.line, pinned: 1 })
}
assert.equal(grouped.size, 17,
`发送口表按 (文件, 整行) 应聚成 17 组(21 处发送口),现在却有 ${grouped.size} 组 —— 表被改形状了,对平判据失去分母`)
assert.equal(grouped.size, 18,
`发送口表按 (文件, 整行) 应聚成 18 组(22 处发送口),现在却有 ${grouped.size} 组 —— 表被改形状了,对平判据失去分母`)
for (const g of grouped.values()) {
const actual = occurrences(stripComments(read(g.file)), g.line)
assert.equal(g.pinned, actual,
@@ -122,7 +122,8 @@ public class CollectDataBrandRecheckInjector {
continue;
}
// 复查去重/无效品牌:补测窗口内该 ASIN 可能已被其他任务采走(first-writer-wins)
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(chunkClean);
CollectDataBatchQuery.FilterResult filtered =
collectDataBatchQuery.filter(chunkClean, task.getUserId());
if (filtered.kept().isEmpty()) {
log.info("[brand-recheck] 补测行全部被去重/无效过滤,不注入 taskId={} rows={}",
task.getId(), chunkClean.size());
@@ -830,7 +830,7 @@ public class CollectDataService {
public CollectDataSubmitResultVo submitResult(Long taskId, CollectDataSubmitResultRequest request) {
if (request == null) {
throw new BusinessException("request is empty");
throw new BusinessException("请求内容为空");
}
ensureRustfsPayloadStorageEnabled();
@@ -861,7 +861,9 @@ public class CollectDataService {
}
}
long prepareStartAt = System.currentTimeMillis();
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(rowsForFiltering);
// 比对集认任务归属账号那一家:chunk 回传的过滤结果不该受别家导入过什么影响
CollectDataBatchQuery.FilterResult filtered =
collectDataBatchQuery.filter(rowsForFiltering, probe.getUserId());
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome = brandBatchFilter.filter(filtered.kept());
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms",
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt);
@@ -2,7 +2,7 @@ package com.nanri.aiimage.modules.collectdata.util;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService;
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
import lombok.RequiredArgsConstructor;
@@ -22,7 +22,7 @@ import java.util.regex.Pattern;
* 多次降为一次,并保证空输入、单元素与超限集合的确定行为。
*
* 过滤语义(与 CollectDataService 原 filterByExistingAsin 一致):
* - ASIN 存在于去重表 → dedupeFiltered(增加计数)
* - ASIN 存在于「本任务归属公司」的去重表 → dedupeFiltered(增加计数)
* - 品牌存在于无效品牌表 → invalidFiltered(增加计数)
* - 其余行保留
*/
@@ -32,21 +32,22 @@ public class CollectDataBatchQuery {
private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final DedupeTotalDataService dedupeTotalDataService;
private final InvalidAsinDataMapper invalidAsinDataMapper;
/**
* 批量集合查询过滤(兼容入口)。返回过滤后的行列表;入参为 null/空时返回空列表。
*/
public List<CollectDataResultRowVo> filterByExistingAsin(List<CollectDataResultRowVo> rows) {
return filter(rows).kept();
}
/**
* 批量集合查询过滤并统计两类命中:去重表命中的 ASIN 行数与无效品牌表
* 命中的品牌行数(null/空行不计数)。入参为 null/空时返回空结果。
*
* <p>去重表那一支的比对集只认 {@code ownerUserId} 那一家:「这个 ASIN 已经有人采过了」不能拿
* 别家导入的值来判。此前它查的是全平台,于是 B 公司导入的 ASIN 会让 A 公司的采集结果少行,而
* A 公司在自己后台根本看不到那行是谁的——V145 头注释 §1.2 点名的就是这一条。推导发生在
* {@code DedupeTotalDataService#findExistingComparableValuesForOwner},不在本文件:</p>
*
* <p>无效品牌那一支<b>不带</b>公司条件,是定案不是漏改:品牌库 biz_invalid_asin_data 已于
* 2026-10-06 退回全平台共享(V151 退唯一键),反向钉在 CompanyIdParamGoesThroughReadScopeTest。</p>
*/
public FilterResult filter(List<CollectDataResultRowVo> rows) {
public FilterResult filter(List<CollectDataResultRowVo> rows, Long ownerUserId) {
if (rows == null || rows.isEmpty()) {
return new FilterResult(List.of(), 0, 0);
}
@@ -55,7 +56,8 @@ public class CollectDataBatchQuery {
.map(CollectDataResultRowVo::getAsin).toList());
Set<String> dedupeValues = new HashSet<>();
if (!asins.isEmpty()) {
List<String> existing = dedupeTotalDataMapper.selectExistingDataValues(asins);
Set<String> existing = dedupeTotalDataService
.findExistingComparableValuesForOwner(asins, ownerUserId);
if (existing != null) {
for (String value : existing) {
String normalized = normalizeAsin(value);
@@ -155,7 +155,11 @@ public class DedupeTotalDataController {
HttpServletRequest httpRequest,
@Valid @RequestBody DedupeTotalDataCreateRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(httpRequest);
return ApiResponse.success("创建成功", dedupeTotalDataService.create(request, operator.id()));
AdminUserEntity user = operator.user();
Long requestedCompanyId = request.getCompanyId();
Long companyId = dataReadScope.readCompanyIdOrOwn(user, requestedCompanyId);
Long operatorId = operator.id();
return ApiResponse.success("创建成功", dedupeTotalDataService.create(request, companyId, operatorId));
}
@PostMapping("/import")
@@ -167,9 +171,14 @@ public class DedupeTotalDataController {
public ApiResponse<DedupeTotalDataImportStartVo> importExcel(
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file,
@Parameter(description = "分组 ID", required = true) @RequestParam Long groupId,
@Parameter(description = "公司 ID") @RequestParam(name = "company_id", required = false) Long companyId,
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
return ApiResponse.success("开始导入", dedupeTotalDataService.startImport(file, groupId, operator.id()));
AdminUserEntity user = operator.user();
Long scopedCompanyId = dataReadScope.readCompanyIdOrOwn(user, companyId);
Long operatorId = operator.id();
return ApiResponse.success("开始导入",
dedupeTotalDataService.startImport(file, groupId, scopedCompanyId, operatorId));
}
@GetMapping("/import/{importId}")
@@ -190,9 +199,14 @@ public class DedupeTotalDataController {
public ApiResponse<DedupeTotalDataImportStartVo> deleteImportExcel(
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file,
@Parameter(description = "分组 ID", required = true) @RequestParam Long groupId,
@Parameter(description = "公司 ID") @RequestParam(name = "company_id", required = false) Long companyId,
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
return ApiResponse.success("开始删除", dedupeTotalDataService.startDeleteImport(file, groupId, operator.id()));
AdminUserEntity user = operator.user();
Long scopedCompanyId = dataReadScope.readCompanyIdOrOwn(user, companyId);
Long operatorId = operator.id();
return ApiResponse.success("开始删除",
dedupeTotalDataService.startDeleteImport(file, groupId, scopedCompanyId, operatorId));
}
@GetMapping("/delete-import/{importId}")
@@ -285,7 +299,8 @@ public class DedupeTotalDataController {
// 公司 tab 的读范围就地算在 handler 里,不留私有收口方法:G2 的断言 A 只在这支 handler 的
// 公司 tab 的读范围与写侧归属(create / import / delete-import)都就地算在 handler 里,
// 不留私有收口方法:G2 的断言 A 只在这支 handler 的
// 块内找 readScope(,私有方法在它自己的闭括号处收口、不算在内(CompanyIdParamGoesThroughReadScopeTest
// 类注释第 5、7 条),挪进去 = 这一页的 tab 没人看守。收窄只给超管:非超管已由 resolveAccessScope
// 按会话裁剪,且他的可见集里含「靠分组可见」的行,叠一层账号 IN 会把那批行静默删掉
@@ -4,18 +4,25 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.SelectProvider;
import java.util.List;
import java.util.StringJoiner;
@Mapper
public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> {
List<String> selectAllDataValues();
@SelectProvider(type = SqlProvider.class, method = "selectExistingDataValues")
List<String> selectExistingDataValues(@Param("values") List<String> values);
/**
* 比对集按公司取:一次去重/一批导入只能看见 {@code companyId} 名下的值。
*
* <p>这条曾是 @SelectProvider 手拼 IN 列表、且不带任何租户条件,后果有两样:去重执行会把别家
* 导入的 ASIN 当「已存在」从输出文件里删掉;导入会把别家已有的值静默跳过,本公司库里永远补不上
* 这一行。本表唯一键自 V145 起就是 uk_company_data_value(company_id, data_value)——同值本来就
* 允许归两家,是代码没跟上库的口径。</p>
*
* <p>{@code companyId} 必须由调用方保证非空:MyBatis 会把它渲染成 {@code company_id = NULL}
* 这条永假条件,静默给出空集,比当场炸更难发现。拦空值的是服务层闸门。</p>
*/
List<String> selectExistingDataValues(@Param("values") List<String> values,
@Param("companyId") Long companyId);
/**
* Import is deliberately a single SQL round trip per batch. The unique
@@ -23,14 +30,4 @@ public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity>
* the same value between the pre-check and this statement.
*/
int insertBatchIgnore(@Param("rows") List<DedupeTotalDataEntity> rows);
class SqlProvider {
public String selectExistingDataValues(@Param("values") List<String> values) {
StringJoiner placeholders = new StringJoiner(", ");
for (int i = 0; i < values.size(); i++) {
placeholders.add("#{values[" + i + "]}");
}
return "SELECT data_value FROM biz_dedupe_total_data WHERE data_value IN (" + placeholders + ")";
}
}
}
@@ -18,4 +18,8 @@ public class DedupeTotalDataCreateRequest {
@NotNull(message = "请选择分组")
@Schema(description = "分组 ID", requiredMode = Schema.RequiredMode.REQUIRED)
private Long groupId;
/** 页面上方公司 tab 选中的归属公司;非超管带什么都不算,超管不带则拒绝写入。 */
@Schema(description = "公司 ID")
private Long companyId;
}
@@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.CompanyScopeSupport;
import com.nanri.aiimage.common.service.NodeSharedStore;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.config.InstanceMetadata;
@@ -79,6 +80,7 @@ public class DedupeRunService {
private final StorageProperties storageProperties;
private final OssStorageService ossStorageService;
private final DedupeTotalDataService dedupeTotalDataService;
private final CompanyScopeSupport companyScopeSupport;
private final InstanceMetadata instanceMetadata;
/**
@@ -102,6 +104,7 @@ public class DedupeRunService {
StorageProperties storageProperties,
OssStorageService ossStorageService,
DedupeTotalDataService dedupeTotalDataService,
CompanyScopeSupport companyScopeSupport,
StringRedisTemplate stringRedisTemplate,
ObjectMapper objectMapper,
InstanceMetadata instanceMetadata) {
@@ -110,6 +113,7 @@ public class DedupeRunService {
this.storageProperties = storageProperties;
this.ossStorageService = ossStorageService;
this.dedupeTotalDataService = dedupeTotalDataService;
this.companyScopeSupport = companyScopeSupport;
this.instanceMetadata = instanceMetadata;
this.runProgressMap = new NodeSharedStore<>("aiimage:dedupe:run:progress", RUN_STATE_TTL,
DedupeRunProgressVo.class, stringRedisTemplate, objectMapper);
@@ -140,6 +144,11 @@ public class DedupeRunService {
throw new BusinessException("请至少选择一种 ID 保留规则");
}
cleanupExpiredProgress();
// 比对集锁在提交人自己那一家:「这个 ASIN 去重库里已经有了」只能拿本公司导入过的值来判,
// 否则别家导入的 ASIN 会把本应保留的行从输出文件里静默删掉。
// 在请求线程上解析:一是解析失败要当场报错,不能任务跑一半才炸;二是清洗跑在虚拟线程上,
// 那时已经没有请求上下文了。
Long companyId = companyScopeSupport.requireCompanyId(request.getUserId());
if (!tryAcquireRunningTaskSlot(request.getUserId())) {
throw new BusinessException("已有其他去重任务正在处理中,请等待完成后再试");
@@ -182,7 +191,7 @@ public class DedupeRunService {
runProgressMap.putNow(runId, snapshot(progress));
try {
Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress));
Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress, companyId));
log.info("dedupe run submitted runId={} taskNo={} userId={} files={} thread={}",
runId, task.getTaskNo(), request.getUserId(), request.getFiles().size(), thread.threadId());
} catch (Exception e) {
@@ -292,7 +301,8 @@ public class DedupeRunService {
return vo;
}
private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request, DedupeRunProgressVo progress) {
private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request,
DedupeRunProgressVo progress, Long companyId) {
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
Map<String, DedupeArchiveEntry> archiveEntries = new ConcurrentHashMap<>();
List<DedupeResultItemVo> outcomeItems = new ArrayList<>();
@@ -304,7 +314,8 @@ public class DedupeRunService {
List<Future<?>> workers = new ArrayList<>(request.getFiles().size());
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
workers.add(fileExecutor.submit(() -> {
DedupeResultItemVo item = processFile(sourceFile, request, folderMode, task, archiveEntries);
DedupeResultItemVo item = processFile(sourceFile, request, folderMode, task, archiveEntries,
companyId);
synchronized (progress) {
progress.setProcessedCount(processedCount.incrementAndGet());
if (item.isSuccess()) {
@@ -466,7 +477,7 @@ public class DedupeRunService {
private DedupeResultItemVo processFile(DedupeSourceFileDto sourceFile, DedupeRunRequest request,
boolean folderMode, FileTaskEntity task,
Map<String, DedupeArchiveEntry> archiveEntries) {
Map<String, DedupeArchiveEntry> archiveEntries, Long companyId) {
DedupeResultItemVo item = new DedupeResultItemVo();
item.setSourceFilename(sourceFile.getOriginalFilename());
try {
@@ -496,7 +507,8 @@ public class DedupeRunService {
request.getSelectedColumns(),
request.isKeepIntegerIds(),
request.isKeepUnderscoreIds(),
request.isKeepIntegerMainIdsWhenNoSubIds()
request.isKeepIntegerMainIdsWhenNoSubIds(),
companyId
);
long cleanNs = elapsedNs(cleanStartNs);
@@ -679,7 +691,7 @@ public class DedupeRunService {
*/
private void cleanExcelByStreamRules(File inputFile, File outputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
boolean keepIntegerMainIdsWhenNoSubIds, Long companyId) throws Exception {
List<String> orderedSelectedColumns = reorderExportColumns(selectedColumns);
long readStartNs = System.nanoTime();
DedupeReadResult readResult = new DedupeReadResult();
@@ -746,7 +758,8 @@ public class DedupeRunService {
}
}
long dbStartNs = System.nanoTime();
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(
candidateAsinValues, companyId);
long dbNs = elapsedNs(dbStartNs);
long writeStartNs = System.nanoTime();
@@ -471,12 +471,12 @@ public class DedupeTotalDataService {
}
@Transactional
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request, Long operatorId) {
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request, Long companyId, Long operatorId) {
AdminUserEntity uploader = getOperator(operatorId);
Long companyId = companyScopeSupport.requireCompanyId(uploader);
requireScopedCompanyId(companyId);
ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), uploader);
String dataValue = normalizeComparableValue(request.getDataValue());
ensureUnique(dataValue, null);
ensureUnique(dataValue, null, companyId);
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setDataValue(dataValue);
entity.setGroupId(group.getId());
@@ -488,14 +488,27 @@ public class DedupeTotalDataService {
return toItemVo(getById(entity.getId()), group.getGroupName());
}
public Set<String> listComparableValues() {
return dedupeTotalDataMapper.selectAllDataValues().stream()
.map(this::normalizeComparableValue)
.filter(value -> !value.isEmpty())
.collect(java.util.stream.Collectors.toSet());
/**
* 去重比对的取数口:只在 {@code companyId} 这一家名下找「已存在的值」。
*
* <p>公司身份必须由调用方给死,null 不放行:{@code company_id = NULL} 在 SQL 里是永假条件,
* 会静默返回空集,「一家都没比对上」和「比对集是空的」在结果上长得一样——那正是这次要拆的形态。
* 同理由删掉了此前的 {@code listComparableValues()}:它把全平台的值整池端出来,且零调用者。</p>
*/
/**
* 采集侧的比对入口:调用方只交任务归属账号,公司由这里推导。
*
* <p>不让调用方自己传 companyId:采集过滤器住在 collectdata 模块,同一个文件里还住着品牌库那一支
* (2026-10-06 定案全平台共享、不许带公司维度),公司字样落进去会让那条反向钉失去分辨力。</p>
*/
public Set<String> findExistingComparableValuesForOwner(Collection<String> values, Long ownerUserId) {
return findExistingComparableValues(values, companyScopeSupport.requireCompanyId(ownerUserId));
}
public Set<String> findExistingComparableValues(Collection<String> values) {
public Set<String> findExistingComparableValues(Collection<String> values, Long companyId) {
if (companyId == null) {
throw new BusinessException("去重比对缺少归属公司,比对集必须锁在一家公司内");
}
if (values == null || values.isEmpty()) {
return Set.of();
}
@@ -510,7 +523,8 @@ public class DedupeTotalDataService {
Set<String> existingValues = new HashSet<>();
for (int start = 0; start < normalizedValues.size(); start += COMPARE_BATCH_SIZE) {
int end = Math.min(start + COMPARE_BATCH_SIZE, normalizedValues.size());
List<String> batch = dedupeTotalDataMapper.selectExistingDataValues(normalizedValues.subList(start, end));
List<String> batch = dedupeTotalDataMapper.selectExistingDataValues(
normalizedValues.subList(start, end), companyId);
if (batch != null) {
batch.stream()
.map(this::normalizeComparableValueOrBlank)
@@ -521,11 +535,8 @@ public class DedupeTotalDataService {
return existingValues;
}
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long operatorId) {
return startImport(file, null, operatorId);
}
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long groupId, Long operatorId) {
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long groupId, Long companyId,
Long operatorId) {
cleanupExpiredProgress();
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
@@ -534,7 +545,7 @@ public class DedupeTotalDataService {
throw new BusinessException("导入文件超过大小上限");
}
AdminUserEntity uploader = getOperator(operatorId);
Long companyId = companyScopeSupport.requireCompanyId(uploader);
requireScopedCompanyId(companyId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
if (!importSlots.tryAcquire()) {
throw new BusinessException("导入任务过多,请稍后重试");
@@ -582,11 +593,8 @@ public class DedupeTotalDataService {
return progress;
}
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long operatorId) {
return startDeleteImport(file, null, operatorId);
}
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long groupId, Long operatorId) {
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long groupId, Long companyId,
Long operatorId) {
cleanupExpiredProgress();
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
@@ -595,6 +603,7 @@ public class DedupeTotalDataService {
throw new BusinessException("导入文件超过大小上限");
}
AdminUserEntity operator = getOperator(operatorId);
requireScopedCompanyId(companyId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
if (!importSlots.tryAcquire()) {
throw new BusinessException("导入任务过多,请稍后重试");
@@ -615,7 +624,7 @@ public class DedupeTotalDataService {
File tempFile = saveMultipartToTempFile(file);
String filename = file.getOriginalFilename();
Thread.ofVirtual().start(() -> runDeleteImportTask(
importId, tempFile, filename, operator.getId(), group.getId()));
importId, tempFile, filename, operator.getId(), group.getId(), companyId));
} catch (Exception e) {
importSlots.release();
deleteImportProgressMap.remove(importId);
@@ -642,7 +651,7 @@ public class DedupeTotalDataService {
}
private void runDeleteImportTask(String importId, File tempFile, String filename,
Long operatorId, Long groupId) {
Long operatorId, Long groupId, Long companyId) {
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
if (progress == null) {
importSlots.release();
@@ -652,7 +661,7 @@ public class DedupeTotalDataService {
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
AccessScope scope = resolveAccessScope(operatorId);
DedupeTotalDataImportVo result = deleteFromExcelInternal(
inputStream, filename, progress, scope, groupId);
inputStream, filename, progress, scope, groupId, companyId);
progress.setTotalRows(result.getTotalRows());
progress.setAsinCount(result.getAsinCount());
progress.setInsertedCount(result.getInsertedCount());
@@ -722,16 +731,13 @@ public class DedupeTotalDataService {
}
}
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long operatorId) {
return importFromExcel(file, null, operatorId);
}
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long groupId, Long operatorId) {
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long groupId, Long companyId,
Long operatorId) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
AdminUserEntity uploader = getOperator(operatorId);
Long companyId = companyScopeSupport.requireCompanyId(uploader);
requireScopedCompanyId(companyId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
try (InputStream inputStream = file.getInputStream()) {
return importFromExcelInternal(
@@ -744,29 +750,6 @@ public class DedupeTotalDataService {
}
}
public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file, Long operatorId) {
return deleteFromExcel(file, null, operatorId);
}
public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file, Long groupId, Long operatorId) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
AdminUserEntity operator = getOperator(operatorId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
AccessScope scope = resolveAccessScope(operatorId);
try (InputStream inputStream = file.getInputStream()) {
return deleteFromExcelInternal(
inputStream, file.getOriginalFilename(), null, scope, group.getId());
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
// 此前丢弃原始异常:排查时只能看到一句「删除 Excel 匹配数据失败」
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败", e);
throw new BusinessException("删除 Excel 匹配数据失败", e);
}
}
private DedupeTotalDataImportVo importFromExcelInternal(InputStream inputStream, String filename,
DedupeTotalDataImportProgressVo progress,
Long uploaderUserId, String uploaderUsername,
@@ -965,7 +948,7 @@ public class DedupeTotalDataService {
if (values == null || values.isEmpty()) {
return new ImportBatchResult(0, 0);
}
Set<String> existingValues = findExistingComparableValues(values);
Set<String> existingValues = findExistingComparableValues(values, companyId);
List<DedupeTotalDataEntity> rows = new ArrayList<>(values.size());
for (String value : values) {
if (value == null || value.isBlank() || existingValues.contains(value)) {
@@ -998,7 +981,7 @@ public class DedupeTotalDataService {
for (DedupeTotalDataEntity row : rows) {
try {
Boolean oneInserted = executeInNewTransaction(() -> {
if (existsDataValue(row.getDataValue())) {
if (existsDataValue(row.getDataValue(), companyId)) {
return false;
}
return dedupeTotalDataMapper.insert(row) > 0;
@@ -1040,14 +1023,14 @@ public class DedupeTotalDataService {
private DedupeTotalDataImportVo deleteFromExcelInternal(InputStream inputStream, String filename,
DedupeTotalDataImportProgressVo progress,
AccessScope scope, Long groupId) {
AccessScope scope, Long groupId, Long companyId) {
String lowerFilename = filename == null ? "" : filename.toLowerCase();
if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) {
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
}
// 2026-09 全维度审查 C5:与导入同样改流式解析,避免 50 万行时 DOM 占 1~2GB 堆
DeleteSession session = new DeleteSession(progress, scope, groupId);
DeleteSession session = new DeleteSession(progress, scope, groupId, companyId);
try {
ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() {
private boolean headerReady;
@@ -1094,6 +1077,7 @@ public class DedupeTotalDataService {
private final DedupeTotalDataImportProgressVo progress;
private final AccessScope scope;
private final Long groupId;
private final Long companyId;
private final Set<String> seenInFile = new HashSet<>();
private final List<String> pendingDeletes = new ArrayList<>();
private Map<String, Integer> headerIndex = Map.of();
@@ -1102,10 +1086,12 @@ public class DedupeTotalDataService {
private int deletedCount;
private int skippedCount;
private DeleteSession(DedupeTotalDataImportProgressVo progress, AccessScope scope, Long groupId) {
private DeleteSession(DedupeTotalDataImportProgressVo progress, AccessScope scope, Long groupId,
Long companyId) {
this.progress = progress;
this.scope = scope;
this.groupId = groupId;
this.companyId = companyId;
}
private void acceptRow(Map<Integer, String> rowMap) {
@@ -1124,7 +1110,7 @@ public class DedupeTotalDataService {
// 批量删除:攒批 + IN 一次删,避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1
pendingDeletes.add(asin);
if (pendingDeletes.size() >= DELETE_BATCH_SIZE) {
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId, companyId);
pendingDeletes.clear();
}
updateProgress();
@@ -1146,7 +1132,7 @@ public class DedupeTotalDataService {
private DedupeTotalDataImportVo finish() {
if (!pendingDeletes.isEmpty()) {
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId, companyId);
pendingDeletes.clear();
}
updateProgress();
@@ -1167,7 +1153,9 @@ public class DedupeTotalDataService {
DedupeTotalDataEntity entity = getAccessibleById(id, operatorId);
AdminUserEntity operator = getOperator(operatorId);
String dataValue = normalizeComparableValue(request.getDataValue());
ensureUnique(dataValue, id);
// 改值时的重名比对认这一行自己的归属,不认操作者的 tab:超管在别家页签里改别家的行,
// 要比的是那一家的库。
ensureUnique(dataValue, id, entity.getCompanyId());
entity.setDataValue(dataValue);
ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), operator);
entity.setGroupId(group.getId());
@@ -1207,32 +1195,58 @@ public class DedupeTotalDataService {
return normalizeExcelText(value).toUpperCase(Locale.ROOT);
}
private void ensureUnique(String dataValue, Long excludeId) {
/**
* 「已存在」只在一家公司内成立:本表唯一键自 V145 起是 uk_company_data_value(company_id,
* data_value),同值本来就允许归两家。此前这一刀不带公司,结果是 B 公司导入过的 ASIN
* 会让 A 公司的「新增」报"数据已存在"——用户看到的是"我加不进去",而库里根本没有他的行。
*/
private void ensureUnique(String dataValue, Long excludeId, Long companyId) {
requireScopedCompanyId(companyId);
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getCompanyId, companyId)
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
.last("LIMIT 1");
DedupeTotalDataEntity exists = dedupeTotalDataMapper.selectOne(query);
if (exists != null && (excludeId == null || !exists.getId().equals(excludeId))) {
throw new BusinessException("数据已存在");
throw new BusinessException("本公司已存在该数据");
}
}
private boolean existsDataValue(String dataValue) {
private boolean existsDataValue(String dataValue, Long companyId) {
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getCompanyId, companyId)
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
.last("LIMIT 1");
return dedupeTotalDataMapper.selectOne(query) != null;
}
/**
* 公司身份闸门:本表每一条按公司取的判据都从这一句过。
*
* <p>值由 controller 在请求线程上算好传进来({@code DataReadScope.readCompanyIdOrOwn}:超管认
* tab、非超管恒本公司)。这里只判空,不补默认——导入与去重跑在虚拟线程上,那时已经没有请求
* 上下文可查了,兜底只能兜出一个"谁都比对不上"的空集。</p>
*/
static void requireScopedCompanyId(Long companyId) {
if (companyId == null) {
throw new BusinessException(
"缺少归属公司:请先在页面上方选择公司,再操作去重数据库");
}
}
/** 批量删除大小:单事务 IN 删除的阈值,兼顾锁窗口与事务日志。 */
private static final int DELETE_BATCH_SIZE = 500;
/** 整批一个独立事务按 IN 一次删除;返回实际删除行数(计入 deletedCount,不再额外计 skipped)。 */
private int deleteBatchByDataValues(List<String> dataValues, AccessScope scope, Long groupId) {
/** 包内可见只为让用例抓到真正的删除谓词(同 {@link #requireCompanyAttribution} 的先例)。 */
int deleteBatchByDataValues(List<String> dataValues, AccessScope scope, Long groupId,
Long companyId) {
requireScopedCompanyId(companyId);
List<String> batch = List.copyOf(dataValues);
Integer deleted = newRequiresNewTemplate().execute(status -> {
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getCompanyId, companyId)
.in(DedupeTotalDataEntity::getDataValue, batch);
applyGroupScope(query, scope, groupId);
return dedupeTotalDataMapper.delete(query);
@@ -1512,6 +1526,6 @@ public class DedupeTotalDataService {
* @param wholeCompany 公司超管:不做分组维度裁剪,userIds = 本公司全体成员,
* 查询条件退化为 uploader_user_id IN (成员)——公司超管的"组"就是整家公司
*/
private record AccessScope(boolean allUsers, Set<Long> groupIds, Set<Long> userIds, boolean wholeCompany) {
record AccessScope(boolean allUsers, Set<Long> groupIds, Set<Long> userIds, boolean wholeCompany) {
}
}
@@ -1,8 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "https://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper">
<!-- 由注解 SQL 平移而来,SQL 文本未改写;语句 id 与接口方法名逐字一致。
等价性由 MigratedSqlEquivalenceTest 按搬迁前的 fixture 逐条比对。 -->
<!-- 由注解 SQL 平移而来,语句 id 与接口方法名逐字一致;等价性由 MigratedSqlEquivalenceTest
按基准账本逐条比对。
一处例外要说清:selectExistingDataValues 不是平移。它当年是 @SelectProvider、没进过这本账,
账本里与它同名的那条基准记的就是它按公司收窄之后的形态;同批摘掉了已零调用者的
selectAllDataValues,所以账本行数与 select 总数双双不动。 -->
<insert id="insertBatchIgnore">
INSERT IGNORE INTO biz_dedupe_total_data
(data_value, country, group_id, uploader_user_id, uploader_username, company_id)
@@ -11,7 +14,10 @@
(#{row.dataValue}, #{row.country}, #{row.groupId}, #{row.uploaderUserId}, #{row.uploaderUsername}, #{row.companyId})
</foreach>
</insert>
<select id="selectAllDataValues" resultType="string">
<select id="selectExistingDataValues" resultType="string">
SELECT data_value FROM biz_dedupe_total_data
WHERE company_id = #{companyId}
AND data_value IN
<foreach collection="values" item="value" open="(" separator="," close=")">#{value}</foreach>
</select>
</mapper>
@@ -71,7 +71,10 @@ class CatchSilenceRatchetTest {
// 随死代码一起离开树;实测重抛数 387、不重抛与只 log 两数未涨 ⇒ 没有任何一处"异常会往外走"
// 被改回吞掉。这一格的本意是"重抛只许涨不许跌",所以调基线必须带着这样的逐处出处;
// 拿"总数变小了"当理由不算——那种情况下真正要看的是不重抛那一侧有没有涨。
static final int RETHROW_FLOOR = 387;
// 10-09 再下调 387 -> 386,出处点名:去重库按公司收口那一笔删掉了 DedupeTotalDataService.deleteFromExcel
// 的两个零调用者重载,其中 `throw new BusinessException("删除 Excel 匹配数据失败", e)` 是 RETHROW 形状,
// 随死代码一起离开树;实测重抛 386、不重抛与只 log 两数未涨(那两格仍绿)=> 没有任何一处「异常会往外走」被改回吞掉。
static final int RETHROW_FLOOR = 386;
static final int STRICT_NOT_RETHROW = 559;
static final int STRICT_LOG_ONLY = 172; // 严格口径:块内每条顶层语句都是 log 调用才算
static final int STRICT_EMPTY = 56;
@@ -119,7 +119,7 @@ class CollectDataBrandRecheckInjectorTest {
String payload = new ObjectMapper().writeValueAsString(List.of(row("B01", "Nike"), row("B02", "Nike")));
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(payload);
CollectDataResultRowVo kept = row("B01", "Nike");
when(collectDataBatchQuery.filter(anyList())).thenReturn(
when(collectDataBatchQuery.filter(anyList(), any())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(kept), 1, 0));
when(transientPayloadStorageService.storeResultPayload(eq("COLLECT_DATA"), eq(100L), eq("hash1"),
eq("recheck-chunk-1"), anyString())).thenReturn("ptr2");
@@ -179,7 +179,7 @@ class CollectDataBrandRecheckInjectorTest {
when(transientPayloadStorageService.resolvePayload(eq("ptr1"), anyString())).thenReturn(
"[{\"asin\":\"B01\",\"brand\":\"Nike\"}]");
// 补测窗口内该 ASIN 已被其他任务采走:复查全滤掉
when(collectDataBatchQuery.filter(anyList())).thenReturn(
when(collectDataBatchQuery.filter(anyList(), any())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(), 1, 0));
GroupOutcome outcome = injector.apply(task(100L), Map.of("nike", RecheckVerdict.CLEAN));
@@ -145,7 +145,7 @@ class CollectDataServiceTxBoundaryTest {
}).when(taskChunkMapper).insert(any(TaskChunkEntity.class));
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin("B0COLLECT1");
lenient().when(collectDataBatchQuery.filter(any())).thenReturn(
lenient().when(collectDataBatchQuery.filter(any(), any())).thenReturn(
new CollectDataBatchQuery.FilterResult(List.of(row), 0, 0));
lenient().when(brandBatchFilter.filter(any())).thenReturn(
new CollectDataBrandBatchFilter.BrandBatchOutcome(List.of(), List.of(), List.of(row)));
@@ -1,7 +1,7 @@
package com.nanri.aiimage.modules.collectdata.util;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService;
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
import org.junit.jupiter.api.BeforeEach;
@@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -28,25 +29,36 @@ import static org.mockito.Mockito.when;
*/
class CollectDataBatchQueryTest {
private DedupeTotalDataMapper dedupeMapper;
/** 采集任务归属账号:比对集由它推导,「推不出公司就不比对」那条闸门钉在 dedupe 侧的用例里。 */
private static final Long OWNER_USER_ID = 7L;
private DedupeTotalDataService dedupeService;
private InvalidAsinDataMapper invalidAsinMapper;
private CollectDataBatchQuery query;
@BeforeEach
void setUp() {
dedupeMapper = mock(DedupeTotalDataMapper.class);
dedupeService = mock(DedupeTotalDataService.class);
invalidAsinMapper = mock(InvalidAsinDataMapper.class);
query = new CollectDataBatchQuery(dedupeMapper, invalidAsinMapper);
query = new CollectDataBatchQuery(dedupeService, invalidAsinMapper);
}
/**
* 生产入口已改成 {@code filter(rows, ownerUserId)};本类逐支测的是过滤语义,
* 公司归属由这一层统一喂,不逐支重复——归属本身由下面那两支新用例专钉。
*/
private List<CollectDataResultRowVo> filterByExistingAsin(List<CollectDataResultRowVo> rows) {
return query.filter(rows, OWNER_USER_ID).kept();
}
@Test
void test_task_045_asin_brand_normal_default_path() {
// 正常输入:去重表已存在的 ASIN 被过滤,无效品牌表中的品牌被过滤,其余保留。
when(dedupeMapper.selectExistingDataValues(anyList()))
.thenReturn(List.of("B000000001"));
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any()))
.thenReturn(Set.of("B000000001"));
when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("Zara")));
List<CollectDataResultRowVo> out = query.filterByExistingAsin(List.of(
List<CollectDataResultRowVo> out = filterByExistingAsin(List.of(
row("B000000001", "Nike"),
row("B000000002", "Zara"),
row("B000000003", "H&M")
@@ -68,14 +80,14 @@ class CollectDataBatchQueryTest {
for (int i = 0; i < 300; i++) {
rows.add(row("B" + String.format("%09d", i + 1), "bad"));
}
List<String> existing = new ArrayList<>();
Set<String> existing = new java.util.HashSet<>();
for (int i = 0; i < 100; i++) {
existing.add("D" + String.format("%09d", i + 1));
}
when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(existing);
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())).thenReturn(existing);
when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("bad")));
List<CollectDataResultRowVo> out = query.filterByExistingAsin(rows);
List<CollectDataResultRowVo> out = filterByExistingAsin(rows);
assertEquals(200, out.size(), "100 行去重 + 300 行无效品牌 = 过滤 400 行");
assertEquals("D000000101", out.get(0).getAsin(), "输出首行保持输入顺序");
@@ -87,51 +99,51 @@ class CollectDataBatchQueryTest {
@Test
void test_task_045_asin_brand_normal_repeated_operation_is_idempotent() {
// 幂等:同一输入重复执行输出一致,两次都发批量查询。
when(dedupeMapper.selectExistingDataValues(anyList()))
.thenReturn(List.of("B000000001"));
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any()))
.thenReturn(Set.of("B000000001"));
when(invalidAsinMapper.selectList(any())).thenReturn(List.of(entity("bad")));
List<CollectDataResultRowVo> rows = List.of(
row("B000000001", "bad"),
row("B000000002", "good")
);
List<CollectDataResultRowVo> first = query.filterByExistingAsin(rows);
List<CollectDataResultRowVo> second = query.filterByExistingAsin(rows);
List<CollectDataResultRowVo> first = filterByExistingAsin(rows);
List<CollectDataResultRowVo> second = filterByExistingAsin(rows);
assertEquals(first, second, "重复执行输出一致");
assertEquals(1, first.size());
verify(dedupeMapper, times(2)).selectExistingDataValues(anyList());
verify(dedupeService, times(2)).findExistingComparableValuesForOwner(anyList(), any());
verify(invalidAsinMapper, times(2)).selectList(any());
}
@Test
void test_task_045_asin_brand_boundary_empty_input() {
// 空输入:空列表/无 ASIN 无品牌的行集合不触发数据库查询,安全跳过。
List<CollectDataResultRowVo> empty = query.filterByExistingAsin(List.of());
List<CollectDataResultRowVo> empty = filterByExistingAsin(List.of());
assertEquals(0, empty.size(), "空列表返回空");
List<CollectDataResultRowVo> blankRows = query.filterByExistingAsin(List.of(
List<CollectDataResultRowVo> blankRows = filterByExistingAsin(List.of(
row("", ""), row(null, null)
));
assertEquals(2, blankRows.size(), "无 ASIN 无品牌行不触发查询且全保留");
verify(dedupeMapper, never()).selectExistingDataValues(anyList());
verify(dedupeService, never()).findExistingComparableValuesForOwner(anyList(), any());
verify(invalidAsinMapper, never()).selectList(any());
}
@Test
void test_task_045_asin_brand_boundary_single_item() {
// 单元素:单行命中/未命中均正确,不依赖批量路径。
when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(List.of());
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())).thenReturn(Set.of());
when(invalidAsinMapper.selectList(any())).thenReturn(List.of());
List<CollectDataResultRowVo> out = query.filterByExistingAsin(List.of(
List<CollectDataResultRowVo> out = filterByExistingAsin(List.of(
row("B000000001", "solo")
));
assertEquals(1, out.size(), "单行未命中保留");
assertEquals("solo", out.get(0).getBrand());
when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(List.of("B000000001"));
List<CollectDataResultRowVo> hit = query.filterByExistingAsin(List.of(
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())).thenReturn(Set.of("B000000001"));
List<CollectDataResultRowVo> hit = filterByExistingAsin(List.of(
row("B000000001", "solo")
));
assertEquals(0, hit.size(), "单行命中去重表被过滤");
@@ -144,26 +156,26 @@ class CollectDataBatchQueryTest {
for (int i = 0; i < 5000; i++) {
rows.add(row("B" + String.format("%09d", i), "brand-" + i));
}
when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(List.of());
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())).thenReturn(Set.of());
when(invalidAsinMapper.selectList(any())).thenReturn(List.of());
List<CollectDataResultRowVo> out = query.filterByExistingAsin(rows);
List<CollectDataResultRowVo> out = filterByExistingAsin(rows);
assertEquals(5000, out.size(), "超大集合全量保留");
verify(dedupeMapper).selectExistingDataValues(anyList());
verify(dedupeService).findExistingComparableValuesForOwner(anyList(), any());
verify(invalidAsinMapper).selectList(any());
}
@Test
void test_task_045_asin_brand_invalid_input_rejected() {
// 非法参数:null 行安全跳过;数据库返回 null 集合安全处理。
when(dedupeMapper.selectExistingDataValues(anyList())).thenReturn(null);
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())).thenReturn(null);
when(invalidAsinMapper.selectList(any())).thenReturn(null);
List<CollectDataResultRowVo> rows = new ArrayList<>();
rows.add(null);
rows.add(row("B000000001", " Nike "));
List<CollectDataResultRowVo> out = query.filterByExistingAsin(rows);
List<CollectDataResultRowVo> out = filterByExistingAsin(rows);
assertEquals(1, out.size(), "null 行不丢弃");
assertEquals("B000000001", out.get(0).getAsin(), "正常行保留");
@@ -173,26 +185,39 @@ class CollectDataBatchQueryTest {
@Test
void test_task_045_asin_brand_dependency_failure_releases_resources() {
// 依赖失败:去重表查询抛错时异常向上传播;恢复后再次执行成功,不残留状态。
when(dedupeMapper.selectExistingDataValues(anyList()))
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any()))
.thenThrow(new RuntimeException("db down"))
.thenReturn(List.of("B000000001"));
.thenReturn(Set.of("B000000001"));
when(invalidAsinMapper.selectList(any())).thenReturn(List.of());
boolean threw = false;
try {
query.filterByExistingAsin(List.of(row("B000000001", "nike")));
filterByExistingAsin(List.of(row("B000000001", "nike")));
} catch (RuntimeException ex) {
threw = "db down".equals(ex.getMessage());
}
assertTrue(threw, "去重查询失败向上抛错");
List<CollectDataResultRowVo> out = query.filterByExistingAsin(List.of(
List<CollectDataResultRowVo> out = filterByExistingAsin(List.of(
row("B000000001", "nike"), row("B000000002", "adidas")
));
assertEquals(1, out.size(), "恢复后过滤正确");
assertEquals("B000000002", out.get(0).getAsin(), "恢复后命中逻辑仍生效");
}
/**
* 采集结果按去重库过滤时,比对集必须由「任务归属账号」那一家推导。
* 这一支钉的是参数真的传下去了:换成常量或漏传,下面这条 eq 就红。
*/
@Test
void comparisonSetIsRequestedForTheTaskOwnerAccount() {
when(dedupeService.findExistingComparableValuesForOwner(anyList(), any())).thenReturn(Set.of());
filterByExistingAsin(List.of(row("B000000001", "nike")));
verify(dedupeService).findExistingComparableValuesForOwner(List.of("B000000001"), OWNER_USER_ID);
}
private static CollectDataResultRowVo row(String asin, String brand) {
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setAsin(asin);
@@ -15,7 +15,8 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* {@code insertBatchIgnore} 生成 SQL 的审计:去重库写侧必须显式带 {@code company_id}。
* 去重库两条自定义语句的 SQL 审计:写侧 {@code insertBatchIgnore} 必须显式落 {@code company_id},
* 比对侧 {@code selectExistingDataValues} 必须按 {@code company_id} 收窄。
*
* <p>为什么必须单独有一支:服务层测试用 mock 捕获的是实体对象,看不到最终 SQL——本表写侧
* 靠的是注解里的列清单(不是 MyBatis-Plus 的实体字段推导),把 {@code company_id} 从清单里漏掉
@@ -43,6 +44,31 @@ class DedupeTotalDataMapperSqlTest {
assertFalse(sql.contains("$"), "不得出现 ${} 拼接面: " + sql);
}
/**
* 比对集的租户谓词审计。断的是最终 SQL 而不是方法签名:这条从 @SelectProvider 迁到 XML 之后,
* 少写一个 {@code AND company_id = #{companyId}} 在 Java 侧照样编译通过、照样返回行,
* 而后果是「别家导入的 ASIN 被当成已存在」——去重执行少行、导入静默跳过。
*/
@Test
void selectExistingValuesAreScopedToOneCompany() throws Exception {
Configuration configuration = MapperXmlSupport.configurationWith(DedupeTotalDataMapper.class);
MappedStatement statement = configuration.getMappedStatement(
DedupeTotalDataMapper.class.getName() + ".selectExistingDataValues");
BoundSql boundSql = statement.getBoundSql(java.util.Map.of(
"values", List.of("B000000001", "B000000002"), "companyId", 3L));
String sql = boundSql.getSql().replaceAll("\\s+", " ");
assertTrue(sql.contains("WHERE company_id = ?"),
"比对集必须按 company_id 收窄,否则一次去重会看见全平台: " + sql);
assertTrue(sql.contains("data_value IN ("),
"值集合仍按 IN 批量比对(收窄不该把它拆成逐行查): " + sql);
assertEquals(3, boundSql.getParameterMappings().size(),
"2 个值 + 1 个公司 = 3 个占位符: " + sql);
assertEquals("companyId", boundSql.getParameterMappings().get(0).getProperty(),
"第一个占位符就是公司(谓词排在 IN 之前,走 uk_company_data_value 前缀): " + sql);
assertFalse(sql.contains("$"), "不得出现 ${} 拼接面: " + sql);
}
private static BoundSql boundSqlFor(int rowCount) throws Exception {
// SQL 已从注解迁到 mapper XML(档 B 口径),原文与语句都要从 XML 那侧取
String script = MapperXmlSupport.statementBody(DedupeTotalDataMapper.class, "insertBatchIgnore");
@@ -69,6 +69,8 @@ class DedupeRunProgressResumeTest {
@Mock
private DedupeTotalDataService dedupeTotalDataService;
@Mock
private com.nanri.aiimage.common.service.CompanyScopeSupport companyScopeSupport;
@Mock
private StringRedisTemplate redisTemplate;
@Mock
private ValueOperations<String, String> valueOperations;
@@ -102,7 +104,7 @@ class DedupeRunProgressResumeTest {
return true;
});
return new DedupeRunService(fileTaskMapper, fileResultMapper, storageProperties,
ossStorageService, dedupeTotalDataService, redisTemplate, objectMapper,
ossStorageService, dedupeTotalDataService, companyScopeSupport, redisTemplate, objectMapper,
new InstanceMetadata("server-110"));
}
@@ -425,7 +427,7 @@ class DedupeRunProgressResumeTest {
// 异步 worker 是否来得及读这个目录取决于线程时序,只做兜底、不参与断言
lenient().when(storageProperties.getLocalTempDir()).thenReturn("D:/definitely-not-a-real-dedupe-dir/xx");
DedupeRunService service = new DedupeRunService(fileTaskMapper, fileResultMapper, storageProperties,
ossStorageService, dedupeTotalDataService, redisTemplate, objectMapper,
ossStorageService, dedupeTotalDataService, companyScopeSupport, redisTemplate, objectMapper,
new InstanceMetadata("server-110"));
DedupeRunProgressVo submitted = service.submitRun(request(1));
@@ -25,6 +25,8 @@ import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
@@ -144,6 +146,20 @@ class DedupeRunServiceCleanRulesTest {
return rows.stream().map(DedupeCandidateRow::asinValue).toList();
}
/**
* 去重执行拿哪一家的库比对:提交人自己那一家。
* 摘掉 DedupeRunService 里往下传的 companyId,这支会红(比对收到的是 null 或别的值)。
*/
@Test
void comparisonSetIsTakenForTheSubmittingAccountCompanyOnly(@TempDir Path tempDir) throws Exception {
runClean(tempDir, List.<String[]>of(
new String[]{"1", "B0AAAAAA01", "英国", "10.00", "FBM", "品牌A", "在售", "2026-09-01"},
new String[]{"1_2", "B0AAAAAA02", "英国", "11.00", "FBM", "品牌A", "在售", "2026-09-01"}),
List.of("id", "ASIN", "国家", "价格", "品牌"));
verify(dedupeTotalDataService).findExistingComparableValues(any(), eq(1L));
}
private File runClean(Path tempDir, List<String[]> rows, List<String> selectedColumns) throws Exception {
File input = tempDir.resolve("input.xlsx").toFile();
File output = tempDir.resolve("output.xlsx").toFile();
@@ -151,8 +167,8 @@ class DedupeRunServiceCleanRulesTest {
DedupeRunService service = new DedupeRunService(
fileTaskMapper, fileResultMapper, storageProperties, ossStorageService, dedupeTotalDataService,
null, null, new com.nanri.aiimage.config.InstanceMetadata("test-instance"));
when(dedupeTotalDataService.findExistingComparableValues(any())).thenReturn(Set.of());
null, null, null, new com.nanri.aiimage.config.InstanceMetadata("test-instance"));
when(dedupeTotalDataService.findExistingComparableValues(any(), any())).thenReturn(Set.of());
when(dedupeTotalDataService.normalizeComparableValueOrBlank(any()))
.thenAnswer(invocation -> {
String value = invocation.getArgument(0);
@@ -160,7 +176,7 @@ class DedupeRunServiceCleanRulesTest {
});
ReflectionTestUtils.invokeMethod(service, "cleanExcelByStreamRules",
input, output, selectedColumns, false, true, true);
input, output, selectedColumns, false, true, true, 1L);
return output;
}
@@ -44,6 +44,7 @@ import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -77,7 +78,6 @@ class DedupeTotalDataServiceTest {
void createRecordsUploaderIdentity() {
AdminUserEntity uploader = user(23L, "normal", "member-a");
when(adminUserMapper.selectById(23L)).thenReturn(uploader);
when(companyScopeSupport.requireCompanyId(any(AdminUserEntity.class))).thenReturn(1L);
stubWritableGroup(23L, 7L);
when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null);
when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class))).thenAnswer(invocation -> {
@@ -98,7 +98,7 @@ class DedupeTotalDataServiceTest {
DedupeTotalDataCreateRequest request = new DedupeTotalDataCreateRequest();
request.setDataValue(" b012345678 ");
request.setGroupId(7L);
DedupeTotalDataItemVo item = service.create(request, 23L);
DedupeTotalDataItemVo item = service.create(request, 1L, 23L);
ArgumentCaptor<DedupeTotalDataEntity> captor = ArgumentCaptor.forClass(DedupeTotalDataEntity.class);
verify(dedupeTotalDataMapper).insert(captor.capture());
@@ -184,13 +184,12 @@ class DedupeTotalDataServiceTest {
@Test
void duplicateExcelValueKeepsOriginalUploader() throws Exception {
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
when(companyScopeSupport.requireCompanyId(any(AdminUserEntity.class))).thenReturn(1L);
stubWritableGroup(23L, 7L);
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678")))
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"), 1L))
.thenReturn(List.of("B012345678"));
MockMultipartFile file = asinWorkbook("B012345678");
var result = service.importFromExcel(file, 7L, 23L);
var result = service.importFromExcel(file, 7L, 1L, 23L);
assertEquals(0, result.getInsertedCount());
assertEquals(1, result.getSkippedCount());
@@ -199,17 +198,15 @@ class DedupeTotalDataServiceTest {
@Test
void importRejectsOperatorWithoutCompany() throws Exception {
AdminUserEntity noCompany = user(23L, "normal", "member-a");
noCompany.setCompanyId(null);
when(adminUserMapper.selectById(23L)).thenReturn(noCompany);
when(companyScopeSupport.requireCompanyId(any(AdminUserEntity.class)))
.thenThrow(new BusinessException(403, "当前账号没有所属公司,无法按公司读写数据"));
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
// 公司身份由 controller 在请求线程上算好传进来(超管认 tab、非超管认自己那一家);
// service 这道闸门钉的是「没算出来就不许写」,它是最后那道。
BusinessException exception = assertThrows(
BusinessException.class,
() -> service.importFromExcel(asinWorkbook("B012345678"), 7L, 23L));
() -> service.importFromExcel(asinWorkbook("B012345678"), 7L, null, 23L));
assertEquals("当前账号没有所属公司,无法按公司读写数据", exception.getMessage());
assertEquals("缺少归属公司:请先在页面上方选择公司,再操作去重数据库", exception.getMessage());
verify(dedupeTotalDataMapper, never()).insertBatchIgnore(any());
verify(dedupeTotalDataMapper, never()).insert(any(DedupeTotalDataEntity.class));
}
@@ -217,13 +214,12 @@ class DedupeTotalDataServiceTest {
@Test
void importReadsAndNormalizesCountryColumn() throws Exception {
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
when(companyScopeSupport.requireCompanyId(any(AdminUserEntity.class))).thenReturn(1L);
stubWritableGroup(23L, 7L);
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"))).thenReturn(List.of());
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"), 1L)).thenReturn(List.of());
when(dedupeTotalDataMapper.insertBatchIgnore(any())).thenReturn(1);
MockMultipartFile file = asinCountryWorkbook("B012345678", "德国, US,United Kingdom");
var result = service.importFromExcel(file, 7L, 23L);
var result = service.importFromExcel(file, 7L, 1L, 23L);
assertEquals(1, result.getInsertedCount());
ArgumentCaptor<List<DedupeTotalDataEntity>> captor = ArgumentCaptor.forClass(List.class);
@@ -237,12 +233,11 @@ class DedupeTotalDataServiceTest {
@Test
void importKeepsCountryBlankWhenHeaderMissing() throws Exception {
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
when(companyScopeSupport.requireCompanyId(any(AdminUserEntity.class))).thenReturn(1L);
stubWritableGroup(23L, 7L);
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"))).thenReturn(List.of());
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"), 1L)).thenReturn(List.of());
when(dedupeTotalDataMapper.insertBatchIgnore(any())).thenReturn(1);
var result = service.importFromExcel(asinWorkbook("B012345678"), 7L, 23L);
var result = service.importFromExcel(asinWorkbook("B012345678"), 7L, 1L, 23L);
assertEquals(1, result.getInsertedCount());
ArgumentCaptor<List<DedupeTotalDataEntity>> captor = ArgumentCaptor.forClass(List.class);
@@ -274,17 +269,16 @@ class DedupeTotalDataServiceTest {
@Test
void concurrentDuplicateDuringImportIsSkipped() throws Exception {
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
when(companyScopeSupport.requireCompanyId(any(AdminUserEntity.class))).thenReturn(1L);
stubWritableGroup(23L, 7L);
when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null);
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"))).thenReturn(List.of());
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"), 1L)).thenReturn(List.of());
when(transactionManager.getTransaction(any())).thenReturn(transactionStatus);
when(dedupeTotalDataMapper.insertBatchIgnore(any()))
.thenThrow(new DuplicateKeyException("duplicate batch"));
when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class)))
.thenThrow(new DuplicateKeyException("duplicate"));
var result = service.importFromExcel(asinWorkbook("b012345678"), 7L, 23L);
var result = service.importFromExcel(asinWorkbook("b012345678"), 7L, 1L, 23L);
assertEquals(0, result.getInsertedCount());
assertEquals(1, result.getSkippedCount());
@@ -360,18 +354,83 @@ class DedupeTotalDataServiceTest {
verify(dedupeTotalDataMapper, never()).selectCount(any());
}
/**
* 比对集只认一家。这条曾经叫 comparableValueLookupRemainsGlobal,显式钉着「不参与公司收窄」
* (2026-10 多公司一期)——那是半套发布的遗留:库里 uk_company_data_value 自 V145 起就允许
* 同值归两家,代码却仍拿全平台当比对集。2026-10-09 按控制方定案反转。
*/
@Test
void comparableValueLookupRemainsGlobal() {
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678")))
void comparableValueLookupIsScopedToOneCompany() {
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"), 1L))
.thenReturn(List.of("b012345678"));
Set<String> values = service.findExistingComparableValues(List.of(" b012345678 "));
Set<String> values = service.findExistingComparableValues(List.of(" b012345678 "), 1L);
assertEquals(Set.of("B012345678"), values);
verify(adminUserMapper, never()).selectById(any(Long.class));
verify(shopManageGroupMapper, never()).selectManagedMemberUserIds(any(Long.class));
}
/**
* 删除导入的谓词必须带公司,且**对平台超管也一样**:allUsers 那一站原本只按分组裁,
* 超管不选分组时删除条件只剩一个 data_value IN——同名 ASIN 在几家库里就删几次。
* 这条走真 wrapper(不是 mock 的返回值):MP 的 IN 段要先把 getSqlSegment 求值一次才铺出来。
*/
@Test
void deleteImportNeverReachesAnotherCompanyRows() {
when(transactionManager.getTransaction(any())).thenReturn(transactionStatus);
when(dedupeTotalDataMapper.delete(any())).thenReturn(1);
service.deleteBatchByDataValues(List.of("B012345678"),
new DedupeTotalDataService.AccessScope(true, java.util.Set.of(), java.util.Set.of(), false),
null, 3L);
ArgumentCaptor<LambdaQueryWrapper<DedupeTotalDataEntity>> captor =
ArgumentCaptor.forClass(LambdaQueryWrapper.class);
verify(dedupeTotalDataMapper).delete(captor.capture());
LambdaQueryWrapper<DedupeTotalDataEntity> wrapper = captor.getValue();
wrapper.getSqlSegment();
assertThat(wrapper.getSqlSegment()).contains("company_id");
assertThat(wrapper.getParamNameValuePairs().values()).contains(3L);
}
/**
* 「数据已存在」只在本公司内成立。这条钉的是重名查询的 wrapper:摘掉 company_id 那一刀,
* B 家导入过的 ASIN 会让 A 家的「新增」报已存在,而 A 家库里根本没有那行。
*/
@Test
void createChecksDuplicatesInsideOneCompanyOnly() {
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
stubWritableGroup(23L, 7L);
when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null);
when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class))).thenAnswer(invocation -> {
DedupeTotalDataEntity entity = invocation.getArgument(0);
entity.setId(91L);
return 1;
});
when(dedupeTotalDataMapper.selectById(91L)).thenReturn(data(91L, 23L));
DedupeTotalDataCreateRequest request = new DedupeTotalDataCreateRequest();
request.setDataValue("B012345678");
request.setGroupId(7L);
service.create(request, 3L, 23L);
ArgumentCaptor<LambdaQueryWrapper<DedupeTotalDataEntity>> captor =
ArgumentCaptor.forClass(LambdaQueryWrapper.class);
verify(dedupeTotalDataMapper).selectOne(captor.capture());
captor.getValue().getSqlSegment();
assertThat(captor.getValue().getSqlSegment()).contains("company_id");
assertThat(captor.getValue().getParamNameValuePairs().values()).contains(3L);
}
/** 没有归属公司就不比对:宁可炸,也不给出一条"比对集是空的"与"一家都没命中"同形的路径。 */
@Test
void comparableValueLookupRefusesACompanylessComparison() {
assertThrows(BusinessException.class,
() -> service.findExistingComparableValues(List.of("B012345678"), null));
verify(dedupeTotalDataMapper, never()).selectExistingDataValues(any(), any());
}
@Test
void exportUsesScopedUsernameAndProducesWorkbook() throws Exception {
when(adminUserMapper.selectById(10L)).thenReturn(user(10L, "admin", "leader"));
@@ -3,7 +3,7 @@ AdminUserMapper selectCompanySuperAdminEarliestExpiry Select PURE SELECT MIN(exp
BrandCrawlTaskMapper selectExpiredTerminalTaskIds Select PURE SELECT id FROM brand_crawl_tasks\nWHERE status IN ('success', 'failed', 'cancelled')\n AND updated_at < #{cutoff}\nORDER BY id ASC\nLIMIT #{batchSize}
CollectDataItemMapper insertBatch Insert PURE <script>\nINSERT INTO biz_collect_data_item\n (task_id, row_index, source_file_key, source_filename, keyword, status_value, extra_json, created_at)\nVALUES\n<foreach collection="rows" item="row" separator=",">\n (#{row.taskId}, #{row.rowIndex}, #{row.sourceFileKey}, #{row.sourceFilename},\n #{row.keyword}, #{row.statusValue}, #{row.extraJson}, #{row.createdAt})\n</foreach>\n</script>
DedupeTotalDataMapper insertBatchIgnore Insert PURE <script>\nINSERT IGNORE INTO biz_dedupe_total_data\n (data_value, country, group_id, uploader_user_id, uploader_username, company_id)\nVALUES\n<foreach collection="rows" item="row" separator=",">\n (#{row.dataValue}, #{row.country}, #{row.groupId}, #{row.uploaderUserId}, #{row.uploaderUsername}, #{row.companyId})\n</foreach>\n</script>
DedupeTotalDataMapper selectAllDataValues Select PURE SELECT data_value FROM biz_dedupe_total_data
DedupeTotalDataMapper selectExistingDataValues Select PURE <script>\nSELECT data_value FROM biz_dedupe_total_data\nWHERE company_id = #{companyId}\n AND data_value IN\n<foreach collection="values" item="value" open="(" separator="," close=")">#{value}</foreach>\n</script>
DeviceLogFileMapper deleteOlderThanBatch Delete PURE DELETE FROM device_log_file\nWHERE log_date < #{cutoff}\nLIMIT #{batchSize}
FileResultMapper selectRowsUpdatedAtByTaskIds Select PURE <script>\nSELECT task_id AS taskId, MAX(updated_at) AS latestRowAt, COUNT(*) AS rowCount\nFROM biz_file_result\nWHERE module_type = #{moduleType}\n AND task_id IN\n <foreach item="taskId" collection="taskIds" open="(" separator="," close=")">#{taskId}</foreach>\nGROUP BY task_id\n</script>
FileTaskMapper selectExpiredTerminalTaskIds Select PURE SELECT id FROM biz_file_task\nWHERE module_type = #{moduleType}\n AND status IN ('SUCCESS', 'FAILED', 'CANCELLED', 'CANCELED')\n AND updated_at < #{cutoff}\nORDER BY id ASC\nLIMIT #{batchSize}
Can't render this file because it contains an unexpected character in line 1 and column 222.