收口(去重库): 五处比对与写入口按 company_id 认公司,不再跨公司去重

读侧(列表/导出)早已按公司收窄,但"去重"这个动作本身一直查全平台:
selectExistingDataValues 是 `WHERE data_value IN (...)`,一个租户条件都没有。
库里唯一键自 V145 起就是 uk_company_data_value(company_id, data_value)——
同值本来就允许归两家,是代码没跟上库的口径。

五处一起收口(第 5 处由编译器抓出,正是 V145 头注释 §1.2 点名的采集侧):
去重执行 ASIN 过滤 / 导入预检"已存在就跳过" / 手工新增与编辑的重名校验
(文案改「本公司已存在该数据」)/ 删除导入(超管不选分组时原先一次删掉全平台
同名行)/ 采集结果过滤(CollectDataBatchQuery 直连 mapper,改为经 dedupe 侧推导)。

超管代导入的归属分裂一并修掉:前端两个导入接口把公司 tab 发下去——原先
group_id 落别家的组、company_id 落超管自己那一家。公司身份一律在请求线程上由
DataReadScope.readCompanyIdOrOwn 解析后传入(导入与去重跑在虚拟线程,那时已无
请求上下文),service 侧 requireScopedCompanyId 只判空不兜底。

删两条零调用者的跨公司通道(selectAllDataValues/listComparableValues 全表读、
deleteFromExcel 两个同步重载)。反转一条反向钉:comparableValueLookupRemainsGlobal
原显式断言"比对不参与公司收窄"(多公司一期的半套遗留),现为 IsScopedToOneCompany。

三把尺子随之改动,均带出处:tsv 基准行搬家(三个标量双双不动)、前端发送口分母
17→18 组/21→22 处、CatchSilenceRatchet 重抛下限 387→386(随死代码离开树那处)。

验证:后端全量 4198 例 0 失败/0 错误/7 跳过;前端 1778 例全绿;变异自证
6/6 全部 KILLED,注入文件按 sha256 还原并复跑绿。
现网只读探针:misfiled=0、同值跨家=0 ⇒ 存量比对结果不变,行为改变只在将来
A 家导入 B 家已有 ASIN 那一刻;两口径差额 1,452 行(0.26%)全属公司 1、
全源于 10-05 账号硬删。
This commit is contained in:
2026-10-09 17:57:37 +08:00
parent 974d0c77d0
commit 5ccf3c896f
22 changed files with 393 additions and 187 deletions
@@ -122,7 +122,8 @@ public class CollectDataBrandRecheckInjector {
continue;
}
// 复查去重/无效品牌:补测窗口内该 ASIN 可能已被其他任务采走(first-writer-wins)
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(chunkClean);
CollectDataBatchQuery.FilterResult filtered =
collectDataBatchQuery.filter(chunkClean, task.getUserId());
if (filtered.kept().isEmpty()) {
log.info("[brand-recheck] 补测行全部被去重/无效过滤,不注入 taskId={} rows={}",
task.getId(), chunkClean.size());
@@ -830,7 +830,7 @@ public class CollectDataService {
public CollectDataSubmitResultVo submitResult(Long taskId, CollectDataSubmitResultRequest request) {
if (request == null) {
throw new BusinessException("request is empty");
throw new BusinessException("请求内容为空");
}
ensureRustfsPayloadStorageEnabled();
@@ -861,7 +861,9 @@ public class CollectDataService {
}
}
long prepareStartAt = System.currentTimeMillis();
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(rowsForFiltering);
// 比对集认任务归属账号那一家:chunk 回传的过滤结果不该受别家导入过什么影响
CollectDataBatchQuery.FilterResult filtered =
collectDataBatchQuery.filter(rowsForFiltering, probe.getUserId());
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome = brandBatchFilter.filter(filtered.kept());
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms",
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt);
@@ -2,7 +2,7 @@ package com.nanri.aiimage.modules.collectdata.util;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService;
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
import lombok.RequiredArgsConstructor;
@@ -22,7 +22,7 @@ import java.util.regex.Pattern;
* 多次降为一次,并保证空输入、单元素与超限集合的确定行为。
*
* 过滤语义(与 CollectDataService 原 filterByExistingAsin 一致):
* - ASIN 存在于去重表 → dedupeFiltered(增加计数)
* - ASIN 存在于「本任务归属公司」的去重表 → dedupeFiltered(增加计数)
* - 品牌存在于无效品牌表 → invalidFiltered(增加计数)
* - 其余行保留
*/
@@ -32,21 +32,22 @@ public class CollectDataBatchQuery {
private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final DedupeTotalDataService dedupeTotalDataService;
private final InvalidAsinDataMapper invalidAsinDataMapper;
/**
* 批量集合查询过滤(兼容入口)。返回过滤后的行列表;入参为 null/空时返回空列表。
*/
public List<CollectDataResultRowVo> filterByExistingAsin(List<CollectDataResultRowVo> rows) {
return filter(rows).kept();
}
/**
* 批量集合查询过滤并统计两类命中:去重表命中的 ASIN 行数与无效品牌表
* 命中的品牌行数(null/空行不计数)。入参为 null/空时返回空结果。
*
* <p>去重表那一支的比对集只认 {@code ownerUserId} 那一家:「这个 ASIN 已经有人采过了」不能拿
* 别家导入的值来判。此前它查的是全平台,于是 B 公司导入的 ASIN 会让 A 公司的采集结果少行,而
* A 公司在自己后台根本看不到那行是谁的——V145 头注释 §1.2 点名的就是这一条。推导发生在
* {@code DedupeTotalDataService#findExistingComparableValuesForOwner},不在本文件:</p>
*
* <p>无效品牌那一支<b>不带</b>公司条件,是定案不是漏改:品牌库 biz_invalid_asin_data 已于
* 2026-10-06 退回全平台共享(V151 退唯一键),反向钉在 CompanyIdParamGoesThroughReadScopeTest。</p>
*/
public FilterResult filter(List<CollectDataResultRowVo> rows) {
public FilterResult filter(List<CollectDataResultRowVo> rows, Long ownerUserId) {
if (rows == null || rows.isEmpty()) {
return new FilterResult(List.of(), 0, 0);
}
@@ -55,7 +56,8 @@ public class CollectDataBatchQuery {
.map(CollectDataResultRowVo::getAsin).toList());
Set<String> dedupeValues = new HashSet<>();
if (!asins.isEmpty()) {
List<String> existing = dedupeTotalDataMapper.selectExistingDataValues(asins);
Set<String> existing = dedupeTotalDataService
.findExistingComparableValuesForOwner(asins, ownerUserId);
if (existing != null) {
for (String value : existing) {
String normalized = normalizeAsin(value);
@@ -155,7 +155,11 @@ public class DedupeTotalDataController {
HttpServletRequest httpRequest,
@Valid @RequestBody DedupeTotalDataCreateRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(httpRequest);
return ApiResponse.success("创建成功", dedupeTotalDataService.create(request, operator.id()));
AdminUserEntity user = operator.user();
Long requestedCompanyId = request.getCompanyId();
Long companyId = dataReadScope.readCompanyIdOrOwn(user, requestedCompanyId);
Long operatorId = operator.id();
return ApiResponse.success("创建成功", dedupeTotalDataService.create(request, companyId, operatorId));
}
@PostMapping("/import")
@@ -167,9 +171,14 @@ public class DedupeTotalDataController {
public ApiResponse<DedupeTotalDataImportStartVo> importExcel(
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file,
@Parameter(description = "分组 ID", required = true) @RequestParam Long groupId,
@Parameter(description = "公司 ID") @RequestParam(name = "company_id", required = false) Long companyId,
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
return ApiResponse.success("开始导入", dedupeTotalDataService.startImport(file, groupId, operator.id()));
AdminUserEntity user = operator.user();
Long scopedCompanyId = dataReadScope.readCompanyIdOrOwn(user, companyId);
Long operatorId = operator.id();
return ApiResponse.success("开始导入",
dedupeTotalDataService.startImport(file, groupId, scopedCompanyId, operatorId));
}
@GetMapping("/import/{importId}")
@@ -190,9 +199,14 @@ public class DedupeTotalDataController {
public ApiResponse<DedupeTotalDataImportStartVo> deleteImportExcel(
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file,
@Parameter(description = "分组 ID", required = true) @RequestParam Long groupId,
@Parameter(description = "公司 ID") @RequestParam(name = "company_id", required = false) Long companyId,
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
return ApiResponse.success("开始删除", dedupeTotalDataService.startDeleteImport(file, groupId, operator.id()));
AdminUserEntity user = operator.user();
Long scopedCompanyId = dataReadScope.readCompanyIdOrOwn(user, companyId);
Long operatorId = operator.id();
return ApiResponse.success("开始删除",
dedupeTotalDataService.startDeleteImport(file, groupId, scopedCompanyId, operatorId));
}
@GetMapping("/delete-import/{importId}")
@@ -285,7 +299,8 @@ public class DedupeTotalDataController {
// 公司 tab 的读范围就地算在 handler 里,不留私有收口方法:G2 的断言 A 只在这支 handler 的
// 公司 tab 的读范围与写侧归属(create / import / delete-import)都就地算在 handler 里,
// 不留私有收口方法:G2 的断言 A 只在这支 handler 的
// 块内找 readScope(,私有方法在它自己的闭括号处收口、不算在内(CompanyIdParamGoesThroughReadScopeTest
// 类注释第 5、7 条),挪进去 = 这一页的 tab 没人看守。收窄只给超管:非超管已由 resolveAccessScope
// 按会话裁剪,且他的可见集里含「靠分组可见」的行,叠一层账号 IN 会把那批行静默删掉
@@ -4,18 +4,25 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.SelectProvider;
import java.util.List;
import java.util.StringJoiner;
@Mapper
public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> {
List<String> selectAllDataValues();
@SelectProvider(type = SqlProvider.class, method = "selectExistingDataValues")
List<String> selectExistingDataValues(@Param("values") List<String> values);
/**
* 比对集按公司取:一次去重/一批导入只能看见 {@code companyId} 名下的值。
*
* <p>这条曾是 @SelectProvider 手拼 IN 列表、且不带任何租户条件,后果有两样:去重执行会把别家
* 导入的 ASIN 当「已存在」从输出文件里删掉;导入会把别家已有的值静默跳过,本公司库里永远补不上
* 这一行。本表唯一键自 V145 起就是 uk_company_data_value(company_id, data_value)——同值本来就
* 允许归两家,是代码没跟上库的口径。</p>
*
* <p>{@code companyId} 必须由调用方保证非空:MyBatis 会把它渲染成 {@code company_id = NULL}
* 这条永假条件,静默给出空集,比当场炸更难发现。拦空值的是服务层闸门。</p>
*/
List<String> selectExistingDataValues(@Param("values") List<String> values,
@Param("companyId") Long companyId);
/**
* Import is deliberately a single SQL round trip per batch. The unique
@@ -23,14 +30,4 @@ public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity>
* the same value between the pre-check and this statement.
*/
int insertBatchIgnore(@Param("rows") List<DedupeTotalDataEntity> rows);
class SqlProvider {
public String selectExistingDataValues(@Param("values") List<String> values) {
StringJoiner placeholders = new StringJoiner(", ");
for (int i = 0; i < values.size(); i++) {
placeholders.add("#{values[" + i + "]}");
}
return "SELECT data_value FROM biz_dedupe_total_data WHERE data_value IN (" + placeholders + ")";
}
}
}
@@ -18,4 +18,8 @@ public class DedupeTotalDataCreateRequest {
@NotNull(message = "请选择分组")
@Schema(description = "分组 ID", requiredMode = Schema.RequiredMode.REQUIRED)
private Long groupId;
/** 页面上方公司 tab 选中的归属公司;非超管带什么都不算,超管不带则拒绝写入。 */
@Schema(description = "公司 ID")
private Long companyId;
}
@@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.CompanyScopeSupport;
import com.nanri.aiimage.common.service.NodeSharedStore;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.config.InstanceMetadata;
@@ -79,6 +80,7 @@ public class DedupeRunService {
private final StorageProperties storageProperties;
private final OssStorageService ossStorageService;
private final DedupeTotalDataService dedupeTotalDataService;
private final CompanyScopeSupport companyScopeSupport;
private final InstanceMetadata instanceMetadata;
/**
@@ -102,6 +104,7 @@ public class DedupeRunService {
StorageProperties storageProperties,
OssStorageService ossStorageService,
DedupeTotalDataService dedupeTotalDataService,
CompanyScopeSupport companyScopeSupport,
StringRedisTemplate stringRedisTemplate,
ObjectMapper objectMapper,
InstanceMetadata instanceMetadata) {
@@ -110,6 +113,7 @@ public class DedupeRunService {
this.storageProperties = storageProperties;
this.ossStorageService = ossStorageService;
this.dedupeTotalDataService = dedupeTotalDataService;
this.companyScopeSupport = companyScopeSupport;
this.instanceMetadata = instanceMetadata;
this.runProgressMap = new NodeSharedStore<>("aiimage:dedupe:run:progress", RUN_STATE_TTL,
DedupeRunProgressVo.class, stringRedisTemplate, objectMapper);
@@ -140,6 +144,11 @@ public class DedupeRunService {
throw new BusinessException("请至少选择一种 ID 保留规则");
}
cleanupExpiredProgress();
// 比对集锁在提交人自己那一家:「这个 ASIN 去重库里已经有了」只能拿本公司导入过的值来判,
// 否则别家导入的 ASIN 会把本应保留的行从输出文件里静默删掉。
// 在请求线程上解析:一是解析失败要当场报错,不能任务跑一半才炸;二是清洗跑在虚拟线程上,
// 那时已经没有请求上下文了。
Long companyId = companyScopeSupport.requireCompanyId(request.getUserId());
if (!tryAcquireRunningTaskSlot(request.getUserId())) {
throw new BusinessException("已有其他去重任务正在处理中,请等待完成后再试");
@@ -182,7 +191,7 @@ public class DedupeRunService {
runProgressMap.putNow(runId, snapshot(progress));
try {
Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress));
Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress, companyId));
log.info("dedupe run submitted runId={} taskNo={} userId={} files={} thread={}",
runId, task.getTaskNo(), request.getUserId(), request.getFiles().size(), thread.threadId());
} catch (Exception e) {
@@ -292,7 +301,8 @@ public class DedupeRunService {
return vo;
}
private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request, DedupeRunProgressVo progress) {
private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request,
DedupeRunProgressVo progress, Long companyId) {
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
Map<String, DedupeArchiveEntry> archiveEntries = new ConcurrentHashMap<>();
List<DedupeResultItemVo> outcomeItems = new ArrayList<>();
@@ -304,7 +314,8 @@ public class DedupeRunService {
List<Future<?>> workers = new ArrayList<>(request.getFiles().size());
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
workers.add(fileExecutor.submit(() -> {
DedupeResultItemVo item = processFile(sourceFile, request, folderMode, task, archiveEntries);
DedupeResultItemVo item = processFile(sourceFile, request, folderMode, task, archiveEntries,
companyId);
synchronized (progress) {
progress.setProcessedCount(processedCount.incrementAndGet());
if (item.isSuccess()) {
@@ -466,7 +477,7 @@ public class DedupeRunService {
private DedupeResultItemVo processFile(DedupeSourceFileDto sourceFile, DedupeRunRequest request,
boolean folderMode, FileTaskEntity task,
Map<String, DedupeArchiveEntry> archiveEntries) {
Map<String, DedupeArchiveEntry> archiveEntries, Long companyId) {
DedupeResultItemVo item = new DedupeResultItemVo();
item.setSourceFilename(sourceFile.getOriginalFilename());
try {
@@ -496,7 +507,8 @@ public class DedupeRunService {
request.getSelectedColumns(),
request.isKeepIntegerIds(),
request.isKeepUnderscoreIds(),
request.isKeepIntegerMainIdsWhenNoSubIds()
request.isKeepIntegerMainIdsWhenNoSubIds(),
companyId
);
long cleanNs = elapsedNs(cleanStartNs);
@@ -679,7 +691,7 @@ public class DedupeRunService {
*/
private void cleanExcelByStreamRules(File inputFile, File outputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
boolean keepIntegerMainIdsWhenNoSubIds, Long companyId) throws Exception {
List<String> orderedSelectedColumns = reorderExportColumns(selectedColumns);
long readStartNs = System.nanoTime();
DedupeReadResult readResult = new DedupeReadResult();
@@ -746,7 +758,8 @@ public class DedupeRunService {
}
}
long dbStartNs = System.nanoTime();
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(
candidateAsinValues, companyId);
long dbNs = elapsedNs(dbStartNs);
long writeStartNs = System.nanoTime();
@@ -471,12 +471,12 @@ public class DedupeTotalDataService {
}
@Transactional
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request, Long operatorId) {
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request, Long companyId, Long operatorId) {
AdminUserEntity uploader = getOperator(operatorId);
Long companyId = companyScopeSupport.requireCompanyId(uploader);
requireScopedCompanyId(companyId);
ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), uploader);
String dataValue = normalizeComparableValue(request.getDataValue());
ensureUnique(dataValue, null);
ensureUnique(dataValue, null, companyId);
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setDataValue(dataValue);
entity.setGroupId(group.getId());
@@ -488,14 +488,27 @@ public class DedupeTotalDataService {
return toItemVo(getById(entity.getId()), group.getGroupName());
}
public Set<String> listComparableValues() {
return dedupeTotalDataMapper.selectAllDataValues().stream()
.map(this::normalizeComparableValue)
.filter(value -> !value.isEmpty())
.collect(java.util.stream.Collectors.toSet());
/**
* 去重比对的取数口:只在 {@code companyId} 这一家名下找「已存在的值」。
*
* <p>公司身份必须由调用方给死,null 不放行:{@code company_id = NULL} 在 SQL 里是永假条件,
* 会静默返回空集,「一家都没比对上」和「比对集是空的」在结果上长得一样——那正是这次要拆的形态。
* 同理由删掉了此前的 {@code listComparableValues()}:它把全平台的值整池端出来,且零调用者。</p>
*/
/**
* 采集侧的比对入口:调用方只交任务归属账号,公司由这里推导。
*
* <p>不让调用方自己传 companyId:采集过滤器住在 collectdata 模块,同一个文件里还住着品牌库那一支
* (2026-10-06 定案全平台共享、不许带公司维度),公司字样落进去会让那条反向钉失去分辨力。</p>
*/
public Set<String> findExistingComparableValuesForOwner(Collection<String> values, Long ownerUserId) {
return findExistingComparableValues(values, companyScopeSupport.requireCompanyId(ownerUserId));
}
public Set<String> findExistingComparableValues(Collection<String> values) {
public Set<String> findExistingComparableValues(Collection<String> values, Long companyId) {
if (companyId == null) {
throw new BusinessException("去重比对缺少归属公司,比对集必须锁在一家公司内");
}
if (values == null || values.isEmpty()) {
return Set.of();
}
@@ -510,7 +523,8 @@ public class DedupeTotalDataService {
Set<String> existingValues = new HashSet<>();
for (int start = 0; start < normalizedValues.size(); start += COMPARE_BATCH_SIZE) {
int end = Math.min(start + COMPARE_BATCH_SIZE, normalizedValues.size());
List<String> batch = dedupeTotalDataMapper.selectExistingDataValues(normalizedValues.subList(start, end));
List<String> batch = dedupeTotalDataMapper.selectExistingDataValues(
normalizedValues.subList(start, end), companyId);
if (batch != null) {
batch.stream()
.map(this::normalizeComparableValueOrBlank)
@@ -521,11 +535,8 @@ public class DedupeTotalDataService {
return existingValues;
}
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long operatorId) {
return startImport(file, null, operatorId);
}
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long groupId, Long operatorId) {
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long groupId, Long companyId,
Long operatorId) {
cleanupExpiredProgress();
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
@@ -534,7 +545,7 @@ public class DedupeTotalDataService {
throw new BusinessException("导入文件超过大小上限");
}
AdminUserEntity uploader = getOperator(operatorId);
Long companyId = companyScopeSupport.requireCompanyId(uploader);
requireScopedCompanyId(companyId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
if (!importSlots.tryAcquire()) {
throw new BusinessException("导入任务过多,请稍后重试");
@@ -582,11 +593,8 @@ public class DedupeTotalDataService {
return progress;
}
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long operatorId) {
return startDeleteImport(file, null, operatorId);
}
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long groupId, Long operatorId) {
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long groupId, Long companyId,
Long operatorId) {
cleanupExpiredProgress();
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
@@ -595,6 +603,7 @@ public class DedupeTotalDataService {
throw new BusinessException("导入文件超过大小上限");
}
AdminUserEntity operator = getOperator(operatorId);
requireScopedCompanyId(companyId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
if (!importSlots.tryAcquire()) {
throw new BusinessException("导入任务过多,请稍后重试");
@@ -615,7 +624,7 @@ public class DedupeTotalDataService {
File tempFile = saveMultipartToTempFile(file);
String filename = file.getOriginalFilename();
Thread.ofVirtual().start(() -> runDeleteImportTask(
importId, tempFile, filename, operator.getId(), group.getId()));
importId, tempFile, filename, operator.getId(), group.getId(), companyId));
} catch (Exception e) {
importSlots.release();
deleteImportProgressMap.remove(importId);
@@ -642,7 +651,7 @@ public class DedupeTotalDataService {
}
private void runDeleteImportTask(String importId, File tempFile, String filename,
Long operatorId, Long groupId) {
Long operatorId, Long groupId, Long companyId) {
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
if (progress == null) {
importSlots.release();
@@ -652,7 +661,7 @@ public class DedupeTotalDataService {
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
AccessScope scope = resolveAccessScope(operatorId);
DedupeTotalDataImportVo result = deleteFromExcelInternal(
inputStream, filename, progress, scope, groupId);
inputStream, filename, progress, scope, groupId, companyId);
progress.setTotalRows(result.getTotalRows());
progress.setAsinCount(result.getAsinCount());
progress.setInsertedCount(result.getInsertedCount());
@@ -722,16 +731,13 @@ public class DedupeTotalDataService {
}
}
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long operatorId) {
return importFromExcel(file, null, operatorId);
}
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long groupId, Long operatorId) {
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long groupId, Long companyId,
Long operatorId) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
AdminUserEntity uploader = getOperator(operatorId);
Long companyId = companyScopeSupport.requireCompanyId(uploader);
requireScopedCompanyId(companyId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
try (InputStream inputStream = file.getInputStream()) {
return importFromExcelInternal(
@@ -744,29 +750,6 @@ public class DedupeTotalDataService {
}
}
public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file, Long operatorId) {
return deleteFromExcel(file, null, operatorId);
}
public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file, Long groupId, Long operatorId) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
AdminUserEntity operator = getOperator(operatorId);
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
AccessScope scope = resolveAccessScope(operatorId);
try (InputStream inputStream = file.getInputStream()) {
return deleteFromExcelInternal(
inputStream, file.getOriginalFilename(), null, scope, group.getId());
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
// 此前丢弃原始异常:排查时只能看到一句「删除 Excel 匹配数据失败」
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败", e);
throw new BusinessException("删除 Excel 匹配数据失败", e);
}
}
private DedupeTotalDataImportVo importFromExcelInternal(InputStream inputStream, String filename,
DedupeTotalDataImportProgressVo progress,
Long uploaderUserId, String uploaderUsername,
@@ -965,7 +948,7 @@ public class DedupeTotalDataService {
if (values == null || values.isEmpty()) {
return new ImportBatchResult(0, 0);
}
Set<String> existingValues = findExistingComparableValues(values);
Set<String> existingValues = findExistingComparableValues(values, companyId);
List<DedupeTotalDataEntity> rows = new ArrayList<>(values.size());
for (String value : values) {
if (value == null || value.isBlank() || existingValues.contains(value)) {
@@ -998,7 +981,7 @@ public class DedupeTotalDataService {
for (DedupeTotalDataEntity row : rows) {
try {
Boolean oneInserted = executeInNewTransaction(() -> {
if (existsDataValue(row.getDataValue())) {
if (existsDataValue(row.getDataValue(), companyId)) {
return false;
}
return dedupeTotalDataMapper.insert(row) > 0;
@@ -1040,14 +1023,14 @@ public class DedupeTotalDataService {
private DedupeTotalDataImportVo deleteFromExcelInternal(InputStream inputStream, String filename,
DedupeTotalDataImportProgressVo progress,
AccessScope scope, Long groupId) {
AccessScope scope, Long groupId, Long companyId) {
String lowerFilename = filename == null ? "" : filename.toLowerCase();
if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) {
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
}
// 2026-09 全维度审查 C5:与导入同样改流式解析,避免 50 万行时 DOM 占 1~2GB 堆
DeleteSession session = new DeleteSession(progress, scope, groupId);
DeleteSession session = new DeleteSession(progress, scope, groupId, companyId);
try {
ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() {
private boolean headerReady;
@@ -1094,6 +1077,7 @@ public class DedupeTotalDataService {
private final DedupeTotalDataImportProgressVo progress;
private final AccessScope scope;
private final Long groupId;
private final Long companyId;
private final Set<String> seenInFile = new HashSet<>();
private final List<String> pendingDeletes = new ArrayList<>();
private Map<String, Integer> headerIndex = Map.of();
@@ -1102,10 +1086,12 @@ public class DedupeTotalDataService {
private int deletedCount;
private int skippedCount;
private DeleteSession(DedupeTotalDataImportProgressVo progress, AccessScope scope, Long groupId) {
private DeleteSession(DedupeTotalDataImportProgressVo progress, AccessScope scope, Long groupId,
Long companyId) {
this.progress = progress;
this.scope = scope;
this.groupId = groupId;
this.companyId = companyId;
}
private void acceptRow(Map<Integer, String> rowMap) {
@@ -1124,7 +1110,7 @@ public class DedupeTotalDataService {
// 批量删除:攒批 + IN 一次删,避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1
pendingDeletes.add(asin);
if (pendingDeletes.size() >= DELETE_BATCH_SIZE) {
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId, companyId);
pendingDeletes.clear();
}
updateProgress();
@@ -1146,7 +1132,7 @@ public class DedupeTotalDataService {
private DedupeTotalDataImportVo finish() {
if (!pendingDeletes.isEmpty()) {
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId, companyId);
pendingDeletes.clear();
}
updateProgress();
@@ -1167,7 +1153,9 @@ public class DedupeTotalDataService {
DedupeTotalDataEntity entity = getAccessibleById(id, operatorId);
AdminUserEntity operator = getOperator(operatorId);
String dataValue = normalizeComparableValue(request.getDataValue());
ensureUnique(dataValue, id);
// 改值时的重名比对认这一行自己的归属,不认操作者的 tab:超管在别家页签里改别家的行,
// 要比的是那一家的库。
ensureUnique(dataValue, id, entity.getCompanyId());
entity.setDataValue(dataValue);
ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), operator);
entity.setGroupId(group.getId());
@@ -1207,32 +1195,58 @@ public class DedupeTotalDataService {
return normalizeExcelText(value).toUpperCase(Locale.ROOT);
}
private void ensureUnique(String dataValue, Long excludeId) {
/**
* 「已存在」只在一家公司内成立:本表唯一键自 V145 起是 uk_company_data_value(company_id,
* data_value),同值本来就允许归两家。此前这一刀不带公司,结果是 B 公司导入过的 ASIN
* 会让 A 公司的「新增」报"数据已存在"——用户看到的是"我加不进去",而库里根本没有他的行。
*/
private void ensureUnique(String dataValue, Long excludeId, Long companyId) {
requireScopedCompanyId(companyId);
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getCompanyId, companyId)
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
.last("LIMIT 1");
DedupeTotalDataEntity exists = dedupeTotalDataMapper.selectOne(query);
if (exists != null && (excludeId == null || !exists.getId().equals(excludeId))) {
throw new BusinessException("数据已存在");
throw new BusinessException("本公司已存在该数据");
}
}
private boolean existsDataValue(String dataValue) {
private boolean existsDataValue(String dataValue, Long companyId) {
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getCompanyId, companyId)
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
.last("LIMIT 1");
return dedupeTotalDataMapper.selectOne(query) != null;
}
/**
* 公司身份闸门:本表每一条按公司取的判据都从这一句过。
*
* <p>值由 controller 在请求线程上算好传进来({@code DataReadScope.readCompanyIdOrOwn}:超管认
* tab、非超管恒本公司)。这里只判空,不补默认——导入与去重跑在虚拟线程上,那时已经没有请求
* 上下文可查了,兜底只能兜出一个"谁都比对不上"的空集。</p>
*/
static void requireScopedCompanyId(Long companyId) {
if (companyId == null) {
throw new BusinessException(
"缺少归属公司:请先在页面上方选择公司,再操作去重数据库");
}
}
/** 批量删除大小:单事务 IN 删除的阈值,兼顾锁窗口与事务日志。 */
private static final int DELETE_BATCH_SIZE = 500;
/** 整批一个独立事务按 IN 一次删除;返回实际删除行数(计入 deletedCount,不再额外计 skipped)。 */
private int deleteBatchByDataValues(List<String> dataValues, AccessScope scope, Long groupId) {
/** 包内可见只为让用例抓到真正的删除谓词(同 {@link #requireCompanyAttribution} 的先例)。 */
int deleteBatchByDataValues(List<String> dataValues, AccessScope scope, Long groupId,
Long companyId) {
requireScopedCompanyId(companyId);
List<String> batch = List.copyOf(dataValues);
Integer deleted = newRequiresNewTemplate().execute(status -> {
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getCompanyId, companyId)
.in(DedupeTotalDataEntity::getDataValue, batch);
applyGroupScope(query, scope, groupId);
return dedupeTotalDataMapper.delete(query);
@@ -1512,6 +1526,6 @@ public class DedupeTotalDataService {
* @param wholeCompany 公司超管:不做分组维度裁剪,userIds = 本公司全体成员,
* 查询条件退化为 uploader_user_id IN (成员)——公司超管的"组"就是整家公司
*/
private record AccessScope(boolean allUsers, Set<Long> groupIds, Set<Long> userIds, boolean wholeCompany) {
record AccessScope(boolean allUsers, Set<Long> groupIds, Set<Long> userIds, boolean wholeCompany) {
}
}