收口(去重库): 五处比对与写入口按 company_id 认公司,不再跨公司去重
读侧(列表/导出)早已按公司收窄,但"去重"这个动作本身一直查全平台: selectExistingDataValues 是 `WHERE data_value IN (...)`,一个租户条件都没有。 库里唯一键自 V145 起就是 uk_company_data_value(company_id, data_value)—— 同值本来就允许归两家,是代码没跟上库的口径。 五处一起收口(第 5 处由编译器抓出,正是 V145 头注释 §1.2 点名的采集侧): 去重执行 ASIN 过滤 / 导入预检"已存在就跳过" / 手工新增与编辑的重名校验 (文案改「本公司已存在该数据」)/ 删除导入(超管不选分组时原先一次删掉全平台 同名行)/ 采集结果过滤(CollectDataBatchQuery 直连 mapper,改为经 dedupe 侧推导)。 超管代导入的归属分裂一并修掉:前端两个导入接口把公司 tab 发下去——原先 group_id 落别家的组、company_id 落超管自己那一家。公司身份一律在请求线程上由 DataReadScope.readCompanyIdOrOwn 解析后传入(导入与去重跑在虚拟线程,那时已无 请求上下文),service 侧 requireScopedCompanyId 只判空不兜底。 删两条零调用者的跨公司通道(selectAllDataValues/listComparableValues 全表读、 deleteFromExcel 两个同步重载)。反转一条反向钉:comparableValueLookupRemainsGlobal 原显式断言"比对不参与公司收窄"(多公司一期的半套遗留),现为 IsScopedToOneCompany。 三把尺子随之改动,均带出处:tsv 基准行搬家(三个标量双双不动)、前端发送口分母 17→18 组/21→22 处、CatchSilenceRatchet 重抛下限 387→386(随死代码离开树那处)。 验证:后端全量 4198 例 0 失败/0 错误/7 跳过;前端 1778 例全绿;变异自证 6/6 全部 KILLED,注入文件按 sha256 还原并复跑绿。 现网只读探针:misfiled=0、同值跨家=0 ⇒ 存量比对结果不变,行为改变只在将来 A 家导入 B 家已有 ASIN 那一刻;两口径差额 1,452 行(0.26%)全属公司 1、 全源于 10-05 账号硬删。
This commit is contained in:
+19
-4
@@ -155,7 +155,11 @@ public class DedupeTotalDataController {
|
||||
HttpServletRequest httpRequest,
|
||||
@Valid @RequestBody DedupeTotalDataCreateRequest request) {
|
||||
RequestOperator operator = requireDedupeTotalDataAccess(httpRequest);
|
||||
return ApiResponse.success("创建成功", dedupeTotalDataService.create(request, operator.id()));
|
||||
AdminUserEntity user = operator.user();
|
||||
Long requestedCompanyId = request.getCompanyId();
|
||||
Long companyId = dataReadScope.readCompanyIdOrOwn(user, requestedCompanyId);
|
||||
Long operatorId = operator.id();
|
||||
return ApiResponse.success("创建成功", dedupeTotalDataService.create(request, companyId, operatorId));
|
||||
}
|
||||
|
||||
@PostMapping("/import")
|
||||
@@ -167,9 +171,14 @@ public class DedupeTotalDataController {
|
||||
public ApiResponse<DedupeTotalDataImportStartVo> importExcel(
|
||||
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file,
|
||||
@Parameter(description = "分组 ID", required = true) @RequestParam Long groupId,
|
||||
@Parameter(description = "公司 ID") @RequestParam(name = "company_id", required = false) Long companyId,
|
||||
HttpServletRequest request) {
|
||||
RequestOperator operator = requireDedupeTotalDataAccess(request);
|
||||
return ApiResponse.success("开始导入", dedupeTotalDataService.startImport(file, groupId, operator.id()));
|
||||
AdminUserEntity user = operator.user();
|
||||
Long scopedCompanyId = dataReadScope.readCompanyIdOrOwn(user, companyId);
|
||||
Long operatorId = operator.id();
|
||||
return ApiResponse.success("开始导入",
|
||||
dedupeTotalDataService.startImport(file, groupId, scopedCompanyId, operatorId));
|
||||
}
|
||||
|
||||
@GetMapping("/import/{importId}")
|
||||
@@ -190,9 +199,14 @@ public class DedupeTotalDataController {
|
||||
public ApiResponse<DedupeTotalDataImportStartVo> deleteImportExcel(
|
||||
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file,
|
||||
@Parameter(description = "分组 ID", required = true) @RequestParam Long groupId,
|
||||
@Parameter(description = "公司 ID") @RequestParam(name = "company_id", required = false) Long companyId,
|
||||
HttpServletRequest request) {
|
||||
RequestOperator operator = requireDedupeTotalDataAccess(request);
|
||||
return ApiResponse.success("开始删除", dedupeTotalDataService.startDeleteImport(file, groupId, operator.id()));
|
||||
AdminUserEntity user = operator.user();
|
||||
Long scopedCompanyId = dataReadScope.readCompanyIdOrOwn(user, companyId);
|
||||
Long operatorId = operator.id();
|
||||
return ApiResponse.success("开始删除",
|
||||
dedupeTotalDataService.startDeleteImport(file, groupId, scopedCompanyId, operatorId));
|
||||
}
|
||||
|
||||
@GetMapping("/delete-import/{importId}")
|
||||
@@ -285,7 +299,8 @@ public class DedupeTotalDataController {
|
||||
|
||||
|
||||
|
||||
// 公司 tab 的读范围就地算在 handler 里,不留私有收口方法:G2 的断言 A 只在这支 handler 的
|
||||
// 公司 tab 的读范围与写侧归属(create / import / delete-import)都就地算在 handler 里,
|
||||
// 不留私有收口方法:G2 的断言 A 只在这支 handler 的
|
||||
// 块内找 readScope(,私有方法在它自己的闭括号处收口、不算在内(CompanyIdParamGoesThroughReadScopeTest
|
||||
// 类注释第 5、7 条),挪进去 = 这一页的 tab 没人看守。收窄只给超管:非超管已由 resolveAccessScope
|
||||
// 按会话裁剪,且他的可见集里含「靠分组可见」的行,叠一层账号 IN 会把那批行静默删掉
|
||||
|
||||
+13
-16
@@ -4,18 +4,25 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.SelectProvider;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.StringJoiner;
|
||||
|
||||
@Mapper
|
||||
public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> {
|
||||
|
||||
List<String> selectAllDataValues();
|
||||
|
||||
@SelectProvider(type = SqlProvider.class, method = "selectExistingDataValues")
|
||||
List<String> selectExistingDataValues(@Param("values") List<String> values);
|
||||
/**
|
||||
* 比对集按公司取:一次去重/一批导入只能看见 {@code companyId} 名下的值。
|
||||
*
|
||||
* <p>这条曾是 @SelectProvider 手拼 IN 列表、且不带任何租户条件,后果有两样:去重执行会把别家
|
||||
* 导入的 ASIN 当「已存在」从输出文件里删掉;导入会把别家已有的值静默跳过,本公司库里永远补不上
|
||||
* 这一行。本表唯一键自 V145 起就是 uk_company_data_value(company_id, data_value)——同值本来就
|
||||
* 允许归两家,是代码没跟上库的口径。</p>
|
||||
*
|
||||
* <p>{@code companyId} 必须由调用方保证非空:MyBatis 会把它渲染成 {@code company_id = NULL}
|
||||
* 这条永假条件,静默给出空集,比当场炸更难发现。拦空值的是服务层闸门。</p>
|
||||
*/
|
||||
List<String> selectExistingDataValues(@Param("values") List<String> values,
|
||||
@Param("companyId") Long companyId);
|
||||
|
||||
/**
|
||||
* Import is deliberately a single SQL round trip per batch. The unique
|
||||
@@ -23,14 +30,4 @@ public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity>
|
||||
* the same value between the pre-check and this statement.
|
||||
*/
|
||||
int insertBatchIgnore(@Param("rows") List<DedupeTotalDataEntity> rows);
|
||||
|
||||
class SqlProvider {
|
||||
public String selectExistingDataValues(@Param("values") List<String> values) {
|
||||
StringJoiner placeholders = new StringJoiner(", ");
|
||||
for (int i = 0; i < values.size(); i++) {
|
||||
placeholders.add("#{values[" + i + "]}");
|
||||
}
|
||||
return "SELECT data_value FROM biz_dedupe_total_data WHERE data_value IN (" + placeholders + ")";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+4
@@ -18,4 +18,8 @@ public class DedupeTotalDataCreateRequest {
|
||||
@NotNull(message = "请选择分组")
|
||||
@Schema(description = "分组 ID", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private Long groupId;
|
||||
|
||||
/** 页面上方公司 tab 选中的归属公司;非超管带什么都不算,超管不带则拒绝写入。 */
|
||||
@Schema(description = "公司 ID")
|
||||
private Long companyId;
|
||||
}
|
||||
|
||||
+20
-7
@@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.service.CompanyScopeSupport;
|
||||
import com.nanri.aiimage.common.service.NodeSharedStore;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
@@ -79,6 +80,7 @@ public class DedupeRunService {
|
||||
private final StorageProperties storageProperties;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final DedupeTotalDataService dedupeTotalDataService;
|
||||
private final CompanyScopeSupport companyScopeSupport;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
|
||||
/**
|
||||
@@ -102,6 +104,7 @@ public class DedupeRunService {
|
||||
StorageProperties storageProperties,
|
||||
OssStorageService ossStorageService,
|
||||
DedupeTotalDataService dedupeTotalDataService,
|
||||
CompanyScopeSupport companyScopeSupport,
|
||||
StringRedisTemplate stringRedisTemplate,
|
||||
ObjectMapper objectMapper,
|
||||
InstanceMetadata instanceMetadata) {
|
||||
@@ -110,6 +113,7 @@ public class DedupeRunService {
|
||||
this.storageProperties = storageProperties;
|
||||
this.ossStorageService = ossStorageService;
|
||||
this.dedupeTotalDataService = dedupeTotalDataService;
|
||||
this.companyScopeSupport = companyScopeSupport;
|
||||
this.instanceMetadata = instanceMetadata;
|
||||
this.runProgressMap = new NodeSharedStore<>("aiimage:dedupe:run:progress", RUN_STATE_TTL,
|
||||
DedupeRunProgressVo.class, stringRedisTemplate, objectMapper);
|
||||
@@ -140,6 +144,11 @@ public class DedupeRunService {
|
||||
throw new BusinessException("请至少选择一种 ID 保留规则");
|
||||
}
|
||||
cleanupExpiredProgress();
|
||||
// 比对集锁在提交人自己那一家:「这个 ASIN 去重库里已经有了」只能拿本公司导入过的值来判,
|
||||
// 否则别家导入的 ASIN 会把本应保留的行从输出文件里静默删掉。
|
||||
// 在请求线程上解析:一是解析失败要当场报错,不能任务跑一半才炸;二是清洗跑在虚拟线程上,
|
||||
// 那时已经没有请求上下文了。
|
||||
Long companyId = companyScopeSupport.requireCompanyId(request.getUserId());
|
||||
|
||||
if (!tryAcquireRunningTaskSlot(request.getUserId())) {
|
||||
throw new BusinessException("已有其他去重任务正在处理中,请等待完成后再试");
|
||||
@@ -182,7 +191,7 @@ public class DedupeRunService {
|
||||
runProgressMap.putNow(runId, snapshot(progress));
|
||||
|
||||
try {
|
||||
Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress));
|
||||
Thread thread = Thread.ofVirtual().start(() -> runAsync(runId, task, request, progress, companyId));
|
||||
log.info("dedupe run submitted runId={} taskNo={} userId={} files={} thread={}",
|
||||
runId, task.getTaskNo(), request.getUserId(), request.getFiles().size(), thread.threadId());
|
||||
} catch (Exception e) {
|
||||
@@ -292,7 +301,8 @@ public class DedupeRunService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request, DedupeRunProgressVo progress) {
|
||||
private void runAsync(String runId, FileTaskEntity task, DedupeRunRequest request,
|
||||
DedupeRunProgressVo progress, Long companyId) {
|
||||
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
|
||||
Map<String, DedupeArchiveEntry> archiveEntries = new ConcurrentHashMap<>();
|
||||
List<DedupeResultItemVo> outcomeItems = new ArrayList<>();
|
||||
@@ -304,7 +314,8 @@ public class DedupeRunService {
|
||||
List<Future<?>> workers = new ArrayList<>(request.getFiles().size());
|
||||
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
|
||||
workers.add(fileExecutor.submit(() -> {
|
||||
DedupeResultItemVo item = processFile(sourceFile, request, folderMode, task, archiveEntries);
|
||||
DedupeResultItemVo item = processFile(sourceFile, request, folderMode, task, archiveEntries,
|
||||
companyId);
|
||||
synchronized (progress) {
|
||||
progress.setProcessedCount(processedCount.incrementAndGet());
|
||||
if (item.isSuccess()) {
|
||||
@@ -466,7 +477,7 @@ public class DedupeRunService {
|
||||
|
||||
private DedupeResultItemVo processFile(DedupeSourceFileDto sourceFile, DedupeRunRequest request,
|
||||
boolean folderMode, FileTaskEntity task,
|
||||
Map<String, DedupeArchiveEntry> archiveEntries) {
|
||||
Map<String, DedupeArchiveEntry> archiveEntries, Long companyId) {
|
||||
DedupeResultItemVo item = new DedupeResultItemVo();
|
||||
item.setSourceFilename(sourceFile.getOriginalFilename());
|
||||
try {
|
||||
@@ -496,7 +507,8 @@ public class DedupeRunService {
|
||||
request.getSelectedColumns(),
|
||||
request.isKeepIntegerIds(),
|
||||
request.isKeepUnderscoreIds(),
|
||||
request.isKeepIntegerMainIdsWhenNoSubIds()
|
||||
request.isKeepIntegerMainIdsWhenNoSubIds(),
|
||||
companyId
|
||||
);
|
||||
long cleanNs = elapsedNs(cleanStartNs);
|
||||
|
||||
@@ -679,7 +691,7 @@ public class DedupeRunService {
|
||||
*/
|
||||
private void cleanExcelByStreamRules(File inputFile, File outputFile, List<String> selectedColumns,
|
||||
boolean keepIntegerIds, boolean keepUnderscoreIds,
|
||||
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
|
||||
boolean keepIntegerMainIdsWhenNoSubIds, Long companyId) throws Exception {
|
||||
List<String> orderedSelectedColumns = reorderExportColumns(selectedColumns);
|
||||
long readStartNs = System.nanoTime();
|
||||
DedupeReadResult readResult = new DedupeReadResult();
|
||||
@@ -746,7 +758,8 @@ public class DedupeRunService {
|
||||
}
|
||||
}
|
||||
long dbStartNs = System.nanoTime();
|
||||
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
|
||||
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(
|
||||
candidateAsinValues, companyId);
|
||||
long dbNs = elapsedNs(dbStartNs);
|
||||
|
||||
long writeStartNs = System.nanoTime();
|
||||
|
||||
+80
-66
@@ -471,12 +471,12 @@ public class DedupeTotalDataService {
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request, Long operatorId) {
|
||||
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request, Long companyId, Long operatorId) {
|
||||
AdminUserEntity uploader = getOperator(operatorId);
|
||||
Long companyId = companyScopeSupport.requireCompanyId(uploader);
|
||||
requireScopedCompanyId(companyId);
|
||||
ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), uploader);
|
||||
String dataValue = normalizeComparableValue(request.getDataValue());
|
||||
ensureUnique(dataValue, null);
|
||||
ensureUnique(dataValue, null, companyId);
|
||||
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
|
||||
entity.setDataValue(dataValue);
|
||||
entity.setGroupId(group.getId());
|
||||
@@ -488,14 +488,27 @@ public class DedupeTotalDataService {
|
||||
return toItemVo(getById(entity.getId()), group.getGroupName());
|
||||
}
|
||||
|
||||
public Set<String> listComparableValues() {
|
||||
return dedupeTotalDataMapper.selectAllDataValues().stream()
|
||||
.map(this::normalizeComparableValue)
|
||||
.filter(value -> !value.isEmpty())
|
||||
.collect(java.util.stream.Collectors.toSet());
|
||||
/**
|
||||
* 去重比对的取数口:只在 {@code companyId} 这一家名下找「已存在的值」。
|
||||
*
|
||||
* <p>公司身份必须由调用方给死,null 不放行:{@code company_id = NULL} 在 SQL 里是永假条件,
|
||||
* 会静默返回空集,「一家都没比对上」和「比对集是空的」在结果上长得一样——那正是这次要拆的形态。
|
||||
* 同理由删掉了此前的 {@code listComparableValues()}:它把全平台的值整池端出来,且零调用者。</p>
|
||||
*/
|
||||
/**
|
||||
* 采集侧的比对入口:调用方只交任务归属账号,公司由这里推导。
|
||||
*
|
||||
* <p>不让调用方自己传 companyId:采集过滤器住在 collectdata 模块,同一个文件里还住着品牌库那一支
|
||||
* (2026-10-06 定案全平台共享、不许带公司维度),公司字样落进去会让那条反向钉失去分辨力。</p>
|
||||
*/
|
||||
public Set<String> findExistingComparableValuesForOwner(Collection<String> values, Long ownerUserId) {
|
||||
return findExistingComparableValues(values, companyScopeSupport.requireCompanyId(ownerUserId));
|
||||
}
|
||||
|
||||
public Set<String> findExistingComparableValues(Collection<String> values) {
|
||||
public Set<String> findExistingComparableValues(Collection<String> values, Long companyId) {
|
||||
if (companyId == null) {
|
||||
throw new BusinessException("去重比对缺少归属公司,比对集必须锁在一家公司内");
|
||||
}
|
||||
if (values == null || values.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
@@ -510,7 +523,8 @@ public class DedupeTotalDataService {
|
||||
Set<String> existingValues = new HashSet<>();
|
||||
for (int start = 0; start < normalizedValues.size(); start += COMPARE_BATCH_SIZE) {
|
||||
int end = Math.min(start + COMPARE_BATCH_SIZE, normalizedValues.size());
|
||||
List<String> batch = dedupeTotalDataMapper.selectExistingDataValues(normalizedValues.subList(start, end));
|
||||
List<String> batch = dedupeTotalDataMapper.selectExistingDataValues(
|
||||
normalizedValues.subList(start, end), companyId);
|
||||
if (batch != null) {
|
||||
batch.stream()
|
||||
.map(this::normalizeComparableValueOrBlank)
|
||||
@@ -521,11 +535,8 @@ public class DedupeTotalDataService {
|
||||
return existingValues;
|
||||
}
|
||||
|
||||
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long operatorId) {
|
||||
return startImport(file, null, operatorId);
|
||||
}
|
||||
|
||||
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long groupId, Long operatorId) {
|
||||
public DedupeTotalDataImportStartVo startImport(MultipartFile file, Long groupId, Long companyId,
|
||||
Long operatorId) {
|
||||
cleanupExpiredProgress();
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new BusinessException("请上传 xlsx 文件");
|
||||
@@ -534,7 +545,7 @@ public class DedupeTotalDataService {
|
||||
throw new BusinessException("导入文件超过大小上限");
|
||||
}
|
||||
AdminUserEntity uploader = getOperator(operatorId);
|
||||
Long companyId = companyScopeSupport.requireCompanyId(uploader);
|
||||
requireScopedCompanyId(companyId);
|
||||
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
|
||||
if (!importSlots.tryAcquire()) {
|
||||
throw new BusinessException("导入任务过多,请稍后重试");
|
||||
@@ -582,11 +593,8 @@ public class DedupeTotalDataService {
|
||||
return progress;
|
||||
}
|
||||
|
||||
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long operatorId) {
|
||||
return startDeleteImport(file, null, operatorId);
|
||||
}
|
||||
|
||||
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long groupId, Long operatorId) {
|
||||
public DedupeTotalDataImportStartVo startDeleteImport(MultipartFile file, Long groupId, Long companyId,
|
||||
Long operatorId) {
|
||||
cleanupExpiredProgress();
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new BusinessException("请上传 xlsx 文件");
|
||||
@@ -595,6 +603,7 @@ public class DedupeTotalDataService {
|
||||
throw new BusinessException("导入文件超过大小上限");
|
||||
}
|
||||
AdminUserEntity operator = getOperator(operatorId);
|
||||
requireScopedCompanyId(companyId);
|
||||
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
|
||||
if (!importSlots.tryAcquire()) {
|
||||
throw new BusinessException("导入任务过多,请稍后重试");
|
||||
@@ -615,7 +624,7 @@ public class DedupeTotalDataService {
|
||||
File tempFile = saveMultipartToTempFile(file);
|
||||
String filename = file.getOriginalFilename();
|
||||
Thread.ofVirtual().start(() -> runDeleteImportTask(
|
||||
importId, tempFile, filename, operator.getId(), group.getId()));
|
||||
importId, tempFile, filename, operator.getId(), group.getId(), companyId));
|
||||
} catch (Exception e) {
|
||||
importSlots.release();
|
||||
deleteImportProgressMap.remove(importId);
|
||||
@@ -642,7 +651,7 @@ public class DedupeTotalDataService {
|
||||
}
|
||||
|
||||
private void runDeleteImportTask(String importId, File tempFile, String filename,
|
||||
Long operatorId, Long groupId) {
|
||||
Long operatorId, Long groupId, Long companyId) {
|
||||
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
|
||||
if (progress == null) {
|
||||
importSlots.release();
|
||||
@@ -652,7 +661,7 @@ public class DedupeTotalDataService {
|
||||
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
|
||||
AccessScope scope = resolveAccessScope(operatorId);
|
||||
DedupeTotalDataImportVo result = deleteFromExcelInternal(
|
||||
inputStream, filename, progress, scope, groupId);
|
||||
inputStream, filename, progress, scope, groupId, companyId);
|
||||
progress.setTotalRows(result.getTotalRows());
|
||||
progress.setAsinCount(result.getAsinCount());
|
||||
progress.setInsertedCount(result.getInsertedCount());
|
||||
@@ -722,16 +731,13 @@ public class DedupeTotalDataService {
|
||||
}
|
||||
}
|
||||
|
||||
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long operatorId) {
|
||||
return importFromExcel(file, null, operatorId);
|
||||
}
|
||||
|
||||
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long groupId, Long operatorId) {
|
||||
public DedupeTotalDataImportVo importFromExcel(MultipartFile file, Long groupId, Long companyId,
|
||||
Long operatorId) {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new BusinessException("请上传 xlsx 文件");
|
||||
}
|
||||
AdminUserEntity uploader = getOperator(operatorId);
|
||||
Long companyId = companyScopeSupport.requireCompanyId(uploader);
|
||||
requireScopedCompanyId(companyId);
|
||||
ShopManageGroupEntity group = resolveWritableGroup(groupId, uploader);
|
||||
try (InputStream inputStream = file.getInputStream()) {
|
||||
return importFromExcelInternal(
|
||||
@@ -744,29 +750,6 @@ public class DedupeTotalDataService {
|
||||
}
|
||||
}
|
||||
|
||||
public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file, Long operatorId) {
|
||||
return deleteFromExcel(file, null, operatorId);
|
||||
}
|
||||
|
||||
public DedupeTotalDataImportVo deleteFromExcel(MultipartFile file, Long groupId, Long operatorId) {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new BusinessException("请上传 xlsx 文件");
|
||||
}
|
||||
AdminUserEntity operator = getOperator(operatorId);
|
||||
ShopManageGroupEntity group = resolveWritableGroup(groupId, operator);
|
||||
AccessScope scope = resolveAccessScope(operatorId);
|
||||
try (InputStream inputStream = file.getInputStream()) {
|
||||
return deleteFromExcelInternal(
|
||||
inputStream, file.getOriginalFilename(), null, scope, group.getId());
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
// 此前丢弃原始异常:排查时只能看到一句「删除 Excel 匹配数据失败」
|
||||
log.error("[dedupe-total-data] 删除 Excel 匹配数据失败", e);
|
||||
throw new BusinessException("删除 Excel 匹配数据失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
private DedupeTotalDataImportVo importFromExcelInternal(InputStream inputStream, String filename,
|
||||
DedupeTotalDataImportProgressVo progress,
|
||||
Long uploaderUserId, String uploaderUsername,
|
||||
@@ -965,7 +948,7 @@ public class DedupeTotalDataService {
|
||||
if (values == null || values.isEmpty()) {
|
||||
return new ImportBatchResult(0, 0);
|
||||
}
|
||||
Set<String> existingValues = findExistingComparableValues(values);
|
||||
Set<String> existingValues = findExistingComparableValues(values, companyId);
|
||||
List<DedupeTotalDataEntity> rows = new ArrayList<>(values.size());
|
||||
for (String value : values) {
|
||||
if (value == null || value.isBlank() || existingValues.contains(value)) {
|
||||
@@ -998,7 +981,7 @@ public class DedupeTotalDataService {
|
||||
for (DedupeTotalDataEntity row : rows) {
|
||||
try {
|
||||
Boolean oneInserted = executeInNewTransaction(() -> {
|
||||
if (existsDataValue(row.getDataValue())) {
|
||||
if (existsDataValue(row.getDataValue(), companyId)) {
|
||||
return false;
|
||||
}
|
||||
return dedupeTotalDataMapper.insert(row) > 0;
|
||||
@@ -1040,14 +1023,14 @@ public class DedupeTotalDataService {
|
||||
|
||||
private DedupeTotalDataImportVo deleteFromExcelInternal(InputStream inputStream, String filename,
|
||||
DedupeTotalDataImportProgressVo progress,
|
||||
AccessScope scope, Long groupId) {
|
||||
AccessScope scope, Long groupId, Long companyId) {
|
||||
String lowerFilename = filename == null ? "" : filename.toLowerCase();
|
||||
if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) {
|
||||
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
|
||||
}
|
||||
|
||||
// 2026-09 全维度审查 C5:与导入同样改流式解析,避免 50 万行时 DOM 占 1~2GB 堆
|
||||
DeleteSession session = new DeleteSession(progress, scope, groupId);
|
||||
DeleteSession session = new DeleteSession(progress, scope, groupId, companyId);
|
||||
try {
|
||||
ExcelStreamReader.readFirstSheet(inputStream, new ExcelStreamReader.SheetRowHandler() {
|
||||
private boolean headerReady;
|
||||
@@ -1094,6 +1077,7 @@ public class DedupeTotalDataService {
|
||||
private final DedupeTotalDataImportProgressVo progress;
|
||||
private final AccessScope scope;
|
||||
private final Long groupId;
|
||||
private final Long companyId;
|
||||
private final Set<String> seenInFile = new HashSet<>();
|
||||
private final List<String> pendingDeletes = new ArrayList<>();
|
||||
private Map<String, Integer> headerIndex = Map.of();
|
||||
@@ -1102,10 +1086,12 @@ public class DedupeTotalDataService {
|
||||
private int deletedCount;
|
||||
private int skippedCount;
|
||||
|
||||
private DeleteSession(DedupeTotalDataImportProgressVo progress, AccessScope scope, Long groupId) {
|
||||
private DeleteSession(DedupeTotalDataImportProgressVo progress, AccessScope scope, Long groupId,
|
||||
Long companyId) {
|
||||
this.progress = progress;
|
||||
this.scope = scope;
|
||||
this.groupId = groupId;
|
||||
this.companyId = companyId;
|
||||
}
|
||||
|
||||
private void acceptRow(Map<Integer, String> rowMap) {
|
||||
@@ -1124,7 +1110,7 @@ public class DedupeTotalDataService {
|
||||
// 批量删除:攒批 + IN 一次删,避免 50 万行 = 50 万个 REQUIRES_NEW 事务的 N+1
|
||||
pendingDeletes.add(asin);
|
||||
if (pendingDeletes.size() >= DELETE_BATCH_SIZE) {
|
||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
|
||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId, companyId);
|
||||
pendingDeletes.clear();
|
||||
}
|
||||
updateProgress();
|
||||
@@ -1146,7 +1132,7 @@ public class DedupeTotalDataService {
|
||||
|
||||
private DedupeTotalDataImportVo finish() {
|
||||
if (!pendingDeletes.isEmpty()) {
|
||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId);
|
||||
deletedCount += deleteBatchByDataValues(pendingDeletes, scope, groupId, companyId);
|
||||
pendingDeletes.clear();
|
||||
}
|
||||
updateProgress();
|
||||
@@ -1167,7 +1153,9 @@ public class DedupeTotalDataService {
|
||||
DedupeTotalDataEntity entity = getAccessibleById(id, operatorId);
|
||||
AdminUserEntity operator = getOperator(operatorId);
|
||||
String dataValue = normalizeComparableValue(request.getDataValue());
|
||||
ensureUnique(dataValue, id);
|
||||
// 改值时的重名比对认这一行自己的归属,不认操作者的 tab:超管在别家页签里改别家的行,
|
||||
// 要比的是那一家的库。
|
||||
ensureUnique(dataValue, id, entity.getCompanyId());
|
||||
entity.setDataValue(dataValue);
|
||||
ShopManageGroupEntity group = resolveWritableGroup(request.getGroupId(), operator);
|
||||
entity.setGroupId(group.getId());
|
||||
@@ -1207,32 +1195,58 @@ public class DedupeTotalDataService {
|
||||
return normalizeExcelText(value).toUpperCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private void ensureUnique(String dataValue, Long excludeId) {
|
||||
/**
|
||||
* 「已存在」只在一家公司内成立:本表唯一键自 V145 起是 uk_company_data_value(company_id,
|
||||
* data_value),同值本来就允许归两家。此前这一刀不带公司,结果是 B 公司导入过的 ASIN
|
||||
* 会让 A 公司的「新增」报"数据已存在"——用户看到的是"我加不进去",而库里根本没有他的行。
|
||||
*/
|
||||
private void ensureUnique(String dataValue, Long excludeId, Long companyId) {
|
||||
requireScopedCompanyId(companyId);
|
||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||
.eq(DedupeTotalDataEntity::getCompanyId, companyId)
|
||||
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
|
||||
.last("LIMIT 1");
|
||||
DedupeTotalDataEntity exists = dedupeTotalDataMapper.selectOne(query);
|
||||
if (exists != null && (excludeId == null || !exists.getId().equals(excludeId))) {
|
||||
throw new BusinessException("数据已存在");
|
||||
throw new BusinessException("本公司已存在该数据");
|
||||
}
|
||||
}
|
||||
|
||||
private boolean existsDataValue(String dataValue) {
|
||||
private boolean existsDataValue(String dataValue, Long companyId) {
|
||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||
.eq(DedupeTotalDataEntity::getCompanyId, companyId)
|
||||
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
|
||||
.last("LIMIT 1");
|
||||
return dedupeTotalDataMapper.selectOne(query) != null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 公司身份闸门:本表每一条按公司取的判据都从这一句过。
|
||||
*
|
||||
* <p>值由 controller 在请求线程上算好传进来({@code DataReadScope.readCompanyIdOrOwn}:超管认
|
||||
* tab、非超管恒本公司)。这里只判空,不补默认——导入与去重跑在虚拟线程上,那时已经没有请求
|
||||
* 上下文可查了,兜底只能兜出一个"谁都比对不上"的空集。</p>
|
||||
*/
|
||||
static void requireScopedCompanyId(Long companyId) {
|
||||
if (companyId == null) {
|
||||
throw new BusinessException(
|
||||
"缺少归属公司:请先在页面上方选择公司,再操作去重数据库");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** 批量删除大小:单事务 IN 删除的阈值,兼顾锁窗口与事务日志。 */
|
||||
private static final int DELETE_BATCH_SIZE = 500;
|
||||
|
||||
/** 整批一个独立事务按 IN 一次删除;返回实际删除行数(计入 deletedCount,不再额外计 skipped)。 */
|
||||
private int deleteBatchByDataValues(List<String> dataValues, AccessScope scope, Long groupId) {
|
||||
/** 包内可见只为让用例抓到真正的删除谓词(同 {@link #requireCompanyAttribution} 的先例)。 */
|
||||
int deleteBatchByDataValues(List<String> dataValues, AccessScope scope, Long groupId,
|
||||
Long companyId) {
|
||||
requireScopedCompanyId(companyId);
|
||||
List<String> batch = List.copyOf(dataValues);
|
||||
Integer deleted = newRequiresNewTemplate().execute(status -> {
|
||||
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
|
||||
.eq(DedupeTotalDataEntity::getCompanyId, companyId)
|
||||
.in(DedupeTotalDataEntity::getDataValue, batch);
|
||||
applyGroupScope(query, scope, groupId);
|
||||
return dedupeTotalDataMapper.delete(query);
|
||||
@@ -1512,6 +1526,6 @@ public class DedupeTotalDataService {
|
||||
* @param wholeCompany 公司超管:不做分组维度裁剪,userIds = 本公司全体成员,
|
||||
* 查询条件退化为 uploader_user_id IN (成员)——公司超管的"组"就是整家公司
|
||||
*/
|
||||
private record AccessScope(boolean allUsers, Set<Long> groupIds, Set<Long> userIds, boolean wholeCompany) {
|
||||
record AccessScope(boolean allUsers, Set<Long> groupIds, Set<Long> userIds, boolean wholeCompany) {
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user