refactor(phase-3): 3C 第六支接入底座——八件十一点,接不上的七格逐格立账
CollectDataService extends AbstractFileTaskPipeline<CollectDataResultRowVo>:
@RequiredArgsConstructor 换成显式 29 枚构造子(参数表逐枚沿用原字段声明序,
两处 29 枚实参的测试调用点一个字未改),五枚底座依赖交 super;钩子两枚
moduleType()/loadTaskForSubmission()(DB 直读派)。
接上的公共件(八件十一点):newTaskHeader/insertTask/newScopeRow×2(建单结果行
+ ensureTaskResult 补建行)/requireSubmissionTask×2(锁外预检+锁内复核,全族
第一次同一格被一支用两次)/rejectTerminalSubmission(IDEMPOTENT)×2/
enqueueResultAssembly。锁外那格的 warn 留在调用点(底座件刻意不打日志)。
接不上、逐格留本地写法并写明理由:admitSubmission 整段(本支两段准入、那一格一次
走完);requireUserId(现网把 request 空判与 userId 非法合并成同一格判据);
requireTaskId(本支没有那一格,golden 已证);acquireSubmissionLockOrThrow(文案
差一个字、warn 还带 waitMillis);listScopeRows(本支要 limit 1 首行);
publishResultFile(底座把上传捆在写列内,本支现网先上传→persistStats→再写列,
那条时序是注释里立着的自愈前提);writeJson 两处未并(要 RETHROW_FLOOR 383→381
的逐处记名授权,本笔未动门禁)。
行数账:显式构造子+钩子 ≈+65、②抵掉 22 ⇒ 本支棘轮 2080→2123(+43),对在册
基线 2209 仍净减 86;基线一格未改、--write 未跑。额度来源是上一笔 A 聚类抽件的
−129,抵平因此跨了两笔,这一点如实登记。
装配面:TaskServiceWiringSliceTest 补第六支——真容器起六支、本支 24 枚私有件逐枚
assertSame、并断言本支不再留 taskFileJobService 同名字段;moduleType() 第五个读数
assemble("COLLECT_DATA")。
门禁:定向 341 例/0 红/2 skipped;wiring 5 例/0 红;后端全量 4528 例/0 红/
14 skipped/BUILD SUCCESS(真实 rc=0,用例数与上一读数相同——本笔只加断言不加
方法);两套 golden 非写模式逐字节复现;ops ruff All checks passed、readability
棘轮 9 passed;格式步未纳管 ⇒ 如实跳过。真机启动验收本格仍欠。
This commit is contained in:
+93
-50
@@ -1,6 +1,5 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.crypto.digest.DigestUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
@@ -10,7 +9,6 @@ import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
|
||||
@@ -65,7 +63,6 @@ import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
@@ -98,11 +95,11 @@ import java.util.regex.Pattern;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy;
|
||||
import com.nanri.aiimage.modules.task.service.pipeline.AbstractFileTaskPipeline;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class CollectDataService {
|
||||
public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResultRowVo> {
|
||||
|
||||
public static final String MODULE_TYPE = "COLLECT_DATA";
|
||||
|
||||
@@ -147,20 +144,15 @@ public class CollectDataService {
|
||||
private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
|
||||
|
||||
private final LocalFileStorageService localFileStorageService;
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final CollectDataItemMapper collectDataItemMapper;
|
||||
private final CollectDataCountryPrefMapper collectDataCountryPrefMapper;
|
||||
private final InvalidAsinDataMapper invalidAsinDataMapper;
|
||||
private final TaskChunkMapper taskChunkMapper;
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final TaskResultItemMapper taskResultItemMapper;
|
||||
private final TaskDistributedLockService taskDistributedLockService;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||
private final CollectDataExcelAssemblyService excelAssemblyService;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TransactionTemplate transactionTemplate;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
@@ -198,6 +190,86 @@ public class CollectDataService {
|
||||
*/
|
||||
private final StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy;
|
||||
|
||||
/**
|
||||
* 显式构造子取代 {@code @RequiredArgsConstructor}:参数表<b>逐枚沿用原字段声明序</b>,
|
||||
* 只是把交给底座的五枚(两支 Mapper+锁+Job+对象存储)转给 {@code super}——
|
||||
* 这样 {@code CollectDataSnapshotTest}/{@code CollectDataSubmitResultSnapshotTest} 那两处
|
||||
* 29 枚实参的调用点一个字都不用改(改顺序等于让 golden 的夹具与被测件重排,白增风险)。
|
||||
* 逐枚接线由 {@code TaskServiceWiringSliceTest} 钉。
|
||||
*/
|
||||
public CollectDataService(LocalFileStorageService localFileStorageService,
|
||||
FileTaskMapper fileTaskMapper,
|
||||
FileResultMapper fileResultMapper,
|
||||
CollectDataItemMapper collectDataItemMapper,
|
||||
CollectDataCountryPrefMapper collectDataCountryPrefMapper,
|
||||
InvalidAsinDataMapper invalidAsinDataMapper,
|
||||
TaskChunkMapper taskChunkMapper,
|
||||
TaskScopeStateMapper taskScopeStateMapper,
|
||||
TaskResultItemMapper taskResultItemMapper,
|
||||
TaskDistributedLockService taskDistributedLockService,
|
||||
TaskFileJobService taskFileJobService,
|
||||
TaskHeartbeatPositionService taskHeartbeatPositionService,
|
||||
TransientPayloadStorageService transientPayloadStorageService,
|
||||
CollectDataExcelAssemblyService excelAssemblyService,
|
||||
OssStorageService ossStorageService,
|
||||
ObjectMapper objectMapper,
|
||||
TransactionTemplate transactionTemplate,
|
||||
DistributedJobLockService distributedJobLockService,
|
||||
CollectDataBatchQuery collectDataBatchQuery,
|
||||
CollectDataBrandBatchFilter brandBatchFilter,
|
||||
CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter,
|
||||
CollectDataResultDetailCodec resultDetailCodec,
|
||||
CollectDataResultItemBatchWriter resultItemBatchWriter,
|
||||
CollectDataResultDetailReader resultDetailReader,
|
||||
TaskProgressLightAssembler taskProgressLightAssembler,
|
||||
NotificationDispatchService notificationDispatchService,
|
||||
AsinResultCacheService asinResultCacheService,
|
||||
CollectDataBrandRecheckService collectDataBrandRecheckService,
|
||||
StaleTaskHeartbeatPolicy staleTaskHeartbeatPolicy) {
|
||||
super(fileTaskMapper, fileResultMapper, taskDistributedLockService, taskFileJobService, ossStorageService);
|
||||
this.localFileStorageService = localFileStorageService;
|
||||
this.collectDataItemMapper = collectDataItemMapper;
|
||||
this.collectDataCountryPrefMapper = collectDataCountryPrefMapper;
|
||||
this.invalidAsinDataMapper = invalidAsinDataMapper;
|
||||
this.taskChunkMapper = taskChunkMapper;
|
||||
this.taskScopeStateMapper = taskScopeStateMapper;
|
||||
this.taskResultItemMapper = taskResultItemMapper;
|
||||
this.taskHeartbeatPositionService = taskHeartbeatPositionService;
|
||||
this.transientPayloadStorageService = transientPayloadStorageService;
|
||||
this.excelAssemblyService = excelAssemblyService;
|
||||
this.objectMapper = objectMapper;
|
||||
this.transactionTemplate = transactionTemplate;
|
||||
this.distributedJobLockService = distributedJobLockService;
|
||||
this.collectDataBatchQuery = collectDataBatchQuery;
|
||||
this.brandBatchFilter = brandBatchFilter;
|
||||
this.invalidAsinBatchWriter = invalidAsinBatchWriter;
|
||||
this.resultDetailCodec = resultDetailCodec;
|
||||
this.resultItemBatchWriter = resultItemBatchWriter;
|
||||
this.resultDetailReader = resultDetailReader;
|
||||
this.taskProgressLightAssembler = taskProgressLightAssembler;
|
||||
this.notificationDispatchService = notificationDispatchService;
|
||||
this.asinResultCacheService = asinResultCacheService;
|
||||
this.collectDataBrandRecheckService = collectDataBrandRecheckService;
|
||||
this.staleTaskHeartbeatPolicy = staleTaskHeartbeatPolicy;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String moduleType() {
|
||||
return MODULE_TYPE;
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载派别=DB 直读(与 AppearancePatent/Publish 同派,见底座 {@code loadTaskForSubmission} 注释)。
|
||||
* 整段 {@code admitSubmission} 套不上:本支是七支里唯一<b>准入两次</b>的(锁外预检+锁内复核),
|
||||
* 而那一格一次走完;锁外那次还要在抛之前 warn 一句现网文案,底座件刻意不打日志。
|
||||
* 泛型 {@code T} 取回传行类型 {@link CollectDataResultRowVo}(本支没有模块缓存服务,
|
||||
* {@code taskCacheService()} 用底座默认的 null,心跳那一格没有承接件)。
|
||||
*/
|
||||
@Override
|
||||
protected FileTaskEntity loadTaskForSubmission(Long taskId) {
|
||||
return fileTaskMapper.selectById(taskId);
|
||||
}
|
||||
|
||||
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
|
||||
private long staleTimeoutMinutes;
|
||||
|
||||
@@ -323,19 +395,10 @@ public class CollectDataService {
|
||||
List<ParsedRow> parsedRows,
|
||||
String aggregateFilename,
|
||||
String deviceId) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType(MODULE_TYPE);
|
||||
task.setTaskMode("PYTHON_QUEUE");
|
||||
task.setStatus(STATUS_PENDING);
|
||||
task.setSourceFileCount(sources.size());
|
||||
FileTaskEntity task = newTaskHeader(STATUS_PENDING, "PYTHON_QUEUE", request.getUserId(), deviceId,
|
||||
sources.size());
|
||||
task.setSuccessFileCount(0);
|
||||
task.setFailedFileCount(0);
|
||||
task.setCreatedBy("user:" + request.getUserId());
|
||||
task.setUserId(request.getUserId());
|
||||
task.setDeviceId(AdminAuthSupport.normalizeDeviceId(deviceId));
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
try {
|
||||
Map<String, Object> requestPayload = new LinkedHashMap<>();
|
||||
requestPayload.put("userId", request.getUserId());
|
||||
@@ -358,7 +421,7 @@ public class CollectDataService {
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("序列化任务信息失败");
|
||||
}
|
||||
fileTaskMapper.insert(task);
|
||||
insertTask(task);
|
||||
|
||||
int rowIndex = 0;
|
||||
LocalDateTime itemCreatedAt = LocalDateTime.now();
|
||||
@@ -390,14 +453,8 @@ public class CollectDataService {
|
||||
collectDataItemMapper.insertBatch(itemBatch);
|
||||
}
|
||||
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setTaskId(task.getId());
|
||||
result.setModuleType(MODULE_TYPE);
|
||||
result.setSourceFilename(aggregateFilename);
|
||||
result.setSourceFileUrl(buildAggregateSourceFileUrl(sources));
|
||||
result.setRowCount(parsedRows.size());
|
||||
result.setUserId(request.getUserId());
|
||||
result.setCreatedAt(LocalDateTime.now());
|
||||
FileResultEntity result = newScopeRow(task.getId(), request.getUserId(), aggregateFilename,
|
||||
buildAggregateSourceFileUrl(sources), parsedRows.size(), null);
|
||||
fileResultMapper.insert(result);
|
||||
|
||||
return new PersistedParse(task, result);
|
||||
@@ -745,14 +802,11 @@ public class CollectDataService {
|
||||
|
||||
// 锁外预检:任务不存在/已结束时立即失败,不为终态任务白跑去重查询与品牌检测。
|
||||
// 只做快速失败,并发正确性仍由锁内的重读复核保证。
|
||||
FileTaskEntity probe = fileTaskMapper.selectById(taskId);
|
||||
if (probe == null || !MODULE_TYPE.equals(probe.getModuleType())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
FileTaskEntity probe = requireSubmissionTask(taskId);
|
||||
if (STATUS_SUCCESS.equals(probe.getStatus()) || STATUS_FAILED.equals(probe.getStatus())) {
|
||||
log.warn("[collect-data] 任务已结束,拒绝重复提交 taskId={} status={}", taskId, probe.getStatus());
|
||||
throw new BusinessException(40901, "任务已结束,拒绝重复提交");
|
||||
}
|
||||
rejectTerminalSubmission(probe.getStatus(), TerminalRejection.IDEMPOTENT);
|
||||
|
||||
// 归一化 / 去重过滤 / 品牌检测放在锁外:品牌检测是同步远程调用,上游 16890 抖动时
|
||||
// 单品牌 10 次重试合计上百秒(taskId 28599 实测:chunk 回传在锁内等品牌检测 103.5 秒,
|
||||
@@ -786,13 +840,8 @@ public class CollectDataService {
|
||||
collectDataBrandRecheckService.enqueue(probe, brandOutcome.queryFailed());
|
||||
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
|
||||
throw new BusinessException(40901, "任务已结束,拒绝重复提交");
|
||||
}
|
||||
FileTaskEntity task = requireSubmissionTask(taskId);
|
||||
rejectTerminalSubmission(task.getStatus(), TerminalRejection.IDEMPOTENT);
|
||||
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||
task.setStatus(STATUS_RUNNING);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
@@ -904,13 +953,7 @@ public class CollectDataService {
|
||||
if (result != null) {
|
||||
return result;
|
||||
}
|
||||
result = new FileResultEntity();
|
||||
result.setTaskId(task.getId());
|
||||
result.setModuleType(MODULE_TYPE);
|
||||
result.setSourceFilename("");
|
||||
result.setRowCount(0);
|
||||
result.setUserId(task.getUserId());
|
||||
result.setCreatedAt(LocalDateTime.now());
|
||||
result = newScopeRow(task.getId(), task.getUserId(), "", null, 0, null);
|
||||
fileResultMapper.insert(result);
|
||||
return result;
|
||||
}
|
||||
@@ -1009,7 +1052,7 @@ public class CollectDataService {
|
||||
fileResultMapper.updateById(result);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(), "task:" + task.getId());
|
||||
enqueueResultAssembly(task.getId(), result.getId(), "task:" + task.getId());
|
||||
}
|
||||
|
||||
private void markTaskFailed(FileTaskEntity task, FileResultEntity result, String error, CollectDataStats stats) {
|
||||
|
||||
Reference in New Issue
Block a user