提交更改

This commit is contained in:
super
2026-06-05 21:25:03 +08:00
parent 95ccb00d25
commit 8ab647c419
39 changed files with 1390 additions and 92 deletions
@@ -3,12 +3,14 @@ package com.nanri.aiimage.modules.collectdata.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataParseRequest;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitResultRequest;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataTaskBatchRequest;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemsPageVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataParseVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
import io.swagger.v3.oas.annotations.Operation;
@@ -65,6 +67,15 @@ public class CollectDataController {
return ApiResponse.success(service.getItemsPage(taskId, userId, page, pageSize));
}
@PostMapping("/tasks/{taskId}/result")
@Operation(summary = "Python 分片回传采集结果", description = "接收任意条数的 Python 回传行;后端按 ASIN 筛库、调用品牌检测并保存最终结果。")
public ApiResponse<CollectDataSubmitResultVo> submitResult(
@Parameter(description = "采集任务 ID", required = true, example = "9001")
@PathVariable Long taskId,
@Valid @RequestBody CollectDataSubmitResultRequest request) {
return ApiResponse.success(service.submitResult(taskId, request));
}
@GetMapping("/dashboard")
@Operation(summary = "查询采集任务总览")
public ApiResponse<CollectDataDashboardVo> dashboard(
@@ -0,0 +1,41 @@
package com.nanri.aiimage.modules.collectdata.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
@Schema(description = "采集数据 Python 分片回传请求")
public class CollectDataSubmitResultRequest {
@JsonProperty("submission_id")
@JsonAlias("submissionId")
@Schema(description = "分片提交批次标识;同一任务可固定一个 submission_id", example = "collect-11877")
private String submissionId;
@JsonProperty("chunk_index")
@JsonAlias("chunkIndex")
@Schema(description = "分片序号,从 1 开始;未传时后端按当前任务顺序递增", example = "1")
private Integer chunkIndex;
@JsonProperty("chunk_total")
@JsonAlias("chunkTotal")
@Schema(description = "总分片数;未知可不传,done=true 时以后端已收数量兜底", example = "10")
private Integer chunkTotal;
@Schema(description = "是否为最后一次回传", example = "false")
private Boolean done;
@Schema(description = "Python 端错误信息;传入时任务会标记失败")
private String error;
@JsonAlias({"rows", "data", "items"})
@Schema(description = "本分片回传数据行")
private List<CollectDataSubmitRowDto> items = new ArrayList<>();
}
@@ -0,0 +1,32 @@
package com.nanri.aiimage.modules.collectdata.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "采集数据 Python 回传行")
public class CollectDataSubmitRowDto {
@JsonAlias({"品牌", "brand_name"})
@Schema(description = "品牌", example = "Nike")
private String brand;
@JsonAlias({"ASIN", "asin"})
@Schema(description = "ASIN", example = "B0XXXXXXX")
private String asin;
@JsonAlias({"价格", "price"})
@Schema(description = "价格", example = "19.99")
private String price;
@JsonProperty("seller_name")
@JsonAlias({"卖家名称", "sellerName", "seller"})
@Schema(description = "卖家名称", example = "Demo Seller")
private String sellerName;
@JsonAlias({"关键词", "key_word"})
@Schema(description = "关键词", example = "phone case")
private String keyword;
}
@@ -37,6 +37,18 @@ public class CollectDataHistoryItemVo {
@Schema(description = "落库行数")
private Integer rowCount;
@Schema(description = "被数据去重总数据过滤的行数")
private Integer dedupeFilteredCount;
@Schema(description = "被不符合 ASIN 数据库过滤的行数")
private Integer invalidFilteredCount;
@Schema(description = "品牌检测不符合的行数")
private Integer brandRejectedCount;
@Schema(description = "最终结果行数")
private Integer finalRowCount;
@Schema(description = "记录创建时间")
private String createdAt;
@@ -0,0 +1,12 @@
package com.nanri.aiimage.modules.collectdata.model.vo;
import lombok.Data;
@Data
public class CollectDataResultRowVo {
private String brand;
private String asin;
private String price;
private String sellerName;
private String keyword;
}
@@ -0,0 +1,22 @@
package com.nanri.aiimage.modules.collectdata.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "采集数据回传处理结果")
public class CollectDataSubmitResultVo {
private Long taskId;
private Long resultId;
private Integer chunkIndex;
private Integer chunkTotal;
private Boolean done;
private Integer receivedRows;
private Integer currentChunkRows;
private Integer dedupeFilteredCount;
private Integer invalidFilteredCount;
private Integer brandRejectedCount;
private Integer brandQueryFailedCount;
private Integer finalRowCount;
private String taskStatus;
}
@@ -0,0 +1,61 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileOutputStream;
import java.util.List;
@Service
@Slf4j
public class CollectDataExcelAssemblyService {
private static final String[] HEADER = {"品牌", "ASIN", "价格", "卖家名称", "关键词"};
public void writeWorkbook(File outputXlsx, List<CollectDataResultRowVo> items) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream outputStream = new FileOutputStream(outputXlsx)) {
Sheet sheet = workbook.createSheet("采集数据结果");
Row headerRow = sheet.createRow(0);
for (int i = 0; i < HEADER.length; i++) {
headerRow.createCell(i).setCellValue(HEADER[i]);
sheet.setColumnWidth(i, (i == 3 ? 24 : 18) * 256);
}
int rowIndex = 1;
if (items != null) {
for (CollectDataResultRowVo item : items) {
if (item == null) {
continue;
}
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(safe(item.getBrand()));
row.createCell(1).setCellValue(safe(item.getAsin()));
row.createCell(2).setCellValue(safe(item.getPrice()));
row.createCell(3).setCellValue(safe(item.getSellerName()));
row.createCell(4).setCellValue(safe(item.getKeyword()));
}
}
workbook.write(outputStream);
} catch (Exception ex) {
log.warn("[collect-data] write workbook failed: {}", ex.getMessage());
throw new BusinessException("生成采集数据 Excel 失败: " + ex.getMessage());
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}
private String safe(String value) {
return value == null ? "" : value;
}
}
@@ -1,17 +1,23 @@
package com.nanri.aiimage.modules.collectdata.service;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.crypto.digest.DigestUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataFiltersDto;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataParseRequest;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSourceFileDto;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitResultRequest;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitRowDto;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataCountryPrefEntity;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo;
@@ -21,14 +27,30 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemsPageVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataParseVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo;
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
@@ -40,18 +62,23 @@ import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.dao.DuplicateKeyException;
import java.io.File;
import java.io.FileInputStream;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Pattern;
@Service
@@ -74,6 +101,9 @@ public class CollectDataService {
private static final String DEFAULT_TASK_TYPE = "collect-data";
private static final int ITEM_INSERT_BATCH_SIZE = 500;
private static final int BRAND_CHECK_BATCH_SIZE = 10;
private static final long TASK_LOCK_WAIT_MILLIS = 5000L;
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final List<String> KEYWORD_HEADER_ALIASES = List.of("关键词", "keyword", "key word");
private static final List<String> STATUS_HEADER_ALIASES = List.of("状态", "status");
@@ -86,6 +116,17 @@ public class CollectDataService {
private final FileResultMapper fileResultMapper;
private final CollectDataItemMapper collectDataItemMapper;
private final CollectDataCountryPrefMapper collectDataCountryPrefMapper;
private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final InvalidAsinDataMapper invalidAsinDataMapper;
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final TaskResultItemMapper taskResultItemMapper;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskFileJobService taskFileJobService;
private final TransientPayloadStorageService transientPayloadStorageService;
private final BrandCheckClient brandCheckClient;
private final CollectDataExcelAssemblyService excelAssemblyService;
private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper;
private final TransactionTemplate transactionTemplate;
@@ -343,6 +384,649 @@ public class CollectDataService {
return vo;
}
public CollectDataSubmitResultVo submitResult(Long taskId, CollectDataSubmitResultRequest request) {
if (request == null) {
throw new BusinessException("request is empty");
}
ensureRustfsPayloadStorageEnabled();
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
}
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
throw new BusinessException(40901, "任务已结束,拒绝重复提交");
}
if (!STATUS_RUNNING.equals(task.getStatus())) {
task.setStatus(STATUS_RUNNING);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(null);
fileTaskMapper.updateById(task);
}
FileResultEntity result = ensureTaskResult(task);
String scopeKey = firstNonBlank(request.getSubmissionId(), "task:" + taskId);
String scopeHash = DigestUtil.sha256Hex(scopeKey);
int chunkIndex = request.getChunkIndex() == null || request.getChunkIndex() <= 0
? nextChunkIndex(taskId, scopeHash)
: request.getChunkIndex();
int chunkTotal = request.getChunkTotal() == null || request.getChunkTotal() <= 0
? Math.max(chunkIndex, 1)
: request.getChunkTotal();
TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
.last("limit 1"));
if (existing != null) {
log.info("[collect-data] duplicate result chunk ignored taskId={} scope={} chunk={}",
taskId, scopeKey, chunkIndex);
return buildSubmitVo(task, result, chunkIndex, chunkTotal, request, 0);
}
List<CollectDataResultRowVo> rows = normalizeSubmitRows(request.getItems());
CollectDataStats stats = loadStats(task);
stats.receivedRows += rows.size();
stats.currentChunkRows = rows.size();
List<CollectDataResultRowVo> candidates = filterByExistingAsin(rows, stats);
List<CollectDataResultRowVo> accepted = filterByBrandCheck(candidates, stats);
for (CollectDataResultRowVo row : accepted) {
upsertResultItem(task.getId(), result.getId(), scopeKey, row);
}
String payloadJson = writeJson(rows, "采集结果序列化失败");
String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(
MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
requireRustfsPayload(storedPayload, "采集结果分片必须写入 RustFS");
persistChunk(taskId, scopeKey, scopeHash, chunkIndex, chunkTotal, storedPayload, payloadJson);
persistScope(taskId, scopeKey, scopeHash, chunkTotal, request);
stats.finalRowCount = countFinalRows(taskId);
persistStats(task, stats);
if (request.getError() != null && !request.getError().isBlank()) {
markTaskFailed(task, result, request.getError(), stats);
} else if (Boolean.TRUE.equals(request.getDone())) {
enqueueFinalWorkbook(task, result, stats);
} else {
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
}
log.info("[collect-data] submit result taskId={} chunk={}/{} rows={} accepted={} dedupeFiltered={} invalidFiltered={} brandRejected={} finalRows={} done={}",
taskId, chunkIndex, chunkTotal, rows.size(), accepted.size(), stats.dedupeFilteredCount,
stats.invalidFilteredCount, stats.brandRejectedCount, stats.finalRowCount, request.getDone());
return buildSubmitVo(fileTaskMapper.selectById(taskId), result, chunkIndex, chunkTotal, request, rows.size());
}
}
private FileResultEntity ensureTaskResult(FileTaskEntity task) {
FileResultEntity result = fileResultMapper.selectOne(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, task.getId())
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.orderByAsc(FileResultEntity::getId)
.last("limit 1"));
if (result != null) {
return result;
}
result = new FileResultEntity();
result.setTaskId(task.getId());
result.setModuleType(MODULE_TYPE);
result.setSourceFilename("");
result.setRowCount(0);
result.setUserId(task.getUserId());
result.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(result);
return result;
}
private List<CollectDataResultRowVo> normalizeSubmitRows(List<CollectDataSubmitRowDto> rawRows) {
List<CollectDataResultRowVo> rows = new ArrayList<>();
if (rawRows == null) {
return rows;
}
for (CollectDataSubmitRowDto raw : rawRows) {
if (raw == null) {
continue;
}
String asin = normalizeAsin(raw.getAsin());
if (asin.isBlank()) {
continue;
}
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setBrand(normalize(raw.getBrand()));
row.setAsin(asin);
row.setPrice(normalize(raw.getPrice()));
row.setSellerName(normalize(raw.getSellerName()));
row.setKeyword(normalize(raw.getKeyword()));
rows.add(row);
}
return rows;
}
private List<CollectDataResultRowVo> filterByExistingAsin(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
List<String> asins = rows.stream()
.map(CollectDataResultRowVo::getAsin)
.filter(value -> value != null && !value.isBlank())
.distinct()
.toList();
Set<String> dedupeValues = new HashSet<>();
if (!asins.isEmpty()) {
List<String> existingDedupeValues = dedupeTotalDataMapper.selectExistingDataValues(asins);
if (existingDedupeValues != null) {
dedupeValues.addAll(existingDedupeValues.stream()
.map(this::normalizeAsin)
.toList());
}
}
Set<String> invalidValues = new HashSet<>();
if (!asins.isEmpty()) {
List<InvalidAsinDataEntity> invalidRows = invalidAsinDataMapper.selectList(new LambdaQueryWrapper<InvalidAsinDataEntity>()
.in(InvalidAsinDataEntity::getDataValue, asins));
if (invalidRows != null) {
for (InvalidAsinDataEntity row : invalidRows) {
invalidValues.add(normalizeAsin(row.getDataValue()));
}
}
}
List<CollectDataResultRowVo> out = new ArrayList<>();
for (CollectDataResultRowVo row : rows) {
String asin = row.getAsin();
if (dedupeValues.contains(asin)) {
stats.dedupeFilteredCount++;
continue;
}
if (invalidValues.contains(asin)) {
stats.invalidFilteredCount++;
continue;
}
out.add(row);
}
return out;
}
private List<CollectDataResultRowVo> filterByBrandCheck(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
List<CollectDataResultRowVo> accepted = new ArrayList<>();
for (int start = 0; start < rows.size(); start += BRAND_CHECK_BATCH_SIZE) {
int end = Math.min(start + BRAND_CHECK_BATCH_SIZE, rows.size());
List<CollectDataResultRowVo> batch = rows.subList(start, end);
List<String> brands = batch.stream()
.map(CollectDataResultRowVo::getBrand)
.filter(value -> value != null && !value.isBlank())
.distinct()
.toList();
BrandCheckClient.BrandCheckBatchResult check = brandCheckClient.checkAll(brands, "Terms");
Set<String> failedBrands = normalizeObjectSet(check == null ? null : check.faildData());
Set<String> queryFailedBrands = normalizeObjectSet(check == null ? null : check.queryFaildData());
for (CollectDataResultRowVo row : batch) {
String brand = normalizeBrand(row.getBrand());
if (brand.isBlank()) {
stats.brandRejectedCount++;
insertInvalidAsin(row);
continue;
}
if (failedBrands.contains(brand)) {
stats.brandRejectedCount++;
insertInvalidAsin(row);
continue;
}
if (queryFailedBrands.contains(brand)) {
stats.brandQueryFailedCount++;
insertInvalidAsin(row);
continue;
}
accepted.add(row);
}
}
return accepted;
}
private Set<String> normalizeObjectSet(List<Object> values) {
Set<String> out = new HashSet<>();
if (values == null) {
return out;
}
for (Object value : values) {
String normalized = normalizeBrand(extractBrandValue(value));
if (!normalized.isBlank()) {
out.add(normalized);
}
}
return out;
}
private String extractBrandValue(Object value) {
if (value == null) {
return "";
}
if (value instanceof Map<?, ?> map) {
for (String key : List.of("brand", "brandName", "brand_name", "name", "value", "data_value")) {
Object candidate = map.get(key);
if (candidate != null && !String.valueOf(candidate).isBlank()) {
return String.valueOf(candidate);
}
}
}
return String.valueOf(value);
}
private void insertInvalidAsin(CollectDataResultRowVo row) {
if (row == null || row.getAsin() == null || row.getAsin().isBlank()) {
return;
}
InvalidAsinDataEntity entity = new InvalidAsinDataEntity();
entity.setDataValue(row.getAsin());
entity.setBrand(row.getBrand());
try {
invalidAsinDataMapper.insert(entity);
} catch (DuplicateKeyException ignored) {
}
}
private void upsertResultItem(Long taskId, Long resultId, String scopeKey, CollectDataResultRowVo row) {
String itemKey = "asin:" + row.getAsin();
String scopeHash = hash(scopeKey);
String payloadJson = writeJson(row, "采集结果明细序列化失败");
String payloadHash = hash(payloadJson);
TaskResultItemEntity existing = taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
.eq(TaskResultItemEntity::getScopeHash, scopeHash)
.eq(TaskResultItemEntity::getItemKey, itemKey)
.last("limit 1"));
if (existing != null && Objects.equals(existing.getPayloadHash(), payloadHash)) {
return;
}
String storedPayload = transientPayloadStorageService.storeResultItemPayload(
MODULE_TYPE, taskId, scopeHash, itemKey, payloadJson);
requireRustfsPayload(storedPayload, "采集结果明细必须写入 RustFS");
LocalDateTime now = LocalDateTime.now();
if (existing == null) {
TaskResultItemEntity entity = new TaskResultItemEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
entity.setResultId(resultId);
entity.setScopeKey(scopeKey);
entity.setScopeHash(scopeHash);
entity.setItemKey(itemKey);
entity.setAsin(row.getAsin());
entity.setStatus("ACCEPTED");
entity.setPayloadJson(storedPayload);
entity.setPayloadHash(payloadHash);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskResultItemMapper.insert(entity);
return;
} catch (DuplicateKeyException ignored) {
existing = taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
.eq(TaskResultItemEntity::getScopeHash, scopeHash)
.eq(TaskResultItemEntity::getItemKey, itemKey)
.last("limit 1"));
}
}
if (existing == null) {
throw new BusinessException("保存采集结果明细失败");
}
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getPayloadJson(), storedPayload);
taskResultItemMapper.update(null, new LambdaUpdateWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getId, existing.getId())
.set(TaskResultItemEntity::getResultId, resultId)
.set(TaskResultItemEntity::getScopeKey, scopeKey)
.set(TaskResultItemEntity::getAsin, row.getAsin())
.set(TaskResultItemEntity::getStatus, "ACCEPTED")
.set(TaskResultItemEntity::getPayloadJson, storedPayload)
.set(TaskResultItemEntity::getPayloadHash, payloadHash)
.set(TaskResultItemEntity::getUpdatedAt, now));
}
private void persistChunk(Long taskId,
String scopeKey,
String scopeHash,
int chunkIndex,
int chunkTotal,
String storedPayload,
String payloadJson) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setTaskId(taskId);
chunk.setModuleType(MODULE_TYPE);
chunk.setScopeKey(scopeKey);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setChunkTotal(chunkTotal);
chunk.setPayloadJson(storedPayload);
chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson));
chunk.setCreatedAt(LocalDateTime.now());
chunk.setUpdatedAt(LocalDateTime.now());
try {
taskChunkMapper.insert(chunk);
} catch (DuplicateKeyException ex) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
log.info("[collect-data] duplicate chunk inserted concurrently taskId={} scope={} chunk={}",
taskId, scopeKey, chunkIndex);
}
}
private void persistScope(Long taskId, String scopeKey, String scopeHash, int chunkTotal, CollectDataSubmitResultRequest request) {
TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.last("limit 1"));
if (scope == null) {
scope = new TaskScopeStateEntity();
scope.setTaskId(taskId);
scope.setModuleType(MODULE_TYPE);
scope.setScopeKey(scopeKey);
scope.setScopeHash(scopeHash);
scope.setCreatedAt(LocalDateTime.now());
}
scope.setChunkTotal(chunkTotal);
scope.setReceivedChunkCount(countReceivedChunks(taskId, scopeHash));
scope.setLastChunkAt(LocalDateTime.now());
scope.setLastError(request.getError());
scope.setCompleted(Boolean.TRUE.equals(request.getDone()) ? 1 : 0);
scope.setStateJson("{\"phase\":\"RECEIVED\"}");
scope.setUpdatedAt(LocalDateTime.now());
if (scope.getId() == null) {
taskScopeStateMapper.insert(scope);
} else {
taskScopeStateMapper.updateById(scope);
}
}
private void enqueueFinalWorkbook(FileTaskEntity task, FileResultEntity result, CollectDataStats stats) {
String filename = buildResultFilename(task);
result.setResultFilename(filename);
result.setResultFileUrl(null);
result.setResultFileSize(0L);
result.setResultContentType(CONTENT_TYPE_XLSX);
result.setRowCount(stats.finalRowCount);
result.setErrorMessage(null);
fileResultMapper.updateById(result);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(), "task:" + task.getId());
}
private void markTaskFailed(FileTaskEntity task, FileResultEntity result, String error, CollectDataStats stats) {
result.setSuccess(0);
result.setErrorMessage(error);
result.setRowCount(stats.finalRowCount);
fileResultMapper.updateById(result);
task.setStatus(STATUS_FAILED);
task.setErrorMessage(error);
task.setFailedFileCount(1);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
persistStats(task, stats);
fileTaskMapper.updateById(task);
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
}
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
throw new BusinessException("结果记录不存在");
}
List<CollectDataResultRowVo> rows = loadFinalRows(task.getId());
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "collect-data-result", String.valueOf(task.getId())));
String filename = buildResultFilename(task);
File xlsx = FileUtil.file(workRoot, filename);
try {
excelAssemblyService.writeWorkbook(xlsx, rows);
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
result.setResultFilename(filename);
result.setResultFileUrl(objectKey);
result.setResultFileSize(xlsx.length());
result.setResultContentType(CONTENT_TYPE_XLSX);
result.setRowCount(rows.size());
result.setSuccess(1);
result.setErrorMessage(null);
fileResultMapper.updateById(result);
CollectDataStats stats = loadStats(task);
stats.finalRowCount = rows.size();
persistStats(task, stats);
task.setStatus(STATUS_SUCCESS);
task.setSuccessFileCount(1);
task.setFailedFileCount(0);
task.setErrorMessage(null);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
} finally {
FileUtil.del(xlsx);
}
}
private List<CollectDataResultRowVo> loadFinalRows(Long taskId) {
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskResultItemEntity::getId));
List<CollectDataResultRowVo> out = new ArrayList<>();
if (rows == null) {
return out;
}
for (TaskResultItemEntity row : rows) {
try {
String payloadJson = transientPayloadStorageService.resolvePayload(row.getPayloadJson(), "read collect data result item failed");
CollectDataResultRowVo value = objectMapper.readValue(payloadJson, CollectDataResultRowVo.class);
if (value != null) {
out.add(value);
}
} catch (Exception ex) {
throw new BusinessException("读取采集结果明细失败");
}
}
return out;
}
private int countFinalRows(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskResultItemMapper.selectCount(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE));
return count == null ? 0 : count.intValue();
}
private void ensureRustfsPayloadStorageEnabled() {
if (!transientPayloadStorageService.isSharedWriteEnabled()) {
throw new BusinessException("RustFS 未配置,采集结果回传暂不可接收");
}
}
private void requireRustfsPayload(String storedPayload, String message) {
String pointer = transientPayloadStorageService.extractPointer(storedPayload);
if (pointer != null && pointer.startsWith("rustfs:")) {
return;
}
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw new BusinessException(message);
}
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(MODULE_TYPE, taskId, TASK_LOCK_WAIT_MILLIS);
if (lockHandle == null) {
throw new BusinessException(40901, "任务正在处理,请稍后再试");
}
return lockHandle;
}
private int nextChunkIndex(Long taskId, String scopeHash) {
TaskChunkEntity latest = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.orderByDesc(TaskChunkEntity::getChunkIndex)
.last("limit 1"));
return latest == null || latest.getChunkIndex() == null ? 1 : latest.getChunkIndex() + 1;
}
private int countReceivedChunks(Long taskId, String scopeHash) {
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash));
return count == null ? 0 : count.intValue();
}
private CollectDataSubmitResultVo buildSubmitVo(FileTaskEntity task,
FileResultEntity result,
int chunkIndex,
int chunkTotal,
CollectDataSubmitResultRequest request,
int currentChunkRows) {
CollectDataStats stats = loadStats(task);
stats.finalRowCount = countFinalRows(task == null ? null : task.getId());
CollectDataSubmitResultVo vo = new CollectDataSubmitResultVo();
vo.setTaskId(task == null ? null : task.getId());
vo.setResultId(result == null ? null : result.getId());
vo.setChunkIndex(chunkIndex);
vo.setChunkTotal(chunkTotal);
vo.setDone(request != null && Boolean.TRUE.equals(request.getDone()));
vo.setReceivedRows(stats.receivedRows);
vo.setCurrentChunkRows(currentChunkRows);
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
vo.setBrandRejectedCount(stats.brandRejectedCount);
vo.setBrandQueryFailedCount(stats.brandQueryFailedCount);
vo.setFinalRowCount(stats.finalRowCount);
vo.setTaskStatus(task == null ? null : task.getStatus());
return vo;
}
private CollectDataStats loadStats(FileTaskEntity task) {
CollectDataStats stats = new CollectDataStats();
if (task == null || task.getResultJson() == null || task.getResultJson().isBlank()) {
return stats;
}
try {
JsonNode root = objectMapper.readTree(task.getResultJson());
stats.receivedRows = root.path("receivedRows").asInt(0);
stats.currentChunkRows = root.path("currentChunkRows").asInt(0);
stats.dedupeFilteredCount = root.path("dedupeFilteredCount").asInt(0);
stats.invalidFilteredCount = root.path("invalidFilteredCount").asInt(0);
stats.brandRejectedCount = root.path("brandRejectedCount").asInt(0);
stats.brandQueryFailedCount = root.path("brandQueryFailedCount").asInt(0);
stats.finalRowCount = root.path("finalRowCount").asInt(0);
} catch (Exception ex) {
log.warn("[collect-data] parse result stats failed taskId={} err={}", task.getId(), ex.getMessage());
}
return stats;
}
private void persistStats(FileTaskEntity task, CollectDataStats stats) {
if (task == null || stats == null) {
return;
}
try {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("receivedRows", stats.receivedRows);
payload.put("currentChunkRows", stats.currentChunkRows);
payload.put("dedupeFilteredCount", stats.dedupeFilteredCount);
payload.put("invalidFilteredCount", stats.invalidFilteredCount);
payload.put("brandRejectedCount", stats.brandRejectedCount);
payload.put("brandQueryFailedCount", stats.brandQueryFailedCount);
payload.put("finalRowCount", stats.finalRowCount);
task.setResultJson(objectMapper.writeValueAsString(payload));
} catch (Exception ex) {
throw new BusinessException("保存采集统计失败");
}
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException(message);
}
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new IllegalStateException("failed to hash collect data payload", ex);
}
}
private String normalizeAsin(String value) {
return normalize(value).toUpperCase(Locale.ROOT);
}
private String normalizeBrand(String value) {
return normalize(value).toLowerCase(Locale.ROOT);
}
private String buildResultFilename(FileTaskEntity task) {
String base = task == null || task.getTaskNo() == null || task.getTaskNo().isBlank()
? "collect-data-" + (task == null ? "result" : task.getId())
: task.getTaskNo();
return base.replaceAll("[\\\\/:*?\"<>|]", "_") + "-result.xlsx";
}
private void deleteTransientTaskPayloads(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
}
}
List<TaskResultItemEntity> items = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.select(TaskResultItemEntity::getPayloadJson)
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE));
if (items != null) {
for (TaskResultItemEntity item : items) {
transientPayloadStorageService.deletePayloadIfPresent(item.getPayloadJson());
}
}
}
private static class CollectDataStats {
private int receivedRows;
private int currentChunkRows;
private int dedupeFilteredCount;
private int invalidFilteredCount;
private int brandRejectedCount;
private int brandQueryFailedCount;
private int finalRowCount;
}
@Transactional
public void deleteTask(Long taskId, Long userId) {
FileTaskEntity task = requireTask(taskId, userId);
@@ -351,6 +1035,17 @@ public class CollectDataService {
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, task.getId())
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
deleteTransientTaskPayloads(task.getId());
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, task.getId())
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, task.getId())
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, task.getId())
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE));
taskFileJobService.deleteTaskJobs(task.getId(), MODULE_TYPE);
fileTaskMapper.deleteById(task.getId());
}
@@ -359,6 +1054,21 @@ public class CollectDataService {
if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) {
throw new BusinessException("记录不存在");
}
List<TaskResultItemEntity> resultItems = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.select(TaskResultItemEntity::getPayloadJson)
.eq(TaskResultItemEntity::getTaskId, row.getTaskId())
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
.eq(TaskResultItemEntity::getResultId, row.getId()));
if (resultItems != null) {
for (TaskResultItemEntity item : resultItems) {
transientPayloadStorageService.deletePayloadIfPresent(item.getPayloadJson());
}
}
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, row.getTaskId())
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
.eq(TaskResultItemEntity::getResultId, row.getId()));
taskFileJobService.deleteResultJobs(row.getTaskId(), MODULE_TYPE, row.getId());
fileResultMapper.deleteById(resultId);
}
@@ -503,21 +1213,41 @@ public class CollectDataService {
vo.setTaskId(row.getTaskId());
vo.setSourceFilename(row.getSourceFilename());
vo.setResultFilename(row.getResultFilename());
vo.setDownloadUrl(buildFreshDownloadUrl(row));
vo.setRowCount(row.getRowCount());
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
vo.setError(row.getErrorMessage());
vo.setCreatedAt(formatTime(row.getCreatedAt()));
if (task != null) {
CollectDataStats stats = loadStats(task);
vo.setTaskNo(task.getTaskNo());
vo.setTaskStatus(task.getStatus());
vo.setStartedAt(formatTime(task.getCreatedAt()));
vo.setFinishedAt(formatTime(task.getFinishedAt()));
vo.setTaskType(extractTaskTypeFromRequest(task));
vo.setFilters(extractFiltersFromRequest(task));
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
vo.setBrandRejectedCount(stats.brandRejectedCount);
vo.setFinalRowCount(stats.finalRowCount);
}
return vo;
}
private String buildFreshDownloadUrl(FileResultEntity row) {
String resultFileUrl = row == null ? null : row.getResultFileUrl();
if (resultFileUrl == null || resultFileUrl.isBlank()) {
return null;
}
try {
return ossStorageService.generateFreshDownloadUrl(resultFileUrl);
} catch (Exception ex) {
log.warn("[collect-data] generate fresh download url failed resultId={} err={}",
row.getId(), ex.getMessage());
return null;
}
}
private CollectDataTaskSummaryVo toTaskSummary(FileTaskEntity task) {
CollectDataTaskSummaryVo vo = new CollectDataTaskSummaryVo();
vo.setId(task.getId());