提交更改
This commit is contained in:
+11
@@ -3,12 +3,14 @@ package com.nanri.aiimage.modules.collectdata.controller;
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataParseRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataTaskBatchRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemsPageVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataParseVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
@@ -65,6 +67,15 @@ public class CollectDataController {
|
||||
return ApiResponse.success(service.getItemsPage(taskId, userId, page, pageSize));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/result")
|
||||
@Operation(summary = "Python 分片回传采集结果", description = "接收任意条数的 Python 回传行;后端按 ASIN 筛库、调用品牌检测并保存最终结果。")
|
||||
public ApiResponse<CollectDataSubmitResultVo> submitResult(
|
||||
@Parameter(description = "采集任务 ID", required = true, example = "9001")
|
||||
@PathVariable Long taskId,
|
||||
@Valid @RequestBody CollectDataSubmitResultRequest request) {
|
||||
return ApiResponse.success(service.submitResult(taskId, request));
|
||||
}
|
||||
|
||||
@GetMapping("/dashboard")
|
||||
@Operation(summary = "查询采集任务总览")
|
||||
public ApiResponse<CollectDataDashboardVo> dashboard(
|
||||
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@Schema(description = "采集数据 Python 分片回传请求")
|
||||
public class CollectDataSubmitResultRequest {
|
||||
|
||||
@JsonProperty("submission_id")
|
||||
@JsonAlias("submissionId")
|
||||
@Schema(description = "分片提交批次标识;同一任务可固定一个 submission_id", example = "collect-11877")
|
||||
private String submissionId;
|
||||
|
||||
@JsonProperty("chunk_index")
|
||||
@JsonAlias("chunkIndex")
|
||||
@Schema(description = "分片序号,从 1 开始;未传时后端按当前任务顺序递增", example = "1")
|
||||
private Integer chunkIndex;
|
||||
|
||||
@JsonProperty("chunk_total")
|
||||
@JsonAlias("chunkTotal")
|
||||
@Schema(description = "总分片数;未知可不传,done=true 时以后端已收数量兜底", example = "10")
|
||||
private Integer chunkTotal;
|
||||
|
||||
@Schema(description = "是否为最后一次回传", example = "false")
|
||||
private Boolean done;
|
||||
|
||||
@Schema(description = "Python 端错误信息;传入时任务会标记失败")
|
||||
private String error;
|
||||
|
||||
@JsonAlias({"rows", "data", "items"})
|
||||
@Schema(description = "本分片回传数据行")
|
||||
private List<CollectDataSubmitRowDto> items = new ArrayList<>();
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据 Python 回传行")
|
||||
public class CollectDataSubmitRowDto {
|
||||
|
||||
@JsonAlias({"品牌", "brand_name"})
|
||||
@Schema(description = "品牌", example = "Nike")
|
||||
private String brand;
|
||||
|
||||
@JsonAlias({"ASIN", "asin"})
|
||||
@Schema(description = "ASIN", example = "B0XXXXXXX")
|
||||
private String asin;
|
||||
|
||||
@JsonAlias({"价格", "price"})
|
||||
@Schema(description = "价格", example = "19.99")
|
||||
private String price;
|
||||
|
||||
@JsonProperty("seller_name")
|
||||
@JsonAlias({"卖家名称", "sellerName", "seller"})
|
||||
@Schema(description = "卖家名称", example = "Demo Seller")
|
||||
private String sellerName;
|
||||
|
||||
@JsonAlias({"关键词", "key_word"})
|
||||
@Schema(description = "关键词", example = "phone case")
|
||||
private String keyword;
|
||||
}
|
||||
+12
@@ -37,6 +37,18 @@ public class CollectDataHistoryItemVo {
|
||||
@Schema(description = "落库行数")
|
||||
private Integer rowCount;
|
||||
|
||||
@Schema(description = "被数据去重总数据过滤的行数")
|
||||
private Integer dedupeFilteredCount;
|
||||
|
||||
@Schema(description = "被不符合 ASIN 数据库过滤的行数")
|
||||
private Integer invalidFilteredCount;
|
||||
|
||||
@Schema(description = "品牌检测不符合的行数")
|
||||
private Integer brandRejectedCount;
|
||||
|
||||
@Schema(description = "最终结果行数")
|
||||
private Integer finalRowCount;
|
||||
|
||||
@Schema(description = "记录创建时间")
|
||||
private String createdAt;
|
||||
|
||||
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class CollectDataResultRowVo {
|
||||
private String brand;
|
||||
private String asin;
|
||||
private String price;
|
||||
private String sellerName;
|
||||
private String keyword;
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据回传处理结果")
|
||||
public class CollectDataSubmitResultVo {
|
||||
private Long taskId;
|
||||
private Long resultId;
|
||||
private Integer chunkIndex;
|
||||
private Integer chunkTotal;
|
||||
private Boolean done;
|
||||
private Integer receivedRows;
|
||||
private Integer currentChunkRows;
|
||||
private Integer dedupeFilteredCount;
|
||||
private Integer invalidFilteredCount;
|
||||
private Integer brandRejectedCount;
|
||||
private Integer brandQueryFailedCount;
|
||||
private Integer finalRowCount;
|
||||
private String taskStatus;
|
||||
}
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class CollectDataExcelAssemblyService {
|
||||
|
||||
private static final String[] HEADER = {"品牌", "ASIN", "价格", "卖家名称", "关键词"};
|
||||
|
||||
public void writeWorkbook(File outputXlsx, List<CollectDataResultRowVo> items) {
|
||||
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
|
||||
workbook.setCompressTempFiles(true);
|
||||
try (FileOutputStream outputStream = new FileOutputStream(outputXlsx)) {
|
||||
Sheet sheet = workbook.createSheet("采集数据结果");
|
||||
Row headerRow = sheet.createRow(0);
|
||||
for (int i = 0; i < HEADER.length; i++) {
|
||||
headerRow.createCell(i).setCellValue(HEADER[i]);
|
||||
sheet.setColumnWidth(i, (i == 3 ? 24 : 18) * 256);
|
||||
}
|
||||
int rowIndex = 1;
|
||||
if (items != null) {
|
||||
for (CollectDataResultRowVo item : items) {
|
||||
if (item == null) {
|
||||
continue;
|
||||
}
|
||||
Row row = sheet.createRow(rowIndex++);
|
||||
row.createCell(0).setCellValue(safe(item.getBrand()));
|
||||
row.createCell(1).setCellValue(safe(item.getAsin()));
|
||||
row.createCell(2).setCellValue(safe(item.getPrice()));
|
||||
row.createCell(3).setCellValue(safe(item.getSellerName()));
|
||||
row.createCell(4).setCellValue(safe(item.getKeyword()));
|
||||
}
|
||||
}
|
||||
workbook.write(outputStream);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] write workbook failed: {}", ex.getMessage());
|
||||
throw new BusinessException("生成采集数据 Excel 失败: " + ex.getMessage());
|
||||
} finally {
|
||||
try {
|
||||
workbook.close();
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
workbook.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private String safe(String value) {
|
||||
return value == null ? "" : value;
|
||||
}
|
||||
}
|
||||
+730
@@ -1,17 +1,23 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.crypto.digest.DigestUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataFiltersDto;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataParseRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSourceFileDto;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitRowDto;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataCountryPrefEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo;
|
||||
@@ -21,14 +27,30 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemsPageVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataParseVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo;
|
||||
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
|
||||
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
|
||||
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
@@ -40,18 +62,23 @@ import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@Service
|
||||
@@ -74,6 +101,9 @@ public class CollectDataService {
|
||||
|
||||
private static final String DEFAULT_TASK_TYPE = "collect-data";
|
||||
private static final int ITEM_INSERT_BATCH_SIZE = 500;
|
||||
private static final int BRAND_CHECK_BATCH_SIZE = 10;
|
||||
private static final long TASK_LOCK_WAIT_MILLIS = 5000L;
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
|
||||
private static final List<String> KEYWORD_HEADER_ALIASES = List.of("关键词", "keyword", "key word");
|
||||
private static final List<String> STATUS_HEADER_ALIASES = List.of("状态", "status");
|
||||
@@ -86,6 +116,17 @@ public class CollectDataService {
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final CollectDataItemMapper collectDataItemMapper;
|
||||
private final CollectDataCountryPrefMapper collectDataCountryPrefMapper;
|
||||
private final DedupeTotalDataMapper dedupeTotalDataMapper;
|
||||
private final InvalidAsinDataMapper invalidAsinDataMapper;
|
||||
private final TaskChunkMapper taskChunkMapper;
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final TaskResultItemMapper taskResultItemMapper;
|
||||
private final TaskDistributedLockService taskDistributedLockService;
|
||||
private final TaskFileJobService taskFileJobService;
|
||||
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||
private final BrandCheckClient brandCheckClient;
|
||||
private final CollectDataExcelAssemblyService excelAssemblyService;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TransactionTemplate transactionTemplate;
|
||||
|
||||
@@ -343,6 +384,649 @@ public class CollectDataService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
public CollectDataSubmitResultVo submitResult(Long taskId, CollectDataSubmitResultRequest request) {
|
||||
if (request == null) {
|
||||
throw new BusinessException("request is empty");
|
||||
}
|
||||
ensureRustfsPayloadStorageEnabled();
|
||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
|
||||
throw new BusinessException(40901, "任务已结束,拒绝重复提交");
|
||||
}
|
||||
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||
task.setStatus(STATUS_RUNNING);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(null);
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
|
||||
FileResultEntity result = ensureTaskResult(task);
|
||||
String scopeKey = firstNonBlank(request.getSubmissionId(), "task:" + taskId);
|
||||
String scopeHash = DigestUtil.sha256Hex(scopeKey);
|
||||
int chunkIndex = request.getChunkIndex() == null || request.getChunkIndex() <= 0
|
||||
? nextChunkIndex(taskId, scopeHash)
|
||||
: request.getChunkIndex();
|
||||
int chunkTotal = request.getChunkTotal() == null || request.getChunkTotal() <= 0
|
||||
? Math.max(chunkIndex, 1)
|
||||
: request.getChunkTotal();
|
||||
|
||||
TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
|
||||
.last("limit 1"));
|
||||
if (existing != null) {
|
||||
log.info("[collect-data] duplicate result chunk ignored taskId={} scope={} chunk={}",
|
||||
taskId, scopeKey, chunkIndex);
|
||||
return buildSubmitVo(task, result, chunkIndex, chunkTotal, request, 0);
|
||||
}
|
||||
|
||||
List<CollectDataResultRowVo> rows = normalizeSubmitRows(request.getItems());
|
||||
CollectDataStats stats = loadStats(task);
|
||||
stats.receivedRows += rows.size();
|
||||
stats.currentChunkRows = rows.size();
|
||||
|
||||
List<CollectDataResultRowVo> candidates = filterByExistingAsin(rows, stats);
|
||||
List<CollectDataResultRowVo> accepted = filterByBrandCheck(candidates, stats);
|
||||
for (CollectDataResultRowVo row : accepted) {
|
||||
upsertResultItem(task.getId(), result.getId(), scopeKey, row);
|
||||
}
|
||||
|
||||
String payloadJson = writeJson(rows, "采集结果序列化失败");
|
||||
String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(
|
||||
MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
|
||||
requireRustfsPayload(storedPayload, "采集结果分片必须写入 RustFS");
|
||||
persistChunk(taskId, scopeKey, scopeHash, chunkIndex, chunkTotal, storedPayload, payloadJson);
|
||||
persistScope(taskId, scopeKey, scopeHash, chunkTotal, request);
|
||||
|
||||
stats.finalRowCount = countFinalRows(taskId);
|
||||
persistStats(task, stats);
|
||||
|
||||
if (request.getError() != null && !request.getError().isBlank()) {
|
||||
markTaskFailed(task, result, request.getError(), stats);
|
||||
} else if (Boolean.TRUE.equals(request.getDone())) {
|
||||
enqueueFinalWorkbook(task, result, stats);
|
||||
} else {
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
|
||||
log.info("[collect-data] submit result taskId={} chunk={}/{} rows={} accepted={} dedupeFiltered={} invalidFiltered={} brandRejected={} finalRows={} done={}",
|
||||
taskId, chunkIndex, chunkTotal, rows.size(), accepted.size(), stats.dedupeFilteredCount,
|
||||
stats.invalidFilteredCount, stats.brandRejectedCount, stats.finalRowCount, request.getDone());
|
||||
return buildSubmitVo(fileTaskMapper.selectById(taskId), result, chunkIndex, chunkTotal, request, rows.size());
|
||||
}
|
||||
}
|
||||
|
||||
private FileResultEntity ensureTaskResult(FileTaskEntity task) {
|
||||
FileResultEntity result = fileResultMapper.selectOne(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, task.getId())
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||
.orderByAsc(FileResultEntity::getId)
|
||||
.last("limit 1"));
|
||||
if (result != null) {
|
||||
return result;
|
||||
}
|
||||
result = new FileResultEntity();
|
||||
result.setTaskId(task.getId());
|
||||
result.setModuleType(MODULE_TYPE);
|
||||
result.setSourceFilename("");
|
||||
result.setRowCount(0);
|
||||
result.setUserId(task.getUserId());
|
||||
result.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<CollectDataResultRowVo> normalizeSubmitRows(List<CollectDataSubmitRowDto> rawRows) {
|
||||
List<CollectDataResultRowVo> rows = new ArrayList<>();
|
||||
if (rawRows == null) {
|
||||
return rows;
|
||||
}
|
||||
for (CollectDataSubmitRowDto raw : rawRows) {
|
||||
if (raw == null) {
|
||||
continue;
|
||||
}
|
||||
String asin = normalizeAsin(raw.getAsin());
|
||||
if (asin.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
CollectDataResultRowVo row = new CollectDataResultRowVo();
|
||||
row.setBrand(normalize(raw.getBrand()));
|
||||
row.setAsin(asin);
|
||||
row.setPrice(normalize(raw.getPrice()));
|
||||
row.setSellerName(normalize(raw.getSellerName()));
|
||||
row.setKeyword(normalize(raw.getKeyword()));
|
||||
rows.add(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
private List<CollectDataResultRowVo> filterByExistingAsin(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> asins = rows.stream()
|
||||
.map(CollectDataResultRowVo::getAsin)
|
||||
.filter(value -> value != null && !value.isBlank())
|
||||
.distinct()
|
||||
.toList();
|
||||
Set<String> dedupeValues = new HashSet<>();
|
||||
if (!asins.isEmpty()) {
|
||||
List<String> existingDedupeValues = dedupeTotalDataMapper.selectExistingDataValues(asins);
|
||||
if (existingDedupeValues != null) {
|
||||
dedupeValues.addAll(existingDedupeValues.stream()
|
||||
.map(this::normalizeAsin)
|
||||
.toList());
|
||||
}
|
||||
}
|
||||
Set<String> invalidValues = new HashSet<>();
|
||||
if (!asins.isEmpty()) {
|
||||
List<InvalidAsinDataEntity> invalidRows = invalidAsinDataMapper.selectList(new LambdaQueryWrapper<InvalidAsinDataEntity>()
|
||||
.in(InvalidAsinDataEntity::getDataValue, asins));
|
||||
if (invalidRows != null) {
|
||||
for (InvalidAsinDataEntity row : invalidRows) {
|
||||
invalidValues.add(normalizeAsin(row.getDataValue()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<CollectDataResultRowVo> out = new ArrayList<>();
|
||||
for (CollectDataResultRowVo row : rows) {
|
||||
String asin = row.getAsin();
|
||||
if (dedupeValues.contains(asin)) {
|
||||
stats.dedupeFilteredCount++;
|
||||
continue;
|
||||
}
|
||||
if (invalidValues.contains(asin)) {
|
||||
stats.invalidFilteredCount++;
|
||||
continue;
|
||||
}
|
||||
out.add(row);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private List<CollectDataResultRowVo> filterByBrandCheck(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<CollectDataResultRowVo> accepted = new ArrayList<>();
|
||||
for (int start = 0; start < rows.size(); start += BRAND_CHECK_BATCH_SIZE) {
|
||||
int end = Math.min(start + BRAND_CHECK_BATCH_SIZE, rows.size());
|
||||
List<CollectDataResultRowVo> batch = rows.subList(start, end);
|
||||
List<String> brands = batch.stream()
|
||||
.map(CollectDataResultRowVo::getBrand)
|
||||
.filter(value -> value != null && !value.isBlank())
|
||||
.distinct()
|
||||
.toList();
|
||||
BrandCheckClient.BrandCheckBatchResult check = brandCheckClient.checkAll(brands, "Terms");
|
||||
Set<String> failedBrands = normalizeObjectSet(check == null ? null : check.faildData());
|
||||
Set<String> queryFailedBrands = normalizeObjectSet(check == null ? null : check.queryFaildData());
|
||||
for (CollectDataResultRowVo row : batch) {
|
||||
String brand = normalizeBrand(row.getBrand());
|
||||
if (brand.isBlank()) {
|
||||
stats.brandRejectedCount++;
|
||||
insertInvalidAsin(row);
|
||||
continue;
|
||||
}
|
||||
if (failedBrands.contains(brand)) {
|
||||
stats.brandRejectedCount++;
|
||||
insertInvalidAsin(row);
|
||||
continue;
|
||||
}
|
||||
if (queryFailedBrands.contains(brand)) {
|
||||
stats.brandQueryFailedCount++;
|
||||
insertInvalidAsin(row);
|
||||
continue;
|
||||
}
|
||||
accepted.add(row);
|
||||
}
|
||||
}
|
||||
return accepted;
|
||||
}
|
||||
|
||||
private Set<String> normalizeObjectSet(List<Object> values) {
|
||||
Set<String> out = new HashSet<>();
|
||||
if (values == null) {
|
||||
return out;
|
||||
}
|
||||
for (Object value : values) {
|
||||
String normalized = normalizeBrand(extractBrandValue(value));
|
||||
if (!normalized.isBlank()) {
|
||||
out.add(normalized);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private String extractBrandValue(Object value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
if (value instanceof Map<?, ?> map) {
|
||||
for (String key : List.of("brand", "brandName", "brand_name", "name", "value", "data_value")) {
|
||||
Object candidate = map.get(key);
|
||||
if (candidate != null && !String.valueOf(candidate).isBlank()) {
|
||||
return String.valueOf(candidate);
|
||||
}
|
||||
}
|
||||
}
|
||||
return String.valueOf(value);
|
||||
}
|
||||
|
||||
private void insertInvalidAsin(CollectDataResultRowVo row) {
|
||||
if (row == null || row.getAsin() == null || row.getAsin().isBlank()) {
|
||||
return;
|
||||
}
|
||||
InvalidAsinDataEntity entity = new InvalidAsinDataEntity();
|
||||
entity.setDataValue(row.getAsin());
|
||||
entity.setBrand(row.getBrand());
|
||||
try {
|
||||
invalidAsinDataMapper.insert(entity);
|
||||
} catch (DuplicateKeyException ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
private void upsertResultItem(Long taskId, Long resultId, String scopeKey, CollectDataResultRowVo row) {
|
||||
String itemKey = "asin:" + row.getAsin();
|
||||
String scopeHash = hash(scopeKey);
|
||||
String payloadJson = writeJson(row, "采集结果明细序列化失败");
|
||||
String payloadHash = hash(payloadJson);
|
||||
TaskResultItemEntity existing = taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskResultItemEntity::getScopeHash, scopeHash)
|
||||
.eq(TaskResultItemEntity::getItemKey, itemKey)
|
||||
.last("limit 1"));
|
||||
if (existing != null && Objects.equals(existing.getPayloadHash(), payloadHash)) {
|
||||
return;
|
||||
}
|
||||
String storedPayload = transientPayloadStorageService.storeResultItemPayload(
|
||||
MODULE_TYPE, taskId, scopeHash, itemKey, payloadJson);
|
||||
requireRustfsPayload(storedPayload, "采集结果明细必须写入 RustFS");
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
if (existing == null) {
|
||||
TaskResultItemEntity entity = new TaskResultItemEntity();
|
||||
entity.setTaskId(taskId);
|
||||
entity.setModuleType(MODULE_TYPE);
|
||||
entity.setResultId(resultId);
|
||||
entity.setScopeKey(scopeKey);
|
||||
entity.setScopeHash(scopeHash);
|
||||
entity.setItemKey(itemKey);
|
||||
entity.setAsin(row.getAsin());
|
||||
entity.setStatus("ACCEPTED");
|
||||
entity.setPayloadJson(storedPayload);
|
||||
entity.setPayloadHash(payloadHash);
|
||||
entity.setCreatedAt(now);
|
||||
entity.setUpdatedAt(now);
|
||||
try {
|
||||
taskResultItemMapper.insert(entity);
|
||||
return;
|
||||
} catch (DuplicateKeyException ignored) {
|
||||
existing = taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskResultItemEntity::getScopeHash, scopeHash)
|
||||
.eq(TaskResultItemEntity::getItemKey, itemKey)
|
||||
.last("limit 1"));
|
||||
}
|
||||
}
|
||||
if (existing == null) {
|
||||
throw new BusinessException("保存采集结果明细失败");
|
||||
}
|
||||
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getPayloadJson(), storedPayload);
|
||||
taskResultItemMapper.update(null, new LambdaUpdateWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getId, existing.getId())
|
||||
.set(TaskResultItemEntity::getResultId, resultId)
|
||||
.set(TaskResultItemEntity::getScopeKey, scopeKey)
|
||||
.set(TaskResultItemEntity::getAsin, row.getAsin())
|
||||
.set(TaskResultItemEntity::getStatus, "ACCEPTED")
|
||||
.set(TaskResultItemEntity::getPayloadJson, storedPayload)
|
||||
.set(TaskResultItemEntity::getPayloadHash, payloadHash)
|
||||
.set(TaskResultItemEntity::getUpdatedAt, now));
|
||||
}
|
||||
|
||||
private void persistChunk(Long taskId,
|
||||
String scopeKey,
|
||||
String scopeHash,
|
||||
int chunkIndex,
|
||||
int chunkTotal,
|
||||
String storedPayload,
|
||||
String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setTaskId(taskId);
|
||||
chunk.setModuleType(MODULE_TYPE);
|
||||
chunk.setScopeKey(scopeKey);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
chunk.setChunkTotal(chunkTotal);
|
||||
chunk.setPayloadJson(storedPayload);
|
||||
chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson));
|
||||
chunk.setCreatedAt(LocalDateTime.now());
|
||||
chunk.setUpdatedAt(LocalDateTime.now());
|
||||
try {
|
||||
taskChunkMapper.insert(chunk);
|
||||
} catch (DuplicateKeyException ex) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||
log.info("[collect-data] duplicate chunk inserted concurrently taskId={} scope={} chunk={}",
|
||||
taskId, scopeKey, chunkIndex);
|
||||
}
|
||||
}
|
||||
|
||||
private void persistScope(Long taskId, String scopeKey, String scopeHash, int chunkTotal, CollectDataSubmitResultRequest request) {
|
||||
TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
|
||||
.last("limit 1"));
|
||||
if (scope == null) {
|
||||
scope = new TaskScopeStateEntity();
|
||||
scope.setTaskId(taskId);
|
||||
scope.setModuleType(MODULE_TYPE);
|
||||
scope.setScopeKey(scopeKey);
|
||||
scope.setScopeHash(scopeHash);
|
||||
scope.setCreatedAt(LocalDateTime.now());
|
||||
}
|
||||
scope.setChunkTotal(chunkTotal);
|
||||
scope.setReceivedChunkCount(countReceivedChunks(taskId, scopeHash));
|
||||
scope.setLastChunkAt(LocalDateTime.now());
|
||||
scope.setLastError(request.getError());
|
||||
scope.setCompleted(Boolean.TRUE.equals(request.getDone()) ? 1 : 0);
|
||||
scope.setStateJson("{\"phase\":\"RECEIVED\"}");
|
||||
scope.setUpdatedAt(LocalDateTime.now());
|
||||
if (scope.getId() == null) {
|
||||
taskScopeStateMapper.insert(scope);
|
||||
} else {
|
||||
taskScopeStateMapper.updateById(scope);
|
||||
}
|
||||
}
|
||||
|
||||
private void enqueueFinalWorkbook(FileTaskEntity task, FileResultEntity result, CollectDataStats stats) {
|
||||
String filename = buildResultFilename(task);
|
||||
result.setResultFilename(filename);
|
||||
result.setResultFileUrl(null);
|
||||
result.setResultFileSize(0L);
|
||||
result.setResultContentType(CONTENT_TYPE_XLSX);
|
||||
result.setRowCount(stats.finalRowCount);
|
||||
result.setErrorMessage(null);
|
||||
fileResultMapper.updateById(result);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(), "task:" + task.getId());
|
||||
}
|
||||
|
||||
private void markTaskFailed(FileTaskEntity task, FileResultEntity result, String error, CollectDataStats stats) {
|
||||
result.setSuccess(0);
|
||||
result.setErrorMessage(error);
|
||||
result.setRowCount(stats.finalRowCount);
|
||||
fileResultMapper.updateById(result);
|
||||
task.setStatus(STATUS_FAILED);
|
||||
task.setErrorMessage(error);
|
||||
task.setFailedFileCount(1);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
persistStats(task, stats);
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
|
||||
public void processResultFileJob(TaskFileJobEntity job) {
|
||||
if (job == null || job.getTaskId() == null) {
|
||||
throw new BusinessException("结果文件任务参数不完整");
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
|
||||
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
|
||||
throw new BusinessException("结果记录不存在");
|
||||
}
|
||||
List<CollectDataResultRowVo> rows = loadFinalRows(task.getId());
|
||||
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "collect-data-result", String.valueOf(task.getId())));
|
||||
String filename = buildResultFilename(task);
|
||||
File xlsx = FileUtil.file(workRoot, filename);
|
||||
try {
|
||||
excelAssemblyService.writeWorkbook(xlsx, rows);
|
||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||
result.setResultFilename(filename);
|
||||
result.setResultFileUrl(objectKey);
|
||||
result.setResultFileSize(xlsx.length());
|
||||
result.setResultContentType(CONTENT_TYPE_XLSX);
|
||||
result.setRowCount(rows.size());
|
||||
result.setSuccess(1);
|
||||
result.setErrorMessage(null);
|
||||
fileResultMapper.updateById(result);
|
||||
|
||||
CollectDataStats stats = loadStats(task);
|
||||
stats.finalRowCount = rows.size();
|
||||
persistStats(task, stats);
|
||||
task.setStatus(STATUS_SUCCESS);
|
||||
task.setSuccessFileCount(1);
|
||||
task.setFailedFileCount(0);
|
||||
task.setErrorMessage(null);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
} finally {
|
||||
FileUtil.del(xlsx);
|
||||
}
|
||||
}
|
||||
|
||||
private List<CollectDataResultRowVo> loadFinalRows(Long taskId) {
|
||||
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
|
||||
.orderByAsc(TaskResultItemEntity::getId));
|
||||
List<CollectDataResultRowVo> out = new ArrayList<>();
|
||||
if (rows == null) {
|
||||
return out;
|
||||
}
|
||||
for (TaskResultItemEntity row : rows) {
|
||||
try {
|
||||
String payloadJson = transientPayloadStorageService.resolvePayload(row.getPayloadJson(), "read collect data result item failed");
|
||||
CollectDataResultRowVo value = objectMapper.readValue(payloadJson, CollectDataResultRowVo.class);
|
||||
if (value != null) {
|
||||
out.add(value);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取采集结果明细失败");
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private int countFinalRows(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return 0;
|
||||
}
|
||||
Long count = taskResultItemMapper.selectCount(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE));
|
||||
return count == null ? 0 : count.intValue();
|
||||
}
|
||||
|
||||
private void ensureRustfsPayloadStorageEnabled() {
|
||||
if (!transientPayloadStorageService.isSharedWriteEnabled()) {
|
||||
throw new BusinessException("RustFS 未配置,采集结果回传暂不可接收");
|
||||
}
|
||||
}
|
||||
|
||||
private void requireRustfsPayload(String storedPayload, String message) {
|
||||
String pointer = transientPayloadStorageService.extractPointer(storedPayload);
|
||||
if (pointer != null && pointer.startsWith("rustfs:")) {
|
||||
return;
|
||||
}
|
||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||
throw new BusinessException(message);
|
||||
}
|
||||
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(MODULE_TYPE, taskId, TASK_LOCK_WAIT_MILLIS);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "任务正在处理,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
private int nextChunkIndex(Long taskId, String scopeHash) {
|
||||
TaskChunkEntity latest = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||
.orderByDesc(TaskChunkEntity::getChunkIndex)
|
||||
.last("limit 1"));
|
||||
return latest == null || latest.getChunkIndex() == null ? 1 : latest.getChunkIndex() + 1;
|
||||
}
|
||||
|
||||
private int countReceivedChunks(Long taskId, String scopeHash) {
|
||||
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskChunkEntity::getScopeHash, scopeHash));
|
||||
return count == null ? 0 : count.intValue();
|
||||
}
|
||||
|
||||
private CollectDataSubmitResultVo buildSubmitVo(FileTaskEntity task,
|
||||
FileResultEntity result,
|
||||
int chunkIndex,
|
||||
int chunkTotal,
|
||||
CollectDataSubmitResultRequest request,
|
||||
int currentChunkRows) {
|
||||
CollectDataStats stats = loadStats(task);
|
||||
stats.finalRowCount = countFinalRows(task == null ? null : task.getId());
|
||||
CollectDataSubmitResultVo vo = new CollectDataSubmitResultVo();
|
||||
vo.setTaskId(task == null ? null : task.getId());
|
||||
vo.setResultId(result == null ? null : result.getId());
|
||||
vo.setChunkIndex(chunkIndex);
|
||||
vo.setChunkTotal(chunkTotal);
|
||||
vo.setDone(request != null && Boolean.TRUE.equals(request.getDone()));
|
||||
vo.setReceivedRows(stats.receivedRows);
|
||||
vo.setCurrentChunkRows(currentChunkRows);
|
||||
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
|
||||
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
|
||||
vo.setBrandRejectedCount(stats.brandRejectedCount);
|
||||
vo.setBrandQueryFailedCount(stats.brandQueryFailedCount);
|
||||
vo.setFinalRowCount(stats.finalRowCount);
|
||||
vo.setTaskStatus(task == null ? null : task.getStatus());
|
||||
return vo;
|
||||
}
|
||||
|
||||
private CollectDataStats loadStats(FileTaskEntity task) {
|
||||
CollectDataStats stats = new CollectDataStats();
|
||||
if (task == null || task.getResultJson() == null || task.getResultJson().isBlank()) {
|
||||
return stats;
|
||||
}
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(task.getResultJson());
|
||||
stats.receivedRows = root.path("receivedRows").asInt(0);
|
||||
stats.currentChunkRows = root.path("currentChunkRows").asInt(0);
|
||||
stats.dedupeFilteredCount = root.path("dedupeFilteredCount").asInt(0);
|
||||
stats.invalidFilteredCount = root.path("invalidFilteredCount").asInt(0);
|
||||
stats.brandRejectedCount = root.path("brandRejectedCount").asInt(0);
|
||||
stats.brandQueryFailedCount = root.path("brandQueryFailedCount").asInt(0);
|
||||
stats.finalRowCount = root.path("finalRowCount").asInt(0);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] parse result stats failed taskId={} err={}", task.getId(), ex.getMessage());
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
private void persistStats(FileTaskEntity task, CollectDataStats stats) {
|
||||
if (task == null || stats == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put("receivedRows", stats.receivedRows);
|
||||
payload.put("currentChunkRows", stats.currentChunkRows);
|
||||
payload.put("dedupeFilteredCount", stats.dedupeFilteredCount);
|
||||
payload.put("invalidFilteredCount", stats.invalidFilteredCount);
|
||||
payload.put("brandRejectedCount", stats.brandRejectedCount);
|
||||
payload.put("brandQueryFailedCount", stats.brandQueryFailedCount);
|
||||
payload.put("finalRowCount", stats.finalRowCount);
|
||||
task.setResultJson(objectMapper.writeValueAsString(payload));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("保存采集统计失败");
|
||||
}
|
||||
}
|
||||
|
||||
private String writeJson(Object value, String message) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(value);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException(message);
|
||||
}
|
||||
}
|
||||
|
||||
private String hash(String value) {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to hash collect data payload", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private String normalizeAsin(String value) {
|
||||
return normalize(value).toUpperCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String normalizeBrand(String value) {
|
||||
return normalize(value).toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String buildResultFilename(FileTaskEntity task) {
|
||||
String base = task == null || task.getTaskNo() == null || task.getTaskNo().isBlank()
|
||||
? "collect-data-" + (task == null ? "result" : task.getId())
|
||||
: task.getTaskNo();
|
||||
return base.replaceAll("[\\\\/:*?\"<>|]", "_") + "-result.xlsx";
|
||||
}
|
||||
|
||||
private void deleteTransientTaskPayloads(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return;
|
||||
}
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.select(TaskChunkEntity::getPayloadJson)
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
if (chunks != null) {
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
|
||||
}
|
||||
}
|
||||
List<TaskResultItemEntity> items = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.select(TaskResultItemEntity::getPayloadJson)
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE));
|
||||
if (items != null) {
|
||||
for (TaskResultItemEntity item : items) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(item.getPayloadJson());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static class CollectDataStats {
|
||||
private int receivedRows;
|
||||
private int currentChunkRows;
|
||||
private int dedupeFilteredCount;
|
||||
private int invalidFilteredCount;
|
||||
private int brandRejectedCount;
|
||||
private int brandQueryFailedCount;
|
||||
private int finalRowCount;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteTask(Long taskId, Long userId) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
@@ -351,6 +1035,17 @@ public class CollectDataService {
|
||||
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, task.getId())
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
||||
deleteTransientTaskPayloads(task.getId());
|
||||
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, task.getId())
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, task.getId())
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, task.getId())
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE));
|
||||
taskFileJobService.deleteTaskJobs(task.getId(), MODULE_TYPE);
|
||||
fileTaskMapper.deleteById(task.getId());
|
||||
}
|
||||
|
||||
@@ -359,6 +1054,21 @@ public class CollectDataService {
|
||||
if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
List<TaskResultItemEntity> resultItems = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.select(TaskResultItemEntity::getPayloadJson)
|
||||
.eq(TaskResultItemEntity::getTaskId, row.getTaskId())
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskResultItemEntity::getResultId, row.getId()));
|
||||
if (resultItems != null) {
|
||||
for (TaskResultItemEntity item : resultItems) {
|
||||
transientPayloadStorageService.deletePayloadIfPresent(item.getPayloadJson());
|
||||
}
|
||||
}
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, row.getTaskId())
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskResultItemEntity::getResultId, row.getId()));
|
||||
taskFileJobService.deleteResultJobs(row.getTaskId(), MODULE_TYPE, row.getId());
|
||||
fileResultMapper.deleteById(resultId);
|
||||
}
|
||||
|
||||
@@ -503,21 +1213,41 @@ public class CollectDataService {
|
||||
vo.setTaskId(row.getTaskId());
|
||||
vo.setSourceFilename(row.getSourceFilename());
|
||||
vo.setResultFilename(row.getResultFilename());
|
||||
vo.setDownloadUrl(buildFreshDownloadUrl(row));
|
||||
vo.setRowCount(row.getRowCount());
|
||||
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
|
||||
vo.setError(row.getErrorMessage());
|
||||
vo.setCreatedAt(formatTime(row.getCreatedAt()));
|
||||
if (task != null) {
|
||||
CollectDataStats stats = loadStats(task);
|
||||
vo.setTaskNo(task.getTaskNo());
|
||||
vo.setTaskStatus(task.getStatus());
|
||||
vo.setStartedAt(formatTime(task.getCreatedAt()));
|
||||
vo.setFinishedAt(formatTime(task.getFinishedAt()));
|
||||
vo.setTaskType(extractTaskTypeFromRequest(task));
|
||||
vo.setFilters(extractFiltersFromRequest(task));
|
||||
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
|
||||
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
|
||||
vo.setBrandRejectedCount(stats.brandRejectedCount);
|
||||
vo.setFinalRowCount(stats.finalRowCount);
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
private String buildFreshDownloadUrl(FileResultEntity row) {
|
||||
String resultFileUrl = row == null ? null : row.getResultFileUrl();
|
||||
if (resultFileUrl == null || resultFileUrl.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return ossStorageService.generateFreshDownloadUrl(resultFileUrl);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] generate fresh download url failed resultId={} err={}",
|
||||
row.getId(), ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private CollectDataTaskSummaryVo toTaskSummary(FileTaskEntity task) {
|
||||
CollectDataTaskSummaryVo vo = new CollectDataTaskSummaryVo();
|
||||
vo.setId(task.getId());
|
||||
|
||||
Reference in New Issue
Block a user