super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

提交更改

Browse Source
This commit is contained in:
super
2026-06-05 21:25:03 +08:00
parent 95ccb00d25
commit 8ab647c419
39 changed files with 1390 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend-java/pom.xml
View File

@@ -171,6 +171,13 @@
</annotationProcessorPaths>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>-XX:+EnableDynamicAgentLoading -Xshare:off</argLine>
</configuration>
</plugin>
</plugins>
</build>
</project>

3
backend-java/src/main/java/com/nanri/aiimage/config/TaskOperationLockConfig.java
View File

@@ -64,6 +64,9 @@ public class TaskOperationLockConfig implements WebMvcConfigurer {
return true;
}
String uri = request.getRequestURI();
if ("POST".equals(request.getMethod()) && TASK_RESULT_PATH.matcher(uri == null ? "" : uri).matches()) {
return true;
}
Matcher matcher = TASK_PATH.matcher(uri == null ? "" : uri);
if (!matcher.matches()) {
return true;

13
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClient.java
View File

@@ -587,9 +587,11 @@ public class AppearancePatentCozeClient {
}
if (isNoTitleRisk(result)) {
row.setTitleRisk(NO_INFRINGEMENT);
} else {
} else if (shouldCheckBrand(result)) {
BrandCheckClient.BrandCheckBatchResult brandCheck = brandCheckClient.checkTitleText(result.title(), "Terms");
row.setTitleRisk(buildTitleRisk(result.title(), brandCheck));
} else {
row.setTitleRisk("");
}
row.setAppearanceRisk(result.appearance());
row.setPatentRisk(result.patent());
@@ -603,8 +605,13 @@ public class AppearancePatentCozeClient {
private boolean isNoTitleRisk(CozeResult result) {
return result != null
&& "无".equals(normalize(result.title()))
&& "无".equals(normalize(result.titleReason()));
&& "无".equals(normalize(result.title()));
}
private boolean shouldCheckBrand(CozeResult result) {
return result != null
&& !normalize(result.title()).isBlank()
&& !normalize(result.appearance()).isBlank();
}
String buildTitleRisk(String cozeTitle, BrandCheckClient.BrandCheckBatchResult brandCheck) {

2
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultRowDto.java
View File

@@ -83,7 +83,7 @@ public class AppearancePatentResultRowDto {
private String patentReason;
@JsonAlias({"score", "Score", "评分"})
@Schema(description = "Coze 回流的评分（外观维度），透传到最终 xlsx 第一张 sheet 的“评分”列。", accessMode = Schema.AccessMode.READ_ONLY)
@Schema(description = "Coze 回流的评分（外观维度），仅保留回流值，不写入最终 xlsx。", accessMode = Schema.AccessMode.READ_ONLY)
private String score;
/**

6
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParsedRowVo.java
View File

@@ -3,6 +3,9 @@ package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.LinkedHashMap;
import java.util.Map;
@Data
@Schema(description = "外观专利检测解析出的单行数据")
public class AppearancePatentParsedRowVo {
@@ -44,4 +47,7 @@ public class AppearancePatentParsedRowVo {
@Schema(description = "商品标题。", example = "Women Floral Dress Summer Casual")
private String title;
@Schema(description = "该 Excel 行的原始列值映射。最终生成 xlsx 时可从这里读取卖家名称、品牌等扩展字段。", example = "{\"id\":\"2_1\",\"asin\":\"B0CJ8SNXXV\",\"国家\":\"英国\"}")
private Map<String, String> values = new LinkedHashMap<>();
}

109
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java
View File

@@ -125,12 +125,13 @@ public class AppearancePatentTaskService {
"id",
"asin",
"国家",
"卖家名称",
"品牌",
"价格",
"标题维度（商标）",
"外观维度（外观设计专利）",
"结论",
"状态",
"评分"
"状态"
);
private final LocalFileStorageService localFileStorageService;
@@ -420,13 +421,7 @@ public class AppearancePatentTaskService {
}
public void submitResult(Long taskId, AppearancePatentSubmitResultRequest request) {
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS);
if (lockHandle == null) {
throw new BusinessException(40902, "任务正在处理上一批结果，请稍后重试");
}
try (lockHandle) {
submitResultLocked(taskId, request);
}
submitResultLocked(taskId, request);
}
private void submitResultLocked(Long taskId, AppearancePatentSubmitResultRequest request) {
@@ -1328,7 +1323,8 @@ public class AppearancePatentTaskService {
}
}
}
boolean failed = finalError != null && !finalError.isBlank();
boolean failedByConclusion = hasInfringingPersistedConclusion(task.getId());
boolean failed = finalError != null && !finalError.isBlank() || failedByConclusion;
boolean waitingForAssemble = !failed && assembleWorkbook && !shouldAssembleSynchronously();
task.setStatus(waitingForAssemble ? STATUS_RUNNING : (failed ? STATUS_FAILED : STATUS_SUCCESS));
task.setSuccessFileCount(failed ? 0 : 1);
@@ -2407,7 +2403,8 @@ public class AppearancePatentTaskService {
saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits - 1, "正在上传结果文件");
fileResultMapper.updateById(result);
boolean taskAlreadyFailed = STATUS_FAILED.equals(task.getStatus())
|| (task.getErrorMessage() != null && !task.getErrorMessage().isBlank());
|| (task.getErrorMessage() != null && !task.getErrorMessage().isBlank())
|| hasInfringingPersistedConclusion(task.getId());
String existingError = task.getErrorMessage();
task.setStatus(taskAlreadyFailed ? STATUS_FAILED : STATUS_SUCCESS);
task.setSuccessFileCount(taskAlreadyFailed ? 0 : 1);
@@ -3300,9 +3297,9 @@ public class AppearancePatentTaskService {
headerStyle.setFont(font);
List<String> resultHeaders = new ArrayList<>(RESULT_HEADERS);
resultHeaders.add(4, "标题");
resultHeaders.add(5, "图片链接");
resultHeaders.add(6, "sku");
resultHeaders.add(6, "标题");
resultHeaders.add(7, "图片链接");
resultHeaders.add(8, "sku");
Row header = sheet.createRow(0);
for (int i = 0; i < resultHeaders.size(); i++) {
Cell cell = header.createCell(i);
@@ -3314,24 +3311,21 @@ public class AppearancePatentTaskService {
List<AppearancePatentParsedRowVo> rowsToWrite = receivedRows == null ? List.of() : receivedRows;
for (AppearancePatentParsedRowVo parsedRow : rowsToWrite) {
AppearancePatentResultRowDto resultRow = findResultRow(parsedRow, resultMap);
String missingReason = "";
if (resultRow == null) {
missingReason = hasPromptFields(parsedRow) ? "未匹配到检测结果" : "未送检：缺少标题或图片";
}
Row row = sheet.createRow(rowIndex++);
int col = 0;
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name", "seller-name", "sellername", "store name", "shop name"));
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "品牌", "brand"));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getPrice(), ""));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getTitle(), "") : firstNonBlank(resultRow.getTitle(), parsedRow.getTitle()));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getUrl(), "") : firstNonBlank(resultRow.getUrl(), parsedRow.getUrl()));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getSku(), "") : firstNonBlank(resultRow.getSku(), parsedRow.getSku()));
row.createCell(col++).setCellValue(resultRow == null ? missingReason : userFacingCozeCellValue(resultRow, resultRow.getTitleRisk()));
row.createCell(col++).setCellValue(resultRow == null ? missingReason : userFacingCozeCellValue(resultRow, resultRow.getAppearanceRisk()));
row.createCell(col++).setCellValue(resultRow == null ? "未送检" : userFacingConclusion(resultRow));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingStatus(resultRow));
row.createCell(col).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getScore(), ""));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getTitleRisk()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getAppearanceRisk()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingConclusion(resultRow));
row.createCell(col).setCellValue(resultRow == null ? "" : userFacingStatus(resultRow));
}
writeReasonSheet(workbook, headerStyle, rowsToWrite, resultMap);
workbook.write(fos);
@@ -3519,6 +3513,7 @@ public class AppearancePatentTaskService {
vo.setSku(skuCol >= 0 ? cell(row, skuCol, formatter) : "");
vo.setUrl(urlCol >= 0 ? cell(row, urlCol, formatter) : "");
vo.setTitle(titleCol >= 0 ? cell(row, titleCol, formatter) : "");
vo.setValues(readRowValues(row, headers, formatter));
parsedRows.add(new ParsedAppearanceRow(vo, statusCol >= 0 ? cell(row, statusCol, formatter) : ""));
}
List<AppearancePatentParsedRowVo> allRows = parsedRows.stream()
@@ -3590,6 +3585,14 @@ public class AppearancePatentTaskService {
return headers;
}
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter) {
Map<String, String> values = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
values.put(headers.get(i), cell(row, i, formatter));
}
return values;
}
private boolean isAppearanceResultWorkbook(List<String> headers) {
return hasHeader(headers, "状态")
&& (hasHeader(headers, "结论")
@@ -3790,7 +3793,8 @@ public class AppearancePatentTaskService {
vo.setDownloadUrl(null);
attachFileJobState(vo, row, job);
vo.setTaskStatus(task == null ? null : task.getStatus());
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
vo.setSuccess(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank()
|| row.getSuccess() != null && row.getSuccess() == 1);
vo.setError(row.getErrorMessage());
vo.setRowCount(row.getRowCount());
vo.setCreatedAt(fmt(row.getCreatedAt()));
@@ -4143,6 +4147,51 @@ public class AppearancePatentTaskService {
return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
}
private String readValueByHeader(AppearancePatentParsedRowVo row, String... candidates) {
if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) {
return "";
}
for (Map.Entry<String, String> entry : row.getValues().entrySet()) {
String header = normalize(entry.getKey()).toLowerCase(Locale.ROOT);
for (String candidate : candidates) {
String expected = normalize(candidate).toLowerCase(Locale.ROOT);
if (!expected.isBlank() && header.contains(expected)) {
return entry.getValue() == null ? "" : entry.getValue();
}
}
}
return "";
}
private boolean hasInfringingPersistedConclusion(Long taskId) {
if (taskId == null) {
return false;
}
return loadPersistedResultRows(taskId).values().stream()
.map(AppearancePatentResultRowDto::getConclusion)
.anyMatch(this::isInfringingConclusion);
}
private boolean isInfringingConclusion(String value) {
String normalized = normalize(value);
return normalized.contains("侵权")
&& !isNoInfringementConclusion(normalized);
}
private boolean isNoInfringementConclusion(String value) {
String normalized = normalize(value);
if (normalized.isBlank()) {
return false;
}
return normalized.contains("无侵权")
|| normalized.contains("未侵权")
|| normalized.contains("不侵权")
|| normalized.contains("未发现") && normalized.contains("侵权")
|| normalized.contains("未见") && normalized.contains("侵权")
|| normalized.contains("不存在") && normalized.contains("侵权")
|| normalized.contains("无明显") && normalized.contains("侵权");
}
private String userFacingCozeCellValue(AppearancePatentResultRowDto row, String value) {
String normalizedValue = normalize(value);
if (!normalizedValue.isBlank() && !isTechnicalCozeFailure(normalizedValue)) {
@@ -4180,11 +4229,13 @@ public class AppearancePatentTaskService {
if (row == null) {
return "";
}
String original = firstNonBlank(row.getStatus(), "");
if (!row.isFailureSyntheticStatus()) {
return original;
}
return firstNonBlank(row.getError(), "");
String conclusion = userFacingConclusion(row);
return resolveResultStatus(conclusion);
}
static String resolveResultStatus(String conclusion) {
String normalized = conclusion == null ? "" : conclusion.trim();
return normalized.isBlank() ? "\u5931\u8d25" : "\u6210\u529f";
}
private boolean isTechnicalCozeFailure(String value) {

11
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/controller/CollectDataController.java
View File

@@ -3,12 +3,14 @@ package com.nanri.aiimage.modules.collectdata.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataParseRequest;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitResultRequest;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataTaskBatchRequest;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemsPageVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataParseVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
import io.swagger.v3.oas.annotations.Operation;
@@ -65,6 +67,15 @@ public class CollectDataController {
return ApiResponse.success(service.getItemsPage(taskId, userId, page, pageSize));
}
@PostMapping("/tasks/{taskId}/result")
@Operation(summary = "Python 分片回传采集结果", description = "接收任意条数的 Python 回传行；后端按 ASIN 筛库、调用品牌检测并保存最终结果。")
public ApiResponse<CollectDataSubmitResultVo> submitResult(
@Parameter(description = "采集任务 ID", required = true, example = "9001")
@PathVariable Long taskId,
@Valid @RequestBody CollectDataSubmitResultRequest request) {
return ApiResponse.success(service.submitResult(taskId, request));
}
@GetMapping("/dashboard")
@Operation(summary = "查询采集任务总览")
public ApiResponse<CollectDataDashboardVo> dashboard(

41
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/dto/CollectDataSubmitResultRequest.java Normal file
View File

@@ -0,0 +1,41 @@
package com.nanri.aiimage.modules.collectdata.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
@Schema(description = "采集数据 Python 分片回传请求")
public class CollectDataSubmitResultRequest {
@JsonProperty("submission_id")
@JsonAlias("submissionId")
@Schema(description = "分片提交批次标识；同一任务可固定一个 submission_id", example = "collect-11877")
private String submissionId;
@JsonProperty("chunk_index")
@JsonAlias("chunkIndex")
@Schema(description = "分片序号，从 1 开始；未传时后端按当前任务顺序递增", example = "1")
private Integer chunkIndex;
@JsonProperty("chunk_total")
@JsonAlias("chunkTotal")
@Schema(description = "总分片数；未知可不传，done=true 时以后端已收数量兜底", example = "10")
private Integer chunkTotal;
@Schema(description = "是否为最后一次回传", example = "false")
private Boolean done;
@Schema(description = "Python 端错误信息；传入时任务会标记失败")
private String error;
@JsonAlias({"rows", "data", "items"})
@Schema(description = "本分片回传数据行")
private List<CollectDataSubmitRowDto> items = new ArrayList<>();
}

32
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/dto/CollectDataSubmitRowDto.java Normal file
View File

@@ -0,0 +1,32 @@
package com.nanri.aiimage.modules.collectdata.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "采集数据 Python 回传行")
public class CollectDataSubmitRowDto {
@JsonAlias({"品牌", "brand_name"})
@Schema(description = "品牌", example = "Nike")
private String brand;
@JsonAlias({"ASIN", "asin"})
@Schema(description = "ASIN", example = "B0XXXXXXX")
private String asin;
@JsonAlias({"价格", "price"})
@Schema(description = "价格", example = "19.99")
private String price;
@JsonProperty("seller_name")
@JsonAlias({"卖家名称", "sellerName", "seller"})
@Schema(description = "卖家名称", example = "Demo Seller")
private String sellerName;
@JsonAlias({"关键词", "key_word"})
@Schema(description = "关键词", example = "phone case")
private String keyword;
}

12
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataHistoryItemVo.java
View File

@@ -37,6 +37,18 @@ public class CollectDataHistoryItemVo {
@Schema(description = "落库行数")
private Integer rowCount;
@Schema(description = "被数据去重总数据过滤的行数")
private Integer dedupeFilteredCount;
@Schema(description = "被不符合 ASIN 数据库过滤的行数")
private Integer invalidFilteredCount;
@Schema(description = "品牌检测不符合的行数")
private Integer brandRejectedCount;
@Schema(description = "最终结果行数")
private Integer finalRowCount;
@Schema(description = "记录创建时间")
private String createdAt;

12
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataResultRowVo.java Normal file
View File

@@ -0,0 +1,12 @@
package com.nanri.aiimage.modules.collectdata.model.vo;
import lombok.Data;
@Data
public class CollectDataResultRowVo {
private String brand;
private String asin;
private String price;
private String sellerName;
private String keyword;
}

22
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataSubmitResultVo.java Normal file
View File

@@ -0,0 +1,22 @@
package com.nanri.aiimage.modules.collectdata.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "采集数据回传处理结果")
public class CollectDataSubmitResultVo {
private Long taskId;
private Long resultId;
private Integer chunkIndex;
private Integer chunkTotal;
private Boolean done;
private Integer receivedRows;
private Integer currentChunkRows;
private Integer dedupeFilteredCount;
private Integer invalidFilteredCount;
private Integer brandRejectedCount;
private Integer brandQueryFailedCount;
private Integer finalRowCount;
private String taskStatus;
}

61
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataExcelAssemblyService.java Normal file
View File

@@ -0,0 +1,61 @@
package com.nanri.aiimage.modules.collectdata.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileOutputStream;
import java.util.List;
@Service
@Slf4j
public class CollectDataExcelAssemblyService {
private static final String[] HEADER = {"品牌", "ASIN", "价格", "卖家名称", "关键词"};
public void writeWorkbook(File outputXlsx, List<CollectDataResultRowVo> items) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream outputStream = new FileOutputStream(outputXlsx)) {
Sheet sheet = workbook.createSheet("采集数据结果");
Row headerRow = sheet.createRow(0);
for (int i = 0; i < HEADER.length; i++) {
headerRow.createCell(i).setCellValue(HEADER[i]);
sheet.setColumnWidth(i, (i == 3 ? 24 : 18) * 256);
}
int rowIndex = 1;
if (items != null) {
for (CollectDataResultRowVo item : items) {
if (item == null) {
continue;
}
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(safe(item.getBrand()));
row.createCell(1).setCellValue(safe(item.getAsin()));
row.createCell(2).setCellValue(safe(item.getPrice()));
row.createCell(3).setCellValue(safe(item.getSellerName()));
row.createCell(4).setCellValue(safe(item.getKeyword()));
}
}
workbook.write(outputStream);
} catch (Exception ex) {
log.warn("[collect-data] write workbook failed: {}", ex.getMessage());
throw new BusinessException("生成采集数据 Excel 失败: " + ex.getMessage());
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}
private String safe(String value) {
return value == null ? "" : value;
}
}

730
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java
View File

@@ -1,17 +1,23 @@
package com.nanri.aiimage.modules.collectdata.service;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.crypto.digest.DigestUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataFiltersDto;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataParseRequest;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSourceFileDto;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitResultRequest;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitRowDto;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataCountryPrefEntity;
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo;
@@ -21,14 +27,30 @@ import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemsPageVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataParseVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo;
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo;
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
@@ -40,18 +62,23 @@ import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.dao.DuplicateKeyException;
import java.io.File;
import java.io.FileInputStream;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Pattern;
@Service
@@ -74,6 +101,9 @@ public class CollectDataService {
private static final String DEFAULT_TASK_TYPE = "collect-data";
private static final int ITEM_INSERT_BATCH_SIZE = 500;
private static final int BRAND_CHECK_BATCH_SIZE = 10;
private static final long TASK_LOCK_WAIT_MILLIS = 5000L;
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final List<String> KEYWORD_HEADER_ALIASES = List.of("关键词", "keyword", "key word");
private static final List<String> STATUS_HEADER_ALIASES = List.of("状态", "status");
@@ -86,6 +116,17 @@ public class CollectDataService {
private final FileResultMapper fileResultMapper;
private final CollectDataItemMapper collectDataItemMapper;
private final CollectDataCountryPrefMapper collectDataCountryPrefMapper;
private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final InvalidAsinDataMapper invalidAsinDataMapper;
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final TaskResultItemMapper taskResultItemMapper;
private final TaskDistributedLockService taskDistributedLockService;
private final TaskFileJobService taskFileJobService;
private final TransientPayloadStorageService transientPayloadStorageService;
private final BrandCheckClient brandCheckClient;
private final CollectDataExcelAssemblyService excelAssemblyService;
private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper;
private final TransactionTemplate transactionTemplate;
@@ -343,6 +384,649 @@ public class CollectDataService {
return vo;
}
public CollectDataSubmitResultVo submitResult(Long taskId, CollectDataSubmitResultRequest request) {
if (request == null) {
throw new BusinessException("request is empty");
}
ensureRustfsPayloadStorageEnabled();
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
}
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
throw new BusinessException(40901, "任务已结束，拒绝重复提交");
}
if (!STATUS_RUNNING.equals(task.getStatus())) {
task.setStatus(STATUS_RUNNING);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(null);
fileTaskMapper.updateById(task);
}
FileResultEntity result = ensureTaskResult(task);
String scopeKey = firstNonBlank(request.getSubmissionId(), "task:" + taskId);
String scopeHash = DigestUtil.sha256Hex(scopeKey);
int chunkIndex = request.getChunkIndex() == null || request.getChunkIndex() <= 0
? nextChunkIndex(taskId, scopeHash)
: request.getChunkIndex();
int chunkTotal = request.getChunkTotal() == null || request.getChunkTotal() <= 0
? Math.max(chunkIndex, 1)
: request.getChunkTotal();
TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
.last("limit 1"));
if (existing != null) {
log.info("[collect-data] duplicate result chunk ignored taskId={} scope={} chunk={}",
taskId, scopeKey, chunkIndex);
return buildSubmitVo(task, result, chunkIndex, chunkTotal, request, 0);
}
List<CollectDataResultRowVo> rows = normalizeSubmitRows(request.getItems());
CollectDataStats stats = loadStats(task);
stats.receivedRows += rows.size();
stats.currentChunkRows = rows.size();
List<CollectDataResultRowVo> candidates = filterByExistingAsin(rows, stats);
List<CollectDataResultRowVo> accepted = filterByBrandCheck(candidates, stats);
for (CollectDataResultRowVo row : accepted) {
upsertResultItem(task.getId(), result.getId(), scopeKey, row);
}
String payloadJson = writeJson(rows, "采集结果序列化失败");
String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(
MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
requireRustfsPayload(storedPayload, "采集结果分片必须写入 RustFS");
persistChunk(taskId, scopeKey, scopeHash, chunkIndex, chunkTotal, storedPayload, payloadJson);
persistScope(taskId, scopeKey, scopeHash, chunkTotal, request);
stats.finalRowCount = countFinalRows(taskId);
persistStats(task, stats);
if (request.getError() != null && !request.getError().isBlank()) {
markTaskFailed(task, result, request.getError(), stats);
} else if (Boolean.TRUE.equals(request.getDone())) {
enqueueFinalWorkbook(task, result, stats);
} else {
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
}
log.info("[collect-data] submit result taskId={} chunk={}/{} rows={} accepted={} dedupeFiltered={} invalidFiltered={} brandRejected={} finalRows={} done={}",
taskId, chunkIndex, chunkTotal, rows.size(), accepted.size(), stats.dedupeFilteredCount,
stats.invalidFilteredCount, stats.brandRejectedCount, stats.finalRowCount, request.getDone());
return buildSubmitVo(fileTaskMapper.selectById(taskId), result, chunkIndex, chunkTotal, request, rows.size());
}
}
private FileResultEntity ensureTaskResult(FileTaskEntity task) {
FileResultEntity result = fileResultMapper.selectOne(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, task.getId())
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.orderByAsc(FileResultEntity::getId)
.last("limit 1"));
if (result != null) {
return result;
}
result = new FileResultEntity();
result.setTaskId(task.getId());
result.setModuleType(MODULE_TYPE);
result.setSourceFilename("");
result.setRowCount(0);
result.setUserId(task.getUserId());
result.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(result);
return result;
}
private List<CollectDataResultRowVo> normalizeSubmitRows(List<CollectDataSubmitRowDto> rawRows) {
List<CollectDataResultRowVo> rows = new ArrayList<>();
if (rawRows == null) {
return rows;
}
for (CollectDataSubmitRowDto raw : rawRows) {
if (raw == null) {
continue;
}
String asin = normalizeAsin(raw.getAsin());
if (asin.isBlank()) {
continue;
}
CollectDataResultRowVo row = new CollectDataResultRowVo();
row.setBrand(normalize(raw.getBrand()));
row.setAsin(asin);
row.setPrice(normalize(raw.getPrice()));
row.setSellerName(normalize(raw.getSellerName()));
row.setKeyword(normalize(raw.getKeyword()));
rows.add(row);
}
return rows;
}
private List<CollectDataResultRowVo> filterByExistingAsin(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
List<String> asins = rows.stream()
.map(CollectDataResultRowVo::getAsin)
.filter(value -> value != null && !value.isBlank())
.distinct()
.toList();
Set<String> dedupeValues = new HashSet<>();
if (!asins.isEmpty()) {
List<String> existingDedupeValues = dedupeTotalDataMapper.selectExistingDataValues(asins);
if (existingDedupeValues != null) {
dedupeValues.addAll(existingDedupeValues.stream()
.map(this::normalizeAsin)
.toList());
}
}
Set<String> invalidValues = new HashSet<>();
if (!asins.isEmpty()) {
List<InvalidAsinDataEntity> invalidRows = invalidAsinDataMapper.selectList(new LambdaQueryWrapper<InvalidAsinDataEntity>()
.in(InvalidAsinDataEntity::getDataValue, asins));
if (invalidRows != null) {
for (InvalidAsinDataEntity row : invalidRows) {
invalidValues.add(normalizeAsin(row.getDataValue()));
}
}
}
List<CollectDataResultRowVo> out = new ArrayList<>();
for (CollectDataResultRowVo row : rows) {
String asin = row.getAsin();
if (dedupeValues.contains(asin)) {
stats.dedupeFilteredCount++;
continue;
}
if (invalidValues.contains(asin)) {
stats.invalidFilteredCount++;
continue;
}
out.add(row);
}
return out;
}
private List<CollectDataResultRowVo> filterByBrandCheck(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
List<CollectDataResultRowVo> accepted = new ArrayList<>();
for (int start = 0; start < rows.size(); start += BRAND_CHECK_BATCH_SIZE) {
int end = Math.min(start + BRAND_CHECK_BATCH_SIZE, rows.size());
List<CollectDataResultRowVo> batch = rows.subList(start, end);
List<String> brands = batch.stream()
.map(CollectDataResultRowVo::getBrand)
.filter(value -> value != null && !value.isBlank())
.distinct()
.toList();
BrandCheckClient.BrandCheckBatchResult check = brandCheckClient.checkAll(brands, "Terms");
Set<String> failedBrands = normalizeObjectSet(check == null ? null : check.faildData());
Set<String> queryFailedBrands = normalizeObjectSet(check == null ? null : check.queryFaildData());
for (CollectDataResultRowVo row : batch) {
String brand = normalizeBrand(row.getBrand());
if (brand.isBlank()) {
stats.brandRejectedCount++;
insertInvalidAsin(row);
continue;
}
if (failedBrands.contains(brand)) {
stats.brandRejectedCount++;
insertInvalidAsin(row);
continue;
}
if (queryFailedBrands.contains(brand)) {
stats.brandQueryFailedCount++;
insertInvalidAsin(row);
continue;
}
accepted.add(row);
}
}
return accepted;
}
private Set<String> normalizeObjectSet(List<Object> values) {
Set<String> out = new HashSet<>();
if (values == null) {
return out;
}
for (Object value : values) {
String normalized = normalizeBrand(extractBrandValue(value));
if (!normalized.isBlank()) {
out.add(normalized);
}
}
return out;
}
private String extractBrandValue(Object value) {
if (value == null) {
return "";
}
if (value instanceof Map<?, ?> map) {
for (String key : List.of("brand", "brandName", "brand_name", "name", "value", "data_value")) {
Object candidate = map.get(key);
if (candidate != null && !String.valueOf(candidate).isBlank()) {
return String.valueOf(candidate);
}
}
}
return String.valueOf(value);
}
private void insertInvalidAsin(CollectDataResultRowVo row) {
if (row == null || row.getAsin() == null || row.getAsin().isBlank()) {
return;
}
InvalidAsinDataEntity entity = new InvalidAsinDataEntity();
entity.setDataValue(row.getAsin());
entity.setBrand(row.getBrand());
try {
invalidAsinDataMapper.insert(entity);
} catch (DuplicateKeyException ignored) {
}
}
private void upsertResultItem(Long taskId, Long resultId, String scopeKey, CollectDataResultRowVo row) {
String itemKey = "asin:" + row.getAsin();
String scopeHash = hash(scopeKey);
String payloadJson = writeJson(row, "采集结果明细序列化失败");
String payloadHash = hash(payloadJson);
TaskResultItemEntity existing = taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
.eq(TaskResultItemEntity::getScopeHash, scopeHash)
.eq(TaskResultItemEntity::getItemKey, itemKey)
.last("limit 1"));
if (existing != null && Objects.equals(existing.getPayloadHash(), payloadHash)) {
return;
}
String storedPayload = transientPayloadStorageService.storeResultItemPayload(
MODULE_TYPE, taskId, scopeHash, itemKey, payloadJson);
requireRustfsPayload(storedPayload, "采集结果明细必须写入 RustFS");
LocalDateTime now = LocalDateTime.now();
if (existing == null) {
TaskResultItemEntity entity = new TaskResultItemEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
entity.setResultId(resultId);
entity.setScopeKey(scopeKey);
entity.setScopeHash(scopeHash);
entity.setItemKey(itemKey);
entity.setAsin(row.getAsin());
entity.setStatus("ACCEPTED");
entity.setPayloadJson(storedPayload);
entity.setPayloadHash(payloadHash);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskResultItemMapper.insert(entity);
return;
} catch (DuplicateKeyException ignored) {
existing = taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
.eq(TaskResultItemEntity::getScopeHash, scopeHash)
.eq(TaskResultItemEntity::getItemKey, itemKey)
.last("limit 1"));
}
}
if (existing == null) {
throw new BusinessException("保存采集结果明细失败");
}
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getPayloadJson(), storedPayload);
taskResultItemMapper.update(null, new LambdaUpdateWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getId, existing.getId())
.set(TaskResultItemEntity::getResultId, resultId)
.set(TaskResultItemEntity::getScopeKey, scopeKey)
.set(TaskResultItemEntity::getAsin, row.getAsin())
.set(TaskResultItemEntity::getStatus, "ACCEPTED")
.set(TaskResultItemEntity::getPayloadJson, storedPayload)
.set(TaskResultItemEntity::getPayloadHash, payloadHash)
.set(TaskResultItemEntity::getUpdatedAt, now));
}
private void persistChunk(Long taskId,
String scopeKey,
String scopeHash,
int chunkIndex,
int chunkTotal,
String storedPayload,
String payloadJson) {
TaskChunkEntity chunk = new TaskChunkEntity();
chunk.setTaskId(taskId);
chunk.setModuleType(MODULE_TYPE);
chunk.setScopeKey(scopeKey);
chunk.setScopeHash(scopeHash);
chunk.setChunkIndex(chunkIndex);
chunk.setChunkTotal(chunkTotal);
chunk.setPayloadJson(storedPayload);
chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson));
chunk.setCreatedAt(LocalDateTime.now());
chunk.setUpdatedAt(LocalDateTime.now());
try {
taskChunkMapper.insert(chunk);
} catch (DuplicateKeyException ex) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
log.info("[collect-data] duplicate chunk inserted concurrently taskId={} scope={} chunk={}",
taskId, scopeKey, chunkIndex);
}
}
private void persistScope(Long taskId, String scopeKey, String scopeHash, int chunkTotal, CollectDataSubmitResultRequest request) {
TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.last("limit 1"));
if (scope == null) {
scope = new TaskScopeStateEntity();
scope.setTaskId(taskId);
scope.setModuleType(MODULE_TYPE);
scope.setScopeKey(scopeKey);
scope.setScopeHash(scopeHash);
scope.setCreatedAt(LocalDateTime.now());
}
scope.setChunkTotal(chunkTotal);
scope.setReceivedChunkCount(countReceivedChunks(taskId, scopeHash));
scope.setLastChunkAt(LocalDateTime.now());
scope.setLastError(request.getError());
scope.setCompleted(Boolean.TRUE.equals(request.getDone()) ? 1 : 0);
scope.setStateJson("{\"phase\":\"RECEIVED\"}");
scope.setUpdatedAt(LocalDateTime.now());
if (scope.getId() == null) {
taskScopeStateMapper.insert(scope);
} else {
taskScopeStateMapper.updateById(scope);
}
}
private void enqueueFinalWorkbook(FileTaskEntity task, FileResultEntity result, CollectDataStats stats) {
String filename = buildResultFilename(task);
result.setResultFilename(filename);
result.setResultFileUrl(null);
result.setResultFileSize(0L);
result.setResultContentType(CONTENT_TYPE_XLSX);
result.setRowCount(stats.finalRowCount);
result.setErrorMessage(null);
fileResultMapper.updateById(result);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(), "task:" + task.getId());
}
private void markTaskFailed(FileTaskEntity task, FileResultEntity result, String error, CollectDataStats stats) {
result.setSuccess(0);
result.setErrorMessage(error);
result.setRowCount(stats.finalRowCount);
fileResultMapper.updateById(result);
task.setStatus(STATUS_FAILED);
task.setErrorMessage(error);
task.setFailedFileCount(1);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
persistStats(task, stats);
fileTaskMapper.updateById(task);
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
}
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
throw new BusinessException("结果记录不存在");
}
List<CollectDataResultRowVo> rows = loadFinalRows(task.getId());
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "collect-data-result", String.valueOf(task.getId())));
String filename = buildResultFilename(task);
File xlsx = FileUtil.file(workRoot, filename);
try {
excelAssemblyService.writeWorkbook(xlsx, rows);
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
result.setResultFilename(filename);
result.setResultFileUrl(objectKey);
result.setResultFileSize(xlsx.length());
result.setResultContentType(CONTENT_TYPE_XLSX);
result.setRowCount(rows.size());
result.setSuccess(1);
result.setErrorMessage(null);
fileResultMapper.updateById(result);
CollectDataStats stats = loadStats(task);
stats.finalRowCount = rows.size();
persistStats(task, stats);
task.setStatus(STATUS_SUCCESS);
task.setSuccessFileCount(1);
task.setFailedFileCount(0);
task.setErrorMessage(null);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
} finally {
FileUtil.del(xlsx);
}
}
private List<CollectDataResultRowVo> loadFinalRows(Long taskId) {
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskResultItemEntity::getId));
List<CollectDataResultRowVo> out = new ArrayList<>();
if (rows == null) {
return out;
}
for (TaskResultItemEntity row : rows) {
try {
String payloadJson = transientPayloadStorageService.resolvePayload(row.getPayloadJson(), "read collect data result item failed");
CollectDataResultRowVo value = objectMapper.readValue(payloadJson, CollectDataResultRowVo.class);
if (value != null) {
out.add(value);
}
} catch (Exception ex) {
throw new BusinessException("读取采集结果明细失败");
}
}
return out;
}
private int countFinalRows(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskResultItemMapper.selectCount(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE));
return count == null ? 0 : count.intValue();
}
private void ensureRustfsPayloadStorageEnabled() {
if (!transientPayloadStorageService.isSharedWriteEnabled()) {
throw new BusinessException("RustFS 未配置，采集结果回传暂不可接收");
}
}
private void requireRustfsPayload(String storedPayload, String message) {
String pointer = transientPayloadStorageService.extractPointer(storedPayload);
if (pointer != null && pointer.startsWith("rustfs:")) {
return;
}
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw new BusinessException(message);
}
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(MODULE_TYPE, taskId, TASK_LOCK_WAIT_MILLIS);
if (lockHandle == null) {
throw new BusinessException(40901, "任务正在处理，请稍后再试");
}
return lockHandle;
}
private int nextChunkIndex(Long taskId, String scopeHash) {
TaskChunkEntity latest = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.orderByDesc(TaskChunkEntity::getChunkIndex)
.last("limit 1"));
return latest == null || latest.getChunkIndex() == null ? 1 : latest.getChunkIndex() + 1;
}
private int countReceivedChunks(Long taskId, String scopeHash) {
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash));
return count == null ? 0 : count.intValue();
}
private CollectDataSubmitResultVo buildSubmitVo(FileTaskEntity task,
FileResultEntity result,
int chunkIndex,
int chunkTotal,
CollectDataSubmitResultRequest request,
int currentChunkRows) {
CollectDataStats stats = loadStats(task);
stats.finalRowCount = countFinalRows(task == null ? null : task.getId());
CollectDataSubmitResultVo vo = new CollectDataSubmitResultVo();
vo.setTaskId(task == null ? null : task.getId());
vo.setResultId(result == null ? null : result.getId());
vo.setChunkIndex(chunkIndex);
vo.setChunkTotal(chunkTotal);
vo.setDone(request != null && Boolean.TRUE.equals(request.getDone()));
vo.setReceivedRows(stats.receivedRows);
vo.setCurrentChunkRows(currentChunkRows);
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
vo.setBrandRejectedCount(stats.brandRejectedCount);
vo.setBrandQueryFailedCount(stats.brandQueryFailedCount);
vo.setFinalRowCount(stats.finalRowCount);
vo.setTaskStatus(task == null ? null : task.getStatus());
return vo;
}
private CollectDataStats loadStats(FileTaskEntity task) {
CollectDataStats stats = new CollectDataStats();
if (task == null || task.getResultJson() == null || task.getResultJson().isBlank()) {
return stats;
}
try {
JsonNode root = objectMapper.readTree(task.getResultJson());
stats.receivedRows = root.path("receivedRows").asInt(0);
stats.currentChunkRows = root.path("currentChunkRows").asInt(0);
stats.dedupeFilteredCount = root.path("dedupeFilteredCount").asInt(0);
stats.invalidFilteredCount = root.path("invalidFilteredCount").asInt(0);
stats.brandRejectedCount = root.path("brandRejectedCount").asInt(0);
stats.brandQueryFailedCount = root.path("brandQueryFailedCount").asInt(0);
stats.finalRowCount = root.path("finalRowCount").asInt(0);
} catch (Exception ex) {
log.warn("[collect-data] parse result stats failed taskId={} err={}", task.getId(), ex.getMessage());
}
return stats;
}
private void persistStats(FileTaskEntity task, CollectDataStats stats) {
if (task == null || stats == null) {
return;
}
try {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("receivedRows", stats.receivedRows);
payload.put("currentChunkRows", stats.currentChunkRows);
payload.put("dedupeFilteredCount", stats.dedupeFilteredCount);
payload.put("invalidFilteredCount", stats.invalidFilteredCount);
payload.put("brandRejectedCount", stats.brandRejectedCount);
payload.put("brandQueryFailedCount", stats.brandQueryFailedCount);
payload.put("finalRowCount", stats.finalRowCount);
task.setResultJson(objectMapper.writeValueAsString(payload));
} catch (Exception ex) {
throw new BusinessException("保存采集统计失败");
}
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException(message);
}
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new IllegalStateException("failed to hash collect data payload", ex);
}
}
private String normalizeAsin(String value) {
return normalize(value).toUpperCase(Locale.ROOT);
}
private String normalizeBrand(String value) {
return normalize(value).toLowerCase(Locale.ROOT);
}
private String buildResultFilename(FileTaskEntity task) {
String base = task == null || task.getTaskNo() == null || task.getTaskNo().isBlank()
? "collect-data-" + (task == null ? "result" : task.getId())
: task.getTaskNo();
return base.replaceAll("[\\\\/:*?\"<>|]", "_") + "-result.xlsx";
}
private void deleteTransientTaskPayloads(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
}
}
List<TaskResultItemEntity> items = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.select(TaskResultItemEntity::getPayloadJson)
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE));
if (items != null) {
for (TaskResultItemEntity item : items) {
transientPayloadStorageService.deletePayloadIfPresent(item.getPayloadJson());
}
}
}
private static class CollectDataStats {
private int receivedRows;
private int currentChunkRows;
private int dedupeFilteredCount;
private int invalidFilteredCount;
private int brandRejectedCount;
private int brandQueryFailedCount;
private int finalRowCount;
}
@Transactional
public void deleteTask(Long taskId, Long userId) {
FileTaskEntity task = requireTask(taskId, userId);
@@ -351,6 +1035,17 @@ public class CollectDataService {
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, task.getId())
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
deleteTransientTaskPayloads(task.getId());
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, task.getId())
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, task.getId())
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, task.getId())
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE));
taskFileJobService.deleteTaskJobs(task.getId(), MODULE_TYPE);
fileTaskMapper.deleteById(task.getId());
}
@@ -359,6 +1054,21 @@ public class CollectDataService {
if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) {
throw new BusinessException("记录不存在");
}
List<TaskResultItemEntity> resultItems = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.select(TaskResultItemEntity::getPayloadJson)
.eq(TaskResultItemEntity::getTaskId, row.getTaskId())
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
.eq(TaskResultItemEntity::getResultId, row.getId()));
if (resultItems != null) {
for (TaskResultItemEntity item : resultItems) {
transientPayloadStorageService.deletePayloadIfPresent(item.getPayloadJson());
}
}
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, row.getTaskId())
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
.eq(TaskResultItemEntity::getResultId, row.getId()));
taskFileJobService.deleteResultJobs(row.getTaskId(), MODULE_TYPE, row.getId());
fileResultMapper.deleteById(resultId);
}
@@ -503,21 +1213,41 @@ public class CollectDataService {
vo.setTaskId(row.getTaskId());
vo.setSourceFilename(row.getSourceFilename());
vo.setResultFilename(row.getResultFilename());
vo.setDownloadUrl(buildFreshDownloadUrl(row));
vo.setRowCount(row.getRowCount());
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
vo.setError(row.getErrorMessage());
vo.setCreatedAt(formatTime(row.getCreatedAt()));
if (task != null) {
CollectDataStats stats = loadStats(task);
vo.setTaskNo(task.getTaskNo());
vo.setTaskStatus(task.getStatus());
vo.setStartedAt(formatTime(task.getCreatedAt()));
vo.setFinishedAt(formatTime(task.getFinishedAt()));
vo.setTaskType(extractTaskTypeFromRequest(task));
vo.setFilters(extractFiltersFromRequest(task));
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
vo.setBrandRejectedCount(stats.brandRejectedCount);
vo.setFinalRowCount(stats.finalRowCount);
}
return vo;
}
private String buildFreshDownloadUrl(FileResultEntity row) {
String resultFileUrl = row == null ? null : row.getResultFileUrl();
if (resultFileUrl == null || resultFileUrl.isBlank()) {
return null;
}
try {
return ossStorageService.generateFreshDownloadUrl(resultFileUrl);
} catch (Exception ex) {
log.warn("[collect-data] generate fresh download url failed resultId={} err={}",
row.getId(), ex.getMessage());
return null;
}
}
private CollectDataTaskSummaryVo toTaskSummary(FileTaskEntity task) {
CollectDataTaskSummaryVo vo = new CollectDataTaskSummaryVo();
vo.setId(task.getId());

7
backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java
View File

@@ -35,6 +35,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -291,7 +292,7 @@ public class ConvertRunService {
long batchId = System.currentTimeMillis();
int[] rowIndex = new int[]{1};
boolean[] headerRead = new boolean[]{false};
Map<String, Integer>[] headerMapHolder = new Map[]{null};
AtomicReference<Map<String, Integer>> headerMapHolder = new AtomicReference<>();
ExcelStreamReader.readFirstSheet(inputFile, new ExcelStreamReader.SheetRowHandler() {
@Override
@@ -310,7 +311,7 @@ public class ConvertRunService {
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
}
headerRead[0] = true;
headerMapHolder[0] = resolvedHeaderMap;
headerMapHolder.set(resolvedHeaderMap);
for (String line : preambleLines) {
writeLineToAll(writers, line);
}
@@ -322,7 +323,7 @@ public class ConvertRunService {
@Override
public void onRow(String sheetName, Integer sheetNo, int excelRowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception {
Map<String, Integer> resolvedHeaderMap = headerMapHolder[0];
Map<String, Integer> resolvedHeaderMap = headerMapHolder.get();
if (resolvedHeaderMap == null) {
throw new BusinessException("Excel header row is empty.");
}

2
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskCacheService.java
View File

@@ -117,7 +117,7 @@ public class DeleteBrandTaskCacheService {
org.springframework.data.redis.serializer.RedisSerializer<String> serializer =
stringRedisTemplate.getStringSerializer();
for (Long taskId : missingTaskIds) {
connection.hGetAll(serializer.serialize(buildProgressKey(taskId)));
connection.hashCommands().hGetAll(serializer.serialize(buildProgressKey(taskId)));
}
return null;
});

10
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java
View File

@@ -50,6 +50,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Locale;
import java.util.concurrent.atomic.AtomicReference;
@Service
@RequiredArgsConstructor
@@ -792,7 +793,7 @@ public class PriceTrackTaskService {
private Map<String, List<Map<String, String>>> parseWorkbookAsinRows(File file, List<String> countryCodes) {
Map<String, List<Map<String, String>>> out = new LinkedHashMap<>();
Map<String, Integer>[] headerIndexHolder = new Map[]{Map.of()};
AtomicReference<Map<String, Integer>> headerIndexHolder = new AtomicReference<>(Map.of());
String[] countryCodeHolder = new String[]{null};
int[] sheetCount = new int[]{0};
try {
@@ -801,7 +802,7 @@ public class PriceTrackTaskService {
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
sheetCount[0]++;
countryCodeHolder[0] = resolveCountryCode(sheetName, countryCodes, false);
headerIndexHolder[0] = buildHeaderIndex(headerMap);
headerIndexHolder.set(buildHeaderIndex(headerMap));
}
@Override
@@ -813,10 +814,11 @@ public class PriceTrackTaskService {
if (countryCode == null) {
return;
}
if (headerIndexHolder[0].isEmpty()) {
Map<String, Integer> headerIndex = headerIndexHolder.get();
if (headerIndex.isEmpty()) {
return;
}
Map<String, String> item = extractAsinRow(rowMap, headerIndexHolder[0]);
Map<String, String> item = extractAsinRow(rowMap, headerIndex);
if (!item.isEmpty()) {
out.computeIfAbsent(countryCode, key -> new ArrayList<>()).add(item);
}

18
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinCozeClient.java
View File

@@ -1264,25 +1264,33 @@ public class SimilarAsinCozeClient {
private Object cozePrice(Object value) {
if (value == null) {
return "";
return BigDecimal.ZERO;
}
if (value instanceof BigDecimal decimal) {
return decimal.stripTrailingZeros();
return normalizeCozeDecimal(decimal);
}
if (value instanceof Number number) {
return new BigDecimal(number.toString()).stripTrailingZeros();
return normalizeCozeDecimal(new BigDecimal(number.toString()));
}
String text = safeText(String.valueOf(value));
if (text.isBlank()) {
return "";
return BigDecimal.ZERO;
}
try {
return new BigDecimal(text).stripTrailingZeros();
return normalizeCozeDecimal(new BigDecimal(text));
} catch (NumberFormatException ex) {
return text;
}
}
private BigDecimal normalizeCozeDecimal(BigDecimal value) {
BigDecimal normalized = value.stripTrailingZeros();
if (normalized.scale() < 0) {
return normalized.setScale(0);
}
return normalized;
}
private boolean isBlankPrice(Object value) {
return value == null || value instanceof String text && text.isBlank();
}

79
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
View File

@@ -606,13 +606,7 @@ public class SimilarAsinTaskService {
}
public void submitResult(Long taskId, SimilarAsinSubmitResultRequest request) {
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS);
if (lockHandle == null) {
throw new BusinessException(40902, "任务正在处理上一批结果，请稍后重试");
}
try (lockHandle) {
submitResultLocked(taskId, request);
}
submitResultLocked(taskId, request);
}
private void submitResultLocked(Long taskId, SimilarAsinSubmitResultRequest request) {
@@ -4532,12 +4526,20 @@ public class SimilarAsinTaskService {
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name", "seller-name", "sellername", "store name", "shop name"));
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "品牌", "brand"));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getIsStock()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getSimilarity()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getIsConform()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getReason()));
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getCategory()));
row.createCell(col).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getStatus(), ""));
String isStock = resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getIsStock());
String similarity = resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getSimilarity());
String isConform = resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getIsConform());
String reason = resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getReason());
String category = resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getCategory());
String status = resultRow == null || isSilentOutputFailure(resultRow)
? ""
: resolveResultStatus(resultRow, isStock, similarity, isConform, reason, category);
row.createCell(col++).setCellValue(isStock);
row.createCell(col++).setCellValue(similarity);
row.createCell(col++).setCellValue(isConform);
row.createCell(col++).setCellValue(reason);
row.createCell(col++).setCellValue(category);
row.createCell(col).setCellValue(status);
if (resultRow != null) {
// 行高固定到 IMAGE_ROW_HEIGHT_POINTS（Excel 上限 409pt）：
row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS);
@@ -4733,6 +4735,39 @@ public class SimilarAsinTaskService {
|| hasUsableCozeField(row.getStatus());
}
static String resolveResultStatus(SimilarAsinResultRowDto row, String... visibleResultValues) {
if (row == null) {
return "\u5931\u8d25";
}
if (hasText(row.getMainUrl()) || hasText(row.getPuzzleImg1()) || hasText(row.getPuzzleImg2())) {
return "\u6210\u529f";
}
String rawStatus = row.getStatus() == null ? "" : row.getStatus().trim();
if (!rawStatus.isBlank()) {
return isFailedStatus(rawStatus) ? "\u5931\u8d25" : "\u6210\u529f";
}
if (visibleResultValues != null) {
for (String value : visibleResultValues) {
if (hasText(value)) {
return "\u6210\u529f";
}
}
}
return hasText(row.getConclusion()) ? "\u6210\u529f" : "\u5931\u8d25";
}
private static boolean isFailedStatus(String status) {
String normalized = status == null ? "" : status.trim().toLowerCase(Locale.ROOT);
return normalized.contains("fail")
|| normalized.contains("error")
|| normalized.contains("\u5931\u8d25")
|| normalized.contains("\u9519\u8bef");
}
private static boolean hasText(String value) {
return value != null && !value.trim().isBlank();
}
private boolean isExportableResultRow(SimilarAsinResultRowDto row) {
if (row == null) {
return false;
@@ -4902,7 +4937,7 @@ public class SimilarAsinTaskService {
.eq(FileTaskEntity::getUserId, userId)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.inSql(FileTaskEntity::getId,
"select distinct task_id from file_result " +
"select distinct task_id from biz_file_result " +
"where module_type = '" + MODULE_TYPE + "' " +
"and user_id = " + userId + " " +
"and result_file_url is not null and result_file_url <> ''"));
@@ -5665,6 +5700,9 @@ public class SimilarAsinTaskService {
private String userFacingCozeCellValue(SimilarAsinResultRowDto row, String value) {
String normalizedValue = normalize(value);
if (isSilentOutputFailure(row) || isSilentOutputFailure(normalizedValue)) {
return "";
}
if (!normalizedValue.isBlank() && !isTechnicalCozeFailure(normalizedValue)) {
return value;
}
@@ -5690,6 +5728,19 @@ public class SimilarAsinTaskService {
return firstNonBlank(row.getConclusion(), "");
}
private boolean isSilentOutputFailure(SimilarAsinResultRowDto row) {
return row != null
&& (isSilentOutputFailure(row.getError())
|| isSilentOutputFailure(row.getReason())
|| isSilentOutputFailure(row.getConclusion()));
}
private boolean isSilentOutputFailure(String value) {
String normalized = normalize(value).toLowerCase(Locale.ROOT);
return normalized.contains("required field missing")
&& normalized.contains("skip coze");
}
private boolean isTechnicalCozeFailure(String value) {
String normalized = normalize(value).toLowerCase(Locale.ROOT);
if (normalized.isBlank()) {

7
backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java
View File

@@ -31,6 +31,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -221,7 +222,7 @@ public class SplitRunService {
results.add(new SplitChunkResult(outputFile, chunkSizes.get(i)));
}
Map<String, Integer>[] headerMapHolder = new Map[]{null};
AtomicReference<Map<String, Integer>> headerMapHolder = new AtomicReference<>();
int[] chunkIndex = new int[]{0};
ExcelStreamReader.readFirstSheet(inputFile, new ExcelStreamReader.SheetRowHandler() {
@Override
@@ -233,7 +234,7 @@ public class SplitRunService {
if (!missing.isEmpty()) {
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
}
headerMapHolder[0] = resolvedHeaderMap;
headerMapHolder.set(resolvedHeaderMap);
}
@Override
@@ -241,7 +242,7 @@ public class SplitRunService {
if (chunkIndex[0] >= writers.size()) {
return;
}
Map<String, Integer> resolvedHeaderMap = headerMapHolder[0];
Map<String, Integer> resolvedHeaderMap = headerMapHolder.get();
if (resolvedHeaderMap == null) {
throw new BusinessException("Excel header row is empty.");
}

2
backend-java/src/main/java/com/nanri/aiimage/modules/task/controller/TaskHeartbeatController.java
View File

@@ -26,7 +26,7 @@ public class TaskHeartbeatController {
@PostMapping("/{taskId}/heartbeat")
@Operation(
summary = "Task heartbeat",
description = "Python calls this during execution. The backend resolves biz_file_task or brand task by taskId. user_id is optional.")
description = "Python calls this during execution. The backend resolves biz_file_task or brand task by taskId only.")
public ApiResponse<TaskHeartbeatVo> heartbeat(
@Parameter(description = "Task ID", required = true, example = "200")
@PathVariable Long taskId,

7
backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskHeartbeatRequest.java
View File

@@ -1,17 +1,14 @@
package com.nanri.aiimage.modules.task.model.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
@Schema(description = "Task heartbeat request")
public class TaskHeartbeatRequest {
@JsonProperty("user_id")
@Schema(description = "Optional task owner user ID. Python workers can omit it.", example = "453")
private Long userId;
@Schema(description = "Optional phase label", example = "crawling")
private String phase;

24
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatService.java
View File

@@ -18,6 +18,7 @@ import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
@@ -26,6 +27,7 @@ import java.util.Map;
@Service
@RequiredArgsConstructor
@Slf4j
public class TaskHeartbeatService {
private static final String STATUS_RUNNING = "RUNNING";
@@ -55,24 +57,18 @@ public class TaskHeartbeatService {
public TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request) {
if (taskId == null || taskId <= 0) {
log.warn("[task-heartbeat] ignored invalid heartbeat taskId={}", taskId);
return TaskHeartbeatVo.notAlive(null, null, "invalid taskId");
}
Long userId = request == null ? null : request.getUserId();
LambdaQueryWrapper<FileTaskEntity> fileQuery = new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, taskId)
.last("limit 1");
if (userId != null && userId > 0) {
fileQuery.eq(FileTaskEntity::getUserId, userId);
}
FileTaskEntity fileTask = fileTaskMapper.selectOne(fileQuery);
LambdaQueryWrapper<BrandCrawlTaskEntity> brandQuery = new LambdaQueryWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
.last("limit 1");
if (userId != null && userId > 0) {
brandQuery.eq(BrandCrawlTaskEntity::getUserId, userId);
}
BrandCrawlTaskEntity brandTask = brandCrawlTaskMapper.selectOne(brandQuery);
TaskHeartbeatVo fileResult = touchFileTaskIfRunning(fileTask, request);
@@ -93,6 +89,7 @@ public class TaskHeartbeatService {
if (brandResult != null) {
return brandResult;
}
log.warn("[task-heartbeat] task not found taskId={} fileFound=false brandFound=false", taskId);
return TaskHeartbeatVo.notAlive(null, null, "task not found");
}
@@ -103,16 +100,20 @@ public class TaskHeartbeatService {
String moduleType = task.getModuleType() == null ? "" : task.getModuleType();
String status = task.getStatus();
if (!STATUS_RUNNING.equals(status)) {
log.warn("[task-heartbeat] file task is not running taskId={} actualUserId={} moduleType={} status={}",
task.getId(), task.getUserId(), moduleType, status);
return TaskHeartbeatVo.notAlive(moduleType, status, "task is not running");
}
LocalDateTime now = LocalDateTime.now();
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
.eq(FileTaskEntity::getUserId, task.getUserId())
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.set(FileTaskEntity::getUpdatedAt, now));
if (updated <= 0) {
FileTaskEntity latest = fileTaskMapper.selectById(task.getId());
log.warn("[task-heartbeat] file task heartbeat update missed taskId={} actualUserId={} moduleType={} status={} latestStatus={}",
task.getId(), task.getUserId(), moduleType, status,
latest == null ? null : latest.getStatus());
return TaskHeartbeatVo.notAlive(moduleType, latest == null ? status : latest.getStatus(), "task is not running");
}
touchModuleHeartbeat(moduleType, task.getId(), request);
@@ -127,15 +128,19 @@ public class TaskHeartbeatService {
}
String status = task.getStatus();
if (!BRAND_STATUS_RUNNING.equals(status)) {
log.warn("[task-heartbeat] brand task is not running taskId={} actualUserId={} status={}",
task.getId(), task.getUserId(), status);
return TaskHeartbeatVo.notAlive(MODULE_BRAND, status, "task is not running");
}
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, task.getId())
.eq(BrandCrawlTaskEntity::getUserId, task.getUserId())
.eq(BrandCrawlTaskEntity::getStatus, BRAND_STATUS_RUNNING)
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()));
if (updated <= 0) {
BrandCrawlTaskEntity latest = brandCrawlTaskMapper.selectById(task.getId());
log.warn("[task-heartbeat] brand task heartbeat update missed taskId={} actualUserId={} status={} latestStatus={}",
task.getId(), task.getUserId(), status,
latest == null ? null : latest.getStatus());
return TaskHeartbeatVo.notAlive(MODULE_BRAND, latest == null ? status : latest.getStatus(), "task is not running");
}
brandTaskProgressCacheService.touchHeartbeat(
@@ -208,4 +213,5 @@ public class TaskHeartbeatService {
values.put(key, String.valueOf(value));
}
}
}

7
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java
View File

@@ -4,6 +4,7 @@ import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandRunService;
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
@@ -45,6 +46,7 @@ public class TaskResultFileJobWorker {
private final SimilarAsinTaskService similarAsinTaskService;
private final DeleteBrandRunService deleteBrandRunService;
private final BrandTaskService brandTaskService;
private final CollectDataService collectDataService;
@Autowired
@Qualifier("cozeTaskExecutor")
private TaskExecutor cozeTaskExecutor;
@@ -274,6 +276,10 @@ public class TaskResultFileJobWorker {
brandTaskService.processResultFileJob(job);
return true;
}
if ("COLLECT_DATA".equals(moduleType)) {
collectDataService.processResultFileJob(job);
return true;
}
throw new IllegalArgumentException("unsupported result file job module: " + moduleType);
}
@@ -297,6 +303,7 @@ public class TaskResultFileJobWorker {
}
if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.cleanupResultFileJob(job);
return;
}
}
}

72
backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClientTest.java
View File

@@ -14,6 +14,7 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class AppearancePatentCozeClientTest {
@@ -40,7 +41,7 @@ class AppearancePatentCozeClientTest {
}
@Test
void skipBrandCheckWhenCozeTitleAndReasonAreNone() throws Exception {
void skipBrandCheckWhenCozeTitleIsNone() throws Exception {
BrandCheckClient brandCheckClient = mock(BrandCheckClient.class);
AppearancePatentCozeClient cozeClient = new AppearancePatentCozeClient(
new AppearancePatentProperties(),
@@ -61,4 +62,73 @@ class AppearancePatentCozeClientTest {
assertThat(merged.get(0).getConclusion()).isEqualTo("无侵权");
verify(brandCheckClient, never()).checkTitleText(anyString(), anyString());
}
@Test
void callBrandCheckOnlyWhenTitleAndAppearanceArePresent() throws Exception {
BrandCheckClient brandCheckClient = mock(BrandCheckClient.class);
when(brandCheckClient.checkTitleText("阿凡达", "Terms"))
.thenReturn(new BrandCheckClient.BrandCheckBatchResult(List.of("阿凡达"), List.of(), List.of()));
AppearancePatentCozeClient cozeClient = new AppearancePatentCozeClient(
new AppearancePatentProperties(),
new ObjectMapper(),
null,
brandCheckClient
);
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
row.setId("1");
String dataText = """
{"data":[{"row_id":"1","title":"阿凡达","appearance":"无侵权","result":"无侵权"}]}
""";
List<AppearancePatentResultRowDto> merged = cozeClient.mergeRowsFromDataText(List.of(row), dataText);
assertThat(merged).hasSize(1);
assertThat(merged.get(0).getTitleRisk()).isEqualTo("无侵权");
verify(brandCheckClient).checkTitleText("阿凡达", "Terms");
}
@Test
void leaveTitleRiskBlankWhenAppearanceIsMissing() throws Exception {
BrandCheckClient brandCheckClient = mock(BrandCheckClient.class);
AppearancePatentCozeClient cozeClient = new AppearancePatentCozeClient(
new AppearancePatentProperties(),
new ObjectMapper(),
null,
brandCheckClient
);
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
row.setId("1");
String dataText = """
{"data":[{"row_id":"1","title":"阿凡达","result":""}]}
""";
List<AppearancePatentResultRowDto> merged = cozeClient.mergeRowsFromDataText(List.of(row), dataText);
assertThat(merged).hasSize(1);
assertThat(merged.get(0).getTitleRisk()).isEmpty();
verify(brandCheckClient, never()).checkTitleText(anyString(), anyString());
}
@Test
void leaveTitleRiskBlankWhenTitleAndAppearanceAreMissing() throws Exception {
BrandCheckClient brandCheckClient = mock(BrandCheckClient.class);
AppearancePatentCozeClient cozeClient = new AppearancePatentCozeClient(
new AppearancePatentProperties(),
new ObjectMapper(),
null,
brandCheckClient
);
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
row.setId("1");
String dataText = """
{"data":[{"row_id":"1","result":""}]}
""";
List<AppearancePatentResultRowDto> merged = cozeClient.mergeRowsFromDataText(List.of(row), dataText);
assertThat(merged).hasSize(1);
assertThat(merged.get(0).getTitleRisk()).isEmpty();
assertThat(merged.get(0).getAppearanceRisk()).isNullOrEmpty();
verify(brandCheckClient, never()).checkTitleText(anyString(), anyString());
}
}

16
backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskServiceTest.java Normal file
View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.appearancepatent.service;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
class AppearancePatentTaskServiceTest {
@Test
void resultStatusFailsOnlyWhenConclusionIsEmpty() {
assertEquals("\u5931\u8d25", AppearancePatentTaskService.resolveResultStatus(null));
assertEquals("\u5931\u8d25", AppearancePatentTaskService.resolveResultStatus(" "));
assertEquals("\u6210\u529f", AppearancePatentTaskService.resolveResultStatus("\u4fb5\u6743"));
assertEquals("\u6210\u529f", AppearancePatentTaskService.resolveResultStatus("\u65e0\u4fb5\u6743"));
}
}

97
backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinCozeClientTest.java
View File

@@ -12,6 +12,7 @@ import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class SimilarAsinCozeClientTest {
@@ -83,4 +84,100 @@ class SimilarAsinCozeClientTest {
assertEquals("https://cbu01.alicdn.com/img/ibank/legacy-a.jpg", alibaba.getFirst().get("url"));
assertEquals(0, new BigDecimal("8.5").compareTo((BigDecimal) alibaba.getFirst().get("price")));
}
@Test
@SuppressWarnings("unchecked")
void buildParametersConvertsBlankAlibabaPricesToZero() throws Exception {
List<SimilarAsinResultRowDto> rows = objectMapper.readValue("""
[
{
"asin": "B0BLANK123",
"url": "https://m.media-amazon.com/images/I/main.jpg",
"alibaba": [
{"url": "https://cbu01.alicdn.com/img/ibank/blank.jpg", "price": ""},
{"url": "https://cbu01.alicdn.com/img/ibank/null.jpg", "price": null}
]
}
]
""", new TypeReference<>() {
});
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null);
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
"buildParameters", List.class, String.class, String.class, boolean.class);
method.setAccessible(true);
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, rows, "", "", true);
List<Map<String, Object>> items = (List<Map<String, Object>>) parameters.get("items");
List<Map<String, Object>> alibaba = (List<Map<String, Object>>) items.getFirst().get("alibaba");
String json = objectMapper.writeValueAsString(parameters);
assertEquals(2, alibaba.size());
assertEquals(0, BigDecimal.ZERO.compareTo((BigDecimal) alibaba.get(0).get("price")));
assertEquals(0, BigDecimal.ZERO.compareTo((BigDecimal) alibaba.get(1).get("price")));
assertTrue(json.contains("\"price\":0"));
assertFalse(json.contains("\"price\":\"\""));
}
@Test
@SuppressWarnings("unchecked")
void buildParametersConvertsBlankFallbackRowPriceToZero() throws Exception {
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
row.setAsin("B0BLANKROW");
row.setUrl("https://m.media-amazon.com/images/I/main.jpg");
row.setUrls(List.of("https://cbu01.alicdn.com/img/ibank/fallback.jpg"));
row.setPrice("");
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null);
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
"buildParameters", List.class, String.class, String.class, boolean.class);
method.setAccessible(true);
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, List.of(row), "", "", true);
List<Map<String, Object>> items = (List<Map<String, Object>>) parameters.get("items");
List<Map<String, Object>> alibaba = (List<Map<String, Object>>) items.getFirst().get("alibaba");
String json = objectMapper.writeValueAsString(parameters);
assertEquals(1, alibaba.size());
assertEquals(0, BigDecimal.ZERO.compareTo((BigDecimal) alibaba.getFirst().get("price")));
assertTrue(json.contains("\"price\":0"));
assertFalse(json.contains("\"price\":\"\""));
}
@Test
@SuppressWarnings("unchecked")
void buildParametersSerializesWholeNumberPricesWithoutScientificNotation() throws Exception {
List<SimilarAsinResultRowDto> rows = objectMapper.readValue("""
[
{
"asin": "B0SCI123",
"url": "https://m.media-amazon.com/images/I/main.jpg",
"alibaba": [
{"url": "https://cbu01.alicdn.com/img/ibank/ten.jpg", "price": 10.0},
{"url": "https://cbu01.alicdn.com/img/ibank/eighty.jpg", "price": 80.0},
{"url": "https://cbu01.alicdn.com/img/ibank/fraction.jpg", "price": 7.5800}
]
}
]
""", new TypeReference<>() {
});
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null);
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
"buildParameters", List.class, String.class, String.class, boolean.class);
method.setAccessible(true);
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, rows, "", "", true);
List<Map<String, Object>> items = (List<Map<String, Object>>) parameters.get("items");
List<Map<String, Object>> alibaba = (List<Map<String, Object>>) items.getFirst().get("alibaba");
String json = objectMapper.writeValueAsString(parameters);
assertEquals("10", ((BigDecimal) alibaba.get(0).get("price")).toPlainString());
assertEquals("80", ((BigDecimal) alibaba.get(1).get("price")).toPlainString());
assertEquals("7.58", ((BigDecimal) alibaba.get(2).get("price")).toPlainString());
assertFalse(json.contains("1E+1"));
assertFalse(json.contains("8E+1"));
assertTrue(json.contains("\"price\":10"));
assertTrue(json.contains("\"price\":80"));
}
}

33
backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskServiceTest.java Normal file
View File

@@ -0,0 +1,33 @@
package com.nanri.aiimage.modules.similarasin.service;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
class SimilarAsinTaskServiceTest {
@Test
void resultStatusUsesReturnedCozeDataAndImages() {
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(null));
SimilarAsinResultRowDto empty = new SimilarAsinResultRowDto();
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(empty));
SimilarAsinResultRowDto withImage = new SimilarAsinResultRowDto();
withImage.setStatus("FAILED");
withImage.setMainUrl("https://example.com/main.jpg");
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withImage));
SimilarAsinResultRowDto withCozeStatus = new SimilarAsinResultRowDto();
withCozeStatus.setStatus("success");
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withCozeStatus));
SimilarAsinResultRowDto failedStatus = new SimilarAsinResultRowDto();
failedStatus.setStatus("FAILED");
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(failedStatus));
SimilarAsinResultRowDto withVisibleResultData = new SimilarAsinResultRowDto();
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withVisibleResultData, "", "80%", "", "", ""));
}
}