super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成上架相关优化、oss迁移

Browse Source
This commit is contained in:
supernijia
2026-07-25 17:57:34 +08:00
parent 8ab36b1aaf
commit 4ddb8b47b0
28 changed files with 964 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/publish/controller/PublishController.java
+1 -1
View File
@@ -36,7 +36,7 @@ public class PublishController {
@PostMapping("/parse")
@Operation(
summary = "匹配店铺、解析 Excel 并创建多文件批次任务",
description = "按文件名（去扩展名）先匹配已管理店铺，再查询紫鸟索引；解析每个非空 Sheet。表头必须依次为 id、ASIN、国家、品牌、价格、状态、同步状态、同步国家。匹配并解析成功的文件为 PENDING，失败文件为 FAILED；只要存在可处理文件，任务为 PENDING，否则任务为 FAILED。")
description = "按文件名（去扩展名）先匹配已管理店铺，再查询紫鸟索引；解析每个非空 Sheet。源文件前五列表头必须依次为 id、ASIN、国家、品牌、价格，第六列及以后不解析；状态、同步状态、同步国家由 Python 回传并写入最终结果文件。匹配并解析成功的文件为 PENDING，失败文件为 FAILED；只要存在可处理文件，任务为 PENDING，否则任务为 FAILED。")
public ApiResponse<PublishParseVo> parse(@Valid @RequestBody PublishParseRequest request) {
return ApiResponse.success(publishTaskService.parseAndCreateTask(request));
}
backend-java/src/main/java/com/nanri/aiimage/modules/publish/model/vo/PublishItemsPageVo.java
+1 -1
View File
@@ -40,6 +40,6 @@ public class PublishItemsPageVo {
private Long total;
@Schema(description = "总页数", example = "14")
private Integer totalPages;
@Schema(description = "按原 Excel 行序返回的八列数据")
@Schema(description = "按原 Excel 行序返回前五列源数据；状态、同步状态、同步国家等待 Python 处理后回传")
private List<PublishRowDto> items = new ArrayList<>();
}
backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishTaskService.java
+53 -3
View File
@@ -7,6 +7,8 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.publish.mapper.PublishFileMapper;
@@ -87,6 +89,7 @@ public class PublishTaskService {
private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper;
private final TransactionTemplate transactionTemplate;
private final InstanceMetadata instanceMetadata;
@Value("${aiimage.publish.stale-timeout-minutes:30}")
private int staleTimeoutMinutes;
@@ -349,6 +352,12 @@ public class PublishTaskService {
List<FileTaskEntity> staleTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.and(owner -> owner
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) IS NULL")
.or()
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = ''")
.or()
.apply("JSON_UNQUOTE(JSON_EXTRACT(result_json, '$.ownerInstanceId')) = {0}", currentInstanceId()))
.lt(FileTaskEntity::getUpdatedAt, threshold)
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit 100"));
@@ -377,6 +386,7 @@ public class PublishTaskService {
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("task not found");
}
ensureTaskOwnedByCurrentInstance(task, "process publish result file");
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
if (result == null || !MODULE_TYPE.equals(result.getModuleType())
|| !task.getId().equals(result.getTaskId())) {
@@ -522,6 +532,7 @@ public class PublishTaskService {
|| task.getUpdatedAt() == null || !task.getUpdatedAt().isBefore(threshold)) {
return;
}
ensureTaskOwnedByCurrentInstance(task, "cleanup stale publish task");
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
return;
}
@@ -551,7 +562,7 @@ public class PublishTaskService {
result.setSuccess(0);
result.setErrorMessage(null);
fileResultMapper.updateById(result);
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, result.getId(), "task:" + taskId);
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, result.getId(), ownerScopeKey(taskId));
}
private PersistedTask persistTask(PublishParseRequest request, List<PreparedFile> preparedFiles) {
@@ -568,7 +579,8 @@ public class PublishTaskService {
task.setSuccessFileCount(0);
task.setFailedFileCount(failedFiles);
task.setRequestJson(writeJson(request, "序列化上架任务失败"));
task.setResultJson("{}");
task.setResultJson(writeJson(Map.of("ownerInstanceId", currentInstanceId()),
"Failed to save publish task instance owner"));
task.setErrorMessage(processableFiles > 0 ? null : "全部文件解析或店铺匹配失败");
task.setCreatedBy("user:" + request.getUserId());
task.setUserId(request.getUserId());
@@ -665,7 +677,7 @@ public class PublishTaskService {
task.setErrorMessage(null);
task.setFinishedAt(null);
fileTaskMapper.updateById(task);
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, result.getId(), "task:" + taskId);
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, result.getId(), ownerScopeKey(taskId));
}
private List<PublishTaskDetailVo> loadTaskDetails(List<FileTaskEntity> tasks) {
@@ -801,9 +813,47 @@ public class PublishTaskService {
|| (userId != null && !userId.equals(task.getUserId()))) {
throw new BusinessException("任务不存在");
}
ensureTaskOwnedByCurrentInstance(task, "access publish task");
return task;
}
public void ensureTaskOwnedByCurrentInstance(FileTaskEntity task, String operation) {
String owner = ownerFromTask(task);
if (owner == null || owner.isBlank() || Objects.equals(owner, currentInstanceId())) {
return;
}
log.warn("[publish] reject task operation because owner is another instance taskId={} operation={} owner={} current={}",
task == null ? null : task.getId(), operation, owner, currentInstanceId());
throw new TaskOwnerMismatchException(
task == null ? null : task.getId(), operation, owner, currentInstanceId());
}
private String ownerFromTask(FileTaskEntity task) {
if (task == null || task.getResultJson() == null || task.getResultJson().isBlank()) {
return null;
}
try {
JsonNode root = objectMapper.readTree(task.getResultJson());
if (root == null) {
return null;
}
String owner = root.path("ownerInstanceId").asText("");
return owner.isBlank() ? null : owner;
} catch (Exception ex) {
log.warn("[publish] read task owner failed taskId={} msg={}", task.getId(), safeMessage(ex));
return null;
}
}
private String currentInstanceId() {
String instanceId = instanceMetadata == null ? null : instanceMetadata.getInstanceId();
return firstNonBlank(instanceId, "unknown-instance");
}
private String ownerScopeKey(Long taskId) {
return "task:" + taskId + ":owner:" + currentInstanceId();
}
private PublishFileEntity requireFile(Long taskId, Long fileId) {
if (fileId == null || fileId <= 0) {
throw new BusinessException("file_id 不合法");
backend-java/src/main/java/com/nanri/aiimage/modules/publish/service/PublishWorkbookService.java
+9 -15
View File
@@ -30,7 +30,9 @@ import java.util.zip.ZipOutputStream;
@Service
public class PublishWorkbookService {
public static final List<String> HEADERS = List.of(
public static final List<String> SOURCE_HEADERS = List.of(
"id", "ASIN", "国家", "品牌", "价格");
public static final List<String> RESULT_HEADERS = List.of(
"id", "ASIN", "国家", "品牌", "价格", "状态", "同步状态", "同步国家");
public static final String XLSX_CONTENT_TYPE =
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
@@ -66,9 +68,9 @@ public class PublishWorkbookService {
if (!validatedSheets.contains(sheetNo)) {
throw new BusinessException("工作表 " + sheetName + " 缺少严格的上架表头");
}
List<String> values = new ArrayList<>(HEADERS.size());
List<String> values = new ArrayList<>(SOURCE_HEADERS.size());
boolean nonEmpty = false;
for (int columnIndex = 0; columnIndex < HEADERS.size(); columnIndex++) {
for (int columnIndex = 0; columnIndex < SOURCE_HEADERS.size(); columnIndex++) {
String value = normalize(rowMap.get(columnIndex));
values.add(value);
nonEmpty = nonEmpty || !value.isBlank();
@@ -163,19 +165,14 @@ public class PublishWorkbookService {
if (headerMap == null || headerMap.isEmpty()) {
throw new BusinessException("Excel 表头为空");
}
for (int index = 0; index < HEADERS.size(); index++) {
for (int index = 0; index < SOURCE_HEADERS.size(); index++) {
String actual = normalize(headerMap.get(index));
String expected = HEADERS.get(index);
String expected = SOURCE_HEADERS.get(index);
if (!expected.equals(actual)) {
throw new BusinessException("Excel 表头不匹配，第 " + (index + 1)
+ " 列应为 " + expected + "，实际为 " + actual);
}
}
for (Map.Entry<Integer, String> entry : headerMap.entrySet()) {
if (entry.getKey() != null && entry.getKey() >= HEADERS.size() && !normalize(entry.getValue()).isBlank()) {
throw new BusinessException("Excel 表头必须严格为: " + String.join("/", HEADERS));
}
}
}
private PublishRowDto toRow(List<String> values) {
@@ -185,9 +182,6 @@ public class PublishWorkbookService {
row.setCountry(values.get(2));
row.setBrand(values.get(3));
row.setPrice(values.get(4));
row.setStatus(values.get(5));
row.setSyncStatus(values.get(6));
row.setSyncCountries(values.get(7));
return row;
}
@@ -210,9 +204,9 @@ public class PublishWorkbookService {
private void writeSheet(Sheet sheet, List<PublishRowDto> rows, CellStyle headerStyle) {
Row header = sheet.createRow(0);
for (int index = 0; index < HEADERS.size(); index++) {
for (int index = 0; index < RESULT_HEADERS.size(); index++) {
Cell cell = header.createCell(index);
cell.setCellValue(HEADERS.get(index));
cell.setCellValue(RESULT_HEADERS.get(index));
cell.setCellStyle(headerStyle);
}
int rowIndex = 1;