task-116: 后台管理新增/导入表单弹窗化 + 分组管理独立菜单与UI优化 + 轻量进度端点
- 后台管理页(admin)所有面板的新增/导入表单改为弹窗操作,原有字段 ID 全部保留、提交逻辑不变;导入删除入口不再触发二次确认拦截 - 分组管理升级为独立菜单(V102 + schema initializer),移除 5 个面板内的管理分组按钮;分组列表改为蓝白主题、增加权限分组横幅 - Python 侧 group-manage 权限守卫(_ensure_backend_menu_access 补充 group-manage) - 引入 V101(biz_task_file_job 复合索引)+ 新增 TaskProgressLight/TaskFileJob 轻量端点与进度聚合作 - 前端 progress-light / page-separated-loads / dispatch-guard 共享模块及单元测试
This commit is contained in:
+10
@@ -7,11 +7,13 @@ import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskLightRequest;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinFilterConditionVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService.ResultDownloadInfo;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
@@ -120,6 +122,14 @@ public class SimilarAsinController {
|
||||
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/light")
|
||||
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
|
||||
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileReady/updatedAt),"
|
||||
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
|
||||
public ApiResponse<SimilarAsinTaskLightBatchVo> progressLight(@Valid @RequestBody SimilarAsinTaskLightRequest request) {
|
||||
return ApiResponse.success(service.progressLight(request.getTaskIds()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/activate")
|
||||
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING。")
|
||||
public ApiResponse<Void> activate(
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package com.nanri.aiimage.modules.similarasin.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Schema(description = "相似ASIN检测任务轻量进度查询请求")
|
||||
public class SimilarAsinTaskLightRequest {
|
||||
@NotNull
|
||||
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<Long> taskIds;
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package com.nanri.aiimage.modules.similarasin.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "相似ASIN检测任务轻量进度响应")
|
||||
public class SimilarAsinTaskLightBatchVo {
|
||||
@Schema(description = "轻量进度项列表。顺序按请求 taskIds 处理。")
|
||||
private List<SimilarAsinTaskLightVo> items = new ArrayList<>();
|
||||
@Schema(description = "未找到或不属于相似ASIN检测模块的任务 ID 列表。", example = "[99999]")
|
||||
private List<Long> missingTaskIds = new ArrayList<>();
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
package com.nanri.aiimage.modules.similarasin.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "相似ASIN检测任务轻量进度项(前端轮询专用,不含明细/result 内容)")
|
||||
public class SimilarAsinTaskLightVo {
|
||||
@Schema(description = "任务 ID。", example = "3938")
|
||||
private Long taskId;
|
||||
@Schema(description = "任务状态:PENDING=已解析待推送,RUNNING=执行中,SUCCESS=成功,FAILED=失败。", example = "RUNNING")
|
||||
private String status;
|
||||
@Schema(description = "业务状态码(预留字段,当前恒为 null)。", example = "null")
|
||||
private String statusCode;
|
||||
@Schema(description = "结果文件组装 Job 状态:SUCCESS/RUNNING/FAILED;无 Job 时为空。", example = "SUCCESS")
|
||||
private String fileStatus;
|
||||
@Schema(description = "文件组装失败时的错误信息;无错误时为空。", example = "组装失败: 文件写入超时")
|
||||
private String fileError;
|
||||
@Schema(description = "结果文件是否已就绪。", example = "true")
|
||||
private Boolean fileReady;
|
||||
@Schema(description = "最后更新时间,ISO 本地时间字符串。", example = "2026-04-26T10:05:00")
|
||||
private String updatedAt;
|
||||
}
|
||||
+84
-447
@@ -35,6 +35,11 @@ import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskDetailVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskItemVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGroupingConverter;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler;
|
||||
import com.nanri.aiimage.modules.similarasin.util.BoundedImageCache;
|
||||
import com.nanri.aiimage.modules.similarasin.util.ExcelCellImageWriter;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
@@ -59,12 +64,9 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.CellStyle;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Font;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
@@ -136,6 +138,7 @@ public class SimilarAsinTaskService {
|
||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
||||
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
|
||||
private static final int MAX_LIGHT_TASK_IDS = 200;
|
||||
/** Task 19:Python 回传逐行日志采样频率(每 N 行记一行)。 */
|
||||
private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L;
|
||||
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
|
||||
@@ -336,6 +339,20 @@ public class SimilarAsinTaskService {
|
||||
* best-effort:service 内部所有异常都已吞掉,不影响主流程。
|
||||
*/
|
||||
private final SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
/**
|
||||
* Task 89:Excel 行解析器(表头/单元格读取、别名匹配、空行跳过、单字段截断)。
|
||||
* 由 parseAndCreateTask 委托;解析语义与搬移前 parseWorkbook 完全一致。
|
||||
*/
|
||||
SimilarAsinExcelParser excelParser = new SimilarAsinExcelParser();
|
||||
/**
|
||||
* Task 91:历史查询 VO 拼装(toHistoryItem + 进度链)搬入独立组件;
|
||||
* 由 history/progressBatch 委托,只读不落库,输出与现状一致。
|
||||
*/
|
||||
private SimilarAsinHistoryAssembler historyAssembler() {
|
||||
return new SimilarAsinHistoryAssembler(
|
||||
taskScopeStateMapper, taskChunkMapper, fileTaskMapper,
|
||||
taskProgressSnapshotService, ossStorageService, transientPayloadStorageService, objectMapper);
|
||||
}
|
||||
@Autowired
|
||||
@Qualifier("taskQueueExecutor")
|
||||
private TaskExecutor taskQueueExecutor;
|
||||
@@ -466,7 +483,7 @@ public class SimilarAsinTaskService {
|
||||
+ " (" + input.length() + " bytes > " + maxBytes + " bytes)");
|
||||
}
|
||||
|
||||
ParsedWorkbook parsed = parseWorkbook(input, source);
|
||||
ParsedWorkbook parsed = parseWorkbookDelegated(input, source);
|
||||
totalRows += parsed.totalRows();
|
||||
droppedRows += parsed.droppedRows();
|
||||
allRows.addAll(parsed.allRows());
|
||||
@@ -487,7 +504,7 @@ public class SimilarAsinTaskService {
|
||||
sourceFiles.size(), allRows.size());
|
||||
}
|
||||
long parsedAt = System.nanoTime();
|
||||
List<SimilarAsinParsedGroupVo> groups = buildParsedGroups(allRows);
|
||||
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(allRows);
|
||||
long groupedAt = System.nanoTime();
|
||||
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
@@ -640,13 +657,49 @@ public class SimilarAsinTaskService {
|
||||
.map(FileResultEntity::getId)
|
||||
.filter(Objects::nonNull)
|
||||
.toList());
|
||||
for (FileResultEntity row : rows) {
|
||||
FileTaskEntity task = taskMap.get(row.getTaskId());
|
||||
String taskStatus = task == null ? null : task.getStatus();
|
||||
if (STATUS_PENDING.equals(taskStatus)) {
|
||||
vo.getItems().addAll(historyAssembler().buildHistoryItems(rows, taskMap, jobMap));
|
||||
return vo;
|
||||
}
|
||||
|
||||
public SimilarAsinTaskLightBatchVo progressLight(List<Long> taskIds) {
|
||||
SimilarAsinTaskLightBatchVo vo = new SimilarAsinTaskLightBatchVo();
|
||||
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
|
||||
.filter(id -> id != null && id > 0)
|
||||
.distinct()
|
||||
.limit(MAX_LIGHT_TASK_IDS)
|
||||
.toList();
|
||||
if (normalizedIds.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||
for (FileTaskEntity task : fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getId,
|
||||
FileTaskEntity::getStatus,
|
||||
FileTaskEntity::getUpdatedAt)
|
||||
.in(FileTaskEntity::getId, normalizedIds))) {
|
||||
if (task != null && MODULE_TYPE.equals(task.getModuleType())) {
|
||||
taskMap.put(task.getId(), task);
|
||||
}
|
||||
}
|
||||
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByTaskIds(MODULE_TYPE, new ArrayList<>(taskMap.keySet()));
|
||||
for (Long taskId : normalizedIds) {
|
||||
FileTaskEntity task = taskMap.get(taskId);
|
||||
if (task == null) {
|
||||
vo.getMissingTaskIds().add(taskId);
|
||||
continue;
|
||||
}
|
||||
vo.getItems().add(toHistoryItem(row, task, jobMap.get(row.getId())));
|
||||
SimilarAsinTaskLightVo item = new SimilarAsinTaskLightVo();
|
||||
item.setTaskId(task.getId());
|
||||
item.setStatus(resolveDisplayTaskStatus(task));
|
||||
TaskFileJobEntity job = jobMap.get(taskId);
|
||||
boolean fileReady = job != null
|
||||
&& job.getResultFileUrl() != null
|
||||
&& !job.getResultFileUrl().isBlank();
|
||||
item.setFileReady(fileReady);
|
||||
item.setFileStatus(job == null ? null : job.getStatus());
|
||||
item.setFileError(job == null ? null : job.getErrorMessage());
|
||||
item.setUpdatedAt(fmt(task.getUpdatedAt()));
|
||||
vo.getItems().add(item);
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
@@ -708,7 +761,8 @@ public class SimilarAsinTaskService {
|
||||
detail.setTask(toTaskItem(task));
|
||||
FileResultEntity resultRow = resultByTaskId.get(taskId);
|
||||
if (resultRow != null) {
|
||||
detail.getItems().add(toHistoryItem(resultRow, task, jobMap.get(resultRow.getId())));
|
||||
detail.getItems().addAll(historyAssembler().buildHistoryItems(
|
||||
List.of(resultRow), taskMap, jobMap));
|
||||
}
|
||||
vo.getItems().add(detail);
|
||||
}
|
||||
@@ -1751,7 +1805,7 @@ public class SimilarAsinTaskService {
|
||||
private Map<String, List<SimilarAsinParsedRowVo>> loadAllRowsByBaseId(FileTaskEntity task) {
|
||||
try {
|
||||
SimilarAsinParsedPayloadDto payload = readParsedPayload(task);
|
||||
return groupRowsByBaseId(resolveAllRows(payload));
|
||||
return SimilarAsinGroupingConverter.groupRowsByBaseId(resolveAllRows(payload));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] read all rows failed taskId={} err={}", task.getId(), ex.getMessage());
|
||||
return new LinkedHashMap<>();
|
||||
@@ -1791,36 +1845,6 @@ public class SimilarAsinTaskService {
|
||||
return candidates;
|
||||
}
|
||||
|
||||
/**
|
||||
* Task 6:分组数据改为索引/范围引用。组内行在原 items 中必然连续
|
||||
* (parseWorkbook 按行遍历、同 baseId 块连续收集),因此只需要
|
||||
* [startIndex, endIndex) 半开区间即可唯一定位组内行,
|
||||
* 行对象仅存在于 items 一次,避免 groups 嵌套复制完整行对象。
|
||||
*/
|
||||
private List<SimilarAsinParsedGroupVo> buildParsedGroups(List<SimilarAsinParsedRowVo> rows) {
|
||||
List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
|
||||
int cursor = 0;
|
||||
for (List<SimilarAsinParsedRowVo> siblings : groupRowsByBaseId(rows).values()) {
|
||||
if (siblings == null || siblings.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
SimilarAsinParsedRowVo first = siblings.getFirst();
|
||||
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
|
||||
group.setSourceFileKey(first.getSourceFileKey());
|
||||
group.setSourceFilename(first.getSourceFilename());
|
||||
group.setGroupKey(firstNonBlank(first.getGroupKey(), buildGroupKey(first.getSourceFileKey(), baseId(first.getDisplayId()), first.getRowIndex())));
|
||||
group.setBaseId(baseId(first.getDisplayId()));
|
||||
group.setDisplayId(firstNonBlank(first.getDisplayId(), first.getSourceId()));
|
||||
group.setItemCount(siblings.size());
|
||||
int end = cursor + siblings.size();
|
||||
group.setStartIndex(cursor);
|
||||
group.setEndIndex(end);
|
||||
cursor = end;
|
||||
groups.add(group);
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
private List<SimilarAsinParsedGroupVo> buildResponsePreviewGroups(List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
|
||||
if (groups == null || groups.isEmpty()) {
|
||||
return List.of();
|
||||
@@ -1887,18 +1911,6 @@ public class SimilarAsinTaskService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
private Map<String, List<SimilarAsinParsedRowVo>> groupRowsByBaseId(List<SimilarAsinParsedRowVo> rows) {
|
||||
Map<String, List<SimilarAsinParsedRowVo>> result = new LinkedHashMap<>();
|
||||
if (rows == null) {
|
||||
return result;
|
||||
}
|
||||
for (SimilarAsinParsedRowVo row : rows) {
|
||||
String key = firstNonBlank(normalize(row.getGroupKey()), baseId(row.getDisplayId()));
|
||||
result.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<SimilarAsinResultRowDto> flattenSubmittedRows(SimilarAsinSubmitResultRequest request) {
|
||||
if (request == null) {
|
||||
return List.of();
|
||||
@@ -4718,8 +4730,11 @@ public class SimilarAsinTaskService {
|
||||
return firstNonBlank(resultRow.getUrl(), fallbackUrl);
|
||||
}
|
||||
|
||||
private ParsedWorkbook parseWorkbook(File input, SimilarAsinSourceFileDto source) {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
/**
|
||||
* Task 89:解析委托 SimilarAsinExcelParser(表头/单元格读取、别名匹配、空行跳过、字段截断),
|
||||
* 保留服务侧编排语义:zip 探针、status 列过滤、类目重试判定、行号/分组键/rowToken 组装。
|
||||
*/
|
||||
private ParsedWorkbook parseWorkbookDelegated(File input, SimilarAsinSourceFileDto source) {
|
||||
try {
|
||||
// Task 8:受控读取——先探测 zip 条目数与解压体积,超限拒绝
|
||||
probeWorkbookZipBounds(input, source.getOriginalFilename());
|
||||
@@ -4729,26 +4744,9 @@ public class SimilarAsinTaskService {
|
||||
// 非 zip 或损坏文件:留给 WorkbookFactory 尝试后由下方 catch 转业务异常
|
||||
log.debug("[similar-asin] workbook zip probe skipped file={} err={}", input, ex.getMessage());
|
||||
}
|
||||
try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) { Sheet sheet = workbook.getSheetAt(0);
|
||||
Row header = sheet.getRow(0);
|
||||
if (header == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
|
||||
List<String> headers = readHeaders(header, formatter);
|
||||
int idCol = findRequiredHeader(headerMap, "id");
|
||||
int asinCol = findRequiredHeader(headerMap, "asin");
|
||||
int countryCol = findRequiredHeader(headerMap, "国家", "country");
|
||||
// 兼容旧 xlsx:RESULT_HEADERS 已不再写出 sku 列,但仍要支持旧版结果簿重新上传,
|
||||
// 因此这里仍按可选列读取并保留到 DTO,enrichRowForLlm 仍可使用,仅导出阶段不再写出。
|
||||
int skuCol = findOptionalHeaderExact(headerMap, "sku", "seller sku", "seller_sku", "msku", "货号");
|
||||
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
|
||||
int urlCol = findOptionalHeaderExact(headerMap,
|
||||
"url", "rul", "link", "image", "img", "pic", "picture",
|
||||
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
|
||||
int titleCol = findOptionalHeaderExact(headerMap,
|
||||
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
|
||||
|
||||
try {
|
||||
SimilarAsinExcelParser.ParsedSheet parsedSheet = excelParser.parse(input, resolveMaxFieldLength());
|
||||
List<String> headers = parsedSheet.headers();
|
||||
int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers);
|
||||
String statusHeader = statusCol >= 0 && statusCol < headers.size() ? headers.get(statusCol) : null;
|
||||
|
||||
@@ -4758,14 +4756,10 @@ public class SimilarAsinTaskService {
|
||||
int validRows = 0;
|
||||
String currentBlockBaseId = "";
|
||||
String currentGroupKey = "";
|
||||
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
|
||||
Row row = sheet.getRow(i);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
String id = cell(row, idCol, formatter);
|
||||
String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT);
|
||||
String country = cell(row, countryCol, formatter);
|
||||
for (SimilarAsinExcelParser.SimilarAsinExcelRow parsedRow : parsedSheet.rows()) {
|
||||
String id = parsedRow.id();
|
||||
String asin = parsedRow.asin();
|
||||
String country = parsedRow.country();
|
||||
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
@@ -4778,7 +4772,7 @@ public class SimilarAsinTaskService {
|
||||
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
|
||||
vo.setSourceFileKey(source.getFileKey());
|
||||
vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName()));
|
||||
vo.setRowIndex(i + 1);
|
||||
vo.setRowIndex(parsedRow.rowIndex());
|
||||
vo.setSourceId(id);
|
||||
vo.setDisplayId(normalizeDisplayId(id));
|
||||
String rowBaseId = baseId(vo.getDisplayId());
|
||||
@@ -4790,11 +4784,11 @@ public class SimilarAsinTaskService {
|
||||
vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex()));
|
||||
vo.setAsin(asin);
|
||||
vo.setCountry(country);
|
||||
vo.setSku(skuCol >= 0 ? cell(row, skuCol, formatter) : "");
|
||||
vo.setPrice(priceCol >= 0 ? cell(row, priceCol, formatter) : "");
|
||||
vo.setUrl(urlCol >= 0 ? cell(row, urlCol, formatter) : "");
|
||||
vo.setTitle(titleCol >= 0 ? cell(row, titleCol, formatter) : "");
|
||||
vo.setValues(readRowValues(row, headers, formatter));
|
||||
vo.setSku(parsedRow.sku());
|
||||
vo.setPrice(parsedRow.price());
|
||||
vo.setUrl(parsedRow.url());
|
||||
vo.setTitle(parsedRow.title());
|
||||
vo.setValues(parsedRow.values());
|
||||
allRows.add(vo);
|
||||
}
|
||||
boolean resultWorkbook = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
|
||||
@@ -4911,86 +4905,6 @@ public class SimilarAsinTaskService {
|
||||
return !normalized.isBlank() && !isTechnicalLlmFailure(normalized);
|
||||
}
|
||||
|
||||
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
|
||||
Map<String, Integer> map = new LinkedHashMap<>();
|
||||
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||
if (!val.isBlank()) {
|
||||
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private List<String> readHeaders(Row header, DataFormatter formatter) {
|
||||
List<String> headers = new ArrayList<>();
|
||||
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||
headers.add(val.isBlank() ? "列" + (i + 1) : val);
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter) {
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
for (int i = 0; i < headers.size(); i++) {
|
||||
values.put(headers.get(i), cell(row, i, formatter));
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private int findRequiredHeader(Map<String, Integer> map, String... names) {
|
||||
int idx = findOptionalHeader(map, names);
|
||||
if (idx < 0) {
|
||||
throw new BusinessException("缺少必要表头: " + String.join("/", names));
|
||||
}
|
||||
return idx;
|
||||
}
|
||||
|
||||
private int findOptionalHeader(Map<String, Integer> map, String... names) {
|
||||
for (Map.Entry<String, Integer> entry : map.entrySet()) {
|
||||
for (String name : names) {
|
||||
if (entry.getKey().contains(name.toLowerCase(Locale.ROOT))) {
|
||||
return entry.getValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private int findOptionalHeaderExact(Map<String, Integer> map, String... names) {
|
||||
for (Map.Entry<String, Integer> entry : map.entrySet()) {
|
||||
String normalizedHeader = normalizeHeaderAlias(entry.getKey());
|
||||
for (String name : names) {
|
||||
if (normalizedHeader.equals(normalizeHeaderAlias(name))) {
|
||||
return entry.getValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private String normalizeHeaderAlias(String value) {
|
||||
String normalized = normalize(value).toLowerCase(Locale.ROOT);
|
||||
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
|
||||
}
|
||||
|
||||
private String cell(Row row, int col, DataFormatter formatter) {
|
||||
if (col < 0) {
|
||||
return "";
|
||||
}
|
||||
String value = normalize(formatter.formatCellValue(row.getCell(col)));
|
||||
if (isSpreadsheetErrorValue(value)) {
|
||||
return "";
|
||||
}
|
||||
// Task 7:单字段长度上限,防止超长单元格导致内存无界增长
|
||||
int maxLen = resolveMaxFieldLength();
|
||||
if (value.length() > maxLen) {
|
||||
return value.substring(0, maxLen);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private static boolean isSpreadsheetErrorValue(String value) {
|
||||
String normalized = value == null ? "" : value.trim();
|
||||
return normalized.startsWith("#") && normalized.endsWith("!");
|
||||
@@ -5133,253 +5047,6 @@ public class SimilarAsinTaskService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
private SimilarAsinHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
||||
SimilarAsinHistoryItemVo vo = new SimilarAsinHistoryItemVo();
|
||||
vo.setResultId(row.getId());
|
||||
vo.setTaskId(row.getTaskId());
|
||||
vo.setSourceFilename(row.getSourceFilename());
|
||||
vo.setResultFilename(row.getResultFilename());
|
||||
vo.setDownloadUrl(buildFreshDownloadUrl(row));
|
||||
vo.setTaskStatus(resolveDisplayTaskStatus(task, row));
|
||||
attachFileJobState(vo, row, job, task);
|
||||
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
|
||||
vo.setError(row.getErrorMessage());
|
||||
vo.setRowCount(row.getRowCount());
|
||||
vo.setCreatedAt(fmt(row.getCreatedAt()));
|
||||
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
|
||||
vo.setStartedAt(fmt(task == null ? row.getCreatedAt() : task.getCreatedAt()));
|
||||
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
|
||||
return vo;
|
||||
}
|
||||
|
||||
private String buildFreshDownloadUrl(FileResultEntity row) {
|
||||
String resultFileUrl = row == null ? null : row.getResultFileUrl();
|
||||
if (resultFileUrl == null || resultFileUrl.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
// bucket 对 result/ 前缀已开放公开读,使用无签名公开 URL,避免签名 1 小时过期后下载 403。
|
||||
// result/ 前缀走 download-endpoint 独立下载域名(带宽/缓存分流),非 result/(如图片)仍走公开端点。
|
||||
return ossStorageService.generateFreshDownloadUrl(resultFileUrl);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] build public download url failed resultId={} err={}",
|
||||
row.getId(), ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void attachFileJobState(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
|
||||
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
|
||||
if (job == null) {
|
||||
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
|
||||
attachFileProgress(vo, row, job, task);
|
||||
return;
|
||||
}
|
||||
vo.setFileJobId(job.getId());
|
||||
vo.setFileStatus(job.getStatus());
|
||||
vo.setFileError(job.getErrorMessage());
|
||||
attachFileProgress(vo, row, job, task);
|
||||
}
|
||||
|
||||
private void attachFileProgress(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
|
||||
if (row == null || row.getTaskId() == null) {
|
||||
return;
|
||||
}
|
||||
Long taskId = row.getTaskId();
|
||||
if (Boolean.TRUE.equals(vo.getFileReady())) {
|
||||
vo.setFileProgressCurrent(1);
|
||||
vo.setFileProgressTotal(1);
|
||||
vo.setFileProgressPercent(100);
|
||||
vo.setFileProgressMessage("结果文件已生成");
|
||||
return;
|
||||
}
|
||||
int llmCompleted = countCompletedLlmStates(taskId);
|
||||
int llmPending = countPendingLlmStates(taskId);
|
||||
boolean uploadComplete = isResultSubmissionComplete(taskId);
|
||||
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(taskId, MODULE_TYPE);
|
||||
if (!uploadComplete) {
|
||||
if (canTreatTaskAsCompleted(task, row, job)) {
|
||||
vo.setFileProgressCurrent(1);
|
||||
vo.setFileProgressTotal(1);
|
||||
vo.setFileProgressPercent(100);
|
||||
vo.setFileProgressMessage("结果文件已生成");
|
||||
return;
|
||||
}
|
||||
if (task != null && !STATUS_RUNNING.equals(task.getStatus())) {
|
||||
return;
|
||||
}
|
||||
if (llmCompleted + llmPending > 0) {
|
||||
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, false);
|
||||
return;
|
||||
}
|
||||
attachPythonUploadProgress(vo, taskId);
|
||||
return;
|
||||
}
|
||||
if (llmCompleted + llmPending > 0) {
|
||||
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, true);
|
||||
return;
|
||||
}
|
||||
if (snapshot == null) {
|
||||
return;
|
||||
}
|
||||
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
|
||||
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
|
||||
if (total <= 0) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : (job == null ? null : job.getUpdatedAt());
|
||||
int percent = calculateDisplayProgressPercent(current, total, job, baseTime);
|
||||
percent = Math.max(percent, extractSnapshotDisplayPercent(snapshot));
|
||||
percent = Boolean.TRUE.equals(vo.getFileReady()) ? 100 : Math.min(99, percent);
|
||||
vo.setFileProgressCurrent(current);
|
||||
vo.setFileProgressTotal(total);
|
||||
vo.setFileProgressPercent(percent);
|
||||
vo.setFileProgressMessage(snapshot.getMessage());
|
||||
}
|
||||
|
||||
private void attachLlmProgress(SimilarAsinHistoryItemVo vo,
|
||||
TaskFileJobEntity job,
|
||||
TaskProgressSnapshotEntity snapshot,
|
||||
int llmCompleted,
|
||||
int llmPending,
|
||||
boolean uploadComplete) {
|
||||
int observed = Math.max(0, llmCompleted) + Math.max(0, llmPending);
|
||||
int total = calculateLlmDisplayTotal(snapshot, observed);
|
||||
if (!uploadComplete) {
|
||||
total = Math.max(total, observed + 1);
|
||||
}
|
||||
int current = Math.max(0, Math.min(llmCompleted, total));
|
||||
vo.setFileProgressCurrent(current);
|
||||
vo.setFileProgressTotal(total);
|
||||
int percent = calculateDisplayProgressPercent(current, total, job, snapshot == null ? null : snapshot.getUpdatedAt());
|
||||
percent = Math.max(percent, calculateSnapshotDisplayPercent(snapshot, job));
|
||||
vo.setFileProgressPercent(Math.min(uploadComplete ? 99 : 98, percent));
|
||||
vo.setFileProgressMessage(uploadComplete
|
||||
? (llmPending > 0
|
||||
? buildLlmProgressMessage(current, total, llmPending)
|
||||
: "LLM 已回流 " + current + "/" + total + " 批次,正在生成结果文件")
|
||||
: buildUploadingLlmProgressMessage(current, total, llmPending));
|
||||
}
|
||||
|
||||
private void attachPythonUploadProgress(SimilarAsinHistoryItemVo vo, Long taskId) {
|
||||
PythonUploadProgress progress = resolvePythonUploadProgress(taskId);
|
||||
int total = progress.total() > 0 ? progress.total() : Math.max(1, progress.current() + 1);
|
||||
int current = Math.max(0, Math.min(progress.current(), total));
|
||||
int percent = current <= 0 ? 1 : Math.min(95, (int) Math.ceil(current * 100.0 / total));
|
||||
vo.setFileProgressCurrent(current);
|
||||
vo.setFileProgressTotal(total);
|
||||
vo.setFileProgressPercent(percent);
|
||||
vo.setFileProgressMessage(buildPythonUploadProgressMessage(current, total, progress.unit()));
|
||||
}
|
||||
|
||||
private PythonUploadProgress resolvePythonUploadProgress(Long taskId) {
|
||||
PythonUploadProgress chunkProgress = resolvePythonChunkProgress(taskId);
|
||||
if (chunkProgress.total() > 0) {
|
||||
return chunkProgress;
|
||||
}
|
||||
if (chunkProgress.current() > 0) {
|
||||
return new PythonUploadProgress(chunkProgress.current(), chunkProgress.current() + 1, chunkProgress.unit());
|
||||
}
|
||||
int totalRows = 0;
|
||||
try {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
totalRows = allRowCount(task);
|
||||
} catch (Exception ignored) {
|
||||
totalRows = 0;
|
||||
}
|
||||
int uploadedRows = countSubmittedRows(taskId);
|
||||
if (totalRows <= 0 && uploadedRows > 0) {
|
||||
totalRows = uploadedRows + 1;
|
||||
}
|
||||
return new PythonUploadProgress(uploadedRows, totalRows, "行");
|
||||
}
|
||||
|
||||
private PythonUploadProgress resolvePythonChunkProgress(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return new PythonUploadProgress(0, 0, "分片");
|
||||
}
|
||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.isNull(TaskScopeStateEntity::getLlmStatus)
|
||||
.orderByDesc(TaskScopeStateEntity::getUpdatedAt));
|
||||
if (states == null || states.isEmpty()) {
|
||||
return resolvePythonChunkProgressFromChunks(taskId);
|
||||
}
|
||||
int current = 0;
|
||||
int total = 0;
|
||||
for (TaskScopeStateEntity state : states) {
|
||||
if (state == null) {
|
||||
continue;
|
||||
}
|
||||
int stateTotal = state.getChunkTotal() == null ? 0 : state.getChunkTotal();
|
||||
int stateCurrent = state.getReceivedChunkCount() == null ? 0 : state.getReceivedChunkCount();
|
||||
stateCurrent = Math.max(stateCurrent, resolveReceivedChunkProgress(taskId, state.getScopeHash(), stateTotal));
|
||||
if (stateTotal > total || total <= 0 && stateCurrent > current) {
|
||||
total = stateTotal;
|
||||
current = stateCurrent;
|
||||
}
|
||||
}
|
||||
return new PythonUploadProgress(current, total, "分片");
|
||||
}
|
||||
|
||||
private PythonUploadProgress resolvePythonChunkProgressFromChunks(Long taskId) {
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return new PythonUploadProgress(0, 0, "分片");
|
||||
}
|
||||
Map<String, Integer> receivedByScope = new LinkedHashMap<>();
|
||||
Map<String, Integer> totalByScope = new LinkedHashMap<>();
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
if (chunk == null) {
|
||||
continue;
|
||||
}
|
||||
String scopeHash = firstNonBlank(chunk.getScopeHash(), "");
|
||||
receivedByScope.merge(scopeHash, 1, Integer::sum);
|
||||
if (chunk.getChunkIndex() != null && chunk.getChunkIndex() > 0) {
|
||||
receivedByScope.merge(scopeHash, chunk.getChunkIndex(), Math::max);
|
||||
}
|
||||
if (chunk.getChunkTotal() != null && chunk.getChunkTotal() > 0) {
|
||||
totalByScope.merge(scopeHash, chunk.getChunkTotal(), Math::max);
|
||||
}
|
||||
}
|
||||
int current = 0;
|
||||
int total = 0;
|
||||
for (Map.Entry<String, Integer> entry : receivedByScope.entrySet()) {
|
||||
int scopeCurrent = entry.getValue() == null ? 0 : entry.getValue();
|
||||
int scopeTotal = totalByScope.getOrDefault(entry.getKey(), 0);
|
||||
if (scopeTotal > total || total <= 0 && scopeCurrent > current) {
|
||||
current = scopeCurrent;
|
||||
total = scopeTotal;
|
||||
}
|
||||
}
|
||||
return new PythonUploadProgress(current, total, "分片");
|
||||
}
|
||||
|
||||
private int countSubmittedRows(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return 0;
|
||||
}
|
||||
int total = 0;
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
total += readChunkRows(chunk).size();
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
private String buildPythonUploadProgressMessage(int current, int total, String unit) {
|
||||
String safeUnit = unit == null || unit.isBlank() ? "分片" : unit;
|
||||
return "等待 Python 回传,已回传 " + Math.max(0, current) + "/" + Math.max(1, total) + " " + safeUnit;
|
||||
}
|
||||
|
||||
private String resolveDisplayTaskStatus(FileTaskEntity task) {
|
||||
return resolveDisplayTaskStatus(task, null);
|
||||
}
|
||||
@@ -5416,36 +5083,6 @@ public class SimilarAsinTaskService {
|
||||
return true;
|
||||
}
|
||||
|
||||
private int calculateLlmDisplayTotal(TaskProgressSnapshotEntity snapshot, int observedLlmStates) {
|
||||
int observed = Math.max(0, observedLlmStates);
|
||||
// 小任务 totalCount <= 3 时直接使用 observed,避免 totalCount - 3 → 0 引起进度条 0/0。
|
||||
if (snapshot == null || snapshot.getTotalCount() == null || snapshot.getTotalCount() <= 3) {
|
||||
return Math.max(1, observed);
|
||||
}
|
||||
int expectedFromSnapshot = Math.max(1, snapshot.getTotalCount() - 3);
|
||||
return Math.max(1, Math.max(observed, expectedFromSnapshot));
|
||||
}
|
||||
|
||||
private String buildLlmProgressMessage(int completed, int total, int pending) {
|
||||
int safeTotal = Math.max(1, total);
|
||||
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
|
||||
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
|
||||
if (safeCompleted <= 0 && submitted > 0) {
|
||||
return "LLM 已提交 " + submitted + "/" + safeTotal + " 批次,等待回流";
|
||||
}
|
||||
return "LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次,等待结果文件";
|
||||
}
|
||||
|
||||
private String buildUploadingLlmProgressMessage(int completed, int total, int pending) {
|
||||
int safeTotal = Math.max(1, total);
|
||||
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
|
||||
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
|
||||
if (pending > 0) {
|
||||
return "Python 仍在回传,LLM 已提交 " + submitted + "/" + safeTotal + " 批次";
|
||||
}
|
||||
return "Python 仍在回传,LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次";
|
||||
}
|
||||
|
||||
private String fmt(LocalDateTime t) {
|
||||
return t == null ? null : t.toString();
|
||||
}
|
||||
|
||||
+245
@@ -0,0 +1,245 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* 任务 83:SimilarAsinExcelParser 行解析器。
|
||||
* POI 读 Excel 行 → 中间行对象(原始单元格值)。语义与 SimilarAsinTaskService.parseWorkbook
|
||||
* 中对应段落逐字节一致:cell 归一化(BOM/全角空格/trim/连续空白折叠)、错误值转空、
|
||||
* 单字段截断 2000、表头别名匹配、空行跳过。
|
||||
*/
|
||||
@Slf4j
|
||||
public class SimilarAsinExcelParser {
|
||||
|
||||
private static final int DEFAULT_MAX_FIELD_LENGTH = 2000;
|
||||
|
||||
public ParsedSheet parse(File input) {
|
||||
return parse(input, DEFAULT_MAX_FIELD_LENGTH);
|
||||
}
|
||||
|
||||
public ParsedSheet parse(File input, int maxFieldLength) {
|
||||
if (input == null) {
|
||||
throw new IllegalArgumentException("input must not be null");
|
||||
}
|
||||
try (FileInputStream fis = new FileInputStream(input);
|
||||
Workbook workbook = WorkbookFactory.create(fis)) {
|
||||
return parseWorkbook(workbook, maxFieldLength);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] parse failed file={} err={}", input.getName(), ex.getMessage());
|
||||
throw new BusinessException("解析 Excel 失败");
|
||||
}
|
||||
}
|
||||
|
||||
public ParsedSheet parse(InputStream input) {
|
||||
if (input == null) {
|
||||
throw new IllegalArgumentException("input must not be null");
|
||||
}
|
||||
try (Workbook workbook = WorkbookFactory.create(input)) {
|
||||
return parseWorkbook(workbook, DEFAULT_MAX_FIELD_LENGTH);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] parse failed input err={}", ex.getMessage());
|
||||
throw new BusinessException("解析 Excel 失败");
|
||||
}
|
||||
}
|
||||
|
||||
private ParsedSheet parseWorkbook(Workbook workbook, int maxFieldLength) {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row header = sheet.getRow(0);
|
||||
if (header == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
|
||||
List<String> headers = readHeaders(header, formatter);
|
||||
int idCol = findRequiredHeader(headerMap, "id");
|
||||
int asinCol = findRequiredHeader(headerMap, "asin");
|
||||
int countryCol = findRequiredHeader(headerMap, "国家", "country");
|
||||
int skuCol = findOptionalHeaderExact(headerMap, "sku", "seller sku", "seller_sku", "msku", "货号");
|
||||
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
|
||||
int urlCol = findOptionalHeaderExact(headerMap,
|
||||
"url", "rul", "link", "image", "img", "pic", "picture",
|
||||
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
|
||||
int titleCol = findOptionalHeaderExact(headerMap,
|
||||
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
|
||||
|
||||
List<SimilarAsinExcelRow> rows = new ArrayList<>();
|
||||
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
|
||||
Row row = sheet.getRow(i);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
String id = cell(row, idCol, formatter, maxFieldLength);
|
||||
String asin = cell(row, asinCol, formatter, maxFieldLength).toUpperCase(Locale.ROOT);
|
||||
String country = cell(row, countryCol, formatter, maxFieldLength);
|
||||
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
rows.add(new SimilarAsinExcelRow(
|
||||
i + 1,
|
||||
id,
|
||||
asin,
|
||||
country,
|
||||
skuCol >= 0 ? cell(row, skuCol, formatter, maxFieldLength) : "",
|
||||
priceCol >= 0 ? cell(row, priceCol, formatter, maxFieldLength) : "",
|
||||
urlCol >= 0 ? cell(row, urlCol, formatter, maxFieldLength) : "",
|
||||
titleCol >= 0 ? cell(row, titleCol, formatter, maxFieldLength) : "",
|
||||
readRowValues(row, headers, formatter, maxFieldLength)));
|
||||
}
|
||||
return new ParsedSheet(headers, rows);
|
||||
}
|
||||
|
||||
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
|
||||
Map<String, Integer> map = new LinkedHashMap<>();
|
||||
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||
if (!val.isBlank()) {
|
||||
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private List<String> readHeaders(Row header, DataFormatter formatter) {
|
||||
List<String> headers = new ArrayList<>();
|
||||
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||
String val = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||
headers.add(val.isBlank() ? "列" + (i + 1) : val);
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter, int maxFieldLength) {
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
for (int i = 0; i < headers.size(); i++) {
|
||||
values.put(headers.get(i), cell(row, i, formatter, maxFieldLength));
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private int findRequiredHeader(Map<String, Integer> map, String... names) {
|
||||
int idx = findOptionalHeader(map, names);
|
||||
if (idx < 0) {
|
||||
throw new BusinessException("缺少必要表头: " + String.join("/", names));
|
||||
}
|
||||
return idx;
|
||||
}
|
||||
|
||||
private int findOptionalHeader(Map<String, Integer> map, String... names) {
|
||||
for (Map.Entry<String, Integer> entry : map.entrySet()) {
|
||||
for (String name : names) {
|
||||
if (entry.getKey().contains(name.toLowerCase(Locale.ROOT))) {
|
||||
return entry.getValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private int findOptionalHeaderExact(Map<String, Integer> map, String... names) {
|
||||
for (Map.Entry<String, Integer> entry : map.entrySet()) {
|
||||
String normalizedHeader = normalizeHeaderAlias(entry.getKey());
|
||||
for (String name : names) {
|
||||
if (normalizedHeader.equals(normalizeHeaderAlias(name))) {
|
||||
return entry.getValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private String normalizeHeaderAlias(String value) {
|
||||
String normalized = normalize(value).toLowerCase(Locale.ROOT);
|
||||
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
|
||||
}
|
||||
|
||||
private String cell(Row row, int col, DataFormatter formatter, int maxFieldLength) {
|
||||
if (col < 0) {
|
||||
return "";
|
||||
}
|
||||
String value = normalize(formatter.formatCellValue(row.getCell(col)));
|
||||
if (isSpreadsheetErrorValue(value)) {
|
||||
return "";
|
||||
}
|
||||
if (value.length() > maxFieldLength) {
|
||||
return value.substring(0, maxFieldLength);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private static boolean isSpreadsheetErrorValue(String value) {
|
||||
String normalized = value == null ? "" : value.trim();
|
||||
return normalized.startsWith("#") && normalized.endsWith("!");
|
||||
}
|
||||
|
||||
private String normalize(String val) {
|
||||
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
public record ParsedSheet(List<String> headers, List<SimilarAsinExcelRow> rows) {
|
||||
|
||||
public ParsedSheet {
|
||||
headers = headers == null ? List.of() : new ArrayList<>(headers);
|
||||
rows = rows == null ? List.of() : new ArrayList<>(rows);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> headers() {
|
||||
return java.util.Collections.unmodifiableList(headers);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<SimilarAsinExcelRow> rows() {
|
||||
return java.util.Collections.unmodifiableList(rows);
|
||||
}
|
||||
}
|
||||
|
||||
public record SimilarAsinExcelRow(int rowIndex, String id, String asin, String country,
|
||||
String sku, String price, String url, String title,
|
||||
Map<String, String> values) {
|
||||
|
||||
public SimilarAsinExcelRow {
|
||||
values = values == null ? Map.of() : new LinkedHashMap<>(values);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (!(o instanceof SimilarAsinExcelRow other)) {
|
||||
return false;
|
||||
}
|
||||
return rowIndex == other.rowIndex
|
||||
&& Objects.equals(id, other.id)
|
||||
&& Objects.equals(asin, other.asin)
|
||||
&& Objects.equals(country, other.country)
|
||||
&& Objects.equals(sku, other.sku)
|
||||
&& Objects.equals(price, other.price)
|
||||
&& Objects.equals(url, other.url)
|
||||
&& Objects.equals(title, other.title)
|
||||
&& values.equals(other.values);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(rowIndex, id, asin, country, sku, price, url, title, values);
|
||||
}
|
||||
}
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 任务 86:SimilarAsinGroupingConverter 分组转换器。
|
||||
* 归一化行 → 分组 DTO;分组规则与现状 SimilarAsinTaskService.buildParsedGroups 完全一致:
|
||||
* 按 groupKey(缺省兜底 displayId 的 baseId)分组、startIndex/endIndex 半开区间游标、
|
||||
* 组内行数、顺序保持行首次出现顺序。纯函数无状态。
|
||||
*/
|
||||
public final class SimilarAsinGroupingConverter {
|
||||
|
||||
private SimilarAsinGroupingConverter() {
|
||||
}
|
||||
|
||||
public static List<SimilarAsinParsedGroupVo> convert(List<SimilarAsinParsedRowVo> rows) {
|
||||
List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return groups;
|
||||
}
|
||||
int cursor = 0;
|
||||
for (List<SimilarAsinParsedRowVo> siblings : groupRowsByBaseId(rows).values()) {
|
||||
if (siblings == null || siblings.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
SimilarAsinParsedRowVo first = siblings.getFirst();
|
||||
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
|
||||
group.setSourceFileKey(first.getSourceFileKey());
|
||||
group.setSourceFilename(first.getSourceFilename());
|
||||
group.setGroupKey(firstNonBlank(first.getGroupKey(),
|
||||
buildGroupKey(first.getSourceFileKey(), baseId(first.getDisplayId()), first.getRowIndex())));
|
||||
group.setBaseId(baseId(first.getDisplayId()));
|
||||
group.setDisplayId(firstNonBlank(first.getDisplayId(), first.getSourceId()));
|
||||
group.setItemCount(siblings.size());
|
||||
int end = cursor + siblings.size();
|
||||
group.setStartIndex(cursor);
|
||||
group.setEndIndex(end);
|
||||
cursor = end;
|
||||
groups.add(group);
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
/** 行 → baseId 分组映射(LinkedHashMap 保序)。供 convert 及服务侧 loadAllRowsByBaseId 复用。 */
|
||||
public static Map<String, List<SimilarAsinParsedRowVo>> groupRowsByBaseId(List<SimilarAsinParsedRowVo> rows) {
|
||||
Map<String, List<SimilarAsinParsedRowVo>> result = new LinkedHashMap<>();
|
||||
for (SimilarAsinParsedRowVo row : rows) {
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
String key = firstNonBlank(SimilarAsinRowNormalizer.normalize(row.getGroupKey()),
|
||||
baseId(row.getDisplayId()));
|
||||
result.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static String baseId(String id) {
|
||||
String s = SimilarAsinRowNormalizer.normalize(id);
|
||||
int idx = s.indexOf('_');
|
||||
return idx > 0 ? s.substring(0, idx) : s;
|
||||
}
|
||||
|
||||
private static String buildGroupKey(String sourceFileKey, String rowBaseId, Integer rowIndex) {
|
||||
return SimilarAsinRowNormalizer.normalize(sourceFileKey) + "::"
|
||||
+ SimilarAsinRowNormalizer.normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex);
|
||||
}
|
||||
|
||||
private static String firstNonBlank(String preferred, String fallback) {
|
||||
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||
}
|
||||
}
|
||||
+757
@@ -0,0 +1,757 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryItemVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* 任务 91:SimilarAsinHistoryAssembler 历史查询组装器。
|
||||
* 历史列表 VO 拼装(toHistoryItem + 进度链:file job 状态、LLM 回流进度、Python 上传进度、
|
||||
* 快照百分比)从 SimilarAsinTaskService 原样搬移;只读不落库;输出与现状逐字段一致。
|
||||
*/
|
||||
@Slf4j
|
||||
public class SimilarAsinHistoryAssembler {
|
||||
|
||||
private static final String MODULE_TYPE = "SIMILAR_ASIN";
|
||||
private static final String STATUS_PENDING = "PENDING";
|
||||
private static final String STATUS_RUNNING = "RUNNING";
|
||||
private static final String STATUS_SUCCESS = "SUCCESS";
|
||||
private static final String LLM_STATUS_SUBMITTED = "SUBMITTED";
|
||||
private static final String LLM_STATUS_RUNNING = "RUNNING";
|
||||
private static final String LLM_STATUS_DONE = "DONE";
|
||||
private static final String LLM_STATUS_FAILED = "FAILED";
|
||||
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final TaskChunkMapper taskChunkMapper;
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public SimilarAsinHistoryAssembler(TaskScopeStateMapper taskScopeStateMapper,
|
||||
TaskChunkMapper taskChunkMapper,
|
||||
FileTaskMapper fileTaskMapper,
|
||||
TaskProgressSnapshotService taskProgressSnapshotService,
|
||||
OssStorageService ossStorageService,
|
||||
TransientPayloadStorageService transientPayloadStorageService,
|
||||
ObjectMapper objectMapper) {
|
||||
this.taskScopeStateMapper = taskScopeStateMapper;
|
||||
this.taskChunkMapper = taskChunkMapper;
|
||||
this.fileTaskMapper = fileTaskMapper;
|
||||
this.taskProgressSnapshotService = taskProgressSnapshotService;
|
||||
this.ossStorageService = ossStorageService;
|
||||
this.transientPayloadStorageService = transientPayloadStorageService;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按 rows 顺序组装历史项;PENDING 任务(history 语义)跳过。
|
||||
* taskMap/jobMap 均为调用方已查好的映射,本方法只读不改输入。
|
||||
*/
|
||||
public List<SimilarAsinHistoryItemVo> buildHistoryItems(List<FileResultEntity> rows,
|
||||
Map<Long, FileTaskEntity> taskMap,
|
||||
Map<Long, TaskFileJobEntity> jobMap) {
|
||||
List<SimilarAsinHistoryItemVo> items = new ArrayList<>();
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return items;
|
||||
}
|
||||
for (FileResultEntity row : rows) {
|
||||
FileTaskEntity task = taskMap.get(row.getTaskId());
|
||||
String taskStatus = task == null ? null : task.getStatus();
|
||||
if (STATUS_PENDING.equals(taskStatus)) {
|
||||
continue;
|
||||
}
|
||||
items.add(toHistoryItem(row, task, jobMap.get(row.getId())));
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
private SimilarAsinHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
||||
SimilarAsinHistoryItemVo vo = new SimilarAsinHistoryItemVo();
|
||||
vo.setResultId(row.getId());
|
||||
vo.setTaskId(row.getTaskId());
|
||||
vo.setSourceFilename(row.getSourceFilename());
|
||||
vo.setResultFilename(row.getResultFilename());
|
||||
vo.setDownloadUrl(buildFreshDownloadUrl(row));
|
||||
vo.setTaskStatus(resolveDisplayTaskStatus(task, row));
|
||||
attachFileJobState(vo, row, job, task);
|
||||
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
|
||||
vo.setError(row.getErrorMessage());
|
||||
vo.setRowCount(row.getRowCount());
|
||||
vo.setCreatedAt(fmt(row.getCreatedAt()));
|
||||
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
|
||||
vo.setStartedAt(fmt(task == null ? row.getCreatedAt() : task.getCreatedAt()));
|
||||
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
|
||||
return vo;
|
||||
}
|
||||
|
||||
private String buildFreshDownloadUrl(FileResultEntity row) {
|
||||
String resultFileUrl = row == null ? null : row.getResultFileUrl();
|
||||
if (resultFileUrl == null || resultFileUrl.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
// bucket 对 result/ 前缀已开放公开读,使用无签名公开 URL,避免签名 1 小时过期后下载 403。
|
||||
// result/ 前缀走 download-endpoint 独立下载域名(带宽/缓存分流),非 result/(如图片)仍走公开端点。
|
||||
return ossStorageService.generateFreshDownloadUrl(resultFileUrl);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] build public download url failed resultId={} err={}",
|
||||
row.getId(), ex.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void attachFileJobState(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
|
||||
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
|
||||
if (job == null) {
|
||||
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
|
||||
attachFileProgress(vo, row, job, task);
|
||||
return;
|
||||
}
|
||||
vo.setFileJobId(job.getId());
|
||||
vo.setFileStatus(job.getStatus());
|
||||
vo.setFileError(job.getErrorMessage());
|
||||
attachFileProgress(vo, row, job, task);
|
||||
}
|
||||
|
||||
private void attachFileProgress(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
|
||||
if (row == null || row.getTaskId() == null) {
|
||||
return;
|
||||
}
|
||||
Long taskId = row.getTaskId();
|
||||
if (Boolean.TRUE.equals(vo.getFileReady())) {
|
||||
vo.setFileProgressCurrent(1);
|
||||
vo.setFileProgressTotal(1);
|
||||
vo.setFileProgressPercent(100);
|
||||
vo.setFileProgressMessage("结果文件已生成");
|
||||
return;
|
||||
}
|
||||
int llmCompleted = countCompletedLlmStates(taskId);
|
||||
int llmPending = countPendingLlmStates(taskId);
|
||||
boolean uploadComplete = isResultSubmissionComplete(taskId);
|
||||
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(taskId, MODULE_TYPE);
|
||||
if (!uploadComplete) {
|
||||
if (canTreatTaskAsCompleted(task, row, job)) {
|
||||
vo.setFileProgressCurrent(1);
|
||||
vo.setFileProgressTotal(1);
|
||||
vo.setFileProgressPercent(100);
|
||||
vo.setFileProgressMessage("结果文件已生成");
|
||||
return;
|
||||
}
|
||||
if (task != null && !STATUS_RUNNING.equals(task.getStatus())) {
|
||||
return;
|
||||
}
|
||||
if (llmCompleted + llmPending > 0) {
|
||||
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, false);
|
||||
return;
|
||||
}
|
||||
attachPythonUploadProgress(vo, taskId);
|
||||
return;
|
||||
}
|
||||
if (llmCompleted + llmPending > 0) {
|
||||
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, true);
|
||||
return;
|
||||
}
|
||||
if (snapshot == null) {
|
||||
return;
|
||||
}
|
||||
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
|
||||
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
|
||||
if (total <= 0) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : (job == null ? null : job.getUpdatedAt());
|
||||
int percent = calculateDisplayProgressPercent(current, total, job, baseTime);
|
||||
percent = Math.max(percent, extractSnapshotDisplayPercent(snapshot));
|
||||
percent = Boolean.TRUE.equals(vo.getFileReady()) ? 100 : Math.min(99, percent);
|
||||
vo.setFileProgressCurrent(current);
|
||||
vo.setFileProgressTotal(total);
|
||||
vo.setFileProgressPercent(percent);
|
||||
vo.setFileProgressMessage(snapshot.getMessage());
|
||||
}
|
||||
|
||||
private void attachLlmProgress(SimilarAsinHistoryItemVo vo,
|
||||
TaskFileJobEntity job,
|
||||
TaskProgressSnapshotEntity snapshot,
|
||||
int llmCompleted,
|
||||
int llmPending,
|
||||
boolean uploadComplete) {
|
||||
int observed = Math.max(0, llmCompleted) + Math.max(0, llmPending);
|
||||
int total = calculateLlmDisplayTotal(snapshot, observed);
|
||||
if (!uploadComplete) {
|
||||
total = Math.max(total, observed + 1);
|
||||
}
|
||||
int current = Math.max(0, Math.min(llmCompleted, total));
|
||||
vo.setFileProgressCurrent(current);
|
||||
vo.setFileProgressTotal(total);
|
||||
int percent = calculateDisplayProgressPercent(current, total, job, snapshot == null ? null : snapshot.getUpdatedAt());
|
||||
percent = Math.max(percent, calculateSnapshotDisplayPercent(snapshot, job));
|
||||
vo.setFileProgressPercent(Math.min(uploadComplete ? 99 : 98, percent));
|
||||
vo.setFileProgressMessage(uploadComplete
|
||||
? (llmPending > 0
|
||||
? buildLlmProgressMessage(current, total, llmPending)
|
||||
: "LLM 已回流 " + current + "/" + total + " 批次,正在生成结果文件")
|
||||
: buildUploadingLlmProgressMessage(current, total, llmPending));
|
||||
}
|
||||
|
||||
private void attachPythonUploadProgress(SimilarAsinHistoryItemVo vo, Long taskId) {
|
||||
PythonUploadProgress progress = resolvePythonUploadProgress(taskId);
|
||||
int total = progress.total() > 0 ? progress.total() : Math.max(1, progress.current() + 1);
|
||||
int current = Math.max(0, Math.min(progress.current(), total));
|
||||
int percent = current <= 0 ? 1 : Math.min(95, (int) Math.ceil(current * 100.0 / total));
|
||||
vo.setFileProgressCurrent(current);
|
||||
vo.setFileProgressTotal(total);
|
||||
vo.setFileProgressPercent(percent);
|
||||
vo.setFileProgressMessage(buildPythonUploadProgressMessage(current, total, progress.unit()));
|
||||
}
|
||||
|
||||
private PythonUploadProgress resolvePythonUploadProgress(Long taskId) {
|
||||
PythonUploadProgress chunkProgress = resolvePythonChunkProgress(taskId);
|
||||
if (chunkProgress.total() > 0) {
|
||||
return chunkProgress;
|
||||
}
|
||||
if (chunkProgress.current() > 0) {
|
||||
return new PythonUploadProgress(chunkProgress.current(), chunkProgress.current() + 1, chunkProgress.unit());
|
||||
}
|
||||
int totalRows = 0;
|
||||
try {
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
totalRows = allRowCount(task);
|
||||
} catch (Exception ignored) {
|
||||
totalRows = 0;
|
||||
}
|
||||
int uploadedRows = countSubmittedRows(taskId);
|
||||
if (totalRows <= 0 && uploadedRows > 0) {
|
||||
totalRows = uploadedRows + 1;
|
||||
}
|
||||
return new PythonUploadProgress(uploadedRows, totalRows, "行");
|
||||
}
|
||||
|
||||
private int allRowCount(FileTaskEntity task) {
|
||||
try {
|
||||
return rowCount(readParsedPayload(task));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin] read all row count failed taskId={} err={}", task.getId(), ex.getMessage());
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private int rowCount(SimilarAsinParsedPayloadDto payload) {
|
||||
if (payload == null) {
|
||||
return 0;
|
||||
}
|
||||
return resolveAllRows(payload).size();
|
||||
}
|
||||
|
||||
private SimilarAsinParsedPayloadDto readParsedPayload(FileTaskEntity task) {
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(task.getResultJson());
|
||||
String pointer = root.path("parsedPayloadRef").asText("");
|
||||
if (!pointer.isBlank()) {
|
||||
String json = transientPayloadStorageService.resolvePayload(pointer, "read similar ASIN parsed payload failed");
|
||||
return hydrateParsedPayloadRows(objectMapper.readValue(json, SimilarAsinParsedPayloadDto.class));
|
||||
}
|
||||
if (root.path("allItems").isArray()) {
|
||||
return hydrateParsedPayloadRows(objectMapper.treeToValue(root, SimilarAsinParsedPayloadDto.class));
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取解析载荷失败");
|
||||
}
|
||||
return hydrateParsedPayloadRows(new SimilarAsinParsedPayloadDto());
|
||||
}
|
||||
|
||||
private SimilarAsinParsedPayloadDto hydrateParsedPayloadRows(SimilarAsinParsedPayloadDto payload) {
|
||||
if (payload == null) {
|
||||
return new SimilarAsinParsedPayloadDto();
|
||||
}
|
||||
List<SimilarAsinParsedRowVo> rows = resolveAllRows(payload);
|
||||
if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) {
|
||||
payload.setAllItems(rows);
|
||||
}
|
||||
if (payload.getItems() == null || payload.getItems().isEmpty()) {
|
||||
payload.setItems(rows);
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从解析载荷统一恢复全量行:优先 items(新规范结构),其次 allItems(旧结构),
|
||||
* 最后 groups 展开(最旧结构)。新旧结构均可恢复完整行集合,供 rowCount、
|
||||
* LLM 候选加载与结果文件组装复用。
|
||||
*/
|
||||
private static List<SimilarAsinParsedRowVo> resolveAllRows(SimilarAsinParsedPayloadDto payload) {
|
||||
if (payload == null) {
|
||||
return List.of();
|
||||
}
|
||||
if (containsGroupRefs(payload)) {
|
||||
// 新格式:分组携带索引引用,信任引用展开结果(越界/非法区间安全跳过)
|
||||
return expandGroupRefs(payload);
|
||||
}
|
||||
List<SimilarAsinParsedRowVo> rows = payload.getItems();
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
rows = payload.getAllItems();
|
||||
}
|
||||
if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) {
|
||||
rows = payload.getGroups().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream())
|
||||
.toList();
|
||||
}
|
||||
return rows == null ? List.of() : rows;
|
||||
}
|
||||
|
||||
private static boolean containsGroupRefs(SimilarAsinParsedPayloadDto payload) {
|
||||
if (payload.getGroups() == null) {
|
||||
return false;
|
||||
}
|
||||
for (SimilarAsinParsedGroupVo group : payload.getGroups()) {
|
||||
if (group != null && (group.getStartIndex() != null || group.getEndIndex() != null)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按分组引用 [startIndex, endIndex) 从 items 展开组内行。
|
||||
* 引用越界或区间非法时安全跳过,不抛异常;展开不修改 payload 内部状态。
|
||||
*/
|
||||
private static List<SimilarAsinParsedRowVo> expandGroupRefs(SimilarAsinParsedPayloadDto payload) {
|
||||
if (payload == null || payload.getGroups() == null || payload.getGroups().isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<SimilarAsinParsedRowVo> rows = payload.getItems();
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
rows = payload.getAllItems();
|
||||
}
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>();
|
||||
for (SimilarAsinParsedGroupVo group : payload.getGroups()) {
|
||||
if (group == null) {
|
||||
continue;
|
||||
}
|
||||
int start = group.getStartIndex() == null ? 0 : group.getStartIndex();
|
||||
int end = group.getEndIndex() == null ? 0 : group.getEndIndex();
|
||||
if (start < 0 || end <= start || end > rows.size()) {
|
||||
continue;
|
||||
}
|
||||
expanded.addAll(rows.subList(start, end));
|
||||
}
|
||||
return expanded;
|
||||
}
|
||||
|
||||
private int countSubmittedRows(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return 0;
|
||||
}
|
||||
int total = 0;
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
total += readChunkRows(chunk).size();
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
private Map<String, SimilarAsinResultRowDto> readChunkRows(TaskChunkEntity chunk) {
|
||||
Map<String, SimilarAsinResultRowDto> rows = new LinkedHashMap<>();
|
||||
if (chunk == null || chunk.getPayloadJson() == null || chunk.getPayloadJson().isBlank()) {
|
||||
return rows;
|
||||
}
|
||||
try {
|
||||
String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read similar ASIN chunk failed");
|
||||
JsonNode array = objectMapper.readTree(payloadJson);
|
||||
if (!array.isArray()) {
|
||||
return rows;
|
||||
}
|
||||
for (JsonNode node : array) {
|
||||
SimilarAsinResultRowDto row = objectMapper.treeToValue(node, SimilarAsinResultRowDto.class);
|
||||
rows.put(rowKey(row), row);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
String msg = ex.getMessage() == null ? "" : ex.getMessage();
|
||||
// P1-4:识别跨实例 local 指针读不到的场景,把"chunk 在另一实例"的元信息
|
||||
// 通过 task_scope_state.last_error 留痕,便于排查"为何 owner 切换后 chunk 读不到"。
|
||||
boolean crossInstance = msg.contains("only exists on instance=");
|
||||
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
|
||||
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
|
||||
recordChunkReadFailure(chunk, crossInstance, msg);
|
||||
throw new BusinessException("similar ASIN chunk payload read failed chunk="
|
||||
+ chunk.getChunkIndex() + ": " + msg, ex);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* 把 chunk 读失败信息聚合到 task_scope_state.last_error,
|
||||
* 形式 "chunk-read-failed[{idx}]" / "chunk-read-failed[{idx}@cross-instance]"。
|
||||
* best-effort:查询 / 更新失败时仅记日志,不抛回主流程。
|
||||
*/
|
||||
private void recordChunkReadFailure(TaskChunkEntity chunk, boolean crossInstance, String msg) {
|
||||
if (chunk == null || chunk.getTaskId() == null || chunk.getScopeHash() == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, chunk.getTaskId())
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskScopeStateEntity::getScopeHash, chunk.getScopeHash())
|
||||
.last("limit 1"));
|
||||
if (scope == null) {
|
||||
return;
|
||||
}
|
||||
String existing = scope.getLastError() == null ? "" : scope.getLastError();
|
||||
String tag = "chunk-read-failed[" + chunk.getChunkIndex() + (crossInstance ? "@cross-instance" : "") + "]";
|
||||
if (existing.contains(tag)) {
|
||||
return;
|
||||
}
|
||||
String updated = existing.isBlank() ? tag : existing + "; " + tag;
|
||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getId, scope.getId())
|
||||
.set(TaskScopeStateEntity::getLastError, updated)
|
||||
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
} catch (Exception ignored) {
|
||||
// best-effort:失败不影响主流程
|
||||
}
|
||||
}
|
||||
|
||||
private String rowKey(SimilarAsinResultRowDto row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
}
|
||||
String rowToken = normalize(row.getRowToken());
|
||||
if (!rowToken.isBlank()) {
|
||||
return rowToken;
|
||||
}
|
||||
return legacyRowKey(row);
|
||||
}
|
||||
|
||||
private String legacyRowKey(SimilarAsinResultRowDto row) {
|
||||
if (row == null) {
|
||||
return "";
|
||||
}
|
||||
return rowKey(row.getId(), row.getAsin(), row.getCountry());
|
||||
}
|
||||
|
||||
private String rowKey(String id, String asin, String country) {
|
||||
return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
|
||||
}
|
||||
|
||||
private String normalize(String val) {
|
||||
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
private PythonUploadProgress resolvePythonChunkProgress(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return new PythonUploadProgress(0, 0, "分片");
|
||||
}
|
||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.isNull(TaskScopeStateEntity::getLlmStatus)
|
||||
.orderByDesc(TaskScopeStateEntity::getUpdatedAt));
|
||||
if (states == null || states.isEmpty()) {
|
||||
return resolvePythonChunkProgressFromChunks(taskId);
|
||||
}
|
||||
int current = 0;
|
||||
int total = 0;
|
||||
for (TaskScopeStateEntity state : states) {
|
||||
if (state == null) {
|
||||
continue;
|
||||
}
|
||||
int stateTotal = state.getChunkTotal() == null ? 0 : state.getChunkTotal();
|
||||
int stateCurrent = state.getReceivedChunkCount() == null ? 0 : state.getReceivedChunkCount();
|
||||
stateCurrent = Math.max(stateCurrent, resolveReceivedChunkProgress(taskId, state.getScopeHash(), stateTotal));
|
||||
if (stateTotal > total || total <= 0 && stateCurrent > current) {
|
||||
total = stateTotal;
|
||||
current = stateCurrent;
|
||||
}
|
||||
}
|
||||
return new PythonUploadProgress(current, total, "分片");
|
||||
}
|
||||
|
||||
private PythonUploadProgress resolvePythonChunkProgressFromChunks(Long taskId) {
|
||||
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return new PythonUploadProgress(0, 0, "分片");
|
||||
}
|
||||
Map<String, Integer> receivedByScope = new LinkedHashMap<>();
|
||||
Map<String, Integer> totalByScope = new LinkedHashMap<>();
|
||||
for (TaskChunkEntity chunk : chunks) {
|
||||
if (chunk == null) {
|
||||
continue;
|
||||
}
|
||||
String scopeHash = firstNonBlank(chunk.getScopeHash(), "");
|
||||
receivedByScope.merge(scopeHash, 1, Integer::sum);
|
||||
if (chunk.getChunkIndex() != null && chunk.getChunkIndex() > 0) {
|
||||
receivedByScope.merge(scopeHash, chunk.getChunkIndex(), Math::max);
|
||||
}
|
||||
if (chunk.getChunkTotal() != null && chunk.getChunkTotal() > 0) {
|
||||
totalByScope.merge(scopeHash, chunk.getChunkTotal(), Math::max);
|
||||
}
|
||||
}
|
||||
int current = 0;
|
||||
int total = 0;
|
||||
for (Map.Entry<String, Integer> entry : receivedByScope.entrySet()) {
|
||||
int scopeCurrent = entry.getValue() == null ? 0 : entry.getValue();
|
||||
int scopeTotal = totalByScope.getOrDefault(entry.getKey(), 0);
|
||||
if (scopeTotal > total || total <= 0 && scopeCurrent > current) {
|
||||
current = scopeCurrent;
|
||||
total = scopeTotal;
|
||||
}
|
||||
}
|
||||
return new PythonUploadProgress(current, total, "分片");
|
||||
}
|
||||
|
||||
private int resolveReceivedChunkProgress(Long taskId, String scopeHash, Integer chunkTotal) {
|
||||
int receivedCount = countChunks(taskId, scopeHash);
|
||||
Integer maxChunkIndex = taskChunkMapper.selectObjs(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.select(TaskChunkEntity::getChunkIndex)
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||
.orderByDesc(TaskChunkEntity::getChunkIndex)
|
||||
.last("limit 1"))
|
||||
.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(value -> {
|
||||
if (value instanceof Number number) {
|
||||
return number.intValue();
|
||||
}
|
||||
try {
|
||||
return Integer.parseInt(value.toString());
|
||||
} catch (Exception ignored) {
|
||||
return 0;
|
||||
}
|
||||
})
|
||||
.findFirst()
|
||||
.orElse(0);
|
||||
int current = Math.max(receivedCount, maxChunkIndex == null ? 0 : maxChunkIndex);
|
||||
if (chunkTotal != null && chunkTotal > 0) {
|
||||
current = Math.min(current, chunkTotal);
|
||||
}
|
||||
return Math.max(0, current);
|
||||
}
|
||||
|
||||
private int countChunks(Long taskId, String scopeHash) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return 0;
|
||||
}
|
||||
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(TaskChunkEntity::getScopeHash, scopeHash));
|
||||
return count == null ? 0 : count.intValue();
|
||||
}
|
||||
|
||||
private String buildPythonUploadProgressMessage(int current, int total, String unit) {
|
||||
String safeUnit = unit == null || unit.isBlank() ? "分片" : unit;
|
||||
return "等待 Python 回传,已回传 " + Math.max(0, current) + "/" + Math.max(1, total) + " " + safeUnit;
|
||||
}
|
||||
|
||||
private String resolveDisplayTaskStatus(FileTaskEntity task, FileResultEntity row) {
|
||||
if (task == null) {
|
||||
return null;
|
||||
}
|
||||
if (canTreatTaskAsCompleted(task, row, null)) {
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
return task.getStatus();
|
||||
}
|
||||
|
||||
private boolean canTreatTaskAsCompleted(FileTaskEntity task, FileResultEntity row, TaskFileJobEntity job) {
|
||||
if (task == null) {
|
||||
return false;
|
||||
}
|
||||
if (STATUS_SUCCESS.equals(task.getStatus())) {
|
||||
return true;
|
||||
}
|
||||
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||
return false;
|
||||
}
|
||||
if (row == null) {
|
||||
return false;
|
||||
}
|
||||
if (row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) {
|
||||
return false;
|
||||
}
|
||||
if (job != null && "FAILED".equals(job.getStatus())) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private int calculateLlmDisplayTotal(TaskProgressSnapshotEntity snapshot, int observedLlmStates) {
|
||||
int observed = Math.max(0, observedLlmStates);
|
||||
// 小任务 totalCount <= 3 时直接使用 observed,避免 totalCount - 3 → 0 引起进度条 0/0。
|
||||
if (snapshot == null || snapshot.getTotalCount() == null || snapshot.getTotalCount() <= 3) {
|
||||
return Math.max(1, observed);
|
||||
}
|
||||
int expectedFromSnapshot = Math.max(1, snapshot.getTotalCount() - 3);
|
||||
return Math.max(1, Math.max(observed, expectedFromSnapshot));
|
||||
}
|
||||
|
||||
private String buildLlmProgressMessage(int completed, int total, int pending) {
|
||||
int safeTotal = Math.max(1, total);
|
||||
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
|
||||
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
|
||||
if (safeCompleted <= 0 && submitted > 0) {
|
||||
return "LLM 已提交 " + submitted + "/" + safeTotal + " 批次,等待回流";
|
||||
}
|
||||
return "LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次,等待结果文件";
|
||||
}
|
||||
|
||||
private String buildUploadingLlmProgressMessage(int completed, int total, int pending) {
|
||||
int safeTotal = Math.max(1, total);
|
||||
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
|
||||
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
|
||||
if (pending > 0) {
|
||||
return "Python 仍在回传,LLM 已提交 " + submitted + "/" + safeTotal + " 批次";
|
||||
}
|
||||
return "Python 仍在回传,LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次";
|
||||
}
|
||||
|
||||
private int calculateDisplayProgressPercent(int current,
|
||||
int total,
|
||||
TaskFileJobEntity job,
|
||||
LocalDateTime baseTime) {
|
||||
if (total <= 0) {
|
||||
return 0;
|
||||
}
|
||||
current = Math.max(0, Math.min(current, total));
|
||||
int percent = Math.max(1, Math.min(99, (int) Math.floor(current * 100.0 / total)));
|
||||
if (job != null && STATUS_RUNNING.equals(job.getStatus())) {
|
||||
long elapsedSeconds = baseTime == null ? 0 : Math.max(0, Duration.between(baseTime, LocalDateTime.now()).getSeconds());
|
||||
if (current <= 0) {
|
||||
int firstRealProgressPercent = Math.max(1, Math.min(99, (int) Math.floor(100.0 / total)));
|
||||
int waitingCap = Math.max(8, Math.min(35, firstRealProgressPercent - 1));
|
||||
percent = Math.max(percent, Math.min(waitingCap, 8 + (int) (elapsedSeconds / 6)));
|
||||
} else if (current < total) {
|
||||
percent = Math.max(percent, Math.min(92, percent + (int) (elapsedSeconds / 10)));
|
||||
}
|
||||
}
|
||||
return percent;
|
||||
}
|
||||
|
||||
private int calculateSnapshotDisplayPercent(TaskProgressSnapshotEntity snapshot, TaskFileJobEntity job) {
|
||||
if (snapshot == null) {
|
||||
return 0;
|
||||
}
|
||||
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
|
||||
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
|
||||
int percent = total <= 0 ? 0 : calculateDisplayProgressPercent(current, total, job, snapshot.getUpdatedAt());
|
||||
return Math.max(percent, extractSnapshotDisplayPercent(snapshot));
|
||||
}
|
||||
|
||||
private int extractSnapshotDisplayPercent(TaskProgressSnapshotEntity snapshot) {
|
||||
if (snapshot == null || snapshot.getSnapshotJson() == null || snapshot.getSnapshotJson().isBlank()) {
|
||||
return 0;
|
||||
}
|
||||
try {
|
||||
JsonNode displayPercent = objectMapper.readTree(snapshot.getSnapshotJson()).path("displayPercent");
|
||||
if (!displayPercent.isNumber()) {
|
||||
return 0;
|
||||
}
|
||||
return Math.max(0, Math.min(100, displayPercent.asInt()));
|
||||
} catch (Exception ignored) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private int countPendingLlmStates(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return 0;
|
||||
}
|
||||
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_SUBMITTED, LLM_STATUS_RUNNING)));
|
||||
return count == null ? 0 : count.intValue();
|
||||
}
|
||||
|
||||
private int countCompletedLlmStates(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return 0;
|
||||
}
|
||||
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_DONE, LLM_STATUS_FAILED)));
|
||||
return count == null ? 0 : count.intValue();
|
||||
}
|
||||
|
||||
private boolean isResultSubmissionComplete(Long taskId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return false;
|
||||
}
|
||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||
.isNull(TaskScopeStateEntity::getLlmStatus)
|
||||
.isNotNull(TaskScopeStateEntity::getLastChunkAt));
|
||||
if (states == null || states.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
boolean hasCompletedScope = false;
|
||||
for (TaskScopeStateEntity state : states) {
|
||||
if (state == null) {
|
||||
continue;
|
||||
}
|
||||
if (Integer.valueOf(1).equals(state.getCompleted())) {
|
||||
hasCompletedScope = true;
|
||||
continue;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return hasCompletedScope;
|
||||
}
|
||||
|
||||
private String fmt(LocalDateTime t) {
|
||||
return t == null ? null : t.toString();
|
||||
}
|
||||
|
||||
private String firstNonBlank(String preferred, String fallback) {
|
||||
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||
}
|
||||
|
||||
private record PythonUploadProgress(int current, int total, String unit) {
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* 任务 85:SimilarAsinRowNormalizer 字段归一化器。
|
||||
* 规则与 SimilarAsinTaskService.normalize 现状逐字节一致(BOM 剥离、全角空格转半角、
|
||||
* trim、连续空白折叠为单空格);normalizeAsin = normalize + 大写。纯函数无状态。
|
||||
*/
|
||||
public final class SimilarAsinRowNormalizer {
|
||||
|
||||
private SimilarAsinRowNormalizer() {
|
||||
}
|
||||
|
||||
public static String normalize(String val) {
|
||||
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
public static String normalizeAsin(String val) {
|
||||
return normalize(val).toUpperCase(Locale.ROOT);
|
||||
}
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 任务 87:SimilarAsinRowValidator 纯校验器。
|
||||
* 必填检查(id/asin/country)、ASIN 格式校验(仅允许字母数字,含校验前大写化)、
|
||||
* 重复检测(asin+country 键,保留首次出现)。返回错误清单不抛异常,由调用方决定处理。
|
||||
* 规则与现状 parseWorkbook 必填丢弃 / dedupeRowsByRowKey 保留首次语义一致。
|
||||
*/
|
||||
public final class SimilarAsinRowValidator {
|
||||
|
||||
public record RowError(int rowIndex, String column, String message) {
|
||||
}
|
||||
|
||||
private SimilarAsinRowValidator() {
|
||||
}
|
||||
|
||||
public static List<RowError> validate(List<SimilarAsinParsedRowVo> rows) {
|
||||
List<RowError> errors = new ArrayList<>();
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return errors;
|
||||
}
|
||||
Set<String> seenKeys = new java.util.HashSet<>();
|
||||
for (SimilarAsinParsedRowVo row : rows) {
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
String id = SimilarAsinRowNormalizer.normalize(row.getSourceId());
|
||||
String asin = SimilarAsinRowNormalizer.normalizeAsin(row.getAsin());
|
||||
String country = SimilarAsinRowNormalizer.normalize(row.getCountry());
|
||||
if (id.isEmpty() && asin.isEmpty() && country.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
int rowIndex = row.getRowIndex() == null ? 0 : row.getRowIndex();
|
||||
if (id.isEmpty()) {
|
||||
errors.add(new RowError(rowIndex, "id", "缺少必要字段: id"));
|
||||
}
|
||||
if (asin.isEmpty()) {
|
||||
errors.add(new RowError(rowIndex, "asin", "缺少必要字段: asin"));
|
||||
}
|
||||
if (country.isEmpty()) {
|
||||
errors.add(new RowError(rowIndex, "country", "缺少必要字段: 国家"));
|
||||
}
|
||||
if (asin.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (!asin.matches("[A-Z0-9]+")) {
|
||||
errors.add(new RowError(rowIndex, "asin",
|
||||
"ASIN 格式不正确(仅允许字母数字): " + asin));
|
||||
continue;
|
||||
}
|
||||
String dupKey = asin + "::" + country;
|
||||
if (!seenKeys.add(dupKey)) {
|
||||
errors.add(new RowError(rowIndex, "asin",
|
||||
"重复 ASIN: " + asin + " 国家: " + country));
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service.support;
|
||||
|
||||
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.CellStyle;
|
||||
import org.apache.poi.ss.usermodel.Font;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 任务 88:SimilarAsinSheetBuilder Sheet 构造器。
|
||||
* 结果 Workbook/Sheet 构造辅助(表头、列顺序、样式);输入数据 → 输出 workbook;不落库、无 IO 依赖。
|
||||
* 表头/列序/样式与现状 assembleResultWorkbook 一致:sheet 名"相似asin检测"、15 列表头、
|
||||
* 加粗表头、数据行从第 1 行、图片列 12-14 列宽、行高 409pt、值兜底与现状一致。
|
||||
* 结果派生列(是否有货/相似度/是否符合类目/不符合理由/产品类目/状态)来自 LLM 结果行,
|
||||
* 本 builder 输入只有解析行,这些列由调用方补写。
|
||||
*/
|
||||
public final class SimilarAsinSheetBuilder {
|
||||
|
||||
public static final List<String> RESULT_HEADERS = List.of(
|
||||
"id",
|
||||
"asin",
|
||||
"国家",
|
||||
"价格",
|
||||
"卖家名称",
|
||||
"品牌",
|
||||
"是否有货",
|
||||
"相似度",
|
||||
"是否符合类目",
|
||||
"不符合理由",
|
||||
"产品类目",
|
||||
"状态",
|
||||
"主图",
|
||||
"阿里巴巴图片1",
|
||||
"阿里巴巴图片2"
|
||||
);
|
||||
|
||||
public static final int IMG_COL_MAIN = 12;
|
||||
public static final int IMG_COL_PUZZLE1 = 13;
|
||||
public static final int IMG_COL_PUZZLE2 = 14;
|
||||
|
||||
private SimilarAsinSheetBuilder() {
|
||||
}
|
||||
|
||||
public static void build(Workbook workbook, List<SimilarAsinParsedRowVo> rows) {
|
||||
Sheet sheet = workbook.createSheet("相似asin检测");
|
||||
CellStyle headerStyle = workbook.createCellStyle();
|
||||
Font font = workbook.createFont();
|
||||
font.setBold(true);
|
||||
headerStyle.setFont(font);
|
||||
|
||||
sheet.setColumnWidth(IMG_COL_MAIN, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256);
|
||||
sheet.setColumnWidth(IMG_COL_PUZZLE1, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256);
|
||||
sheet.setColumnWidth(IMG_COL_PUZZLE2, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256);
|
||||
|
||||
Row header = sheet.createRow(0);
|
||||
for (int i = 0; i < RESULT_HEADERS.size(); i++) {
|
||||
Cell cell = header.createCell(i);
|
||||
cell.setCellValue(RESULT_HEADERS.get(i));
|
||||
cell.setCellStyle(headerStyle);
|
||||
}
|
||||
|
||||
if (rows == null) {
|
||||
return;
|
||||
}
|
||||
int rowIndex = 1;
|
||||
for (SimilarAsinParsedRowVo parsedRow : rows) {
|
||||
if (parsedRow == null) {
|
||||
continue;
|
||||
}
|
||||
Row row = sheet.createRow(rowIndex);
|
||||
int col = 0;
|
||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
|
||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
|
||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
|
||||
row.createCell(col++).setCellValue(firstNonBlank(
|
||||
parsedRow.getPrice(), readValueByHeader(parsedRow, "价格", "price")));
|
||||
row.createCell(col++).setCellValue(readValueByHeader(
|
||||
parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name",
|
||||
"seller-name", "sellername", "store name", "shop name"));
|
||||
row.createCell(col).setCellValue(readValueByHeader(parsedRow, "品牌", "brand"));
|
||||
row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS);
|
||||
rowIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
private static String readValueByHeader(SimilarAsinParsedRowVo row, String... candidates) {
|
||||
if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) {
|
||||
return "";
|
||||
}
|
||||
for (Map.Entry<String, String> entry : row.getValues().entrySet()) {
|
||||
String header = SimilarAsinRowNormalizer.normalize(entry.getKey()).toLowerCase(Locale.ROOT);
|
||||
for (String candidate : candidates) {
|
||||
String expected = SimilarAsinRowNormalizer.normalize(candidate).toLowerCase(Locale.ROOT);
|
||||
if (!expected.isBlank() && header.contains(expected)) {
|
||||
return entry.getValue() == null ? "" : entry.getValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private static String firstNonBlank(String preferred, String fallback) {
|
||||
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user