task-116: 后台管理新增/导入表单弹窗化 + 分组管理独立菜单与UI优化 + 轻量进度端点

- 后台管理页(admin)所有面板的新增/导入表单改为弹窗操作,原有字段 ID 全部保留、提交逻辑不变;导入删除入口不再触发二次确认拦截
- 分组管理升级为独立菜单(V102 + schema initializer),移除 5 个面板内的管理分组按钮;分组列表改为蓝白主题、增加权限分组横幅
- Python 侧 group-manage 权限守卫(_ensure_backend_menu_access 补充 group-manage)
- 引入 V101(biz_task_file_job 复合索引)+ 新增 TaskProgressLight/TaskFileJob 轻量端点与进度聚合作
- 前端 progress-light / page-separated-loads / dispatch-guard 共享模块及单元测试
This commit is contained in:
2026-09-01 12:54:13 +08:00
parent 652ec1bbc9
commit b148dc57eb
139 changed files with 10540 additions and 8834 deletions
@@ -7,11 +7,13 @@ import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParseRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinSubmitResultRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskBatchRequest;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinTaskLightRequest;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinDashboardVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinFilterConditionVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskService.ResultDownloadInfo;
import io.swagger.v3.oas.annotations.Operation;
@@ -120,6 +122,14 @@ public class SimilarAsinController {
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
}
@PostMapping("/tasks/progress/light")
@Operation(summary = "批量查询任务轻量进度(前端轮询专用)",
description = "与 progress/batch 同输入,只返回轻量字段(taskId/status/statusCode/fileStatus/fileReady/updatedAt),"
+ "不查明细/result 行,响应体更小;旧 progress/batch 端点保留不动。")
public ApiResponse<SimilarAsinTaskLightBatchVo> progressLight(@Valid @RequestBody SimilarAsinTaskLightRequest request) {
return ApiResponse.success(service.progressLight(request.getTaskIds()));
}
@PostMapping("/tasks/{taskId}/activate")
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING。")
public ApiResponse<Void> activate(
@@ -0,0 +1,19 @@
package com.nanri.aiimage.modules.similarasin.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Schema(description = "相似ASIN检测任务轻量进度查询请求")
public class SimilarAsinTaskLightRequest {
@NotNull
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds;
}
@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "相似ASIN检测任务轻量进度响应")
public class SimilarAsinTaskLightBatchVo {
@Schema(description = "轻量进度项列表。顺序按请求 taskIds 处理。")
private List<SimilarAsinTaskLightVo> items = new ArrayList<>();
@Schema(description = "未找到或不属于相似ASIN检测模块的任务 ID 列表。", example = "[99999]")
private List<Long> missingTaskIds = new ArrayList<>();
}
@@ -0,0 +1,23 @@
package com.nanri.aiimage.modules.similarasin.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "相似ASIN检测任务轻量进度项(前端轮询专用,不含明细/result 内容)")
public class SimilarAsinTaskLightVo {
@Schema(description = "任务 ID。", example = "3938")
private Long taskId;
@Schema(description = "任务状态:PENDING=已解析待推送,RUNNING=执行中,SUCCESS=成功,FAILED=失败。", example = "RUNNING")
private String status;
@Schema(description = "业务状态码(预留字段,当前恒为 null)。", example = "null")
private String statusCode;
@Schema(description = "结果文件组装 Job 状态:SUCCESS/RUNNING/FAILED;无 Job 时为空。", example = "SUCCESS")
private String fileStatus;
@Schema(description = "文件组装失败时的错误信息;无错误时为空。", example = "组装失败: 文件写入超时")
private String fileError;
@Schema(description = "结果文件是否已就绪。", example = "true")
private Boolean fileReady;
@Schema(description = "最后更新时间,ISO 本地时间字符串。", example = "2026-04-26T10:05:00")
private String updatedAt;
}
@@ -35,6 +35,11 @@ import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParseVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskBatchVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskDetailVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskItemVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightBatchVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinTaskLightVo;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelParser;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGroupingConverter;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler;
import com.nanri.aiimage.modules.similarasin.util.BoundedImageCache;
import com.nanri.aiimage.modules.similarasin.util.ExcelCellImageWriter;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
@@ -59,12 +64,9 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Font;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
@@ -136,6 +138,7 @@ public class SimilarAsinTaskService {
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String CONTENT_TYPE_ZIP = "application/zip";
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
private static final int MAX_LIGHT_TASK_IDS = 200;
/** Task 19:Python 回传逐行日志采样频率(每 N 行记一行)。 */
private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L;
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
@@ -336,6 +339,20 @@ public class SimilarAsinTaskService {
* best-effort:service 内部所有异常都已吞掉,不影响主流程。
*/
private final SimilarAsinImagePrefetchService imagePrefetchService;
/**
* Task 89:Excel 行解析器(表头/单元格读取、别名匹配、空行跳过、单字段截断)。
* 由 parseAndCreateTask 委托;解析语义与搬移前 parseWorkbook 完全一致。
*/
SimilarAsinExcelParser excelParser = new SimilarAsinExcelParser();
/**
* Task 91:历史查询 VO 拼装(toHistoryItem + 进度链)搬入独立组件;
* 由 history/progressBatch 委托,只读不落库,输出与现状一致。
*/
private SimilarAsinHistoryAssembler historyAssembler() {
return new SimilarAsinHistoryAssembler(
taskScopeStateMapper, taskChunkMapper, fileTaskMapper,
taskProgressSnapshotService, ossStorageService, transientPayloadStorageService, objectMapper);
}
@Autowired
@Qualifier("taskQueueExecutor")
private TaskExecutor taskQueueExecutor;
@@ -466,7 +483,7 @@ public class SimilarAsinTaskService {
+ " (" + input.length() + " bytes > " + maxBytes + " bytes)");
}
ParsedWorkbook parsed = parseWorkbook(input, source);
ParsedWorkbook parsed = parseWorkbookDelegated(input, source);
totalRows += parsed.totalRows();
droppedRows += parsed.droppedRows();
allRows.addAll(parsed.allRows());
@@ -487,7 +504,7 @@ public class SimilarAsinTaskService {
sourceFiles.size(), allRows.size());
}
long parsedAt = System.nanoTime();
List<SimilarAsinParsedGroupVo> groups = buildParsedGroups(allRows);
List<SimilarAsinParsedGroupVo> groups = SimilarAsinGroupingConverter.convert(allRows);
long groupedAt = System.nanoTime();
FileTaskEntity task = new FileTaskEntity();
@@ -640,13 +657,49 @@ public class SimilarAsinTaskService {
.map(FileResultEntity::getId)
.filter(Objects::nonNull)
.toList());
for (FileResultEntity row : rows) {
FileTaskEntity task = taskMap.get(row.getTaskId());
String taskStatus = task == null ? null : task.getStatus();
if (STATUS_PENDING.equals(taskStatus)) {
vo.getItems().addAll(historyAssembler().buildHistoryItems(rows, taskMap, jobMap));
return vo;
}
public SimilarAsinTaskLightBatchVo progressLight(List<Long> taskIds) {
SimilarAsinTaskLightBatchVo vo = new SimilarAsinTaskLightBatchVo();
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.limit(MAX_LIGHT_TASK_IDS)
.toList();
if (normalizedIds.isEmpty()) {
return vo;
}
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
for (FileTaskEntity task : fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getStatus,
FileTaskEntity::getUpdatedAt)
.in(FileTaskEntity::getId, normalizedIds))) {
if (task != null && MODULE_TYPE.equals(task.getModuleType())) {
taskMap.put(task.getId(), task);
}
}
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByTaskIds(MODULE_TYPE, new ArrayList<>(taskMap.keySet()));
for (Long taskId : normalizedIds) {
FileTaskEntity task = taskMap.get(taskId);
if (task == null) {
vo.getMissingTaskIds().add(taskId);
continue;
}
vo.getItems().add(toHistoryItem(row, task, jobMap.get(row.getId())));
SimilarAsinTaskLightVo item = new SimilarAsinTaskLightVo();
item.setTaskId(task.getId());
item.setStatus(resolveDisplayTaskStatus(task));
TaskFileJobEntity job = jobMap.get(taskId);
boolean fileReady = job != null
&& job.getResultFileUrl() != null
&& !job.getResultFileUrl().isBlank();
item.setFileReady(fileReady);
item.setFileStatus(job == null ? null : job.getStatus());
item.setFileError(job == null ? null : job.getErrorMessage());
item.setUpdatedAt(fmt(task.getUpdatedAt()));
vo.getItems().add(item);
}
return vo;
}
@@ -708,7 +761,8 @@ public class SimilarAsinTaskService {
detail.setTask(toTaskItem(task));
FileResultEntity resultRow = resultByTaskId.get(taskId);
if (resultRow != null) {
detail.getItems().add(toHistoryItem(resultRow, task, jobMap.get(resultRow.getId())));
detail.getItems().addAll(historyAssembler().buildHistoryItems(
List.of(resultRow), taskMap, jobMap));
}
vo.getItems().add(detail);
}
@@ -1751,7 +1805,7 @@ public class SimilarAsinTaskService {
private Map<String, List<SimilarAsinParsedRowVo>> loadAllRowsByBaseId(FileTaskEntity task) {
try {
SimilarAsinParsedPayloadDto payload = readParsedPayload(task);
return groupRowsByBaseId(resolveAllRows(payload));
return SimilarAsinGroupingConverter.groupRowsByBaseId(resolveAllRows(payload));
} catch (Exception ex) {
log.warn("[similar-asin] read all rows failed taskId={} err={}", task.getId(), ex.getMessage());
return new LinkedHashMap<>();
@@ -1791,36 +1845,6 @@ public class SimilarAsinTaskService {
return candidates;
}
/**
* Task 6:分组数据改为索引/范围引用。组内行在原 items 中必然连续
* (parseWorkbook 按行遍历、同 baseId 块连续收集),因此只需要
* [startIndex, endIndex) 半开区间即可唯一定位组内行,
* 行对象仅存在于 items 一次,避免 groups 嵌套复制完整行对象。
*/
private List<SimilarAsinParsedGroupVo> buildParsedGroups(List<SimilarAsinParsedRowVo> rows) {
List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
int cursor = 0;
for (List<SimilarAsinParsedRowVo> siblings : groupRowsByBaseId(rows).values()) {
if (siblings == null || siblings.isEmpty()) {
continue;
}
SimilarAsinParsedRowVo first = siblings.getFirst();
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
group.setSourceFileKey(first.getSourceFileKey());
group.setSourceFilename(first.getSourceFilename());
group.setGroupKey(firstNonBlank(first.getGroupKey(), buildGroupKey(first.getSourceFileKey(), baseId(first.getDisplayId()), first.getRowIndex())));
group.setBaseId(baseId(first.getDisplayId()));
group.setDisplayId(firstNonBlank(first.getDisplayId(), first.getSourceId()));
group.setItemCount(siblings.size());
int end = cursor + siblings.size();
group.setStartIndex(cursor);
group.setEndIndex(end);
cursor = end;
groups.add(group);
}
return groups;
}
private List<SimilarAsinParsedGroupVo> buildResponsePreviewGroups(List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
if (groups == null || groups.isEmpty()) {
return List.of();
@@ -1887,18 +1911,6 @@ public class SimilarAsinTaskService {
return vo;
}
private Map<String, List<SimilarAsinParsedRowVo>> groupRowsByBaseId(List<SimilarAsinParsedRowVo> rows) {
Map<String, List<SimilarAsinParsedRowVo>> result = new LinkedHashMap<>();
if (rows == null) {
return result;
}
for (SimilarAsinParsedRowVo row : rows) {
String key = firstNonBlank(normalize(row.getGroupKey()), baseId(row.getDisplayId()));
result.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row);
}
return result;
}
private List<SimilarAsinResultRowDto> flattenSubmittedRows(SimilarAsinSubmitResultRequest request) {
if (request == null) {
return List.of();
@@ -4718,8 +4730,11 @@ public class SimilarAsinTaskService {
return firstNonBlank(resultRow.getUrl(), fallbackUrl);
}
private ParsedWorkbook parseWorkbook(File input, SimilarAsinSourceFileDto source) {
DataFormatter formatter = new DataFormatter();
/**
* Task 89:解析委托 SimilarAsinExcelParser(表头/单元格读取、别名匹配、空行跳过、字段截断),
* 保留服务侧编排语义:zip 探针、status 列过滤、类目重试判定、行号/分组键/rowToken 组装。
*/
private ParsedWorkbook parseWorkbookDelegated(File input, SimilarAsinSourceFileDto source) {
try {
// Task 8:受控读取——先探测 zip 条目数与解压体积,超限拒绝
probeWorkbookZipBounds(input, source.getOriginalFilename());
@@ -4729,26 +4744,9 @@ public class SimilarAsinTaskService {
// 非 zip 或损坏文件:留给 WorkbookFactory 尝试后由下方 catch 转业务异常
log.debug("[similar-asin] workbook zip probe skipped file={} err={}", input, ex.getMessage());
}
try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) { Sheet sheet = workbook.getSheetAt(0);
Row header = sheet.getRow(0);
if (header == null) {
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
List<String> headers = readHeaders(header, formatter);
int idCol = findRequiredHeader(headerMap, "id");
int asinCol = findRequiredHeader(headerMap, "asin");
int countryCol = findRequiredHeader(headerMap, "国家", "country");
// 兼容旧 xlsx:RESULT_HEADERS 已不再写出 sku 列,但仍要支持旧版结果簿重新上传,
// 因此这里仍按可选列读取并保留到 DTO,enrichRowForLlm 仍可使用,仅导出阶段不再写出。
int skuCol = findOptionalHeaderExact(headerMap, "sku", "seller sku", "seller_sku", "msku", "货号");
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
int urlCol = findOptionalHeaderExact(headerMap,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
int titleCol = findOptionalHeaderExact(headerMap,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
try {
SimilarAsinExcelParser.ParsedSheet parsedSheet = excelParser.parse(input, resolveMaxFieldLength());
List<String> headers = parsedSheet.headers();
int statusCol = FailedStatusRowFilter.findStatusColumnIndex(headers);
String statusHeader = statusCol >= 0 && statusCol < headers.size() ? headers.get(statusCol) : null;
@@ -4758,14 +4756,10 @@ public class SimilarAsinTaskService {
int validRows = 0;
String currentBlockBaseId = "";
String currentGroupKey = "";
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
}
String id = cell(row, idCol, formatter);
String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT);
String country = cell(row, countryCol, formatter);
for (SimilarAsinExcelParser.SimilarAsinExcelRow parsedRow : parsedSheet.rows()) {
String id = parsedRow.id();
String asin = parsedRow.asin();
String country = parsedRow.country();
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
}
@@ -4778,7 +4772,7 @@ public class SimilarAsinTaskService {
SimilarAsinParsedRowVo vo = new SimilarAsinParsedRowVo();
vo.setSourceFileKey(source.getFileKey());
vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName()));
vo.setRowIndex(i + 1);
vo.setRowIndex(parsedRow.rowIndex());
vo.setSourceId(id);
vo.setDisplayId(normalizeDisplayId(id));
String rowBaseId = baseId(vo.getDisplayId());
@@ -4790,11 +4784,11 @@ public class SimilarAsinTaskService {
vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex()));
vo.setAsin(asin);
vo.setCountry(country);
vo.setSku(skuCol >= 0 ? cell(row, skuCol, formatter) : "");
vo.setPrice(priceCol >= 0 ? cell(row, priceCol, formatter) : "");
vo.setUrl(urlCol >= 0 ? cell(row, urlCol, formatter) : "");
vo.setTitle(titleCol >= 0 ? cell(row, titleCol, formatter) : "");
vo.setValues(readRowValues(row, headers, formatter));
vo.setSku(parsedRow.sku());
vo.setPrice(parsedRow.price());
vo.setUrl(parsedRow.url());
vo.setTitle(parsedRow.title());
vo.setValues(parsedRow.values());
allRows.add(vo);
}
boolean resultWorkbook = statusCol >= 0 && isSimilarAsinResultWorkbook(headers);
@@ -4911,86 +4905,6 @@ public class SimilarAsinTaskService {
return !normalized.isBlank() && !isTechnicalLlmFailure(normalized);
}
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
Map<String, Integer> map = new LinkedHashMap<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
if (!val.isBlank()) {
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
}
}
return map;
}
private List<String> readHeaders(Row header, DataFormatter formatter) {
List<String> headers = new ArrayList<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
headers.add(val.isBlank() ? "列" + (i + 1) : val);
}
return headers;
}
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter) {
Map<String, String> values = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
values.put(headers.get(i), cell(row, i, formatter));
}
return values;
}
private int findRequiredHeader(Map<String, Integer> map, String... names) {
int idx = findOptionalHeader(map, names);
if (idx < 0) {
throw new BusinessException("缺少必要表头: " + String.join("/", names));
}
return idx;
}
private int findOptionalHeader(Map<String, Integer> map, String... names) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
for (String name : names) {
if (entry.getKey().contains(name.toLowerCase(Locale.ROOT))) {
return entry.getValue();
}
}
}
return -1;
}
private int findOptionalHeaderExact(Map<String, Integer> map, String... names) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
String normalizedHeader = normalizeHeaderAlias(entry.getKey());
for (String name : names) {
if (normalizedHeader.equals(normalizeHeaderAlias(name))) {
return entry.getValue();
}
}
}
return -1;
}
private String normalizeHeaderAlias(String value) {
String normalized = normalize(value).toLowerCase(Locale.ROOT);
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
}
private String cell(Row row, int col, DataFormatter formatter) {
if (col < 0) {
return "";
}
String value = normalize(formatter.formatCellValue(row.getCell(col)));
if (isSpreadsheetErrorValue(value)) {
return "";
}
// Task 7:单字段长度上限,防止超长单元格导致内存无界增长
int maxLen = resolveMaxFieldLength();
if (value.length() > maxLen) {
return value.substring(0, maxLen);
}
return value;
}
private static boolean isSpreadsheetErrorValue(String value) {
String normalized = value == null ? "" : value.trim();
return normalized.startsWith("#") && normalized.endsWith("!");
@@ -5133,253 +5047,6 @@ public class SimilarAsinTaskService {
return vo;
}
private SimilarAsinHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
SimilarAsinHistoryItemVo vo = new SimilarAsinHistoryItemVo();
vo.setResultId(row.getId());
vo.setTaskId(row.getTaskId());
vo.setSourceFilename(row.getSourceFilename());
vo.setResultFilename(row.getResultFilename());
vo.setDownloadUrl(buildFreshDownloadUrl(row));
vo.setTaskStatus(resolveDisplayTaskStatus(task, row));
attachFileJobState(vo, row, job, task);
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
vo.setError(row.getErrorMessage());
vo.setRowCount(row.getRowCount());
vo.setCreatedAt(fmt(row.getCreatedAt()));
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
vo.setStartedAt(fmt(task == null ? row.getCreatedAt() : task.getCreatedAt()));
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
return vo;
}
private String buildFreshDownloadUrl(FileResultEntity row) {
String resultFileUrl = row == null ? null : row.getResultFileUrl();
if (resultFileUrl == null || resultFileUrl.isBlank()) {
return null;
}
try {
// bucket 对 result/ 前缀已开放公开读,使用无签名公开 URL,避免签名 1 小时过期后下载 403。
// result/ 前缀走 download-endpoint 独立下载域名(带宽/缓存分流),非 result/(如图片)仍走公开端点。
return ossStorageService.generateFreshDownloadUrl(resultFileUrl);
} catch (Exception ex) {
log.warn("[similar-asin] build public download url failed resultId={} err={}",
row.getId(), ex.getMessage());
return null;
}
}
private void attachFileJobState(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
attachFileProgress(vo, row, job, task);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(job.getErrorMessage());
attachFileProgress(vo, row, job, task);
}
private void attachFileProgress(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
if (row == null || row.getTaskId() == null) {
return;
}
Long taskId = row.getTaskId();
if (Boolean.TRUE.equals(vo.getFileReady())) {
vo.setFileProgressCurrent(1);
vo.setFileProgressTotal(1);
vo.setFileProgressPercent(100);
vo.setFileProgressMessage("结果文件已生成");
return;
}
int llmCompleted = countCompletedLlmStates(taskId);
int llmPending = countPendingLlmStates(taskId);
boolean uploadComplete = isResultSubmissionComplete(taskId);
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(taskId, MODULE_TYPE);
if (!uploadComplete) {
if (canTreatTaskAsCompleted(task, row, job)) {
vo.setFileProgressCurrent(1);
vo.setFileProgressTotal(1);
vo.setFileProgressPercent(100);
vo.setFileProgressMessage("结果文件已生成");
return;
}
if (task != null && !STATUS_RUNNING.equals(task.getStatus())) {
return;
}
if (llmCompleted + llmPending > 0) {
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, false);
return;
}
attachPythonUploadProgress(vo, taskId);
return;
}
if (llmCompleted + llmPending > 0) {
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, true);
return;
}
if (snapshot == null) {
return;
}
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
if (total <= 0) {
return;
}
LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : (job == null ? null : job.getUpdatedAt());
int percent = calculateDisplayProgressPercent(current, total, job, baseTime);
percent = Math.max(percent, extractSnapshotDisplayPercent(snapshot));
percent = Boolean.TRUE.equals(vo.getFileReady()) ? 100 : Math.min(99, percent);
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
vo.setFileProgressPercent(percent);
vo.setFileProgressMessage(snapshot.getMessage());
}
private void attachLlmProgress(SimilarAsinHistoryItemVo vo,
TaskFileJobEntity job,
TaskProgressSnapshotEntity snapshot,
int llmCompleted,
int llmPending,
boolean uploadComplete) {
int observed = Math.max(0, llmCompleted) + Math.max(0, llmPending);
int total = calculateLlmDisplayTotal(snapshot, observed);
if (!uploadComplete) {
total = Math.max(total, observed + 1);
}
int current = Math.max(0, Math.min(llmCompleted, total));
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
int percent = calculateDisplayProgressPercent(current, total, job, snapshot == null ? null : snapshot.getUpdatedAt());
percent = Math.max(percent, calculateSnapshotDisplayPercent(snapshot, job));
vo.setFileProgressPercent(Math.min(uploadComplete ? 99 : 98, percent));
vo.setFileProgressMessage(uploadComplete
? (llmPending > 0
? buildLlmProgressMessage(current, total, llmPending)
: "LLM 已回流 " + current + "/" + total + " 批次,正在生成结果文件")
: buildUploadingLlmProgressMessage(current, total, llmPending));
}
private void attachPythonUploadProgress(SimilarAsinHistoryItemVo vo, Long taskId) {
PythonUploadProgress progress = resolvePythonUploadProgress(taskId);
int total = progress.total() > 0 ? progress.total() : Math.max(1, progress.current() + 1);
int current = Math.max(0, Math.min(progress.current(), total));
int percent = current <= 0 ? 1 : Math.min(95, (int) Math.ceil(current * 100.0 / total));
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
vo.setFileProgressPercent(percent);
vo.setFileProgressMessage(buildPythonUploadProgressMessage(current, total, progress.unit()));
}
private PythonUploadProgress resolvePythonUploadProgress(Long taskId) {
PythonUploadProgress chunkProgress = resolvePythonChunkProgress(taskId);
if (chunkProgress.total() > 0) {
return chunkProgress;
}
if (chunkProgress.current() > 0) {
return new PythonUploadProgress(chunkProgress.current(), chunkProgress.current() + 1, chunkProgress.unit());
}
int totalRows = 0;
try {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
totalRows = allRowCount(task);
} catch (Exception ignored) {
totalRows = 0;
}
int uploadedRows = countSubmittedRows(taskId);
if (totalRows <= 0 && uploadedRows > 0) {
totalRows = uploadedRows + 1;
}
return new PythonUploadProgress(uploadedRows, totalRows, "行");
}
private PythonUploadProgress resolvePythonChunkProgress(Long taskId) {
if (taskId == null || taskId <= 0) {
return new PythonUploadProgress(0, 0, "分片");
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNull(TaskScopeStateEntity::getLlmStatus)
.orderByDesc(TaskScopeStateEntity::getUpdatedAt));
if (states == null || states.isEmpty()) {
return resolvePythonChunkProgressFromChunks(taskId);
}
int current = 0;
int total = 0;
for (TaskScopeStateEntity state : states) {
if (state == null) {
continue;
}
int stateTotal = state.getChunkTotal() == null ? 0 : state.getChunkTotal();
int stateCurrent = state.getReceivedChunkCount() == null ? 0 : state.getReceivedChunkCount();
stateCurrent = Math.max(stateCurrent, resolveReceivedChunkProgress(taskId, state.getScopeHash(), stateTotal));
if (stateTotal > total || total <= 0 && stateCurrent > current) {
total = stateTotal;
current = stateCurrent;
}
}
return new PythonUploadProgress(current, total, "分片");
}
private PythonUploadProgress resolvePythonChunkProgressFromChunks(Long taskId) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks == null || chunks.isEmpty()) {
return new PythonUploadProgress(0, 0, "分片");
}
Map<String, Integer> receivedByScope = new LinkedHashMap<>();
Map<String, Integer> totalByScope = new LinkedHashMap<>();
for (TaskChunkEntity chunk : chunks) {
if (chunk == null) {
continue;
}
String scopeHash = firstNonBlank(chunk.getScopeHash(), "");
receivedByScope.merge(scopeHash, 1, Integer::sum);
if (chunk.getChunkIndex() != null && chunk.getChunkIndex() > 0) {
receivedByScope.merge(scopeHash, chunk.getChunkIndex(), Math::max);
}
if (chunk.getChunkTotal() != null && chunk.getChunkTotal() > 0) {
totalByScope.merge(scopeHash, chunk.getChunkTotal(), Math::max);
}
}
int current = 0;
int total = 0;
for (Map.Entry<String, Integer> entry : receivedByScope.entrySet()) {
int scopeCurrent = entry.getValue() == null ? 0 : entry.getValue();
int scopeTotal = totalByScope.getOrDefault(entry.getKey(), 0);
if (scopeTotal > total || total <= 0 && scopeCurrent > current) {
current = scopeCurrent;
total = scopeTotal;
}
}
return new PythonUploadProgress(current, total, "分片");
}
private int countSubmittedRows(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
int total = 0;
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks == null || chunks.isEmpty()) {
return 0;
}
for (TaskChunkEntity chunk : chunks) {
total += readChunkRows(chunk).size();
}
return total;
}
private String buildPythonUploadProgressMessage(int current, int total, String unit) {
String safeUnit = unit == null || unit.isBlank() ? "分片" : unit;
return "等待 Python 回传,已回传 " + Math.max(0, current) + "/" + Math.max(1, total) + " " + safeUnit;
}
private String resolveDisplayTaskStatus(FileTaskEntity task) {
return resolveDisplayTaskStatus(task, null);
}
@@ -5416,36 +5083,6 @@ public class SimilarAsinTaskService {
return true;
}
private int calculateLlmDisplayTotal(TaskProgressSnapshotEntity snapshot, int observedLlmStates) {
int observed = Math.max(0, observedLlmStates);
// 小任务 totalCount <= 3 时直接使用 observed,避免 totalCount - 3 → 0 引起进度条 0/0。
if (snapshot == null || snapshot.getTotalCount() == null || snapshot.getTotalCount() <= 3) {
return Math.max(1, observed);
}
int expectedFromSnapshot = Math.max(1, snapshot.getTotalCount() - 3);
return Math.max(1, Math.max(observed, expectedFromSnapshot));
}
private String buildLlmProgressMessage(int completed, int total, int pending) {
int safeTotal = Math.max(1, total);
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
if (safeCompleted <= 0 && submitted > 0) {
return "LLM 已提交 " + submitted + "/" + safeTotal + " 批次,等待回流";
}
return "LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次,等待结果文件";
}
private String buildUploadingLlmProgressMessage(int completed, int total, int pending) {
int safeTotal = Math.max(1, total);
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
if (pending > 0) {
return "Python 仍在回传,LLM 已提交 " + submitted + "/" + safeTotal + " 批次";
}
return "Python 仍在回传,LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次";
}
private String fmt(LocalDateTime t) {
return t == null ? null : t.toString();
}
@@ -0,0 +1,245 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.common.exception.BusinessException;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
/**
* 任务 83:SimilarAsinExcelParser 行解析器。
* POI 读 Excel 行 → 中间行对象(原始单元格值)。语义与 SimilarAsinTaskService.parseWorkbook
* 中对应段落逐字节一致:cell 归一化(BOM/全角空格/trim/连续空白折叠)、错误值转空、
* 单字段截断 2000、表头别名匹配、空行跳过。
*/
@Slf4j
public class SimilarAsinExcelParser {
private static final int DEFAULT_MAX_FIELD_LENGTH = 2000;
public ParsedSheet parse(File input) {
return parse(input, DEFAULT_MAX_FIELD_LENGTH);
}
public ParsedSheet parse(File input, int maxFieldLength) {
if (input == null) {
throw new IllegalArgumentException("input must not be null");
}
try (FileInputStream fis = new FileInputStream(input);
Workbook workbook = WorkbookFactory.create(fis)) {
return parseWorkbook(workbook, maxFieldLength);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
log.warn("[similar-asin] parse failed file={} err={}", input.getName(), ex.getMessage());
throw new BusinessException("解析 Excel 失败");
}
}
public ParsedSheet parse(InputStream input) {
if (input == null) {
throw new IllegalArgumentException("input must not be null");
}
try (Workbook workbook = WorkbookFactory.create(input)) {
return parseWorkbook(workbook, DEFAULT_MAX_FIELD_LENGTH);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
log.warn("[similar-asin] parse failed input err={}", ex.getMessage());
throw new BusinessException("解析 Excel 失败");
}
}
private ParsedSheet parseWorkbook(Workbook workbook, int maxFieldLength) {
DataFormatter formatter = new DataFormatter();
Sheet sheet = workbook.getSheetAt(0);
Row header = sheet.getRow(0);
if (header == null) {
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
List<String> headers = readHeaders(header, formatter);
int idCol = findRequiredHeader(headerMap, "id");
int asinCol = findRequiredHeader(headerMap, "asin");
int countryCol = findRequiredHeader(headerMap, "国家", "country");
int skuCol = findOptionalHeaderExact(headerMap, "sku", "seller sku", "seller_sku", "msku", "货号");
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
int urlCol = findOptionalHeaderExact(headerMap,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
int titleCol = findOptionalHeaderExact(headerMap,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
List<SimilarAsinExcelRow> rows = new ArrayList<>();
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
}
String id = cell(row, idCol, formatter, maxFieldLength);
String asin = cell(row, asinCol, formatter, maxFieldLength).toUpperCase(Locale.ROOT);
String country = cell(row, countryCol, formatter, maxFieldLength);
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
}
rows.add(new SimilarAsinExcelRow(
i + 1,
id,
asin,
country,
skuCol >= 0 ? cell(row, skuCol, formatter, maxFieldLength) : "",
priceCol >= 0 ? cell(row, priceCol, formatter, maxFieldLength) : "",
urlCol >= 0 ? cell(row, urlCol, formatter, maxFieldLength) : "",
titleCol >= 0 ? cell(row, titleCol, formatter, maxFieldLength) : "",
readRowValues(row, headers, formatter, maxFieldLength)));
}
return new ParsedSheet(headers, rows);
}
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
Map<String, Integer> map = new LinkedHashMap<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
if (!val.isBlank()) {
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
}
}
return map;
}
private List<String> readHeaders(Row header, DataFormatter formatter) {
List<String> headers = new ArrayList<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
headers.add(val.isBlank() ? "列" + (i + 1) : val);
}
return headers;
}
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter, int maxFieldLength) {
Map<String, String> values = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
values.put(headers.get(i), cell(row, i, formatter, maxFieldLength));
}
return values;
}
private int findRequiredHeader(Map<String, Integer> map, String... names) {
int idx = findOptionalHeader(map, names);
if (idx < 0) {
throw new BusinessException("缺少必要表头: " + String.join("/", names));
}
return idx;
}
private int findOptionalHeader(Map<String, Integer> map, String... names) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
for (String name : names) {
if (entry.getKey().contains(name.toLowerCase(Locale.ROOT))) {
return entry.getValue();
}
}
}
return -1;
}
private int findOptionalHeaderExact(Map<String, Integer> map, String... names) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
String normalizedHeader = normalizeHeaderAlias(entry.getKey());
for (String name : names) {
if (normalizedHeader.equals(normalizeHeaderAlias(name))) {
return entry.getValue();
}
}
}
return -1;
}
private String normalizeHeaderAlias(String value) {
String normalized = normalize(value).toLowerCase(Locale.ROOT);
return normalized.replaceAll("[\\s_\\-()()\\[\\]{}::/\\\\]+", "");
}
private String cell(Row row, int col, DataFormatter formatter, int maxFieldLength) {
if (col < 0) {
return "";
}
String value = normalize(formatter.formatCellValue(row.getCell(col)));
if (isSpreadsheetErrorValue(value)) {
return "";
}
if (value.length() > maxFieldLength) {
return value.substring(0, maxFieldLength);
}
return value;
}
private static boolean isSpreadsheetErrorValue(String value) {
String normalized = value == null ? "" : value.trim();
return normalized.startsWith("#") && normalized.endsWith("!");
}
private String normalize(String val) {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
public record ParsedSheet(List<String> headers, List<SimilarAsinExcelRow> rows) {
public ParsedSheet {
headers = headers == null ? List.of() : new ArrayList<>(headers);
rows = rows == null ? List.of() : new ArrayList<>(rows);
}
@Override
public List<String> headers() {
return java.util.Collections.unmodifiableList(headers);
}
@Override
public List<SimilarAsinExcelRow> rows() {
return java.util.Collections.unmodifiableList(rows);
}
}
public record SimilarAsinExcelRow(int rowIndex, String id, String asin, String country,
String sku, String price, String url, String title,
Map<String, String> values) {
public SimilarAsinExcelRow {
values = values == null ? Map.of() : new LinkedHashMap<>(values);
}
@Override
public boolean equals(Object o) {
if (!(o instanceof SimilarAsinExcelRow other)) {
return false;
}
return rowIndex == other.rowIndex
&& Objects.equals(id, other.id)
&& Objects.equals(asin, other.asin)
&& Objects.equals(country, other.country)
&& Objects.equals(sku, other.sku)
&& Objects.equals(price, other.price)
&& Objects.equals(url, other.url)
&& Objects.equals(title, other.title)
&& values.equals(other.values);
}
@Override
public int hashCode() {
return Objects.hash(rowIndex, id, asin, country, sku, price, url, title, values);
}
}
}
@@ -0,0 +1,78 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* 任务 86:SimilarAsinGroupingConverter 分组转换器。
* 归一化行 → 分组 DTO;分组规则与现状 SimilarAsinTaskService.buildParsedGroups 完全一致:
* 按 groupKey(缺省兜底 displayId 的 baseId)分组、startIndex/endIndex 半开区间游标、
* 组内行数、顺序保持行首次出现顺序。纯函数无状态。
*/
public final class SimilarAsinGroupingConverter {
private SimilarAsinGroupingConverter() {
}
public static List<SimilarAsinParsedGroupVo> convert(List<SimilarAsinParsedRowVo> rows) {
List<SimilarAsinParsedGroupVo> groups = new ArrayList<>();
if (rows == null || rows.isEmpty()) {
return groups;
}
int cursor = 0;
for (List<SimilarAsinParsedRowVo> siblings : groupRowsByBaseId(rows).values()) {
if (siblings == null || siblings.isEmpty()) {
continue;
}
SimilarAsinParsedRowVo first = siblings.getFirst();
SimilarAsinParsedGroupVo group = new SimilarAsinParsedGroupVo();
group.setSourceFileKey(first.getSourceFileKey());
group.setSourceFilename(first.getSourceFilename());
group.setGroupKey(firstNonBlank(first.getGroupKey(),
buildGroupKey(first.getSourceFileKey(), baseId(first.getDisplayId()), first.getRowIndex())));
group.setBaseId(baseId(first.getDisplayId()));
group.setDisplayId(firstNonBlank(first.getDisplayId(), first.getSourceId()));
group.setItemCount(siblings.size());
int end = cursor + siblings.size();
group.setStartIndex(cursor);
group.setEndIndex(end);
cursor = end;
groups.add(group);
}
return groups;
}
/** 行 → baseId 分组映射(LinkedHashMap 保序)。供 convert 及服务侧 loadAllRowsByBaseId 复用。 */
public static Map<String, List<SimilarAsinParsedRowVo>> groupRowsByBaseId(List<SimilarAsinParsedRowVo> rows) {
Map<String, List<SimilarAsinParsedRowVo>> result = new LinkedHashMap<>();
for (SimilarAsinParsedRowVo row : rows) {
if (row == null) {
continue;
}
String key = firstNonBlank(SimilarAsinRowNormalizer.normalize(row.getGroupKey()),
baseId(row.getDisplayId()));
result.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row);
}
return result;
}
private static String baseId(String id) {
String s = SimilarAsinRowNormalizer.normalize(id);
int idx = s.indexOf('_');
return idx > 0 ? s.substring(0, idx) : s;
}
private static String buildGroupKey(String sourceFileKey, String rowBaseId, Integer rowIndex) {
return SimilarAsinRowNormalizer.normalize(sourceFileKey) + "::"
+ SimilarAsinRowNormalizer.normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex);
}
private static String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
}
@@ -0,0 +1,757 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinParsedPayloadDto;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinHistoryItemVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedGroupVo;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.extern.slf4j.Slf4j;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
/**
* 任务 91:SimilarAsinHistoryAssembler 历史查询组装器。
* 历史列表 VO 拼装(toHistoryItem + 进度链:file job 状态、LLM 回流进度、Python 上传进度、
* 快照百分比)从 SimilarAsinTaskService 原样搬移;只读不落库;输出与现状逐字段一致。
*/
@Slf4j
public class SimilarAsinHistoryAssembler {
private static final String MODULE_TYPE = "SIMILAR_ASIN";
private static final String STATUS_PENDING = "PENDING";
private static final String STATUS_RUNNING = "RUNNING";
private static final String STATUS_SUCCESS = "SUCCESS";
private static final String LLM_STATUS_SUBMITTED = "SUBMITTED";
private static final String LLM_STATUS_RUNNING = "RUNNING";
private static final String LLM_STATUS_DONE = "DONE";
private static final String LLM_STATUS_FAILED = "FAILED";
private final TaskScopeStateMapper taskScopeStateMapper;
private final TaskChunkMapper taskChunkMapper;
private final FileTaskMapper fileTaskMapper;
private final TaskProgressSnapshotService taskProgressSnapshotService;
private final OssStorageService ossStorageService;
private final TransientPayloadStorageService transientPayloadStorageService;
private final ObjectMapper objectMapper;
public SimilarAsinHistoryAssembler(TaskScopeStateMapper taskScopeStateMapper,
TaskChunkMapper taskChunkMapper,
FileTaskMapper fileTaskMapper,
TaskProgressSnapshotService taskProgressSnapshotService,
OssStorageService ossStorageService,
TransientPayloadStorageService transientPayloadStorageService,
ObjectMapper objectMapper) {
this.taskScopeStateMapper = taskScopeStateMapper;
this.taskChunkMapper = taskChunkMapper;
this.fileTaskMapper = fileTaskMapper;
this.taskProgressSnapshotService = taskProgressSnapshotService;
this.ossStorageService = ossStorageService;
this.transientPayloadStorageService = transientPayloadStorageService;
this.objectMapper = objectMapper;
}
/**
* 按 rows 顺序组装历史项;PENDING 任务(history 语义)跳过。
* taskMap/jobMap 均为调用方已查好的映射,本方法只读不改输入。
*/
public List<SimilarAsinHistoryItemVo> buildHistoryItems(List<FileResultEntity> rows,
Map<Long, FileTaskEntity> taskMap,
Map<Long, TaskFileJobEntity> jobMap) {
List<SimilarAsinHistoryItemVo> items = new ArrayList<>();
if (rows == null || rows.isEmpty()) {
return items;
}
for (FileResultEntity row : rows) {
FileTaskEntity task = taskMap.get(row.getTaskId());
String taskStatus = task == null ? null : task.getStatus();
if (STATUS_PENDING.equals(taskStatus)) {
continue;
}
items.add(toHistoryItem(row, task, jobMap.get(row.getId())));
}
return items;
}
private SimilarAsinHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
SimilarAsinHistoryItemVo vo = new SimilarAsinHistoryItemVo();
vo.setResultId(row.getId());
vo.setTaskId(row.getTaskId());
vo.setSourceFilename(row.getSourceFilename());
vo.setResultFilename(row.getResultFilename());
vo.setDownloadUrl(buildFreshDownloadUrl(row));
vo.setTaskStatus(resolveDisplayTaskStatus(task, row));
attachFileJobState(vo, row, job, task);
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
vo.setError(row.getErrorMessage());
vo.setRowCount(row.getRowCount());
vo.setCreatedAt(fmt(row.getCreatedAt()));
// 任务开始时间复用 biz_file_task.created_at;缺 task 时回退到 result.createdAt 兜底
vo.setStartedAt(fmt(task == null ? row.getCreatedAt() : task.getCreatedAt()));
vo.setFinishedAt(fmt(task == null ? null : task.getFinishedAt()));
return vo;
}
private String buildFreshDownloadUrl(FileResultEntity row) {
String resultFileUrl = row == null ? null : row.getResultFileUrl();
if (resultFileUrl == null || resultFileUrl.isBlank()) {
return null;
}
try {
// bucket 对 result/ 前缀已开放公开读,使用无签名公开 URL,避免签名 1 小时过期后下载 403。
// result/ 前缀走 download-endpoint 独立下载域名(带宽/缓存分流),非 result/(如图片)仍走公开端点。
return ossStorageService.generateFreshDownloadUrl(resultFileUrl);
} catch (Exception ex) {
log.warn("[similar-asin] build public download url failed resultId={} err={}",
row.getId(), ex.getMessage());
return null;
}
}
private void attachFileJobState(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
attachFileProgress(vo, row, job, task);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(job.getErrorMessage());
attachFileProgress(vo, row, job, task);
}
private void attachFileProgress(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
if (row == null || row.getTaskId() == null) {
return;
}
Long taskId = row.getTaskId();
if (Boolean.TRUE.equals(vo.getFileReady())) {
vo.setFileProgressCurrent(1);
vo.setFileProgressTotal(1);
vo.setFileProgressPercent(100);
vo.setFileProgressMessage("结果文件已生成");
return;
}
int llmCompleted = countCompletedLlmStates(taskId);
int llmPending = countPendingLlmStates(taskId);
boolean uploadComplete = isResultSubmissionComplete(taskId);
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(taskId, MODULE_TYPE);
if (!uploadComplete) {
if (canTreatTaskAsCompleted(task, row, job)) {
vo.setFileProgressCurrent(1);
vo.setFileProgressTotal(1);
vo.setFileProgressPercent(100);
vo.setFileProgressMessage("结果文件已生成");
return;
}
if (task != null && !STATUS_RUNNING.equals(task.getStatus())) {
return;
}
if (llmCompleted + llmPending > 0) {
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, false);
return;
}
attachPythonUploadProgress(vo, taskId);
return;
}
if (llmCompleted + llmPending > 0) {
attachLlmProgress(vo, job, snapshot, llmCompleted, llmPending, true);
return;
}
if (snapshot == null) {
return;
}
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
if (total <= 0) {
return;
}
LocalDateTime baseTime = snapshot.getUpdatedAt() != null ? snapshot.getUpdatedAt() : (job == null ? null : job.getUpdatedAt());
int percent = calculateDisplayProgressPercent(current, total, job, baseTime);
percent = Math.max(percent, extractSnapshotDisplayPercent(snapshot));
percent = Boolean.TRUE.equals(vo.getFileReady()) ? 100 : Math.min(99, percent);
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
vo.setFileProgressPercent(percent);
vo.setFileProgressMessage(snapshot.getMessage());
}
private void attachLlmProgress(SimilarAsinHistoryItemVo vo,
TaskFileJobEntity job,
TaskProgressSnapshotEntity snapshot,
int llmCompleted,
int llmPending,
boolean uploadComplete) {
int observed = Math.max(0, llmCompleted) + Math.max(0, llmPending);
int total = calculateLlmDisplayTotal(snapshot, observed);
if (!uploadComplete) {
total = Math.max(total, observed + 1);
}
int current = Math.max(0, Math.min(llmCompleted, total));
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
int percent = calculateDisplayProgressPercent(current, total, job, snapshot == null ? null : snapshot.getUpdatedAt());
percent = Math.max(percent, calculateSnapshotDisplayPercent(snapshot, job));
vo.setFileProgressPercent(Math.min(uploadComplete ? 99 : 98, percent));
vo.setFileProgressMessage(uploadComplete
? (llmPending > 0
? buildLlmProgressMessage(current, total, llmPending)
: "LLM 已回流 " + current + "/" + total + " 批次,正在生成结果文件")
: buildUploadingLlmProgressMessage(current, total, llmPending));
}
private void attachPythonUploadProgress(SimilarAsinHistoryItemVo vo, Long taskId) {
PythonUploadProgress progress = resolvePythonUploadProgress(taskId);
int total = progress.total() > 0 ? progress.total() : Math.max(1, progress.current() + 1);
int current = Math.max(0, Math.min(progress.current(), total));
int percent = current <= 0 ? 1 : Math.min(95, (int) Math.ceil(current * 100.0 / total));
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
vo.setFileProgressPercent(percent);
vo.setFileProgressMessage(buildPythonUploadProgressMessage(current, total, progress.unit()));
}
private PythonUploadProgress resolvePythonUploadProgress(Long taskId) {
PythonUploadProgress chunkProgress = resolvePythonChunkProgress(taskId);
if (chunkProgress.total() > 0) {
return chunkProgress;
}
if (chunkProgress.current() > 0) {
return new PythonUploadProgress(chunkProgress.current(), chunkProgress.current() + 1, chunkProgress.unit());
}
int totalRows = 0;
try {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
totalRows = allRowCount(task);
} catch (Exception ignored) {
totalRows = 0;
}
int uploadedRows = countSubmittedRows(taskId);
if (totalRows <= 0 && uploadedRows > 0) {
totalRows = uploadedRows + 1;
}
return new PythonUploadProgress(uploadedRows, totalRows, "行");
}
private int allRowCount(FileTaskEntity task) {
try {
return rowCount(readParsedPayload(task));
} catch (Exception ex) {
log.warn("[similar-asin] read all row count failed taskId={} err={}", task.getId(), ex.getMessage());
return 0;
}
}
private int rowCount(SimilarAsinParsedPayloadDto payload) {
if (payload == null) {
return 0;
}
return resolveAllRows(payload).size();
}
private SimilarAsinParsedPayloadDto readParsedPayload(FileTaskEntity task) {
try {
JsonNode root = objectMapper.readTree(task.getResultJson());
String pointer = root.path("parsedPayloadRef").asText("");
if (!pointer.isBlank()) {
String json = transientPayloadStorageService.resolvePayload(pointer, "read similar ASIN parsed payload failed");
return hydrateParsedPayloadRows(objectMapper.readValue(json, SimilarAsinParsedPayloadDto.class));
}
if (root.path("allItems").isArray()) {
return hydrateParsedPayloadRows(objectMapper.treeToValue(root, SimilarAsinParsedPayloadDto.class));
}
} catch (Exception ex) {
throw new BusinessException("读取解析载荷失败");
}
return hydrateParsedPayloadRows(new SimilarAsinParsedPayloadDto());
}
private SimilarAsinParsedPayloadDto hydrateParsedPayloadRows(SimilarAsinParsedPayloadDto payload) {
if (payload == null) {
return new SimilarAsinParsedPayloadDto();
}
List<SimilarAsinParsedRowVo> rows = resolveAllRows(payload);
if (payload.getAllItems() == null || payload.getAllItems().isEmpty()) {
payload.setAllItems(rows);
}
if (payload.getItems() == null || payload.getItems().isEmpty()) {
payload.setItems(rows);
}
return payload;
}
/**
* 从解析载荷统一恢复全量行:优先 items(新规范结构),其次 allItems(旧结构),
* 最后 groups 展开(最旧结构)。新旧结构均可恢复完整行集合,供 rowCount、
* LLM 候选加载与结果文件组装复用。
*/
private static List<SimilarAsinParsedRowVo> resolveAllRows(SimilarAsinParsedPayloadDto payload) {
if (payload == null) {
return List.of();
}
if (containsGroupRefs(payload)) {
// 新格式:分组携带索引引用,信任引用展开结果(越界/非法区间安全跳过)
return expandGroupRefs(payload);
}
List<SimilarAsinParsedRowVo> rows = payload.getItems();
if (rows == null || rows.isEmpty()) {
rows = payload.getAllItems();
}
if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) {
rows = payload.getGroups().stream()
.filter(Objects::nonNull)
.flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream())
.toList();
}
return rows == null ? List.of() : rows;
}
private static boolean containsGroupRefs(SimilarAsinParsedPayloadDto payload) {
if (payload.getGroups() == null) {
return false;
}
for (SimilarAsinParsedGroupVo group : payload.getGroups()) {
if (group != null && (group.getStartIndex() != null || group.getEndIndex() != null)) {
return true;
}
}
return false;
}
/**
* 按分组引用 [startIndex, endIndex) 从 items 展开组内行。
* 引用越界或区间非法时安全跳过,不抛异常;展开不修改 payload 内部状态。
*/
private static List<SimilarAsinParsedRowVo> expandGroupRefs(SimilarAsinParsedPayloadDto payload) {
if (payload == null || payload.getGroups() == null || payload.getGroups().isEmpty()) {
return List.of();
}
List<SimilarAsinParsedRowVo> rows = payload.getItems();
if (rows == null || rows.isEmpty()) {
rows = payload.getAllItems();
}
if (rows == null || rows.isEmpty()) {
return List.of();
}
List<SimilarAsinParsedRowVo> expanded = new ArrayList<>();
for (SimilarAsinParsedGroupVo group : payload.getGroups()) {
if (group == null) {
continue;
}
int start = group.getStartIndex() == null ? 0 : group.getStartIndex();
int end = group.getEndIndex() == null ? 0 : group.getEndIndex();
if (start < 0 || end <= start || end > rows.size()) {
continue;
}
expanded.addAll(rows.subList(start, end));
}
return expanded;
}
private int countSubmittedRows(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
int total = 0;
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks == null || chunks.isEmpty()) {
return 0;
}
for (TaskChunkEntity chunk : chunks) {
total += readChunkRows(chunk).size();
}
return total;
}
private Map<String, SimilarAsinResultRowDto> readChunkRows(TaskChunkEntity chunk) {
Map<String, SimilarAsinResultRowDto> rows = new LinkedHashMap<>();
if (chunk == null || chunk.getPayloadJson() == null || chunk.getPayloadJson().isBlank()) {
return rows;
}
try {
String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read similar ASIN chunk failed");
JsonNode array = objectMapper.readTree(payloadJson);
if (!array.isArray()) {
return rows;
}
for (JsonNode node : array) {
SimilarAsinResultRowDto row = objectMapper.treeToValue(node, SimilarAsinResultRowDto.class);
rows.put(rowKey(row), row);
}
} catch (Exception ex) {
String msg = ex.getMessage() == null ? "" : ex.getMessage();
// P1-4:识别跨实例 local 指针读不到的场景,把"chunk 在另一实例"的元信息
// 通过 task_scope_state.last_error 留痕,便于排查"为何 owner 切换后 chunk 读不到"。
boolean crossInstance = msg.contains("only exists on instance=");
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
recordChunkReadFailure(chunk, crossInstance, msg);
throw new BusinessException("similar ASIN chunk payload read failed chunk="
+ chunk.getChunkIndex() + ": " + msg, ex);
}
return rows;
}
/**
* 把 chunk 读失败信息聚合到 task_scope_state.last_error,
* 形式 "chunk-read-failed[{idx}]" / "chunk-read-failed[{idx}@cross-instance]"。
* best-effort:查询 / 更新失败时仅记日志,不抛回主流程。
*/
private void recordChunkReadFailure(TaskChunkEntity chunk, boolean crossInstance, String msg) {
if (chunk == null || chunk.getTaskId() == null || chunk.getScopeHash() == null) {
return;
}
try {
TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, chunk.getTaskId())
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, chunk.getScopeHash())
.last("limit 1"));
if (scope == null) {
return;
}
String existing = scope.getLastError() == null ? "" : scope.getLastError();
String tag = "chunk-read-failed[" + chunk.getChunkIndex() + (crossInstance ? "@cross-instance" : "") + "]";
if (existing.contains(tag)) {
return;
}
String updated = existing.isBlank() ? tag : existing + "; " + tag;
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, scope.getId())
.set(TaskScopeStateEntity::getLastError, updated)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
} catch (Exception ignored) {
// best-effort:失败不影响主流程
}
}
private String rowKey(SimilarAsinResultRowDto row) {
if (row == null) {
return "";
}
String rowToken = normalize(row.getRowToken());
if (!rowToken.isBlank()) {
return rowToken;
}
return legacyRowKey(row);
}
private String legacyRowKey(SimilarAsinResultRowDto row) {
if (row == null) {
return "";
}
return rowKey(row.getId(), row.getAsin(), row.getCountry());
}
private String rowKey(String id, String asin, String country) {
return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
}
private String normalize(String val) {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
private PythonUploadProgress resolvePythonChunkProgress(Long taskId) {
if (taskId == null || taskId <= 0) {
return new PythonUploadProgress(0, 0, "分片");
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNull(TaskScopeStateEntity::getLlmStatus)
.orderByDesc(TaskScopeStateEntity::getUpdatedAt));
if (states == null || states.isEmpty()) {
return resolvePythonChunkProgressFromChunks(taskId);
}
int current = 0;
int total = 0;
for (TaskScopeStateEntity state : states) {
if (state == null) {
continue;
}
int stateTotal = state.getChunkTotal() == null ? 0 : state.getChunkTotal();
int stateCurrent = state.getReceivedChunkCount() == null ? 0 : state.getReceivedChunkCount();
stateCurrent = Math.max(stateCurrent, resolveReceivedChunkProgress(taskId, state.getScopeHash(), stateTotal));
if (stateTotal > total || total <= 0 && stateCurrent > current) {
total = stateTotal;
current = stateCurrent;
}
}
return new PythonUploadProgress(current, total, "分片");
}
private PythonUploadProgress resolvePythonChunkProgressFromChunks(Long taskId) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks == null || chunks.isEmpty()) {
return new PythonUploadProgress(0, 0, "分片");
}
Map<String, Integer> receivedByScope = new LinkedHashMap<>();
Map<String, Integer> totalByScope = new LinkedHashMap<>();
for (TaskChunkEntity chunk : chunks) {
if (chunk == null) {
continue;
}
String scopeHash = firstNonBlank(chunk.getScopeHash(), "");
receivedByScope.merge(scopeHash, 1, Integer::sum);
if (chunk.getChunkIndex() != null && chunk.getChunkIndex() > 0) {
receivedByScope.merge(scopeHash, chunk.getChunkIndex(), Math::max);
}
if (chunk.getChunkTotal() != null && chunk.getChunkTotal() > 0) {
totalByScope.merge(scopeHash, chunk.getChunkTotal(), Math::max);
}
}
int current = 0;
int total = 0;
for (Map.Entry<String, Integer> entry : receivedByScope.entrySet()) {
int scopeCurrent = entry.getValue() == null ? 0 : entry.getValue();
int scopeTotal = totalByScope.getOrDefault(entry.getKey(), 0);
if (scopeTotal > total || total <= 0 && scopeCurrent > current) {
current = scopeCurrent;
total = scopeTotal;
}
}
return new PythonUploadProgress(current, total, "分片");
}
private int resolveReceivedChunkProgress(Long taskId, String scopeHash, Integer chunkTotal) {
int receivedCount = countChunks(taskId, scopeHash);
Integer maxChunkIndex = taskChunkMapper.selectObjs(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getChunkIndex)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.orderByDesc(TaskChunkEntity::getChunkIndex)
.last("limit 1"))
.stream()
.filter(Objects::nonNull)
.map(value -> {
if (value instanceof Number number) {
return number.intValue();
}
try {
return Integer.parseInt(value.toString());
} catch (Exception ignored) {
return 0;
}
})
.findFirst()
.orElse(0);
int current = Math.max(receivedCount, maxChunkIndex == null ? 0 : maxChunkIndex);
if (chunkTotal != null && chunkTotal > 0) {
current = Math.min(current, chunkTotal);
}
return Math.max(0, current);
}
private int countChunks(Long taskId, String scopeHash) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash));
return count == null ? 0 : count.intValue();
}
private String buildPythonUploadProgressMessage(int current, int total, String unit) {
String safeUnit = unit == null || unit.isBlank() ? "分片" : unit;
return "等待 Python 回传,已回传 " + Math.max(0, current) + "/" + Math.max(1, total) + " " + safeUnit;
}
private String resolveDisplayTaskStatus(FileTaskEntity task, FileResultEntity row) {
if (task == null) {
return null;
}
if (canTreatTaskAsCompleted(task, row, null)) {
return STATUS_SUCCESS;
}
return task.getStatus();
}
private boolean canTreatTaskAsCompleted(FileTaskEntity task, FileResultEntity row, TaskFileJobEntity job) {
if (task == null) {
return false;
}
if (STATUS_SUCCESS.equals(task.getStatus())) {
return true;
}
if (!STATUS_RUNNING.equals(task.getStatus())) {
return false;
}
if (row == null) {
return false;
}
if (row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) {
return false;
}
if (job != null && "FAILED".equals(job.getStatus())) {
return false;
}
return true;
}
private int calculateLlmDisplayTotal(TaskProgressSnapshotEntity snapshot, int observedLlmStates) {
int observed = Math.max(0, observedLlmStates);
// 小任务 totalCount <= 3 时直接使用 observed,避免 totalCount - 3 → 0 引起进度条 0/0。
if (snapshot == null || snapshot.getTotalCount() == null || snapshot.getTotalCount() <= 3) {
return Math.max(1, observed);
}
int expectedFromSnapshot = Math.max(1, snapshot.getTotalCount() - 3);
return Math.max(1, Math.max(observed, expectedFromSnapshot));
}
private String buildLlmProgressMessage(int completed, int total, int pending) {
int safeTotal = Math.max(1, total);
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
if (safeCompleted <= 0 && submitted > 0) {
return "LLM 已提交 " + submitted + "/" + safeTotal + " 批次,等待回流";
}
return "LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次,等待结果文件";
}
private String buildUploadingLlmProgressMessage(int completed, int total, int pending) {
int safeTotal = Math.max(1, total);
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
if (pending > 0) {
return "Python 仍在回传,LLM 已提交 " + submitted + "/" + safeTotal + " 批次";
}
return "Python 仍在回传,LLM 已回流 " + safeCompleted + "/" + safeTotal + " 批次";
}
private int calculateDisplayProgressPercent(int current,
int total,
TaskFileJobEntity job,
LocalDateTime baseTime) {
if (total <= 0) {
return 0;
}
current = Math.max(0, Math.min(current, total));
int percent = Math.max(1, Math.min(99, (int) Math.floor(current * 100.0 / total)));
if (job != null && STATUS_RUNNING.equals(job.getStatus())) {
long elapsedSeconds = baseTime == null ? 0 : Math.max(0, Duration.between(baseTime, LocalDateTime.now()).getSeconds());
if (current <= 0) {
int firstRealProgressPercent = Math.max(1, Math.min(99, (int) Math.floor(100.0 / total)));
int waitingCap = Math.max(8, Math.min(35, firstRealProgressPercent - 1));
percent = Math.max(percent, Math.min(waitingCap, 8 + (int) (elapsedSeconds / 6)));
} else if (current < total) {
percent = Math.max(percent, Math.min(92, percent + (int) (elapsedSeconds / 10)));
}
}
return percent;
}
private int calculateSnapshotDisplayPercent(TaskProgressSnapshotEntity snapshot, TaskFileJobEntity job) {
if (snapshot == null) {
return 0;
}
int total = snapshot.getTotalCount() == null ? 0 : snapshot.getTotalCount();
int current = snapshot.getSuccessCount() == null ? 0 : snapshot.getSuccessCount();
int percent = total <= 0 ? 0 : calculateDisplayProgressPercent(current, total, job, snapshot.getUpdatedAt());
return Math.max(percent, extractSnapshotDisplayPercent(snapshot));
}
private int extractSnapshotDisplayPercent(TaskProgressSnapshotEntity snapshot) {
if (snapshot == null || snapshot.getSnapshotJson() == null || snapshot.getSnapshotJson().isBlank()) {
return 0;
}
try {
JsonNode displayPercent = objectMapper.readTree(snapshot.getSnapshotJson()).path("displayPercent");
if (!displayPercent.isNumber()) {
return 0;
}
return Math.max(0, Math.min(100, displayPercent.asInt()));
} catch (Exception ignored) {
return 0;
}
}
private int countPendingLlmStates(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_SUBMITTED, LLM_STATUS_RUNNING)));
return count == null ? 0 : count.intValue();
}
private int countCompletedLlmStates(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.in(TaskScopeStateEntity::getLlmStatus, List.of(LLM_STATUS_DONE, LLM_STATUS_FAILED)));
return count == null ? 0 : count.intValue();
}
private boolean isResultSubmissionComplete(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNull(TaskScopeStateEntity::getLlmStatus)
.isNotNull(TaskScopeStateEntity::getLastChunkAt));
if (states == null || states.isEmpty()) {
return false;
}
boolean hasCompletedScope = false;
for (TaskScopeStateEntity state : states) {
if (state == null) {
continue;
}
if (Integer.valueOf(1).equals(state.getCompleted())) {
hasCompletedScope = true;
continue;
}
return false;
}
return hasCompletedScope;
}
private String fmt(LocalDateTime t) {
return t == null ? null : t.toString();
}
private String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
private record PythonUploadProgress(int current, int total, String unit) {
}
}
@@ -0,0 +1,22 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import java.util.Locale;
/**
* 任务 85:SimilarAsinRowNormalizer 字段归一化器。
* 规则与 SimilarAsinTaskService.normalize 现状逐字节一致(BOM 剥离、全角空格转半角、
* trim、连续空白折叠为单空格);normalizeAsin = normalize + 大写。纯函数无状态。
*/
public final class SimilarAsinRowNormalizer {
private SimilarAsinRowNormalizer() {
}
public static String normalize(String val) {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
public static String normalizeAsin(String val) {
return normalize(val).toUpperCase(Locale.ROOT);
}
}
@@ -0,0 +1,66 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Set;
/**
* 任务 87:SimilarAsinRowValidator 纯校验器。
* 必填检查(id/asin/country)、ASIN 格式校验(仅允许字母数字,含校验前大写化)、
* 重复检测(asin+country 键,保留首次出现)。返回错误清单不抛异常,由调用方决定处理。
* 规则与现状 parseWorkbook 必填丢弃 / dedupeRowsByRowKey 保留首次语义一致。
*/
public final class SimilarAsinRowValidator {
public record RowError(int rowIndex, String column, String message) {
}
private SimilarAsinRowValidator() {
}
public static List<RowError> validate(List<SimilarAsinParsedRowVo> rows) {
List<RowError> errors = new ArrayList<>();
if (rows == null || rows.isEmpty()) {
return errors;
}
Set<String> seenKeys = new java.util.HashSet<>();
for (SimilarAsinParsedRowVo row : rows) {
if (row == null) {
continue;
}
String id = SimilarAsinRowNormalizer.normalize(row.getSourceId());
String asin = SimilarAsinRowNormalizer.normalizeAsin(row.getAsin());
String country = SimilarAsinRowNormalizer.normalize(row.getCountry());
if (id.isEmpty() && asin.isEmpty() && country.isEmpty()) {
continue;
}
int rowIndex = row.getRowIndex() == null ? 0 : row.getRowIndex();
if (id.isEmpty()) {
errors.add(new RowError(rowIndex, "id", "缺少必要字段: id"));
}
if (asin.isEmpty()) {
errors.add(new RowError(rowIndex, "asin", "缺少必要字段: asin"));
}
if (country.isEmpty()) {
errors.add(new RowError(rowIndex, "country", "缺少必要字段: 国家"));
}
if (asin.isEmpty()) {
continue;
}
if (!asin.matches("[A-Z0-9]+")) {
errors.add(new RowError(rowIndex, "asin",
"ASIN 格式不正确(仅允许字母数字): " + asin));
continue;
}
String dupKey = asin + "::" + country;
if (!seenKeys.add(dupKey)) {
errors.add(new RowError(rowIndex, "asin",
"重复 ASIN: " + asin + " 国家: " + country));
}
}
return errors;
}
}
@@ -0,0 +1,112 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.Font;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import java.util.List;
import java.util.Locale;
import java.util.Map;
/**
* 任务 88:SimilarAsinSheetBuilder Sheet 构造器。
* 结果 Workbook/Sheet 构造辅助(表头、列顺序、样式);输入数据 → 输出 workbook;不落库、无 IO 依赖。
* 表头/列序/样式与现状 assembleResultWorkbook 一致:sheet 名"相似asin检测"、15 列表头、
* 加粗表头、数据行从第 1 行、图片列 12-14 列宽、行高 409pt、值兜底与现状一致。
* 结果派生列(是否有货/相似度/是否符合类目/不符合理由/产品类目/状态)来自 LLM 结果行,
* 本 builder 输入只有解析行,这些列由调用方补写。
*/
public final class SimilarAsinSheetBuilder {
public static final List<String> RESULT_HEADERS = List.of(
"id",
"asin",
"国家",
"价格",
"卖家名称",
"品牌",
"是否有货",
"相似度",
"是否符合类目",
"不符合理由",
"产品类目",
"状态",
"主图",
"阿里巴巴图片1",
"阿里巴巴图片2"
);
public static final int IMG_COL_MAIN = 12;
public static final int IMG_COL_PUZZLE1 = 13;
public static final int IMG_COL_PUZZLE2 = 14;
private SimilarAsinSheetBuilder() {
}
public static void build(Workbook workbook, List<SimilarAsinParsedRowVo> rows) {
Sheet sheet = workbook.createSheet("相似asin检测");
CellStyle headerStyle = workbook.createCellStyle();
Font font = workbook.createFont();
font.setBold(true);
headerStyle.setFont(font);
sheet.setColumnWidth(IMG_COL_MAIN, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256);
sheet.setColumnWidth(IMG_COL_PUZZLE1, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256);
sheet.setColumnWidth(IMG_COL_PUZZLE2, SimilarAsinImageEmbedder.IMAGE_COL_WIDTH_CHARS * 256);
Row header = sheet.createRow(0);
for (int i = 0; i < RESULT_HEADERS.size(); i++) {
Cell cell = header.createCell(i);
cell.setCellValue(RESULT_HEADERS.get(i));
cell.setCellStyle(headerStyle);
}
if (rows == null) {
return;
}
int rowIndex = 1;
for (SimilarAsinParsedRowVo parsedRow : rows) {
if (parsedRow == null) {
continue;
}
Row row = sheet.createRow(rowIndex);
int col = 0;
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
row.createCell(col++).setCellValue(firstNonBlank(
parsedRow.getPrice(), readValueByHeader(parsedRow, "价格", "price")));
row.createCell(col++).setCellValue(readValueByHeader(
parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name",
"seller-name", "sellername", "store name", "shop name"));
row.createCell(col).setCellValue(readValueByHeader(parsedRow, "品牌", "brand"));
row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS);
rowIndex++;
}
}
private static String readValueByHeader(SimilarAsinParsedRowVo row, String... candidates) {
if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) {
return "";
}
for (Map.Entry<String, String> entry : row.getValues().entrySet()) {
String header = SimilarAsinRowNormalizer.normalize(entry.getKey()).toLowerCase(Locale.ROOT);
for (String candidate : candidates) {
String expected = SimilarAsinRowNormalizer.normalize(candidate).toLowerCase(Locale.ROOT);
if (!expected.isBlank() && header.contains(expected)) {
return entry.getValue() == null ? "" : entry.getValue();
}
}
}
return "";
}
private static String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
}