fix(backlog): 品牌出口统一、join 截止与进行中任务补回

- BrandLabel 出口守卫扩到全仓 entity.setBrand,撞款/去重/品牌库统一清洗
- FutureDeadline 替换五处无界 join,回传口锁等待降为 1s
- 采集/跟价/外观专利 history 补回窗口外进行中与续跑任务
- 工作台客户端版本强制单调;catch 棘轮按新增出口逐处记名
This commit is contained in:
2026-10-25 13:27:46 +08:00
parent 1dd9e66d21
commit 5928072c6a
30 changed files with 558 additions and 75 deletions
@@ -0,0 +1,29 @@
package com.nanri.aiimage.common.support;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* 带截止的 future 等待。
*
* <p>{@code join()} 没有时间界:JDK HTTP 客户端的超时只封顶到响应头,响应体读取一旦卡住,
* 行线程/任务线程会永久 park。凡是把 worker 线程 join 回主线程的地方都走这里,超时即取消
* 该 future 并把超时作为该行/该批的失败收口,不拖住整条任务。</p>
*/
public final class FutureDeadline {
private FutureDeadline() {
}
public static <T> T await(CompletableFuture<T> future, long timeoutMillis, String label)
throws InterruptedException, ExecutionException, TimeoutException {
try {
return future.get(Math.max(1L, timeoutMillis), TimeUnit.MILLISECONDS);
} catch (TimeoutException ex) {
future.cancel(true);
throw new TimeoutException(label + " 超过 " + timeoutMillis + "ms 未完成,已取消");
}
}
}
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.retry.LlmKeyGate;
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
import com.nanri.aiimage.common.retry.UnreachableCooldown;
import com.nanri.aiimage.common.support.FutureDeadline;
import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.config.HttpClientPool;
import com.nanri.aiimage.metrics.ExternalCallMetricsRecorder;
@@ -32,9 +33,12 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.regex.Matcher;
@@ -376,6 +380,13 @@ public class AppearancePatentLlmClient {
return chunks;
}
/** 单批等待上限:按一次读超时 × 重试次数再留一倍余量,封顶 2 小时。 */
private long futureDeadlineMillis() {
long perAttempt = Math.max(1_000L, properties.getLlmReadTimeoutMillis());
long total = perAttempt * (Math.max(1, properties.getLlmRetryTimes()) + 1L);
return Math.min(TimeUnit.HOURS.toMillis(2), Math.max(TimeUnit.MINUTES.toMillis(5), total * 2));
}
private <T> List<BatchOutcome<T>> runBatches(List<List<Integer>> chunks, BatchInvoker<T> invoker) {
Semaphore concurrency = new Semaphore(Math.max(1, properties.getLlmRowConcurrency()));
ExecutorService executor = rowExecutor();
@@ -401,12 +412,16 @@ public class AppearancePatentLlmClient {
List<BatchOutcome<T>> outcomes = new ArrayList<>(chunks.size());
for (int i = 0; i < futures.size(); i++) {
try {
outcomes.add(futures.get(i).join());
} catch (Exception ex) {
// 调度层炸了(invoker 本该自己 catch 干净):按传输失败收口,宁可整批记失败也不逐行放大
outcomes.add(FutureDeadline.await(futures.get(i), futureDeadlineMillis(), "外观专利 LLM 批次"));
} catch (TimeoutException | ExecutionException ex) {
// 调度层炸了或单批超过等待上限(invoker 本该自己 catch 干净):按传输失败收口
log.warn("[appearance-patent] llm batch crashed rows={} err={}",
chunks.get(i).size(), failureMessage(ex));
outcomes.add(new BatchOutcome<T>(Map.of(), failureMessage(ex), false));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
log.warn("[appearance-patent] llm batch wait interrupted rows={}", chunks.get(i).size());
outcomes.add(new BatchOutcome<T>(Map.of(), "LLM 批次等待被中断", false));
}
}
return outcomes;
@@ -63,6 +63,18 @@ public class AppearancePatentHistoryAssembler {
return vo;
}
/** 结果行不在返回窗口里的在跑/续跑任务:只带前端渲染当前任务所需字段。 */
public AppearancePatentHistoryItemVo buildActiveItem(FileTaskEntity task) {
AppearancePatentHistoryItemVo vo = new AppearancePatentHistoryItemVo();
vo.setTaskId(task.getId());
vo.setTaskStatus(task.getStatus());
vo.setSuccess(false);
vo.setCreatedAt(fmt(task.getCreatedAt()));
vo.setStartedAt(fmt(task.getCreatedAt()));
vo.setFinishedAt(fmt(task.getFinishedAt()));
return vo;
}
public int historyPriority(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
String taskStatus = task == null ? null : task.getStatus();
if (STATUS_PENDING.equals(taskStatus) || STATUS_RUNNING.equals(taskStatus) || isHistoryFileBuilding(row, taskStatus, job)) {
@@ -125,9 +125,46 @@ public class AppearancePatentProgressSupport {
FileTaskEntity task = taskMap.get(row.getTaskId());
vo.getItems().add(toHistoryItem(row, task, jobMap.get(row.getId())));
}
prependActiveTasksOutsideWindow(vo, userId, taskIds);
return vo;
}
/**
* 结果行被 top-N 窗口挤出后,仍在执行/续跑排队的任务会从列表彻底消失
* (页面重载即无兜底)。这里按任务头补回最前面;已有结果行的任务不重复补。
*/
private void prependActiveTasksOutsideWindow(AppearancePatentHistoryVo vo, Long userId, List<Long> coveredTaskIds) {
List<FileTaskEntity> activeTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getStatus,
FileTaskEntity::getResumeOfTaskId,
FileTaskEntity::getCreatedAt,
FileTaskEntity::getFinishedAt)
.eq(FileTaskEntity::getModuleType, moduleType)
.eq(FileTaskEntity::getUserId, userId)
.in(FileTaskEntity::getStatus, TaskStatuses.FileTask.ACTIVE_ORDERED)
.orderByDesc(FileTaskEntity::getCreatedAt)
.last("limit 50"));
if (activeTasks == null || activeTasks.isEmpty()) {
return;
}
AppearancePatentHistoryAssembler assembler = historyAssembler();
List<AppearancePatentHistoryItemVo> activeItems = new ArrayList<>();
for (FileTaskEntity task : activeTasks) {
if (task == null || task.getId() == null || coveredTaskIds.contains(task.getId())) {
continue;
}
// 普通 PENDING 是未启动的解析预览,不进列表;续跑克隆件必须留(与相似 ASIN 同口径)
if (TaskStatuses.FileTask.PENDING.equals(task.getStatus()) && task.getResumeOfTaskId() == null) {
continue;
}
activeItems.add(assembler.buildActiveItem(task));
}
if (!activeItems.isEmpty()) {
vo.getItems().addAll(0, activeItems);
}
}
public AppearancePatentTaskBatchVo progressBatch(List<Long> taskIds) {
AppearancePatentTaskBatchVo vo = new AppearancePatentTaskBatchVo();
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
@@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.security.AdminAuthSupport;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.common.support.FutureDeadline;
import com.nanri.aiimage.common.text.BrandLabel;
import com.nanri.aiimage.common.excel.ExcelStreamReader;
import com.nanri.aiimage.config.BrandProgressProperties;
@@ -71,9 +72,11 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -118,6 +121,8 @@ public class BrandTaskService {
private static final int RESULT_UPLOAD_CONCURRENCY = 3;
// 有界队列线程池:newFixedThreadPool 用的是无界队列,任务堆积时不会拒绝、会把内存吃满
/** 结果文件上传是收尾腿,任何一片卡死都不能让任务线程无限 join。 */
private static final long RESULT_UPLOAD_DEADLINE_MILLIS = TimeUnit.MINUTES.toMillis(30);
private final ExecutorService resultUploadExecutor = com.nanri.aiimage.common.support.ThreadPools
.boundedFixed("brand-result-upload", RESULT_UPLOAD_CONCURRENCY);
@@ -1382,8 +1387,13 @@ public class BrandTaskService {
}
for (CompletableFuture<String> future : uploadFutures) {
try {
fullUrls.add(future.join());
} catch (CompletionException ex) {
fullUrls.add(FutureDeadline.await(future, RESULT_UPLOAD_DEADLINE_MILLIS, "结果文件上传"));
} catch (TimeoutException ex) {
throw new IOException("结果文件上传超时(" + RESULT_UPLOAD_DEADLINE_MILLIS + "ms)", ex);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IOException("结果文件上传被中断", ex);
} catch (ExecutionException ex) {
Throwable cause = ex.getCause() == null ? ex : ex.getCause();
if (cause instanceof IOException ioe) {
throw ioe;
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.brandgating.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.text.BrandLabel;
import com.nanri.aiimage.modules.brandgating.mapper.BrandGatingMapper;
import com.nanri.aiimage.modules.brandgating.model.dto.BrandGatingAddRequest;
import com.nanri.aiimage.modules.brandgating.model.dto.BrandGatingFilterQuery;
@@ -91,7 +92,7 @@ public class BrandGatingService {
public BrandGatingAddResultVo add(BrandGatingAddRequest request) {
BrandGatingAddResultVo vo = new BrandGatingAddResultVo();
String brand = trimToEmpty(request.getBrand());
String brand = trimToEmpty(BrandLabel.strip(request.getBrand()));
if (brand.isEmpty()) {
throw new BusinessException(400, "品牌不能为空");
}
@@ -133,7 +134,7 @@ public class BrandGatingService {
BrandGatingImportResultVo vo = new BrandGatingImportResultVo();
Set<String> seen = new HashSet<>();
for (BrandGatingImportRequest.Item item : items) {
String brand = trimToEmpty(item.getBrand());
String brand = trimToEmpty(BrandLabel.strip(item.getBrand()));
String country = firstNonBlank(item.getCountry(), DEFAULT_COUNTRY);
String gatingType = firstNonBlank(item.getGatingType(), DEFAULT_GATING_TYPE);
String asin = trimToEmpty(item.getAsin());
@@ -148,7 +149,8 @@ public class BrandGatingService {
continue;
}
BrandGatingEntity entity = new BrandGatingEntity();
entity.setBrand(brand);
// 与 add() 同口径:唯一键/去重键用的是洗后的 brand,写出再兜一次,防止调用方直传脏值
entity.setBrand(BrandLabel.strip(brand));
entity.setAsin(asin.isEmpty() ? null : asin);
entity.setCountry(country);
entity.setGatingType(gatingType);
@@ -702,15 +702,60 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
.eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt)
.last("limit " + safeLimit));
if (rows == null || rows.isEmpty()) {
return vo;
}
List<Long> taskIds = rows.stream().map(FileResultEntity::getTaskId).filter(Objects::nonNull).distinct().toList();
List<Long> taskIds = rows == null ? List.of() : rows.stream()
.map(FileResultEntity::getTaskId).filter(Objects::nonNull).distinct().toList();
Map<Long, FileTaskEntity> taskMap = loadTaskMap(taskIds);
for (FileResultEntity row : rows) {
FileTaskEntity task = row.getTaskId() == null ? null : taskMap.get(row.getTaskId());
vo.getItems().add(toHistoryItem(row, task));
if (rows != null) {
for (FileResultEntity row : rows) {
FileTaskEntity task = row.getTaskId() == null ? null : taskMap.get(row.getTaskId());
vo.getItems().add(toHistoryItem(row, task));
}
}
prependActiveTasksOutsideWindow(vo, userId, taskIds);
return vo;
}
/**
* 结果行被 top-N 窗口挤出后,仍在执行/排队中的任务会从列表彻底消失(页面重载即无兜底)。
* 这里按任务头补回最前面;已有结果行的任务不重复补。
*/
private void prependActiveTasksOutsideWindow(CollectDataHistoryVo vo, Long userId, List<Long> coveredTaskIds) {
List<FileTaskEntity> activeTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getTaskNo,
FileTaskEntity::getStatus,
FileTaskEntity::getErrorMessage,
FileTaskEntity::getResumeOfTaskId,
FileTaskEntity::getRequestJson,
FileTaskEntity::getResultJson,
FileTaskEntity::getCreatedAt,
FileTaskEntity::getFinishedAt)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getUserId, userId)
.in(FileTaskEntity::getStatus, List.of(STATUS_PENDING, STATUS_RUNNING))
.orderByDesc(FileTaskEntity::getCreatedAt)
.last("limit 50"));
if (activeTasks == null || activeTasks.isEmpty()) {
return;
}
List<CollectDataHistoryItemVo> activeItems = new ArrayList<>();
for (FileTaskEntity task : activeTasks) {
if (task == null || task.getId() == null || coveredTaskIds.contains(task.getId())) {
continue;
}
activeItems.add(toActiveHistoryItem(task));
}
if (!activeItems.isEmpty()) {
vo.getItems().addAll(0, activeItems);
}
}
private CollectDataHistoryItemVo toActiveHistoryItem(FileTaskEntity task) {
CollectDataHistoryItemVo vo = new CollectDataHistoryItemVo();
vo.setTaskId(task.getId());
vo.setSuccess(false);
vo.setError(task.getErrorMessage());
fillTaskHistoryFields(vo, task);
return vo;
}
@@ -1745,31 +1790,35 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
vo.setError(row.getErrorMessage());
vo.setCreatedAt(formatTime(row.getCreatedAt()));
if (task != null) {
CollectDataStats stats = loadStats(task);
vo.setTaskNo(task.getTaskNo());
vo.setTaskStatus(task.getStatus());
vo.setStartedAt(formatTime(task.getCreatedAt()));
vo.setFinishedAt(formatTime(task.getFinishedAt()));
vo.setTaskType(extractTaskTypeFromRequest(task));
vo.setFilters(extractFiltersFromRequest(task));
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
vo.setBrandRejectedCount(stats.brandRejectedCount);
vo.setFinalRowCount(stats.finalRowCount);
vo.setTotalRows(stats.totalRows);
vo.setReceivedRows(stats.receivedRows);
vo.setProcessedRows(stats.processedRows);
vo.setCollectStage(stats.collectStage);
vo.setCurrentKeyword(stats.currentKeyword);
vo.setSearchCurrentPage(stats.searchCurrentPage);
vo.setSearchTotalPages(stats.searchTotalPages);
vo.setDetailProcessedAsins(stats.detailProcessedAsins);
vo.setDetailTotalAsins(stats.detailTotalAsins);
vo.setProgressPercent(calculateProgressPercent(task.getStatus(), stats.totalRows, stats.processedRows));
fillTaskHistoryFields(vo, task);
}
return vo;
}
private void fillTaskHistoryFields(CollectDataHistoryItemVo vo, FileTaskEntity task) {
CollectDataStats stats = loadStats(task);
vo.setTaskNo(task.getTaskNo());
vo.setTaskStatus(task.getStatus());
vo.setStartedAt(formatTime(task.getCreatedAt()));
vo.setFinishedAt(formatTime(task.getFinishedAt()));
vo.setTaskType(extractTaskTypeFromRequest(task));
vo.setFilters(extractFiltersFromRequest(task));
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
vo.setBrandRejectedCount(stats.brandRejectedCount);
vo.setFinalRowCount(stats.finalRowCount);
vo.setTotalRows(stats.totalRows);
vo.setReceivedRows(stats.receivedRows);
vo.setProcessedRows(stats.processedRows);
vo.setCollectStage(stats.collectStage);
vo.setCurrentKeyword(stats.currentKeyword);
vo.setSearchCurrentPage(stats.searchCurrentPage);
vo.setSearchTotalPages(stats.searchTotalPages);
vo.setDetailProcessedAsins(stats.detailProcessedAsins);
vo.setDetailTotalAsins(stats.detailTotalAsins);
vo.setProgressPercent(calculateProgressPercent(task.getStatus(), stats.totalRows, stats.processedRows));
}
private int calculateProgressPercent(String status, int totalRows, int receivedRows) {
if (STATUS_SUCCESS.equals(status)) {
return 100;
@@ -66,6 +66,9 @@ public class DedupeRunService {
private static final List<String> EXPORT_COLUMN_PRIORITY = List.of("id", "ASIN", "国家", "价格", "品牌");
/** 去重源文件的列集由用户勾选,品牌列只能按表头认出来,其余列一律原文透传。 */
private static final String BRAND_COLUMN_HEADER = "品牌";
/** 源表可能导出自不同模板,品牌列别名必须在这里收口;比对键仍用正名。 */
private static final List<String> BRAND_COLUMN_HEADERS =
List.of(BRAND_COLUMN_HEADER, "Brand", "品牌(英文)", "品牌(英文)");
/** 同时处理的文件数上限,避免多文件并行时打满内存与 OSS 上传带宽 */
private static final int MAX_PARALLEL_FILES = 4;
/**
@@ -866,6 +869,11 @@ public class DedupeRunService {
return headerIndex;
}
private static boolean isBrandColumn(String column) {
return column != null && BRAND_COLUMN_HEADERS.stream().anyMatch(
alias -> alias.equalsIgnoreCase(column.strip()));
}
private List<String> reorderExportColumns(List<String> selectedColumns) {
if (selectedColumns == null || selectedColumns.isEmpty()) {
return selectedColumns;
@@ -891,7 +899,7 @@ public class DedupeRunService {
String value = normalizeCellText(cellText(rowMap, headerIndex.get(selectedColumn)));
// 品牌列的值可能是用户从亚马逊详情页复制来的署名("Brand: XXX"):在这里剥一次,
// 下方出件写 cell 用的就是同一个字符串,比对与交付不会拿到两个版本
selectedValues.add(BRAND_COLUMN_HEADER.equals(selectedColumn) ? BrandLabel.strip(value) : value);
selectedValues.add(isBrandColumn(selectedColumn) ? BrandLabel.strip(value) : value);
}
if (asinColumnIndex == null) {
asinColumnIndex = headerIndex.get("ASIN");
@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.pricetrack.service;
import com.nanri.aiimage.common.task.TaskStatuses;
import com.nanri.aiimage.common.text.UserMessages;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
@@ -245,18 +246,19 @@ public class PriceTrackTaskService extends AbstractShopPayloadTaskPipeline<Price
.orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) {
vo.setItems(List.of());
log.debug("[price-track] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
Map<Long, FileTaskEntity> taskById = new LinkedHashMap<>();
List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId)
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (entities.isEmpty()) {
vo.setItems(new ArrayList<>());
prependActiveTasksOutsideWindow(vo, userId, taskIds);
log.debug("[price-track] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
Map<Long, FileTaskEntity> taskById = new LinkedHashMap<>();
if (!taskIds.isEmpty()) {
List<FileTaskEntity> tasks = selectTaskHistoryFieldsByIdsInBatches(taskIds);
for (FileTaskEntity t : tasks) {
@@ -275,6 +277,7 @@ public class PriceTrackTaskService extends AbstractShopPayloadTaskPipeline<Price
items.add(toHistoryItemVo(entity, taskById.get(entity.getTaskId()), jobMap.get(entity.getId()), false));
}
vo.setItems(items);
prependActiveTasksOutsideWindow(vo, userId, taskIds);
long finishedAt = System.nanoTime();
log.debug("[price-track] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
@@ -289,6 +292,45 @@ public class PriceTrackTaskService extends AbstractShopPayloadTaskPipeline<Price
return vo;
}
/**
* 结果行被 top-N 窗口挤出后,仍在执行/排队中的任务会从列表彻底消失(页面重载即无兜底)。
* 这里按任务头补回最前面;已有结果行的任务不重复补。
*/
private void prependActiveTasksOutsideWindow(PriceTrackHistoryVo vo, Long userId, List<Long> coveredTaskIds) {
List<FileTaskEntity> activeTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getTaskNo,
FileTaskEntity::getStatus,
FileTaskEntity::getErrorMessage,
FileTaskEntity::getCreatedAt,
FileTaskEntity::getFinishedAt)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getUserId, userId)
.in(FileTaskEntity::getStatus, TaskStatuses.FileTask.ACTIVE_ORDERED)
.orderByDesc(FileTaskEntity::getCreatedAt)
.last("limit 50"));
if (activeTasks == null || activeTasks.isEmpty()) {
return;
}
List<PriceTrackResultItemVo> activeItems = new ArrayList<>();
for (FileTaskEntity task : activeTasks) {
if (task == null || task.getId() == null || coveredTaskIds.contains(task.getId())) {
continue;
}
PriceTrackResultItemVo item = new PriceTrackResultItemVo();
item.setTaskId(task.getId());
item.setTaskStatus(task.getStatus());
item.setSuccess(false);
item.setError(task.getErrorMessage());
item.setCreatedAt(fmt(task.getCreatedAt()));
item.setFinishedAt(fmt(task.getFinishedAt()));
activeItems.add(item);
}
if (!activeItems.isEmpty()) {
vo.getItems().addAll(0, activeItems);
}
}
/**
* 事务边界:Redis 任务锁在事务外获取(自旋等待不再占用 DB 连接),
* DB 段(复核 + 删除 + 重算任务状态)仍在同一个事务内。
@@ -78,7 +78,9 @@ public final class ShopDataCrawlRowNormalizer {
}
public static boolean rowEmpty(ShopDataCrawlRowDto row) {
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getBrand()) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
// 品牌要按出口口径判空:纯 "Brand:" 这类标签值洗完是空串,若不先洗,
// 一行只有标签的脏行会通过 rowEmpty,随后 copyRow 把它洗成整行空白落件。
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(brand(row.getBrand())) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
&& blank(row.getSalesRank()) && blank(row.getPageViews()) && blank(row.getUnitsSold())
&& blank(row.getPrice()) && blank(row.getRecommendedOffer()));
}
@@ -143,18 +143,23 @@ public class ShopDataDuplicateCheckQueryService {
return result;
}
/** 逐 ASIN 裁剪 occurrences 并重算 shop_count/record_count;items 保持缓存序。 */
/**
* 逐 ASIN 裁剪 occurrences 并重算 shop_count/record_count;items 保持缓存序。
*
* <p>legacy payload 里的 brand 可能仍带 "Brand:" 前缀(存量扫描结果在下次重写前不会变),
* 所有页面出口都经过这里,所以在这一层统一洗一次;可见范围为空时也要洗,不能直接短路返回原对象。</p>
*/
public List<DuplicateItem> cropItems(List<DuplicateItem> items, Set<String> visibleKeys) {
if (visibleKeys == null || items == null) {
return items == null ? List.of() : items;
if (items == null) {
return List.of();
}
List<DuplicateItem> result = new ArrayList<>(items.size());
for (DuplicateItem item : items) {
List<DuplicateOccurrence> kept = new ArrayList<>();
Set<String> shops = new LinkedHashSet<>();
for (DuplicateOccurrence occurrence : item.occurrences()) {
if (visibleKeys.contains(shopKey(occurrence.shopName()))) {
kept.add(occurrence);
if (visibleKeys == null || visibleKeys.contains(shopKey(occurrence.shopName()))) {
kept.add(withCleanBrand(occurrence));
shops.add(occurrence.shopName());
}
}
@@ -166,6 +171,15 @@ public class ShopDataDuplicateCheckQueryService {
return result;
}
private static DuplicateOccurrence withCleanBrand(DuplicateOccurrence occ) {
String clean = BrandLabel.strip(occ.brand());
if (java.util.Objects.equals(clean, occ.brand())) {
return occ;
}
return new DuplicateOccurrence(occ.asin(), occ.date(), occ.price(), clean, occ.shopName(),
occ.groupName(), occ.countryCodes(), occ.country());
}
public Map<String, Object> overviewData(DuplicateScanView view, Set<String> visibleKeys) {
List<DuplicateShop> shops = cropShops(view.payload().shops(), visibleKeys);
List<DuplicateItem> items = cropItems(view.payload().items(), visibleKeys);
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.service.support.PuzzleImageMerger;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinImageEmbedder;
import com.nanri.aiimage.common.support.FutureDeadline;
import com.nanri.aiimage.modules.usersecret.support.SecretUsageContext;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -30,9 +31,12 @@ import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* 货源查询直连 LLM 编排服务:复刻原工作流 similarity_asin
@@ -259,7 +263,7 @@ public class SimilarAsinLlmService {
List<SimilarAsinResultRowDto> merged = new ArrayList<>(rows.size());
for (int i = 0; i < futures.size(); i++) {
try {
merged.add(futures.get(i).join());
merged.add(FutureDeadline.await(futures.get(i), futureDeadlineMillis(), "LLM 单行处理"));
} catch (Exception ex) {
log.warn("[similar-asin][llm] row failed index={} asin={} err={}",
i, rows.get(i).getAsin(), failureMessage(ex));
@@ -269,6 +273,13 @@ public class SimilarAsinLlmService {
return merged;
}
/** 单行/单批等待上限:按一次读超时 × 重试次数再留一倍余量,封顶 2 小时。 */
private long futureDeadlineMillis() {
long perAttempt = Math.max(1_000L, properties.getLlmReadTimeoutMillis());
long total = perAttempt * (Math.max(1, properties.getLlmRetryTimes()) + 1L);
return Math.min(TimeUnit.HOURS.toMillis(2), Math.max(TimeUnit.MINUTES.toMillis(5), total * 2));
}
private SimilarAsinResultRowDto inspectRow(SimilarAsinResultRowDto source,
int rowIndex,
CategoryPrepass prepass,
@@ -460,7 +471,16 @@ public class SimilarAsinLlmService {
}, usageContext), executor));
}
for (CompletableFuture<Void> future : futures) {
future.join(); // matchCategoryChunk 内部自捕获,join 只会因中断失败
try {
FutureDeadline.await(future, futureDeadlineMillis(), "LLM 类目批次");
} catch (TimeoutException | ExecutionException ex) {
// 超时批次的结果按缺失处理:缺行在下游走单行回退,不让一个批次拖住整批
log.warn("[similar-asin][llm] category batch closed without result err={}", failureMessage(ex));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
log.warn("[similar-asin][llm] category batch wait interrupted");
break;
}
}
return new CategoryPrepass(paths, errors);
}
@@ -737,7 +757,15 @@ public class SimilarAsinLlmService {
}, usageContext), executor));
}
for (CompletableFuture<Void> future : futures) {
future.join(); // checkConformChunk 内部自捕获,join 只会因中断失败
try {
FutureDeadline.await(future, futureDeadlineMillis(), "LLM 合规批次");
} catch (TimeoutException | ExecutionException ex) {
log.warn("[similar-asin][llm] conform batch closed without result err={}", failureMessage(ex));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
log.warn("[similar-asin][llm] conform batch wait interrupted");
break;
}
}
return new ConformPrepass(byRow, errors);
}
@@ -126,6 +126,11 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L;
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
private static final long TASK_LOCK_WAIT_MILLIS = 10000L;
/**
* 回传口撞锁时只等 1 秒:锁被 LLM 批合法持有几分钟是常态,客户端 result_feed 本来就会按龄重试,
* 服务端替它等满 10 秒只会白占一个 Tomcat 线程(B15 §3 第 2 条)。
*/
static final long RESULT_SUBMIT_LOCK_WAIT_MILLIS = 1000L;
/**
* Task 8:WorkbookFactory 打开前的受控读取。xlsx 本质是 zip,
* 先用 ZipFile 中央目录探测条目数与解压总字节(不读取压缩内容),
@@ -791,7 +796,8 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
}
public void submitResult(Long taskId, SimilarAsinSubmitResultRequest request) {
try (TaskDistributedLockService.LockHandle ignored = ownershipSupport().requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) {
try (TaskDistributedLockService.LockHandle ignored =
ownershipSupport().requireTaskLock(taskId, RESULT_SUBMIT_LOCK_WAIT_MILLIS)) {
submitResultLocked(taskId, request);
}
}
@@ -9,6 +9,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.math.BigInteger;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
@@ -191,6 +192,13 @@ public class WorkbenchClientPackageService {
normalizedVersion, objectKey);
throw new BusinessException("该版本已存在,请提升版本号或先删除旧包");
}
WorkbenchClientPackageEntity latest = newest();
if (latest != null && versionIsLower(normalizedVersion, latest.getVersion())) {
// 按上传时间取最新时,低版本后传会顶掉当前更新目标,已升级装机反而被判"无需更新"
log.warn("[workbench-client] 直传确认失败:版本低于当前最新 version={} latest={} objectKey={}",
normalizedVersion, latest.getVersion(), objectKey);
throw new BusinessException("版本号不能低于当前最新版本 " + latest.getVersion());
}
String sha256 = ossStorageService.workbenchClientObjectSha256(objectKey);
if (sha256 == null || sha256.length() != 64) {
log.warn("[workbench-client] 直传确认失败:摘要计算异常 objectKey={} sha256={}", objectKey, sha256);
@@ -244,6 +252,43 @@ public class WorkbenchClientPackageService {
return deleted;
}
/** 仅对「数字点分」版本做单调性判断;命名不合规时跳过,不臆造顺序。 */
static boolean versionIsLower(String candidate, String current) {
List<BigInteger> left = versionSegments(candidate);
List<BigInteger> right = versionSegments(current);
if (left == null || right == null) {
return false;
}
int size = Math.max(left.size(), right.size());
for (int i = 0; i < size; i++) {
BigInteger a = i < left.size() ? left.get(i) : BigInteger.ZERO;
BigInteger b = i < right.size() ? right.get(i) : BigInteger.ZERO;
int cmp = a.compareTo(b);
if (cmp != 0) {
return cmp < 0;
}
}
return false;
}
private static List<BigInteger> versionSegments(String value) {
String normalized = value == null ? "" : value.trim();
if (normalized.startsWith("v") || normalized.startsWith("V")) {
normalized = normalized.substring(1);
}
if (normalized.isEmpty()) {
return null;
}
List<BigInteger> segments = new ArrayList<>();
for (String part : normalized.split("\\.", -1)) {
if (!part.matches("\\d+")) {
return null;
}
segments.add(new BigInteger(part));
}
return segments;
}
private WorkbenchClientPackageEntity newest() {
return packageMapper.selectOne(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
.orderByDesc(WorkbenchClientPackageEntity::getCreatedAt)