fix(backlog): 品牌出口统一、join 截止与进行中任务补回
- BrandLabel 出口守卫扩到全仓 entity.setBrand,撞款/去重/品牌库统一清洗 - FutureDeadline 替换五处无界 join,回传口锁等待降为 1s - 采集/跟价/外观专利 history 补回窗口外进行中与续跑任务 - 工作台客户端版本强制单调;catch 棘轮按新增出口逐处记名
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
package com.nanri.aiimage.common.support;
|
||||
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
/**
|
||||
* 带截止的 future 等待。
|
||||
*
|
||||
* <p>{@code join()} 没有时间界:JDK HTTP 客户端的超时只封顶到响应头,响应体读取一旦卡住,
|
||||
* 行线程/任务线程会永久 park。凡是把 worker 线程 join 回主线程的地方都走这里,超时即取消
|
||||
* 该 future 并把超时作为该行/该批的失败收口,不拖住整条任务。</p>
|
||||
*/
|
||||
public final class FutureDeadline {
|
||||
|
||||
private FutureDeadline() {
|
||||
}
|
||||
|
||||
public static <T> T await(CompletableFuture<T> future, long timeoutMillis, String label)
|
||||
throws InterruptedException, ExecutionException, TimeoutException {
|
||||
try {
|
||||
return future.get(Math.max(1L, timeoutMillis), TimeUnit.MILLISECONDS);
|
||||
} catch (TimeoutException ex) {
|
||||
future.cancel(true);
|
||||
throw new TimeoutException(label + " 超过 " + timeoutMillis + "ms 未完成,已取消");
|
||||
}
|
||||
}
|
||||
}
|
||||
+18
-3
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.retry.LlmKeyGate;
|
||||
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
|
||||
import com.nanri.aiimage.common.retry.UnreachableCooldown;
|
||||
import com.nanri.aiimage.common.support.FutureDeadline;
|
||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||
import com.nanri.aiimage.config.HttpClientPool;
|
||||
import com.nanri.aiimage.metrics.ExternalCallMetricsRecorder;
|
||||
@@ -32,9 +33,12 @@ import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.regex.Matcher;
|
||||
@@ -376,6 +380,13 @@ public class AppearancePatentLlmClient {
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/** 单批等待上限:按一次读超时 × 重试次数再留一倍余量,封顶 2 小时。 */
|
||||
private long futureDeadlineMillis() {
|
||||
long perAttempt = Math.max(1_000L, properties.getLlmReadTimeoutMillis());
|
||||
long total = perAttempt * (Math.max(1, properties.getLlmRetryTimes()) + 1L);
|
||||
return Math.min(TimeUnit.HOURS.toMillis(2), Math.max(TimeUnit.MINUTES.toMillis(5), total * 2));
|
||||
}
|
||||
|
||||
private <T> List<BatchOutcome<T>> runBatches(List<List<Integer>> chunks, BatchInvoker<T> invoker) {
|
||||
Semaphore concurrency = new Semaphore(Math.max(1, properties.getLlmRowConcurrency()));
|
||||
ExecutorService executor = rowExecutor();
|
||||
@@ -401,12 +412,16 @@ public class AppearancePatentLlmClient {
|
||||
List<BatchOutcome<T>> outcomes = new ArrayList<>(chunks.size());
|
||||
for (int i = 0; i < futures.size(); i++) {
|
||||
try {
|
||||
outcomes.add(futures.get(i).join());
|
||||
} catch (Exception ex) {
|
||||
// 调度层炸了(invoker 本该自己 catch 干净):按传输失败收口,宁可整批记失败也不逐行放大
|
||||
outcomes.add(FutureDeadline.await(futures.get(i), futureDeadlineMillis(), "外观专利 LLM 批次"));
|
||||
} catch (TimeoutException | ExecutionException ex) {
|
||||
// 调度层炸了或单批超过等待上限(invoker 本该自己 catch 干净):按传输失败收口
|
||||
log.warn("[appearance-patent] llm batch crashed rows={} err={}",
|
||||
chunks.get(i).size(), failureMessage(ex));
|
||||
outcomes.add(new BatchOutcome<T>(Map.of(), failureMessage(ex), false));
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
log.warn("[appearance-patent] llm batch wait interrupted rows={}", chunks.get(i).size());
|
||||
outcomes.add(new BatchOutcome<T>(Map.of(), "LLM 批次等待被中断", false));
|
||||
}
|
||||
}
|
||||
return outcomes;
|
||||
|
||||
+12
@@ -63,6 +63,18 @@ public class AppearancePatentHistoryAssembler {
|
||||
return vo;
|
||||
}
|
||||
|
||||
/** 结果行不在返回窗口里的在跑/续跑任务:只带前端渲染当前任务所需字段。 */
|
||||
public AppearancePatentHistoryItemVo buildActiveItem(FileTaskEntity task) {
|
||||
AppearancePatentHistoryItemVo vo = new AppearancePatentHistoryItemVo();
|
||||
vo.setTaskId(task.getId());
|
||||
vo.setTaskStatus(task.getStatus());
|
||||
vo.setSuccess(false);
|
||||
vo.setCreatedAt(fmt(task.getCreatedAt()));
|
||||
vo.setStartedAt(fmt(task.getCreatedAt()));
|
||||
vo.setFinishedAt(fmt(task.getFinishedAt()));
|
||||
return vo;
|
||||
}
|
||||
|
||||
public int historyPriority(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
|
||||
String taskStatus = task == null ? null : task.getStatus();
|
||||
if (STATUS_PENDING.equals(taskStatus) || STATUS_RUNNING.equals(taskStatus) || isHistoryFileBuilding(row, taskStatus, job)) {
|
||||
|
||||
+37
@@ -125,9 +125,46 @@ public class AppearancePatentProgressSupport {
|
||||
FileTaskEntity task = taskMap.get(row.getTaskId());
|
||||
vo.getItems().add(toHistoryItem(row, task, jobMap.get(row.getId())));
|
||||
}
|
||||
prependActiveTasksOutsideWindow(vo, userId, taskIds);
|
||||
return vo;
|
||||
}
|
||||
|
||||
/**
|
||||
* 结果行被 top-N 窗口挤出后,仍在执行/续跑排队的任务会从列表彻底消失
|
||||
* (页面重载即无兜底)。这里按任务头补回最前面;已有结果行的任务不重复补。
|
||||
*/
|
||||
private void prependActiveTasksOutsideWindow(AppearancePatentHistoryVo vo, Long userId, List<Long> coveredTaskIds) {
|
||||
List<FileTaskEntity> activeTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getId,
|
||||
FileTaskEntity::getStatus,
|
||||
FileTaskEntity::getResumeOfTaskId,
|
||||
FileTaskEntity::getCreatedAt,
|
||||
FileTaskEntity::getFinishedAt)
|
||||
.eq(FileTaskEntity::getModuleType, moduleType)
|
||||
.eq(FileTaskEntity::getUserId, userId)
|
||||
.in(FileTaskEntity::getStatus, TaskStatuses.FileTask.ACTIVE_ORDERED)
|
||||
.orderByDesc(FileTaskEntity::getCreatedAt)
|
||||
.last("limit 50"));
|
||||
if (activeTasks == null || activeTasks.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
AppearancePatentHistoryAssembler assembler = historyAssembler();
|
||||
List<AppearancePatentHistoryItemVo> activeItems = new ArrayList<>();
|
||||
for (FileTaskEntity task : activeTasks) {
|
||||
if (task == null || task.getId() == null || coveredTaskIds.contains(task.getId())) {
|
||||
continue;
|
||||
}
|
||||
// 普通 PENDING 是未启动的解析预览,不进列表;续跑克隆件必须留(与相似 ASIN 同口径)
|
||||
if (TaskStatuses.FileTask.PENDING.equals(task.getStatus()) && task.getResumeOfTaskId() == null) {
|
||||
continue;
|
||||
}
|
||||
activeItems.add(assembler.buildActiveItem(task));
|
||||
}
|
||||
if (!activeItems.isEmpty()) {
|
||||
vo.getItems().addAll(0, activeItems);
|
||||
}
|
||||
}
|
||||
|
||||
public AppearancePatentTaskBatchVo progressBatch(List<Long> taskIds) {
|
||||
AppearancePatentTaskBatchVo vo = new AppearancePatentTaskBatchVo();
|
||||
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
|
||||
|
||||
+13
-3
@@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.common.support.FutureDeadline;
|
||||
import com.nanri.aiimage.common.text.BrandLabel;
|
||||
import com.nanri.aiimage.common.excel.ExcelStreamReader;
|
||||
import com.nanri.aiimage.config.BrandProgressProperties;
|
||||
@@ -71,9 +72,11 @@ import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
@@ -118,6 +121,8 @@ public class BrandTaskService {
|
||||
private static final int RESULT_UPLOAD_CONCURRENCY = 3;
|
||||
|
||||
// 有界队列线程池:newFixedThreadPool 用的是无界队列,任务堆积时不会拒绝、会把内存吃满
|
||||
/** 结果文件上传是收尾腿,任何一片卡死都不能让任务线程无限 join。 */
|
||||
private static final long RESULT_UPLOAD_DEADLINE_MILLIS = TimeUnit.MINUTES.toMillis(30);
|
||||
private final ExecutorService resultUploadExecutor = com.nanri.aiimage.common.support.ThreadPools
|
||||
.boundedFixed("brand-result-upload", RESULT_UPLOAD_CONCURRENCY);
|
||||
|
||||
@@ -1382,8 +1387,13 @@ public class BrandTaskService {
|
||||
}
|
||||
for (CompletableFuture<String> future : uploadFutures) {
|
||||
try {
|
||||
fullUrls.add(future.join());
|
||||
} catch (CompletionException ex) {
|
||||
fullUrls.add(FutureDeadline.await(future, RESULT_UPLOAD_DEADLINE_MILLIS, "结果文件上传"));
|
||||
} catch (TimeoutException ex) {
|
||||
throw new IOException("结果文件上传超时(" + RESULT_UPLOAD_DEADLINE_MILLIS + "ms)", ex);
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IOException("结果文件上传被中断", ex);
|
||||
} catch (ExecutionException ex) {
|
||||
Throwable cause = ex.getCause() == null ? ex : ex.getCause();
|
||||
if (cause instanceof IOException ioe) {
|
||||
throw ioe;
|
||||
|
||||
+5
-3
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.brandgating.service;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.text.BrandLabel;
|
||||
import com.nanri.aiimage.modules.brandgating.mapper.BrandGatingMapper;
|
||||
import com.nanri.aiimage.modules.brandgating.model.dto.BrandGatingAddRequest;
|
||||
import com.nanri.aiimage.modules.brandgating.model.dto.BrandGatingFilterQuery;
|
||||
@@ -91,7 +92,7 @@ public class BrandGatingService {
|
||||
|
||||
public BrandGatingAddResultVo add(BrandGatingAddRequest request) {
|
||||
BrandGatingAddResultVo vo = new BrandGatingAddResultVo();
|
||||
String brand = trimToEmpty(request.getBrand());
|
||||
String brand = trimToEmpty(BrandLabel.strip(request.getBrand()));
|
||||
if (brand.isEmpty()) {
|
||||
throw new BusinessException(400, "品牌不能为空");
|
||||
}
|
||||
@@ -133,7 +134,7 @@ public class BrandGatingService {
|
||||
BrandGatingImportResultVo vo = new BrandGatingImportResultVo();
|
||||
Set<String> seen = new HashSet<>();
|
||||
for (BrandGatingImportRequest.Item item : items) {
|
||||
String brand = trimToEmpty(item.getBrand());
|
||||
String brand = trimToEmpty(BrandLabel.strip(item.getBrand()));
|
||||
String country = firstNonBlank(item.getCountry(), DEFAULT_COUNTRY);
|
||||
String gatingType = firstNonBlank(item.getGatingType(), DEFAULT_GATING_TYPE);
|
||||
String asin = trimToEmpty(item.getAsin());
|
||||
@@ -148,7 +149,8 @@ public class BrandGatingService {
|
||||
continue;
|
||||
}
|
||||
BrandGatingEntity entity = new BrandGatingEntity();
|
||||
entity.setBrand(brand);
|
||||
// 与 add() 同口径:唯一键/去重键用的是洗后的 brand,写出再兜一次,防止调用方直传脏值
|
||||
entity.setBrand(BrandLabel.strip(brand));
|
||||
entity.setAsin(asin.isEmpty() ? null : asin);
|
||||
entity.setCountry(country);
|
||||
entity.setGatingType(gatingType);
|
||||
|
||||
+77
-28
@@ -702,15 +702,60 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
.eq(FileResultEntity::getUserId, userId)
|
||||
.orderByDesc(FileResultEntity::getCreatedAt)
|
||||
.last("limit " + safeLimit));
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
List<Long> taskIds = rows.stream().map(FileResultEntity::getTaskId).filter(Objects::nonNull).distinct().toList();
|
||||
List<Long> taskIds = rows == null ? List.of() : rows.stream()
|
||||
.map(FileResultEntity::getTaskId).filter(Objects::nonNull).distinct().toList();
|
||||
Map<Long, FileTaskEntity> taskMap = loadTaskMap(taskIds);
|
||||
for (FileResultEntity row : rows) {
|
||||
FileTaskEntity task = row.getTaskId() == null ? null : taskMap.get(row.getTaskId());
|
||||
vo.getItems().add(toHistoryItem(row, task));
|
||||
if (rows != null) {
|
||||
for (FileResultEntity row : rows) {
|
||||
FileTaskEntity task = row.getTaskId() == null ? null : taskMap.get(row.getTaskId());
|
||||
vo.getItems().add(toHistoryItem(row, task));
|
||||
}
|
||||
}
|
||||
prependActiveTasksOutsideWindow(vo, userId, taskIds);
|
||||
return vo;
|
||||
}
|
||||
|
||||
/**
|
||||
* 结果行被 top-N 窗口挤出后,仍在执行/排队中的任务会从列表彻底消失(页面重载即无兜底)。
|
||||
* 这里按任务头补回最前面;已有结果行的任务不重复补。
|
||||
*/
|
||||
private void prependActiveTasksOutsideWindow(CollectDataHistoryVo vo, Long userId, List<Long> coveredTaskIds) {
|
||||
List<FileTaskEntity> activeTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getId,
|
||||
FileTaskEntity::getTaskNo,
|
||||
FileTaskEntity::getStatus,
|
||||
FileTaskEntity::getErrorMessage,
|
||||
FileTaskEntity::getResumeOfTaskId,
|
||||
FileTaskEntity::getRequestJson,
|
||||
FileTaskEntity::getResultJson,
|
||||
FileTaskEntity::getCreatedAt,
|
||||
FileTaskEntity::getFinishedAt)
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getUserId, userId)
|
||||
.in(FileTaskEntity::getStatus, List.of(STATUS_PENDING, STATUS_RUNNING))
|
||||
.orderByDesc(FileTaskEntity::getCreatedAt)
|
||||
.last("limit 50"));
|
||||
if (activeTasks == null || activeTasks.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
List<CollectDataHistoryItemVo> activeItems = new ArrayList<>();
|
||||
for (FileTaskEntity task : activeTasks) {
|
||||
if (task == null || task.getId() == null || coveredTaskIds.contains(task.getId())) {
|
||||
continue;
|
||||
}
|
||||
activeItems.add(toActiveHistoryItem(task));
|
||||
}
|
||||
if (!activeItems.isEmpty()) {
|
||||
vo.getItems().addAll(0, activeItems);
|
||||
}
|
||||
}
|
||||
|
||||
private CollectDataHistoryItemVo toActiveHistoryItem(FileTaskEntity task) {
|
||||
CollectDataHistoryItemVo vo = new CollectDataHistoryItemVo();
|
||||
vo.setTaskId(task.getId());
|
||||
vo.setSuccess(false);
|
||||
vo.setError(task.getErrorMessage());
|
||||
fillTaskHistoryFields(vo, task);
|
||||
return vo;
|
||||
}
|
||||
|
||||
@@ -1745,31 +1790,35 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
|
||||
vo.setError(row.getErrorMessage());
|
||||
vo.setCreatedAt(formatTime(row.getCreatedAt()));
|
||||
if (task != null) {
|
||||
CollectDataStats stats = loadStats(task);
|
||||
vo.setTaskNo(task.getTaskNo());
|
||||
vo.setTaskStatus(task.getStatus());
|
||||
vo.setStartedAt(formatTime(task.getCreatedAt()));
|
||||
vo.setFinishedAt(formatTime(task.getFinishedAt()));
|
||||
vo.setTaskType(extractTaskTypeFromRequest(task));
|
||||
vo.setFilters(extractFiltersFromRequest(task));
|
||||
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
|
||||
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
|
||||
vo.setBrandRejectedCount(stats.brandRejectedCount);
|
||||
vo.setFinalRowCount(stats.finalRowCount);
|
||||
vo.setTotalRows(stats.totalRows);
|
||||
vo.setReceivedRows(stats.receivedRows);
|
||||
vo.setProcessedRows(stats.processedRows);
|
||||
vo.setCollectStage(stats.collectStage);
|
||||
vo.setCurrentKeyword(stats.currentKeyword);
|
||||
vo.setSearchCurrentPage(stats.searchCurrentPage);
|
||||
vo.setSearchTotalPages(stats.searchTotalPages);
|
||||
vo.setDetailProcessedAsins(stats.detailProcessedAsins);
|
||||
vo.setDetailTotalAsins(stats.detailTotalAsins);
|
||||
vo.setProgressPercent(calculateProgressPercent(task.getStatus(), stats.totalRows, stats.processedRows));
|
||||
fillTaskHistoryFields(vo, task);
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
private void fillTaskHistoryFields(CollectDataHistoryItemVo vo, FileTaskEntity task) {
|
||||
CollectDataStats stats = loadStats(task);
|
||||
vo.setTaskNo(task.getTaskNo());
|
||||
vo.setTaskStatus(task.getStatus());
|
||||
vo.setStartedAt(formatTime(task.getCreatedAt()));
|
||||
vo.setFinishedAt(formatTime(task.getFinishedAt()));
|
||||
vo.setTaskType(extractTaskTypeFromRequest(task));
|
||||
vo.setFilters(extractFiltersFromRequest(task));
|
||||
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
|
||||
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
|
||||
vo.setBrandRejectedCount(stats.brandRejectedCount);
|
||||
vo.setFinalRowCount(stats.finalRowCount);
|
||||
vo.setTotalRows(stats.totalRows);
|
||||
vo.setReceivedRows(stats.receivedRows);
|
||||
vo.setProcessedRows(stats.processedRows);
|
||||
vo.setCollectStage(stats.collectStage);
|
||||
vo.setCurrentKeyword(stats.currentKeyword);
|
||||
vo.setSearchCurrentPage(stats.searchCurrentPage);
|
||||
vo.setSearchTotalPages(stats.searchTotalPages);
|
||||
vo.setDetailProcessedAsins(stats.detailProcessedAsins);
|
||||
vo.setDetailTotalAsins(stats.detailTotalAsins);
|
||||
vo.setProgressPercent(calculateProgressPercent(task.getStatus(), stats.totalRows, stats.processedRows));
|
||||
}
|
||||
|
||||
private int calculateProgressPercent(String status, int totalRows, int receivedRows) {
|
||||
if (STATUS_SUCCESS.equals(status)) {
|
||||
return 100;
|
||||
|
||||
+9
-1
@@ -66,6 +66,9 @@ public class DedupeRunService {
|
||||
private static final List<String> EXPORT_COLUMN_PRIORITY = List.of("id", "ASIN", "国家", "价格", "品牌");
|
||||
/** 去重源文件的列集由用户勾选,品牌列只能按表头认出来,其余列一律原文透传。 */
|
||||
private static final String BRAND_COLUMN_HEADER = "品牌";
|
||||
/** 源表可能导出自不同模板,品牌列别名必须在这里收口;比对键仍用正名。 */
|
||||
private static final List<String> BRAND_COLUMN_HEADERS =
|
||||
List.of(BRAND_COLUMN_HEADER, "Brand", "品牌(英文)", "品牌(英文)");
|
||||
/** 同时处理的文件数上限,避免多文件并行时打满内存与 OSS 上传带宽 */
|
||||
private static final int MAX_PARALLEL_FILES = 4;
|
||||
/**
|
||||
@@ -866,6 +869,11 @@ public class DedupeRunService {
|
||||
return headerIndex;
|
||||
}
|
||||
|
||||
private static boolean isBrandColumn(String column) {
|
||||
return column != null && BRAND_COLUMN_HEADERS.stream().anyMatch(
|
||||
alias -> alias.equalsIgnoreCase(column.strip()));
|
||||
}
|
||||
|
||||
private List<String> reorderExportColumns(List<String> selectedColumns) {
|
||||
if (selectedColumns == null || selectedColumns.isEmpty()) {
|
||||
return selectedColumns;
|
||||
@@ -891,7 +899,7 @@ public class DedupeRunService {
|
||||
String value = normalizeCellText(cellText(rowMap, headerIndex.get(selectedColumn)));
|
||||
// 品牌列的值可能是用户从亚马逊详情页复制来的署名("Brand: XXX"):在这里剥一次,
|
||||
// 下方出件写 cell 用的就是同一个字符串,比对与交付不会拿到两个版本
|
||||
selectedValues.add(BRAND_COLUMN_HEADER.equals(selectedColumn) ? BrandLabel.strip(value) : value);
|
||||
selectedValues.add(isBrandColumn(selectedColumn) ? BrandLabel.strip(value) : value);
|
||||
}
|
||||
if (asinColumnIndex == null) {
|
||||
asinColumnIndex = headerIndex.get("ASIN");
|
||||
|
||||
+49
-7
@@ -1,5 +1,6 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.service;
|
||||
|
||||
import com.nanri.aiimage.common.task.TaskStatuses;
|
||||
import com.nanri.aiimage.common.text.UserMessages;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
@@ -245,18 +246,19 @@ public class PriceTrackTaskService extends AbstractShopPayloadTaskPipeline<Price
|
||||
.orderByDesc(FileResultEntity::getCreatedAt)
|
||||
.last("limit 100"));
|
||||
long resultRowsLoadedAt = System.nanoTime();
|
||||
if (entities.isEmpty()) {
|
||||
vo.setItems(List.of());
|
||||
log.debug("[price-track] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
|
||||
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
|
||||
return vo;
|
||||
}
|
||||
Map<Long, FileTaskEntity> taskById = new LinkedHashMap<>();
|
||||
List<Long> taskIds = entities.stream()
|
||||
.map(FileResultEntity::getTaskId)
|
||||
.filter(id -> id != null && id > 0)
|
||||
.distinct()
|
||||
.toList();
|
||||
if (entities.isEmpty()) {
|
||||
vo.setItems(new ArrayList<>());
|
||||
prependActiveTasksOutsideWindow(vo, userId, taskIds);
|
||||
log.debug("[price-track] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
|
||||
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
|
||||
return vo;
|
||||
}
|
||||
Map<Long, FileTaskEntity> taskById = new LinkedHashMap<>();
|
||||
if (!taskIds.isEmpty()) {
|
||||
List<FileTaskEntity> tasks = selectTaskHistoryFieldsByIdsInBatches(taskIds);
|
||||
for (FileTaskEntity t : tasks) {
|
||||
@@ -275,6 +277,7 @@ public class PriceTrackTaskService extends AbstractShopPayloadTaskPipeline<Price
|
||||
items.add(toHistoryItemVo(entity, taskById.get(entity.getTaskId()), jobMap.get(entity.getId()), false));
|
||||
}
|
||||
vo.setItems(items);
|
||||
prependActiveTasksOutsideWindow(vo, userId, taskIds);
|
||||
long finishedAt = System.nanoTime();
|
||||
log.debug("[price-track] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
|
||||
userId,
|
||||
@@ -289,6 +292,45 @@ public class PriceTrackTaskService extends AbstractShopPayloadTaskPipeline<Price
|
||||
return vo;
|
||||
}
|
||||
|
||||
/**
|
||||
* 结果行被 top-N 窗口挤出后,仍在执行/排队中的任务会从列表彻底消失(页面重载即无兜底)。
|
||||
* 这里按任务头补回最前面;已有结果行的任务不重复补。
|
||||
*/
|
||||
private void prependActiveTasksOutsideWindow(PriceTrackHistoryVo vo, Long userId, List<Long> coveredTaskIds) {
|
||||
List<FileTaskEntity> activeTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.select(FileTaskEntity::getId,
|
||||
FileTaskEntity::getTaskNo,
|
||||
FileTaskEntity::getStatus,
|
||||
FileTaskEntity::getErrorMessage,
|
||||
FileTaskEntity::getCreatedAt,
|
||||
FileTaskEntity::getFinishedAt)
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getUserId, userId)
|
||||
.in(FileTaskEntity::getStatus, TaskStatuses.FileTask.ACTIVE_ORDERED)
|
||||
.orderByDesc(FileTaskEntity::getCreatedAt)
|
||||
.last("limit 50"));
|
||||
if (activeTasks == null || activeTasks.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
List<PriceTrackResultItemVo> activeItems = new ArrayList<>();
|
||||
for (FileTaskEntity task : activeTasks) {
|
||||
if (task == null || task.getId() == null || coveredTaskIds.contains(task.getId())) {
|
||||
continue;
|
||||
}
|
||||
PriceTrackResultItemVo item = new PriceTrackResultItemVo();
|
||||
item.setTaskId(task.getId());
|
||||
item.setTaskStatus(task.getStatus());
|
||||
item.setSuccess(false);
|
||||
item.setError(task.getErrorMessage());
|
||||
item.setCreatedAt(fmt(task.getCreatedAt()));
|
||||
item.setFinishedAt(fmt(task.getFinishedAt()));
|
||||
activeItems.add(item);
|
||||
}
|
||||
if (!activeItems.isEmpty()) {
|
||||
vo.getItems().addAll(0, activeItems);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 事务边界:Redis 任务锁在事务外获取(自旋等待不再占用 DB 连接),
|
||||
* DB 段(复核 + 删除 + 重算任务状态)仍在同一个事务内。
|
||||
|
||||
+3
-1
@@ -78,7 +78,9 @@ public final class ShopDataCrawlRowNormalizer {
|
||||
}
|
||||
|
||||
public static boolean rowEmpty(ShopDataCrawlRowDto row) {
|
||||
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getBrand()) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
|
||||
// 品牌要按出口口径判空:纯 "Brand:" 这类标签值洗完是空串,若不先洗,
|
||||
// 一行只有标签的脏行会通过 rowEmpty,随后 copyRow 把它洗成整行空白落件。
|
||||
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(brand(row.getBrand())) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
|
||||
&& blank(row.getSalesRank()) && blank(row.getPageViews()) && blank(row.getUnitsSold())
|
||||
&& blank(row.getPrice()) && blank(row.getRecommendedOffer()));
|
||||
}
|
||||
|
||||
+19
-5
@@ -143,18 +143,23 @@ public class ShopDataDuplicateCheckQueryService {
|
||||
return result;
|
||||
}
|
||||
|
||||
/** 逐 ASIN 裁剪 occurrences 并重算 shop_count/record_count;items 保持缓存序。 */
|
||||
/**
|
||||
* 逐 ASIN 裁剪 occurrences 并重算 shop_count/record_count;items 保持缓存序。
|
||||
*
|
||||
* <p>legacy payload 里的 brand 可能仍带 "Brand:" 前缀(存量扫描结果在下次重写前不会变),
|
||||
* 所有页面出口都经过这里,所以在这一层统一洗一次;可见范围为空时也要洗,不能直接短路返回原对象。</p>
|
||||
*/
|
||||
public List<DuplicateItem> cropItems(List<DuplicateItem> items, Set<String> visibleKeys) {
|
||||
if (visibleKeys == null || items == null) {
|
||||
return items == null ? List.of() : items;
|
||||
if (items == null) {
|
||||
return List.of();
|
||||
}
|
||||
List<DuplicateItem> result = new ArrayList<>(items.size());
|
||||
for (DuplicateItem item : items) {
|
||||
List<DuplicateOccurrence> kept = new ArrayList<>();
|
||||
Set<String> shops = new LinkedHashSet<>();
|
||||
for (DuplicateOccurrence occurrence : item.occurrences()) {
|
||||
if (visibleKeys.contains(shopKey(occurrence.shopName()))) {
|
||||
kept.add(occurrence);
|
||||
if (visibleKeys == null || visibleKeys.contains(shopKey(occurrence.shopName()))) {
|
||||
kept.add(withCleanBrand(occurrence));
|
||||
shops.add(occurrence.shopName());
|
||||
}
|
||||
}
|
||||
@@ -166,6 +171,15 @@ public class ShopDataDuplicateCheckQueryService {
|
||||
return result;
|
||||
}
|
||||
|
||||
private static DuplicateOccurrence withCleanBrand(DuplicateOccurrence occ) {
|
||||
String clean = BrandLabel.strip(occ.brand());
|
||||
if (java.util.Objects.equals(clean, occ.brand())) {
|
||||
return occ;
|
||||
}
|
||||
return new DuplicateOccurrence(occ.asin(), occ.date(), occ.price(), clean, occ.shopName(),
|
||||
occ.groupName(), occ.countryCodes(), occ.country());
|
||||
}
|
||||
|
||||
public Map<String, Object> overviewData(DuplicateScanView view, Set<String> visibleKeys) {
|
||||
List<DuplicateShop> shops = cropShops(view.payload().shops(), visibleKeys);
|
||||
List<DuplicateItem> items = cropItems(view.payload().items(), visibleKeys);
|
||||
|
||||
+31
-3
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.PuzzleImageMerger;
|
||||
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.common.support.FutureDeadline;
|
||||
import com.nanri.aiimage.modules.usersecret.support.SecretUsageContext;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -30,9 +31,12 @@ import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
/**
|
||||
* 货源查询直连 LLM 编排服务:复刻原工作流 similarity_asin
|
||||
@@ -259,7 +263,7 @@ public class SimilarAsinLlmService {
|
||||
List<SimilarAsinResultRowDto> merged = new ArrayList<>(rows.size());
|
||||
for (int i = 0; i < futures.size(); i++) {
|
||||
try {
|
||||
merged.add(futures.get(i).join());
|
||||
merged.add(FutureDeadline.await(futures.get(i), futureDeadlineMillis(), "LLM 单行处理"));
|
||||
} catch (Exception ex) {
|
||||
log.warn("[similar-asin][llm] row failed index={} asin={} err={}",
|
||||
i, rows.get(i).getAsin(), failureMessage(ex));
|
||||
@@ -269,6 +273,13 @@ public class SimilarAsinLlmService {
|
||||
return merged;
|
||||
}
|
||||
|
||||
/** 单行/单批等待上限:按一次读超时 × 重试次数再留一倍余量,封顶 2 小时。 */
|
||||
private long futureDeadlineMillis() {
|
||||
long perAttempt = Math.max(1_000L, properties.getLlmReadTimeoutMillis());
|
||||
long total = perAttempt * (Math.max(1, properties.getLlmRetryTimes()) + 1L);
|
||||
return Math.min(TimeUnit.HOURS.toMillis(2), Math.max(TimeUnit.MINUTES.toMillis(5), total * 2));
|
||||
}
|
||||
|
||||
private SimilarAsinResultRowDto inspectRow(SimilarAsinResultRowDto source,
|
||||
int rowIndex,
|
||||
CategoryPrepass prepass,
|
||||
@@ -460,7 +471,16 @@ public class SimilarAsinLlmService {
|
||||
}, usageContext), executor));
|
||||
}
|
||||
for (CompletableFuture<Void> future : futures) {
|
||||
future.join(); // matchCategoryChunk 内部自捕获,join 只会因中断失败
|
||||
try {
|
||||
FutureDeadline.await(future, futureDeadlineMillis(), "LLM 类目批次");
|
||||
} catch (TimeoutException | ExecutionException ex) {
|
||||
// 超时批次的结果按缺失处理:缺行在下游走单行回退,不让一个批次拖住整批
|
||||
log.warn("[similar-asin][llm] category batch closed without result err={}", failureMessage(ex));
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
log.warn("[similar-asin][llm] category batch wait interrupted");
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new CategoryPrepass(paths, errors);
|
||||
}
|
||||
@@ -737,7 +757,15 @@ public class SimilarAsinLlmService {
|
||||
}, usageContext), executor));
|
||||
}
|
||||
for (CompletableFuture<Void> future : futures) {
|
||||
future.join(); // checkConformChunk 内部自捕获,join 只会因中断失败
|
||||
try {
|
||||
FutureDeadline.await(future, futureDeadlineMillis(), "LLM 合规批次");
|
||||
} catch (TimeoutException | ExecutionException ex) {
|
||||
log.warn("[similar-asin][llm] conform batch closed without result err={}", failureMessage(ex));
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
log.warn("[similar-asin][llm] conform batch wait interrupted");
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new ConformPrepass(byRow, errors);
|
||||
}
|
||||
|
||||
+7
-1
@@ -126,6 +126,11 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L;
|
||||
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
|
||||
private static final long TASK_LOCK_WAIT_MILLIS = 10000L;
|
||||
/**
|
||||
* 回传口撞锁时只等 1 秒:锁被 LLM 批合法持有几分钟是常态,客户端 result_feed 本来就会按龄重试,
|
||||
* 服务端替它等满 10 秒只会白占一个 Tomcat 线程(B15 §3 第 2 条)。
|
||||
*/
|
||||
static final long RESULT_SUBMIT_LOCK_WAIT_MILLIS = 1000L;
|
||||
/**
|
||||
* Task 8:WorkbookFactory 打开前的受控读取。xlsx 本质是 zip,
|
||||
* 先用 ZipFile 中央目录探测条目数与解压总字节(不读取压缩内容),
|
||||
@@ -791,7 +796,8 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
|
||||
}
|
||||
|
||||
public void submitResult(Long taskId, SimilarAsinSubmitResultRequest request) {
|
||||
try (TaskDistributedLockService.LockHandle ignored = ownershipSupport().requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) {
|
||||
try (TaskDistributedLockService.LockHandle ignored =
|
||||
ownershipSupport().requireTaskLock(taskId, RESULT_SUBMIT_LOCK_WAIT_MILLIS)) {
|
||||
submitResultLocked(taskId, request);
|
||||
}
|
||||
}
|
||||
|
||||
+45
@@ -9,6 +9,7 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
@@ -191,6 +192,13 @@ public class WorkbenchClientPackageService {
|
||||
normalizedVersion, objectKey);
|
||||
throw new BusinessException("该版本已存在,请提升版本号或先删除旧包");
|
||||
}
|
||||
WorkbenchClientPackageEntity latest = newest();
|
||||
if (latest != null && versionIsLower(normalizedVersion, latest.getVersion())) {
|
||||
// 按上传时间取最新时,低版本后传会顶掉当前更新目标,已升级装机反而被判"无需更新"
|
||||
log.warn("[workbench-client] 直传确认失败:版本低于当前最新 version={} latest={} objectKey={}",
|
||||
normalizedVersion, latest.getVersion(), objectKey);
|
||||
throw new BusinessException("版本号不能低于当前最新版本 " + latest.getVersion());
|
||||
}
|
||||
String sha256 = ossStorageService.workbenchClientObjectSha256(objectKey);
|
||||
if (sha256 == null || sha256.length() != 64) {
|
||||
log.warn("[workbench-client] 直传确认失败:摘要计算异常 objectKey={} sha256={}", objectKey, sha256);
|
||||
@@ -244,6 +252,43 @@ public class WorkbenchClientPackageService {
|
||||
return deleted;
|
||||
}
|
||||
|
||||
/** 仅对「数字点分」版本做单调性判断;命名不合规时跳过,不臆造顺序。 */
|
||||
static boolean versionIsLower(String candidate, String current) {
|
||||
List<BigInteger> left = versionSegments(candidate);
|
||||
List<BigInteger> right = versionSegments(current);
|
||||
if (left == null || right == null) {
|
||||
return false;
|
||||
}
|
||||
int size = Math.max(left.size(), right.size());
|
||||
for (int i = 0; i < size; i++) {
|
||||
BigInteger a = i < left.size() ? left.get(i) : BigInteger.ZERO;
|
||||
BigInteger b = i < right.size() ? right.get(i) : BigInteger.ZERO;
|
||||
int cmp = a.compareTo(b);
|
||||
if (cmp != 0) {
|
||||
return cmp < 0;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static List<BigInteger> versionSegments(String value) {
|
||||
String normalized = value == null ? "" : value.trim();
|
||||
if (normalized.startsWith("v") || normalized.startsWith("V")) {
|
||||
normalized = normalized.substring(1);
|
||||
}
|
||||
if (normalized.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
List<BigInteger> segments = new ArrayList<>();
|
||||
for (String part : normalized.split("\\.", -1)) {
|
||||
if (!part.matches("\\d+")) {
|
||||
return null;
|
||||
}
|
||||
segments.add(new BigInteger(part));
|
||||
}
|
||||
return segments;
|
||||
}
|
||||
|
||||
private WorkbenchClientPackageEntity newest() {
|
||||
return packageMapper.selectOne(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
|
||||
.orderByDesc(WorkbenchClientPackageEntity::getCreatedAt)
|
||||
|
||||
Reference in New Issue
Block a user