fix(backlog): 品牌出口统一、join 截止与进行中任务补回

- BrandLabel 出口守卫扩到全仓 entity.setBrand,撞款/去重/品牌库统一清洗
- FutureDeadline 替换五处无界 join,回传口锁等待降为 1s
- 采集/跟价/外观专利 history 补回窗口外进行中与续跑任务
- 工作台客户端版本强制单调;catch 棘轮按新增出口逐处记名
This commit is contained in:
2026-10-25 13:27:46 +08:00
parent 1dd9e66d21
commit 5928072c6a
30 changed files with 558 additions and 75 deletions
@@ -0,0 +1,29 @@
package com.nanri.aiimage.common.support;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* 带截止的 future 等待。
*
* <p>{@code join()} 没有时间界:JDK HTTP 客户端的超时只封顶到响应头,响应体读取一旦卡住,
* 行线程/任务线程会永久 park。凡是把 worker 线程 join 回主线程的地方都走这里,超时即取消
* 该 future 并把超时作为该行/该批的失败收口,不拖住整条任务。</p>
*/
public final class FutureDeadline {
private FutureDeadline() {
}
public static <T> T await(CompletableFuture<T> future, long timeoutMillis, String label)
throws InterruptedException, ExecutionException, TimeoutException {
try {
return future.get(Math.max(1L, timeoutMillis), TimeUnit.MILLISECONDS);
} catch (TimeoutException ex) {
future.cancel(true);
throw new TimeoutException(label + " 超过 " + timeoutMillis + "ms 未完成,已取消");
}
}
}
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.retry.LlmKeyGate;
import com.nanri.aiimage.common.retry.LlmRetryBackoff;
import com.nanri.aiimage.common.retry.UnreachableCooldown;
import com.nanri.aiimage.common.support.FutureDeadline;
import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.config.HttpClientPool;
import com.nanri.aiimage.metrics.ExternalCallMetricsRecorder;
@@ -32,9 +33,12 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.regex.Matcher;
@@ -376,6 +380,13 @@ public class AppearancePatentLlmClient {
return chunks;
}
/** 单批等待上限:按一次读超时 × 重试次数再留一倍余量,封顶 2 小时。 */
private long futureDeadlineMillis() {
long perAttempt = Math.max(1_000L, properties.getLlmReadTimeoutMillis());
long total = perAttempt * (Math.max(1, properties.getLlmRetryTimes()) + 1L);
return Math.min(TimeUnit.HOURS.toMillis(2), Math.max(TimeUnit.MINUTES.toMillis(5), total * 2));
}
private <T> List<BatchOutcome<T>> runBatches(List<List<Integer>> chunks, BatchInvoker<T> invoker) {
Semaphore concurrency = new Semaphore(Math.max(1, properties.getLlmRowConcurrency()));
ExecutorService executor = rowExecutor();
@@ -401,12 +412,16 @@ public class AppearancePatentLlmClient {
List<BatchOutcome<T>> outcomes = new ArrayList<>(chunks.size());
for (int i = 0; i < futures.size(); i++) {
try {
outcomes.add(futures.get(i).join());
} catch (Exception ex) {
// 调度层炸了(invoker 本该自己 catch 干净):按传输失败收口,宁可整批记失败也不逐行放大
outcomes.add(FutureDeadline.await(futures.get(i), futureDeadlineMillis(), "外观专利 LLM 批次"));
} catch (TimeoutException | ExecutionException ex) {
// 调度层炸了或单批超过等待上限(invoker 本该自己 catch 干净):按传输失败收口
log.warn("[appearance-patent] llm batch crashed rows={} err={}",
chunks.get(i).size(), failureMessage(ex));
outcomes.add(new BatchOutcome<T>(Map.of(), failureMessage(ex), false));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
log.warn("[appearance-patent] llm batch wait interrupted rows={}", chunks.get(i).size());
outcomes.add(new BatchOutcome<T>(Map.of(), "LLM 批次等待被中断", false));
}
}
return outcomes;
@@ -63,6 +63,18 @@ public class AppearancePatentHistoryAssembler {
return vo;
}
/** 结果行不在返回窗口里的在跑/续跑任务:只带前端渲染当前任务所需字段。 */
public AppearancePatentHistoryItemVo buildActiveItem(FileTaskEntity task) {
AppearancePatentHistoryItemVo vo = new AppearancePatentHistoryItemVo();
vo.setTaskId(task.getId());
vo.setTaskStatus(task.getStatus());
vo.setSuccess(false);
vo.setCreatedAt(fmt(task.getCreatedAt()));
vo.setStartedAt(fmt(task.getCreatedAt()));
vo.setFinishedAt(fmt(task.getFinishedAt()));
return vo;
}
public int historyPriority(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
String taskStatus = task == null ? null : task.getStatus();
if (STATUS_PENDING.equals(taskStatus) || STATUS_RUNNING.equals(taskStatus) || isHistoryFileBuilding(row, taskStatus, job)) {
@@ -125,9 +125,46 @@ public class AppearancePatentProgressSupport {
FileTaskEntity task = taskMap.get(row.getTaskId());
vo.getItems().add(toHistoryItem(row, task, jobMap.get(row.getId())));
}
prependActiveTasksOutsideWindow(vo, userId, taskIds);
return vo;
}
/**
* 结果行被 top-N 窗口挤出后,仍在执行/续跑排队的任务会从列表彻底消失
* (页面重载即无兜底)。这里按任务头补回最前面;已有结果行的任务不重复补。
*/
private void prependActiveTasksOutsideWindow(AppearancePatentHistoryVo vo, Long userId, List<Long> coveredTaskIds) {
List<FileTaskEntity> activeTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getStatus,
FileTaskEntity::getResumeOfTaskId,
FileTaskEntity::getCreatedAt,
FileTaskEntity::getFinishedAt)
.eq(FileTaskEntity::getModuleType, moduleType)
.eq(FileTaskEntity::getUserId, userId)
.in(FileTaskEntity::getStatus, TaskStatuses.FileTask.ACTIVE_ORDERED)
.orderByDesc(FileTaskEntity::getCreatedAt)
.last("limit 50"));
if (activeTasks == null || activeTasks.isEmpty()) {
return;
}
AppearancePatentHistoryAssembler assembler = historyAssembler();
List<AppearancePatentHistoryItemVo> activeItems = new ArrayList<>();
for (FileTaskEntity task : activeTasks) {
if (task == null || task.getId() == null || coveredTaskIds.contains(task.getId())) {
continue;
}
// 普通 PENDING 是未启动的解析预览,不进列表;续跑克隆件必须留(与相似 ASIN 同口径)
if (TaskStatuses.FileTask.PENDING.equals(task.getStatus()) && task.getResumeOfTaskId() == null) {
continue;
}
activeItems.add(assembler.buildActiveItem(task));
}
if (!activeItems.isEmpty()) {
vo.getItems().addAll(0, activeItems);
}
}
public AppearancePatentTaskBatchVo progressBatch(List<Long> taskIds) {
AppearancePatentTaskBatchVo vo = new AppearancePatentTaskBatchVo();
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
@@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.security.AdminAuthSupport;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.common.support.FutureDeadline;
import com.nanri.aiimage.common.text.BrandLabel;
import com.nanri.aiimage.common.excel.ExcelStreamReader;
import com.nanri.aiimage.config.BrandProgressProperties;
@@ -71,9 +72,11 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -118,6 +121,8 @@ public class BrandTaskService {
private static final int RESULT_UPLOAD_CONCURRENCY = 3;
// 有界队列线程池:newFixedThreadPool 用的是无界队列,任务堆积时不会拒绝、会把内存吃满
/** 结果文件上传是收尾腿,任何一片卡死都不能让任务线程无限 join。 */
private static final long RESULT_UPLOAD_DEADLINE_MILLIS = TimeUnit.MINUTES.toMillis(30);
private final ExecutorService resultUploadExecutor = com.nanri.aiimage.common.support.ThreadPools
.boundedFixed("brand-result-upload", RESULT_UPLOAD_CONCURRENCY);
@@ -1382,8 +1387,13 @@ public class BrandTaskService {
}
for (CompletableFuture<String> future : uploadFutures) {
try {
fullUrls.add(future.join());
} catch (CompletionException ex) {
fullUrls.add(FutureDeadline.await(future, RESULT_UPLOAD_DEADLINE_MILLIS, "结果文件上传"));
} catch (TimeoutException ex) {
throw new IOException("结果文件上传超时(" + RESULT_UPLOAD_DEADLINE_MILLIS + "ms)", ex);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IOException("结果文件上传被中断", ex);
} catch (ExecutionException ex) {
Throwable cause = ex.getCause() == null ? ex : ex.getCause();
if (cause instanceof IOException ioe) {
throw ioe;
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.brandgating.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.text.BrandLabel;
import com.nanri.aiimage.modules.brandgating.mapper.BrandGatingMapper;
import com.nanri.aiimage.modules.brandgating.model.dto.BrandGatingAddRequest;
import com.nanri.aiimage.modules.brandgating.model.dto.BrandGatingFilterQuery;
@@ -91,7 +92,7 @@ public class BrandGatingService {
public BrandGatingAddResultVo add(BrandGatingAddRequest request) {
BrandGatingAddResultVo vo = new BrandGatingAddResultVo();
String brand = trimToEmpty(request.getBrand());
String brand = trimToEmpty(BrandLabel.strip(request.getBrand()));
if (brand.isEmpty()) {
throw new BusinessException(400, "品牌不能为空");
}
@@ -133,7 +134,7 @@ public class BrandGatingService {
BrandGatingImportResultVo vo = new BrandGatingImportResultVo();
Set<String> seen = new HashSet<>();
for (BrandGatingImportRequest.Item item : items) {
String brand = trimToEmpty(item.getBrand());
String brand = trimToEmpty(BrandLabel.strip(item.getBrand()));
String country = firstNonBlank(item.getCountry(), DEFAULT_COUNTRY);
String gatingType = firstNonBlank(item.getGatingType(), DEFAULT_GATING_TYPE);
String asin = trimToEmpty(item.getAsin());
@@ -148,7 +149,8 @@ public class BrandGatingService {
continue;
}
BrandGatingEntity entity = new BrandGatingEntity();
entity.setBrand(brand);
// 与 add() 同口径:唯一键/去重键用的是洗后的 brand,写出再兜一次,防止调用方直传脏值
entity.setBrand(BrandLabel.strip(brand));
entity.setAsin(asin.isEmpty() ? null : asin);
entity.setCountry(country);
entity.setGatingType(gatingType);
@@ -702,15 +702,60 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
.eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt)
.last("limit " + safeLimit));
if (rows == null || rows.isEmpty()) {
return vo;
}
List<Long> taskIds = rows.stream().map(FileResultEntity::getTaskId).filter(Objects::nonNull).distinct().toList();
List<Long> taskIds = rows == null ? List.of() : rows.stream()
.map(FileResultEntity::getTaskId).filter(Objects::nonNull).distinct().toList();
Map<Long, FileTaskEntity> taskMap = loadTaskMap(taskIds);
for (FileResultEntity row : rows) {
FileTaskEntity task = row.getTaskId() == null ? null : taskMap.get(row.getTaskId());
vo.getItems().add(toHistoryItem(row, task));
if (rows != null) {
for (FileResultEntity row : rows) {
FileTaskEntity task = row.getTaskId() == null ? null : taskMap.get(row.getTaskId());
vo.getItems().add(toHistoryItem(row, task));
}
}
prependActiveTasksOutsideWindow(vo, userId, taskIds);
return vo;
}
/**
* 结果行被 top-N 窗口挤出后,仍在执行/排队中的任务会从列表彻底消失(页面重载即无兜底)。
* 这里按任务头补回最前面;已有结果行的任务不重复补。
*/
private void prependActiveTasksOutsideWindow(CollectDataHistoryVo vo, Long userId, List<Long> coveredTaskIds) {
List<FileTaskEntity> activeTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getTaskNo,
FileTaskEntity::getStatus,
FileTaskEntity::getErrorMessage,
FileTaskEntity::getResumeOfTaskId,
FileTaskEntity::getRequestJson,
FileTaskEntity::getResultJson,
FileTaskEntity::getCreatedAt,
FileTaskEntity::getFinishedAt)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getUserId, userId)
.in(FileTaskEntity::getStatus, List.of(STATUS_PENDING, STATUS_RUNNING))
.orderByDesc(FileTaskEntity::getCreatedAt)
.last("limit 50"));
if (activeTasks == null || activeTasks.isEmpty()) {
return;
}
List<CollectDataHistoryItemVo> activeItems = new ArrayList<>();
for (FileTaskEntity task : activeTasks) {
if (task == null || task.getId() == null || coveredTaskIds.contains(task.getId())) {
continue;
}
activeItems.add(toActiveHistoryItem(task));
}
if (!activeItems.isEmpty()) {
vo.getItems().addAll(0, activeItems);
}
}
private CollectDataHistoryItemVo toActiveHistoryItem(FileTaskEntity task) {
CollectDataHistoryItemVo vo = new CollectDataHistoryItemVo();
vo.setTaskId(task.getId());
vo.setSuccess(false);
vo.setError(task.getErrorMessage());
fillTaskHistoryFields(vo, task);
return vo;
}
@@ -1745,31 +1790,35 @@ public class CollectDataService extends AbstractFileTaskPipeline<CollectDataResu
vo.setError(row.getErrorMessage());
vo.setCreatedAt(formatTime(row.getCreatedAt()));
if (task != null) {
CollectDataStats stats = loadStats(task);
vo.setTaskNo(task.getTaskNo());
vo.setTaskStatus(task.getStatus());
vo.setStartedAt(formatTime(task.getCreatedAt()));
vo.setFinishedAt(formatTime(task.getFinishedAt()));
vo.setTaskType(extractTaskTypeFromRequest(task));
vo.setFilters(extractFiltersFromRequest(task));
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
vo.setBrandRejectedCount(stats.brandRejectedCount);
vo.setFinalRowCount(stats.finalRowCount);
vo.setTotalRows(stats.totalRows);
vo.setReceivedRows(stats.receivedRows);
vo.setProcessedRows(stats.processedRows);
vo.setCollectStage(stats.collectStage);
vo.setCurrentKeyword(stats.currentKeyword);
vo.setSearchCurrentPage(stats.searchCurrentPage);
vo.setSearchTotalPages(stats.searchTotalPages);
vo.setDetailProcessedAsins(stats.detailProcessedAsins);
vo.setDetailTotalAsins(stats.detailTotalAsins);
vo.setProgressPercent(calculateProgressPercent(task.getStatus(), stats.totalRows, stats.processedRows));
fillTaskHistoryFields(vo, task);
}
return vo;
}
private void fillTaskHistoryFields(CollectDataHistoryItemVo vo, FileTaskEntity task) {
CollectDataStats stats = loadStats(task);
vo.setTaskNo(task.getTaskNo());
vo.setTaskStatus(task.getStatus());
vo.setStartedAt(formatTime(task.getCreatedAt()));
vo.setFinishedAt(formatTime(task.getFinishedAt()));
vo.setTaskType(extractTaskTypeFromRequest(task));
vo.setFilters(extractFiltersFromRequest(task));
vo.setDedupeFilteredCount(stats.dedupeFilteredCount);
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
vo.setBrandRejectedCount(stats.brandRejectedCount);
vo.setFinalRowCount(stats.finalRowCount);
vo.setTotalRows(stats.totalRows);
vo.setReceivedRows(stats.receivedRows);
vo.setProcessedRows(stats.processedRows);
vo.setCollectStage(stats.collectStage);
vo.setCurrentKeyword(stats.currentKeyword);
vo.setSearchCurrentPage(stats.searchCurrentPage);
vo.setSearchTotalPages(stats.searchTotalPages);
vo.setDetailProcessedAsins(stats.detailProcessedAsins);
vo.setDetailTotalAsins(stats.detailTotalAsins);
vo.setProgressPercent(calculateProgressPercent(task.getStatus(), stats.totalRows, stats.processedRows));
}
private int calculateProgressPercent(String status, int totalRows, int receivedRows) {
if (STATUS_SUCCESS.equals(status)) {
return 100;
@@ -66,6 +66,9 @@ public class DedupeRunService {
private static final List<String> EXPORT_COLUMN_PRIORITY = List.of("id", "ASIN", "国家", "价格", "品牌");
/** 去重源文件的列集由用户勾选,品牌列只能按表头认出来,其余列一律原文透传。 */
private static final String BRAND_COLUMN_HEADER = "品牌";
/** 源表可能导出自不同模板,品牌列别名必须在这里收口;比对键仍用正名。 */
private static final List<String> BRAND_COLUMN_HEADERS =
List.of(BRAND_COLUMN_HEADER, "Brand", "品牌(英文)", "品牌(英文)");
/** 同时处理的文件数上限,避免多文件并行时打满内存与 OSS 上传带宽 */
private static final int MAX_PARALLEL_FILES = 4;
/**
@@ -866,6 +869,11 @@ public class DedupeRunService {
return headerIndex;
}
private static boolean isBrandColumn(String column) {
return column != null && BRAND_COLUMN_HEADERS.stream().anyMatch(
alias -> alias.equalsIgnoreCase(column.strip()));
}
private List<String> reorderExportColumns(List<String> selectedColumns) {
if (selectedColumns == null || selectedColumns.isEmpty()) {
return selectedColumns;
@@ -891,7 +899,7 @@ public class DedupeRunService {
String value = normalizeCellText(cellText(rowMap, headerIndex.get(selectedColumn)));
// 品牌列的值可能是用户从亚马逊详情页复制来的署名("Brand: XXX"):在这里剥一次,
// 下方出件写 cell 用的就是同一个字符串,比对与交付不会拿到两个版本
selectedValues.add(BRAND_COLUMN_HEADER.equals(selectedColumn) ? BrandLabel.strip(value) : value);
selectedValues.add(isBrandColumn(selectedColumn) ? BrandLabel.strip(value) : value);
}
if (asinColumnIndex == null) {
asinColumnIndex = headerIndex.get("ASIN");
@@ -1,5 +1,6 @@
package com.nanri.aiimage.modules.pricetrack.service;
import com.nanri.aiimage.common.task.TaskStatuses;
import com.nanri.aiimage.common.text.UserMessages;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
@@ -245,18 +246,19 @@ public class PriceTrackTaskService extends AbstractShopPayloadTaskPipeline<Price
.orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) {
vo.setItems(List.of());
log.debug("[price-track] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
Map<Long, FileTaskEntity> taskById = new LinkedHashMap<>();
List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId)
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (entities.isEmpty()) {
vo.setItems(new ArrayList<>());
prependActiveTasksOutsideWindow(vo, userId, taskIds);
log.debug("[price-track] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
Map<Long, FileTaskEntity> taskById = new LinkedHashMap<>();
if (!taskIds.isEmpty()) {
List<FileTaskEntity> tasks = selectTaskHistoryFieldsByIdsInBatches(taskIds);
for (FileTaskEntity t : tasks) {
@@ -275,6 +277,7 @@ public class PriceTrackTaskService extends AbstractShopPayloadTaskPipeline<Price
items.add(toHistoryItemVo(entity, taskById.get(entity.getTaskId()), jobMap.get(entity.getId()), false));
}
vo.setItems(items);
prependActiveTasksOutsideWindow(vo, userId, taskIds);
long finishedAt = System.nanoTime();
log.debug("[price-track] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
@@ -289,6 +292,45 @@ public class PriceTrackTaskService extends AbstractShopPayloadTaskPipeline<Price
return vo;
}
/**
* 结果行被 top-N 窗口挤出后,仍在执行/排队中的任务会从列表彻底消失(页面重载即无兜底)。
* 这里按任务头补回最前面;已有结果行的任务不重复补。
*/
private void prependActiveTasksOutsideWindow(PriceTrackHistoryVo vo, Long userId, List<Long> coveredTaskIds) {
List<FileTaskEntity> activeTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId,
FileTaskEntity::getTaskNo,
FileTaskEntity::getStatus,
FileTaskEntity::getErrorMessage,
FileTaskEntity::getCreatedAt,
FileTaskEntity::getFinishedAt)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getUserId, userId)
.in(FileTaskEntity::getStatus, TaskStatuses.FileTask.ACTIVE_ORDERED)
.orderByDesc(FileTaskEntity::getCreatedAt)
.last("limit 50"));
if (activeTasks == null || activeTasks.isEmpty()) {
return;
}
List<PriceTrackResultItemVo> activeItems = new ArrayList<>();
for (FileTaskEntity task : activeTasks) {
if (task == null || task.getId() == null || coveredTaskIds.contains(task.getId())) {
continue;
}
PriceTrackResultItemVo item = new PriceTrackResultItemVo();
item.setTaskId(task.getId());
item.setTaskStatus(task.getStatus());
item.setSuccess(false);
item.setError(task.getErrorMessage());
item.setCreatedAt(fmt(task.getCreatedAt()));
item.setFinishedAt(fmt(task.getFinishedAt()));
activeItems.add(item);
}
if (!activeItems.isEmpty()) {
vo.getItems().addAll(0, activeItems);
}
}
/**
* 事务边界:Redis 任务锁在事务外获取(自旋等待不再占用 DB 连接),
* DB 段(复核 + 删除 + 重算任务状态)仍在同一个事务内。
@@ -78,7 +78,9 @@ public final class ShopDataCrawlRowNormalizer {
}
public static boolean rowEmpty(ShopDataCrawlRowDto row) {
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(row.getBrand()) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
// 品牌要按出口口径判空:纯 "Brand:" 这类标签值洗完是空串,若不先洗,
// 一行只有标签的脏行会通过 rowEmpty,随后 copyRow 把它洗成整行空白落件。
return row == null || (blank(row.getDate()) && blank(row.getAsin()) && blank(brand(row.getBrand())) && blank(row.getCommodityImage()) && blank(row.getInventorySales())
&& blank(row.getSalesRank()) && blank(row.getPageViews()) && blank(row.getUnitsSold())
&& blank(row.getPrice()) && blank(row.getRecommendedOffer()));
}
@@ -143,18 +143,23 @@ public class ShopDataDuplicateCheckQueryService {
return result;
}
/** 逐 ASIN 裁剪 occurrences 并重算 shop_count/record_count;items 保持缓存序。 */
/**
* 逐 ASIN 裁剪 occurrences 并重算 shop_count/record_count;items 保持缓存序。
*
* <p>legacy payload 里的 brand 可能仍带 "Brand:" 前缀(存量扫描结果在下次重写前不会变),
* 所有页面出口都经过这里,所以在这一层统一洗一次;可见范围为空时也要洗,不能直接短路返回原对象。</p>
*/
public List<DuplicateItem> cropItems(List<DuplicateItem> items, Set<String> visibleKeys) {
if (visibleKeys == null || items == null) {
return items == null ? List.of() : items;
if (items == null) {
return List.of();
}
List<DuplicateItem> result = new ArrayList<>(items.size());
for (DuplicateItem item : items) {
List<DuplicateOccurrence> kept = new ArrayList<>();
Set<String> shops = new LinkedHashSet<>();
for (DuplicateOccurrence occurrence : item.occurrences()) {
if (visibleKeys.contains(shopKey(occurrence.shopName()))) {
kept.add(occurrence);
if (visibleKeys == null || visibleKeys.contains(shopKey(occurrence.shopName()))) {
kept.add(withCleanBrand(occurrence));
shops.add(occurrence.shopName());
}
}
@@ -166,6 +171,15 @@ public class ShopDataDuplicateCheckQueryService {
return result;
}
private static DuplicateOccurrence withCleanBrand(DuplicateOccurrence occ) {
String clean = BrandLabel.strip(occ.brand());
if (java.util.Objects.equals(clean, occ.brand())) {
return occ;
}
return new DuplicateOccurrence(occ.asin(), occ.date(), occ.price(), clean, occ.shopName(),
occ.groupName(), occ.countryCodes(), occ.country());
}
public Map<String, Object> overviewData(DuplicateScanView view, Set<String> visibleKeys) {
List<DuplicateShop> shops = cropShops(view.payload().shops(), visibleKeys);
List<DuplicateItem> items = cropItems(view.payload().items(), visibleKeys);
@@ -11,6 +11,7 @@ import com.nanri.aiimage.modules.similarasin.client.SimilarAsinLlmClient;
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
import com.nanri.aiimage.modules.similarasin.service.support.PuzzleImageMerger;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinImageEmbedder;
import com.nanri.aiimage.common.support.FutureDeadline;
import com.nanri.aiimage.modules.usersecret.support.SecretUsageContext;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -30,9 +31,12 @@ import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* 货源查询直连 LLM 编排服务:复刻原工作流 similarity_asin
@@ -259,7 +263,7 @@ public class SimilarAsinLlmService {
List<SimilarAsinResultRowDto> merged = new ArrayList<>(rows.size());
for (int i = 0; i < futures.size(); i++) {
try {
merged.add(futures.get(i).join());
merged.add(FutureDeadline.await(futures.get(i), futureDeadlineMillis(), "LLM 单行处理"));
} catch (Exception ex) {
log.warn("[similar-asin][llm] row failed index={} asin={} err={}",
i, rows.get(i).getAsin(), failureMessage(ex));
@@ -269,6 +273,13 @@ public class SimilarAsinLlmService {
return merged;
}
/** 单行/单批等待上限:按一次读超时 × 重试次数再留一倍余量,封顶 2 小时。 */
private long futureDeadlineMillis() {
long perAttempt = Math.max(1_000L, properties.getLlmReadTimeoutMillis());
long total = perAttempt * (Math.max(1, properties.getLlmRetryTimes()) + 1L);
return Math.min(TimeUnit.HOURS.toMillis(2), Math.max(TimeUnit.MINUTES.toMillis(5), total * 2));
}
private SimilarAsinResultRowDto inspectRow(SimilarAsinResultRowDto source,
int rowIndex,
CategoryPrepass prepass,
@@ -460,7 +471,16 @@ public class SimilarAsinLlmService {
}, usageContext), executor));
}
for (CompletableFuture<Void> future : futures) {
future.join(); // matchCategoryChunk 内部自捕获,join 只会因中断失败
try {
FutureDeadline.await(future, futureDeadlineMillis(), "LLM 类目批次");
} catch (TimeoutException | ExecutionException ex) {
// 超时批次的结果按缺失处理:缺行在下游走单行回退,不让一个批次拖住整批
log.warn("[similar-asin][llm] category batch closed without result err={}", failureMessage(ex));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
log.warn("[similar-asin][llm] category batch wait interrupted");
break;
}
}
return new CategoryPrepass(paths, errors);
}
@@ -737,7 +757,15 @@ public class SimilarAsinLlmService {
}, usageContext), executor));
}
for (CompletableFuture<Void> future : futures) {
future.join(); // checkConformChunk 内部自捕获,join 只会因中断失败
try {
FutureDeadline.await(future, futureDeadlineMillis(), "LLM 合规批次");
} catch (TimeoutException | ExecutionException ex) {
log.warn("[similar-asin][llm] conform batch closed without result err={}", failureMessage(ex));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
log.warn("[similar-asin][llm] conform batch wait interrupted");
break;
}
}
return new ConformPrepass(byRow, errors);
}
@@ -126,6 +126,11 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
private static final long PYTHON_INBOUND_LOG_EVERY_N = 20L;
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
private static final long TASK_LOCK_WAIT_MILLIS = 10000L;
/**
* 回传口撞锁时只等 1 秒:锁被 LLM 批合法持有几分钟是常态,客户端 result_feed 本来就会按龄重试,
* 服务端替它等满 10 秒只会白占一个 Tomcat 线程(B15 §3 第 2 条)。
*/
static final long RESULT_SUBMIT_LOCK_WAIT_MILLIS = 1000L;
/**
* Task 8:WorkbookFactory 打开前的受控读取。xlsx 本质是 zip,
* 先用 ZipFile 中央目录探测条目数与解压总字节(不读取压缩内容),
@@ -791,7 +796,8 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost {
}
public void submitResult(Long taskId, SimilarAsinSubmitResultRequest request) {
try (TaskDistributedLockService.LockHandle ignored = ownershipSupport().requireTaskLock(taskId, TASK_LOCK_WAIT_MILLIS)) {
try (TaskDistributedLockService.LockHandle ignored =
ownershipSupport().requireTaskLock(taskId, RESULT_SUBMIT_LOCK_WAIT_MILLIS)) {
submitResultLocked(taskId, request);
}
}
@@ -9,6 +9,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.math.BigInteger;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
@@ -191,6 +192,13 @@ public class WorkbenchClientPackageService {
normalizedVersion, objectKey);
throw new BusinessException("该版本已存在,请提升版本号或先删除旧包");
}
WorkbenchClientPackageEntity latest = newest();
if (latest != null && versionIsLower(normalizedVersion, latest.getVersion())) {
// 按上传时间取最新时,低版本后传会顶掉当前更新目标,已升级装机反而被判"无需更新"
log.warn("[workbench-client] 直传确认失败:版本低于当前最新 version={} latest={} objectKey={}",
normalizedVersion, latest.getVersion(), objectKey);
throw new BusinessException("版本号不能低于当前最新版本 " + latest.getVersion());
}
String sha256 = ossStorageService.workbenchClientObjectSha256(objectKey);
if (sha256 == null || sha256.length() != 64) {
log.warn("[workbench-client] 直传确认失败:摘要计算异常 objectKey={} sha256={}", objectKey, sha256);
@@ -244,6 +252,43 @@ public class WorkbenchClientPackageService {
return deleted;
}
/** 仅对「数字点分」版本做单调性判断;命名不合规时跳过,不臆造顺序。 */
static boolean versionIsLower(String candidate, String current) {
List<BigInteger> left = versionSegments(candidate);
List<BigInteger> right = versionSegments(current);
if (left == null || right == null) {
return false;
}
int size = Math.max(left.size(), right.size());
for (int i = 0; i < size; i++) {
BigInteger a = i < left.size() ? left.get(i) : BigInteger.ZERO;
BigInteger b = i < right.size() ? right.get(i) : BigInteger.ZERO;
int cmp = a.compareTo(b);
if (cmp != 0) {
return cmp < 0;
}
}
return false;
}
private static List<BigInteger> versionSegments(String value) {
String normalized = value == null ? "" : value.trim();
if (normalized.startsWith("v") || normalized.startsWith("V")) {
normalized = normalized.substring(1);
}
if (normalized.isEmpty()) {
return null;
}
List<BigInteger> segments = new ArrayList<>();
for (String part : normalized.split("\\.", -1)) {
if (!part.matches("\\d+")) {
return null;
}
segments.add(new BigInteger(part));
}
return segments;
}
private WorkbenchClientPackageEntity newest() {
return packageMapper.selectOne(new LambdaQueryWrapper<WorkbenchClientPackageEntity>()
.orderByDesc(WorkbenchClientPackageEntity::getCreatedAt)
@@ -0,0 +1,29 @@
package com.nanri.aiimage.common.support;
import org.junit.jupiter.api.Test;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeoutException;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
class FutureDeadlineTest {
@Test
void timesOutAndCancelsTheStuckFuture() {
CompletableFuture<String> never = new CompletableFuture<>();
assertThrows(TimeoutException.class,
() -> FutureDeadline.await(never, 20L, "测试行"));
assertTrue(never.isCancelled(), "超时后必须取消 future,不能让卡住的 worker 继续占线程");
}
@Test
void returnsCompletedValue() throws Exception {
CompletableFuture<String> done = CompletableFuture.completedFuture("ok");
assertTrue("ok".equals(FutureDeadline.await(done, 1000L, "测试行")));
}
}
@@ -9,6 +9,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
@@ -91,6 +92,53 @@ class BrandLabelTest {
.isEmpty();
}
/**
* 品牌列不只是 publish 一处在写:所有实体出口都要经过 BrandLabel。两处身份键例外
* (复核队列与无效 ASIN 表的 brand 是匹配键,改值会重键历史行),必须点名而非静默跳过。
*/
@Test
void everyEntityBrandWriteGoesThroughOutletOrIsAnIdentityKey() throws IOException {
Set<String> identityKeyFiles = Set.of(
"CollectDataBrandRecheckService.java",
"CollectDataInvalidAsinBatchWriter.java");
List<String> offenders = new ArrayList<>();
List<String> written = new ArrayList<>();
for (Path file : mainSources()) {
if (file.getFileName().toString().equals(OUTLET_CLASS)) {
continue;
}
String[] lines = read(file).split("\r?\n", -1);
for (int i = 0; i < lines.length; i++) {
String line = lines[i].trim();
if (line.startsWith("//") || line.startsWith("*") || line.startsWith("/*")) {
continue;
}
if (!line.matches("^entity\\.setBrand\\(.*\\);")) {
continue;
}
written.add(file.getFileName() + ":" + (i + 1));
if (line.contains("BrandLabel.strip(")
|| identityKeyFiles.contains(file.getFileName().toString())
|| hasStripInPrecedingLines(lines, i)) {
continue;
}
offenders.add(file.getFileName() + ":" + (i + 1) + " " + line);
}
}
assertThat(written).as("全仓 entity.setBrand 写入点应被扫到").isNotEmpty();
assertThat(offenders).as("绕过 BrandLabel.strip 的实体品牌写入点").isEmpty();
}
/** strip 可能先赋给局部变量再写出(如 InvalidAsinDataService),允许向上看一小段。 */
private static boolean hasStripInPrecedingLines(String[] lines, int index) {
for (int i = Math.max(0, index - 6); i < index; i++) {
if (lines[i].contains("BrandLabel.strip(")) {
return true;
}
}
return false;
}
@Test
void labelRegexLivesInExactlyOneMainSource() throws IOException {
List<String> carriers = new ArrayList<>();
@@ -48,8 +48,9 @@ class CatchSilenceRatchetTest {
// 10-17 心跳证据两处 fail-open 抬 559 -> 561,逐处出处见 STRICT_NOT_RETHROW 上方注释。
// 10-18 外观专利失败兜底自吞抬 561 -> 562,逐处出处同下。
// 10-24 损耗监控三处带锚点的 fail-open 抬 562 -> 564,逐处出处见 STRICT_NOT_RETHROW 上方注释。
// 10-25 B15 join 无界收口抬 564 -> 569,逐处出处见 STRICT_NOT_RETHROW 上方注释。
static final int GIVEN_TOTAL = 1000;
static final int GIVEN_NOT_RETHROW = 564;
static final int GIVEN_NOT_RETHROW = 569;
static final int GIVEN_LOG_ONLY = 249;
static final int GIVEN_EMPTY = 56;
@@ -104,8 +105,23 @@ class CatchSilenceRatchetTest {
* </ul>
* 这 4 处是替换 EasyExcel 监听器后的等价出口(原实现的 catch 只在 {@code runRead} 一处,
* 但监听器内部另有 POI 的异常分诊);实测 total 1008。
*
* <p>10-25 B15 任务活性收口再 +10(同时替换掉 2 处 join 侧的旧 catch,整树实测 1015),
* 逐处点名——{@code join()} 没有时间界,下面四类出口改成带截止的等待后,超时必须各有去处:
* <ul>
* <li>{@code FutureDeadline.await} 一枚:超时先 cancel 该 future,再重抛带标签的
* TimeoutException(RETHROW);</li>
* <li>{@code AppearancePatentLlmClient.runBatches} 两枚:超时/执行失败与中断都收口成该批
* {@code BatchOutcome} 失败(NOT_RETHROW,替换原来那一枚 {@code catch (Exception)});</li>
* <li>{@code BrandTaskService.buildAndUploadResult} 三枚:超时/中断/执行失败按 IOException
* 或原异常重抛(RETHROW,替换原来那一枚 {@code catch (CompletionException)});</li>
* <li>{@code SimilarAsinLlmService} 的类目与合规两个 prepass 各两枚:超时/执行失败只 WARN
* 并继续(LOG_ONLY),中断复位中断位后退出等待(NOT_RETHROW)。这两处预计算本就是
* 逐行链路的缓存,缺结果的行会走单行回退,所以等待侧超时不需要炸掉整条任务。</li>
* </ul>
* 同批实测 notRethrow 569、logOnly 176、empty 56;气味三格只按这三处实测值抬,形状未放宽。
*/
static final int STRICT_TOTAL = 1008;
static final int STRICT_TOTAL = 1015;
/**
* 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。
* 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。
@@ -187,8 +203,12 @@ class CatchSilenceRatchetTest {
* 全体用户的合法任务一起冻住,真失控由下一轮扫描重新置位(方向是「少冻一个号」)。
* 其中 1)、2) 块体只有一句 log,故同时进 STRICT_LOG_ONLY;3) 是 log+return,不进。
* 同批先写后摘、没留在树上的 6 处静默出口见 total 那一格的点名。
* 10-25 抬 564 -> 569,逐处记名(B15 join 无界收口,共 6 处新增不重抛、替换掉 1 处旧的):
* {@code AppearancePatentLlmClient.runBatches} 两枚(超时/执行失败、中断各收口成该批失败),
* {@code SimilarAsinLlmService} 类目与合规两个 prepass 各两枚(超时/执行失败只 WARN,
* 中断复位中断位后退出等待)。两处预计算的缺行都会走单行回退,不是把已有出口改回吞掉。
*/
static final int STRICT_NOT_RETHROW = 564;
static final int STRICT_NOT_RETHROW = 569;
/**
* 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}:
* 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出——
@@ -197,8 +217,12 @@ class CatchSilenceRatchetTest {
* 10-24 抬 173 -> 175,出处即上面 STRICT_NOT_RETHROW 里点名过的 cost-guard 两处:
* {@code CostGuardScheduler.scan()} 与 {@code UserTaskGuardService.pause()}——块体各只有一句
* {@code log.warn}/{@code log.error},都是带锚点的 fail-open,不是把已有出口改回吞掉。
* 10-25 抬 175 -> 176,出处即上面点名过的 {@code SimilarAsinLlmService} 类目/合规 prepass
* 两个超时出口之一(两处形状相同,其中一处按块内语句判定落进本格,另一处因复位中断位
* 多一条语句只进 NOT_RETHROW)。块体只有一句带 {@code [similar-asin][llm]} 锚点的 WARN,
* 缺结果的行走单行回退,不是无声吞掉。
*/
static final int STRICT_LOG_ONLY = 175; // 严格口径:块内每条顶层语句都是 log 调用才算
static final int STRICT_LOG_ONLY = 176; // 严格口径:块内每条顶层语句都是 log 调用才算
static final int STRICT_EMPTY = 56;
/**
@@ -168,6 +168,13 @@ class AppearancePatentTaskServiceHistoryBatchTest {
LambdaQueryWrapper<FileTaskEntity> q = invocation.getArgument(0);
String segment = q.getSqlSegment() == null ? "" : q.getSqlSegment();
List<Object> params = paramValuesOf(q, segment);
if (segment.contains("status")) {
// 非终态补回查询:返回在跑/续跑件,已有结果行的由 coveredTaskIds 去重
return taskDb.stream()
.filter(t -> "RUNNING".equals(t.getStatus())
|| ("PENDING".equals(t.getStatus()) && t.getResumeOfTaskId() != null))
.toList();
}
List<Long> wanted = new ArrayList<>();
for (Object value : params) {
if (value instanceof List<?> list) {
@@ -361,15 +368,29 @@ class AppearancePatentTaskServiceHistoryBatchTest {
assertEquals(202L, vo.getItems().getFirst().getResultId());
}
@Test
void historyPrependsRunningTaskOutsideResultWindow() throws Exception {
resultDb.add(result(201L, 1001L, 1L, LocalDateTime.of(2026, 8, 1, 10, 0)));
taskDb.add(task(1001L, "SUCCESS", LocalDateTime.of(2026, 8, 1, 9, 0)));
taskDb.add(task(1003L, "RUNNING", LocalDateTime.of(2026, 8, 1, 9, 30)));
AppearancePatentHistoryVo vo = service.history(1L, 50);
assertEquals(2, vo.getItems().size());
assertEquals(1003L, vo.getItems().getFirst().getTaskId(), "窗口外在跑任务必须补回最前");
assertNull(vo.getItems().getFirst().getResultId(), "没有结果行时不伪造 resultId");
assertEquals("RUNNING", vo.getItems().getFirst().getTaskStatus());
}
@Test
void historySqlCountConstant() throws Exception {
seed(20, 4, 20);
service.history(1L, 50);
verify(fileResultMapper, times(1)).selectList(any());
verify(fileTaskMapper, times(1)).selectList(any());
verify(fileTaskMapper, times(2)).selectList(any());
verify(taskFileJobService, times(1)).findAssembleJobsByResultIds(eq(MODULE), anyList());
assertEquals(1, resultSelectCount.get(), "结果 1 次 IN 查询");
assertEquals(1, taskSelectCount.get(), "任务 1 次 IN 查询");
assertEquals(2, taskSelectCount.get(), "任务 2 次固定查询(IN 批查 + 非终态补回)");
assertEquals(1, jobSelectCount.get(), "Job 1 次 IN 查询");
assertEquals(20, resultSelectCount.get() * 20, "查询次数与结果行数无关(无 N+1)");
}
@@ -113,6 +113,19 @@ class DedupeRunServiceCleanRulesTest {
"非品牌列不参与清洗");
}
/** 品牌列别名(Brand/品牌(英文))同样必须剥标签,不能只认正名「品牌」。 */
@Test
@DisplayName("品牌列别名也要剥标签前缀")
void stripsBrandLabelPrefixFromBrandColumnAliases(@TempDir Path tempDir) throws Exception {
String[] headers = {"id", "ASIN", "国家", "价格", "Brand"};
File output = runCleanWithHeaders(tempDir, headers, List.of(
new String[]{"1", "B0ALIAS001", "英国", "10.00", "Brand: NIKE"},
new String[]{"2", "B0ALIAS002", "英国", "11.00", "Brands: ADIDAS"}),
List.of("id", "ASIN", "国家", "价格", "Brand"));
assertEquals(List.of("NIKE", "Brands: ADIDAS"), readOutputColumn(output, 4));
}
/** 纯规则单测:保留主链接开关打开时,整数行不再受子链接影响。 */
@Test
@DisplayName("开启保留主链接时,整数行不受子链接影响")
@@ -181,9 +194,14 @@ class DedupeRunServiceCleanRulesTest {
}
private File runClean(Path tempDir, List<String[]> rows, List<String> selectedColumns) throws Exception {
return runCleanWithHeaders(tempDir, HEADERS, rows, selectedColumns);
}
private File runCleanWithHeaders(Path tempDir, String[] headers, List<String[]> rows,
List<String> selectedColumns) throws Exception {
File input = tempDir.resolve("input.xlsx").toFile();
File output = tempDir.resolve("output.xlsx").toFile();
writeInput(input, rows);
writeInput(input, headers, rows);
DedupeRunService service = new DedupeRunService(
fileTaskMapper, fileResultMapper, storageProperties, ossStorageService, dedupeTotalDataService,
@@ -200,12 +218,12 @@ class DedupeRunServiceCleanRulesTest {
return output;
}
private void writeInput(File file, List<String[]> rows) throws Exception {
private void writeInput(File file, String[] headers, List<String[]> rows) throws Exception {
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
Sheet sheet = workbook.createSheet("variant_collection");
Row headerRow = sheet.createRow(0);
for (int i = 0; i < HEADERS.length; i++) {
headerRow.createCell(i).setCellValue(HEADERS[i]);
for (int i = 0; i < headers.length; i++) {
headerRow.createCell(i).setCellValue(headers[i]);
}
for (int r = 0; r < rows.size(); r++) {
Row row = sheet.createRow(r + 1);
@@ -144,6 +144,11 @@ class ShopDataCrawlRowNormalizerTest {
blank.setAsin("B001");
assertFalse(ShopDataCrawlRowNormalizer.rowEmpty(blank), "任一字段有值即非空");
ShopDataCrawlRowDto labelOnly = new ShopDataCrawlRowDto();
labelOnly.setBrand("Brand:");
assertTrue(ShopDataCrawlRowNormalizer.rowEmpty(labelOnly),
"纯标签品牌洗完是空串,不能让它单独撑出一行空白落件");
}
// ---- sameRow ----
@@ -39,6 +39,18 @@ class ShopDataDuplicateCheckQueryServiceTest {
return ShopDataDuplicateCheckQueryService.Filters.clean("", "", "", "", "", "");
}
@Test
void legacyBrandPrefixIsStrippedBeforeAnyPageOutput() {
DuplicateOccurrence dirty = new DuplicateOccurrence(
"A0000001", "2026-09-01", "10.00", "Brand: NIKE", "ShopA", "G1", List.of("UK"), "英国");
DuplicateItem item = new DuplicateItem("A0000001", 1, 1, List.of(dirty));
List<DuplicateItem> clean = service.cropItems(List.of(item), null);
assertThat(clean).hasSize(1);
assertThat(clean.get(0).occurrences()).extracting(DuplicateOccurrence::brand).containsExactly("NIKE");
}
@Test
void overviewSuperAdminSeesAll() {
Map<String, Object> data = service.overviewData(view, null);
@@ -110,7 +110,7 @@ class SimilarAsinTaskServiceCleanupAfterCommitTest {
void setUpTransactionAndLock() {
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(taskDistributedLockService.acquire(
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), eq(10_000L)))
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), eq(SimilarAsinTaskService.RESULT_SUBMIT_LOCK_WAIT_MILLIS)))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenAnswer(invocation -> {
@@ -117,7 +117,7 @@ class SimilarAsinTaskServiceDtoAssemblyTest {
void setUpTransactionAndLock() {
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(taskDistributedLockService.acquire(
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), eq(10_000L)))
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), eq(SimilarAsinTaskService.RESULT_SUBMIT_LOCK_WAIT_MILLIS)))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenAnswer(invocation -> {
@@ -117,7 +117,7 @@ class SimilarAsinTaskServiceSubmitTest {
eq(SimilarAsinTaskService.MODULE_TYPE),
anyLong(),
any(Duration.class),
eq(10_000L)))
eq(SimilarAsinTaskService.RESULT_SUBMIT_LOCK_WAIT_MILLIS)))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenAnswer(invocation -> {
@@ -117,7 +117,7 @@ class SimilarAsinTaskServiceTxBoundaryTest {
void setUpTransactionAndLock() {
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(taskDistributedLockService.acquire(
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), eq(10_000L)))
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), eq(SimilarAsinTaskService.RESULT_SUBMIT_LOCK_WAIT_MILLIS)))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenAnswer(invocation -> {
@@ -112,7 +112,7 @@ class CleanupErrorContractTest {
void setUp() {
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(taskDistributedLockService.acquire(
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), eq(10_000L)))
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), anyLong()))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenAnswer(invocation -> {
@@ -111,7 +111,7 @@ class DoneFinalizationContractTest {
void setUp() {
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(taskDistributedLockService.acquire(
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), eq(10_000L)))
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), anyLong()))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenReturn(transactionStatus);
@@ -121,7 +121,7 @@ class RollbackSemanticsContractTest {
void setUp() {
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(taskDistributedLockService.acquire(
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), eq(10_000L)))
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), anyLong()))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenAnswer(invocation -> {
@@ -109,7 +109,7 @@ class TxDurationBenchmarkTest {
void setUp() {
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
lenient().when(taskDistributedLockService.acquire(
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), eq(10_000L)))
eq(SimilarAsinTaskService.MODULE_TYPE), anyLong(), any(Duration.class), anyLong()))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(transactionManager.getTransaction(any(TransactionDefinition.class)))
.thenAnswer(invocation -> {
@@ -203,10 +203,11 @@ class TxDurationBenchmarkTest {
long[] first = measureSingleSubmission();
long[] second = measureSingleSubmission();
// 两次测量的事务段应在 ±3× 内(mock 环境抖动容忍)
// 两次测量的事务段应在 ±3× 内;mock 调用只有百纳秒级,纯比例在亚微秒读数上会被
// 调度抖动放大成假红(实测 100ns vs 400ns),所以再给 10ms 绝对下限兜底
long lower = Math.min(first[2], second[2]);
long upper = Math.max(first[2], second[2]);
assertTrue(upper <= Math.max(1, lower * 3),
assertTrue(upper <= Math.max(java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(10), lower * 3),
"两次测量偏差过大: " + lower + " vs " + upper);
}
@@ -162,6 +162,32 @@ class WorkbenchClientPackageServiceTest {
verify(packageMapper, never()).insert(any(WorkbenchClientPackageEntity.class));
}
@Test
void confirmRejectsVersionLowerThanCurrentLatest() {
String objectKey = PREFIX + "20261024100000123-uuid-1.2.2.zip";
when(ossStorageService.workbenchClientObjectSize(objectKey)).thenReturn(1234L);
when(packageMapper.selectCount(any())).thenReturn(0L);
// 第一次 selectOne 是幂等查(无同对象),第二次才是取当前最新版本
when(packageMapper.selectOne(any()))
.thenReturn(null, row(99, "1.2.3", PREFIX + "old.zip"));
BusinessException ex = assertThrows(BusinessException.class, () -> service().confirmPackage(
objectKey, "ShuFuAI-Workbench-1.2.2.zip", "1.2.2"));
assertTrue(ex.getMessage().contains("1.2.3"));
verify(ossStorageService, never()).workbenchClientObjectSha256(anyString());
verify(packageMapper, never()).insert(any(WorkbenchClientPackageEntity.class));
}
@Test
void versionComparisonHandlesDifferentSegmentCounts() {
assertTrue(WorkbenchClientPackageService.versionIsLower("1.2.3", "1.2.10"));
assertTrue(WorkbenchClientPackageService.versionIsLower("1.2", "1.2.1"));
assertTrue(!WorkbenchClientPackageService.versionIsLower("v1.10.0", "1.9.9"));
assertTrue(!WorkbenchClientPackageService.versionIsLower("release-x", "1.9.9"),
"非数字版本不臆造顺序");
}
@Test
void confirmRejectsVersionThatDoesNotMatchObjectKey() {
String objectKey = PREFIX + "20261024100000123-1.2.3.zip";