super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-38

Browse Source
This commit is contained in:
黄自达
2026-08-29 23:11:59 +08:00
parent a9a72c7bdf
commit 36bed2bdc6
5 changed files with 653 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/ShopDataCrawlTaskController.java
+8 -4
View File
@@ -127,14 +127,18 @@ public class ShopDataCrawlTaskController {
@GetMapping("/history")
@Operation(
summary = "查询抓取记录",
description = "返回当前用户最近 100 条店铺抓取结果，按创建时间倒序排列，包含执行状态和异步结果文件状态。",
summary = "分页查询抓取记录",
description = "返回当前用户的店铺抓取结果，按创建时间倒序分页返回（limit 收敛到 [1,100]），包含执行状态和异步结果文件状态。",
responses = @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功"))
public ApiResponse<ShopDataCrawlHistoryVo> history(
@Parameter(name = "user_id", description = "当前用户 ID", required = true,
in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(taskService.listHistory(userId));
@RequestParam("user_id") Long userId,
@Parameter(name = "page", description = "页号，从 1 开始", in = ParameterIn.QUERY, example = "1")
@RequestParam(value = "page", defaultValue = "1") int page,
@Parameter(name = "limit", description = "每页条数，最大 100；默认 100 与旧版返回最近 100 条的行为一致", in = ParameterIn.QUERY, example = "20")
@RequestParam(value = "limit", defaultValue = "100") int limit) {
return ApiResponse.success(taskService.listHistory(userId, page, limit));
}
@PostMapping("/tasks/progress/batch")
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/vo/ShopDataCrawlHistoryVo.java
+11 -2
View File
@@ -7,10 +7,19 @@ import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "当前用户的店铺数据抓取历史记录")
@Schema(description = "当前用户的店铺数据抓取历史记录（分页）")
public class ShopDataCrawlHistoryVo {
@Schema(description = "历史记录项，按创建时间倒序返回，最多返回最近 100 条")
@Schema(description = "历史记录项，按创建时间倒序返回，本页最多返回 limit 条")
private List<ShopDataCrawlResultItemVo> items = new ArrayList<>();
@Schema(description = "当前页号，从 1 开始；空数据时返回 0", example = "1")
private long page;
@Schema(description = "本页实际返回条数上限，收敛到 [1,100]", example = "20")
private int limit;
@Schema(description = "当前用户历史记录总数", example = "128")
private long total;
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java
+88 -10
View File
@@ -85,6 +85,8 @@ public class ShopDataCrawlTaskService {
private static final int ID_BATCH_SIZE = 500;
/** Task 36：每日累计归档版本 CAS 冲突的最大重试次数（每次重试都释放店铺级锁）。 */
private static final int MAX_DAILY_AGGREGATION_ATTEMPTS = 3;
/** 历史列表单页条数上限。 */
private static final int HISTORY_MAX_PAGE_SIZE = 100;
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
private static final String PARTIAL_RESULT_MESSAGE = "Python 中断，已保留已回传的部分数据";
private static final String RESULT_CHUNK_SCOPE_PREFIX = "result-chunks:";
@@ -223,9 +225,35 @@ public class ShopDataCrawlTaskService {
}
public ShopDataCrawlHistoryVo listHistory(Long userId) {
return listHistory(userId, 1, HISTORY_MAX_PAGE_SIZE);
}
/**
* 分页查询当前用户历史记录：结果行按创建时间倒序，字段裁剪，任务与文件作业状态批量加载。
* page 必须 ≥ 1；limit 收敛到 [1, {@value #HISTORY_MAX_PAGE_SIZE}]；越界页返回空页。
*/
public ShopDataCrawlHistoryVo listHistory(Long userId, int page, int limit) {
long startedAt = System.nanoTime();
validateUserId(userId);
if (page < 1) {
throw new BusinessException("page 必须大于等于 1");
}
if (limit < 1) {
throw new BusinessException("limit 必须大于等于 1");
}
int effectiveLimit = Math.min(limit, HISTORY_MAX_PAGE_SIZE);
ShopDataCrawlHistoryVo vo = new ShopDataCrawlHistoryVo();
vo.setPage(page);
vo.setLimit(effectiveLimit);
long total = countHistoryRows(userId);
vo.setTotal(total);
if (total == 0L || (long) (page - 1) * effectiveLimit >= total) {
vo.setItems(List.of());
vo.setPage(total == 0L ? 0L : page);
log.info("[shop-data-crawl] history timing userId={} rows=0 total={} page={} limit={} totalMs={}",
userId, total, page, effectiveLimit, elapsedMs(startedAt, System.nanoTime()));
return vo;
}
List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
@@ -241,12 +269,12 @@ public class ShopDataCrawlTaskService {
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100"));
.last("LIMIT " + (long) (page - 1) * effectiveLimit + ", " + effectiveLimit));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) {
vo.setItems(List.of());
log.info("[shop-data-crawl] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
log.info("[shop-data-crawl] history timing userId={} rows=0 total={} page={} limit={} totalMs={}",
userId, total, page, effectiveLimit, elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
@@ -265,11 +293,14 @@ public class ShopDataCrawlTaskService {
}
vo.setItems(items);
long finishedAt = System.nanoTime();
log.info("[shop-data-crawl] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
log.info("[shop-data-crawl] history timing userId={} rows={} tasks={} jobs={} total={} page={} limit={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
taskMap.size(),
jobMap.size(),
total,
page,
effectiveLimit,
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
@@ -278,6 +309,13 @@ public class ShopDataCrawlTaskService {
return vo;
}
private long countHistoryRows(Long userId) {
Long count = fileResultMapper.selectCount(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId));
return count == null ? 0L : count;
}
public ShopDataCrawlTaskBatchVo getTaskProgressBatch(List<Long> taskIds) {
ShopDataCrawlTaskBatchVo batch = new ShopDataCrawlTaskBatchVo();
if (taskIds == null || taskIds.isEmpty()) {
@@ -294,10 +332,18 @@ public class ShopDataCrawlTaskService {
}
Map<Long, FileTaskEntity> taskMap = loadTaskMapByIds(normalizedTaskIds);
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.in(FileResultEntity::getTaskId, normalizedTaskIds)
.orderByAsc(FileResultEntity::getId));
List<FileResultEntity> rows;
try {
rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.in(FileResultEntity::getTaskId, normalizedTaskIds)
.orderByAsc(FileResultEntity::getId));
} catch (Exception ex) {
throw new BusinessException("任务进度查询失败", ex);
}
if (rows == null) {
throw new BusinessException("任务进度查询失败");
}
Map<Long, List<FileResultEntity>> rowsByTaskId = new LinkedHashMap<>();
for (FileResultEntity row : rows) {
rowsByTaskId.computeIfAbsent(row.getTaskId(), ignored -> new ArrayList<>()).add(row);
@@ -1571,8 +1617,8 @@ public class ShopDataCrawlTaskService {
return;
}
snapshot.setShopName(firstNonBlank(payload.getShopName(), snapshot.getShopName()));
// 按国家增量合并：本次未回传的国家保留快照中的旧结果，同一国家按行去重合并
snapshot.setCountryResults(mergeCountryResults(snapshot.getCountryResults(), payload.getCountryResults()));
// 按国家增量合并：本次未回传的国家保留快照中的旧结果；已回传的国家以本次提交为准（客户端按国家整体 upsert）
snapshot.setCountryResults(mergeSnapshotCountries(snapshot.getCountryResults(), payload.getCountryResults()));
if (!blank(payload.getError())) {
snapshot.setError(payload.getError().trim());
}
@@ -1638,6 +1684,22 @@ public class ShopDataCrawlTaskService {
return new ArrayList<>(map.values());
}
/**
* 快照国家的增量合并：只合并本次提交含有的国家（新国家追加、已含国家以本次为准），
* 本次未提交的国家完整保留 —— 与 mergeCountryResults 的行级并集语义不同，
* 这里不跨提交做行去重累积，避免客户端按国家整体回传时旧行残留。
*/
private List<ShopDataCrawlCountryResultDto> mergeSnapshotCountries(List<ShopDataCrawlCountryResultDto> base, List<ShopDataCrawlCountryResultDto> incoming) {
Map<String, ShopDataCrawlCountryResultDto> map = new LinkedHashMap<>();
for (ShopDataCrawlCountryResultDto item : copyCountryResults(base)) {
map.put(item.getCountry(), item);
}
for (ShopDataCrawlCountryResultDto item : copyCountryResults(incoming)) {
map.put(item.getCountry(), item);
}
return new ArrayList<>(map.values());
}
private boolean hasResultRows(ShopDataCrawlShopPayloadDto payload) {
return payload != null && hasResultRows(payload.getCountryResults());
}
@@ -2490,14 +2552,30 @@ public class ShopDataCrawlTaskService {
/**
* RUNNING 期间只写轻量进度字段（successFileCount/failedFileCount/status/updatedAt），
* 避免每次分片接收都序列化并落库完整结果 JSON；仅终态（全部结果行完成）才写完整快照。
* 例外：本次提交携带了新国家数据且 resultJson 尚无任何国家行（缓存中断后恢复的增量提交），
* 此时将合并后的国家写回 resultJson，保证跨提交的国家累积持久化，任务中断不再丢失已上报国家。
*/
private void persistProgressOrSnapshot(FileTaskEntity task, List<ShopDataCrawlResultItemVo> snapshots, boolean terminal) {
if (terminal) {
persistSnapshotJson(task, snapshots);
} else if (snapshotCarriesCountryRows(snapshots) && !snapshotJsonHasCountryRows(task)) {
persistSnapshotJson(task, snapshots);
}
fileTaskMapper.updateById(task);
}
private boolean snapshotCarriesCountryRows(List<ShopDataCrawlResultItemVo> snapshots) {
if (snapshots == null) {
return false;
}
for (ShopDataCrawlResultItemVo snapshot : snapshots) {
if (snapshot != null && hasResultRows(snapshot.getCountryResults())) {
return true;
}
}
return false;
}
private void syncSnapshotTables(FileTaskEntity task, List<ShopDataCrawlResultItemVo> snapshots) {
List<ShopDataCrawlResultItemVo> safe = snapshots == null ? List.of() : snapshots;
taskResultItemService.replaceTaskSnapshots(task.getId(), MODULE_TYPE, safe, new TaskResultItemService.SnapshotKeyResolver() {