super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新需求更新 同步更新

Browse Source
This commit is contained in:
supernijia
2026-08-06 01:11:54 +08:00
parent 9048bbb7f8
commit 28e7fce11c
112 changed files with 7739 additions and 637 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/controller/CollectDataController.java
+13
View File
@@ -53,6 +53,19 @@ public class CollectDataController {
return ApiResponse.success(null);
}
@PostMapping("/tasks/{taskId}/fail")
@Operation(summary = "标记采集任务失败", description = "桌面端入队失败或无法继续执行时，将任务收敛为 FAILED。")
public ApiResponse<Void> fail(
@Parameter(description = "采集任务 ID", required = true, example = "9001")
@PathVariable Long taskId,
@Parameter(description = "当前用户 ID", required = true, example = "1")
@RequestParam("user_id") Long userId,
@Parameter(description = "失败原因")
@RequestParam(value = "error", required = false) String error) {
service.failTask(taskId, userId, error);
return ApiResponse.success(null);
}
@GetMapping("/tasks/{taskId}/items")
@Operation(summary = "分页获取任务明细数据", description = "供 Python 端拉取，默认每页 50 条；返回任务关联的筛选条件，便于 Python 端按筛选条件采集。")
public ApiResponse<CollectDataItemsPageVo> items(
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/dto/CollectDataSummaryRowDto.java
+3
View File
@@ -51,4 +51,7 @@ public class CollectDataSummaryRowDto {
@JsonAlias({"page", "totalPage", "页数", "max_page", "maxPage"})
@Schema(description = "该关键词总页数（Python 端取所有命中行的最大页码）", example = "10")
private Integer totalPage;
@Schema(description = "Python 端 ASIN 过滤数量", example = "3")
private Integer asinFilter;
}
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/vo/CollectDataHistoryItemVo.java
+30
View File
@@ -49,6 +49,36 @@ public class CollectDataHistoryItemVo {
@Schema(description = "最终结果行数")
private Integer finalRowCount;
@Schema(description = "解析任务总行数")
private Integer totalRows;
@Schema(description = "Python 已回传行数")
private Integer receivedRows;
@Schema(description = "Python 已处理关键词数")
private Integer processedRows;
@Schema(description = "当前采集阶段：search / detail")
private String collectStage;
@Schema(description = "当前关键词")
private String currentKeyword;
@Schema(description = "搜索页当前页码")
private Integer searchCurrentPage;
@Schema(description = "搜索页总页数")
private Integer searchTotalPages;
@Schema(description = "详情页已处理 ASIN 数")
private Integer detailProcessedAsins;
@Schema(description = "详情页 ASIN 总数")
private Integer detailTotalAsins;
@Schema(description = "任务进度百分比，0-100")
private Integer progressPercent;
@Schema(description = "记录创建时间")
private String createdAt;
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataExcelAssemblyService.java
+2 -1
View File
@@ -24,7 +24,7 @@ public class CollectDataExcelAssemblyService {
private static final String[] SHEET_DETAIL_HEADER = {"品牌", "ASIN", "价格", "卖家名称", "关键词", "配送方式"};
private static final String SHEET_SUMMARY_NAME = "结果文件";
private static final String[] SHEET_SUMMARY_HEADER = {"关键词", "FBA", "FBM", "AMZ", "无配送方式", "页数"};
private static final String[] SHEET_SUMMARY_HEADER = {"关键词", "FBA", "FBM", "AMZ", "无配送方式", "页数", "ASIN过滤"};
/**
* 生成采集结果工作簿。
@@ -115,6 +115,7 @@ public class CollectDataExcelAssemblyService {
} else {
row.createCell(5).setCellValue("");
}
row.createCell(6).setCellValue(intOrZero(item.getAsinFilter()));
}
return;
}
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/service/CollectDataService.java
+212 -2
View File
@@ -43,6 +43,7 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
@@ -60,6 +61,8 @@ import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionTemplate;
@@ -105,6 +108,7 @@ public class CollectDataService {
private static final int BRAND_CHECK_BATCH_SIZE = 10;
private static final long TASK_LOCK_WAIT_MILLIS = 5000L;
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String STALE_TASK_ERROR = "长时间未收到 Python 心跳，任务已自动失败";
private static final List<String> KEYWORD_HEADER_ALIASES = List.of("关键词", "keyword", "key word");
private static final List<String> STATUS_HEADER_ALIASES = List.of("状态", "status");
@@ -131,6 +135,9 @@ public class CollectDataService {
private final ObjectMapper objectMapper;
private final TransactionTemplate transactionTemplate;
@Value("${aiimage.collect-data.stale-timeout-minutes:30}")
private long staleTimeoutMinutes;
public CollectDataParseVo parseAndCreateTask(CollectDataParseRequest request) {
long startedAt = System.currentTimeMillis();
if (request == null || request.getUserId() == null || request.getUserId() <= 0) {
@@ -216,7 +223,18 @@ public class CollectDataService {
requestPayload.put("filters", filters);
requestPayload.put("files", sources);
task.setRequestJson(objectMapper.writeValueAsString(requestPayload));
task.setResultJson("{}");
Map<String, Object> initialStats = new LinkedHashMap<>();
initialStats.put("totalRows", parsedRows.size());
initialStats.put("receivedRows", 0);
initialStats.put("processedRows", 0);
initialStats.put("currentChunkRows", 0);
initialStats.put("dedupeFilteredCount", 0);
initialStats.put("invalidFilteredCount", 0);
initialStats.put("brandRejectedCount", 0);
initialStats.put("brandQueryFailedCount", 0);
initialStats.put("finalRowCount", 0);
initialStats.put("summaries", List.of());
task.setResultJson(objectMapper.writeValueAsString(initialStats));
} catch (Exception ex) {
throw new BusinessException("序列化任务信息失败");
}
@@ -274,9 +292,138 @@ public class CollectDataService {
fileTaskMapper.updateById(task);
}
@Transactional
public void failTask(Long taskId, Long userId, String error) {
FileTaskEntity task = requireTask(taskId, userId);
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
return;
}
String message = firstNonBlank(error, "collect-data task dispatch failed");
FileResultEntity result = ensureTaskResult(task);
CollectDataStats stats = loadStats(task);
result.setSuccess(0);
result.setErrorMessage(message);
result.setRowCount(stats.finalRowCount);
fileResultMapper.updateById(result);
task.setStatus(STATUS_FAILED);
task.setErrorMessage(message);
task.setFailedFileCount(1);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
persistStats(task, stats);
fileTaskMapper.updateById(task);
}
@Transactional
public void updateProgress(Long taskId, TaskHeartbeatRequest request) {
if (taskId == null || taskId <= 0 || request == null) {
return;
}
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) {
return;
}
CollectDataStats stats = loadStats(task);
boolean changed = false;
Integer current = request.getCurrent();
Integer total = request.getTotal();
if (current != null && total != null && total > 0) {
int totalRows = stats.totalRows > 0 ? stats.totalRows : total;
int processedRows = Math.max(stats.processedRows, Math.min(Math.max(current, 0), totalRows));
if (stats.totalRows != totalRows || stats.processedRows != processedRows) {
stats.totalRows = totalRows;
stats.processedRows = processedRows;
changed = true;
}
}
if (request.getCollectStage() != null && !java.util.Objects.equals(stats.collectStage, request.getCollectStage())) {
stats.collectStage = request.getCollectStage();
changed = true;
}
if (request.getCurrentKeyword() != null && !java.util.Objects.equals(stats.currentKeyword, request.getCurrentKeyword())) {
stats.currentKeyword = request.getCurrentKeyword();
changed = true;
}
if (request.getSearchCurrentPage() != null && stats.searchCurrentPage != Math.max(0, request.getSearchCurrentPage())) {
stats.searchCurrentPage = Math.max(0, request.getSearchCurrentPage());
changed = true;
}
if (request.getSearchTotalPages() != null && stats.searchTotalPages != Math.max(0, request.getSearchTotalPages())) {
stats.searchTotalPages = Math.max(0, request.getSearchTotalPages());
changed = true;
}
if (request.getDetailProcessedAsins() != null && stats.detailProcessedAsins != Math.max(0, request.getDetailProcessedAsins())) {
stats.detailProcessedAsins = Math.max(0, request.getDetailProcessedAsins());
changed = true;
}
if (request.getDetailTotalAsins() != null && stats.detailTotalAsins != Math.max(0, request.getDetailTotalAsins())) {
stats.detailTotalAsins = Math.max(0, request.getDetailTotalAsins());
changed = true;
}
if (!changed) {
return;
}
persistStats(task, stats);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
}
}
@Scheduled(cron = "${aiimage.collect-data.stale-check-cron:*/30 * * * * *}")
public void finalizeStaleTasks() {
long timeoutMinutes = Math.max(1L, staleTimeoutMinutes);
LocalDateTime threshold = LocalDateTime.now().minusMinutes(timeoutMinutes);
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.lt(FileTaskEntity::getUpdatedAt, threshold)
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit 200"));
for (FileTaskEntity task : tasks) {
finalizeStaleTask(task.getId(), threshold, timeoutMinutes);
}
}
private void finalizeStaleTask(Long taskId, LocalDateTime threshold, long timeoutMinutes) {
try (TaskDistributedLockService.LockHandle lock =
taskDistributedLockService.acquire(MODULE_TYPE, taskId, 0L)) {
if (lock == null) {
return;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null
|| !MODULE_TYPE.equals(task.getModuleType())
|| !STATUS_RUNNING.equals(task.getStatus())
|| task.getUpdatedAt() == null
|| !task.getUpdatedAt().isBefore(threshold)) {
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(taskId, MODULE_TYPE) > 0L) {
return;
}
FileResultEntity result = ensureTaskResult(task);
CollectDataStats stats = loadStats(task);
if (hasReceivedChunks(taskId)) {
enqueueFinalWorkbook(task, result, stats);
log.warn("[collect-data] stale task enqueued partial workbook taskId={} timeoutMinutes={} finalRows={}",
taskId, timeoutMinutes, stats.finalRowCount);
return;
}
markTaskFailed(task, result, STALE_TASK_ERROR, stats);
log.warn("[collect-data] stale task failed without result chunks taskId={} timeoutMinutes={}",
taskId, timeoutMinutes);
} catch (Exception ex) {
log.warn("[collect-data] stale task finalization failed taskId={} msg={}",
taskId, ex.getMessage(), ex);
}
}
public CollectDataDashboardVo dashboard(Long userId) {
CollectDataDashboardVo vo = new CollectDataDashboardVo();
vo.setPendingTaskCount(countTask(userId, STATUS_RUNNING));
vo.setPendingTaskCount(countActiveTasks(userId));
vo.setSuccessTaskCount(countTask(userId, STATUS_SUCCESS));
vo.setFailedTaskCount(countTask(userId, STATUS_FAILED));
long processed = (vo.getSuccessTaskCount() == null ? 0 : vo.getSuccessTaskCount())
@@ -967,6 +1114,13 @@ public class CollectDataService {
return count == null ? 0 : count.intValue();
}
private boolean hasReceivedChunks(Long taskId) {
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
return count != null && count > 0L;
}
private CollectDataSubmitResultVo buildSubmitVo(FileTaskEntity task,
FileResultEntity result,
int chunkIndex,
@@ -999,7 +1153,15 @@ public class CollectDataService {
}
try {
JsonNode root = objectMapper.readTree(task.getResultJson());
stats.totalRows = root.path("totalRows").asInt(0);
stats.receivedRows = root.path("receivedRows").asInt(0);
stats.processedRows = root.path("processedRows").asInt(0);
stats.collectStage = root.path("collectStage").asText(null);
stats.currentKeyword = root.path("currentKeyword").asText(null);
stats.searchCurrentPage = root.path("searchCurrentPage").asInt(0);
stats.searchTotalPages = root.path("searchTotalPages").asInt(0);
stats.detailProcessedAsins = root.path("detailProcessedAsins").asInt(0);
stats.detailTotalAsins = root.path("detailTotalAsins").asInt(0);
stats.currentChunkRows = root.path("currentChunkRows").asInt(0);
stats.dedupeFilteredCount = root.path("dedupeFilteredCount").asInt(0);
stats.invalidFilteredCount = root.path("invalidFilteredCount").asInt(0);
@@ -1028,7 +1190,15 @@ public class CollectDataService {
}
try {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("totalRows", stats.totalRows);
payload.put("receivedRows", stats.receivedRows);
payload.put("processedRows", stats.processedRows);
payload.put("collectStage", stats.collectStage);
payload.put("currentKeyword", stats.currentKeyword);
payload.put("searchCurrentPage", stats.searchCurrentPage);
payload.put("searchTotalPages", stats.searchTotalPages);
payload.put("detailProcessedAsins", stats.detailProcessedAsins);
payload.put("detailTotalAsins", stats.detailTotalAsins);
payload.put("currentChunkRows", stats.currentChunkRows);
payload.put("dedupeFilteredCount", stats.dedupeFilteredCount);
payload.put("invalidFilteredCount", stats.invalidFilteredCount);
@@ -1142,7 +1312,15 @@ public class CollectDataService {
}
private static class CollectDataStats {
private int totalRows;
private int receivedRows;
private int processedRows;
private String collectStage;
private String currentKeyword;
private int searchCurrentPage;
private int searchTotalPages;
private int detailProcessedAsins;
private int detailTotalAsins;
private int currentChunkRows;
private int dedupeFilteredCount;
private int invalidFilteredCount;
@@ -1316,6 +1494,17 @@ public class CollectDataService {
return count == null ? 0L : count;
}
private long countActiveTasks(Long userId) {
if (userId == null || userId <= 0) {
return 0L;
}
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getUserId, userId)
.in(FileTaskEntity::getStatus, List.of(STATUS_PENDING, STATUS_RUNNING)));
return count == null ? 0L : count;
}
private Map<Long, FileTaskEntity> loadTaskMap(List<Long> taskIds) {
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
@@ -1356,10 +1545,31 @@ public class CollectDataService {
vo.setInvalidFilteredCount(stats.invalidFilteredCount);
vo.setBrandRejectedCount(stats.brandRejectedCount);
vo.setFinalRowCount(stats.finalRowCount);
vo.setTotalRows(stats.totalRows);
vo.setReceivedRows(stats.receivedRows);
vo.setProcessedRows(stats.processedRows);
vo.setCollectStage(stats.collectStage);
vo.setCurrentKeyword(stats.currentKeyword);
vo.setSearchCurrentPage(stats.searchCurrentPage);
vo.setSearchTotalPages(stats.searchTotalPages);
vo.setDetailProcessedAsins(stats.detailProcessedAsins);
vo.setDetailTotalAsins(stats.detailTotalAsins);
vo.setProgressPercent(calculateProgressPercent(task.getStatus(), stats.totalRows, stats.processedRows));
}
return vo;
}
private int calculateProgressPercent(String status, int totalRows, int receivedRows) {
if (STATUS_SUCCESS.equals(status)) {
return 100;
}
if (totalRows <= 0) {
return 0;
}
int received = Math.max(0, Math.min(receivedRows, totalRows));
return Math.min(99, received * 100 / totalRows);
}
private String resolveDisplayResultFilename(FileResultEntity row, FileTaskEntity task) {
if (row == null) {
return null;