super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

异常记录检测修改

Browse Source
This commit is contained in:
super
2026-05-04 19:04:55 +08:00
parent f46c231323
commit 111f30d150
70 changed files with 7182 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/controller/QueryAsinTaskController.java
+2 -5
View File
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.queryasin.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskCandidateAddRequest;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskMatchShopsRequest;
import com.nanri.aiimage.modules.productrisk.model.vo.ProductRiskCandidateVo;
@@ -23,7 +24,6 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
@@ -37,7 +37,6 @@ import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream;
import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.List;
@@ -269,10 +268,8 @@ public class QueryAsinTaskController {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
}
try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
DownloadHeaderUtil.setAttachment(response, filename);
try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536];
int read;
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java
+73 -6
View File
@@ -135,27 +135,58 @@ public class QueryAsinTaskService {
}
public QueryAsinHistoryVo listHistory(Long userId) {
long startedAt = System.nanoTime();
validateUserId(userId);
QueryAsinHistoryVo vo = new QueryAsinHistoryVo();
List<FileResultEntity> entities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.select(FileResultEntity::getId,
FileResultEntity::getTaskId,
FileResultEntity::getModuleType,
FileResultEntity::getSourceFilename,
FileResultEntity::getSourceFileUrl,
FileResultEntity::getResultFilename,
FileResultEntity::getResultFileUrl,
FileResultEntity::getSuccess,
FileResultEntity::getErrorMessage,
FileResultEntity::getUserId,
FileResultEntity::getCreatedAt)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getUserId, userId)
.orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100"));
long resultRowsLoadedAt = System.nanoTime();
if (entities.isEmpty()) {
vo.setItems(List.of());
log.info("[query-asin] history timing userId={} rows=0 totalMs={} resultQueryMs={} taskQueryMs=0 jobQueryMs=0 buildMs=0",
userId, elapsedMs(startedAt, resultRowsLoadedAt), elapsedMs(startedAt, resultRowsLoadedAt));
return vo;
}
Map<Long, FileTaskEntity> taskMap = loadTaskMap(entities);
Map<Long, Map<Long, QueryAsinResultItemVo>> snapshotMap = buildSnapshotMap(taskMap);
Map<Long, FileTaskEntity> taskMap = loadHistoryTaskMap(entities);
long tasksLoadedAt = System.nanoTime();
Map<Long, TaskFileJobEntity> jobMap = taskFileJobService.findAssembleJobsByResultIds(MODULE_TYPE, entities.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.distinct()
.toList());
long jobsLoadedAt = System.nanoTime();
List<QueryAsinResultItemVo> items = new ArrayList<>();
for (FileResultEntity entity : entities) {
FileTaskEntity task = taskMap.get(entity.getTaskId());
QueryAsinResultItemVo snapshot = snapshotMap.getOrDefault(entity.getTaskId(), Map.of()).get(entity.getId());
items.add(toHistoryItem(entity, task, snapshot));
items.add(toHistoryItem(entity, task, null, jobMap.get(entity.getId())));
}
vo.setItems(items);
long finishedAt = System.nanoTime();
log.info("[query-asin] history timing userId={} rows={} tasks={} jobs={} totalMs={} resultQueryMs={} taskQueryMs={} jobQueryMs={} buildMs={}",
userId,
entities.size(),
taskMap.size(),
jobMap.size(),
elapsedMs(startedAt, finishedAt),
elapsedMs(startedAt, resultRowsLoadedAt),
elapsedMs(resultRowsLoadedAt, tasksLoadedAt),
elapsedMs(tasksLoadedAt, jobsLoadedAt),
elapsedMs(jobsLoadedAt, finishedAt));
return vo;
}
@@ -510,6 +541,31 @@ public class QueryAsinTaskService {
return taskMap;
}
private Map<Long, FileTaskEntity> loadHistoryTaskMap(List<FileResultEntity> entities) {
List<Long> taskIds = entities.stream()
.map(FileResultEntity::getTaskId)
.filter(id -> id != null && id > 0)
.distinct()
.toList();
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
if (taskIds.isEmpty()) {
return taskMap;
}
int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize());
for (int start = 0; start < taskIds.size(); start += batchSize) {
int end = Math.min(start + batchSize, taskIds.size());
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.select(FileTaskEntity::getId, FileTaskEntity::getStatus, FileTaskEntity::getFinishedAt)
.in(FileTaskEntity::getId, taskIds.subList(start, end)));
for (FileTaskEntity task : tasks) {
if (task != null) {
taskMap.put(task.getId(), task);
}
}
}
return taskMap;
}
private Map<Long, Map<Long, QueryAsinResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
Map<Long, Map<Long, QueryAsinResultItemVo>> out = new LinkedHashMap<>();
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
@@ -523,6 +579,10 @@ public class QueryAsinTaskService {
}
private QueryAsinResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, QueryAsinResultItemVo snapshot) {
return toHistoryItem(entity, task, snapshot, null);
}
private QueryAsinResultItemVo toHistoryItem(FileResultEntity entity, FileTaskEntity task, QueryAsinResultItemVo snapshot, TaskFileJobEntity job) {
QueryAsinResultItemVo item = snapshot != null ? snapshot : new QueryAsinResultItemVo();
item.setResultId(entity.getId());
item.setTaskId(entity.getTaskId());
@@ -535,7 +595,7 @@ public class QueryAsinTaskService {
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
item.setDownloadUrl(null);
attachFileJobState(item, entity);
attachFileJobState(item, entity, job);
if (item.getCountryResults() == null) {
item.setCountryResults(new ArrayList<>());
}
@@ -546,7 +606,10 @@ public class QueryAsinTaskService {
}
private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
attachFileJobState(item, entity, taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()));
}
private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity, TaskFileJobEntity job) {
item.setFileReady(!blank(entity.getResultFileUrl()));
if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
@@ -1061,6 +1124,10 @@ public class QueryAsinTaskService {
return value == null || value.isBlank();
}
private static long elapsedMs(long startInclusive, long endExclusive) {
return Math.max(0L, (endExclusive - startInclusive) / 1_000_000L);
}
private String blankToNull(String value) {
return blank(value) ? null : value.trim();
}