super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

处理后台管理系统、修复BUG、处理权限

Browse Source
This commit is contained in:
super
2026-04-22 01:09:28 +08:00
parent d691e53886
commit 236b73755c
77 changed files with 5157 additions and 522 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java
+4 -1
View File
@@ -223,7 +223,10 @@ public class PriceTrackController {
example = "200"
)
@PathVariable Long taskId,
@Valid @RequestBody PriceTrackSubmitResultRequest request) {
@Valid @RequestBody PriceTrackSubmitResultRequest request,
jakarta.servlet.http.HttpServletResponse response) {
response.setCharacterEncoding(StandardCharsets.UTF_8.name());
response.setContentType("application/json;charset=UTF-8");
priceTrackTaskService.submitResult(taskId, request);
return ApiResponse.success(null);
}
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackCreateTaskVo.java
+3
View File
@@ -20,4 +20,7 @@ public class PriceTrackCreateTaskVo {
@Schema(description = "parsed asin rows by country")
private Map<String, List<Map<String, String>>> asinRowsByCountry;
@Schema(description = "minimum price grouped by country code and asin")
private Map<String, Map<String, String>> minimumPriceByCountryAndAsin;
}
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackMatchShopsVo.java
+3
View File
@@ -20,6 +20,9 @@ public class PriceTrackMatchShopsVo {
@Schema(description = "Parsed asin rows grouped by country code")
private Map<String, List<Map<String, String>>> asinRowsByCountry;
@Schema(description = "Minimum price grouped by country code and asin")
private Map<String, Map<String, String>> minimumPriceByCountryAndAsin;
@Data
@Schema(description = "Single matched shop item")
public static class PriceTrackShopQueueItem {
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackService.java
+1
View File
@@ -158,6 +158,7 @@ public class PriceTrackService {
PriceTrackMatchShopsVo vo = new PriceTrackMatchShopsVo();
vo.setSkipAsinsByCountry(skipAsinsByCountry);
vo.setAsinRowsByCountry(asinRowsByCountry);
vo.setMinimumPriceByCountryAndAsin(priceTrackTaskService.buildMinimumPriceLookupForMatch(asinRowsByCountry));
for (String shopName : ordered) {
vo.getItems().add(matchOneShop(shopName, skipAsinsByCountry));
}
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskCacheService.java
+81 -1
View File
@@ -2,24 +2,29 @@ package com.nanri.aiimage.modules.pricetrack.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequest;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
public class PriceTrackTaskCacheService {
private static final long HEARTBEAT_TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public void touchTaskHeartbeat(Long taskId) {
if (taskId == null || taskId <= 0) {
@@ -112,8 +117,72 @@ public class PriceTrackTaskCacheService {
if (taskId == null || taskId <= 0) {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildShopPayloadKey(taskId));
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
}
public void saveTaskCache(FileTaskEntity task) {
if (task == null || task.getId() == null) {
return;
}
long now = System.currentTimeMillis();
taskEntityLocalCache.put(task.getId(), new LocalTaskEntityCacheEntry(
now,
objectMapper.convertValue(task, FileTaskEntity.class)
));
try {
stringRedisTemplate.opsForValue().set(
buildTaskEntityKey(task.getId()),
objectMapper.writeValueAsString(task),
Duration.ofHours(HEARTBEAT_TTL_HOURS)
);
} catch (Exception ignored) {
}
}
public Map<Long, FileTaskEntity> getTaskCacheBatch(java.util.List<Long> taskIds) {
Map<Long, FileTaskEntity> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
java.util.List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
long now = System.currentTimeMillis();
java.util.List<Long> missingIds = new ArrayList<>();
for (Long taskId : normalized) {
LocalTaskEntityCacheEntry cached = taskEntityLocalCache.get(taskId);
if (isLocalCacheFresh(cached, now)) {
result.put(taskId, objectMapper.convertValue(cached.task(), FileTaskEntity.class));
} else {
missingIds.add(taskId);
}
}
if (missingIds.isEmpty()) {
return result;
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);
String val = values != null && i < values.size() ? values.get(i) : null;
if (val == null || val.isBlank()) {
continue;
}
try {
FileTaskEntity task = objectMapper.readValue(val, FileTaskEntity.class);
result.put(taskId, task);
taskEntityLocalCache.put(taskId, new LocalTaskEntityCacheEntry(now, task));
} catch (Exception ignored) {
}
}
return result;
}
private String buildShopPayloadKey(Long taskId) {
@@ -123,4 +192,15 @@ public class PriceTrackTaskCacheService {
private String buildTaskHeartbeatKey(Long taskId) {
return "price-track:task:heartbeat:" + taskId;
}
private String buildTaskEntityKey(Long taskId) {
return "price-track:task:entity:" + taskId;
}
private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) {
return cached != null
&& now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis());
}
private record LocalTaskEntityCacheEntry(long cachedAtMillis, FileTaskEntity task) {}
}
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java
+111 -11
View File
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode;
import com.nanri.aiimage.modules.pricetrack.mapper.PriceTrackShopCandidateMapper;
@@ -64,6 +65,62 @@ public class PriceTrackTaskService {
private final PriceTrackExcelAssemblyService excelAssemblyService;
private final PriceTrackTaskCacheService priceTrackTaskCacheService;
private final PriceTrackLoopRunService priceTrackLoopRunService;
private final TaskPressureProperties taskPressureProperties;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
FileTaskEntity cached = cachedTasks.get(taskId);
if (cached != null) {
return cached;
}
FileTaskEntity dbTask = fileTaskMapper.selectById(taskId);
if (dbTask != null && MODULE_TYPE.equals(dbTask.getModuleType())) {
priceTrackTaskCacheService.saveTaskCache(dbTask);
}
return dbTask;
}
private Map<Long, FileTaskEntity> loadTaskMapByIds(List<Long> taskIds) {
Map<Long, FileTaskEntity> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
List<Long> normalizedTaskIds = taskIds.stream()
.filter(taskId -> taskId != null && taskId > 0)
.distinct()
.toList();
if (normalizedTaskIds.isEmpty()) {
return result;
}
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(normalizedTaskIds);
result.putAll(cachedTasks);
List<Long> missingTaskIds = normalizedTaskIds.stream()
.filter(taskId -> !cachedTasks.containsKey(taskId))
.toList();
for (FileTaskEntity dbTask : selectTasksByIdsInBatches(missingTaskIds)) {
if (dbTask == null || !MODULE_TYPE.equals(dbTask.getModuleType())) {
continue;
}
result.put(dbTask.getId(), dbTask);
if ("RUNNING".equals(dbTask.getStatus())) {
priceTrackTaskCacheService.saveTaskCache(dbTask);
}
}
return result;
}
private List<FileTaskEntity> selectTasksByIdsInBatches(List<Long> taskIds) {
List<FileTaskEntity> tasks = new ArrayList<>();
if (taskIds == null || taskIds.isEmpty()) {
return tasks;
}
int batchSize = Math.max(1, taskPressureProperties.getDbSelectBatchSize());
for (int start = 0; start < taskIds.size(); start += batchSize) {
int end = Math.min(start + batchSize, taskIds.size());
tasks.addAll(fileTaskMapper.selectBatchIds(taskIds.subList(start, end)));
}
return tasks;
}
public PriceTrackDashboardVo dashboard(Long userId) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
@@ -104,7 +161,7 @@ public class PriceTrackTaskService {
.distinct()
.toList();
if (!taskIds.isEmpty()) {
List<FileTaskEntity> tasks = fileTaskMapper.selectBatchIds(taskIds);
List<FileTaskEntity> tasks = selectTasksByIdsInBatches(taskIds);
for (FileTaskEntity t : tasks) {
if (t != null) statusByTaskId.put(t.getId(), t.getStatus());
}
@@ -134,7 +191,7 @@ public class PriceTrackTaskService {
@Transactional
public void deleteTask(Long taskId, Long userId) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
FileTaskEntity task = fileTaskMapper.selectById(taskId);
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
throw new BusinessException("任务不存在");
}
@@ -164,7 +221,7 @@ public class PriceTrackTaskService {
vo.setRemoved(false);
for (FileResultEntity fr : candidates) {
if (fr.getTaskId() == null) continue;
FileTaskEntity task = fileTaskMapper.selectById(fr.getTaskId());
FileTaskEntity task = loadTaskForExecution(fr.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) continue;
if (!"RUNNING".equals(task.getStatus())) continue;
fileResultMapper.deleteById(fr.getId());
@@ -178,10 +235,11 @@ public class PriceTrackTaskService {
public PriceTrackTaskBatchVo getTaskDetailsBatch(List<Long> taskIds) {
PriceTrackTaskBatchVo batch = new PriceTrackTaskBatchVo();
if (taskIds == null || taskIds.isEmpty()) return batch;
Map<Long, FileTaskEntity> taskMap = loadTaskMapByIds(taskIds);
for (Long taskId : taskIds) {
if (taskId == null || taskId <= 0) continue;
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
FileTaskEntity task = taskMap.get(taskId);
if (task == null) {
batch.getMissingTaskIds().add(taskId);
continue;
}
@@ -195,12 +253,13 @@ public class PriceTrackTaskService {
if (taskIds == null || taskIds.isEmpty()) {
return batch;
}
Map<Long, FileTaskEntity> taskMap = loadTaskMapByIds(taskIds);
for (Long taskId : taskIds) {
if (taskId == null || taskId <= 0) {
continue;
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
FileTaskEntity task = taskMap.get(taskId);
if (task == null) {
batch.getMissingTaskIds().add(taskId);
continue;
}
@@ -237,6 +296,8 @@ public class PriceTrackTaskService {
Map<String, List<Map<String, String>>> asinRowsByCountry = request.isAsinMode()
? parseAsinRowsByCountry(request.getAsinFiles(), request.getCountryCodes())
: new LinkedHashMap<>();
Map<String, Map<String, String>> minimumPriceByCountryAndAsin =
buildMinimumPriceByCountryAndAsin(asinRowsByCountry);
log.info("[price-track] createTask skipAsins countries={} asinMode={}",
skipAsinsByCountry.keySet(), request.isAsinMode());
@@ -254,6 +315,7 @@ public class PriceTrackTaskService {
task.setCreatedAt(LocalDateTime.now());
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.insert(task);
priceTrackTaskCacheService.saveTaskCache(task);
priceTrackTaskCacheService.touchTaskHeartbeat(task.getId());
if (request.getLoopRunId() != null) {
priceTrackLoopRunService.bindChildTask(request.getLoopRunId(), task.getId(), request.getRoundIndex(), request.getShopIndex());
@@ -287,6 +349,7 @@ public class PriceTrackTaskService {
ctx.put("countryCodes", request.getCountryCodes());
ctx.put("skipAsinsByCountry", skipAsinsByCountry);
ctx.put("asinRowsByCountry", asinRowsByCountry);
ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin);
ctx.put("items", uniqueItems);
ctx.put("loopRunId", request.getLoopRunId());
ctx.put("roundIndex", request.getRoundIndex());
@@ -306,6 +369,7 @@ public class PriceTrackTaskService {
vo.setItems(snapshot);
vo.setSkipAsinsByCountry(skipAsinsByCountry);
vo.setAsinRowsByCountry(asinRowsByCountry);
vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin);
return vo;
}
@@ -315,7 +379,7 @@ public class PriceTrackTaskService {
if (request == null || request.getShops() == null || request.getShops().isEmpty()) {
throw new BusinessException("shops 不能为空");
}
FileTaskEntity task = fileTaskMapper.selectById(taskId);
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
}
@@ -386,7 +450,7 @@ public class PriceTrackTaskService {
@Transactional
public boolean tryFinalizeTask(Long taskId, boolean fromCompensation) {
if (taskId == null || taskId <= 0) return false;
FileTaskEntity task = fileTaskMapper.selectById(taskId);
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) return false;
if (!"RUNNING".equals(task.getStatus())) {
log.warn("[price-track] stale finalize skipped taskId={} status={} fromCompensation={}",
@@ -490,7 +554,7 @@ public class PriceTrackTaskService {
// ---- private helpers ----
private void reconcileTaskAfterResultRemoval(Long taskId) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) return;
List<FileResultEntity> latest = fileResultMapper.selectList(
new LambdaQueryWrapper<FileResultEntity>()
@@ -552,6 +616,11 @@ public class PriceTrackTaskService {
return parseAsinRowsByCountry(asinFiles, countryCodes);
}
public Map<String, Map<String, String>> buildMinimumPriceLookupForMatch(
Map<String, List<Map<String, String>>> asinRowsByCountry) {
return buildMinimumPriceByCountryAndAsin(asinRowsByCountry);
}
private Map<String, List<Map<String, String>>> parseAsinRowsByCountry(List<String> asinFiles, List<String> countryCodes) {
Map<String, List<Map<String, String>>> merged = new LinkedHashMap<>();
if (asinFiles == null || asinFiles.isEmpty()) {
@@ -654,6 +723,37 @@ public class PriceTrackTaskService {
}
}
private Map<String, Map<String, String>> buildMinimumPriceByCountryAndAsin(
Map<String, List<Map<String, String>>> asinRowsByCountry) {
Map<String, Map<String, String>> out = new LinkedHashMap<>();
if (asinRowsByCountry == null || asinRowsByCountry.isEmpty()) {
return out;
}
for (Map.Entry<String, List<Map<String, String>>> entry : asinRowsByCountry.entrySet()) {
String countryCode = entry.getKey();
if (countryCode == null || countryCode.isBlank()) {
continue;
}
Map<String, String> minimumPriceByAsin = new LinkedHashMap<>();
List<Map<String, String>> rows = entry.getValue();
if (rows != null) {
for (Map<String, String> row : rows) {
if (row == null || row.isEmpty()) {
continue;
}
String asin = normalizeCellText(row.get("asin")).toUpperCase(Locale.ROOT);
if (asin.isBlank()) {
continue;
}
String minimumPrice = normalizeCellText(row.get("minimumPrice"));
minimumPriceByAsin.put(asin, minimumPrice);
}
}
out.put(countryCode, minimumPriceByAsin);
}
return out;
}
private String resolveCountryCode(String sheetName, List<String> countryCodes, boolean singleSheetFallback) {
String normalized = normalizeHeaderText(sheetName).toUpperCase(Locale.ROOT);
for (ProductRiskCountryCode code : ProductRiskCountryCode.values()) {
@@ -832,7 +932,7 @@ public class PriceTrackTaskService {
}
private void mergeQueueFieldsFromRequest(PriceTrackResultItemVo vo, Long taskId) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || task.getRequestJson() == null || task.getRequestJson().isBlank()) return;
try {
Map<String, Object> payload = objectMapper.readValue(task.getRequestJson(), new TypeReference<Map<String, Object>>() {});