super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成后端架构重构等

Browse Source
This commit is contained in:
super
2026-04-23 15:25:41 +08:00
parent 46f46039ea
commit 34bc980eea
76 changed files with 3242 additions and 1111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/dto/PriceTrackSubmitResultRequest.java
+9
View File
@@ -93,5 +93,14 @@ public class PriceTrackSubmitResultRequest {
@Schema(description = "状态", example = "SUCCESS")
private String status;
@Schema(description = "Python 确认是否需要从 skip ASIN 表删除当前 ASIN", example = "true")
private Boolean deleteSkipAsin;
@Schema(description = "需要删除的 skip ASIN；不传时默认使用 asin 字段", example = "B0ABCDE123")
private String removeAsin;
@Schema(description = "删除确认原因，便于排查日志", example = "CURRENT_PRICE_BELOW_MINIMUM")
private String deleteReason;
}
}
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackCreateTaskVo.java
+3
View File
@@ -18,6 +18,9 @@ public class PriceTrackCreateTaskVo {
@Schema(description = "all skip asins grouped by country")
private Map<String, List<String>> skipAsinsByCountry;
@Schema(description = "all skip asin details grouped by country code, each item includes asin and minimumPrice")
private Map<String, List<Map<String, String>>> skipAsinDetailsByCountry;
@Schema(description = "parsed asin rows by country")
private Map<String, List<Map<String, String>>> asinRowsByCountry;
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackMatchShopsVo.java
+3
View File
@@ -17,6 +17,9 @@ public class PriceTrackMatchShopsVo {
@Schema(description = "All skip ASINs grouped by country code")
private Map<String, List<String>> skipAsinsByCountry;
@Schema(description = "All skip ASIN details grouped by country code, each item includes asin and minimumPrice")
private Map<String, List<Map<String, String>>> skipAsinDetailsByCountry;
@Schema(description = "Parsed asin rows grouped by country code")
private Map<String, List<Map<String, String>>> asinRowsByCountry;
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackExcelAssemblyService.java
+30 -18
View File
@@ -6,7 +6,7 @@ import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import java.io.File;
@@ -20,25 +20,27 @@ import java.util.Map;
public class PriceTrackExcelAssemblyService {
private static final String[] RESULT_HEADER = {
"\u5e97\u94fa\u5546\u57ce\u540d\u79f0",
"店铺商城名称",
"ASIN",
"\u4ef7\u683c",
"\u63a8\u8350\u4ef7",
"\u6700\u4f4e\u4ef7",
"\u7b2c\u4e00\u540d",
"\u7b2c\u4e8c\u540d",
"\u8d2d\u7269\u8f66\u5e97\u94fa\u540d",
"\u6539\u4ef7\u60c5\u51b5",
"\u4fee\u6539\u6b21\u6570",
"\u72b6\u6001"
"价格",
"推荐价",
"最低价",
"第一名",
"第二名",
"购物车店铺名",
"改价情况",
"修改次数",
"状态"
};
public void writeWorkbook(File outputXlsx, Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> countries) {
Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> safe = countries == null ? Map.of() : countries;
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream fos = new FileOutputStream(outputXlsx)) {
for (ProductRiskCountryCode code : ProductRiskCountryCode.values()) {
String sheetName = code.name();
Sheet sheet = wb.createSheet(sheetName);
Sheet sheet = workbook.createSheet(sheetName);
Row headerRow = sheet.createRow(0);
for (int i = 0; i < RESULT_HEADER.length; i++) {
headerRow.createCell(i).setCellValue(RESULT_HEADER[i]);
@@ -62,14 +64,18 @@ public class PriceTrackExcelAssemblyService {
row.createCell(9).setCellValue(valueOf(item.getModifyCount()));
row.createCell(10).setCellValue(valueOf(item.getStatus()));
}
for (int i = 0; i < RESULT_HEADER.length; i++) {
sheet.autoSizeColumn(i);
}
applyDefaultColumnWidths(sheet);
}
wb.write(fos);
workbook.write(fos);
} catch (Exception ex) {
log.warn("[price-track] write workbook failed: {}", ex.getMessage());
throw new BusinessException("鐢熸垚 Excel 澶辫触: " + ex.getMessage());
throw new BusinessException("generate price-track excel failed: " + ex.getMessage());
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}
@@ -105,4 +111,10 @@ public class PriceTrackExcelAssemblyService {
return value == null ? "" : value;
}
private void applyDefaultColumnWidths(Sheet sheet) {
int[] widths = {22, 18, 12, 12, 12, 20, 20, 22, 18, 12, 12};
for (int i = 0; i < widths.length; i++) {
sheet.setColumnWidth(i, widths[i] * 256);
}
}
}
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackService.java
+4
View File
@@ -148,6 +148,9 @@ public class PriceTrackService {
Map<String, List<String>> skipAsinsByCountry = asinMode
? new LinkedHashMap<>()
: skipPriceAsinService.listAllSkipAsinsByCountry();
Map<String, List<Map<String, String>>> skipAsinDetailsByCountry = asinMode
? new LinkedHashMap<>()
: skipPriceAsinService.listAllSkipAsinDetailsByCountry();
Map<String, List<Map<String, String>>> asinRowsByCountry =
!asinMode
? new LinkedHashMap<>()
@@ -157,6 +160,7 @@ public class PriceTrackService {
PriceTrackMatchShopsVo vo = new PriceTrackMatchShopsVo();
vo.setSkipAsinsByCountry(skipAsinsByCountry);
vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry);
vo.setAsinRowsByCountry(asinRowsByCountry);
vo.setMinimumPriceByCountryAndAsin(priceTrackTaskService.buildMinimumPriceLookupForMatch(asinRowsByCountry));
for (String shopName : ordered) {
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskCacheService.java
+10 -50
View File
@@ -1,10 +1,10 @@
package com.nanri.aiimage.modules.pricetrack.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequest;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -20,10 +20,12 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor
public class PriceTrackTaskCacheService {
private static final String MODULE_TYPE = "PRICE_TRACK";
private static final long HEARTBEAT_TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public void touchTaskHeartbeat(Long taskId) {
@@ -52,65 +54,27 @@ public class PriceTrackTaskCacheService {
}
public PriceTrackSubmitResultRequest.ShopResult getShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return null;
}
Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey);
if (!(raw instanceof String json) || json.isBlank()) {
return null;
}
try {
return objectMapper.readValue(json, PriceTrackSubmitResultRequest.ShopResult.class);
} catch (Exception ex) {
throw new BusinessException("read price-track cache failed");
}
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PriceTrackSubmitResultRequest.ShopResult.class);
}
public void saveShopMergedPayload(Long taskId, String shopKey, PriceTrackSubmitResultRequest.ShopResult payload) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
return;
}
try {
stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload));
stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(HEARTBEAT_TTL_HOURS));
touchTaskHeartbeat(taskId);
} catch (Exception ex) {
throw new BusinessException("save price-track cache failed");
}
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
touchTaskHeartbeat(taskId);
}
public void removeShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return;
}
stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey);
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
}
public Map<String, PriceTrackSubmitResultRequest.ShopResult> getAllShopMergedPayload(Long taskId) {
try {
Map<Object, Object> raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId));
if (raw == null || raw.isEmpty()) {
return Map.of();
}
LinkedHashMap<String, PriceTrackSubmitResultRequest.ShopResult> out = new LinkedHashMap<>();
for (Map.Entry<Object, Object> entry : raw.entrySet()) {
if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) {
continue;
}
out.put(key, objectMapper.readValue(val, PriceTrackSubmitResultRequest.ShopResult.class));
}
return out;
} catch (Exception ex) {
throw new BusinessException("read price-track cache failed");
}
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, PriceTrackSubmitResultRequest.ShopResult.class);
}
public boolean hasAnyShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
return size != null && size > 0;
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
}
public void deleteTaskCache(Long taskId) {
@@ -118,9 +82,9 @@ public class PriceTrackTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildShopPayloadKey(taskId));
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
public void saveTaskCache(FileTaskEntity task) {
@@ -185,10 +149,6 @@ public class PriceTrackTaskCacheService {
return result;
}
private String buildShopPayloadKey(Long taskId) {
return "price-track:task:shop-payload:" + taskId;
}
private String buildTaskHeartbeatKey(Long taskId) {
return "price-track:task:heartbeat:" + taskId;
}
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java
+42 -3
View File
@@ -206,7 +206,7 @@ public class PriceTrackTaskService {
@Transactional
public PriceTrackPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 涓嶈兘涓虹┖");
if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 不能为空");
String norm = ziniaoShopSwitchService.normalizeShopName(shopName);
if (norm.isBlank()) throw new BusinessException("店铺名称无效");
@@ -293,6 +293,9 @@ public class PriceTrackTaskService {
Map<String, List<String>> skipAsinsByCountry = request.isAsinMode()
? new LinkedHashMap<>()
: skipPriceAsinService.listAllSkipAsinsByCountry();
Map<String, List<Map<String, String>>> skipAsinDetailsByCountry = request.isAsinMode()
? new LinkedHashMap<>()
: skipPriceAsinService.listAllSkipAsinDetailsByCountry();
Map<String, List<Map<String, String>>> asinRowsByCountry = request.isAsinMode()
? parseAsinRowsByCountry(request.getAsinFiles(), request.getCountryCodes())
: new LinkedHashMap<>();
@@ -348,6 +351,7 @@ public class PriceTrackTaskService {
ctx.put("asinMode", request.isAsinMode());
ctx.put("countryCodes", request.getCountryCodes());
ctx.put("skipAsinsByCountry", skipAsinsByCountry);
ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry);
ctx.put("asinRowsByCountry", asinRowsByCountry);
ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin);
ctx.put("items", uniqueItems);
@@ -368,6 +372,7 @@ public class PriceTrackTaskService {
vo.setTaskId(task.getId());
vo.setItems(snapshot);
vo.setSkipAsinsByCountry(skipAsinsByCountry);
vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry);
vo.setAsinRowsByCountry(asinRowsByCountry);
vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin);
return vo;
@@ -419,6 +424,7 @@ public class PriceTrackTaskService {
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
continue;
}
handleSkipAsinDeletionSignals(shopKey, payload);
PriceTrackSubmitResultRequest.ShopResult merged = mergeShopPayload(taskId, shopKey, payload);
if (!Boolean.TRUE.equals(merged.getSuccess())) {
continue;
@@ -833,8 +839,8 @@ public class PriceTrackTaskService {
if (value == null) {
return "";
}
return value.replace("\ufeff", "")
.replace("\u3000", " ")
return value.replace("", "")
.replace("　", " ")
.replace("\r", " ")
.replace("\n", " ")
.trim()
@@ -1075,6 +1081,11 @@ public class PriceTrackTaskService {
merged.setPriceChangeStatus(firstNonBlank(incoming.getPriceChangeStatus(), merged.getPriceChangeStatus()));
merged.setModifyCount(firstNonBlank(incoming.getModifyCount(), merged.getModifyCount()));
merged.setStatus(firstNonBlank(incoming.getStatus(), merged.getStatus()));
if (incoming.getDeleteSkipAsin() != null) {
merged.setDeleteSkipAsin(incoming.getDeleteSkipAsin());
}
merged.setRemoveAsin(firstNonBlank(incoming.getRemoveAsin(), merged.getRemoveAsin()));
merged.setDeleteReason(firstNonBlank(incoming.getDeleteReason(), merged.getDeleteReason()));
return merged;
}
private PriceTrackSubmitResultRequest.AsinResult cloneRow(PriceTrackSubmitResultRequest.AsinResult row) {
@@ -1093,8 +1104,36 @@ public class PriceTrackTaskService {
out.setPriceChangeStatus(row.getPriceChangeStatus());
out.setModifyCount(row.getModifyCount());
out.setStatus(row.getStatus());
out.setDeleteSkipAsin(row.getDeleteSkipAsin());
out.setRemoveAsin(row.getRemoveAsin());
out.setDeleteReason(row.getDeleteReason());
return out;
}
private void handleSkipAsinDeletionSignals(String shopKey,
PriceTrackSubmitResultRequest.ShopResult payload) {
if (payload == null || payload.getCountries() == null || payload.getCountries().isEmpty()) {
return;
}
for (Map.Entry<String, List<PriceTrackSubmitResultRequest.AsinResult>> entry : payload.getCountries().entrySet()) {
String countryCode = entry.getKey();
List<PriceTrackSubmitResultRequest.AsinResult> rows = entry.getValue();
if (countryCode == null || countryCode.isBlank() || rows == null || rows.isEmpty()) {
continue;
}
for (PriceTrackSubmitResultRequest.AsinResult row : rows) {
if (row == null || !Boolean.TRUE.equals(row.getDeleteSkipAsin())) {
continue;
}
String targetAsin = firstNonBlank(row.getRemoveAsin(), row.getAsin());
if (targetAsin == null || targetAsin.isBlank()) {
continue;
}
boolean removed = skipPriceAsinService.removeByShopCountryAndAsin(shopKey, countryCode, targetAsin);
log.info("[price-track] skip asin delete signal taskShop={} country={} asin={} removed={} reason={}",
shopKey, countryCode, targetAsin, removed, row.getDeleteReason());
}
}
}
private int countPayloadRows(PriceTrackSubmitResultRequest.ShopResult payload) {
if (payload == null) {
return 0;