super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://git.aishufu.top/super/crawler-plugin

Browse Source
This commit is contained in:
黄自达
2026-08-23 22:06:11 +08:00
parent b00ea71c1b 159d52194c
commit 5a750be066
32 changed files with 1239 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/config/TaskPressureProperties.java
+2
View File
@@ -7,6 +7,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "aiimage.task-pressure")
public class TaskPressureProperties {
private long localTaskEntityCacheMillis = 3000;
// 本地文件缓存有效时长，超过该时长视为过期、强制回查 DB，避免陈旧 RUNNING 被复活
private long localTaskEntityFileCacheMillis = 60000;
private int dbSelectBatchSize = 200;
private long scopePayloadFlushIntervalMillis = 15000;
private long scopePayloadBufferRetentionHours = 24;
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java
+22 -5
View File
@@ -42,6 +42,7 @@ import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBo
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.YearMonth;
import java.time.format.DateTimeFormatter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -58,6 +59,8 @@ public class DedupeTotalDataController {
private static final String DEDUPE_TOTAL_DATA_ROUTE_PATH = "dedupe-total-data";
private static final DateTimeFormatter EXPORT_FILENAME_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
private static final String XLSX_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String ZIP_CONTENT_TYPE = "application/zip";
private final DedupeTotalDataService dedupeTotalDataService;
private final AdminAuthSupport adminAuthSupport;
@@ -91,7 +94,7 @@ public class DedupeTotalDataController {
}
@GetMapping("/export")
@Operation(summary = "导出总数据", description = "按上传用户名和创建日期导出当前用户可访问的总数据。")
@Operation(summary = "导出总数据", description = "按上传用户名和创建日期导出当前用户可访问的总数据；跨自然月时按月生成 XLSX 并打包为 ZIP。")
public ResponseEntity<StreamingResponseBody> export(
@Parameter(description = "用户名模糊搜索关键字") @RequestParam(required = false) String username,
@Parameter(description = "开始日期（包含）")
@@ -100,16 +103,30 @@ public class DedupeTotalDataController {
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
HttpServletRequest request) {
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
throw new BusinessException(400, "invalid export date range");
}
RequestOperator operator = requireDedupeTotalDataAccess(request);
String filename = "dedupe-total-data-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER) + ".xlsx";
StreamingResponseBody body = outputStream -> dedupeTotalDataService.writeExport(
outputStream, username, startDate, endDate, groupId, operator.id());
boolean monthlyZip = isCrossMonthRange(startDate, endDate);
String filename = "dedupe-total-data-" + LocalDateTime.now().format(EXPORT_FILENAME_FORMATTER)
+ (monthlyZip ? ".zip" : ".xlsx");
StreamingResponseBody body = monthlyZip
? outputStream -> dedupeTotalDataService.writeMonthlyZipExport(
outputStream, username, startDate, endDate, groupId, operator.id())
: outputStream -> dedupeTotalDataService.writeExport(
outputStream, username, startDate, endDate, groupId, operator.id());
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename))
.contentType(MediaType.parseMediaType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"))
.contentType(MediaType.parseMediaType(monthlyZip ? ZIP_CONTENT_TYPE : XLSX_CONTENT_TYPE))
.body(body);
}
private boolean isCrossMonthRange(LocalDate startDate, LocalDate endDate) {
return startDate != null
&& endDate != null
&& !YearMonth.from(startDate).equals(YearMonth.from(endDate));
}
@PostMapping
@Operation(summary = "新增总数据", description = "新增一条数据去重总数据。")
@ApiResponses({
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java
+82 -3
View File
@@ -40,6 +40,7 @@ import java.io.OutputStream;
import java.nio.file.Files;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.YearMonth;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
@@ -53,6 +54,9 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Supplier;
import java.util.zip.Deflater;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Service
@RequiredArgsConstructor
@@ -157,12 +161,67 @@ public class DedupeTotalDataService {
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, scope);
}
public void writeMonthlyZipExport(OutputStream outputStream,
String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
Long operatorId) {
if (outputStream == null) {
throw new BusinessException("export output stream cannot be null");
}
if (startDate == null || endDate == null || YearMonth.from(startDate).equals(YearMonth.from(endDate))) {
writeExport(outputStream, username, startDate, endDate, groupId, operatorId);
return;
}
if (startDate.isAfter(endDate)) {
throw new BusinessException("invalid export date range");
}
String safeUsername = username == null ? "" : username.trim();
AccessScope scope = resolveAccessScope(operatorId);
boolean groupPrevalidated = groupId != null && groupId > 0;
if (groupPrevalidated) {
resolveAccessibleGroup(groupId, scope);
}
try (ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream)) {
zipOutputStream.setLevel(Deflater.NO_COMPRESSION);
YearMonth currentMonth = YearMonth.from(startDate);
YearMonth endMonth = YearMonth.from(endDate);
while (!currentMonth.isAfter(endMonth)) {
LocalDate monthStart = currentMonth.atDay(1);
LocalDate monthEnd = currentMonth.atEndOfMonth();
LocalDate entryStartDate = startDate.isAfter(monthStart) ? startDate : monthStart;
LocalDate entryEndDate = endDate.isBefore(monthEnd) ? endDate : monthEnd;
zipOutputStream.putNextEntry(new ZipEntry(monthlyExportEntryName(currentMonth)));
buildExportWorkbook(zipOutputStream, safeUsername, entryStartDate, entryEndDate,
groupId, scope, groupPrevalidated);
zipOutputStream.closeEntry();
currentMonth = currentMonth.plusMonths(1);
}
zipOutputStream.finish();
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("export dedupe total data failed");
}
}
private void buildExportWorkbook(OutputStream outputStream,
String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
AccessScope scope) {
buildExportWorkbook(outputStream, username, startDate, endDate, groupId, scope, false);
}
private void buildExportWorkbook(OutputStream outputStream,
String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
AccessScope scope,
boolean groupPrevalidated) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(100)) {
Sheet sheet = workbook.createSheet("DedupeTotalData");
Row header = sheet.createRow(0);
@@ -175,7 +234,7 @@ public class DedupeTotalDataService {
int rowIndex = 1;
while (true) {
LambdaQueryWrapper<DedupeTotalDataEntity> pageQuery = buildExportQuery(
username, startDate, endDate, groupId, scope);
username, startDate, endDate, groupId, scope, groupPrevalidated);
if (lastId != null) {
pageQuery.lt(DedupeTotalDataEntity::getId, lastId);
}
@@ -221,6 +280,15 @@ public class DedupeTotalDataService {
LocalDate endDate,
Long groupId,
AccessScope scope) {
return buildExportQuery(username, startDate, endDate, groupId, scope, false);
}
private LambdaQueryWrapper<DedupeTotalDataEntity> buildExportQuery(String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
AccessScope scope,
boolean groupPrevalidated) {
String safeUsername = username == null ? "" : username.trim();
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
@@ -229,7 +297,7 @@ public class DedupeTotalDataService {
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
endDate == null ? null : endDate.plusDays(1).atStartOfDay())
.orderByDesc(DedupeTotalDataEntity::getId);
applyGroupScope(query, scope, groupId);
applyGroupScope(query, scope, groupId, groupPrevalidated);
return query;
}
@@ -237,6 +305,10 @@ public class DedupeTotalDataService {
return value == null ? "" : value.format(EXPORT_TIME_FORMATTER);
}
private String monthlyExportEntryName(YearMonth month) {
return "dedupe-total-data-" + month + ".xlsx";
}
@Transactional
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request, Long operatorId) {
AdminUserEntity uploader = getOperator(operatorId);
@@ -885,8 +957,15 @@ public class DedupeTotalDataService {
private void applyGroupScope(LambdaQueryWrapper<DedupeTotalDataEntity> query,
AccessScope scope, Long groupId) {
applyGroupScope(query, scope, groupId, false);
}
private void applyGroupScope(LambdaQueryWrapper<DedupeTotalDataEntity> query,
AccessScope scope, Long groupId, boolean groupPrevalidated) {
if (groupId != null && groupId > 0) {
resolveAccessibleGroup(groupId, scope);
if (!groupPrevalidated) {
resolveAccessibleGroup(groupId, scope);
}
query.eq(DedupeTotalDataEntity::getGroupId, groupId);
return;
}
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java
+101
View File
@@ -815,11 +815,112 @@ public class PriceTrackTaskService {
private void enqueueResultFileAssembly(FileResultEntity result,
String shopKey,
PriceTrackSubmitResultRequest.ShopResult payload) {
applyServerModifyCounts(result.getTaskId(), payload);
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
markResultFilePending(result, shopKey, payload);
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
}
private void applyServerModifyCounts(
Long taskId,
PriceTrackSubmitResultRequest.ShopResult payload) {
if (payload == null || payload.getCountries() == null || payload.getCountries().isEmpty()) {
return;
}
Map<String, Integer> baselineByAsin = buildModifyCountBaseline(taskId);
for (Map.Entry<String, List<PriceTrackSubmitResultRequest.AsinResult>> entry : payload.getCountries().entrySet()) {
String countryCode = normalizeCountryCode(entry.getKey());
if (entry.getValue() == null) {
continue;
}
for (PriceTrackSubmitResultRequest.AsinResult row : entry.getValue()) {
if (row == null) {
continue;
}
String asin = normalizeAsin(row.getAsin());
int baseline = baselineByAsin.getOrDefault(buildModifyCountKey(countryCode, asin), 0);
row.setModifyCount(String.valueOf(addOneIfChanged(baseline, row)));
}
}
}
private Map<String, Integer> buildModifyCountBaseline(Long taskId) {
Map<String, List<Map<String, String>>> originalRows = loadTaskAsinRowsPayload(taskId);
if (originalRows == null || originalRows.isEmpty()) {
return Map.of();
}
Map<String, Integer> out = new LinkedHashMap<>();
for (Map.Entry<String, List<Map<String, String>>> entry : originalRows.entrySet()) {
String countryCode = normalizeCountryCode(entry.getKey());
if (entry.getValue() == null) {
continue;
}
for (Map<String, String> row : entry.getValue()) {
if (row == null) {
continue;
}
String asin = normalizeAsin(row.get("asin"));
if (asin.isBlank()) {
continue;
}
out.putIfAbsent(
buildModifyCountKey(countryCode, asin),
parseModifyCountBaseline(row.get("modifyCount")));
}
}
return out;
}
private String buildModifyCountKey(String countryCode, String asin) {
return countryCode + "|" + asin;
}
private String normalizeCountryCode(String countryCode) {
return countryCode == null ? "" : countryCode.trim().toUpperCase(Locale.ROOT);
}
private String normalizeAsin(String asin) {
return asin == null ? "" : asin.trim().toUpperCase(Locale.ROOT);
}
private int parseModifyCountBaseline(String value) {
String normalized = normalizeCellText(value);
if (normalized.isBlank()) {
return 0;
}
int decimalPoint = normalized.indexOf('.');
if (decimalPoint >= 0) {
for (int i = decimalPoint + 1; i < normalized.length(); i++) {
if (normalized.charAt(i) != '0') {
return 0;
}
}
normalized = normalized.substring(0, decimalPoint);
}
try {
long parsed = Long.parseLong(normalized);
if (parsed <= 0) {
return 0;
}
return parsed > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) parsed;
} catch (NumberFormatException ex) {
return 0;
}
}
private int addOneIfChanged(int baseline, PriceTrackSubmitResultRequest.AsinResult row) {
if (!isPriceUpdateSuccessStatus(row.getPriceChangeStatus())) {
return baseline;
}
return baseline >= Integer.MAX_VALUE ? Integer.MAX_VALUE : baseline + 1;
}
private boolean isPriceUpdateSuccessStatus(String value) {
String normalized = normalizeCellText(value);
return "\u6539\u4ef7\u6210\u529f".equals(normalized)
|| "UPDATED".equalsIgnoreCase(normalized);
}
private void markResultFilePending(FileResultEntity result,
String shopKey,
PriceTrackSubmitResultRequest.ShopResult payload) {
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java
+34 -1
View File
@@ -618,11 +618,15 @@ public class ShopDataCrawlTaskService {
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
List<FileResultEntity> taskRows = listTaskRows(taskId);
try (DailyLockSet dailyLocks = acquireDailyLocks(task.getUserId(), taskRows)) {
ensureDailySyncCompletedBeforeDelete(taskRows);
Set<Long> removedResultIds = taskRows.stream()
.map(FileResultEntity::getId)
.filter(id -> id != null && id > 0)
.collect(java.util.stream.Collectors.toSet());
DailyDeletionResult dailyResult = prepareDailyForDeletion(removedResultIds);
// A task deletion is only a frontend task-record cleanup. The daily
// workbook is an independent backend aggregate and must not roll
// back when its source task is removed.
DailyDeletionResult dailyResult = preserveDailyForTaskDeletion(removedResultIds);
registerUploadedObjectRollback(dailyResult.uploadedObjectKeys());
List<String> resultFileUrls = new ArrayList<>(dailyResult.obsoleteObjectKeys());
resultFileUrls.addAll(taskRows.stream()
@@ -642,6 +646,24 @@ public class ShopDataCrawlTaskService {
}
}
private void ensureDailySyncCompletedBeforeDelete(List<FileResultEntity> taskRows) {
if (taskRows == null || taskRows.isEmpty()) {
throw new BusinessException("后台店铺数据尚未完成同步，暂不能删除任务");
}
for (FileResultEntity row : taskRows) {
if (row == null || !isResultFinished(row)) {
throw new BusinessException("后台店铺数据尚未完成同步，暂不能删除任务");
}
if (!Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
continue;
}
if (!taskFileJobService.hasSuccessfulAssembleJob(row.getTaskId(), MODULE_TYPE, row.getId())
|| dailyFileService.findMembersByResultId(row.getId()).isEmpty()) {
throw new BusinessException("后台店铺数据尚未完成同步，暂不能删除任务");
}
}
}
@Transactional
public void deleteHistory(Long resultId, Long userId) {
validateUserId(userId);
@@ -756,6 +778,17 @@ public class ShopDataCrawlTaskService {
}
}
private DailyDeletionResult preserveDailyForTaskDeletion(Set<Long> removedResultIds) {
if (removedResultIds == null || removedResultIds.isEmpty()) {
return new DailyDeletionResult(List.of(), List.of());
}
// Keep the already assembled daily object and its row count intact. Only
// remove task membership links so the deleted task is not retained as a
// frontend history record and the next crawl can append normally.
dailyFileService.deleteMembersForResults(removedResultIds);
return new DailyDeletionResult(List.of(), List.of());
}
private List<DailyMemberData> loadDailyMemberData(ShopDataCrawlDailyFileEntity dailyFile,
Set<Long> removedResultIds) {
List<ShopDataCrawlDailyMemberEntity> memberRows = dailyFileService.listMembers(dailyFile.getId());
backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/dto/ShopManageCreateRequest.java
+6 -1
View File
@@ -1,8 +1,9 @@
package com.nanri.aiimage.modules.shopkey.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Size;
import lombok.Data;
@Data
@@ -20,6 +21,10 @@ public class ShopManageCreateRequest {
@NotBlank(message = "商城名称不能为空")
private String mallName;
@Schema(description = "自动化账号")
@Size(max = 128, message = "自动化账号长度不能超过128个字符")
private String znUsername;
@Schema(description = "登录账号", requiredMode = Schema.RequiredMode.REQUIRED)
@NotBlank(message = "账号不能为空")
private String account;
backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/dto/ShopManageUpdateRequest.java
+6 -1
View File
@@ -1,8 +1,9 @@
package com.nanri.aiimage.modules.shopkey.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Size;
import lombok.Data;
@Data
@@ -20,6 +21,10 @@ public class ShopManageUpdateRequest {
@NotBlank(message = "商城名称不能为空")
private String mallName;
@Schema(description = "自动化账号")
@Size(max = 128, message = "自动化账号长度不能超过128个字符")
private String znUsername;
@Schema(description = "登录账号", requiredMode = Schema.RequiredMode.REQUIRED)
@NotBlank(message = "账号不能为空")
private String account;
backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/ShopKeyEntity.java
+1
View File
@@ -18,6 +18,7 @@ public class ShopKeyEntity {
private String remarkName;
private String ziniaoAccountName;
private String ziniaoToken;
private String ziniaoTokenHash;
private String ipWhitelistStatus;
@TableField(updateStrategy = FieldStrategy.ALWAYS)
private LocalDateTime ipWhitelistCheckedAt;
backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/ShopManageEntity.java
+1
View File
@@ -21,6 +21,7 @@ public class ShopManageEntity {
private Long createdById;
@TableField("mall_name")
private String mallName;
private String znUsername;
private String account;
private String password;
private LocalDateTime createdAt;
backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/vo/ShopManageCredentialVo.java
+1
View File
@@ -9,6 +9,7 @@ public class ShopManageCredentialVo {
private String groupName;
private String shopName;
private String mallName;
private String znUsername;
private String account;
private String password;
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/vo/ShopManageItemVo.java
+1
View File
@@ -12,6 +12,7 @@ public class ShopManageItemVo {
private String groupName;
private String shopName;
private String mallName;
private String znUsername;
private String account;
private String password;
private String passwordMasked;
backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopKeyService.java
+65 -5
View File
@@ -11,9 +11,12 @@ import com.nanri.aiimage.modules.shopkey.model.vo.ShopKeyPageVo;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.List;
@Service
@@ -47,13 +50,16 @@ public class ShopKeyService {
@Transactional
public ShopKeyItemVo create(ShopKeyCreateRequest request) {
String ziniaoAccountName = normalizeRequired(request.getZiniaoAccountName(), "紫鸟账号名称不能为空");
String ziniaoToken = normalizeRequired(request.getZiniaoToken(), "紫鸟令牌不能为空");
String ziniaoToken = normalizeToken(request.getZiniaoToken());
String ziniaoTokenHash = hashToken(ziniaoToken);
ensureTokenAvailable(ziniaoTokenHash, null);
ShopKeyEntity entity = new ShopKeyEntity();
entity.setRemarkName(normalizeOptional(request.getRemarkName()));
entity.setZiniaoAccountName(ziniaoAccountName);
entity.setZiniaoToken(ziniaoToken);
entity.setZiniaoTokenHash(ziniaoTokenHash);
entity.setIpWhitelistStatus(IP_WHITELIST_STATUS_UNKNOWN);
shopKeyMapper.insert(entity);
insertOrThrowDuplicateToken(entity);
triggerShopIndexRefresh();
return toItemVo(getById(entity.getId()));
}
@@ -62,17 +68,20 @@ public class ShopKeyService {
public ShopKeyItemVo update(Long id, ShopKeyUpdateRequest request) {
ShopKeyEntity entity = getById(id);
String ziniaoAccountName = normalizeRequired(request.getZiniaoAccountName(), "紫鸟账号名称不能为空");
String ziniaoToken = normalizeRequired(request.getZiniaoToken(), "紫鸟令牌不能为空");
boolean tokenChanged = !ziniaoToken.equals(entity.getZiniaoToken());
String ziniaoToken = normalizeToken(request.getZiniaoToken());
String ziniaoTokenHash = hashToken(ziniaoToken);
ensureTokenAvailable(ziniaoTokenHash, id);
boolean tokenChanged = !ziniaoTokenHash.equals(entity.getZiniaoTokenHash());
entity.setRemarkName(normalizeOptional(request.getRemarkName()));
entity.setZiniaoAccountName(ziniaoAccountName);
entity.setZiniaoToken(ziniaoToken);
entity.setZiniaoTokenHash(ziniaoTokenHash);
if (tokenChanged) {
entity.setIpWhitelistStatus(IP_WHITELIST_STATUS_UNKNOWN);
entity.setIpWhitelistCheckedAt(null);
entity.setIpWhitelistMessage(null);
}
shopKeyMapper.updateById(entity);
updateOrThrowDuplicateToken(entity);
triggerShopIndexRefresh();
return toItemVo(getById(id));
}
@@ -125,4 +134,55 @@ public class ShopKeyService {
log.warn("[ziniao-index] refresh trigger failed after shop key change: {}", ex.getMessage());
}
}
private void ensureTokenAvailable(String tokenHash, Long currentId) {
ShopKeyEntity existing = shopKeyMapper.selectOne(new LambdaQueryWrapper<ShopKeyEntity>()
.eq(ShopKeyEntity::getZiniaoTokenHash, tokenHash)
.ne(currentId != null, ShopKeyEntity::getId, currentId)
.last("LIMIT 1"));
if (existing != null) {
throw new BusinessException("紫鸟令牌已存在，不能重复维护");
}
}
private void insertOrThrowDuplicateToken(ShopKeyEntity entity) {
try {
shopKeyMapper.insert(entity);
} catch (DuplicateKeyException ex) {
throw new BusinessException("紫鸟令牌已存在，不能重复维护", ex);
}
}
private void updateOrThrowDuplicateToken(ShopKeyEntity entity) {
try {
shopKeyMapper.updateById(entity);
} catch (DuplicateKeyException ex) {
throw new BusinessException("紫鸟令牌已存在，不能重复维护", ex);
}
}
private String normalizeToken(String value) {
String normalized = normalizeRequired(value, "紫鸟令牌不能为空");
if (normalized.regionMatches(true, 0, "Bearer ", 0, 7)) {
normalized = normalized.substring(7).trim();
}
if (normalized.isEmpty()) {
throw new BusinessException("紫鸟令牌不能为空");
}
return normalized;
}
private String hashToken(String token) {
try {
byte[] digest = MessageDigest.getInstance("SHA-256")
.digest(token.getBytes(StandardCharsets.UTF_8));
StringBuilder result = new StringBuilder(digest.length * 2);
for (byte value : digest) {
result.append(String.format("%02x", value));
}
return result.toString();
} catch (Exception ex) {
throw new BusinessException("紫鸟令牌指纹生成失败");
}
}
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopManageService.java
+10
View File
@@ -94,6 +94,7 @@ public class ShopManageService {
entity.setShopName(shopName);
entity.setCreatedById(createdById);
entity.setMallName(mallName);
entity.setZnUsername(normalizeOptional(request.getZnUsername()));
entity.setAccount(normalizeRequired(request.getAccount(), "账号不能为空"));
entity.setPassword(shopCredentialCryptoService.encrypt(normalizeRequired(request.getPassword(), "密码不能为空")));
shopManageMapper.insert(entity);
@@ -114,6 +115,9 @@ public class ShopManageService {
entity.setGroupName(group.getGroupName());
entity.setShopName(shopName);
entity.setMallName(mallName);
if (request.getZnUsername() != null) {
entity.setZnUsername(normalizeOptional(request.getZnUsername()));
}
entity.setAccount(normalizeRequired(request.getAccount(), "账号不能为空"));
entity.setPassword(shopCredentialCryptoService.encrypt(normalizeRequired(request.getPassword(), "密码不能为空")));
shopManageMapper.updateById(entity);
@@ -141,6 +145,7 @@ public class ShopManageService {
vo.setGroupId(entity.getGroupId());
vo.setShopName(entity.getShopName());
vo.setMallName(entity.getMallName());
vo.setZnUsername(entity.getZnUsername());
vo.setAccount(entity.getAccount());
vo.setPassword(shopCredentialCryptoService.decrypt(entity.getPassword()));
try {
@@ -195,6 +200,10 @@ public class ShopManageService {
return normalized;
}
private String normalizeOptional(String value) {
return value == null ? "" : value.trim();
}
private Long normalizePositiveId(Long value, String message) {
if (value == null || value <= 0) {
throw new BusinessException(message);
@@ -221,6 +230,7 @@ public class ShopManageService {
vo.setGroupName(groupName == null ? "" : groupName);
vo.setShopName(entity.getShopName());
vo.setMallName(entity.getMallName());
vo.setZnUsername(entity.getZnUsername());
vo.setAccount(entity.getAccount());
String masked = entity.getPassword() == null || entity.getPassword().isBlank() ? "" : "******";
vo.setPassword(masked);
backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java
+16 -1
View File
@@ -221,9 +221,24 @@ public class SkipPriceAsinService {
entity.setGroupId(group.getId());
entity.setShopName(shopName);
SkipPriceAsinEntity firstExisting = null;
boolean acceptedAny = false;
for (Map.Entry<String, String> entry : countryAsinMap.entrySet()) {
String country = entry.getKey();
setCountryData(entity, country, entry.getValue(), countryMinimumPriceMap.get(country));
String asin = entry.getValue();
SkipPriceAsinEntity existing = findCountryAsin(group.getId(), shopName, country, asin);
if (existing != null) {
if (firstExisting == null) {
firstExisting = existing;
}
continue;
}
setCountryData(entity, country, asin, countryMinimumPriceMap.get(country));
acceptedAny = true;
}
if (!acceptedAny) {
return toItemVo(firstExisting, group.getGroupName());
}
skipPriceAsinMapper.insert(entity);
backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java
+25
View File
@@ -214,6 +214,14 @@ public class ShopMatchTaskCacheService {
if (!Files.isRegularFile(file)) {
continue;
}
if (!isFileCacheFresh(file, now)) {
// 文件超过 TTL，视为过期并删除，避免被 poll 线程无限信任导致前端轮询不收敛
try {
Files.deleteIfExists(file);
} catch (IOException ignored) {
}
continue;
}
try {
FileTaskEntity task = objectMapper.readValue(Files.readString(file), FileTaskEntity.class);
result.put(taskId, task);
@@ -245,5 +253,22 @@ public class ShopMatchTaskCacheService {
&& now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis());
}
/**
* 文件缓存新鲜度判断：超过 {@code localTaskEntityFileCacheMillis} 视为过期。
* 通过文件 mtime 判断，避免在 finalize 与 poll 线程的竞态下把陈旧的 RUNNING 写回后被永久信任。
*/
private boolean isFileCacheFresh(Path file, long now) {
long ttl = Math.max(0L, taskPressureProperties.getLocalTaskEntityFileCacheMillis());
if (ttl <= 0L) {
return false;
}
try {
long modified = Files.getLastModifiedTime(file).toMillis();
return now - modified <= ttl;
} catch (IOException ex) {
return false;
}
}
private record LocalTaskEntityCacheEntry(long cachedAtMillis, FileTaskEntity task) {}
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java
+20 -3
View File
@@ -123,14 +123,31 @@ public class ShopMatchTaskService {
if (dbTask == null || !MODULE_TYPE.equals(dbTask.getModuleType())) {
continue;
}
result.put(dbTask.getId(), dbTask);
if ("RUNNING".equals(dbTask.getStatus()) || "SCHEDULED".equals(dbTask.getStatus())) {
shopMatchTaskCacheService.saveTaskCache(dbTask);
// 复活防护：避免陈旧 RUNNING 把 finalize 后的 SUCCESS 覆盖掉
if (!isCacheNewerThanDb(result.get(dbTask.getId()), dbTask)) {
result.put(dbTask.getId(), dbTask);
if ("RUNNING".equals(dbTask.getStatus()) || "SCHEDULED".equals(dbTask.getStatus())) {
shopMatchTaskCacheService.saveTaskCache(dbTask);
}
}
}
return result;
}
/**
* 当缓存中已存在的实体比本次 DB 读取的结果"更新"（updatedAt 不晚于 DB）时，
* 认为缓存为权威值，避免陈旧的 DB 读把缓存中的终态覆盖回 RUNNING。
*/
private boolean isCacheNewerThanDb(FileTaskEntity cached, FileTaskEntity dbTask) {
if (cached == null || dbTask == null) {
return false;
}
if (cached.getUpdatedAt() == null || dbTask.getUpdatedAt() == null) {
return false;
}
return !cached.getUpdatedAt().isBefore(dbTask.getUpdatedAt());
}
private List<FileTaskEntity> selectTasksByIdsInBatches(List<Long> taskIds) {
List<FileTaskEntity> tasks = new ArrayList<>();
if (taskIds == null || taskIds.isEmpty()) {
backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/memory/service/ZiniaoMemoryStoreService.java
+7
View File
@@ -181,6 +181,13 @@ public class ZiniaoMemoryStoreService {
return deleted;
}
@Transactional
public int deleteAllByType(String cacheType) {
String normalizedType = normalizeRequired(cacheType, "cacheType 不能为空");
return ziniaoMemoryStoreMapper.delete(new LambdaQueryWrapper<ZiniaoMemoryStoreEntity>()
.eq(ZiniaoMemoryStoreEntity::getCacheType, normalizedType));
}
@Transactional
public int deleteExpired(int limit) {
int safeLimit = Math.max(limit, 1);
backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/memory/service/ZiniaoTransientCacheService.java
+13
View File
@@ -105,6 +105,19 @@ public class ZiniaoTransientCacheService {
log.trace("[ziniao-transient] delete type={}", normalizedType);
}
public int deleteByType(String cacheType) {
String normalizedType = normalizeRequired(cacheType, "cacheType 不能为空");
String prefix = normalizedType + SEP;
int deleted = 0;
for (String key : map.keySet()) {
if (key.startsWith(prefix) && map.remove(key) != null) {
deleted++;
}
}
log.trace("[ziniao-transient] delete by type={} count={}", normalizedType, deleted);
return deleted;
}
@Scheduled(fixedDelayString = "${aiimage.ziniao.transient-cache-cleanup-delay-ms:60000}")
void cleanupExpiredEntriesScheduled() {
cleanupExpiredEntries(LocalDateTime.now());
backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java
+17 -2
View File
@@ -44,6 +44,11 @@ public class ZiniaoShopIndexService {
private static final String CACHE_TYPE_SHOP_INDEX_SCOPE_SNAPSHOT = "SHOP_INDEX_SCOPE_SNAPSHOT";
private static final String CACHE_TYPE_SHOP_INDEX_REFRESH_CURSOR = "SHOP_INDEX_REFRESH_CURSOR";
private static final String CACHE_TYPE_COMPANY_ID = "COMPANY_ID";
private static final String CACHE_TYPE_STAFF_LIST = "STAFF_LIST";
private static final String CACHE_TYPE_USER_STORES = "USER_STORES";
private static final String CACHE_TYPE_INVALID_USER_STORES = "INVALID_USER_STORES";
private static final String CACHE_TYPE_SHOP_MATCH = "SHOP_MATCH";
/** 有 shopId 时唯一键，避免同一店铺因不同员工/哈希产生多行。 */
private static final String SHOP_ENTRY_KEY_SHOP_PREFIX = "s:";
/** 无 shopId（冲突占位等）时仍按规范化店名存一行。 */
@@ -414,8 +419,18 @@ public class ZiniaoShopIndexService {
}
public void invalidateIndex() {
ziniaoTransientCacheService.delete(CACHE_TYPE_SHOP_INDEX_REFRESH_CURSOR, "global");
log.info("[ziniao-index] cursor invalidated (transient only; shop rows unchanged)");
int persistentDeleted = ziniaoMemoryStoreService.deleteAllByType(
ZiniaoMemoryStoreService.CACHE_TYPE_SHOP_INDEX_ENTRY);
int transientDeleted = 0;
transientDeleted += ziniaoTransientCacheService.deleteByType(CACHE_TYPE_SHOP_INDEX_SCOPE_SNAPSHOT);
transientDeleted += ziniaoTransientCacheService.deleteByType(CACHE_TYPE_SHOP_INDEX_REFRESH_CURSOR);
transientDeleted += ziniaoTransientCacheService.deleteByType(CACHE_TYPE_COMPANY_ID);
transientDeleted += ziniaoTransientCacheService.deleteByType(CACHE_TYPE_STAFF_LIST);
transientDeleted += ziniaoTransientCacheService.deleteByType(CACHE_TYPE_USER_STORES);
transientDeleted += ziniaoTransientCacheService.deleteByType(CACHE_TYPE_INVALID_USER_STORES);
transientDeleted += ziniaoTransientCacheService.deleteByType(CACHE_TYPE_SHOP_MATCH);
log.info("[ziniao-index] invalidated persistentRows={} transientEntries={} next refresh starts from offset 0",
persistentDeleted, transientDeleted);
}
private void markIpWhitelistAllowedSafely(ZiniaoApiKeyProvider.ApiKeyAccount account) {
backend-java/src/main/resources/application.yml
+10
View File
@@ -8,6 +8,9 @@ spring:
multipart:
max-file-size: 2GB
max-request-size: 2GB
mvc:
async:
request-timeout: ${AIIMAGE_MVC_ASYNC_REQUEST_TIMEOUT:30m}
jackson:
time-zone: Asia/Shanghai
datasource:
@@ -24,6 +27,13 @@ spring:
max-lifetime: ${AIIMAGE_DB_POOL_MAX_LIFETIME_MS:1500000}
keepalive-time: ${AIIMAGE_DB_POOL_KEEPALIVE_TIME_MS:120000}
leak-detection-threshold: ${AIIMAGE_DB_POOL_LEAK_DETECT_MS:15000}
flyway:
enabled: ${AIIMAGE_FLYWAY_ENABLED:true}
locations: classpath:db
baseline-on-migrate: ${AIIMAGE_FLYWAY_BASELINE_ON_MIGRATE:true}
baseline-version: ${AIIMAGE_FLYWAY_BASELINE_VERSION:88}
baseline-description: ${AIIMAGE_FLYWAY_BASELINE_DESCRIPTION:existing-schema}
validate-on-migrate: true
data:
redis:
username: ${AIIMAGE_REDIS_USERNAME:}
backend-java/src/main/resources/db/V25__shop_manage_group_bind_user.sql → backend-java/src/main/resources/db/V25_1__shop_manage_group_bind_user.sql
View File
backend-java/src/main/resources/db/V89__shop_manage_zn_username_and_shop_key_token_hash.sql
+20
View File
@@ -0,0 +1,20 @@
ALTER TABLE biz_shop_manage
ADD COLUMN zn_username VARCHAR(128) NOT NULL DEFAULT '' COMMENT '自动化账号' AFTER mall_name;
ALTER TABLE biz_shop_key
ADD COLUMN ziniao_token_hash CHAR(64) NOT NULL DEFAULT '' COMMENT '规范化紫鸟令牌 SHA-256' AFTER ziniao_token;
UPDATE biz_shop_key
SET ziniao_token = TRIM(
CASE
WHEN LOWER(LEFT(TRIM(ziniao_token), 7)) = 'bearer '
THEN SUBSTRING(TRIM(ziniao_token), 8)
ELSE TRIM(ziniao_token)
END
);
UPDATE biz_shop_key
SET ziniao_token_hash = SHA2(ziniao_token, 256);
ALTER TABLE biz_shop_key
ADD UNIQUE KEY uk_ziniao_token_hash (ziniao_token_hash);