完成后端架构重构等

This commit is contained in:
super
2026-04-23 15:25:41 +08:00
parent 6894f9cc57
commit 0391cb223f
86 changed files with 10843 additions and 1757 deletions

View File

@@ -8,4 +8,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
public class TaskPressureProperties {
private long localTaskEntityCacheMillis = 3000;
private int dbSelectBatchSize = 200;
private long scopePayloadFlushIntervalMillis = 15000;
private long scopePayloadBufferRetentionHours = 24;
private int scopePayloadRecoveryMaxFiles = 200;
private String scopePayloadCleanupCron = "15 */30 * * * *";
}

View File

@@ -1,25 +1,13 @@
package com.nanri.aiimage.modules.brand.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.BrandProgressProperties;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
@@ -33,6 +21,7 @@ public class BrandTaskProgressCacheService {
private final StringRedisTemplate stringRedisTemplate;
private final BrandProgressProperties brandProgressProperties;
@SuppressWarnings("unused")
private final ObjectMapper objectMapper;
public BrandTaskProgressCacheService(StringRedisTemplate stringRedisTemplate,
@@ -96,121 +85,9 @@ public class BrandTaskProgressCacheService {
return stringRedisTemplate.opsForHash().entries(buildKey(taskId));
}
public void saveParsedPayload(Long taskId, Object payload) {
try {
Files.createDirectories(buildTaskDir(taskId));
Files.writeString(
buildPayloadFile(taskId),
objectMapper.writeValueAsString(payload),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE
);
} catch (Exception ex) {
throw new BusinessException("鏆傚瓨鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
public <T> T getParsedPayload(Long taskId, TypeReference<T> typeReference) {
Path payloadFile = buildPayloadFile(taskId);
if (!Files.isRegularFile(payloadFile)) {
return null;
}
try {
return objectMapper.readValue(Files.readString(payloadFile), typeReference);
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
public ChunkStoreResult storeChunk(Long taskId, BrandCrawlResultFileDto file) {
validateChunk(file);
String fileUrl = normalizeFileUrl(file.getFileUrl());
String chunkDigest = digestChunk(file);
String digestField = String.valueOf(file.getChunkIndex());
String digestKey = buildChunkDigestKey(taskId, fileUrl);
Object existingDigest = stringRedisTemplate.opsForHash().get(digestKey, digestField);
if (existingDigest instanceof String existing && !existing.isBlank()) {
refreshFileKeys(taskId, fileUrl);
if (existing.equals(chunkDigest)) {
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl);
int finishedFiles = countCompletedFiles(taskId);
return new ChunkStoreResult(false, false, finishedFiles, aggregate);
}
throw new BusinessException("鍚屼竴鍒嗙墖閲嶅鎻愪氦浣嗗唴瀹逛笉涓€鑷? " + fileUrl + "#" + file.getChunkIndex());
}
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl);
if (aggregate == null) {
aggregate = createAggregate(file);
} else {
ensureChunkConsistency(aggregate, file);
}
mergeAggregate(aggregate, file);
stringRedisTemplate.opsForSet().add(buildChunkReceiptKey(taskId, fileUrl), digestField);
Long receivedSize = stringRedisTemplate.opsForSet().size(buildChunkReceiptKey(taskId, fileUrl));
int receivedCount = receivedSize == null ? 0 : receivedSize.intValue();
if (receivedCount <= 0) {
receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
}
aggregate.setReceivedChunkCount(Math.max(receivedCount, aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount()));
boolean fileCompleted = aggregate.getChunkTotal() != null
&& aggregate.getChunkTotal() > 0
&& aggregate.getReceivedChunkCount() >= aggregate.getChunkTotal();
aggregate.setCompleted(fileCompleted);
saveFileAggregate(taskId, aggregate);
stringRedisTemplate.opsForHash().put(digestKey, digestField, chunkDigest);
refreshFileKeys(taskId, fileUrl);
boolean newlyCompleted = false;
int finishedFiles = countCompletedFiles(taskId);
if (fileCompleted) {
Long added = stringRedisTemplate.opsForSet().add(buildCompletedFilesKey(taskId), fileUrl);
refreshTaskKey(buildCompletedFilesKey(taskId));
newlyCompleted = added != null && added > 0;
if (newlyCompleted) {
finishedFiles = countCompletedFiles(taskId);
}
}
return new ChunkStoreResult(true, newlyCompleted, finishedFiles, aggregate);
}
public BrandFileAggregateCacheDto getFileAggregate(Long taskId, String fileUrl) {
String normalizedFileUrl = normalizeFileUrl(fileUrl);
Object raw = stringRedisTemplate.opsForHash().get(buildFileAggregateKey(taskId), normalizedFileUrl);
if (!(raw instanceof String json) || json.isBlank()) {
return null;
}
try {
return objectMapper.readValue(json, BrandFileAggregateCacheDto.class);
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍝佺墝鏂囦欢鑱氬悎缂撳瓨澶辫触");
}
}
public Map<String, BrandFileAggregateCacheDto> getAllFileAggregates(Long taskId) {
Map<Object, Object> stored = stringRedisTemplate.opsForHash().entries(buildFileAggregateKey(taskId));
Map<String, BrandFileAggregateCacheDto> result = new LinkedHashMap<>();
for (Map.Entry<Object, Object> entry : stored.entrySet()) {
if (!(entry.getKey() instanceof String fileUrl) || !(entry.getValue() instanceof String json) || json.isBlank()) {
continue;
}
try {
result.put(fileUrl, objectMapper.readValue(json, BrandFileAggregateCacheDto.class));
} catch (Exception ignored) {
}
}
return result;
}
public int countCompletedFiles(Long taskId) {
Long count = stringRedisTemplate.opsForSet().size(buildCompletedFilesKey(taskId));
return count == null ? 0 : count.intValue();
}
public boolean acquireFinalizeLock(Long taskId) {
Boolean ok = stringRedisTemplate.opsForValue().setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
Boolean ok = stringRedisTemplate.opsForValue()
.setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
return Boolean.TRUE.equals(ok);
}
@@ -220,18 +97,7 @@ public class BrandTaskProgressCacheService {
public void delete(Long taskId) {
stringRedisTemplate.delete(buildKey(taskId));
stringRedisTemplate.delete(buildFileAggregateKey(taskId));
stringRedisTemplate.delete(buildCompletedFilesKey(taskId));
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
stringRedisTemplate.delete(buildLegacyResultKey(taskId));
deleteTaskDirQuietly(taskId);
}
public void deleteFileState(Long taskId, String fileUrl) {
String normalizedFileUrl = normalizeFileUrl(fileUrl);
stringRedisTemplate.opsForHash().delete(buildFileAggregateKey(taskId), normalizedFileUrl);
stringRedisTemplate.delete(buildChunkReceiptKey(taskId, normalizedFileUrl));
stringRedisTemplate.delete(buildChunkDigestKey(taskId, normalizedFileUrl));
}
public String buildKey(Long taskId) {
@@ -242,182 +108,14 @@ public class BrandTaskProgressCacheService {
return brandProgressProperties.getHeartbeatTimeoutMinutes();
}
private void saveFileAggregate(Long taskId, BrandFileAggregateCacheDto aggregate) {
try {
stringRedisTemplate.opsForHash().put(buildFileAggregateKey(taskId), normalizeFileUrl(aggregate.getFileUrl()), objectMapper.writeValueAsString(aggregate));
refreshTaskKey(buildFileAggregateKey(taskId));
} catch (Exception ex) {
throw new BusinessException("鏆傚瓨鍝佺墝鏂囦欢鑱氬悎缁撴灉澶辫触");
}
}
private void validateChunk(BrandCrawlResultFileDto file) {
if (file == null) {
throw new BusinessException("鍒嗙墖涓嶈兘涓虹┖");
}
if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0 || file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) {
throw new BusinessException("鍒嗙墖鍙傛暟涓嶅悎娉?");
}
if (file.getFileUrl() == null || file.getFileUrl().isBlank()) {
throw new BusinessException("fileUrl 涓嶈兘涓虹┖");
}
}
private BrandFileAggregateCacheDto createAggregate(BrandCrawlResultFileDto file) {
BrandFileAggregateCacheDto aggregate = new BrandFileAggregateCacheDto();
aggregate.setFileUrl(normalizeFileUrl(file.getFileUrl()));
aggregate.setOriginalFilename(file.getOriginalFilename());
aggregate.setRelativePath(file.getRelativePath());
aggregate.setMainSheetName(file.getMainSheetName());
aggregate.setChunkTotal(file.getChunkTotal());
aggregate.setTotalLines(safePositive(file.getTotalLines()));
aggregate.setReceivedChunkCount(0);
aggregate.setProcessedLineCount(0);
aggregate.setCompleted(false);
aggregate.setInvalidBrands(new ArrayList<>());
aggregate.setQueryFailedBrands(new ArrayList<>());
return aggregate;
}
private void ensureChunkConsistency(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
if (aggregate.getChunkTotal() != null && file.getChunkTotal() != null && !aggregate.getChunkTotal().equals(file.getChunkTotal())) {
throw new BusinessException("鍚屼竴鏂囦欢鐨?chunkTotal 涓嶄竴鑷? " + aggregate.getFileUrl());
}
if (aggregate.getChunkTotal() == null) {
aggregate.setChunkTotal(file.getChunkTotal());
}
int incomingTotalLines = safePositive(file.getTotalLines());
if (incomingTotalLines > 0) {
aggregate.setTotalLines(Math.max(safePositive(aggregate.getTotalLines()), incomingTotalLines));
}
if ((aggregate.getOriginalFilename() == null || aggregate.getOriginalFilename().isBlank()) && file.getOriginalFilename() != null && !file.getOriginalFilename().isBlank()) {
aggregate.setOriginalFilename(file.getOriginalFilename());
}
if ((aggregate.getRelativePath() == null || aggregate.getRelativePath().isBlank()) && file.getRelativePath() != null && !file.getRelativePath().isBlank()) {
aggregate.setRelativePath(file.getRelativePath());
}
if ((aggregate.getMainSheetName() == null || aggregate.getMainSheetName().isBlank()) && file.getMainSheetName() != null && !file.getMainSheetName().isBlank()) {
aggregate.setMainSheetName(file.getMainSheetName());
}
}
private void mergeAggregate(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
int processed = safePositive(aggregate.getProcessedLineCount())
+ sizeOf(file.getKeptRows())
+ sizeOf(file.getInvalidBrands())
+ sizeOf(file.getQueryFailedBrands());
aggregate.setProcessedLineCount(processed);
if (file.getInvalidBrands() != null && !file.getInvalidBrands().isEmpty()) {
aggregate.getInvalidBrands().addAll(file.getInvalidBrands());
}
if (file.getQueryFailedBrands() != null && !file.getQueryFailedBrands().isEmpty()) {
aggregate.getQueryFailedBrands().addAll(file.getQueryFailedBrands());
}
}
private int sizeOf(List<?> list) {
return list == null ? 0 : list.size();
}
private int safePositive(Integer value) {
return value == null || value <= 0 ? 0 : value;
}
private String digestChunk(BrandCrawlResultFileDto file) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(objectMapper.writeValueAsString(file).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new BusinessException("璁$畻鍝佺墝缁撴灉鍒嗙墖鎽樿澶辫触");
}
}
private void refreshFileKeys(Long taskId, String fileUrl) {
refreshTaskKey(buildChunkReceiptKey(taskId, fileUrl));
refreshTaskKey(buildChunkDigestKey(taskId, fileUrl));
refreshTaskKey(buildFileAggregateKey(taskId));
}
private void refreshTaskKey(String key) {
stringRedisTemplate.expire(key, ttl());
}
private Duration ttl() {
return Duration.ofHours(brandProgressProperties.getTtlHours());
}
private Path buildTaskDir(Long taskId) {
return Path.of(System.getProperty("java.io.tmpdir"), "brand-task-cache", String.valueOf(taskId));
}
private Path buildPayloadFile(Long taskId) {
return buildTaskDir(taskId).resolve("parsed-payload.json");
}
private void deleteTaskDirQuietly(Long taskId) {
Path taskDir = buildTaskDir(taskId);
if (!Files.exists(taskDir)) {
return;
}
try (var walk = Files.walk(taskDir)) {
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (IOException ignored) {
}
});
} catch (IOException ignored) {
}
}
private String buildFileAggregateKey(Long taskId) {
return "brand:task:file-aggregate:" + taskId;
}
private String buildChunkReceiptKey(Long taskId, String fileUrl) {
return "brand:task:file-chunks:" + taskId + ":" + hashFileUrl(fileUrl);
}
private String buildChunkDigestKey(Long taskId, String fileUrl) {
return "brand:task:file-chunk-digests:" + taskId + ":" + hashFileUrl(fileUrl);
}
private String buildCompletedFilesKey(Long taskId) {
return "brand:task:completed-files:" + taskId;
}
private String buildFinalizeLockKey(Long taskId) {
return "brand:task:finalizing:" + taskId;
}
private String buildLegacyResultKey(Long taskId) {
return "brand:task:result-chunks:" + taskId;
}
private String normalizeFileUrl(String fileUrl) {
return fileUrl == null ? "" : fileUrl.trim();
}
private String hashFileUrl(String fileUrl) {
String normalized = normalizeFileUrl(fileUrl);
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalized.getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new BusinessException("鐢熸垚鏂囦欢缂撳瓨閿け璐?");
}
}
private String normalizePhase(String phase) {
if (phase == null || phase.isBlank()) {
return PHASE_CRAWLING;
@@ -432,7 +130,4 @@ public class BrandTaskProgressCacheService {
private String blankToEmpty(String value) {
return value == null ? "" : value.trim();
}
public record ChunkStoreResult(boolean stored, boolean newlyCompletedFile, int finishedFiles, BrandFileAggregateCacheDto aggregate) {
}
}

View File

@@ -39,7 +39,7 @@ import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -83,6 +83,7 @@ public class BrandTaskService {
private final StorageProperties storageProperties;
private final BrandProgressProperties brandProgressProperties;
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
private final BrandTaskStorageService brandTaskStorageService;
private final DistributedJobLockService distributedJobLockService;
private final ObjectMapper objectMapper;
@@ -193,7 +194,7 @@ public class BrandTaskService {
cachedFiles.add(cacheDto);
}
brandTaskProgressCacheService.saveParsedPayload(taskId, cachedFiles);
brandTaskStorageService.saveParsedPayload(taskId, cachedFiles);
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
@@ -226,9 +227,7 @@ public class BrandTaskService {
sourceByUrl.put(file.getFileUrl(), file);
sourceIndexByUrl.put(file.getFileUrl(), i + 1);
}
List<BrandParsedFileCacheDto> cachedFiles = brandTaskProgressCacheService.getParsedPayload(taskId,
new TypeReference<List<BrandParsedFileCacheDto>>() {
});
List<BrandParsedFileCacheDto> cachedFiles = brandTaskStorageService.getParsedPayload(taskId);
if (cachedFiles == null || cachedFiles.isEmpty()) {
throw new BusinessException("任务原始数据已过期,请重新创建任务");
}
@@ -252,7 +251,7 @@ public class BrandTaskService {
totalCount,
Thread.currentThread().getName());
int finishedCount = brandTaskProgressCacheService.countCompletedFiles(taskId);
int finishedCount = brandTaskStorageService.countCompletedFiles(taskId);
for (BrandCrawlResultFileDto resultFile : resultFiles) {
String fileUrl = blankToNull(resultFile.getFileUrl());
if (fileUrl == null) {
@@ -272,7 +271,7 @@ public class BrandTaskService {
} else if (totalLines > 0 && resultFile.getTotalLines() > totalLines) {
throw new BusinessException("totalLines 超过原始文件行数: " + fileUrl);
}
BrandTaskProgressCacheService.ChunkStoreResult storeResult = brandTaskProgressCacheService.storeChunk(taskId, resultFile);
BrandTaskStorageService.ChunkStoreResult storeResult = brandTaskStorageService.storeChunk(taskId, resultFile);
finishedCount = Math.max(finishedCount, storeResult.finishedFiles());
log.info("[brand-submit] taskId={} fileUrl={} chunk={}/{} stored={} newlyCompletedFile={} finishedFiles={}",
taskId,
@@ -316,6 +315,7 @@ public class BrandTaskService {
if (updated == 0) {
throw new BusinessException("任务不存在或无法取消");
}
brandTaskStorageService.deleteTaskData(taskId);
brandTaskProgressCacheService.delete(taskId);
}
@@ -327,6 +327,7 @@ public class BrandTaskService {
if (deleted == 0) {
throw new BusinessException("任务不存在或正在执行中无法删除");
}
brandTaskStorageService.deleteTaskData(taskId);
brandTaskProgressCacheService.delete(taskId);
}
@@ -555,7 +556,7 @@ public class BrandTaskService {
if (STATUS_FAILED.equalsIgnoreCase(freshStatus)) {
throw new BusinessException("任务已结束,不能继续组装结果");
}
Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskProgressCacheService.getAllFileAggregates(taskId);
Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskStorageService.getAllFileAggregates(taskId);
log.info("[brand-finalize] taskId={} aggregateFiles={} expectedFiles={} thread={}",
taskId,
aggregates.size(),
@@ -600,9 +601,7 @@ public class BrandTaskService {
if (updated == 0) {
throw new BusinessException("任务已取消");
}
for (BrandSourceFileDto sourceFile : sourceFiles) {
brandTaskProgressCacheService.deleteFileState(taskId, sourceFile.getFileUrl());
}
brandTaskStorageService.deleteTaskData(taskId);
brandTaskProgressCacheService.delete(taskId);
log.info("[brand-finalize] taskId={} finalized success files={} elapsedMs={}",
taskId,
@@ -614,7 +613,7 @@ public class BrandTaskService {
.eq(BrandCrawlTaskEntity::getId, taskId)
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
.set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED)
.set(BrandCrawlTaskEntity::getProgressCurrent, brandTaskProgressCacheService.countCompletedFiles(taskId))
.set(BrandCrawlTaskEntity::getProgressCurrent, brandTaskStorageService.countCompletedFiles(taskId))
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
.set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage()));
brandTaskProgressCacheService.markFailed(taskId, ex.getMessage());
@@ -622,6 +621,8 @@ public class BrandTaskService {
throw businessException;
}
throw new BusinessException(ex.getMessage());
} finally {
cleanupBrandResultTempFiles(outputEntries, outputDir);
}
}
@@ -746,7 +747,7 @@ public class BrandTaskService {
String value = index == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(index)));
rowData.put(column, value);
}
String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("\u54c1\u724c", ""), ""));
String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), ""));
if (brand.isBlank()) {
rowData.put("__rowIndex", rowNum + 1);
rows.add(rowData);
@@ -827,7 +828,9 @@ public class BrandTaskService {
BrandParsedFileCacheDto cachedFile,
BrandFileAggregateCacheDto resultFile) throws IOException {
String actualStrategy = normalizeStrategy(strategy);
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try {
String mainSheetName = blankToDefault(cachedFile.getSheetName(), "Sheet1");
var mainSheet = workbook.createSheet(mainSheetName);
var headerRow = mainSheet.createRow(0);
@@ -878,8 +881,35 @@ public class BrandTaskService {
}
try (FileOutputStream outputStream = new FileOutputStream(outputFile)) {
applyBrandSheetWidths(mainSheet, columns.size());
applyFixedColumnWidths(invalidSheet, new int[]{20, 12, 18});
applyFixedColumnWidths(queryFailedSheet, new int[]{20, 18});
workbook.write(outputStream);
}
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}
private void applyBrandSheetWidths(Sheet sheet, int columnCount) {
if (sheet == null || columnCount <= 0) {
return;
}
for (int i = 0; i < columnCount; i++) {
sheet.setColumnWidth(i, 20 * 256);
}
}
private void applyFixedColumnWidths(Sheet sheet, int[] widths) {
if (sheet == null || widths == null) {
return;
}
for (int i = 0; i < widths.length; i++) {
sheet.setColumnWidth(i, widths[i] * 256);
}
}
@@ -946,6 +976,43 @@ public class BrandTaskService {
return zipFile;
}
private void cleanupBrandResultTempFiles(List<OutputEntry> entries, File outputDir) {
if (entries != null) {
for (OutputEntry entry : entries) {
if (entry == null) {
continue;
}
deleteQuietly(entry.resultFile());
if (isManagedBrandSourceTempFile(entry.sourceFile())) {
deleteQuietly(entry.sourceFile());
}
}
}
deleteQuietly(outputDir);
}
private boolean isManagedBrandSourceTempFile(File file) {
if (file == null) {
return false;
}
File managedDir = FileUtil.file(storageProperties.getLocalTempDir(), "brand-source-download");
try {
return file.getCanonicalPath().startsWith(managedDir.getCanonicalPath());
} catch (IOException ex) {
return false;
}
}
private void deleteQuietly(File file) {
if (file == null || !file.exists()) {
return;
}
try {
FileUtil.del(file);
} catch (Exception ignored) {
}
}
private File buildNamedOutputFile(File outputDir, String filename) {
File candidate = FileUtil.file(outputDir, filename);
if (!candidate.exists()) {

View File

@@ -0,0 +1,475 @@
package com.nanri.aiimage.modules.brand.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
@RequiredArgsConstructor
public class BrandTaskStorageService {
private static final String MODULE_TYPE = "BRAND";
private static final String OSS_POINTER_PREFIX = "oss:";
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final ObjectMapper objectMapper;
private final OssStorageService ossStorageService;
@Transactional
public void saveParsedPayload(Long taskId, List<BrandParsedFileCacheDto> payload) {
if (taskId == null || taskId <= 0 || payload == null || payload.isEmpty()) {
return;
}
LocalDateTime now = LocalDateTime.now();
for (BrandParsedFileCacheDto item : payload) {
if (item == null || isBlank(item.getFileUrl())) {
continue;
}
String scopeKey = normalize(item.getFileUrl());
String scopeHash = hash(scopeKey);
String parsedJson = storeParsedPayload(taskId, scopeHash, writeJson(item, "Save brand task parsed payload failed"));
TaskScopeStateEntity state = getScopeState(taskId, scopeHash);
if (state == null) {
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
entity.setScopeKey(scopeKey);
entity.setScopeHash(scopeHash);
entity.setParsedPayloadJson(parsedJson);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(entity);
continue;
} catch (DuplicateKeyException ignored) {
state = getScopeState(taskId, scopeHash);
}
}
if (state == null) {
throw new BusinessException("Save brand task parsed payload failed");
}
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedJson)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
}
public List<BrandParsedFileCacheDto> getParsedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return List.of();
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
if (states == null || states.isEmpty()) {
return List.of();
}
List<BrandParsedFileCacheDto> result = new ArrayList<>();
for (TaskScopeStateEntity state : states) {
String payloadJson = resolveParsedPayload(state.getParsedPayloadJson());
if (isBlank(payloadJson)) {
continue;
}
try {
result.add(objectMapper.readValue(payloadJson, BrandParsedFileCacheDto.class));
} catch (Exception ex) {
throw new BusinessException("Read brand task parsed payload failed");
}
}
return result;
}
@Transactional
public ChunkStoreResult storeChunk(Long taskId, BrandCrawlResultFileDto file) {
validateChunk(file);
String scopeKey = normalize(file.getFileUrl());
String scopeHash = hash(scopeKey);
String payloadJson = writeJson(file, "Save brand task chunk failed");
String payloadHash = hash(payloadJson);
LocalDateTime now = LocalDateTime.now();
TaskChunkEntity existingChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex())
.last("limit 1"));
if (existingChunk != null) {
if (payloadHash.equals(existingChunk.getPayloadHash())) {
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate);
}
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
}
TaskChunkEntity entity = new TaskChunkEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
entity.setScopeKey(scopeKey);
entity.setScopeHash(scopeHash);
entity.setChunkIndex(file.getChunkIndex());
entity.setChunkTotal(file.getChunkTotal());
entity.setPayloadJson(payloadJson);
entity.setPayloadHash(payloadHash);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskChunkMapper.insert(entity);
} catch (DuplicateKeyException ex) {
TaskChunkEntity concurrentChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex())
.last("limit 1"));
if (concurrentChunk != null && payloadHash.equals(concurrentChunk.getPayloadHash())) {
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate);
}
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
}
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
boolean completed = Boolean.TRUE.equals(aggregate.getCompleted());
saveAggregate(taskId, scopeKey, scopeHash, aggregate, now);
return new ChunkStoreResult(true, completed, countCompletedFiles(taskId), aggregate);
}
public BrandFileAggregateCacheDto getFileAggregate(Long taskId, String fileUrl) {
TaskScopeStateEntity state = getScopeState(taskId, hash(normalize(fileUrl)));
if (state == null || isBlank(state.getStateJson())) {
return null;
}
try {
return objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class);
} catch (Exception ex) {
throw new BusinessException("Read brand task aggregate failed");
}
}
public Map<String, BrandFileAggregateCacheDto> getAllFileAggregates(Long taskId) {
if (taskId == null || taskId <= 0) {
return Map.of();
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNotNull(TaskScopeStateEntity::getStateJson));
if (states == null || states.isEmpty()) {
return Map.of();
}
Map<String, BrandFileAggregateCacheDto> result = new LinkedHashMap<>();
for (TaskScopeStateEntity state : states) {
if (isBlank(state.getStateJson()) || isBlank(state.getScopeKey())) {
continue;
}
try {
result.put(state.getScopeKey(), objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class));
} catch (Exception ex) {
throw new BusinessException("Read brand task aggregate failed");
}
}
return result;
}
public int countCompletedFiles(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getCompleted, 1));
return count == null ? 0 : count.intValue();
}
@Transactional
public void deleteTaskData(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (states != null) {
for (TaskScopeStateEntity state : states) {
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null);
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
}
private void saveAggregate(Long taskId,
String scopeKey,
String scopeHash,
BrandFileAggregateCacheDto aggregate,
LocalDateTime now) {
String aggregateJson = writeJson(aggregate, "Save brand task aggregate failed");
TaskScopeStateEntity state = getScopeState(taskId, scopeHash);
if (state == null) {
throw new BusinessException("Brand task aggregate state missing: " + scopeKey);
}
int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal();
int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getStateJson, aggregateJson)
.set(TaskScopeStateEntity::getChunkTotal, chunkTotal)
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.set(TaskScopeStateEntity::getCompleted, Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getLastError, null)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
private TaskScopeStateEntity getScopeState(Long taskId, String scopeHash) {
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.last("limit 1"));
}
private BrandFileAggregateCacheDto createAggregate(BrandCrawlResultFileDto file) {
BrandFileAggregateCacheDto aggregate = new BrandFileAggregateCacheDto();
aggregate.setFileUrl(normalize(file.getFileUrl()));
aggregate.setOriginalFilename(file.getOriginalFilename());
aggregate.setRelativePath(file.getRelativePath());
aggregate.setMainSheetName(file.getMainSheetName());
aggregate.setChunkTotal(file.getChunkTotal());
aggregate.setTotalLines(safePositive(file.getTotalLines()));
aggregate.setReceivedChunkCount(0);
aggregate.setProcessedLineCount(0);
aggregate.setCompleted(false);
aggregate.setInvalidBrands(new ArrayList<>());
aggregate.setQueryFailedBrands(new ArrayList<>());
return aggregate;
}
private void ensureChunkConsistency(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
if (aggregate.getChunkTotal() != null && file.getChunkTotal() != null && !aggregate.getChunkTotal().equals(file.getChunkTotal())) {
throw new BusinessException("Brand task chunkTotal mismatch: " + aggregate.getFileUrl());
}
if (aggregate.getChunkTotal() == null) {
aggregate.setChunkTotal(file.getChunkTotal());
}
int incomingTotalLines = safePositive(file.getTotalLines());
if (incomingTotalLines > 0) {
aggregate.setTotalLines(Math.max(safePositive(aggregate.getTotalLines()), incomingTotalLines));
}
if (isBlank(aggregate.getOriginalFilename()) && !isBlank(file.getOriginalFilename())) {
aggregate.setOriginalFilename(file.getOriginalFilename());
}
if (isBlank(aggregate.getRelativePath()) && !isBlank(file.getRelativePath())) {
aggregate.setRelativePath(file.getRelativePath());
}
if (isBlank(aggregate.getMainSheetName()) && !isBlank(file.getMainSheetName())) {
aggregate.setMainSheetName(file.getMainSheetName());
}
}
private void mergeAggregate(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
int processed = safePositive(aggregate.getProcessedLineCount())
+ sizeOf(file.getKeptRows())
+ sizeOf(file.getInvalidBrands())
+ sizeOf(file.getQueryFailedBrands());
aggregate.setProcessedLineCount(processed);
if (file.getInvalidBrands() != null && !file.getInvalidBrands().isEmpty()) {
aggregate.getInvalidBrands().addAll(file.getInvalidBrands());
}
if (file.getQueryFailedBrands() != null && !file.getQueryFailedBrands().isEmpty()) {
aggregate.getQueryFailedBrands().addAll(file.getQueryFailedBrands());
}
}
private BrandFileAggregateCacheDto rebuildAggregate(Long taskId, String scopeKey, String scopeHash) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.orderByAsc(TaskChunkEntity::getChunkIndex, TaskChunkEntity::getId));
if (chunks == null || chunks.isEmpty()) {
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, scopeKey);
if (aggregate != null) {
aggregate.setFileUrl(scopeKey);
aggregate.setReceivedChunkCount(0);
aggregate.setProcessedLineCount(0);
aggregate.setCompleted(false);
aggregate.setInvalidBrands(new ArrayList<>());
aggregate.setQueryFailedBrands(new ArrayList<>());
}
return aggregate;
}
BrandFileAggregateCacheDto aggregate = null;
for (TaskChunkEntity chunk : chunks) {
BrandCrawlResultFileDto chunkPayload = readChunkPayload(chunk.getPayloadJson());
if (aggregate == null) {
aggregate = createAggregate(chunkPayload);
} else {
ensureChunkConsistency(aggregate, chunkPayload);
}
mergeAggregate(aggregate, chunkPayload);
}
if (aggregate == null) {
throw new BusinessException("Read brand task aggregate data failed");
}
aggregate.setFileUrl(scopeKey);
aggregate.setReceivedChunkCount(chunks.size());
aggregate.setCompleted(aggregate.getChunkTotal() != null
&& aggregate.getChunkTotal() > 0
&& chunks.size() >= aggregate.getChunkTotal());
return aggregate;
}
private BrandCrawlResultFileDto readChunkPayload(String payloadJson) {
try {
return objectMapper.readValue(payloadJson, BrandCrawlResultFileDto.class);
} catch (Exception ex) {
throw new BusinessException("Read brand task chunk failed");
}
}
private void validateChunk(BrandCrawlResultFileDto file) {
if (file == null) {
throw new BusinessException("Brand task chunk must not be null");
}
if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0
|| file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) {
throw new BusinessException("Brand task chunk parameters are invalid");
}
if (isBlank(file.getFileUrl())) {
throw new BusinessException("fileUrl must not be blank");
}
}
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
try {
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson);
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new BusinessException("Save brand task parsed payload failed");
}
}
private String resolveParsedPayload(String value) {
if (isBlank(value)) {
return value;
}
String ossPointer = extractOssPointer(value);
if (ossPointer == null) {
return value;
}
try {
return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length()));
} catch (Exception ex) {
throw new BusinessException("Read brand task parsed payload failed");
}
}
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractOssPointer(oldValue);
String newPointer = extractOssPointer(newValue);
if (oldPointer == null || oldPointer.equals(newPointer)) {
return;
}
try {
ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length()));
} catch (Exception ignored) {
}
}
private String extractOssPointer(String value) {
if (isBlank(value)) {
return null;
}
if (value.startsWith(OSS_POINTER_PREFIX)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException(message);
}
}
private String normalize(String value) {
return value == null ? "" : value.trim();
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
private int sizeOf(List<?> list) {
return list == null ? 0 : list.size();
}
private int safePositive(Integer value) {
return value == null || value <= 0 ? 0 : value;
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new IllegalStateException("failed to hash brand scope", ex);
}
}
public record ChunkStoreResult(boolean stored, boolean newlyCompletedFile, int finishedFiles, BrandFileAggregateCacheDto aggregate) {
}
}

View File

@@ -44,11 +44,11 @@ public class ConvertRunService {
private static final String MODULE_TYPE = "CONVERT";
private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer";
private static final List<String> FIVE_COUNTRY_OUTPUT_FILES = List.of(
"\u82f1\u56fd.txt",
"\u6cd5\u56fd.txt",
"\u5fb7\u56fd.txt",
"\u897f\u73ed\u7259.txt",
"\u610f\u5927\u5229.txt"
"英国.txt",
"法国.txt",
"德国.txt",
"西班牙.txt",
"意大利.txt"
);
private final FileTaskMapper fileTaskMapper;

View File

@@ -461,8 +461,8 @@ public class DedupeRunService {
if (value == null) {
return "";
}
return value.replace("\ufeff", "")
.replace("\u3000", " ")
return value.replace("", "")
.replace(" ", " ")
.replace("\r\n", " ")
.replace("\r", " ")
.replace("\n", " ")

View File

@@ -229,7 +229,7 @@ public class DedupeTotalDataService {
progress.setStatus("success");
} catch (Exception e) {
progress.setStatus("failed");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "鍒犻櫎 Excel 鍖归厤鏁版嵁澶辫触");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "删除 Excel 匹配数据失败");
} finally {
deleteQuietly(tempFile);
}
@@ -251,7 +251,7 @@ public class DedupeTotalDataService {
progress.setStatus("success");
} catch (Exception e) {
progress.setStatus("failed");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "瀵煎叆 Excel 澶辫触");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "导入 Excel 失败");
} finally {
deleteQuietly(tempFile);
}
@@ -590,8 +590,8 @@ public class DedupeTotalDataService {
if (value == null) {
return "";
}
return value.replace("\ufeff", "")
.replace("\u3000", " ")
return value.replace("", "")
.replace(" ", " ")
.replace("\r\n", " ")
.replace("\r", " ")
.replace("\n", " ")

View File

@@ -135,9 +135,10 @@ public class DeleteBrandRunController {
try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
String asciiFilename = buildAsciiDownloadFilename(filename, taskId);
response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
"attachment; filename=\"" + asciiFilename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536];
int read;
@@ -150,4 +151,24 @@ public class DeleteBrandRunController {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
}
}
private String buildAsciiDownloadFilename(String filename, Long taskId) {
String sanitized = filename == null ? "" : filename.replaceAll("[^A-Za-z0-9._-]", "_");
sanitized = sanitized.replaceAll("_+", "_");
sanitized = sanitized.replaceAll("^[_\\.]+|[_\\.]+$", "");
if (!sanitized.isBlank() && sanitized.contains(".")) {
return sanitized;
}
String ext = "xlsx";
if (filename != null) {
int dotIndex = filename.lastIndexOf('.');
if (dotIndex >= 0 && dotIndex < filename.length() - 1) {
String rawExt = filename.substring(dotIndex + 1).replaceAll("[^A-Za-z0-9]", "");
if (!rawExt.isBlank()) {
ext = rawExt;
}
}
}
return "delete-brand_" + taskId + "." + ext;
}
}

View File

@@ -44,7 +44,7 @@ import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -74,6 +74,7 @@ public class DeleteBrandRunService {
private final FileResultMapper fileResultMapper;
private final LocalFileStorageService localFileStorageService;
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
private final DeleteBrandTaskStorageService deleteBrandTaskStorageService;
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper;
@@ -221,9 +222,7 @@ public class DeleteBrandRunService {
deleteBrandTaskCacheService.saveTaskCache(task);
}
if (!parsedPayloadByFileIdentity.isEmpty()) {
deleteBrandTaskCacheService.saveParsedPayload(task.getId(), parsedPayloadByFileIdentity);
}
deleteBrandTaskStorageService.saveParsedPayload(task.getId(), parsedPayloadByFileIdentity);
DeleteBrandRunVo vo = new DeleteBrandRunVo();
vo.setTotal(request.getFiles().size());
@@ -277,7 +276,7 @@ public class DeleteBrandRunService {
item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
item.setError(entity.getErrorMessage());
// 以 task.resultJson 中的项为准补 matched/shopId/platform/openStoreUrl/错误信息
// 以 task.resultJson 中的项为准补 matched/shopId/platform/openStoreUrl/错误信息
// 真实补充
if (entity.getTaskId() != null) {
item.setTaskStatus(statusByTaskId.get(entity.getTaskId()));
@@ -291,7 +290,7 @@ public class DeleteBrandRunService {
item.setMatched(candidate.isMatched());
item.setMatchStatus(candidate.getMatchStatus());
item.setMatchMessage(candidate.getMatchMessage());
// 如果物理结果已经成功,说明已经处理过,强制为匹配成功
// 如果物理结果已经成功,说明已经处理过,强制标记为匹配成功
if (item.isSuccess() && candidate.getShopId() != null && !candidate.getShopId().isBlank()) {
item.setMatched(true);
item.setMatchStatus(ZiniaoShopIndexService.MATCH_STATUS_MATCHED);
@@ -303,8 +302,8 @@ public class DeleteBrandRunService {
item.setPlatform(candidate.getPlatform());
item.setOpenStoreUrl(candidate.getOpenStoreUrl());
// 历史表可能只有通用 errorMessage优先返回任务当时的真实 error
// 但是,如果物理结果成功,就不应该显示所谓“未匹配”报错
// 历史表可能只有通用 errorMessage优先返回任务当时的真实 error
// 但如果物理结果成功,就不应该显示所谓“未匹配”报错
if (item.isSuccess()) {
item.setError(null);
} else if ((item.getError() == null || item.getError().isBlank()) && candidate.getError() != null && !candidate.getError().isBlank()) {
@@ -344,7 +343,16 @@ public class DeleteBrandRunService {
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("记录不存在");
}
Long taskId = entity.getTaskId();
fileResultMapper.deleteById(resultId);
if (taskId != null && taskId > 0) {
Long remaining = fileResultMapper.selectCount(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
if (remaining == null || remaining <= 0) {
deleteBrandTaskStorageService.deleteTaskData(taskId);
}
}
}
private ParsedDeleteBrandFile parseDeleteBrandFile(File inputFile) {
@@ -468,8 +476,8 @@ public class DeleteBrandRunService {
if (value == null) {
return "";
}
return value.replace("\ufeff", "")
.replace("\u3000", " ")
return value.replace("", "")
.replace(" ", " ")
.replace("\r\n", " ")
.replace("\r", " ")
.replace("\n", " ")
@@ -640,7 +648,7 @@ public class DeleteBrandRunService {
taskVo.setCreatedAt(task.getCreatedAt());
taskVo.setUpdatedAt(task.getUpdatedAt());
taskVo.setFinishedAt(task.getFinishedAt());
// 轮询期避免每次都查 biz_file_result(会放大 DB 压力;仅任务成功后再提供下载信息
// 轮询期避免每次都查 biz_file_result放大 DB 压力;仅任务成功后再提供下载信息
if ("SUCCESS".equals(task.getStatus())) {
taskVo.setDownloadUrl(resolveTaskDownloadUrl(task.getId()));
taskVo.setDownloadFilename(resolveTaskDownloadFilename(task.getId()));
@@ -996,9 +1004,7 @@ public class DeleteBrandRunService {
throw new BusinessException("任务已结束,拒绝继续提交结果");
}
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskCacheService.getParsedPayload(taskId,
new TypeReference<Map<String, DeleteBrandParsedFileCacheDto>>() {
});
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId);
if (parsedPayload == null || parsedPayload.isEmpty()) {
throw new BusinessException("任务原始数据已过期,请重新执行解析");
}
@@ -1029,10 +1035,10 @@ public class DeleteBrandRunService {
throw new BusinessException("文件标识不匹配: " + fileIdentity);
}
// 更早 duplicate fast-path尽量在较重校验/进度写入前直接短路
// 更早执行 duplicate fast-path尽量在较重校验进度写入前直接短路
validateChunk(fileDto, parsedFile);
validateResultFileAgainstParsed(fileDto, parsedFile);
boolean stored = deleteBrandTaskCacheService.storeResultChunkIfChanged(taskId, fileIdentity, fileDto.getChunkIndex(), fileDto);
boolean stored = deleteBrandTaskStorageService.storeResultChunkIfChanged(taskId, fileIdentity, fileDto.getChunkIndex(), fileDto);
if (!stored) {
log.info("[DeleteBrand] Duplicate chunk ignored for taskId: {}, file: {}, chunkIndex: {}", taskId, fileIdentity, fileDto.getChunkIndex());
continue;
@@ -1051,12 +1057,12 @@ public class DeleteBrandRunService {
progress.put("last_heartbeat_at", String.valueOf(System.currentTimeMillis()));
deleteBrandTaskCacheService.saveProgress(taskId, progress, true);
// 如果该文件已完成,立即更新成品计数的 Redis 缓存「提示,让后续 tryFinalizeTask 能更精
// 如果该文件已完成,立即更新成品计数的 Redis 提示,让后续 tryFinalizeTask 更准
if (isFileCompleted(fileDto)) {
log.info("[DeleteBrand] File chunk transmission fully reached via chunkTotal for taskId: {}, file: {}", taskId, fileDto.getSourceFilename());
shouldTryFinalize = true;
// 这里我们暂不直接 increment而是标记需要重算或在 tryFinalizeTask 中重新计算
// 为了保险,我们直接在 tryFinalizeTask 里重新遍历分片状态
// 这里暂不直接 increment而是标记需要在 tryFinalizeTask 中重新计算
// 为了保险,直接在 tryFinalizeTask 里重新遍历分片状态
}
}
@@ -1079,26 +1085,24 @@ public class DeleteBrandRunService {
return;
}
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskCacheService.getParsedPayload(taskId,
new TypeReference<Map<String, DeleteBrandParsedFileCacheDto>>() {
});
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId);
if (parsedPayload == null || parsedPayload.isEmpty()) {
return;
}
int expectedFiles = parsedPayload.size();
// 移除 finishedFilesHint 强行返回逻辑,因为 Redis 里的 finished_files 可能是旧的,
// 既然进入 tryFinalizeTask就说明有新片到达应该以 loadMergedChunks 为准
// 移除 finishedFilesHint 的强依赖,因为 Redis 里的 finished_files 可能是旧
// 既然进入 tryFinalizeTask就说明有新片到达,应该以 loadMergedChunks 为准
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = loadMergedChunks(taskId);
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(taskId);
int finishedFiles = countCompletedFiles(mergedByFile);
if (finishedFiles < expectedFiles) {
Map<String, String> progress = new LinkedHashMap<>();
progress.put("finished_files", String.valueOf(finishedFiles));
// 定时补偿只负责“再试一次 finalize”不能把缺片任务重新续命
// 否则 stale-check 永远看不到超时任务
// 定时补偿只负责“再试一次 finalize”不能把缺片任务重新续命
// 否则 stale-check 会一直看不到超时任务
if (!fromCompensation) {
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
}
@@ -1133,24 +1137,6 @@ public class DeleteBrandRunService {
finalizeTask(task, parsedPayload, mergedByFile);
}
private Map<String, List<DeleteBrandResultFileDto>> loadMergedChunks(Long taskId) {
Map<String, List<String>> groupedJson = deleteBrandTaskCacheService.groupResultChunkJsonByFile(taskId);
Map<String, List<DeleteBrandResultFileDto>> merged = new LinkedHashMap<>();
for (Map.Entry<String, List<String>> entry : groupedJson.entrySet()) {
List<DeleteBrandResultFileDto> chunks = new ArrayList<>();
for (String raw : entry.getValue()) {
try {
chunks.add(objectMapper.readValue(raw, DeleteBrandResultFileDto.class));
} catch (Exception ex) {
throw new BusinessException("读取删除品牌结果分片失败");
}
}
chunks.sort(Comparator.comparing(DeleteBrandResultFileDto::getChunkIndex, Comparator.nullsLast(Integer::compareTo)));
merged.put(entry.getKey(), chunks);
}
return merged;
}
private int countCompletedFiles(Map<String, List<DeleteBrandResultFileDto>> mergedByFile) {
int finishedFiles = 0;
for (List<DeleteBrandResultFileDto> chunks : mergedByFile.values()) {
@@ -1187,6 +1173,7 @@ public class DeleteBrandRunService {
private void finalizeTask(FileTaskEntity task,
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload,
Map<String, List<DeleteBrandResultFileDto>> mergedByFile) {
File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(task.getId())));
try {
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
"phase", DeleteBrandTaskCacheService.PHASE_ASSEMBLING,
@@ -1259,6 +1246,8 @@ public class DeleteBrandRunService {
item.setPlatform(parsedFile.getPlatform());
item.setOpenStoreUrl(parsedFile.getOpenStoreUrl());
item.setMatched(parsedFile.getShopId() != null && !parsedFile.getShopId().isBlank());
item.setSuccess(true);
item.setError(null);
item.setMatchStatus(item.isMatched() ? ZiniaoShopIndexService.MATCH_STATUS_MATCHED : ZiniaoShopIndexService.MATCH_STATUS_PENDING);
item.setMatchMessage(item.isMatched() ? null : "店铺索引信息缺失,请重新匹配索引");
finalItems.add(item);
@@ -1267,12 +1256,14 @@ public class DeleteBrandRunService {
task.setStatus("SUCCESS");
task.setSuccessFileCount(successCount);
task.setFailedFileCount(Math.max(0, parsedPayload.size() - successCount));
task.setErrorMessage(null);
task.setResultJson(toCompactTaskResultJsonForDb(finalItems));
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
deleteBrandTaskCacheService.saveTaskCache(task);
deleteBrandTaskStorageService.deleteTaskData(task.getId());
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
"phase", "success",
@@ -1297,6 +1288,8 @@ public class DeleteBrandRunService {
throw businessException;
}
throw new BusinessException("删除品牌结果组装失败");
} finally {
deleteQuietly(outputDir);
}
}
@@ -1355,7 +1348,9 @@ public class DeleteBrandRunService {
File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(taskId)));
File outputFile = buildNamedOutputFile(outputDir, blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx"));
try (Workbook workbook = new XSSFWorkbook(); FileOutputStream outputStream = new FileOutputStream(outputFile)) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200);
FileOutputStream outputStream = new FileOutputStream(outputFile)) {
workbook.setCompressTempFiles(true);
Sheet sheet = workbook.createSheet("删除品牌结果");
Row headerRow = sheet.createRow(0);
headerRow.createCell(0).setCellValue("国家");
@@ -1371,9 +1366,7 @@ public class DeleteBrandRunService {
createTextCell(row, 2, item.getStatus());
}
}
for (int i = 0; i < 3; i++) {
sheet.autoSizeColumn(i);
}
applyDeleteBrandColumnWidths(sheet, 3);
workbook.write(outputStream);
return outputFile;
} catch (Exception ex) {
@@ -1387,7 +1380,9 @@ public class DeleteBrandRunService {
File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(taskId)));
File outputFile = buildNamedOutputFile(outputDir, blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx"));
try (Workbook workbook = new XSSFWorkbook(); FileOutputStream outputStream = new FileOutputStream(outputFile)) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200);
FileOutputStream outputStream = new FileOutputStream(outputFile)) {
workbook.setCompressTempFiles(true);
Sheet sheet = workbook.createSheet("删除品牌结果");
Row countryRow = sheet.createRow(0);
Row headerRow = sheet.createRow(1);
@@ -1417,9 +1412,7 @@ public class DeleteBrandRunService {
}
}
for (int i = 0; i < mergedFile.countries().size() * 2; i++) {
sheet.autoSizeColumn(i);
}
applyDeleteBrandColumnWidths(sheet, mergedFile.countries().size() * 2);
workbook.write(outputStream);
return outputFile;
} catch (Exception ex) {
@@ -1432,6 +1425,22 @@ public class DeleteBrandRunService {
cell.setCellValue(value == null ? "" : value);
}
private void applyDeleteBrandColumnWidths(Sheet sheet, int columnCount) {
for (int i = 0; i < columnCount; i++) {
sheet.setColumnWidth(i, 18 * 256);
}
}
private void deleteQuietly(File file) {
if (file == null || !file.exists()) {
return;
}
try {
FileUtil.del(file);
} catch (Exception ignored) {
}
}
private File buildNamedOutputFile(File outputDir, String filename) {
File candidate = FileUtil.file(outputDir, filename);
if (!candidate.exists()) {
@@ -1478,10 +1487,9 @@ public class DeleteBrandRunService {
String normalized = blankToEmpty(value);
return normalized.isBlank() ? defaultValue : normalized;
}
/**
* 落库用 result_json去掉国家分组、预览行等大字段保留匹配摘要与计数。
* 索引命中后单次任务 JSON 体积会陡增,易触发 max_allowed_packet / 更新失败,进而前端 unwrap 报「请求失败」
* 索引命中后单次任务 JSON 体积会明显膨胀,容易触发 max_allowed_packet 更新失败。
*/
private String toCompactTaskResultJsonForDb(List<DeleteBrandResultItemVo> items) {
if (items == null || items.isEmpty()) {

View File

@@ -1,16 +1,11 @@
package com.nanri.aiimage.modules.deletebrand.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.Duration;
import java.util.concurrent.ConcurrentHashMap;
@@ -24,63 +19,16 @@ public class DeleteBrandTaskCacheService {
public static final String PHASE_FAILED = "failed";
private static final long PAYLOAD_TTL_HOURS = 24;
private static final long LOCAL_PARSED_PAYLOAD_CACHE_MILLIS = 3000;
private static final long LOCAL_PROGRESS_CACHE_MILLIS = 1500;
private static final long MIN_PROGRESS_REDIS_FLUSH_MILLIS = 1200;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final ConcurrentHashMap<Long, LocalParsedPayloadCacheEntry> parsedPayloadLocalCache = new ConcurrentHashMap<>();
private final ConcurrentHashMap<Long, LocalProgressCacheEntry> progressLocalCache = new ConcurrentHashMap<>();
private final ConcurrentHashMap<Long, Long> progressRedisFlushAt = new ConcurrentHashMap<>();
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public void saveParsedPayload(Long taskId, Object payload) {
try {
Files.createDirectories(buildTaskDir(taskId));
Files.writeString(
buildPayloadFile(taskId),
objectMapper.writeValueAsString(payload),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE
);
parsedPayloadLocalCache.put(taskId, new LocalParsedPayloadCacheEntry(System.currentTimeMillis(), payload));
} catch (Exception ex) {
throw new BusinessException("鏆傚瓨鍒犻櫎鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
public <T> T getParsedPayload(Long taskId, TypeReference<T> typeReference) {
if (taskId == null || taskId <= 0) {
return null;
}
long now = System.currentTimeMillis();
LocalParsedPayloadCacheEntry cached = parsedPayloadLocalCache.get(taskId);
if (cached != null && cached.isFresh(now) && cached.value() != null) {
try {
return objectMapper.convertValue(cached.value(), typeReference);
} catch (Exception ignored) {
// fall through to file
}
}
Path payloadFile = buildPayloadFile(taskId);
if (!Files.isRegularFile(payloadFile)) {
parsedPayloadLocalCache.remove(taskId);
return null;
}
try {
T parsed = objectMapper.readValue(Files.readString(payloadFile), typeReference);
parsedPayloadLocalCache.put(taskId, new LocalParsedPayloadCacheEntry(now, parsed));
return parsed;
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍒犻櫎鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
public void saveProgress(Long taskId, java.util.Map<String, String> values) {
saveProgress(taskId, values, false);
}
@@ -150,13 +98,15 @@ public class DeleteBrandTaskCacheService {
return result;
}
java.util.List<Object> pipelineResults = stringRedisTemplate.executePipelined((org.springframework.data.redis.core.RedisCallback<Object>) connection -> {
org.springframework.data.redis.serializer.RedisSerializer<String> serializer = stringRedisTemplate.getStringSerializer();
for (Long taskId : missingTaskIds) {
connection.hGetAll(serializer.serialize(buildProgressKey(taskId)));
}
return null;
});
java.util.List<Object> pipelineResults = stringRedisTemplate.executePipelined(
(org.springframework.data.redis.core.RedisCallback<Object>) connection -> {
org.springframework.data.redis.serializer.RedisSerializer<String> serializer =
stringRedisTemplate.getStringSerializer();
for (Long taskId : missingTaskIds) {
connection.hGetAll(serializer.serialize(buildProgressKey(taskId)));
}
return null;
});
for (int i = 0; i < missingTaskIds.size(); i++) {
Long taskId = missingTaskIds.get(i);
@@ -175,83 +125,12 @@ public class DeleteBrandTaskCacheService {
return result;
}
public boolean hasSameResultChunk(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
if (chunkIndex == null || chunkIndex <= 0) {
throw new BusinessException("chunkIndex 涓嶅悎娉?");
}
if (fileIdentity == null || fileIdentity.isBlank()) {
throw new BusinessException("fileIdentity 涓嶈兘涓虹┖");
}
String field = fileIdentity.trim() + "#" + chunkIndex;
String chunkJson;
try {
chunkJson = objectMapper.writeValueAsString(chunk);
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍒犻櫎鍝佺墝缁撴灉鍒嗙墖澶辫触");
}
Object existing = stringRedisTemplate.opsForHash().get(buildResultChunksKey(taskId), field);
if (chunkJson.equals(existing)) {
stringRedisTemplate.expire(buildResultChunksKey(taskId), Duration.ofHours(PAYLOAD_TTL_HOURS));
return true;
}
return false;
}
public boolean mergeResultChunk(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
String resultKey = buildResultChunksKey(taskId);
if (chunkIndex == null || chunkIndex <= 0) {
throw new BusinessException("chunkIndex 涓嶅悎娉?");
}
if (fileIdentity == null || fileIdentity.isBlank()) {
throw new BusinessException("fileIdentity 涓嶈兘涓虹┖");
}
String field = fileIdentity.trim() + "#" + chunkIndex;
String chunkJson;
try {
chunkJson = objectMapper.writeValueAsString(chunk);
} catch (Exception ex) {
throw new BusinessException("鏆傚瓨鍒犻櫎鍝佺墝缁撴灉鍒嗙墖澶辫触");
}
Object existing = stringRedisTemplate.opsForHash().get(resultKey, field);
if (chunkJson.equals(existing)) {
stringRedisTemplate.expire(resultKey, Duration.ofHours(PAYLOAD_TTL_HOURS));
return false;
}
stringRedisTemplate.opsForHash().put(resultKey, field, chunkJson);
stringRedisTemplate.expire(resultKey, Duration.ofHours(PAYLOAD_TTL_HOURS));
return true;
}
public boolean storeResultChunkIfChanged(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
return mergeResultChunk(taskId, fileIdentity, chunkIndex, chunk);
}
public java.util.Map<String, java.util.List<String>> groupResultChunkJsonByFile(Long taskId) {
java.util.Map<Object, Object> stored = stringRedisTemplate.opsForHash().entries(buildResultChunksKey(taskId));
java.util.Map<String, java.util.List<String>> grouped = new java.util.LinkedHashMap<>();
for (java.util.Map.Entry<Object, Object> entry : stored.entrySet()) {
if (!(entry.getKey() instanceof String field) || !(entry.getValue() instanceof String raw) || raw.isBlank()) {
continue;
}
int sep = field.lastIndexOf('#');
if (sep <= 0) {
continue;
}
String fileIdentity = field.substring(0, sep);
grouped.computeIfAbsent(fileIdentity, ignored -> new java.util.ArrayList<>()).add(raw);
}
return grouped;
}
public void delete(Long taskId) {
parsedPayloadLocalCache.remove(taskId);
progressLocalCache.remove(taskId);
progressRedisFlushAt.remove(taskId);
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildProgressKey(taskId));
stringRedisTemplate.delete(buildResultChunksKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
deleteTaskDirQuietly(taskId);
}
public void saveTaskCache(com.nanri.aiimage.modules.task.model.entity.FileTaskEntity task) {
@@ -264,13 +143,18 @@ public class DeleteBrandTaskCacheService {
objectMapper.convertValue(task, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class)
));
try {
stringRedisTemplate.opsForValue().set(buildTaskEntityKey(task.getId()), objectMapper.writeValueAsString(task), Duration.ofHours(PAYLOAD_TTL_HOURS));
stringRedisTemplate.opsForValue().set(
buildTaskEntityKey(task.getId()),
objectMapper.writeValueAsString(task),
Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ignored) {
}
}
public java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> getTaskCacheBatch(java.util.List<Long> taskIds) {
java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> result = new java.util.LinkedHashMap<>();
public java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> getTaskCacheBatch(
java.util.List<Long> taskIds) {
java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> result =
new java.util.LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
@@ -285,7 +169,8 @@ public class DeleteBrandTaskCacheService {
for (Long taskId : normalized) {
LocalTaskEntityCacheEntry cached = taskEntityLocalCache.get(taskId);
if (isLocalTaskEntityCacheFresh(cached, now)) {
result.put(taskId, objectMapper.convertValue(cached.task(), com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class));
result.put(taskId, objectMapper.convertValue(
cached.task(), com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class));
} else {
missingIds.add(taskId);
}
@@ -313,40 +198,10 @@ public class DeleteBrandTaskCacheService {
return result;
}
private Path buildTaskDir(Long taskId) {
return Path.of(System.getProperty("java.io.tmpdir"), "delete-brand-task-cache", String.valueOf(taskId));
}
private Path buildPayloadFile(Long taskId) {
return buildTaskDir(taskId).resolve("parsed-payload.json");
}
private void deleteTaskDirQuietly(Long taskId) {
Path taskDir = buildTaskDir(taskId);
if (!Files.exists(taskDir)) {
return;
}
try (var walk = Files.walk(taskDir)) {
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (Exception ignored) {
}
});
} catch (Exception ignored) {
}
}
private String buildTaskEntityKey(Long taskId) {
return "delete-brand:task:entity:" + taskId;
}
private record LocalParsedPayloadCacheEntry(long cachedAtMillis, Object value) {
private boolean isFresh(long now) {
return now - cachedAtMillis <= LOCAL_PARSED_PAYLOAD_CACHE_MILLIS;
}
}
private record LocalProgressCacheEntry(long cachedAtMillis, java.util.Map<String, String> values) {
private boolean isFresh(long now) {
return now - cachedAtMillis <= LOCAL_PROGRESS_CACHE_MILLIS;
@@ -377,8 +232,4 @@ public class DeleteBrandTaskCacheService {
private String buildProgressKey(Long taskId) {
return "delete-brand:task:progress:" + taskId;
}
private String buildResultChunksKey(Long taskId) {
return "delete-brand:task:result-chunks:" + taskId;
}
}

View File

@@ -0,0 +1,404 @@
package com.nanri.aiimage.modules.deletebrand.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.deletebrand.model.cache.DeleteBrandParsedFileCacheDto;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandResultFileDto;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
@RequiredArgsConstructor
@Slf4j
public class DeleteBrandTaskStorageService {
private static final String MODULE_TYPE = "DELETE_BRAND";
private static final String OSS_POINTER_PREFIX = "oss:";
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final ObjectMapper objectMapper;
private final OssStorageService ossStorageService;
@Transactional
public void saveParsedPayload(Long taskId, Map<String, DeleteBrandParsedFileCacheDto> parsedPayloadByScope) {
if (taskId == null || taskId <= 0 || parsedPayloadByScope == null || parsedPayloadByScope.isEmpty()) {
return;
}
LocalDateTime now = LocalDateTime.now();
Map<String, TaskScopeStateEntity> existingByHash = loadScopeStateByHash(taskId);
for (Map.Entry<String, DeleteBrandParsedFileCacheDto> entry : parsedPayloadByScope.entrySet()) {
String scopeKey = normalizeScopeKey(entry.getKey());
if (scopeKey.isBlank() || entry.getValue() == null) {
continue;
}
String scopeHash = hash(scopeKey);
TaskScopeStateEntity existing = existingByHash.get(scopeHash);
String payloadJson = writeJson(entry.getValue(), "failed to save delete-brand parsed payload");
String parsedPayloadPointer = storeParsedPayload(taskId, scopeHash, payloadJson);
if (existing == null) {
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
entity.setScopeKey(scopeKey);
entity.setScopeHash(scopeHash);
entity.setParsedPayloadJson(parsedPayloadPointer);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
entity.setLastChunkAt(null);
entity.setLastError(null);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(entity);
existingByHash.put(scopeHash, entity);
continue;
} catch (DuplicateKeyException ex) {
log.info("[delete-brand-storage] parsed payload scope inserted concurrently taskId={} scopeKey={}", taskId, scopeKey);
existing = getScopeState(taskId, scopeHash);
}
}
if (existing == null) {
throw new BusinessException("failed to save delete-brand parsed payload");
}
deleteParsedPayloadIfNeeded(existing.getParsedPayloadJson(), parsedPayloadPointer);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, existing.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadPointer)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
}
public Map<String, DeleteBrandParsedFileCacheDto> loadParsedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return Map.of();
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getParsedPayloadJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
if (states == null || states.isEmpty()) {
return Map.of();
}
Map<String, DeleteBrandParsedFileCacheDto> result = new LinkedHashMap<>();
for (TaskScopeStateEntity state : states) {
if (isBlank(state.getScopeKey())) {
continue;
}
String payloadJson = resolveParsedPayload(state.getParsedPayloadJson());
if (isBlank(payloadJson)) {
continue;
}
try {
result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, DeleteBrandParsedFileCacheDto.class));
} catch (Exception ex) {
throw new BusinessException("failed to load delete-brand parsed payload");
}
}
return result;
}
@Transactional
public boolean storeResultChunkIfChanged(Long taskId, String scopeKey, Integer chunkIndex, DeleteBrandResultFileDto chunk) {
if (taskId == null || taskId <= 0) {
throw new BusinessException("taskId is invalid");
}
if (chunkIndex == null || chunkIndex <= 0) {
throw new BusinessException("chunkIndex is invalid");
}
String normalizedScopeKey = normalizeScopeKey(scopeKey);
if (normalizedScopeKey.isBlank()) {
throw new BusinessException("scopeKey must not be blank");
}
String scopeHash = hash(normalizedScopeKey);
String payloadJson = writeJson(chunk, "failed to save delete-brand result chunk");
String payloadHash = hash(payloadJson);
LocalDateTime now = LocalDateTime.now();
TaskChunkEntity existingChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
.last("limit 1"));
boolean changed = true;
if (existingChunk == null) {
TaskChunkEntity entity = new TaskChunkEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
entity.setScopeKey(normalizedScopeKey);
entity.setScopeHash(scopeHash);
entity.setChunkIndex(chunkIndex);
entity.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
entity.setPayloadJson(payloadJson);
entity.setPayloadHash(payloadHash);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskChunkMapper.insert(entity);
} catch (DuplicateKeyException ex) {
TaskChunkEntity concurrentChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
.last("limit 1"));
if (concurrentChunk != null) {
changed = !payloadHash.equals(concurrentChunk.getPayloadHash());
concurrentChunk.setScopeKey(normalizedScopeKey);
concurrentChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
concurrentChunk.setPayloadJson(payloadJson);
concurrentChunk.setPayloadHash(payloadHash);
concurrentChunk.setUpdatedAt(now);
taskChunkMapper.updateById(concurrentChunk);
} else {
throw ex;
}
}
} else {
changed = !payloadHash.equals(existingChunk.getPayloadHash());
existingChunk.setScopeKey(normalizedScopeKey);
existingChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
existingChunk.setPayloadJson(payloadJson);
existingChunk.setPayloadHash(payloadHash);
existingChunk.setUpdatedAt(now);
taskChunkMapper.updateById(existingChunk);
}
refreshScopeState(taskId, normalizedScopeKey, scopeHash, chunk.getChunkTotal(), now);
return changed;
}
public Map<String, List<DeleteBrandResultFileDto>> loadMergedChunks(Long taskId) {
if (taskId == null || taskId <= 0) {
return Map.of();
}
List<TaskChunkEntity> entities = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskChunkEntity::getScopeHash)
.orderByAsc(TaskChunkEntity::getChunkIndex)
.orderByAsc(TaskChunkEntity::getId));
if (entities == null || entities.isEmpty()) {
return Map.of();
}
Map<String, List<DeleteBrandResultFileDto>> grouped = new LinkedHashMap<>();
for (TaskChunkEntity entity : entities) {
if (entity.getPayloadJson() == null || entity.getPayloadJson().isBlank()) {
continue;
}
try {
DeleteBrandResultFileDto dto = objectMapper.readValue(entity.getPayloadJson(), DeleteBrandResultFileDto.class);
grouped.computeIfAbsent(entity.getScopeKey(), ignored -> new ArrayList<>()).add(dto);
} catch (Exception ex) {
throw new BusinessException("failed to load delete-brand result chunks");
}
}
for (List<DeleteBrandResultFileDto> chunks : grouped.values()) {
chunks.sort(Comparator.comparing(DeleteBrandResultFileDto::getChunkIndex, Comparator.nullsLast(Integer::compareTo)));
}
return grouped;
}
@Transactional
public void deleteTaskData(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (states != null) {
for (TaskScopeStateEntity state : states) {
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null);
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
}
private void refreshScopeState(Long taskId, String scopeKey, String scopeHash, Integer chunkTotal, LocalDateTime now) {
Long receivedCountRaw = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash));
int receivedCount = receivedCountRaw == null ? 0 : receivedCountRaw.intValue();
int normalizedChunkTotal = chunkTotal == null || chunkTotal < 0 ? 0 : chunkTotal;
TaskScopeStateEntity state = getScopeState(taskId, scopeHash);
if (state == null) {
state = new TaskScopeStateEntity();
state.setTaskId(taskId);
state.setModuleType(MODULE_TYPE);
state.setScopeKey(scopeKey);
state.setScopeHash(scopeHash);
state.setParsedPayloadJson(null);
state.setChunkTotal(normalizedChunkTotal);
state.setReceivedChunkCount(receivedCount);
state.setCompleted(normalizedChunkTotal > 0 && receivedCount >= normalizedChunkTotal ? 1 : 0);
state.setLastChunkAt(now);
state.setLastError(null);
state.setCreatedAt(now);
state.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(state);
} catch (DuplicateKeyException ex) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getChunkTotal, normalizedChunkTotal)
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.set(TaskScopeStateEntity::getCompleted, normalizedChunkTotal > 0 && receivedCount >= normalizedChunkTotal ? 1 : 0)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getLastError, null)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
return;
}
int mergedChunkTotal = Math.max(state.getChunkTotal() == null ? 0 : state.getChunkTotal(), normalizedChunkTotal);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getChunkTotal, mergedChunkTotal)
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.set(TaskScopeStateEntity::getCompleted, mergedChunkTotal > 0 && receivedCount >= mergedChunkTotal ? 1 : 0)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getLastError, null)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
private TaskScopeStateEntity getScopeState(Long taskId, String scopeHash) {
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.last("limit 1"));
}
private Map<String, TaskScopeStateEntity> loadScopeStateByHash(Long taskId) {
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
Map<String, TaskScopeStateEntity> result = new LinkedHashMap<>();
if (states == null) {
return result;
}
for (TaskScopeStateEntity state : states) {
if (!isBlank(state.getScopeHash())) {
result.put(state.getScopeHash(), state);
}
}
return result;
}
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
try {
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson);
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new BusinessException("failed to save delete-brand parsed payload");
}
}
private String resolveParsedPayload(String value) {
if (isBlank(value)) {
return value;
}
String ossPointer = extractOssPointer(value);
if (ossPointer == null) {
return value;
}
try {
return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length()));
} catch (Exception ex) {
throw new BusinessException("failed to load delete-brand parsed payload");
}
}
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractOssPointer(oldValue);
String newPointer = extractOssPointer(newValue);
if (oldPointer == null || oldPointer.equals(newPointer)) {
return;
}
try {
ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length()));
} catch (Exception ignored) {
}
}
private String extractOssPointer(String value) {
if (isBlank(value)) {
return null;
}
if (value.startsWith(OSS_POINTER_PREFIX)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException(message);
}
}
private String normalizeScopeKey(String scopeKey) {
return scopeKey == null ? "" : scopeKey.trim();
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalizeScopeKey(value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new IllegalStateException("failed to hash delete-brand scope", ex);
}
}
}

View File

@@ -2,14 +2,18 @@ package com.nanri.aiimage.modules.file.service.oss;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.model.OSSObject;
import com.nanri.aiimage.config.OssProperties;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.Objects;
import java.util.UUID;
@Service
@@ -33,6 +37,68 @@ public class OssStorageService {
}
}
public String uploadText(String objectKey, String content) {
if (objectKey == null || objectKey.isBlank()) {
throw new IllegalArgumentException("objectKey must not be blank");
}
OSS ossClient = buildClient();
try (ByteArrayInputStream stream = new ByteArrayInputStream(
Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8))) {
ossClient.putObject(ossProperties.getBucket(), objectKey, stream);
return objectKey;
} catch (Exception ex) {
throw new IllegalStateException("failed to upload text to oss", ex);
} finally {
ossClient.shutdown();
}
}
public String uploadTaskScopePayload(String moduleType, Long taskId, String scopeHash, String content) {
String normalizedModuleType = moduleType == null || moduleType.isBlank()
? "unknown"
: moduleType.trim().toLowerCase();
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
String objectKey = String.format("task-scope/%s/%s/%s.json", normalizedModuleType, taskId, normalizedScopeHash);
return uploadText(objectKey, content);
}
public String uploadTaskParsedPayload(String moduleType, Long taskId, String scopeHash, String content) {
String normalizedModuleType = moduleType == null || moduleType.isBlank()
? "unknown"
: moduleType.trim().toLowerCase();
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
String objectKey = String.format("task-parsed/%s/%s/%s.json", normalizedModuleType, taskId, normalizedScopeHash);
return uploadText(objectKey, content);
}
public String readObjectAsString(String value) {
String objectKey = resolveObjectKey(value);
if (objectKey == null || objectKey.isBlank()) {
return null;
}
OSS ossClient = buildClient();
try (OSSObject ossObject = ossClient.getObject(ossProperties.getBucket(), objectKey)) {
return new String(ossObject.getObjectContent().readAllBytes(), StandardCharsets.UTF_8);
} catch (Exception ex) {
throw new IllegalStateException("failed to read object from oss", ex);
} finally {
ossClient.shutdown();
}
}
public void deleteObject(String value) {
String objectKey = resolveObjectKey(value);
if (objectKey == null || objectKey.isBlank()) {
return;
}
OSS ossClient = buildClient();
try {
ossClient.deleteObject(ossProperties.getBucket(), objectKey);
} finally {
ossClient.shutdown();
}
}
/**
* 根据 objectKey 生成预签名下载 URL1小时有效
*/

View File

@@ -8,7 +8,7 @@ import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteResultItemVo;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import java.io.File;
@@ -40,8 +40,9 @@ public class PatrolDeleteExcelAssemblyService {
};
public void writeWorkbook(File outputXlsx, List<PatrolDeleteResultItemVo> items) {
try (XSSFWorkbook workbook = new XSSFWorkbook();
FileOutputStream outputStream = new FileOutputStream(outputXlsx)) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream outputStream = new FileOutputStream(outputXlsx)) {
Sheet sheet = workbook.createSheet("巡店删除结果");
Row headerRow = sheet.createRow(0);
for (int columnIndex = 0; columnIndex < HEADER.length; columnIndex++) {
@@ -73,13 +74,17 @@ public class PatrolDeleteExcelAssemblyService {
rowIndex++;
}
for (int columnIndex = 0; columnIndex < HEADER.length; columnIndex++) {
sheet.autoSizeColumn(columnIndex);
}
applyDefaultColumnWidths(sheet);
workbook.write(outputStream);
} catch (Exception ex) {
log.warn("[patrol-delete] write workbook failed: {}", ex.getMessage());
throw new BusinessException("生成巡店删除结果 Excel 失败: " + ex.getMessage());
throw new BusinessException("generate patrol-delete excel failed: " + ex.getMessage());
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}
@@ -141,4 +146,23 @@ public class PatrolDeleteExcelAssemblyService {
private String safe(String value) {
return value == null ? "" : value;
}
private void applyDefaultColumnWidths(Sheet sheet) {
int[] widths = {
20,
14, 10, 12, 16,
14, 10, 12, 16,
14, 10, 12, 16,
14, 10, 12, 16,
14, 10, 12, 16,
10, 16,
10, 16,
10, 16,
10, 16,
10, 16
};
for (int i = 0; i < widths.length; i++) {
sheet.setColumnWidth(i, widths[i] * 256);
}
}
}

View File

@@ -1,10 +1,10 @@
package com.nanri.aiimage.modules.patroldelete.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteShopPayloadDto;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -20,80 +20,40 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor
public class PatrolDeleteTaskCacheService {
private static final String MODULE_TYPE = "PATROL_DELETE";
private static final long PAYLOAD_TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public PatrolDeleteShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return null;
}
Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey);
if (!(raw instanceof String json) || json.isBlank()) {
return null;
}
try {
return objectMapper.readValue(json, PatrolDeleteShopPayloadDto.class);
} catch (Exception ex) {
throw new BusinessException("读取巡店删除店铺缓存失败");
}
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PatrolDeleteShopPayloadDto.class);
}
public void saveShopMergedPayload(Long taskId, String shopKey, PatrolDeleteShopPayloadDto payload) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
return;
}
try {
stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload));
stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(PAYLOAD_TTL_HOURS));
touchTaskHeartbeat(taskId);
} catch (Exception ex) {
throw new BusinessException("暂存巡店删除店铺缓存失败");
}
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
touchTaskHeartbeat(taskId);
}
public void removeShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return;
}
stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey);
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
}
public Map<String, PatrolDeleteShopPayloadDto> getAllShopMergedPayload(Long taskId) {
try {
Map<Object, Object> raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId));
if (raw == null || raw.isEmpty()) {
return Map.of();
}
Map<String, PatrolDeleteShopPayloadDto> out = new LinkedHashMap<>();
for (Map.Entry<Object, Object> entry : raw.entrySet()) {
if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) {
continue;
}
out.put(key, objectMapper.readValue(val, PatrolDeleteShopPayloadDto.class));
}
return out;
} catch (Exception ex) {
throw new BusinessException("读取巡店删除缓存失败");
}
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, PatrolDeleteShopPayloadDto.class);
}
public boolean hasAnyShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
return size != null && size > 0;
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
}
public long countShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
return size == null ? 0L : size;
return taskScopePayloadStorageService.countScopePayload(taskId, MODULE_TYPE);
}
public long getTaskHeartbeatMillis(Long taskId) {
@@ -123,9 +83,9 @@ public class PatrolDeleteTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildShopPayloadKey(taskId));
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
public void saveTaskCache(FileTaskEntity task) {
@@ -190,10 +150,6 @@ public class PatrolDeleteTaskCacheService {
return result;
}
private String buildShopPayloadKey(Long taskId) {
return "patrol-delete:task:shop-payload:" + taskId;
}
private String buildTaskHeartbeatKey(Long taskId) {
return "patrol-delete:task:heartbeat:" + taskId;
}

View File

@@ -93,5 +93,14 @@ public class PriceTrackSubmitResultRequest {
@Schema(description = "状态", example = "SUCCESS")
private String status;
@Schema(description = "Python 确认是否需要从 skip ASIN 表删除当前 ASIN", example = "true")
private Boolean deleteSkipAsin;
@Schema(description = "需要删除的 skip ASIN不传时默认使用 asin 字段", example = "B0ABCDE123")
private String removeAsin;
@Schema(description = "删除确认原因,便于排查日志", example = "CURRENT_PRICE_BELOW_MINIMUM")
private String deleteReason;
}
}

View File

@@ -18,6 +18,9 @@ public class PriceTrackCreateTaskVo {
@Schema(description = "all skip asins grouped by country")
private Map<String, List<String>> skipAsinsByCountry;
@Schema(description = "all skip asin details grouped by country code, each item includes asin and minimumPrice")
private Map<String, List<Map<String, String>>> skipAsinDetailsByCountry;
@Schema(description = "parsed asin rows by country")
private Map<String, List<Map<String, String>>> asinRowsByCountry;

View File

@@ -17,6 +17,9 @@ public class PriceTrackMatchShopsVo {
@Schema(description = "All skip ASINs grouped by country code")
private Map<String, List<String>> skipAsinsByCountry;
@Schema(description = "All skip ASIN details grouped by country code, each item includes asin and minimumPrice")
private Map<String, List<Map<String, String>>> skipAsinDetailsByCountry;
@Schema(description = "Parsed asin rows grouped by country code")
private Map<String, List<Map<String, String>>> asinRowsByCountry;

View File

@@ -6,7 +6,7 @@ import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import java.io.File;
@@ -20,25 +20,27 @@ import java.util.Map;
public class PriceTrackExcelAssemblyService {
private static final String[] RESULT_HEADER = {
"\u5e97\u94fa\u5546\u57ce\u540d\u79f0",
"店铺商城名称",
"ASIN",
"\u4ef7\u683c",
"\u63a8\u8350\u4ef7",
"\u6700\u4f4e\u4ef7",
"\u7b2c\u4e00\u540d",
"\u7b2c\u4e8c\u540d",
"\u8d2d\u7269\u8f66\u5e97\u94fa\u540d",
"\u6539\u4ef7\u60c5\u51b5",
"\u4fee\u6539\u6b21\u6570",
"\u72b6\u6001"
"价格",
"推荐价",
"最低价",
"第一名",
"第二名",
"购物车店铺名",
"改价情况",
"修改次数",
"状态"
};
public void writeWorkbook(File outputXlsx, Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> countries) {
Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> safe = countries == null ? Map.of() : countries;
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream fos = new FileOutputStream(outputXlsx)) {
for (ProductRiskCountryCode code : ProductRiskCountryCode.values()) {
String sheetName = code.name();
Sheet sheet = wb.createSheet(sheetName);
Sheet sheet = workbook.createSheet(sheetName);
Row headerRow = sheet.createRow(0);
for (int i = 0; i < RESULT_HEADER.length; i++) {
headerRow.createCell(i).setCellValue(RESULT_HEADER[i]);
@@ -62,14 +64,18 @@ public class PriceTrackExcelAssemblyService {
row.createCell(9).setCellValue(valueOf(item.getModifyCount()));
row.createCell(10).setCellValue(valueOf(item.getStatus()));
}
for (int i = 0; i < RESULT_HEADER.length; i++) {
sheet.autoSizeColumn(i);
}
applyDefaultColumnWidths(sheet);
}
wb.write(fos);
workbook.write(fos);
} catch (Exception ex) {
log.warn("[price-track] write workbook failed: {}", ex.getMessage());
throw new BusinessException("鐢熸垚 Excel 澶辫触: " + ex.getMessage());
throw new BusinessException("generate price-track excel failed: " + ex.getMessage());
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}
@@ -105,4 +111,10 @@ public class PriceTrackExcelAssemblyService {
return value == null ? "" : value;
}
private void applyDefaultColumnWidths(Sheet sheet) {
int[] widths = {22, 18, 12, 12, 12, 20, 20, 22, 18, 12, 12};
for (int i = 0; i < widths.length; i++) {
sheet.setColumnWidth(i, widths[i] * 256);
}
}
}

View File

@@ -148,6 +148,9 @@ public class PriceTrackService {
Map<String, List<String>> skipAsinsByCountry = asinMode
? new LinkedHashMap<>()
: skipPriceAsinService.listAllSkipAsinsByCountry();
Map<String, List<Map<String, String>>> skipAsinDetailsByCountry = asinMode
? new LinkedHashMap<>()
: skipPriceAsinService.listAllSkipAsinDetailsByCountry();
Map<String, List<Map<String, String>>> asinRowsByCountry =
!asinMode
? new LinkedHashMap<>()
@@ -157,6 +160,7 @@ public class PriceTrackService {
PriceTrackMatchShopsVo vo = new PriceTrackMatchShopsVo();
vo.setSkipAsinsByCountry(skipAsinsByCountry);
vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry);
vo.setAsinRowsByCountry(asinRowsByCountry);
vo.setMinimumPriceByCountryAndAsin(priceTrackTaskService.buildMinimumPriceLookupForMatch(asinRowsByCountry));
for (String shopName : ordered) {

View File

@@ -1,10 +1,10 @@
package com.nanri.aiimage.modules.pricetrack.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequest;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -20,10 +20,12 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor
public class PriceTrackTaskCacheService {
private static final String MODULE_TYPE = "PRICE_TRACK";
private static final long HEARTBEAT_TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public void touchTaskHeartbeat(Long taskId) {
@@ -52,65 +54,27 @@ public class PriceTrackTaskCacheService {
}
public PriceTrackSubmitResultRequest.ShopResult getShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return null;
}
Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey);
if (!(raw instanceof String json) || json.isBlank()) {
return null;
}
try {
return objectMapper.readValue(json, PriceTrackSubmitResultRequest.ShopResult.class);
} catch (Exception ex) {
throw new BusinessException("read price-track cache failed");
}
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PriceTrackSubmitResultRequest.ShopResult.class);
}
public void saveShopMergedPayload(Long taskId, String shopKey, PriceTrackSubmitResultRequest.ShopResult payload) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
return;
}
try {
stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload));
stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(HEARTBEAT_TTL_HOURS));
touchTaskHeartbeat(taskId);
} catch (Exception ex) {
throw new BusinessException("save price-track cache failed");
}
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
touchTaskHeartbeat(taskId);
}
public void removeShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return;
}
stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey);
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
}
public Map<String, PriceTrackSubmitResultRequest.ShopResult> getAllShopMergedPayload(Long taskId) {
try {
Map<Object, Object> raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId));
if (raw == null || raw.isEmpty()) {
return Map.of();
}
LinkedHashMap<String, PriceTrackSubmitResultRequest.ShopResult> out = new LinkedHashMap<>();
for (Map.Entry<Object, Object> entry : raw.entrySet()) {
if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) {
continue;
}
out.put(key, objectMapper.readValue(val, PriceTrackSubmitResultRequest.ShopResult.class));
}
return out;
} catch (Exception ex) {
throw new BusinessException("read price-track cache failed");
}
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, PriceTrackSubmitResultRequest.ShopResult.class);
}
public boolean hasAnyShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
return size != null && size > 0;
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
}
public void deleteTaskCache(Long taskId) {
@@ -118,9 +82,9 @@ public class PriceTrackTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildShopPayloadKey(taskId));
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
public void saveTaskCache(FileTaskEntity task) {
@@ -185,10 +149,6 @@ public class PriceTrackTaskCacheService {
return result;
}
private String buildShopPayloadKey(Long taskId) {
return "price-track:task:shop-payload:" + taskId;
}
private String buildTaskHeartbeatKey(Long taskId) {
return "price-track:task:heartbeat:" + taskId;
}

View File

@@ -206,7 +206,7 @@ public class PriceTrackTaskService {
@Transactional
public PriceTrackPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 涓嶈兘涓虹┖");
if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 不能为空");
String norm = ziniaoShopSwitchService.normalizeShopName(shopName);
if (norm.isBlank()) throw new BusinessException("店铺名称无效");
@@ -293,6 +293,9 @@ public class PriceTrackTaskService {
Map<String, List<String>> skipAsinsByCountry = request.isAsinMode()
? new LinkedHashMap<>()
: skipPriceAsinService.listAllSkipAsinsByCountry();
Map<String, List<Map<String, String>>> skipAsinDetailsByCountry = request.isAsinMode()
? new LinkedHashMap<>()
: skipPriceAsinService.listAllSkipAsinDetailsByCountry();
Map<String, List<Map<String, String>>> asinRowsByCountry = request.isAsinMode()
? parseAsinRowsByCountry(request.getAsinFiles(), request.getCountryCodes())
: new LinkedHashMap<>();
@@ -348,6 +351,7 @@ public class PriceTrackTaskService {
ctx.put("asinMode", request.isAsinMode());
ctx.put("countryCodes", request.getCountryCodes());
ctx.put("skipAsinsByCountry", skipAsinsByCountry);
ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry);
ctx.put("asinRowsByCountry", asinRowsByCountry);
ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin);
ctx.put("items", uniqueItems);
@@ -368,6 +372,7 @@ public class PriceTrackTaskService {
vo.setTaskId(task.getId());
vo.setItems(snapshot);
vo.setSkipAsinsByCountry(skipAsinsByCountry);
vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry);
vo.setAsinRowsByCountry(asinRowsByCountry);
vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin);
return vo;
@@ -419,6 +424,7 @@ public class PriceTrackTaskService {
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
continue;
}
handleSkipAsinDeletionSignals(shopKey, payload);
PriceTrackSubmitResultRequest.ShopResult merged = mergeShopPayload(taskId, shopKey, payload);
if (!Boolean.TRUE.equals(merged.getSuccess())) {
continue;
@@ -833,8 +839,8 @@ public class PriceTrackTaskService {
if (value == null) {
return "";
}
return value.replace("\ufeff", "")
.replace("\u3000", " ")
return value.replace("", "")
.replace(" ", " ")
.replace("\r", " ")
.replace("\n", " ")
.trim()
@@ -1075,6 +1081,11 @@ public class PriceTrackTaskService {
merged.setPriceChangeStatus(firstNonBlank(incoming.getPriceChangeStatus(), merged.getPriceChangeStatus()));
merged.setModifyCount(firstNonBlank(incoming.getModifyCount(), merged.getModifyCount()));
merged.setStatus(firstNonBlank(incoming.getStatus(), merged.getStatus()));
if (incoming.getDeleteSkipAsin() != null) {
merged.setDeleteSkipAsin(incoming.getDeleteSkipAsin());
}
merged.setRemoveAsin(firstNonBlank(incoming.getRemoveAsin(), merged.getRemoveAsin()));
merged.setDeleteReason(firstNonBlank(incoming.getDeleteReason(), merged.getDeleteReason()));
return merged;
}
private PriceTrackSubmitResultRequest.AsinResult cloneRow(PriceTrackSubmitResultRequest.AsinResult row) {
@@ -1093,8 +1104,36 @@ public class PriceTrackTaskService {
out.setPriceChangeStatus(row.getPriceChangeStatus());
out.setModifyCount(row.getModifyCount());
out.setStatus(row.getStatus());
out.setDeleteSkipAsin(row.getDeleteSkipAsin());
out.setRemoveAsin(row.getRemoveAsin());
out.setDeleteReason(row.getDeleteReason());
return out;
}
private void handleSkipAsinDeletionSignals(String shopKey,
PriceTrackSubmitResultRequest.ShopResult payload) {
if (payload == null || payload.getCountries() == null || payload.getCountries().isEmpty()) {
return;
}
for (Map.Entry<String, List<PriceTrackSubmitResultRequest.AsinResult>> entry : payload.getCountries().entrySet()) {
String countryCode = entry.getKey();
List<PriceTrackSubmitResultRequest.AsinResult> rows = entry.getValue();
if (countryCode == null || countryCode.isBlank() || rows == null || rows.isEmpty()) {
continue;
}
for (PriceTrackSubmitResultRequest.AsinResult row : rows) {
if (row == null || !Boolean.TRUE.equals(row.getDeleteSkipAsin())) {
continue;
}
String targetAsin = firstNonBlank(row.getRemoveAsin(), row.getAsin());
if (targetAsin == null || targetAsin.isBlank()) {
continue;
}
boolean removed = skipPriceAsinService.removeByShopCountryAndAsin(shopKey, countryCode, targetAsin);
log.info("[price-track] skip asin delete signal taskShop={} country={} asin={} removed={} reason={}",
shopKey, countryCode, targetAsin, removed, row.getDeleteReason());
}
}
}
private int countPayloadRows(PriceTrackSubmitResultRequest.ShopResult payload) {
if (payload == null) {
return 0;

View File

@@ -7,7 +7,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import java.io.File;
@@ -35,9 +35,11 @@ public class ProductRiskExcelAssemblyService {
public void writeWorkbook(File outputXlsx, String shopDisplayName, Map<String, List<ProductRiskRowDto>> countries) {
Map<String, List<ProductRiskRowDto>> safe = countries == null ? Map.of() : countries;
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream fos = new FileOutputStream(outputXlsx)) {
for (String sheetName : SHEET_ORDER) {
Sheet sheet = wb.createSheet(sheetName);
Sheet sheet = workbook.createSheet(sheetName);
Row headerRow = sheet.createRow(0);
for (int c = 0; c < HEADER.length; c++) {
headerRow.createCell(c).setCellValue(HEADER[c]);
@@ -53,16 +55,20 @@ public class ProductRiskExcelAssemblyService {
createTextCell(row, 4, dto == null ? null : dto.getRemoveAsin());
createTextCell(row, 5, dto == null ? null : dto.getRemoveStatus());
}
for (int i = 0; i < HEADER.length; i++) {
sheet.autoSizeColumn(i);
}
applyDefaultColumnWidths(sheet);
}
wb.write(fos);
workbook.write(fos);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
log.warn("[product-risk] write workbook failed: {}", ex.getMessage());
throw new BusinessException("生成 Excel 失败: " + ex.getMessage());
throw new BusinessException("generate product-risk excel failed: " + ex.getMessage());
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}
@@ -79,9 +85,6 @@ public class ProductRiskExcelAssemblyService {
return n;
}
/**
* 将接口中的 {@link ProductRiskCountryCode} 键转为 Excel 中文工作表名 → 行列表。
*/
public Map<String, List<ProductRiskRowDto>> normalizeCountriesMap(Map<ProductRiskCountryCode, List<ProductRiskRowDto>> raw) {
if (raw == null || raw.isEmpty()) {
return new LinkedHashMap<>();
@@ -107,4 +110,11 @@ public class ProductRiskExcelAssemblyService {
}
return fallback == null ? "" : fallback;
}
private void applyDefaultColumnWidths(Sheet sheet) {
int[] widths = {20, 22, 18, 10, 18, 18};
for (int i = 0; i < widths.length; i++) {
sheet.setColumnWidth(i, widths[i] * 256);
}
}
}

View File

@@ -34,7 +34,7 @@ import java.util.Objects;
public class ProductRiskResolveService {
/**
* 默认处理顺序:德国 英国 法国 意大利 西班牙与前端一致
* 默认国家偏好顺序:德国 -> 英国 -> 法国 -> 意大利 -> 西班牙与前端保持一致。
*/
public static final List<String> DEFAULT_COUNTRY_PREFERENCE_ORDER = List.of("DE", "UK", "FR", "IT", "ES");
@@ -78,7 +78,7 @@ public class ProductRiskResolveService {
if (indexHit == null || !indexHit.isMatched()) {
String hint = indexHit != null && indexHit.getMatchMessage() != null && !indexHit.getMatchMessage().isBlank()
? indexHit.getMatchMessage()
: "店铺索引未命中,无法加入备选";
: "店铺索引未命中,无法加入待处理列表";
throw new BusinessException(hint);
}
if (ZiniaoShopIndexService.MATCH_STATUS_CONFLICT.equals(indexHit.getMatchStatus())) {
@@ -135,7 +135,7 @@ public class ProductRiskResolveService {
}
}
if (ordered.isEmpty()) {
throw new BusinessException("shop_names 有效店铺名");
throw new BusinessException("shop_names 没有有效店铺名");
}
ProductRiskMatchShopsVo vo = new ProductRiskMatchShopsVo();
for (String shopName : ordered) {
@@ -227,7 +227,7 @@ public class ProductRiskResolveService {
}
/**
* 仅保留合法枚举名,去重并保持顺序;非法静默丢弃(用于读取库数据
* 仅保留合法国家代码,去重并保持顺序;非法静默忽略,用于读取库存量数据。
*/
private static List<String> parseValidCountryCodes(List<String> raw) {
if (raw == null || raw.isEmpty()) {

View File

@@ -1,10 +1,10 @@
package com.nanri.aiimage.modules.productrisk.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskShopPayloadDto;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -20,80 +20,40 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor
public class ProductRiskTaskCacheService {
private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE";
private static final long PAYLOAD_TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public ProductRiskShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return null;
}
Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey);
if (!(raw instanceof String json) || json.isBlank()) {
return null;
}
try {
return objectMapper.readValue(json, ProductRiskShopPayloadDto.class);
} catch (Exception ex) {
throw new BusinessException("读取商品风险店铺缓存失败");
}
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ProductRiskShopPayloadDto.class);
}
public void saveShopMergedPayload(Long taskId, String shopKey, ProductRiskShopPayloadDto payload) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
return;
}
try {
stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload));
stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(PAYLOAD_TTL_HOURS));
touchTaskHeartbeat(taskId);
} catch (Exception ex) {
throw new BusinessException("暂存商品风险店铺缓存失败");
}
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
touchTaskHeartbeat(taskId);
}
public void removeShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return;
}
stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey);
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
}
public Map<String, ProductRiskShopPayloadDto> getAllShopMergedPayload(Long taskId) {
try {
Map<Object, Object> raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId));
if (raw == null || raw.isEmpty()) {
return Map.of();
}
java.util.LinkedHashMap<String, ProductRiskShopPayloadDto> out = new java.util.LinkedHashMap<>();
for (Map.Entry<Object, Object> entry : raw.entrySet()) {
if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) {
continue;
}
out.put(key, objectMapper.readValue(val, ProductRiskShopPayloadDto.class));
}
return out;
} catch (Exception ex) {
throw new BusinessException("读取商品风险缓存失败");
}
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, ProductRiskShopPayloadDto.class);
}
public boolean hasAnyShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
return size != null && size > 0;
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
}
public long countShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
return size == null ? 0L : size;
return taskScopePayloadStorageService.countScopePayload(taskId, MODULE_TYPE);
}
public long getTaskHeartbeatMillis(Long taskId) {
@@ -116,9 +76,9 @@ public class ProductRiskTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildShopPayloadKey(taskId));
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
public void saveTaskCache(FileTaskEntity task) {
@@ -183,10 +143,6 @@ public class ProductRiskTaskCacheService {
return result;
}
private String buildShopPayloadKey(Long taskId) {
return "product-risk:task:shop-payload:" + taskId;
}
public void touchTaskHeartbeat(Long taskId) {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),

View File

@@ -189,7 +189,7 @@ public class ProductRiskTaskService {
}
/**
* 删除整条商品风险任务及其下所有店铺结果运行中已结束均可,需归属当前用户
* 删除整条商品风险任务及其下所有店铺结果运行中已结束任务都允许删除,但必须归属当前用户。
*/
@Transactional
public void deleteTask(Long taskId, Long userId) {
@@ -208,8 +208,8 @@ public class ProductRiskTaskService {
}
/**
* 从用户当前运行中的商品风险任务里,按规范化店铺名删除一条结果用于前端移除匹配列表后同步后端
* 若无对应记录则 removed=false。
* 从用户当前运行中的商品风险任务里,按规范化店铺名删除一条结果用于前端移除匹配列表后同步后端。
* 如果没有对应记录,则返回 removed=false。
*/
@Transactional
public ProductRiskPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
@@ -252,7 +252,7 @@ public class ProductRiskTaskService {
}
/**
* 删除一条 file_result 后重算父任务状态;若无剩余结果则删除任务。
* 删除一条 file_result 后重算父任务状态;如果没有剩余结果则删除任务。
*/
private void reconcileTaskAfterResultRemoval(Long taskId) {
FileTaskEntity task = loadTaskForExecution(taskId);
@@ -303,11 +303,11 @@ public class ProductRiskTaskService {
task.setFinishedAt(LocalDateTime.now());
} else if (ok > 0) {
task.setStatus("SUCCESS");
task.setErrorMessage(String.join("", allErrors));
task.setErrorMessage(String.join("; ", allErrors));
task.setFinishedAt(LocalDateTime.now());
} else {
task.setStatus("FAILED");
task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("", allErrors));
task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("; ", allErrors));
task.setFinishedAt(LocalDateTime.now());
}
try {
@@ -507,7 +507,7 @@ public class ProductRiskTaskService {
String shopKey = fr.getSourceFilename();
ProductRiskShopPayloadDto payload = shopKey == null ? null : payloadByShop.get(shopKey);
// 兼容单店任务:规范化店名偶发不一致(空格/符号差异)导致未命中,且任务与回传都只有 1 个店时按唯一店回填。
// 兼容单店任务:如果规范化店名偶发不一致导致未命中,且任务与回传都只有 1 个店时按唯一店回填。
if (payload == null && resultRows.size() == 1 && payloadByShop.size() == 1) {
payload = payloadByShop.values().iterator().next();
fallbackMatchedCount++;
@@ -517,7 +517,7 @@ public class ProductRiskTaskService {
if (payload == null) {
skippedUnmatchedCount++;
// 与删除品牌一致:支持队列逐条处理、分次回传,未在本次 payload 的店铺保持待处理
// 与删除品牌一致:支持队列逐条处理、分次回传,本次 payload 未包含的店铺保持待处理
continue;
}
matchedShopCount++;
@@ -602,11 +602,11 @@ public class ProductRiskTaskService {
task.setFinishedAt(LocalDateTime.now());
} else if (ok > 0) {
task.setStatus("SUCCESS");
task.setErrorMessage(String.join("", allErrors.isEmpty() ? batchErrors : allErrors));
task.setErrorMessage(String.join("; ", allErrors.isEmpty() ? batchErrors : allErrors));
task.setFinishedAt(LocalDateTime.now());
} else {
task.setStatus("FAILED");
task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("", allErrors));
task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("; ", allErrors));
task.setFinishedAt(LocalDateTime.now());
}
try {
@@ -1034,7 +1034,7 @@ public class ProductRiskTaskService {
continue;
}
// 兼容 Python 末次“空行 done=true”心跳视为当前国家已有行全部完成。
// 兼容 Python 末次“空行 + done=true”心跳视为当前国家已有行全部完成。
if (isDoneValue(row.getDone()) && isEmptyBusinessRow(row) && !byKey.isEmpty()) {
for (Map.Entry<String, ProductRiskRowDto> entry : byKey.entrySet()) {
ProductRiskRowDto existed = entry.getValue();
@@ -1103,8 +1103,8 @@ public class ProductRiskTaskService {
if (row == null) {
return true;
}
// 注意shopName 只是上下文信息,不作为业务行是否为空的判定条件。
// Python 末次心跳行通常只带 shopName + done=true业务字段为空。
// 注意shopName 只是上下文信息,不作为业务行是否为空的判定条件。
// Python 末次心跳行通常只带 shopName + done=true业务字段为空。
return isBlank(row.getProductAsinSku())
&& isBlank(row.getRemoveAsin())
&& isBlank(row.getStatus())

View File

@@ -71,6 +71,7 @@ public class SkipPriceAsinController {
id,
country,
request.getAsin(),
request.getMinimumPrice(),
operatorId,
Boolean.TRUE.equals(superAdmin)));
}

View File

@@ -4,10 +4,15 @@ import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
import java.math.BigDecimal;
@Data
public class SkipPriceAsinCountryUpdateRequest {
@Schema(description = "ASIN", requiredMode = Schema.RequiredMode.REQUIRED)
@NotBlank(message = "ASIN 不能为空")
private String asin;
@Schema(description = "最低价", example = "18.50")
private BigDecimal minimumPrice;
}

View File

@@ -5,6 +5,7 @@ import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
@@ -29,4 +30,8 @@ public class SkipPriceAsinCreateRequest {
* 新请求: 每个国家对应独立 ASIN例如 {"DE":"B0...", "UK":"B0..."}。
*/
private Map<String, String> asinMappings;
private BigDecimal minimumPrice;
private Map<String, BigDecimal> minimumPriceMappings;
}

View File

@@ -6,6 +6,7 @@ import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.math.BigDecimal;
import java.time.LocalDateTime;
@Data
@@ -24,18 +25,33 @@ public class SkipPriceAsinEntity {
@TableField("asin_de")
private String asinDe;
@TableField("minimum_price_de")
private BigDecimal minimumPriceDe;
@TableField("asin_uk")
private String asinUk;
@TableField("minimum_price_uk")
private BigDecimal minimumPriceUk;
@TableField("asin_fr")
private String asinFr;
@TableField("minimum_price_fr")
private BigDecimal minimumPriceFr;
@TableField("asin_it")
private String asinIt;
@TableField("minimum_price_it")
private BigDecimal minimumPriceIt;
@TableField("asin_es")
private String asinEs;
@TableField("minimum_price_es")
private BigDecimal minimumPriceEs;
@TableField("created_at")
private LocalDateTime createdAt;

View File

@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.shopkey.model.vo;
import lombok.Data;
import java.math.BigDecimal;
import java.time.LocalDateTime;
@Data
@@ -12,10 +13,15 @@ public class SkipPriceAsinItemVo {
private String groupName;
private String shopName;
private String asinDe;
private BigDecimal minimumPriceDe;
private String asinUk;
private BigDecimal minimumPriceUk;
private String asinFr;
private BigDecimal minimumPriceFr;
private String asinIt;
private BigDecimal minimumPriceIt;
private String asinEs;
private BigDecimal minimumPriceEs;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

View File

@@ -12,6 +12,9 @@ import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -79,6 +82,7 @@ public class SkipPriceAsinService {
String shopName = normalizeRequired(request.getShopName(), "店铺名不能为空");
Set<String> countries = normalizeCountries(request.getCountries());
Map<String, String> countryAsinMap = normalizeCountryAsinMap(countries, request);
Map<String, BigDecimal> countryMinimumPriceMap = normalizeCountryMinimumPriceMap(countries, request);
SkipPriceAsinEntity entity = skipPriceAsinMapper.selectOne(new LambdaQueryWrapper<SkipPriceAsinEntity>()
.eq(SkipPriceAsinEntity::getGroupId, group.getId())
@@ -91,7 +95,8 @@ public class SkipPriceAsinService {
}
for (Map.Entry<String, String> entry : countryAsinMap.entrySet()) {
setCountryAsin(entity, entry.getKey(), entry.getValue());
String country = entry.getKey();
setCountryData(entity, country, entry.getValue(), countryMinimumPriceMap.get(country));
}
if (entity.getId() == null) {
@@ -108,7 +113,7 @@ public class SkipPriceAsinService {
SkipPriceAsinEntity entity = getById(id);
validateGroupAccess(entity, operatorId, superAdmin);
String normalizedCountry = normalizeCountry(country);
setCountryAsin(entity, normalizedCountry, "");
setCountryData(entity, normalizedCountry, "", null);
if (isEmptyRow(entity)) {
skipPriceAsinMapper.deleteById(entity.getId());
return;
@@ -117,12 +122,13 @@ public class SkipPriceAsinService {
}
@Transactional
public SkipPriceAsinItemVo updateCountry(Long id, String country, String asin, Long operatorId, boolean superAdmin) {
public SkipPriceAsinItemVo updateCountry(Long id, String country, String asin, BigDecimal minimumPrice,
Long operatorId, boolean superAdmin) {
SkipPriceAsinEntity entity = getById(id);
validateGroupAccess(entity, operatorId, superAdmin);
String normalizedCountry = normalizeCountry(country);
String normalizedAsin = normalizeAsin(asin);
setCountryAsin(entity, normalizedCountry, normalizedAsin);
setCountryData(entity, normalizedCountry, normalizedAsin, normalizeMinimumPrice(minimumPrice));
skipPriceAsinMapper.updateById(entity);
SkipPriceAsinEntity saved = getById(entity.getId());
String groupName = "";
@@ -154,10 +160,15 @@ public class SkipPriceAsinService {
vo.setGroupName(groupName == null ? "" : groupName);
vo.setShopName(entity.getShopName());
vo.setAsinDe(blankToEmpty(entity.getAsinDe()));
vo.setMinimumPriceDe(entity.getMinimumPriceDe());
vo.setAsinUk(blankToEmpty(entity.getAsinUk()));
vo.setMinimumPriceUk(entity.getMinimumPriceUk());
vo.setAsinFr(blankToEmpty(entity.getAsinFr()));
vo.setMinimumPriceFr(entity.getMinimumPriceFr());
vo.setAsinIt(blankToEmpty(entity.getAsinIt()));
vo.setMinimumPriceIt(entity.getMinimumPriceIt());
vo.setAsinEs(blankToEmpty(entity.getAsinEs()));
vo.setMinimumPriceEs(entity.getMinimumPriceEs());
vo.setCreatedAt(entity.getCreatedAt());
vo.setUpdatedAt(entity.getUpdatedAt());
return vo;
@@ -213,6 +224,27 @@ public class SkipPriceAsinService {
return normalized;
}
private Map<String, BigDecimal> normalizeCountryMinimumPriceMap(Set<String> countries, SkipPriceAsinCreateRequest request) {
LinkedHashMap<String, BigDecimal> normalized = new LinkedHashMap<>();
Map<String, BigDecimal> requestMappings = request.getMinimumPriceMappings();
if (requestMappings != null && !requestMappings.isEmpty()) {
for (String country : countries) {
BigDecimal minimumPrice = requestMappings.get(country);
if (minimumPrice == null) {
minimumPrice = requestMappings.get(country.toLowerCase(Locale.ROOT));
}
normalized.put(country, normalizeMinimumPrice(minimumPrice));
}
return normalized;
}
BigDecimal minimumPrice = normalizeMinimumPrice(request.getMinimumPrice());
for (String country : countries) {
normalized.put(country, minimumPrice);
}
return normalized;
}
private String normalizeCountry(String country) {
String normalized = normalizeBlank(country).toUpperCase(Locale.ROOT);
if (!SUPPORTED_COUNTRIES.contains(normalized)) {
@@ -229,6 +261,16 @@ public class SkipPriceAsinService {
return normalized;
}
private BigDecimal normalizeMinimumPrice(BigDecimal minimumPrice) {
if (minimumPrice == null) {
return null;
}
if (minimumPrice.compareTo(BigDecimal.ZERO) < 0) {
throw new BusinessException("最低价不能小于 0");
}
return minimumPrice.setScale(2, RoundingMode.HALF_UP);
}
private String normalizeRequired(String value, String message) {
String normalized = normalizeBlank(value);
if (normalized.isEmpty()) {
@@ -245,14 +287,29 @@ public class SkipPriceAsinService {
return value == null ? "" : value;
}
private void setCountryAsin(SkipPriceAsinEntity entity, String country, String asin) {
private void setCountryData(SkipPriceAsinEntity entity, String country, String asin, BigDecimal minimumPrice) {
String value = normalizeBlank(asin);
switch (country) {
case "DE" -> entity.setAsinDe(value);
case "UK" -> entity.setAsinUk(value);
case "FR" -> entity.setAsinFr(value);
case "IT" -> entity.setAsinIt(value);
case "ES" -> entity.setAsinEs(value);
case "DE" -> {
entity.setAsinDe(value);
entity.setMinimumPriceDe(minimumPrice);
}
case "UK" -> {
entity.setAsinUk(value);
entity.setMinimumPriceUk(minimumPrice);
}
case "FR" -> {
entity.setAsinFr(value);
entity.setMinimumPriceFr(minimumPrice);
}
case "IT" -> {
entity.setAsinIt(value);
entity.setMinimumPriceIt(minimumPrice);
}
case "ES" -> {
entity.setAsinEs(value);
entity.setMinimumPriceEs(minimumPrice);
}
default -> throw new BusinessException("国家参数不支持: " + country);
}
}
@@ -262,7 +319,12 @@ public class SkipPriceAsinService {
&& normalizeBlank(entity.getAsinUk()).isEmpty()
&& normalizeBlank(entity.getAsinFr()).isEmpty()
&& normalizeBlank(entity.getAsinIt()).isEmpty()
&& normalizeBlank(entity.getAsinEs()).isEmpty();
&& normalizeBlank(entity.getAsinEs()).isEmpty()
&& entity.getMinimumPriceDe() == null
&& entity.getMinimumPriceUk() == null
&& entity.getMinimumPriceFr() == null
&& entity.getMinimumPriceIt() == null
&& entity.getMinimumPriceEs() == null;
}
public Map<String, Map<String, String>> listSkipAsinsByShops(List<String> shopNames) {
@@ -312,10 +374,78 @@ public class SkipPriceAsinService {
return result;
}
public Map<String, List<Map<String, String>>> listAllSkipAsinDetailsByCountry() {
List<SkipPriceAsinEntity> rows = skipPriceAsinMapper.selectList(
new LambdaQueryWrapper<SkipPriceAsinEntity>()
.orderByDesc(SkipPriceAsinEntity::getId));
Map<String, List<Map<String, String>>> grouped = new LinkedHashMap<>();
grouped.put("DE", new java.util.ArrayList<>());
grouped.put("UK", new java.util.ArrayList<>());
grouped.put("FR", new java.util.ArrayList<>());
grouped.put("IT", new java.util.ArrayList<>());
grouped.put("ES", new java.util.ArrayList<>());
for (SkipPriceAsinEntity row : rows) {
addCountryAsinDetail(grouped.get("DE"), row.getAsinDe(), row.getMinimumPriceDe());
addCountryAsinDetail(grouped.get("UK"), row.getAsinUk(), row.getMinimumPriceUk());
addCountryAsinDetail(grouped.get("FR"), row.getAsinFr(), row.getMinimumPriceFr());
addCountryAsinDetail(grouped.get("IT"), row.getAsinIt(), row.getMinimumPriceIt());
addCountryAsinDetail(grouped.get("ES"), row.getAsinEs(), row.getMinimumPriceEs());
}
return grouped;
}
@Transactional
public boolean removeByShopCountryAndAsin(String shopName, String country, String asin) {
String normalizedShopName = normalizeBlank(shopName);
String normalizedCountry = normalizeCountry(country);
String normalizedAsin = normalizeAsin(asin);
if (normalizedShopName.isEmpty() || normalizedAsin.isEmpty()) {
return false;
}
List<SkipPriceAsinEntity> rows = skipPriceAsinMapper.selectList(
new LambdaQueryWrapper<SkipPriceAsinEntity>()
.eq(SkipPriceAsinEntity::getShopName, normalizedShopName));
boolean changed = false;
for (SkipPriceAsinEntity row : rows) {
String existingAsin = switch (normalizedCountry) {
case "DE" -> normalizeBlank(row.getAsinDe());
case "UK" -> normalizeBlank(row.getAsinUk());
case "FR" -> normalizeBlank(row.getAsinFr());
case "IT" -> normalizeBlank(row.getAsinIt());
case "ES" -> normalizeBlank(row.getAsinEs());
default -> "";
};
if (!normalizedAsin.equals(existingAsin)) {
continue;
}
setCountryData(row, normalizedCountry, "", null);
row.setUpdatedAt(LocalDateTime.now());
if (isEmptyRow(row)) {
skipPriceAsinMapper.deleteById(row.getId());
} else {
skipPriceAsinMapper.updateById(row);
}
changed = true;
}
return changed;
}
private void addCountryAsin(Set<String> bucket, String asin) {
String normalized = normalizeBlank(asin);
if (!normalized.isEmpty()) {
bucket.add(normalized);
}
}
private void addCountryAsinDetail(List<Map<String, String>> bucket, String asin, BigDecimal minimumPrice) {
String normalizedAsin = normalizeBlank(asin);
if (normalizedAsin.isEmpty()) {
return;
}
Map<String, String> detail = new LinkedHashMap<>();
detail.put("asin", normalizedAsin);
detail.put("minimumPrice", minimumPrice == null ? "" : minimumPrice.toPlainString());
bucket.add(detail);
}
}

View File

@@ -6,7 +6,7 @@ import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchRowDto;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import java.io.File;
@@ -23,10 +23,12 @@ public class ShopMatchExcelAssemblyService {
public void writeWorkbook(File outputXlsx, Map<String, List<ShopMatchRowDto>> countries) {
Map<String, List<ShopMatchRowDto>> safe = countries == null ? Map.of() : countries;
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream fos = new FileOutputStream(outputXlsx)) {
for (ProductRiskCountryCode code : ProductRiskCountryCode.values()) {
String sheetName = code.getSheetName();
Sheet sheet = wb.createSheet(sheetName);
Sheet sheet = workbook.createSheet(sheetName);
Row header = sheet.createRow(0);
header.createCell(0).setCellValue(HEADER[0]);
header.createCell(1).setCellValue(HEADER[1]);
@@ -40,13 +42,19 @@ public class ShopMatchExcelAssemblyService {
row.createCell(0).setCellValue(item.getAsin() == null ? "" : item.getAsin());
row.createCell(1).setCellValue(item.getStatus() == null ? "" : item.getStatus());
}
sheet.autoSizeColumn(0);
sheet.autoSizeColumn(1);
sheet.setColumnWidth(0, 20 * 256);
sheet.setColumnWidth(1, 18 * 256);
}
wb.write(fos);
workbook.write(fos);
} catch (Exception ex) {
log.warn("[shop-match] write workbook failed: {}", ex.getMessage());
throw new BusinessException("生成 Excel 失败: " + ex.getMessage());
throw new BusinessException("generate shop-match excel failed: " + ex.getMessage());
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}

View File

@@ -1,24 +1,20 @@
package com.nanri.aiimage.modules.shopmatch.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchShopPayloadDto;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import com.nanri.aiimage.config.TaskPressureProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HexFormat;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -28,104 +24,34 @@ import java.util.concurrent.ConcurrentHashMap;
@Slf4j
public class ShopMatchTaskCacheService {
private static final String MODULE_TYPE = "SHOP_MATCH";
private static final long PAYLOAD_TTL_HOURS = 24;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return null;
}
Path file = buildShopPayloadFile(taskId, shopKey);
if (!Files.isRegularFile(file)) {
return null;
}
try {
PayloadFile payloadFile = objectMapper.readValue(Files.readString(file), PayloadFile.class);
if (payloadFile == null || payloadFile.payload == null) {
return null;
}
if (payloadFile.shopKey == null || !shopKey.equals(payloadFile.shopKey)) {
log.warn("[shop-match] payload file shop key mismatch taskId={} shopKey={} storedKey={}",
taskId, shopKey, payloadFile.shopKey);
return null;
}
return payloadFile.payload;
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍖归厤搴楅摵缂撳瓨澶辫触");
}
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class);
}
public void saveShopMergedPayload(Long taskId, String shopKey, ShopMatchShopPayloadDto payload) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
return;
}
try {
Files.createDirectories(buildTaskDir(taskId));
Files.writeString(
buildShopPayloadFile(taskId, shopKey),
objectMapper.writeValueAsString(new PayloadFile(shopKey, payload)),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE
);
} catch (Exception ex) {
throw new BusinessException("鏆傚瓨鍖归厤搴楅摵缁撴灉澶辫触");
}
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
}
public void removeShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return;
}
try {
Files.deleteIfExists(buildShopPayloadFile(taskId, shopKey));
} catch (IOException ex) {
log.warn("[shop-match] delete shop payload failed taskId={} shopKey={} msg={}", taskId, shopKey, ex.getMessage());
}
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
}
public Map<String, ShopMatchShopPayloadDto> getAllShopMergedPayload(Long taskId) {
try {
Path taskDir = buildTaskDir(taskId);
if (!Files.isDirectory(taskDir)) {
return Map.of();
}
LinkedHashMap<String, ShopMatchShopPayloadDto> out = new LinkedHashMap<>();
try (var stream = Files.list(taskDir)) {
for (Path path : stream
.filter(file -> file.getFileName().toString().endsWith(".json"))
.sorted(Comparator.comparing(file -> file.getFileName().toString()))
.toList()) {
PayloadFile payloadFile = objectMapper.readValue(Files.readString(path), PayloadFile.class);
if (payloadFile == null || payloadFile.shopKey == null || payloadFile.shopKey.isBlank() || payloadFile.payload == null) {
continue;
}
out.put(payloadFile.shopKey, payloadFile.payload);
}
}
return out;
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍖归厤搴楅摵缂撳瓨澶辫触");
}
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, ShopMatchShopPayloadDto.class);
}
public boolean hasAnyShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
Path taskDir = buildTaskDir(taskId);
if (!Files.isDirectory(taskDir)) {
return false;
}
try (var stream = Files.list(taskDir)) {
return stream.anyMatch(file -> Files.isRegularFile(file)
&& file.getFileName().toString().endsWith(".json"));
} catch (IOException ex) {
log.warn("[shop-match] scan task cache failed taskId={} msg={}", taskId, ex.getMessage());
return false;
}
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
}
public void deleteTaskCache(Long taskId) {
@@ -133,6 +59,7 @@ public class ShopMatchTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
try {
Path taskDir = buildTaskDir(taskId);
if (!Files.exists(taskDir)) {
@@ -140,7 +67,7 @@ public class ShopMatchTaskCacheService {
return;
}
try (var walk = Files.walk(taskDir)) {
walk.sorted(Comparator.reverseOrder()).forEach(path -> {
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (IOException ex) {
@@ -225,37 +152,10 @@ public class ShopMatchTaskCacheService {
return Path.of(System.getProperty("java.io.tmpdir"), "shop-match-cache", String.valueOf(taskId));
}
private Path buildShopPayloadFile(Long taskId, String shopKey) {
return buildTaskDir(taskId).resolve(hashShopKey(shopKey) + ".json");
}
private Path buildTaskEntityFile(Long taskId) {
return buildTaskDir(taskId).resolve("_task-entity.json");
}
private String hashShopKey(String shopKey) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(shopKey.trim().getBytes(StandardCharsets.UTF_8));
return HexFormat.of().formatHex(bytes);
} catch (Exception ex) {
throw new IllegalStateException("failed to hash shop key", ex);
}
}
private static class PayloadFile {
public String shopKey;
public ShopMatchShopPayloadDto payload;
public PayloadFile() {
}
public PayloadFile(String shopKey, ShopMatchShopPayloadDto payload) {
this.shopKey = shopKey;
this.payload = payload;
}
}
private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) {
return cached != null
&& now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis());

View File

@@ -44,8 +44,8 @@ public class SplitRunService {
private static final String ZIP_CONTENT_TYPE = "application/zip";
private static final String SPLIT_MODE_ROWS_PER_FILE = "rows_per_file";
private static final String SPLIT_MODE_PARTS = "parts";
private static final String HEADER_SUFFIX_MARKER = "idASIN\u56fd\u5bb6\u72b6\u6001\u4ef7\u683c\u53d8\u4f53\u6570\u91cf";
private static final String HEADER_STOP_COLUMN = "\u7f29\u7565\u56fe\u5730\u57408";
private static final String HEADER_SUFFIX_MARKER = "idASIN国家状态价格变体数量";
private static final String HEADER_STOP_COLUMN = "缩略图地址8";
private static final DateTimeFormatter ZIP_NAME_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd");
private final FileTaskMapper fileTaskMapper;

View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskChunkMapper extends BaseMapper<TaskChunkEntity> {
}

View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskScopeStateMapper extends BaseMapper<TaskScopeStateEntity> {
}

View File

@@ -0,0 +1,26 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_chunk")
public class TaskChunkEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private String scopeKey;
private String scopeHash;
private Integer chunkIndex;
private Integer chunkTotal;
private String payloadJson;
private String payloadHash;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

View File

@@ -0,0 +1,29 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_scope_state")
public class TaskScopeStateEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private String scopeKey;
private String scopeHash;
private String parsedPayloadJson;
private String stateJson;
private Integer chunkTotal;
private Integer receivedChunkCount;
private Integer completed;
private LocalDateTime lastChunkAt;
private String lastError;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

View File

@@ -0,0 +1,172 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.TaskPressureProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileTime;
import java.time.Duration;
import java.time.Instant;
import java.util.Comparator;
import java.util.List;
@Service
@RequiredArgsConstructor
@Slf4j
public class TaskScopePayloadBufferMaintenanceService {
private static final Duration RECOVERY_LOCK_TTL = Duration.ofMinutes(10);
private static final Duration CLEANUP_LOCK_TTL = Duration.ofMinutes(10);
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final TaskPressureProperties taskPressureProperties;
private final DistributedJobLockService distributedJobLockService;
@EventListener(ApplicationReadyEvent.class)
public void recoverBufferedPayloadsOnStartup() {
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("task-scope-buffer:recover", RECOVERY_LOCK_TTL);
if (lockHandle == null) {
log.info("[task-scope-buffer] skip startup recovery because another instance holds the lock");
return;
}
try (lockHandle) {
Path root = taskScopePayloadStorageService.getBufferedPayloadRoot();
if (!Files.isDirectory(root)) {
return;
}
int limit = Math.max(0, taskPressureProperties.getScopePayloadRecoveryMaxFiles());
if (limit == 0) {
return;
}
int recovered = 0;
int scanned = 0;
for (Path file : listBufferedPayloadFiles(root, limit)) {
scanned++;
try {
BufferFileRef ref = parseBufferFile(root, file);
if (ref == null) {
Files.deleteIfExists(file);
continue;
}
if (taskScopePayloadStorageService.recoverBufferedScopePayload(ref.taskId(), ref.moduleType(), ref.scopeHash())) {
recovered++;
}
} catch (Exception ex) {
log.warn("[task-scope-buffer] startup recovery failed path={} msg={}", file, ex.getMessage());
}
}
if (scanned > 0) {
log.info("[task-scope-buffer] startup recovery finished scanned={} recovered={} limit={}", scanned, recovered, limit);
}
}
}
@Scheduled(cron = "${aiimage.task-pressure.scope-payload-cleanup-cron:15 */30 * * * *}")
public void cleanupExpiredBufferedPayloads() {
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("task-scope-buffer:cleanup", CLEANUP_LOCK_TTL);
if (lockHandle == null) {
log.info("[task-scope-buffer] skip cleanup because another instance holds the lock");
return;
}
try (lockHandle) {
Path root = taskScopePayloadStorageService.getBufferedPayloadRoot();
if (!Files.isDirectory(root)) {
return;
}
Instant cutoff = Instant.now().minus(Duration.ofHours(Math.max(1L, taskPressureProperties.getScopePayloadBufferRetentionHours())));
int deleted = 0;
for (Path file : listBufferedPayloadFiles(root, Integer.MAX_VALUE)) {
try {
FileTime lastModified = Files.getLastModifiedTime(file);
if (lastModified.toInstant().isAfter(cutoff)) {
continue;
}
Files.deleteIfExists(file);
deleted++;
deleteEmptyParents(file.getParent(), root);
} catch (Exception ex) {
log.warn("[task-scope-buffer] cleanup failed path={} msg={}", file, ex.getMessage());
}
}
if (deleted > 0) {
log.info("[task-scope-buffer] cleanup finished deleted={} root={}", deleted, root);
}
}
}
private List<Path> listBufferedPayloadFiles(Path root, int limit) {
try (var walk = Files.walk(root, 4)) {
return walk
.filter(Files::isRegularFile)
.filter(path -> path.getFileName().toString().endsWith(".json"))
.sorted(Comparator.comparing(this::safeLastModified))
.limit(limit)
.toList();
} catch (IOException ex) {
log.warn("[task-scope-buffer] scan failed root={} msg={}", root, ex.getMessage());
return List.of();
}
}
private Instant safeLastModified(Path path) {
try {
return Files.getLastModifiedTime(path).toInstant();
} catch (IOException ex) {
return Instant.EPOCH;
}
}
private BufferFileRef parseBufferFile(Path root, Path file) {
Path relative = root.relativize(file);
if (relative.getNameCount() != 3) {
return null;
}
String moduleType = relative.getName(0).toString().toUpperCase();
String taskIdRaw = relative.getName(1).toString();
String fileName = relative.getName(2).toString();
if (!fileName.endsWith(".json")) {
return null;
}
try {
long taskId = Long.parseLong(taskIdRaw);
String scopeHash = fileName.substring(0, fileName.length() - 5);
if (scopeHash.isBlank()) {
return null;
}
return new BufferFileRef(moduleType, taskId, scopeHash);
} catch (NumberFormatException ex) {
return null;
}
}
private void deleteEmptyParents(Path start, Path root) {
Path current = start;
while (current != null && !current.equals(root)) {
try (var list = Files.list(current)) {
if (list.findAny().isPresent()) {
return;
}
} catch (IOException ex) {
return;
}
try {
Files.deleteIfExists(current);
} catch (IOException ex) {
return;
}
current = current.getParent();
}
}
private record BufferFileRef(String moduleType, Long taskId, String scopeHash) {}
}

View File

@@ -0,0 +1,542 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class TaskScopePayloadStorageService {
private static final Set<String> OSS_BACKED_STATE_MODULES = new HashSet<>(Set.of(
"SHOP_MATCH",
"PRODUCT_RISK_RESOLVE",
"PRICE_TRACK",
"PATROL_DELETE"
));
private static final String OSS_POINTER_PREFIX = "oss:";
private static final String LOCAL_BUFFER_DIR = "task-scope-buffer";
private final TaskScopeStateMapper taskScopeStateMapper;
private final ObjectMapper objectMapper;
private final OssStorageService ossStorageService;
private final TaskPressureProperties taskPressureProperties;
@Transactional
public <T> void saveScopePayload(Long taskId, String moduleType, String scopeKey, T payload) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey) || payload == null) {
return;
}
String normalizedScopeKey = normalize(scopeKey);
String scopeHash = hash(normalizedScopeKey);
String payloadJson = writeJson(payload, "写入任务范围载荷失败");
LocalDateTime now = LocalDateTime.now();
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
if (shouldStoreStatePayloadInOss(moduleType)) {
saveOssBackedScopePayload(taskId, moduleType, normalizedScopeKey, scopeHash, payloadJson, now, state);
return;
}
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
if (state == null) {
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setScopeKey(normalizedScopeKey);
entity.setScopeHash(scopeHash);
entity.setStateJson(stateJson);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
entity.setLastChunkAt(now);
entity.setLastError(null);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(entity);
return;
} catch (DuplicateKeyException ignored) {
state = getScopeState(taskId, moduleType, scopeHash);
}
}
if (state == null) {
throw new BusinessException("写入任务范围载荷失败");
}
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
.set(TaskScopeStateEntity::getStateJson, stateJson)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
public <T> T getScopePayload(Long taskId, String moduleType, String scopeKey, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
return null;
}
TaskScopeStateEntity state = getScopeState(taskId, moduleType, hash(normalize(scopeKey)));
String payloadJson = resolveStatePayload(state);
if (payloadJson == null) {
return null;
}
try {
return objectMapper.readValue(payloadJson, clazz);
} catch (Exception ex) {
throw new BusinessException("读取任务范围载荷失败");
}
}
public <T> Map<String, T> getAllScopePayload(Long taskId, String moduleType, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return Map.of();
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getScopeHash, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType)
.isNotNull(TaskScopeStateEntity::getStateJson));
if (states == null || states.isEmpty()) {
return Map.of();
}
Map<String, T> result = new LinkedHashMap<>();
for (TaskScopeStateEntity state : states) {
String payloadJson = resolveStatePayload(state);
if (payloadJson == null || isBlank(state.getScopeKey())) {
continue;
}
try {
result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, clazz));
} catch (Exception ex) {
throw new BusinessException("读取任务范围载荷失败");
}
}
return result;
}
@Transactional
public void removeScopePayload(Long taskId, String moduleType, String scopeKey) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
return;
}
TaskScopeStateEntity state = getScopeState(taskId, moduleType, hash(normalize(scopeKey)));
if (state == null) {
return;
}
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
deleteStatePayloadIfNeeded(state.getStateJson());
boolean hasParsedPayload = !isBlank(state.getParsedPayloadJson());
if (hasParsedPayload) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getStateJson, null)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
return;
}
taskScopeStateMapper.deleteById(state.getId());
}
public boolean hasAnyScopePayload(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return false;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType)
.isNotNull(TaskScopeStateEntity::getStateJson));
return count != null && count > 0;
}
public long countScopePayload(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return 0L;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType)
.isNotNull(TaskScopeStateEntity::getStateJson));
return count == null ? 0L : count;
}
@Transactional
public void deleteTaskScopePayloads(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return;
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType));
if (states == null || states.isEmpty()) {
return;
}
for (TaskScopeStateEntity state : states) {
if (state == null || state.getId() == null) {
continue;
}
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
deleteStatePayloadIfNeeded(state.getStateJson());
if (!isBlank(state.getParsedPayloadJson())) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getStateJson, null)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
} else {
taskScopeStateMapper.deleteById(state.getId());
}
}
}
@Transactional
public <T> void saveParsedPayloadMap(Long taskId, String moduleType, Map<String, T> payloadByScope) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || payloadByScope == null || payloadByScope.isEmpty()) {
return;
}
LocalDateTime now = LocalDateTime.now();
for (Map.Entry<String, T> entry : payloadByScope.entrySet()) {
if (isBlank(entry.getKey()) || entry.getValue() == null) {
continue;
}
String scopeKey = normalize(entry.getKey());
String scopeHash = hash(scopeKey);
String parsedPayloadJson = writeJson(entry.getValue(), "写入任务解析载荷失败");
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
if (state == null) {
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setScopeKey(scopeKey);
entity.setScopeHash(scopeHash);
entity.setParsedPayloadJson(parsedPayloadJson);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(entity);
continue;
} catch (DuplicateKeyException ignored) {
state = getScopeState(taskId, moduleType, scopeHash);
}
}
if (state == null) {
throw new BusinessException("写入任务解析载荷失败");
}
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedPayloadJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadJson)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
}
public <T> Map<String, T> getParsedPayloadMap(Long taskId, String moduleType, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return Map.of();
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getParsedPayloadJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType)
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
if (states == null || states.isEmpty()) {
return Map.of();
}
Map<String, T> result = new LinkedHashMap<>();
for (TaskScopeStateEntity state : states) {
if (isBlank(state.getParsedPayloadJson()) || isBlank(state.getScopeKey())) {
continue;
}
try {
result.put(state.getScopeKey(), objectMapper.readValue(resolveParsedPayload(state.getParsedPayloadJson()), clazz));
} catch (Exception ex) {
throw new BusinessException("读取任务原始载荷失败");
}
}
return result;
}
@Transactional
public boolean recoverBufferedScopePayload(Long taskId, String moduleType, String scopeHash) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeHash)) {
return false;
}
String payloadJson = readBufferedPayload(taskId, moduleType, scopeHash);
if (payloadJson == null) {
return false;
}
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
if (state == null) {
deleteBufferedPayload(taskId, moduleType, scopeHash);
return false;
}
if (!shouldStoreStatePayloadInOss(moduleType)) {
deleteBufferedPayload(taskId, moduleType, scopeHash);
return false;
}
LocalDateTime now = LocalDateTime.now();
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getStateJson, stateJson)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now));
deleteBufferedPayload(taskId, moduleType, scopeHash);
return true;
}
public Path getBufferedPayloadRoot() {
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR);
}
private TaskScopeStateEntity getScopeState(Long taskId, String moduleType, String scopeHash) {
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.last("limit 1"));
}
private String storeStatePayload(String moduleType, Long taskId, String scopeHash, String payloadJson) {
if (!shouldStoreStatePayloadInOss(moduleType)) {
return payloadJson;
}
try {
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskScopePayload(moduleType, taskId, scopeHash, payloadJson);
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new BusinessException("写入任务范围载荷失败");
}
}
private String resolveStatePayload(TaskScopeStateEntity state) {
if (state == null || isBlank(state.getStateJson())) {
return null;
}
String bufferedPayload = readBufferedPayload(state.getTaskId(), state.getModuleType(), state.getScopeHash());
if (bufferedPayload != null) {
return bufferedPayload;
}
String ossPointer = extractOssPointer(state.getStateJson());
if (ossPointer == null) {
return state.getStateJson();
}
try {
return ossStorageService.readObjectAsString(stripOssPointerPrefix(ossPointer));
} catch (Exception ex) {
throw new BusinessException("读取任务范围载荷失败");
}
}
private void deleteStatePayloadIfNeeded(String value) {
String ossPointer = extractOssPointer(value);
if (ossPointer == null) {
return;
}
try {
ossStorageService.deleteObject(stripOssPointerPrefix(ossPointer));
} catch (Exception ignored) {
}
}
private String resolveParsedPayload(String value) {
if (isBlank(value) || !isOssPointer(value)) {
return value;
}
try {
return ossStorageService.readObjectAsString(stripOssPointerPrefix(value));
} catch (Exception ex) {
throw new BusinessException("读取任务解析载荷失败");
}
}
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
if (isBlank(oldValue) || oldValue.equals(newValue) || !isOssPointer(oldValue)) {
return;
}
try {
ossStorageService.deleteObject(stripOssPointerPrefix(oldValue));
} catch (Exception ignored) {
}
}
private void deleteReplacedStatePayloadIfNeeded(String oldValue, String newValue) {
if (isBlank(oldValue) || oldValue.equals(newValue)) {
return;
}
deleteStatePayloadIfNeeded(oldValue);
}
private void saveOssBackedScopePayload(Long taskId,
String moduleType,
String normalizedScopeKey,
String scopeHash,
String payloadJson,
LocalDateTime now,
TaskScopeStateEntity state) {
if (state == null) {
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setScopeKey(normalizedScopeKey);
entity.setScopeHash(scopeHash);
entity.setStateJson(stateJson);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
entity.setLastChunkAt(now);
entity.setLastError(null);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(entity);
deleteBufferedPayload(taskId, moduleType, scopeHash);
return;
} catch (DuplicateKeyException ignored) {
state = getScopeState(taskId, moduleType, scopeHash);
}
}
if (state == null) {
throw new BusinessException("刷新任务范围缓冲载荷失败");
}
writeBufferedPayload(taskId, moduleType, scopeHash, payloadJson);
if (!shouldFlushBufferedPayload(state, now)) {
return;
}
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
.set(TaskScopeStateEntity::getStateJson, stateJson)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now));
deleteBufferedPayload(taskId, moduleType, scopeHash);
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException(message);
}
}
private String normalize(String value) {
return value == null ? "" : value.trim();
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
private boolean shouldStoreStatePayloadInOss(String moduleType) {
return OSS_BACKED_STATE_MODULES.contains(normalize(moduleType).toUpperCase());
}
private boolean isOssPointer(String value) {
return !isBlank(value) && value.startsWith(OSS_POINTER_PREFIX);
}
private String extractOssPointer(String value) {
if (isBlank(value)) {
return null;
}
if (isOssPointer(value)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return isOssPointer(decoded) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private String stripOssPointerPrefix(String value) {
return value.substring(OSS_POINTER_PREFIX.length());
}
private boolean shouldFlushBufferedPayload(TaskScopeStateEntity state, LocalDateTime now) {
long flushIntervalMillis = Math.max(0L, taskPressureProperties.getScopePayloadFlushIntervalMillis());
if (flushIntervalMillis <= 0L || state.getUpdatedAt() == null) {
return true;
}
return java.time.Duration.between(state.getUpdatedAt(), now).toMillis() >= flushIntervalMillis;
}
private void writeBufferedPayload(Long taskId, String moduleType, String scopeHash, String payloadJson) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
Files.createDirectories(path.getParent());
Files.writeString(path, payloadJson, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
} catch (IOException ex) {
throw new BusinessException("写入任务范围缓冲载荷失败");
}
}
private String readBufferedPayload(Long taskId, String moduleType, String scopeHash) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
if (!Files.isRegularFile(path)) {
return null;
}
return Files.readString(path);
} catch (IOException ex) {
return null;
}
}
private void deleteBufferedPayload(Long taskId, String moduleType, String scopeHash) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
Files.deleteIfExists(path);
} catch (IOException ignored) {
}
}
private Path buildBufferedPayloadPath(Long taskId, String moduleType, String scopeHash) {
String normalizedModuleType = normalize(moduleType).toLowerCase();
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR, normalizedModuleType, String.valueOf(taskId), scopeHash + ".json");
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new IllegalStateException("failed to hash scope key", ex);
}
}
}

View File

@@ -50,7 +50,7 @@ public class ZiniaoShopSwitchService {
if (value == null) {
return "";
}
return value.replace("\u3000", " ").trim();
return value.replace(" ", " ").trim();
}
public boolean isSkippableMatchError(BusinessException ex) {
@@ -59,9 +59,9 @@ public class ZiniaoShopSwitchService {
return false;
}
return (message.contains("code=40004")
|| message.contains("userId\u5b58\u5728\u65e0\u6548\u7684\u53c2\u6570\u503c")
|| message.contains("\u65e0\u6743\u9650")
|| message.contains("\u6ca1\u6709\u6743\u9650"))
&& !message.contains("\u767d\u540d\u5355");
|| message.contains("userId存在无效的参数值")
|| message.contains("无权限")
|| message.contains("没有权限"))
&& !message.contains("白名单");
}
}

View File

@@ -99,6 +99,10 @@ aiimage:
task-pressure:
local-task-entity-cache-millis: ${AIIMAGE_TASK_LOCAL_ENTITY_CACHE_MILLIS:3000}
db-select-batch-size: ${AIIMAGE_TASK_DB_SELECT_BATCH_SIZE:200}
scope-payload-flush-interval-millis: ${AIIMAGE_TASK_SCOPE_PAYLOAD_FLUSH_INTERVAL_MILLIS:15000}
scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24}
scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200}
scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *}
security:
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}

View File

@@ -0,0 +1,76 @@
SET @schema_name = DATABASE();
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_de'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_de DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''DE minimum price'' AFTER asin_de'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_uk'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_uk DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''UK minimum price'' AFTER asin_uk'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_fr'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_fr DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''FR minimum price'' AFTER asin_fr'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_it'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_it DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''IT minimum price'' AFTER asin_it'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_es'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_es DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''ES minimum price'' AFTER asin_es'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;

View File

@@ -0,0 +1,36 @@
CREATE TABLE IF NOT EXISTS biz_task_scope_state (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier',
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
parsed_payload_json JSON NULL COMMENT 'parsed source payload',
state_json JSON NULL COMMENT 'merged intermediate state',
chunk_total INT NOT NULL DEFAULT 0 COMMENT 'expected chunk count',
received_chunk_count INT NOT NULL DEFAULT 0 COMMENT 'received chunk count',
completed TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'whether scope is complete',
last_chunk_at DATETIME NULL COMMENT 'last chunk received time',
last_error VARCHAR(1000) NULL COMMENT 'last error message',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_scope (task_id, module_type, scope_hash),
KEY idx_task_completed (task_id, module_type, completed),
KEY idx_scope_hash (scope_hash)
) COMMENT='task scope state';
CREATE TABLE IF NOT EXISTS biz_task_chunk (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier',
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
chunk_index INT NOT NULL COMMENT 'chunk index starting from 1',
chunk_total INT NOT NULL DEFAULT 0 COMMENT 'total chunk count',
payload_json JSON NOT NULL COMMENT 'chunk payload',
payload_hash CHAR(64) NOT NULL COMMENT 'hash of payload json',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_scope_chunk (task_id, module_type, scope_hash, chunk_index),
KEY idx_task_scope (task_id, module_type, scope_hash),
KEY idx_task_module (task_id, module_type)
) COMMENT='task chunk detail';