super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

店铺数据抓取累计文件改为店铺级共享，按国家覆盖更新

Browse Source 
同店同日在不同账号下产生多份互不相干的累计文件（唯一键含 user_id），
后台管理页每店只显示最新一份，其他账号抓的国家看起来丢失。

- V95 迁移：加 shop_key/country_codes_json/compensation_done 列，按店铺
  合并存量 daily_file 与成员行，唯一键改为 (shop_key, business_date)
- DailyFileService：findForUpdate/findOlder/acquireLock 去 user 维度，
  店铺级锁跨账号串行
- TaskService：聚合按店铺定位；applyCountryCoverage 按国家覆盖（本次
  回传的国家替换旧行，未更新的国家保留）；启动补偿组件按成员快照重建
  合并文件并回填国家列表
- 管理页：country_codes 改从 daily_file.country_codes_json 取，行数用
  累计文件实际值
This commit is contained in:
黄自达
2026-08-30 16:30:09 +08:00
parent ae95b294ec
commit 97ae8a7c51
11 changed files with 495 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlDailyFileEntity.java
+2
View File
@@ -24,8 +24,10 @@ public class ShopDataCrawlDailyFileEntity {
private String resultFileUrl;
private Long resultFileSize;
private String resultContentType;
private String countryCodesJson;
private Integer rowCount;
private Long version;
private Integer compensationDone;
private LocalDateTime lastSuccessAt;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlDailyCompensationRunner.java
+29
View File
@@ -0,0 +1,29 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.stereotype.Component;
/**
* V95 店铺级累计文件迁移的启动补偿：对 compensation_done=0 的每日累计文件，
* 按成员 result 快照重建合并工作簿（覆盖国家语义），补齐国家列表并清理旧对象。
* 失败仅记日志，不阻断应用启动与后续补偿。
*/
@Component
@RequiredArgsConstructor
@Slf4j
public class ShopDataCrawlDailyCompensationRunner implements ApplicationRunner {
private final ShopDataCrawlTaskService taskService;
@Override
public void run(ApplicationArguments args) {
try {
taskService.compensateDailyWorkbooksAfterMigration();
} catch (Exception ex) {
log.warn("[shop-data-crawl] daily compensation runner failed: {}", ex.getMessage(), ex);
}
}
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlDailyFileService.java
+24 -11
View File
@@ -73,34 +73,47 @@ public class ShopDataCrawlDailyFileService {
return HexFormat.of().formatHex(digest);
}
public TaskDistributedLockService.LockHandle acquireLock(Long userId, String shopKey) {
if (userId == null || userId <= 0 || shopKey == null || shopKey.isBlank()) {
public TaskDistributedLockService.LockHandle acquireLock(String shopKey) {
if (shopKey == null || shopKey.isBlank()) {
return null;
}
// The lock protects the shop's whole daily-file lifecycle. Including the
// date would allow yesterday and today to update the same shop together.
String identity = userId + "|" + shopKey;
// 店铺级锁：只按店铺串行，跨账号同店并发抓取时互斥（锁在 Redis，与 DB 行解耦）
String identity = shopKey;
String lockModule = DAILY_LOCK_MODULE_PREFIX + shopKeyHash(identity);
return taskDistributedLockService.acquire(lockModule, 1L, DAILY_LOCK_TTL, DAILY_LOCK_WAIT_MILLIS);
}
public ShopDataCrawlDailyFileEntity findForUpdate(Long userId, String shopKeyHash, LocalDate businessDate) {
if (userId == null || shopKeyHash == null || businessDate == null) {
public ShopDataCrawlDailyFileEntity findForUpdate(String shopKeyHash, LocalDate businessDate) {
if (shopKeyHash == null || businessDate == null) {
return null;
}
return dailyFileMapper.selectOne(new LambdaQueryWrapper<ShopDataCrawlDailyFileEntity>()
.eq(ShopDataCrawlDailyFileEntity::getUserId, userId)
.eq(ShopDataCrawlDailyFileEntity::getShopKeyHash, shopKeyHash)
.eq(ShopDataCrawlDailyFileEntity::getBusinessDate, businessDate)
.last("FOR UPDATE"));
}
public List<ShopDataCrawlDailyFileEntity> findOlder(Long userId, String shopKeyHash, LocalDate businessDate) {
if (userId == null || shopKeyHash == null || businessDate == null) {
public List<ShopDataCrawlDailyFileEntity> findForCompensation() {
return dailyFileMapper.selectList(new LambdaQueryWrapper<ShopDataCrawlDailyFileEntity>()
.eq(ShopDataCrawlDailyFileEntity::getCompensationDone, 0)
.orderByAsc(ShopDataCrawlDailyFileEntity::getBusinessDate)
.orderByAsc(ShopDataCrawlDailyFileEntity::getId)
.last("LIMIT 500"));
}
public void markCompensationDone(ShopDataCrawlDailyFileEntity entity) {
if (entity == null || entity.getId() == null) {
return;
}
entity.setCompensationDone(1);
update(entity);
}
public List<ShopDataCrawlDailyFileEntity> findOlder(String shopKeyHash, LocalDate businessDate) {
if (shopKeyHash == null || businessDate == null) {
return List.of();
}
return dailyFileMapper.selectList(new LambdaQueryWrapper<ShopDataCrawlDailyFileEntity>()
.eq(ShopDataCrawlDailyFileEntity::getUserId, userId)
.eq(ShopDataCrawlDailyFileEntity::getShopKeyHash, shopKeyHash)
.lt(ShopDataCrawlDailyFileEntity::getBusinessDate, businessDate)
.orderByDesc(ShopDataCrawlDailyFileEntity::getBusinessDate)
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java
+249 -90
View File
@@ -63,6 +63,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -676,7 +677,7 @@ public class ShopDataCrawlTaskService {
ensureTaskOwnedByCurrentInstance(task, "delete shop data crawl task");
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
List<FileResultEntity> taskRows = listTaskRows(taskId);
try (DailyLockSet dailyLocks = acquireDailyLocks(task.getUserId(), taskRows)) {
try (DailyLockSet dailyLocks = acquireDailyLocks(taskRows)) {
ensureDailySyncCompletedBeforeDelete(taskRows);
Set<Long> removedResultIds = taskRows.stream()
.map(FileResultEntity::getId)
@@ -739,7 +740,7 @@ public class ShopDataCrawlTaskService {
if (latestEntity == null || !MODULE_TYPE.equals(latestEntity.getModuleType()) || !userId.equals(latestEntity.getUserId())) {
throw new BusinessException("记录不存在");
}
try (DailyLockSet dailyLocks = acquireDailyLocks(userId, List.of(latestEntity))) {
try (DailyLockSet dailyLocks = acquireDailyLocks(List.of(latestEntity))) {
deleteResultHistoryRow(latestEntity);
}
}
@@ -816,7 +817,7 @@ public class ShopDataCrawlTaskService {
continue;
}
ShopDataCrawlDailyFileEntity lockedDailyFile = dailyFileService.findForUpdate(
dailyFile.getUserId(), dailyFile.getShopKeyHash(), dailyFile.getBusinessDate());
dailyFile.getShopKeyHash(), dailyFile.getBusinessDate());
if (lockedDailyFile != null) {
dailyFile = lockedDailyFile;
}
@@ -960,6 +961,7 @@ public class ShopDataCrawlTaskService {
dailyFile.setResultFileSize(latest.result().getResultFileSize());
dailyFile.setResultContentType(CONTENT_TYPE_XLSX);
dailyFile.setRowCount(rowCount);
dailyFile.setCountryCodesJson(countryCodesJson(collectCountryCodes(snapshots)));
dailyFile.setVersion(Math.max(0L, Objects.requireNonNullElse(dailyFile.getVersion(), 0L)) + 1L);
dailyFile.setLastSuccessAt(now);
dailyFile.setUpdatedAt(now);
@@ -1801,10 +1803,9 @@ public class ShopDataCrawlTaskService {
FileResultEntity row,
ShopDataCrawlResultItemVo snapshot,
LocalDate businessDate) {
Long userId = row.getUserId() != null ? row.getUserId() : task.getUserId();
String shopKey = dailyFileService.shopKey(row);
String shopKeyHash = dailyFileService.shopKeyHash(shopKey);
if (userId == null || shopKeyHash == null) {
if (shopKeyHash == null) {
throw new BusinessException("店铺累计文件归属信息不完整");
}
// Task 36：版本号/CAS 短临界区。店铺级锁只覆盖“准备/提交”两个毫秒级短事务
@@ -1814,13 +1815,13 @@ public class ShopDataCrawlTaskService {
DailyWorkbookArtifact artifact = null;
for (int attempt = 1; attempt <= MAX_DAILY_AGGREGATION_ATTEMPTS; attempt++) {
DailyAggregationPreparation preparation;
TaskDistributedLockService.LockHandle prepareLock = dailyFileService.acquireLock(userId, shopKey);
TaskDistributedLockService.LockHandle prepareLock = dailyFileService.acquireLock(shopKey);
if (prepareLock == null) {
throw new BusinessException("店铺当天累计文件正在处理中，请稍后重试");
}
try {
preparation = executeShortTransaction(
() -> prepareDailyAggregation(userId, shopKeyHash, businessDate, row));
() -> prepareDailyAggregation(shopKeyHash, businessDate, row));
} finally {
prepareLock.close();
}
@@ -1828,7 +1829,7 @@ public class ShopDataCrawlTaskService {
return new DailyAggregationResult(List.of(), false);
}
ShopDataCrawlDailyFileEntity baseForAttempt = resolveBaseDailyFile(
preparation.dailyFile(), userId, shopKeyHash, businessDate);
preparation.dailyFile(), shopKeyHash, businessDate);
// 组装在锁外执行。每次尝试都用本次准备阶段读到的最新 base 组装
// （冲突重试时 base 已变化，复用过期的组装结果会把并发写入的行丢在对象外）；
// 零行引用对象不重复上传。
@@ -1836,7 +1837,7 @@ public class ShopDataCrawlTaskService {
artifact = assembleDailyWorkbook(task, snapshot, baseForAttempt,
excelAssemblyService.countRows(List.of(snapshot)));
}
TaskDistributedLockService.LockHandle commitLock = dailyFileService.acquireLock(userId, shopKey);
TaskDistributedLockService.LockHandle commitLock = dailyFileService.acquireLock(shopKey);
if (commitLock == null) {
throw new BusinessException("店铺当天累计文件正在处理中，请稍后重试");
}
@@ -1845,7 +1846,7 @@ public class ShopDataCrawlTaskService {
DailyWorkbookArtifact artifactForAttempt = artifact;
try {
persistedResult = executeShortTransaction(() -> persistDailyAggregation(
task, row, userId, shopKey, shopKeyHash, businessDate,
task, row, shopKey, shopKeyHash, businessDate,
preparationForAttempt, artifactForAttempt, snapshot,
preparationForAttempt.dailyFile()));
if (persistedResult.discardUploadedObject() && artifactForAttempt.uploaded()) {
@@ -1876,12 +1877,11 @@ public class ShopDataCrawlTaskService {
throw new BusinessException("店铺累计文件并发更新冲突，请稍后重试");
}
private DailyAggregationPreparation prepareDailyAggregation(Long userId,
String shopKeyHash,
private DailyAggregationPreparation prepareDailyAggregation(String shopKeyHash,
LocalDate businessDate,
FileResultEntity row) {
ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findForUpdate(
userId, shopKeyHash, businessDate);
shopKeyHash, businessDate);
if (handleExistingDailyMembership(row, dailyFile)) {
return new DailyAggregationPreparation(dailyFile, true);
}
@@ -1889,13 +1889,12 @@ public class ShopDataCrawlTaskService {
}
private ShopDataCrawlDailyFileEntity resolveBaseDailyFile(ShopDataCrawlDailyFileEntity currentDailyFile,
Long userId,
String shopKeyHash,
LocalDate businessDate) {
if (currentDailyFile != null) {
return currentDailyFile;
}
List<ShopDataCrawlDailyFileEntity> older = dailyFileService.findOlder(userId, shopKeyHash, businessDate);
List<ShopDataCrawlDailyFileEntity> older = dailyFileService.findOlder(shopKeyHash, businessDate);
return older == null || older.isEmpty() ? null : older.get(0);
}
@@ -1916,7 +1915,8 @@ public class ShopDataCrawlTaskService {
Math.max(0L, Objects.requireNonNullElse(baseDailyFile.getResultFileSize(), 0L)),
false,
filename,
Math.max(0, Objects.requireNonNullElse(baseDailyFile.getRowCount(), 0)));
Math.max(0, Objects.requireNonNullElse(baseDailyFile.getRowCount(), 0)),
parseCountryCodesJson(baseDailyFile.getCountryCodesJson()));
}
File workRoot = FileUtil.mkdir(FileUtil.file(
@@ -1935,7 +1935,8 @@ public class ShopDataCrawlTaskService {
if (blank(objectKey)) {
throw new BusinessException("累计文件上传后未返回文件地址");
}
return new DailyWorkbookArtifact(objectKey, outputXlsx.length(), true, filename, rowCount);
List<String> countryCodes = collectCountryCodes(accumulatedItems);
return new DailyWorkbookArtifact(objectKey, outputXlsx.length(), true, filename, rowCount, countryCodes);
} finally {
FileUtil.del(outputXlsx);
FileUtil.del(workRoot);
@@ -1943,8 +1944,10 @@ public class ShopDataCrawlTaskService {
}
/**
* 从数据层累积重建每日累计文件的快照列表：既有成员行按 (createdAt, id) 升序读取，
* 优先用行级 payload；payload 缺失（历史数据）时按结果快照兜底；新结果追加在末尾。
* 数据层增量模型：既有成员行按 (createdAt, id) 升序读取，优先用行级 payload；
* payload 缺失（历史数据）时按结果快照兜底；新结果追加在末尾。
* 随后按国家覆盖：同一国家以最后一次出现的成员结果为准（本次回传的国家替换旧行，
* 未回传的国家保留），item 结构与顺序保持稳定。
*/
private List<ShopDataCrawlResultItemVo> buildDailyFileFromData(ShopDataCrawlDailyFileEntity baseDailyFile,
List<ShopDataCrawlResultItemVo> appended) {
@@ -1967,7 +1970,93 @@ public class ShopDataCrawlTaskService {
accumulated.add(item);
}
}
return accumulated;
return applyCountryCoverage(accumulated);
}
/**
* 按国家覆盖：从后往前记录每个国家最后一次出现的成员下标；每个成员只保留
* "该国最后一次出现" 的国家结果（非该次出现的行被剔除）。空结果国家同样覆盖旧行。
* item 结构（resultId/shopName 等元数据）与顺序保持稳定。
*/
private List<ShopDataCrawlResultItemVo> applyCountryCoverage(List<ShopDataCrawlResultItemVo> items) {
if (items == null || items.isEmpty()) {
return items == null ? List.of() : items;
}
Map<String, Integer> lastCountryItemIndex = new HashMap<>();
for (int i = 0; i < items.size(); i++) {
ShopDataCrawlResultItemVo item = items.get(i);
if (item == null) {
continue;
}
for (ShopDataCrawlCountryResultDto countryResult : item.getCountryResults() == null
? List.<ShopDataCrawlCountryResultDto>of() : item.getCountryResults()) {
if (countryResult != null && !blank(countryResult.getCountry())) {
lastCountryItemIndex.put(countryResult.getCountry().trim().toUpperCase(), i);
}
}
}
if (lastCountryItemIndex.isEmpty()) {
return items;
}
for (int i = 0; i < items.size(); i++) {
ShopDataCrawlResultItemVo item = items.get(i);
if (item == null || item.getCountryResults() == null || item.getCountryResults().isEmpty()) {
continue;
}
List<ShopDataCrawlCountryResultDto> kept = new ArrayList<>();
for (ShopDataCrawlCountryResultDto countryResult : item.getCountryResults()) {
if (countryResult == null || blank(countryResult.getCountry())) {
continue;
}
if (Objects.equals(lastCountryItemIndex.get(countryResult.getCountry().trim().toUpperCase()), i)) {
kept.add(countryResult);
}
}
item.setCountryResults(kept);
}
return items;
}
private List<String> collectCountryCodes(List<ShopDataCrawlResultItemVo> items) {
LinkedHashSet<String> codes = new LinkedHashSet<>();
if (items != null) {
for (ShopDataCrawlResultItemVo item : items) {
if (item == null) {
continue;
}
for (ShopDataCrawlCountryResultDto countryResult : item.getCountryResults() == null
? List.<ShopDataCrawlCountryResultDto>of() : item.getCountryResults()) {
if (countryResult != null && !blank(countryResult.getCountry())) {
codes.add(countryResult.getCountry().trim().toUpperCase());
}
}
}
}
return new ArrayList<>(codes);
}
private String countryCodesJson(List<String> countryCodes) {
if (countryCodes == null || countryCodes.isEmpty()) {
return "[]";
}
try {
return objectMapper.writeValueAsString(countryCodes);
} catch (Exception ex) {
return "[]";
}
}
private List<String> parseCountryCodesJson(String json) {
if (blank(json)) {
return List.of();
}
try {
List<String> parsed = objectMapper.readValue(json, new TypeReference<List<String>>() {
});
return parsed == null ? List.of() : parsed;
} catch (Exception ex) {
return List.of();
}
}
private ShopDataCrawlResultItemVo snapshotFromPayload(ShopDataCrawlDailyMemberEntity member) {
@@ -2023,7 +2112,6 @@ public class ShopDataCrawlTaskService {
private DailyAggregationResult persistDailyAggregation(FileTaskEntity task,
FileResultEntity row,
Long userId,
String shopKey,
String shopKeyHash,
LocalDate businessDate,
@@ -2032,7 +2120,7 @@ public class ShopDataCrawlTaskService {
ShopDataCrawlResultItemVo snapshot,
ShopDataCrawlDailyFileEntity expectedBase) {
ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findForUpdate(
userId, shopKeyHash, businessDate);
shopKeyHash, businessDate);
if (handleExistingDailyMembership(row, dailyFile)) {
return new DailyAggregationResult(List.of(), true);
}
@@ -2044,7 +2132,7 @@ public class ShopDataCrawlTaskService {
}
List<ShopDataCrawlDailyFileEntity> olderFiles = dailyFileService.findOlder(
userId, shopKeyHash, businessDate);
shopKeyHash, businessDate);
Set<String> obsoleteObjectKeys = new HashSet<>();
collectObjectKey(obsoleteObjectKeys, dailyFile == null ? null : dailyFile.getResultFileUrl());
for (ShopDataCrawlDailyFileEntity older : olderFiles) {
@@ -2053,8 +2141,6 @@ public class ShopDataCrawlTaskService {
String objectKey = artifact.objectKey();
String filename = artifact.filename();
List<FileResultEntity> shopRows = findShopResultRows(userId, row);
clearShopResultPointers(shopRows, row.getId());
row.setResultFilename(filename);
row.setResultFileUrl(objectKey);
row.setResultFileSize(artifact.fileSize());
@@ -2065,7 +2151,6 @@ public class ShopDataCrawlTaskService {
LocalDateTime now = dailyFileService.currentBusinessDateTime();
if (dailyFile == null) {
dailyFile = new ShopDataCrawlDailyFileEntity();
dailyFile.setUserId(userId);
dailyFile.setShopKeyHash(shopKeyHash);
dailyFile.setShopKey(shopKey);
dailyFile.setBusinessDate(businessDate);
@@ -2081,6 +2166,8 @@ public class ShopDataCrawlTaskService {
dailyFile.setResultFileSize(row.getResultFileSize());
dailyFile.setResultContentType(CONTENT_TYPE_XLSX);
dailyFile.setRowCount(row.getRowCount());
dailyFile.setCountryCodesJson(countryCodesJson(artifact.countryCodes()));
dailyFile.setCompensationDone(1);
dailyFile.setLastSuccessAt(now);
dailyFile.setUpdatedAt(now);
if (dailyFile.getId() == null) {
@@ -2179,60 +2266,6 @@ public class ShopDataCrawlTaskService {
return template.execute(status -> action.get());
}
private List<FileResultEntity> findShopResultRows(Long userId, FileResultEntity sourceRow) {
if (userId == null || sourceRow == null) {
return List.of();
}
String shopId = trimToNull(sourceRow.getSourceFileUrl());
String shopName = trimToNull(sourceRow.getSourceFilename());
LambdaQueryWrapper<FileResultEntity> wrapper = new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.and(owner -> owner.eq(FileResultEntity::getUserId, userId)
.or().isNull(FileResultEntity::getUserId));
if (shopId != null) {
wrapper.eq(FileResultEntity::getSourceFileUrl, shopId);
} else if (shopName != null) {
wrapper.eq(FileResultEntity::getSourceFilename, shopName)
.apply("TRIM(COALESCE(source_file_url, '')) = ''");
} else {
return List.of();
}
List<FileResultEntity> candidates = fileResultMapper.selectList(wrapper);
if (candidates == null || candidates.isEmpty()) {
return List.of();
}
Map<Long, FileTaskEntity> legacyTaskOwners = loadTaskMapByIds(candidates.stream()
.filter(candidate -> candidate != null && candidate.getUserId() == null)
.map(FileResultEntity::getTaskId)
.filter(Objects::nonNull)
.distinct()
.toList());
return candidates.stream()
.filter(Objects::nonNull)
.filter(candidate -> Objects.equals(userId, candidate.getUserId())
|| Objects.equals(userId,
legacyTaskOwners.get(candidate.getTaskId()) == null
? null
: legacyTaskOwners.get(candidate.getTaskId()).getUserId()))
.toList();
}
private void clearShopResultPointers(List<FileResultEntity> rows, Long keepResultId) {
if (rows == null) {
return;
}
for (FileResultEntity candidate : rows) {
if (candidate == null || Objects.equals(candidate.getId(), keepResultId)
|| blank(candidate.getResultFileUrl())) {
continue;
}
candidate.setResultFileUrl(null);
candidate.setResultFileSize(null);
candidate.setResultContentType(null);
fileResultMapper.updateById(candidate);
}
}
private void collectObjectKey(Set<String> target, String value) {
if (target == null || blank(value)) {
return;
@@ -2286,19 +2319,18 @@ public class ShopDataCrawlTaskService {
}
}
private DailyLockSet acquireDailyLocks(Long fallbackUserId, List<FileResultEntity> rows) {
private DailyLockSet acquireDailyLocks(List<FileResultEntity> rows) {
Map<String, DailyLockRequest> requests = new TreeMap<>();
if (rows != null) {
for (FileResultEntity row : rows) {
if (row == null) {
continue;
}
Long userId = row.getUserId() != null ? row.getUserId() : fallbackUserId;
String shopKey = dailyFileService.shopKey(row);
if (userId == null || userId <= 0 || blank(shopKey)) {
if (blank(shopKey)) {
continue;
}
requests.putIfAbsent(userId + "|" + shopKey, new DailyLockRequest(userId, shopKey));
requests.putIfAbsent(shopKey, new DailyLockRequest(shopKey));
}
}
if (requests.isEmpty()) {
@@ -2307,8 +2339,7 @@ public class ShopDataCrawlTaskService {
List<TaskDistributedLockService.LockHandle> handles = new ArrayList<>();
try {
for (DailyLockRequest request : requests.values()) {
TaskDistributedLockService.LockHandle handle = dailyFileService.acquireLock(
request.userId(), request.shopKey());
TaskDistributedLockService.LockHandle handle = dailyFileService.acquireLock(request.shopKey());
if (handle == null) {
throw new BusinessException("店铺累计文件正在处理中，请稍后重试");
}
@@ -2321,7 +2352,7 @@ public class ShopDataCrawlTaskService {
}
}
private record DailyLockRequest(Long userId, String shopKey) {
private record DailyLockRequest(String shopKey) {
}
private record DailyMemberData(ShopDataCrawlDailyMemberEntity member,
@@ -2374,10 +2405,11 @@ public class ShopDataCrawlTaskService {
long fileSize,
boolean uploaded,
String filename,
int rowCount) {
int rowCount,
List<String> countryCodes) {
DailyWorkbookArtifact(String objectKey, long fileSize, boolean uploaded, String filename) {
this(objectKey, fileSize, uploaded, filename, 0);
DailyWorkbookArtifact(String objectKey, long fileSize, boolean uploaded, String filename, int rowCount) {
this(objectKey, fileSize, uploaded, filename, rowCount, List.of());
}
}
@@ -2443,6 +2475,133 @@ public class ShopDataCrawlTaskService {
}
}
/**
* V95 迁移后启动补偿：对 compensation_done=0 的每日累计文件，
* 按成员 result 快照重建一份合并工作簿（覆盖国家语义），上传新对象、
* 更新 daily_file 与最新 result 行指针、清理不再被引用的旧对象。
* 单文件失败仅记日志，不阻断其余文件与启动。
*/
public void compensateDailyWorkbooksAfterMigration() {
List<ShopDataCrawlDailyFileEntity> pending = dailyFileService.findForCompensation();
if (pending.isEmpty()) {
return;
}
log.info("[shop-data-crawl] daily compensation start, pending={}", pending.size());
int processed = 0;
for (ShopDataCrawlDailyFileEntity dailyFile : pending) {
if (dailyFile == null || dailyFile.getId() == null) {
continue;
}
try {
compensateOneDailyWorkbook(dailyFile);
processed++;
} catch (Exception ex) {
log.warn("[shop-data-crawl] daily compensation failed dailyFileId={} shopKey={} date={} msg={}",
dailyFile.getId(), dailyFile.getShopKey(), dailyFile.getBusinessDate(), safeMessage(ex));
}
}
log.info("[shop-data-crawl] daily compensation done, processed={}/{}", processed, pending.size());
}
private void compensateOneDailyWorkbook(ShopDataCrawlDailyFileEntity dailyFile) {
List<ShopDataCrawlDailyMemberEntity> members = dailyFileService.listMembers(dailyFile.getId());
List<ShopDataCrawlDailyMemberEntity> sorted = sortedDailyMembers(members);
if (sorted.isEmpty()) {
// 无成员行：标记完成，避免每次启动重试
dailyFileService.markCompensationDone(dailyFile);
return;
}
List<ShopDataCrawlResultItemVo> snapshots = new ArrayList<>();
for (ShopDataCrawlDailyMemberEntity member : sorted) {
FileResultEntity result = fileResultMapper.selectById(member.getResultId());
if (result == null) {
log.warn("[shop-data-crawl] compensation member result missing dailyFileId={} resultId={}",
dailyFile.getId(), member.getResultId());
continue;
}
ShopDataCrawlResultItemVo snapshot = loadSnapshotForDailyMember(result);
if (snapshot == null) {
log.warn("[shop-data-crawl] compensation snapshot missing dailyFileId={} resultId={} skip",
dailyFile.getId(), member.getResultId());
continue;
}
snapshots.add(snapshot);
}
if (snapshots.isEmpty()) {
dailyFileService.markCompensationDone(dailyFile);
return;
}
String filename = blank(dailyFile.getResultFilename())
? buildTaskWorkbookFilename(loadCompensationTask(dailyFile))
: dailyFile.getResultFilename();
String oldObjectKey = dailyFile.getResultFileUrl();
File workRoot = FileUtil.mkdir(FileUtil.file(
System.getProperty("java.io.tmpdir"),
"shop-data-crawl-result",
"compensation-" + dailyFile.getId()));
File outputXlsx = FileUtil.file(workRoot, filename);
try {
int rowCount = excelAssemblyService.writeWorkbook(outputXlsx, snapshots);
String newObjectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
if (blank(newObjectKey)) {
throw new BusinessException("累计文件补偿上传结果为空");
}
LocalDateTime now = dailyFileService.currentBusinessDateTime();
dailyFile.setLatestTaskId(dailyFile.getLatestTaskId());
dailyFile.setResultFilename(filename);
dailyFile.setResultFileUrl(newObjectKey);
dailyFile.setResultFileSize(outputXlsx.length());
dailyFile.setResultContentType(CONTENT_TYPE_XLSX);
dailyFile.setRowCount(rowCount);
dailyFile.setCountryCodesJson(countryCodesJson(collectCountryCodes(snapshots)));
dailyFile.setVersion(Math.max(0L, Objects.requireNonNullElse(dailyFile.getVersion(), 0L)) + 1L);
dailyFile.setCompensationDone(1);
dailyFile.setLastSuccessAt(now);
dailyFile.setUpdatedAt(now);
dailyFileService.update(dailyFile);
FileResultEntity latestResult = fileResultMapper.selectById(dailyFile.getLatestResultId());
if (latestResult != null) {
latestResult.setResultFilename(filename);
latestResult.setResultFileUrl(newObjectKey);
latestResult.setResultFileSize(outputXlsx.length());
latestResult.setResultContentType(CONTENT_TYPE_XLSX);
latestResult.setRowCount(rowCount);
fileResultMapper.updateById(latestResult);
}
// 清理旧对象（引用计数：其他 result/daily_file 仍引用则保留）
if (!blank(oldObjectKey) && !Objects.equals(oldObjectKey, newObjectKey)) {
deleteResultObjectNowIfUnreferenced(oldObjectKey);
}
log.info("[shop-data-crawl] compensation rebuilt dailyFileId={} shopKey={} rows={} object={}",
dailyFile.getId(), dailyFile.getShopKey(), rowCount, newObjectKey);
} finally {
FileUtil.del(outputXlsx);
FileUtil.del(workRoot);
}
}
private FileTaskEntity loadCompensationTask(ShopDataCrawlDailyFileEntity dailyFile) {
if (dailyFile.getLatestTaskId() != null) {
FileTaskEntity task = fileTaskMapper.selectById(dailyFile.getLatestTaskId());
if (task != null) {
return task;
}
}
List<ShopDataCrawlDailyMemberEntity> members = dailyFileService.listMembers(dailyFile.getId());
for (ShopDataCrawlDailyMemberEntity member : sortedDailyMembers(members)) {
FileTaskEntity task = fileTaskMapper.selectById(member.getTaskId());
if (task != null) {
return task;
}
}
FileTaskEntity fallback = new FileTaskEntity();
fallback.setId(0L);
fallback.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
return fallback;
}
private boolean isTaskWorkbookPending(FileTaskEntity task, List<FileResultEntity> rows) {
if (task == null || task.getId() == null || rows == null || rows.isEmpty()) {
return false;