super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(shop-data-crawl): 后台店铺数据记录改读每店最新累计档，删除任务不再连坐后台

Browse Source 
后台“店铺数据记录”此前以每次抓取的 file_result/file_task 为源，前台删除任务会物理删
这两张表，导致该店在后台退档/消失。现改为以采集归档维护、删除任务时保留的
biz_shop_data_crawl_daily_file 每店最新一档为数据源：
- V113：daily_file 冗余 shop_name 列并回填；归档写档三处入口同步写入
- 列表/批量下载/单档下载/删除端点改按每日累计档 id（daily-files/{id}）
- 管理端删除=真删该店这条数据记录（档+文件+明细），前台任务历史保留但清文件指针
- 明细行 daily_file_id 跨天滚动删旧档时改指新档，修存量悬空引用（另附修复脚本）
This commit is contained in:
黄自达
2026-09-08 15:02:25 +08:00
parent b12acf7793
commit 630bfc3eee
10 changed files with 278 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/AdminShopDataCrawlTasksController.java
+32
View File
@@ -18,6 +18,8 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@@ -98,6 +100,36 @@ public class AdminShopDataCrawlTasksController {
return ApiResponse.success(payload);
}
@GetMapping("/daily-files/{dailyFileId}/download")
@Operation(summary = "下载该店最新一次采集的数据文件（按累计档 id）")
public void downloadDailyFile(
@PathVariable Long dailyFileId,
HttpServletRequest request,
HttpServletResponse response) {
requireShopDataCrawlTaskAccess(request);
ShopDataCrawlAdminTasksService.DailyDownload daily = adminTasksService.resolveDailyDownload(dailyFileId);
try {
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
DownloadHeaderUtil.setAttachment(response, daily.filename());
try (InputStream input = URI.create(daily.url()).toURL().openStream()) {
input.transferTo(response.getOutputStream());
}
} catch (Exception ex) {
log.warn("[shop-data-crawl-admin] 下载店铺数据文件失败 dailyFileId={} msg={}", dailyFileId, ex.getMessage());
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
}
}
@DeleteMapping("/daily-files/{dailyFileId}")
@Operation(summary = "删除该店这条店铺数据记录（累计档 + 文件 + 明细，管理端显式操作）")
public ApiResponse<Void> deleteDailyFile(
@PathVariable Long dailyFileId,
HttpServletRequest request) {
requireShopDataCrawlTaskAccess(request);
adminTasksService.deleteAdminDailyFile(dailyFileId);
return ApiResponse.success(null);
}
@PostMapping("/download-zip")
@Operation(summary = "批量下载选中结果文件为 zip（文件部分失败经响应头 X-Archive-Error-Count 提示）")
public void downloadZip(
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlAdminTasksMapper.java
+61 -64
View File
@@ -6,15 +6,18 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminGroup
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminRow;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlDownloadRowDto;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.SelectProvider;
import java.util.List;
import java.util.Map;
/**
* 店铺数据抓取记录"按店铺分组列表"与批量下载的数据源查询，
* 语义逐条对照 Flask admin_api.py 中 list_shop_data_crawl_tasks /
* _load_shop_data_crawl_download_rows 的 SQL。
* 店铺数据记录"按店铺分组列表"与批量下载的数据源查询。
*
* <p>数据源为<b>每店最新一次采集的累计文件</b>（biz_shop_data_crawl_daily_file）：
* 该表由采集归档维护、删除前台任务时被保留，因此后台记录不会因客户端删除任务而消失；
* 同一店铺再次采集会生成/滚动到更新的一档，列表只取每店最新一档 —— 覆盖式语义，与日期无关。
*
* <p>参数统一走单参 Map：筛选与分页字段用语义键（shopName/groupName/country/
* createdFrom/createdTo/pageLimit/pageOffset），IN 列表键 shopNames / resultIds，
@@ -23,15 +26,15 @@ import java.util.Map;
@Mapper
public interface ShopDataCrawlAdminTasksMapper {
/** 店铺分组总数（按 TRIM(source_filename) 去重后的组数）。 */
/** 店铺分组总数（按 TRIM(shop_name) 去重后的组数）。 */
@SelectProvider(type = SqlProvider.class, method = "countShopGroups")
long countShopGroups(Map<String, Object> p);
/** 当前页店铺分组（每店最新归档时间）。 */
/** 当前页店铺分组（每店最新采集时间）。 */
@SelectProvider(type = SqlProvider.class, method = "selectShopGroupPage")
List<ShopDataCrawlAdminGroupRow> selectShopGroupPage(Map<String, Object> p);
/** 当前页每家店铺的最新结果行（窗口函数 row_number=1），需先铺平 shopNames。 */
/** 当前页每家店铺的最新档（窗口函数 row_number=1），需先铺平 shopNames。 */
@SelectProvider(type = SqlProvider.class, method = "selectLatestRowsForShops")
List<ShopDataCrawlAdminRow> selectLatestRowsForShops(Map<String, Object> p);
@@ -39,39 +42,48 @@ public interface ShopDataCrawlAdminTasksMapper {
@SelectProvider(type = SqlProvider.class, method = "selectGroupLabels")
List<ShopDataCrawlAdminGroupLabelDto> selectGroupLabels(Map<String, Object> p);
/** 每个结果文件最新一条 ASSEMBLE_RESULT 任务（按 job id 取最大），需先铺平 resultIds。 */
@SelectProvider(type = SqlProvider.class, method = "selectLatestAssembleJobs")
List<ShopDataCrawlAdminFileJobBriefDto> selectLatestAssembleJobs(Map<String, Object> p);
/** 批量下载 zip 所需结果行，需先铺平 resultIds。 */
/** 批量下载 zip 所需档行（按每日累计档 id），需先铺平 resultIds。 */
@SelectProvider(type = SqlProvider.class, method = "selectDownloadRows")
List<ShopDataCrawlDownloadRowDto> selectDownloadRows(Map<String, Object> p);
/** 按每日累计档 id 取单行（下载/删除用）。 */
@Select("""
SELECT df.id AS resultId, df.latest_task_id AS taskId, df.user_id AS userId,
df.shop_name AS sourceFilename, df.result_filename AS resultFilename,
df.result_file_url AS resultFileUrl
FROM biz_shop_data_crawl_daily_file df
WHERE df.id = #{dailyFileId}
""")
ShopDataCrawlDownloadRowDto selectDownloadRowById(@org.apache.ibatis.annotations.Param("dailyFileId") Long dailyFileId);
/** 某一累计对象 key 还被多少 daily_file 行引用。 */
@Select("""
SELECT COUNT(1) FROM biz_shop_data_crawl_daily_file
WHERE result_file_url = #{objectKey}
""")
long countDailyFileObjectReferences(@org.apache.ibatis.annotations.Param("objectKey") String objectKey);
class SqlProvider {
private static final String LATEST_TIME = "COALESCE(df.last_success_at, df.updated_at, "
+ "t.finished_at, t.updated_at, t.created_at)";
private static final String SHOP_NAME = "TRIM(COALESCE(df.shop_name, ''))";
private static final String SHOP_KEY = "TRIM(COALESCE(r.source_filename, ''))";
private static final String LATEST_TIME = "COALESCE(df.last_success_at, df.updated_at, df.created_at)";
private static final String FROM_WHERE_BASE =
" FROM biz_file_result r "
+ "JOIN biz_file_task t ON t.id = r.task_id "
+ "LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id "
+ "LEFT JOIN users u ON u.id = r.user_id "
+ "WHERE r.module_type = 'SHOP_DATA_CRAWL' "
+ "AND t.module_type = 'SHOP_DATA_CRAWL' "
+ "AND TRIM(COALESCE(r.result_file_url, '')) <> ''";
" FROM biz_shop_data_crawl_daily_file df "
+ "LEFT JOIN users u ON u.id = df.user_id "
+ "WHERE " + SHOP_NAME + " <> '' "
+ "AND df.result_file_url IS NOT NULL AND TRIM(df.result_file_url) <> ''";
public String countShopGroups(Map<String, Object> p) {
return "SELECT COUNT(*) AS total FROM (SELECT " + SHOP_KEY + " AS shopKey"
+ FROM_WHERE_BASE + whereTail(p, "") + " GROUP BY " + SHOP_KEY + ") grouped_shops";
return "SELECT COUNT(*) AS total FROM (SELECT " + SHOP_NAME + " AS shopKey"
+ FROM_WHERE_BASE + whereTail(p, "") + " GROUP BY " + SHOP_NAME + ") grouped_shops";
}
public String selectShopGroupPage(Map<String, Object> p) {
return "SELECT " + SHOP_KEY + " AS shopName, MAX(" + LATEST_TIME + ") AS latestCreatedAt"
return "SELECT " + SHOP_NAME + " AS shopName, MAX(" + LATEST_TIME + ") AS latestCreatedAt"
+ FROM_WHERE_BASE + whereTail(p, "")
+ " GROUP BY " + SHOP_KEY
+ " GROUP BY " + SHOP_NAME
+ " ORDER BY latestCreatedAt DESC, shopName ASC"
+ " LIMIT #{pageLimit} OFFSET #{pageOffset}";
}
@@ -84,23 +96,22 @@ public interface ShopDataCrawlAdminTasksMapper {
+ " ranked.taskNo, ranked.taskStatus, ranked.requestJson, ranked.taskError,"
+ " ranked.createdAt, ranked.updatedAt, ranked.finishedAt,"
+ " ranked.latestFileUpdatedAt, ranked.countryCodesJson, ranked.rowCountDisplay"
+ " FROM (SELECT r.id AS resultId, r.task_id AS taskId, r.user_id AS userId,"
+ " FROM (SELECT df.id AS resultId, df.latest_task_id AS taskId, df.user_id AS userId,"
+ " u.username AS username,"
+ " r.source_filename AS shopName, r.source_file_url AS shopId,"
+ " r.result_filename AS resultFilename, r.result_file_url AS resultFileUrl,"
+ " r.result_file_size AS resultFileSize, r.result_content_type AS resultContentType,"
+ " r.row_count AS rowCount, r.success AS resultSuccess,"
+ " r.error_message AS resultError, r.created_at AS resultCreatedAt,"
+ " t.task_no AS taskNo, t.status AS taskStatus, t.request_json AS requestJson,"
+ " t.error_message AS taskError, t.created_at AS createdAt,"
+ " t.updated_at AS updatedAt, t.finished_at AS finishedAt,"
+ " df.shop_name AS shopName, df.shop_key AS shopId,"
+ " df.result_filename AS resultFilename, df.result_file_url AS resultFileUrl,"
+ " df.result_file_size AS resultFileSize, df.result_content_type AS resultContentType,"
+ " df.row_count AS rowCount,"
+ " 1 AS resultSuccess, NULL AS resultError, df.created_at AS resultCreatedAt,"
+ " NULL AS taskNo, 'SUCCESS' AS taskStatus, NULL AS requestJson, NULL AS taskError,"
+ " df.created_at AS createdAt, df.updated_at AS updatedAt, df.last_success_at AS finishedAt,"
+ " " + LATEST_TIME + " AS latestFileUpdatedAt,"
+ " df.country_codes_json AS countryCodesJson,"
+ " COALESCE(df.row_count, r.row_count) AS rowCountDisplay,"
+ " ROW_NUMBER() OVER (PARTITION BY " + SHOP_KEY
+ " ORDER BY " + LATEST_TIME + " DESC, r.id DESC) AS rowNo"
+ " df.row_count AS rowCountDisplay,"
+ " ROW_NUMBER() OVER (PARTITION BY " + SHOP_NAME
+ " ORDER BY " + LATEST_TIME + " DESC, df.id DESC) AS rowNo"
+ FROM_WHERE_BASE
+ whereTail(p, " AND " + SHOP_KEY + " IN (" + inPlaceholders(p, "shopNames", "sn") + ")")
+ whereTail(p, " AND " + SHOP_NAME + " IN (" + inPlaceholders(p, "shopNames", "sn") + ")")
+ ") ranked WHERE ranked.rowNo = 1"
+ " ORDER BY ranked.latestFileUpdatedAt DESC, ranked.resultId DESC";
}
@@ -115,60 +126,46 @@ public interface ShopDataCrawlAdminTasksMapper {
+ " GROUP BY TRIM(sm.shop_name)";
}
public String selectLatestAssembleJobs(Map<String, Object> p) {
return "SELECT fj.result_id AS resultId, fj.id AS fileJobId,"
+ " fj.status AS fileStatus, fj.error_message AS fileError"
+ " FROM biz_task_file_job fj"
+ " INNER JOIN (SELECT result_id, MAX(id) AS max_id FROM biz_task_file_job"
+ " WHERE module_type = 'SHOP_DATA_CRAWL' AND job_type = 'ASSEMBLE_RESULT'"
+ " AND result_id IN (" + inPlaceholders(p, "resultIds", "ri") + ")"
+ " GROUP BY result_id) latest ON fj.id = latest.max_id";
}
public String selectDownloadRows(Map<String, Object> p) {
return "SELECT r.id AS resultId, r.task_id AS taskId, r.user_id AS userId,"
+ " r.source_filename AS sourceFilename, r.result_filename AS resultFilename,"
+ " r.result_file_url AS resultFileUrl, t.status AS taskStatus"
+ " FROM biz_file_result r"
+ " JOIN biz_file_task t ON t.id = r.task_id"
+ " WHERE r.module_type = 'SHOP_DATA_CRAWL' AND t.module_type = 'SHOP_DATA_CRAWL'"
+ " AND r.id IN (" + inPlaceholders(p, "resultIds", "ri") + ")";
return "SELECT df.id AS resultId, df.latest_task_id AS taskId, df.user_id AS userId,"
+ " df.shop_name AS sourceFilename, df.result_filename AS resultFilename,"
+ " df.result_file_url AS resultFileUrl, 'SUCCESS' AS taskStatus"
+ " FROM biz_shop_data_crawl_daily_file df"
+ " WHERE df.id IN (" + inPlaceholders(p, "resultIds", "ri") + ")";
}
/** 追加可选的店铺/分组/国家/时间筛选。 */
/** 追加可选的店铺/分组/国家/时间筛选（全部基于每日累计档列）。 */
private static String whereTail(Map<String, Object> p, String extraCondition) {
StringBuilder sql = new StringBuilder(extraCondition == null ? "" : extraCondition);
String shopName = (String) p.get("shopName");
if (shopName != null && !shopName.isBlank()) {
sql.append(" AND r.source_filename LIKE CONCAT('%', #{shopName}, '%')");
sql.append(" AND df.shop_name LIKE CONCAT('%', #{shopName}, '%')");
}
String groupName = (String) p.get("groupName");
if (groupName != null && !groupName.isBlank()) {
sql.append(" AND EXISTS (SELECT 1 FROM biz_shop_manage sm"
+ " LEFT JOIN biz_shop_manage_group g ON g.id = sm.group_id"
+ " WHERE TRIM(COALESCE(sm.shop_name, '')) = "
+ "TRIM(COALESCE(r.source_filename, ''))"
+ "TRIM(COALESCE(df.shop_name, ''))"
+ " AND COALESCE(NULLIF(g.group_name, ''), NULLIF(sm.group_name, ''))"
+ " LIKE CONCAT('%', #{groupName}, '%'))");
}
if (p.get("country") != null && !((String) p.get("country")).isBlank()) {
sql.append(" AND JSON_CONTAINS(COALESCE(df.country_codes_json,"
+ " JSON_EXTRACT(t.request_json, '$.countryCodes'),"
+ " JSON_EXTRACT(t.request_json, '$.country_codes'), '[]'),"
sql.append(" AND JSON_CONTAINS(COALESCE(df.country_codes_json, '[]'),"
+ " CONCAT('\"', #{country}, '\"'))");
}
if (p.get("createdFrom") != null) {
sql.append(" AND t.created_at >= #{createdFrom}");
sql.append(" AND df.created_at >= #{createdFrom}");
}
if (p.get("createdTo") != null) {
sql.append(" AND t.created_at <= #{createdTo}");
sql.append(" AND df.created_at <= #{createdTo}");
}
Object visibleShopKeys = p.get("visibleShopKeys");
if (visibleShopKeys instanceof List<?> keys) {
if (keys.isEmpty()) {
sql.append(" AND 1 = 0");
} else {
sql.append(" AND LOWER(TRIM(COALESCE(r.source_filename, ''))) IN (")
sql.append(" AND LOWER(TRIM(COALESCE(df.shop_name, ''))) IN (")
.append(inPlaceholders(p, "visibleShopKeys", "lv")).append(')');
}
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlItemMapper.java
+7
View File
@@ -15,4 +15,11 @@ public interface ShopDataCrawlItemMapper extends BaseMapper<ShopDataCrawlItemEnt
WHERE shop_name = #{shopName} AND business_date = #{businessDate}
""")
int deleteBatch(@Param("shopName") String shopName, @Param("businessDate") java.time.LocalDate businessDate);
/** 删除归属某累计档的全部明细行（管理端真删店铺数据记录时清理，避免留下悬空引用）。 */
@Delete("""
DELETE FROM biz_shop_data_crawl_item
WHERE daily_file_id = #{dailyFileId}
""")
int deleteByDailyFileId(@Param("dailyFileId") Long dailyFileId);
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/model/entity/ShopDataCrawlDailyFileEntity.java
+2
View File
@@ -17,6 +17,8 @@ public class ShopDataCrawlDailyFileEntity {
private Long userId;
private String shopKeyHash;
private String shopKey;
/** 店铺显示名（冗余，供后台记录按店分组/筛选；采集归档时写入）。 */
private String shopName;
private LocalDate businessDate;
private Long latestTaskId;
private Long latestResultId;
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java
+105 -51
View File
@@ -3,13 +3,19 @@ package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlAdminTasksMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminFileJobBriefDto;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlItemMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminGroupLabelDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminGroupRow;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminRow;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlDownloadRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckSourceMapper;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@@ -19,11 +25,13 @@ import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
/**
@@ -40,8 +48,92 @@ public class ShopDataCrawlAdminTasksService {
private final ShopDataCrawlAdminTasksMapper adminTasksMapper;
private final ShopDuplicateCheckSourceMapper duplicateCheckSourceMapper;
private final ShopDataCrawlItemMapper itemMapper;
private final FileResultMapper fileResultMapper;
private final ShopDataCrawlDailyFileService dailyFileService;
private final OssStorageService ossStorageService;
private final ShopDataCrawlTaskService taskService;
private final ObjectMapper objectMapper;
/** 单档下载描述。 */
public record DailyDownload(String url, String filename) {
}
/** 按累计档 id 解析可下载地址（生成新鲜签名 URL）与文件名。 */
public DailyDownload resolveDailyDownload(Long dailyFileId) {
ShopDataCrawlDownloadRowDto row = adminTasksMapper.selectDownloadRowById(dailyFileId);
if (row == null || row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) {
throw new BusinessException("记录不存在或文件未就绪");
}
String url = ossStorageService.generateFreshDownloadUrl(row.getResultFileUrl());
String source = row.getSourceFilename() == null ? "" : row.getSourceFilename().trim();
String filename = row.getResultFilename() == null || row.getResultFilename().isBlank()
? (source.isEmpty() ? "shop-data-" + dailyFileId : source) + ".xlsx"
: row.getResultFilename();
return new DailyDownload(url, filename);
}
/** 管理端真删该店这条店铺数据记录（累计档 + 文件 + 明细），仅允许删除已生成文件的终态记录。 */
public void deleteAdminDailyFile(Long dailyFileId) {
if (dailyFileId == null || dailyFileId <= 0) {
throw new BusinessException("参数不合法");
}
ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findById(dailyFileId);
if (dailyFile == null) {
throw new BusinessException("记录不存在");
}
TaskDistributedLockService.LockHandle lockHandle = dailyFileService.acquireLock(dailyFile.getShopKey());
if (lockHandle == null) {
throw new BusinessException("店铺数据正在处理中，请稍后重试");
}
try (lockHandle) {
ShopDataCrawlDailyFileEntity latest = dailyFileService.findById(dailyFileId);
if (latest == null) {
throw new BusinessException("记录不存在");
}
String objectKey = latest.getResultFileUrl();
Set<Long> resultIds = new HashSet<>();
List<ShopDataCrawlDailyMemberEntity> members = dailyFileService.listMembers(dailyFileId);
for (ShopDataCrawlDailyMemberEntity member : members == null ? List.<ShopDataCrawlDailyMemberEntity>of() : members) {
if (member != null && member.getResultId() != null) {
resultIds.add(member.getResultId());
}
}
if (latest.getLatestResultId() != null) {
resultIds.add(latest.getLatestResultId());
}
// 该档归属的明细行一并清掉，避免悬空引用
itemMapper.deleteByDailyFileId(dailyFileId);
// 前台任务历史仍保留，但清掉指向被删文件的下载指针（避免下载 404）
if (objectKey != null && !objectKey.isBlank()) {
for (Long resultId : resultIds) {
detachResultFileIfSameObject(resultId, objectKey);
}
}
// 成员行 + 档行
dailyFileService.deleteDailyFile(dailyFileId);
// 文件对象在确无其它引用时回收
if (objectKey != null && !objectKey.isBlank()) {
taskService.deleteResultObjectIfUnreferenced(objectKey);
}
}
}
private void detachResultFileIfSameObject(Long resultId, String objectKey) {
if (resultId == null || resultId <= 0) {
return;
}
FileResultEntity result = fileResultMapper.selectById(resultId);
if (result == null || result.getResultFileUrl() == null
|| !result.getResultFileUrl().equals(objectKey)) {
return;
}
result.setResultFileUrl(null);
result.setResultFileSize(null);
result.setResultContentType(null);
fileResultMapper.updateById(result);
}
/**
* 分页列表组装（组总数 → 当前页店铺组 → 每店最新结果行 + 文件任务/分组名补充）。
*/
@@ -122,24 +214,20 @@ public class ShopDataCrawlAdminTasksService {
labelByShop.put(normalizeShopKey(label.getShopName()), label.getGroupName() == null ? "" : label.getGroupName());
}
// 当前页每家店铺最新结果行，按店铺归一化名聚组
// 当前页每家店铺最新一次采集的累计档，按店铺归一化名聚组
List<ShopDataCrawlAdminRow> rows = adminTasksMapper.selectLatestRowsForShops(p);
Map<String, List<ShopDataCrawlAdminRow>> rowsByShop = new LinkedHashMap<>();
for (ShopDataCrawlAdminRow row : rows) {
rowsByShop.computeIfAbsent(normalizeShopKey(row.getShopName()), key -> new ArrayList<>()).add(row);
}
// 结果文件最近一次 ASSEMBLE_RESULT 任务（文件组装状态/错误）
Map<Long, ShopDataCrawlAdminFileJobBriefDto> jobByResult = loadAssembleJobMap(rows);
List<Map<String, Object>> items = new ArrayList<>(groups.size());
for (ShopDataCrawlAdminGroupRow group : groups) {
List<ShopDataCrawlAdminRow> children = rowsByShop.getOrDefault(
normalizeShopKey(group.getShopName()), List.of());
List<Map<String, Object>> results = new ArrayList<>();
if (!children.isEmpty()) {
results.add(toItemMap(children.get(0), jobByResult.get(children.get(0).getResultId()),
labelByShop));
results.add(toItemMap(children.get(0), labelByShop));
}
LocalDateTime latest = group.getLatestCreatedAt();
if (latest == null && !children.isEmpty()) {
@@ -160,57 +248,30 @@ public class ShopDataCrawlAdminTasksService {
return items;
}
private Map<Long, ShopDataCrawlAdminFileJobBriefDto> loadAssembleJobMap(List<ShopDataCrawlAdminRow> rows) {
if (rows.isEmpty()) {
return Map.of();
}
List<Long> resultIds = new ArrayList<>();
for (ShopDataCrawlAdminRow row : rows) {
if (row.getResultId() != null && row.getResultId() > 0) {
resultIds.add(row.getResultId());
}
}
List<Long> distinctIds = new ArrayList<>(new LinkedHashSet<>(resultIds));
if (distinctIds.isEmpty()) {
return Map.of();
}
Map<String, Object> p = new HashMap<>();
flattenList(p, distinctIds, "resultIds", "ri");
Map<Long, ShopDataCrawlAdminFileJobBriefDto> jobByResult = new HashMap<>();
for (ShopDataCrawlAdminFileJobBriefDto job : adminTasksMapper.selectLatestAssembleJobs(p)) {
if (job.getResultId() != null) {
jobByResult.put(job.getResultId(), job);
}
}
return jobByResult;
}
/** 单个"最新结果行"→ 前端字段 Map（键 snake_case，对齐 Flask _shop_data_crawl_admin_item）。 */
/** 单个"最新一次采集档" → 前端字段 Map（键 snake_case，result_id 语义为每日累计档 id）。 */
private Map<String, Object> toItemMap(ShopDataCrawlAdminRow row,
ShopDataCrawlAdminFileJobBriefDto job,
Map<String, String> labelByShop) {
boolean fileReady = row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank();
Map<String, Object> m = new LinkedHashMap<>();
m.put("daily_file_id", row.getResultId());
m.put("result_id", row.getResultId());
m.put("task_id", row.getTaskId());
m.put("task_no", defaultText(row.getTaskNo()));
m.put("result_id", row.getResultId());
m.put("user_id", row.getUserId());
m.put("username", defaultText(row.getUsername()));
m.put("shop_name", defaultText(row.getShopName()));
m.put("shop_id", defaultText(row.getShopId()));
m.put("group_name", groupLabel(labelByShop, row.getShopName()));
m.put("status", defaultText(row.getTaskStatus()));
Integer success = row.getResultSuccess();
m.put("success", success == null || success < 0 ? null : Integer.valueOf(1).equals(success));
m.put("error", firstNonBlank(row.getResultError(), row.getTaskError(),
job == null ? null : job.getFileError()));
m.put("country_codes", countryCodesOf(row.getCountryCodesJson(), row.getRequestJson()));
m.put("status", "SUCCESS");
m.put("success", Boolean.TRUE);
m.put("error", firstNonBlank(row.getResultError(), row.getTaskError()));
m.put("country_codes", countryCodesOf(row.getCountryCodesJson(), null));
m.put("output_filename", defaultText(row.getResultFilename()));
m.put("result_file_url", defaultText(row.getResultFileUrl()));
m.put("file_ready", fileReady);
m.put("file_job_id", job == null ? null : job.getFileJobId());
m.put("file_status", fileStatusOf(job, fileReady));
m.put("file_error", job == null || job.getFileError() == null ? "" : job.getFileError());
m.put("file_job_id", null);
m.put("file_status", fileReady ? "SUCCESS" : "");
m.put("file_error", "");
m.put("file_size", row.getResultFileSize() == null ? 0L : row.getResultFileSize());
Integer displayRowCount = row.getRowCountDisplay() == null ? row.getRowCount() : row.getRowCountDisplay();
m.put("row_count", displayRowCount == null ? 0 : displayRowCount);
@@ -221,13 +282,6 @@ public class ShopDataCrawlAdminTasksService {
return m;
}
private String fileStatusOf(ShopDataCrawlAdminFileJobBriefDto job, boolean fileReady) {
if (job != null && job.getFileStatus() != null && !job.getFileStatus().isBlank()) {
return job.getFileStatus();
}
return fileReady ? "SUCCESS" : "";
}
private String groupLabel(Map<String, String> labelByShop, String shopName) {
String value = labelByShop.get(normalizeShopKey(shopName));
return value == null ? "" : value;
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlItemStoreService.java
+15
View File
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlItemMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
@@ -77,6 +78,20 @@ public class ShopDataCrawlItemStoreService {
shopName, businessDate, deleted, inserted, taskId);
}
/**
* 跨天滚动删除旧累计档前，把仍指向旧档的明细行改指到新档，避免明细悬空引用被删档。
*/
@Transactional(rollbackFor = Exception.class)
public void repointDailyFile(Long fromDailyFileId, Long toDailyFileId) {
if (fromDailyFileId == null || fromDailyFileId <= 0 || toDailyFileId == null || toDailyFileId <= 0) {
return;
}
ShopDataCrawlItemEntity update = new ShopDataCrawlItemEntity();
update.setDailyFileId(toDailyFileId);
itemMapper.update(update, new LambdaUpdateWrapper<ShopDataCrawlItemEntity>()
.eq(ShopDataCrawlItemEntity::getDailyFileId, fromDailyFileId));
}
private ShopDataCrawlItemEntity toEntity(String shopName, LocalDate businessDate, String country,
ShopDataCrawlRowDto item, Long resultId, Long taskId, Long dailyFileId) {
ShopDataCrawlItemEntity entity = new ShopDataCrawlItemEntity();
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java
+10
View File
@@ -983,6 +983,7 @@ public class ShopDataCrawlTaskService {
LocalDateTime now = dailyFileService.currentBusinessDateTime();
dailyFile.setLatestTaskId(latest.result().getTaskId());
dailyFile.setShopName(trimToNull(latest.result().getSourceFilename()));
dailyFile.setLatestResultId(latest.result().getId());
dailyFile.setResultFilename(filename);
dailyFile.setResultFileUrl(newObjectKey);
@@ -2170,6 +2171,9 @@ public class ShopDataCrawlTaskService {
} else {
dailyFile.setVersion(Math.max(0L, Objects.requireNonNullElse(dailyFile.getVersion(), 0L)) + 1L);
}
dailyFile.setShopKeyHash(shopKeyHash);
dailyFile.setShopKey(shopKey);
dailyFile.setShopName(trimToNull(row.getSourceFilename()));
dailyFile.setLatestTaskId(row.getTaskId());
dailyFile.setLatestResultId(row.getId());
dailyFile.setResultFilename(filename);
@@ -2193,6 +2197,8 @@ public class ShopDataCrawlTaskService {
reassignOlderMembers(dailyFile, olderFiles);
for (ShopDataCrawlDailyFileEntity older : olderFiles) {
// 明细行跟随成员一起迁往新档，避免删旧档后留下悬空 daily_file_id
shopDataCrawlItemStoreService.repointDailyFile(older.getId(), dailyFile.getId());
dailyFileService.deleteDailyFile(older.getId());
}
obsoleteObjectKeys.remove(objectKey);
@@ -2559,6 +2565,10 @@ public class ShopDataCrawlTaskService {
throw new BusinessException("累计文件补偿上传结果为空");
}
LocalDateTime now = dailyFileService.currentBusinessDateTime();
if (blank(dailyFile.getShopName()) && !snapshots.isEmpty()
&& !blank(snapshots.get(0).getShopName())) {
dailyFile.setShopName(trimToNull(snapshots.get(0).getShopName()));
}
dailyFile.setLatestTaskId(dailyFile.getLatestTaskId());
dailyFile.setResultFilename(filename);
dailyFile.setResultFileUrl(newObjectKey);
backend-java/src/main/resources/db/V113__shop_data_crawl_daily_file_shop_name.sql
+39
View File
@@ -0,0 +1,39 @@
-- 后台"店铺数据记录"页的数据源改为该店最新一次采集的累计文件（biz_shop_data_crawl_daily_file）。
-- daily_file 需冗余店铺显示名：删除前台任务后 file_result 记录可能已物理删除，
-- 不能再依赖 file_result.source_filename 反查店名。
ALTER TABLE biz_shop_data_crawl_daily_file
ADD COLUMN shop_name VARCHAR(255) NULL COMMENT '店铺显示名（冗余，供后台记录按店分组/筛选；采集归档时写入）'
AFTER shop_key;
CREATE INDEX idx_sdc_daily_shop_name ON biz_shop_data_crawl_daily_file (shop_name);
-- 回填 1：该档成员结果行的店铺名（同档店名唯一，取任意非空）
UPDATE biz_shop_data_crawl_daily_file df
JOIN (
SELECT m.daily_file_id AS did, MAX(TRIM(r.source_filename)) AS nm
FROM biz_shop_data_crawl_daily_member m
JOIN biz_file_result r ON r.id = m.result_id
WHERE r.source_filename IS NOT NULL AND TRIM(r.source_filename) <> ''
GROUP BY m.daily_file_id
) s ON s.did = df.id
SET df.shop_name = s.nm
WHERE df.shop_name IS NULL;
-- 回填 2：回退到该档 latest_result 指向的结果行店铺名
UPDATE biz_shop_data_crawl_daily_file df
JOIN biz_file_result r ON r.id = df.latest_result_id
SET df.shop_name = TRIM(r.source_filename)
WHERE df.shop_name IS NULL
AND r.source_filename IS NOT NULL AND TRIM(r.source_filename) <> '';
-- 回填 3：再回退到归属该档的采集明细行店铺名
UPDATE biz_shop_data_crawl_daily_file df
JOIN (
SELECT i.daily_file_id AS did, MAX(TRIM(i.shop_name)) AS nm
FROM biz_shop_data_crawl_item i
WHERE i.daily_file_id IS NOT NULL
AND i.shop_name IS NOT NULL AND TRIM(i.shop_name) <> ''
GROUP BY i.daily_file_id
) s ON s.did = df.id
SET df.shop_name = s.nm
WHERE df.shop_name IS NULL;