From 630bfc3eee3e06e73cfc177410702f7ea6f76307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Tue, 8 Sep 2026 15:02:25 +0800 Subject: [PATCH] =?UTF-8?q?fix(shop-data-crawl):=20=E5=90=8E=E5=8F=B0?= =?UTF-8?q?=E5=BA=97=E9=93=BA=E6=95=B0=E6=8D=AE=E8=AE=B0=E5=BD=95=E6=94=B9?= =?UTF-8?q?=E8=AF=BB=E6=AF=8F=E5=BA=97=E6=9C=80=E6=96=B0=E7=B4=AF=E8=AE=A1?= =?UTF-8?q?=E6=A1=A3=EF=BC=8C=E5=88=A0=E9=99=A4=E4=BB=BB=E5=8A=A1=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E8=BF=9E=E5=9D=90=E5=90=8E=E5=8F=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 后台“店铺数据记录”此前以每次抓取的 file_result/file_task 为源，前台删除任务会物理删 这两张表，导致该店在后台退档/消失。现改为以采集归档维护、删除任务时保留的 biz_shop_data_crawl_daily_file 每店最新一档为数据源： - V113：daily_file 冗余 shop_name 列并回填；归档写档三处入口同步写入 - 列表/批量下载/单档下载/删除端点改按每日累计档 id（daily-files/{id}） - 管理端删除=真删该店这条数据记录（档+文件+明细），前台任务历史保留但清文件指针 - 明细行 daily_file_id 跨天滚动删旧档时改指新档，修存量悬空引用（另附修复脚本） --- .../src/pages/tasks/ShopDataTasksPage.vue | 6 +- .../src/pages/tasks/shop-data-api.ts | 8 +- .../AdminShopDataCrawlTasksController.java | 32 ++++ .../mapper/ShopDataCrawlAdminTasksMapper.java | 125 +++++++------- .../mapper/ShopDataCrawlItemMapper.java | 7 + .../entity/ShopDataCrawlDailyFileEntity.java | 2 + .../ShopDataCrawlAdminTasksService.java | 156 ++++++++++++------ .../ShopDataCrawlItemStoreService.java | 15 ++ .../service/ShopDataCrawlTaskService.java | 10 ++ ...__shop_data_crawl_daily_file_shop_name.sql | 39 +++++ 10 files changed, 278 insertions(+), 122 deletions(-) create mode 100644 backend-java/src/main/resources/db/V113__shop_data_crawl_daily_file_shop_name.sql diff --git a/admin-frontend-vue/src/pages/tasks/ShopDataTasksPage.vue b/admin-frontend-vue/src/pages/tasks/ShopDataTasksPage.vue index a021bb63..ed529e42 100644 --- a/admin-frontend-vue/src/pages/tasks/ShopDataTasksPage.vue +++ b/admin-frontend-vue/src/pages/tasks/ShopDataTasksPage.vue @@ -216,15 +216,15 @@ async function downloadBatch() { } async function removeRowByRow(row: ShopDataResultRow) { - // 对齐旧版：原生 confirm。 - if (!window.confirm(`确认删除店铺“${row.shopName}”的任务 ${row.taskId || row.taskNo || row.resultId} 及结果文件？`)) return + // 管理端真删：删除该店这条店铺数据记录（累计档 + 文件 + 明细），前台任务记录仍保留但文件不可再下载 + if (!window.confirm(`确认删除店铺“${row.shopName}”这条店铺数据记录及其数据文件？此操作不可恢复。`)) return deletingKey.value = row.resultId try { await deleteShopDataResultHistory(row.resultId) ElMessage.success('删除成功') await load() } catch (error) { - ElMessage.error(error instanceof Error ? error.message : `正在删除任务 ${row.taskId || row.taskNo || row.resultId}...`) + ElMessage.error(error instanceof Error ? error.message : `删除店铺“${row.shopName}”的记录失败`) } finally { deletingKey.value = '' } diff --git a/admin-frontend-vue/src/pages/tasks/shop-data-api.ts b/admin-frontend-vue/src/pages/tasks/shop-data-api.ts index 53c6c99f..2a062c47 100644 --- a/admin-frontend-vue/src/pages/tasks/shop-data-api.ts +++ b/admin-frontend-vue/src/pages/tasks/shop-data-api.ts @@ -41,18 +41,18 @@ export interface ShopDataResultDownload { filename: string } -/** 下载单个结果文件（服务端原文件名）：GET /api/admin/shop-data-crawl/results/{resultId}/download。 */ +/** 下载单店最新采集文件（按每日累计档 id）：GET /api/admin/shop-data-crawl-tasks/daily-files/{id}/download。 */ export async function fetchShopDataResultDownload(resultId: number | string): Promise { - const { data, headers } = await http.get(`${SHOP_DATA_SINGLE_ENDPOINT}/results/${resultId}/download`, { + const { data, headers } = await http.get(`${SHOP_DATA_CRAWL_TASKS_ENDPOINT}/daily-files/${resultId}/download`, { responseType: 'blob', }) const disposition = (headers as Record | undefined)?.['content-disposition'] return { blob: data, filename: shopDataFilenameFromDisposition(disposition, `shop-data-task-${resultId}.xlsx`) } } -/** 删除单条结果历史（仅终态）：DELETE /api/admin/shop-data-crawl/history/{resultId}。 */ +/** 删除该店这条店铺数据记录（按每日累计档 id，管理端显式操作）：DELETE .../daily-files/{id}。 */ export async function deleteShopDataResultHistory(resultId: number | string): Promise { - await http.delete(`${SHOP_DATA_SINGLE_ENDPOINT}/history/${resultId}`) + await http.delete(`${SHOP_DATA_CRAWL_TASKS_ENDPOINT}/daily-files/${resultId}`) } /** 加载店铺数据任务数据范围授权用户：GET /api/admin/shop-data-crawl-task-permissions。 */ diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/AdminShopDataCrawlTasksController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/AdminShopDataCrawlTasksController.java index dfa823c5..a6de2b35 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/AdminShopDataCrawlTasksController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/AdminShopDataCrawlTasksController.java @@ -18,6 +18,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; @@ -98,6 +100,36 @@ public class AdminShopDataCrawlTasksController { return ApiResponse.success(payload); } + @GetMapping("/daily-files/{dailyFileId}/download") + @Operation(summary = "下载该店最新一次采集的数据文件（按累计档 id）") + public void downloadDailyFile( + @PathVariable Long dailyFileId, + HttpServletRequest request, + HttpServletResponse response) { + requireShopDataCrawlTaskAccess(request); + ShopDataCrawlAdminTasksService.DailyDownload daily = adminTasksService.resolveDailyDownload(dailyFileId); + try { + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + DownloadHeaderUtil.setAttachment(response, daily.filename()); + try (InputStream input = URI.create(daily.url()).toURL().openStream()) { + input.transferTo(response.getOutputStream()); + } + } catch (Exception ex) { + log.warn("[shop-data-crawl-admin] 下载店铺数据文件失败 dailyFileId={} msg={}", dailyFileId, ex.getMessage()); + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败"); + } + } + + @DeleteMapping("/daily-files/{dailyFileId}") + @Operation(summary = "删除该店这条店铺数据记录（累计档 + 文件 + 明细，管理端显式操作）") + public ApiResponse deleteDailyFile( + @PathVariable Long dailyFileId, + HttpServletRequest request) { + requireShopDataCrawlTaskAccess(request); + adminTasksService.deleteAdminDailyFile(dailyFileId); + return ApiResponse.success(null); + } + @PostMapping("/download-zip") @Operation(summary = "批量下载选中结果文件为 zip（文件部分失败经响应头 X-Archive-Error-Count 提示）") public void downloadZip( diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlAdminTasksMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlAdminTasksMapper.java index 2c6d3c50..29be588a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlAdminTasksMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlAdminTasksMapper.java @@ -6,15 +6,18 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminGroup import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminRow; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlDownloadRowDto; import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Select; import org.apache.ibatis.annotations.SelectProvider; import java.util.List; import java.util.Map; /** - * 店铺数据抓取记录"按店铺分组列表"与批量下载的数据源查询， - * 语义逐条对照 Flask admin_api.py 中 list_shop_data_crawl_tasks / - * _load_shop_data_crawl_download_rows 的 SQL。 + * 店铺数据记录"按店铺分组列表"与批量下载的数据源查询。 + * + *

数据源为每店最新一次采集的累计文件（biz_shop_data_crawl_daily_file）： + * 该表由采集归档维护、删除前台任务时被保留，因此后台记录不会因客户端删除任务而消失； + * 同一店铺再次采集会生成/滚动到更新的一档，列表只取每店最新一档 —— 覆盖式语义，与日期无关。 * *

参数统一走单参 Map：筛选与分页字段用语义键（shopName/groupName/country/ * createdFrom/createdTo/pageLimit/pageOffset），IN 列表键 shopNames / resultIds， @@ -23,15 +26,15 @@ import java.util.Map; @Mapper public interface ShopDataCrawlAdminTasksMapper { - /** 店铺分组总数（按 TRIM(source_filename) 去重后的组数）。 */ + /** 店铺分组总数（按 TRIM(shop_name) 去重后的组数）。 */ @SelectProvider(type = SqlProvider.class, method = "countShopGroups") long countShopGroups(Map p); - /** 当前页店铺分组（每店最新归档时间）。 */ + /** 当前页店铺分组（每店最新采集时间）。 */ @SelectProvider(type = SqlProvider.class, method = "selectShopGroupPage") List selectShopGroupPage(Map p); - /** 当前页每家店铺的最新结果行（窗口函数 row_number=1），需先铺平 shopNames。 */ + /** 当前页每家店铺的最新档（窗口函数 row_number=1），需先铺平 shopNames。 */ @SelectProvider(type = SqlProvider.class, method = "selectLatestRowsForShops") List selectLatestRowsForShops(Map p); @@ -39,39 +42,48 @@ public interface ShopDataCrawlAdminTasksMapper { @SelectProvider(type = SqlProvider.class, method = "selectGroupLabels") List selectGroupLabels(Map p); - /** 每个结果文件最新一条 ASSEMBLE_RESULT 任务（按 job id 取最大），需先铺平 resultIds。 */ - @SelectProvider(type = SqlProvider.class, method = "selectLatestAssembleJobs") - List selectLatestAssembleJobs(Map p); - - /** 批量下载 zip 所需结果行，需先铺平 resultIds。 */ + /** 批量下载 zip 所需档行（按每日累计档 id），需先铺平 resultIds。 */ @SelectProvider(type = SqlProvider.class, method = "selectDownloadRows") List selectDownloadRows(Map p); + /** 按每日累计档 id 取单行（下载/删除用）。 */ + @Select(""" + SELECT df.id AS resultId, df.latest_task_id AS taskId, df.user_id AS userId, + df.shop_name AS sourceFilename, df.result_filename AS resultFilename, + df.result_file_url AS resultFileUrl + FROM biz_shop_data_crawl_daily_file df + WHERE df.id = #{dailyFileId} + """) + ShopDataCrawlDownloadRowDto selectDownloadRowById(@org.apache.ibatis.annotations.Param("dailyFileId") Long dailyFileId); + + /** 某一累计对象 key 还被多少 daily_file 行引用。 */ + @Select(""" + SELECT COUNT(1) FROM biz_shop_data_crawl_daily_file + WHERE result_file_url = #{objectKey} + """) + long countDailyFileObjectReferences(@org.apache.ibatis.annotations.Param("objectKey") String objectKey); + class SqlProvider { - private static final String LATEST_TIME = "COALESCE(df.last_success_at, df.updated_at, " - + "t.finished_at, t.updated_at, t.created_at)"; + private static final String SHOP_NAME = "TRIM(COALESCE(df.shop_name, ''))"; - private static final String SHOP_KEY = "TRIM(COALESCE(r.source_filename, ''))"; + private static final String LATEST_TIME = "COALESCE(df.last_success_at, df.updated_at, df.created_at)"; private static final String FROM_WHERE_BASE = - " FROM biz_file_result r " - + "JOIN biz_file_task t ON t.id = r.task_id " - + "LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id " - + "LEFT JOIN users u ON u.id = r.user_id " - + "WHERE r.module_type = 'SHOP_DATA_CRAWL' " - + "AND t.module_type = 'SHOP_DATA_CRAWL' " - + "AND TRIM(COALESCE(r.result_file_url, '')) <> ''"; + " FROM biz_shop_data_crawl_daily_file df " + + "LEFT JOIN users u ON u.id = df.user_id " + + "WHERE " + SHOP_NAME + " <> '' " + + "AND df.result_file_url IS NOT NULL AND TRIM(df.result_file_url) <> ''"; public String countShopGroups(Map p) { - return "SELECT COUNT(*) AS total FROM (SELECT " + SHOP_KEY + " AS shopKey" - + FROM_WHERE_BASE + whereTail(p, "") + " GROUP BY " + SHOP_KEY + ") grouped_shops"; + return "SELECT COUNT(*) AS total FROM (SELECT " + SHOP_NAME + " AS shopKey" + + FROM_WHERE_BASE + whereTail(p, "") + " GROUP BY " + SHOP_NAME + ") grouped_shops"; } public String selectShopGroupPage(Map p) { - return "SELECT " + SHOP_KEY + " AS shopName, MAX(" + LATEST_TIME + ") AS latestCreatedAt" + return "SELECT " + SHOP_NAME + " AS shopName, MAX(" + LATEST_TIME + ") AS latestCreatedAt" + FROM_WHERE_BASE + whereTail(p, "") - + " GROUP BY " + SHOP_KEY + + " GROUP BY " + SHOP_NAME + " ORDER BY latestCreatedAt DESC, shopName ASC" + " LIMIT #{pageLimit} OFFSET #{pageOffset}"; } @@ -84,23 +96,22 @@ public interface ShopDataCrawlAdminTasksMapper { + " ranked.taskNo, ranked.taskStatus, ranked.requestJson, ranked.taskError," + " ranked.createdAt, ranked.updatedAt, ranked.finishedAt," + " ranked.latestFileUpdatedAt, ranked.countryCodesJson, ranked.rowCountDisplay" - + " FROM (SELECT r.id AS resultId, r.task_id AS taskId, r.user_id AS userId," + + " FROM (SELECT df.id AS resultId, df.latest_task_id AS taskId, df.user_id AS userId," + " u.username AS username," - + " r.source_filename AS shopName, r.source_file_url AS shopId," - + " r.result_filename AS resultFilename, r.result_file_url AS resultFileUrl," - + " r.result_file_size AS resultFileSize, r.result_content_type AS resultContentType," - + " r.row_count AS rowCount, r.success AS resultSuccess," - + " r.error_message AS resultError, r.created_at AS resultCreatedAt," - + " t.task_no AS taskNo, t.status AS taskStatus, t.request_json AS requestJson," - + " t.error_message AS taskError, t.created_at AS createdAt," - + " t.updated_at AS updatedAt, t.finished_at AS finishedAt," + + " df.shop_name AS shopName, df.shop_key AS shopId," + + " df.result_filename AS resultFilename, df.result_file_url AS resultFileUrl," + + " df.result_file_size AS resultFileSize, df.result_content_type AS resultContentType," + + " df.row_count AS rowCount," + + " 1 AS resultSuccess, NULL AS resultError, df.created_at AS resultCreatedAt," + + " NULL AS taskNo, 'SUCCESS' AS taskStatus, NULL AS requestJson, NULL AS taskError," + + " df.created_at AS createdAt, df.updated_at AS updatedAt, df.last_success_at AS finishedAt," + " " + LATEST_TIME + " AS latestFileUpdatedAt," + " df.country_codes_json AS countryCodesJson," - + " COALESCE(df.row_count, r.row_count) AS rowCountDisplay," - + " ROW_NUMBER() OVER (PARTITION BY " + SHOP_KEY - + " ORDER BY " + LATEST_TIME + " DESC, r.id DESC) AS rowNo" + + " df.row_count AS rowCountDisplay," + + " ROW_NUMBER() OVER (PARTITION BY " + SHOP_NAME + + " ORDER BY " + LATEST_TIME + " DESC, df.id DESC) AS rowNo" + FROM_WHERE_BASE - + whereTail(p, " AND " + SHOP_KEY + " IN (" + inPlaceholders(p, "shopNames", "sn") + ")") + + whereTail(p, " AND " + SHOP_NAME + " IN (" + inPlaceholders(p, "shopNames", "sn") + ")") + ") ranked WHERE ranked.rowNo = 1" + " ORDER BY ranked.latestFileUpdatedAt DESC, ranked.resultId DESC"; } @@ -115,60 +126,46 @@ public interface ShopDataCrawlAdminTasksMapper { + " GROUP BY TRIM(sm.shop_name)"; } - public String selectLatestAssembleJobs(Map p) { - return "SELECT fj.result_id AS resultId, fj.id AS fileJobId," - + " fj.status AS fileStatus, fj.error_message AS fileError" - + " FROM biz_task_file_job fj" - + " INNER JOIN (SELECT result_id, MAX(id) AS max_id FROM biz_task_file_job" - + " WHERE module_type = 'SHOP_DATA_CRAWL' AND job_type = 'ASSEMBLE_RESULT'" - + " AND result_id IN (" + inPlaceholders(p, "resultIds", "ri") + ")" - + " GROUP BY result_id) latest ON fj.id = latest.max_id"; - } - public String selectDownloadRows(Map p) { - return "SELECT r.id AS resultId, r.task_id AS taskId, r.user_id AS userId," - + " r.source_filename AS sourceFilename, r.result_filename AS resultFilename," - + " r.result_file_url AS resultFileUrl, t.status AS taskStatus" - + " FROM biz_file_result r" - + " JOIN biz_file_task t ON t.id = r.task_id" - + " WHERE r.module_type = 'SHOP_DATA_CRAWL' AND t.module_type = 'SHOP_DATA_CRAWL'" - + " AND r.id IN (" + inPlaceholders(p, "resultIds", "ri") + ")"; + return "SELECT df.id AS resultId, df.latest_task_id AS taskId, df.user_id AS userId," + + " df.shop_name AS sourceFilename, df.result_filename AS resultFilename," + + " df.result_file_url AS resultFileUrl, 'SUCCESS' AS taskStatus" + + " FROM biz_shop_data_crawl_daily_file df" + + " WHERE df.id IN (" + inPlaceholders(p, "resultIds", "ri") + ")"; } - /** 追加可选的店铺/分组/国家/时间筛选。 */ + /** 追加可选的店铺/分组/国家/时间筛选（全部基于每日累计档列）。 */ private static String whereTail(Map p, String extraCondition) { StringBuilder sql = new StringBuilder(extraCondition == null ? "" : extraCondition); String shopName = (String) p.get("shopName"); if (shopName != null && !shopName.isBlank()) { - sql.append(" AND r.source_filename LIKE CONCAT('%', #{shopName}, '%')"); + sql.append(" AND df.shop_name LIKE CONCAT('%', #{shopName}, '%')"); } String groupName = (String) p.get("groupName"); if (groupName != null && !groupName.isBlank()) { sql.append(" AND EXISTS (SELECT 1 FROM biz_shop_manage sm" + " LEFT JOIN biz_shop_manage_group g ON g.id = sm.group_id" + " WHERE TRIM(COALESCE(sm.shop_name, '')) = " - + "TRIM(COALESCE(r.source_filename, ''))" + + "TRIM(COALESCE(df.shop_name, ''))" + " AND COALESCE(NULLIF(g.group_name, ''), NULLIF(sm.group_name, ''))" + " LIKE CONCAT('%', #{groupName}, '%'))"); } if (p.get("country") != null && !((String) p.get("country")).isBlank()) { - sql.append(" AND JSON_CONTAINS(COALESCE(df.country_codes_json," - + " JSON_EXTRACT(t.request_json, '$.countryCodes')," - + " JSON_EXTRACT(t.request_json, '$.country_codes'), '[]')," + sql.append(" AND JSON_CONTAINS(COALESCE(df.country_codes_json, '[]')," + " CONCAT('\"', #{country}, '\"'))"); } if (p.get("createdFrom") != null) { - sql.append(" AND t.created_at >= #{createdFrom}"); + sql.append(" AND df.created_at >= #{createdFrom}"); } if (p.get("createdTo") != null) { - sql.append(" AND t.created_at <= #{createdTo}"); + sql.append(" AND df.created_at <= #{createdTo}"); } Object visibleShopKeys = p.get("visibleShopKeys"); if (visibleShopKeys instanceof List keys) { if (keys.isEmpty()) { sql.append(" AND 1 = 0"); } else { - sql.append(" AND LOWER(TRIM(COALESCE(r.source_filename, ''))) IN (") + sql.append(" AND LOWER(TRIM(COALESCE(df.shop_name, ''))) IN (") .append(inPlaceholders(p, "visibleShopKeys", "lv")).append(')'); } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlItemMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlItemMapper.java index 3da49f98..e250b39d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlItemMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlItemMapper.java @@ -15,4 +15,11 @@ public interface ShopDataCrawlItemMapper extends BaseMapper resultIds = new HashSet<>(); + List members = dailyFileService.listMembers(dailyFileId); + for (ShopDataCrawlDailyMemberEntity member : members == null ? List.of() : members) { + if (member != null && member.getResultId() != null) { + resultIds.add(member.getResultId()); + } + } + if (latest.getLatestResultId() != null) { + resultIds.add(latest.getLatestResultId()); + } + // 该档归属的明细行一并清掉，避免悬空引用 + itemMapper.deleteByDailyFileId(dailyFileId); + // 前台任务历史仍保留，但清掉指向被删文件的下载指针（避免下载 404） + if (objectKey != null && !objectKey.isBlank()) { + for (Long resultId : resultIds) { + detachResultFileIfSameObject(resultId, objectKey); + } + } + // 成员行 + 档行 + dailyFileService.deleteDailyFile(dailyFileId); + // 文件对象在确无其它引用时回收 + if (objectKey != null && !objectKey.isBlank()) { + taskService.deleteResultObjectIfUnreferenced(objectKey); + } + } + } + + private void detachResultFileIfSameObject(Long resultId, String objectKey) { + if (resultId == null || resultId <= 0) { + return; + } + FileResultEntity result = fileResultMapper.selectById(resultId); + if (result == null || result.getResultFileUrl() == null + || !result.getResultFileUrl().equals(objectKey)) { + return; + } + result.setResultFileUrl(null); + result.setResultFileSize(null); + result.setResultContentType(null); + fileResultMapper.updateById(result); + } + /** * 分页列表组装（组总数 → 当前页店铺组 → 每店最新结果行 + 文件任务/分组名补充）。 */ @@ -122,24 +214,20 @@ public class ShopDataCrawlAdminTasksService { labelByShop.put(normalizeShopKey(label.getShopName()), label.getGroupName() == null ? "" : label.getGroupName()); } - // 当前页每家店铺最新结果行，按店铺归一化名聚组 + // 当前页每家店铺最新一次采集的累计档，按店铺归一化名聚组 List rows = adminTasksMapper.selectLatestRowsForShops(p); Map> rowsByShop = new LinkedHashMap<>(); for (ShopDataCrawlAdminRow row : rows) { rowsByShop.computeIfAbsent(normalizeShopKey(row.getShopName()), key -> new ArrayList<>()).add(row); } - // 结果文件最近一次 ASSEMBLE_RESULT 任务（文件组装状态/错误） - Map jobByResult = loadAssembleJobMap(rows); - List> items = new ArrayList<>(groups.size()); for (ShopDataCrawlAdminGroupRow group : groups) { List children = rowsByShop.getOrDefault( normalizeShopKey(group.getShopName()), List.of()); List> results = new ArrayList<>(); if (!children.isEmpty()) { - results.add(toItemMap(children.get(0), jobByResult.get(children.get(0).getResultId()), - labelByShop)); + results.add(toItemMap(children.get(0), labelByShop)); } LocalDateTime latest = group.getLatestCreatedAt(); if (latest == null && !children.isEmpty()) { @@ -160,57 +248,30 @@ public class ShopDataCrawlAdminTasksService { return items; } - private Map loadAssembleJobMap(List rows) { - if (rows.isEmpty()) { - return Map.of(); - } - List resultIds = new ArrayList<>(); - for (ShopDataCrawlAdminRow row : rows) { - if (row.getResultId() != null && row.getResultId() > 0) { - resultIds.add(row.getResultId()); - } - } - List distinctIds = new ArrayList<>(new LinkedHashSet<>(resultIds)); - if (distinctIds.isEmpty()) { - return Map.of(); - } - Map p = new HashMap<>(); - flattenList(p, distinctIds, "resultIds", "ri"); - Map jobByResult = new HashMap<>(); - for (ShopDataCrawlAdminFileJobBriefDto job : adminTasksMapper.selectLatestAssembleJobs(p)) { - if (job.getResultId() != null) { - jobByResult.put(job.getResultId(), job); - } - } - return jobByResult; - } - - /** 单个"最新结果行"→ 前端字段 Map（键 snake_case，对齐 Flask _shop_data_crawl_admin_item）。 */ + /** 单个"最新一次采集档" → 前端字段 Map（键 snake_case，result_id 语义为每日累计档 id）。 */ private Map toItemMap(ShopDataCrawlAdminRow row, - ShopDataCrawlAdminFileJobBriefDto job, Map labelByShop) { boolean fileReady = row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank(); Map m = new LinkedHashMap<>(); + m.put("daily_file_id", row.getResultId()); + m.put("result_id", row.getResultId()); m.put("task_id", row.getTaskId()); m.put("task_no", defaultText(row.getTaskNo())); - m.put("result_id", row.getResultId()); m.put("user_id", row.getUserId()); m.put("username", defaultText(row.getUsername())); m.put("shop_name", defaultText(row.getShopName())); m.put("shop_id", defaultText(row.getShopId())); m.put("group_name", groupLabel(labelByShop, row.getShopName())); - m.put("status", defaultText(row.getTaskStatus())); - Integer success = row.getResultSuccess(); - m.put("success", success == null || success < 0 ? null : Integer.valueOf(1).equals(success)); - m.put("error", firstNonBlank(row.getResultError(), row.getTaskError(), - job == null ? null : job.getFileError())); - m.put("country_codes", countryCodesOf(row.getCountryCodesJson(), row.getRequestJson())); + m.put("status", "SUCCESS"); + m.put("success", Boolean.TRUE); + m.put("error", firstNonBlank(row.getResultError(), row.getTaskError())); + m.put("country_codes", countryCodesOf(row.getCountryCodesJson(), null)); m.put("output_filename", defaultText(row.getResultFilename())); m.put("result_file_url", defaultText(row.getResultFileUrl())); m.put("file_ready", fileReady); - m.put("file_job_id", job == null ? null : job.getFileJobId()); - m.put("file_status", fileStatusOf(job, fileReady)); - m.put("file_error", job == null || job.getFileError() == null ? "" : job.getFileError()); + m.put("file_job_id", null); + m.put("file_status", fileReady ? "SUCCESS" : ""); + m.put("file_error", ""); m.put("file_size", row.getResultFileSize() == null ? 0L : row.getResultFileSize()); Integer displayRowCount = row.getRowCountDisplay() == null ? row.getRowCount() : row.getRowCountDisplay(); m.put("row_count", displayRowCount == null ? 0 : displayRowCount); @@ -221,13 +282,6 @@ public class ShopDataCrawlAdminTasksService { return m; } - private String fileStatusOf(ShopDataCrawlAdminFileJobBriefDto job, boolean fileReady) { - if (job != null && job.getFileStatus() != null && !job.getFileStatus().isBlank()) { - return job.getFileStatus(); - } - return fileReady ? "SUCCESS" : ""; - } - private String groupLabel(Map labelByShop, String shopName) { String value = labelByShop.get(normalizeShopKey(shopName)); return value == null ? "" : value; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlItemStoreService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlItemStoreService.java index 04d8f8c5..8628b0b3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlItemStoreService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlItemStoreService.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.shopdatacrawl.service; import com.fasterxml.jackson.databind.ObjectMapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlItemMapper; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; @@ -77,6 +78,20 @@ public class ShopDataCrawlItemStoreService { shopName, businessDate, deleted, inserted, taskId); } + /** + * 跨天滚动删除旧累计档前，把仍指向旧档的明细行改指到新档，避免明细悬空引用被删档。 + */ + @Transactional(rollbackFor = Exception.class) + public void repointDailyFile(Long fromDailyFileId, Long toDailyFileId) { + if (fromDailyFileId == null || fromDailyFileId <= 0 || toDailyFileId == null || toDailyFileId <= 0) { + return; + } + ShopDataCrawlItemEntity update = new ShopDataCrawlItemEntity(); + update.setDailyFileId(toDailyFileId); + itemMapper.update(update, new LambdaUpdateWrapper() + .eq(ShopDataCrawlItemEntity::getDailyFileId, fromDailyFileId)); + } + private ShopDataCrawlItemEntity toEntity(String shopName, LocalDate businessDate, String country, ShopDataCrawlRowDto item, Long resultId, Long taskId, Long dailyFileId) { ShopDataCrawlItemEntity entity = new ShopDataCrawlItemEntity(); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java index 6c7a0321..f9600a2e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskService.java @@ -983,6 +983,7 @@ public class ShopDataCrawlTaskService { LocalDateTime now = dailyFileService.currentBusinessDateTime(); dailyFile.setLatestTaskId(latest.result().getTaskId()); + dailyFile.setShopName(trimToNull(latest.result().getSourceFilename())); dailyFile.setLatestResultId(latest.result().getId()); dailyFile.setResultFilename(filename); dailyFile.setResultFileUrl(newObjectKey); @@ -2170,6 +2171,9 @@ public class ShopDataCrawlTaskService { } else { dailyFile.setVersion(Math.max(0L, Objects.requireNonNullElse(dailyFile.getVersion(), 0L)) + 1L); } + dailyFile.setShopKeyHash(shopKeyHash); + dailyFile.setShopKey(shopKey); + dailyFile.setShopName(trimToNull(row.getSourceFilename())); dailyFile.setLatestTaskId(row.getTaskId()); dailyFile.setLatestResultId(row.getId()); dailyFile.setResultFilename(filename); @@ -2193,6 +2197,8 @@ public class ShopDataCrawlTaskService { reassignOlderMembers(dailyFile, olderFiles); for (ShopDataCrawlDailyFileEntity older : olderFiles) { + // 明细行跟随成员一起迁往新档，避免删旧档后留下悬空 daily_file_id + shopDataCrawlItemStoreService.repointDailyFile(older.getId(), dailyFile.getId()); dailyFileService.deleteDailyFile(older.getId()); } obsoleteObjectKeys.remove(objectKey); @@ -2559,6 +2565,10 @@ public class ShopDataCrawlTaskService { throw new BusinessException("累计文件补偿上传结果为空"); } LocalDateTime now = dailyFileService.currentBusinessDateTime(); + if (blank(dailyFile.getShopName()) && !snapshots.isEmpty() + && !blank(snapshots.get(0).getShopName())) { + dailyFile.setShopName(trimToNull(snapshots.get(0).getShopName())); + } dailyFile.setLatestTaskId(dailyFile.getLatestTaskId()); dailyFile.setResultFilename(filename); dailyFile.setResultFileUrl(newObjectKey); diff --git a/backend-java/src/main/resources/db/V113__shop_data_crawl_daily_file_shop_name.sql b/backend-java/src/main/resources/db/V113__shop_data_crawl_daily_file_shop_name.sql new file mode 100644 index 00000000..ea2d9d32 --- /dev/null +++ b/backend-java/src/main/resources/db/V113__shop_data_crawl_daily_file_shop_name.sql @@ -0,0 +1,39 @@ +-- 后台"店铺数据记录"页的数据源改为该店最新一次采集的累计文件（biz_shop_data_crawl_daily_file）。 +-- daily_file 需冗余店铺显示名：删除前台任务后 file_result 记录可能已物理删除， +-- 不能再依赖 file_result.source_filename 反查店名。 +ALTER TABLE biz_shop_data_crawl_daily_file + ADD COLUMN shop_name VARCHAR(255) NULL COMMENT '店铺显示名（冗余，供后台记录按店分组/筛选；采集归档时写入）' + AFTER shop_key; + +CREATE INDEX idx_sdc_daily_shop_name ON biz_shop_data_crawl_daily_file (shop_name); + +-- 回填 1：该档成员结果行的店铺名（同档店名唯一，取任意非空） +UPDATE biz_shop_data_crawl_daily_file df +JOIN ( + SELECT m.daily_file_id AS did, MAX(TRIM(r.source_filename)) AS nm + FROM biz_shop_data_crawl_daily_member m + JOIN biz_file_result r ON r.id = m.result_id + WHERE r.source_filename IS NOT NULL AND TRIM(r.source_filename) <> '' + GROUP BY m.daily_file_id +) s ON s.did = df.id +SET df.shop_name = s.nm +WHERE df.shop_name IS NULL; + +-- 回填 2：回退到该档 latest_result 指向的结果行店铺名 +UPDATE biz_shop_data_crawl_daily_file df +JOIN biz_file_result r ON r.id = df.latest_result_id +SET df.shop_name = TRIM(r.source_filename) +WHERE df.shop_name IS NULL + AND r.source_filename IS NOT NULL AND TRIM(r.source_filename) <> ''; + +-- 回填 3：再回退到归属该档的采集明细行店铺名 +UPDATE biz_shop_data_crawl_daily_file df +JOIN ( + SELECT i.daily_file_id AS did, MAX(TRIM(i.shop_name)) AS nm + FROM biz_shop_data_crawl_item i + WHERE i.daily_file_id IS NOT NULL + AND i.shop_name IS NOT NULL AND TRIM(i.shop_name) <> '' + GROUP BY i.daily_file_id +) s ON s.did = df.id +SET df.shop_name = s.nm +WHERE df.shop_name IS NULL;