fix(shop-data-crawl): 后台店铺数据记录改读每店最新累计档,删除任务不再连坐后台

后台“店铺数据记录”此前以每次抓取的 file_result/file_task 为源,前台删除任务会物理删
这两张表,导致该店在后台退档/消失。现改为以采集归档维护、删除任务时保留的
biz_shop_data_crawl_daily_file 每店最新一档为数据源:
- V113:daily_file 冗余 shop_name 列并回填;归档写档三处入口同步写入
- 列表/批量下载/单档下载/删除端点改按每日累计档 id(daily-files/{id})
- 管理端删除=真删该店这条数据记录(档+文件+明细),前台任务历史保留但清文件指针
- 明细行 daily_file_id 跨天滚动删旧档时改指新档,修存量悬空引用(另附修复脚本)
This commit is contained in:
2026-09-08 15:02:25 +08:00
parent b12acf7793
commit 630bfc3eee
10 changed files with 278 additions and 122 deletions
@@ -216,15 +216,15 @@ async function downloadBatch() {
} }
async function removeRowByRow(row: ShopDataResultRow) { async function removeRowByRow(row: ShopDataResultRow) {
// 对齐旧版:原生 confirm。 // 管理端真删:删除该店这条店铺数据记录(累计档 + 文件 + 明细),前台任务记录仍保留但文件不可再下载
if (!window.confirm(`确认删除店铺“${row.shopName}的任务 ${row.taskId || row.taskNo || row.resultId} 及结果文件?`)) return if (!window.confirm(`确认删除店铺“${row.shopName}这条店铺数据记录及其数据文件?此操作不可恢复。`)) return
deletingKey.value = row.resultId deletingKey.value = row.resultId
try { try {
await deleteShopDataResultHistory(row.resultId) await deleteShopDataResultHistory(row.resultId)
ElMessage.success('删除成功') ElMessage.success('删除成功')
await load() await load()
} catch (error) { } catch (error) {
ElMessage.error(error instanceof Error ? error.message : `正在删除任务 ${row.taskId || row.taskNo || row.resultId}...`) ElMessage.error(error instanceof Error ? error.message : `删除店铺“${row.shopName}”的记录失败`)
} finally { } finally {
deletingKey.value = '' deletingKey.value = ''
} }
@@ -41,18 +41,18 @@ export interface ShopDataResultDownload {
filename: string filename: string
} }
/** 下载单个结果文件(服务端原文件名):GET /api/admin/shop-data-crawl/results/{resultId}/download。 */ /** 下载单店最新采集文件(按每日累计档 id):GET /api/admin/shop-data-crawl-tasks/daily-files/{id}/download。 */
export async function fetchShopDataResultDownload(resultId: number | string): Promise<ShopDataResultDownload> { export async function fetchShopDataResultDownload(resultId: number | string): Promise<ShopDataResultDownload> {
const { data, headers } = await http.get<Blob>(`${SHOP_DATA_SINGLE_ENDPOINT}/results/${resultId}/download`, { const { data, headers } = await http.get<Blob>(`${SHOP_DATA_CRAWL_TASKS_ENDPOINT}/daily-files/${resultId}/download`, {
responseType: 'blob', responseType: 'blob',
}) })
const disposition = (headers as Record<string, string> | undefined)?.['content-disposition'] const disposition = (headers as Record<string, string> | undefined)?.['content-disposition']
return { blob: data, filename: shopDataFilenameFromDisposition(disposition, `shop-data-task-${resultId}.xlsx`) } return { blob: data, filename: shopDataFilenameFromDisposition(disposition, `shop-data-task-${resultId}.xlsx`) }
} }
/** 删除单条结果历史(仅终态):DELETE /api/admin/shop-data-crawl/history/{resultId}。 */ /** 删除该店这条店铺数据记录(按每日累计档 id,管理端显式操作):DELETE .../daily-files/{id}。 */
export async function deleteShopDataResultHistory(resultId: number | string): Promise<void> { export async function deleteShopDataResultHistory(resultId: number | string): Promise<void> {
await http.delete<unknown>(`${SHOP_DATA_SINGLE_ENDPOINT}/history/${resultId}`) await http.delete<unknown>(`${SHOP_DATA_CRAWL_TASKS_ENDPOINT}/daily-files/${resultId}`)
} }
/** 加载店铺数据任务数据范围授权用户:GET /api/admin/shop-data-crawl-task-permissions。 */ /** 加载店铺数据任务数据范围授权用户:GET /api/admin/shop-data-crawl-task-permissions。 */
@@ -18,6 +18,8 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
@@ -98,6 +100,36 @@ public class AdminShopDataCrawlTasksController {
return ApiResponse.success(payload); return ApiResponse.success(payload);
} }
@GetMapping("/daily-files/{dailyFileId}/download")
@Operation(summary = "下载该店最新一次采集的数据文件(按累计档 id)")
public void downloadDailyFile(
@PathVariable Long dailyFileId,
HttpServletRequest request,
HttpServletResponse response) {
requireShopDataCrawlTaskAccess(request);
ShopDataCrawlAdminTasksService.DailyDownload daily = adminTasksService.resolveDailyDownload(dailyFileId);
try {
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
DownloadHeaderUtil.setAttachment(response, daily.filename());
try (InputStream input = URI.create(daily.url()).toURL().openStream()) {
input.transferTo(response.getOutputStream());
}
} catch (Exception ex) {
log.warn("[shop-data-crawl-admin] 下载店铺数据文件失败 dailyFileId={} msg={}", dailyFileId, ex.getMessage());
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
}
}
@DeleteMapping("/daily-files/{dailyFileId}")
@Operation(summary = "删除该店这条店铺数据记录(累计档 + 文件 + 明细,管理端显式操作)")
public ApiResponse<Void> deleteDailyFile(
@PathVariable Long dailyFileId,
HttpServletRequest request) {
requireShopDataCrawlTaskAccess(request);
adminTasksService.deleteAdminDailyFile(dailyFileId);
return ApiResponse.success(null);
}
@PostMapping("/download-zip") @PostMapping("/download-zip")
@Operation(summary = "批量下载选中结果文件为 zip(文件部分失败经响应头 X-Archive-Error-Count 提示)") @Operation(summary = "批量下载选中结果文件为 zip(文件部分失败经响应头 X-Archive-Error-Count 提示)")
public void downloadZip( public void downloadZip(
@@ -6,15 +6,18 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminGroup
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminRow; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminRow;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlDownloadRowDto; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlDownloadRowDto;
import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.SelectProvider; import org.apache.ibatis.annotations.SelectProvider;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
/** /**
* 店铺数据抓取记录"按店铺分组列表"与批量下载的数据源查询 * 店铺数据记录"按店铺分组列表"与批量下载的数据源查询
* 语义逐条对照 Flask admin_api.py 中 list_shop_data_crawl_tasks / *
* _load_shop_data_crawl_download_rows 的 SQL。 * <p>数据源为<b>每店最新一次采集的累计文件</b>biz_shop_data_crawl_daily_file):
* 该表由采集归档维护、删除前台任务时被保留,因此后台记录不会因客户端删除任务而消失;
* 同一店铺再次采集会生成/滚动到更新的一档,列表只取每店最新一档 —— 覆盖式语义,与日期无关。
* *
* <p>参数统一走单参 Map:筛选与分页字段用语义键(shopName/groupName/country/ * <p>参数统一走单参 Map:筛选与分页字段用语义键(shopName/groupName/country/
* createdFrom/createdTo/pageLimit/pageOffset),IN 列表键 shopNames / resultIds * createdFrom/createdTo/pageLimit/pageOffset),IN 列表键 shopNames / resultIds
@@ -23,15 +26,15 @@ import java.util.Map;
@Mapper @Mapper
public interface ShopDataCrawlAdminTasksMapper { public interface ShopDataCrawlAdminTasksMapper {
/** 店铺分组总数(按 TRIM(source_filename) 去重后的组数)。 */ /** 店铺分组总数(按 TRIM(shop_name) 去重后的组数)。 */
@SelectProvider(type = SqlProvider.class, method = "countShopGroups") @SelectProvider(type = SqlProvider.class, method = "countShopGroups")
long countShopGroups(Map<String, Object> p); long countShopGroups(Map<String, Object> p);
/** 当前页店铺分组(每店最新归档时间)。 */ /** 当前页店铺分组(每店最新采集时间)。 */
@SelectProvider(type = SqlProvider.class, method = "selectShopGroupPage") @SelectProvider(type = SqlProvider.class, method = "selectShopGroupPage")
List<ShopDataCrawlAdminGroupRow> selectShopGroupPage(Map<String, Object> p); List<ShopDataCrawlAdminGroupRow> selectShopGroupPage(Map<String, Object> p);
/** 当前页每家店铺的最新结果行(窗口函数 row_number=1),需先铺平 shopNames。 */ /** 当前页每家店铺的最新(窗口函数 row_number=1),需先铺平 shopNames。 */
@SelectProvider(type = SqlProvider.class, method = "selectLatestRowsForShops") @SelectProvider(type = SqlProvider.class, method = "selectLatestRowsForShops")
List<ShopDataCrawlAdminRow> selectLatestRowsForShops(Map<String, Object> p); List<ShopDataCrawlAdminRow> selectLatestRowsForShops(Map<String, Object> p);
@@ -39,39 +42,48 @@ public interface ShopDataCrawlAdminTasksMapper {
@SelectProvider(type = SqlProvider.class, method = "selectGroupLabels") @SelectProvider(type = SqlProvider.class, method = "selectGroupLabels")
List<ShopDataCrawlAdminGroupLabelDto> selectGroupLabels(Map<String, Object> p); List<ShopDataCrawlAdminGroupLabelDto> selectGroupLabels(Map<String, Object> p);
/** 每个结果文件最新一条 ASSEMBLE_RESULT 任务(按 job id 取最大),需先铺平 resultIds。 */ /** 批量下载 zip 所需档行(按每日累计档 id),需先铺平 resultIds。 */
@SelectProvider(type = SqlProvider.class, method = "selectLatestAssembleJobs")
List<ShopDataCrawlAdminFileJobBriefDto> selectLatestAssembleJobs(Map<String, Object> p);
/** 批量下载 zip 所需结果行,需先铺平 resultIds。 */
@SelectProvider(type = SqlProvider.class, method = "selectDownloadRows") @SelectProvider(type = SqlProvider.class, method = "selectDownloadRows")
List<ShopDataCrawlDownloadRowDto> selectDownloadRows(Map<String, Object> p); List<ShopDataCrawlDownloadRowDto> selectDownloadRows(Map<String, Object> p);
/** 按每日累计档 id 取单行(下载/删除用)。 */
@Select("""
SELECT df.id AS resultId, df.latest_task_id AS taskId, df.user_id AS userId,
df.shop_name AS sourceFilename, df.result_filename AS resultFilename,
df.result_file_url AS resultFileUrl
FROM biz_shop_data_crawl_daily_file df
WHERE df.id = #{dailyFileId}
""")
ShopDataCrawlDownloadRowDto selectDownloadRowById(@org.apache.ibatis.annotations.Param("dailyFileId") Long dailyFileId);
/** 某一累计对象 key 还被多少 daily_file 行引用。 */
@Select("""
SELECT COUNT(1) FROM biz_shop_data_crawl_daily_file
WHERE result_file_url = #{objectKey}
""")
long countDailyFileObjectReferences(@org.apache.ibatis.annotations.Param("objectKey") String objectKey);
class SqlProvider { class SqlProvider {
private static final String LATEST_TIME = "COALESCE(df.last_success_at, df.updated_at, " private static final String SHOP_NAME = "TRIM(COALESCE(df.shop_name, ''))";
+ "t.finished_at, t.updated_at, t.created_at)";
private static final String SHOP_KEY = "TRIM(COALESCE(r.source_filename, ''))"; private static final String LATEST_TIME = "COALESCE(df.last_success_at, df.updated_at, df.created_at)";
private static final String FROM_WHERE_BASE = private static final String FROM_WHERE_BASE =
" FROM biz_file_result r " " FROM biz_shop_data_crawl_daily_file df "
+ "JOIN biz_file_task t ON t.id = r.task_id " + "LEFT JOIN users u ON u.id = df.user_id "
+ "LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id " + "WHERE " + SHOP_NAME + " <> '' "
+ "LEFT JOIN users u ON u.id = r.user_id " + "AND df.result_file_url IS NOT NULL AND TRIM(df.result_file_url) <> ''";
+ "WHERE r.module_type = 'SHOP_DATA_CRAWL' "
+ "AND t.module_type = 'SHOP_DATA_CRAWL' "
+ "AND TRIM(COALESCE(r.result_file_url, '')) <> ''";
public String countShopGroups(Map<String, Object> p) { public String countShopGroups(Map<String, Object> p) {
return "SELECT COUNT(*) AS total FROM (SELECT " + SHOP_KEY + " AS shopKey" return "SELECT COUNT(*) AS total FROM (SELECT " + SHOP_NAME + " AS shopKey"
+ FROM_WHERE_BASE + whereTail(p, "") + " GROUP BY " + SHOP_KEY + ") grouped_shops"; + FROM_WHERE_BASE + whereTail(p, "") + " GROUP BY " + SHOP_NAME + ") grouped_shops";
} }
public String selectShopGroupPage(Map<String, Object> p) { public String selectShopGroupPage(Map<String, Object> p) {
return "SELECT " + SHOP_KEY + " AS shopName, MAX(" + LATEST_TIME + ") AS latestCreatedAt" return "SELECT " + SHOP_NAME + " AS shopName, MAX(" + LATEST_TIME + ") AS latestCreatedAt"
+ FROM_WHERE_BASE + whereTail(p, "") + FROM_WHERE_BASE + whereTail(p, "")
+ " GROUP BY " + SHOP_KEY + " GROUP BY " + SHOP_NAME
+ " ORDER BY latestCreatedAt DESC, shopName ASC" + " ORDER BY latestCreatedAt DESC, shopName ASC"
+ " LIMIT #{pageLimit} OFFSET #{pageOffset}"; + " LIMIT #{pageLimit} OFFSET #{pageOffset}";
} }
@@ -84,23 +96,22 @@ public interface ShopDataCrawlAdminTasksMapper {
+ " ranked.taskNo, ranked.taskStatus, ranked.requestJson, ranked.taskError," + " ranked.taskNo, ranked.taskStatus, ranked.requestJson, ranked.taskError,"
+ " ranked.createdAt, ranked.updatedAt, ranked.finishedAt," + " ranked.createdAt, ranked.updatedAt, ranked.finishedAt,"
+ " ranked.latestFileUpdatedAt, ranked.countryCodesJson, ranked.rowCountDisplay" + " ranked.latestFileUpdatedAt, ranked.countryCodesJson, ranked.rowCountDisplay"
+ " FROM (SELECT r.id AS resultId, r.task_id AS taskId, r.user_id AS userId," + " FROM (SELECT df.id AS resultId, df.latest_task_id AS taskId, df.user_id AS userId,"
+ " u.username AS username," + " u.username AS username,"
+ " r.source_filename AS shopName, r.source_file_url AS shopId," + " df.shop_name AS shopName, df.shop_key AS shopId,"
+ " r.result_filename AS resultFilename, r.result_file_url AS resultFileUrl," + " df.result_filename AS resultFilename, df.result_file_url AS resultFileUrl,"
+ " r.result_file_size AS resultFileSize, r.result_content_type AS resultContentType," + " df.result_file_size AS resultFileSize, df.result_content_type AS resultContentType,"
+ " r.row_count AS rowCount, r.success AS resultSuccess," + " df.row_count AS rowCount,"
+ " r.error_message AS resultError, r.created_at AS resultCreatedAt," + " 1 AS resultSuccess, NULL AS resultError, df.created_at AS resultCreatedAt,"
+ " t.task_no AS taskNo, t.status AS taskStatus, t.request_json AS requestJson," + " NULL AS taskNo, 'SUCCESS' AS taskStatus, NULL AS requestJson, NULL AS taskError,"
+ " t.error_message AS taskError, t.created_at AS createdAt," + " df.created_at AS createdAt, df.updated_at AS updatedAt, df.last_success_at AS finishedAt,"
+ " t.updated_at AS updatedAt, t.finished_at AS finishedAt,"
+ " " + LATEST_TIME + " AS latestFileUpdatedAt," + " " + LATEST_TIME + " AS latestFileUpdatedAt,"
+ " df.country_codes_json AS countryCodesJson," + " df.country_codes_json AS countryCodesJson,"
+ " COALESCE(df.row_count, r.row_count) AS rowCountDisplay," + " df.row_count AS rowCountDisplay,"
+ " ROW_NUMBER() OVER (PARTITION BY " + SHOP_KEY + " ROW_NUMBER() OVER (PARTITION BY " + SHOP_NAME
+ " ORDER BY " + LATEST_TIME + " DESC, r.id DESC) AS rowNo" + " ORDER BY " + LATEST_TIME + " DESC, df.id DESC) AS rowNo"
+ FROM_WHERE_BASE + FROM_WHERE_BASE
+ whereTail(p, " AND " + SHOP_KEY + " IN (" + inPlaceholders(p, "shopNames", "sn") + ")") + whereTail(p, " AND " + SHOP_NAME + " IN (" + inPlaceholders(p, "shopNames", "sn") + ")")
+ ") ranked WHERE ranked.rowNo = 1" + ") ranked WHERE ranked.rowNo = 1"
+ " ORDER BY ranked.latestFileUpdatedAt DESC, ranked.resultId DESC"; + " ORDER BY ranked.latestFileUpdatedAt DESC, ranked.resultId DESC";
} }
@@ -115,60 +126,46 @@ public interface ShopDataCrawlAdminTasksMapper {
+ " GROUP BY TRIM(sm.shop_name)"; + " GROUP BY TRIM(sm.shop_name)";
} }
public String selectLatestAssembleJobs(Map<String, Object> p) {
return "SELECT fj.result_id AS resultId, fj.id AS fileJobId,"
+ " fj.status AS fileStatus, fj.error_message AS fileError"
+ " FROM biz_task_file_job fj"
+ " INNER JOIN (SELECT result_id, MAX(id) AS max_id FROM biz_task_file_job"
+ " WHERE module_type = 'SHOP_DATA_CRAWL' AND job_type = 'ASSEMBLE_RESULT'"
+ " AND result_id IN (" + inPlaceholders(p, "resultIds", "ri") + ")"
+ " GROUP BY result_id) latest ON fj.id = latest.max_id";
}
public String selectDownloadRows(Map<String, Object> p) { public String selectDownloadRows(Map<String, Object> p) {
return "SELECT r.id AS resultId, r.task_id AS taskId, r.user_id AS userId," return "SELECT df.id AS resultId, df.latest_task_id AS taskId, df.user_id AS userId,"
+ " r.source_filename AS sourceFilename, r.result_filename AS resultFilename," + " df.shop_name AS sourceFilename, df.result_filename AS resultFilename,"
+ " r.result_file_url AS resultFileUrl, t.status AS taskStatus" + " df.result_file_url AS resultFileUrl, 'SUCCESS' AS taskStatus"
+ " FROM biz_file_result r" + " FROM biz_shop_data_crawl_daily_file df"
+ " JOIN biz_file_task t ON t.id = r.task_id" + " WHERE df.id IN (" + inPlaceholders(p, "resultIds", "ri") + ")";
+ " WHERE r.module_type = 'SHOP_DATA_CRAWL' AND t.module_type = 'SHOP_DATA_CRAWL'"
+ " AND r.id IN (" + inPlaceholders(p, "resultIds", "ri") + ")";
} }
/** 追加可选的店铺/分组/国家/时间筛选。 */ /** 追加可选的店铺/分组/国家/时间筛选(全部基于每日累计档列)。 */
private static String whereTail(Map<String, Object> p, String extraCondition) { private static String whereTail(Map<String, Object> p, String extraCondition) {
StringBuilder sql = new StringBuilder(extraCondition == null ? "" : extraCondition); StringBuilder sql = new StringBuilder(extraCondition == null ? "" : extraCondition);
String shopName = (String) p.get("shopName"); String shopName = (String) p.get("shopName");
if (shopName != null && !shopName.isBlank()) { if (shopName != null && !shopName.isBlank()) {
sql.append(" AND r.source_filename LIKE CONCAT('%', #{shopName}, '%')"); sql.append(" AND df.shop_name LIKE CONCAT('%', #{shopName}, '%')");
} }
String groupName = (String) p.get("groupName"); String groupName = (String) p.get("groupName");
if (groupName != null && !groupName.isBlank()) { if (groupName != null && !groupName.isBlank()) {
sql.append(" AND EXISTS (SELECT 1 FROM biz_shop_manage sm" sql.append(" AND EXISTS (SELECT 1 FROM biz_shop_manage sm"
+ " LEFT JOIN biz_shop_manage_group g ON g.id = sm.group_id" + " LEFT JOIN biz_shop_manage_group g ON g.id = sm.group_id"
+ " WHERE TRIM(COALESCE(sm.shop_name, '')) = " + " WHERE TRIM(COALESCE(sm.shop_name, '')) = "
+ "TRIM(COALESCE(r.source_filename, ''))" + "TRIM(COALESCE(df.shop_name, ''))"
+ " AND COALESCE(NULLIF(g.group_name, ''), NULLIF(sm.group_name, ''))" + " AND COALESCE(NULLIF(g.group_name, ''), NULLIF(sm.group_name, ''))"
+ " LIKE CONCAT('%', #{groupName}, '%'))"); + " LIKE CONCAT('%', #{groupName}, '%'))");
} }
if (p.get("country") != null && !((String) p.get("country")).isBlank()) { if (p.get("country") != null && !((String) p.get("country")).isBlank()) {
sql.append(" AND JSON_CONTAINS(COALESCE(df.country_codes_json," sql.append(" AND JSON_CONTAINS(COALESCE(df.country_codes_json, '[]'),"
+ " JSON_EXTRACT(t.request_json, '$.countryCodes'),"
+ " JSON_EXTRACT(t.request_json, '$.country_codes'), '[]'),"
+ " CONCAT('\"', #{country}, '\"'))"); + " CONCAT('\"', #{country}, '\"'))");
} }
if (p.get("createdFrom") != null) { if (p.get("createdFrom") != null) {
sql.append(" AND t.created_at >= #{createdFrom}"); sql.append(" AND df.created_at >= #{createdFrom}");
} }
if (p.get("createdTo") != null) { if (p.get("createdTo") != null) {
sql.append(" AND t.created_at <= #{createdTo}"); sql.append(" AND df.created_at <= #{createdTo}");
} }
Object visibleShopKeys = p.get("visibleShopKeys"); Object visibleShopKeys = p.get("visibleShopKeys");
if (visibleShopKeys instanceof List<?> keys) { if (visibleShopKeys instanceof List<?> keys) {
if (keys.isEmpty()) { if (keys.isEmpty()) {
sql.append(" AND 1 = 0"); sql.append(" AND 1 = 0");
} else { } else {
sql.append(" AND LOWER(TRIM(COALESCE(r.source_filename, ''))) IN (") sql.append(" AND LOWER(TRIM(COALESCE(df.shop_name, ''))) IN (")
.append(inPlaceholders(p, "visibleShopKeys", "lv")).append(')'); .append(inPlaceholders(p, "visibleShopKeys", "lv")).append(')');
} }
} }
@@ -15,4 +15,11 @@ public interface ShopDataCrawlItemMapper extends BaseMapper<ShopDataCrawlItemEnt
WHERE shop_name = #{shopName} AND business_date = #{businessDate} WHERE shop_name = #{shopName} AND business_date = #{businessDate}
""") """)
int deleteBatch(@Param("shopName") String shopName, @Param("businessDate") java.time.LocalDate businessDate); int deleteBatch(@Param("shopName") String shopName, @Param("businessDate") java.time.LocalDate businessDate);
/** 删除归属某累计档的全部明细行(管理端真删店铺数据记录时清理,避免留下悬空引用)。 */
@Delete("""
DELETE FROM biz_shop_data_crawl_item
WHERE daily_file_id = #{dailyFileId}
""")
int deleteByDailyFileId(@Param("dailyFileId") Long dailyFileId);
} }
@@ -17,6 +17,8 @@ public class ShopDataCrawlDailyFileEntity {
private Long userId; private Long userId;
private String shopKeyHash; private String shopKeyHash;
private String shopKey; private String shopKey;
/** 店铺显示名(冗余,供后台记录按店分组/筛选;采集归档时写入)。 */
private String shopName;
private LocalDate businessDate; private LocalDate businessDate;
private Long latestTaskId; private Long latestTaskId;
private Long latestResultId; private Long latestResultId;
@@ -3,13 +3,19 @@ package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlAdminTasksMapper; import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlAdminTasksMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminFileJobBriefDto; import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlItemMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminGroupLabelDto; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminGroupLabelDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminGroupRow; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminGroupRow;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminRow; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlAdminRow;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlDownloadRowDto; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlDownloadRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyFileEntity;
import com.nanri.aiimage.modules.shopdatacrawl.model.entity.ShopDataCrawlDailyMemberEntity;
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckSourceMapper; import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckSourceMapper;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -19,11 +25,13 @@ import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException; import java.time.format.DateTimeParseException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import java.util.Set; import java.util.Set;
/** /**
@@ -40,8 +48,92 @@ public class ShopDataCrawlAdminTasksService {
private final ShopDataCrawlAdminTasksMapper adminTasksMapper; private final ShopDataCrawlAdminTasksMapper adminTasksMapper;
private final ShopDuplicateCheckSourceMapper duplicateCheckSourceMapper; private final ShopDuplicateCheckSourceMapper duplicateCheckSourceMapper;
private final ShopDataCrawlItemMapper itemMapper;
private final FileResultMapper fileResultMapper;
private final ShopDataCrawlDailyFileService dailyFileService;
private final OssStorageService ossStorageService;
private final ShopDataCrawlTaskService taskService;
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
/** 单档下载描述。 */
public record DailyDownload(String url, String filename) {
}
/** 按累计档 id 解析可下载地址(生成新鲜签名 URL)与文件名。 */
public DailyDownload resolveDailyDownload(Long dailyFileId) {
ShopDataCrawlDownloadRowDto row = adminTasksMapper.selectDownloadRowById(dailyFileId);
if (row == null || row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) {
throw new BusinessException("记录不存在或文件未就绪");
}
String url = ossStorageService.generateFreshDownloadUrl(row.getResultFileUrl());
String source = row.getSourceFilename() == null ? "" : row.getSourceFilename().trim();
String filename = row.getResultFilename() == null || row.getResultFilename().isBlank()
? (source.isEmpty() ? "shop-data-" + dailyFileId : source) + ".xlsx"
: row.getResultFilename();
return new DailyDownload(url, filename);
}
/** 管理端真删该店这条店铺数据记录(累计档 + 文件 + 明细),仅允许删除已生成文件的终态记录。 */
public void deleteAdminDailyFile(Long dailyFileId) {
if (dailyFileId == null || dailyFileId <= 0) {
throw new BusinessException("参数不合法");
}
ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findById(dailyFileId);
if (dailyFile == null) {
throw new BusinessException("记录不存在");
}
TaskDistributedLockService.LockHandle lockHandle = dailyFileService.acquireLock(dailyFile.getShopKey());
if (lockHandle == null) {
throw new BusinessException("店铺数据正在处理中,请稍后重试");
}
try (lockHandle) {
ShopDataCrawlDailyFileEntity latest = dailyFileService.findById(dailyFileId);
if (latest == null) {
throw new BusinessException("记录不存在");
}
String objectKey = latest.getResultFileUrl();
Set<Long> resultIds = new HashSet<>();
List<ShopDataCrawlDailyMemberEntity> members = dailyFileService.listMembers(dailyFileId);
for (ShopDataCrawlDailyMemberEntity member : members == null ? List.<ShopDataCrawlDailyMemberEntity>of() : members) {
if (member != null && member.getResultId() != null) {
resultIds.add(member.getResultId());
}
}
if (latest.getLatestResultId() != null) {
resultIds.add(latest.getLatestResultId());
}
// 该档归属的明细行一并清掉,避免悬空引用
itemMapper.deleteByDailyFileId(dailyFileId);
// 前台任务历史仍保留,但清掉指向被删文件的下载指针(避免下载 404)
if (objectKey != null && !objectKey.isBlank()) {
for (Long resultId : resultIds) {
detachResultFileIfSameObject(resultId, objectKey);
}
}
// 成员行 + 档行
dailyFileService.deleteDailyFile(dailyFileId);
// 文件对象在确无其它引用时回收
if (objectKey != null && !objectKey.isBlank()) {
taskService.deleteResultObjectIfUnreferenced(objectKey);
}
}
}
private void detachResultFileIfSameObject(Long resultId, String objectKey) {
if (resultId == null || resultId <= 0) {
return;
}
FileResultEntity result = fileResultMapper.selectById(resultId);
if (result == null || result.getResultFileUrl() == null
|| !result.getResultFileUrl().equals(objectKey)) {
return;
}
result.setResultFileUrl(null);
result.setResultFileSize(null);
result.setResultContentType(null);
fileResultMapper.updateById(result);
}
/** /**
* 分页列表组装(组总数 → 当前页店铺组 → 每店最新结果行 + 文件任务/分组名补充)。 * 分页列表组装(组总数 → 当前页店铺组 → 每店最新结果行 + 文件任务/分组名补充)。
*/ */
@@ -122,24 +214,20 @@ public class ShopDataCrawlAdminTasksService {
labelByShop.put(normalizeShopKey(label.getShopName()), label.getGroupName() == null ? "" : label.getGroupName()); labelByShop.put(normalizeShopKey(label.getShopName()), label.getGroupName() == null ? "" : label.getGroupName());
} }
// 当前页每家店铺最新结果行,按店铺归一化名聚组 // 当前页每家店铺最新一次采集的累计档,按店铺归一化名聚组
List<ShopDataCrawlAdminRow> rows = adminTasksMapper.selectLatestRowsForShops(p); List<ShopDataCrawlAdminRow> rows = adminTasksMapper.selectLatestRowsForShops(p);
Map<String, List<ShopDataCrawlAdminRow>> rowsByShop = new LinkedHashMap<>(); Map<String, List<ShopDataCrawlAdminRow>> rowsByShop = new LinkedHashMap<>();
for (ShopDataCrawlAdminRow row : rows) { for (ShopDataCrawlAdminRow row : rows) {
rowsByShop.computeIfAbsent(normalizeShopKey(row.getShopName()), key -> new ArrayList<>()).add(row); rowsByShop.computeIfAbsent(normalizeShopKey(row.getShopName()), key -> new ArrayList<>()).add(row);
} }
// 结果文件最近一次 ASSEMBLE_RESULT 任务(文件组装状态/错误)
Map<Long, ShopDataCrawlAdminFileJobBriefDto> jobByResult = loadAssembleJobMap(rows);
List<Map<String, Object>> items = new ArrayList<>(groups.size()); List<Map<String, Object>> items = new ArrayList<>(groups.size());
for (ShopDataCrawlAdminGroupRow group : groups) { for (ShopDataCrawlAdminGroupRow group : groups) {
List<ShopDataCrawlAdminRow> children = rowsByShop.getOrDefault( List<ShopDataCrawlAdminRow> children = rowsByShop.getOrDefault(
normalizeShopKey(group.getShopName()), List.of()); normalizeShopKey(group.getShopName()), List.of());
List<Map<String, Object>> results = new ArrayList<>(); List<Map<String, Object>> results = new ArrayList<>();
if (!children.isEmpty()) { if (!children.isEmpty()) {
results.add(toItemMap(children.get(0), jobByResult.get(children.get(0).getResultId()), results.add(toItemMap(children.get(0), labelByShop));
labelByShop));
} }
LocalDateTime latest = group.getLatestCreatedAt(); LocalDateTime latest = group.getLatestCreatedAt();
if (latest == null && !children.isEmpty()) { if (latest == null && !children.isEmpty()) {
@@ -160,57 +248,30 @@ public class ShopDataCrawlAdminTasksService {
return items; return items;
} }
private Map<Long, ShopDataCrawlAdminFileJobBriefDto> loadAssembleJobMap(List<ShopDataCrawlAdminRow> rows) { /** 单个"最新一次采集档" → 前端字段 Map(键 snake_caseresult_id 语义为每日累计档 id)。 */
if (rows.isEmpty()) {
return Map.of();
}
List<Long> resultIds = new ArrayList<>();
for (ShopDataCrawlAdminRow row : rows) {
if (row.getResultId() != null && row.getResultId() > 0) {
resultIds.add(row.getResultId());
}
}
List<Long> distinctIds = new ArrayList<>(new LinkedHashSet<>(resultIds));
if (distinctIds.isEmpty()) {
return Map.of();
}
Map<String, Object> p = new HashMap<>();
flattenList(p, distinctIds, "resultIds", "ri");
Map<Long, ShopDataCrawlAdminFileJobBriefDto> jobByResult = new HashMap<>();
for (ShopDataCrawlAdminFileJobBriefDto job : adminTasksMapper.selectLatestAssembleJobs(p)) {
if (job.getResultId() != null) {
jobByResult.put(job.getResultId(), job);
}
}
return jobByResult;
}
/** 单个"最新结果行"→ 前端字段 Map(键 snake_case,对齐 Flask _shop_data_crawl_admin_item)。 */
private Map<String, Object> toItemMap(ShopDataCrawlAdminRow row, private Map<String, Object> toItemMap(ShopDataCrawlAdminRow row,
ShopDataCrawlAdminFileJobBriefDto job,
Map<String, String> labelByShop) { Map<String, String> labelByShop) {
boolean fileReady = row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank(); boolean fileReady = row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank();
Map<String, Object> m = new LinkedHashMap<>(); Map<String, Object> m = new LinkedHashMap<>();
m.put("daily_file_id", row.getResultId());
m.put("result_id", row.getResultId());
m.put("task_id", row.getTaskId()); m.put("task_id", row.getTaskId());
m.put("task_no", defaultText(row.getTaskNo())); m.put("task_no", defaultText(row.getTaskNo()));
m.put("result_id", row.getResultId());
m.put("user_id", row.getUserId()); m.put("user_id", row.getUserId());
m.put("username", defaultText(row.getUsername())); m.put("username", defaultText(row.getUsername()));
m.put("shop_name", defaultText(row.getShopName())); m.put("shop_name", defaultText(row.getShopName()));
m.put("shop_id", defaultText(row.getShopId())); m.put("shop_id", defaultText(row.getShopId()));
m.put("group_name", groupLabel(labelByShop, row.getShopName())); m.put("group_name", groupLabel(labelByShop, row.getShopName()));
m.put("status", defaultText(row.getTaskStatus())); m.put("status", "SUCCESS");
Integer success = row.getResultSuccess(); m.put("success", Boolean.TRUE);
m.put("success", success == null || success < 0 ? null : Integer.valueOf(1).equals(success)); m.put("error", firstNonBlank(row.getResultError(), row.getTaskError()));
m.put("error", firstNonBlank(row.getResultError(), row.getTaskError(), m.put("country_codes", countryCodesOf(row.getCountryCodesJson(), null));
job == null ? null : job.getFileError()));
m.put("country_codes", countryCodesOf(row.getCountryCodesJson(), row.getRequestJson()));
m.put("output_filename", defaultText(row.getResultFilename())); m.put("output_filename", defaultText(row.getResultFilename()));
m.put("result_file_url", defaultText(row.getResultFileUrl())); m.put("result_file_url", defaultText(row.getResultFileUrl()));
m.put("file_ready", fileReady); m.put("file_ready", fileReady);
m.put("file_job_id", job == null ? null : job.getFileJobId()); m.put("file_job_id", null);
m.put("file_status", fileStatusOf(job, fileReady)); m.put("file_status", fileReady ? "SUCCESS" : "");
m.put("file_error", job == null || job.getFileError() == null ? "" : job.getFileError()); m.put("file_error", "");
m.put("file_size", row.getResultFileSize() == null ? 0L : row.getResultFileSize()); m.put("file_size", row.getResultFileSize() == null ? 0L : row.getResultFileSize());
Integer displayRowCount = row.getRowCountDisplay() == null ? row.getRowCount() : row.getRowCountDisplay(); Integer displayRowCount = row.getRowCountDisplay() == null ? row.getRowCount() : row.getRowCountDisplay();
m.put("row_count", displayRowCount == null ? 0 : displayRowCount); m.put("row_count", displayRowCount == null ? 0 : displayRowCount);
@@ -221,13 +282,6 @@ public class ShopDataCrawlAdminTasksService {
return m; return m;
} }
private String fileStatusOf(ShopDataCrawlAdminFileJobBriefDto job, boolean fileReady) {
if (job != null && job.getFileStatus() != null && !job.getFileStatus().isBlank()) {
return job.getFileStatus();
}
return fileReady ? "SUCCESS" : "";
}
private String groupLabel(Map<String, String> labelByShop, String shopName) { private String groupLabel(Map<String, String> labelByShop, String shopName) {
String value = labelByShop.get(normalizeShopKey(shopName)); String value = labelByShop.get(normalizeShopKey(shopName));
return value == null ? "" : value; return value == null ? "" : value;
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.shopdatacrawl.service; package com.nanri.aiimage.modules.shopdatacrawl.service;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlItemMapper; import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlItemMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
@@ -77,6 +78,20 @@ public class ShopDataCrawlItemStoreService {
shopName, businessDate, deleted, inserted, taskId); shopName, businessDate, deleted, inserted, taskId);
} }
/**
* 跨天滚动删除旧累计档前,把仍指向旧档的明细行改指到新档,避免明细悬空引用被删档。
*/
@Transactional(rollbackFor = Exception.class)
public void repointDailyFile(Long fromDailyFileId, Long toDailyFileId) {
if (fromDailyFileId == null || fromDailyFileId <= 0 || toDailyFileId == null || toDailyFileId <= 0) {
return;
}
ShopDataCrawlItemEntity update = new ShopDataCrawlItemEntity();
update.setDailyFileId(toDailyFileId);
itemMapper.update(update, new LambdaUpdateWrapper<ShopDataCrawlItemEntity>()
.eq(ShopDataCrawlItemEntity::getDailyFileId, fromDailyFileId));
}
private ShopDataCrawlItemEntity toEntity(String shopName, LocalDate businessDate, String country, private ShopDataCrawlItemEntity toEntity(String shopName, LocalDate businessDate, String country,
ShopDataCrawlRowDto item, Long resultId, Long taskId, Long dailyFileId) { ShopDataCrawlRowDto item, Long resultId, Long taskId, Long dailyFileId) {
ShopDataCrawlItemEntity entity = new ShopDataCrawlItemEntity(); ShopDataCrawlItemEntity entity = new ShopDataCrawlItemEntity();
@@ -983,6 +983,7 @@ public class ShopDataCrawlTaskService {
LocalDateTime now = dailyFileService.currentBusinessDateTime(); LocalDateTime now = dailyFileService.currentBusinessDateTime();
dailyFile.setLatestTaskId(latest.result().getTaskId()); dailyFile.setLatestTaskId(latest.result().getTaskId());
dailyFile.setShopName(trimToNull(latest.result().getSourceFilename()));
dailyFile.setLatestResultId(latest.result().getId()); dailyFile.setLatestResultId(latest.result().getId());
dailyFile.setResultFilename(filename); dailyFile.setResultFilename(filename);
dailyFile.setResultFileUrl(newObjectKey); dailyFile.setResultFileUrl(newObjectKey);
@@ -2170,6 +2171,9 @@ public class ShopDataCrawlTaskService {
} else { } else {
dailyFile.setVersion(Math.max(0L, Objects.requireNonNullElse(dailyFile.getVersion(), 0L)) + 1L); dailyFile.setVersion(Math.max(0L, Objects.requireNonNullElse(dailyFile.getVersion(), 0L)) + 1L);
} }
dailyFile.setShopKeyHash(shopKeyHash);
dailyFile.setShopKey(shopKey);
dailyFile.setShopName(trimToNull(row.getSourceFilename()));
dailyFile.setLatestTaskId(row.getTaskId()); dailyFile.setLatestTaskId(row.getTaskId());
dailyFile.setLatestResultId(row.getId()); dailyFile.setLatestResultId(row.getId());
dailyFile.setResultFilename(filename); dailyFile.setResultFilename(filename);
@@ -2193,6 +2197,8 @@ public class ShopDataCrawlTaskService {
reassignOlderMembers(dailyFile, olderFiles); reassignOlderMembers(dailyFile, olderFiles);
for (ShopDataCrawlDailyFileEntity older : olderFiles) { for (ShopDataCrawlDailyFileEntity older : olderFiles) {
// 明细行跟随成员一起迁往新档,避免删旧档后留下悬空 daily_file_id
shopDataCrawlItemStoreService.repointDailyFile(older.getId(), dailyFile.getId());
dailyFileService.deleteDailyFile(older.getId()); dailyFileService.deleteDailyFile(older.getId());
} }
obsoleteObjectKeys.remove(objectKey); obsoleteObjectKeys.remove(objectKey);
@@ -2559,6 +2565,10 @@ public class ShopDataCrawlTaskService {
throw new BusinessException("累计文件补偿上传结果为空"); throw new BusinessException("累计文件补偿上传结果为空");
} }
LocalDateTime now = dailyFileService.currentBusinessDateTime(); LocalDateTime now = dailyFileService.currentBusinessDateTime();
if (blank(dailyFile.getShopName()) && !snapshots.isEmpty()
&& !blank(snapshots.get(0).getShopName())) {
dailyFile.setShopName(trimToNull(snapshots.get(0).getShopName()));
}
dailyFile.setLatestTaskId(dailyFile.getLatestTaskId()); dailyFile.setLatestTaskId(dailyFile.getLatestTaskId());
dailyFile.setResultFilename(filename); dailyFile.setResultFilename(filename);
dailyFile.setResultFileUrl(newObjectKey); dailyFile.setResultFileUrl(newObjectKey);
@@ -0,0 +1,39 @@
-- 后台"店铺数据记录"页的数据源改为该店最新一次采集的累计文件(biz_shop_data_crawl_daily_file)。
-- daily_file 需冗余店铺显示名:删除前台任务后 file_result 记录可能已物理删除,
-- 不能再依赖 file_result.source_filename 反查店名。
ALTER TABLE biz_shop_data_crawl_daily_file
ADD COLUMN shop_name VARCHAR(255) NULL COMMENT '店铺显示名(冗余,供后台记录按店分组/筛选;采集归档时写入)'
AFTER shop_key;
CREATE INDEX idx_sdc_daily_shop_name ON biz_shop_data_crawl_daily_file (shop_name);
-- 回填 1:该档成员结果行的店铺名(同档店名唯一,取任意非空)
UPDATE biz_shop_data_crawl_daily_file df
JOIN (
SELECT m.daily_file_id AS did, MAX(TRIM(r.source_filename)) AS nm
FROM biz_shop_data_crawl_daily_member m
JOIN biz_file_result r ON r.id = m.result_id
WHERE r.source_filename IS NOT NULL AND TRIM(r.source_filename) <> ''
GROUP BY m.daily_file_id
) s ON s.did = df.id
SET df.shop_name = s.nm
WHERE df.shop_name IS NULL;
-- 回填 2:回退到该档 latest_result 指向的结果行店铺名
UPDATE biz_shop_data_crawl_daily_file df
JOIN biz_file_result r ON r.id = df.latest_result_id
SET df.shop_name = TRIM(r.source_filename)
WHERE df.shop_name IS NULL
AND r.source_filename IS NOT NULL AND TRIM(r.source_filename) <> '';
-- 回填 3:再回退到归属该档的采集明细行店铺名
UPDATE biz_shop_data_crawl_daily_file df
JOIN (
SELECT i.daily_file_id AS did, MAX(TRIM(i.shop_name)) AS nm
FROM biz_shop_data_crawl_item i
WHERE i.daily_file_id IS NOT NULL
AND i.shop_name IS NOT NULL AND TRIM(i.shop_name) <> ''
GROUP BY i.daily_file_id
) s ON s.did = df.id
SET df.shop_name = s.nm
WHERE df.shop_name IS NULL;