店铺数据抓取累计文件改为店铺级共享,按国家覆盖更新
同店同日在不同账号下产生多份互不相干的累计文件(唯一键含 user_id), 后台管理页每店只显示最新一份,其他账号抓的国家看起来丢失。 - V95 迁移:加 shop_key/country_codes_json/compensation_done 列,按店铺 合并存量 daily_file 与成员行,唯一键改为 (shop_key, business_date) - DailyFileService:findForUpdate/findOlder/acquireLock 去 user 维度, 店铺级锁跨账号串行 - TaskService:聚合按店铺定位;applyCountryCoverage 按国家覆盖(本次 回传的国家替换旧行,未更新的国家保留);启动补偿组件按成员快照重建 合并文件并回填国家列表 - 管理页:country_codes 改从 daily_file.country_codes_json 取,行数用 累计文件实际值
This commit is contained in:
@@ -1362,6 +1362,7 @@ _SHOP_DATA_CRAWL_ADMIN_COLUMNS = f"""
|
||||
t.task_no, t.status AS task_status, t.request_json, t.result_json,
|
||||
t.error_message AS task_error, t.created_at, t.updated_at, t.finished_at,
|
||||
{_SHOP_DATA_CRAWL_LATEST_TIME_SQL} AS latest_file_updated_at,
|
||||
df.country_codes_json, COALESCE(df.row_count, r.row_count) AS row_count_display,
|
||||
u.username
|
||||
"""
|
||||
|
||||
@@ -1378,6 +1379,14 @@ def _shop_data_crawl_country_codes(request_json):
|
||||
return [str(value).strip().upper() for value in raw if str(value or '').strip()]
|
||||
|
||||
|
||||
def _shop_data_crawl_country_codes_from_json(json_value):
|
||||
"""从 daily_file.country_codes_json 解析国家代码列表(店铺累计文件实际包含的国家)。"""
|
||||
value = _parse_json_value(json_value, None)
|
||||
if isinstance(value, list):
|
||||
return [str(item).strip().upper() for item in value if str(item or '').strip()]
|
||||
return []
|
||||
|
||||
|
||||
def _shop_data_crawl_group_names(cursor, rows):
|
||||
shop_names = sorted({
|
||||
_shop_data_crawl_shop_key(row.get('shop_name'))
|
||||
@@ -1437,7 +1446,8 @@ def _shop_data_crawl_admin_item(row, group_names=None):
|
||||
'status': row.get('task_status') or '',
|
||||
'success': success,
|
||||
'error': row.get('result_error') or row.get('task_error') or row.get('file_error') or '',
|
||||
'country_codes': _shop_data_crawl_country_codes(row.get('request_json')),
|
||||
'country_codes': _shop_data_crawl_country_codes_from_json(row.get('country_codes_json'))
|
||||
or _shop_data_crawl_country_codes(row.get('request_json')),
|
||||
'output_filename': row.get('result_filename') or '',
|
||||
'result_file_url': row.get('result_file_url') or '',
|
||||
'file_ready': file_ready,
|
||||
@@ -1445,7 +1455,7 @@ def _shop_data_crawl_admin_item(row, group_names=None):
|
||||
'file_status': file_status,
|
||||
'file_error': row.get('file_error') or '',
|
||||
'file_size': int(row.get('result_file_size') or 0),
|
||||
'row_count': int(row.get('row_count') or 0),
|
||||
'row_count': int(row.get('row_count_display') or row.get('row_count') or 0),
|
||||
'created_at': _format_admin_datetime(row.get('created_at') or row.get('result_created_at')),
|
||||
'updated_at': _format_admin_datetime(row.get('updated_at')),
|
||||
'finished_at': _format_admin_datetime(row.get('finished_at')),
|
||||
|
||||
Reference in New Issue
Block a user