feat(admin): A1 管理后台收敛 Java 单后台完整实现并修复 guard 误拦内部令牌
Build Backend JAR / build (push) Has been cancelled
Build Backend JAR / build (push) Has been cancelled
- Java 补齐后台全部迁移差集:shopduplicatecheck 店铺数据重复检查模块(V108 扫描表+查询/扫描服务)、 PinyinAbbrUtil 拼音缩写、ImageHistory 接口调整为内部可用、AdminUser 支持内部令牌操作并放宽列表上限 - Flask 后台 admin_api.py 路由收敛转发 Java、admin.html/admin.js 适配新后台形态 - AdminApiGuardFilter 对可信 X-Internal-Token 放行(controller 自校验兜底),修复客户端仅凭 内部令牌调用 /api/admin/shop-manages/credential 被误拦 401 - 测试:AdminApiGuardFilterTest 补可信/假令牌用例;AdminUserServiceTest 补菜单权限 mock; shopduplicatecheck 新增查询/聚合/CSV 单测
This commit is contained in:
+118
-741
@@ -15,8 +15,6 @@ from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
import requests
|
||||
from openpyxl import load_workbook
|
||||
from openpyxl.utils.exceptions import InvalidFileException
|
||||
from requests.adapters import HTTPAdapter
|
||||
from flask import (
|
||||
Blueprint,
|
||||
@@ -436,7 +434,7 @@ class _PermissionProxyError(Exception):
|
||||
|
||||
|
||||
def _proxy_permission_java(
|
||||
method, path, *, params=None, json_data=None, files=None, data=None, current_row=None):
|
||||
method, path, *, params=None, json_data=None, files=None, data=None, current_row=None, timeout=10):
|
||||
"""Call Java permission APIs using either forwarded JWT or trusted Flask identity."""
|
||||
proxy_params = {}
|
||||
request_row = getattr(g, '_current_user_row', None) if has_request_context() else None
|
||||
@@ -458,6 +456,7 @@ def _proxy_permission_java(
|
||||
files=files,
|
||||
data=data,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
)
|
||||
return result, error_response, status
|
||||
|
||||
@@ -1826,477 +1825,6 @@ def _load_shop_data_crawl_download_rows(result_ids):
|
||||
conn.close()
|
||||
|
||||
|
||||
_SHOP_DATA_CRAWL_ASIN_ANALYSIS_MAX_BYTES = 256 * 1024 * 1024
|
||||
_SHOP_DATA_CRAWL_ASIN_ANALYSIS_TIMEOUT = (10, 60)
|
||||
|
||||
# 重复 ASIN 扫描缓存：结果存 MySQL（shop_data_duplicate_scan 表），
|
||||
# 定时任务每天凌晨执行一次全量扫描，页面读取最近一次结果，避免实时拉取 Excel 消耗资源。
|
||||
_DUPLICATE_SCAN_TABLE = 'shop_data_duplicate_scan'
|
||||
_duplicate_scan_lock = threading.Lock()
|
||||
_duplicate_scan_running = False
|
||||
|
||||
|
||||
def _ensure_duplicate_scan_table():
|
||||
"""按需建表：重复 ASIN 扫描结果缓存表。"""
|
||||
conn = get_db()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""
|
||||
CREATE TABLE IF NOT EXISTS {_DUPLICATE_SCAN_TABLE} (
|
||||
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
status VARCHAR(16) NOT NULL,
|
||||
error_text TEXT NULL,
|
||||
summary_json MEDIUMTEXT NULL,
|
||||
payload_json MEDIUMTEXT NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
finished_at DATETIME NULL,
|
||||
KEY idx_status_created (status, created_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
"""
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _save_duplicate_scan(status, summary=None, payload=None, error_text=None):
|
||||
"""保存一次扫描结果记录；status: SUCCESS / FAILED。"""
|
||||
import json as _json
|
||||
conn = get_db()
|
||||
scan_id = None
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f'INSERT INTO {_DUPLICATE_SCAN_TABLE} (status, error_text, summary_json, payload_json, created_at, finished_at) '
|
||||
'VALUES (%s, %s, %s, %s, NOW(), NOW())',
|
||||
(
|
||||
status,
|
||||
error_text,
|
||||
_json.dumps(summary or {}, ensure_ascii=False),
|
||||
_json.dumps(payload or [], ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
scan_id = cur.lastrowid
|
||||
conn.commit()
|
||||
return scan_id
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _latest_duplicate_scan():
|
||||
"""最近一次成功扫描记录：{'scanned_at': str, 'summary': dict, 'items': list, 'shops': list}|None。
|
||||
|
||||
兼容两种缓存格式：
|
||||
- 新版（重复检查页）：payload = {'shops': [...], 'items': [...]}
|
||||
- 旧版（跨店重复明细）：payload 直接为 items 列表，shops 由明细推导。
|
||||
"""
|
||||
import json as _json
|
||||
conn = get_db()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f'SELECT id, summary_json, payload_json, created_at FROM {_DUPLICATE_SCAN_TABLE} '
|
||||
"WHERE status = 'SUCCESS' ORDER BY id DESC LIMIT 1"
|
||||
)
|
||||
row = cur.fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
if not row:
|
||||
return None
|
||||
summary_raw = row.get('summary_json') or {}
|
||||
payload = row.get('payload_json') or '[]'
|
||||
# summary_json / payload_json 在 MySQL 中为 JSON 字符串，读取后需反序列化
|
||||
summary = _json.loads(summary_raw) if isinstance(summary_raw, str) else (summary_raw or {})
|
||||
summary = summary if isinstance(summary, dict) else {}
|
||||
payload = _json.loads(payload) if isinstance(payload, str) else (payload or [])
|
||||
if isinstance(payload, dict):
|
||||
shops = payload.get('shops') or []
|
||||
items = payload.get('items') or []
|
||||
else:
|
||||
# 旧格式缓存：兼容旧页面，店铺概览由明细推导
|
||||
items = payload if isinstance(payload, list) else []
|
||||
shop_seen = {}
|
||||
for item in items:
|
||||
for occ in (item.get('occurrences') or []):
|
||||
shop_name = occ.get('shop_name') or ''
|
||||
shop_seen[shop_name] = shop_seen.get(shop_name, 0) + 1
|
||||
shops = [
|
||||
{'shop_name': shop_name, 'asin_count': 0, 'group_name': ''}
|
||||
for shop_name in sorted(shop_seen, key=lambda name: (-shop_seen[name], name))
|
||||
]
|
||||
return {
|
||||
'scanned_at': str(row.get('created_at') or ''),
|
||||
'summary': summary,
|
||||
'items': items if isinstance(items, list) else [],
|
||||
'shops': shops if isinstance(shops, list) else [],
|
||||
}
|
||||
|
||||
|
||||
def _shop_data_crawl_fetch_result_bytes(row, timeout=None):
|
||||
"""从 Java 下载接口拉取结果文件字节流（仅内存，不落盘）。"""
|
||||
# row 支持 id（下载行）或 result_id（管理列集别名）
|
||||
result_id = row.get('id') or row.get('result_id') or 0
|
||||
url = f"{backend_java_base_url}/api/admin/shop-data-crawl/results/{int(result_id)}/download"
|
||||
headers, params = _backend_java_internal_request()
|
||||
response = _get_backend_java_session().get(
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
stream=True,
|
||||
timeout=timeout or _SHOP_DATA_CRAWL_ASIN_ANALYSIS_TIMEOUT,
|
||||
)
|
||||
try:
|
||||
response.raise_for_status()
|
||||
total = 0
|
||||
chunks = []
|
||||
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
||||
if not chunk:
|
||||
continue
|
||||
total += len(chunk)
|
||||
if total > _SHOP_DATA_CRAWL_ASIN_ANALYSIS_MAX_BYTES:
|
||||
raise ValueError('结果文件过大，无法分析')
|
||||
chunks.append(chunk)
|
||||
raw = b''.join(chunks)
|
||||
if not raw:
|
||||
raise ValueError(f'结果文件为空 result_id={result_id}')
|
||||
# Java 内部端点鉴权/归属失败时返回 JSON 错误体（HTTP 200），解析报错更直观
|
||||
if raw.lstrip().startswith(b'{'):
|
||||
try:
|
||||
err_payload = json.loads(raw.decode('utf-8', 'replace'))
|
||||
message = err_payload.get('message') or err_payload.get('error') or '未知错误'
|
||||
except ValueError:
|
||||
message = raw[:200].decode('utf-8', 'replace')
|
||||
raise ValueError(f'结果文件下载失败 result_id={result_id}: {message}')
|
||||
return raw
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
|
||||
def _shop_data_crawl_cell_text(cell):
|
||||
"""读取单元格文本：日期/数字等统一转字符串，None 返回空串。"""
|
||||
if cell is None:
|
||||
return ''
|
||||
value = cell.value
|
||||
if value is None:
|
||||
return ''
|
||||
if isinstance(value, datetime):
|
||||
return value.strftime('%Y-%m-%d')
|
||||
if isinstance(value, float) and value.is_integer():
|
||||
return str(int(value))
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def _shop_data_date_key(date_text):
|
||||
"""把各种日期文本归一化为 YYYY-MM-DD，供范围比较。
|
||||
|
||||
支持：ISO（2026-08-19）、中文（2026年8月19日 上午4:34）、
|
||||
yyyy.m.d / yyyy/m/d 等常见格式；无法识别时返回原字符串。
|
||||
"""
|
||||
text = (date_text or '').strip()
|
||||
if not text:
|
||||
return ''
|
||||
if len(text) >= 10 and text[4] == '-' and text[7] == '-':
|
||||
return text[:10]
|
||||
year = month = day = None
|
||||
m = re.search(r'(\d{4})\s*[年./-]\s*(\d{1,2})\s*[月./-]\s*(\d{1,2})', text)
|
||||
if m:
|
||||
year, month, day = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||||
if year and month and day:
|
||||
try:
|
||||
return f'{year:04d}-{month:02d}-{day:02d}'
|
||||
except ValueError:
|
||||
return text
|
||||
return text
|
||||
|
||||
|
||||
_SHOP_DATA_SHEET_COUNTRY_MAP = {
|
||||
'英国': 'UK', 'uk': 'UK', 'u.k.': 'UK', 'united kingdom': 'UK',
|
||||
'德国': 'DE', 'de': 'DE', 'germany': 'DE',
|
||||
'法国': 'FR', 'fr': 'FR', 'france': 'FR',
|
||||
'意大利': 'IT', 'it': 'IT', 'italy': 'IT',
|
||||
'西班牙': 'ES', 'es': 'ES', 'spain': 'ES',
|
||||
}
|
||||
|
||||
|
||||
def _shop_data_sheet_country(sheet_name):
|
||||
"""结果文件 sheet 名 → 国家码（英国/德国/法国…）；无法识别返回空串。"""
|
||||
key = str(sheet_name or '').strip().lower()
|
||||
return _SHOP_DATA_SHEET_COUNTRY_MAP.get(key, '')
|
||||
|
||||
|
||||
def _shop_data_crawl_parse_workbook(workbook):
|
||||
"""从结果 Workbook 提取 ASIN 行：字典列表，表头按 日期/ASIN/…/价格/…/品牌 识别，跳过无 ASIN 行。
|
||||
|
||||
每行附带所在 sheet 映射的国家码 country（记录级站点，供重复检查页按站点筛选/展示）。
|
||||
"""
|
||||
rows = []
|
||||
for sheet in workbook.worksheets:
|
||||
header_cells = list(next(sheet.iter_rows(min_row=1, max_row=1), []))
|
||||
header = [_shop_data_crawl_cell_text(cell) for cell in header_cells]
|
||||
try:
|
||||
asin_col = header.index('ASIN')
|
||||
except ValueError:
|
||||
continue
|
||||
date_col = header.index('日期') if '日期' in header else None
|
||||
price_col = header.index('价格') if '价格' in header else None
|
||||
brand_col = header.index('品牌') if '品牌' in header else None
|
||||
sheet_country = _shop_data_sheet_country(sheet.title)
|
||||
for sheet_row in sheet.iter_rows(min_row=2):
|
||||
asin = _shop_data_crawl_cell_text(sheet_row[asin_col])
|
||||
if not asin:
|
||||
continue
|
||||
rows.append({
|
||||
'asin': asin,
|
||||
'date': _shop_data_crawl_cell_text(sheet_row[date_col]) if date_col is not None else '',
|
||||
'price': _shop_data_crawl_cell_text(sheet_row[price_col]) if price_col is not None else '',
|
||||
'brand': _shop_data_crawl_cell_text(sheet_row[brand_col]) if brand_col is not None else '',
|
||||
'country': sheet_country,
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def _shops_with_latest_results():
|
||||
"""查询每家店铺最新结果行（用于重复 ASIN 扫描）。返回 shop_items 列表。"""
|
||||
conditions = [
|
||||
"r.module_type = 'SHOP_DATA_CRAWL'",
|
||||
"t.module_type = 'SHOP_DATA_CRAWL'",
|
||||
"TRIM(COALESCE(r.result_file_url, '')) <> ''",
|
||||
]
|
||||
where_sql = ' AND '.join(conditions)
|
||||
conn = get_db()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
shop_key_sql = "TRIM(COALESCE(r.source_filename, ''))"
|
||||
grouped_from_sql = (
|
||||
' FROM biz_file_result r '
|
||||
'JOIN biz_file_task t ON t.id = r.task_id '
|
||||
'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id '
|
||||
'LEFT JOIN users u ON u.id = r.user_id '
|
||||
'WHERE ' + where_sql
|
||||
)
|
||||
cur.execute(
|
||||
'SELECT ' + shop_key_sql + ' AS shop_name, MAX('
|
||||
+ _SHOP_DATA_CRAWL_LATEST_TIME_SQL + ') AS latest_created_at'
|
||||
+ grouped_from_sql +
|
||||
' GROUP BY ' + shop_key_sql +
|
||||
' ORDER BY latest_created_at DESC, shop_name ASC',
|
||||
)
|
||||
group_rows = cur.fetchall()
|
||||
group_names = _shop_data_crawl_group_names(cur, group_rows)
|
||||
result_rows_by_shop = {}
|
||||
selected_shop_names = [row.get('shop_name') for row in group_rows]
|
||||
if selected_shop_names:
|
||||
placeholders = ','.join(['%s'] * len(selected_shop_names))
|
||||
cur.execute(
|
||||
'SELECT ranked.* FROM (SELECT ' + _SHOP_DATA_CRAWL_ADMIN_COLUMNS +
|
||||
', ROW_NUMBER() OVER (PARTITION BY ' + shop_key_sql +
|
||||
' ORDER BY ' + _SHOP_DATA_CRAWL_LATEST_TIME_SQL
|
||||
+ ' DESC, r.id DESC) AS shop_row_number '
|
||||
' FROM biz_file_result r '
|
||||
'JOIN biz_file_task t ON t.id = r.task_id '
|
||||
'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id '
|
||||
'LEFT JOIN users u ON u.id = r.user_id '
|
||||
'WHERE ' + where_sql +
|
||||
f' AND {shop_key_sql} IN ({placeholders})' +
|
||||
') ranked WHERE ranked.shop_row_number <= 1 '
|
||||
'ORDER BY ranked.latest_file_updated_at DESC, ranked.result_id DESC',
|
||||
tuple(selected_shop_names),
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
shop_key = _shop_data_crawl_shop_key(row.get('shop_name'))
|
||||
result_rows_by_shop.setdefault(shop_key, []).append(row)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# 逐店读取结果文件并解析
|
||||
shop_items = []
|
||||
for rows in result_rows_by_shop.values():
|
||||
for row in rows:
|
||||
result_id = int(row.get('result_id') or 0)
|
||||
if result_id <= 0:
|
||||
continue
|
||||
try:
|
||||
raw = _shop_data_crawl_fetch_result_bytes(row)
|
||||
try:
|
||||
parsed = _shop_data_crawl_parse_workbook(
|
||||
load_workbook(io.BytesIO(raw), read_only=True, data_only=True))
|
||||
except (InvalidFileException, KeyError, ValueError, zipfile.BadZipFile) as exc:
|
||||
current_app.logger.warning(
|
||||
'[shop-data-crawl] 解析结果文件失败 result_id=%s: %s', result_id, exc)
|
||||
continue
|
||||
shop_items.append({
|
||||
'shop_name': row.get('shop_name') or '未命名',
|
||||
'group_name': _shop_data_crawl_group_name(group_names, row.get('shop_name')),
|
||||
'country_codes': _shop_data_crawl_country_codes_from_json(row.get('country_codes_json'))
|
||||
or _shop_data_crawl_country_codes(row.get('request_json')),
|
||||
'rows': parsed,
|
||||
})
|
||||
except (requests.RequestException, ValueError) as exc:
|
||||
current_app.logger.warning(
|
||||
'[shop-data-crawl] 拉取结果文件失败 result_id=%s: %s', result_id, exc)
|
||||
return shop_items
|
||||
|
||||
|
||||
def _build_duplicate_scan_cache(shop_items, source='job'):
|
||||
"""由 shop_items 构建全量重复检查缓存：shops 概览 + 明细 items + 指标统计。
|
||||
|
||||
返回 {'shops': [...], 'items': [...], 'summary': {唯一ASIN/上架记录/重复ASIN…}}
|
||||
items 内 occurrences 记录带 country（结果文件 sheet 映射的站点）。
|
||||
"""
|
||||
asin_occurrences = {}
|
||||
shop_agg = {}
|
||||
for shop_item in shop_items:
|
||||
shop_name = shop_item['shop_name'] or '未命名'
|
||||
group_name = shop_item['group_name'] or ''
|
||||
country_codes = shop_item['country_codes'] or []
|
||||
shop_agg[shop_name] = {'group_name': group_name, 'country_codes': country_codes}
|
||||
for row in shop_item['rows']:
|
||||
asin = row['asin'].strip().upper()
|
||||
if not asin:
|
||||
continue
|
||||
asin_occurrences.setdefault(asin, []).append({
|
||||
'asin': asin,
|
||||
'date': row['date'],
|
||||
'price': row['price'],
|
||||
'brand': row['brand'],
|
||||
'shop_name': shop_name,
|
||||
'group_name': group_name,
|
||||
'country_codes': country_codes,
|
||||
'country': row.get('country') or '',
|
||||
})
|
||||
items = []
|
||||
total_records = 0
|
||||
for asin, occurrences in asin_occurrences.items():
|
||||
shop_count = len({item['shop_name'] for item in occurrences})
|
||||
total_records += len(occurrences)
|
||||
items.append({
|
||||
'asin': asin,
|
||||
'shop_count': shop_count,
|
||||
'record_count': len(occurrences),
|
||||
'occurrences': occurrences,
|
||||
})
|
||||
items.sort(key=lambda item: (-item['shop_count'], item['asin']))
|
||||
repeated = sum(1 for item in items if item['shop_count'] >= 2)
|
||||
|
||||
shops = []
|
||||
for shop_name, agg in sorted(shop_agg.items()):
|
||||
shop_asin_set = {
|
||||
occ['asin']
|
||||
for occ_list in (oo for oo in asin_occurrences.values())
|
||||
for occ in occ_list
|
||||
if occ['shop_name'] == shop_name
|
||||
}
|
||||
shops.append({
|
||||
'shop_name': shop_name,
|
||||
'group_name': agg['group_name'],
|
||||
'country_codes': agg['country_codes'],
|
||||
'asin_count': len(shop_asin_set),
|
||||
'record_count': sum(1 for occ_list in asin_occurrences.values() for occ in occ_list if occ['shop_name'] == shop_name),
|
||||
})
|
||||
shops.sort(key=lambda shop: (-shop['asin_count'], shop['shop_name']))
|
||||
|
||||
summary = {
|
||||
'shop_count': len(shops),
|
||||
'asin_total': len(items),
|
||||
'record_total': total_records,
|
||||
'duplicate_asin_total': repeated,
|
||||
'duplicate_shop_count': len({occ['shop_name'] for item in items if item['shop_count'] >= 2 for occ in item['occurrences']}),
|
||||
'site_count': len({occ['country'] for occ_list in asin_occurrences.values() for occ in occ_list if occ.get('country')}),
|
||||
'asin_per_shop': round(float(total_records) / len(shops), 1) if shops else 0.0,
|
||||
'source': source,
|
||||
}
|
||||
return {'shops': shops, 'items': items, 'summary': summary}
|
||||
|
||||
|
||||
def _build_duplicate_scan_items(shop_items):
|
||||
"""由 shop_items 聚合跨店重复 ASIN 完整结果（不含筛选，返回全量列表）。
|
||||
|
||||
保持旧接口语义（仅跨店重复），供导入模拟重分析等旧逻辑复用；
|
||||
新页面走 _build_duplicate_scan_cache 全量缓存。
|
||||
"""
|
||||
cache = _build_duplicate_scan_cache(shop_items)
|
||||
return cache['items']
|
||||
|
||||
|
||||
def _run_duplicate_scan_job():
|
||||
"""执行一次全量重复检查扫描并落库（新格式：shops + items + summary）。返回 (ok, scanned_at, summary)。"""
|
||||
global _duplicate_scan_running
|
||||
if _duplicate_scan_running:
|
||||
return False, '', '扫描进行中'
|
||||
_duplicate_scan_running = True
|
||||
try:
|
||||
shop_items = _shops_with_latest_results()
|
||||
cache = _build_duplicate_scan_cache(shop_items, source='job')
|
||||
_save_duplicate_scan('SUCCESS', summary=cache['summary'], payload={
|
||||
'shops': cache['shops'],
|
||||
'items': cache['items'],
|
||||
})
|
||||
return True, datetime.now().strftime('%Y-%m-%d %H:%M:%S'), cache['summary']
|
||||
except Exception as exc:
|
||||
current_app.logger.exception('[shop-data-crawl] 重复检查定时扫描失败: %s', exc)
|
||||
try:
|
||||
_save_duplicate_scan('FAILED', error_text=str(exc))
|
||||
except Exception:
|
||||
current_app.logger.exception('[shop-data-crawl] 重复检查扫描失败记录落库失败')
|
||||
return False, '', '执行失败：%s' % exc
|
||||
finally:
|
||||
_duplicate_scan_running = False
|
||||
|
||||
|
||||
def start_duplicate_scan_scheduler(app):
|
||||
"""每天凌晨 03:10 执行一次重复 ASIN 全量扫描（后台守护线程）。"""
|
||||
_ensure_duplicate_scan_table()
|
||||
|
||||
def _loop():
|
||||
while True:
|
||||
now = datetime.now()
|
||||
# 下一个 03:10（含今天，若已过则明天）
|
||||
next_run = now.replace(hour=3, minute=10, second=0, microsecond=0)
|
||||
if next_run <= now:
|
||||
next_run = next_run.replace(day=next_run.day) + timedelta(days=1)
|
||||
delay = (next_run - now).total_seconds()
|
||||
app.logger.info('[shop-data-crawl] 重复ASIN定时扫描将于 %s 执行（%d 秒后）', next_run, int(delay))
|
||||
time.sleep(delay)
|
||||
try:
|
||||
with app.app_context():
|
||||
ok, scanned_at, summary = _run_duplicate_scan_job()
|
||||
app.logger.info('[shop-data-crawl] 重复ASIN定时扫描结束 ok=%s summary=%s', ok, summary)
|
||||
except Exception as exc:
|
||||
app.logger.exception('[shop-data-crawl] 重复ASIN定时扫描异常: %s', exc)
|
||||
|
||||
threading.Thread(target=_loop, daemon=True, name='duplicate-scan-scheduler').start()
|
||||
|
||||
|
||||
def _duplicate_check_filter_cache(cache, role, current_row):
|
||||
"""按角色数据范围裁剪重复检查缓存：超管返回全量，主管只返回自己组的店铺。
|
||||
|
||||
返回 (shops, items)：shops 为可见店铺概览；items 为裁剪 occurrences 后的明细
|
||||
（裁剪后同一 ASIN 只剩 1 家店时也会保留 —— 展示完整台账，是否重复由 shop_count>=2 标记）。
|
||||
"""
|
||||
visible_shops = _shop_data_managed_shop_names(role, current_row)
|
||||
if visible_shops is None:
|
||||
return cache.get('shops') or [], cache.get('items') or []
|
||||
shops = []
|
||||
for shop in (cache.get('shops') or []):
|
||||
if _shop_data_crawl_shop_key(shop.get('shop_name')) in visible_shops:
|
||||
shops.append(shop)
|
||||
items = []
|
||||
for item in (cache.get('items') or []):
|
||||
occurrences = [
|
||||
occ for occ in (item.get('occurrences') or [])
|
||||
if _shop_data_crawl_shop_key(occ.get('shop_name')) in visible_shops
|
||||
]
|
||||
if occurrences:
|
||||
items.append({
|
||||
'asin': item.get('asin'),
|
||||
'shop_count': len({occ.get('shop_name') for occ in occurrences}),
|
||||
'record_count': len(occurrences),
|
||||
'occurrences': occurrences,
|
||||
})
|
||||
return shops, items
|
||||
|
||||
|
||||
@admin_api.route('/shop-data-crawl/duplicate-check-overview')
|
||||
@login_required
|
||||
def shop_data_crawl_duplicate_check_overview():
|
||||
@@ -2307,58 +1835,17 @@ def shop_data_crawl_duplicate_check_overview():
|
||||
if denied:
|
||||
return denied
|
||||
try:
|
||||
force = (request.args.get('force') or '').strip() in ('1', 'true', 'yes')
|
||||
if force:
|
||||
if not _duplicate_scan_lock.acquire(blocking=False):
|
||||
return jsonify({'success': False, 'error': '扫描进行中，请稍后刷新'}), 409
|
||||
try:
|
||||
ok, scanned_at, summary = _run_duplicate_scan_job()
|
||||
if not ok:
|
||||
return jsonify({'success': False, 'error': summary}), 500
|
||||
cache = _latest_duplicate_scan()
|
||||
finally:
|
||||
_duplicate_scan_lock.release()
|
||||
else:
|
||||
cache = _latest_duplicate_scan()
|
||||
if not cache:
|
||||
return jsonify({
|
||||
'success': True, 'pending': True,
|
||||
'scanned_at': '', 'source': '',
|
||||
'summary': {}, 'shops': [],
|
||||
})
|
||||
shops, items = _duplicate_check_filter_cache(cache, role, current_row)
|
||||
asin_total = len(items)
|
||||
record_total = sum(item.get('record_count') or 0 for item in items)
|
||||
repeated = [item for item in items if item['shop_count'] >= 2]
|
||||
shop_asin_totals = {}
|
||||
duplicate_shops = set()
|
||||
site_codes = set()
|
||||
for item in items:
|
||||
occs = item.get('occurrences') or []
|
||||
for occ in occs:
|
||||
shop_asin_totals[occ.get('shop_name')] = shop_asin_totals.get(occ.get('shop_name'), set())
|
||||
shop_asin_totals[occ.get('shop_name')].add(item['asin'])
|
||||
if item['shop_count'] >= 2:
|
||||
duplicate_shops.add(occ.get('shop_name'))
|
||||
if occ.get('country'):
|
||||
site_codes.add(occ['country'])
|
||||
summary = {
|
||||
'shop_count': len(shops),
|
||||
'asin_total': asin_total,
|
||||
'record_total': record_total,
|
||||
'duplicate_asin_total': len(repeated),
|
||||
'duplicate_shop_count': len(duplicate_shops),
|
||||
'site_count': len(site_codes),
|
||||
'asin_per_shop': round(float(record_total) / len(shops), 1) if shops else 0.0,
|
||||
'source': (cache.get('summary') or {}).get('source', ''),
|
||||
}
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'pending': False,
|
||||
'scanned_at': cache['scanned_at'],
|
||||
'summary': summary,
|
||||
'shops': shops,
|
||||
})
|
||||
force_raw = (request.args.get('force') or '').strip()
|
||||
params = {}
|
||||
if force_raw in ('1', 'true', 'yes'):
|
||||
params['force'] = '1'
|
||||
result, error_response, status = _proxy_permission_java(
|
||||
'GET', '/api/admin/shop-data-crawl/duplicate-check-overview',
|
||||
params=params, timeout=(10, 1800))
|
||||
if error_response is not None:
|
||||
return error_response, status
|
||||
payload = result.get('data') or {}
|
||||
return jsonify({'success': True, **payload})
|
||||
except Exception as exc:
|
||||
return _internal_error(exc)
|
||||
|
||||
@@ -2373,78 +1860,68 @@ def shop_data_crawl_duplicate_check_items():
|
||||
if denied:
|
||||
return denied
|
||||
try:
|
||||
role, current_row = get_current_admin_role()
|
||||
page = max(1, int(request.args.get('page', 1)))
|
||||
page_size = min(100, max(10, int(request.args.get('page_size', 20))))
|
||||
view = (request.args.get('view') or 'monitor').strip().lower()
|
||||
asin_filter = (request.args.get('asin') or '').strip().upper()
|
||||
shop_name_filter = (request.args.get('shop_name') or request.args.get('shop') or '').strip()
|
||||
country_filter = (request.args.get('country') or '').strip().upper()
|
||||
site_filter = (request.args.get('site') or '').strip().upper()
|
||||
date_from = (request.args.get('date_from') or '').strip()[:10]
|
||||
date_to = (request.args.get('date_to') or '').strip()[:10]
|
||||
|
||||
cache = _latest_duplicate_scan()
|
||||
if not cache:
|
||||
return jsonify({
|
||||
'success': True, 'pending': True,
|
||||
'items': [], 'shops': [],
|
||||
'total': 0, 'page': page, 'page_size': page_size,
|
||||
'scanned_at': '',
|
||||
})
|
||||
shops, all_items = _duplicate_check_filter_cache(cache, role, current_row)
|
||||
has_filter = bool(asin_filter or shop_name_filter or country_filter or site_filter or date_from or date_to)
|
||||
matched = []
|
||||
for item in all_items:
|
||||
if asin_filter and asin_filter not in item['asin']:
|
||||
continue
|
||||
if view == 'monitor' and item['shop_count'] < 2:
|
||||
continue
|
||||
if not has_filter:
|
||||
matched.append(item)
|
||||
continue
|
||||
hit = False
|
||||
for occ in item['occurrences']:
|
||||
if shop_name_filter and shop_name_filter.lower() not in (occ.get('shop_name') or '').lower() \
|
||||
and shop_name_filter.lower() not in (occ.get('group_name') or '').lower():
|
||||
continue
|
||||
if country_filter and country_filter not in [c.upper() for c in (occ.get('country_codes') or [])]:
|
||||
continue
|
||||
if site_filter and site_filter not in [c.upper() for c in [occ.get('country') or '']]:
|
||||
continue
|
||||
row_date_key = _shop_data_date_key((occ.get('date') or '').strip())
|
||||
if date_from and row_date_key and row_date_key < date_from:
|
||||
continue
|
||||
if date_to and row_date_key and row_date_key > date_to:
|
||||
continue
|
||||
hit = True
|
||||
break
|
||||
if hit:
|
||||
matched.append(item)
|
||||
total = len(matched)
|
||||
offset = (page - 1) * page_size
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'pending': False,
|
||||
'items': matched[offset:offset + page_size],
|
||||
'shops': shops,
|
||||
'total': total,
|
||||
'page': page,
|
||||
'page_size': page_size,
|
||||
'scanned_at': cache['scanned_at'],
|
||||
})
|
||||
params = {'page': str(page), 'pageSize': str(page_size), 'view': view}
|
||||
shop_name = (request.args.get('shop_name') or request.args.get('shop') or '').strip()
|
||||
for key, value in (
|
||||
('asin', (request.args.get('asin') or '').strip()),
|
||||
('shopName', shop_name),
|
||||
('country', (request.args.get('country') or '').strip()),
|
||||
('site', (request.args.get('site') or '').strip()),
|
||||
('dateFrom', (request.args.get('date_from') or '').strip()),
|
||||
('dateTo', (request.args.get('date_to') or '').strip())):
|
||||
if value:
|
||||
params[key] = value
|
||||
result, error_response, status = _proxy_permission_java(
|
||||
'GET', '/api/admin/shop-data-crawl/duplicate-check-items', params=params)
|
||||
if error_response is not None:
|
||||
return error_response, status
|
||||
payload = result.get('data') or {}
|
||||
return jsonify({'success': True, **payload})
|
||||
except ValueError as exc:
|
||||
return jsonify({'success': False, 'error': str(exc)}), 400
|
||||
except Exception as exc:
|
||||
return _internal_error(exc)
|
||||
|
||||
|
||||
def _duplicate_check_effective_site(country, country_codes):
|
||||
"""导出用站点：优先记录级 country，其次店铺 country_codes 汇总。"""
|
||||
if country:
|
||||
return country.upper()
|
||||
codes = [c.upper() for c in (country_codes or []) if c]
|
||||
return '、'.join(codes) if codes else ''
|
||||
@admin_api.route('/shop-data-crawl/duplicate-check-detail')
|
||||
@login_required
|
||||
def shop_data_crawl_duplicate_check_detail():
|
||||
"""撞款详情（跨店重复 ASIN 卡片区）：分页返回 shop_count>=2 的 ASIN 明细。
|
||||
|
||||
筛选条件与矩阵接口一致（遵循当前筛选），排序：店铺数倒序 → 最早上架时间倒序 → ASIN 升序。
|
||||
"""
|
||||
_, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks')
|
||||
if not denied:
|
||||
_, _, denied = _ensure_shop_data_crawl_data_access()
|
||||
if denied:
|
||||
return denied
|
||||
try:
|
||||
page = max(1, int(request.args.get('page', 1)))
|
||||
page_size = min(24, max(1, int(request.args.get('page_size', 6))))
|
||||
params = {'page': str(page), 'pageSize': str(page_size)}
|
||||
shop_name = (request.args.get('shop_name') or request.args.get('shop') or '').strip()
|
||||
for key, value in (
|
||||
('asin', (request.args.get('asin') or '').strip()),
|
||||
('shopName', shop_name),
|
||||
('country', (request.args.get('country') or '').strip()),
|
||||
('site', (request.args.get('site') or '').strip()),
|
||||
('dateFrom', (request.args.get('date_from') or '').strip()),
|
||||
('dateTo', (request.args.get('date_to') or '').strip())):
|
||||
if value:
|
||||
params[key] = value
|
||||
result, error_response, status = _proxy_permission_java(
|
||||
'GET', '/api/admin/shop-data-crawl/duplicate-check-detail', params=params)
|
||||
if error_response is not None:
|
||||
return error_response, status
|
||||
payload = result.get('data') or {}
|
||||
return jsonify({'success': True, **payload})
|
||||
except ValueError as exc:
|
||||
return jsonify({'success': False, 'error': str(exc)}), 400
|
||||
except Exception as exc:
|
||||
return _internal_error(exc)
|
||||
|
||||
|
||||
@admin_api.route('/shop-data-crawl/duplicate-check-export')
|
||||
@@ -2459,161 +1936,61 @@ def shop_data_crawl_duplicate_check_export():
|
||||
try:
|
||||
role, current_row = get_current_admin_role()
|
||||
view = (request.args.get('view') or 'monitor').strip().lower()
|
||||
asin_filter = (request.args.get('asin') or '').strip().upper()
|
||||
shop_name_filter = (request.args.get('shop_name') or request.args.get('shop') or '').strip()
|
||||
country_filter = (request.args.get('country') or '').strip().upper()
|
||||
site_filter = (request.args.get('site') or '').strip().upper()
|
||||
date_from = (request.args.get('date_from') or '').strip()[:10]
|
||||
date_to = (request.args.get('date_to') or '').strip()[:10]
|
||||
|
||||
cache = _latest_duplicate_scan()
|
||||
if not cache:
|
||||
return jsonify({'success': False, 'error': '暂无扫描结果，请先点击「重新分析」'}), 400
|
||||
shops, all_items = _duplicate_check_filter_cache(cache, role, current_row)
|
||||
has_filter = bool(asin_filter or shop_name_filter or country_filter or site_filter or date_from or date_to)
|
||||
# 展平为逐行记录（每行=一条上架记录），筛选逻辑与矩阵接口一致
|
||||
flat = []
|
||||
for item in all_items:
|
||||
if asin_filter and asin_filter not in item['asin']:
|
||||
continue
|
||||
if view == 'monitor' and item['shop_count'] < 2:
|
||||
continue
|
||||
for occ in item['occurrences']:
|
||||
if shop_name_filter and shop_name_filter.lower() not in (occ.get('shop_name') or '').lower() \
|
||||
and shop_name_filter.lower() not in (occ.get('group_name') or '').lower():
|
||||
continue
|
||||
if country_filter and country_filter not in [c.upper() for c in (occ.get('country_codes') or [])]:
|
||||
continue
|
||||
if site_filter and site_filter not in [c.upper() for c in [occ.get('country') or '']]:
|
||||
continue
|
||||
row_date_key = _shop_data_date_key((occ.get('date') or '').strip())
|
||||
if date_from and row_date_key and row_date_key < date_from:
|
||||
continue
|
||||
if date_to and row_date_key and row_date_key > date_to:
|
||||
continue
|
||||
flat.append({
|
||||
'asin': item['asin'],
|
||||
'shop_count': item['shop_count'],
|
||||
'shop_name': occ.get('shop_name') or '',
|
||||
'group_name': occ.get('group_name') or '',
|
||||
'country': _duplicate_check_effective_site(occ.get('country'), occ.get('country_codes')),
|
||||
'date': occ.get('date') or '',
|
||||
'price': occ.get('price') or '',
|
||||
'brand': occ.get('brand') or '',
|
||||
})
|
||||
flat.sort(key=lambda r: (-r['shop_count'], r['asin'], r['shop_name'], r['date']))
|
||||
import csv
|
||||
stream = io.StringIO()
|
||||
writer = csv.writer(stream)
|
||||
writer.writerow(['ASIN', '店铺数', '店铺', '分组', '站点', '上架时间', '价格', '品牌'])
|
||||
for row in flat:
|
||||
writer.writerow([row['asin'], row['shop_count'], row['shop_name'], row['group_name'],
|
||||
row['country'], row['date'], row['price'], row['brand']])
|
||||
payload = ('' + stream.getvalue()).encode('utf-8')
|
||||
response = Response(payload, mimetype='text/csv; charset=utf-8')
|
||||
response.headers['Content-Disposition'] = 'attachment; filename="shop-data-duplicate-check.csv"'
|
||||
response.headers['Cache-Control'] = 'no-store'
|
||||
return response
|
||||
except Exception as exc:
|
||||
return _internal_error(exc)
|
||||
|
||||
|
||||
@admin_api.route('/shop-data-crawl/duplicate-asins')
|
||||
@login_required
|
||||
def shop_data_crawl_duplicate_asins():
|
||||
"""按当前筛选条件展示跨店铺重复的 ASIN 明细。
|
||||
|
||||
默认读取最近一次成功扫描的缓存结果（定时任务每天凌晨全量扫描），
|
||||
页面点「重新分析」携带 force=1 触发一次实时扫描（锁防并发，重复触发返回 409）。
|
||||
"""
|
||||
_, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks')
|
||||
if not denied:
|
||||
_, _, denied = _ensure_shop_data_crawl_data_access()
|
||||
if denied:
|
||||
return denied
|
||||
try:
|
||||
page = max(1, int(request.args.get('page', 1)))
|
||||
page_size = min(100, max(10, int(request.args.get('page_size', 20))))
|
||||
force = (request.args.get('force') or '').strip() in ('1', 'true', 'yes')
|
||||
shop_name_filter = (request.args.get('shop_name') or request.args.get('shop') or '').strip()
|
||||
country_filter = (request.args.get('country') or '').strip().upper()
|
||||
asin_filter = (request.args.get('asin') or '').strip().upper()
|
||||
date_from = (request.args.get('date_from') or '').strip()[:10]
|
||||
date_to = (request.args.get('date_to') or '').strip()[:10]
|
||||
|
||||
if force:
|
||||
# 实时扫描：持锁执行，避免与定时任务/其他请求并发
|
||||
if not _duplicate_scan_lock.acquire(blocking=False):
|
||||
return jsonify({'success': False, 'error': '扫描进行中，请稍后刷新'}), 409
|
||||
operator_id = (current_row or {}).get('id')
|
||||
if not operator_id and has_request_context():
|
||||
operator_id = session.get('user_id')
|
||||
params = {'view': view}
|
||||
if operator_id:
|
||||
params['operatorId'] = operator_id
|
||||
shop_name = (request.args.get('shop_name') or request.args.get('shop') or '').strip()
|
||||
for key, value in (
|
||||
('asin', (request.args.get('asin') or '').strip()),
|
||||
('shopName', shop_name),
|
||||
('country', (request.args.get('country') or '').strip()),
|
||||
('site', (request.args.get('site') or '').strip()),
|
||||
('dateFrom', (request.args.get('date_from') or '').strip()),
|
||||
('dateTo', (request.args.get('date_to') or '').strip())):
|
||||
if value:
|
||||
params[key] = value
|
||||
url = f"{backend_java_base_url}/api/admin/shop-data-crawl/duplicate-check-export"
|
||||
try:
|
||||
resp = _get_backend_java_session().get(
|
||||
url,
|
||||
params=params,
|
||||
headers={'X-Internal-Token': _resolve_internal_token()},
|
||||
stream=True,
|
||||
timeout=(10, 1800),
|
||||
)
|
||||
except requests.RequestException:
|
||||
return jsonify({'success': False, 'error': 'backend-java 服务不可用'}), 502
|
||||
if resp.status_code >= 400:
|
||||
try:
|
||||
ok, scanned_at, summary = _run_duplicate_scan_job()
|
||||
if not ok:
|
||||
return jsonify({'success': False, 'error': summary}), 500
|
||||
cache = _latest_duplicate_scan()
|
||||
data = resp.json()
|
||||
error = data.get('message') or data.get('error') or '导出失败'
|
||||
except ValueError:
|
||||
error = '导出失败'
|
||||
resp.close()
|
||||
return jsonify({'success': False, 'error': error}), resp.status_code
|
||||
|
||||
headers = {}
|
||||
disposition = resp.headers.get('Content-Disposition')
|
||||
if disposition:
|
||||
headers['Content-Disposition'] = disposition
|
||||
|
||||
def generate():
|
||||
try:
|
||||
for chunk in resp.iter_content(chunk_size=1024 * 1024):
|
||||
if chunk:
|
||||
yield chunk
|
||||
finally:
|
||||
_duplicate_scan_lock.release()
|
||||
else:
|
||||
cache = _latest_duplicate_scan()
|
||||
resp.close()
|
||||
|
||||
if not cache:
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'items': [],
|
||||
'total': 0,
|
||||
'page': page,
|
||||
'page_size': page_size,
|
||||
'analyzed_shop_count': 0,
|
||||
'analyzed_result_count': 0,
|
||||
'scanned_at': '',
|
||||
'pending': True,
|
||||
})
|
||||
|
||||
all_items = cache['items'] or []
|
||||
# 旧接口只展示跨店重复：由全量缓存裁剪出 shop_count>=2 的 ASIN
|
||||
all_items = [item for item in all_items if (item.get('shop_count') or 0) >= 2]
|
||||
has_filter = bool(asin_filter or shop_name_filter or country_filter or date_from or date_to)
|
||||
matched = []
|
||||
for item in all_items:
|
||||
if asin_filter and asin_filter not in item['asin']:
|
||||
continue
|
||||
if not has_filter:
|
||||
matched.append(item)
|
||||
continue
|
||||
# 有筛选时：任一 occurrence 命中即保留该 ASIN（展示完整记录）
|
||||
hit = False
|
||||
for occ in item['occurrences']:
|
||||
if shop_name_filter and shop_name_filter.lower() not in (occ.get('shop_name') or '').lower() \
|
||||
and shop_name_filter.lower() not in (occ.get('group_name') or '').lower():
|
||||
continue
|
||||
if country_filter and country_filter not in [c.upper() for c in (occ.get('country_codes') or [])]:
|
||||
continue
|
||||
row_date_key = _shop_data_date_key((occ.get('date') or '').strip())
|
||||
if date_from and row_date_key and row_date_key < date_from:
|
||||
continue
|
||||
if date_to and row_date_key and row_date_key > date_to:
|
||||
continue
|
||||
hit = True
|
||||
break
|
||||
if hit:
|
||||
matched.append(item)
|
||||
|
||||
total_details = len(matched)
|
||||
offset = (page - 1) * page_size
|
||||
paged_details = matched[offset:offset + page_size]
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'items': paged_details,
|
||||
'total': total_details,
|
||||
'page': page,
|
||||
'page_size': page_size,
|
||||
'analyzed_shop_count': cache['summary'].get('shop_count', 0) if cache.get('summary') else 0,
|
||||
'analyzed_result_count': len(all_items),
|
||||
'scanned_at': cache['scanned_at'],
|
||||
'source': cache.get('summary', {}).get('source', ''),
|
||||
})
|
||||
except ValueError as exc:
|
||||
return jsonify({'success': False, 'error': str(exc)}), 400
|
||||
return Response(
|
||||
stream_with_context(generate()),
|
||||
status=resp.status_code,
|
||||
headers=headers,
|
||||
content_type=resp.headers.get('Content-Type', 'text/csv; charset=utf-8'),
|
||||
)
|
||||
except Exception as exc:
|
||||
return _internal_error(exc)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user