后台店铺数据记录页新增 Tab 切换与重复 ASIN 分析;task-169/170 HTTP 客户端配置接入;dedupe/brand 相关改造
Build Backend JAR / build (push) Has been cancelled

- 管理后台:店铺数据记录改为列表分页展示(去任务号/文件列/累计标题,'最新'改'更新时间'),新增'重复 ASIN' Tab 按跨店聚合展示 ASIN/店铺/分组/国家/日期/价格;修复失败/进行中任务被过滤不显示的问题
- task-169: BrandCheckHttpConfigResolver 从 aiimage.http-client.* 读取品牌检查超时/重试(默认同现状、钳制),附 8 条测试
- task-170: surefire 内存调整为 1536m
- dedupe: DedupeRunService 重构(事务边界/进度)、新增 DedupeRunProgressVo、Controller 与结果 VO 调整,PermissionMenuService 相应适配
- brand/dedupe 前端:BrandDedupeTab/BrandPatrolDeleteTab 改造、新增 CountrySelector 组件与 country-options、类型与接口端对齐、测试更新
- 移除无引用文件:backend/static/logo.jpg、prompts/
This commit is contained in:
2026-09-03 01:57:41 +08:00
parent 152ea6eec0
commit 417a2bf831
20 changed files with 1883 additions and 475 deletions
+265
View File
@@ -1,6 +1,7 @@
"""
管理员 API 蓝图:用户管理、生成历史、版本管理(后台)
"""
import io
import json
import os
import re
@@ -13,6 +14,8 @@ from pathlib import Path
from urllib.parse import quote
import requests
from openpyxl import load_workbook
from openpyxl.utils.exceptions import InvalidFileException
from requests.adapters import HTTPAdapter
from flask import (
Blueprint,
@@ -1690,6 +1693,268 @@ def _load_shop_data_crawl_download_rows(result_ids):
conn.close()
_SHOP_DATA_CRAWL_ASIN_ANALYSIS_MAX_BYTES = 256 * 1024 * 1024
_SHOP_DATA_CRAWL_ASIN_ANALYSIS_TIMEOUT = (10, 60)
def _shop_data_crawl_fetch_result_bytes(row, timeout=None):
"""从 Java 下载接口拉取结果文件字节流(仅内存,不落盘)。"""
url = f"{backend_java_base_url}/api/admin/shop-data-crawl/results/{int(row['id'])}/download"
headers, params = _backend_java_internal_request()
response = _get_backend_java_session().get(
url,
params=params,
headers=headers,
stream=True,
timeout=timeout or _SHOP_DATA_CRAWL_ASIN_ANALYSIS_TIMEOUT,
)
try:
response.raise_for_status()
total = 0
chunks = []
for chunk in response.iter_content(chunk_size=1024 * 1024):
if not chunk:
continue
total += len(chunk)
if total > _SHOP_DATA_CRAWL_ASIN_ANALYSIS_MAX_BYTES:
raise ValueError('结果文件过大,无法分析')
chunks.append(chunk)
return b''.join(chunks)
finally:
response.close()
def _shop_data_crawl_cell_text(cell):
"""读取单元格文本:日期/数字等统一转字符串,None 返回空串。"""
if cell is None:
return ''
value = cell.value
if value is None:
return ''
if isinstance(value, datetime):
return value.strftime('%Y-%m-%d')
if isinstance(value, float) and value.is_integer():
return str(int(value))
return str(value).strip()
def _shop_data_crawl_parse_workbook(workbook):
"""从结果 Workbook 提取 ASIN 行:字典列表,表头按 日期/ASIN/…/价格/…/品牌 识别,跳过无 ASIN 行。"""
rows = []
for sheet in workbook.worksheets:
header_cells = list(next(sheet.iter_rows(min_row=1, max_row=1), []))
header = [_shop_data_crawl_cell_text(cell) for cell in header_cells]
try:
asin_col = header.index('ASIN')
except ValueError:
continue
date_col = header.index('日期') if '日期' in header else None
price_col = header.index('价格') if '价格' in header else None
brand_col = header.index('品牌') if '品牌' in header else None
for sheet_row in sheet.iter_rows(min_row=2):
asin = _shop_data_crawl_cell_text(sheet_row[asin_col])
if not asin:
continue
rows.append({
'asin': asin,
'date': _shop_data_crawl_cell_text(sheet_row[date_col]) if date_col is not None else '',
'price': _shop_data_crawl_cell_text(sheet_row[price_col]) if price_col is not None else '',
'brand': _shop_data_crawl_cell_text(sheet_row[brand_col]) if brand_col is not None else '',
})
return rows
@admin_api.route('/shop-data-crawl/duplicate-asins')
@login_required
def shop_data_crawl_duplicate_asins():
"""按当前店铺列表筛选条件,分析跨店铺重复的 ASIN 明细。
读取当前页每家店铺最新结果 Excel(经 Java 下载接口拉取),按 ASIN 聚合
其出现的店铺、国家与「日期/价格/品牌」细节,仅返回出现在 2 家及以上店铺
的 ASIN(按店铺数降序、ASIN 升序),空 pagination 参数时返回全量用于导出。
"""
_, _, denied = _ensure_backend_menu_access('shop-data-crawl-tasks')
if not denied:
_, _, denied = _ensure_shop_data_crawl_data_access()
if denied:
return denied
try:
page = max(1, int(request.args.get('page', 1)))
page_size = min(100, max(10, int(request.args.get('page_size', 20))))
shop_name = (request.args.get('shop_name') or request.args.get('shop') or '').strip()
group_name = (request.args.get('group_name') or request.args.get('group') or '').strip()
country = (request.args.get('country') or '').strip().upper()
created_from = _parse_admin_datetime_arg('created_from')
created_to = _parse_admin_datetime_arg('created_to')
conditions = [
"r.module_type = 'SHOP_DATA_CRAWL'",
"t.module_type = 'SHOP_DATA_CRAWL'",
"TRIM(COALESCE(r.result_file_url, '')) <> ''",
]
params = []
if shop_name:
conditions.append('r.source_filename LIKE %s')
params.append('%' + shop_name + '%')
if group_name:
conditions.append(
'EXISTS (SELECT 1 FROM biz_shop_manage sm '
'LEFT JOIN biz_shop_manage_group g ON g.id = sm.group_id '
'WHERE TRIM(COALESCE(sm.shop_name, \'\')) = '
'TRIM(COALESCE(r.source_filename, \'\')) '
"AND COALESCE(NULLIF(g.group_name, ''), NULLIF(sm.group_name, '')) LIKE %s)"
)
params.append('%' + group_name + '%')
if country:
if country not in ('DE', 'UK', 'FR', 'IT', 'ES'):
raise ValueError('不支持的国家代码: ' + country)
conditions.append(
'JSON_CONTAINS('
'COALESCE(df.country_codes_json, '
'JSON_EXTRACT(t.request_json, \'$.countryCodes\'), '
'JSON_EXTRACT(t.request_json, \'$.country_codes\'), \'[]\'), %s)'
)
params.append('"' + country + '"')
if created_from:
conditions.append('t.created_at >= %s')
params.append(created_from)
if created_to:
conditions.append('t.created_at <= %s')
params.append(created_to)
where_sql = ' AND '.join(conditions)
offset = (page - 1) * page_size
conn = get_db()
try:
with conn.cursor() as cur:
shop_key_sql = "TRIM(COALESCE(r.source_filename, ''))"
grouped_from_sql = (
' FROM biz_file_result r '
'JOIN biz_file_task t ON t.id = r.task_id '
'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id '
'LEFT JOIN users u ON u.id = r.user_id '
'WHERE ' + where_sql
)
cur.execute(
'SELECT COUNT(*) AS total FROM ('
'SELECT ' + shop_key_sql + ' AS shop_key' + grouped_from_sql +
' GROUP BY ' + shop_key_sql +
') shop_groups',
tuple(params),
)
total = int((cur.fetchone() or {}).get('total') or 0)
cur.execute(
'SELECT ' + shop_key_sql + ' AS shop_name, MAX('
+ _SHOP_DATA_CRAWL_LATEST_TIME_SQL + ') AS latest_created_at'
+ grouped_from_sql +
' GROUP BY ' + shop_key_sql +
' ORDER BY latest_created_at DESC, shop_name ASC LIMIT %s OFFSET %s',
tuple(params + [page_size, offset]),
)
group_rows = cur.fetchall()
group_names = _shop_data_crawl_group_names(cur, group_rows)
result_rows_by_shop = {}
selected_shop_names = [row.get('shop_name') for row in group_rows]
if selected_shop_names:
placeholders = ','.join(['%s'] * len(selected_shop_names))
cur.execute(
'SELECT ranked.* FROM (SELECT ' + _SHOP_DATA_CRAWL_ADMIN_COLUMNS +
', ROW_NUMBER() OVER (PARTITION BY ' + shop_key_sql +
' ORDER BY ' + _SHOP_DATA_CRAWL_LATEST_TIME_SQL
+ ' DESC, r.id DESC) AS shop_row_number '
' FROM biz_file_result r '
'JOIN biz_file_task t ON t.id = r.task_id '
'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id '
'LEFT JOIN users u ON u.id = r.user_id '
'WHERE ' + where_sql +
f' AND {shop_key_sql} IN ({placeholders})' +
') ranked WHERE ranked.shop_row_number <= 1 '
'ORDER BY ranked.latest_file_updated_at DESC, ranked.result_id DESC',
tuple(params + selected_shop_names),
)
for row in cur.fetchall():
shop_key = _shop_data_crawl_shop_key(row.get('shop_name'))
result_rows_by_shop.setdefault(shop_key, []).append(row)
finally:
conn.close()
# 逐店读取结果文件并解析(每店最多一个结果文件,共 page_size 个)
shop_items = []
for rows in result_rows_by_shop.values():
for row in rows:
result_id = int(row.get('result_id') or 0)
if result_id <= 0:
continue
try:
raw = _shop_data_crawl_fetch_result_bytes(row)
try:
parsed = _shop_data_crawl_parse_workbook(
load_workbook(io.BytesIO(raw), read_only=True, data_only=True))
except (InvalidFileException, KeyError, ValueError) as exc:
current_app.logger.warning(
'[shop-data-crawl] 解析结果文件失败 result_id=%s: %s', result_id, exc)
continue
shop_items.append({
'shop_name': row.get('shop_name') or '未命名',
'group_name': _shop_data_crawl_group_name(group_names, row.get('shop_name')),
'country_codes': _shop_data_crawl_country_codes_from_json(row.get('country_codes_json'))
or _shop_data_crawl_country_codes(row.get('request_json')),
'rows': parsed,
})
except (requests.RequestException, ValueError) as exc:
current_app.logger.warning(
'[shop-data-crawl] 拉取结果文件失败 result_id=%s: %s', result_id, exc)
# 按 ASIN 聚合其出现的店铺/分组/国家与行细节
asin_occurrences = {}
for shop_item in shop_items:
shop_name = shop_item['shop_name'] or '未命名'
for row in shop_item['rows']:
asin = row['asin'].strip().upper()
if not asin:
continue
asin_occurrences.setdefault(asin, []).append({
'asin': asin,
'date': row['date'],
'price': row['price'],
'brand': row['brand'],
'shop_name': shop_name,
'group_name': shop_item['group_name'],
'country_codes': shop_item['country_codes'],
})
occurrences_list = []
for asin, occurrences in asin_occurrences.items():
shop_count = len({item['shop_name'] for item in occurrences})
if shop_count < 2:
continue
occurrences_list.append({
'asin': asin,
'shop_count': shop_count,
'occurrences': occurrences,
})
occurrences_list.sort(key=lambda item: (-item['shop_count'], item['asin']))
total_details = len(occurrences_list)
paged_details = occurrences_list[offset:offset + page_size]
return jsonify({
'success': True,
'items': paged_details,
'total': total_details,
'page': page,
'page_size': page_size,
'analyzed_shop_count': len(shop_items),
'analyzed_result_count': len(shop_items),
})
except ValueError as exc:
return jsonify({'success': False, 'error': str(exc)}), 400
except Exception as exc:
return _internal_error(exc)
def _open_shop_data_crawl_download(row):
url = f"{backend_java_base_url}/api/admin/shop-data-crawl/results/{int(row['id'])}/download"
headers, params = _backend_java_internal_request()
+186 -52
View File
@@ -1722,7 +1722,8 @@
if (!rawResult || typeof rawResult !== 'object') return;
var result = shopDataNormalizeResult(groupBase, rawResult);
var resultId = shopDataResultId(result);
if (!resultId || !result.file_ready) return;
// 失败/进行中的任务没有结果文件也保留显示(禁用下载、可删除)
if (!resultId) return;
if (resultId && group.results.some(function (existing) { return shopDataResultId(existing) === resultId; })) return;
group.results.push(result);
if (!group.shop_name) group.shop_name = result.shop_name || '';
@@ -1747,55 +1748,58 @@
function renderShopDataStatus(item, status) {
var normalized = String(status || '-').toUpperCase();
var errorTitle = item && item.error ? ' title="' + escapeHtml(item.error) + '"' : '';
return '<span class="image-video-status ' + escapeHtml(normalized) + '"' + errorTitle + '>' + escapeHtml(imageVideoStatusLabel(normalized)) + '</span>';
var cls = normalized === 'SUCCESS' || normalized === 'COMPLETED'
? 'success'
: (normalized === 'FAILED' || normalized === 'CANCELLED' ? 'failed' : 'running');
return '<span class="shop-data-status ' + cls + '"' + errorTitle + '>' + escapeHtml(imageVideoStatusLabel(normalized)) + '</span>';
}
function renderShopDataTaskResult(item) {
function renderShopDataRecordRow(item) {
var resultId = shopDataResultId(item);
var selected = resultId > 0 && selectedShopDataResultIds.has(resultId);
var status = String(item.status || item.file_status || '').toUpperCase();
var terminal = ['SUCCESS', 'FAILED', 'CANCELLED'].indexOf(status) >= 0;
var countryCodes = item.country_codes != null ? item.country_codes : item.countryCodes;
var countries = countryListLabel(countryCodes);
var filename = item.output_filename || '-';
var updatedAt = item.updated_at || item.latest_created_at || item.created_at || item.finished_at || '-';
var checkbox = '<input type="checkbox" data-shop-data-select="' + (resultId || '') + '"' +
(selected ? ' checked' : '') + (item.file_ready && resultId > 0 ? '' : ' disabled') + '>';
return '<div class="shop-data-result' + (selected ? ' selected' : '') + '" data-shop-data-card="' + (resultId || '') + '">' +
'<div class="shop-data-result-head">' +
'<label class="shop-data-task-title">' + checkbox +
'<span title="任务 ' + escapeHtml(item.task_id || item.task_no || '-') + '">任务 ' + escapeHtml(item.task_id || item.task_no || '-') + '</span>' +
'</label>' +
renderShopDataStatus(item, status || item.file_status) +
'</div>' +
'<div class="image-video-card-info">' +
'<div class="image-video-info-row"><label>国家</label><span>' + escapeHtml(countries) + '</span></div>' +
'<div class="image-video-info-row"><label>文件</label><span title="' + escapeHtml(filename) + '">' + escapeHtml(filename) + '</span></div>' +
'</div>' +
'<div class="image-video-card-actions">' +
'<button class="image-video-card-action" type="button" data-shop-data-download="' + (resultId || '') + '"' + (item.file_ready && resultId > 0 ? '' : ' disabled') + '>' + imageVideoDownloadIcon() + '下载文件</button>' +
'<button class="image-video-card-action shop-data-delete-action" type="button" data-shop-data-delete="' + (resultId || '') + '"' + (terminal && resultId > 0 ? '' : ' disabled') + '>' + shopDataDeleteIcon() + '删除</button>' +
'</div>' +
'</div>';
return '<tr data-shop-data-card="' + (resultId || '') + '">' +
'<td style="width:36px;">' + checkbox + '</td>' +
'<td class="dup-shop" title="' + escapeHtml(item.shop_name || '-') + '">' + escapeHtml(item.shop_name || '-') + '</td>' +
'<td class="muted">' + escapeHtml(item.group_name || '-') + '</td>' +
'<td class="dup-country">' + escapeHtml(countryListLabel(countryCodes)) + '</td>' +
'<td>' + renderShopDataStatus(item, status || item.file_status) + '</td>' +
'<td class="dup-date">' + escapeHtml(updatedAt) + '</td>' +
'<td style="width:170px;">' +
'<button class="shop-data-record-action" type="button" data-shop-data-download="' + (resultId || '') + '"' + (item.file_ready && resultId > 0 ? '' : ' disabled') + '>' + imageVideoDownloadIcon() + '下载文件</button>' +
'<button class="shop-data-record-action danger" type="button" data-shop-data-delete="' + (resultId || '') + '"' + (terminal && resultId > 0 ? '' : ' disabled') + ' style="margin-left:8px;">' + shopDataDeleteIcon() + '删除</button>' +
'</td>' +
'</tr>';
}
function renderShopDataTaskCard(group) {
var results = Array.isArray(group.results) ? group.results : [];
var latest = group.latest_created_at || (results[0] && (results[0].created_at || results[0].finished_at)) || '-';
return '<article class="image-video-card shop-data-task-card" data-shop-data-group="' + escapeHtml(group.key || '') + '">' +
'<div class="image-video-card-body">' +
'<div class="image-video-card-head shop-data-group-head">' +
'<div class="shop-data-task-title"><span title="' + escapeHtml(group.shop_name || '-') + '">' + escapeHtml(group.shop_name || '-') + '</span></div>' +
'<span class="shop-data-group-meta">' + results.length + '/1 份当日累计文件</span>' +
'</div>' +
'<div class="image-video-card-info">' +
'<div class="image-video-info-row"><label>分组</label><span>' + escapeHtml(group.group_name || '-') + '</span></div>' +
'<div class="image-video-info-row"><label>最新</label><span>' + escapeHtml(latest) + '</span></div>' +
'</div>' +
'<div class="shop-data-result-list">' +
(results.length ? results.map(renderShopDataTaskResult).join('') : '<div class="image-video-empty">暂无结果</div>') +
'</div>' +
'</div>' +
'</article>';
function renderShopDataRecordTable() {
var rows = shopDataTasks.map(renderShopDataRecordRow).join('');
if (!shopDataTasks.length) {
return '<div class="shop-data-empty-hint">暂无符合条件的店铺数据任务</div>';
}
return '<table>' +
'<thead><tr>' +
'<th style="width:36px;"></th>' +
'<th style="width:18%;">店铺</th>' +
'<th style="width:14%;">分组</th>' +
'<th style="width:16%;">国家</th>' +
'<th style="width:10%;">状态</th>' +
'<th style="width:18%;">更新时间</th>' +
'<th style="width:170px;">操作</th>' +
'</tr></thead>' +
'<tbody>' + rows + '</tbody>' +
'</table>';
}
function renderShopDataTasks() {
var grid = document.getElementById('shopDataTaskGrid');
grid.innerHTML = renderShopDataRecordTable();
syncShopDataSelectionUi();
}
function syncShopDataSelectionUi() {
@@ -1811,9 +1815,11 @@
return selectedShopDataResultIds.has(shopDataResultId(item));
}).length;
var selectAll = document.getElementById('shopDataTaskSelectAll');
selectAll.checked = selectable.length > 0 && selectedCount === selectable.length;
selectAll.indeterminate = selectedCount > 0 && selectedCount < selectable.length;
selectAll.disabled = shopDataDownloadInProgress || selectable.length === 0;
if (selectAll) {
selectAll.checked = selectable.length > 0 && selectedCount === selectable.length;
selectAll.indeterminate = selectedCount > 0 && selectedCount < selectable.length;
selectAll.disabled = shopDataDownloadInProgress || selectable.length === 0;
}
var batch = document.getElementById('btnBatchDownloadShopDataTasks');
batch.disabled = shopDataDownloadInProgress || selectedCount === 0;
batch.innerHTML = imageVideoDownloadIcon() + (shopDataDownloadInProgress
@@ -1821,14 +1827,6 @@
: '批量下载' + (selectedCount ? ' (' + selectedCount + ')' : ''));
}
function renderShopDataTasks() {
var grid = document.getElementById('shopDataTaskGrid');
grid.innerHTML = shopDataTaskGroups.length
? shopDataTaskGroups.map(renderShopDataTaskCard).join('')
: '<div class="image-video-empty">暂无符合条件的店铺数据任务</div>';
syncShopDataSelectionUi();
}
function loadShopDataCrawlTasks(page) {
shopDataTaskPage = page || 1;
selectedShopDataResultIds.clear();
@@ -1847,7 +1845,7 @@
var total = payload.total != null ? Number(payload.total) : shopDataTaskGroups.length;
var responsePage = payload.page || page;
var responsePageSize = payload.page_size || shopDataTaskPageSize;
document.getElementById('shopDataTaskTotal').textContent = '共 ' + (total || 0) + ' 家店铺 · 每家店铺保留 1 份当日累计文件';
document.getElementById('shopDataTaskTotal').textContent = '共 ' + (total || 0) + ' 家店铺';
renderShopDataTasks();
renderPagination('shopDataTaskPagination', total, responsePage, responsePageSize, loadShopDataCrawlTasks);
})
@@ -1860,6 +1858,131 @@
});
}
// ========== 重复 ASIN 分析 ==========
var shopDataDuplicatePage = 1, shopDataDuplicatePageSize = 10;
var shopDataDuplicateItems = [];
var shopDataDuplicateTotal = 0;
var shopDataDuplicateAnalyzed = { shopCount: 0, resultCount: 0 };
var shopDataDuplicateLoading = false;
function buildShopDataDuplicateQuery(page) {
var params = new URLSearchParams();
params.set('page', String(page || 1));
params.set('page_size', String(shopDataDuplicatePageSize));
var values = {
shop_name: document.getElementById('shopDataTaskFilterShop').value.trim(),
group_name: document.getElementById('shopDataTaskFilterGroup').value.trim(),
country: document.getElementById('shopDataTaskFilterCountry').value.trim(),
created_from: document.getElementById('shopDataTaskFilterFrom').value,
created_to: document.getElementById('shopDataTaskFilterTo').value
};
Object.keys(values).forEach(function (key) {
if (values[key]) params.set(key, values[key]);
});
return params.toString();
}
function shopDataDuplicateHeader(header) {
return '<thead><tr>' + header.map(function (col) {
return '<th style="width:' + (col.width || '') + ';">' + col.label + '</th>';
}).join('') + '</tr></thead>';
}
function renderShopDataDuplicateCard(item) {
var occurrences = Array.isArray(item.occurrences) ? item.occurrences : [];
var brand = '';
occurrences.forEach(function (occ) { if (occ.brand && !brand) brand = occ.brand; });
var rows = occurrences.map(function (occ) {
var countries = countryListLabel(occ.country_codes);
return '<tr>' +
'<td class="dup-shop">' + escapeHtml(occ.shop_name || '-') + '</td>' +
'<td class="dup-country">' + escapeHtml(occ.group_name || '-') + '</td>' +
'<td class="dup-country">' + escapeHtml(countries) + '</td>' +
'<td class="dup-date">' + escapeHtml(occ.date || '-') + '</td>' +
'<td class="dup-date">' + escapeHtml(occ.price || '-') + '</td>' +
'</tr>';
}).join('');
return '<article class="duplicate-asin-card" data-duplicate-asin="' + escapeHtml(item.asin) + '">' +
'<div class="duplicate-asin-card-head">' +
'<span class="dup-asin">' + escapeHtml(item.asin) + '</span>' +
'<span class="dup-count">' + item.shop_count + ' 家店铺</span>' +
'<span class="dup-brand" title="' + escapeHtml(brand) + '">' + escapeHtml(brand || '') + '</span>' +
'</div>' +
'<table class="duplicate-asin-table">' +
shopDataDuplicateHeader([
{ label: '店铺', width: '18%' },
{ label: '分组', width: '14%' },
{ label: '国家', width: '16%' },
{ label: '日期', width: '14%' },
{ label: '价格', width: '12%' }
]) +
'<tbody>' + rows + '</tbody>' +
'</table>' +
'</article>';
}
function renderShopDataDuplicateList() {
var list = document.getElementById('shopDataDuplicateList');
if (!shopDataDuplicateItems.length) {
list.innerHTML = '<div class="shop-data-empty-hint">暂无重复 ASIN。请在左侧筛选条件后点击"查询"生效范围,再点击"重新分析"。</div>';
return;
}
list.innerHTML = shopDataDuplicateItems.map(renderShopDataDuplicateCard).join('');
}
function loadShopDataDuplicateAsins(page) {
if (shopDataDuplicateLoading) return;
shopDataDuplicatePage = page || 1;
shopDataDuplicateLoading = true;
var list = document.getElementById('shopDataDuplicateList');
var progress = document.getElementById('shopDataDuplicateProgress');
var button = document.getElementById('btnRefreshShopDataDuplicates');
progress.textContent = '正在读取各店铺结果文件并分析,请稍候...';
button.disabled = true;
list.innerHTML = '<div class="shop-data-empty-hint">分析中...</div>';
fetch('/api/admin/shop-data-crawl/duplicate-asins?' + buildShopDataDuplicateQuery(shopDataDuplicatePage))
.then(function (response) { return response.json(); })
.then(function (res) {
if (!res.success) throw new Error(res.error || '分析失败');
shopDataDuplicateItems = res.items || [];
shopDataDuplicateTotal = Number(res.total) || 0;
shopDataDuplicateAnalyzed.shopCount = Number(res.analyzed_shop_count) || 0;
shopDataDuplicateAnalyzed.resultCount = Number(res.analyzed_result_count) || 0;
var totalEl = document.getElementById('shopDataDuplicateTotal');
totalEl.textContent = '共 ' + shopDataDuplicateTotal + ' 个重复 ASIN · 已分析 ' + shopDataDuplicateAnalyzed.shopCount + ' 家店铺';
renderShopDataDuplicateList();
renderPagination('shopDataDuplicatePagination', shopDataDuplicateTotal, shopDataDuplicatePage, shopDataDuplicatePageSize, loadShopDataDuplicateAsins);
})
.catch(function (error) {
shopDataDuplicateItems = [];
shopDataDuplicateTotal = 0;
list.innerHTML = '<div class="shop-data-empty-hint">分析失败:' + escapeHtml(error.message || '') + '</div>';
document.getElementById('shopDataDuplicateTotal').textContent = '';
})
.finally(function () {
shopDataDuplicateLoading = false;
progress.textContent = '';
button.disabled = false;
});
}
function switchShopDataSubTab(view) {
var recordsView = document.getElementById('shopDataRecordsView');
var duplicatesView = document.getElementById('shopDataDuplicatesView');
var recordsTab = document.getElementById('shopDataSubTabRecords');
var duplicatesTab = document.getElementById('shopDataSubTabDuplicates');
var recordsActive = view === 'records';
recordsView.style.display = recordsActive ? '' : 'none';
duplicatesView.style.display = recordsActive ? 'none' : '';
recordsTab.classList.toggle('active', recordsActive);
recordsTab.setAttribute('aria-selected', recordsActive ? 'true' : 'false');
duplicatesTab.classList.toggle('active', !recordsActive);
duplicatesTab.setAttribute('aria-selected', !recordsActive ? 'true' : 'false');
if (!recordsActive) {
loadShopDataDuplicateAsins(1);
}
}
function downloadShopDataTask(item) {
var resultId = shopDataResultId(item);
if (!item || !item.file_ready || !resultId) return;
@@ -2037,12 +2160,23 @@
});
}
document.getElementById('btnFilterShopDataTasks').onclick = function () { loadShopDataCrawlTasks(1); };
document.getElementById('btnFilterShopDataTasks').onclick = function () {
loadShopDataCrawlTasks(1);
if (document.getElementById('shopDataDuplicatesView').style.display !== 'none') {
loadShopDataDuplicateAsins(1);
}
};
document.getElementById('btnResetShopDataTasks').onclick = function () {
['shopDataTaskFilterShop', 'shopDataTaskFilterGroup', 'shopDataTaskFilterCountry', 'shopDataTaskFilterFrom', 'shopDataTaskFilterTo']
.forEach(function (id) { document.getElementById(id).value = ''; });
loadShopDataCrawlTasks(1);
if (document.getElementById('shopDataDuplicatesView').style.display !== 'none') {
loadShopDataDuplicateAsins(1);
}
};
document.getElementById('shopDataSubTabRecords').onclick = function () { switchShopDataSubTab('records'); };
document.getElementById('shopDataSubTabDuplicates').onclick = function () { switchShopDataSubTab('duplicates'); };
document.getElementById('btnRefreshShopDataDuplicates').onclick = function () { loadShopDataDuplicateAsins(1); };
document.getElementById('shopDataTaskSelectAll').onchange = function (event) {
shopDataTasks.forEach(function (item) {
var resultId = shopDataResultId(item);
@@ -0,0 +1,178 @@
"""重复 ASIN 分析接口单元测试:模拟数据库行与结果文件,验证跨店铺重复聚合逻辑。"""
import sys
import unittest
from datetime import datetime
from io import BytesIO
from pathlib import Path
from unittest.mock import patch
from openpyxl import Workbook
from flask import Flask
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from blueprints import admin_api
def _make_workbook(rows_by_sheet):
"""构造结果 Workbookrows_by_sheet = {sheet名: [(日期, ASIN, 价格, 品牌), ...]}"""
wb = Workbook()
wb.remove(wb.active)
for sheet_name, rows in rows_by_sheet.items():
ws = wb.create_sheet(sheet_name)
ws.append(['日期', 'ASIN', '商品图片', '库存销量', '销售排名',
'页面浏览量', '售出件数', '价格', '推荐报价', '品牌'])
for date, asin, price, brand in rows:
row = [date, asin, '', '', '', '', '', price, '', brand]
ws.append(row)
return wb
class ShopDataDuplicateAsinTest(unittest.TestCase):
def setUp(self):
self.app = Flask(__name__)
self.group_rows = [
{'shop_name': 'Shop A', 'latest_created_at': datetime(2026, 8, 31, 1, 15, 1)},
{'shop_name': 'Shop B', 'latest_created_at': datetime(2026, 8, 31, 1, 15, 2)},
{'shop_name': 'Shop C', 'latest_created_at': datetime(2026, 8, 31, 1, 15, 3)},
]
# 每家店一个结果文件:Shop A 与 Shop B 共享 ASIN1Shop C 单独 ASIN3
self.shop_files = {
'Shop A': _make_workbook({
'英国': [('2026-08-30', 'B0ABC111', 'GBP 9.99', 'BrandA'),
('2026-08-30', 'B0UNIQUE1', 'GBP 5.00', 'BrandA')],
'德国': [('2026-08-30', 'B0ABC111', 'EUR 10.99', 'BrandA')],
}),
'Shop B': _make_workbook({
'英国': [('2026-08-31', 'B0ABC111', 'GBP 8.50', 'BrandA'),
('2026-08-31', 'B0ABC222', 'GBP 12.00', 'BrandB')],
}),
'Shop C': _make_workbook({
'法国': [('2026-08-29', 'B0ABC333', 'EUR 7.50', 'BrandC')],
}),
}
def _result_row(self, result_id, shop_name, country_codes_json=None):
return {
'result_id': result_id,
'task_id': result_id + 100,
'user_id': 7,
'shop_name': shop_name,
'shop_id': shop_name.lower(),
'task_no': f'task-{result_id}',
'task_status': 'SUCCESS',
'result_success': 1,
'result_error': None,
'task_error': None,
'file_error': None,
'result_file_url': f'object-{result_id}',
'result_filename': f'result-{result_id}.xlsx',
'result_file_size': 10,
'row_count': 2,
'request_json': '{}',
'country_codes_json': country_codes_json,
'created_at': '2026-08-31T01:15:00',
'updated_at': '2026-08-31T01:15:00',
'finished_at': '2026-08-31T01:15:00',
'latest_file_updated_at': '2026-08-31T01:15:00',
'file_job_id': None,
'file_status': 'SUCCESS',
'username': 'operator',
}
class _FakeCursor:
def __init__(self, group_rows, result_rows):
self.group_rows = group_rows
self.result_rows = result_rows
self.kind = None
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def execute(self, sql, params=()):
if 'COUNT(*) AS total' in sql:
self.kind = 'count'
elif 'AS latest_created_at' in sql and 'GROUP BY' in sql:
self.kind = 'groups'
elif 'GROUP_CONCAT' in sql:
self.kind = 'group_names'
else:
self.kind = 'results'
def fetchone(self):
return {'total': len(self.group_rows)}
def fetchall(self):
if self.kind == 'groups':
return self.group_rows
if self.kind == 'group_names':
return [{'shop_name': row['shop_name'], 'group_name': 'Group-' + row['shop_name']}
for row in self.group_rows]
return self.result_rows
class _FakeConnection:
def __init__(self, cursor):
self.cursor_value = cursor
def cursor(self):
return self.cursor_value
def close(self):
pass
def _make_workbook_bytes(self, shop_name):
stream = BytesIO()
self.shop_files[shop_name].save(stream)
return stream.getvalue()
def _run_request(self, query=''):
cursor = self._FakeCursor(
self.group_rows,
[self._result_row(i + 1, row['shop_name'], '["UK","DE"]') for i, row in enumerate(self.group_rows)],
)
connection = self._FakeConnection(cursor)
with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-asins?' + query):
with patch.object(admin_api, 'get_db', return_value=connection), \
patch.object(admin_api, '_ensure_backend_menu_access', return_value=(None, None, None)), \
patch.object(admin_api, '_ensure_shop_data_crawl_data_access', return_value=(None, None, None)), \
patch.object(admin_api, '_shop_data_crawl_fetch_result_bytes',
side_effect=lambda row: self._make_workbook_bytes(row['shop_name'])):
return admin_api.shop_data_crawl_duplicate_asins.__wrapped__()
def test_detects_duplicate_asins_across_shops(self):
response = self._run_request('page=1&page_size=10')
self.assertEqual(response.status_code, 200)
body = response.get_json()
self.assertTrue(body['success'])
self.assertEqual(body['total'], 1) # 只有 B0ABC111 跨店重复
self.assertEqual(body['analyzed_shop_count'], 3)
item = body['items'][0]
self.assertEqual(item['asin'], 'B0ABC111')
self.assertEqual(item['shop_count'], 2)
shops = {occ['shop_name'] for occ in item['occurrences']}
self.assertEqual(shops, {'Shop A', 'Shop B'})
# 国家与日期从行/表头正确映射
shop_a = next(occ for occ in item['occurrences'] if occ['shop_name'] == 'Shop A')
self.assertIn('UK', shop_a['country_codes'])
self.assertEqual(shop_a['date'], '2026-08-30')
def test_pagination_when_page_out_of_range(self):
response = self._run_request('page=2&page_size=10')
body = response.get_json()
self.assertEqual(body['total'], 1)
self.assertEqual(body['items'], [])
def test_parse_workbook_skips_unknown_sheets(self):
wb = _make_workbook({'英国': [('2026-08-30', 'B0TEST01', 'GBP 1.00', '')]})
# 手工追加一个无标准表头的 sheet,模拟未知表
ws = wb.create_sheet('未知表')
ws.append(['随便', '某列'])
ws.append(['2026-08-30', 'B0NOHEADER'])
rows = admin_api._shop_data_crawl_parse_workbook(wb)
self.assertEqual([row['asin'] for row in rows], ['B0TEST01'])
if __name__ == '__main__':
unittest.main()
+285 -130
View File
@@ -841,6 +841,244 @@
box-shadow: 0 0 0 3px rgba(99, 102, 241, 0.12);
}
/* ===== 店铺数据记录 ===== */
.shop-data-sub-tabs {
display: inline-flex;
gap: 6px;
padding: 4px;
margin-bottom: 18px;
background: #eef0f6;
border-radius: 10px;
}
.shop-data-sub-tab {
padding: 8px 22px;
border: 0;
border-radius: 8px;
background: transparent;
color: var(--c-text-2);
font: inherit;
font-size: 14px;
cursor: pointer;
transition: color 0.15s ease, background 0.15s ease, box-shadow 0.15s ease;
}
.shop-data-sub-tab:hover {
color: var(--c-text);
}
.shop-data-sub-tab.active {
background: #fff;
color: var(--c-primary);
font-weight: 700;
box-shadow: var(--shadow-card);
}
.shop-data-record-table-scroll {
margin-top: 4px;
max-height: none;
overflow-x: auto;
}
.shop-data-record-table-scroll table {
width: 100% !important;
table-layout: fixed;
}
.shop-data-record-table-scroll th,
.shop-data-record-table-scroll td {
vertical-align: middle;
text-align: left;
padding: 10px 12px;
border-bottom: 1px solid var(--c-border);
}
.shop-data-record-table-scroll td {
color: var(--c-text);
}
.shop-data-record-table-scroll td.muted {
color: var(--c-text-2);
}
.shop-data-record-table-scroll .shop-data-status {
display: inline-block;
min-width: 62px;
padding: 3px 10px;
border-radius: 999px;
font-size: 12px;
text-align: center;
white-space: nowrap;
}
.shop-data-record-table-scroll .shop-data-status.success {
color: var(--c-success);
background: #e6f6ec;
}
.shop-data-record-table-scroll .shop-data-status.failed {
color: var(--c-danger);
background: var(--c-danger-soft);
}
.shop-data-record-table-scroll .shop-data-status.running {
color: var(--c-warning);
background: #fdf0dc;
}
.shop-data-record-action {
display: inline-flex;
align-items: center;
gap: 6px;
padding: 5px 10px;
border: 1px solid var(--c-border);
border-radius: var(--radius-sm);
background: #fff;
color: var(--c-text);
font: inherit;
font-size: 12.5px;
cursor: pointer;
transition: color 0.15s ease, border-color 0.15s ease, background 0.15s ease;
}
.shop-data-record-action:hover:not(:disabled) {
color: var(--c-primary);
border-color: #b9bcf3;
background: var(--c-primary-soft);
}
.shop-data-record-action.danger:hover:not(:disabled) {
color: var(--c-danger);
border-color: #f3b4b6;
background: var(--c-danger-soft);
}
.shop-data-record-action:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.shop-data-record-action svg {
width: 14px;
height: 14px;
fill: none;
stroke: currentColor;
stroke-width: 2;
stroke-linecap: round;
stroke-linejoin: round;
}
/* ===== 重复 ASIN 列表 ===== */
.duplicate-asin-list {
display: flex;
flex-direction: column;
gap: 14px;
}
.duplicate-asin-card {
min-width: 0;
background: #fff;
border: 1px solid var(--c-border);
border-radius: 10px;
box-shadow: var(--shadow-card);
overflow: hidden;
}
.duplicate-asin-card-head {
display: flex;
align-items: center;
gap: 12px;
padding: 12px 16px;
background: var(--c-primary-soft);
border-bottom: 1px solid var(--c-border);
}
.duplicate-asin-card-head .dup-asin {
font-family: Consolas, Menlo, monospace;
font-size: 15px;
font-weight: 800;
color: var(--c-primary-strong);
letter-spacing: 0.02em;
}
.duplicate-asin-card-head .dup-count {
padding: 2px 10px;
border-radius: 999px;
background: #fff;
color: var(--c-primary-strong);
font-size: 12px;
font-weight: 700;
white-space: nowrap;
}
.duplicate-asin-card-head .dup-brand {
margin-left: auto;
color: var(--c-text-2);
font-size: 12.5px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.duplicate-asin-table {
width: 100%;
border-collapse: collapse;
font-size: 13px;
}
.duplicate-asin-table th,
.duplicate-asin-table td {
padding: 9px 16px;
text-align: left;
border-bottom: 1px solid var(--c-border);
}
.duplicate-asin-table th {
background: #fbfcfe;
color: var(--c-text-2);
font-size: 12.5px;
font-weight: 700;
}
.duplicate-asin-table tr:last-child td {
border-bottom: 0;
}
.duplicate-asin-table td.dup-shop {
font-weight: 700;
}
.duplicate-asin-table td.dup-country {
color: var(--c-text-2);
}
.duplicate-asin-table .dup-date {
color: var(--c-text-2);
font-variant-numeric: tabular-nums;
}
.shop-data-empty-hint {
padding: 28px 16px;
text-align: center;
color: var(--c-text-3);
font-size: 13px;
background: #fff;
border: 1px dashed var(--c-border);
border-radius: 10px;
}
.shop-data-refresh-btn {
background: #fff;
border: 1px solid var(--c-border);
color: var(--c-text);
}
.shop-data-refresh-btn:hover {
color: var(--c-primary);
border-color: #b9bcf3;
background: var(--c-primary-soft);
}
.thumb {
width: 56px;
height: 56px;
@@ -1864,113 +2102,6 @@
background: var(--c-primary-soft);
}
.shop-data-task-card .image-video-card-body {
min-height: 0;
display: flex;
flex-direction: column;
}
.shop-data-task-card .image-video-card-info {
flex: none;
}
.shop-data-group-head {
margin-bottom: 8px;
}
.shop-data-group-meta {
display: flex;
align-items: center;
gap: 8px;
color: var(--c-text-2);
font-size: 12px;
white-space: nowrap;
}
.shop-data-result-list {
margin-top: 8px;
border-top: 1px solid #eef1f6;
}
.shop-data-result {
padding: 13px 0 12px;
border-bottom: 1px solid #eef1f6;
}
.shop-data-result:last-child {
border-bottom: 0;
padding-bottom: 0;
}
.shop-data-result.selected {
margin-left: -8px;
margin-right: -8px;
padding-left: 8px;
padding-right: 8px;
border-radius: 4px;
background: #f4f6fe;
}
.shop-data-result-head {
display: flex;
align-items: center;
justify-content: space-between;
gap: 10px;
}
.shop-data-result .shop-data-task-title {
font-size: 13px;
}
.shop-data-result .image-video-card-info {
margin-top: 9px;
}
.shop-data-result .image-video-card-actions {
margin-top: 11px;
}
.shop-data-task-title {
min-width: 0;
display: flex;
align-items: center;
gap: 9px;
color: var(--c-text);
font-size: 15px;
font-weight: 600;
cursor: pointer;
}
.shop-data-task-title input {
flex: 0 0 auto;
width: 17px;
height: 17px;
accent-color: #6366f1;
}
.shop-data-task-title span {
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.shop-data-result .image-video-status[title] {
cursor: help;
}
.shop-data-delete-action {
margin-left: auto;
border-color: #f0b2b4;
color: var(--c-danger);
}
.shop-data-delete-action:hover:not(:disabled) {
border-color: var(--c-danger);
background: var(--c-danger);
color: #fff;
}
.image-video-permission-btn {
display: none;
min-height: 36px;
@@ -4580,7 +4711,14 @@
<div id="panel-shop-data-crawl-tasks" class="tab-panel">
<div class="form-box">
<h3 style="margin-bottom:16px;">店铺数据记录筛选</h3>
<h3 style="margin-bottom:16px;">店铺数据记录</h3>
<div class="shop-data-sub-tabs" role="tablist" aria-label="店铺数据记录视图">
<button class="shop-data-sub-tab active" type="button" id="shopDataSubTabRecords"
role="tab" aria-selected="true">店铺数据</button>
<button class="shop-data-sub-tab" type="button" id="shopDataSubTabDuplicates"
role="tab" aria-selected="false">重复 ASIN</button>
</div>
<h4 style="margin-bottom:14px;">筛选条件</h4>
<div class="form-row">
<div class="form-group" style="min-width:150px;"><label>店铺</label><input type="text"
id="shopDataTaskFilterShop" placeholder="模糊搜索店铺名"></div>
@@ -4604,30 +4742,47 @@
</div>
</div>
<div class="panel-box image-video-panel-box">
<div class="image-video-results">
<div class="image-video-toolbar">
<div class="image-video-toolbar-main">
<button class="btn btn-secondary image-video-permission-btn"
id="btnOpenShopDataTaskPermissions" type="button">权限配置</button>
<button class="btn image-video-batch-btn" id="btnBatchDownloadShopDataTasks"
type="button" disabled>
<svg viewBox="0 0 24 24" aria-hidden="true">
<path d="M12 3v12m0 0 4-4m-4 4-4-4M5 21h14"></path>
</svg>
批量下载
</button>
<label class="image-video-select-all">
<input type="checkbox" id="shopDataTaskSelectAll">
全选当前页
</label>
<span class="image-video-download-progress" id="shopDataTaskDownloadProgress"
aria-live="polite"></span>
<div id="shopDataRecordsView">
<div class="image-video-results">
<div class="image-video-toolbar">
<div class="image-video-toolbar-main">
<button class="btn btn-secondary image-video-permission-btn"
id="btnOpenShopDataTaskPermissions" type="button">权限配置</button>
<button class="btn image-video-batch-btn" id="btnBatchDownloadShopDataTasks"
type="button" disabled>
<svg viewBox="0 0 24 24" aria-hidden="true">
<path d="M12 3v12m0 0 4-4m-4 4-4-4M5 21h14"></path>
</svg>
批量下载
</button>
<label class="image-video-select-all">
<input type="checkbox" id="shopDataTaskSelectAll">
全选当前页
</label>
<span class="image-video-download-progress" id="shopDataTaskDownloadProgress"
aria-live="polite"></span>
</div>
<span class="image-video-summary" id="shopDataTaskTotal"></span>
</div>
<span class="image-video-summary" id="shopDataTaskTotal"></span>
<div class="table-scroll shop-data-record-table-scroll" id="shopDataTaskGrid" aria-live="polite"></div>
</div>
<div class="image-video-grid" id="shopDataTaskGrid" aria-live="polite"></div>
<div class="pagination" id="shopDataTaskPagination"></div>
</div>
<div id="shopDataDuplicatesView" style="display:none;">
<div class="image-video-results">
<div class="image-video-toolbar">
<div class="image-video-toolbar-main">
<button class="btn shop-data-refresh-btn" id="btnRefreshShopDataDuplicates"
type="button">重新分析</button>
<span class="image-video-download-progress" id="shopDataDuplicateProgress"
aria-live="polite"></span>
</div>
<span class="image-video-summary" id="shopDataDuplicateTotal"></span>
</div>
<div class="duplicate-asin-list" id="shopDataDuplicateList" aria-live="polite"></div>
</div>
<div class="pagination" id="shopDataDuplicatePagination"></div>
</div>
<div class="pagination" id="shopDataTaskPagination"></div>
</div>
</div>
@@ -5528,7 +5683,7 @@
window.__initAdminMenuCollapse();
})();
</script>
<script src="/static/admin.js?v=drop-shop-check"></script>
<script src="/static/admin.js?v=shop-data-tabs"></script>
<div class="admin-toast-region" id="adminToastRegion" role="status" aria-live="polite" aria-atomic="true"></div>
<div class="admin-confirm-mask" id="adminConfirmModal" aria-hidden="true">
<section class="admin-confirm-dialog" role="alertdialog" aria-modal="true" aria-labelledby="adminConfirmTitle" aria-describedby="adminConfirmMessage">