diff --git a/backend/blueprints/admin_api.py b/backend/blueprints/admin_api.py index b4cacef5..cdff9425 100644 --- a/backend/blueprints/admin_api.py +++ b/backend/blueprints/admin_api.py @@ -1699,7 +1699,9 @@ _SHOP_DATA_CRAWL_ASIN_ANALYSIS_TIMEOUT = (10, 60) def _shop_data_crawl_fetch_result_bytes(row, timeout=None): """从 Java 下载接口拉取结果文件字节流（仅内存，不落盘）。""" - url = f"{backend_java_base_url}/api/admin/shop-data-crawl/results/{int(row['id'])}/download" + # row 支持 id（下载行）或 result_id（管理列集别名） + result_id = row.get('id') or row.get('result_id') or 0 + url = f"{backend_java_base_url}/api/admin/shop-data-crawl/results/{int(result_id)}/download" headers, params = _backend_java_internal_request() response = _get_backend_java_session().get( url, @@ -1738,6 +1740,29 @@ def _shop_data_crawl_cell_text(cell): return str(value).strip() +def _shop_data_date_key(date_text): + """把各种日期文本归一化为 YYYY-MM-DD，供范围比较。 + + 支持：ISO（2026-08-19）、中文（2026年8月19日 上午4:34）、 + yyyy.m.d / yyyy/m/d 等常见格式；无法识别时返回原字符串。 + """ + text = (date_text or '').strip() + if not text: + return '' + if len(text) >= 10 and text[4] == '-' and text[7] == '-': + return text[:10] + year = month = day = None + m = re.search(r'(\d{4})\s*[年./-]\s*(\d{1,2})\s*[月./-]\s*(\d{1,2})', text) + if m: + year, month, day = int(m.group(1)), int(m.group(2)), int(m.group(3)) + if year and month and day: + try: + return f'{year:04d}-{month:02d}-{day:02d}' + except ValueError: + return text + return text + + def _shop_data_crawl_parse_workbook(workbook): """从结果 Workbook 提取 ASIN 行：字典列表，表头按 日期/ASIN/…/价格/…/品牌 识别，跳过无 ASIN 行。""" rows = [] @@ -1781,11 +1806,11 @@ def shop_data_crawl_duplicate_asins(): try: page = max(1, int(request.args.get('page', 1))) page_size = min(100, max(10, int(request.args.get('page_size', 20)))) - shop_name = (request.args.get('shop_name') or request.args.get('shop') or '').strip() - group_name = (request.args.get('group_name') or request.args.get('group') or '').strip() - country = (request.args.get('country') or '').strip().upper() - created_from = _parse_admin_datetime_arg('created_from') - created_to = _parse_admin_datetime_arg('created_to') + shop_name_filter = (request.args.get('shop_name') or request.args.get('shop') or '').strip() + country_filter = (request.args.get('country') or '').strip().upper() + asin_filter = (request.args.get('asin') or '').strip().upper() + date_from = (request.args.get('date_from') or '').strip()[:10] + date_to = (request.args.get('date_to') or '').strip()[:10] conditions = [ "r.module_type = 'SHOP_DATA_CRAWL'", @@ -1793,34 +1818,6 @@ def shop_data_crawl_duplicate_asins(): "TRIM(COALESCE(r.result_file_url, '')) <> ''", ] params = [] - if shop_name: - conditions.append('r.source_filename LIKE %s') - params.append('%' + shop_name + '%') - if group_name: - conditions.append( - 'EXISTS (SELECT 1 FROM biz_shop_manage sm ' - 'LEFT JOIN biz_shop_manage_group g ON g.id = sm.group_id ' - 'WHERE TRIM(COALESCE(sm.shop_name, \'\')) = ' - 'TRIM(COALESCE(r.source_filename, \'\')) ' - "AND COALESCE(NULLIF(g.group_name, ''), NULLIF(sm.group_name, '')) LIKE %s)" - ) - params.append('%' + group_name + '%') - if country: - if country not in ('DE', 'UK', 'FR', 'IT', 'ES'): - raise ValueError('不支持的国家代码: ' + country) - conditions.append( - 'JSON_CONTAINS(' - 'COALESCE(df.country_codes_json, ' - 'JSON_EXTRACT(t.request_json, \'$.countryCodes\'), ' - 'JSON_EXTRACT(t.request_json, \'$.country_codes\'), \'[]\'), %s)' - ) - params.append('"' + country + '"') - if created_from: - conditions.append('t.created_at >= %s') - params.append(created_from) - if created_to: - conditions.append('t.created_at <= %s') - params.append(created_to) where_sql = ' AND '.join(conditions) offset = (page - 1) * page_size @@ -1907,10 +1904,13 @@ def shop_data_crawl_duplicate_asins(): current_app.logger.warning( '[shop-data-crawl] 拉取结果文件失败 result_id=%s: %s', result_id, exc) - # 按 ASIN 聚合其出现的店铺/分组/国家与行细节 + # 先全量聚合所有行，再用筛选条件圈定「命中 ASIN」； + # 命中且跨店重复的 ASIN 展示完整 occurrences（方便看该 ASIN 与哪些店重复）。 asin_occurrences = {} + matched_asins = set() for shop_item in shop_items: shop_name = shop_item['shop_name'] or '未命名' + country_codes = shop_item['country_codes'] or [] for row in shop_item['rows']: asin = row['asin'].strip().upper() if not asin: @@ -1924,15 +1924,31 @@ def shop_data_crawl_duplicate_asins(): 'group_name': shop_item['group_name'], 'country_codes': shop_item['country_codes'], }) + # 行级筛选：命中则将此 ASIN 加入待展示集合 + if asin_filter and asin_filter not in asin: + continue + if shop_name_filter and shop_name_filter.lower() not in (shop_name or '').lower() \ + and shop_name_filter.lower() not in (shop_item['group_name'] or '').lower(): + continue + if country_filter and country_filter not in [c.upper() for c in country_codes]: + continue + row_date_key = _shop_data_date_key((row['date'] or '').strip()) + if date_from and row_date_key and row_date_key < date_from: + continue + if date_to and row_date_key and row_date_key > date_to: + continue + matched_asins.add(asin) occurrences_list = [] - for asin, occurrences in asin_occurrences.items(): + for asin in matched_asins: + occurrences = asin_occurrences.get(asin, []) shop_count = len({item['shop_name'] for item in occurrences}) if shop_count < 2: continue occurrences_list.append({ 'asin': asin, 'shop_count': shop_count, + 'record_count': len(occurrences), 'occurrences': occurrences, }) occurrences_list.sort(key=lambda item: (-item['shop_count'], item['asin'])) diff --git a/backend/static/admin.js b/backend/static/admin.js index b01e10e8..db47d605 100644 --- a/backend/static/admin.js +++ b/backend/static/admin.js @@ -1784,7 +1784,7 @@ } return '' + '' + - '' + + '' + '' + '' + '' + @@ -1870,11 +1870,11 @@ params.set('page', String(page || 1)); params.set('page_size', String(shopDataDuplicatePageSize)); var values = { - shop_name: document.getElementById('shopDataTaskFilterShop').value.trim(), - group_name: document.getElementById('shopDataTaskFilterGroup').value.trim(), - country: document.getElementById('shopDataTaskFilterCountry').value.trim(), - created_from: document.getElementById('shopDataTaskFilterFrom').value, - created_to: document.getElementById('shopDataTaskFilterTo').value + asin: document.getElementById('shopDataDupFilterAsin').value.trim(), + shop_name: document.getElementById('shopDataDupFilterShop').value.trim(), + country: document.getElementById('shopDataDupFilterCountry').value.trim(), + date_from: document.getElementById('shopDataDupFilterDateFrom').value, + date_to: document.getElementById('shopDataDupFilterDateTo').value }; Object.keys(values).forEach(function (key) { if (values[key]) params.set(key, values[key]); @@ -1924,10 +1924,69 @@ function renderShopDataDuplicateList() { var list = document.getElementById('shopDataDuplicateList'); if (!shopDataDuplicateItems.length) { - list.innerHTML = '
暂无重复 ASIN。请在左侧筛选条件后点击"查询"生效范围，再点击"重新分析"。
'; + list.innerHTML = '
暂无重复 ASIN。请在筛选条件后点击"查询"，或"重新分析"。' + + '
'; return; } - list.innerHTML = shopDataDuplicateItems.map(renderShopDataDuplicateCard).join(''); + var header = [ + { label: 'ASIN', width: '14%' }, + { label: '店铺数', width: '8%' }, + { label: '记录条数', width: '10%' }, + { label: '品牌', width: '12%' }, + { label: '对应店铺', width: '22%' }, + { label: '国家', width: '14%' }, + { label: '上架时间', width: '12%' }, + { label: '操作', width: '8%' } + ]; + var tbody = shopDataDuplicateItems.map(function (item) { + return renderShopDataDuplicateRow(item); + }).join(''); + list.innerHTML = '
店铺分组国家
' + shopDataDuplicateHeader(header) + '' + tbody + '
'; + } + + function renderShopDataDuplicateRow(item) { + var occurrences = Array.isArray(item.occurrences) ? item.occurrences : []; + var brand = ''; + var countries = {}; + var shops = {}; + occurrences.forEach(function (occ) { + if (occ.brand && !brand) brand = occ.brand; + (occ.country_codes || []).forEach(function (c) { countries[c] = true; }); + shops[occ.shop_name || '-'] = true; + }); + var countryLabel = countryListLabel(Object.keys(countries)); + var shopList = Object.keys(shops).join('、'); + var dateList = occurrences.map(function (occ) { return occ.date; }) + .filter(function (d) { return d; }).join('、'); + return '' + + '' + escapeHtml(item.asin) + '' + + '' + (item.shop_count || 1) + ' 家' + + '' + (item.record_count || occurrences.length) + ' 条' + + '' + escapeHtml(brand || '-') + '' + + '' + escapeHtml(shopList || '-') + '' + + '' + escapeHtml(countryLabel || '-') + '' + + '' + escapeHtml(dateList || '-') + '' + + '' + + '' + renderShopDataDuplicateDetailRows(item); + } + + function renderShopDataDuplicateDetailRows(item) { + var occurrences = Array.isArray(item.occurrences) ? item.occurrences : []; + if (!occurrences.length) return ''; + var rows = occurrences.map(function (occ) { + var countries = countryListLabel(occ.country_codes); + return '' + + '' + + '' + escapeHtml(occ.shop_name || '-') + '' + + '' + escapeHtml(occ.group_name || '-') + '' + + '' + escapeHtml(countries) + '' + + '' + escapeHtml(occ.date || '-') + '' + + '' + escapeHtml(occ.price || '-') + '' + + '' + escapeHtml(occ.brand || '-') + '' + + '' + + ''; + }).join(''); + return rows; } function loadShopDataDuplicateAsins(page) { @@ -1950,6 +2009,8 @@ shopDataDuplicateAnalyzed.resultCount = Number(res.analyzed_result_count) || 0; var totalEl = document.getElementById('shopDataDuplicateTotal'); totalEl.textContent = '共 ' + shopDataDuplicateTotal + ' 个重复 ASIN · 已分析 ' + shopDataDuplicateAnalyzed.shopCount + ' 家店铺'; + var tabCount = document.getElementById('shopDataDuplicateTabCount'); + if (tabCount) tabCount.textContent = shopDataDuplicateTotal; renderShopDataDuplicateList(); renderPagination('shopDataDuplicatePagination', shopDataDuplicateTotal, shopDataDuplicatePage, shopDataDuplicatePageSize, loadShopDataDuplicateAsins); }) @@ -2177,24 +2238,32 @@ document.getElementById('shopDataSubTabRecords').onclick = function () { switchShopDataSubTab('records'); }; document.getElementById('shopDataSubTabDuplicates').onclick = function () { switchShopDataSubTab('duplicates'); }; document.getElementById('btnRefreshShopDataDuplicates').onclick = function () { loadShopDataDuplicateAsins(1); }; - document.getElementById('shopDataTaskSelectAll').onchange = function (event) { - shopDataTasks.forEach(function (item) { - var resultId = shopDataResultId(item); - if (!item.file_ready || !resultId) return; - if (event.target.checked) selectedShopDataResultIds.add(resultId); - else selectedShopDataResultIds.delete(resultId); - }); - syncShopDataSelectionUi(); + document.getElementById('btnFilterShopDataDuplicates').onclick = function () { loadShopDataDuplicateAsins(1); }; + document.getElementById('btnResetShopDataDuplicates').onclick = function () { + ['shopDataDupFilterAsin', 'shopDataDupFilterShop', 'shopDataDupFilterCountry', 'shopDataDupFilterDateFrom', 'shopDataDupFilterDateTo'] + .forEach(function (id) { document.getElementById(id).value = ''; }); + loadShopDataDuplicateAsins(1); }; - document.getElementById('btnBatchDownloadShopDataTasks').onclick = downloadShopDataTasksZip; + // 全选移到表格表头后为动态元素，用事件委托 document.getElementById('shopDataTaskGrid').onchange = function (event) { var checkbox = event.target.closest('[data-shop-data-select]'); - if (!checkbox) return; - var resultId = Number(checkbox.dataset.shopDataSelect); - if (!resultId) return; - if (checkbox.checked) selectedShopDataResultIds.add(resultId); - else selectedShopDataResultIds.delete(resultId); - syncShopDataSelectionUi(); + if (checkbox) { + var resultId = Number(checkbox.dataset.shopDataSelect); + if (!resultId) return; + if (checkbox.checked) selectedShopDataResultIds.add(resultId); + else selectedShopDataResultIds.delete(resultId); + syncShopDataSelectionUi(); + return; + } + if (event.target.closest('#shopDataTaskSelectAll')) { + shopDataTasks.forEach(function (item) { + var sid = shopDataResultId(item); + if (!item.file_ready || !sid) return; + if (event.target.checked) selectedShopDataResultIds.add(sid); + else selectedShopDataResultIds.delete(sid); + }); + syncShopDataSelectionUi(); + } }; document.getElementById('shopDataTaskGrid').onclick = function (event) { var downloadButton = event.target.closest('[data-shop-data-download]'); @@ -2209,6 +2278,29 @@ deleteShopDataTask(deleteItem); } }; + document.getElementById('btnBatchDownloadShopDataTasks').onclick = downloadShopDataTasksZip; + document.getElementById('shopDataDuplicateList').onclick = function (event) { + var toggle = event.target.closest('[data-dup-toggle]'); + if (!toggle) return; + var row = toggle.closest('tr'); + var allDetailRows = document.querySelectorAll('.dup-detail-row'); + // 若当前 ASIN 的详情行已展开，则本次点击收起 + var cur = row ? row.nextElementSibling : null; + var wasExpanded = false; + while (cur && cur.classList && cur.classList.contains('dup-detail-row')) { + if (cur.style.display !== 'none') { wasExpanded = true; break; } + cur = cur.nextElementSibling; + } + allDetailRows.forEach(function (dr) { dr.style.display = 'none'; }); + if (row && !wasExpanded) { + // 展开当前 ASIN 的详情行（紧随其后） + cur = row.nextElementSibling; + while (cur && cur.classList && cur.classList.contains('dup-detail-row')) { + cur.style.display = ''; + cur = cur.nextElementSibling; + } + } + }; document.getElementById('btnOpenShopDataTaskPermissions').onclick = openShopDataTaskPermissions; document.getElementById('btnCloseShopDataTaskPermissions').onclick = closeShopDataTaskPermissions; document.getElementById('btnCancelShopDataTaskPermissions').onclick = closeShopDataTaskPermissions; @@ -6408,8 +6500,6 @@ currentUserRole = item.role || currentUserRole; currentUserUsername = item.username || currentUserUsername; nameEl.textContent = item.username || '管理员'; - var avatarEl = document.getElementById('adminUserAvatar'); - if (avatarEl) avatarEl.textContent = (item.username || '管').charAt(0).toUpperCase(); if (item.role) { roleEl.textContent = item.role === 'super_admin' ? '超级管理员' diff --git a/backend/static/logo.jpg b/backend/static/logo.jpg new file mode 100644 index 00000000..ab825d78 Binary files /dev/null and b/backend/static/logo.jpg differ diff --git a/backend/tests/test_admin_shop_data_duplicate_asins.py b/backend/tests/test_admin_shop_data_duplicate_asins.py index 82d865ac..c9331adf 100644 --- a/backend/tests/test_admin_shop_data_duplicate_asins.py +++ b/backend/tests/test_admin_shop_data_duplicate_asins.py @@ -36,6 +36,12 @@ class ShopDataDuplicateAsinTest(unittest.TestCase): {'shop_name': 'Shop C', 'latest_created_at': datetime(2026, 8, 31, 1, 15, 3)}, ] # 每家店一个结果文件：Shop A 与 Shop B 共享 ASIN1；Shop C 单独 ASIN3 + # 国家码按店铺实际 sheet：A=UK+DE、B=仅UK、C=FR + self.shop_country = { + 'Shop A': '["UK","DE"]', + 'Shop B': '["UK"]', + 'Shop C': '["FR"]', + } self.shop_files = { 'Shop A': _make_workbook({ '英国': [('2026-08-30', 'B0ABC111', 'GBP 9.99', 'BrandA'), @@ -130,7 +136,8 @@ class ShopDataDuplicateAsinTest(unittest.TestCase): def _run_request(self, query=''): cursor = self._FakeCursor( self.group_rows, - [self._result_row(i + 1, row['shop_name'], '["UK","DE"]') for i, row in enumerate(self.group_rows)], + [self._result_row(i + 1, row['shop_name'], self.shop_country[row['shop_name']]) + for i, row in enumerate(self.group_rows)], ) connection = self._FakeConnection(cursor) with self.app.test_request_context('/api/admin/shop-data-crawl/duplicate-asins?' + query): @@ -151,6 +158,7 @@ class ShopDataDuplicateAsinTest(unittest.TestCase): item = body['items'][0] self.assertEqual(item['asin'], 'B0ABC111') self.assertEqual(item['shop_count'], 2) + self.assertEqual(item['record_count'], 3) # ShopA 英国/德国 + ShopB 英国 共 3 条 shops = {occ['shop_name'] for occ in item['occurrences']} self.assertEqual(shops, {'Shop A', 'Shop B'}) # 国家与日期从行/表头正确映射 @@ -158,6 +166,38 @@ class ShopDataDuplicateAsinTest(unittest.TestCase): self.assertIn('UK', shop_a['country_codes']) self.assertEqual(shop_a['date'], '2026-08-30') + def test_filters_asin_and_date_range(self): + # asin 模糊：匹配 B0ABC222 的只有 Shop B 一家，不构成跨店重复 → total 0 + response = self._run_request('page=1&page_size=10&asin=B0ABC222') + body = response.get_json() + self.assertEqual(body['total'], 0) + # 日期范围 08-31：B0ABC111 在 Shop B 有该日记录 → 命中，展示完整记录（跨店仍 2 家） + response = self._run_request('page=1&page_size=10&date_from=2026-08-31&date_to=2026-08-31') + body = response.get_json() + self.assertEqual(body['total'], 1) + self.assertEqual(body['items'][0]['asin'], 'B0ABC111') + self.assertEqual(body['items'][0]['shop_count'], 2) # 完整记录仍跨店 + # 日期范围 08-30：Shop A 两条命中，同样返回完整记录 + response = self._run_request('page=1&page_size=10&date_from=2026-08-30T00:00&date_to=2026-08-30T23:59') + body = response.get_json() + self.assertEqual(body['total'], 1) + + def test_filters_shop_and_country(self): + # 店铺过滤 Shop B：命中含 Shop B 记录的 ASIN → B0ABC111，展示完整记录跨店 + response = self._run_request('page=1&page_size=10&shop_name=Shop+B') + body = response.get_json() + self.assertEqual(body['total'], 1) + self.assertEqual(body['items'][0]['asin'], 'B0ABC111') + # 国家过滤 FR：只有 Shop C 有 FR，其 ASIN 仅单店 → 不构成重复 → total 0 + response = self._run_request('page=1&page_size=10&country=FR') + body = response.get_json() + self.assertEqual(body['total'], 0) + # 国家过滤 DE：Shop A 命中 → B0ABC111 完整记录 → total 1 + response = self._run_request('page=1&page_size=10&country=DE') + body = response.get_json() + self.assertEqual(body['total'], 1) + self.assertEqual(body['items'][0]['asin'], 'B0ABC111') + def test_pagination_when_page_out_of_range(self): response = self._run_request('page=2&page_size=10') body = response.get_json() @@ -173,6 +213,20 @@ class ShopDataDuplicateAsinTest(unittest.TestCase): rows = admin_api._shop_data_crawl_parse_workbook(wb) self.assertEqual([row['asin'] for row in rows], ['B0TEST01']) + def test_date_key_normalizes_chinese_datetime(self): + # 中文日期归一化为 ISO 供范围比较 + cases = { + '2026年8月18日 上午4:34': '2026-08-18', + '2026年8月19日 05:48': '2026-08-19', + '2026-08-19': '2026-08-19', + '2026.8.19': '2026-08-19', + '2026/8/19': '2026-08-19', + '': '', + 'abc': 'abc', # 无法识别时原样返回，范围比较自然失败 + } + for raw, expected in cases.items(): + self.assertEqual(admin_api._shop_data_date_key(raw), expected) + if __name__ == '__main__': unittest.main() diff --git a/backend/web_source/admin.html b/backend/web_source/admin.html index 3325d309..0d787af2 100644 --- a/backend/web_source/admin.html +++ b/backend/web_source/admin.html @@ -75,16 +75,18 @@ width: 32px; height: 32px; border-radius: 9px; - background: linear-gradient(135deg, #6366f1, #7a8bf0); - color: #fff; - display: flex; - align-items: center; - justify-content: center; - font-size: 15px; - font-weight: 700; + overflow: hidden; + background: #fff; box-shadow: 0 4px 10px rgba(99, 102, 241, 0.35); } + .admin-brand-logo { + display: block; + width: 100%; + height: 100%; + object-fit: cover; + } + .admin-brand-text { min-width: 0; line-height: 1.2; @@ -302,21 +304,6 @@ flex: 0 0 auto; } - .admin-user-avatar { - width: 30px; - height: 30px; - border-radius: 50%; - background: linear-gradient(135deg, #6366f1, #7a8bf0); - color: #fff; - display: flex; - align-items: center; - justify-content: center; - font-size: 13px; - font-weight: 600; - flex: 0 0 auto; - box-shadow: 0 2px 6px rgba(99, 102, 241, 0.3); - } - .admin-user-name { font-size: 13px; font-weight: 500; @@ -2532,8 +2519,7 @@ width: 36px; height: 36px; border-radius: 11px; - background: linear-gradient(135deg, #818cf8, #4f46e5); - color: #eef2ff; + background: #fff; box-shadow: 0 8px 22px rgba(79, 70, 229, 0.36); } @@ -2680,19 +2666,6 @@ gap: 12px; } - .admin-user-avatar { - width: 34px; - height: 34px; - background: linear-gradient(135deg, #818cf8, #4f46e5); - color: #eef2ff; - box-shadow: 0 6px 16px rgba(79, 70, 229, 0.28); - } - - .admin-user-name { - color: #dbe4f1; - font-weight: 550; - } - .admin-user-role { color: #c7d2fe; background: rgba(129, 140, 248, 0.15); @@ -3484,10 +3457,8 @@ } .admin-brand { border-bottom-color: #d5ded7; } - .admin-brand-mark, - .admin-user-avatar { - background: linear-gradient(135deg, #7f998a, #607a6d); - color: #ffffff; + .admin-brand-mark { + background: #fff; box-shadow: 0 8px 20px rgba(96, 122, 109, 0.25); } .admin-brand-name { color: var(--c-text); } @@ -3685,10 +3656,8 @@ box-shadow: 14px 0 34px -28px rgba(39, 67, 94, 0.34); } .admin-brand { border-bottom-color: #d4e0eb; } - .admin-brand-mark, - .admin-user-avatar { - background: linear-gradient(135deg, #7094ba, #4f78a5); - color: #ffffff; + .admin-brand-mark { + background: #fff; box-shadow: 0 8px 20px rgba(79, 120, 165, 0.24); } .admin-brand-name { color: var(--c-text); } @@ -4022,7 +3991,7 @@
- 加载中... @@ -4716,7 +4684,7 @@ + role="tab" aria-selected="false">重复ASIN(0)

筛选条件

@@ -4755,10 +4723,6 @@ 批量下载 -
@@ -4769,6 +4733,30 @@