后台店铺数据重复ASIN：全选移至表头、Tab总数、独立筛选、按ASIN分组表格展示

- 列表全选复选框从工具栏移到表格表头，事件委托处理动态元素 - 重复ASIN Tab 标题加总数（XX 为重复 ASIN 数量） - 重复ASIN 筛选独立：ASIN/店铺/国家/上架时间（开始+结束），与原店铺数据筛选互不影响 - 后端支持 asin/shop_name/country/date_from/date_to 参数：行级筛选命中 ASIN 后展示完整记录（含其他店铺，跨店语义更直观） - 日期文本归一化（支持 2026年8月19日上午4:34 中文格式） - 按 ASIN 分组表格展示：ASIN/店铺数/记录条数/品牌/对应店铺/国家/上架时间/操作，详情展开显示每条记录的店铺/分组/国家/日期/价格/品牌 - 修复 _shop_data_crawl_fetch_result_bytes 兼容 result_id 列别名（生产首个真实调用报 KeyError） - 新增单测：筛选语义、日期归一化、record_count
2026-09-03 09:32:25 +08:00
parent 417a2bf831
commit 672b9f9b46
5 changed files with 263 additions and 115 deletions
@@ -1699,7 +1699,9 @@ _SHOP_DATA_CRAWL_ASIN_ANALYSIS_TIMEOUT = (10, 60)

 def _shop_data_crawl_fetch_result_bytes(row, timeout=None):
    """从 Java 下载接口拉取结果文件字节流（仅内存，不落盘）。"""
-    url = f"{backend_java_base_url}/api/admin/shop-data-crawl/results/{int(row['id'])}/download"
+    # row 支持 id（下载行）或 result_id（管理列集别名）
+    result_id = row.get('id') or row.get('result_id') or 0
+    url = f"{backend_java_base_url}/api/admin/shop-data-crawl/results/{int(result_id)}/download"
    headers, params = _backend_java_internal_request()
    response = _get_backend_java_session().get(
        url,
@@ -1738,6 +1740,29 @@ def _shop_data_crawl_cell_text(cell):
    return str(value).strip()


+def _shop_data_date_key(date_text):
+    """把各种日期文本归一化为 YYYY-MM-DD，供范围比较。
+
+    支持：ISO（2026-08-19）、中文（2026年8月19日 上午4:34）、
+    yyyy.m.d / yyyy/m/d 等常见格式；无法识别时返回原字符串。
+    """
+    text = (date_text or '').strip()
+    if not text:
+        return ''
+    if len(text) >= 10 and text[4] == '-' and text[7] == '-':
+        return text[:10]
+    year = month = day = None
+    m = re.search(r'(\d{4})\s*[年./-]\s*(\d{1,2})\s*[月./-]\s*(\d{1,2})', text)
+    if m:
+        year, month, day = int(m.group(1)), int(m.group(2)), int(m.group(3))
+    if year and month and day:
+        try:
+            return f'{year:04d}-{month:02d}-{day:02d}'
+        except ValueError:
+            return text
+    return text
+
+
 def _shop_data_crawl_parse_workbook(workbook):
    """从结果 Workbook 提取 ASIN 行：字典列表，表头按 日期/ASIN/…/价格/…/品牌 识别，跳过无 ASIN 行。"""
    rows = []
@@ -1781,11 +1806,11 @@ def shop_data_crawl_duplicate_asins():
    try:
        page = max(1, int(request.args.get('page', 1)))
        page_size = min(100, max(10, int(request.args.get('page_size', 20))))
-        shop_name = (request.args.get('shop_name') or request.args.get('shop') or '').strip()
-        group_name = (request.args.get('group_name') or request.args.get('group') or '').strip()
-        country = (request.args.get('country') or '').strip().upper()
-        created_from = _parse_admin_datetime_arg('created_from')
-        created_to = _parse_admin_datetime_arg('created_to')
+        shop_name_filter = (request.args.get('shop_name') or request.args.get('shop') or '').strip()
+        country_filter = (request.args.get('country') or '').strip().upper()
+        asin_filter = (request.args.get('asin') or '').strip().upper()
+        date_from = (request.args.get('date_from') or '').strip()[:10]
+        date_to = (request.args.get('date_to') or '').strip()[:10]

        conditions = [
            "r.module_type = 'SHOP_DATA_CRAWL'",
@@ -1793,34 +1818,6 @@ def shop_data_crawl_duplicate_asins():
            "TRIM(COALESCE(r.result_file_url, '')) <> ''",
        ]
        params = []
-        if shop_name:
-            conditions.append('r.source_filename LIKE %s')
-            params.append('%' + shop_name + '%')
-        if group_name:
-            conditions.append(
-                'EXISTS (SELECT 1 FROM biz_shop_manage sm '
-                'LEFT JOIN biz_shop_manage_group g ON g.id = sm.group_id '
-                'WHERE TRIM(COALESCE(sm.shop_name, \'\')) = '
-                'TRIM(COALESCE(r.source_filename, \'\')) '
-                "AND COALESCE(NULLIF(g.group_name, ''), NULLIF(sm.group_name, '')) LIKE %s)"
-            )
-            params.append('%' + group_name + '%')
-        if country:
-            if country not in ('DE', 'UK', 'FR', 'IT', 'ES'):
-                raise ValueError('不支持的国家代码: ' + country)
-            conditions.append(
-                'JSON_CONTAINS('
-                'COALESCE(df.country_codes_json, '
-                'JSON_EXTRACT(t.request_json, \'$.countryCodes\'), '
-                'JSON_EXTRACT(t.request_json, \'$.country_codes\'), \'[]\'), %s)'
-            )
-            params.append('"' + country + '"')
-        if created_from:
-            conditions.append('t.created_at >= %s')
-            params.append(created_from)
-        if created_to:
-            conditions.append('t.created_at <= %s')
-            params.append(created_to)
        where_sql = ' AND '.join(conditions)
        offset = (page - 1) * page_size

@@ -1907,10 +1904,13 @@ def shop_data_crawl_duplicate_asins():
                    current_app.logger.warning(
                        '[shop-data-crawl] 拉取结果文件失败 result_id=%s: %s', result_id, exc)

-        # 按 ASIN 聚合其出现的店铺/分组/国家与行细节
+        # 先全量聚合所有行，再用筛选条件圈定「命中 ASIN」；
+        # 命中且跨店重复的 ASIN 展示完整 occurrences（方便看该 ASIN 与哪些店重复）。
        asin_occurrences = {}
+        matched_asins = set()
        for shop_item in shop_items:
            shop_name = shop_item['shop_name'] or '未命名'
+            country_codes = shop_item['country_codes'] or []
            for row in shop_item['rows']:
                asin = row['asin'].strip().upper()
                if not asin:
@@ -1924,15 +1924,31 @@ def shop_data_crawl_duplicate_asins():
                    'group_name': shop_item['group_name'],
                    'country_codes': shop_item['country_codes'],
                })
+                # 行级筛选：命中则将此 ASIN 加入待展示集合
+                if asin_filter and asin_filter not in asin:
+                    continue
+                if shop_name_filter and shop_name_filter.lower() not in (shop_name or '').lower() \
+                        and shop_name_filter.lower() not in (shop_item['group_name'] or '').lower():
+                    continue
+                if country_filter and country_filter not in [c.upper() for c in country_codes]:
+                    continue
+                row_date_key = _shop_data_date_key((row['date'] or '').strip())
+                if date_from and row_date_key and row_date_key < date_from:
+                    continue
+                if date_to and row_date_key and row_date_key > date_to:
+                    continue
+                matched_asins.add(asin)

        occurrences_list = []
-        for asin, occurrences in asin_occurrences.items():
+        for asin in matched_asins:
+            occurrences = asin_occurrences.get(asin, [])
            shop_count = len({item['shop_name'] for item in occurrences})
            if shop_count < 2:
                continue
            occurrences_list.append({
                'asin': asin,
                'shop_count': shop_count,
+                'record_count': len(occurrences),
                'occurrences': occurrences,
            })
        occurrences_list.sort(key=lambda item: (-item['shop_count'], item['asin']))