diff --git a/.gitignore b/.gitignore index 16ec542..0eb1730 100644 --- a/.gitignore +++ b/.gitignore @@ -104,4 +104,6 @@ xlsx/ .omx/ .codex .rtk -OPS_REDIS_MYSQL_OPTIMIZATION_NOTES.md \ No newline at end of file +OPS_REDIS_MYSQL_OPTIMIZATION_NOTES.md +架构.md +*ts.% diff --git a/app/__pycache__/config.cpython-312.pyc b/app/__pycache__/config.cpython-312.pyc index 921936d..0e592c6 100644 Binary files a/app/__pycache__/config.cpython-312.pyc and b/app/__pycache__/config.cpython-312.pyc differ diff --git a/app/amazon/__pycache__/approve.cpython-312.pyc b/app/amazon/__pycache__/approve.cpython-312.pyc index 859dd8c..1859b0c 100644 Binary files a/app/amazon/__pycache__/approve.cpython-312.pyc and b/app/amazon/__pycache__/approve.cpython-312.pyc differ diff --git a/app/amazon/__pycache__/approve.cpython-39.pyc b/app/amazon/__pycache__/approve.cpython-39.pyc index d75c697..d86cdd4 100644 Binary files a/app/amazon/__pycache__/approve.cpython-39.pyc and b/app/amazon/__pycache__/approve.cpython-39.pyc differ diff --git a/app/amazon/__pycache__/asin_status.cpython-312.pyc b/app/amazon/__pycache__/asin_status.cpython-312.pyc new file mode 100644 index 0000000..1224c5c Binary files /dev/null and b/app/amazon/__pycache__/asin_status.cpython-312.pyc differ diff --git a/app/amazon/__pycache__/detail_spider.cpython-312.pyc b/app/amazon/__pycache__/detail_spider.cpython-312.pyc new file mode 100644 index 0000000..7b450aa Binary files /dev/null and b/app/amazon/__pycache__/detail_spider.cpython-312.pyc differ diff --git a/app/amazon/__pycache__/detail_spider.cpython-39.pyc b/app/amazon/__pycache__/detail_spider.cpython-39.pyc index 9a8f5b8..abf9037 100644 Binary files a/app/amazon/__pycache__/detail_spider.cpython-39.pyc and b/app/amazon/__pycache__/detail_spider.cpython-39.pyc differ diff --git a/app/amazon/__pycache__/main.cpython-312.pyc b/app/amazon/__pycache__/main.cpython-312.pyc index 7e8147d..d1560d0 100644 Binary files a/app/amazon/__pycache__/main.cpython-312.pyc and b/app/amazon/__pycache__/main.cpython-312.pyc differ diff --git a/app/amazon/__pycache__/match_action.cpython-312.pyc b/app/amazon/__pycache__/match_action.cpython-312.pyc index b049682..da45664 100644 Binary files a/app/amazon/__pycache__/match_action.cpython-312.pyc and b/app/amazon/__pycache__/match_action.cpython-312.pyc differ diff --git a/app/amazon/__pycache__/price_match.cpython-312.pyc b/app/amazon/__pycache__/price_match.cpython-312.pyc index b3bad90..a643421 100644 Binary files a/app/amazon/__pycache__/price_match.cpython-312.pyc and b/app/amazon/__pycache__/price_match.cpython-312.pyc differ diff --git a/app/amazon/__pycache__/price_match.cpython-39.pyc b/app/amazon/__pycache__/price_match.cpython-39.pyc index 95e7dc2..bff556f 100644 Binary files a/app/amazon/__pycache__/price_match.cpython-39.pyc and b/app/amazon/__pycache__/price_match.cpython-39.pyc differ diff --git a/app/amazon/approve.py b/app/amazon/approve.py index 54d8e1e..db65147 100644 --- a/app/amazon/approve.py +++ b/app/amazon/approve.py @@ -475,7 +475,8 @@ class AmzoneApprove(AmamzonBase): retry_num = 0 already_asin = set() now_page = None - while retry_num < 3: # 最多重试3次 + max_retry_num = 5 + while retry_num < max_retry_num: # 最多重试3次 # if num > 3: #测试 # return # 等待加载完成 @@ -505,8 +506,11 @@ class AmzoneApprove(AmamzonBase): sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10) print(f"获取到 {len(sku_ls)}") if len(sku_ls) == 0: - print(f"搜索不出SKU停止") - break + retry_num += 1 + print(f"没有获取到 ASIN 商品,重试{retry_num}/{max_retry_num}") + self.tab.refresh() + self.tab.wait.doc_loaded(raise_err=False, timeout=120) + continue # for sku_ele in sku_ls[0:2]: for sku_ele in sku_ls[1:]: # solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]') diff --git a/app/amazon/detail_spider.py b/app/amazon/detail_spider.py index 0558bbb..1a07e86 100644 --- a/app/amazon/detail_spider.py +++ b/app/amazon/detail_spider.py @@ -9,6 +9,7 @@ import re import traceback from datetime import datetime from DrissionPage import Chromium, ChromiumOptions +from collections import defaultdict import requests from amazon.del_brand import AmamzonBase, kill_process @@ -144,7 +145,7 @@ class ChromeAmzone: error_msg = f"运行出错: {traceback.format_exc()}" print(error_msg) show_notification(f"采集失败: {str(e)}", "error") - return None + return {} def _set_zip_code(self, zip_code, mark=None): """ @@ -286,6 +287,26 @@ class SpiderTask: show_notification(message, "error") print(f"[{timestamp}] [PriceTask] [{level}] {message}") + @staticmethod + def group_by_id_prefix(data): + """ + 根据每条数据的 id 字段进行分组： + - 如果 id 为 "2_1"，则取 "_" 前面的 "2" 作为分组 key + - 如果 id 为 "1"，则分组 key 就是 "1" + - 相同 key 的数据归为同一组 + + :param data: 原始列表数据 + :return: 二维列表，按 id 前缀分组 + """ + grouped = defaultdict(list) + + for item in data: + item_id = str(item.get("id", "")) + group_key = item_id.split("_")[0] + grouped[group_key].append(item) + + return list(grouped.values()) + def process_task(self, task_data: dict): """处理审批任务主入口 @@ -295,7 +316,8 @@ class SpiderTask: try: data = task_data.get("data", {}) task_id = data.get("taskId") - items = data.get("rows", []) + groups = data.get("groups") + items = groups[0].get("items", []) # 用于测试 limit = data.get("limit", None) @@ -331,35 +353,64 @@ class SpiderTask: show_notification(f"开始爬取数据", "info") chrome = ChromeAmzone() try: - for index,i in enumerate(items): - asin = i.get("asin") - country = i.get("country") - return_data = None - for _ in range(max_retry): - try: - return_data = chrome.run(country, asin) - self.log(f"抓取结果->{return_data}") + # 数据整理 + new_items = self.group_by_id_prefix(items) + + result = [] + for index,value_ls in enumerate(new_items): + + return_data = { + 'image_url': "", + 'title': "" + } + for value in value_ls: + asin = value.get("asin") + country = value.get("country") + return_data = None + for _ in range(max_retry): + try: + return_data = chrome.run(country, asin) + self.log(f"抓取结果->{return_data}") + break + except Exception as e: + if "与页面的连接已断开" in str(e): + chrome = ChromeAmzone() + if return_data.get("image_url"): break - except Exception as e: - if "与页面的连接已断开" in str(e): - chrome = ChromeAmzone() + # 提交结果 # 'image_url': "", - # 'title': "" + # 'title': "" - item_data = { - "id": i.get("id"), - "asin": asin, - "country": country, - "url": return_data.get("image_url"), - "title": return_data.get("title"), - } + group_item = [ + { + "sourceFileKey": i.get("sourceFileKey"), + "sourceFilename": i.get("sourceFilename"), + "rowToken": i.get("rowToken"), + "groupKey": i.get("groupKey"), + "id": i.get("id"), + "asin": i.get("asin"), + "country": i.get("country"), + "url": return_data.get("image_url"), + "title": return_data.get("title"), + } + for i in value_ls + ] # task_id: int, chunkIndex:int,chunkTotal: int, country_code: str, asin: str, status: dict,error:str="", # item_data:dict={}, + result.append({ + "sourceFileKey": groups[0].get("sourceFileKey"), + "sourceFilename": groups[0].get("sourceFilename"), + "groupKey":groups[0].get("groupKey"), + "baseId": groups[0].get("baseId"), + "displayId": groups[0].get("displayId"), + "items": group_item + }) is_done = index == len(items)-1 - self.post_result(task_id=task_id,chunkIndex=index+1,chunkTotal=len(items), - asin=asin,item_data=item_data,is_done=is_done) - + if len(result) > 20 or is_done: + self.post_result(task_id=task_id,chunkIndex=index,chunkTotal=len(items), + asin=asin,item_data=result,is_done=is_done) + result = [] try: chrome.close() except Exception as e: @@ -398,8 +449,9 @@ class SpiderTask: url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result" + submission_id = f"appearance-patent:{task_id}" payload ={ - "submissionId": f"{int(time.time())}", + "submissionId": submission_id, "chunkIndex": chunkIndex, "chunkTotal": chunkTotal, "error": error, @@ -412,6 +464,7 @@ class SpiderTask: max_retries = 3 for retry in range(max_retries): try: + request_timeout = 300 if is_done else 30 print("================【详情采集】=====================") self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)") self.log(f"回传URL: {url}") @@ -420,7 +473,7 @@ class SpiderTask: url, json=payload, headers={"Content-Type": "application/json"}, - timeout=30, + timeout=request_timeout, verify=False ) self.log(f"回传结果: {response.text}") @@ -443,6465 +496,35 @@ class SpiderTask: self.log(f"已达到最大重试次数，结果回传最终失败", "ERROR") raise RuntimeError("已达到最大重试次数，结果回传最终失败") -if __name__ == '__main__': - spide = SpiderTask() - task_data = { - "success": True, - "message": "操作成功", - "data": { - "taskId": 4188, - "sourceFilename": "17.xlsx", - "totalRows": 1601, - "acceptedRows": 716, - "droppedRows": 885, - "aiPrompt": "请帮我排查以下亚马逊商品在英国及欧洲地区是否存在知识产权侵权风险。 请直接给出明确结论（有侵权风险 或 未发现明显侵权风险），并严格按照以下三个维度提供精简的排查理由： 外观维度： 评估产品外形、图案设计是否与欧洲/英国常见外观专利雷同。 专利维度： 评估产品的核心技术或物理结构是否存在侵权可能。 标题维度： 排查标题中是否包含大牌商标、敏感词或版权保护词汇。 结论：XX", - "items": [ - { - "rowIndex": 2, - "sourceId": "1", - "displayId": "1", - "asin": "B0D792ND9V", - "country": "英国", - "url": "", - "title": "低" - }, - { - "rowIndex": 3, - "sourceId": "2_1", - "displayId": "2_1", - "asin": "B0D14L34S7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 7, - "sourceId": "3_1", - "displayId": "3_1", - "asin": "B0CWQFZMGY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 9, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0CX87XXWJ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 10, - "sourceId": "5_1", - "displayId": "5_1", - "asin": "B0CR3PMR8L", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 11, - "sourceId": "6", - "displayId": "6", - "asin": "B0DK3L6FD4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 12, - "sourceId": "8", - "displayId": "8", - "asin": "B0D9QNV6FT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 13, - "sourceId": "10_1", - "displayId": "10_1", - "asin": "B0DBZG716J", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 14, - "sourceId": "11_1", - "displayId": "11_1", - "asin": "B0CX8K4F8Z", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 15, - "sourceId": "12_1", - "displayId": "12_1", - "asin": "B0C4KXJWW1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 28, - "sourceId": "13", - "displayId": "13", - "asin": "B0D62LNNKQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 29, - "sourceId": "14", - "displayId": "14", - "asin": "B0DK3K3GF5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 30, - "sourceId": "15_1", - "displayId": "15_1", - "asin": "B0C1HN8GY6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 57, - "sourceId": "16_1", - "displayId": "16_1", - "asin": "B0B71WMDDF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 58, - "sourceId": "18_1", - "displayId": "18_1", - "asin": "B0DSZNF73N", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 60, - "sourceId": "19", - "displayId": "19", - "asin": "B0897Y5QDC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 61, - "sourceId": "22_1", - "displayId": "22_1", - "asin": "B0FNCLNS68", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 62, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B009U2SCMW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 67, - "sourceId": "24", - "displayId": "24", - "asin": "B0FBLY2ZRD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 68, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B08LP7DW3M", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 69, - "sourceId": "28_1", - "displayId": "28_1", - "asin": "B0FPBZLD6R", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 80, - "sourceId": "30", - "displayId": "30", - "asin": "B09LV1TNHK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 81, - "sourceId": "1_1", - "displayId": "1_1", - "asin": "B0DSKS214Y", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 86, - "sourceId": "2_1", - "displayId": "2_1", - "asin": "B0BZBW8KL5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 88, - "sourceId": "3", - "displayId": "3", - "asin": "B0CR3LS3R7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 89, - "sourceId": "4", - "displayId": "4", - "asin": "B097TS2MNQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 90, - "sourceId": "5", - "displayId": "5", - "asin": "B09LV4WGTR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 91, - "sourceId": "6_1", - "displayId": "6_1", - "asin": "B0916ZQP1V", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 93, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0C9WHZ67N", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 96, - "sourceId": "9", - "displayId": "9", - "asin": "B0F8QKVS1D", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 97, - "sourceId": "10", - "displayId": "10", - "asin": "B0D49974HV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 98, - "sourceId": "11", - "displayId": "11", - "asin": "B0F2DXMKVF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 99, - "sourceId": "12_1", - "displayId": "12_1", - "asin": "B08YX48NZ2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 100, - "sourceId": "14", - "displayId": "14", - "asin": "B0BGM3HF9C", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 101, - "sourceId": "15", - "displayId": "15", - "asin": "B0CJJM9KZV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 102, - "sourceId": "16_1", - "displayId": "16_1", - "asin": "B09HDSTGT4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 103, - "sourceId": "17", - "displayId": "17", - "asin": "B0CX9HK5NL", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 104, - "sourceId": "18", - "displayId": "18", - "asin": "B0C2BV83D3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 105, - "sourceId": "19_1", - "displayId": "19_1", - "asin": "B0D5WDK7GW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 107, - "sourceId": "20", - "displayId": "20", - "asin": "B09NR1PZYD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 108, - "sourceId": "21", - "displayId": "21", - "asin": "B0DC45N1Y8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 109, - "sourceId": "23", - "displayId": "23", - "asin": "B01AVHJQY2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 110, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0F3VSJ9Q2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 117, - "sourceId": "27", - "displayId": "27", - "asin": "B0CLXCXW6W", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 118, - "sourceId": "28", - "displayId": "28", - "asin": "B07PSPN1XH", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 119, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B09PHQYR6Z", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 122, - "sourceId": "30", - "displayId": "30", - "asin": "B0CT2H94WV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 123, - "sourceId": "1", - "displayId": "1", - "asin": "B0FRRK9V73", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 124, - "sourceId": "3_1", - "displayId": "3_1", - "asin": "B0B3RNHXV8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 125, - "sourceId": "4", - "displayId": "4", - "asin": "B09FF115TW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 126, - "sourceId": "5", - "displayId": "5", - "asin": "B0CYGXDWBT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 127, - "sourceId": "6", - "displayId": "6", - "asin": "B0F9FT3ZPY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 128, - "sourceId": "7", - "displayId": "7", - "asin": "B0F295JTQZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 129, - "sourceId": "8", - "displayId": "8", - "asin": "B0FPJZ1N51", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 130, - "sourceId": "9_1", - "displayId": "9_1", - "asin": "B08B8SP7XT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 131, - "sourceId": "12_1", - "displayId": "12_1", - "asin": "B002P7HGEO", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 132, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B09X268FXC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 134, - "sourceId": "14_1", - "displayId": "14_1", - "asin": "B0170OT3X8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 136, - "sourceId": "15_1", - "displayId": "15_1", - "asin": "B0859793H1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 138, - "sourceId": "17", - "displayId": "17", - "asin": "B0FP8KHCPX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 139, - "sourceId": "18_1", - "displayId": "18_1", - "asin": "B09H6KR87N", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 146, - "sourceId": "19_1", - "displayId": "19_1", - "asin": "B0D9778Q65", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 147, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0CPJCBZR8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 148, - "sourceId": "21", - "displayId": "21", - "asin": "B0CZ63NGNQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 149, - "sourceId": "22", - "displayId": "22", - "asin": "B0CR4795W6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 150, - "sourceId": "23", - "displayId": "23", - "asin": "B0CR45ZB5Y", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 151, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B0CYB255ST", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 153, - "sourceId": "25", - "displayId": "25", - "asin": "B0BMYWFKTF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 154, - "sourceId": "26", - "displayId": "26", - "asin": "B0BMYTPRD6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 155, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B0DK1MGDZM", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 157, - "sourceId": "28", - "displayId": "28", - "asin": "B006FUXS30", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 158, - "sourceId": "30", - "displayId": "30", - "asin": "B00BLPUXCI", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 159, - "sourceId": "1", - "displayId": "1", - "asin": "B09N79DR4C", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 160, - "sourceId": "2", - "displayId": "2", - "asin": "B006O74WTI", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 161, - "sourceId": "3", - "displayId": "3", - "asin": "B006O75DIC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 162, - "sourceId": "4", - "displayId": "4", - "asin": "B09323MXYP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 163, - "sourceId": "5", - "displayId": "5", - "asin": "B00K2NTEPC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 164, - "sourceId": "6", - "displayId": "6", - "asin": "B09C19Y6B3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 165, - "sourceId": "7", - "displayId": "7", - "asin": "B00L2HK0S2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 166, - "sourceId": "8", - "displayId": "8", - "asin": "B00666DQBW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 167, - "sourceId": "9", - "displayId": "9", - "asin": "B00666FLII", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 168, - "sourceId": "10", - "displayId": "10", - "asin": "B00666GVM8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 169, - "sourceId": "11_1", - "displayId": "11_1", - "asin": "B07KWJCVVN", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 173, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B00F0KAGKO", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 179, - "sourceId": "14_1", - "displayId": "14_1", - "asin": "B0C695NBMR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 180, - "sourceId": "15_1", - "displayId": "15_1", - "asin": "B0F2FXYCG4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 182, - "sourceId": "16", - "displayId": "16", - "asin": "B0FPGC8Z84", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 183, - "sourceId": "17_1", - "displayId": "17_1", - "asin": "B0D8HXBGGB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 184, - "sourceId": "18_1", - "displayId": "18_1", - "asin": "B0DDG8GJP9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 188, - "sourceId": "19", - "displayId": "19", - "asin": "B0FLCZVDDH", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 189, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B07WZZW6V9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 191, - "sourceId": "21", - "displayId": "21", - "asin": "B0GT3SJ6J1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 192, - "sourceId": "22", - "displayId": "22", - "asin": "B077R1GRW1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 193, - "sourceId": "23", - "displayId": "23", - "asin": "B0D2TMYX23", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 194, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B00AXTKYCM", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 195, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0D45CB5SS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 197, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0852JR7RD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 198, - "sourceId": "27", - "displayId": "27", - "asin": "B0GR9F93JL", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 199, - "sourceId": "28_1", - "displayId": "28_1", - "asin": "B0CCJ3NY78", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 200, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B0D1V26VBG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 201, - "sourceId": "1", - "displayId": "1", - "asin": "B0CN5VVWB2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 202, - "sourceId": "2", - "displayId": "2", - "asin": "B0DLNJTKFB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 203, - "sourceId": "3", - "displayId": "3", - "asin": "B0CSD7S964", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 204, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0GLP8L4TT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 205, - "sourceId": "5", - "displayId": "5", - "asin": "B0FGJB6LLV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 206, - "sourceId": "6", - "displayId": "6", - "asin": "B0DZ67MYZN", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 207, - "sourceId": "7", - "displayId": "7", - "asin": "B0G3XP9VYB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 208, - "sourceId": "8", - "displayId": "8", - "asin": "B0FNWMRMSZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 209, - "sourceId": "9", - "displayId": "9", - "asin": "B0FNWP3K72", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 210, - "sourceId": "10_1", - "displayId": "10_1", - "asin": "B0DQTMRNLC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 212, - "sourceId": "11", - "displayId": "11", - "asin": "B0DZ62KT5N", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 213, - "sourceId": "13", - "displayId": "13", - "asin": "B0DZ65VCDC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 214, - "sourceId": "16", - "displayId": "16", - "asin": "B0DZ67P85P", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 215, - "sourceId": "17_1", - "displayId": "17_1", - "asin": "B0F6LRCZXW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 216, - "sourceId": "18", - "displayId": "18", - "asin": "B0DPYVJMLK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 217, - "sourceId": "19", - "displayId": "19", - "asin": "B0FDKZJTY5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 218, - "sourceId": "20", - "displayId": "20", - "asin": "B0D9VFN9QW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 219, - "sourceId": "21_1", - "displayId": "21_1", - "asin": "B0BW46XZ2G", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 222, - "sourceId": "22", - "displayId": "22", - "asin": "B0FLCBQ92M", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 223, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B0F6JW61SR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 225, - "sourceId": "24", - "displayId": "24", - "asin": "B0CNZ27HCF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 226, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B001DKRB8A", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 232, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B00NH5DWIW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 236, - "sourceId": "28", - "displayId": "28", - "asin": "B01IOKEOGS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 237, - "sourceId": "29", - "displayId": "29", - "asin": "B08YJVVPBK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 238, - "sourceId": "30", - "displayId": "30", - "asin": "B0FG29NPDK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 239, - "sourceId": "1", - "displayId": "1", - "asin": "B0FRFHFXXZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 240, - "sourceId": "3", - "displayId": "3", - "asin": "B0FMQ5G18V", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 241, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0DPDKD5PB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 242, - "sourceId": "6_1", - "displayId": "6_1", - "asin": "B0FS6JDGNW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 248, - "sourceId": "8", - "displayId": "8", - "asin": "B0DZBM5VRX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 249, - "sourceId": "11_1", - "displayId": "11_1", - "asin": "B0D4Q2KB8X", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 250, - "sourceId": "12_1", - "displayId": "12_1", - "asin": "B098B3XKRS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 251, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B09F9JM8F9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 252, - "sourceId": "14", - "displayId": "14", - "asin": "B08H5M3ZQ7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 253, - "sourceId": "15", - "displayId": "15", - "asin": "B0DG3NXCCF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 254, - "sourceId": "16_1", - "displayId": "16_1", - "asin": "B0FJX2FSM1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 256, - "sourceId": "17", - "displayId": "17", - "asin": "B089KHDH15", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 257, - "sourceId": "18_1", - "displayId": "18_1", - "asin": "B0CPWP6W5M", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 264, - "sourceId": "19_1", - "displayId": "19_1", - "asin": "B0D4LXTCSZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 265, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0CK682F4T", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 267, - "sourceId": "21_1", - "displayId": "21_1", - "asin": "B0B9CGJDBV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 268, - "sourceId": "23", - "displayId": "23", - "asin": "B0DV55H9DD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 269, - "sourceId": "25", - "displayId": "25", - "asin": "B0FJ8B8RJS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 270, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0D426K41D", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 275, - "sourceId": "27", - "displayId": "27", - "asin": "B0CTD4BF7S", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 276, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B0CGXGQ43L", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 279, - "sourceId": "30_1", - "displayId": "30_1", - "asin": "B0BDM3B6LY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 283, - "sourceId": "1", - "displayId": "1", - "asin": "B0FS1SJGJ5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 284, - "sourceId": "2", - "displayId": "2", - "asin": "B0FJ1RH4JV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 285, - "sourceId": "3", - "displayId": "3", - "asin": "B0B9MBTX28", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 286, - "sourceId": "4", - "displayId": "4", - "asin": "B0FV2VLSX1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 287, - "sourceId": "5_1", - "displayId": "5_1", - "asin": "B0BRYKW1GD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 289, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B0CT5G9YWM", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 291, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0D927DZ6B", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 294, - "sourceId": "9", - "displayId": "9", - "asin": "B0DSDY3G29", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 295, - "sourceId": "10_1", - "displayId": "10_1", - "asin": "B0CWRRSV9H", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 296, - "sourceId": "11", - "displayId": "11", - "asin": "B0DNDTT9Z4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 297, - "sourceId": "13", - "displayId": "13", - "asin": "B0CP5VD5SG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 298, - "sourceId": "14_1", - "displayId": "14_1", - "asin": "B0BV2Q24YK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 300, - "sourceId": "15_1", - "displayId": "15_1", - "asin": "B0F3JD29KW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 302, - "sourceId": "16", - "displayId": "16", - "asin": "B0C818XVYT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 303, - "sourceId": "17", - "displayId": "17", - "asin": "B0FJRYQ3QV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 304, - "sourceId": "18", - "displayId": "18", - "asin": "B0CP3NYG8J", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 305, - "sourceId": "19", - "displayId": "19", - "asin": "B0F6C7XH75", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 306, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0FXX9N5VH", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 308, - "sourceId": "22_1", - "displayId": "22_1", - "asin": "B0CFJPCSDP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 310, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B0FQJHS1MZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 311, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0D5VJN99L", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 313, - "sourceId": "26", - "displayId": "26", - "asin": "B07MCDWVJG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 314, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B0CKYJCJGD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 319, - "sourceId": "28_1", - "displayId": "28_1", - "asin": "B0CQXPR4H4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 321, - "sourceId": "29", - "displayId": "29", - "asin": "B08S2913J7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 322, - "sourceId": "30_1", - "displayId": "30_1", - "asin": "B0CXPL3GSW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 323, - "sourceId": "1", - "displayId": "1", - "asin": "B0D7JVZWZF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 324, - "sourceId": "2", - "displayId": "2", - "asin": "B0FKMXP3ZR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 325, - "sourceId": "3", - "displayId": "3", - "asin": "B0FGP7YJC8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 326, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0F9PDZTCJ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 328, - "sourceId": "5_1", - "displayId": "5_1", - "asin": "B0CTLXM3C6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 331, - "sourceId": "6_1", - "displayId": "6_1", - "asin": "B0CSYB8BPK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 335, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B01I1CEEXW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 340, - "sourceId": "8", - "displayId": "8", - "asin": "B0G2Y49LCY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 341, - "sourceId": "9_1", - "displayId": "9_1", - "asin": "B0F3HK55L6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 343, - "sourceId": "10", - "displayId": "10", - "asin": "B0GWJ69JK1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 344, - "sourceId": "11_1", - "displayId": "11_1", - "asin": "B09Z877C17", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 345, - "sourceId": "12", - "displayId": "12", - "asin": "B0GLDX9NB1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 346, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B0FPC1Y19B", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 348, - "sourceId": "14", - "displayId": "14", - "asin": "B0C65438CP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 349, - "sourceId": "17", - "displayId": "17", - "asin": "B0F6MNZB6F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 350, - "sourceId": "18", - "displayId": "18", - "asin": "B0BVPLLLRP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 351, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0FK2B5FK2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 353, - "sourceId": "21_1", - "displayId": "21_1", - "asin": "B0BPNFMKQV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 354, - "sourceId": "22", - "displayId": "22", - "asin": "B0G4BLRDW6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 355, - "sourceId": "23", - "displayId": "23", - "asin": "B0F93QCQF1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 356, - "sourceId": "24", - "displayId": "24", - "asin": "B0FHHPYP9K", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 357, - "sourceId": "28", - "displayId": "28", - "asin": "B0F1K9KDX6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 358, - "sourceId": "29", - "displayId": "29", - "asin": "B0F52ZXD55", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 359, - "sourceId": "30_1", - "displayId": "30_1", - "asin": "B07GDGSKK6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 361, - "sourceId": "1_1", - "displayId": "1_1", - "asin": "B08BPHJ34D", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 362, - "sourceId": "2", - "displayId": "2", - "asin": "B07ZFWRH75", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 363, - "sourceId": "3", - "displayId": "3", - "asin": "B0G5PVP3Y3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 364, - "sourceId": "4", - "displayId": "4", - "asin": "B0FXXLPC1T", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 365, - "sourceId": "5", - "displayId": "5", - "asin": "B0GQB1K38X", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 366, - "sourceId": "6", - "displayId": "6", - "asin": "B0C1TCDZ5N", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 367, - "sourceId": "8", - "displayId": "8", - "asin": "B0FHD3QCLW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 368, - "sourceId": "9_1", - "displayId": "9_1", - "asin": "B0F1THT9XW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 370, - "sourceId": "11", - "displayId": "11", - "asin": "B0DYJ8743N", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 371, - "sourceId": "12_1", - "displayId": "12_1", - "asin": "B0FD3Q2RGP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 373, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B0BV66T3PQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 374, - "sourceId": "14_1", - "displayId": "14_1", - "asin": "B0CDVYLTB4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 383, - "sourceId": "16", - "displayId": "16", - "asin": "B0CNKGFWKR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 384, - "sourceId": "17", - "displayId": "17", - "asin": "B0D4VQF22F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 385, - "sourceId": "18_1", - "displayId": "18_1", - "asin": "B0C5TBR8LB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 387, - "sourceId": "19", - "displayId": "19", - "asin": "B0DDH3G9PG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 388, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0D8VRMWP2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 390, - "sourceId": "22_1", - "displayId": "22_1", - "asin": "B0BFWXWNTC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 395, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B096ZCL4JX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 399, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B0CFDXR5FC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 400, - "sourceId": "25", - "displayId": "25", - "asin": "B0G6DN5QC7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 401, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0CNRL9R1F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 402, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B0BJK6D2D4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 403, - "sourceId": "28_1", - "displayId": "28_1", - "asin": "B0DPFDTL7W", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 405, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B0G5N9KGXC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 406, - "sourceId": "30_1", - "displayId": "30_1", - "asin": "B0F73X9X8D", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 408, - "sourceId": "1", - "displayId": "1", - "asin": "B0CJTJB8PV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 409, - "sourceId": "2", - "displayId": "2", - "asin": "B0FBGD3QJ2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 410, - "sourceId": "3", - "displayId": "3", - "asin": "B0F98TGSQ1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 411, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0F329ZXH5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 412, - "sourceId": "5", - "displayId": "5", - "asin": "B0G191SX5F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 413, - "sourceId": "6", - "displayId": "6", - "asin": "B0F7XT5TNY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 414, - "sourceId": "7", - "displayId": "7", - "asin": "B0FVS7K78Q", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 415, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0FYFZBQLT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 418, - "sourceId": "10", - "displayId": "10", - "asin": "B0FG2LHWB3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 419, - "sourceId": "11", - "displayId": "11", - "asin": "B0F9WX88T1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 420, - "sourceId": "12", - "displayId": "12", - "asin": "B0F7XNDBNN", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 421, - "sourceId": "14", - "displayId": "14", - "asin": "B0DT7GV9NS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 422, - "sourceId": "15_1", - "displayId": "15_1", - "asin": "B0DKF26TYN", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 423, - "sourceId": "18_1", - "displayId": "18_1", - "asin": "B0D1FP339J", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 424, - "sourceId": "20", - "displayId": "20", - "asin": "B0CNCBQCMG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 425, - "sourceId": "21_1", - "displayId": "21_1", - "asin": "B0CHRR45X7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 427, - "sourceId": "23", - "displayId": "23", - "asin": "B0C1RPB8BQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 428, - "sourceId": "24", - "displayId": "24", - "asin": "B0BVPPBH7D", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 429, - "sourceId": "25", - "displayId": "25", - "asin": "B0BHQKQMG8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 430, - "sourceId": "26", - "displayId": "26", - "asin": "B0C7QQPNC9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 431, - "sourceId": "27", - "displayId": "27", - "asin": "B0FHKT5CBC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 432, - "sourceId": "29", - "displayId": "29", - "asin": "B0DR5XVMN7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 433, - "sourceId": "30", - "displayId": "30", - "asin": "B08HPS68Q4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 434, - "sourceId": "1", - "displayId": "1", - "asin": "B0C4KLQPDZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 435, - "sourceId": "2", - "displayId": "2", - "asin": "B0F6TT21VN", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 436, - "sourceId": "3", - "displayId": "3", - "asin": "B08MFB5W4Q", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 437, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B07C9Z5CG5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 441, - "sourceId": "5", - "displayId": "5", - "asin": "B07589LNBP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 442, - "sourceId": "6", - "displayId": "6", - "asin": "B08HRLZVX2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 443, - "sourceId": "7", - "displayId": "7", - "asin": "B08K7B3T6P", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 444, - "sourceId": "8", - "displayId": "8", - "asin": "B09T9YYYTS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 445, - "sourceId": "9", - "displayId": "9", - "asin": "B081ZVNH98", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 446, - "sourceId": "10", - "displayId": "10", - "asin": "B0F6T4XMXY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 447, - "sourceId": "11_1", - "displayId": "11_1", - "asin": "B07QH4X4Y5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 449, - "sourceId": "12_1", - "displayId": "12_1", - "asin": "B096S27HMH", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 460, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B0FB9DWYXF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 462, - "sourceId": "14_1", - "displayId": "14_1", - "asin": "B08R58FBXZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 494, - "sourceId": "15_1", - "displayId": "15_1", - "asin": "B0CRK8YNNY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 496, - "sourceId": "16_1", - "displayId": "16_1", - "asin": "B09C8F7J2H", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 504, - "sourceId": "17", - "displayId": "17", - "asin": "B0F6TK2KP3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 505, - "sourceId": "18_1", - "displayId": "18_1", - "asin": "B0BYJHNBWM", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 515, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0DJBZXCDP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 516, - "sourceId": "21_1", - "displayId": "21_1", - "asin": "B0FMRGW63W", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 518, - "sourceId": "22_1", - "displayId": "22_1", - "asin": "B0D175WMJC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 519, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B07FLD65NM", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 521, - "sourceId": "24", - "displayId": "24", - "asin": "B0DRFWYNNV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 522, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0DZX3JWQP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 524, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0F5X5G34F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 530, - "sourceId": "28_1", - "displayId": "28_1", - "asin": "B0BJDL1D9W", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 544, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B0BYJMMMLX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 545, - "sourceId": "30_1", - "displayId": "30_1", - "asin": "B092M4RJVV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 552, - "sourceId": "1_1", - "displayId": "1_1", - "asin": "B092M49F1Q", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 565, - "sourceId": "2_1", - "displayId": "2_1", - "asin": "B0FMRFHRZR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 566, - "sourceId": "3_1", - "displayId": "3_1", - "asin": "B0DZWVVM6F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 570, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0BCNZZ32X", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 580, - "sourceId": "6_1", - "displayId": "6_1", - "asin": "B0BRC8PXMW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 581, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B07RL77RM9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 582, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0FMRBN7H4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 584, - "sourceId": "9_1", - "displayId": "9_1", - "asin": "B0FMRGHCPK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 587, - "sourceId": "10", - "displayId": "10", - "asin": "B0G4BWPCK7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 588, - "sourceId": "11_1", - "displayId": "11_1", - "asin": "B0CJJ4Y8SB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 589, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B0DYY436WG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 591, - "sourceId": "14", - "displayId": "14", - "asin": "B0G64PMQ94", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 592, - "sourceId": "16_1", - "displayId": "16_1", - "asin": "B0CST698J6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 593, - "sourceId": "17_1", - "displayId": "17_1", - "asin": "B00U7LUWR8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 594, - "sourceId": "18", - "displayId": "18", - "asin": "B0DHRJX1VS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 595, - "sourceId": "20", - "displayId": "20", - "asin": "B0CY9NB8VS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 596, - "sourceId": "21_1", - "displayId": "21_1", - "asin": "B08ML38H24", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 599, - "sourceId": "22_1", - "displayId": "22_1", - "asin": "B0BXVBVJ6G", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 605, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B08HS69FQG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 608, - "sourceId": "25", - "displayId": "25", - "asin": "B0F281WHDB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 609, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0007RN382", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 610, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B0F2JDR3Y5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 612, - "sourceId": "28", - "displayId": "28", - "asin": "B0FX9L2JJ4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 613, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B0FD9NVNFT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 615, - "sourceId": "30", - "displayId": "30", - "asin": "B0B72HD5B9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 616, - "sourceId": "2_1", - "displayId": "2_1", - "asin": "B0F53FQ9Y4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 618, - "sourceId": "3_1", - "displayId": "3_1", - "asin": "B0GHDHPQ7J", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 629, - "sourceId": "5_1", - "displayId": "5_1", - "asin": "B07B9ZP8TK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 631, - "sourceId": "6", - "displayId": "6", - "asin": "B0DPWW9P2V", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 632, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B0CR43DTKJ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 659, - "sourceId": "8", - "displayId": "8", - "asin": "B0G8K4C3D6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 660, - "sourceId": "10_1", - "displayId": "10_1", - "asin": "B0F52TLKVM", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 662, - "sourceId": "11", - "displayId": "11", - "asin": "B0BGH6Y6KB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 663, - "sourceId": "12_1", - "displayId": "12_1", - "asin": "B0D28VG1C3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 667, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B0D3LLKF7C", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 670, - "sourceId": "14", - "displayId": "14", - "asin": "B09KT2VBJL", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 671, - "sourceId": "15_1", - "displayId": "15_1", - "asin": "B07F6P6C3F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 672, - "sourceId": "18_1", - "displayId": "18_1", - "asin": "B0GCZH5V2P", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 709, - "sourceId": "19_1", - "displayId": "19_1", - "asin": "B0FH5RNFZQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 710, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0CSFMDTSR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 714, - "sourceId": "21", - "displayId": "21", - "asin": "B0GR9TS4N8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 715, - "sourceId": "22", - "displayId": "22", - "asin": "B0D9GWVPGJ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 716, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B0GF2BKHJK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 717, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B0GQZ8SPZZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 719, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0D3LQF6P5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 720, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0GF2ML2NY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 722, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B0C28BCVH6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 723, - "sourceId": "28_1", - "displayId": "28_1", - "asin": "B0C1CRZPMS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 730, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B0F8HKX897", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 734, - "sourceId": "30_1", - "displayId": "30_1", - "asin": "B0GSKSC1KR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 735, - "sourceId": "2_1", - "displayId": "2_1", - "asin": "B0GSQGF76K", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 736, - "sourceId": "3_1", - "displayId": "3_1", - "asin": "B0F9WTNVXP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 738, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0F47S987P", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 740, - "sourceId": "5_1", - "displayId": "5_1", - "asin": "B0F4Y4F5TC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 741, - "sourceId": "6_1", - "displayId": "6_1", - "asin": "B0DYNW29NB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 748, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B0GF29HX77", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 749, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0GM2WBVXF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 751, - "sourceId": "9", - "displayId": "9", - "asin": "B0GGWW658M", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 752, - "sourceId": "10", - "displayId": "10", - "asin": "B0F32R7JTF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 753, - "sourceId": "11", - "displayId": "11", - "asin": "B0DZ67TBHQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 754, - "sourceId": "12_1", - "displayId": "12_1", - "asin": "B0CLC6KSZH", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 755, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B0F8QW1FS1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 760, - "sourceId": "14_1", - "displayId": "14_1", - "asin": "B0F6LHFVJ3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 768, - "sourceId": "16", - "displayId": "16", - "asin": "B0DZH72GF6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 769, - "sourceId": "17_1", - "displayId": "17_1", - "asin": "B099N1FB4K", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 771, - "sourceId": "18", - "displayId": "18", - "asin": "B0BS678GBK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 772, - "sourceId": "19_1", - "displayId": "19_1", - "asin": "B0F5Y4KSQJ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 778, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0D584KC9W", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 805, - "sourceId": "21_1", - "displayId": "21_1", - "asin": "B0GK1D75ZF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 810, - "sourceId": "22_1", - "displayId": "22_1", - "asin": "B0DPWXNWV5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 812, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B0G64ZCGTV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 813, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B07VHK16YV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 815, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0GS3QV5F2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 816, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0DC7TR9B5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 821, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B0FH634932", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 824, - "sourceId": "28_1", - "displayId": "28_1", - "asin": "B0CY2XN6JF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 826, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B0GKR5WMXK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 829, - "sourceId": "30", - "displayId": "30", - "asin": "B0GH4KYZH5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 830, - "sourceId": "1_1", - "displayId": "1_1", - "asin": "B0FD76KJ9P", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 831, - "sourceId": "3", - "displayId": "3", - "asin": "B0836NDGM2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 832, - "sourceId": "4", - "displayId": "4", - "asin": "B0DYJWTS4N", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 833, - "sourceId": "5", - "displayId": "5", - "asin": "B089X4VCPY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 834, - "sourceId": "6", - "displayId": "6", - "asin": "B0CQZR1LZX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 835, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B0GSVTDQBX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 837, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0D9TK3VMD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 839, - "sourceId": "9_1", - "displayId": "9_1", - "asin": "B0GDZWHF6D", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 841, - "sourceId": "10_1", - "displayId": "10_1", - "asin": "B0C8N1LZT9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 842, - "sourceId": "12", - "displayId": "12", - "asin": "B0GWHQHKJS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 843, - "sourceId": "13", - "displayId": "13", - "asin": "B0BTTJKZDC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 844, - "sourceId": "14", - "displayId": "14", - "asin": "B0DZPDJ7Q4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 845, - "sourceId": "15_1", - "displayId": "15_1", - "asin": "B0D5W9F2DS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 849, - "sourceId": "16_1", - "displayId": "16_1", - "asin": "B0CGCPLWV1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 850, - "sourceId": "17_1", - "displayId": "17_1", - "asin": "B09VT58BM4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 852, - "sourceId": "19_1", - "displayId": "19_1", - "asin": "B07XZT797H", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 854, - "sourceId": "20", - "displayId": "20", - "asin": "B0GW9TR89X", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 856, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B09ZKT7QW6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 859, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0CJYRW7RL", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 861, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0DSZ4V9PJ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 862, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B09TKN8F9W", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 863, - "sourceId": "28_1", - "displayId": "28_1", - "asin": "B09SW82GSP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 871, - "sourceId": "2_1", - "displayId": "2_1", - "asin": "B001F4OSJY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 878, - "sourceId": "3", - "displayId": "3", - "asin": "B091BK7ZF3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 879, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B07HSL53DY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 883, - "sourceId": "5", - "displayId": "5", - "asin": "B07HSDTYC8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 884, - "sourceId": "6_1", - "displayId": "6_1", - "asin": "B0FKN7JZ6B", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 886, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B071XJDSBK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 889, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B006UITUJ8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 890, - "sourceId": "10_1", - "displayId": "10_1", - "asin": "B0CBPWL6NZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 891, - "sourceId": "11", - "displayId": "11", - "asin": "B0G3Q9B48L", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 892, - "sourceId": "12", - "displayId": "12", - "asin": "B0BYJV8P98", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 893, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B0G8DTQP1F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 897, - "sourceId": "14_1", - "displayId": "14_1", - "asin": "B0CYP192B6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 898, - "sourceId": "15_1", - "displayId": "15_1", - "asin": "B0CCN8TJCD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 903, - "sourceId": "16_1", - "displayId": "16_1", - "asin": "B0FWKJVMNK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 906, - "sourceId": "19", - "displayId": "19", - "asin": "B0GD11NDH2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 907, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0BC2NFJBP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 911, - "sourceId": "21_1", - "displayId": "21_1", - "asin": "B0D9RRCC93", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 913, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B0DLSZFMM2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 918, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B0GFBG4HSR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 919, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B07TBCFGPT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 920, - "sourceId": "27", - "displayId": "27", - "asin": "B0GSZRFZGZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 921, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B008OB1AZ6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 922, - "sourceId": "30_1", - "displayId": "30_1", - "asin": "B09XLVPYPR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 930, - "sourceId": "1_1", - "displayId": "1_1", - "asin": "B0FZK5NCWK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 933, - "sourceId": "2_1", - "displayId": "2_1", - "asin": "B0DYJTRT6K", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 934, - "sourceId": "3_1", - "displayId": "3_1", - "asin": "B0CL664TZX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 936, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0DN6XQ5W4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 938, - "sourceId": "5_1", - "displayId": "5_1", - "asin": "B002HNNWLM", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 943, - "sourceId": "6_1", - "displayId": "6_1", - "asin": "B0F5HHHR65", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 946, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B0G5739C1J", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 951, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0G11TTQPT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 952, - "sourceId": "9_1", - "displayId": "9_1", - "asin": "B0G579K29G", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 957, - "sourceId": "10", - "displayId": "10", - "asin": "B0G57DKM85", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 958, - "sourceId": "11", - "displayId": "11", - "asin": "B0DGFW14PB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 959, - "sourceId": "12_1", - "displayId": "12_1", - "asin": "B0DK3YXKS5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 960, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B0FGTHRS1R", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 961, - "sourceId": "14_1", - "displayId": "14_1", - "asin": "B0FP2FDZPZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 963, - "sourceId": "15", - "displayId": "15", - "asin": "B002HO7QQ8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 964, - "sourceId": "16_1", - "displayId": "16_1", - "asin": "B002HNS1TU", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 966, - "sourceId": "17_1", - "displayId": "17_1", - "asin": "B004KL50HS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 967, - "sourceId": "18_1", - "displayId": "18_1", - "asin": "B07QK3B4Q5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 972, - "sourceId": "19", - "displayId": "19", - "asin": "B08DVGHKLL", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 973, - "sourceId": "20", - "displayId": "20", - "asin": "B08K3ZYRGC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 974, - "sourceId": "21", - "displayId": "21", - "asin": "B0DZWVCVK1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 975, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B0CT29V1D5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 977, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B01N7G9ALO", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 978, - "sourceId": "25", - "displayId": "25", - "asin": "B0CQLN3T6N", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 979, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B0FJKTYHSV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 981, - "sourceId": "28", - "displayId": "28", - "asin": "B0FRFNVTKX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 982, - "sourceId": "1", - "displayId": "1", - "asin": "B0DJQWP543", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 983, - "sourceId": "3", - "displayId": "3", - "asin": "B0D7M543CT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 984, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0F21QQQL1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 987, - "sourceId": "5", - "displayId": "5", - "asin": "B0CMZR1JJD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 988, - "sourceId": "6", - "displayId": "6", - "asin": "B0F66NTL6T", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 989, - "sourceId": "7", - "displayId": "7", - "asin": "B0D976JCLF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 990, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0FP2DMFMG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 991, - "sourceId": "9_1", - "displayId": "9_1", - "asin": "B0CTDM36PB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 994, - "sourceId": "10", - "displayId": "10", - "asin": "B0DY76XFLY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 995, - "sourceId": "11", - "displayId": "11", - "asin": "B0CQ4PS256", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 996, - "sourceId": "12_1", - "displayId": "12_1", - "asin": "B0C15RDVDK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 997, - "sourceId": "13", - "displayId": "13", - "asin": "B0F5Y8Y58T", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 998, - "sourceId": "14", - "displayId": "14", - "asin": "B0B5CBKC17", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 999, - "sourceId": "16_1", - "displayId": "16_1", - "asin": "B0FMXVRQM2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1000, - "sourceId": "18", - "displayId": "18", - "asin": "B0DC2WX8Q2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1001, - "sourceId": "19_1", - "displayId": "19_1", - "asin": "B0FVR6L7NP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1004, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0DFQMCBS7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1006, - "sourceId": "21", - "displayId": "21", - "asin": "B07TSMHP3T", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1007, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B09DY2KCLB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1008, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0GRTXKMGZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1010, - "sourceId": "26", - "displayId": "26", - "asin": "B0BG4TTWWW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1011, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B0FC2QBR9T", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1012, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B09Y2H27PS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1013, - "sourceId": "30_1", - "displayId": "30_1", - "asin": "B0CTGLQFCD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1016, - "sourceId": "2_1", - "displayId": "2_1", - "asin": "B01MCVO50E", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1017, - "sourceId": "3", - "displayId": "3", - "asin": "B0FJXM3T1S", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1018, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0C8MG6NF7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1019, - "sourceId": "5_1", - "displayId": "5_1", - "asin": "B0C7NK648Y", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1020, - "sourceId": "6", - "displayId": "6", - "asin": "B0DYFDXD8N", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1021, - "sourceId": "7", - "displayId": "7", - "asin": "B0GGMLRZWF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1022, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0D45VVD1Y", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1024, - "sourceId": "9_1", - "displayId": "9_1", - "asin": "B0CC6KP279", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1046, - "sourceId": "12", - "displayId": "12", - "asin": "B0FRZYQYGT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1047, - "sourceId": "13", - "displayId": "13", - "asin": "B0DK447HBK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1048, - "sourceId": "14", - "displayId": "14", - "asin": "B0788GRQT1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1049, - "sourceId": "15", - "displayId": "15", - "asin": "B0F93QPPVD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1050, - "sourceId": "16_1", - "displayId": "16_1", - "asin": "B0BRQG6CKG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1051, - "sourceId": "17", - "displayId": "17", - "asin": "B09DCW6XN1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1052, - "sourceId": "18", - "displayId": "18", - "asin": "B0FSQDHC9V", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1053, - "sourceId": "19_1", - "displayId": "19_1", - "asin": "B0D25DHV1T", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1054, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0DJF4Z91G", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1056, - "sourceId": "21_1", - "displayId": "21_1", - "asin": "B0F3J3B1R2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1060, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B0F3J6WJZJ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1062, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B0FQTWP8GM", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1065, - "sourceId": "28", - "displayId": "28", - "asin": "B0FPQL1ZW9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1066, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B0B77PQ12W", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1068, - "sourceId": "30", - "displayId": "30", - "asin": "B0FXWTDY16", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1069, - "sourceId": "1_1", - "displayId": "1_1", - "asin": "B0D6ZV5F22", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1070, - "sourceId": "2", - "displayId": "2", - "asin": "B0BF13CW1W", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1071, - "sourceId": "3", - "displayId": "3", - "asin": "B0F52CM3X1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1072, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0043RJSSG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1095, - "sourceId": "5_1", - "displayId": "5_1", - "asin": "B07V2G46J7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1096, - "sourceId": "6_1", - "displayId": "6_1", - "asin": "B07X4V22PW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1098, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B0GQXDVW6C", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1099, - "sourceId": "8", - "displayId": "8", - "asin": "B0BK888623", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1100, - "sourceId": "9_1", - "displayId": "9_1", - "asin": "B0BFWC5SZQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1102, - "sourceId": "10_1", - "displayId": "10_1", - "asin": "B0F1C9ZQKD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1106, - "sourceId": "11", - "displayId": "11", - "asin": "B09F5VV8DW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1107, - "sourceId": "12", - "displayId": "12", - "asin": "B0BWMT9LFC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1108, - "sourceId": "14_1", - "displayId": "14_1", - "asin": "B0FJLQLRNT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1112, - "sourceId": "15", - "displayId": "15", - "asin": "B08ZCNVYTK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1113, - "sourceId": "16", - "displayId": "16", - "asin": "B0CMZLKY9T", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1114, - "sourceId": "17", - "displayId": "17", - "asin": "B0GJZXZ2F9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1115, - "sourceId": "18", - "displayId": "18", - "asin": "B07BMFCKCR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1116, - "sourceId": "19_1", - "displayId": "19_1", - "asin": "B0DK46T977", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1118, - "sourceId": "20", - "displayId": "20", - "asin": "B08HCPX1MB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1119, - "sourceId": "21_1", - "displayId": "21_1", - "asin": "B0D8L6WRCM", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1121, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B09NDN8NV1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1122, - "sourceId": "24", - "displayId": "24", - "asin": "B0GT2527XG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1123, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0FNRCLHNS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1127, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0DQS8GKLG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1129, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B003NGLFYI", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1132, - "sourceId": "28", - "displayId": "28", - "asin": "B0CR4FTMGG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1133, - "sourceId": "29", - "displayId": "29", - "asin": "B0DK9ZBBKS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1134, - "sourceId": "30", - "displayId": "30", - "asin": "B0D1XX28FN", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1135, - "sourceId": "1_1", - "displayId": "1_1", - "asin": "B003KHEEY8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1151, - "sourceId": "2_1", - "displayId": "2_1", - "asin": "B0F9WRKC84", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1153, - "sourceId": "3", - "displayId": "3", - "asin": "B08BHKGYJ7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1154, - "sourceId": "4", - "displayId": "4", - "asin": "B0FT1562DX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1155, - "sourceId": "5", - "displayId": "5", - "asin": "B0F8QRJ9JB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1156, - "sourceId": "6_1", - "displayId": "6_1", - "asin": "B0070OARSO", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1168, - "sourceId": "7", - "displayId": "7", - "asin": "B07XJ4W6SW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1169, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0C1V86KMX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1170, - "sourceId": "9_1", - "displayId": "9_1", - "asin": "B0CGM1Z2VG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1171, - "sourceId": "10_1", - "displayId": "10_1", - "asin": "B004699SWM", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1220, - "sourceId": "11", - "displayId": "11", - "asin": "B0GT281VQK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1221, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B09BCSR7NQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1222, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B003YCKTIE", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1223, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B092CGQN2K", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1224, - "sourceId": "30_1", - "displayId": "30_1", - "asin": "B001WSDMBI", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1225, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B079C2CDLD", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1227, - "sourceId": "5", - "displayId": "5", - "asin": "B077WX43MH", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1228, - "sourceId": "6_1", - "displayId": "6_1", - "asin": "B0FQ2YC11V", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1229, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B0DRFQJ6CX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1231, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0BX65MF93", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1233, - "sourceId": "9", - "displayId": "9", - "asin": "B0CT5ZR247", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1234, - "sourceId": "10", - "displayId": "10", - "asin": "B0BFXLLR2F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1235, - "sourceId": "11_1", - "displayId": "11_1", - "asin": "B0FDQT4X7W", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1236, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B0FQ2XD95Z", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1238, - "sourceId": "15", - "displayId": "15", - "asin": "B094RDMZ3Z", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1239, - "sourceId": "16_1", - "displayId": "16_1", - "asin": "B0DMZHTJY2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1240, - "sourceId": "18", - "displayId": "18", - "asin": "B0CS2TQGM5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1241, - "sourceId": "19", - "displayId": "19", - "asin": "B0DG2XK945", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1242, - "sourceId": "21", - "displayId": "21", - "asin": "B0G2LNGQPK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1243, - "sourceId": "22", - "displayId": "22", - "asin": "B0G4R1RHYH", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1244, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B0GSRZM4XB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1245, - "sourceId": "25", - "displayId": "25", - "asin": "B0D926NJN9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1246, - "sourceId": "26", - "displayId": "26", - "asin": "B01LZBHEKJ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1247, - "sourceId": "27", - "displayId": "27", - "asin": "B09LQL4S9Q", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1248, - "sourceId": "29_1", - "displayId": "29_1", - "asin": "B0CQTPF6BJ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1256, - "sourceId": "30", - "displayId": "30", - "asin": "B0DZVY2F7M", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1257, - "sourceId": "1", - "displayId": "1", - "asin": "B0GMYGQ2G6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1258, - "sourceId": "3_1", - "displayId": "3_1", - "asin": "B0C5WQ6DFB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1261, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0DJVK3SC4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1262, - "sourceId": "5_1", - "displayId": "5_1", - "asin": "B000N324DU", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1267, - "sourceId": "6", - "displayId": "6", - "asin": "B074F6YQZW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1268, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B09MD5W6Z5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1269, - "sourceId": "8", - "displayId": "8", - "asin": "B0F94GJX9D", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1270, - "sourceId": "9", - "displayId": "9", - "asin": "B0DP298J3J", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1271, - "sourceId": "10", - "displayId": "10", - "asin": "B010W3V6Z2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1272, - "sourceId": "11_1", - "displayId": "11_1", - "asin": "B0F24BSV4V", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1273, - "sourceId": "12", - "displayId": "12", - "asin": "B0B9MKBPH9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1274, - "sourceId": "13", - "displayId": "13", - "asin": "B0BHHKC5JQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1275, - "sourceId": "14", - "displayId": "14", - "asin": "B0B4RFNK72", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1276, - "sourceId": "15", - "displayId": "15", - "asin": "B01LXD3KE6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1277, - "sourceId": "17", - "displayId": "17", - "asin": "B010W3V1W0", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1278, - "sourceId": "18", - "displayId": "18", - "asin": "B095JNPP53", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1279, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0D7DSBNNH", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1281, - "sourceId": "21", - "displayId": "21", - "asin": "B0B935PV25", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1282, - "sourceId": "22", - "displayId": "22", - "asin": "B0B67XDHPQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1283, - "sourceId": "24", - "displayId": "24", - "asin": "B072Z1PQM2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1284, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0CT1J5HWX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1285, - "sourceId": "26", - "displayId": "26", - "asin": "B097G2DXSQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1286, - "sourceId": "27", - "displayId": "27", - "asin": "B010W3VLE8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1287, - "sourceId": "28", - "displayId": "28", - "asin": "B00R39MP4A", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1288, - "sourceId": "29", - "displayId": "29", - "asin": "B095JMYSQ9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1289, - "sourceId": "30", - "displayId": "30", - "asin": "B0F8F5H3MG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1290, - "sourceId": "1", - "displayId": "1", - "asin": "B0DNM4R43P", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1291, - "sourceId": "2", - "displayId": "2", - "asin": "B0FCFB3X78", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1292, - "sourceId": "3", - "displayId": "3", - "asin": "B0CN4KXVWF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1293, - "sourceId": "4", - "displayId": "4", - "asin": "B0D455HLWT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1294, - "sourceId": "5", - "displayId": "5", - "asin": "B0B1HYLLYT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1295, - "sourceId": "6_1", - "displayId": "6_1", - "asin": "B0F1TKQG14", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1296, - "sourceId": "7", - "displayId": "7", - "asin": "B0F32CLT9F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1297, - "sourceId": "8", - "displayId": "8", - "asin": "B0DJXYDPFC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1298, - "sourceId": "9", - "displayId": "9", - "asin": "B0B3RRLNCT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1299, - "sourceId": "10", - "displayId": "10", - "asin": "B0F2DR94NT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1300, - "sourceId": "11", - "displayId": "11", - "asin": "B0BL7HRF5R", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1301, - "sourceId": "12", - "displayId": "12", - "asin": "B0B6PT6D7Q", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1302, - "sourceId": "13", - "displayId": "13", - "asin": "B0FF4FLHM4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1303, - "sourceId": "14", - "displayId": "14", - "asin": "B07L2476DB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1304, - "sourceId": "15", - "displayId": "15", - "asin": "B0CK4379VH", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1305, - "sourceId": "16", - "displayId": "16", - "asin": "B0BRFW6XF4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1306, - "sourceId": "17", - "displayId": "17", - "asin": "B07BZT7699", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1307, - "sourceId": "18_1", - "displayId": "18_1", - "asin": "B0D8W4JRTY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1308, - "sourceId": "19", - "displayId": "19", - "asin": "B0F9NQMWGC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1309, - "sourceId": "20", - "displayId": "20", - "asin": "B07CF8KD48", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1310, - "sourceId": "21", - "displayId": "21", - "asin": "B0F9NWBVD8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1311, - "sourceId": "22", - "displayId": "22", - "asin": "B0DSZK41Z9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1312, - "sourceId": "23", - "displayId": "23", - "asin": "B0DHP1TPQL", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1313, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B0DPHSPQRP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1314, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0BVZ3VKNX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1320, - "sourceId": "26", - "displayId": "26", - "asin": "B0FPX7363M", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1321, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B0F9KCHB7F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1323, - "sourceId": "28_1", - "displayId": "28_1", - "asin": "B07L3H5DDW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1325, - "sourceId": "30", - "displayId": "30", - "asin": "B003K5S4NW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1326, - "sourceId": "1_1", - "displayId": "1_1", - "asin": "B0BNVHXLDN", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1328, - "sourceId": "2", - "displayId": "2", - "asin": "B0FC2R299V", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1329, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0BNVJPSDT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1330, - "sourceId": "5", - "displayId": "5", - "asin": "B0CCRNRHYW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1331, - "sourceId": "6_1", - "displayId": "6_1", - "asin": "B0FP2NQM67", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1334, - "sourceId": "7", - "displayId": "7", - "asin": "B0DRSD66T3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1335, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0DM6CS4KG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1336, - "sourceId": "9_1", - "displayId": "9_1", - "asin": "B0GQ2YGFTR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1338, - "sourceId": "10", - "displayId": "10", - "asin": "B0F1LNXRSH", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1339, - "sourceId": "11", - "displayId": "11", - "asin": "B0GJ59NQTT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1340, - "sourceId": "12", - "displayId": "12", - "asin": "B0GKY1T54N", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1341, - "sourceId": "13", - "displayId": "13", - "asin": "B0FZNMHLDY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1342, - "sourceId": "14_1", - "displayId": "14_1", - "asin": "B0DHWYL3JP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1344, - "sourceId": "16_1", - "displayId": "16_1", - "asin": "B0CW5DD6SC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1351, - "sourceId": "17_1", - "displayId": "17_1", - "asin": "B0CRH7T5WG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1353, - "sourceId": "19_1", - "displayId": "19_1", - "asin": "B0CRR6F9JX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1354, - "sourceId": "20", - "displayId": "20", - "asin": "B09D3XG1PT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1355, - "sourceId": "22", - "displayId": "22", - "asin": "B00AFGCL42", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1356, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B0CXP3619X", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1357, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0CW5GFQX3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1363, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0CZDK561H", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1364, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B0FKLRMG14", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1373, - "sourceId": "28_1", - "displayId": "28_1", - "asin": "B0DFQWC297", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1383, - "sourceId": "1_1", - "displayId": "1_1", - "asin": "B0DGPV9QY8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1386, - "sourceId": "2_1", - "displayId": "2_1", - "asin": "B0GV3W42MJ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1388, - "sourceId": "3_1", - "displayId": "3_1", - "asin": "B0DK4JV5YY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1402, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0CV4K3HSW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1403, - "sourceId": "9", - "displayId": "9", - "asin": "B0FN5YD344", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1404, - "sourceId": "10", - "displayId": "10", - "asin": "B01HY4XZE2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1407, - "sourceId": "13", - "displayId": "13", - "asin": "B0D485JG3T", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1408, - "sourceId": "15", - "displayId": "15", - "asin": "B0GC584NTL", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1409, - "sourceId": "16", - "displayId": "16", - "asin": "B0FDQLKJL3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1410, - "sourceId": "18_1", - "displayId": "18_1", - "asin": "B0D6RBQR4Q", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1418, - "sourceId": "19_1", - "displayId": "19_1", - "asin": "B0CSLMZ4XS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1419, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0FC64KYP9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1421, - "sourceId": "21_1", - "displayId": "21_1", - "asin": "B0FH234TL6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1445, - "sourceId": "22_1", - "displayId": "22_1", - "asin": "B0F6D4Z3WB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1455, - "sourceId": "23_1", - "displayId": "23_1", - "asin": "B0F8RCZ91V", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1465, - "sourceId": "24_1", - "displayId": "24_1", - "asin": "B0FP5Q7B7X", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1476, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0FM88JBRX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1477, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0CKMT72MX", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1486, - "sourceId": "27", - "displayId": "27", - "asin": "B0C4JT7PHB", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1487, - "sourceId": "28", - "displayId": "28", - "asin": "B0C1RMVGTZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1488, - "sourceId": "29", - "displayId": "29", - "asin": "B07BFQJ3G5", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1489, - "sourceId": "30", - "displayId": "30", - "asin": "B074VGHYF8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1490, - "sourceId": "1", - "displayId": "1", - "asin": "B015AK9JXI", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1491, - "sourceId": "2_1", - "displayId": "2_1", - "asin": "B084WSGN9C", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1492, - "sourceId": "3_1", - "displayId": "3_1", - "asin": "B0FHWJ42Q8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1497, - "sourceId": "4_1", - "displayId": "4_1", - "asin": "B0CZDTGFSQ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1498, - "sourceId": "5_1", - "displayId": "5_1", - "asin": "B0F7XKNY9L", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1499, - "sourceId": "6", - "displayId": "6", - "asin": "B0DWLFKPNW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1500, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B0CWDPSJG4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1512, - "sourceId": "8_1", - "displayId": "8_1", - "asin": "B0G488Z8JG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1514, - "sourceId": "9_1", - "displayId": "9_1", - "asin": "B0BYRQ5TBY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1515, - "sourceId": "10", - "displayId": "10", - "asin": "B0CWH1MRCV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1516, - "sourceId": "11", - "displayId": "11", - "asin": "B0FSCXLBHZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1517, - "sourceId": "12", - "displayId": "12", - "asin": "B0GH6YJX2L", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1518, - "sourceId": "13_1", - "displayId": "13_1", - "asin": "B0GMQ29YQ9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1519, - "sourceId": "14_1", - "displayId": "14_1", - "asin": "B0BQLX4QP3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1521, - "sourceId": "15", - "displayId": "15", - "asin": "B0FXVHQNQ4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1522, - "sourceId": "16", - "displayId": "16", - "asin": "B072DVNZYH", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1523, - "sourceId": "17_1", - "displayId": "17_1", - "asin": "B0G198MZH3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1524, - "sourceId": "18", - "displayId": "18", - "asin": "B0F7HGQHLP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1525, - "sourceId": "19", - "displayId": "19", - "asin": "B0B9M83MZV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1526, - "sourceId": "20", - "displayId": "20", - "asin": "B074P9YRGW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1527, - "sourceId": "21_1", - "displayId": "21_1", - "asin": "B0G3YJ32Q4", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1528, - "sourceId": "22", - "displayId": "22", - "asin": "B0D8RXPCH6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1529, - "sourceId": "24", - "displayId": "24", - "asin": "B0CBPKKT98", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1530, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0DGG82L25", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1531, - "sourceId": "26", - "displayId": "26", - "asin": "B0FXVNDFHC", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1532, - "sourceId": "27", - "displayId": "27", - "asin": "B0CYY2YJ2S", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1533, - "sourceId": "28", - "displayId": "28", - "asin": "B0FC34Z9P6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1534, - "sourceId": "29", - "displayId": "29", - "asin": "B0018MQIVO", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1535, - "sourceId": "30", - "displayId": "30", - "asin": "B018OQ66AE", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1536, - "sourceId": "1", - "displayId": "1", - "asin": "B0FNMK6R7Z", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1537, - "sourceId": "3", - "displayId": "3", - "asin": "B0CTJ8WRWZ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1538, - "sourceId": "4", - "displayId": "4", - "asin": "B0CTJ9522Y", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1539, - "sourceId": "5", - "displayId": "5", - "asin": "B0CTJ88CKR", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1540, - "sourceId": "6", - "displayId": "6", - "asin": "B07DL3PBL7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1541, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B0C96V7HFN", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1543, - "sourceId": "8", - "displayId": "8", - "asin": "B079HQKBP3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1544, - "sourceId": "9", - "displayId": "9", - "asin": "B0BZHW8VQ6", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1545, - "sourceId": "10_1", - "displayId": "10_1", - "asin": "B019RKRGWS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1547, - "sourceId": "11", - "displayId": "11", - "asin": "B0DCLSFGQS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1548, - "sourceId": "12", - "displayId": "12", - "asin": "B07FL8Y4BP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1549, - "sourceId": "14", - "displayId": "14", - "asin": "B01NBVCL86", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1550, - "sourceId": "15_1", - "displayId": "15_1", - "asin": "B0C8J5Z7VN", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1551, - "sourceId": "16", - "displayId": "16", - "asin": "B0CP73M8ST", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1552, - "sourceId": "18", - "displayId": "18", - "asin": "B0G2XVTF2R", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1553, - "sourceId": "19_1", - "displayId": "19_1", - "asin": "B08GCVHM9S", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1555, - "sourceId": "20_1", - "displayId": "20_1", - "asin": "B0B5X3YGMP", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1557, - "sourceId": "22", - "displayId": "22", - "asin": "B07VM7QS8C", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1558, - "sourceId": "23", - "displayId": "23", - "asin": "B0F93G7KFF", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1559, - "sourceId": "24", - "displayId": "24", - "asin": "B098B8QTM1", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1560, - "sourceId": "25_1", - "displayId": "25_1", - "asin": "B0F4WSK73F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1561, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0DC5M4XT7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1564, - "sourceId": "27", - "displayId": "27", - "asin": "B07SD5DWFT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1565, - "sourceId": "28", - "displayId": "28", - "asin": "B082ZNZG3X", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1566, - "sourceId": "29", - "displayId": "29", - "asin": "B084M4ZGC3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1567, - "sourceId": "30", - "displayId": "30", - "asin": "B0FXH7SVWV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1568, - "sourceId": "3", - "displayId": "3", - "asin": "B07RM4MB3H", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1569, - "sourceId": "4", - "displayId": "4", - "asin": "B0154H395G", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1570, - "sourceId": "5", - "displayId": "5", - "asin": "B08CHFDRJT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1571, - "sourceId": "6", - "displayId": "6", - "asin": "B09YQ3D7CW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1572, - "sourceId": "7_1", - "displayId": "7_1", - "asin": "B087YTMRK8", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1574, - "sourceId": "8", - "displayId": "8", - "asin": "B08WKDDV96", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1575, - "sourceId": "9", - "displayId": "9", - "asin": "B0GFMG4SFT", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1576, - "sourceId": "10", - "displayId": "10", - "asin": "B07KWY3XLW", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1577, - "sourceId": "11", - "displayId": "11", - "asin": "B07RQTKPSK", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1578, - "sourceId": "12", - "displayId": "12", - "asin": "B0CRHC42M2", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1579, - "sourceId": "14", - "displayId": "14", - "asin": "B07RGG2TCV", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1580, - "sourceId": "15", - "displayId": "15", - "asin": "B0CTJNQ5MH", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1581, - "sourceId": "16", - "displayId": "16", - "asin": "B0GGZSVY7G", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1582, - "sourceId": "17", - "displayId": "17", - "asin": "B0G3WTPGMG", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1583, - "sourceId": "18", - "displayId": "18", - "asin": "B0GH136M59", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1584, - "sourceId": "19", - "displayId": "19", - "asin": "B0G3WJP9HJ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1585, - "sourceId": "20", - "displayId": "20", - "asin": "B0814RZNHY", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1586, - "sourceId": "21", - "displayId": "21", - "asin": "B09ZXZH432", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1587, - "sourceId": "22_1", - "displayId": "22_1", - "asin": "B0BWY99XXS", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1589, - "sourceId": "23", - "displayId": "23", - "asin": "B0849Y6VYJ", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1590, - "sourceId": "24", - "displayId": "24", - "asin": "B0FKBBNN2G", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1591, - "sourceId": "25", - "displayId": "25", - "asin": "B07SBWB2X9", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1592, - "sourceId": "26_1", - "displayId": "26_1", - "asin": "B0DK4JNBM3", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1593, - "sourceId": "27_1", - "displayId": "27_1", - "asin": "B0FFZ3636F", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1594, - "sourceId": "29", - "displayId": "29", - "asin": "B0BRGZKXN7", - "country": "英国", - "url": "", - "title": "" - }, - { - "rowIndex": 1602, - "sourceId": "2", - "displayId": "2", - "asin": "B0GSTM85H3", - "country": "英国", - "url": "", - "title": "" - } - ] - }, - "code": None -} - spide.process_task(task_data) \ No newline at end of file +if __name__ == '__main__': + spide = SpiderTask() + task_data = { + "data": { + "taskId": 1, + "groups": [ + { + "sourceFileKey": "", + "sourceFilename": "", + "groupKey": "", + "baseId": "", + "displayId": "", + "items": [ + { + "sourceFileKey": "", + "sourceFilename": "", + "rowToken": "", + "groupKey": "", + "id": "1", + "asin": "B0D792ND9V", + "country": "英国", + "url": "", + "title": "低" + } + ] + } + ], + "limit": None + }, + "code": None + } + spide.process_task(task_data) diff --git a/app/amazon/price_match.py b/app/amazon/price_match.py index ebdaa18..b836edb 100644 --- a/app/amazon/price_match.py +++ b/app/amazon/price_match.py @@ -18,7 +18,7 @@ from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE class RepricingLogic: - + @staticmethod def situation_1_general_decrease(my_price, rank1_price, is_first_time=False): """ @@ -27,9 +27,9 @@ class RepricingLogic: """ if is_first_time: return rank1_price - 0.3 - - diff = abs(my_price - rank1_price) - + + diff = my_price - rank1_price + # 精简了原图中长串的阶梯逻辑 if diff <= 0.3: return rank1_price - 0.5 @@ -47,63 +47,63 @@ class RepricingLogic: 情况2：已经是自己购物车 逻辑：判断与第二名的差价，如果拉开足够差距，则稍微降一点点(减0.5)保持优势，防止卖太便宜。 """ - diff = abs(rank2_price - my_price) - + diff = rank2_price - my_price + # 1. 绝对差价2欧以上 if diff >= 2.0: return rank2_price - 0.3 - + # 2. 产品售价区间判定 (是否要提高价格) if (20 <= my_price < 30 and diff >= 4) or \ - (30 <= my_price < 60 and diff >= 8) or \ - (60 <= my_price <= 150 and diff >= 15): + (30 <= my_price < 60 and diff >= 8) or \ + (60 <= my_price <= 150 and diff >= 15): return rank2_price - 0.3 # 满足区间大差价，提价至第二名之下 - - return my_price # 不满足条件则保持原价 + + return my_price # 不满足条件则保持原价 @staticmethod def situation_3_not_own_buybox(my_price, rank1_price): """ 情况3：不是自己购物车 (常规情况) """ - diff = abs(my_price - rank1_price) - + diff = my_price - rank1_price + if diff <= 0.3: return rank1_price - 0.5 elif 0.5 <= diff <= 0.8: return rank1_price - 0.7 elif 0.8 < diff <= 2.5: return rank1_price - 1.0 - elif diff > 2.5: # 2.5-3.5以上跳过 - return None - - return rank1_price - 0.3 # 默认按第一名减0.3 + elif diff > 2.5: # 2.5-3.5以上跳过 + return None + + return rank1_price - 0.3 # 默认按第一名减0.3 @staticmethod def situation_4_encounter_amz_us_1st(my_price, amz_us_price): """ 情况4：第一名是 Amazon US 卖家 """ - diff = abs(my_price - amz_us_price) - + diff = my_price - amz_us_price + if diff <= 3: return amz_us_price - 2 elif 4 <= diff <= 6: return amz_us_price - 3 elif 8 <= diff <= 12: - return None # 跳过不跟价 - - return amz_us_price - 3 # 默认直接按照 Amazon US 减去3 + return None # 跳过不跟价 + + return amz_us_price - 3 # 默认直接按照 Amazon US 减去3 @staticmethod def situation_5_im_1st_amz_us_2nd(my_price, amz_us_price): """ 情况5：自己是第一名，第二名是 Amazon US 卖家 """ - diff = abs(amz_us_price - my_price) - + diff = amz_us_price - my_price + if diff <= 7: - return None # 相差5以内跳过 (保持原价) + return None # 相差5以内跳过 (保持原价) else: return amz_us_price - 5 @@ -127,7 +127,12 @@ def calculate_target_price( # --- Step 2: 紫鸟后台特殊判定 (最高优先级) --- if recommended_shipping > 0: price_1 = float(front_end_data.get("top_sellers")[0].get("price")) # 实际第一名价格 - return RepricingLogic.situation_1_general_decrease(my_price,price_1,is_my_buybox) + if is_my_buybox: + price_col = price_1 + else: + price_col = recommended_price + recommended_shipping + print(f"【紫鸟后台价格】{is_my_buybox} 推荐价格: {recommended_price}，推荐运费: {recommended_shipping}，总和: {price_col}") + return RepricingLogic.situation_1_general_decrease(my_price,price_col,is_my_buybox) # backend_base_price = recommended_price # backend_shipping = recommended_shipping # total_price = backend_base_price + backend_shipping @@ -151,7 +156,7 @@ def calculate_target_price( # 分支 A：第一名是 Amazon US 卖家 (特殊强敌优先) price_1 = float(front_end_data.get("top_sellers")[0].get("price")) # 实际第一名价格 if "Amazon" in shop_name_1: - return RepricingLogic.isituation_4_encounter_amz_us_1st(my_price,price_1) + return RepricingLogic.situation_4_encounter_amz_us_1st(my_price,price_1) # 分支 B：不是 Amazon US，且目前是自己的购物车 @@ -619,6 +624,11 @@ class AmzonePriceMatch(AmamzonBase): page_pamel = self.tab.eles('xpath://kat-pagination', timeout=5) if len(page_pamel) > 0: current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text + #测试===== + # if int(current_page) > 3: + # break + # 测试=========== + # yield (None,{"page":int(current_page.strip())}) # 总页数 total_page = page_pamel[0].sr.eles( @@ -653,9 +663,9 @@ class AmzonePriceMatch(AmamzonBase): 'xpath:.//div[contains(@class,"JanusSplitBox-module__container")]//div[contains(@class,"JanusSplitBox-module__panel--") and contains(string(.),"ASIN")]/..//div[last()]', timeout=3).text print(f"【{self.mark_name}】ASIN {asin} 找到....") - if asin in already_asin: - print(f"【{self.mark_name}】{asin} 已经处理过了，跳过") - continue + # if asin in already_asin: + # print(f"【{self.mark_name}】{asin} 已经处理过了，跳过") + # continue if asin in skip_asin: yield (asin, { "statu": "跳过,需要跳过的ASIN" @@ -779,7 +789,8 @@ class AmzonePriceMatch(AmamzonBase): "secondPlace": price_2, "cartShopName" : cartShopName, "shippingFee" : f"{shipping_fee}", - "priceChangeStatus" : "改价成功", + "priceChangeStatus" : f"{adjust_prices}", + "modifyCount" : "1", }) else: recommendedPrice = recommend_price + shipping_fee @@ -797,7 +808,8 @@ class AmzonePriceMatch(AmamzonBase): "secondPlace": price_2, "cartShopName": cartShopName, "shippingFee": f"{shipping_fee}", - "priceChangeStatus": "跳过，无需改价", + "priceChangeStatus": "", + "modifyCount": "0", }) already_asin.add(asin) @@ -1322,30 +1334,35 @@ class PriceTask: url = f"{DELETE_BRAND_API_BASE}/api/price-track/tasks/{task_id}/result" + country_aliases = {name: code for code, name in self.country_info.items()} + country_key = country_aliases.get(str(country_code).strip(), str(country_code).strip()) + + countries = {} + if asin or status: + countries[country_key] = [ + { + "shopMallName": shopMallName, + "asin": asin, + "price": status.get("currentPrice") if status.get("currentPrice") else "", + "recommendedPrice": status.get("recommendedPrice") if status.get("recommendedPrice") else "", + "minimumPrice": status.get("minimumPrice") if status.get("minimumPrice") else "", + "firstPlace": status.get("firstPlace") if status.get("firstPlace") else "", + "secondPlace": status.get("secondPlace") if status.get("secondPlace") else "", + "cartShopName": status.get("cartShopName") if status.get("cartShopName") else "", + "priceChangeStatus": status.get("priceChangeStatus") if status.get("priceChangeStatus") else "", + "deleteSkipAsin": status.get("deleteSkipAsin", False), + "removeAsin": status.get("removeAsin") if status.get("removeAsin") else "", + "modifyCount": status.get("modifyCount") if status.get("modifyCount") else "", + "shippingFee": status.get("shippingFee") if status.get("shippingFee") else "", + "status": status.get("statu") + } + ] + payload = { "shops": [ { "shopName": shop_name, - "countries": { - "additionalProperties1": [ - { - "shopMallName": shopMallName, - "asin": asin, - "price": status.get("currentPrice") if status.get("currentPrice") else "", - "recommendedPrice": status.get("recommendedPrice") if status.get("recommendedPrice") else "", - "minimumPrice": status.get("minimumPrice") if status.get("minimumPrice") else "", - "firstPlace": status.get("firstPlace") if status.get("firstPlace") else "", - "secondPlace": status.get("secondPlace") if status.get("secondPlace") else "", - "cartShopName": status.get("cartShopName") if status.get("cartShopName") else "", - "priceChangeStatus": "UPDATED", - "deleteSkipAsin": status.get("deleteSkipAsin",False), - "removeAsin": status.get("removeAsin") if status.get("removeAsin") else "", - # "modifyCount": "2", - "shippingFee" : status.get("shippingFee") if status.get("shippingFee") else "", - "status": status.get("statu") - } - ] - }, + "countries": countries, "error": "" } ] @@ -1391,11 +1408,11 @@ class PriceTask: if __name__ == '__main__': # 使用示例 # user_info = { - # "company": "尾号5578的公司115", + # "company": "rongchuang123", # "username": "自动化_Robot", # "password": "#20zsg25" # } - # shop_name = "刘建煌" + # shop_name = "郭亚芳" # country = "德国" # kill_process('v6') # driver = AmzonePriceMatch(user_info) @@ -1403,8 +1420,8 @@ if __name__ == '__main__': # sw_suc = driver.SwitchingCountries(country) # driver.SwitchPage() # risk_listing_filter = "Active" - # _shopMallName = "Jianhuang 888" - # ap_asin ="B0BGHRP6BS" + # _shopMallName = "yafang123" + # ap_asin ="B0BTHBJDKG" # skip_asin = [] # for _ in range(3): # try: @@ -1422,12 +1439,14 @@ if __name__ == '__main__': # print(f"ASIN {asin} 的处理结果: {status}") # print("已完成操作") - front_end_data = {"top_sellers": [{"rank": 1, "price": "36.81", "stock": "5", "shop_name": "Windera"}, {"rank": 2, "price": "36.50", "stock": "100", "shop_name": "Jianhuang 888"}], "cart_seller": "Windera", "timestamp": "2026-04-25 09:32:58"} - current_Price = 36.50 - current_shop_name = "Jianhuang 888" - recommended_price = 36.81 - recommended_shipping = 0 - calculate_target_price( + front_end_data = {'top_sellers': [{'rank': 1, 'price': '50.24', 'stock': '26', 'shop_name': 'yafang123'}, {'rank': 2, 'price': '44.35', 'stock': '100', 'shop_name': 'QinPimy'}], 'cart_seller': 'Amazon US', 'timestamp': '2026-04-27 16:17:42'} + + current_Price = 44.35 + current_shop_name = "yafang123" + recommended_price = 44.35 + recommended_shipping =0.0 + res = calculate_target_price( front_end_data, current_Price, current_shop_name, recommended_price, recommended_shipping - ) \ No newline at end of file + ) + print(res) diff --git a/app/blueprints/__pycache__/admin.cpython-39.pyc b/app/blueprints/__pycache__/admin.cpython-39.pyc index 0421db9..4dbaa56 100644 Binary files a/app/blueprints/__pycache__/admin.cpython-39.pyc and b/app/blueprints/__pycache__/admin.cpython-39.pyc differ diff --git a/app/blueprints/__pycache__/brand.cpython-39.pyc b/app/blueprints/__pycache__/brand.cpython-39.pyc index e12fac5..c155f6a 100644 Binary files a/app/blueprints/__pycache__/brand.cpython-39.pyc and b/app/blueprints/__pycache__/brand.cpython-39.pyc differ diff --git a/app/blueprints/__pycache__/main.cpython-312.pyc b/app/blueprints/__pycache__/main.cpython-312.pyc index 61aa0d3..fe704e0 100644 Binary files a/app/blueprints/__pycache__/main.cpython-312.pyc and b/app/blueprints/__pycache__/main.cpython-312.pyc differ diff --git a/app/blueprints/__pycache__/main.cpython-39.pyc b/app/blueprints/__pycache__/main.cpython-39.pyc index eabc04d..dce162a 100644 Binary files a/app/blueprints/__pycache__/main.cpython-39.pyc and b/app/blueprints/__pycache__/main.cpython-39.pyc differ diff --git a/app/blueprints/main.py b/app/blueprints/main.py index 85cfd05..6881dc0 100644 --- a/app/blueprints/main.py +++ b/app/blueprints/main.py @@ -1,8 +1,8 @@ """ 主页面蓝图：首页、home、图片工作台、品牌页、静态文件、Logo """ -import os -from flask import Blueprint, send_file, render_template_string +import os +from flask import Blueprint, send_file, render_template_string from app_common import ( get_db, @@ -18,8 +18,8 @@ from flask import redirect, url_for, session from flask import Flask, request, Response, stream_with_context import requests -from config import base_url,version,JAVA_API_BASE - +from config import base_url,version,JAVA_API_BASE + main_bp = Blueprint('main', __name__) @@ -130,9 +130,9 @@ def brand_page_legacy(): '
', '
', 1, - ) - content = content.replace('height: calc(100vh - 56px);', 'height: 100vh;', 1) - return render_template_string(content, user_id=session.get('user_id')) + ) + content = content.replace('height: calc(100vh - 56px);', 'height: 100vh;', 1) + return render_template_string(content, user_id=session.get('user_id')) @@ -147,7 +147,7 @@ def serve_static(filename): return '', 404 return send_file(file_abs, as_attachment=False) - + @main_bp.route('/assets/') def serve_assets(filename): """提供 static 目录及子目录下的静态文件访问。""" @@ -167,25 +167,18 @@ def serve_assets(filename): return '', 404 -@main_bp.route('/new_web_source/') -def serve_new_web_source(filename): - """提供 new_web_source 目录下的静态页面文件访问。""" - candidate_dirs = [ - os.path.abspath(os.path.join(BASE_DIR, 'new_web_source')), - os.path.abspath(os.path.join(BASE_DIR, '..', 'new_web_source')), - ] - for static_abs in candidate_dirs: - filepath = os.path.normpath(os.path.join(static_abs, filename)) - file_abs = os.path.abspath(filepath) - if not file_abs.startswith(static_abs): - continue - if os.path.isfile(file_abs): - return send_file(file_abs, as_attachment=False) - return '', 404 - - -@main_bp.route('/logo.jpg', methods=['GET']) -def get_logo_image(): +@main_bp.route('/new_web_source/') +def serve_new_web_source(filename): + """提供 static 目录及子目录下的静态文件访问。""" + filepath = os.path.normpath(os.path.join("new_web_source", filename)) + static_abs = os.path.abspath("new_web_source") + file_abs = os.path.abspath(filepath) + if not file_abs.startswith(static_abs) or not os.path.isfile(file_abs): + return '', 404 + return send_file(file_abs, as_attachment=False) + +@main_bp.route('/logo.jpg', methods=['GET']) +def get_logo_image(): return send_file(os.path.join(BASE_DIR, "logo.jpg"), mimetype='image/jpeg') @@ -253,4 +246,4 @@ def proxy(path): return Response("无法连接到后端服务", status=502) except Exception as e: print(f"代理请求失败: {e}") - return Response(f"代理错误: {str(e)}", status=500) + return Response(f"代理错误: {str(e)}", status=500) diff --git a/app/brand_spider/__pycache__/main.cpython-39.pyc b/app/brand_spider/__pycache__/main.cpython-39.pyc index 4b1ab72..a7113dc 100644 Binary files a/app/brand_spider/__pycache__/main.cpython-39.pyc and b/app/brand_spider/__pycache__/main.cpython-39.pyc differ diff --git a/app/config.py b/app/config.py index db6b0e8..0fbff9e 100644 --- a/app/config.py +++ b/app/config.py @@ -51,7 +51,7 @@ os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6" -debug = False +debug = os.getenv("debug", "false").strip().lower() in ("1", "true", "yes", "on") version = "1.0.56" APP_UPDATE_URL = f"{_base_url}/api/version/latest" # 检测更新接口地址，返回 { "file_url": "...", "version": "x.x.x" } os.environ['APP_VERSION'] = version diff --git a/app/main.py b/app/main.py index be4fba7..1f0b379 100644 --- a/app/main.py +++ b/app/main.py @@ -50,8 +50,8 @@ class WindowAPI: def __init__(self, window): self._window = window self._is_maximized = False - window.events.maximized += lambda _: setattr(self, '_is_maximized', True) - window.events.restored += lambda _: setattr(self, '_is_maximized', False) + window.events.maximized += lambda *args: setattr(self, '_is_maximized', True) + window.events.restored += lambda *args: setattr(self, '_is_maximized', False) def close(self): """关闭窗口，异步执行清理逻辑""" diff --git a/app/tool/__pycache__/devices.cpython-39.pyc b/app/tool/__pycache__/devices.cpython-39.pyc index bf96713..2663452 100644 Binary files a/app/tool/__pycache__/devices.cpython-39.pyc and b/app/tool/__pycache__/devices.cpython-39.pyc differ diff --git a/backend-java/pom.xml b/backend-java/pom.xml index 71e4523..2fab57a 100644 --- a/backend-java/pom.xml +++ b/backend-java/pom.xml @@ -23,6 +23,8 @@ 4.0.3 3.17.4 5.8.36 + 2.3.5 + 8.5.17 @@ -81,6 +83,16 @@ hutool-all ${hutool.version} + + org.apache.rocketmq + rocketmq-spring-boot-starter + ${rocketmq-spring.version} + + + io.minio + minio + ${minio.version} + org.projectlombok lombok diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/exception/GlobalExceptionHandler.java b/backend-java/src/main/java/com/nanri/aiimage/common/exception/GlobalExceptionHandler.java index 73d65c6..ae7caf4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/common/exception/GlobalExceptionHandler.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/exception/GlobalExceptionHandler.java @@ -13,6 +13,9 @@ public class GlobalExceptionHandler { @ExceptionHandler(BusinessException.class) public ApiResponse handleBusinessException(BusinessException ex) { + if (Integer.valueOf(40901).equals(ex.getCode())) { + return ApiResponse.success("任务已结束，忽略重复提交", null); + } return ex.getCode() == null ? ApiResponse.fail(ex.getMessage()) : ApiResponse.fail(ex.getCode(), ex.getMessage()); diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/service/DistributedJobLockService.java b/backend-java/src/main/java/com/nanri/aiimage/common/service/DistributedJobLockService.java index 5ae7736..40f1772 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/common/service/DistributedJobLockService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/common/service/DistributedJobLockService.java @@ -36,7 +36,13 @@ public class DistributedJobLockService { Duration actualTtl = (ttl == null || ttl.isNegative() || ttl.isZero()) ? Duration.ofMinutes(10) : ttl; String key = buildLockKey(jobName); String token = ownerPrefix + ":" + UUID.randomUUID(); - Boolean locked = stringRedisTemplate.opsForValue().setIfAbsent(key, token, actualTtl); + Boolean locked; + try { + locked = stringRedisTemplate.opsForValue().setIfAbsent(key, token, actualTtl); + } catch (Exception ex) { + log.warn("[job-lock] acquire skipped jobName={} key={} msg={}", jobName, key, ex.getMessage()); + return null; + } if (!Boolean.TRUE.equals(locked)) { return null; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java new file mode 100644 index 0000000..6ab2db0 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java @@ -0,0 +1,21 @@ +package com.nanri.aiimage.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "aiimage.appearance-patent") +public class AppearancePatentProperties { + private String cozeBaseUrl = "https://api.coze.cn"; + private String cozeWorkflowPath = "/v1/workflow/run"; + private String cozeWorkflowHistoryPath = "/v1/workflows/{workflow_id}/run_histories/{execute_id}"; + private String cozeWorkflowId = "7632683471312355338"; + private String cozeToken = ""; + private int cozeBatchSize = 10; + private int cozeConnectTimeoutMillis = 10000; + private int cozeReadTimeoutMillis = 60000; + private int cozePollIntervalMillis = 2000; + private int cozePollTimeoutMillis = 120000; + private int staleTimeoutMinutes = 20; + private String staleFinalizeCron = "0 */2 * * * *"; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/DeleteBrandProgressProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/DeleteBrandProgressProperties.java index 3e17430..7c79101 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/DeleteBrandProgressProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/DeleteBrandProgressProperties.java @@ -7,6 +7,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "aiimage.delete-brand-progress") public class DeleteBrandProgressProperties { private long heartbeatTimeoutMinutes = 15; + private long deleteBrandInitialTimeoutMinutes = 20; private String staleCheckCron = "*/30 * * * * *"; /** diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java index 6a48a49..5d1e0d5 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java @@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Configuration; @Configuration -@EnableConfigurationProperties({OssProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class}) +@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class}) public class PropertiesConfig { } diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java new file mode 100644 index 0000000..6cee12c --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java @@ -0,0 +1,27 @@ +package com.nanri.aiimage.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.TaskExecutor; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +@Configuration +public class TaskFileJobConfig { + + @Bean("taskFileJobDispatchExecutor") + public TaskExecutor taskFileJobDispatchExecutor( + @Value("${aiimage.result-file-job.local-dispatch-pool-size:2}") int poolSize, + @Value("${aiimage.result-file-job.local-dispatch-queue-capacity:200}") int queueCapacity) { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + int normalizedPoolSize = Math.max(1, poolSize); + executor.setCorePoolSize(normalizedPoolSize); + executor.setMaxPoolSize(normalizedPoolSize); + executor.setQueueCapacity(Math.max(10, queueCapacity)); + executor.setThreadNamePrefix("task-file-job-dispatch-"); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(30); + executor.initialize(); + return executor; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/TransientStorageProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/TransientStorageProperties.java new file mode 100644 index 0000000..8458f3e --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/config/TransientStorageProperties.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "aiimage.transient-storage") +public class TransientStorageProperties { + private boolean enabled = false; + private String endpoint; + private String bucket; + private String accessKeyId; + private String accessKeySecret; + private String region = "us-east-1"; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClient.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClient.java new file mode 100644 index 0000000..12cd99d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClient.java @@ -0,0 +1,746 @@ +package com.nanri.aiimage.modules.appearancepatent.client; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.config.AppearancePatentProperties; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.MediaType; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.stereotype.Component; +import org.springframework.util.StreamUtils; +import org.springframework.web.client.RestClient; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +@Component +@RequiredArgsConstructor +@Slf4j +public class AppearancePatentCozeClient { + + private final AppearancePatentProperties properties; + private final ObjectMapper objectMapper; + + public List inspect(List rows, String prompt) { + if (rows == null || rows.isEmpty()) { + return List.of(); + } + if (properties.getCozeToken() == null || properties.getCozeToken().isBlank()) { + log.warn("[appearance-patent] coze token not configured, keep raw rows size={}", rows.size()); + return rows.stream().map(this::copy).toList(); + } + try { + return inspectWithFallback(rows, prompt); + } catch (Exception ex) { + String failureMessage = failureMessage(ex); + log.warn("[appearance-patent] coze batch failed size={} err={}", rows.size(), failureMessage); + return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList(); + } + } + + private List inspectWithFallback(List rows, String prompt) { + try { + if (rows.size() == 1) { + return inspectSingleRowWithRetry(rows, prompt); + } + InspectAttempt attempt = inspectOnce(rows, prompt); + if (attempt.resolvedCount() < rows.size()) { + throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount()); + } + return attempt.mergedRows(); + } catch (Exception ex) { + if (shouldSplitBatch(rows, ex)) { + int middle = rows.size() / 2; + log.warn("[appearance-patent] coze batch fallback split size={} left={} right={} err={}", + rows.size(), middle, rows.size() - middle, failureMessage(ex)); + List merged = new ArrayList<>(rows.size()); + merged.addAll(inspectWithFallback(rows.subList(0, middle), prompt)); + merged.addAll(inspectWithFallback(rows.subList(middle, rows.size()), prompt)); + return merged; + } + throw propagate(ex); + } + } + + private List inspectSingleRowWithRetry(List rows, String prompt) throws Exception { + AppearancePatentResultRowDto row = rows.getFirst(); + PartialCozeResultException lastFailure = null; + for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) { + try { + InspectAttempt attempt = inspectOnce(rows, prompt); + if (attempt.resolvedCount() == rows.size()) { + return attempt.mergedRows(); + } + log.warn("[appearance-patent] coze single unresolved attempt={} rowId={} asin={} country={} title={} url={} raw={}", + attemptIndex, + row.getId(), + row.getAsin(), + row.getCountry(), + abbreviate(row.getTitle(), 120), + abbreviate(row.getUrl(), 120), + abbreviate(attempt.raw(), 500)); + lastFailure = new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount()); + } catch (Exception ex) { + if (attemptIndex >= 3 || !isRetryableBatchFailure(ex)) { + throw ex; + } + log.warn("[appearance-patent] coze single retryable failure attempt={} rowId={} asin={} country={} err={}", + attemptIndex, + row.getId(), + row.getAsin(), + row.getCountry(), + failureMessage(ex)); + } + if (attemptIndex < 3) { + sleepBeforeRetry(attemptIndex); + } + } + throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure; + } + + private InspectAttempt inspectOnce(List rows, String prompt) throws Exception { + String raw = runWorkflowAsyncAndWait(rows, prompt); + List results = parseResults(raw); + if (rows.size() > 1 && !results.isEmpty() && results.stream().noneMatch(this::hasIdentity)) { + throw new PartialCozeResultException(0, rows.size(), results.size()); + } + List merged = mergeRows(rows, results); + return new InspectAttempt(raw, merged, resolvedCount(merged), results.size()); + } + + private String runWorkflowAsyncAndWait(List rows, String prompt) throws Exception { + JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt)); + ensureSuccess(submitRoot); + + String immediateData = extractResultDataText(submitRoot); + if (!immediateData.isBlank()) { + return wrapDataPayload(immediateData); + } + + String executeId = extractExecuteId(submitRoot); + if (executeId == null || executeId.isBlank()) { + throw new IllegalStateException("Coze async execute_id missing"); + } + + long deadline = System.currentTimeMillis() + Math.max(10000, properties.getCozePollTimeoutMillis()); + while (System.currentTimeMillis() < deadline) { + JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId)); + ensureSuccess(pollRoot); + + String dataText = extractResultDataText(pollRoot); + if (!dataText.isBlank()) { + return wrapDataPayload(dataText); + } + + String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT); + if (status.contains("FAIL") || status.contains("ERROR") || status.contains("CANCEL")) { + throw new IllegalStateException(firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed")); + } + sleepQuietly(Math.max(200, properties.getCozePollIntervalMillis())); + } + + throw new IllegalStateException("Coze async workflow poll timeout"); + } + + private String postWorkflow(List rows, String prompt) { + Map parameters = buildParameters(rows, prompt); + Map body = new LinkedHashMap<>(); + body.put("workflow_id", properties.getCozeWorkflowId()); + body.put("parameters", parameters); + body.put("is_async", Boolean.TRUE); + + RestClient.RequestBodySpec request = restClient().post() + .uri(joinUrl(properties.getCozeBaseUrl(), properties.getCozeWorkflowPath())) + .headers(headers -> { + headers.setBearerAuth(stripBearer(properties.getCozeToken())); + headers.setContentType(MediaType.APPLICATION_JSON); + }); + request.body(body); + return request.exchange((clientRequest, clientResponse) -> { + byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody()); + return responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8); + }); + } + + private String getWorkflowHistory(String executeId) { + String path = properties.getCozeWorkflowHistoryPath() + .replace("{workflow_id}", properties.getCozeWorkflowId()) + .replace("{execute_id}", executeId); + return restClient().get() + .uri(joinUrl(properties.getCozeBaseUrl(), path)) + .headers(headers -> { + headers.setBearerAuth(stripBearer(properties.getCozeToken())); + headers.setContentType(MediaType.APPLICATION_JSON); + }) + .exchange((clientRequest, clientResponse) -> { + byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody()); + return responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8); + }); + } + + private Map buildParameters(List rows, String prompt) { + List groupKeys = rows.stream().map(row -> nonBlank(row.getGroupKey(), rowKey(row))).toList(); + List rowIds = rows.stream().map(row -> nonBlank(row.getId(), "")).toList(); + List asins = rows.stream().map(row -> nonBlank(row.getAsin(), "")).toList(); + List countries = rows.stream().map(row -> nonBlank(row.getCountry(), "")).toList(); + List titles = rows.stream().map(row -> nonBlank(row.getTitle(), row.getAsin())).toList(); + List urls = rows.stream().map(row -> nonBlank(row.getUrl(), "")).toList(); + + Map parameters = new LinkedHashMap<>(); + parameters.put("title_list", titles); + parameters.put("url_list", urls); + parameters.put("items", buildItemObjects(rows, groupKeys, rowIds, asins, countries, titles, urls)); + parameters.put("prompt", prompt == null ? "" : prompt); + return parameters; + } + + private List> buildItemObjects(List rows, + List groupKeys, + List rowIds, + List asins, + List countries, + List titles, + List urls) { + List> items = new ArrayList<>(rows.size()); + for (int i = 0; i < rows.size(); i++) { + Map item = new LinkedHashMap<>(); + item.put("group_key", groupKeys.get(i)); + item.put("row_id", rowIds.get(i)); + item.put("asin", asins.get(i)); + item.put("country", countries.get(i)); + item.put("title", titles.get(i)); + item.put("url", urls.get(i)); + items.add(item); + } + return items; + } + + private List parseResults(String raw) throws Exception { + JsonNode root = objectMapper.readTree(raw); + ensureSuccess(root); + String dataText = extractResultDataText(root); + if (dataText.isBlank()) { + return List.of(); + } + JsonNode dataRoot = objectMapper.readTree(dataText); + JsonNode array = dataRoot.path("data"); + List results = new ArrayList<>(); + if (array.isArray()) { + for (JsonNode node : array) { + JsonNode itemNode = resultItemNode(node); + results.add(new CozeResult( + text(firstNonNull( + firstNonNull(node.get("group_key"), node.get("groupKey")), + firstNonNull(itemNode.get("group_key"), itemNode.get("groupKey")))), + text(firstNonNull( + firstNonNull(node.get("row_id"), firstNonNull(node.get("rowId"), node.get("id"))), + firstNonNull(itemNode.get("row_id"), firstNonNull(itemNode.get("rowId"), itemNode.get("id"))))), + text(firstNonNull(node.get("asin"), itemNode.get("asin"))), + text(firstNonNull( + firstNonNull(node.get("country"), node.get("site")), + firstNonNull(itemNode.get("country"), itemNode.get("site")))), + text(node.get("title")), + text(node.get("appearance")), + text(firstNonNull(node.get("patent"), node.get("patent "))), + text(node.get("result")) + )); + } + } + return results; + } + + private JsonNode resultItemNode(JsonNode node) { + if (node == null || node.isMissingNode() || node.isNull()) { + return objectMapper.missingNode(); + } + JsonNode item = firstNonNull(node.get("item"), node.get("items")); + if (item == null || item.isMissingNode() || item.isNull()) { + return objectMapper.missingNode(); + } + if (item.isArray()) { + return item.isEmpty() ? objectMapper.missingNode() : item.get(0); + } + return item; + } + + private List mergeRows(List rows, List results) { + Map resultByGroupKey = new LinkedHashMap<>(); + Map resultByCompositeKey = new LinkedHashMap<>(); + Map resultByAsinCountry = new LinkedHashMap<>(); + Map resultByAsin = new LinkedHashMap<>(); + Map resultByRowId = new LinkedHashMap<>(); + for (CozeResult result : results) { + String groupKey = normalize(result.groupKey()); + if (!groupKey.isBlank()) { + resultByGroupKey.putIfAbsent(groupKey, result); + } + String compositeKey = rowKey(result.rowId(), result.asin(), result.country()); + if (!compositeKey.isBlank()) { + resultByCompositeKey.putIfAbsent(compositeKey, result); + } + String asinCountryKey = asinCountryKey(result.asin(), result.country()); + if (!asinCountryKey.isBlank()) { + resultByAsinCountry.putIfAbsent(asinCountryKey, result); + } + String asinKey = normalize(result.asin()).toUpperCase(Locale.ROOT); + if (!asinKey.isBlank()) { + resultByAsin.putIfAbsent(asinKey, result); + } + String rowIdKey = normalize(result.rowId()); + if (!rowIdKey.isBlank()) { + resultByRowId.putIfAbsent(rowIdKey, result); + } + } + + List merged = new ArrayList<>(rows.size()); + boolean allowIndexFallback = results.size() == rows.size() && results.stream().noneMatch(this::hasIdentity); + for (int i = 0; i < rows.size(); i++) { + AppearancePatentResultRowDto row = copy(rows.get(i)); + CozeResult result = resultByGroupKey.get(normalize(row.getGroupKey())); + if (result == null) { + result = resultByCompositeKey.get(rowKey(row.getId(), row.getAsin(), row.getCountry())); + } + if (result == null) { + result = resultByAsinCountry.get(asinCountryKey(row.getAsin(), row.getCountry())); + } + if (result == null) { + result = resultByAsin.get(normalize(row.getAsin()).toUpperCase(Locale.ROOT)); + } + if (result == null) { + result = resultByRowId.get(normalize(row.getId())); + } + if (result == null && allowIndexFallback && i < results.size()) { + result = results.get(i); + } + applyResult(row, result); + merged.add(row); + } + return merged; + } + + private String asinCountryKey(String asin, String country) { + String normalizedAsin = normalize(asin).toUpperCase(Locale.ROOT); + if (normalizedAsin.isBlank()) { + return ""; + } + return normalizedAsin + "::" + normalize(country); + } + + private boolean hasIdentity(CozeResult result) { + if (result == null) { + return false; + } + return !normalize(result.groupKey()).isBlank() + || !normalize(result.rowId()).isBlank() + || !normalize(result.asin()).isBlank(); + } + + private void applyResult(AppearancePatentResultRowDto row, CozeResult result) { + if (row == null || result == null) { + return; + } + row.setTitleRisk(result.title()); + row.setAppearanceRisk(result.appearance()); + row.setPatentRisk(result.patent()); + row.setConclusion(result.result()); + } + + private RestClient restClient() { + SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); + requestFactory.setConnectTimeout(properties.getCozeConnectTimeoutMillis()); + requestFactory.setReadTimeout(properties.getCozeReadTimeoutMillis()); + return RestClient.builder().requestFactory(requestFactory).build(); + } + + private AppearancePatentResultRowDto copy(AppearancePatentResultRowDto source) { + AppearancePatentResultRowDto row = new AppearancePatentResultRowDto(); + row.setSourceFileKey(source.getSourceFileKey()); + row.setSourceFilename(source.getSourceFilename()); + row.setRowToken(source.getRowToken()); + row.setGroupKey(source.getGroupKey()); + row.setId(source.getId()); + row.setAsin(source.getAsin()); + row.setCountry(source.getCountry()); + row.setUrl(source.getUrl()); + row.setTitle(source.getTitle()); + row.setError(source.getError()); + row.setDone(source.getDone()); + row.setTitleRisk(source.getTitleRisk()); + row.setAppearanceRisk(source.getAppearanceRisk()); + row.setPatentRisk(source.getPatentRisk()); + row.setConclusion(source.getConclusion()); + return row; + } + + private AppearancePatentResultRowDto markFailed(AppearancePatentResultRowDto row, String failureMessage) { + if (row.getError() == null || row.getError().isBlank()) { + row.setError(failureMessage); + } + if (row.getConclusion() == null || row.getConclusion().isBlank()) { + row.setConclusion(failureMessage); + } + return row; + } + + private boolean shouldSplitBatch(List rows, Exception ex) { + return rows != null && rows.size() > 1 && isRetryableBatchFailure(ex); + } + + private boolean isRetryableBatchFailure(Exception ex) { + if (ex instanceof PartialCozeResultException) { + return true; + } + String message = ex == null ? "" : nonBlank(ex.getMessage(), ""); + return message.contains("Workflow node execution limit exceeded") + || message.contains("Read timed out") + || message.contains("Connection reset") + || message.contains("I/O error on POST request") + || message.toLowerCase(Locale.ROOT).contains("timeout"); + } + + private int resolvedCount(List rows) { + int resolved = 0; + for (AppearancePatentResultRowDto row : rows) { + if (hasResolvedCozeFields(row)) { + resolved++; + } + } + return resolved; + } + + private boolean hasResolvedCozeFields(AppearancePatentResultRowDto row) { + if (row == null) { + return false; + } + return !normalize(row.getTitleRisk()).isBlank() + || !normalize(row.getAppearanceRisk()).isBlank() + || !normalize(row.getPatentRisk()).isBlank() + || !normalize(row.getConclusion()).isBlank(); + } + + private void ensureSuccess(JsonNode root) { + if (root.path("code").asInt(-1) != 0) { + throw new IllegalStateException(root.path("msg").asText("Coze response code is not 0")); + } + } + + private String extractResultDataText(JsonNode root) { + if (root == null || root.isMissingNode() || root.isNull()) { + return ""; + } + + JsonNode dataNode = root.path("data"); + if (dataNode.isTextual()) { + String value = dataNode.asText(""); + if (looksLikeResultDataPayload(value)) { + return value; + } + return discoverEmbeddedData(parseJsonOrMissing(value)); + } + if (dataNode.isObject()) { + String nested = text(firstNonNull(dataNode.get("data"), firstNonNull(dataNode.get("output"), dataNode.get("result")))); + if (nested != null && !nested.isBlank() && looksLikeResultDataPayload(nested)) { + return nested; + } + JsonNode outputs = firstNonNull(dataNode.get("outputs"), dataNode.get("details")); + String discovered = discoverEmbeddedData(outputs); + if (!discovered.isBlank()) { + return discovered; + } + } + + return discoverEmbeddedData(root); + } + + private String discoverEmbeddedData(JsonNode node) { + if (node == null || node.isNull() || node.isMissingNode()) { + return ""; + } + if (node.isTextual()) { + String value = node.asText(""); + if (looksLikeResultDataPayload(value)) { + return value; + } + return discoverEmbeddedData(parseJsonOrMissing(value)); + } + if (node.isArray()) { + for (JsonNode child : node) { + String discovered = discoverEmbeddedData(child); + if (!discovered.isBlank()) { + return discovered; + } + } + return ""; + } + if (node.isObject()) { + if (isResultDataPayload(node)) { + return node.toString(); + } + for (java.util.Iterator> it = node.fields(); it.hasNext(); ) { + Map.Entry entry = it.next(); + String discovered = discoverEmbeddedData(entry.getValue()); + if (!discovered.isBlank()) { + return discovered; + } + } + } + return ""; + } + + private boolean looksLikeResultDataPayload(String value) { + JsonNode parsed = parseJsonOrMissing(value); + return isResultDataPayload(parsed); + } + + private boolean isResultDataPayload(JsonNode node) { + if (node == null || node.isMissingNode() || node.isNull()) { + return false; + } + JsonNode array = node.path("data"); + if (!array.isArray()) { + return false; + } + for (JsonNode item : array) { + if (item.has("appearance") || item.has("patent") || item.has("patent ") || item.has("result")) { + return true; + } + } + return false; + } + + private JsonNode parseJsonOrMissing(String value) { + String normalized = normalize(value); + if (!(normalized.startsWith("{") || normalized.startsWith("["))) { + return objectMapper.missingNode(); + } + try { + return objectMapper.readTree(normalized); + } catch (Exception ignored) { + return objectMapper.missingNode(); + } + } + + private String resolveWorkflowStatus(JsonNode root) { + JsonNode dataNode = root.path("data"); + JsonNode statusNode = firstNonNull( + firstNonNull(dataNode.get("status"), dataNode.get("execute_status")), + firstNonNull(root.get("status"), root.get("execute_status"))); + String status = text(statusNode); + if (status != null && !status.isBlank()) { + return status; + } + return findTextByFieldName(root, "execute_status", "status"); + } + + private String resolveFailureMessage(JsonNode root) { + JsonNode dataNode = root.path("data"); + String message = text(firstNonNull(dataNode.get("error_message"), firstNonNull(dataNode.get("msg"), root.get("msg")))); + return message == null ? "" : message; + } + + private String extractExecuteId(JsonNode root) { + return findTextByFieldName(root, "execute_id", "executeId"); + } + + private String findTextByFieldName(JsonNode node, String... names) { + if (node == null || node.isMissingNode() || node.isNull()) { + return ""; + } + if (node.isTextual()) { + return findTextByFieldName(parseJsonOrMissing(node.asText("")), names); + } + if (node.isArray()) { + for (JsonNode child : node) { + String found = findTextByFieldName(child, names); + if (!found.isBlank()) { + return found; + } + } + return ""; + } + if (node.isObject()) { + for (String name : names) { + String value = text(node.get(name)); + if (value != null && !value.isBlank()) { + return value; + } + } + JsonNode dataNode = node.get("data"); + if (dataNode != null && dataNode.isTextual()) { + String found = findTextByFieldName(parseJsonOrMissing(dataNode.asText("")), names); + if (!found.isBlank()) { + return found; + } + } + for (java.util.Iterator> it = node.fields(); it.hasNext(); ) { + Map.Entry entry = it.next(); + String found = findTextByFieldName(entry.getValue(), names); + if (!found.isBlank()) { + return found; + } + } + } + return ""; + } + + private String wrapDataPayload(String dataText) { + Map payload = new LinkedHashMap<>(); + payload.put("code", 0); + payload.put("data", dataText); + return writeJson(payload); + } + + private String writeJson(Object value) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new IllegalStateException("Failed to serialize Coze payload", ex); + } + } + + private String abbreviate(String value, int maxLength) { + String normalized = value == null ? "" : value.trim(); + if (normalized.length() <= maxLength) { + return normalized; + } + return normalized.substring(0, Math.max(0, maxLength - 3)) + "..."; + } + + private void sleepBeforeRetry(int attemptIndex) { + long delayMillis = Math.max(1, attemptIndex) * 1500L; + sleepQuietly(delayMillis); + } + + private void sleepQuietly(long delayMillis) { + try { + Thread.sleep(delayMillis); + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + } + } + + private RuntimeException propagate(Exception ex) { + if (ex instanceof RuntimeException runtimeException) { + return runtimeException; + } + return new IllegalStateException(nonBlank(ex.getMessage(), "Coze call failed"), ex); + } + + private JsonNode firstNonNull(JsonNode left, JsonNode right) { + return left == null || left.isNull() ? right : left; + } + + private String text(JsonNode node) { + return node == null || node.isNull() ? null : node.asText(); + } + + private String nonBlank(String value, String fallback) { + return value == null || value.isBlank() ? fallback : value; + } + + private String firstNonBlank(String preferred, String fallback) { + return preferred == null || preferred.isBlank() ? fallback : preferred.trim(); + } + + private String normalize(String value) { + return value == null ? "" : value.replace("\ufeff", "").replace("\u3000", " ").trim(); + } + + private String rowKey(AppearancePatentResultRowDto row) { + if (row == null) { + return ""; + } + return rowKey(row.getId(), row.getAsin(), row.getCountry()); + } + + private String rowKey(String rowId, String asin, String country) { + return normalize(rowId) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country); + } + + private String failureMessage(Exception ex) { + if (ex instanceof PartialCozeResultException partial) { + return "Coze result incomplete(" + partial.resolvedCount() + "/" + partial.expectedCount() + ")"; + } + String message = ex == null ? null : ex.getMessage(); + if (message == null || message.isBlank()) { + return "Coze call failed"; + } + if (message.contains("Workflow node execution limit exceeded")) { + return "Coze workflow node execution limit exceeded"; + } + if (message.contains("Read timed out")) { + return "Coze call timed out"; + } + return "Coze call failed: " + message; + } + + private String stripBearer(String token) { + String normalized = token == null ? "" : token.trim(); + return normalized.regionMatches(true, 0, "Bearer ", 0, 7) ? normalized.substring(7).trim() : normalized; + } + + private String joinUrl(String baseUrl, String path) { + String base = baseUrl == null ? "" : baseUrl.trim(); + String suffix = path == null ? "" : path.trim(); + if (base.endsWith("/") && suffix.startsWith("/")) { + return base + suffix.substring(1); + } + if (!base.endsWith("/") && !suffix.startsWith("/")) { + return base + "/" + suffix; + } + return base + suffix; + } + + private record CozeResult( + String groupKey, + String rowId, + String asin, + String country, + String title, + String appearance, + String patent, + String result + ) { + } + + private record InspectAttempt( + String raw, + List mergedRows, + int resolvedCount, + int rawResultCount + ) { + } + + private static final class PartialCozeResultException extends RuntimeException { + + private final int resolvedCount; + private final int expectedCount; + private final int rawResultCount; + + private PartialCozeResultException(int resolvedCount, int expectedCount, int rawResultCount) { + super("partial-result resolved=" + resolvedCount + "/" + expectedCount + " raw=" + rawResultCount); + this.resolvedCount = resolvedCount; + this.expectedCount = expectedCount; + this.rawResultCount = rawResultCount; + } + + private int resolvedCount() { + return resolvedCount; + } + + private int expectedCount() { + return expectedCount; + } + + @SuppressWarnings("unused") + private int rawResultCount() { + return rawResultCount; + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java new file mode 100644 index 0000000..8a7157a --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java @@ -0,0 +1,143 @@ +package com.nanri.aiimage.modules.appearancepatent.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentTaskBatchRequest; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashboardVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParseVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskBatchVo; +import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +import java.io.InputStream; +import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/appearance-patent") +@Tag(name = "外观专利检测", description = "外观专利检测任务接口。前端上传 Excel 后由 Java 解析并创建任务；Python 回传商品数据；Java 负责攒批调用 Coze、补齐子行、生成最终 xlsx 并上传 OSS。") +public class AppearancePatentController { + + private final AppearancePatentTaskService service; + + @PostMapping("/parse") + @Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件，提取 id、ASIN、国家、URL、标题等字段。返回给前端的数据只包含整数 id 和 n_1 行；n_2、n_3 等子行会保存在 OSS 解析载荷中，用于最终结果补齐。创建后的任务状态为 PENDING，不会自动推送 Python。") + public ApiResponse parse(@Valid @RequestBody AppearancePatentParseRequest request) { + return ApiResponse.success(service.parseAndCreateTask(request)); + } + + @GetMapping("/dashboard") + @Operation(summary = "查询外观专利检测总览", description = "查询当前用户的运行中、成功、失败和已结束任务数量。页面进入时请求一次即可，不需要持续轮询。") + public ApiResponse dashboard( + @Parameter(description = "当前用户 ID，用于隔离不同用户的任务和历史记录。", required = true, example = "1") + @RequestParam("user_id") Long userId) { + return ApiResponse.success(service.dashboard(userId)); + } + + @GetMapping("/history") + @Operation(summary = "查询外观专利检测历史", description = "查询当前用户最近的外观专利检测历史记录，包含源文件名、任务状态、行数、错误信息和最终 xlsx 下载地址。") + public ApiResponse history( + @Parameter(description = "当前用户 ID，用于查询该用户自己的历史记录。", required = true, example = "1") + @RequestParam("user_id") Long userId) { + return ApiResponse.success(service.history(userId)); + } + + @PostMapping("/tasks/progress/batch") + @Operation(summary = "批量查询任务进度", description = "前端只对活跃任务调用该接口，建议 6 秒一次。接口只返回轻量任务状态，不返回明细结果。") + public ApiResponse progress(@Valid @RequestBody AppearancePatentTaskBatchRequest request) { + return ApiResponse.success(service.progressBatch(request.getTaskIds())); + } + + @PostMapping("/tasks/{taskId}/activate") + @Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用，将任务从 PENDING 改为 RUNNING，并记录后端内部活跃时间。后续活跃时间由 Python 回传结果接口自动刷新，不需要单独心跳接口。") + public ApiResponse activate( + @Parameter(description = "外观专利检测任务 ID，即解析接口返回的 taskId。", required = true, example = "3938") + @PathVariable Long taskId, + @Parameter(description = "当前用户 ID，必须与创建任务的用户一致。", required = true, example = "1") + @RequestParam("user_id") Long userId) { + service.activateTask(taskId, userId); + return ApiResponse.success(null); + } + + @PostMapping("/tasks/{taskId}/result") + @Operation(summary = "提交 Python 回传结果", description = "Python 回传商品数据接口。items 可以是一条或多条；Java 先原样保存回传数据，再内部攒够 10 条调用 Coze。done=true 表示 Python 已完成全部回传，Java 会强制处理剩余不足 10 条的数据并生成最终 xlsx。") + public ApiResponse result( + @Parameter(description = "外观专利检测任务 ID。任务必须处于 RUNNING 状态。", required = true, example = "3938") + @PathVariable Long taskId, + @Valid @RequestBody AppearancePatentSubmitResultRequest request) { + service.submitResult(taskId, request); + return ApiResponse.success(null); + } + + @DeleteMapping("/tasks/{taskId}") + @Operation(summary = "删除任务", description = "删除当前用户的一条外观专利检测任务，同时清理任务结果、scope 状态和分片记录。") + public ApiResponse deleteTask( + @Parameter(description = "外观专利检测任务 ID。", required = true, example = "3938") + @PathVariable Long taskId, + @Parameter(description = "当前用户 ID，必须与创建任务的用户一致。", required = true, example = "1") + @RequestParam("user_id") Long userId) { + service.deleteTask(taskId, userId); + return ApiResponse.success(null); + } + + @DeleteMapping("/history/{resultId}") + @Operation(summary = "删除历史记录", description = "删除当前用户的一条外观专利检测历史记录。只删除 biz_file_result 记录，不主动删除任务主记录。") + public ApiResponse deleteHistory( + @Parameter(description = "历史结果记录 ID，即 history 接口返回的 resultId。", required = true, example = "1001") + @PathVariable Long resultId, + @Parameter(description = "当前用户 ID。", required = true, example = "1") + @RequestParam("user_id") Long userId) { + service.deleteHistory(resultId, userId); + return ApiResponse.success(null); + } + + @GetMapping("/results/{resultId}/download") + @Operation(summary = "下载外观专利检测结果文件") + public void downloadResult( + @Parameter(description = "结果记录 ID", required = true, example = "1001") + @PathVariable Long resultId, + @Parameter(description = "当前用户 ID", required = true, example = "1") + @RequestParam("user_id") Long userId, + jakarta.servlet.http.HttpServletResponse response) { + String url = service.resolveResultDownloadUrl(resultId, userId); + String filename = service.resolveResultDownloadFilename(resultId, userId); + if (url == null || url.isBlank()) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果"); + } + try { + String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20"); + response.setContentType("application/octet-stream"); + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, + "attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename); + try (InputStream in = URI.create(url).toURL().openStream()) { + byte[] buffer = new byte[65536]; + int read; + while ((read = in.read(buffer)) != -1) { + response.getOutputStream().write(buffer, 0, read); + } + response.getOutputStream().flush(); + } + } catch (Exception ex) { + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败"); + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParseRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParseRequest.java new file mode 100644 index 0000000..71fd5a1 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParseRequest.java @@ -0,0 +1,28 @@ +package com.nanri.aiimage.modules.appearancepatent.model.dto; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "外观专利检测解析请求") +public class AppearancePatentParseRequest { + @JsonProperty("user_id") + @NotNull + @Schema(description = "当前用户 ID。后端会把创建的任务、历史记录和结果文件归属到该用户。", example = "1", requiredMode = Schema.RequiredMode.REQUIRED) + private Long userId; + + @NotEmpty + @Schema(description = "已上传的 Excel 文件列表。支持多文件聚合解析；文件对象来自统一上传接口返回值。", requiredMode = Schema.RequiredMode.REQUIRED) + private List files; + + @JsonProperty("ai_prompt") + @JsonAlias({"aiPrompt", "prompt"}) + @Schema(description = "AI 提示词。非必填；为空时前端会使用默认提示词。后端会保存该提示词，并在调用 Coze workflow 时作为 prompt 参数传入。", example = "请排查这些亚马逊商品在英国及欧洲地区是否存在知识产权侵权风险。") + private String aiPrompt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParsedPayloadDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParsedPayloadDto.java new file mode 100644 index 0000000..95abf6f --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParsedPayloadDto.java @@ -0,0 +1,31 @@ +package com.nanri.aiimage.modules.appearancepatent.model.dto; + +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "外观专利解析载荷") +public class AppearancePatentParsedPayloadDto { + @Schema(description = "AI 提示词") + private String aiPrompt; + + @Schema(description = "本次解析的源文件列表") + private List sourceFiles = new ArrayList<>(); + + @Schema(description = "Excel 原始表头列表") + private List headers = new ArrayList<>(); + + @Schema(description = "兼容旧链路的平铺有效行，现为全部有效行") + private List items = new ArrayList<>(); + + @Schema(description = "按相邻主 ID 块分组后的完整数据") + private List groups = new ArrayList<>(); + + @Schema(description = "完整有效行") + private List allItems = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultGroupDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultGroupDto.java new file mode 100644 index 0000000..7423e8d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultGroupDto.java @@ -0,0 +1,29 @@ +package com.nanri.aiimage.modules.appearancepatent.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "Python 回传的外观专利分组结果") +public class AppearancePatentResultGroupDto { + @Schema(description = "来源文件 key") + private String sourceFileKey; + + @Schema(description = "来源文件名") + private String sourceFilename; + + @Schema(description = "相邻主 ID 块的分组 key") + private String groupKey; + + @Schema(description = "主 ID，例如 1、2、10") + private String baseId; + + @Schema(description = "分组首条展示 ID") + private String displayId; + + @Schema(description = "分组内全部抓取结果") + private List items = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultRowDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultRowDto.java new file mode 100644 index 0000000..10d40a0 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultRowDto.java @@ -0,0 +1,55 @@ +package com.nanri.aiimage.modules.appearancepatent.model.dto; + +import com.fasterxml.jackson.annotation.JsonAlias; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "外观专利检测单行商品数据") +public class AppearancePatentResultRowDto { + @Schema(description = "来源文件 key。多文件回传时用于避免行结果串到别的文件。", example = "uploads/20260426/appearance_patent_17.xlsx") + private String sourceFileKey; + + @Schema(description = "来源文件名。仅用于调试与追踪。", example = "17.xlsx") + private String sourceFilename; + + @Schema(description = "行级唯一 token。Python 应从解析结果原样透传。", example = "uploads/20260426/appearance_patent_17.xlsx::row::2") + private String rowToken; + + @Schema(description = "主数据分组 key。Python 应从解析结果原样透传，用于把同组子行补回。", example = "uploads/20260426/appearance_patent_17.xlsx::2@2") + private String groupKey; + + @Schema(description = "Excel 中的 id。代表行通常是整数 id 或 n_1，例如 2_1；最终生成 xlsx 时，2_2、2_3 会复用同组 2_1 的 Coze 检测结果。", example = "2_1") + private String id; + + @Schema(description = "亚马逊 ASIN。后端会统一按大写处理和匹配。", example = "B0CJ8SNXXV") + private String asin; + + @Schema(description = "站点或国家。来自 Excel 的国家列，例如英国、德国、法国。", example = "英国") + private String country; + + @Schema(description = "商品主图或待检测图片 URL。Java 调用 Coze 时会放入 url_list。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg") + private String url; + + @Schema(description = "商品标题。Java 调用 Coze 时会放入 title_list；为空时会回退使用 ASIN。", example = "Women Floral Dress Summer Casual") + private String title; + + @Schema(description = "单行错误信息。通常用于记录 Python 单行处理异常；Coze 失败时后端会尽量保留原始行，不强行写入风险结果。", example = "图片地址为空") + private String error; + + @Schema(description = "单行完成标记。当前主要使用请求体顶层 done 控制任务收尾，该字段仅作兼容。", example = "true") + private Boolean done; + + @Schema(description = "Java 调用 Coze 后生成的标题维度检测结果，对应最终 xlsx 的“标题维度（商标）”列。Python 回传请求中不要传该字段；即使传入，后端也会以 Java/Coze 处理结果为准。", example = "标题未发现明显商标侵权风险。", accessMode = Schema.AccessMode.READ_ONLY) + private String titleRisk; + + @Schema(description = "Java 调用 Coze 后生成的外观维度检测结果，对应最终 xlsx 的“外观维度（外观设计专利）”列。Python 回传请求中不要传该字段。", example = "未发现明显外观设计专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY) + private String appearanceRisk; + + @JsonAlias({"patent ", "patent"}) + @Schema(description = "Java 调用 Coze 后生成的专利维度检测结果，对应最终 xlsx 的“专利维度（发明/实用新型专利）”列。兼容 Coze 返回字段 patent 和 patent 后带空格的情况；Python 回传请求中不要传该字段。", example = "未发现明显发明或实用新型专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY) + private String patentRisk; + + @Schema(description = "Java 调用 Coze 后生成的最终结论，对应最终 xlsx 的“结论”列。Python 回传请求中不要传该字段。", example = "未发现明显侵权风险", accessMode = Schema.AccessMode.READ_ONLY) + private String conclusion; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentSourceFileDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentSourceFileDto.java new file mode 100644 index 0000000..8a92056 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentSourceFileDto.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.appearancepatent.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "外观专利检测源文件信息") +public class AppearancePatentSourceFileDto { + @Schema(description = "上传接口返回的临时文件 key。后端根据该 key 查找本地临时 Excel 文件并解析。", example = "uploads/20260426/appearance_patent_17.xlsx", requiredMode = Schema.RequiredMode.REQUIRED) + private String fileKey; + @Schema(description = "原始文件名。用于历史记录展示和最终结果文件命名。", example = "17.xlsx") + private String originalFilename; + @Schema(description = "相对目录路径。当前仅记录来源，外观专利检测不依赖该字段处理。", example = "xlsx/17.xlsx") + private String relativePath; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentSubmitResultRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentSubmitResultRequest.java new file mode 100644 index 0000000..5417196 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentSubmitResultRequest.java @@ -0,0 +1,32 @@ +package com.nanri.aiimage.modules.appearancepatent.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "Python 回传外观专利检测结果请求") +public class AppearancePatentSubmitResultRequest { + @Schema(description = "本次 Python 回传的提交批次标识", example = "appearance-patent-3938") + private String submissionId; + + @Schema(description = "当前回传分片序号", example = "1") + private Integer chunkIndex; + + @Schema(description = "本任务预计总分片数", example = "36") + private Integer chunkTotal; + + @Schema(description = "是否为最后一次回传", example = "false") + private Boolean done; + + @Schema(description = "Python 侧任务级错误信息", example = "浏览器执行异常，任务提前结束") + private String error; + + @Schema(description = "本次回传的分组结果列表，推荐优先使用") + private List groups = new ArrayList<>(); + + @Schema(description = "兼容旧链路的平铺结果列表") + private List items = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentTaskBatchRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentTaskBatchRequest.java new file mode 100644 index 0000000..799b1f8 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentTaskBatchRequest.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.appearancepatent.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "外观专利检测批量进度查询请求") +public class AppearancePatentTaskBatchRequest { + @NotEmpty + @Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务；后端会批量查询，避免每个任务单独请求。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED) + private List taskIds; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentDashboardVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentDashboardVo.java new file mode 100644 index 0000000..c8c2254 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentDashboardVo.java @@ -0,0 +1,17 @@ +package com.nanri.aiimage.modules.appearancepatent.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "外观专利检测总览统计") +public class AppearancePatentDashboardVo { + @Schema(description = "运行中任务数量。这里统计 RUNNING 状态任务。", example = "1") + private Long pendingTaskCount; + @Schema(description = "已结束任务数量，等于成功任务数加失败任务数。", example = "12") + private Long processedTaskCount; + @Schema(description = "成功任务数量。", example = "10") + private Long successTaskCount; + @Schema(description = "失败任务数量。", example = "2") + private Long failedTaskCount; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryItemVo.java new file mode 100644 index 0000000..cdd9e99 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryItemVo.java @@ -0,0 +1,33 @@ +package com.nanri.aiimage.modules.appearancepatent.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "外观专利检测历史记录项") +public class AppearancePatentHistoryItemVo { + @Schema(description = "结果记录 ID。删除历史、下载结果时使用。", example = "1001") + private Long resultId; + @Schema(description = "任务 ID。", example = "3938") + private Long taskId; + @Schema(description = "源 Excel 文件名。", example = "17.xlsx") + private String sourceFilename; + @Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx") + private String resultFilename; + @Schema(description = "最终结果文件下载地址。后端基于 OSS objectKey 生成的新鲜预签名 URL。", example = "https://bucket.oss-cn-hangzhou.aliyuncs.com/result/appearance_patent/xxx/17-result.xlsx?Expires=...") + private String downloadUrl; + private Long fileJobId; + private String fileStatus; + private String fileError; + private Boolean fileReady; + @Schema(description = "任务状态：PENDING=已解析待推送，RUNNING=执行中，SUCCESS=成功，FAILED=失败。", example = "SUCCESS") + private String taskStatus; + @Schema(description = "结果是否成功。true 表示任务完成并生成结果文件；false 表示失败或未完成。", example = "true") + private Boolean success; + @Schema(description = "错误信息。任务失败时返回，例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final appearance patent result") + private String error; + @Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716") + private Integer rowCount; + @Schema(description = "历史记录创建时间，ISO 本地时间字符串。", example = "2026-04-26T10:30:00") + private String createdAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryVo.java new file mode 100644 index 0000000..dfa788e --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryVo.java @@ -0,0 +1,14 @@ +package com.nanri.aiimage.modules.appearancepatent.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "外观专利检测历史记录列表") +public class AppearancePatentHistoryVo { + @Schema(description = "历史记录项列表，默认返回最近 100 条。") + private List items = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParseVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParseVo.java new file mode 100644 index 0000000..4ff3222 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParseVo.java @@ -0,0 +1,41 @@ +package com.nanri.aiimage.modules.appearancepatent.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "外观专利检测解析结果") +public class AppearancePatentParseVo { + @Schema(description = "新创建的任务 ID", example = "3938") + private Long taskId; + + @Schema(description = "来源 Excel 文件名，多个文件时为聚合展示文案", example = "17.xlsx 等 2 个文件") + private String sourceFilename; + + @Schema(description = "参与本次解析的源文件数量", example = "2") + private Integer sourceFileCount; + + @Schema(description = "Excel 中检测到的非空数据总行数", example = "716") + private Integer totalRows; + + @Schema(description = "解析出的有效行数", example = "458") + private Integer acceptedRows; + + @Schema(description = "因缺少必要字段而被丢弃的行数", example = "12") + private Integer droppedRows; + + @Schema(description = "按相邻主 ID 块生成的分组数", example = "36") + private Integer groupCount; + + @Schema(description = "本任务最终使用的 AI 提示词") + private String aiPrompt; + + @Schema(description = "兼容旧前端的平铺有效行列表，现为全部有效行") + private List items = new ArrayList<>(); + + @Schema(description = "返回前端和推送 Python 的分组列表") + private List groups = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParsedGroupVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParsedGroupVo.java new file mode 100644 index 0000000..032b020 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParsedGroupVo.java @@ -0,0 +1,32 @@ +package com.nanri.aiimage.modules.appearancepatent.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "外观专利解析后的分组数据") +public class AppearancePatentParsedGroupVo { + @Schema(description = "来源文件 key") + private String sourceFileKey; + + @Schema(description = "来源文件名") + private String sourceFilename; + + @Schema(description = "相邻主 ID 块的分组 key") + private String groupKey; + + @Schema(description = "主 ID，例如 1、2、10") + private String baseId; + + @Schema(description = "分组首条展示 ID，例如 1 或 2_1") + private String displayId; + + @Schema(description = "分组内行数") + private Integer itemCount; + + @Schema(description = "分组内全部行，顺序与原 Excel 保持一致") + private List items = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParsedRowVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParsedRowVo.java new file mode 100644 index 0000000..52418d3 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParsedRowVo.java @@ -0,0 +1,47 @@ +package com.nanri.aiimage.modules.appearancepatent.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.LinkedHashMap; +import java.util.Map; + +@Data +@Schema(description = "外观专利检测解析出的单行数据") +public class AppearancePatentParsedRowVo { + @Schema(description = "来源文件 key。用于多文件场景下区分同名/同 ID 行。", example = "uploads/20260426/appearance_patent_17.xlsx") + private String sourceFileKey; + + @Schema(description = "来源文件名。用于调试和结果追踪。", example = "17.xlsx") + private String sourceFilename; + + @Schema(description = "Excel 原始行号，从 1 开始。", example = "2") + private Integer rowIndex; + + @Schema(description = "Excel 中原始 id 值。", example = "2_1") + private String sourceId; + + @Schema(description = "前端展示和 Python 回传使用的 id。整数 id 原样保留，子数据第一条如 2_1 原样保留。", example = "2_1") + private String displayId; + + @Schema(description = "行级唯一 token。多文件、重复主数据时用于精确匹配回传结果。", example = "uploads/20260426/appearance_patent_17.xlsx::row::2") + private String rowToken; + + @Schema(description = "同一主数据块的分组 key。用于把 2_1、2_2、2_3 等子行重新补回同一组结果。", example = "uploads/20260426/appearance_patent_17.xlsx::2@2") + private String groupKey; + + @Schema(description = "亚马逊 ASIN。", example = "B0CJ8SNXXV") + private String asin; + + @Schema(description = "国家或站点。", example = "英国") + private String country; + + @Schema(description = "商品图片 URL 或商品 URL，供 Coze 检测使用。", example = "https://webstatic.aiproxy.vip/output/demo.jpg") + private String url; + + @Schema(description = "商品标题。", example = "Women Floral Dress Summer Casual") + private String title; + + @Schema(description = "该 Excel 行的原始列值映射。最终生成 xlsx 时可以从这里读取价格等字段。", example = "{\"id\":\"2_1\",\"asin\":\"B0CJ8SNXXV\",\"国家\":\"英国\",\"价格\":\"12.99\"}") + private Map values = new LinkedHashMap<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentTaskBatchVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentTaskBatchVo.java new file mode 100644 index 0000000..7dd79c1 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentTaskBatchVo.java @@ -0,0 +1,16 @@ +package com.nanri.aiimage.modules.appearancepatent.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "外观专利检测批量进度响应") +public class AppearancePatentTaskBatchVo { + @Schema(description = "查询到的任务详情列表。顺序按请求 taskIds 处理。") + private List items = new ArrayList<>(); + @Schema(description = "未找到或不属于外观专利检测模块的任务 ID 列表。", example = "[99999]") + private List missingTaskIds = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentTaskDetailVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentTaskDetailVo.java new file mode 100644 index 0000000..f71e369 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentTaskDetailVo.java @@ -0,0 +1,16 @@ +package com.nanri.aiimage.modules.appearancepatent.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Schema(description = "外观专利检测任务进度详情") +public class AppearancePatentTaskDetailVo { + @Schema(description = "任务主记录轻量信息。") + private AppearancePatentTaskItemVo task; + @Schema(description = "预留的任务明细列表。当前进度接口主要返回任务轻量状态，不返回完整结果明细。") + private List items = new ArrayList<>(); +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentTaskItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentTaskItemVo.java new file mode 100644 index 0000000..955213a --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentTaskItemVo.java @@ -0,0 +1,23 @@ +package com.nanri.aiimage.modules.appearancepatent.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "外观专利检测任务轻量信息") +public class AppearancePatentTaskItemVo { + @Schema(description = "任务 ID。", example = "3938") + private Long id; + @Schema(description = "任务编号，后端自动生成。", example = "APPEARANCE_PATENT-1780000000000000000") + private String taskNo; + @Schema(description = "任务状态：PENDING=已解析待推送，RUNNING=执行中，SUCCESS=成功，FAILED=失败。", example = "RUNNING") + private String status; + @Schema(description = "任务级错误信息。失败时返回。", example = "生成外观专利检测结果失败") + private String errorMessage; + @Schema(description = "创建时间，ISO 本地时间字符串。", example = "2026-04-26T10:00:00") + private String createdAt; + @Schema(description = "最后更新时间，通常由 Python 回传结果或任务收尾更新。", example = "2026-04-26T10:05:00") + private String updatedAt; + @Schema(description = "完成时间。任务未结束时为空。", example = "2026-04-26T10:10:00") + private String finishedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskCacheService.java new file mode 100644 index 0000000..cded051 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskCacheService.java @@ -0,0 +1,144 @@ +package com.nanri.aiimage.modules.appearancepatent.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +@Service +@RequiredArgsConstructor +@Slf4j +public class AppearancePatentTaskCacheService { + + private static final long TTL_HOURS = 24; + + private final StringRedisTemplate stringRedisTemplate; + private final ObjectMapper objectMapper; + + public void appendPendingRow(Long taskId, String scopeHash, Integer chunkIndex, AppearancePatentResultRowDto row) { + if (taskId == null || taskId <= 0 || scopeHash == null || scopeHash.isBlank() || chunkIndex == null || row == null) { + return; + } + try { + stringRedisTemplate.opsForList().rightPush( + pendingRowsKey(taskId), + objectMapper.writeValueAsString(new PendingRow(scopeHash, chunkIndex, row)) + ); + stringRedisTemplate.expire(pendingRowsKey(taskId), Duration.ofHours(TTL_HOURS)); + touchTaskHeartbeat(taskId); + } catch (Exception ignored) { + } + } + + public long pendingRowCount(Long taskId) { + Long size; + try { + size = stringRedisTemplate.opsForList().size(pendingRowsKey(taskId)); + } catch (Exception ex) { + log.warn("[appearance-patent-cache] pending row count degraded taskId={} msg={}", taskId, ex.getMessage()); + return 0L; + } + return size == null ? 0L : size; + } + + public List drainPendingRows(Long taskId, int limit) { + if (taskId == null || taskId <= 0 || limit <= 0) { + return List.of(); + } + String key = pendingRowsKey(taskId); + List values; + try { + values = stringRedisTemplate.opsForList().range(key, 0, limit - 1L); + } catch (Exception ex) { + log.warn("[appearance-patent-cache] drain range degraded taskId={} msg={}", taskId, ex.getMessage()); + return List.of(); + } + if (values == null || values.isEmpty()) { + return List.of(); + } + try { + stringRedisTemplate.opsForList().trim(key, values.size(), -1); + } catch (Exception ex) { + log.warn("[appearance-patent-cache] drain trim degraded taskId={} msg={}", taskId, ex.getMessage()); + return List.of(); + } + List rows = new ArrayList<>(); + for (String value : values) { + if (value == null || value.isBlank()) { + continue; + } + try { + rows.add(objectMapper.readValue(value, PendingRow.class)); + } catch (Exception ignored) { + } + } + touchTaskHeartbeat(taskId); + return rows; + } + + public void touchTaskHeartbeat(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + try { + stringRedisTemplate.opsForValue().set( + heartbeatKey(taskId), + String.valueOf(Instant.now().toEpochMilli()), + Duration.ofHours(TTL_HOURS) + ); + } catch (Exception ex) { + log.warn("[appearance-patent-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + } + } + + public long getTaskHeartbeatMillis(Long taskId) { + if (taskId == null || taskId <= 0) { + return 0L; + } + String raw; + try { + raw = stringRedisTemplate.opsForValue().get(heartbeatKey(taskId)); + } catch (Exception ex) { + log.warn("[appearance-patent-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + return 0L; + } + if (raw == null || raw.isBlank()) { + return 0L; + } + try { + return Long.parseLong(raw); + } catch (NumberFormatException ignored) { + return 0L; + } + } + + public void deleteTaskCache(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + try { + stringRedisTemplate.delete(pendingRowsKey(taskId)); + stringRedisTemplate.delete(heartbeatKey(taskId)); + } catch (Exception ex) { + log.warn("[appearance-patent-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage()); + } + } + + private String pendingRowsKey(Long taskId) { + return "appearance-patent:task:pending-rows:" + taskId; + } + + private String heartbeatKey(Long taskId) { + return "appearance-patent:task:heartbeat:" + taskId; + } + + public record PendingRow(String scopeHash, Integer chunkIndex, AppearancePatentResultRowDto row) { + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java new file mode 100644 index 0000000..9d751c2 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -0,0 +1,1608 @@ +package com.nanri.aiimage.modules.appearancepatent.service; + +import cn.hutool.core.util.IdUtil; +import cn.hutool.crypto.digest.DigestUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.AppearancePatentProperties; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentCozeClient; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultGroupDto; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSourceFileDto; +import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashboardVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryItemVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParseVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskBatchVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskDetailVo; +import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskItemVo; +import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; + +@Service +@RequiredArgsConstructor +@Slf4j +public class AppearancePatentTaskService { + + public static final String MODULE_TYPE = "APPEARANCE_PATENT"; + private static final String STATUS_PENDING = "PENDING"; + private static final String STATUS_RUNNING = "RUNNING"; + private static final String STATUS_SUCCESS = "SUCCESS"; + private static final String STATUS_FAILED = "FAILED"; + private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + private static final List RESULT_HEADERS = List.of( + "id", + "asin", + "国家", + "价格", + "标题维度（商标）", + "外观维度（外观设计专利）", + "专利维度（发明/实用新型专利）", + "结论" + ); + + private final LocalFileStorageService localFileStorageService; + private final OssStorageService ossStorageService; + private final StorageProperties storageProperties; + private final FileTaskMapper fileTaskMapper; + private final FileResultMapper fileResultMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final TaskChunkMapper taskChunkMapper; + private final ObjectMapper objectMapper; + private final AppearancePatentCozeClient cozeClient; + private final AppearancePatentTaskCacheService taskCacheService; + private final AppearancePatentProperties properties; + private final TaskFileJobService taskFileJobService; + private final TransientPayloadStorageService transientPayloadStorageService; + private final PlatformTransactionManager transactionManager; + + @Transactional + public AppearancePatentParseVo parseAndCreateTask(AppearancePatentParseRequest request) { + if (request.getUserId() == null || request.getUserId() <= 0) { + throw new BusinessException("user_id 不合法"); + } + if (request.getFiles() == null || request.getFiles().isEmpty()) { + throw new BusinessException("请先上传 Excel 文件"); + } + List sourceFiles = request.getFiles().stream() + .filter(Objects::nonNull) + .toList(); + if (sourceFiles.isEmpty()) { + throw new BusinessException("璇峰厛涓婁紶 Excel 鏂囦欢"); + } + List allRows = new ArrayList<>(); + List mergedHeaders = new ArrayList<>(); + int totalRows = 0; + int droppedRows = 0; + for (AppearancePatentSourceFileDto source : sourceFiles) { + if (source.getFileKey() == null || source.getFileKey().isBlank()) { + throw new BusinessException("fileKey 不能为空"); + } + File input = localFileStorageService.findLocalSourceFile(source.getFileKey()); + if (input == null || !input.exists()) { + throw new BusinessException("源文件不存在"); + } + + ParsedWorkbook parsed = parseWorkbook(input, source); + totalRows += parsed.totalRows(); + droppedRows += parsed.droppedRows(); + allRows.addAll(parsed.allRows()); + mergeHeaders(mergedHeaders, parsed.headers()); + } + if (allRows.isEmpty()) { + throw new BusinessException("未解析到有效 ASIN 数据"); + } + List groups = buildParsedGroups(allRows); + + FileTaskEntity task = new FileTaskEntity(); + task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr()); + task.setModuleType(MODULE_TYPE); + task.setTaskMode("PYTHON_QUEUE"); + task.setStatus(STATUS_PENDING); + task.setSourceFileCount(sourceFiles.size()); + task.setSuccessFileCount(0); + task.setFailedFileCount(0); + task.setCreatedBy("user:" + request.getUserId()); + task.setUserId(request.getUserId()); + task.setCreatedAt(LocalDateTime.now()); + task.setUpdatedAt(LocalDateTime.now()); + try { + task.setRequestJson(objectMapper.writeValueAsString(request)); + task.setResultJson("{}"); + } catch (Exception ex) { + throw new BusinessException("序列化解析结果失败"); + } + fileTaskMapper.insert(task); + + String aggregateScopeKey = buildAggregateScopeKey(sourceFiles); + String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey); + String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), sourceFiles, mergedHeaders, groups, allRows); + String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload); + task.setResultJson(buildTaskResultJson(request.getAiPrompt(), sourceFiles, parsedPayloadPointer)); + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + + String sourceFilename = buildAggregateSourceFilenameLabel(sourceFiles); + FileResultEntity result = new FileResultEntity(); + result.setTaskId(task.getId()); + result.setModuleType(MODULE_TYPE); + result.setSourceFilename(sourceFilename); + result.setSourceFileUrl(aggregateScopeKey); + result.setRowCount(allRows.size()); + result.setUserId(request.getUserId()); + result.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(result); + + TaskScopeStateEntity scope = new TaskScopeStateEntity(); + scope.setTaskId(task.getId()); + scope.setModuleType(MODULE_TYPE); + scope.setScopeKey(aggregateScopeKey); + scope.setScopeHash(sourceScopeHash); + scope.setParsedPayloadJson(writeJson(parsedPayloadPointer, "保存解析载荷指针失败")); + scope.setStateJson("{\"phase\":\"PARSED\"}"); + scope.setChunkTotal(0); + scope.setReceivedChunkCount(0); + scope.setCompleted(0); + scope.setCreatedAt(LocalDateTime.now()); + scope.setUpdatedAt(LocalDateTime.now()); + taskScopeStateMapper.insert(scope); + + AppearancePatentParseVo vo = new AppearancePatentParseVo(); + vo.setTaskId(task.getId()); + vo.setSourceFilename(sourceFilename); + vo.setSourceFileCount(sourceFiles.size()); + vo.setTotalRows(totalRows); + vo.setAcceptedRows(allRows.size()); + vo.setDroppedRows(droppedRows); + vo.setGroupCount(groups.size()); + vo.setAiPrompt(normalize(request.getAiPrompt())); + vo.setItems(allRows); + vo.setGroups(groups); + return vo; + } + + @Transactional + public void activateTask(Long taskId, Long userId) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) { + throw new BusinessException("任务不存在"); + } + if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) { + throw new BusinessException("任务已结束"); + } + task.setStatus(STATUS_RUNNING); + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + taskCacheService.touchTaskHeartbeat(taskId); + } + + public AppearancePatentDashboardVo dashboard(Long userId) { + AppearancePatentDashboardVo vo = new AppearancePatentDashboardVo(); + vo.setPendingTaskCount(countTask(userId, STATUS_RUNNING)); + vo.setSuccessTaskCount(countTask(userId, STATUS_SUCCESS)); + vo.setFailedTaskCount(countTask(userId, STATUS_FAILED)); + vo.setProcessedTaskCount(vo.getSuccessTaskCount() + vo.getFailedTaskCount()); + return vo; + } + + public AppearancePatentHistoryVo history(Long userId) { + AppearancePatentHistoryVo vo = new AppearancePatentHistoryVo(); + List rows = fileResultMapper.selectList(new LambdaQueryWrapper() + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .eq(FileResultEntity::getUserId, userId) + .orderByDesc(FileResultEntity::getCreatedAt) + .last("limit 100")); + Map statusMap = new LinkedHashMap<>(); + List taskIds = rows.stream().map(FileResultEntity::getTaskId).filter(Objects::nonNull).distinct().toList(); + if (!taskIds.isEmpty()) { + for (FileTaskEntity task : fileTaskMapper.selectBatchIds(taskIds)) { + statusMap.put(task.getId(), task.getStatus()); + } + } + for (FileResultEntity row : rows) { + vo.getItems().add(toHistoryItem(row, statusMap.get(row.getTaskId()))); + } + return vo; + } + + public AppearancePatentTaskBatchVo progressBatch(List taskIds) { + AppearancePatentTaskBatchVo vo = new AppearancePatentTaskBatchVo(); + List normalizedIds = taskIds == null ? List.of() : taskIds.stream() + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + if (normalizedIds.isEmpty()) { + return vo; + } + Map taskMap = new LinkedHashMap<>(); + for (FileTaskEntity task : fileTaskMapper.selectBatchIds(normalizedIds)) { + if (task != null && MODULE_TYPE.equals(task.getModuleType())) { + taskMap.put(task.getId(), task); + } + } + for (Long taskId : normalizedIds) { + FileTaskEntity task = taskMap.get(taskId); + if (task == null) { + vo.getMissingTaskIds().add(taskId); + continue; + } + AppearancePatentTaskDetailVo detail = new AppearancePatentTaskDetailVo(); + detail.setTask(toTaskItem(task)); + vo.getItems().add(detail); + } + return vo; + } + + public void submitResult(Long taskId, AppearancePatentSubmitResultRequest request) { + if (transactionManager != null) { + SubmitContext context = inNewTransaction(() -> persistSubmittedChunk(taskId, request)); + inNewTransaction(() -> { + completeSubmittedChunk(context); + return null; + }); + return; + } + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + if (!STATUS_RUNNING.equals(task.getStatus())) { + throw new BusinessException("任务不是运行中状态"); + } + + int chunkIndex = request.getChunkIndex() == null ? 0 : request.getChunkIndex(); + int chunkTotal = request.getChunkTotal() == null ? 1 : request.getChunkTotal(); + String scopeKey = firstNonBlank(request.getSubmissionId(), "task:" + taskId); + String scopeHash = DigestUtil.sha256Hex(scopeKey); + taskCacheService.touchTaskHeartbeat(taskId); + + TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, chunkIndex) + .last("limit 1")); + if (existing == null) { + List rawRows = flattenSubmittedRows(request); + String payloadJson = writeJson(rawRows, "结果序列化失败"); + + TaskChunkEntity chunk = new TaskChunkEntity(); + chunk.setTaskId(taskId); + chunk.setModuleType(MODULE_TYPE); + chunk.setScopeKey(scopeKey); + chunk.setScopeHash(scopeHash); + chunk.setChunkIndex(chunkIndex); + chunk.setChunkTotal(chunkTotal); + String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); + chunk.setPayloadJson(storedPayload); + chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson)); + chunk.setCreatedAt(LocalDateTime.now()); + chunk.setUpdatedAt(LocalDateTime.now()); + try { + taskChunkMapper.insert(chunk); + } catch (DuplicateKeyException ex) { + // storeChunkPayload uses a deterministic key like chunk-{index}. When two + // concurrent callbacks submit the same chunk, deleting the loser payload here + // can also remove the winner's shared object and break later assembly. + log.info("[appearance-patent] duplicate chunk inserted concurrently taskId={} scope={} chunk={}", taskId, scopeKey, chunkIndex); + } + } else { + log.info("[appearance-patent] duplicate chunk ignored taskId={} scope={} chunk={}", taskId, scopeKey, chunkIndex); + } + + TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + if (scope == null) { + scope = new TaskScopeStateEntity(); + scope.setTaskId(taskId); + scope.setModuleType(MODULE_TYPE); + scope.setScopeKey(scopeKey); + scope.setScopeHash(scopeHash); + scope.setCreatedAt(LocalDateTime.now()); + } + scope.setChunkTotal(chunkTotal); + scope.setReceivedChunkCount(countChunks(taskId, scopeHash)); + scope.setLastChunkAt(LocalDateTime.now()); + scope.setLastError(request.getError()); + scope.setCompleted(Boolean.TRUE.equals(request.getDone()) ? 1 : 0); + scope.setUpdatedAt(LocalDateTime.now()); + scope.setStateJson("{\"phase\":\"RECEIVED\",\"coze\":\"PENDING\"}"); + if (scope.getId() == null) { + taskScopeStateMapper.insert(scope); + } else { + taskScopeStateMapper.updateById(scope); + } + + if (Boolean.TRUE.equals(request.getDone()) || request.getError() != null && !request.getError().isBlank()) { + finalizeTask(task, request.getError(), allRowCount(task), true); + } else { + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + } + } + + @Transactional + public void deleteTask(Long taskId, Long userId) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) { + throw new BusinessException("任务不存在"); + } + fileResultMapper.delete(new LambdaQueryWrapper().eq(FileResultEntity::getTaskId, taskId).eq(FileResultEntity::getModuleType, MODULE_TYPE)); + deleteTransientTaskPayloads(taskId); + taskScopeStateMapper.delete(new LambdaQueryWrapper().eq(TaskScopeStateEntity::getTaskId, taskId).eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + taskChunkMapper.delete(new LambdaQueryWrapper().eq(TaskChunkEntity::getTaskId, taskId).eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + taskCacheService.deleteTaskCache(taskId); + fileTaskMapper.deleteById(taskId); + } + + public void deleteHistory(Long resultId, Long userId) { + FileResultEntity row = fileResultMapper.selectById(resultId); + if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) { + throw new BusinessException("记录不存在"); + } + fileResultMapper.deleteById(resultId); + } + + public String resolveResultDownloadUrl(Long resultId, Long userId) { + FileResultEntity row = fileResultMapper.selectById(resultId); + if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) { + throw new BusinessException("记录不存在"); + } + if (row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) { + throw new BusinessException("暂无可下载文件"); + } + return ossStorageService.generateFreshDownloadUrl(row.getResultFileUrl()); + } + + public String resolveResultDownloadFilename(Long resultId, Long userId) { + FileResultEntity row = fileResultMapper.selectById(resultId); + if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) { + throw new BusinessException("记录不存在"); + } + return row.getResultFilename() == null || row.getResultFilename().isBlank() + ? "appearance-patent-" + resultId + ".xlsx" + : row.getResultFilename(); + } + + @Scheduled(cron = "${aiimage.appearance-patent.stale-finalize-cron:0 */2 * * * *}") + public void finalizeStaleTasks() { + if (transactionManager != null) { + LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes())); + long thresholdMillis = threshold.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + List tasks = fileTaskMapper.selectList(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .lt(FileTaskEntity::getUpdatedAt, threshold) + .last("limit 50")); + for (FileTaskEntity task : tasks) { + long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId()); + if (heartbeatMillis > thresholdMillis) { + continue; + } + inNewTransaction(() -> { + finalizeStaleTask(task.getId(), "Python interrupted before uploading final appearance patent result"); + return null; + }); + } + return; + } + LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes())); + long thresholdMillis = threshold.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + List tasks = fileTaskMapper.selectList(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getStatus, STATUS_RUNNING) + .lt(FileTaskEntity::getUpdatedAt, threshold) + .last("limit 50")); + for (FileTaskEntity task : tasks) { + long heartbeatMillis = taskCacheService.getTaskHeartbeatMillis(task.getId()); + if (heartbeatMillis > thresholdMillis) { + continue; + } + finalizeTask(task, "Python interrupted before uploading final appearance patent result", allRowCount(task), true); + } + } + + private SubmitContext persistSubmittedChunk(Long taskId, AppearancePatentSubmitResultRequest request) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("浠诲姟涓嶅瓨鍦?"); + } + if (!STATUS_RUNNING.equals(task.getStatus())) { + throw new BusinessException("浠诲姟涓嶆槸杩愯涓姸鎬?"); + } + + int chunkIndex = request.getChunkIndex() == null ? 0 : request.getChunkIndex(); + int chunkTotal = request.getChunkTotal() == null ? 1 : request.getChunkTotal(); + boolean done = Boolean.TRUE.equals(request.getDone()); + String scopeKey = firstNonBlank(request.getSubmissionId(), "task:" + taskId); + String scopeHash = DigestUtil.sha256Hex(scopeKey); + taskCacheService.touchTaskHeartbeat(taskId); + + TaskChunkEntity existing = taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, chunkIndex) + .last("limit 1")); + if (existing == null) { + List rawRows = flattenSubmittedRows(request); + String payloadJson = writeJson(rawRows, "缁撴灉搴忓垪鍖栧け璐?"); + + TaskChunkEntity chunk = new TaskChunkEntity(); + chunk.setTaskId(taskId); + chunk.setModuleType(MODULE_TYPE); + chunk.setScopeKey(scopeKey); + chunk.setScopeHash(scopeHash); + chunk.setChunkIndex(chunkIndex); + chunk.setChunkTotal(chunkTotal); + String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); + chunk.setPayloadJson(storedPayload); + chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson)); + chunk.setCreatedAt(LocalDateTime.now()); + chunk.setUpdatedAt(LocalDateTime.now()); + try { + taskChunkMapper.insert(chunk); + } catch (DuplicateKeyException ex) { + log.info("[appearance-patent] duplicate chunk inserted concurrently taskId={} scope={} chunk={}", taskId, scopeKey, chunkIndex); + } + } else { + log.info("[appearance-patent] duplicate chunk ignored taskId={} scope={} chunk={}", taskId, scopeKey, chunkIndex); + } + + upsertScopeState(taskId, scopeKey, scopeHash, chunkTotal, request.getError(), done, false); + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + return new SubmitContext(task, scopeKey, scopeHash, done, request.getError()); + } + + private void completeSubmittedChunk(SubmitContext context) { + FileTaskEntity task = fileTaskMapper.selectById(context.task().getId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("浠诲姟涓嶅瓨鍦?"); + } + if (!STATUS_RUNNING.equals(task.getStatus())) { + log.info("[appearance-patent] skip completion because task already finalized taskId={} status={}", + task.getId(), task.getStatus()); + return; + } + upsertScopeState(task.getId(), context.scopeKey(), context.scopeHash(), null, context.error(), context.forceFlush(), false); + if (context.forceFlush() || context.error() != null && !context.error().isBlank()) { + finalizeTask(task, context.error(), allRowCount(task), true); + return; + } + task.setUpdatedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + } + + private void finalizeStaleTask(Long taskId, String error) { + FileTaskEntity task = fileTaskMapper.selectById(taskId); + if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !STATUS_RUNNING.equals(task.getStatus())) { + return; + } + finalizeTask(task, error, allRowCount(task), true); + } + + private void upsertScopeState(Long taskId, + String scopeKey, + String scopeHash, + Integer chunkTotal, + String error, + boolean completed, + boolean cozeDone) { + TaskScopeStateEntity scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + LocalDateTime now = LocalDateTime.now(); + if (scope == null) { + scope = new TaskScopeStateEntity(); + scope.setTaskId(taskId); + scope.setModuleType(MODULE_TYPE); + scope.setScopeKey(scopeKey); + scope.setScopeHash(scopeHash); + scope.setCreatedAt(now); + } + if (chunkTotal != null) { + scope.setChunkTotal(chunkTotal); + } + scope.setReceivedChunkCount(countChunks(taskId, scopeHash)); + scope.setLastChunkAt(now); + scope.setLastError(error); + scope.setCompleted(completed ? 1 : 0); + scope.setUpdatedAt(now); + scope.setStateJson(cozeDone + ? "{\"phase\":\"RECEIVED\",\"coze\":\"DONE\"}" + : "{\"phase\":\"RECEIVED\",\"coze\":\"PENDING\"}"); + if (scope.getId() == null) { + try { + taskScopeStateMapper.insert(scope); + return; + } catch (DuplicateKeyException ex) { + log.info("[appearance-patent] duplicate scope state inserted concurrently taskId={} scope={}", taskId, scopeKey); + scope = taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + if (scope == null) { + log.info("[appearance-patent] scope state winner not committed yet, skip duplicate updater taskId={} scope={}", + taskId, scopeKey); + return; + } + if (chunkTotal != null) { + scope.setChunkTotal(chunkTotal); + } + scope.setReceivedChunkCount(countChunks(taskId, scopeHash)); + scope.setLastChunkAt(now); + scope.setLastError(error); + scope.setCompleted(completed ? 1 : 0); + scope.setUpdatedAt(now); + scope.setStateJson(cozeDone + ? "{\"phase\":\"RECEIVED\",\"coze\":\"DONE\"}" + : "{\"phase\":\"RECEIVED\",\"coze\":\"PENDING\"}"); + } + } + taskScopeStateMapper.updateById(scope); + } + + private T inNewTransaction(Supplier action) { + TransactionTemplate template = new TransactionTemplate(transactionManager); + template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + return template.execute(status -> action.get()); + } + + private List applyCozeInBatches(List items, FileTaskEntity task) { + return applyCozeInBatches(items, task, null); + } + + private List applyCozeInBatches(List items, + FileTaskEntity task, + Runnable progressHook) { + if (items == null || items.isEmpty()) { + return List.of(); + } + String prompt = readAiPrompt(task); + int batchSize = Math.max(1, properties.getCozeBatchSize()); + List result = new ArrayList<>(); + for (int i = 0; i < items.size(); i += batchSize) { + if (progressHook != null) { + progressHook.run(); + } + result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt)); + } + return result; + } + + private void applyCozeToPersistedChunks(FileTaskEntity task, Runnable progressHook) { + if (task == null || task.getId() == null) { + return; + } + Map> allRowsByBaseId = loadAllRowsByBaseId(task); + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, task.getId()) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + log.info("[appearance-patent] async coze start taskId={} chunks={}", task.getId(), chunks.size()); + for (TaskChunkEntity chunk : chunks) { + if (progressHook != null) { + progressHook.run(); + } + Map persistedRows = readChunkRows(chunk); + if (persistedRows.isEmpty()) { + continue; + } + List unresolvedRows = pickGroupRepresentativesForCoze(persistedRows.values()); + if (unresolvedRows.isEmpty()) { + continue; + } + List cozeRows = applyCozeInBatches(unresolvedRows, task, progressHook); + Map mergedRows = new LinkedHashMap<>(); + for (AppearancePatentResultRowDto resultRow : cozeRows) { + for (AppearancePatentResultRowDto expandedRow : expandRows(List.of(resultRow), allRowsByBaseId)) { + mergedRows.put(rowKey(expandedRow), expandedRow); + } + } + mergeChunkPayload(task.getId(), chunk.getScopeHash(), chunk.getChunkIndex(), new ArrayList<>(mergedRows.values())); + log.info("[appearance-patent] async coze chunk merged taskId={} chunk={} unresolved={} merged={}", + task.getId(), chunk.getChunkIndex(), unresolvedRows.size(), mergedRows.size()); + } + } + + private void mergeChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, List rows) { + TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, chunkIndex) + .last("limit 1")); + if (chunk == null || rows == null || rows.isEmpty()) { + return; + } + Map persistedRows = readChunkRows(chunk); + for (AppearancePatentResultRowDto row : rows) { + persistedRows.put(rowKey(row), row); + } + String payloadJson = writeJson(rows == null ? List.of() : rows, "结果序列化失败"); + payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "appearance patent chunk payload merge failed"); + String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(chunk.getPayloadJson(), storedPayload); + chunk.setPayloadJson(storedPayload); + chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson)); + chunk.setUpdatedAt(LocalDateTime.now()); + taskChunkMapper.updateById(chunk); + } + + private List expandRows(List representatives, + Map> allRowsByBaseId) { + if (representatives == null || representatives.isEmpty()) { + return List.of(); + } + List result = new ArrayList<>(); + Set emittedKeys = new LinkedHashSet<>(); + for (AppearancePatentResultRowDto representative : representatives) { + List siblings = resolveSiblingRows(representative, allRowsByBaseId); + if (siblings == null || siblings.isEmpty()) { + result.add(representative); + continue; + } + for (AppearancePatentParsedRowVo sibling : siblings) { + String key = rowKey(sibling); + if (!emittedKeys.add(key)) { + continue; + } + result.add(copyResultToSibling(representative, sibling)); + } + } + return result; + } + + private List resolveSiblingRows(AppearancePatentResultRowDto representative, + Map> groupedRows) { + if (representative == null || groupedRows == null || groupedRows.isEmpty()) { + return List.of(); + } + String representativeGroupKey = normalize(representative.getGroupKey()); + if (!representativeGroupKey.isBlank()) { + return groupedRows.getOrDefault(representativeGroupKey, List.of()); + } + String representativeKey = rowKey(representative); + String representativeLegacyKey = legacyRowKey(representative); + for (List rows : groupedRows.values()) { + for (AppearancePatentParsedRowVo row : rows) { + if (Objects.equals(representativeKey, rowKey(row)) + || Objects.equals(representativeLegacyKey, legacyRowKey(row))) { + return rows; + } + } + } + return List.of(); + } + + private Map> loadAllRowsByBaseId(FileTaskEntity task) { + try { + AppearancePatentParsedPayloadDto payload = readParsedPayload(task); + List rows = payload.getAllItems(); + if (rows == null || rows.isEmpty()) { + rows = payload.getItems(); + } + if ((rows == null || rows.isEmpty()) && payload.getGroups() != null) { + rows = payload.getGroups().stream() + .filter(Objects::nonNull) + .flatMap(group -> group.getItems() == null ? java.util.stream.Stream.empty() : group.getItems().stream()) + .toList(); + } + return groupRowsByBaseId(rows); + } catch (Exception ex) { + log.warn("[appearance-patent] read all rows failed taskId={} err={}", task.getId(), ex.getMessage()); + return new LinkedHashMap<>(); + } + } + + private List pickGroupRepresentativesForCoze(java.util.Collection rows) { + Map> groupedRows = new LinkedHashMap<>(); + if (rows == null) { + return List.of(); + } + for (AppearancePatentResultRowDto row : rows) { + if (row == null) { + continue; + } + String key = firstNonBlank(normalize(row.getGroupKey()), rowKey(row)); + groupedRows.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row); + } + List representatives = new ArrayList<>(); + for (List siblings : groupedRows.values()) { + boolean alreadyResolved = siblings.stream().anyMatch(this::hasResolvedCozeFields); + if (alreadyResolved) { + continue; + } + AppearancePatentResultRowDto candidate = null; + for (AppearancePatentResultRowDto sibling : siblings) { + if (shouldInspectRow(sibling)) { + candidate = sibling; + break; + } + } + if (candidate != null) { + representatives.add(candidate); + } + } + return representatives; + } + + private List buildParsedGroups(List rows) { + List groups = new ArrayList<>(); + for (List siblings : groupRowsByBaseId(rows).values()) { + if (siblings == null || siblings.isEmpty()) { + continue; + } + AppearancePatentParsedRowVo first = siblings.getFirst(); + AppearancePatentParsedGroupVo group = new AppearancePatentParsedGroupVo(); + group.setSourceFileKey(first.getSourceFileKey()); + group.setSourceFilename(first.getSourceFilename()); + group.setGroupKey(firstNonBlank(first.getGroupKey(), buildGroupKey(first.getSourceFileKey(), baseId(first.getDisplayId()), first.getRowIndex()))); + group.setBaseId(baseId(first.getDisplayId())); + group.setDisplayId(firstNonBlank(first.getDisplayId(), first.getSourceId())); + group.setItemCount(siblings.size()); + group.setItems(new ArrayList<>(siblings)); + groups.add(group); + } + return groups; + } + + private Map> groupRowsByBaseId(List rows) { + Map> result = new LinkedHashMap<>(); + if (rows == null) { + return result; + } + for (AppearancePatentParsedRowVo row : rows) { + String key = firstNonBlank(normalize(row.getGroupKey()), baseId(row.getDisplayId())); + result.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row); + } + return result; + } + + private List flattenSubmittedRows(AppearancePatentSubmitResultRequest request) { + if (request == null) { + return List.of(); + } + List groups = request.getGroups(); + if (groups != null && !groups.isEmpty()) { + List rows = new ArrayList<>(); + for (AppearancePatentResultGroupDto group : groups) { + if (group == null || group.getItems() == null || group.getItems().isEmpty()) { + continue; + } + for (AppearancePatentResultRowDto row : group.getItems()) { + if (row == null) { + continue; + } + if (normalize(row.getGroupKey()).isBlank()) { + row.setGroupKey(group.getGroupKey()); + } + if (normalize(row.getSourceFileKey()).isBlank()) { + row.setSourceFileKey(group.getSourceFileKey()); + } + if (normalize(row.getSourceFilename()).isBlank()) { + row.setSourceFilename(group.getSourceFilename()); + } + rows.add(row); + } + } + return rows; + } + return request.getItems() == null ? List.of() : request.getItems(); + } + + private int allRowCount(FileTaskEntity task) { + try { + AppearancePatentParsedPayloadDto payload = readParsedPayload(task); + if (payload.getAllItems() != null && !payload.getAllItems().isEmpty()) { + return payload.getAllItems().size(); + } + if (payload.getItems() != null && !payload.getItems().isEmpty()) { + return payload.getItems().size(); + } + if (payload.getGroups() != null && !payload.getGroups().isEmpty()) { + return payload.getGroups().stream() + .map(AppearancePatentParsedGroupVo::getItems) + .filter(Objects::nonNull) + .mapToInt(List::size) + .sum(); + } + return 0; + } catch (Exception ex) { + log.warn("[appearance-patent] read all row count failed taskId={} err={}", task.getId(), ex.getMessage()); + return 0; + } + } + + private AppearancePatentResultRowDto copyResultToSibling(AppearancePatentResultRowDto representative, AppearancePatentParsedRowVo sibling) { + AppearancePatentResultRowDto row = new AppearancePatentResultRowDto(); + row.setSourceFileKey(sibling.getSourceFileKey()); + row.setSourceFilename(sibling.getSourceFilename()); + row.setRowToken(sibling.getRowToken()); + row.setGroupKey(sibling.getGroupKey()); + row.setId(sibling.getDisplayId()); + row.setAsin(sibling.getAsin()); + row.setCountry(sibling.getCountry()); + row.setUrl(firstNonBlank(representative.getUrl(), sibling.getUrl())); + row.setTitle(firstNonBlank(representative.getTitle(), sibling.getTitle())); + row.setError(representative.getError()); + row.setTitleRisk(representative.getTitleRisk()); + row.setAppearanceRisk(representative.getAppearanceRisk()); + row.setPatentRisk(representative.getPatentRisk()); + row.setConclusion(representative.getConclusion()); + row.setDone(representative.getDone()); + return row; + } + + private String readAiPrompt(FileTaskEntity task) { + try { + return readParsedPayload(task).getAiPrompt(); + } catch (Exception ignored) { + return ""; + } + } + + private void finalizeTask(FileTaskEntity task, String error, int rowCount, boolean assembleWorkbook) { + String finalError = error; + FileResultEntity result = null; + List rows = fileResultMapper.selectList(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, task.getId()) + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .last("limit 1")); + if (!rows.isEmpty()) { + result = rows.getFirst(); + if (assembleWorkbook && shouldAssembleSynchronously()) { + try { + assembleResultWorkbook(task, result); + } catch (Exception ex) { + log.warn("[appearance-patent] assemble result workbook failed taskId={} err={}", task.getId(), ex.getMessage()); + finalError = firstNonBlank(finalError, "生成外观专利检测结果失败"); + } + } + } + boolean failed = finalError != null && !finalError.isBlank(); + task.setStatus(failed ? STATUS_FAILED : STATUS_SUCCESS); + task.setSuccessFileCount(failed ? 0 : 1); + task.setFailedFileCount(failed ? 1 : 0); + task.setErrorMessage(finalError); + task.setUpdatedAt(LocalDateTime.now()); + task.setFinishedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + if (result != null) { + result.setSuccess(failed ? 0 : 1); + result.setErrorMessage(finalError); + result.setRowCount(rowCount > 0 ? rowCount : result.getRowCount()); + if (!failed && assembleWorkbook) { + result.setResultFilename(safeFileStem(result.getSourceFilename()) + "-result.xlsx"); + result.setResultFileUrl(null); + result.setResultFileSize(0L); + result.setResultContentType(CONTENT_TYPE_XLSX); + } + fileResultMapper.updateById(result); + if (!failed && assembleWorkbook) { + taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(), "task:" + task.getId()); + } + } + taskCacheService.deleteTaskCache(task.getId()); + } + + private boolean shouldAssembleSynchronously() { + return false; + } + + public void processResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null || job.getResultId() == null) { + throw new BusinessException("结果文件任务参数不完整"); + } + FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + FileResultEntity result = fileResultMapper.selectById(job.getResultId()); + if (result == null || !MODULE_TYPE.equals(result.getModuleType())) { + throw new BusinessException("结果记录不存在"); + } + Runnable progressHook = () -> taskFileJobService.touchRunning(job.getId()); + applyCozeToPersistedChunks(task, progressHook); + progressHook.run(); + assembleResultWorkbook(task, result); + progressHook.run(); + fileResultMapper.updateById(result); + } + + public void cleanupResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null) { + return; + } + deleteTransientTaskPayloads(job.getTaskId()); + taskChunkMapper.delete(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, job.getTaskId()) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + taskScopeStateMapper.delete(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, job.getTaskId()) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + } + + private void assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) { + AppearancePatentParsedPayloadDto parsed = readParsedPayload(task); + Map resultMap = loadPersistedResultRows(task.getId()); + File outputDir = new File(storageProperties.getLocalTempDir(), "appearance-patent-result"); + if (!outputDir.exists() && !outputDir.mkdirs()) { + throw new BusinessException("创建结果目录失败"); + } + String filename = safeFileStem(result.getSourceFilename()) + "-result.xlsx"; + File xlsx = new File(outputDir, filename); + try { + writeResultWorkbook(xlsx, parsed, resultMap); + String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE); + result.setResultFilename(filename); + result.setResultFileUrl(objectKey); + result.setResultFileSize(xlsx.length()); + result.setResultContentType(CONTENT_TYPE_XLSX); + result.setRowCount(parsed.getAllItems().size()); + } finally { + if (xlsx.exists() && !xlsx.delete()) { + log.warn("[appearance-patent] delete temp xlsx failed file={}", xlsx); + } + } + } + + private Map loadPersistedResultRows(Long taskId) { + Map result = new LinkedHashMap<>(); + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .orderByAsc(TaskChunkEntity::getChunkIndex)); + for (TaskChunkEntity chunk : chunks) { + result.putAll(readChunkRows(chunk)); + } + return result; + } + + private void writeResultWorkbook(File xlsx, AppearancePatentParsedPayloadDto parsed, Map resultMap) { + try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) { + Sheet sheet = workbook.createSheet("外观专利检测结果"); + CellStyle headerStyle = workbook.createCellStyle(); + Font font = workbook.createFont(); + font.setBold(true); + headerStyle.setFont(font); + + List resultHeaders = new ArrayList<>(RESULT_HEADERS); + resultHeaders.add(4, "title"); + resultHeaders.add(5, "url"); + Row header = sheet.createRow(0); + for (int i = 0; i < resultHeaders.size(); i++) { + Cell cell = header.createCell(i); + cell.setCellValue(resultHeaders.get(i)); + cell.setCellStyle(headerStyle); + } + + int rowIndex = 1; + for (AppearancePatentParsedRowVo parsedRow : parsed.getAllItems()) { + AppearancePatentResultRowDto resultRow = findResultRow(parsedRow, resultMap); + String missingReason = ""; + if (resultRow == null) { + missingReason = hasPromptFields(parsedRow) ? "未匹配到检测结果" : "未送检：缺少标题或图片"; + } + Row row = sheet.createRow(rowIndex++); + int col = 0; + row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId())); + row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), "")); + row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), "")); + row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "价格", "price")); + row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getTitle(), "") : firstNonBlank(resultRow.getTitle(), parsedRow.getTitle())); + row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getUrl(), "") : firstNonBlank(resultRow.getUrl(), parsedRow.getUrl())); + row.createCell(col++).setCellValue(resultRow == null ? missingReason : userFacingCozeCellValue(resultRow, resultRow.getTitleRisk())); + row.createCell(col++).setCellValue(resultRow == null ? missingReason : userFacingCozeCellValue(resultRow, resultRow.getAppearanceRisk())); + row.createCell(col++).setCellValue(resultRow == null ? missingReason : userFacingCozeCellValue(resultRow, resultRow.getPatentRisk())); + row.createCell(col).setCellValue(resultRow == null ? "未送检" : userFacingConclusion(resultRow)); + } + workbook.write(fos); + workbook.dispose(); + } catch (Exception ex) { + throw new BusinessException("生成外观专利检测结果失败"); + } + } + + private ParsedWorkbook parseWorkbook(File input, AppearancePatentSourceFileDto source) { + DataFormatter formatter = new DataFormatter(); + try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) { + Sheet sheet = workbook.getSheetAt(0); + Row header = sheet.getRow(0); + if (header == null) { + throw new BusinessException("Excel 表头为空"); + } + Map headerMap = buildHeaderMap(header, formatter); + List headers = readHeaders(header, formatter); + int idCol = findRequiredHeader(headerMap, "id"); + int asinCol = findRequiredHeader(headerMap, "asin"); + int countryCol = findRequiredHeader(headerMap, "国家", "country"); + int urlCol = findOptionalHeaderExact(headerMap, + "url", "rul", "link", "image", "img", "pic", "picture", + "链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接"); + int titleCol = findOptionalHeaderExact(headerMap, + "标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称"); + + List allRows = new ArrayList<>(); + int total = 0; + int dropped = 0; + String currentBlockBaseId = ""; + String currentGroupKey = ""; + for (int i = 1; i <= sheet.getLastRowNum(); i++) { + Row row = sheet.getRow(i); + if (row == null) { + continue; + } + String id = cell(row, idCol, formatter); + String asin = cell(row, asinCol, formatter).toUpperCase(Locale.ROOT); + String country = cell(row, countryCol, formatter); + if (id.isBlank() && asin.isBlank() && country.isBlank()) { + continue; + } + total++; + if (id.isBlank() || asin.isBlank() || country.isBlank()) { + dropped++; + continue; + } + AppearancePatentParsedRowVo vo = new AppearancePatentParsedRowVo(); + vo.setSourceFileKey(source.getFileKey()); + vo.setSourceFilename(firstNonBlank(source.getOriginalFilename(), input.getName())); + vo.setRowIndex(i + 1); + vo.setSourceId(id); + vo.setDisplayId(normalizeDisplayId(id)); + String rowBaseId = baseId(vo.getDisplayId()); + if (!Objects.equals(currentBlockBaseId, rowBaseId)) { + currentBlockBaseId = rowBaseId; + currentGroupKey = buildGroupKey(source.getFileKey(), rowBaseId, vo.getRowIndex()); + } + vo.setGroupKey(currentGroupKey); + vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex())); + vo.setAsin(asin); + vo.setCountry(country); + vo.setUrl(urlCol >= 0 ? cell(row, urlCol, formatter) : ""); + vo.setTitle(titleCol >= 0 ? cell(row, titleCol, formatter) : ""); + vo.setValues(readRowValues(row, headers, formatter)); + allRows.add(vo); + } + hydratePromptFields(allRows); + if (allRows.isEmpty()) { + throw new BusinessException("no valid appearance patent rows"); + } + return new ParsedWorkbook(total, dropped, headers, allRows); + } catch (BusinessException ex) { + throw ex; + } catch (Exception ex) { + log.warn("[appearance-patent] parse failed file={} err={}", input, ex.getMessage()); + throw new BusinessException("解析 Excel 失败"); + } + } + + private void hydratePromptFields(List rows) { + if (rows == null || rows.isEmpty()) { + return; + } + Map> rowsByBaseId = groupRowsByBaseId(rows); + for (List siblings : rowsByBaseId.values()) { + String title = ""; + String url = ""; + for (AppearancePatentParsedRowVo sibling : siblings) { + title = firstNonBlank(title, sibling.getTitle()); + url = firstNonBlank(url, sibling.getUrl()); + } + for (AppearancePatentParsedRowVo sibling : siblings) { + if (normalize(sibling.getTitle()).isBlank()) { + sibling.setTitle(title); + } + if (normalize(sibling.getUrl()).isBlank()) { + sibling.setUrl(url); + } + } + } + } + + private boolean hasPromptFields(AppearancePatentParsedRowVo row) { + return row != null && (!normalize(row.getTitle()).isBlank() || !normalize(row.getUrl()).isBlank()); + } + + private boolean shouldInspectRow(AppearancePatentResultRowDto row) { + return row != null && (!normalize(row.getTitle()).isBlank() || !normalize(row.getUrl()).isBlank()); + } + + private boolean hasResolvedCozeFields(AppearancePatentResultRowDto row) { + if (row == null) { + return false; + } + return hasUsableCozeField(row.getTitleRisk()) + || hasUsableCozeField(row.getAppearanceRisk()) + || hasUsableCozeField(row.getPatentRisk()) + || hasUsableCozeField(row.getConclusion()); + } + + private boolean hasUsableCozeField(String value) { + String normalized = normalize(value); + return !normalized.isBlank() && !isTechnicalCozeFailure(normalized); + } + + private Map buildHeaderMap(Row header, DataFormatter formatter) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < header.getLastCellNum(); i++) { + String val = normalize(formatter.formatCellValue(header.getCell(i))); + if (!val.isBlank()) { + map.putIfAbsent(val.toLowerCase(Locale.ROOT), i); + } + } + return map; + } + + private List readHeaders(Row header, DataFormatter formatter) { + List headers = new ArrayList<>(); + for (int i = 0; i < header.getLastCellNum(); i++) { + String val = normalize(formatter.formatCellValue(header.getCell(i))); + headers.add(val.isBlank() ? "列" + (i + 1) : val); + } + return headers; + } + + private Map readRowValues(Row row, List headers, DataFormatter formatter) { + Map values = new LinkedHashMap<>(); + for (int i = 0; i < headers.size(); i++) { + values.put(headers.get(i), cell(row, i, formatter)); + } + return values; + } + + private int findRequiredHeader(Map map, String... names) { + int idx = findOptionalHeader(map, names); + if (idx < 0) { + throw new BusinessException("缺少必要表头: " + String.join("/", names)); + } + return idx; + } + + private int findOptionalHeader(Map map, String... names) { + for (Map.Entry entry : map.entrySet()) { + for (String name : names) { + if (entry.getKey().contains(name.toLowerCase(Locale.ROOT))) { + return entry.getValue(); + } + } + } + return -1; + } + + private int findOptionalHeaderExact(Map map, String... names) { + for (Map.Entry entry : map.entrySet()) { + String normalizedHeader = normalizeHeaderAlias(entry.getKey()); + for (String name : names) { + if (normalizedHeader.equals(normalizeHeaderAlias(name))) { + return entry.getValue(); + } + } + } + return -1; + } + + private String normalizeHeaderAlias(String value) { + String normalized = normalize(value).toLowerCase(Locale.ROOT); + return normalized.replaceAll("[\\s_\\-()（）\\[\\]{}:：/\\\\]+", ""); + } + + private String cell(Row row, int col, DataFormatter formatter) { + return col < 0 ? "" : normalize(formatter.formatCellValue(row.getCell(col))); + } + + private String baseId(String id) { + String s = normalize(id); + int idx = s.indexOf('_'); + return idx > 0 ? s.substring(0, idx) : s; + } + + private String normalizeDisplayId(String id) { + return id == null ? "" : id.trim(); + } + + private void mergeHeaders(List mergedHeaders, List headers) { + if (mergedHeaders == null || headers == null || headers.isEmpty()) { + return; + } + Set existing = new LinkedHashSet<>(mergedHeaders); + for (String header : headers) { + if (existing.add(header)) { + mergedHeaders.add(header); + } + } + } + + private String buildAggregateScopeKey(List sourceFiles) { + List fileKeys = sourceFiles == null ? List.of() : sourceFiles.stream() + .map(AppearancePatentSourceFileDto::getFileKey) + .filter(Objects::nonNull) + .map(String::trim) + .filter(value -> !value.isBlank()) + .toList(); + if (fileKeys.isEmpty()) { + return ""; + } + if (fileKeys.size() == 1) { + return fileKeys.get(0); + } + return "aggregate:" + DigestUtil.sha256Hex(String.join("|", fileKeys)); + } + + private String buildAggregateSourceFilenameLabel(List sourceFiles) { + if (sourceFiles == null || sourceFiles.isEmpty()) { + return "appearance-patent"; + } + String firstFilename = sourceFiles.stream() + .map(AppearancePatentSourceFileDto::getOriginalFilename) + .filter(Objects::nonNull) + .map(String::trim) + .filter(value -> !value.isBlank()) + .findFirst() + .orElse("appearance-patent"); + if (sourceFiles.size() == 1) { + return firstFilename; + } + return firstFilename + " 等 " + sourceFiles.size() + " 个文件"; + } + + private String buildRowToken(String sourceFileKey, Integer rowIndex) { + return normalize(sourceFileKey) + "::row::" + (rowIndex == null ? 0 : rowIndex); + } + + private String buildGroupKey(String sourceFileKey, String rowBaseId, Integer rowIndex) { + return normalize(sourceFileKey) + "::" + normalize(rowBaseId) + "@" + (rowIndex == null ? 0 : rowIndex); + } + + private long countTask(Long userId, String status) { + Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getUserId, userId) + .eq(FileTaskEntity::getStatus, status)); + return count == null ? 0L : count; + } + + private int countChunks(Long taskId, String scopeHash) { + Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash)); + return count == null ? 0 : count.intValue(); + } + + private AppearancePatentTaskItemVo toTaskItem(FileTaskEntity task) { + AppearancePatentTaskItemVo vo = new AppearancePatentTaskItemVo(); + vo.setId(task.getId()); + vo.setTaskNo(task.getTaskNo()); + vo.setStatus(task.getStatus()); + vo.setErrorMessage(task.getErrorMessage()); + vo.setCreatedAt(fmt(task.getCreatedAt())); + vo.setUpdatedAt(fmt(task.getUpdatedAt())); + vo.setFinishedAt(fmt(task.getFinishedAt())); + return vo; + } + + private AppearancePatentHistoryItemVo toHistoryItem(FileResultEntity row, String taskStatus) { + AppearancePatentHistoryItemVo vo = new AppearancePatentHistoryItemVo(); + vo.setResultId(row.getId()); + vo.setTaskId(row.getTaskId()); + vo.setSourceFilename(row.getSourceFilename()); + vo.setResultFilename(row.getResultFilename()); + vo.setDownloadUrl(null); + attachFileJobState(vo, row); + vo.setTaskStatus(taskStatus); + vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1); + vo.setError(row.getErrorMessage()); + vo.setRowCount(row.getRowCount()); + vo.setCreatedAt(fmt(row.getCreatedAt())); + return vo; + } + + private void attachFileJobState(AppearancePatentHistoryItemVo vo, FileResultEntity row) { + TaskFileJobEntity job = taskFileJobService.findAssembleJob(row.getTaskId(), MODULE_TYPE, row.getId()); + vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank()); + if (job == null) { + vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); + return; + } + vo.setFileJobId(job.getId()); + vo.setFileStatus(job.getStatus()); + vo.setFileError(job.getErrorMessage()); + } + + private String fmt(LocalDateTime t) { + return t == null ? null : t.toString(); + } + + private String firstNonBlank(String preferred, String fallback) { + return preferred == null || preferred.isBlank() ? fallback : preferred.trim(); + } + + private String normalize(String val) { + return val == null ? "" : val.replace("\ufeff", "").replace("\u3000", " ").trim().replaceAll("\\s+", " "); + } + + private String buildParsedPayloadJson(String aiPrompt, List sourceFiles, List headers, List groups, List allRows) { + AppearancePatentParsedPayloadDto payload = new AppearancePatentParsedPayloadDto(); + payload.setAiPrompt(normalize(aiPrompt)); + payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles); + payload.setHeaders(headers == null ? List.of() : headers); + payload.setItems(allRows == null ? List.of() : allRows); + payload.setGroups(groups == null ? List.of() : groups); + payload.setAllItems(allRows == null ? List.of() : allRows); + return writeJson(payload, "保存解析结果失败"); + } + + private String buildTaskResultJson(String aiPrompt, List sourceFiles, String parsedPayloadPointer) { + Map payload = new LinkedHashMap<>(); + payload.put("aiPrompt", normalize(aiPrompt)); + payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream() + .map(AppearancePatentSourceFileDto::getFileKey) + .filter(Objects::nonNull) + .toList()); + payload.put("parsedPayloadRef", parsedPayloadPointer); + return writeJson(payload, "保存任务结果索引失败"); + } + + private String storeParsedPayload(Long taskId, String scopeHash, String parsedPayloadJson) { + return transientPayloadStorageService.storeParsedPayload(MODULE_TYPE, taskId, scopeHash, parsedPayloadJson, true); + } + + private AppearancePatentParsedPayloadDto readParsedPayload(FileTaskEntity task) { + try { + JsonNode root = objectMapper.readTree(task.getResultJson()); + String pointer = root.path("parsedPayloadRef").asText(""); + if (!pointer.isBlank()) { + String json = transientPayloadStorageService.resolvePayload(pointer, "read appearance patent parsed payload failed"); + return objectMapper.readValue(json, AppearancePatentParsedPayloadDto.class); + } + if (root.path("allItems").isArray()) { + return objectMapper.treeToValue(root, AppearancePatentParsedPayloadDto.class); + } + } catch (Exception ex) { + throw new BusinessException("读取解析载荷失败"); + } + return new AppearancePatentParsedPayloadDto(); + } + + private Map readChunkRows(TaskChunkEntity chunk) { + Map rows = new LinkedHashMap<>(); + if (chunk == null || chunk.getPayloadJson() == null || chunk.getPayloadJson().isBlank()) { + return rows; + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read appearance patent chunk failed"); + JsonNode array = objectMapper.readTree(payloadJson); + if (!array.isArray()) { + return rows; + } + for (JsonNode node : array) { + AppearancePatentResultRowDto row = objectMapper.treeToValue(node, AppearancePatentResultRowDto.class); + rows.put(rowKey(row), row); + } + } catch (Exception ex) { + log.warn("[appearance-patent] read chunk payload failed taskId={} chunk={} err={}", + chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage()); + } + return rows; + } + + private String rowKey(AppearancePatentParsedRowVo row) { + if (row == null) { + return ""; + } + String rowToken = normalize(row.getRowToken()); + if (!rowToken.isBlank()) { + return rowToken; + } + return legacyRowKey(row); + } + + private String rowKey(AppearancePatentResultRowDto row) { + if (row == null) { + return ""; + } + String rowToken = normalize(row.getRowToken()); + if (!rowToken.isBlank()) { + return rowToken; + } + return legacyRowKey(row); + } + + private AppearancePatentResultRowDto findResultRow(AppearancePatentParsedRowVo parsedRow, + Map resultMap) { + if (parsedRow == null || resultMap == null || resultMap.isEmpty()) { + return null; + } + AppearancePatentResultRowDto resultRow = resultMap.get(rowKey(parsedRow)); + if (resultRow != null) { + return resultRow; + } + String legacyKey = legacyRowKey(parsedRow); + if (legacyKey.isBlank()) { + return null; + } + return resultMap.get(legacyKey); + } + + private String legacyRowKey(AppearancePatentParsedRowVo row) { + if (row == null) { + return ""; + } + return rowKey(row.getDisplayId(), row.getAsin(), row.getCountry()); + } + + private String legacyRowKey(AppearancePatentResultRowDto row) { + if (row == null) { + return ""; + } + return rowKey(row.getId(), row.getAsin(), row.getCountry()); + } + + private String rowKey(String id, String asin, String country) { + return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country); + } + + private String readValueByHeader(AppearancePatentParsedRowVo row, String... candidates) { + if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) { + return ""; + } + for (Map.Entry entry : row.getValues().entrySet()) { + String header = normalize(entry.getKey()).toLowerCase(Locale.ROOT); + for (String candidate : candidates) { + String expected = normalize(candidate).toLowerCase(Locale.ROOT); + if (!expected.isBlank() && header.contains(expected)) { + return entry.getValue() == null ? "" : entry.getValue(); + } + } + } + return ""; + } + + private String userFacingCozeCellValue(AppearancePatentResultRowDto row, String value) { + String normalizedValue = normalize(value); + if (!normalizedValue.isBlank() && !isTechnicalCozeFailure(normalizedValue)) { + return value; + } + if (row != null && isTechnicalCozeFailure(row.getError())) { + return "待人工复核"; + } + return firstNonBlank(value, ""); + } + + private String userFacingConclusion(AppearancePatentResultRowDto row) { + if (row == null) { + return ""; + } + String conclusion = normalize(row.getConclusion()); + if (!conclusion.isBlank() && !isTechnicalCozeFailure(conclusion)) { + return row.getConclusion(); + } + if (isTechnicalCozeFailure(row.getError())) { + return "待人工复核"; + } + return firstNonBlank(row.getConclusion(), ""); + } + + private boolean isTechnicalCozeFailure(String value) { + String normalized = normalize(value).toLowerCase(Locale.ROOT); + return normalized.contains("coze") + || normalized.contains("结果不完整") + || normalized.contains("工作流节点执行超限") + || normalized.contains("调用超时") + || normalized.contains("timeout"); + } + + private String safeFileStem(String filename) { + String name = filename == null || filename.isBlank() ? "appearance-patent" : filename; + int idx = name.lastIndexOf('.'); + if (idx > 0) { + name = name.substring(0, idx); + } + String safe = name.replaceAll("[\\\\/:*?\"<>|]", "_").trim(); + return safe.isBlank() ? "appearance-patent" : safe; + } + + private String writeJson(Object value, String message) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new BusinessException(message); + } + } + + private void deleteTransientTaskPayloads(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + List scopes = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + if (scopes != null) { + for (TaskScopeStateEntity scope : scopes) { + transientPayloadStorageService.deletePayloadIfPresent(scope.getParsedPayloadJson()); + transientPayloadStorageService.deletePayloadIfPresent(scope.getStateJson()); + } + } + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .select(TaskChunkEntity::getPayloadJson) + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + if (chunks != null) { + for (TaskChunkEntity chunk : chunks) { + transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); + } + } + } + + private record SubmitContext(FileTaskEntity task, + String scopeKey, + String scopeHash, + boolean forceFlush, + String error) { + } + + private record ParsedWorkbook(int totalRows, int droppedRows, List headers, List allRows) { + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java index be8759c..26f35bd 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.brand.service; import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.config.BrandProgressProperties; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; @@ -11,6 +12,7 @@ import java.util.LinkedHashMap; import java.util.Map; @Service +@Slf4j public class BrandTaskProgressCacheService { public static final String PHASE_CRAWLING = "crawling"; @@ -53,8 +55,12 @@ public class BrandTaskProgressCacheService { values.put("finished_files", String.valueOf(Math.max(finishedFiles, 0))); values.put("updated_at", now); values.put("last_heartbeat_at", now); - stringRedisTemplate.opsForHash().putAll(key, values); - stringRedisTemplate.expire(key, ttl()); + try { + stringRedisTemplate.opsForHash().putAll(key, values); + stringRedisTemplate.expire(key, ttl()); + } catch (Exception ex) { + log.warn("[brand-progress-cache] save progress degraded taskId={} msg={}", taskId, ex.getMessage()); + } } public void updatePhase(Long taskId, String phase, int finishedFiles, int fileTotal) { @@ -66,8 +72,12 @@ public class BrandTaskProgressCacheService { values.put("file_total", String.valueOf(Math.max(fileTotal, 0))); values.put("updated_at", now); values.put("last_heartbeat_at", now); - stringRedisTemplate.opsForHash().putAll(key, values); - stringRedisTemplate.expire(key, ttl()); + try { + stringRedisTemplate.opsForHash().putAll(key, values); + stringRedisTemplate.expire(key, ttl()); + } catch (Exception ex) { + log.warn("[brand-progress-cache] update phase degraded taskId={} msg={}", taskId, ex.getMessage()); + } } public void markFailed(Long taskId, String message) { @@ -77,27 +87,49 @@ public class BrandTaskProgressCacheService { values.put("phase", PHASE_FAILED); values.put("updated_at", now); values.put("error_message", blankToEmpty(message)); - stringRedisTemplate.opsForHash().putAll(key, values); - stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getFailedTtlHours())); + try { + stringRedisTemplate.opsForHash().putAll(key, values); + stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getFailedTtlHours())); + } catch (Exception ex) { + log.warn("[brand-progress-cache] mark failed degraded taskId={} msg={}", taskId, ex.getMessage()); + } } public Map getProgress(Long taskId) { - return stringRedisTemplate.opsForHash().entries(buildKey(taskId)); + try { + return stringRedisTemplate.opsForHash().entries(buildKey(taskId)); + } catch (Exception ex) { + log.warn("[brand-progress-cache] get progress degraded taskId={} msg={}", taskId, ex.getMessage()); + return Map.of(); + } } public boolean acquireFinalizeLock(Long taskId) { - Boolean ok = stringRedisTemplate.opsForValue() - .setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL); - return Boolean.TRUE.equals(ok); + try { + Boolean ok = stringRedisTemplate.opsForValue() + .setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL); + return Boolean.TRUE.equals(ok); + } catch (Exception ex) { + log.warn("[brand-progress-cache] acquire finalize lock degraded taskId={} msg={}", taskId, ex.getMessage()); + return false; + } } public void releaseFinalizeLock(Long taskId) { - stringRedisTemplate.delete(buildFinalizeLockKey(taskId)); + try { + stringRedisTemplate.delete(buildFinalizeLockKey(taskId)); + } catch (Exception ex) { + log.warn("[brand-progress-cache] release finalize lock degraded taskId={} msg={}", taskId, ex.getMessage()); + } } public void delete(Long taskId) { - stringRedisTemplate.delete(buildKey(taskId)); - stringRedisTemplate.delete(buildFinalizeLockKey(taskId)); + try { + stringRedisTemplate.delete(buildKey(taskId)); + stringRedisTemplate.delete(buildFinalizeLockKey(taskId)); + } catch (Exception ex) { + log.warn("[brand-progress-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage()); + } } public String buildKey(Long taskId) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java index 7d3b590..6361298 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java @@ -31,6 +31,9 @@ import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskDetailVo; import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskItemVo; import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskListVo; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.poi.ss.usermodel.Cell; @@ -77,6 +80,7 @@ public class BrandTaskService { private static final String STATUS_SUCCESS = "success"; private static final String STATUS_FAILED = "failed"; private static final String STATUS_CANCELLED = "cancelled"; + private static final String MODULE_TYPE = "BRAND"; private final BrandCrawlTaskMapper brandCrawlTaskMapper; private final OssStorageService ossStorageService; @@ -86,6 +90,8 @@ public class BrandTaskService { private final BrandTaskStorageService brandTaskStorageService; private final DistributedJobLockService distributedJobLockService; private final ObjectMapper objectMapper; + private final TaskFileJobService taskFileJobService; + private final TaskProgressSnapshotService taskProgressSnapshotService; public BrandTaskCreateVo createTask(Long userId, BrandTaskCreateRequest request) { if (userId == null || userId <= 0) { @@ -244,6 +250,7 @@ public class BrandTaskService { int totalCount = sourceFiles.size(); markTaskRunning(taskId, totalCount); + afterTaskStarted(taskId, totalCount); log.info("[brand-submit] taskId={} chunks={} files={} thread={}", taskId, @@ -295,6 +302,7 @@ public class BrandTaskService { currentTotalLines, finishedCount); updateTaskProgress(taskId, finishedCount, totalCount); + saveBrandProgressSnapshot(taskId, STATUS_RUNNING, totalCount, finishedCount, 0, null); } tryFinalizeTask(taskId, request.getStrategy(), sourceFiles, cachedByUrl, finishedCount, totalCount); @@ -357,6 +365,23 @@ public class BrandTaskService { return ossStorageService.generateFreshDownloadUrl(stored); } + public String resolveResultObjectKey(Long taskId) { + BrandCrawlTaskEntity task = requireTask(taskId); + Object raw = parseJsonValue(task.getResultPaths()); + if (!(raw instanceof Map map)) { + return null; + } + Object zipObjectKey = map.get("zip_object_key"); + if (zipObjectKey instanceof String key && !key.isBlank()) { + return key; + } + Object zipUrl = map.get("zip_url"); + if (zipUrl instanceof String url && !url.isBlank()) { + return url; + } + return null; + } + private List listTaskEntities(Long userId) { if (userId == null || userId <= 0) { throw new BusinessException("userId 不合法"); @@ -496,7 +521,7 @@ public class BrandTaskService { private void markTaskRunning(Long taskId, int totalCount) { int started = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() .eq(BrandCrawlTaskEntity::getId, taskId) - .ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED) + .in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING) .set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING) .set(BrandCrawlTaskEntity::getProgressTotal, totalCount) .set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()) @@ -505,6 +530,13 @@ public class BrandTaskService { throw new BusinessException("任务已取消"); } } + private void afterTaskStarted(Long taskId, int totalCount) { + saveBrandProgressSnapshot(taskId, STATUS_RUNNING, totalCount, 0, 0, null); + } + + private void saveBrandProgressSnapshot(Long taskId, String status, int totalCount, int successCount, int failedCount, String message) { + taskProgressSnapshotService.save(taskId, MODULE_TYPE, status, totalCount, successCount, failedCount, null, message, null); + } private BrandCrawlTaskEntity requireActiveTask(Long taskId) { BrandCrawlTaskEntity task = requireTask(taskId); @@ -533,12 +565,33 @@ public class BrandTaskService { return; } try { - finalizeTask(taskId, strategy, sourceFiles, cachedByUrl, totalCount); + enqueueFinalizeTask(taskId, totalCount); } finally { brandTaskProgressCacheService.releaseFinalizeLock(taskId); } } + private void enqueueFinalizeTask(Long taskId, int totalCount) { + brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_ASSEMBLING, totalCount, totalCount); + taskProgressSnapshotService.save(taskId, MODULE_TYPE, STATUS_RUNNING, totalCount, totalCount, 0, + null, "result file assembly queued", null); + taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, taskId, "task:" + taskId); + } + + public void processResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null) { + throw new BusinessException("结果文件任务参数不完整"); + } + BrandCrawlTaskEntity task = requireTask(job.getTaskId()); + List sourceFiles = parseSourceFiles(task.getFilePaths()); + if (sourceFiles.isEmpty()) { + throw new BusinessException("任务没有源文件"); + } + List cachedFiles = brandTaskStorageService.getParsedPayload(task.getId()); + Map cachedByUrl = indexCachedFiles(cachedFiles); + finalizeTask(task.getId(), normalizeStrategy(task.getStrategy()), sourceFiles, cachedByUrl, sourceFiles.size()); + } + private void finalizeTask(Long taskId, String strategy, List sourceFiles, @@ -601,6 +654,7 @@ public class BrandTaskService { if (updated == 0) { throw new BusinessException("任务已取消"); } + saveBrandProgressSnapshot(taskId, STATUS_SUCCESS, totalCount, totalCount, 0, null); brandTaskStorageService.deleteTaskData(taskId); brandTaskProgressCacheService.delete(taskId); log.info("[brand-finalize] taskId={} finalized success files={} elapsedMs={}", @@ -617,6 +671,7 @@ public class BrandTaskService { .set(BrandCrawlTaskEntity::getProgressTotal, totalCount) .set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage())); brandTaskProgressCacheService.markFailed(taskId, ex.getMessage()); + saveBrandProgressSnapshot(taskId, STATUS_FAILED, totalCount, brandTaskStorageService.countCompletedFiles(taskId), 1, ex.getMessage()); if (ex instanceof BusinessException businessException) { throw businessException; } @@ -643,10 +698,23 @@ public class BrandTaskService { } } + private Map indexCachedFiles(List cachedFiles) { + Map cachedByUrl = new LinkedHashMap<>(); + if (cachedFiles == null) { + return cachedByUrl; + } + for (BrandParsedFileCacheDto cachedFile : cachedFiles) { + if (cachedFile != null && cachedFile.getFileUrl() != null) { + cachedByUrl.put(cachedFile.getFileUrl(), cachedFile); + } + } + return cachedByUrl; + } + private void updateTaskProgress(Long taskId, int finishedCount, int totalCount) { brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() .eq(BrandCrawlTaskEntity::getId, taskId) - .ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED) + .in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING) .set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING) .set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount) .set(BrandCrawlTaskEntity::getProgressTotal, totalCount) @@ -951,13 +1019,14 @@ public class BrandTaskService { List fullUrls = new ArrayList<>(); for (OutputEntry entry : entries) { // 存储完整公开地址，下载时会自动通过 resolveObjectKey 提取并重新签名（见 OssStorageService.resolveObjectKey） - String objectKey = ossStorageService.uploadResultFile(entry.resultFile(), "BRAND"); + String objectKey = ossStorageService.uploadResultFile(entry.resultFile(), MODULE_TYPE); fullUrls.add(ossStorageService.getPublicUrl(objectKey)); } Map result = new LinkedHashMap<>(); result.put("urls", fullUrls); File zipFile = packageAsZip(taskId, entries); - String zipObjectKey = ossStorageService.uploadResultFile(zipFile, "BRAND"); + String zipObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE); + result.put("zip_object_key", zipObjectKey); result.put("zip_url", ossStorageService.getPublicUrl(zipObjectKey)); return result; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java index 9deb686..2da2092 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java @@ -7,11 +7,11 @@ import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto; import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto; import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto; -import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import lombok.RequiredArgsConstructor; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; @@ -30,12 +30,11 @@ import java.util.Map; public class BrandTaskStorageService { private static final String MODULE_TYPE = "BRAND"; - private static final String OSS_POINTER_PREFIX = "oss:"; private final TaskChunkMapper taskChunkMapper; private final TaskScopeStateMapper taskScopeStateMapper; private final ObjectMapper objectMapper; - private final OssStorageService ossStorageService; + private final TransientPayloadStorageService transientPayloadStorageService; @Transactional public void saveParsedPayload(Long taskId, List payload) { @@ -73,7 +72,7 @@ public class BrandTaskStorageService { if (state == null) { throw new BusinessException("Save brand task parsed payload failed"); } - deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedJson); + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getParsedPayloadJson(), parsedJson); taskScopeStateMapper.update(null, new LambdaUpdateWrapper() .eq(TaskScopeStateEntity::getId, state.getId()) .set(TaskScopeStateEntity::getScopeKey, scopeKey) @@ -132,6 +131,7 @@ public class BrandTaskStorageService { throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex()); } + String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, file.getChunkIndex(), payloadJson); TaskChunkEntity entity = new TaskChunkEntity(); entity.setTaskId(taskId); entity.setModuleType(MODULE_TYPE); @@ -139,7 +139,7 @@ public class BrandTaskStorageService { entity.setScopeHash(scopeHash); entity.setChunkIndex(file.getChunkIndex()); entity.setChunkTotal(file.getChunkTotal()); - entity.setPayloadJson(payloadJson); + entity.setPayloadJson(storedPayload); entity.setPayloadHash(payloadHash); entity.setCreatedAt(now); entity.setUpdatedAt(now); @@ -153,9 +153,11 @@ public class BrandTaskStorageService { .eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex()) .last("limit 1")); if (concurrentChunk != null && payloadHash.equals(concurrentChunk.getPayloadHash())) { + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash); return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate); } + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex()); } @@ -171,7 +173,8 @@ public class BrandTaskStorageService { return null; } try { - return objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class); + String aggregateJson = transientPayloadStorageService.resolvePayload(state.getStateJson(), "read brand task aggregate failed"); + return objectMapper.readValue(aggregateJson, BrandFileAggregateCacheDto.class); } catch (Exception ex) { throw new BusinessException("Read brand task aggregate failed"); } @@ -195,7 +198,8 @@ public class BrandTaskStorageService { continue; } try { - result.put(state.getScopeKey(), objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class)); + String aggregateJson = transientPayloadStorageService.resolvePayload(state.getStateJson(), "read brand task aggregate failed"); + result.put(state.getScopeKey(), objectMapper.readValue(aggregateJson, BrandFileAggregateCacheDto.class)); } catch (Exception ex) { throw new BusinessException("Read brand task aggregate failed"); } @@ -220,12 +224,22 @@ public class BrandTaskStorageService { return; } List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() - .select(TaskScopeStateEntity::getParsedPayloadJson) + .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) .eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); if (states != null) { for (TaskScopeStateEntity state : states) { - deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null); + transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson()); + transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson()); + } + } + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .select(TaskChunkEntity::getPayloadJson) + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + if (chunks != null) { + for (TaskChunkEntity chunk : chunks) { + transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); } } taskChunkMapper.delete(new LambdaQueryWrapper() @@ -246,12 +260,14 @@ public class BrandTaskStorageService { if (state == null) { throw new BusinessException("Brand task aggregate state missing: " + scopeKey); } + String storedAggregate = transientPayloadStorageService.storeScopePayload(MODULE_TYPE, taskId, scopeHash, aggregateJson, true); int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal(); int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount(); + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getStateJson(), storedAggregate); taskScopeStateMapper.update(null, new LambdaUpdateWrapper() .eq(TaskScopeStateEntity::getId, state.getId()) .set(TaskScopeStateEntity::getScopeKey, scopeKey) - .set(TaskScopeStateEntity::getStateJson, aggregateJson) + .set(TaskScopeStateEntity::getStateJson, storedAggregate) .set(TaskScopeStateEntity::getChunkTotal, chunkTotal) .set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount) .set(TaskScopeStateEntity::getCompleted, Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0) @@ -362,7 +378,8 @@ public class BrandTaskStorageService { private BrandCrawlResultFileDto readChunkPayload(String payloadJson) { try { - return objectMapper.readValue(payloadJson, BrandCrawlResultFileDto.class); + String resolvedPayload = transientPayloadStorageService.resolvePayload(payloadJson, "read brand task chunk failed"); + return objectMapper.readValue(resolvedPayload, BrandCrawlResultFileDto.class); } catch (Exception ex) { throw new BusinessException("Read brand task chunk failed"); } @@ -382,56 +399,17 @@ public class BrandTaskStorageService { } private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) { - try { - String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson); - return objectMapper.writeValueAsString(pointer); - } catch (Exception ex) { - throw new BusinessException("Save brand task parsed payload failed"); - } + return transientPayloadStorageService.storeParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson, true); } private String resolveParsedPayload(String value) { - if (isBlank(value)) { - return value; - } - String ossPointer = extractOssPointer(value); - if (ossPointer == null) { - return value; - } try { - return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length())); + return transientPayloadStorageService.resolvePayload(value, "Read brand task parsed payload failed"); } catch (Exception ex) { throw new BusinessException("Read brand task parsed payload failed"); } } - private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) { - String oldPointer = extractOssPointer(oldValue); - String newPointer = extractOssPointer(newValue); - if (oldPointer == null || oldPointer.equals(newPointer)) { - return; - } - try { - ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length())); - } catch (Exception ignored) { - } - } - - private String extractOssPointer(String value) { - if (isBlank(value)) { - return null; - } - if (value.startsWith(OSS_POINTER_PREFIX)) { - return value; - } - try { - String decoded = objectMapper.readValue(value, String.class); - return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null; - } catch (Exception ignored) { - return null; - } - } - private String writeJson(Object value, String message) { try { return objectMapper.writeValueAsString(value); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java index b8d3fd2..a8161ed 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java @@ -205,9 +205,7 @@ public class ConvertRunService { vo.setResultId(entity.getId()); vo.setSourceFilename(entity.getSourceFilename()); vo.setOutputFilename(entity.getResultFilename()); - vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1 - ? ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()) - : null); + vo.setDownloadUrl(null); vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); vo.setError(entity.getErrorMessage()); return vo; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java index 2d1ad09..804130e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java @@ -229,9 +229,7 @@ public class DedupeRunService { vo.setResultId(entity.getId()); vo.setSourceFilename(entity.getSourceFilename()); vo.setOutputFilename(entity.getResultFilename()); - vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1 - ? ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()) - : null); + vo.setDownloadUrl(null); vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); vo.setError(entity.getErrorMessage()); return vo; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/model/vo/DeleteBrandResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/model/vo/DeleteBrandResultItemVo.java index 5f50a2c..22e8c29 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/model/vo/DeleteBrandResultItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/model/vo/DeleteBrandResultItemVo.java @@ -50,6 +50,10 @@ public class DeleteBrandResultItemVo { @Schema(description = "下载地址") private String downloadUrl; + private Long fileJobId; + private String fileStatus; + private String fileError; + private Boolean fileReady; @Schema(description = "任务ID") private Long taskId; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java index f7d5a63..52e2e58 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java @@ -34,6 +34,8 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; @@ -80,6 +82,7 @@ public class DeleteBrandRunService { private final ObjectMapper objectMapper; private final DeleteBrandProgressProperties deleteBrandProgressProperties; private final TaskPressureProperties taskPressureProperties; + private final TaskFileJobService taskFileJobService; public DeleteBrandRunVo run(DeleteBrandRunRequest request) { if (request.getUserId() == null || request.getUserId() <= 0) { @@ -271,7 +274,8 @@ public class DeleteBrandRunService { item.setSourceFilename(entity.getSourceFilename()); item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename())); item.setOutputFilename(entity.getResultFilename()); - item.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.getPublicUrl(entity.getResultFileUrl())); + item.setDownloadUrl(null); + attachFileJobState(item, entity); item.setTotalRows(entity.getRowCount()); item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); item.setError(entity.getErrorMessage()); @@ -320,6 +324,18 @@ public class DeleteBrandRunService { return vo; } + private void attachFileJobState(DeleteBrandResultItemVo item, FileResultEntity entity) { + TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); + item.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); + if (job == null) { + item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null); + return; + } + item.setFileJobId(job.getId()); + item.setFileStatus(job.getStatus()); + item.setFileError(job.getErrorMessage()); + } + public DeleteBrandTaskDeletionStatusVo getTaskDeletionStatus(Long taskId, Long userId) { FileTaskEntity task = loadTaskForExecution(taskId); if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) { @@ -373,12 +389,19 @@ public class DeleteBrandRunService { Map> grouped = new LinkedHashMap<>(); Map displayCountryNames = new LinkedHashMap<>(); - for (int rowNum = 2; rowNum <= sheet.getLastRowNum(); rowNum++) { + int firstDataRow = pairs.stream() + .mapToInt(CountryColumnPair::dataStartRowIndex) + .min() + .orElse(2); + for (int rowNum = firstDataRow; rowNum <= sheet.getLastRowNum(); rowNum++) { Row row = sheet.getRow(rowNum); if (row == null) { continue; } for (CountryColumnPair pair : pairs) { + if (rowNum < pair.dataStartRowIndex()) { + continue; + } String country = pair.country(); String asin = normalizeCellText(formatter.formatCellValue(row.getCell(pair.asinColumnIndex()))); String status = normalizeCellText(formatter.formatCellValue(row.getCell(pair.statusColumnIndex()))); @@ -457,7 +480,10 @@ public class DeleteBrandRunService { continue; } if ("删除ASIN".equals(firstHeader) && "状态".equals(secondHeader)) { - pairs.add(new CountryColumnPair(title, i, i + 1)); + pairs.add(new CountryColumnPair(title, i, i + 1, 2)); + i++; + } else if (!"店铺名".equals(title) && "状态".equals(secondHeader)) { + pairs.add(new CountryColumnPair(title, i, i + 1, 1)); i++; } } @@ -486,7 +512,7 @@ public class DeleteBrandRunService { .replaceAll("\\s+", " "); } - private record CountryColumnPair(String country, int asinColumnIndex, int statusColumnIndex) { + private record CountryColumnPair(String country, int asinColumnIndex, int statusColumnIndex, int dataStartRowIndex) { } private String resolveFileIdentity(DeleteBrandSourceFileDto sourceFile) { @@ -1085,58 +1111,111 @@ public class DeleteBrandRunService { return; } - Map parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId); - if (parsedPayload == null || parsedPayload.isEmpty()) { + int expectedFiles = deleteBrandTaskStorageService.countParsedPayloadScopes(taskId); + if (expectedFiles <= 0) { + tryFinalizeTaskFromTerminalResults(task); return; } - - int expectedFiles = parsedPayload.size(); // 移除 finishedFilesHint 的强依赖，因为 Redis 里的 finished_files 可能是旧值 // 既然进入 tryFinalizeTask，就说明有新分片到达，应该以 loadMergedChunks 为准 - Map> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(taskId); - int finishedFiles = countCompletedFiles(mergedByFile); + int finishedFiles = deleteBrandTaskStorageService.countCompletedScopes(taskId); if (finishedFiles < expectedFiles) { - Map progress = new LinkedHashMap<>(); - progress.put("finished_files", String.valueOf(finishedFiles)); - // 定时补偿只负责“再试一次 finalize”，不能把缺片任务重新续命 - // 否则 stale-check 会一直看不到超时任务 - if (!fromCompensation) { - progress.put("updated_at", String.valueOf(System.currentTimeMillis())); - } - deleteBrandTaskCacheService.saveProgress(taskId, progress); - - task.setSuccessFileCount(finishedFiles); - if (!fromCompensation) { - task.setUpdatedAt(LocalDateTime.now()); - } - fileTaskMapper.updateById(task); - deleteBrandTaskCacheService.saveTaskCache(task); + updateTaskFinalizeProgress(task, finishedFiles, fromCompensation); log.info("[DeleteBrand] tryFinalizeTask skipping -> finishedFiles ({}) < expectedFiles ({})", finishedFiles, expectedFiles); return; } - Map progress = new LinkedHashMap<>(); - progress.put("finished_files", String.valueOf(finishedFiles)); - progress.put("updated_at", String.valueOf(System.currentTimeMillis())); - if (fromCompensation) { - progress.put("last_heartbeat_at", String.valueOf(System.currentTimeMillis())); + updateTaskFinalizeProgress(task, finishedFiles, true); + Map parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId); + if (parsedPayload == null || parsedPayload.isEmpty()) { + tryFinalizeTaskFromTerminalResults(task); + return; } - deleteBrandTaskCacheService.saveProgress(taskId, progress, true); + Map> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(taskId); - task.setSuccessFileCount(finishedFiles); - task.setUpdatedAt(LocalDateTime.now()); - fileTaskMapper.updateById(task); - deleteBrandTaskCacheService.saveTaskCache(task); - log.info("[DeleteBrand] tryFinalizeTask calling finalizeTask -> finishedFiles: {}, expectedFiles: {}", finishedFiles, expectedFiles); finalizeTask(task, parsedPayload, mergedByFile); } + private void updateTaskFinalizeProgress(FileTaskEntity task, int finishedFiles, boolean fromCompensation) { + Map progress = new LinkedHashMap<>(); + progress.put("finished_files", String.valueOf(finishedFiles)); + if (!fromCompensation) { + progress.put("updated_at", String.valueOf(System.currentTimeMillis())); + } + deleteBrandTaskCacheService.saveProgress(task.getId(), progress); + + Integer currentFinished = task.getSuccessFileCount(); + if (fromCompensation && currentFinished != null && currentFinished == finishedFiles) { + return; + } + task.setSuccessFileCount(finishedFiles); + if (!fromCompensation) { + task.setUpdatedAt(LocalDateTime.now()); + } + fileTaskMapper.updateById(task); + deleteBrandTaskCacheService.saveTaskCache(task); + } + + private void tryFinalizeTaskFromTerminalResults(FileTaskEntity task) { + if (task == null || task.getId() == null) { + return; + } + List resultEntities = fileResultMapper.selectList(new LambdaQueryWrapper() + .eq(FileResultEntity::getModuleType, MODULE_TYPE) + .eq(FileResultEntity::getTaskId, task.getId()) + .orderByAsc(FileResultEntity::getId)); + if (resultEntities == null || resultEntities.isEmpty()) { + return; + } + if (taskFileJobService.countUnfinishedAssembleJobs(task.getId(), MODULE_TYPE) > 0L) { + return; + } + + int successCount = 0; + int failedCount = 0; + for (FileResultEntity entity : resultEntities) { + boolean success = (entity.getSuccess() != null && entity.getSuccess() == 1) + || (entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); + boolean failed = !success && entity.getErrorMessage() != null && !entity.getErrorMessage().isBlank(); + if (!success && !failed) { + return; + } + if (success) { + successCount++; + } else { + failedCount++; + } + } + + if (task.getSourceFileCount() != null + && task.getSourceFileCount() > 0 + && successCount + failedCount < task.getSourceFileCount()) { + return; + } + + task.setStatus(successCount > 0 ? "SUCCESS" : "FAILED"); + task.setSuccessFileCount(successCount); + task.setFailedFileCount(failedCount); + task.setErrorMessage(successCount > 0 ? null : "任务中间数据缺失，已按结果行状态自动收尾"); + task.setUpdatedAt(LocalDateTime.now()); + task.setFinishedAt(LocalDateTime.now()); + fileTaskMapper.updateById(task); + deleteBrandTaskCacheService.saveTaskCache(task); + deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of( + "phase", successCount > 0 ? "success" : DeleteBrandTaskCacheService.PHASE_FAILED, + "finished_files", String.valueOf(successCount), + "updated_at", String.valueOf(System.currentTimeMillis()) + ), true); + log.info("[DeleteBrand] finalized orphan running task from terminal results -> taskId: {}, status: {}, successFiles: {}, failedFiles: {}", + task.getId(), task.getStatus(), successCount, failedCount); + } + private int countCompletedFiles(Map> mergedByFile) { int finishedFiles = 0; for (List chunks : mergedByFile.values()) { @@ -1205,28 +1284,22 @@ public class DeleteBrandRunService { if (!isMergedFileCompleted(chunks)) { throw new BusinessException("存在未完成的结果分片: " + parsedFile.getSourceFilename()); } - MergedDeleteBrandFile mergedFile = mergeChunks(parsedFile, chunks); - File outputFile = buildResultWorkbookPreserveLayout(task.getId(), parsedFile, mergedFile); + mergeChunks(parsedFile, chunks); - deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of( - "phase", DeleteBrandTaskCacheService.PHASE_UPLOADING, - "file_name", blankToEmpty(parsedFile.getSourceFilename()), - "updated_at", String.valueOf(System.currentTimeMillis()) - )); - - String objectKey = ossStorageService.uploadResultFile(outputFile, MODULE_TYPE); FileResultEntity resultEntity = resultEntityByIdentity.get(fileIdentity); if (resultEntity == null) { throw new BusinessException("结果记录不存在: " + parsedFile.getSourceFilename()); } - resultEntity.setResultFilename(outputFile.getName()); - resultEntity.setResultFileUrl(objectKey); - resultEntity.setResultFileSize(outputFile.length()); + String outputFilename = buildResultFilename(parsedFile); + resultEntity.setResultFilename(outputFilename); + resultEntity.setResultFileUrl(null); + resultEntity.setResultFileSize(0L); resultEntity.setResultContentType(CONTENT_TYPE_XLSX); resultEntity.setRowCount(parsedFile.getTotalRows()); resultEntity.setSuccess(1); resultEntity.setErrorMessage(null); fileResultMapper.updateById(resultEntity); + taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity); DeleteBrandResultItemVo item = new DeleteBrandResultItemVo(); item.setResultId(resultEntity.getId()); @@ -1235,8 +1308,8 @@ public class DeleteBrandRunService { item.setSourceFilename(parsedFile.getSourceFilename()); item.setShopName(parsedFile.getShopName()); item.setCompanyName(parsedFile.getCompanyName()); - item.setOutputFilename(outputFile.getName()); - item.setDownloadUrl(ossStorageService.getPublicUrl(objectKey)); + item.setOutputFilename(outputFilename); + item.setDownloadUrl(null); item.setTotalRows(parsedFile.getTotalRows()); item.setCountryCount(parsedFile.getCountries() == null ? 0 : parsedFile.getCountries().size()); item.setCountries(parsedFile.getCountries()); @@ -1263,8 +1336,6 @@ public class DeleteBrandRunService { task.setFinishedAt(LocalDateTime.now()); fileTaskMapper.updateById(task); deleteBrandTaskCacheService.saveTaskCache(task); - deleteBrandTaskStorageService.deleteTaskData(task.getId()); - deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of( "phase", "success", "file_index", String.valueOf(parsedPayload.size()), @@ -1293,6 +1364,75 @@ public class DeleteBrandRunService { } } + public void processResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null || job.getResultId() == null) { + throw new BusinessException("结果文件任务参数不完整"); + } + FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + FileResultEntity resultEntity = fileResultMapper.selectById(job.getResultId()); + if (resultEntity == null || !MODULE_TYPE.equals(resultEntity.getModuleType())) { + throw new BusinessException("结果记录不存在"); + } + Map parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(job.getTaskId()); + Map> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(job.getTaskId()); + String fileIdentity = blankToEmpty(job.getScopeKey()); + if (fileIdentity.isBlank()) { + fileIdentity = blankToEmpty(resultEntity.getSourceFileUrl()); + } + if (fileIdentity.isBlank()) { + fileIdentity = blankToEmpty(resultEntity.getSourceFilename()); + } + DeleteBrandParsedFileCacheDto parsedFile = parsedPayload.get(fileIdentity); + if (parsedFile == null) { + throw new BusinessException("任务原始数据不存在: " + fileIdentity); + } + List chunks = mergedByFile.get(fileIdentity); + if (!isMergedFileCompleted(chunks)) { + throw new BusinessException("结果分片未完整: " + fileIdentity); + } + MergedDeleteBrandFile mergedFile = mergeChunks(parsedFile, chunks); + File outputFile = buildResultWorkbookPreserveLayout(job.getTaskId(), parsedFile, mergedFile); + try { + deleteBrandTaskCacheService.saveProgress(job.getTaskId(), Map.of( + "phase", DeleteBrandTaskCacheService.PHASE_UPLOADING, + "file_name", blankToEmpty(parsedFile.getSourceFilename()), + "updated_at", String.valueOf(System.currentTimeMillis()) + )); + String objectKey = ossStorageService.uploadResultFile(outputFile, MODULE_TYPE); + resultEntity.setResultFilename(outputFile.getName()); + resultEntity.setResultFileUrl(objectKey); + resultEntity.setResultFileSize(outputFile.length()); + resultEntity.setResultContentType(CONTENT_TYPE_XLSX); + resultEntity.setRowCount(parsedFile.getTotalRows()); + resultEntity.setSuccess(1); + resultEntity.setErrorMessage(null); + fileResultMapper.updateById(resultEntity); + } finally { + deleteQuietly(outputFile); + } + } + + public void cleanupResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null) { + return; + } + if (taskFileJobService.countUnfinishedAssembleJobs(job.getTaskId(), MODULE_TYPE) == 0L) { + deleteBrandTaskStorageService.deleteTaskData(job.getTaskId()); + } + } + + private String buildResultFilename(DeleteBrandParsedFileCacheDto parsedFile) { + String filename = blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx"); + String lower = filename.toLowerCase(Locale.ROOT); + if (!lower.endsWith(".xlsx") && !lower.endsWith(".xls")) { + return filename + ".xlsx"; + } + return filename; + } + private MergedDeleteBrandFile mergeChunks(DeleteBrandParsedFileCacheDto parsedFile, List chunks) { Integer chunkTotal = chunks.get(0).getChunkTotal(); for (DeleteBrandResultFileDto chunk : chunks) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java index 9ebe69b..50ee53c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java @@ -27,7 +27,6 @@ import java.nio.file.attribute.FileTime; import java.time.Duration; import java.time.Instant; import java.time.LocalDateTime; -import java.time.ZoneId; import java.util.List; import java.util.Map; @@ -47,6 +46,7 @@ public class DeleteBrandStaleTaskService { private final FileTaskMapper fileTaskMapper; private final DeleteBrandTaskCacheService deleteBrandTaskCacheService; + private final DeleteBrandTaskStorageService deleteBrandTaskStorageService; private final DeleteBrandRunService deleteBrandRunService; private final ProductRiskTaskService productRiskTaskService; private final ProductRiskTaskCacheService productRiskTaskCacheService; @@ -110,7 +110,13 @@ public class DeleteBrandStaleTaskService { } private void failStaleDeleteBrandTasks() { - LocalDateTime threshold = LocalDateTime.now().minusMinutes(deleteBrandProgressProperties.getHeartbeatTimeoutMinutes()); + long minutes = Math.max(1L, deleteBrandProgressProperties.getHeartbeatTimeoutMinutes()); + long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getDeleteBrandInitialTimeoutMinutes()); + long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis(); + long nowMillis = System.currentTimeMillis(); + LocalDateTime now = LocalDateTime.now(); + LocalDateTime threshold = now.minusMinutes(minutes); + LocalDateTime initialThreshold = now.minusMinutes(initialMinutes); java.util.List runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper() .eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND) @@ -120,36 +126,35 @@ public class DeleteBrandStaleTaskService { for (FileTaskEntity task : runningTasks) { Map progress = deleteBrandTaskCacheService.getProgress(task.getId()); - boolean hasProgress = progress != null && !progress.isEmpty() && progress.containsKey("last_heartbeat_at"); - - boolean isActive = false; - if (progress != null && progress.containsKey("has_progress")) { - Object hpObj = progress.get("has_progress"); - if (hpObj instanceof Boolean) { - isActive = (Boolean) hpObj; - } else if (hpObj instanceof String) { - isActive = Boolean.parseBoolean((String) hpObj); - } - } - long lastHeartbeatAt = 0L; - if (hasProgress) { + if (progress != null && !progress.isEmpty() && progress.containsKey("last_heartbeat_at")) { try { lastHeartbeatAt = Long.parseLong(String.valueOf(progress.get("last_heartbeat_at"))); } catch (Exception ignored) { } } + boolean hasStartedProgress = lastHeartbeatAt > 0L; + if (!hasStartedProgress) { + hasStartedProgress = deleteBrandTaskStorageService.countCompletedScopes(task.getId()) > 0 + || deleteBrandTaskStorageService.countParsedPayloadScopes(task.getId()) > 0; + } - if (lastHeartbeatAt > 0 && isActive) { - LocalDateTime lastActivityTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastHeartbeatAt), ZoneId.systemDefault()); - if (lastActivityTime.isAfter(threshold)) { - continue; - } - } else { - LocalDateTime queuedThreshold = LocalDateTime.now().minusHours(12); - if (task.getCreatedAt() != null && task.getCreatedAt().isAfter(queuedThreshold)) { + if (lastHeartbeatAt > 0L && nowMillis - lastHeartbeatAt < staleTimeoutMillis) { + continue; + } + if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { + continue; + } + + try { + deleteBrandRunService.tryFinalizeTask(task.getId(), true); + FileTaskEntity refreshed = fileTaskMapper.selectById(task.getId()); + if (refreshed != null && !"RUNNING".equals(refreshed.getStatus())) { + deleteBrandTaskCacheService.saveTaskCache(refreshed); continue; } + } catch (Exception ex) { + log.warn("[stale-check] delete-brand finalize threw taskId={} msg={}", task.getId(), ex.getMessage()); } int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper() @@ -185,20 +190,24 @@ public class DeleteBrandStaleTaskService { if (runningTasks.isEmpty()) { return stats; } + Map heartbeatByTaskId = productRiskTaskCacheService.getTaskHeartbeatMillisBatch( + runningTasks.stream().map(FileTaskEntity::getId).toList()); log.info("[stale-check] product-risk candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}", runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes); for (FileTaskEntity task : runningTasks) { - boolean hasUploadedPayload = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId()); - long lastPayloadHeartbeatMillis = productRiskTaskCacheService.getTaskHeartbeatMillis(task.getId()); + long lastPayloadHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L); boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0) || (task.getFailedFileCount() != null && task.getFailedFileCount() > 0); - boolean hasStartedProgress = hasUploadedPayload || hasFinishedRows; if (lastPayloadHeartbeatMillis > 0L && nowMillis - lastPayloadHeartbeatMillis < staleTimeoutMillis) { stats.skippedTaskCount++; log.info("[stale-check] product-risk skip recent-task-heartbeat taskId={} lastPayloadHeartbeatMillis={} timeoutMinutes={} updatedAt={}", task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt()); continue; } + boolean hasStartedProgress = lastPayloadHeartbeatMillis > 0L || hasFinishedRows; + if (!hasStartedProgress) { + hasStartedProgress = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId()); + } if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { stats.skippedTaskCount++; log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}", @@ -250,18 +259,22 @@ public class DeleteBrandStaleTaskService { if (runningTasks.isEmpty()) { return stats; } + Map heartbeatByTaskId = priceTrackTaskCacheService.getTaskHeartbeatMillisBatch( + runningTasks.stream().map(FileTaskEntity::getId).toList()); log.info("[stale-check] price-track candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}", runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes); for (FileTaskEntity task : runningTasks) { - long lastHeartbeatMillis = priceTrackTaskCacheService.getTaskHeartbeatMillis(task.getId()); - boolean hasUploadedPayload = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId()); + long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L); boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0) || (task.getFailedFileCount() != null && task.getFailedFileCount() > 0); - boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows; if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) { stats.skippedTaskCount++; continue; } + boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows; + if (!hasStartedProgress) { + hasStartedProgress = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId()); + } if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { stats.skippedTaskCount++; continue; @@ -312,20 +325,24 @@ public class DeleteBrandStaleTaskService { if (runningTasks.isEmpty()) { return stats; } + Map heartbeatByTaskId = shopMatchTaskCacheService.getTaskHeartbeatMillisBatch( + runningTasks.stream().map(FileTaskEntity::getId).toList()); log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}", runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes); for (FileTaskEntity task : runningTasks) { - long lastHeartbeatMillis = shopMatchTaskCacheService.getTaskHeartbeatMillis(task.getId()); - boolean hasUploadedPayload = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId()); + long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L); boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0) || (task.getFailedFileCount() != null && task.getFailedFileCount() > 0); - boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows; if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) { stats.skippedTaskCount++; log.info("[stale-check] shop-match skip recent-task-heartbeat taskId={} lastHeartbeatMillis={} timeoutMinutes={} updatedAt={}", task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt()); continue; } + boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows; + if (!hasStartedProgress) { + hasStartedProgress = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId()); + } if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { stats.skippedTaskCount++; log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}", @@ -378,16 +395,20 @@ public class DeleteBrandStaleTaskService { if (runningTasks.isEmpty()) { return stats; } + Map heartbeatByTaskId = patrolDeleteTaskCacheService.getTaskHeartbeatMillisBatch( + runningTasks.stream().map(FileTaskEntity::getId).toList()); for (FileTaskEntity task : runningTasks) { - long lastHeartbeatMillis = patrolDeleteTaskCacheService.getTaskHeartbeatMillis(task.getId()); - boolean hasUploadedPayload = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId()); + long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L); boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0) || (task.getFailedFileCount() != null && task.getFailedFileCount() > 0); - boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows; if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) { stats.skippedTaskCount++; continue; } + boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows; + if (!hasStartedProgress) { + hasStartedProgress = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId()); + } if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) { stats.skippedTaskCount++; continue; @@ -431,6 +452,10 @@ public class DeleteBrandStaleTaskService { java.util.List runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper() .eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND) .eq(FileTaskEntity::getStatus, "RUNNING") + .and(wrapper -> wrapper + .gt(FileTaskEntity::getSuccessFileCount, 0) + .or() + .gt(FileTaskEntity::getFailedFileCount, 0)) .orderByAsc(FileTaskEntity::getUpdatedAt) .last("limit 100")); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskCacheService.java index 2552055..39bd653 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskCacheService.java @@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.deletebrand.service; import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.config.TaskPressureProperties; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; @@ -11,6 +12,7 @@ import java.util.concurrent.ConcurrentHashMap; @Service @RequiredArgsConstructor +@Slf4j public class DeleteBrandTaskCacheService { public static final String PHASE_CRAWLING = "crawling"; @@ -52,9 +54,13 @@ public class DeleteBrandTaskCacheService { } String key = buildProgressKey(taskId); - stringRedisTemplate.opsForHash().putAll(key, merged); - stringRedisTemplate.expire(key, Duration.ofHours(PAYLOAD_TTL_HOURS)); - progressRedisFlushAt.put(taskId, now); + try { + stringRedisTemplate.opsForHash().putAll(key, merged); + stringRedisTemplate.expire(key, Duration.ofHours(PAYLOAD_TTL_HOURS)); + progressRedisFlushAt.put(taskId, now); + } catch (Exception ex) { + log.warn("[delete-brand-cache] save progress degraded taskId={} msg={}", taskId, ex.getMessage()); + } } public java.util.Map getProgress(Long taskId) { @@ -63,7 +69,13 @@ public class DeleteBrandTaskCacheService { if (cached != null && cached.isFresh(now)) { return new java.util.LinkedHashMap<>(cached.values()); } - java.util.Map values = stringRedisTemplate.opsForHash().entries(buildProgressKey(taskId)); + java.util.Map values; + try { + values = stringRedisTemplate.opsForHash().entries(buildProgressKey(taskId)); + } catch (Exception ex) { + log.warn("[delete-brand-cache] load progress degraded taskId={} msg={}", taskId, ex.getMessage()); + return java.util.Collections.emptyMap(); + } if (!values.isEmpty()) { progressLocalCache.put(taskId, new LocalProgressCacheEntry(now, toStringMap(values))); } @@ -98,15 +110,24 @@ public class DeleteBrandTaskCacheService { return result; } - java.util.List pipelineResults = stringRedisTemplate.executePipelined( - (org.springframework.data.redis.core.RedisCallback) connection -> { - org.springframework.data.redis.serializer.RedisSerializer serializer = - stringRedisTemplate.getStringSerializer(); - for (Long taskId : missingTaskIds) { - connection.hGetAll(serializer.serialize(buildProgressKey(taskId))); - } - return null; - }); + java.util.List pipelineResults; + try { + pipelineResults = stringRedisTemplate.executePipelined( + (org.springframework.data.redis.core.RedisCallback) connection -> { + org.springframework.data.redis.serializer.RedisSerializer serializer = + stringRedisTemplate.getStringSerializer(); + for (Long taskId : missingTaskIds) { + connection.hGetAll(serializer.serialize(buildProgressKey(taskId))); + } + return null; + }); + } catch (Exception ex) { + log.warn("[delete-brand-cache] batch load progress degraded taskIds={} msg={}", missingTaskIds, ex.getMessage()); + for (Long taskId : missingTaskIds) { + result.put(taskId, java.util.Collections.emptyMap()); + } + return result; + } for (int i = 0; i < missingTaskIds.size(); i++) { Long taskId = missingTaskIds.get(i); @@ -129,8 +150,12 @@ public class DeleteBrandTaskCacheService { progressLocalCache.remove(taskId); progressRedisFlushAt.remove(taskId); taskEntityLocalCache.remove(taskId); - stringRedisTemplate.delete(buildProgressKey(taskId)); - stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + try { + stringRedisTemplate.delete(buildProgressKey(taskId)); + stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + } catch (Exception ex) { + log.warn("[delete-brand-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage()); + } } public void saveTaskCache(com.nanri.aiimage.modules.task.model.entity.FileTaskEntity task) { @@ -180,7 +205,13 @@ public class DeleteBrandTaskCacheService { } java.util.List keys = missingIds.stream().map(this::buildTaskEntityKey).toList(); - java.util.List values = stringRedisTemplate.opsForValue().multiGet(keys); + java.util.List values; + try { + values = stringRedisTemplate.opsForValue().multiGet(keys); + } catch (Exception ex) { + log.warn("[delete-brand-cache] batch load task cache degraded taskIds={} msg={}", missingIds, ex.getMessage()); + return result; + } for (int i = 0; i < missingIds.size(); i++) { Long taskId = missingIds.get(i); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java index 3cc4857..64388ff 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java @@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.modules.deletebrand.model.cache.DeleteBrandParsedFileCacheDto; import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandResultFileDto; -import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.dao.DuplicateKeyException; @@ -32,12 +32,11 @@ import java.util.Map; public class DeleteBrandTaskStorageService { private static final String MODULE_TYPE = "DELETE_BRAND"; - private static final String OSS_POINTER_PREFIX = "oss:"; private final TaskChunkMapper taskChunkMapper; private final TaskScopeStateMapper taskScopeStateMapper; private final ObjectMapper objectMapper; - private final OssStorageService ossStorageService; + private final TransientPayloadStorageService transientPayloadStorageService; @Transactional public void saveParsedPayload(Long taskId, Map parsedPayloadByScope) { @@ -81,7 +80,7 @@ public class DeleteBrandTaskStorageService { if (existing == null) { throw new BusinessException("failed to save delete-brand parsed payload"); } - deleteParsedPayloadIfNeeded(existing.getParsedPayloadJson(), parsedPayloadPointer); + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getParsedPayloadJson(), parsedPayloadPointer); taskScopeStateMapper.update(null, new LambdaUpdateWrapper() .eq(TaskScopeStateEntity::getId, existing.getId()) .set(TaskScopeStateEntity::getScopeKey, scopeKey) @@ -120,6 +119,28 @@ public class DeleteBrandTaskStorageService { return result; } + public int countParsedPayloadScopes(Long taskId) { + if (taskId == null || taskId <= 0) { + return 0; + } + Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .isNotNull(TaskScopeStateEntity::getParsedPayloadJson)); + return count == null ? 0 : count.intValue(); + } + + public int countCompletedScopes(Long taskId) { + if (taskId == null || taskId <= 0) { + return 0; + } + Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getCompleted, 1)); + return count == null ? 0 : count.intValue(); + } + @Transactional public boolean storeResultChunkIfChanged(Long taskId, String scopeKey, Integer chunkIndex, DeleteBrandResultFileDto chunk) { if (taskId == null || taskId <= 0) { @@ -145,6 +166,7 @@ public class DeleteBrandTaskStorageService { .last("limit 1")); boolean changed = true; if (existingChunk == null) { + String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); TaskChunkEntity entity = new TaskChunkEntity(); entity.setTaskId(taskId); entity.setModuleType(MODULE_TYPE); @@ -152,7 +174,7 @@ public class DeleteBrandTaskStorageService { entity.setScopeHash(scopeHash); entity.setChunkIndex(chunkIndex); entity.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal()); - entity.setPayloadJson(payloadJson); + entity.setPayloadJson(storedPayload); entity.setPayloadHash(payloadHash); entity.setCreatedAt(now); entity.setUpdatedAt(now); @@ -169,11 +191,13 @@ public class DeleteBrandTaskStorageService { changed = !payloadHash.equals(concurrentChunk.getPayloadHash()); concurrentChunk.setScopeKey(normalizedScopeKey); concurrentChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal()); - concurrentChunk.setPayloadJson(payloadJson); + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(concurrentChunk.getPayloadJson(), storedPayload); + concurrentChunk.setPayloadJson(storedPayload); concurrentChunk.setPayloadHash(payloadHash); concurrentChunk.setUpdatedAt(now); taskChunkMapper.updateById(concurrentChunk); } else { + transientPayloadStorageService.deletePayloadIfPresent(storedPayload); throw ex; } } @@ -181,7 +205,9 @@ public class DeleteBrandTaskStorageService { changed = !payloadHash.equals(existingChunk.getPayloadHash()); existingChunk.setScopeKey(normalizedScopeKey); existingChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal()); - existingChunk.setPayloadJson(payloadJson); + String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existingChunk.getPayloadJson(), storedPayload); + existingChunk.setPayloadJson(storedPayload); existingChunk.setPayloadHash(payloadHash); existingChunk.setUpdatedAt(now); taskChunkMapper.updateById(existingChunk); @@ -210,7 +236,8 @@ public class DeleteBrandTaskStorageService { continue; } try { - DeleteBrandResultFileDto dto = objectMapper.readValue(entity.getPayloadJson(), DeleteBrandResultFileDto.class); + String payloadJson = transientPayloadStorageService.resolvePayload(entity.getPayloadJson(), "failed to load delete-brand result chunks"); + DeleteBrandResultFileDto dto = objectMapper.readValue(payloadJson, DeleteBrandResultFileDto.class); grouped.computeIfAbsent(entity.getScopeKey(), ignored -> new ArrayList<>()).add(dto); } catch (Exception ex) { throw new BusinessException("failed to load delete-brand result chunks"); @@ -228,12 +255,22 @@ public class DeleteBrandTaskStorageService { return; } List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() - .select(TaskScopeStateEntity::getParsedPayloadJson) + .select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson) .eq(TaskScopeStateEntity::getTaskId, taskId) .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); if (states != null) { for (TaskScopeStateEntity state : states) { - deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null); + transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson()); + transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson()); + } + } + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .select(TaskChunkEntity::getPayloadJson) + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + if (chunks != null) { + for (TaskChunkEntity chunk : chunks) { + transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson()); } } taskChunkMapper.delete(new LambdaQueryWrapper() @@ -322,56 +359,17 @@ public class DeleteBrandTaskStorageService { } private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) { - try { - String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson); - return objectMapper.writeValueAsString(pointer); - } catch (Exception ex) { - throw new BusinessException("failed to save delete-brand parsed payload"); - } + return transientPayloadStorageService.storeParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson, true); } private String resolveParsedPayload(String value) { - if (isBlank(value)) { - return value; - } - String ossPointer = extractOssPointer(value); - if (ossPointer == null) { - return value; - } try { - return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length())); + return transientPayloadStorageService.resolvePayload(value, "failed to load delete-brand parsed payload"); } catch (Exception ex) { throw new BusinessException("failed to load delete-brand parsed payload"); } } - private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) { - String oldPointer = extractOssPointer(oldValue); - String newPointer = extractOssPointer(newValue); - if (oldPointer == null || oldPointer.equals(newPointer)) { - return; - } - try { - ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length())); - } catch (Exception ignored) { - } - } - - private String extractOssPointer(String value) { - if (isBlank(value)) { - return null; - } - if (value.startsWith(OSS_POINTER_PREFIX)) { - return value; - } - try { - String decoded = objectMapper.readValue(value, String.class); - return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null; - } catch (Exception ignored) { - return null; - } - } - private String writeJson(Object value, String message) { try { return objectMapper.writeValueAsString(value); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsObjectStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsObjectStorageService.java new file mode 100644 index 0000000..b2a9f07 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsObjectStorageService.java @@ -0,0 +1,85 @@ +package com.nanri.aiimage.modules.file.service.object; + +import com.nanri.aiimage.config.TransientStorageProperties; +import io.minio.GetObjectArgs; +import io.minio.MinioClient; +import io.minio.PutObjectArgs; +import io.minio.RemoveObjectArgs; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +@Service +@RequiredArgsConstructor +public class RustfsObjectStorageService { + + private final TransientStorageProperties properties; + + public boolean isConfigured() { + return notBlank(properties.getEndpoint()) + && notBlank(properties.getBucket()) + && notBlank(properties.getAccessKeyId()) + && notBlank(properties.getAccessKeySecret()); + } + + public String uploadText(String objectKey, String content) { + if (!isConfigured()) { + throw new IllegalStateException("transient storage is not configured"); + } + try (ByteArrayInputStream stream = new ByteArrayInputStream( + Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8))) { + buildClient().putObject(PutObjectArgs.builder() + .bucket(properties.getBucket()) + .object(objectKey) + .stream(stream, stream.available(), -1) + .contentType("application/json") + .build()); + return objectKey; + } catch (Exception ex) { + throw new IllegalStateException("failed to upload payload to transient storage", ex); + } + } + + public String readObjectAsString(String objectKey) { + if (!isConfigured()) { + throw new IllegalStateException("transient storage is not configured"); + } + try (var stream = buildClient().getObject(GetObjectArgs.builder() + .bucket(properties.getBucket()) + .object(objectKey) + .build())) { + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } catch (Exception ex) { + throw new IllegalStateException("failed to read payload from transient storage", ex); + } + } + + public void deleteObject(String objectKey) { + if (!isConfigured() || !notBlank(objectKey)) { + return; + } + try { + buildClient().removeObject(RemoveObjectArgs.builder() + .bucket(properties.getBucket()) + .object(objectKey) + .build()); + } catch (Exception ex) { + throw new IllegalStateException("failed to delete payload from transient storage", ex); + } + } + + private MinioClient buildClient() { + return MinioClient.builder() + .endpoint(properties.getEndpoint()) + .credentials(properties.getAccessKeyId(), properties.getAccessKeySecret()) + .region(properties.getRegion()) + .build(); + } + + private boolean notBlank(String value) { + return value != null && !value.isBlank(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/model/vo/PatrolDeleteResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/model/vo/PatrolDeleteResultItemVo.java index 9b827e2..838cff7 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/model/vo/PatrolDeleteResultItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/model/vo/PatrolDeleteResultItemVo.java @@ -43,6 +43,10 @@ public class PatrolDeleteResultItemVo { private LocalDateTime finishedAt; private String outputFilename; private String downloadUrl; + private Long fileJobId; + private String fileStatus; + private String fileError; + private Boolean fileReady; @JsonProperty("countrySections") private List countrySections = new ArrayList<>(); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskCacheService.java index 19212f1..083e02d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskCacheService.java @@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteShopPayloadD import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; @@ -13,11 +14,13 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @Service @RequiredArgsConstructor +@Slf4j public class PatrolDeleteTaskCacheService { private static final String MODULE_TYPE = "PATROL_DELETE"; @@ -60,7 +63,13 @@ public class PatrolDeleteTaskCacheService { if (taskId == null || taskId <= 0) { return 0L; } - String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId)); + String raw; + try { + raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId)); + } catch (Exception ex) { + log.warn("[patrol-delete-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + return 0L; + } if (raw == null || raw.isBlank()) { return 0L; } @@ -71,11 +80,50 @@ public class PatrolDeleteTaskCacheService { } } + public Map getTaskHeartbeatMillisBatch(List taskIds) { + Map result = new LinkedHashMap<>(); + if (taskIds == null || taskIds.isEmpty()) { + return result; + } + List normalized = taskIds.stream() + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + if (normalized.isEmpty()) { + return result; + } + List keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList(); + List values; + try { + values = stringRedisTemplate.opsForValue().multiGet(keys); + } catch (Exception ex) { + log.warn("[patrol-delete-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage()); + return result; + } + for (int i = 0; i < normalized.size(); i++) { + String raw = values != null && i < values.size() ? values.get(i) : null; + if (raw == null || raw.isBlank()) { + result.put(normalized.get(i), 0L); + continue; + } + try { + result.put(normalized.get(i), Long.parseLong(raw)); + } catch (NumberFormatException ignored) { + result.put(normalized.get(i), 0L); + } + } + return result; + } + public void touchTaskHeartbeat(Long taskId) { - stringRedisTemplate.opsForValue().set( - buildTaskHeartbeatKey(taskId), - String.valueOf(Instant.now().toEpochMilli()), - Duration.ofHours(PAYLOAD_TTL_HOURS)); + try { + stringRedisTemplate.opsForValue().set( + buildTaskHeartbeatKey(taskId), + String.valueOf(Instant.now().toEpochMilli()), + Duration.ofHours(PAYLOAD_TTL_HOURS)); + } catch (Exception ex) { + log.warn("[patrol-delete-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + } } public void deleteTaskCache(Long taskId) { @@ -83,8 +131,12 @@ public class PatrolDeleteTaskCacheService { return; } taskEntityLocalCache.remove(taskId); - stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); - stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + try { + stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); + stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + } catch (Exception ex) { + log.warn("[patrol-delete-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage()); + } taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE); } @@ -133,7 +185,13 @@ public class PatrolDeleteTaskCacheService { return result; } java.util.List keys = missingIds.stream().map(this::buildTaskEntityKey).toList(); - java.util.List values = stringRedisTemplate.opsForValue().multiGet(keys); + java.util.List values; + try { + values = stringRedisTemplate.opsForValue().multiGet(keys); + } catch (Exception ex) { + log.warn("[patrol-delete-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage()); + return result; + } for (int i = 0; i < missingIds.size(); i++) { Long taskId = missingIds.get(i); String val = values != null && i < values.size() ? values.get(i) : null; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java index ffd31a8..21298b0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java @@ -24,6 +24,10 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -45,6 +49,9 @@ public class PatrolDeleteTaskService { private static final String MODULE_TYPE = "PATROL_DELETE"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + private static final int RESULT_PENDING = -1; + private static final int RESULT_FAILED = 0; + private static final int RESULT_SUCCESS = 1; private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断"; private final FileTaskMapper fileTaskMapper; @@ -56,6 +63,9 @@ public class PatrolDeleteTaskService { private final ZiniaoShopSwitchService ziniaoShopSwitchService; private final ObjectMapper objectMapper; private final TaskPressureProperties taskPressureProperties; + private final TaskFileJobService taskFileJobService; + private final TaskResultItemService taskResultItemService; + private final TaskProgressSnapshotService taskProgressSnapshotService; private FileTaskEntity loadTaskForExecution(Long taskId) { Map cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId)); @@ -185,8 +195,7 @@ public class PatrolDeleteTaskService { batch.getMissingTaskIds().add(taskId); continue; } - List snapshots = buildSnapshotFromDb(task, taskRows); - batch.getItems().addAll(snapshots); + batch.getItems().addAll(buildProgressItems(task, taskRows)); } return batch; } @@ -258,12 +267,13 @@ public class PatrolDeleteTaskService { result.setSourceFilename(normalizedShopName); result.setSourceFileUrl(item.getShopId()); result.setUserId(request.getUserId()); - result.setSuccess(null); + result.setSuccess(RESULT_PENDING); result.setCreatedAt(now); fileResultMapper.insert(result); snapshots.add(toSnapshotVo(result, item, task.getStatus(), null)); } persistTaskJson(task, uniqueItems, snapshots); + taskCacheService.saveTaskCache(task); PatrolDeleteCreateTaskVo vo = new PatrolDeleteCreateTaskVo(); vo.setTaskId(task.getId()); @@ -320,6 +330,7 @@ public class PatrolDeleteTaskService { persistSnapshotJson(task, snapshots); fileTaskMapper.updateById(task); + taskCacheService.saveTaskCache(task); tryFinalizeTask(taskId, false); } @@ -396,6 +407,7 @@ public class PatrolDeleteTaskService { persistSnapshotJson(task, snapshots); fileTaskMapper.updateById(task); + taskCacheService.saveTaskCache(task); return changed; } @@ -442,6 +454,7 @@ public class PatrolDeleteTaskService { updateTaskStatusFromRows(task, rows); persistSnapshotJson(task, buildSnapshotFromDb(task, rows)); fileTaskMapper.updateById(task); + taskCacheService.saveTaskCache(task); } private long countTasks(Long userId, List statuses) { @@ -496,7 +509,11 @@ public class PatrolDeleteTaskService { private Map> buildSnapshotMap(Map taskMap) { Map> out = new LinkedHashMap<>(); for (Map.Entry entry : taskMap.entrySet()) { - out.put(entry.getKey(), indexSnapshotByResultId(parseTaskSnapshots(entry.getValue().getResultJson()))); + List snapshots = parseTaskSnapshots(entry.getValue().getResultJson()); + if (snapshots.isEmpty()) { + snapshots = taskResultItemService.listResultSnapshots(entry.getKey(), MODULE_TYPE, PatrolDeleteResultItemVo.class); + } + out.put(entry.getKey(), indexSnapshotByResultId(snapshots)); } return out; } @@ -508,14 +525,13 @@ public class PatrolDeleteTaskService { item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename())); item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl())); item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus()); - item.setSuccess(entity.getSuccess() == null ? item.getSuccess() : entity.getSuccess() == 1); + item.setSuccess(toSuccessFlag(entity.getSuccess(), item.getSuccess())); item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError()); item.setCreatedAt(entity.getCreatedAt()); item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt()); item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename())); - item.setDownloadUrl(blank(entity.getResultFileUrl()) - ? item.getDownloadUrl() - : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())); + item.setDownloadUrl(null); + attachFileJobState(item, entity); if (item.getCountrySections() == null) { item.setCountrySections(new ArrayList<>()); } @@ -525,6 +541,18 @@ public class PatrolDeleteTaskService { return item; } + private void attachFileJobState(PatrolDeleteResultItemVo item, FileResultEntity entity) { + TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); + item.setFileReady(!blank(entity.getResultFileUrl())); + if (job == null) { + item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null); + return; + } + item.setFileJobId(job.getId()); + item.setFileStatus(job.getStatus()); + item.setFileError(job.getErrorMessage()); + } + private List dedupeItems(List items) { LinkedHashMap map = new LinkedHashMap<>(); for (PatrolDeleteTaskItemDto item : items) { @@ -554,7 +582,7 @@ public class PatrolDeleteTaskService { vo.setMatchStatus(item.getMatchStatus()); vo.setMatchMessage(item.getMatchMessage()); vo.setTaskStatus(taskStatus); - vo.setSuccess(result.getSuccess() == null ? null : result.getSuccess() == 1); + vo.setSuccess(toSuccessFlag(result.getSuccess(), null)); vo.setError(result.getErrorMessage()); vo.setCreatedAt(result.getCreatedAt()); vo.setFinishedAt(finishedAt); @@ -569,6 +597,7 @@ public class PatrolDeleteTaskService { try { task.setRequestJson(objectMapper.writeValueAsString(requestItems)); task.setResultJson(objectMapper.writeValueAsString(snapshots)); + syncSnapshotTables(task, snapshots); fileTaskMapper.updateById(task); } catch (Exception ex) { throw new BusinessException("巡店删除任务快照保存失败"); @@ -585,6 +614,27 @@ public class PatrolDeleteTaskService { return list; } + private List buildProgressItems(FileTaskEntity task, List rows) { + List list = new ArrayList<>(); + for (FileResultEntity row : rows) { + PatrolDeleteResultItemVo item = new PatrolDeleteResultItemVo(); + item.setResultId(row.getId()); + item.setTaskId(row.getTaskId()); + item.setShopName(row.getSourceFilename()); + item.setShopId(row.getSourceFileUrl()); + item.setTaskStatus(task == null ? null : task.getStatus()); + item.setSuccess(toSuccessFlag(row.getSuccess(), null)); + item.setError(row.getErrorMessage()); + item.setCreatedAt(row.getCreatedAt()); + item.setFinishedAt(task == null ? null : task.getFinishedAt()); + item.setOutputFilename(row.getResultFilename()); + item.setDownloadUrl(null); + attachFileJobState(item, row); + list.add(item); + } + return list; + } + private Map indexSnapshotByResultId(List snapshots) { Map map = new LinkedHashMap<>(); if (snapshots == null) { @@ -599,8 +649,8 @@ public class PatrolDeleteTaskService { } private void updateTaskStatusFromRows(FileTaskEntity task, List rows) { - long successCount = rows.stream().filter(row -> Integer.valueOf(1).equals(row.getSuccess())).count(); - long failedCount = rows.stream().filter(row -> Integer.valueOf(0).equals(row.getSuccess())).count(); + long successCount = rows.stream().filter(row -> Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())).count(); + long failedCount = rows.stream().filter(row -> Integer.valueOf(RESULT_FAILED).equals(row.getSuccess())).count(); long pendingCount = rows.stream().filter(row -> !isResultFinished(row)).count(); task.setSuccessFileCount((int) successCount); task.setFailedFileCount((int) failedCount); @@ -753,27 +803,25 @@ public class PatrolDeleteTaskService { .filter(item -> Boolean.TRUE.equals(item.getSuccess())) .toList(); if (!successItems.isEmpty()) { - File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "patrol-delete-result", String.valueOf(task.getId()))); - String filename = safeFileStem("巡店删除-" + task.getId()) + ".xlsx"; - File xlsx = FileUtil.file(workRoot, filename); - try { - excelAssemblyService.writeWorkbook(xlsx, successItems); - String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE); - long fileSize = xlsx.length(); - int rowCount = excelAssemblyService.countRows(successItems); + String filename = buildTaskWorkbookFilename(task); + int rowCount = excelAssemblyService.countRows(successItems); + FileResultEntity firstSuccessRow = null; for (FileResultEntity row : rows) { - if (Integer.valueOf(1).equals(row.getSuccess())) { + if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) { row.setResultFilename(filename); - row.setResultFileUrl(objectKey); - row.setResultFileSize(fileSize); + row.setResultFileUrl(null); + row.setResultFileSize(0L); row.setResultContentType(CONTENT_TYPE_XLSX); row.setRowCount(rowCount); fileResultMapper.updateById(row); + if (firstSuccessRow == null) { + firstSuccessRow = row; + } } } + if (firstSuccessRow != null) { + taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId()); snapshots = buildSnapshotFromDb(task, rows); - } finally { - FileUtil.del(xlsx); } } @@ -782,14 +830,56 @@ public class PatrolDeleteTaskService { taskCacheService.deleteTaskCache(task.getId()); } + public void processResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null) { + throw new BusinessException("结果文件任务参数不完整"); + } + FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + List rows = listTaskRows(job.getTaskId()); + List snapshots = taskResultItemService.listResultSnapshots(job.getTaskId(), MODULE_TYPE, PatrolDeleteResultItemVo.class); + if (snapshots.isEmpty()) { + snapshots = buildSnapshotFromDb(task, rows); + } + List successItems = snapshots.stream() + .filter(item -> Boolean.TRUE.equals(item.getSuccess())) + .toList(); + if (successItems.isEmpty()) { + throw new BusinessException("没有可生成的巡店删除结果"); + } + File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "patrol-delete-result", String.valueOf(task.getId()))); + String filename = buildTaskWorkbookFilename(task); + File xlsx = FileUtil.file(workRoot, filename); + try { + excelAssemblyService.writeWorkbook(xlsx, successItems); + String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE); + long fileSize = xlsx.length(); + int rowCount = excelAssemblyService.countRows(successItems); + for (FileResultEntity row : rows) { + if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) { + row.setResultFilename(filename); + row.setResultFileUrl(objectKey); + row.setResultFileSize(fileSize); + row.setResultContentType(CONTENT_TYPE_XLSX); + row.setRowCount(rowCount); + fileResultMapper.updateById(row); + } + } + } finally { + FileUtil.del(xlsx); + } + } + private void markResultSuccess(FileResultEntity row) { - row.setSuccess(1); + row.setSuccess(RESULT_SUCCESS); row.setErrorMessage(null); fileResultMapper.updateById(row); } private void markResultFailed(FileResultEntity row, String message) { - row.setSuccess(0); + row.setSuccess(RESULT_FAILED); row.setErrorMessage(blankToNull(message)); row.setResultFilename(null); row.setResultFileUrl(null); @@ -800,7 +890,14 @@ public class PatrolDeleteTaskService { } private boolean isResultFinished(FileResultEntity row) { - return Integer.valueOf(1).equals(row.getSuccess()) || Integer.valueOf(0).equals(row.getSuccess()); + return Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess()) || Integer.valueOf(RESULT_FAILED).equals(row.getSuccess()); + } + + private Boolean toSuccessFlag(Integer dbValue, Boolean fallback) { + if (dbValue == null || Integer.valueOf(RESULT_PENDING).equals(dbValue)) { + return fallback; + } + return Integer.valueOf(RESULT_SUCCESS).equals(dbValue); } private List parseTaskSnapshots(String json) { @@ -818,11 +915,39 @@ public class PatrolDeleteTaskService { private void persistSnapshotJson(FileTaskEntity task, List snapshots) { try { task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots)); + syncSnapshotTables(task, snapshots); } catch (Exception ex) { throw new BusinessException("巡店删除任务快照保存失败"); } } + private void syncSnapshotTables(FileTaskEntity task, List snapshots) { + List safe = snapshots == null ? List.of() : snapshots; + taskResultItemService.replaceTaskSnapshots(task.getId(), MODULE_TYPE, safe, new TaskResultItemService.SnapshotKeyResolver() { + @Override + public Long resultId(Object snapshot) { + return ((PatrolDeleteResultItemVo) snapshot).getResultId(); + } + + @Override + public String scopeKey(Object snapshot) { + PatrolDeleteResultItemVo item = (PatrolDeleteResultItemVo) snapshot; + return firstNonBlank(item.getShopName(), "result:" + item.getResultId()); + } + }); + int successCount = 0; + int failedCount = 0; + for (PatrolDeleteResultItemVo item : safe) { + if (Boolean.TRUE.equals(item.getSuccess())) { + successCount++; + } else if (Boolean.FALSE.equals(item.getSuccess())) { + failedCount++; + } + } + taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, firstNonBlank(task.getStatus(), "RUNNING"), + safe.size(), successCount, failedCount, null, task.getErrorMessage(), null); + } + private List copyCountrySections(List sections) { List copy = new ArrayList<>(); if (sections == null) { @@ -861,6 +986,11 @@ public class PatrolDeleteTaskService { return !blank(first) ? first : second; } + private String buildTaskWorkbookFilename(FileTaskEntity task) { + Long taskId = task == null ? null : task.getId(); + return safeFileStem("巡店删除-" + (taskId == null ? "result" : taskId)) + ".xlsx"; + } + private String safeFileStem(String value) { String raw = value == null ? "result" : value.trim(); String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_"); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java index 346beac..c74e581 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java @@ -1,16 +1,20 @@ package com.nanri.aiimage.modules.permission.service; -import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; import java.util.List; +import java.util.concurrent.CompletableFuture; @Component @RequiredArgsConstructor @Slf4j +@ConditionalOnProperty(prefix = "aiimage.permission-schema-init", name = "enabled", havingValue = "true") public class PermissionMenuSchemaInitializer { private final JdbcTemplate jdbcTemplate; @@ -33,8 +37,12 @@ public class PermissionMenuSchemaInitializer { new DefaultAdminMenu("版本管理", "admin_version", "version", 80) ); - @PostConstruct + @EventListener(ApplicationReadyEvent.class) public void initialize() { + CompletableFuture.runAsync(this::initializeInternal); + } + + private void initializeInternal() { executeQuietly(""" CREATE TABLE IF NOT EXISTS columns ( id INT AUTO_INCREMENT PRIMARY KEY, diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java index 8a8a811..8caa38d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java @@ -215,7 +215,7 @@ public class PriceTrackController { + "未完成/处理中：本次只传增量行数据，shop.success 不传或传 null，后端仅合并缓存继续等待。" + "已完成：传最后一批数据并设置 shop.success=true，后端会使用该店铺累计后的完整数据出结果。" + "失败：传 shop.success=false 或传非空 shop.error。" - + "行级数据只保留表头字段：shopMallName、asin、price、recommendedPrice、minimumPrice、firstPlace、secondPlace、cartShopName、priceChangeStatus、modifyCount、status。" + + "行级数据只保留表头字段：shopMallName、asin、price、recommendedPrice、shippingFee、minimumPrice、firstPlace、secondPlace、cartShopName、priceChangeStatus、modifyCount、status。" ) public ApiResponse submitResult( @Parameter( diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/dto/PriceTrackSubmitResultRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/dto/PriceTrackSubmitResultRequest.java index c9f8762..e0aeff8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/dto/PriceTrackSubmitResultRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/dto/PriceTrackSubmitResultRequest.java @@ -73,6 +73,9 @@ public class PriceTrackSubmitResultRequest { @Schema(description = "推荐价", example = "18.99") private String recommendedPrice; + @Schema(description = "运费", example = "2.50") + private String shippingFee; + @Schema(description = "最低价", example = "18.50") private String minimumPrice; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackResultItemVo.java index 444e775..0e3b3d7 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackResultItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackResultItemVo.java @@ -52,6 +52,10 @@ public class PriceTrackResultItemVo { @Schema(description = "预签名下载 URL，列表可能带回，也可通过下载接口获取") private String downloadUrl; + private Long fileJobId; + private String fileStatus; + private String fileError; + private Boolean fileReady; @Schema(description = "所属循环任务 ID", example = "1") private Long loopRunId; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackExcelAssemblyService.java index 2741df9..6d313a1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackExcelAssemblyService.java @@ -24,6 +24,7 @@ public class PriceTrackExcelAssemblyService { "ASIN", "价格", "推荐价", + "运费", "最低价", "第一名", "第二名", @@ -56,13 +57,14 @@ public class PriceTrackExcelAssemblyService { row.createCell(1).setCellValue(valueOf(item.getAsin())); row.createCell(2).setCellValue(valueOf(item.getPrice())); row.createCell(3).setCellValue(valueOf(item.getRecommendedPrice())); - row.createCell(4).setCellValue(valueOf(item.getMinimumPrice())); - row.createCell(5).setCellValue(valueOf(item.getFirstPlace())); - row.createCell(6).setCellValue(valueOf(item.getSecondPlace())); - row.createCell(7).setCellValue(valueOf(item.getCartShopName())); - row.createCell(8).setCellValue(valueOf(item.getPriceChangeStatus())); - row.createCell(9).setCellValue(valueOf(item.getModifyCount())); - row.createCell(10).setCellValue(valueOf(item.getStatus())); + row.createCell(4).setCellValue(valueOf(item.getShippingFee())); + row.createCell(5).setCellValue(valueOf(item.getMinimumPrice())); + row.createCell(6).setCellValue(valueOf(item.getFirstPlace())); + row.createCell(7).setCellValue(valueOf(item.getSecondPlace())); + row.createCell(8).setCellValue(valueOf(item.getCartShopName())); + row.createCell(9).setCellValue(valueOf(item.getPriceChangeStatus())); + row.createCell(10).setCellValue(valueOf(item.getModifyCount())); + row.createCell(11).setCellValue(valueOf(item.getStatus())); } applyDefaultColumnWidths(sheet); } @@ -112,7 +114,7 @@ public class PriceTrackExcelAssemblyService { } private void applyDefaultColumnWidths(Sheet sheet) { - int[] widths = {22, 18, 12, 12, 12, 20, 20, 22, 18, 12, 12}; + int[] widths = {22, 18, 12, 12, 12, 12, 20, 20, 22, 18, 12, 12}; for (int i = 0; i < widths.length; i++) { sheet.setColumnWidth(i, widths[i] * 256); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskCacheService.java index 6570567..13c104f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskCacheService.java @@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequ import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; @@ -13,11 +14,13 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @Service @RequiredArgsConstructor +@Slf4j public class PriceTrackTaskCacheService { private static final String MODULE_TYPE = "PRICE_TRACK"; @@ -32,17 +35,27 @@ public class PriceTrackTaskCacheService { if (taskId == null || taskId <= 0) { return; } - stringRedisTemplate.opsForValue().set( - buildTaskHeartbeatKey(taskId), - String.valueOf(Instant.now().toEpochMilli()), - Duration.ofHours(HEARTBEAT_TTL_HOURS)); + try { + stringRedisTemplate.opsForValue().set( + buildTaskHeartbeatKey(taskId), + String.valueOf(Instant.now().toEpochMilli()), + Duration.ofHours(HEARTBEAT_TTL_HOURS)); + } catch (Exception ex) { + log.warn("[price-track-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + } } public long getTaskHeartbeatMillis(Long taskId) { if (taskId == null || taskId <= 0) { return 0L; } - String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId)); + String raw; + try { + raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId)); + } catch (Exception ex) { + log.warn("[price-track-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + return 0L; + } if (raw == null || raw.isBlank()) { return 0L; } @@ -53,6 +66,41 @@ public class PriceTrackTaskCacheService { } } + public Map getTaskHeartbeatMillisBatch(List taskIds) { + Map result = new LinkedHashMap<>(); + if (taskIds == null || taskIds.isEmpty()) { + return result; + } + List normalized = taskIds.stream() + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + if (normalized.isEmpty()) { + return result; + } + List keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList(); + List values; + try { + values = stringRedisTemplate.opsForValue().multiGet(keys); + } catch (Exception ex) { + log.warn("[price-track-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage()); + return result; + } + for (int i = 0; i < normalized.size(); i++) { + String raw = values != null && i < values.size() ? values.get(i) : null; + if (raw == null || raw.isBlank()) { + result.put(normalized.get(i), 0L); + continue; + } + try { + result.put(normalized.get(i), Long.parseLong(raw)); + } catch (NumberFormatException ignored) { + result.put(normalized.get(i), 0L); + } + } + return result; + } + public PriceTrackSubmitResultRequest.ShopResult getShopMergedPayload(Long taskId, String shopKey) { return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PriceTrackSubmitResultRequest.ShopResult.class); } @@ -82,8 +130,12 @@ public class PriceTrackTaskCacheService { return; } taskEntityLocalCache.remove(taskId); - stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); - stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + try { + stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); + stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + } catch (Exception ex) { + log.warn("[price-track-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage()); + } taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE); } @@ -132,7 +184,13 @@ public class PriceTrackTaskCacheService { return result; } java.util.List keys = missingIds.stream().map(this::buildTaskEntityKey).toList(); - java.util.List values = stringRedisTemplate.opsForValue().multiGet(keys); + java.util.List values; + try { + values = stringRedisTemplate.opsForValue().multiGet(keys); + } catch (Exception ex) { + log.warn("[price-track-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage()); + return result; + } for (int i = 0; i < missingIds.size(); i++) { Long taskId = missingIds.get(i); String val = values != null && i < values.size() ? values.get(i) : null; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java index 6c43fb3..e9ece5c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java @@ -26,6 +26,9 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskResultPayloadService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import cn.hutool.core.util.IdUtil; import lombok.RequiredArgsConstructor; @@ -66,6 +69,8 @@ public class PriceTrackTaskService { private final PriceTrackTaskCacheService priceTrackTaskCacheService; private final PriceTrackLoopRunService priceTrackLoopRunService; private final TaskPressureProperties taskPressureProperties; + private final TaskResultPayloadService taskResultPayloadService; + private final TaskFileJobService taskFileJobService; private FileTaskEntity loadTaskForExecution(Long taskId) { Map cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId)); @@ -435,7 +440,7 @@ public class PriceTrackTaskService { continue; } try { - assembleShopResult(fr, shopKey, merged); + enqueueResultFileAssembly(fr, shopKey, merged); priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey); } catch (Exception ex) { markResultFailed(fr, ex.getMessage() == null ? "assemble price-track result failed" : ex.getMessage()); @@ -509,7 +514,7 @@ public class PriceTrackTaskService { continue; } try { - assembleShopResult(fr, shopKey, cachedPayload); + enqueueResultFileAssembly(fr, shopKey, cachedPayload); priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey); changed = true; log.warn("[price-track] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}", @@ -618,6 +623,49 @@ public class PriceTrackTaskService { } } + public void processResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null || job.getResultId() == null || job.getScopeKey() == null) { + throw new BusinessException("结果文件任务参数不完整"); + } + FileResultEntity result = fileResultMapper.selectById(job.getResultId()); + if (result == null || !MODULE_TYPE.equals(result.getModuleType())) { + throw new BusinessException("结果记录不存在"); + } + PriceTrackSubmitResultRequest.ShopResult payload = taskResultPayloadService.getLatest( + job.getTaskId(), MODULE_TYPE, job.getScopeKey(), PriceTrackSubmitResultRequest.ShopResult.class); + if (payload == null) { + throw new BusinessException("结果文件载荷不存在"); + } + assembleShopResult(result, job.getScopeKey(), payload); + } + + private void enqueueResultFileAssembly(FileResultEntity result, + String shopKey, + PriceTrackSubmitResultRequest.ShopResult payload) { + taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload); + markResultFilePending(result, shopKey, payload); + taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey); + } + + private void markResultFilePending(FileResultEntity result, + String shopKey, + PriceTrackSubmitResultRequest.ShopResult payload) { + Map> countries = + excelAssemblyService.normalizeCountriesMap(payload.getCountries()); + String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() + ? payload.getShopName().trim() + : shopKey; + String stem = safeFileStem(displayName); + result.setSuccess(1); + result.setErrorMessage(null); + result.setResultFilename(stem + ".xlsx"); + result.setResultFileUrl(null); + result.setResultFileSize(0L); + result.setResultContentType(CONTENT_TYPE_XLSX); + result.setRowCount(excelAssemblyService.countRows(countries)); + fileResultMapper.updateById(result); + } + public Map>> parseMatchAsinRowsByCountry(List asinFiles, List countryCodes) { return parseAsinRowsByCountry(asinFiles, countryCodes); } @@ -901,8 +949,8 @@ public class PriceTrackTaskService { vo.setSuccess(ok); vo.setError(entity.getErrorMessage()); vo.setOutputFilename(entity.getResultFilename()); - vo.setDownloadUrl(entity.getResultFileUrl() == null || entity.getResultFileUrl().isBlank() - ? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())); + vo.setDownloadUrl(null); + attachFileJobState(vo, entity); vo.setMatched(true); if (entity.getTaskId() != null) { mergeQueueFieldsFromRequest(vo, entity.getTaskId()); @@ -910,6 +958,18 @@ public class PriceTrackTaskService { return vo; } + private void attachFileJobState(PriceTrackResultItemVo vo, FileResultEntity entity) { + TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); + vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); + if (job == null) { + vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); + return; + } + vo.setFileJobId(job.getId()); + vo.setFileStatus(job.getStatus()); + vo.setFileError(job.getErrorMessage()); + } + private PriceTrackTaskDetailVo buildTaskDetail(FileTaskEntity task) { PriceTrackTaskDetailVo detail = new PriceTrackTaskDetailVo(); detail.setTask(toTaskItemVo(task)); @@ -1074,6 +1134,7 @@ public class PriceTrackTaskService { merged.setAsin(firstNonBlank(incoming.getAsin(), merged.getAsin())); merged.setPrice(firstNonBlank(incoming.getPrice(), merged.getPrice())); merged.setRecommendedPrice(firstNonBlank(incoming.getRecommendedPrice(), merged.getRecommendedPrice())); + merged.setShippingFee(firstNonBlank(incoming.getShippingFee(), merged.getShippingFee())); merged.setMinimumPrice(firstNonBlank(incoming.getMinimumPrice(), merged.getMinimumPrice())); merged.setFirstPlace(firstNonBlank(incoming.getFirstPlace(), merged.getFirstPlace())); merged.setSecondPlace(firstNonBlank(incoming.getSecondPlace(), merged.getSecondPlace())); @@ -1097,6 +1158,7 @@ public class PriceTrackTaskService { out.setAsin(row.getAsin()); out.setPrice(row.getPrice()); out.setRecommendedPrice(row.getRecommendedPrice()); + out.setShippingFee(row.getShippingFee()); out.setMinimumPrice(row.getMinimumPrice()); out.setFirstPlace(row.getFirstPlace()); out.setSecondPlace(row.getSecondPlace()); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskResultItemVo.java index 41a6096..2504c58 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskResultItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskResultItemVo.java @@ -57,6 +57,10 @@ public class ProductRiskResultItemVo { @JsonProperty("downloadUrl") @Schema(description = "预签名下载 URL（列表里可能带；直链下载也可用 GET /results/{resultId}/download）") private String downloadUrl; + private Long fileJobId; + private String fileStatus; + private String fileError; + private Boolean fileReady; @JsonProperty("scheduledAt") @Schema(description = "定时执行时间，未设置时为空") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskCacheService.java index db0e5c9..14c6f56 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskCacheService.java @@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskShopPayloadDto import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; @@ -13,11 +14,13 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @Service @RequiredArgsConstructor +@Slf4j public class ProductRiskTaskCacheService { private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE"; @@ -60,7 +63,13 @@ public class ProductRiskTaskCacheService { if (taskId == null || taskId <= 0) { return 0L; } - String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId)); + String raw; + try { + raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId)); + } catch (Exception ex) { + log.warn("[product-risk-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + return 0L; + } if (raw == null || raw.isBlank()) { return 0L; } @@ -71,13 +80,52 @@ public class ProductRiskTaskCacheService { } } + public Map getTaskHeartbeatMillisBatch(List taskIds) { + Map result = new LinkedHashMap<>(); + if (taskIds == null || taskIds.isEmpty()) { + return result; + } + List normalized = taskIds.stream() + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + if (normalized.isEmpty()) { + return result; + } + List keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList(); + List values; + try { + values = stringRedisTemplate.opsForValue().multiGet(keys); + } catch (Exception ex) { + log.warn("[product-risk-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage()); + return result; + } + for (int i = 0; i < normalized.size(); i++) { + String raw = values != null && i < values.size() ? values.get(i) : null; + if (raw == null || raw.isBlank()) { + result.put(normalized.get(i), 0L); + continue; + } + try { + result.put(normalized.get(i), Long.parseLong(raw)); + } catch (NumberFormatException ignored) { + result.put(normalized.get(i), 0L); + } + } + return result; + } + public void deleteTaskCache(Long taskId) { if (taskId == null || taskId <= 0) { return; } taskEntityLocalCache.remove(taskId); - stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); - stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + try { + stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); + stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + } catch (Exception ex) { + log.warn("[product-risk-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage()); + } taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE); } @@ -126,7 +174,13 @@ public class ProductRiskTaskCacheService { return result; } java.util.List keys = missingIds.stream().map(this::buildTaskEntityKey).toList(); - java.util.List values = stringRedisTemplate.opsForValue().multiGet(keys); + java.util.List values; + try { + values = stringRedisTemplate.opsForValue().multiGet(keys); + } catch (Exception ex) { + log.warn("[product-risk-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage()); + return result; + } for (int i = 0; i < missingIds.size(); i++) { Long taskId = missingIds.get(i); String val = values != null && i < values.size() ? values.get(i) : null; @@ -144,10 +198,14 @@ public class ProductRiskTaskCacheService { } public void touchTaskHeartbeat(Long taskId) { - stringRedisTemplate.opsForValue().set( - buildTaskHeartbeatKey(taskId), - String.valueOf(Instant.now().toEpochMilli()), - Duration.ofHours(PAYLOAD_TTL_HOURS)); + try { + stringRedisTemplate.opsForValue().set( + buildTaskHeartbeatKey(taskId), + String.valueOf(Instant.now().toEpochMilli()), + Duration.ofHours(PAYLOAD_TTL_HOURS)); + } catch (Exception ex) { + log.warn("[product-risk-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + } } private String buildTaskHeartbeatKey(Long taskId) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java index bb00826..1744916 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java @@ -24,6 +24,11 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import com.nanri.aiimage.modules.task.service.TaskResultItemService; +import com.nanri.aiimage.modules.task.service.TaskResultPayloadService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.IdUtil; @@ -59,6 +64,10 @@ public class ProductRiskTaskService { private final ObjectMapper objectMapper; private final ProductRiskTaskCacheService productRiskTaskCacheService; private final TaskPressureProperties taskPressureProperties; + private final TaskResultPayloadService taskResultPayloadService; + private final TaskFileJobService taskFileJobService; + private final TaskResultItemService taskResultItemService; + private final TaskProgressSnapshotService taskProgressSnapshotService; private FileTaskEntity loadTaskForExecution(Long taskId) { Map cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId)); @@ -553,7 +562,7 @@ public class ProductRiskTaskService { } try { - assembleShopResult(fr, shopKey, mergedPayload, workRoot); + enqueueResultFileAssembly(fr, shopKey, mergedPayload); assembledCount++; log.info("[product-risk] shop assembled taskId={} shop={} rowCount={} file={}", taskId, shopKey, fr.getRowCount(), fr.getResultFilename()); @@ -698,7 +707,7 @@ public class ProductRiskTaskService { } try { - assembleShopResult(fr, shopKey, cachedPayload, workRoot); + enqueueResultFileAssembly(fr, shopKey, cachedPayload); productRiskTaskCacheService.removeShopMergedPayload(taskId, shopKey); changed = true; log.warn("[product-risk] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}", @@ -778,6 +787,49 @@ public class ProductRiskTaskService { } } + public void processResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null || job.getResultId() == null || job.getScopeKey() == null) { + throw new BusinessException("结果文件任务参数不完整"); + } + FileResultEntity result = fileResultMapper.selectById(job.getResultId()); + if (result == null || !MODULE_TYPE.equals(result.getModuleType())) { + throw new BusinessException("结果记录不存在"); + } + ProductRiskShopPayloadDto payload = taskResultPayloadService.getLatest( + job.getTaskId(), MODULE_TYPE, job.getScopeKey(), ProductRiskShopPayloadDto.class); + if (payload == null) { + payload = taskResultItemService.getResultSnapshot(job.getTaskId(), MODULE_TYPE, job.getResultId(), ProductRiskShopPayloadDto.class); + } + if (payload == null) { + throw new BusinessException("结果文件载荷不存在"); + } + File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "product-risk-result", String.valueOf(job.getTaskId()))); + assembleShopResult(result, job.getScopeKey(), payload, workRoot); + } + + private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) { + taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload); + taskResultItemService.replaceResultSnapshot(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey, payload); + markResultFilePending(result, shopKey, payload); + taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey); + } + + private void markResultFilePending(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) { + Map> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries()); + String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() + ? payload.getShopName().trim() + : shopKey; + String stem = safeFileStem(displayName); + result.setResultFilename(stem + ".zip"); + result.setResultFileUrl(null); + result.setResultFileSize(0L); + result.setResultContentType(CONTENT_TYPE_ZIP); + result.setRowCount(excelAssemblyService.countRows(countries)); + result.setSuccess(1); + result.setErrorMessage(null); + fileResultMapper.updateById(result); + } + private void updateTaskStatusFromLatestRows(FileTaskEntity task, List latest, List batchErrors) { @@ -826,6 +878,8 @@ public class ProductRiskTaskService { log.warn("[product-risk] compact result json failed: {}", ex.getMessage()); } fileTaskMapper.updateById(task); + taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, task.getStatus(), + latest.size(), ok, fail, null, task.getErrorMessage(), null); log.warn("[product-risk] status updated from latest rows taskId={} oldStatus={} newStatus={} ok={} fail={} allDone={} batchErrors={} errorMessage={}", task.getId(), oldStatus, task.getStatus(), ok, fail, allDone, batchErrors, task.getErrorMessage()); cleanupTaskCacheIfTerminal(task.getId(), task.getStatus()); @@ -910,9 +964,8 @@ public class ProductRiskTaskService { vo.setSuccess(ok); vo.setError(entity.getErrorMessage()); vo.setOutputFilename(entity.getResultFilename()); - vo.setDownloadUrl(entity.getResultFileUrl() == null || entity.getResultFileUrl().isBlank() - ? null - : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())); + vo.setDownloadUrl(null); + attachFileJobState(vo, entity); vo.setMatched(true); if (entity.getTaskId() != null) { mergeQueueFieldsFromRequest(vo, entity.getTaskId()); @@ -920,6 +973,18 @@ public class ProductRiskTaskService { return vo; } + private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) { + TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); + vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); + if (job == null) { + vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); + return; + } + vo.setFileJobId(job.getId()); + vo.setFileStatus(job.getStatus()); + vo.setFileError(job.getErrorMessage()); + } + private void mergeQueueFieldsFromRequest(ProductRiskResultItemVo vo, Long taskId) { FileTaskEntity t = loadTaskForExecution(taskId); if (t == null || t.getRequestJson() == null || t.getRequestJson().isBlank()) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/vo/QueryAsinResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/vo/QueryAsinResultItemVo.java index 14aa5d0..3a3d47c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/vo/QueryAsinResultItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/vo/QueryAsinResultItemVo.java @@ -43,6 +43,10 @@ public class QueryAsinResultItemVo { private LocalDateTime finishedAt; private String outputFilename; private String downloadUrl; + private Long fileJobId; + private String fileStatus; + private String fileError; + private Boolean fileReady; @JsonProperty("queryAsins") private List queryAsins = new ArrayList<>(); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskCacheService.java index 4c8c4db..45afebb 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskCacheService.java @@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinShopPayloadDto; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; @@ -18,6 +19,7 @@ import java.util.concurrent.ConcurrentHashMap; @Service @RequiredArgsConstructor +@Slf4j public class QueryAsinTaskCacheService { private static final String MODULE_TYPE = "QUERY_ASIN"; @@ -60,7 +62,13 @@ public class QueryAsinTaskCacheService { if (taskId == null || taskId <= 0) { return 0L; } - String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId)); + String raw; + try { + raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId)); + } catch (Exception ex) { + log.warn("[query-asin-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + return 0L; + } if (raw == null || raw.isBlank()) { return 0L; } @@ -72,10 +80,14 @@ public class QueryAsinTaskCacheService { } public void touchTaskHeartbeat(Long taskId) { - stringRedisTemplate.opsForValue().set( - buildTaskHeartbeatKey(taskId), - String.valueOf(Instant.now().toEpochMilli()), - Duration.ofHours(PAYLOAD_TTL_HOURS)); + try { + stringRedisTemplate.opsForValue().set( + buildTaskHeartbeatKey(taskId), + String.valueOf(Instant.now().toEpochMilli()), + Duration.ofHours(PAYLOAD_TTL_HOURS)); + } catch (Exception ex) { + log.warn("[query-asin-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + } } public void deleteTaskCache(Long taskId) { @@ -83,8 +95,12 @@ public class QueryAsinTaskCacheService { return; } taskEntityLocalCache.remove(taskId); - stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); - stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + try { + stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); + stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + } catch (Exception ex) { + log.warn("[query-asin-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage()); + } taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE); } @@ -133,7 +149,13 @@ public class QueryAsinTaskCacheService { return result; } java.util.List keys = missingIds.stream().map(this::buildTaskEntityKey).toList(); - java.util.List values = stringRedisTemplate.opsForValue().multiGet(keys); + java.util.List values; + try { + values = stringRedisTemplate.opsForValue().multiGet(keys); + } catch (Exception ex) { + log.warn("[query-asin-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage()); + return result; + } for (int i = 0; i < missingIds.size(); i++) { Long taskId = missingIds.get(i); String val = values != null && i < values.size() ? values.get(i) : null; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java index e9b1eb6..b266054 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java @@ -24,6 +24,10 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import com.nanri.aiimage.modules.task.service.TaskResultItemService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -45,6 +49,9 @@ public class QueryAsinTaskService { private static final String MODULE_TYPE = "QUERY_ASIN"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + private static final int RESULT_PENDING = -1; + private static final int RESULT_FAILED = 0; + private static final int RESULT_SUCCESS = 1; private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断"; private final FileTaskMapper fileTaskMapper; @@ -56,6 +63,9 @@ public class QueryAsinTaskService { private final ZiniaoShopSwitchService ziniaoShopSwitchService; private final ObjectMapper objectMapper; private final TaskPressureProperties taskPressureProperties; + private final TaskFileJobService taskFileJobService; + private final TaskResultItemService taskResultItemService; + private final TaskProgressSnapshotService taskProgressSnapshotService; private FileTaskEntity loadTaskForExecution(Long taskId) { Map cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId)); @@ -185,8 +195,7 @@ public class QueryAsinTaskService { batch.getMissingTaskIds().add(taskId); continue; } - List snapshots = buildSnapshotFromDb(task, taskRows); - batch.getItems().addAll(snapshots); + batch.getItems().addAll(buildProgressItems(task, taskRows)); } return batch; } @@ -264,12 +273,13 @@ public class QueryAsinTaskService { result.setSourceFilename(normalizedShopName); result.setSourceFileUrl(item.getShopId()); result.setUserId(request.getUserId()); - result.setSuccess(null); + result.setSuccess(RESULT_PENDING); result.setCreatedAt(now); fileResultMapper.insert(result); snapshots.add(toSnapshotVo(result, item, task.getStatus(), null)); } persistTaskJson(task, uniqueItems, snapshots); + taskCacheService.saveTaskCache(task); QueryAsinCreateTaskVo vo = new QueryAsinCreateTaskVo(); vo.setTaskId(task.getId()); @@ -326,6 +336,7 @@ public class QueryAsinTaskService { persistSnapshotJson(task, snapshots); fileTaskMapper.updateById(task); + taskCacheService.saveTaskCache(task); tryFinalizeTask(taskId, false); } @@ -402,6 +413,7 @@ public class QueryAsinTaskService { persistSnapshotJson(task, snapshots); fileTaskMapper.updateById(task); + taskCacheService.saveTaskCache(task); return changed; } @@ -448,6 +460,7 @@ public class QueryAsinTaskService { updateTaskStatusFromRows(task, rows); persistSnapshotJson(task, buildSnapshotFromDb(task, rows)); fileTaskMapper.updateById(task); + taskCacheService.saveTaskCache(task); } private long countTasks(Long userId, List statuses) { @@ -502,7 +515,11 @@ public class QueryAsinTaskService { private Map> buildSnapshotMap(Map taskMap) { Map> out = new LinkedHashMap<>(); for (Map.Entry entry : taskMap.entrySet()) { - out.put(entry.getKey(), indexSnapshotByResultId(parseTaskSnapshots(entry.getValue().getResultJson()))); + List snapshots = parseTaskSnapshots(entry.getValue().getResultJson()); + if (snapshots.isEmpty()) { + snapshots = taskResultItemService.listResultSnapshots(entry.getKey(), MODULE_TYPE, QueryAsinResultItemVo.class); + } + out.put(entry.getKey(), indexSnapshotByResultId(snapshots)); } return out; } @@ -514,14 +531,13 @@ public class QueryAsinTaskService { item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename())); item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl())); item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus()); - item.setSuccess(entity.getSuccess() == null ? item.getSuccess() : entity.getSuccess() == 1); + item.setSuccess(toSuccessFlag(entity.getSuccess(), item.getSuccess())); item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError()); item.setCreatedAt(entity.getCreatedAt()); item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt()); item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename())); - item.setDownloadUrl(blank(entity.getResultFileUrl()) - ? item.getDownloadUrl() - : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())); + item.setDownloadUrl(null); + attachFileJobState(item, entity); if (item.getCountryResults() == null) { item.setCountryResults(new ArrayList<>()); } @@ -531,6 +547,18 @@ public class QueryAsinTaskService { return item; } + private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity) { + TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); + item.setFileReady(!blank(entity.getResultFileUrl())); + if (job == null) { + item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null); + return; + } + item.setFileJobId(job.getId()); + item.setFileStatus(job.getStatus()); + item.setFileError(job.getErrorMessage()); + } + private List dedupeItems(List items) { LinkedHashMap map = new LinkedHashMap<>(); for (QueryAsinTaskItemDto item : items) { @@ -560,7 +588,7 @@ public class QueryAsinTaskService { vo.setMatchStatus(item.getMatchStatus()); vo.setMatchMessage(item.getMatchMessage()); vo.setTaskStatus(taskStatus); - vo.setSuccess(result.getSuccess() == null ? null : result.getSuccess() == 1); + vo.setSuccess(toSuccessFlag(result.getSuccess(), null)); vo.setError(result.getErrorMessage()); vo.setCreatedAt(result.getCreatedAt()); vo.setFinishedAt(finishedAt); @@ -575,6 +603,7 @@ public class QueryAsinTaskService { try { task.setRequestJson(objectMapper.writeValueAsString(requestItems)); task.setResultJson(objectMapper.writeValueAsString(snapshots)); + syncSnapshotTables(task, snapshots); fileTaskMapper.updateById(task); } catch (Exception ex) { throw new BusinessException("查询ASIN任务快照保存失败"); @@ -591,6 +620,27 @@ public class QueryAsinTaskService { return list; } + private List buildProgressItems(FileTaskEntity task, List rows) { + List list = new ArrayList<>(); + for (FileResultEntity row : rows) { + QueryAsinResultItemVo item = new QueryAsinResultItemVo(); + item.setResultId(row.getId()); + item.setTaskId(row.getTaskId()); + item.setShopName(row.getSourceFilename()); + item.setShopId(row.getSourceFileUrl()); + item.setTaskStatus(task == null ? null : task.getStatus()); + item.setSuccess(toSuccessFlag(row.getSuccess(), null)); + item.setError(row.getErrorMessage()); + item.setCreatedAt(row.getCreatedAt()); + item.setFinishedAt(task == null ? null : task.getFinishedAt()); + item.setOutputFilename(row.getResultFilename()); + item.setDownloadUrl(null); + attachFileJobState(item, row); + list.add(item); + } + return list; + } + private Map indexSnapshotByResultId(List snapshots) { Map map = new LinkedHashMap<>(); if (snapshots == null) { @@ -605,8 +655,8 @@ public class QueryAsinTaskService { } private void updateTaskStatusFromRows(FileTaskEntity task, List rows) { - long successCount = rows.stream().filter(row -> Integer.valueOf(1).equals(row.getSuccess())).count(); - long failedCount = rows.stream().filter(row -> Integer.valueOf(0).equals(row.getSuccess())).count(); + long successCount = rows.stream().filter(row -> Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())).count(); + long failedCount = rows.stream().filter(row -> Integer.valueOf(RESULT_FAILED).equals(row.getSuccess())).count(); long pendingCount = rows.stream().filter(row -> !isResultFinished(row)).count(); task.setSuccessFileCount((int) successCount); task.setFailedFileCount((int) failedCount); @@ -773,27 +823,25 @@ public class QueryAsinTaskService { .filter(item -> Boolean.TRUE.equals(item.getSuccess())) .toList(); if (!successItems.isEmpty()) { - File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "query-asin-result", String.valueOf(task.getId()))); - String filename = safeFileStem("查询ASIN-" + task.getId()) + ".xlsx"; - File xlsx = FileUtil.file(workRoot, filename); - try { - excelAssemblyService.writeWorkbook(xlsx, successItems); - String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE); - long fileSize = xlsx.length(); - int rowCount = excelAssemblyService.countRows(successItems); + String filename = buildTaskWorkbookFilename(task); + int rowCount = excelAssemblyService.countRows(successItems); + FileResultEntity firstSuccessRow = null; for (FileResultEntity row : rows) { - if (Integer.valueOf(1).equals(row.getSuccess())) { + if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) { row.setResultFilename(filename); - row.setResultFileUrl(objectKey); - row.setResultFileSize(fileSize); + row.setResultFileUrl(null); + row.setResultFileSize(0L); row.setResultContentType(CONTENT_TYPE_XLSX); row.setRowCount(rowCount); fileResultMapper.updateById(row); + if (firstSuccessRow == null) { + firstSuccessRow = row; + } } } + if (firstSuccessRow != null) { + taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId()); snapshots = buildSnapshotFromDb(task, rows); - } finally { - FileUtil.del(xlsx); } } @@ -802,14 +850,56 @@ public class QueryAsinTaskService { taskCacheService.deleteTaskCache(task.getId()); } + public void processResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null) { + throw new BusinessException("结果文件任务参数不完整"); + } + FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId()); + if (task == null || !MODULE_TYPE.equals(task.getModuleType())) { + throw new BusinessException("任务不存在"); + } + List rows = listTaskRows(job.getTaskId()); + List snapshots = taskResultItemService.listResultSnapshots(job.getTaskId(), MODULE_TYPE, QueryAsinResultItemVo.class); + if (snapshots.isEmpty()) { + snapshots = buildSnapshotFromDb(task, rows); + } + List successItems = snapshots.stream() + .filter(item -> Boolean.TRUE.equals(item.getSuccess())) + .toList(); + if (successItems.isEmpty()) { + throw new BusinessException("没有可生成的查询ASIN结果"); + } + File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "query-asin-result", String.valueOf(task.getId()))); + String filename = buildTaskWorkbookFilename(task); + File xlsx = FileUtil.file(workRoot, filename); + try { + excelAssemblyService.writeWorkbook(xlsx, successItems); + String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE); + long fileSize = xlsx.length(); + int rowCount = excelAssemblyService.countRows(successItems); + for (FileResultEntity row : rows) { + if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) { + row.setResultFilename(filename); + row.setResultFileUrl(objectKey); + row.setResultFileSize(fileSize); + row.setResultContentType(CONTENT_TYPE_XLSX); + row.setRowCount(rowCount); + fileResultMapper.updateById(row); + } + } + } finally { + FileUtil.del(xlsx); + } + } + private void markResultSuccess(FileResultEntity row) { - row.setSuccess(1); + row.setSuccess(RESULT_SUCCESS); row.setErrorMessage(null); fileResultMapper.updateById(row); } private void markResultFailed(FileResultEntity row, String message) { - row.setSuccess(0); + row.setSuccess(RESULT_FAILED); row.setErrorMessage(blankToNull(message)); row.setResultFilename(null); row.setResultFileUrl(null); @@ -820,7 +910,14 @@ public class QueryAsinTaskService { } private boolean isResultFinished(FileResultEntity row) { - return Integer.valueOf(1).equals(row.getSuccess()) || Integer.valueOf(0).equals(row.getSuccess()); + return Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess()) || Integer.valueOf(RESULT_FAILED).equals(row.getSuccess()); + } + + private Boolean toSuccessFlag(Integer dbValue, Boolean fallback) { + if (dbValue == null || Integer.valueOf(RESULT_PENDING).equals(dbValue)) { + return fallback; + } + return Integer.valueOf(RESULT_SUCCESS).equals(dbValue); } private List parseTaskSnapshots(String json) { @@ -838,11 +935,39 @@ public class QueryAsinTaskService { private void persistSnapshotJson(FileTaskEntity task, List snapshots) { try { task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots)); + syncSnapshotTables(task, snapshots); } catch (Exception ex) { throw new BusinessException("查询ASIN任务快照保存失败"); } } + private void syncSnapshotTables(FileTaskEntity task, List snapshots) { + List safe = snapshots == null ? List.of() : snapshots; + taskResultItemService.replaceTaskSnapshots(task.getId(), MODULE_TYPE, safe, new TaskResultItemService.SnapshotKeyResolver() { + @Override + public Long resultId(Object snapshot) { + return ((QueryAsinResultItemVo) snapshot).getResultId(); + } + + @Override + public String scopeKey(Object snapshot) { + QueryAsinResultItemVo item = (QueryAsinResultItemVo) snapshot; + return firstNonBlank(item.getShopName(), "result:" + item.getResultId()); + } + }); + int successCount = 0; + int failedCount = 0; + for (QueryAsinResultItemVo item : safe) { + if (Boolean.TRUE.equals(item.getSuccess())) { + successCount++; + } else if (Boolean.FALSE.equals(item.getSuccess())) { + failedCount++; + } + } + taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, firstNonBlank(task.getStatus(), "RUNNING"), + safe.size(), successCount, failedCount, null, task.getErrorMessage(), null); + } + private List copyCountryResults(List results) { List copy = new ArrayList<>(); if (results == null) { @@ -923,6 +1048,11 @@ public class QueryAsinTaskService { return asin == null ? "" : asin.trim().toUpperCase(); } + private String buildTaskWorkbookFilename(FileTaskEntity task) { + Long taskId = task == null ? null : task.getId(); + return safeFileStem("查询ASIN-" + (taskId == null ? "result" : taskId)) + ".xlsx"; + } + private String safeFileStem(String value) { String raw = value == null ? "result" : value.trim(); String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_"); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/QueryAsinService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/QueryAsinService.java index e8a39d7..2e89945 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/QueryAsinService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/QueryAsinService.java @@ -15,6 +15,7 @@ import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinItemVo; import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinPageVo; import com.nanri.aiimage.modules.shopkey.model.vo.ShopManageGroupItemVo; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; @@ -30,9 +31,11 @@ import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Predicate; @Service @RequiredArgsConstructor +@Slf4j public class QueryAsinService { private static final List SUPPORTED_COUNTRIES = List.of("DE", "UK", "FR", "IT", "ES"); @@ -364,13 +367,32 @@ public class QueryAsinService { } private String normalizeCountry(String country) { - String normalized = normalizeBlank(country).toUpperCase(Locale.ROOT); + String normalized = normalizeCountryAlias(country); if (!SUPPORTED_COUNTRIES.contains(normalized)) { throw new BusinessException("国家参数不支持: " + country); } return normalized; } + private String normalizeCountryAlias(String country) { + String normalized = normalizeBlank(country); + if (normalized.isEmpty()) { + return ""; + } + String upper = normalized.toUpperCase(Locale.ROOT); + if (SUPPORTED_COUNTRIES.contains(upper)) { + return upper; + } + return switch (normalizeHeaderKey(normalized)) { + case "\u5fb7\u56fd", "\u5fb7", "de", "germany", "german", "deutschland" -> "DE"; + case "\u82f1\u56fd", "\u82f1", "uk", "gb", "greatbritain", "britain", "unitedkingdom" -> "UK"; + case "\u6cd5\u56fd", "\u6cd5", "fr", "france", "french" -> "FR"; + case "\u610f\u5927\u5229", "\u610f", "it", "italy", "italian" -> "IT"; + case "\u897f\u73ed\u7259", "\u897f", "es", "spain", "spanish", "espana" -> "ES"; + default -> ""; + }; + } + private String normalizeAsin(String asin) { String normalized = normalizeRequired(asin, "ASIN 不能为空").toUpperCase(Locale.ROOT); if (normalized.length() > 64) { @@ -505,13 +527,19 @@ public class QueryAsinService { } } headerLoaded = true; - if (findGroupIndex() == null && (fallbackGroupId == null || fallbackGroupId <= 0)) { + Integer resolvedGroupIndex = resolveGroupHeaderIndex(); + Integer resolvedShopIndex = resolveShopHeaderIndex(); + boolean hasWideCountries = hasWideCountryColumns(); + boolean hasSingleCountryAsin = hasSingleCountryAsinColumns(); + log.info("[query-asin-import] headerMap={} normalizedKeys={} groupIndex={} shopIndex={} hasWideCountries={} hasSingleCountryAsin={}", + headerMap, headerIndexByKey.keySet(), resolvedGroupIndex, resolvedShopIndex, hasWideCountries, hasSingleCountryAsin); + if (resolvedGroupIndex == null && (fallbackGroupId == null || fallbackGroupId <= 0)) { throw new BusinessException("Excel 缺少分组列，请在页面选择分组或提供分组列"); } - if (findShopIndex() == null) { + if (resolvedShopIndex == null) { throw new BusinessException("Excel 缺少店铺名列"); } - if (SUPPORTED_COUNTRIES.stream().noneMatch(country -> findCountryIndex(country) != null)) { + if (!hasWideCountries && !hasSingleCountryAsin) { throw new BusinessException("Excel 至少需要包含一个国家 ASIN 列"); } } @@ -524,7 +552,7 @@ public class QueryAsinService { progress.setProcessedRows(rowIndex); Long groupId = resolveGroupId(rowMap); - String shopName = cell(rowMap, findShopIndex()); + String shopName = cell(rowMap, resolveShopHeaderIndex()); if (groupId == null || shopName.isEmpty()) { skippedCount++; updateProgress(); @@ -532,27 +560,8 @@ public class QueryAsinService { } QueryAsinEntity row = newImportEntity(groupId, shopName); - int rowAsinCount = 0; - int rowAcceptedCount = 0; - for (String country : SUPPORTED_COUNTRIES) { - Integer index = findCountryIndex(country); - if (index == null) { - continue; - } - String asin = normalizeBlank(cell(rowMap, index)).toUpperCase(Locale.ROOT); - if (asin.isEmpty()) { - continue; - } - if (asin.length() > 64) { - skippedCount++; - continue; - } - asinCount++; - rowAsinCount++; - setCountryAsin(row, country, asin); - rowAcceptedCount++; - } - if (rowAsinCount == 0) { + int rowAcceptedCount = collectRowCountryAsins(row, rowMap); + if (countCountryCells(row) == 0) { skippedCount++; } if (rowAcceptedCount > 0) { @@ -754,7 +763,7 @@ public class QueryAsinService { } private Long resolveGroupId(Map rowMap) { - Integer groupIndex = findGroupIndex(); + Integer groupIndex = resolveGroupHeaderIndex(); String groupText = cell(rowMap, groupIndex); if (groupText.isEmpty()) { return fallbackGroupId != null && fallbackGroupId > 0 ? fallbackGroupId : null; @@ -767,6 +776,51 @@ public class QueryAsinService { return id; } + private int collectRowCountryAsins(QueryAsinEntity row, Map rowMap) { + int acceptedCount = 0; + for (String country : SUPPORTED_COUNTRIES) { + Integer index = resolveCountryHeaderIndex(country); + if (index == null) { + continue; + } + acceptedCount += acceptCountryAsin(row, country, cell(rowMap, index)); + } + Integer singleCountryIndex = findSingleCountryIndex(); + Integer singleAsinIndex = findSingleAsinIndex(); + if (singleCountryIndex != null && singleAsinIndex != null) { + acceptedCount += acceptCountryAsin( + row, + normalizeCountryAlias(cell(rowMap, singleCountryIndex)), + cell(rowMap, singleAsinIndex)); + } + return acceptedCount; + } + + private int acceptCountryAsin(QueryAsinEntity row, String country, String asin) { + String normalizedAsin = normalizeBlank(asin).toUpperCase(Locale.ROOT); + if (normalizedAsin.isEmpty()) { + return 0; + } + asinCount++; + if (country == null || country.isEmpty() || !SUPPORTED_COUNTRIES.contains(country)) { + skippedCount++; + return 0; + } + if (normalizedAsin.length() > 64) { + skippedCount++; + return 0; + } + String existing = getCountryAsin(row, country); + if (!existing.isEmpty()) { + if (!existing.equals(normalizedAsin)) { + skippedCount++; + } + return 0; + } + setCountryAsin(row, country, normalizedAsin); + return 1; + } + private void loadGroups() { if (groupsLoaded) { return; @@ -801,6 +855,88 @@ public class QueryAsinService { }; } + private Integer resolveGroupHeaderIndex() { + Integer index = findGroupIndex(); + if (index != null) { + return index; + } + index = firstHeader("groupname", "\u5206\u7ec4id", "\u5206\u7ec4"); + if (index != null) { + return index; + } + return findHeaderByPredicate(key -> key.contains("\u5206\u7ec4") || key.contains("group")); + } + + private Integer resolveShopHeaderIndex() { + Integer index = findShopIndex(); + if (index != null) { + return index; + } + index = firstHeader( + "\u5e97\u94fa\u540d", "\u5e97\u94fa\u540d\u79f0", "\u5e97\u94fa\u8d26\u53f7", "\u8d26\u53f7", + "account", "selleraccount", "storeaccount", "storename"); + if (index != null) { + return index; + } + return findHeaderByPredicate(key -> + (key.contains("\u5e97\u94fa") && (key.contains("\u540d") || key.contains("\u8d26\u53f7"))) + || (key.contains("shop") && (key.contains("name") || key.contains("account"))) + || key.contains("selleraccount") + || key.contains("storeaccount")); + } + + private Integer resolveCountryHeaderIndex(String country) { + Integer index = findCountryIndex(country); + if (index != null) { + return index; + } + return switch (country) { + case "DE" -> firstHeader("\u5fb7\u56fd", "\u5fb7", "\u5fb7\u56fdasin"); + case "UK" -> firstHeader("\u82f1\u56fd", "\u82f1", "\u82f1\u56fdasin"); + case "FR" -> firstHeader("\u6cd5\u56fd", "\u6cd5", "\u6cd5\u56fdasin"); + case "IT" -> firstHeader("\u610f\u5927\u5229", "\u610f", "\u610f\u5927\u5229asin"); + case "ES" -> firstHeader("\u897f\u73ed\u7259", "\u897f", "\u897f\u73ed\u7259asin"); + default -> null; + }; + } + + private Integer findSingleCountryIndex() { + Integer index = firstHeader( + "\u4e2d\u6587\u56fd\u5bb6\u540d", "\u56fd\u5bb6", "\u56fd\u5bb6\u540d", "\u56fd\u5bb6\u7ad9\u70b9", + "\u7ad9\u70b9", "country", "countryname", "marketplace", "site"); + if (index != null) { + return index; + } + return findHeaderByPredicate(key -> + key.contains("\u56fd\u5bb6") + || key.contains("\u7ad9\u70b9") + || key.contains("country") + || key.contains("marketplace") + || key.equals("site")); + } + + private Integer findSingleAsinIndex() { + return firstHeader("asin", "\u67e5\u8be2asin", "\u4ea7\u54c1asin", "\u94fe\u63a5asin"); + } + + private boolean hasWideCountryColumns() { + return SUPPORTED_COUNTRIES.stream().anyMatch(country -> resolveCountryHeaderIndex(country) != null); + } + + private boolean hasSingleCountryAsinColumns() { + return findSingleCountryIndex() != null && findSingleAsinIndex() != null; + } + + private Integer findHeaderByPredicate(Predicate predicate) { + for (Map.Entry entry : headerIndexByKey.entrySet()) { + String key = entry.getKey(); + if (key != null && predicate.test(key)) { + return entry.getValue(); + } + } + return null; + } + private Integer firstHeader(String... names) { for (String name : names) { Integer index = headerIndexByKey.get(normalizeHeaderKey(name)); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopManageGroupService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopManageGroupService.java index 6ced207..204beea 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopManageGroupService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/ShopManageGroupService.java @@ -81,13 +81,32 @@ public class ShopManageGroupService { .map(ShopManageGroupEntity::getId) .filter(id -> id != null && id > 0) .collect(Collectors.toCollection(LinkedHashSet::new)); - groupIds.addAll(groupMemberMapper.selectList(new LambdaQueryWrapper() + List memberGroupIds = groupMemberMapper.selectList(new LambdaQueryWrapper() .select(ShopManageGroupMemberEntity::getGroupId) .eq(ShopManageGroupMemberEntity::getUserId, normalizedOperatorId)) .stream() .map(ShopManageGroupMemberEntity::getGroupId) .filter(id -> id != null && id > 0) - .collect(Collectors.toCollection(LinkedHashSet::new))); + .toList(); + groupIds.addAll(memberGroupIds); + if (!memberGroupIds.isEmpty()) { + LinkedHashSet leaderIds = groupMapper.selectList(new LambdaQueryWrapper() + .select(ShopManageGroupEntity::getCreatedById) + .in(ShopManageGroupEntity::getId, memberGroupIds)) + .stream() + .map(ShopManageGroupEntity::getCreatedById) + .filter(id -> id != null && id > 0) + .collect(Collectors.toCollection(LinkedHashSet::new)); + if (!leaderIds.isEmpty()) { + groupIds.addAll(groupMapper.selectList(new LambdaQueryWrapper() + .select(ShopManageGroupEntity::getId) + .in(ShopManageGroupEntity::getCreatedById, leaderIds)) + .stream() + .map(ShopManageGroupEntity::getId) + .filter(id -> id != null && id > 0) + .toList()); + } + } return groupIds; } @@ -178,9 +197,7 @@ public class ShopManageGroupService { return; } Long normalizedOperatorId = normalizePositiveId(operatorId, "操作人不能为空"); - boolean createdBySelf = entity.getCreatedById() != null && entity.getCreatedById().equals(normalizedOperatorId); - boolean memberOfGroup = isGroupMember(entity.getId(), normalizedOperatorId); - if (!createdBySelf && !memberOfGroup) { + if (!listAccessibleGroupIds(normalizedOperatorId, false).contains(entity.getId())) { throw new BusinessException("只能操作自己有权限的分组数据"); } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java index b823dc8..a186396 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java @@ -18,6 +18,7 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -38,17 +39,27 @@ public class ShopMatchTaskCacheService { if (taskId == null || taskId <= 0) { return; } - stringRedisTemplate.opsForValue().set( - buildTaskHeartbeatKey(taskId), - String.valueOf(Instant.now().toEpochMilli()), - Duration.ofHours(PAYLOAD_TTL_HOURS)); + try { + stringRedisTemplate.opsForValue().set( + buildTaskHeartbeatKey(taskId), + String.valueOf(Instant.now().toEpochMilli()), + Duration.ofHours(PAYLOAD_TTL_HOURS)); + } catch (Exception ex) { + log.warn("[shop-match-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + } } public long getTaskHeartbeatMillis(Long taskId) { if (taskId == null || taskId <= 0) { return 0L; } - String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId)); + String raw; + try { + raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId)); + } catch (Exception ex) { + log.warn("[shop-match-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + return 0L; + } if (raw == null || raw.isBlank()) { return 0L; } @@ -59,6 +70,41 @@ public class ShopMatchTaskCacheService { } } + public Map getTaskHeartbeatMillisBatch(List taskIds) { + Map result = new LinkedHashMap<>(); + if (taskIds == null || taskIds.isEmpty()) { + return result; + } + List normalized = taskIds.stream() + .filter(id -> id != null && id > 0) + .distinct() + .toList(); + if (normalized.isEmpty()) { + return result; + } + List keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList(); + List values; + try { + values = stringRedisTemplate.opsForValue().multiGet(keys); + } catch (Exception ex) { + log.warn("[shop-match-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage()); + return result; + } + for (int i = 0; i < normalized.size(); i++) { + String raw = values != null && i < values.size() ? values.get(i) : null; + if (raw == null || raw.isBlank()) { + result.put(normalized.get(i), 0L); + continue; + } + try { + result.put(normalized.get(i), Long.parseLong(raw)); + } catch (NumberFormatException ignored) { + result.put(normalized.get(i), 0L); + } + } + return result; + } + public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) { return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class); } @@ -88,7 +134,11 @@ public class ShopMatchTaskCacheService { return; } taskEntityLocalCache.remove(taskId); - stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); + try { + stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); + } catch (Exception ex) { + log.warn("[shop-match-cache] delete heartbeat degraded taskId={} msg={}", taskId, ex.getMessage()); + } taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE); try { Path taskDir = buildTaskDir(taskId); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java index 32f33f0..4f18f2d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java @@ -28,6 +28,9 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskResultPayloadService; import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -62,6 +65,8 @@ public class ShopMatchTaskService { private final ObjectMapper objectMapper; private final ShopMatchTaskCacheService shopMatchTaskCacheService; private final TaskPressureProperties taskPressureProperties; + private final TaskResultPayloadService taskResultPayloadService; + private final TaskFileJobService taskFileJobService; private FileTaskEntity loadTaskForExecution(Long taskId) { Map cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId)); @@ -409,6 +414,12 @@ public class ShopMatchTaskService { if (state.getScheduleTimes() == null || stageIndex >= state.getScheduleTimes().size()) { throw new BusinessException("轮次配置缺失"); } + FileTaskEntity runningTask = findOtherRunningTask(userId, taskId); + if (runningTask != null) { + log.warn("[shop-match] activate serialized taskId={} stageIndex={} blockedByTaskId={} blockedUpdatedAt={}", + taskId, stageIndex, runningTask.getId(), runningTask.getUpdatedAt()); + throw new BusinessException("当前已有定时匹配任务执行中，请等待上一任务执行结束后再重试"); + } state.setActiveStageIndex(stageIndex); task.setStatus("RUNNING"); task.setUpdatedAt(now); @@ -511,7 +522,7 @@ public class ShopMatchTaskService { continue; } try { - assembleShopResult(result, shopKey, merged, workRoot); + enqueueResultFileAssembly(result, shopKey, merged); shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey); } catch (Exception ex) { markResultFailed(result, ex.getMessage() == null ? "结果组装失败" : ex.getMessage()); @@ -603,7 +614,7 @@ public class ShopMatchTaskService { } try { - assembleShopResult(result, shopKey, cachedPayload, workRoot); + enqueueResultFileAssembly(result, shopKey, cachedPayload); shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey); changed = true; log.warn("[shop-match] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}", @@ -651,6 +662,43 @@ public class ShopMatchTaskService { } } + public void processResultFileJob(TaskFileJobEntity job) { + if (job == null || job.getTaskId() == null || job.getResultId() == null || job.getScopeKey() == null) { + throw new BusinessException("结果文件任务参数不完整"); + } + FileResultEntity result = fileResultMapper.selectById(job.getResultId()); + if (result == null || !MODULE_TYPE.equals(result.getModuleType())) { + throw new BusinessException("结果记录不存在"); + } + ShopMatchShopPayloadDto payload = taskResultPayloadService.getLatest( + job.getTaskId(), MODULE_TYPE, job.getScopeKey(), ShopMatchShopPayloadDto.class); + if (payload == null) { + throw new BusinessException("结果文件载荷不存在"); + } + File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "shop-match-result", String.valueOf(job.getTaskId()))); + assembleShopResult(result, job.getScopeKey(), payload, workRoot); + } + + private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) { + taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload); + markResultFilePending(result, shopKey, payload); + taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey); + } + + private void markResultFilePending(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) { + Map> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries()); + String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() ? payload.getShopName().trim() : shopKey; + String stem = safeFileStem(displayName); + result.setResultFilename(stem + ".xlsx"); + result.setResultFileUrl(null); + result.setResultFileSize(0L); + result.setResultContentType(CONTENT_TYPE_XLSX); + result.setRowCount(excelAssemblyService.countRows(countries)); + result.setSuccess(1); + result.setErrorMessage(null); + fileResultMapper.updateById(result); + } + private void markResultFailed(FileResultEntity result, String message) { result.setSuccess(0); result.setErrorMessage(message); @@ -816,9 +864,8 @@ public class ShopMatchTaskService { vo.setSuccess(success); vo.setError(entity.getErrorMessage()); vo.setOutputFilename(entity.getResultFilename()); - vo.setDownloadUrl(entity.getResultFileUrl() == null || entity.getResultFileUrl().isBlank() - ? null - : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())); + vo.setDownloadUrl(null); + attachFileJobState(vo, entity); vo.setMatched(true); if (entity.getTaskId() != null) { mergeQueueFieldsFromRequest(vo, entity.getTaskId()); @@ -826,6 +873,18 @@ public class ShopMatchTaskService { return vo; } + private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) { + TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId()); + vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank()); + if (job == null) { + vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null); + return; + } + vo.setFileJobId(job.getId()); + vo.setFileStatus(job.getStatus()); + vo.setFileError(job.getErrorMessage()); + } + private List buildStageVos(FileTaskEntity task, ShopMatchCreateTaskRequest request) { List stages = new ArrayList<>(); List scheduleTimes = request.getScheduleTimes() == null ? List.of() : request.getScheduleTimes(); @@ -942,6 +1001,19 @@ public class ShopMatchTaskService { shopMatchTaskCacheService.saveTaskCache(task); } + private FileTaskEntity findOtherRunningTask(Long userId, Long taskId) { + if (userId == null || userId <= 0) { + return null; + } + return fileTaskMapper.selectOne(new LambdaQueryWrapper() + .eq(FileTaskEntity::getModuleType, MODULE_TYPE) + .eq(FileTaskEntity::getUserId, userId) + .eq(FileTaskEntity::getStatus, "RUNNING") + .ne(taskId != null, FileTaskEntity::getId, taskId) + .orderByAsc(FileTaskEntity::getUpdatedAt) + .last("limit 1")); + } + private static String fmt(LocalDateTime time) { return time == null ? null : time.toString(); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java index c47f882..474e36e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java @@ -173,9 +173,7 @@ public class SplitRunService { vo.setResultId(entity.getId()); vo.setSourceFilename(entity.getSourceFilename()); vo.setOutputFilename(entity.getResultFilename()); - vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1 - ? ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()) - : null); + vo.setDownloadUrl(null); vo.setRowCount(entity.getRowCount()); vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); vo.setError(entity.getErrorMessage()); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/controller/TaskFileJobController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/controller/TaskFileJobController.java new file mode 100644 index 0000000..3217cb2 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/controller/TaskFileJobController.java @@ -0,0 +1,52 @@ +package com.nanri.aiimage.modules.task.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/task-file-jobs") +@Tag(name = "任务结果文件 Job") +public class TaskFileJobController { + + private final TaskFileJobService taskFileJobService; + + @GetMapping + @Operation(summary = "查询结果文件生成 Job") + public ApiResponse> list( + @RequestParam(value = "status", required = false) String status, + @RequestParam(value = "module_type", required = false) String moduleType, + @RequestParam(value = "task_id", required = false) Long taskId, + @RequestParam(value = "limit", defaultValue = "100") int limit) { + return ApiResponse.success(taskFileJobService.listJobs(status, moduleType, taskId, limit)); + } + + @PostMapping("/{jobId}/retry") + @Operation(summary = "手动重试结果文件生成 Job") + public ApiResponse> retry(@PathVariable Long jobId) { + boolean updated = taskFileJobService.retry(jobId); + return ApiResponse.success(Map.of("jobId", jobId, "updated", updated)); + } + + @PostMapping("/stuck/reset") + @Operation(summary = "手动扫描并复位卡住的 RUNNING Job") + public ApiResponse> resetStuck( + @RequestParam(value = "timeout_minutes", defaultValue = "30") int timeoutMinutes, + @RequestParam(value = "limit", defaultValue = "100") int limit) { + int reset = taskFileJobService.resetStuckRunningJobs(timeoutMinutes, limit); + return ApiResponse.success(Map.of("reset", reset)); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskFileJobMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskFileJobMapper.java new file mode 100644 index 0000000..af5dfea --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskFileJobMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.task.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface TaskFileJobMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskProgressSnapshotMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskProgressSnapshotMapper.java new file mode 100644 index 0000000..b931e69 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskProgressSnapshotMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.task.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface TaskProgressSnapshotMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskResultItemMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskResultItemMapper.java new file mode 100644 index 0000000..b0fef8c --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskResultItemMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.task.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface TaskResultItemMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskResultPayloadMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskResultPayloadMapper.java new file mode 100644 index 0000000..fda1f17 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskResultPayloadMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.task.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface TaskResultPayloadMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskFileJobDispatchEvent.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskFileJobDispatchEvent.java new file mode 100644 index 0000000..2145d47 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskFileJobDispatchEvent.java @@ -0,0 +1,14 @@ +package com.nanri.aiimage.modules.task.model.dto; + +import java.time.LocalDateTime; + +public record TaskFileJobDispatchEvent( + Long jobId, + Long taskId, + String moduleType, + Long resultId, + String scopeKey, + String jobType, + LocalDateTime createdAt +) { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskFileJobMessage.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskFileJobMessage.java new file mode 100644 index 0000000..9b5465b --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskFileJobMessage.java @@ -0,0 +1,16 @@ +package com.nanri.aiimage.modules.task.model.dto; + +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +public class TaskFileJobMessage { + private Long jobId; + private Long taskId; + private String moduleType; + private Long resultId; + private String scopeKey; + private String jobType; + private LocalDateTime createdAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskFileJobEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskFileJobEntity.java new file mode 100644 index 0000000..9f95af6 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskFileJobEntity.java @@ -0,0 +1,28 @@ +package com.nanri.aiimage.modules.task.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("biz_task_file_job") +public class TaskFileJobEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private Long taskId; + private String moduleType; + private Long resultId; + private String scopeKey; + private String jobType; + private String status; + private Integer retryCount; + private String errorMessage; + private String resultFileUrl; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; + private LocalDateTime finishedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskProgressSnapshotEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskProgressSnapshotEntity.java new file mode 100644 index 0000000..3a23c21 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskProgressSnapshotEntity.java @@ -0,0 +1,28 @@ +package com.nanri.aiimage.modules.task.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("biz_task_progress_snapshot") +public class TaskProgressSnapshotEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private Long taskId; + private String moduleType; + private String status; + private Integer totalCount; + private Integer successCount; + private Integer failedCount; + private Integer pendingCount; + private String currentScopeKey; + private String message; + private String snapshotJson; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskResultItemEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskResultItemEntity.java new file mode 100644 index 0000000..6562f90 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskResultItemEntity.java @@ -0,0 +1,29 @@ +package com.nanri.aiimage.modules.task.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("biz_task_result_item") +public class TaskResultItemEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private Long taskId; + private String moduleType; + private Long resultId; + private String scopeKey; + private String scopeHash; + private String itemKey; + private String countryCode; + private String asin; + private String status; + private String payloadJson; + private String payloadHash; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskResultPayloadEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskResultPayloadEntity.java new file mode 100644 index 0000000..711b6e1 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskResultPayloadEntity.java @@ -0,0 +1,26 @@ +package com.nanri.aiimage.modules.task.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("biz_task_result_payload") +public class TaskResultPayloadEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private Long taskId; + private String moduleType; + private String scopeKey; + private String scopeHash; + private String submissionId; + private String payloadJson; + private String payloadHash; + private Long version; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobConsumer.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobConsumer.java new file mode 100644 index 0000000..b340bcb --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobConsumer.java @@ -0,0 +1,39 @@ +package com.nanri.aiimage.modules.task.service; + +import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper; +import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.rocketmq.spring.annotation.RocketMQMessageListener; +import org.apache.rocketmq.spring.core.RocketMQListener; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "aiimage.result-file-job", name = "mq-enabled", havingValue = "true") +@RocketMQMessageListener( + topic = "${aiimage.result-file-job.topic:aiimage-result-file-job}", + consumerGroup = "${aiimage.result-file-job.consumer-group:aiimage-result-file-job-consumer}" +) +public class TaskFileJobConsumer implements RocketMQListener { + + private final TaskFileJobMapper taskFileJobMapper; + private final TaskResultFileJobWorker taskResultFileJobWorker; + + @Override + public void onMessage(TaskFileJobMessage message) { + if (message == null || message.getJobId() == null) { + return; + } + TaskFileJobEntity job = taskFileJobMapper.selectById(message.getJobId()); + if (job == null) { + log.warn("[task-file-job] mq message ignored, job not found jobId={} taskId={} moduleType={}", + message.getJobId(), message.getTaskId(), message.getModuleType()); + return; + } + taskResultFileJobWorker.process(job); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobDispatchCoordinator.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobDispatchCoordinator.java new file mode 100644 index 0000000..f797853 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobDispatchCoordinator.java @@ -0,0 +1,42 @@ +package com.nanri.aiimage.modules.task.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent; +import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.transaction.event.TransactionPhase; +import org.springframework.transaction.event.TransactionalEventListener; + +@Slf4j +@Component +@RequiredArgsConstructor +public class TaskFileJobDispatchCoordinator { + + private final TaskFileJobPublisher taskFileJobPublisher; + private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher; + + @TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT, fallbackExecution = true) + public void onTaskFileJobDispatch(TaskFileJobDispatchEvent event) { + if (event == null || event.jobId() == null) { + return; + } + TaskFileJobMessage message = new TaskFileJobMessage(); + message.setJobId(event.jobId()); + message.setTaskId(event.taskId()); + message.setModuleType(event.moduleType()); + message.setResultId(event.resultId()); + message.setScopeKey(event.scopeKey()); + message.setJobType(event.jobType()); + message.setCreatedAt(event.createdAt()); + boolean published = taskFileJobPublisher.publish(message); + if (published) { + return; + } + boolean dispatched = taskFileJobLocalDispatcher.dispatch(event.jobId(), event.taskId(), event.moduleType()); + if (dispatched) { + log.info("[task-file-job] local async dispatch scheduled jobId={} taskId={} moduleType={}", + event.jobId(), event.taskId(), event.moduleType()); + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobLocalDispatcher.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobLocalDispatcher.java new file mode 100644 index 0000000..9cfea22 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobLocalDispatcher.java @@ -0,0 +1,53 @@ +package com.nanri.aiimage.modules.task.service; + +import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.task.TaskExecutor; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class TaskFileJobLocalDispatcher { + + private final TaskFileJobMapper taskFileJobMapper; + private final ObjectProvider taskResultFileJobWorkerProvider; + + @Qualifier("taskFileJobDispatchExecutor") + private final TaskExecutor taskFileJobDispatchExecutor; + + @Value("${aiimage.result-file-job.local-dispatch-enabled:true}") + private boolean localDispatchEnabled; + + public boolean dispatch(Long jobId, Long taskId, String moduleType) { + if (!localDispatchEnabled || jobId == null || jobId <= 0) { + return false; + } + taskFileJobDispatchExecutor.execute(() -> { + try { + TaskFileJobEntity job = taskFileJobMapper.selectById(jobId); + if (job == null) { + log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}", + jobId, taskId, moduleType); + return; + } + TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable(); + if (taskResultFileJobWorker == null) { + log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}", + jobId, taskId, moduleType); + return; + } + taskResultFileJobWorker.process(job); + } catch (Exception ex) { + log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}", + jobId, taskId, moduleType, ex.getMessage(), ex); + } + }); + return true; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobPublisher.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobPublisher.java new file mode 100644 index 0000000..3d06a59 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobPublisher.java @@ -0,0 +1,50 @@ +package com.nanri.aiimage.modules.task.service; + +import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.rocketmq.spring.core.RocketMQTemplate; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class TaskFileJobPublisher { + + private final ObjectProvider rocketMQTemplateProvider; + + @Value("${aiimage.result-file-job.mq-enabled:false}") + private boolean mqEnabled; + + @Value("${aiimage.result-file-job.topic:aiimage-result-file-job}") + private String topic; + + public boolean publish(TaskFileJobMessage message) { + if (message == null || message.getJobId() == null) { + return false; + } + if (!mqEnabled) { + log.info("[task-file-job] mq disabled, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={}", + message.getJobId(), message.getTaskId(), message.getModuleType()); + return false; + } + RocketMQTemplate template = rocketMQTemplateProvider.getIfAvailable(); + if (template == null) { + log.warn("[task-file-job] RocketMQTemplate unavailable, jobId={} taskId={} moduleType={}", + message.getJobId(), message.getTaskId(), message.getModuleType()); + return false; + } + try { + template.convertAndSend(topic, message); + log.info("[task-file-job] mq published jobId={} taskId={} moduleType={} topic={}", + message.getJobId(), message.getTaskId(), message.getModuleType(), topic); + return true; + } catch (Exception ex) { + log.warn("[task-file-job] publish RocketMQ failed, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} msg={}", + message.getJobId(), message.getTaskId(), message.getModuleType(), ex.getMessage()); + return false; + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java new file mode 100644 index 0000000..8166c2a --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java @@ -0,0 +1,236 @@ +package com.nanri.aiimage.modules.task.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper; +import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import lombok.RequiredArgsConstructor; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; +import java.util.List; + +@Service +@RequiredArgsConstructor +public class TaskFileJobService { + + public static final String JOB_TYPE_ASSEMBLE_RESULT = "ASSEMBLE_RESULT"; + + private final TaskFileJobMapper taskFileJobMapper; + private final ApplicationEventPublisher applicationEventPublisher; + + @Transactional + public TaskFileJobEntity enqueueAssembleResult(Long taskId, String moduleType, Long resultId, String scopeKey) { + if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank() || resultId == null || resultId <= 0) { + return null; + } + LocalDateTime now = LocalDateTime.now(); + TaskFileJobEntity existing = findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT); + if (existing == null) { + TaskFileJobEntity entity = new TaskFileJobEntity(); + entity.setTaskId(taskId); + entity.setModuleType(moduleType); + entity.setResultId(resultId); + entity.setScopeKey(scopeKey); + entity.setJobType(JOB_TYPE_ASSEMBLE_RESULT); + entity.setStatus("PENDING"); + entity.setRetryCount(0); + entity.setCreatedAt(now); + entity.setUpdatedAt(now); + try { + taskFileJobMapper.insert(entity); + publishDispatchEvent(entity); + return entity; + } catch (DuplicateKeyException ignored) { + existing = findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT); + } + } + if (existing == null) { + return null; + } + if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus())) { + return existing; + } + taskFileJobMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskFileJobEntity::getId, existing.getId()) + .set(TaskFileJobEntity::getScopeKey, scopeKey) + .set(TaskFileJobEntity::getStatus, "PENDING") + .set(TaskFileJobEntity::getErrorMessage, null) + .set(TaskFileJobEntity::getUpdatedAt, now) + .set(TaskFileJobEntity::getFinishedAt, null)); + TaskFileJobEntity refreshed = taskFileJobMapper.selectById(existing.getId()); + publishDispatchEvent(refreshed); + return refreshed; + } + + public List listRunnableJobs(int limit) { + return taskFileJobMapper.selectList(new LambdaQueryWrapper() + .in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED")) + .lt(TaskFileJobEntity::getRetryCount, 5) + .orderByAsc(TaskFileJobEntity::getUpdatedAt) + .last("limit " + Math.max(1, Math.min(limit, 100)))); + } + + public List listJobs(String status, String moduleType, Long taskId, int limit) { + LambdaQueryWrapper wrapper = new LambdaQueryWrapper() + .orderByDesc(TaskFileJobEntity::getUpdatedAt) + .last("limit " + Math.max(1, Math.min(limit, 500))); + if (status != null && !status.isBlank()) { + wrapper.eq(TaskFileJobEntity::getStatus, status.trim()); + } + if (moduleType != null && !moduleType.isBlank()) { + wrapper.eq(TaskFileJobEntity::getModuleType, moduleType.trim()); + } + if (taskId != null && taskId > 0) { + wrapper.eq(TaskFileJobEntity::getTaskId, taskId); + } + return taskFileJobMapper.selectList(wrapper); + } + + @Transactional + public boolean retry(Long jobId) { + if (jobId == null || jobId <= 0) { + return false; + } + int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskFileJobEntity::getId, jobId) + .ne(TaskFileJobEntity::getStatus, "SUCCESS") + .set(TaskFileJobEntity::getStatus, "PENDING") + .set(TaskFileJobEntity::getRetryCount, 0) + .set(TaskFileJobEntity::getErrorMessage, null) + .set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()) + .set(TaskFileJobEntity::getFinishedAt, null)); + if (updated > 0) { + TaskFileJobEntity refreshed = taskFileJobMapper.selectById(jobId); + publishDispatchEvent(refreshed); + return true; + } + return false; + } + + @Transactional + public int resetStuckRunningJobs(int stuckMinutes, int limit) { + LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes)); + List jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper() + .eq(TaskFileJobEntity::getStatus, "RUNNING") + .lt(TaskFileJobEntity::getUpdatedAt, threshold) + .orderByAsc(TaskFileJobEntity::getUpdatedAt) + .last("limit " + Math.max(1, Math.min(limit, 200)))); + int reset = 0; + for (TaskFileJobEntity job : jobs) { + int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount(); + if (retryCount >= 5) { + taskFileJobMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskFileJobEntity::getId, job.getId()) + .eq(TaskFileJobEntity::getStatus, "RUNNING") + .set(TaskFileJobEntity::getStatus, "FAILED") + .set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时") + .set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())); + continue; + } + int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskFileJobEntity::getId, job.getId()) + .eq(TaskFileJobEntity::getStatus, "RUNNING") + .set(TaskFileJobEntity::getStatus, "PENDING") + .set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时，已重新排队") + .set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())); + if (updated > 0) { + reset++; + publishDispatchEvent(taskFileJobMapper.selectById(job.getId())); + } + } + return reset; + } + + @Transactional + public boolean markRunning(Long jobId) { + if (jobId == null || jobId <= 0) { + return false; + } + return taskFileJobMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskFileJobEntity::getId, jobId) + .in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED")) + .set(TaskFileJobEntity::getStatus, "RUNNING") + .set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())) > 0; + } + + @Transactional + public void touchRunning(Long jobId) { + if (jobId == null || jobId <= 0) { + return; + } + taskFileJobMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskFileJobEntity::getId, jobId) + .eq(TaskFileJobEntity::getStatus, "RUNNING") + .set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())); + } + + @Transactional + public void markSuccess(TaskFileJobEntity job, String resultFileUrl) { + taskFileJobMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskFileJobEntity::getId, job.getId()) + .set(TaskFileJobEntity::getStatus, "SUCCESS") + .set(TaskFileJobEntity::getErrorMessage, null) + .set(TaskFileJobEntity::getResultFileUrl, resultFileUrl) + .set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()) + .set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now())); + } + + @Transactional + public void markFailed(TaskFileJobEntity job, String message) { + int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount() + 1; + taskFileJobMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskFileJobEntity::getId, job.getId()) + .set(TaskFileJobEntity::getStatus, "FAILED") + .set(TaskFileJobEntity::getRetryCount, retryCount) + .set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message) + .set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())); + } + + public TaskFileJobEntity findAssembleJob(Long taskId, String moduleType, Long resultId) { + return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT); + } + + public long countUnfinishedAssembleJobs(Long taskId, String moduleType) { + if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) { + return 0L; + } + Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskFileJobEntity::getTaskId, taskId) + .eq(TaskFileJobEntity::getModuleType, moduleType) + .eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT) + .ne(TaskFileJobEntity::getStatus, "SUCCESS")); + return count == null ? 0L : count; + } + + private TaskFileJobEntity findJob(Long taskId, String moduleType, Long resultId, String jobType) { + if (taskId == null || moduleType == null || moduleType.isBlank() || resultId == null || jobType == null || jobType.isBlank()) { + return null; + } + return taskFileJobMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskFileJobEntity::getTaskId, taskId) + .eq(TaskFileJobEntity::getModuleType, moduleType) + .eq(TaskFileJobEntity::getResultId, resultId) + .eq(TaskFileJobEntity::getJobType, jobType) + .last("limit 1")); + } + + private void publishDispatchEvent(TaskFileJobEntity entity) { + if (entity == null || entity.getId() == null) { + return; + } + applicationEventPublisher.publishEvent(new TaskFileJobDispatchEvent( + entity.getId(), + entity.getTaskId(), + entity.getModuleType(), + entity.getResultId(), + entity.getScopeKey(), + entity.getJobType(), + LocalDateTime.now() + )); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskProgressSnapshotService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskProgressSnapshotService.java new file mode 100644 index 0000000..d310ab5 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskProgressSnapshotService.java @@ -0,0 +1,98 @@ +package com.nanri.aiimage.modules.task.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.task.mapper.TaskProgressSnapshotMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity; +import lombok.RequiredArgsConstructor; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; + +@Service +@RequiredArgsConstructor +public class TaskProgressSnapshotService { + + private final TaskProgressSnapshotMapper taskProgressSnapshotMapper; + private final ObjectMapper objectMapper; + + @Transactional + public void save(Long taskId, + String moduleType, + String status, + int totalCount, + int successCount, + int failedCount, + String currentScopeKey, + String message, + Object snapshot) { + if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(status)) { + return; + } + int pendingCount = Math.max(0, totalCount - successCount - failedCount); + String snapshotJson = snapshot == null ? null : writeJson(snapshot); + LocalDateTime now = LocalDateTime.now(); + TaskProgressSnapshotEntity existing = find(taskId, moduleType); + if (existing == null) { + TaskProgressSnapshotEntity entity = new TaskProgressSnapshotEntity(); + entity.setTaskId(taskId); + entity.setModuleType(moduleType); + entity.setStatus(status); + entity.setTotalCount(totalCount); + entity.setSuccessCount(successCount); + entity.setFailedCount(failedCount); + entity.setPendingCount(pendingCount); + entity.setCurrentScopeKey(currentScopeKey); + entity.setMessage(message); + entity.setSnapshotJson(snapshotJson); + entity.setCreatedAt(now); + entity.setUpdatedAt(now); + try { + taskProgressSnapshotMapper.insert(entity); + return; + } catch (DuplicateKeyException ignored) { + existing = find(taskId, moduleType); + } + } + if (existing == null) { + throw new BusinessException("保存任务进度快照失败"); + } + taskProgressSnapshotMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskProgressSnapshotEntity::getId, existing.getId()) + .set(TaskProgressSnapshotEntity::getStatus, status) + .set(TaskProgressSnapshotEntity::getTotalCount, totalCount) + .set(TaskProgressSnapshotEntity::getSuccessCount, successCount) + .set(TaskProgressSnapshotEntity::getFailedCount, failedCount) + .set(TaskProgressSnapshotEntity::getPendingCount, pendingCount) + .set(TaskProgressSnapshotEntity::getCurrentScopeKey, currentScopeKey) + .set(TaskProgressSnapshotEntity::getMessage, message) + .set(TaskProgressSnapshotEntity::getSnapshotJson, snapshotJson) + .set(TaskProgressSnapshotEntity::getUpdatedAt, now)); + } + + public TaskProgressSnapshotEntity find(Long taskId, String moduleType) { + if (taskId == null || taskId <= 0 || isBlank(moduleType)) { + return null; + } + return taskProgressSnapshotMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskProgressSnapshotEntity::getTaskId, taskId) + .eq(TaskProgressSnapshotEntity::getModuleType, moduleType) + .last("limit 1")); + } + + private String writeJson(Object value) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new BusinessException("序列化任务进度快照失败"); + } + } + + private boolean isBlank(String value) { + return value == null || value.isBlank(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java new file mode 100644 index 0000000..2a83f7f --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java @@ -0,0 +1,166 @@ +package com.nanri.aiimage.modules.task.service; + +import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService; +import com.nanri.aiimage.modules.brand.service.BrandTaskService; +import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandRunService; +import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService; +import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService; +import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService; +import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService; +import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Slf4j +@Service +@RequiredArgsConstructor +public class TaskResultFileJobWorker { + + private final TaskFileJobService taskFileJobService; + private final TaskResultPayloadService taskResultPayloadService; + private final FileResultMapper fileResultMapper; + private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher; + private final ShopMatchTaskService shopMatchTaskService; + private final PriceTrackTaskService priceTrackTaskService; + private final ProductRiskTaskService productRiskTaskService; + private final QueryAsinTaskService queryAsinTaskService; + private final PatrolDeleteTaskService patrolDeleteTaskService; + private final AppearancePatentTaskService appearancePatentTaskService; + private final DeleteBrandRunService deleteBrandRunService; + private final BrandTaskService brandTaskService; + + @Value("${aiimage.result-file-job.local-worker-enabled:true}") + private boolean localWorkerEnabled; + + @Value("${aiimage.result-file-job.batch-size:20}") + private int batchSize; + + @Value("${aiimage.result-file-job.stuck-timeout-minutes:30}") + private int stuckTimeoutMinutes; + + @Scheduled(fixedDelayString = "${aiimage.result-file-job.local-worker-delay-ms:15000}") + public void runPendingJobs() { + if (!localWorkerEnabled) { + return; + } + List jobs = taskFileJobService.listRunnableJobs(batchSize); + if (jobs == null || jobs.isEmpty()) { + return; + } + log.info("[task-file-job] scheduled worker picked jobs count={}", jobs.size()); + for (TaskFileJobEntity job : jobs) { + boolean dispatched = taskFileJobLocalDispatcher.dispatch( + job.getId(), + job.getTaskId(), + job.getModuleType() + ); + if (!dispatched) { + process(job); + } + } + } + + @Scheduled(fixedDelayString = "${aiimage.result-file-job.stuck-scan-delay-ms:60000}") + public void resetStuckJobs() { + int reset = taskFileJobService.resetStuckRunningJobs(stuckTimeoutMinutes, batchSize); + if (reset > 0) { + log.warn("[task-file-job] reset stuck running jobs count={} timeoutMinutes={}", reset, stuckTimeoutMinutes); + } + } + + public void process(TaskFileJobEntity job) { + if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) { + return; + } + long startedAt = System.currentTimeMillis(); + try { + dispatch(job); + String resultFileUrl = resolveResultFileUrl(job); + taskFileJobService.markSuccess(job, resultFileUrl); + cleanupAfterSuccess(job); + log.info("[task-file-job] process success jobId={} taskId={} moduleType={} resultId={} elapsedMs={} resultFileUrl={}", + job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), + System.currentTimeMillis() - startedAt, resultFileUrl); + } catch (Exception ex) { + String message = ex.getMessage() == null ? "结果文件生成失败" : ex.getMessage(); + log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}", + job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), message); + taskFileJobService.markFailed(job, message); + } + } + + private String resolveResultFileUrl(TaskFileJobEntity job) { + if ("BRAND".equals(job.getModuleType())) { + return brandTaskService.resolveResultObjectKey(job.getTaskId()); + } + if (job.getResultId() == null) { + return null; + } + FileResultEntity result = fileResultMapper.selectById(job.getResultId()); + return result == null ? null : result.getResultFileUrl(); + } + + private void dispatch(TaskFileJobEntity job) { + String moduleType = job.getModuleType(); + if ("SHOP_MATCH".equals(moduleType)) { + shopMatchTaskService.processResultFileJob(job); + return; + } + if ("PRICE_TRACK".equals(moduleType)) { + priceTrackTaskService.processResultFileJob(job); + return; + } + if ("PRODUCT_RISK_RESOLVE".equals(moduleType)) { + productRiskTaskService.processResultFileJob(job); + return; + } + if ("QUERY_ASIN".equals(moduleType)) { + queryAsinTaskService.processResultFileJob(job); + return; + } + if ("PATROL_DELETE".equals(moduleType)) { + patrolDeleteTaskService.processResultFileJob(job); + return; + } + if ("APPEARANCE_PATENT".equals(moduleType)) { + appearancePatentTaskService.processResultFileJob(job); + return; + } + if ("DELETE_BRAND".equals(moduleType)) { + deleteBrandRunService.processResultFileJob(job); + return; + } + if ("BRAND".equals(moduleType)) { + brandTaskService.processResultFileJob(job); + return; + } + throw new IllegalArgumentException("unsupported result file job module: " + moduleType); + } + + private void cleanupAfterSuccess(TaskFileJobEntity job) { + String moduleType = job.getModuleType(); + if ("SHOP_MATCH".equals(moduleType) + || "PRICE_TRACK".equals(moduleType) + || "PRODUCT_RISK_RESOLVE".equals(moduleType) + || "QUERY_ASIN".equals(moduleType) + || "PATROL_DELETE".equals(moduleType)) { + taskResultPayloadService.deleteLatest(job.getTaskId(), moduleType, job.getScopeKey()); + return; + } + if ("APPEARANCE_PATENT".equals(moduleType)) { + appearancePatentTaskService.cleanupResultFileJob(job); + return; + } + if ("DELETE_BRAND".equals(moduleType)) { + deleteBrandRunService.cleanupResultFileJob(job); + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java new file mode 100644 index 0000000..d955967 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java @@ -0,0 +1,258 @@ +package com.nanri.aiimage.modules.task.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity; +import lombok.RequiredArgsConstructor; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; + +@Service +@RequiredArgsConstructor +public class TaskResultItemService { + + private final TaskResultItemMapper taskResultItemMapper; + private final ObjectMapper objectMapper; + private final TransientPayloadStorageService transientPayloadStorageService; + + @Transactional + public void replaceResultSnapshot(Long taskId, String moduleType, Long resultId, String scopeKey, Object snapshot) { + if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0 || snapshot == null) { + return; + } + String normalizedScopeKey = firstNonBlank(scopeKey, "result:" + resultId).trim(); + String itemKey = "result:" + resultId; + upsert(taskId, moduleType, resultId, normalizedScopeKey, itemKey, null, null, null, snapshot); + } + + @Transactional + public void replaceTaskSnapshots(Long taskId, String moduleType, List snapshots, SnapshotKeyResolver resolver) { + if (taskId == null || taskId <= 0 || isBlank(moduleType) || snapshots == null) { + return; + } + for (Object snapshot : snapshots) { + if (snapshot == null) { + continue; + } + Long resultId = resolver == null ? null : resolver.resultId(snapshot); + if (resultId == null || resultId <= 0) { + continue; + } + replaceResultSnapshot(taskId, moduleType, resultId, resolver.scopeKey(snapshot), snapshot); + } + } + + public List listResultSnapshots(Long taskId, String moduleType, Class clazz) { + if (taskId == null || taskId <= 0 || isBlank(moduleType)) { + return List.of(); + } + List rows = taskResultItemMapper.selectList(new LambdaQueryWrapper() + .eq(TaskResultItemEntity::getTaskId, taskId) + .eq(TaskResultItemEntity::getModuleType, moduleType) + .likeRight(TaskResultItemEntity::getItemKey, "result:") + .orderByAsc(TaskResultItemEntity::getResultId) + .orderByAsc(TaskResultItemEntity::getId)); + List out = new ArrayList<>(); + for (TaskResultItemEntity row : rows) { + T value = readPayload(row, clazz); + if (value != null) { + out.add(value); + } + } + return out; + } + + public T getResultSnapshot(Long taskId, String moduleType, Long resultId, Class clazz) { + if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0) { + return null; + } + TaskResultItemEntity row = taskResultItemMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskResultItemEntity::getTaskId, taskId) + .eq(TaskResultItemEntity::getModuleType, moduleType) + .eq(TaskResultItemEntity::getResultId, resultId) + .eq(TaskResultItemEntity::getItemKey, "result:" + resultId) + .last("limit 1")); + return readPayload(row, clazz); + } + + public long countResultSnapshots(Long taskId, String moduleType) { + if (taskId == null || taskId <= 0 || isBlank(moduleType)) { + return 0L; + } + Long count = taskResultItemMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskResultItemEntity::getTaskId, taskId) + .eq(TaskResultItemEntity::getModuleType, moduleType) + .likeRight(TaskResultItemEntity::getItemKey, "result:")); + return count == null ? 0L : count; + } + + @Transactional + public void deleteTaskItems(Long taskId, String moduleType) { + if (taskId == null || taskId <= 0 || isBlank(moduleType)) { + return; + } + List rows = taskResultItemMapper.selectList(new LambdaQueryWrapper() + .select(TaskResultItemEntity::getId, TaskResultItemEntity::getPayloadJson) + .eq(TaskResultItemEntity::getTaskId, taskId) + .eq(TaskResultItemEntity::getModuleType, moduleType)); + if (rows != null) { + for (TaskResultItemEntity row : rows) { + transientPayloadStorageService.deletePayloadIfPresent(row.getPayloadJson()); + } + } + taskResultItemMapper.delete(new LambdaQueryWrapper() + .eq(TaskResultItemEntity::getTaskId, taskId) + .eq(TaskResultItemEntity::getModuleType, moduleType)); + } + + private void upsert(Long taskId, + String moduleType, + Long resultId, + String scopeKey, + String itemKey, + String countryCode, + String asin, + String status, + Object payload) { + String payloadJson = writeJson(payload); + String payloadHash = hash(payloadJson); + String normalizedScopeKey = firstNonBlank(scopeKey, "task:" + taskId).trim(); + String normalizedItemKey = firstNonBlank(itemKey, "result:" + resultId).trim(); + String scopeHash = hash(normalizedScopeKey); + LocalDateTime now = LocalDateTime.now(); + + TaskResultItemEntity existing = find(taskId, moduleType, scopeHash, normalizedItemKey); + if (isUnchanged(existing, resultId, normalizedScopeKey, countryCode, asin, status, payloadHash)) { + return; + } + String storedPayload = transientPayloadStorageService.storeResultItemPayload( + moduleType, taskId, scopeHash, normalizedItemKey, payloadJson); + if (existing == null) { + TaskResultItemEntity entity = new TaskResultItemEntity(); + entity.setTaskId(taskId); + entity.setModuleType(moduleType); + entity.setResultId(resultId); + entity.setScopeKey(normalizedScopeKey); + entity.setScopeHash(scopeHash); + entity.setItemKey(normalizedItemKey); + entity.setCountryCode(countryCode); + entity.setAsin(asin); + entity.setStatus(status); + entity.setPayloadJson(storedPayload); + entity.setPayloadHash(payloadHash); + entity.setCreatedAt(now); + entity.setUpdatedAt(now); + try { + taskResultItemMapper.insert(entity); + return; + } catch (DuplicateKeyException ignored) { + existing = find(taskId, moduleType, scopeHash, normalizedItemKey); + } + } + if (existing == null) { + throw new BusinessException("保存任务结果明细失败"); + } + if (isUnchanged(existing, resultId, normalizedScopeKey, countryCode, asin, status, payloadHash)) { + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(storedPayload, existing.getPayloadJson()); + return; + } + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getPayloadJson(), storedPayload); + taskResultItemMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskResultItemEntity::getId, existing.getId()) + .set(TaskResultItemEntity::getResultId, resultId) + .set(TaskResultItemEntity::getScopeKey, normalizedScopeKey) + .set(TaskResultItemEntity::getCountryCode, countryCode) + .set(TaskResultItemEntity::getAsin, asin) + .set(TaskResultItemEntity::getStatus, status) + .set(TaskResultItemEntity::getPayloadJson, storedPayload) + .set(TaskResultItemEntity::getPayloadHash, payloadHash) + .set(TaskResultItemEntity::getUpdatedAt, now)); + } + + private boolean isUnchanged(TaskResultItemEntity existing, + Long resultId, + String scopeKey, + String countryCode, + String asin, + String status, + String payloadHash) { + if (existing == null) { + return false; + } + return java.util.Objects.equals(existing.getResultId(), resultId) + && java.util.Objects.equals(existing.getScopeKey(), scopeKey) + && java.util.Objects.equals(existing.getCountryCode(), countryCode) + && java.util.Objects.equals(existing.getAsin(), asin) + && java.util.Objects.equals(existing.getStatus(), status) + && java.util.Objects.equals(existing.getPayloadHash(), payloadHash); + } + + private TaskResultItemEntity find(Long taskId, String moduleType, String scopeHash, String itemKey) { + return taskResultItemMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskResultItemEntity::getTaskId, taskId) + .eq(TaskResultItemEntity::getModuleType, moduleType) + .eq(TaskResultItemEntity::getScopeHash, scopeHash) + .eq(TaskResultItemEntity::getItemKey, itemKey) + .last("limit 1")); + } + + private T readPayload(TaskResultItemEntity row, Class clazz) { + if (row == null || isBlank(row.getPayloadJson())) { + return null; + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload(row.getPayloadJson(), "read task result item failed"); + return objectMapper.readValue(payloadJson, clazz); + } catch (BusinessException ex) { + throw ex; + } catch (Exception ex) { + throw new BusinessException("读取任务结果明细失败"); + } + } + + private String writeJson(Object value) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new BusinessException("序列化任务结果明细失败"); + } + } + + private String hash(String value) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (Exception ex) { + throw new IllegalStateException("failed to hash task result item", ex); + } + } + + private String firstNonBlank(String primary, String fallback) { + return primary == null || primary.isBlank() ? fallback : primary; + } + + private boolean isBlank(String value) { + return value == null || value.isBlank(); + } + + public interface SnapshotKeyResolver { + Long resultId(Object snapshot); + + String scopeKey(Object snapshot); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultPayloadService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultPayloadService.java new file mode 100644 index 0000000..e6fdf36 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultPayloadService.java @@ -0,0 +1,139 @@ +package com.nanri.aiimage.modules.task.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.task.mapper.TaskResultPayloadMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity; +import lombok.RequiredArgsConstructor; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.LocalDateTime; + +@Service +@RequiredArgsConstructor +public class TaskResultPayloadService { + + private static final String LATEST_SUBMISSION_ID = "latest"; + + private final TaskResultPayloadMapper taskResultPayloadMapper; + private final ObjectMapper objectMapper; + private final TransientPayloadStorageService transientPayloadStorageService; + + @Transactional + public void saveLatest(Long taskId, String moduleType, String scopeKey, Object payload) { + if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey) || payload == null) { + return; + } + String normalizedScopeKey = scopeKey.trim(); + String scopeHash = hash(normalizedScopeKey); + String payloadJson = writeJson(payload); + String storedPayload = transientPayloadStorageService.storeResultPayload( + moduleType, taskId, scopeHash, LATEST_SUBMISSION_ID, payloadJson); + String payloadHash = hash(payloadJson); + LocalDateTime now = LocalDateTime.now(); + + TaskResultPayloadEntity existing = findLatest(taskId, moduleType, scopeHash); + if (existing == null) { + TaskResultPayloadEntity entity = new TaskResultPayloadEntity(); + entity.setTaskId(taskId); + entity.setModuleType(moduleType); + entity.setScopeKey(normalizedScopeKey); + entity.setScopeHash(scopeHash); + entity.setSubmissionId(LATEST_SUBMISSION_ID); + entity.setPayloadJson(storedPayload); + entity.setPayloadHash(payloadHash); + entity.setVersion(1L); + entity.setCreatedAt(now); + entity.setUpdatedAt(now); + try { + taskResultPayloadMapper.insert(entity); + return; + } catch (DuplicateKeyException ignored) { + existing = findLatest(taskId, moduleType, scopeHash); + } + } + if (existing == null) { + throw new BusinessException("保存任务结果载荷失败"); + } + long nextVersion = existing.getVersion() == null ? 1L : existing.getVersion() + 1L; + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getPayloadJson(), storedPayload); + taskResultPayloadMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskResultPayloadEntity::getId, existing.getId()) + .set(TaskResultPayloadEntity::getScopeKey, normalizedScopeKey) + .set(TaskResultPayloadEntity::getPayloadJson, storedPayload) + .set(TaskResultPayloadEntity::getPayloadHash, payloadHash) + .set(TaskResultPayloadEntity::getVersion, nextVersion) + .set(TaskResultPayloadEntity::getUpdatedAt, now)); + } + + public T getLatest(Long taskId, String moduleType, String scopeKey, Class clazz) { + if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) { + return null; + } + TaskResultPayloadEntity entity = findLatest(taskId, moduleType, hash(scopeKey.trim())); + if (entity == null || isBlank(entity.getPayloadJson())) { + return null; + } + try { + String payloadJson = transientPayloadStorageService.resolvePayload(entity.getPayloadJson(), "read task result payload failed"); + return objectMapper.readValue(payloadJson, clazz); + } catch (BusinessException ex) { + throw ex; + } catch (Exception ex) { + throw new BusinessException("读取任务结果载荷失败"); + } + } + + @Transactional + public void deleteLatest(Long taskId, String moduleType, String scopeKey) { + if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) { + return; + } + TaskResultPayloadEntity existing = findLatest(taskId, moduleType, hash(scopeKey.trim())); + if (existing != null) { + transientPayloadStorageService.deletePayloadIfPresent(existing.getPayloadJson()); + taskResultPayloadMapper.deleteById(existing.getId()); + } + } + + private TaskResultPayloadEntity findLatest(Long taskId, String moduleType, String scopeHash) { + return taskResultPayloadMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskResultPayloadEntity::getTaskId, taskId) + .eq(TaskResultPayloadEntity::getModuleType, moduleType) + .eq(TaskResultPayloadEntity::getScopeHash, scopeHash) + .eq(TaskResultPayloadEntity::getSubmissionId, LATEST_SUBMISSION_ID) + .last("limit 1")); + } + + private String writeJson(Object value) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new BusinessException("序列化任务结果载荷失败"); + } + } + + private String hash(String value) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (Exception ex) { + throw new IllegalStateException("failed to hash task result payload", ex); + } + } + + private boolean isBlank(String value) { + return value == null || value.isBlank(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java index 362f1c4..5c4aeb5 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java @@ -3,9 +3,7 @@ package com.nanri.aiimage.modules.task.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.fasterxml.jackson.databind.ObjectMapper; -import com.nanri.aiimage.config.TaskPressureProperties; import com.nanri.aiimage.common.exception.BusinessException; -import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; import lombok.RequiredArgsConstructor; @@ -13,37 +11,23 @@ import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.security.MessageDigest; import java.time.LocalDateTime; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; @Service @RequiredArgsConstructor public class TaskScopePayloadStorageService { - private static final Set OSS_BACKED_STATE_MODULES = new HashSet<>(Set.of( - "SHOP_MATCH", - "PRODUCT_RISK_RESOLVE", - "PRICE_TRACK", - "PATROL_DELETE", - "QUERY_ASIN" - )); - private static final String OSS_POINTER_PREFIX = "oss:"; private static final String LOCAL_BUFFER_DIR = "task-scope-buffer"; private final TaskScopeStateMapper taskScopeStateMapper; private final ObjectMapper objectMapper; - private final OssStorageService ossStorageService; - private final TaskPressureProperties taskPressureProperties; + private final TransientPayloadStorageService transientPayloadStorageService; @Transactional public void saveScopePayload(Long taskId, String moduleType, String scopeKey, T payload) { @@ -53,13 +37,11 @@ public class TaskScopePayloadStorageService { String normalizedScopeKey = normalize(scopeKey); String scopeHash = hash(normalizedScopeKey); String payloadJson = writeJson(payload, "写入任务范围载荷失败"); + String stateJson = transientPayloadStorageService.storeScopePayload( + moduleType, taskId, scopeHash, payloadJson, true); LocalDateTime now = LocalDateTime.now(); + TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash); - if (shouldStoreStatePayloadInOss(moduleType)) { - saveOssBackedScopePayload(taskId, moduleType, normalizedScopeKey, scopeHash, payloadJson, now, state); - return; - } - String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson); if (state == null) { TaskScopeStateEntity entity = new TaskScopeStateEntity(); entity.setTaskId(taskId); @@ -84,7 +66,7 @@ public class TaskScopePayloadStorageService { if (state == null) { throw new BusinessException("写入任务范围载荷失败"); } - deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson); + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getStateJson(), stateJson); taskScopeStateMapper.update(null, new LambdaUpdateWrapper() .eq(TaskScopeStateEntity::getId, state.getId()) .set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey) @@ -145,8 +127,7 @@ public class TaskScopePayloadStorageService { if (state == null) { return; } - deleteBufferedPayload(taskId, moduleType, state.getScopeHash()); - deleteStatePayloadIfNeeded(state.getStateJson()); + transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson()); boolean hasParsedPayload = !isBlank(state.getParsedPayloadJson()); if (hasParsedPayload) { taskScopeStateMapper.update(null, new LambdaUpdateWrapper() @@ -195,8 +176,7 @@ public class TaskScopePayloadStorageService { if (state == null || state.getId() == null) { continue; } - deleteBufferedPayload(taskId, moduleType, state.getScopeHash()); - deleteStatePayloadIfNeeded(state.getStateJson()); + transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson()); if (!isBlank(state.getParsedPayloadJson())) { taskScopeStateMapper.update(null, new LambdaUpdateWrapper() .eq(TaskScopeStateEntity::getId, state.getId()) @@ -221,6 +201,9 @@ public class TaskScopePayloadStorageService { String scopeKey = normalize(entry.getKey()); String scopeHash = hash(scopeKey); String parsedPayloadJson = writeJson(entry.getValue(), "写入任务解析载荷失败"); + String storedParsedPayload = transientPayloadStorageService.storeParsedPayload( + moduleType, taskId, scopeHash, parsedPayloadJson, true); + TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash); if (state == null) { TaskScopeStateEntity entity = new TaskScopeStateEntity(); @@ -228,7 +211,7 @@ public class TaskScopePayloadStorageService { entity.setModuleType(moduleType); entity.setScopeKey(scopeKey); entity.setScopeHash(scopeHash); - entity.setParsedPayloadJson(parsedPayloadJson); + entity.setParsedPayloadJson(storedParsedPayload); entity.setChunkTotal(0); entity.setReceivedChunkCount(0); entity.setCompleted(0); @@ -244,11 +227,11 @@ public class TaskScopePayloadStorageService { if (state == null) { throw new BusinessException("写入任务解析载荷失败"); } - deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedPayloadJson); + transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getParsedPayloadJson(), storedParsedPayload); taskScopeStateMapper.update(null, new LambdaUpdateWrapper() .eq(TaskScopeStateEntity::getId, state.getId()) .set(TaskScopeStateEntity::getScopeKey, scopeKey) - .set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadJson) + .set(TaskScopeStateEntity::getParsedPayloadJson, storedParsedPayload) .set(TaskScopeStateEntity::getUpdatedAt, now)); } } @@ -271,7 +254,9 @@ public class TaskScopePayloadStorageService { continue; } try { - result.put(state.getScopeKey(), objectMapper.readValue(resolveParsedPayload(state.getParsedPayloadJson()), clazz)); + String payloadJson = transientPayloadStorageService.resolvePayload( + state.getParsedPayloadJson(), "read parsed payload failed"); + result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, clazz)); } catch (Exception ex) { throw new BusinessException("读取任务原始载荷失败"); } @@ -281,32 +266,7 @@ public class TaskScopePayloadStorageService { @Transactional public boolean recoverBufferedScopePayload(Long taskId, String moduleType, String scopeHash) { - if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeHash)) { - return false; - } - String payloadJson = readBufferedPayload(taskId, moduleType, scopeHash); - if (payloadJson == null) { - return false; - } - TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash); - if (state == null) { - deleteBufferedPayload(taskId, moduleType, scopeHash); - return false; - } - if (!shouldStoreStatePayloadInOss(moduleType)) { - deleteBufferedPayload(taskId, moduleType, scopeHash); - return false; - } - LocalDateTime now = LocalDateTime.now(); - String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson); - deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson); - taskScopeStateMapper.update(null, new LambdaUpdateWrapper() - .eq(TaskScopeStateEntity::getId, state.getId()) - .set(TaskScopeStateEntity::getStateJson, stateJson) - .set(TaskScopeStateEntity::getLastChunkAt, now) - .set(TaskScopeStateEntity::getUpdatedAt, now)); - deleteBufferedPayload(taskId, moduleType, scopeHash); - return true; + return false; } public Path getBufferedPayloadRoot() { @@ -321,126 +281,17 @@ public class TaskScopePayloadStorageService { .last("limit 1")); } - private String storeStatePayload(String moduleType, Long taskId, String scopeHash, String payloadJson) { - if (!shouldStoreStatePayloadInOss(moduleType)) { - return payloadJson; - } - try { - String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskScopePayload(moduleType, taskId, scopeHash, payloadJson); - return objectMapper.writeValueAsString(pointer); - } catch (Exception ex) { - throw new BusinessException("写入任务范围载荷失败"); - } - } - private String resolveStatePayload(TaskScopeStateEntity state) { if (state == null || isBlank(state.getStateJson())) { return null; } - String bufferedPayload = readBufferedPayload(state.getTaskId(), state.getModuleType(), state.getScopeHash()); - if (bufferedPayload != null) { - return bufferedPayload; - } - String ossPointer = extractOssPointer(state.getStateJson()); - if (ossPointer == null) { - return state.getStateJson(); - } try { - return ossStorageService.readObjectAsString(stripOssPointerPrefix(ossPointer)); + return transientPayloadStorageService.resolvePayload(state.getStateJson(), "read task scope payload failed"); } catch (Exception ex) { throw new BusinessException("读取任务范围载荷失败"); } } - private void deleteStatePayloadIfNeeded(String value) { - String ossPointer = extractOssPointer(value); - if (ossPointer == null) { - return; - } - try { - ossStorageService.deleteObject(stripOssPointerPrefix(ossPointer)); - } catch (Exception ignored) { - } - } - - private String resolveParsedPayload(String value) { - if (isBlank(value) || !isOssPointer(value)) { - return value; - } - try { - return ossStorageService.readObjectAsString(stripOssPointerPrefix(value)); - } catch (Exception ex) { - throw new BusinessException("读取任务解析载荷失败"); - } - } - - private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) { - if (isBlank(oldValue) || oldValue.equals(newValue) || !isOssPointer(oldValue)) { - return; - } - try { - ossStorageService.deleteObject(stripOssPointerPrefix(oldValue)); - } catch (Exception ignored) { - } - } - - private void deleteReplacedStatePayloadIfNeeded(String oldValue, String newValue) { - if (isBlank(oldValue) || oldValue.equals(newValue)) { - return; - } - deleteStatePayloadIfNeeded(oldValue); - } - - private void saveOssBackedScopePayload(Long taskId, - String moduleType, - String normalizedScopeKey, - String scopeHash, - String payloadJson, - LocalDateTime now, - TaskScopeStateEntity state) { - if (state == null) { - String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson); - TaskScopeStateEntity entity = new TaskScopeStateEntity(); - entity.setTaskId(taskId); - entity.setModuleType(moduleType); - entity.setScopeKey(normalizedScopeKey); - entity.setScopeHash(scopeHash); - entity.setStateJson(stateJson); - entity.setChunkTotal(0); - entity.setReceivedChunkCount(0); - entity.setCompleted(0); - entity.setLastChunkAt(now); - entity.setLastError(null); - entity.setCreatedAt(now); - entity.setUpdatedAt(now); - try { - taskScopeStateMapper.insert(entity); - deleteBufferedPayload(taskId, moduleType, scopeHash); - return; - } catch (DuplicateKeyException ignored) { - state = getScopeState(taskId, moduleType, scopeHash); - } - } - if (state == null) { - throw new BusinessException("刷新任务范围缓冲载荷失败"); - } - - writeBufferedPayload(taskId, moduleType, scopeHash, payloadJson); - if (!shouldFlushBufferedPayload(state, now)) { - return; - } - - String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson); - deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson); - taskScopeStateMapper.update(null, new LambdaUpdateWrapper() - .eq(TaskScopeStateEntity::getId, state.getId()) - .set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey) - .set(TaskScopeStateEntity::getStateJson, stateJson) - .set(TaskScopeStateEntity::getLastChunkAt, now) - .set(TaskScopeStateEntity::getUpdatedAt, now)); - deleteBufferedPayload(taskId, moduleType, scopeHash); - } - private String writeJson(Object value, String message) { try { return objectMapper.writeValueAsString(value); @@ -457,76 +308,6 @@ public class TaskScopePayloadStorageService { return value == null || value.isBlank(); } - private boolean shouldStoreStatePayloadInOss(String moduleType) { - return OSS_BACKED_STATE_MODULES.contains(normalize(moduleType).toUpperCase()); - } - - private boolean isOssPointer(String value) { - return !isBlank(value) && value.startsWith(OSS_POINTER_PREFIX); - } - - private String extractOssPointer(String value) { - if (isBlank(value)) { - return null; - } - if (isOssPointer(value)) { - return value; - } - try { - String decoded = objectMapper.readValue(value, String.class); - return isOssPointer(decoded) ? decoded : null; - } catch (Exception ignored) { - return null; - } - } - - private String stripOssPointerPrefix(String value) { - return value.substring(OSS_POINTER_PREFIX.length()); - } - - private boolean shouldFlushBufferedPayload(TaskScopeStateEntity state, LocalDateTime now) { - long flushIntervalMillis = Math.max(0L, taskPressureProperties.getScopePayloadFlushIntervalMillis()); - if (flushIntervalMillis <= 0L || state.getUpdatedAt() == null) { - return true; - } - return java.time.Duration.between(state.getUpdatedAt(), now).toMillis() >= flushIntervalMillis; - } - - private void writeBufferedPayload(Long taskId, String moduleType, String scopeHash, String payloadJson) { - try { - Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash); - Files.createDirectories(path.getParent()); - Files.writeString(path, payloadJson, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); - } catch (IOException ex) { - throw new BusinessException("写入任务范围缓冲载荷失败"); - } - } - - private String readBufferedPayload(Long taskId, String moduleType, String scopeHash) { - try { - Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash); - if (!Files.isRegularFile(path)) { - return null; - } - return Files.readString(path); - } catch (IOException ex) { - return null; - } - } - - private void deleteBufferedPayload(Long taskId, String moduleType, String scopeHash) { - try { - Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash); - Files.deleteIfExists(path); - } catch (IOException ignored) { - } - } - - private Path buildBufferedPayloadPath(Long taskId, String moduleType, String scopeHash) { - String normalizedModuleType = normalize(moduleType).toLowerCase(); - return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR, normalizedModuleType, String.valueOf(taskId), scopeHash + ".json"); - } - private String hash(String value) { try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java new file mode 100644 index 0000000..022fcab --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java @@ -0,0 +1,149 @@ +package com.nanri.aiimage.modules.task.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.config.TransientStorageProperties; +import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.Locale; +import java.util.UUID; + +@Service +@RequiredArgsConstructor +public class TransientPayloadStorageService { + + private static final String RUSTFS_POINTER_PREFIX = "rustfs:"; + private static final String OSS_POINTER_PREFIX = "oss:"; + + private final TransientStorageProperties properties; + private final RustfsObjectStorageService rustfsObjectStorageService; + private final OssStorageService ossStorageService; + private final ObjectMapper objectMapper; + + public boolean isWriteEnabled() { + return properties.isEnabled() && rustfsObjectStorageService.isConfigured(); + } + + public String storeScopePayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) { + return store("task-scope", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString); + } + + public String storeParsedPayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) { + return store("task-parsed", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString); + } + + public String storeResultPayload(String moduleType, Long taskId, String scopeHash, String submissionId, String content) { + return store("task-result-payload", moduleType, taskId, scopeHash, submissionId, content, true); + } + + public String storeResultItemPayload(String moduleType, Long taskId, String scopeHash, String itemKey, String content) { + return store("task-result-item", moduleType, taskId, scopeHash, itemKey, content, true); + } + + public String storeChunkPayload(String moduleType, Long taskId, String scopeHash, Integer chunkIndex, String content) { + String entryKey = chunkIndex == null ? UUID.randomUUID().toString() : "chunk-" + chunkIndex; + return store("task-chunk", moduleType, taskId, scopeHash, entryKey, content, true); + } + + public String resolvePayload(String value, String errorMessage) { + String pointer = extractPointer(value); + if (pointer == null) { + return value; + } + try { + if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) { + return rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length())); + } + if (pointer.startsWith(OSS_POINTER_PREFIX)) { + return ossStorageService.readObjectAsString(pointer.substring(OSS_POINTER_PREFIX.length())); + } + } catch (Exception ex) { + throw new IllegalStateException(errorMessage, ex); + } + return value; + } + + public void deletePayloadIfPresent(String value) { + String pointer = extractPointer(value); + if (pointer == null) { + return; + } + if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) { + rustfsObjectStorageService.deleteObject(pointer.substring(RUSTFS_POINTER_PREFIX.length())); + return; + } + if (pointer.startsWith(OSS_POINTER_PREFIX)) { + ossStorageService.deleteObject(pointer.substring(OSS_POINTER_PREFIX.length())); + } + } + + public void deleteReplacedPayloadIfNeeded(String oldValue, String newValue) { + String oldPointer = extractPointer(oldValue); + String newPointer = extractPointer(newValue); + if (oldPointer == null || oldPointer.equals(newPointer)) { + return; + } + deletePayloadIfPresent(oldPointer); + } + + public String extractPointer(String value) { + if (value == null || value.isBlank()) { + return null; + } + if (isPointer(value)) { + return value; + } + try { + String decoded = objectMapper.readValue(value, String.class); + return isPointer(decoded) ? decoded : null; + } catch (Exception ignored) { + return null; + } + } + + private boolean isPointer(String value) { + return value != null && (value.startsWith(RUSTFS_POINTER_PREFIX) || value.startsWith(OSS_POINTER_PREFIX)); + } + + private String store(String category, + String moduleType, + Long taskId, + String scopeHash, + String entryKey, + String content, + boolean encodeAsJsonString) { + if (!isWriteEnabled()) { + return content; + } + String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey); + String pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content); + if (!encodeAsJsonString) { + return pointer; + } + try { + return objectMapper.writeValueAsString(pointer); + } catch (Exception ex) { + throw new IllegalStateException("failed to encode transient storage pointer", ex); + } + } + + private String buildObjectKey(String category, String moduleType, Long taskId, String scopeHash, String entryKey) { + String normalizedCategory = normalize(category, "unknown"); + String normalizedModuleType = normalize(moduleType, "unknown"); + String normalizedScopeHash = normalize(scopeHash, UUID.randomUUID().toString()); + String normalizedEntryKey = normalize(entryKey, "latest"); + return String.format("%s/%s/%s/%s/%s.json", + normalizedCategory, + normalizedModuleType, + taskId == null ? 0L : taskId, + normalizedScopeHash, + normalizedEntryKey); + } + + private String normalize(String value, String fallback) { + String normalized = value == null ? "" : value.trim().toLowerCase(Locale.ROOT); + return normalized.isBlank() ? fallback : normalized.replaceAll("[^a-z0-9._-]+", "_"); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoSessionCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoSessionCacheService.java index 61b4a51..71099c7 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoSessionCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoSessionCacheService.java @@ -2,11 +2,11 @@ package com.nanri.aiimage.modules.ziniao.service; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.config.ZiniaoProperties; import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoSessionCacheDto; import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopCacheDto; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; @@ -15,6 +15,7 @@ import java.util.List; @Service @RequiredArgsConstructor +@Slf4j public class ZiniaoSessionCacheService { private final StringRedisTemplate stringRedisTemplate; @@ -22,62 +23,96 @@ public class ZiniaoSessionCacheService { private final ZiniaoProperties ziniaoProperties; public void saveSession(ZiniaoSessionCacheDto session) { + if (session == null || session.getSessionId() == null || session.getSessionId().isBlank()) { + return; + } try { - stringRedisTemplate.opsForValue().set(buildSessionKey(session.getSessionId()), objectMapper.writeValueAsString(session), Duration.ofHours(ziniaoProperties.getSessionTtlHours())); + stringRedisTemplate.opsForValue().set( + buildSessionKey(session.getSessionId()), + objectMapper.writeValueAsString(session), + Duration.ofHours(ziniaoProperties.getSessionTtlHours())); } catch (Exception ex) { - throw new BusinessException("保存紫鸟会话失败"); + log.warn("[ziniao-session-cache] save session degraded sessionId={} msg={}", + session.getSessionId(), ex.getMessage()); } } public ZiniaoSessionCacheDto getSession(String sessionId) { - String raw = stringRedisTemplate.opsForValue().get(buildSessionKey(sessionId)); + String raw; + try { + raw = stringRedisTemplate.opsForValue().get(buildSessionKey(sessionId)); + } catch (Exception ex) { + log.warn("[ziniao-session-cache] get session degraded sessionId={} msg={}", sessionId, ex.getMessage()); + return null; + } if (raw == null || raw.isBlank()) { return null; } try { return objectMapper.readValue(raw, ZiniaoSessionCacheDto.class); } catch (Exception ex) { - throw new BusinessException("读取紫鸟会话失败"); + log.warn("[ziniao-session-cache] parse session degraded sessionId={} msg={}", sessionId, ex.getMessage()); + return null; } } public void saveShops(String sessionId, List shops) { try { - stringRedisTemplate.opsForValue().set(buildShopsKey(sessionId), objectMapper.writeValueAsString(shops), Duration.ofMinutes(ziniaoProperties.getShopsCacheMinutes())); + stringRedisTemplate.opsForValue().set( + buildShopsKey(sessionId), + objectMapper.writeValueAsString(shops), + Duration.ofMinutes(ziniaoProperties.getShopsCacheMinutes())); } catch (Exception ex) { - throw new BusinessException("保存紫鸟店铺列表失败"); + log.warn("[ziniao-session-cache] save shops degraded sessionId={} msg={}", sessionId, ex.getMessage()); } } public List getShops(String sessionId) { - String raw = stringRedisTemplate.opsForValue().get(buildShopsKey(sessionId)); + String raw; + try { + raw = stringRedisTemplate.opsForValue().get(buildShopsKey(sessionId)); + } catch (Exception ex) { + log.warn("[ziniao-session-cache] get shops degraded sessionId={} msg={}", sessionId, ex.getMessage()); + return List.of(); + } if (raw == null || raw.isBlank()) { return List.of(); } try { return objectMapper.readValue(raw, new TypeReference>() {}); } catch (Exception ex) { - throw new BusinessException("读取紫鸟店铺列表失败"); + log.warn("[ziniao-session-cache] parse shops degraded sessionId={} msg={}", sessionId, ex.getMessage()); + return List.of(); } } public void saveCurrentShop(String sessionId, ZiniaoShopCacheDto shop) { try { - stringRedisTemplate.opsForValue().set(buildCurrentShopKey(sessionId), objectMapper.writeValueAsString(shop), Duration.ofHours(ziniaoProperties.getSessionTtlHours())); + stringRedisTemplate.opsForValue().set( + buildCurrentShopKey(sessionId), + objectMapper.writeValueAsString(shop), + Duration.ofHours(ziniaoProperties.getSessionTtlHours())); } catch (Exception ex) { - throw new BusinessException("保存当前紫鸟店铺失败"); + log.warn("[ziniao-session-cache] save current shop degraded sessionId={} msg={}", sessionId, ex.getMessage()); } } public ZiniaoShopCacheDto getCurrentShop(String sessionId) { - String raw = stringRedisTemplate.opsForValue().get(buildCurrentShopKey(sessionId)); + String raw; + try { + raw = stringRedisTemplate.opsForValue().get(buildCurrentShopKey(sessionId)); + } catch (Exception ex) { + log.warn("[ziniao-session-cache] get current shop degraded sessionId={} msg={}", sessionId, ex.getMessage()); + return null; + } if (raw == null || raw.isBlank()) { return null; } try { return objectMapper.readValue(raw, ZiniaoShopCacheDto.class); } catch (Exception ex) { - throw new BusinessException("读取当前紫鸟店铺失败"); + log.warn("[ziniao-session-cache] parse current shop degraded sessionId={} msg={}", sessionId, ex.getMessage()); + return null; } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java index 40339bb..a446d9a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java @@ -101,6 +101,8 @@ public class ZiniaoShopIndexService { return pendingResult("店铺索引信息不完整，请等待后台刷新"); } + boolean treatAsFresh = isFresh(entry) || shouldBypassFreshnessBecauseWhitelistFailure(entry); + if (!buildOpenStoreUrl) { ZiniaoShopMatchResultVo vo = new ZiniaoShopMatchResultVo(); vo.setMatched(true); @@ -110,8 +112,8 @@ public class ZiniaoShopIndexService { vo.setPlatform(entry.getPlatform()); vo.setMatchedUserId(entry.getMatchedUserId()); vo.setOpenStoreUrl(null); - vo.setMatchStatus(isFresh(entry) ? MATCH_STATUS_MATCHED : MATCH_STATUS_STALE); - vo.setMatchMessage(isFresh(entry) ? null : "店铺索引已过保鲜期，请等待后台刷新"); + vo.setMatchStatus(treatAsFresh ? MATCH_STATUS_MATCHED : MATCH_STATUS_STALE); + vo.setMatchMessage(treatAsFresh ? null : "店铺索引已过保鲜期，请等待后台刷新"); return vo; } @@ -145,8 +147,8 @@ public class ZiniaoShopIndexService { vo.setMatchStatus(MATCH_STATUS_STALE); vo.setMatchMessage("打开店铺链接暂时无法生成，请稍后重试"); } else { - vo.setMatchStatus(isFresh(entry) ? MATCH_STATUS_MATCHED : MATCH_STATUS_STALE); - vo.setMatchMessage(isFresh(entry) ? null : "店铺索引已过保鲜期，请等待后台刷新"); + vo.setMatchStatus(treatAsFresh ? MATCH_STATUS_MATCHED : MATCH_STATUS_STALE); + vo.setMatchMessage(treatAsFresh ? null : "店铺索引已过保鲜期，请等待后台刷新"); } return vo; } @@ -406,7 +408,19 @@ public class ZiniaoShopIndexService { private ZiniaoShopMatchResultVo staleOrPendingResult(String defaultMessage, ZiniaoShopIndexEntryDto entry) { if (entry != null && STATUS_STALE.equals(entry.getStatus())) { ZiniaoShopMatchResultVo vo = new ZiniaoShopMatchResultVo(); - vo.setMatched(false); + boolean hasReusableMatch = entry.getShopId() != null + && !entry.getShopId().isBlank() + && entry.getMatchedUserId() != null + && entry.getMatchedUserId() > 0; + vo.setMatched(hasReusableMatch); + if (hasReusableMatch) { + vo.setShopId(entry.getShopId()); + vo.setShopName(entry.getShopName()); + vo.setCompanyName(entry.getCompanyName()); + vo.setPlatform(entry.getPlatform()); + vo.setMatchedUserId(entry.getMatchedUserId()); + vo.setOpenStoreUrl(null); + } vo.setMatchStatus(MATCH_STATUS_STALE); vo.setMatchMessage(defaultMessage); return vo; @@ -538,6 +552,25 @@ public class ZiniaoShopIndexService { return Instant.now().toEpochMilli() - entry.getLastRefreshedAt() <= freshWindowMillis; } + private boolean shouldBypassFreshnessBecauseWhitelistFailure(ZiniaoShopIndexEntryDto entry) { + if (entry == null || !STATUS_ACTIVE.equals(entry.getStatus())) { + return false; + } + if (entry.getLastRefreshedAt() == null || entry.getLastRefreshedAt() <= 0) { + return false; + } + ZiniaoShopIndexRefreshCursorDto cursor = getRefreshCursor(); + if (cursor == null || !"FAILED".equals(cursor.getStatus())) { + return false; + } + String message = cursor.getMessage(); + if (message == null || !message.contains("白名单")) { + return false; + } + Long lastFinishedAt = cursor.getLastFinishedAt(); + return lastFinishedAt != null && lastFinishedAt >= entry.getLastRefreshedAt(); + } + private Duration resolveEntryTtl() { Integer ttlHours = ziniaoProperties.getShopIndexEntryTtlHours(); if (ttlHours == null || ttlHours <= 0) { diff --git a/backend-java/src/main/resources/application-local.example.yml b/backend-java/src/main/resources/application-local.example.yml index d76c3a9..f734238 100644 --- a/backend-java/src/main/resources/application-local.example.yml +++ b/backend-java/src/main/resources/application-local.example.yml @@ -3,15 +3,22 @@ AIIMAGE_SERVER_PORT=18080 -AIIMAGE_DB_URL=jdbc:mysql://159.75.121.33:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false -AIIMAGE_DB_USERNAME=AIimage -AIIMAGE_DB_PASSWORD=WTFrb5y6hNLz6hNy +AIIMAGE_DB_URL=jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false +AIIMAGE_DB_USERNAME=change-me +AIIMAGE_DB_PASSWORD=change-me AIIMAGE_OSS_REGION=cn-hangzhou AIIMAGE_OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com -AIIMAGE_OSS_BUCKET=nanri-ai-images -AIIMAGE_OSS_ACCESS_KEY_ID=LTAI5tNpyvzMNz9f2dHarsm8 -AIIMAGE_OSS_ACCESS_KEY_SECRET=bQSZnFH455i8tzyOgeahJmUzwmhynz +AIIMAGE_OSS_BUCKET=change-me +AIIMAGE_OSS_ACCESS_KEY_ID=change-me +AIIMAGE_OSS_ACCESS_KEY_SECRET=change-me + +AIIMAGE_TRANSIENT_STORAGE_ENABLED=true +AIIMAGE_TRANSIENT_STORAGE_ENDPOINT=http://change-me +AIIMAGE_TRANSIENT_STORAGE_BUCKET=change-me +AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_ID=change-me +AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_SECRET=change-me +AIIMAGE_TRANSIENT_STORAGE_REGION=us-east-1 AIIMAGE_STORAGE_LOCAL_TEMP_DIR=F:/project/crawler-plugin/backend-java/data/tmp @@ -19,6 +26,23 @@ AIIMAGE_MODULE_CLEANUP_ENABLED=true AIIMAGE_MODULE_CLEANUP_CRON=0 0 0 * * * AIIMAGE_MODULE_CLEANUP_MODULE_TYPES=DEDUPE,SPLIT,CONVERT,DELETE_BRAND +AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL=https://api.coze.cn +AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH=/v1/workflow/run +AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_ID=7632683471312355338 +AIIMAGE_APPEARANCE_PATENT_COZE_TOKEN= +AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE=10 +AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS=60000 +AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES=20 + +AIIMAGE_ROCKETMQ_NAME_SERVER=121.196.149.225:9876 +AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED=true +AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_ENABLED=true +AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_POOL_SIZE=2 +AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_QUEUE_CAPACITY=200 +AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_ENABLED=false +AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS=10000 +AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES=30 + AIIMAGE_ZINIAO_ENABLED=false AIIMAGE_ZINIAO_BASE_URL=https://sbappstoreapi.ziniao.com/openapi-router AIIMAGE_ZINIAO_API_KEY=change-me diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index b740648..8184634 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -28,6 +28,12 @@ spring: database: ${AIIMAGE_REDIS_DATABASE:0} timeout: ${AIIMAGE_REDIS_TIMEOUT:5s} +rocketmq: + name-server: ${AIIMAGE_ROCKETMQ_NAME_SERVER:121.196.149.225:9876} + producer: + group: ${AIIMAGE_ROCKETMQ_PRODUCER_GROUP:aiimage-backend-producer} + send-message-timeout: ${AIIMAGE_ROCKETMQ_SEND_TIMEOUT_MS:3000} + management: health: db: @@ -67,6 +73,13 @@ aiimage: bucket: ${AIIMAGE_OSS_BUCKET:change-me} access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:change-me} access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:change-me} + transient-storage: + enabled: ${AIIMAGE_TRANSIENT_STORAGE_ENABLED:false} + endpoint: ${AIIMAGE_TRANSIENT_STORAGE_ENDPOINT:} + bucket: ${AIIMAGE_TRANSIENT_STORAGE_BUCKET:} + access-key-id: ${AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_ID:} + access-key-secret: ${AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_SECRET:} + region: ${AIIMAGE_TRANSIENT_STORAGE_REGION:us-east-1} storage: local-temp-dir: ${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp} cleanup-enabled: ${AIIMAGE_STORAGE_CLEANUP_ENABLED:true} @@ -96,6 +109,8 @@ aiimage: enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true} cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *} module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE} + permission-schema-init: + enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false} task-pressure: local-task-entity-cache-millis: ${AIIMAGE_TASK_LOCAL_ENTITY_CACHE_MILLIS:3000} db-select-batch-size: ${AIIMAGE_TASK_DB_SELECT_BATCH_SIZE:200} @@ -103,6 +118,28 @@ aiimage: scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24} scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200} scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *} + result-file-job: + mq-enabled: ${AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED:false} + topic: ${AIIMAGE_RESULT_FILE_JOB_TOPIC:aiimage-result-file-job} + consumer-group: ${AIIMAGE_RESULT_FILE_JOB_CONSUMER_GROUP:aiimage-result-file-job-consumer} + local-dispatch-enabled: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_ENABLED:true} + local-dispatch-pool-size: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_POOL_SIZE:2} + local-dispatch-queue-capacity: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_QUEUE_CAPACITY:200} + local-worker-enabled: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_ENABLED:true} + local-worker-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS:10000} + stuck-scan-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_STUCK_SCAN_DELAY_MS:60000} + stuck-timeout-minutes: ${AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES:30} + batch-size: ${AIIMAGE_RESULT_FILE_JOB_BATCH_SIZE:20} + appearance-patent: + coze-base-url: ${AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL:https://api.coze.cn} + coze-workflow-path: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH:/v1/workflow/run} + coze-workflow-id: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_ID:7632683471312355338} + coze-token: ${AIIMAGE_APPEARANCE_PATENT_COZE_TOKEN:Bearer sat_gsInckdsN0qqhnZSK7K4Zevw6neJCHmYHyIbm85oGgJIGEmSdkH5OFctSKLsAHvT} + coze-batch-size: ${AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE:10} + coze-connect-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_CONNECT_TIMEOUT_MILLIS:10000} + coze-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS:60000} + stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:20} + stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *} security: shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key} internal-token: ${AIIMAGE_INTERNAL_TOKEN:} diff --git a/backend-java/src/main/resources/db/V34__appearance_patent.sql b/backend-java/src/main/resources/db/V34__appearance_patent.sql new file mode 100644 index 0000000..bd8aa66 --- /dev/null +++ b/backend-java/src/main/resources/db/V34__appearance_patent.sql @@ -0,0 +1,85 @@ +-- 外观专利检测模块复用通用任务表： +-- biz_file_task 保存任务主记录，module_type = APPEARANCE_PATENT +-- biz_file_result 保存最终 xlsx 结果文件和历史记录 +-- biz_task_scope_state 保存解析载荷 OSS 指针、scope 状态和心跳辅助状态 +-- biz_task_chunk 保存 Python 回传分片和 Java/Coze 处理后的分片结果 + +-- 兼容老库：如果通用任务表缺少 user_id，则补齐用户归属字段。 +SET @biz_file_task_user_id_exists := ( + SELECT COUNT(1) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_file_task' + AND COLUMN_NAME = 'user_id' +); +SET @sql_add_biz_file_task_user_id := IF( + @biz_file_task_user_id_exists = 0, + 'ALTER TABLE biz_file_task ADD COLUMN user_id BIGINT NULL COMMENT ''用户ID'' AFTER created_by', + 'SELECT 1' +); +PREPARE stmt_add_biz_file_task_user_id FROM @sql_add_biz_file_task_user_id; +EXECUTE stmt_add_biz_file_task_user_id; +DEALLOCATE PREPARE stmt_add_biz_file_task_user_id; + +SET @biz_file_result_user_id_exists := ( + SELECT COUNT(1) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_file_result' + AND COLUMN_NAME = 'user_id' +); +SET @sql_add_biz_file_result_user_id := IF( + @biz_file_result_user_id_exists = 0, + 'ALTER TABLE biz_file_result ADD COLUMN user_id BIGINT NULL COMMENT ''用户ID'' AFTER error_message', + 'SELECT 1' +); +PREPARE stmt_add_biz_file_result_user_id FROM @sql_add_biz_file_result_user_id; +EXECUTE stmt_add_biz_file_result_user_id; +DEALLOCATE PREPARE stmt_add_biz_file_result_user_id; + +-- 外观专利检测常用查询索引：按用户查任务、按用户查历史。 +SET @idx_file_task_module_user_created_exists := ( + SELECT COUNT(1) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_file_task' + AND INDEX_NAME = 'idx_biz_file_task_module_user_created' +); +SET @sql_add_idx_file_task_module_user_created := IF( + @idx_file_task_module_user_created_exists = 0, + 'ALTER TABLE biz_file_task ADD INDEX idx_biz_file_task_module_user_created (module_type, user_id, created_at)', + 'SELECT 1' +); +PREPARE stmt_add_idx_file_task_module_user_created FROM @sql_add_idx_file_task_module_user_created; +EXECUTE stmt_add_idx_file_task_module_user_created; +DEALLOCATE PREPARE stmt_add_idx_file_task_module_user_created; + +SET @idx_file_result_module_user_created_exists := ( + SELECT COUNT(1) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'biz_file_result' + AND INDEX_NAME = 'idx_biz_file_result_module_user_created' +); +SET @sql_add_idx_file_result_module_user_created := IF( + @idx_file_result_module_user_created_exists = 0, + 'ALTER TABLE biz_file_result ADD INDEX idx_biz_file_result_module_user_created (module_type, user_id, created_at)', + 'SELECT 1' +); +PREPARE stmt_add_idx_file_result_module_user_created FROM @sql_add_idx_file_result_module_user_created; +EXECUTE stmt_add_idx_file_result_module_user_created; +DEALLOCATE PREPARE stmt_add_idx_file_result_module_user_created; + +-- 前端栏目权限兜底：外观专利检测属于前端工具下的页面。 +INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`) +SELECT '外观专利检测', 'appearance_patent', 'app', 'appearance-patent', 111 +WHERE NOT EXISTS ( + SELECT 1 FROM `columns` WHERE `column_key` = 'appearance_patent' +); + +UPDATE `columns` +SET `name` = '外观专利检测', + `menu_type` = 'app', + `route_path` = 'appearance-patent', + `sort_order` = 111 +WHERE `column_key` = 'appearance_patent'; diff --git a/backend-java/src/main/resources/db/V35__task_result_payload_and_file_job.sql b/backend-java/src/main/resources/db/V35__task_result_payload_and_file_job.sql new file mode 100644 index 0000000..2c38347 --- /dev/null +++ b/backend-java/src/main/resources/db/V35__task_result_payload_and_file_job.sql @@ -0,0 +1,74 @@ +CREATE TABLE IF NOT EXISTS biz_task_result_payload ( + id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key', + task_id BIGINT NOT NULL COMMENT 'task id', + module_type VARCHAR(32) NOT NULL COMMENT 'module type', + scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier, such as shop name or file key', + scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key', + submission_id VARCHAR(128) NULL COMMENT 'idempotency key from submitter when available', + payload_json JSON NOT NULL COMMENT 'raw or normalized callback payload', + payload_hash CHAR(64) NOT NULL COMMENT 'sha256 of payload json', + version BIGINT NOT NULL DEFAULT 1 COMMENT 'scope payload version', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time', + UNIQUE KEY uk_task_scope_submission (task_id, module_type, scope_hash, submission_id), + KEY idx_task_module_scope (task_id, module_type, scope_hash), + KEY idx_task_module_updated (task_id, module_type, updated_at) +) COMMENT='task result callback payload detail'; + +CREATE TABLE IF NOT EXISTS biz_task_file_job ( + id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key', + task_id BIGINT NOT NULL COMMENT 'task id', + module_type VARCHAR(32) NOT NULL COMMENT 'module type', + result_id BIGINT NULL COMMENT 'biz_file_result id when job is scoped to one result row', + scope_key VARCHAR(1000) NULL COMMENT 'scope identifier', + job_type VARCHAR(32) NOT NULL DEFAULT 'ASSEMBLE_RESULT' COMMENT 'job type', + status VARCHAR(32) NOT NULL DEFAULT 'PENDING' COMMENT 'PENDING/RUNNING/SUCCESS/FAILED', + retry_count INT NOT NULL DEFAULT 0 COMMENT 'retry count', + error_message VARCHAR(1000) NULL COMMENT 'last error', + result_file_url VARCHAR(1000) NULL COMMENT 'generated file object key or url', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time', + finished_at DATETIME NULL COMMENT 'finished time', + UNIQUE KEY uk_file_job_scope (task_id, module_type, result_id, job_type), + KEY idx_file_job_status (status, updated_at), + KEY idx_file_job_task (task_id, module_type) +) COMMENT='async result file assembly job'; + +CREATE TABLE IF NOT EXISTS biz_task_result_item ( + id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key', + task_id BIGINT NOT NULL COMMENT 'task id', + module_type VARCHAR(32) NOT NULL COMMENT 'module type', + result_id BIGINT NULL COMMENT 'biz_file_result id when available', + scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier, such as shop name or file key', + scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key', + item_key VARCHAR(512) NOT NULL COMMENT 'dedupe key, such as country + asin', + country_code VARCHAR(32) NULL COMMENT 'country code', + asin VARCHAR(128) NULL COMMENT 'asin or item id', + status VARCHAR(64) NULL COMMENT 'business status', + payload_json JSON NOT NULL COMMENT 'normalized row payload', + payload_hash CHAR(64) NOT NULL COMMENT 'sha256 of payload json', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time', + UNIQUE KEY uk_task_scope_item (task_id, module_type, scope_hash, item_key), + KEY idx_result_item_result (result_id), + KEY idx_result_item_task (task_id, module_type), + KEY idx_result_item_country_asin (task_id, module_type, country_code, asin) +) COMMENT='normalized task result item detail'; + +CREATE TABLE IF NOT EXISTS biz_task_progress_snapshot ( + id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key', + task_id BIGINT NOT NULL COMMENT 'task id', + module_type VARCHAR(32) NOT NULL COMMENT 'module type', + status VARCHAR(32) NOT NULL COMMENT 'task status', + total_count INT NOT NULL DEFAULT 0 COMMENT 'total item count', + success_count INT NOT NULL DEFAULT 0 COMMENT 'success count', + failed_count INT NOT NULL DEFAULT 0 COMMENT 'failed count', + pending_count INT NOT NULL DEFAULT 0 COMMENT 'pending count', + current_scope_key VARCHAR(1000) NULL COMMENT 'current scope identifier', + message VARCHAR(1000) NULL COMMENT 'progress message', + snapshot_json JSON NULL COMMENT 'small progress snapshot', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time', + UNIQUE KEY uk_task_progress_snapshot (task_id, module_type), + KEY idx_progress_module_status (module_type, status, updated_at) +) COMMENT='lightweight task progress snapshot'; diff --git a/backend/__pycache__/config.cpython-312.pyc b/backend/__pycache__/config.cpython-312.pyc index 08477f7..0afb1db 100644 Binary files a/backend/__pycache__/config.cpython-312.pyc and b/backend/__pycache__/config.cpython-312.pyc differ diff --git a/backend/blueprints/__pycache__/admin_api.cpython-312.pyc b/backend/blueprints/__pycache__/admin_api.cpython-312.pyc index d659909..e207bee 100644 Binary files a/backend/blueprints/__pycache__/admin_api.cpython-312.pyc and b/backend/blueprints/__pycache__/admin_api.cpython-312.pyc differ diff --git a/backend/blueprints/admin_api.py b/backend/blueprints/admin_api.py index 8ca619d..1652c5c 100644 --- a/backend/blueprints/admin_api.py +++ b/backend/blueprints/admin_api.py @@ -1441,6 +1441,77 @@ def delete_dedupe_total_data(item_id): # ---------- 店铺管理 ---------- +def _format_shop_manage_item(item): + return { + 'id': item.get('id'), + 'group_id': item.get('groupId'), + 'group_name': item.get('groupName') or '', + 'shop_name': item.get('shopName') or '', + 'mall_name': item.get('mallName') or '', + 'account': item.get('account') or '', + 'password': item.get('passwordMasked') or '', + 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], + 'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16], + } + + +def _format_shop_manage_group_item(item): + return { + 'id': item.get('id'), + 'group_name': item.get('groupName') or '', + 'leader_user_id': item.get('leaderUserId'), + 'leader_username': item.get('leaderUsername') or '', + 'member_user_ids': item.get('memberUserIds') or [], + 'member_usernames': item.get('memberUsernames') or [], + 'member_count': item.get('memberCount') or 0, + 'user_id': item.get('userId'), + 'username': item.get('username') or '', + 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], + 'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16], + } + + +def _load_expanded_shop_manage_groups(role, current_row): + params = {} + if current_row and current_row.get('id'): + params['operatorId'] = current_row.get('id') + params['superAdmin'] = 'true' if role == 'super_admin' else 'false' + result, error_response, status = _proxy_backend_java( + 'GET', + '/api/admin/shop-manages/groups', + params=params, + ) + if error_response is not None: + return None, error_response, status + + direct_groups = result.get('data') or [] + if role == 'super_admin': + return direct_groups, None, 200 + + direct_ids = {item.get('id') for item in direct_groups if item.get('id')} + leader_ids = {item.get('leaderUserId') for item in direct_groups if item.get('leaderUserId')} + if not leader_ids: + return direct_groups, None, 200 + + all_result, error_response, status = _proxy_backend_java( + 'GET', + '/api/admin/shop-manages/groups', + params={'superAdmin': 'true'}, + ) + if error_response is not None: + return None, error_response, status + + merged = [] + seen_ids = set() + for item in all_result.get('data') or []: + item_id = item.get('id') + if item_id in direct_ids or item.get('leaderUserId') in leader_ids: + if item_id not in seen_ids: + merged.append(item) + seen_ids.add(item_id) + return merged, None, 200 + + @admin_api.route('/shop-manages') @login_required def list_shop_manages(): @@ -1452,6 +1523,59 @@ def list_shop_manages(): group_id_raw = (request.args.get('group_id') or '').strip() shop_name = (request.args.get('shop_name') or '').strip() + if role != 'super_admin': + groups, error_response, status = _load_expanded_shop_manage_groups(role, current_row) + if error_response is not None: + return error_response, status + accessible_group_ids = [item.get('id') for item in (groups or []) if item.get('id')] + selected_group_id = None + if group_id_raw: + try: + selected_group_id = int(group_id_raw) + except (TypeError, ValueError): + selected_group_id = None + if selected_group_id and selected_group_id not in accessible_group_ids: + return jsonify({'success': True, 'items': [], 'total': 0, 'page': page, 'page_size': page_size}) + query_group_ids = [selected_group_id] if selected_group_id else accessible_group_ids + all_items = [] + for group_id in query_group_ids: + group_page = 1 + while True: + group_params = { + 'page': group_page, + 'pageSize': 100, + 'groupId': group_id, + 'superAdmin': 'true', + } + if current_row and current_row.get('id'): + group_params['operatorId'] = current_row.get('id') + if shop_name: + group_params['shopName'] = shop_name + result, error_response, status = _proxy_backend_java( + 'GET', + '/api/admin/shop-manages', + params=group_params, + ) + if error_response is not None: + return error_response, status + payload = result.get('data') or {} + rows = payload.get('items') or [] + all_items.extend(rows) + total = payload.get('total') or 0 + if group_page * 100 >= total or not rows: + break + group_page += 1 + all_items.sort(key=lambda item: item.get('id') or 0, reverse=True) + total = len(all_items) + offset = (page - 1) * page_size + return jsonify({ + 'success': True, + 'items': [_format_shop_manage_item(item) for item in all_items[offset:offset + page_size]], + 'total': total, + 'page': page, + 'page_size': page_size, + }) + params = {'page': page, 'pageSize': page_size} if current_row and current_row.get('id'): params['operatorId'] = current_row.get('id') @@ -1474,23 +1598,9 @@ def list_shop_manages(): if error_response is not None: return error_response, status payload = result.get('data') or {} - items = [ - { - 'id': item.get('id'), - 'group_id': item.get('groupId'), - 'group_name': item.get('groupName') or '', - 'shop_name': item.get('shopName') or '', - 'mall_name': item.get('mallName') or '', - 'account': item.get('account') or '', - 'password': item.get('passwordMasked') or '', - 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], - 'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16], - } - for item in (payload.get('items') or []) - ] return jsonify({ 'success': True, - 'items': items, + 'items': [_format_shop_manage_item(item) for item in (payload.get('items') or [])], 'total': payload.get('total') or 0, 'page': payload.get('page') or page, 'page_size': payload.get('pageSize') or page_size, @@ -1611,36 +1721,10 @@ def list_shop_manage_groups(): role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin', 'query-asin') if denied: return denied - params = {} - if current_row and current_row.get('id'): - params['operatorId'] = current_row.get('id') - params['superAdmin'] = 'true' if role == 'super_admin' else 'false' - if role != 'super_admin' and current_row and current_row.get('id'): - params['createdById'] = current_row.get('id') - result, error_response, status = _proxy_backend_java( - 'GET', - '/api/admin/shop-manages/groups', - params=params, - ) + groups, error_response, status = _load_expanded_shop_manage_groups(role, current_row) if error_response is not None: return error_response, status - items = [ - { - 'id': item.get('id'), - 'group_name': item.get('groupName') or '', - 'leader_user_id': item.get('leaderUserId'), - 'leader_username': item.get('leaderUsername') or '', - 'member_user_ids': item.get('memberUserIds') or [], - 'member_usernames': item.get('memberUsernames') or [], - 'member_count': item.get('memberCount') or 0, - 'user_id': item.get('userId'), - 'username': item.get('username') or '', - 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], - 'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16], - } - for item in (result.get('data') or []) - ] - return jsonify({'success': True, 'items': items}) + return jsonify({'success': True, 'items': [_format_shop_manage_group_item(item) for item in (groups or [])]}) @admin_api.route('/shop-manage-group', methods=['POST']) diff --git a/backend/config.py b/backend/config.py index 6692838..4e042f5 100644 --- a/backend/config.py +++ b/backend/config.py @@ -24,7 +24,8 @@ file_url_pre = f"https://{bucket}.oss-cn-hangzhou.aliyuncs.com/" import os # backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/') -backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/') +# backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/') +backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://47.111.163.154:18080').rstrip('/') os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6" diff --git a/backend/web_source/admin.html b/backend/web_source/admin.html index 6094daf..ccfb20b 100644 --- a/backend/web_source/admin.html +++ b/backend/web_source/admin.html @@ -862,7 +862,7 @@

导入添加

- +
@@ -879,7 +879,7 @@

导入删除

- +
diff --git a/frontend-vue/appearance-patent.html b/frontend-vue/appearance-patent.html new file mode 100644 index 0000000..8aca6ce --- /dev/null +++ b/frontend-vue/appearance-patent.html @@ -0,0 +1,12 @@ + + + + + + 外观专利检测 + + +
+ + + diff --git a/frontend-vue/src/appearance-patent-main.ts b/frontend-vue/src/appearance-patent-main.ts new file mode 100644 index 0000000..8cb25e1 --- /dev/null +++ b/frontend-vue/src/appearance-patent-main.ts @@ -0,0 +1,7 @@ +import { createApp } from 'vue' +import ElementPlus from 'element-plus' +import 'element-plus/dist/index.css' +import '@/styles/main.css' +import BrandAppearancePatentTab from '@/pages/brand/components/BrandAppearancePatentTab.vue' + +createApp(BrandAppearancePatentTab).use(ElementPlus).mount('#app') diff --git a/frontend-vue/src/pages/brand/components/BrandAppearancePatentTab.vue b/frontend-vue/src/pages/brand/components/BrandAppearancePatentTab.vue new file mode 100644 index 0000000..91a6535 --- /dev/null +++ b/frontend-vue/src/pages/brand/components/BrandAppearancePatentTab.vue @@ -0,0 +1,605 @@ +