From 080625567bb61a2411204bdd840e577814b8e9ac Mon Sep 17 00:00:00 2001 From: super <2903208875@qq.com> Date: Fri, 29 May 2026 19:47:23 +0800 Subject: [PATCH] =?UTF-8?q?=E5=90=8C=E6=AD=A5=20Gitee=20=E5=BA=94=E7=94=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/amazon/approve.py | 17 +++++++++++++---- app/amazon/chrome_base.py | 12 +++++++++++- app/amazon/detail_spider.py | 21 +++++++++++++++++--- app/amazon/match_action.py | 38 +++++++++++++++++++++++++------------ app/amazon/price_match.py | 30 +++++++++++++++++------------ app/amazon/similar_asin.py | 18 ++++++++++++++++-- app/blueprints/auth.py | 1 + 7 files changed, 103 insertions(+), 34 deletions(-) diff --git a/app/amazon/approve.py b/app/amazon/approve.py index d079001..f851de5 100644 --- a/app/amazon/approve.py +++ b/app/amazon/approve.py @@ -308,8 +308,9 @@ class AmzoneApprove(AmamzonBase): self.log("开始执行...") num = 0 retry_num = 0 - max_retry_num = 5 + total_page = 0 + current_page = 0 while retry_num < max_retry_num: # 最多重试3次 # if num > 3: #测试 # return @@ -325,13 +326,13 @@ class AmzoneApprove(AmamzonBase): page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5) if len(page_pamel) > 0: current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text + current_page = int(current_page.strip()) # 总页数 total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]') if len(total_page) > 0: - total_page = total_page[-1].text - else: - total_page = 0 + total_page = int(total_page[-1].text.strip()) + self.log(f"当前页码: {current_page} / 总页数: {total_page}") except Exception as e: self.log(f"获取页码失败:{e}") @@ -543,6 +544,14 @@ class AmzoneApprove(AmamzonBase): self.tab.refresh() self.tab.wait.doc_loaded(raise_err=False,timeout=120) + #检查页数是否相等，不相等则继续 + self.log(f"开始检查页数，当前页数 {current_page} / {total_page}") + if total_page != 0 and current_page!= 0 and current_page < total_page: + self.log(f"检查到页数还未完成，重启浏览器继续") + raise RuntimeError(f"与页面的连接已断开，检查到页数还未完成，重启浏览器继续") + + + class ApproveTask(TaskBase): """审批任务处理类：负责处理产品风险审批任务""" task_name = "产品风险审批-TASK" diff --git a/app/amazon/chrome_base.py b/app/amazon/chrome_base.py index cc7d756..84b39f8 100644 --- a/app/amazon/chrome_base.py +++ b/app/amazon/chrome_base.py @@ -193,4 +193,14 @@ class ChromeAmzoneBase: self.browser.quit() print("浏览器已关闭") except Exception as e: - print(f"关闭浏览器时出错: {str(e)}") \ No newline at end of file + print(f"关闭浏览器时出错: {str(e)}") + + + + + + + + + + diff --git a/app/amazon/detail_spider.py b/app/amazon/detail_spider.py index 6db0607..6afa92e 100644 --- a/app/amazon/detail_spider.py +++ b/app/amazon/detail_spider.py @@ -13,7 +13,7 @@ from amazon.amazon_base import TaskBase from amazon.chrome_base import ChromeAmzoneBase -from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE +from config import runing_task,runing_shop,base_dir,DELETE_BRAND_API_BASE class ChromeAmzone(ChromeAmzoneBase): @@ -92,7 +92,8 @@ class ChromeAmzone(ChromeAmzoneBase): """ data = { 'image_url': "", - 'title': "" + 'title': "", + "sku" : "" } try: @@ -103,26 +104,39 @@ class ChromeAmzone(ChromeAmzoneBase): data["title"] = title sku_ele_ls = self.tab.eles('xpath://ul[@class="a-unordered-list a-vertical a-spacing-mini"]', timeout=20) if len(sku_ele_ls) > 0: - data["title"] += sku_ele_ls[0].text + data["sku"] = sku_ele_ls[0].text imge_ele = self.tab.ele('xpath://div[@id="imgTagWrapperId"]//img',timeout=20) image_url = "" + min_image_url = "" data_a_dynamic_image = imge_ele.attr("data-a-dynamic-image") if data_a_dynamic_image: dynamic_image_json = json.loads(data_a_dynamic_image) self.log(f"图片信息:{dynamic_image_json}") max_area = 0 + min_area = 0 for url, (width, height) in dynamic_image_json.items(): area = width * height if area > max_area: max_area = area image_url = url + if min_area == 0: + min_area = area + min_image_url = url + if area < min_area: + min_area = area + min_image_url = url if not image_url: image_url = imge_ele.attr("src") + if not min_image_url: + min_image_url = imge_ele.attr("src") + data["image_url"] = image_url + data["min_image_url"] = min_image_url + return data except Exception as e: @@ -266,6 +280,7 @@ class SpiderTask(TaskBase): "country": i.get("country"), "url": return_data.get("image_url"), "title": return_data.get("title"), + "sku": return_data.get("sku") } for i in items ] diff --git a/app/amazon/match_action.py b/app/amazon/match_action.py index 9ea7cde..1faa795 100644 --- a/app/amazon/match_action.py +++ b/app/amazon/match_action.py @@ -35,9 +35,12 @@ class AmzoneMatchAction(AmamzonBase): self.log("开始执行...") num = 0 retry_num = 0 - get_page_faild = 0 - while retry_num < 3: # 最多重试3次 + max_retry_num = 5 + total_page = 0 + current_page = 0 + + while retry_num < max_retry_num: # if num > 3: #测试 # return # 等待加载完成 @@ -45,6 +48,7 @@ class AmzoneMatchAction(AmamzonBase): load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5) if len(load_ele) > 0: load_ele[0].wait.deleted(timeout=3, raise_err=False) + time.sleep(0.5) # 获取当前页码 @@ -52,23 +56,17 @@ class AmzoneMatchAction(AmamzonBase): page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5) if len(page_pamel) > 0: current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text + current_page = int(current_page.strip()) # 测试 # if current_page > 1: # break # 总页数 total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]') if len(total_page) > 0: - total_page = total_page[-1].text - else: - total_page = 0 + total_page = int(total_page[-1].text.strip()) self.log(f"【{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}") - get_page_faild = 0 except Exception as e: self.log(f"【{self.mark_name}】获取页码失败", e) - get_page_faild+= 1 - if get_page_faild > 2: - show_notification(f"【{self.mark_name}】获取页码失败超3次停止任务！") - break # 保存当前URL，用于失败重试时访问 try: @@ -81,6 +79,15 @@ class AmzoneMatchAction(AmamzonBase): sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10) self.log(f"【{self.mark_name}】获取到 {len(sku_ls)}") + if len(sku_ls) == 0: + retry_num += 1 + self.log(f"没有获取到SKU列表，重试{retry_num}/{max_retry_num}") + self.tab.refresh() + self.tab.wait.doc_loaded(raise_err=False, timeout=120) + if retry_num == max_retry_num: + raise RuntimeError(f"与页面的连接已断开，没有获取到SKU列表,重试{retry_num}/{max_retry_num}") + continue + # for sku_ele in sku_ls[0:2]: for sku_ele in sku_ls: # solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]') @@ -140,6 +147,12 @@ class AmzoneMatchAction(AmamzonBase): self.tab.refresh() self.tab.wait.doc_loaded(raise_err=False,timeout=120) + #检查页数是否相等，不相等则继续 + self.log(f"开始检查页数，当前页数 {current_page} / {total_page}") + if total_page != 0 and current_page!= 0 and current_page < total_page: + self.log(f"检查到页数还未完成，重启浏览器继续") + raise RuntimeError(f"与页面的连接已断开，检查到页数还未完成，重启浏览器继续") + class MatchTak(TaskBase): task_name = "匹配价格-TASK" @@ -261,7 +274,7 @@ class MatchTak(TaskBase): self.log(f"检测到任务 {task_id} 的暂停请求，停止处理国家", "WARNING") break - skip_asin = skip_asins_by_country.get(country_code) + skip_asin = skip_asins_by_country.get(country_code,[]) if skipAsinDetailsByCountry : skipAsinDetails = {i.get("asin"):i.get("minimumPrice") for i in skipAsinDetailsByCountry.get(country_code)} else: @@ -276,6 +289,7 @@ class MatchTak(TaskBase): return current_url = None + max_retries = 200 for _ in range(max_retries): try: self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter,limit, @@ -366,8 +380,8 @@ class MatchTak(TaskBase): if task_id in runing_task: runing_task[task_id]["current_asin"] = asin runing_task[task_id]["processed_asins"] += 1 - runing_task[task_id]["failed_count"] += 1 + minimumPrice = "" if "有最低价跳过" in status: minimumPrice = skipAsinDetails.get(asin) diff --git a/app/amazon/price_match.py b/app/amazon/price_match.py index a513b33..a5bca12 100644 --- a/app/amazon/price_match.py +++ b/app/amazon/price_match.py @@ -11,7 +11,7 @@ from amazon.tool import show_notification,get_shop_info,remove_special_character from config import runing_task, runing_shop,DELETE_BRAND_API_BASE class RepricingLogic: - + @staticmethod def situation_1_general_decrease(my_price, rank1_price, is_first_time=False): """ @@ -460,6 +460,8 @@ class AmzonePriceMatch(AmamzonBase): super().__init__(user_info,socket_port) self.already_asin = set() + + def reset_already_asin(self): self.already_asin = set() self.log("already_asin 已重置") @@ -513,14 +515,15 @@ class AmzonePriceMatch(AmamzonBase): self.log("开始执行...") num = 0 retry_num = 0 - already_asin = set() - get_page_faild = 0 + already_asin = self.already_asin #获取当前国家 current_country_ele = self.tab.ele('xpath://div[@class="dropdown-account-switcher-header-label"]/span[last()]', timeout=20) current_country = current_country_ele.text.strip() max_retry_num = 5 - while retry_num < max_retry_num: # 最多重试3次 + total_page = 0 + current_page = 0 + while retry_num < max_retry_num: try: if appoint_asin is not None: self.log(f"指定asin操作,{appoint_asin}") @@ -536,6 +539,7 @@ class AmzonePriceMatch(AmamzonBase): page_pamel = self.tab.eles('xpath://kat-pagination', timeout=5) if len(page_pamel) > 0: current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text + current_page = int(current_page.strip()) #测试===== # if int(current_page) > 1: # break @@ -546,17 +550,12 @@ class AmzonePriceMatch(AmamzonBase): total_page = page_pamel[0].sr.eles( 'xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]') if len(total_page) > 0: - total_page = total_page[-1].text - else: - total_page = 0 + total_page = int(total_page[-1].text.strip()) + self.log(f"【{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}") - get_page_faild = 0 except Exception as e: self.log(f"获取页码失败,{e}") - get_page_faild += 1 - if get_page_faild > 2: - show_notification(f"【{self.mark_name}】获取页码失败超3次停止任务！") - break + # 保存当前URL，用于失败重试时访问 try: @@ -767,6 +766,12 @@ class AmzonePriceMatch(AmamzonBase): retry_num += 1 self.tab.refresh() self.tab.wait.doc_loaded(raise_err=False, timeout=120) + #检查页数是否相等，不相等则继续 + self.log(f"开始检查页数，当前页数 {current_page} / {total_page}") + if total_page != 0 and current_page!= 0 and current_page < total_page: + self.log(f"检查到页数还未完成，重启浏览器继续") + raise RuntimeError(f"与页面的连接已断开，检查到页数还未完成，重启浏览器继续") + class PriceTask(TaskBase): task_name = "跟价-TASK" @@ -922,6 +927,7 @@ class PriceTask(TaskBase): miniprice_info = minimum_price_by_country_and_asin.get(country_code,{}) current_url = None + max_retries = 200 for _ in range(max_retries): try: self.process_country(driver, country_code, task_id, shop_name, risk_listing_filter, diff --git a/app/amazon/similar_asin.py b/app/amazon/similar_asin.py index 181e47b..1864542 100644 --- a/app/amazon/similar_asin.py +++ b/app/amazon/similar_asin.py @@ -257,22 +257,35 @@ class ChromeAmzone(ChromeAmzoneBase): imge_ele = self.tab.ele('xpath://div[@id="imgTagWrapperId"]//img',timeout=20) image_url = "" + min_image_url = "" data_a_dynamic_image = imge_ele.attr("data-a-dynamic-image") if data_a_dynamic_image: dynamic_image_json = json.loads(data_a_dynamic_image) self.log(f"图片信息:{dynamic_image_json}") max_area = 0 + min_area = 0 for url, (width, height) in dynamic_image_json.items(): area = width * height if area > max_area: max_area = area image_url = url + if min_area == 0: + min_area = area + min_image_url = url + if area < min_area: + min_area = area + min_image_url = url if not image_url: image_url = imge_ele.attr("src") + if not min_image_url: + min_image_url = imge_ele.attr("src") + + data["image_url"] = image_url + data["min_image_url"] = min_image_url # sku sku_ele_ls = self.tab.eles('xpath://ul[@class="a-unordered-list a-vertical a-spacing-mini"]',timeout=20) @@ -340,9 +353,10 @@ class ChromeAmzone(ChromeAmzoneBase): # 判断是否采集标题出错 new_size = "220,220" pattern = r"\._(?:[A-Z]+)?(\d+)_\." - replacement = f"._{new_size}_.jpg" + # replacement = f"._{new_size}_.jpg" # 使用正则替换 - image_new_url = re.sub(pattern, lambda m: replacement, data["image_url"]) + # image_new_url = re.sub(pattern, lambda m: replacement, data["image_url"]) + image_new_url = data["min_image_url"] print(image_new_url) diff --git a/app/blueprints/auth.py b/app/blueprints/auth.py index d1ced64..f4fa1dd 100644 --- a/app/blueprints/auth.py +++ b/app/blueprints/auth.py @@ -28,6 +28,7 @@ def api_auth_sync(): """前端拿到 Java 返回的 JWT 后调用，把 token 写进 Python 同源 cookie。""" data = request.get_json(silent=True) or {} token = (data.get('token') or '').strip() + if not token: return jsonify({'success': False, 'error': '缺少 token'}), 400 payload, reason = parse_token_with_reason(token)