super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

同步 Gitee 应用代码更新

Browse Source
This commit is contained in:
super
2026-05-29 19:47:23 +08:00
parent 266c0f17c1
commit 080625567b
7 changed files with 103 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
app/amazon/match_action.py
View File

@@ -35,9 +35,12 @@ class AmzoneMatchAction(AmamzonBase):
self.log("开始执行...")
num = 0
retry_num = 0
get_page_faild = 0
while retry_num < 3: # 最多重试3次
max_retry_num = 5
total_page = 0
current_page = 0
while retry_num < max_retry_num:
# if num > 3: #测试
# return
# 等待加载完成
@@ -45,6 +48,7 @@ class AmzoneMatchAction(AmamzonBase):
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
# 获取当前页码
@@ -52,23 +56,17 @@ class AmzoneMatchAction(AmamzonBase):
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) > 0:
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
current_page = int(current_page.strip())
# 测试
# if current_page > 1:
# break
# 总页数
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
if len(total_page) > 0:
total_page = total_page[-1].text
else:
total_page = 0
total_page = int(total_page[-1].text.strip())
self.log(f"【{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}")
get_page_faild = 0
except Exception as e:
self.log(f"【{self.mark_name}】获取页码失败", e)
get_page_faild+= 1
if get_page_faild > 2:
show_notification(f"【{self.mark_name}】获取页码失败超3次停止任务！")
break
# 保存当前URL，用于失败重试时访问
try:
@@ -81,6 +79,15 @@ class AmzoneMatchAction(AmamzonBase):
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
self.log(f"【{self.mark_name}】获取到 {len(sku_ls)}")
if len(sku_ls) == 0:
retry_num += 1
self.log(f"没有获取到SKU列表，重试{retry_num}/{max_retry_num}")
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False, timeout=120)
if retry_num == max_retry_num:
raise RuntimeError(f"与页面的连接已断开，没有获取到SKU列表,重试{retry_num}/{max_retry_num}")
continue
# for sku_ele in sku_ls[0:2]:
for sku_ele in sku_ls:
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]')
@@ -140,6 +147,12 @@ class AmzoneMatchAction(AmamzonBase):
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False,timeout=120)
#检查页数是否相等，不相等则继续
self.log(f"开始检查页数，当前页数 {current_page} / {total_page}")
if total_page != 0 and current_page!= 0 and current_page < total_page:
self.log(f"检查到页数还未完成，重启浏览器继续")
raise RuntimeError(f"与页面的连接已断开，检查到页数还未完成，重启浏览器继续")
class MatchTak(TaskBase):
task_name = "匹配价格-TASK"
@@ -261,7 +274,7 @@ class MatchTak(TaskBase):
self.log(f"检测到任务 {task_id} 的暂停请求，停止处理国家", "WARNING")
break
skip_asin = skip_asins_by_country.get(country_code)
skip_asin = skip_asins_by_country.get(country_code,[])
if skipAsinDetailsByCountry :
skipAsinDetails = {i.get("asin"):i.get("minimumPrice") for i in skipAsinDetailsByCountry.get(country_code)}
else:
@@ -276,6 +289,7 @@ class MatchTak(TaskBase):
return
current_url = None
max_retries = 200
for _ in range(max_retries):
try:
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter,limit,
@@ -366,8 +380,8 @@ class MatchTak(TaskBase):
if task_id in runing_task:
runing_task[task_id]["current_asin"] = asin
runing_task[task_id]["processed_asins"] += 1
runing_task[task_id]["failed_count"] += 1
minimumPrice = ""
if "有最低价跳过" in status:
minimumPrice = skipAsinDetails.get(asin)