super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

同步 Gitee 应用代码更新

Browse Source
This commit is contained in:
super
2026-05-29 19:47:23 +08:00
parent 266c0f17c1
commit 080625567b
7 changed files with 103 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
app/amazon/approve.py
View File

@@ -308,8 +308,9 @@ class AmzoneApprove(AmamzonBase):
self.log("开始执行...")
num = 0
retry_num = 0
max_retry_num = 5
total_page = 0
current_page = 0
while retry_num < max_retry_num: # 最多重试3次
# if num > 3: #测试
# return
@@ -325,13 +326,13 @@ class AmzoneApprove(AmamzonBase):
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) > 0:
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
current_page = int(current_page.strip())
# 总页数
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
if len(total_page) > 0:
total_page = total_page[-1].text
else:
total_page = 0
total_page = int(total_page[-1].text.strip())
self.log(f"当前页码: {current_page} / 总页数: {total_page}")
except Exception as e:
self.log(f"获取页码失败:{e}")
@@ -543,6 +544,14 @@ class AmzoneApprove(AmamzonBase):
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False,timeout=120)
#检查页数是否相等，不相等则继续
self.log(f"开始检查页数，当前页数 {current_page} / {total_page}")
if total_page != 0 and current_page!= 0 and current_page < total_page:
self.log(f"检查到页数还未完成，重启浏览器继续")
raise RuntimeError(f"与页面的连接已断开，检查到页数还未完成，重启浏览器继续")
class ApproveTask(TaskBase):
"""审批任务处理类：负责处理产品风险审批任务"""
task_name = "产品风险审批-TASK"

10
app/amazon/chrome_base.py
View File

@@ -194,3 +194,13 @@ class ChromeAmzoneBase:
print("浏览器已关闭")
except Exception as e:
print(f"关闭浏览器时出错: {str(e)}")

19
app/amazon/detail_spider.py
View File

@@ -92,7 +92,8 @@ class ChromeAmzone(ChromeAmzoneBase):
"""
data = {
'image_url': "",
'title': ""
'title': "",
"sku" : ""
}
try:
@@ -103,26 +104,39 @@ class ChromeAmzone(ChromeAmzoneBase):
data["title"] = title
sku_ele_ls = self.tab.eles('xpath://ul[@class="a-unordered-list a-vertical a-spacing-mini"]', timeout=20)
if len(sku_ele_ls) > 0:
data["title"] += sku_ele_ls[0].text
data["sku"] = sku_ele_ls[0].text
imge_ele = self.tab.ele('xpath://div[@id="imgTagWrapperId"]//img',timeout=20)
image_url = ""
min_image_url = ""
data_a_dynamic_image = imge_ele.attr("data-a-dynamic-image")
if data_a_dynamic_image:
dynamic_image_json = json.loads(data_a_dynamic_image)
self.log(f"图片信息:{dynamic_image_json}")
max_area = 0
min_area = 0
for url, (width, height) in dynamic_image_json.items():
area = width * height
if area > max_area:
max_area = area
image_url = url
if min_area == 0:
min_area = area
min_image_url = url
if area < min_area:
min_area = area
min_image_url = url
if not image_url:
image_url = imge_ele.attr("src")
if not min_image_url:
min_image_url = imge_ele.attr("src")
data["image_url"] = image_url
data["min_image_url"] = min_image_url
return data
except Exception as e:
@@ -266,6 +280,7 @@ class SpiderTask(TaskBase):
"country": i.get("country"),
"url": return_data.get("image_url"),
"title": return_data.get("title"),
"sku": return_data.get("sku")
}
for i in items
]

38
app/amazon/match_action.py
View File

@@ -35,9 +35,12 @@ class AmzoneMatchAction(AmamzonBase):
self.log("开始执行...")
num = 0
retry_num = 0
get_page_faild = 0
while retry_num < 3: # 最多重试3次
max_retry_num = 5
total_page = 0
current_page = 0
while retry_num < max_retry_num:
# if num > 3: #测试
# return
# 等待加载完成
@@ -45,6 +48,7 @@ class AmzoneMatchAction(AmamzonBase):
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
# 获取当前页码
@@ -52,23 +56,17 @@ class AmzoneMatchAction(AmamzonBase):
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) > 0:
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
current_page = int(current_page.strip())
# 测试
# if current_page > 1:
# break
# 总页数
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
if len(total_page) > 0:
total_page = total_page[-1].text
else:
total_page = 0
total_page = int(total_page[-1].text.strip())
self.log(f"【{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}")
get_page_faild = 0
except Exception as e:
self.log(f"【{self.mark_name}】获取页码失败", e)
get_page_faild+= 1
if get_page_faild > 2:
show_notification(f"【{self.mark_name}】获取页码失败超3次停止任务！")
break
# 保存当前URL，用于失败重试时访问
try:
@@ -81,6 +79,15 @@ class AmzoneMatchAction(AmamzonBase):
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
self.log(f"【{self.mark_name}】获取到 {len(sku_ls)}")
if len(sku_ls) == 0:
retry_num += 1
self.log(f"没有获取到SKU列表，重试{retry_num}/{max_retry_num}")
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False, timeout=120)
if retry_num == max_retry_num:
raise RuntimeError(f"与页面的连接已断开，没有获取到SKU列表,重试{retry_num}/{max_retry_num}")
continue
# for sku_ele in sku_ls[0:2]:
for sku_ele in sku_ls:
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]')
@@ -140,6 +147,12 @@ class AmzoneMatchAction(AmamzonBase):
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False,timeout=120)
#检查页数是否相等，不相等则继续
self.log(f"开始检查页数，当前页数 {current_page} / {total_page}")
if total_page != 0 and current_page!= 0 and current_page < total_page:
self.log(f"检查到页数还未完成，重启浏览器继续")
raise RuntimeError(f"与页面的连接已断开，检查到页数还未完成，重启浏览器继续")
class MatchTak(TaskBase):
task_name = "匹配价格-TASK"
@@ -261,7 +274,7 @@ class MatchTak(TaskBase):
self.log(f"检测到任务 {task_id} 的暂停请求，停止处理国家", "WARNING")
break
skip_asin = skip_asins_by_country.get(country_code)
skip_asin = skip_asins_by_country.get(country_code,[])
if skipAsinDetailsByCountry :
skipAsinDetails = {i.get("asin"):i.get("minimumPrice") for i in skipAsinDetailsByCountry.get(country_code)}
else:
@@ -276,6 +289,7 @@ class MatchTak(TaskBase):
return
current_url = None
max_retries = 200
for _ in range(max_retries):
try:
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter,limit,
@@ -366,8 +380,8 @@ class MatchTak(TaskBase):
if task_id in runing_task:
runing_task[task_id]["current_asin"] = asin
runing_task[task_id]["processed_asins"] += 1
runing_task[task_id]["failed_count"] += 1
minimumPrice = ""
if "有最低价跳过" in status:
minimumPrice = skipAsinDetails.get(asin)

28
app/amazon/price_match.py
View File

@@ -460,6 +460,8 @@ class AmzonePriceMatch(AmamzonBase):
super().__init__(user_info,socket_port)
self.already_asin = set()
def reset_already_asin(self):
self.already_asin = set()
self.log("already_asin 已重置")
@@ -513,14 +515,15 @@ class AmzonePriceMatch(AmamzonBase):
self.log("开始执行...")
num = 0
retry_num = 0
already_asin = set()
get_page_faild = 0
already_asin = self.already_asin
#获取当前国家
current_country_ele = self.tab.ele('xpath://div[@class="dropdown-account-switcher-header-label"]/span[last()]',
timeout=20)
current_country = current_country_ele.text.strip()
max_retry_num = 5
while retry_num < max_retry_num: # 最多重试3次
total_page = 0
current_page = 0
while retry_num < max_retry_num:
try:
if appoint_asin is not None:
self.log(f"指定asin操作,{appoint_asin}")
@@ -536,6 +539,7 @@ class AmzonePriceMatch(AmamzonBase):
page_pamel = self.tab.eles('xpath://kat-pagination', timeout=5)
if len(page_pamel) > 0:
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
current_page = int(current_page.strip())
#测试=====
# if int(current_page) > 1:
# break
@@ -546,17 +550,12 @@ class AmzonePriceMatch(AmamzonBase):
total_page = page_pamel[0].sr.eles(
'xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
if len(total_page) > 0:
total_page = total_page[-1].text
else:
total_page = 0
total_page = int(total_page[-1].text.strip())
self.log(f"【{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}")
get_page_faild = 0
except Exception as e:
self.log(f"获取页码失败,{e}")
get_page_faild += 1
if get_page_faild > 2:
show_notification(f"【{self.mark_name}】获取页码失败超3次停止任务！")
break
# 保存当前URL，用于失败重试时访问
try:
@@ -767,6 +766,12 @@ class AmzonePriceMatch(AmamzonBase):
retry_num += 1
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False, timeout=120)
#检查页数是否相等，不相等则继续
self.log(f"开始检查页数，当前页数 {current_page} / {total_page}")
if total_page != 0 and current_page!= 0 and current_page < total_page:
self.log(f"检查到页数还未完成，重启浏览器继续")
raise RuntimeError(f"与页面的连接已断开，检查到页数还未完成，重启浏览器继续")
class PriceTask(TaskBase):
task_name = "跟价-TASK"
@@ -922,6 +927,7 @@ class PriceTask(TaskBase):
miniprice_info = minimum_price_by_country_and_asin.get(country_code,{})
current_url = None
max_retries = 200
for _ in range(max_retries):
try:
self.process_country(driver, country_code, task_id, shop_name, risk_listing_filter,

18
app/amazon/similar_asin.py
View File

@@ -257,22 +257,35 @@ class ChromeAmzone(ChromeAmzoneBase):
imge_ele = self.tab.ele('xpath://div[@id="imgTagWrapperId"]//img',timeout=20)
image_url = ""
min_image_url = ""
data_a_dynamic_image = imge_ele.attr("data-a-dynamic-image")
if data_a_dynamic_image:
dynamic_image_json = json.loads(data_a_dynamic_image)
self.log(f"图片信息:{dynamic_image_json}")
max_area = 0
min_area = 0
for url, (width, height) in dynamic_image_json.items():
area = width * height
if area > max_area:
max_area = area
image_url = url
if min_area == 0:
min_area = area
min_image_url = url
if area < min_area:
min_area = area
min_image_url = url
if not image_url:
image_url = imge_ele.attr("src")
if not min_image_url:
min_image_url = imge_ele.attr("src")
data["image_url"] = image_url
data["min_image_url"] = min_image_url
# sku
sku_ele_ls = self.tab.eles('xpath://ul[@class="a-unordered-list a-vertical a-spacing-mini"]',timeout=20)
@@ -340,9 +353,10 @@ class ChromeAmzone(ChromeAmzoneBase):
# 判断是否采集标题出错
new_size = "220,220"
pattern = r"\._(?:[A-Z]+)?(\d+)_\."
replacement = f"._{new_size}_.jpg"
# replacement = f"._{new_size}_.jpg"
# 使用正则替换
image_new_url = re.sub(pattern, lambda m: replacement, data["image_url"])
# image_new_url = re.sub(pattern, lambda m: replacement, data["image_url"])
image_new_url = data["min_image_url"]
print(image_new_url)

1
app/blueprints/auth.py
View File

@@ -28,6 +28,7 @@ def api_auth_sync():
"""前端拿到 Java 返回的 JWT 后调用，把 token 写进 Python 同源 cookie。"""
data = request.get_json(silent=True) or {}
token = (data.get('token') or '').strip()
if not token:
return jsonify({'success': False, 'error': '缺少 token'}), 400
payload, reason = parse_token_with_reason(token)