super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

提交货源图片和相似ASIN优化

Browse Source
This commit is contained in:
super
2026-05-24 21:59:21 +08:00
parent 532438faba
commit 7503e3fa8b
14 changed files with 1265 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
app/amazon/amazon_base.py
View File

@@ -417,8 +417,8 @@ class AmamzonBase(ZiniaoDriver):
"""
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
# if level == "ERROR":
# show_notification(message, "error")
print(f"[{timestamp}] [{self.mark_name}] [{level}] {message}")
except Exception as e:
print(f"输出出错,{e}")
@@ -689,8 +689,8 @@ class TaskBase:
"""
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
# if level == "ERROR":
# show_notification(message, "error")
print(f"[{timestamp}] [{self.task_name}] [{level}] {message}")
except Exception as e:
print(f"输出出错,{e}")
@@ -739,7 +739,7 @@ class TaskBase:
if not shop_data:
mes = f"获取店铺凭证失败，响应数据: {shop_data.get('message', '未知错误')}"
self.log(mes, "ERROR")
show_notification(mes, "ERROR")
# show_notification(mes, "ERROR")
continue
password = shop_data["data"]["password"]

4
app/amazon/approve.py
View File

@@ -674,12 +674,12 @@ class ApproveTask(TaskBase):
return
current_url = None
max_retries = 200 #只要没有完成，一直重试
for _ in range(max_retries):
try:
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter,target_url=current_url)
driver.reset_already_asin()
driver.close_store()
break
except Exception as e:
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR")
@@ -777,7 +777,7 @@ class ApproveTask(TaskBase):
}
result.append(country_data)
if len(result) > 20:
if len(result) > 10:
self.post_result_batch(task_id, shop_name, country_code, result)
result = []

9
app/amazon/chrome_base.py
View File

@@ -9,7 +9,7 @@ from datetime import datetime
from DrissionPage import Chromium, ChromiumOptions
from collections import defaultdict
from config import base_dir
from config import base_dir,debug
from amazon.tool import get_shop_info,show_notification
@@ -46,7 +46,8 @@ class ChromeAmzoneBase:
杀死当前谷歌浏览器进程，并使用 drissionpage 启动谷歌浏览器，使用系统安装的浏览器默认用户文件夹
"""
print("正在关闭现有的chromium浏览器进程...")
os.system('taskkill /f /t /im chrome.exe')
if not debug:
os.system('taskkill /f /t /im chrome.exe')
time.sleep(2)
print("正在启动chromium浏览器...")
@@ -70,8 +71,8 @@ class ChromeAmzoneBase:
"""
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
# if level == "ERROR":
# show_notification(message, "error")
print(f"[{timestamp}] [{self.mark_name}] [{level}] {message}")
except Exception as e:
print(f"输出出错,{e}")

26
app/amazon/detail_spider.py
View File

@@ -79,7 +79,8 @@ class ChromeAmzone(ChromeAmzoneBase):
except Exception as e:
error_msg = f"运行出错: {traceback.format_exc()}"
self.log(error_msg)
show_notification(f"采集失败: {str(e)}", "error")
# show_notification(f"采集失败: {str(e)}", "error")
raise RuntimeError(error_msg)
return {}
def _scrape_data(self):
@@ -100,8 +101,27 @@ class ChromeAmzone(ChromeAmzoneBase):
title_ele = self.tab.ele('xpath://h1[@id="title"]',timeout=30)
title = title_ele.text
data["title"] = title
sku_ele_ls = self.tab.eles('xpath://ul[@class="a-unordered-list a-vertical a-spacing-mini"]', timeout=20)
if len(sku_ele_ls) > 0:
data["title"] += sku_ele_ls[0].text
imge_ele = self.tab.ele('xpath://div[@id="imgTagWrapperId"]//img',timeout=20)
image_url = imge_ele.attr("src")
image_url = ""
data_a_dynamic_image = imge_ele.attr("data-a-dynamic-image")
if data_a_dynamic_image:
dynamic_image_json = json.loads(data_a_dynamic_image)
self.log(f"图片信息:{dynamic_image_json}")
max_area = 0
for url, (width, height) in dynamic_image_json.items():
area = width * height
if area > max_area:
max_area = area
image_url = url
if not image_url:
image_url = imge_ele.attr("src")
data["image_url"] = image_url
return data
@@ -263,7 +283,7 @@ class SpiderTask(TaskBase):
result.append(res)
print("================")
is_done = gp_index == len(groups)-1
if len(result) > 20 or is_done:
if len(result) > 10 or is_done:
self.post_result(task_id=task_id,chunkIndex=gp_index+1,chunkTotal=len(groups),
asin=asin,item_data=result,is_done=is_done)
result = []

2
app/amazon/match_action.py
View File

@@ -378,7 +378,7 @@ class MatchTak(TaskBase):
"done": False
})
if len(result) > 20:
if len(result) > 10:
self.post_result_batch(task_id, shop_name, country_code,result)
result = []

9
app/amazon/price_match.py
View File

File diff suppressed because one or more lines are too long

127
app/amazon/similar_asin.py
View File

@@ -10,9 +10,10 @@ from datetime import datetime
from collections import defaultdict
import requests
from urllib.parse import quote
import os
from curl_cffi import requests as requests_frp
from config import base_dir
from amazon.tool import show_notification,get_shop_info,remove_special_characters,split_currency_values
@@ -76,7 +77,9 @@ class ChromeAmzone(ChromeAmzoneBase):
将图片链接（URL 或本地文件路径）转换为 Base64 编码的字符串。
"""
if image_source.startswith(('http://', 'https://')):
response = requests.get(image_source, timeout=10)
response = requests.get(image_source, timeout=10,headers={
"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0 18444"
})
response.raise_for_status() # 非 2xx 状态码将抛出异常
image_data = response.content
else:
@@ -153,7 +156,7 @@ class ChromeAmzone(ChromeAmzoneBase):
# 发送第一次请求
try:
response = requests_frp.post(url, headers=headers, params=params, json=data, impersonate="chrome101",
cookies=cookie, proxies=proxies)
cookies=cookie, proxies=proxies,verify=False)
response.encoding = "utf-8"
# 检查响应状态码和数据有效性
@@ -176,6 +179,7 @@ class ChromeAmzone(ChromeAmzoneBase):
need_retry = True
# 如果需要重试且配置了代理URL
# need_retry = True
if need_retry and CONFIG_PROXY_URL and not proxies:
try:
proxy_resp = requests.get(CONFIG_PROXY_URL, timeout=10)
@@ -204,7 +208,7 @@ class ChromeAmzone(ChromeAmzoneBase):
if proxy_ip:
proxies = {
"http": f"http://{proxy_ip}",
"https": f"http://{proxy_ip}",
"https": f"https://{proxy_ip}",
}
if proxies:
@@ -251,7 +255,23 @@ class ChromeAmzone(ChromeAmzoneBase):
title = title_ele.text
data["title"] = title
imge_ele = self.tab.ele('xpath://div[@id="imgTagWrapperId"]//img',timeout=20)
image_url = imge_ele.attr("src")
image_url = ""
data_a_dynamic_image = imge_ele.attr("data-a-dynamic-image")
if data_a_dynamic_image:
dynamic_image_json = json.loads(data_a_dynamic_image)
self.log(f"图片信息:{dynamic_image_json}")
max_area = 0
for url, (width, height) in dynamic_image_json.items():
area = width * height
if area > max_area:
max_area = area
image_url = url
if not image_url:
image_url = imge_ele.attr("src")
data["image_url"] = image_url
# sku
@@ -277,12 +297,13 @@ class ChromeAmzone(ChromeAmzoneBase):
Returns:
dict: 包含采集到的数据
"""
return_data = {}
try:
# 验证国家是否支持
if country not in self.country_info:
error_msg = f"不支持的国家: {country}，支持的国家有: {list(self.country_info.keys())}"
print(error_msg)
show_notification(error_msg, "error")
# show_notification(error_msg, "error")
return None
# 获取国家配置
@@ -314,6 +335,8 @@ class ChromeAmzone(ChromeAmzoneBase):
self.log("正在抓取商品数据...")
data = self._scrape_data()
return_data.update(data)
# 判断是否采集标题出错
new_size = "220,220"
pattern = r"\._(?:[A-Z]+)?(\d+)_\."
@@ -321,22 +344,25 @@ class ChromeAmzone(ChromeAmzoneBase):
# 使用正则替换
image_new_url = re.sub(pattern, lambda m: replacement, data["image_url"])
print(image_new_url)
iamge_base64 = self.image_to_base64(image_new_url)
similar_data = []
# 4、请求获取插件的数据
for page_num in range(total_page):
resp_data = self.get_aliprice_data(
page=page_num,
title = data["title"],
domain=domain,
category=data["category"],
imageBase64 = iamge_base64
)
self.log(f"Aliprice 扩展数据获取:{str(resp_data)[0:200]}")
if len(resp_data.get("data",[])) > 0:
for i in resp_data.get("data"):
similar_data.append(i)
if image_new_url:
for page_num in range(total_page):
resp_data = self.get_aliprice_data(
page=page_num+1,
title = data["title"],
domain=domain,
category=data["category"],
imageBase64 = iamge_base64
)
self.log(f"Aliprice 扩展数据获取:{str(resp_data)[0:200]}")
if len(resp_data.get("data",[])) > 0:
for i in resp_data.get("data"):
similar_data.append(i)
data["similar_data"] = similar_data
@@ -346,14 +372,17 @@ class ChromeAmzone(ChromeAmzoneBase):
data['url'] = product_url
data['timestamp'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f"数据抓取完成: {json.dumps(data)}")
return data
print(f"数据抓取完成: {json.dumps(return_data)}")
return_data.update(data)
return return_data
except Exception as e:
error_msg = f"运行出错: {traceback.format_exc()}"
print(error_msg)
show_notification(f"采集失败: {str(e)}", "error")
return {}
# show_notification(f"采集失败: {str(e)}", "error")
raise RuntimeError(f"{traceback.format_exc()}")
return return_data
class SimilarAsinTask(TaskBase):
@@ -478,10 +507,9 @@ class SimilarAsinTask(TaskBase):
result = []
for gp_index,gp in enumerate(groups):
items = gp.get("items", [])
return_data = {}
group_item = []
for index,value in enumerate(items):
print(value)
return_data = {
'image_url': "",
'title': "",
@@ -494,37 +522,36 @@ class SimilarAsinTask(TaskBase):
country = value.get("country")
for _ in range(max_retry):
try:
return_data = chrome.run(country, asin,total_page=2) or {}
return_data = chrome.run(country, asin,total_page=1) or {}
self.log(f"抓取结果->{return_data}")
break
except Exception as e:
# if "与页面的连接已断开" in str(e):
chrome = ChromeAmzone()
self.log(f"{asin}抓取数据报错,{e}")
if not isinstance(return_data, dict):
return_data = {}
if return_data.get("image_url"):
break
# if not isinstance(return_data, dict):
# return_data = {}
# if return_data.get("image_url"):
# break
group_item.append({
"sourceFileKey": value.get("sourceFileKey",""),
"sourceFilename": value.get("sourceFilename",""),
"rowToken": value.get("rowToken",""),
"groupKey": value.get("groupKey",""),
"id": value.get("values",{}).get("id",""),
"asin": value.get("asin",""),
"sku" : return_data.get("sku",""),
"country": value.get("country",""),
"url": return_data.get("image_url",""),
"title": return_data.get("title",""),
"done": False,
"urls" : [i.get("ori_picture") for i in return_data.get("similar_data",[])[:16]]
})
# task_id: int, chunkIndex:int,chunkTotal: int, country_code: str, asin: str, status: dict,error:str="",
# item_data:dict={},
group_item = [
{
"sourceFileKey": i.get("sourceFileKey"),
"sourceFilename": i.get("sourceFilename"),
"rowToken": i.get("rowToken"),
"groupKey": i.get("groupKey"),
"id": i.get("id"),
"asin": i.get("asin"),
"sku" : return_data.get("sku"),
"country": i.get("country"),
"url": return_data.get("image_url"),
"title": return_data.get("title"),
"done": False,
"urls" : [i.get("ori_picture") for i in return_data.get("similar_data")]
}
for i in items
]
# task_id: int, chunkIndex:int,chunkTotal: int, country_code: str, asin: str, status: dict,error:str="",
# item_data:dict={},
res = {
"sourceFileKey": gp.get("sourceFileKey"),
"sourceFilename": gp.get("sourceFilename"),
@@ -629,7 +656,7 @@ class SimilarAsinTask(TaskBase):
time.sleep(2)
self.log(f"已达到最大重试次数，结果回传最终失败", "ERROR")
raise RuntimeError("已达到最大重试次数，结果回传最终失败")
# raise RuntimeError("已达到最大重试次数，结果回传最终失败")
if __name__ == '__main__':
spide = ChromeAmzone()