new file: amazon/amazon_base.py
modified: amazon/approve.py modified: amazon/asin_status.py new file: amazon/chrome_base.py modified: amazon/del_brand.py modified: amazon/detail_spider.py new file: "amazon/detail_spider_\345\244\207\344\273\275.py" modified: amazon/main.py modified: amazon/match_action.py modified: amazon/price_match.py new file: amazon/similar_asin.py modified: app.py new file: assets/appearance-patent-Br1dtmol.css new file: assets/appearance-patent-D1DgeYhe.css new file: assets/delete-brand-BfMLVSQU.css new file: assets/patrol-delete-BAzbYtgc.css new file: assets/price-track-Hozgt_em.css new file: assets/product-risk-CbX7bwZi.css new file: assets/query-asin-B4RsOiza.css new file: assets/shop-match-B2HWAgBY.css modified: main.py
This commit is contained in:
铭坤
505
app/amazon/similar_asin.py
Normal file
505
app/amazon/similar_asin.py
Normal file
@@ -0,0 +1,505 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import base64
|
||||
import time
|
||||
import re
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from collections import defaultdict
|
||||
import requests
|
||||
from urllib.parse import quote
|
||||
|
||||
from curl_cffi import requests as requests_frp
|
||||
|
||||
|
||||
|
||||
from amazon.tool import show_notification,get_shop_info,remove_special_characters,split_currency_values
|
||||
from amazon.amazon_base import TaskBase
|
||||
|
||||
from amazon.chrome_base import ChromeAmzoneBase
|
||||
|
||||
from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE
|
||||
|
||||
|
||||
class ChromeAmzone(ChromeAmzoneBase):
|
||||
mark_name = "亚马逊相似ASIN"
|
||||
|
||||
def wait_load_complete(self):
|
||||
try:
|
||||
for _ in range(3):
|
||||
content_area = self.tab.eles('xpath://div[@class="ap-sbi-aside__main"]', timeout=20)
|
||||
if len(content_area) > 0:
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"等待加载完成失败,{e},等待5秒")
|
||||
time.sleep(5)
|
||||
|
||||
def encode_custom(self,input_):
|
||||
if not input_ or not isinstance(input_, str):
|
||||
return ""
|
||||
|
||||
transformed = ""
|
||||
|
||||
# 第一个字符：字符编码 + 字符串长度
|
||||
transformed += chr((ord(input_[0]) + len(input_)) & 0xFFFF)
|
||||
|
||||
# 后续字符：当前字符编码 + 前一个字符编码
|
||||
for i in range(1, len(input_)):
|
||||
current_code = ord(input_[i])
|
||||
prev_code = ord(input_[i - 1])
|
||||
transformed += chr((current_code + prev_code) & 0xFFFF)
|
||||
|
||||
# 对应 JavaScript 的 encodeURIComponent
|
||||
encoded = quote(transformed, safe="-_.!~*'()")
|
||||
|
||||
# 对应 replace(/[!'()*]/g, ...)
|
||||
for ch in ["!", "'", "(", ")", "*"]:
|
||||
encoded = encoded.replace(ch, "%" + format(ord(ch), "x"))
|
||||
return encoded
|
||||
|
||||
def image_to_base64(self,image_source: str) -> str:
|
||||
"""
|
||||
将图片链接（URL 或本地文件路径）转换为 Base64 编码的字符串。
|
||||
"""
|
||||
if image_source.startswith(('http://', 'https://')):
|
||||
response = requests.get(image_source, timeout=10)
|
||||
response.raise_for_status() # 非 2xx 状态码将抛出异常
|
||||
image_data = response.content
|
||||
else:
|
||||
path = Path(image_source)
|
||||
if not path.is_file():
|
||||
raise ValueError(f"本地文件不存在: {image_source}")
|
||||
with open(path, 'rb') as f:
|
||||
image_data = f.read()
|
||||
base64_str = base64.b64encode(image_data).decode('utf-8')
|
||||
return base64_str
|
||||
|
||||
def get_aliprice_data(self,page=1,title="",domain="",category="",imageBase64=""):
|
||||
"""
|
||||
"fishkeeper Quick Aquarium Siphon Pump Gravel Cleaner - 256GPH Adjustable Powerful Fish Tank Vacuum Gravel Cleaning Kit for Aquarium Water Changer, Sand Cleaner, Dirt Removal : Amazon.co.uk: Pet Supplies"
|
||||
标题 :
|
||||
|
||||
"""
|
||||
headers = {
|
||||
"accept": "application/json, text/plain, */*",
|
||||
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"browser": "chrome",
|
||||
"cache-control": "no-cache",
|
||||
"channel": "chrome_offline",
|
||||
"content-type": "application/json;charset=UTF-8",
|
||||
"ext-id": "10100",
|
||||
"ext_id": "10100",
|
||||
"origin": "chrome-extension://ephkdklmdkaakeleplfpahjphaokcllh",
|
||||
"platform": "1688",
|
||||
"pragma": "no-cache",
|
||||
"priority": "u=1, i",
|
||||
"sec-fetch-dest": "empty",
|
||||
"sec-fetch-mode": "cors",
|
||||
"sec-fetch-site": "none",
|
||||
"sec-fetch-storage-access": "active",
|
||||
"version": "3.7.4",
|
||||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
|
||||
}
|
||||
cookie = {
|
||||
"e-info": "[{\"e-name\":\"1688\",\"adid\":\"100\",\"version\":\"3.7.4\",\"ext_id\":\"10100\"}]",
|
||||
"language": "chinese",
|
||||
"province_code": "Guangdong",
|
||||
"_ct": "czg",
|
||||
"_e_ct": "czg",
|
||||
"is_reto": "1",
|
||||
"crossborder": "1",
|
||||
"agent": "0",
|
||||
"first_view": "1",
|
||||
"currency": "USD",
|
||||
"is_coo": "1",
|
||||
"plugin": "chrome",
|
||||
"plugin_ext": "100,129",
|
||||
"m-info": "[{\"platform\":\"1688\",\"version\":\"3.7.4\",\"browser\":\"chrome\",\"m\":\"uni\",\"t\":1777902599578}]"
|
||||
}
|
||||
url = "https://api.aliprice.com/index.php/chrome/items/imageAnalysis"
|
||||
itemTitle = f"{title} : {domain}: {category}"
|
||||
|
||||
params = {"page": page, "size": 20, "website": "1688_lite2", "language": "zh-CN", "currency": "USD", "from": "",
|
||||
"itemTitle": itemTitle, "domain": domain }
|
||||
params['sign'] = self.encode_custom(json.dumps(params, ensure_ascii=False))
|
||||
data = {
|
||||
"imageBase64" : imageBase64
|
||||
}
|
||||
response = requests.post(url, headers=headers, params=params, json=data, impersonate="chrome101",
|
||||
cookies=cookie)
|
||||
response.encoding = "utf-8"
|
||||
data = response.json()
|
||||
return data
|
||||
|
||||
def _scrape_data(self):
|
||||
"""
|
||||
抓取商品数据
|
||||
|
||||
Returns:
|
||||
dict: 抓取到的数据
|
||||
"""
|
||||
data = {
|
||||
'image_url': "",
|
||||
'title': "",
|
||||
'category' : '',
|
||||
'success' : True
|
||||
}
|
||||
|
||||
try:
|
||||
# 等待页面加载
|
||||
# time.sleep(3)
|
||||
title_ele = self.tab.ele('xpath://h1[@id="title"]',timeout=30)
|
||||
title = title_ele.text
|
||||
data["title"] = title
|
||||
imge_ele = self.tab.ele('xpath://div[@id="imgTagWrapperId"]//img',timeout=20)
|
||||
image_url = imge_ele.attr("src")
|
||||
data["image_url"] = image_url
|
||||
category = self.tab.eles('xpath://div[@id="wayfinding-breadcrumbs_feature_div"]//span[@class="a-list-item"]//a[@class="a-link-normal a-color-tertiary"]',timeout=20)
|
||||
if len(category) > 0:
|
||||
data["category"] = category[0].text
|
||||
return data
|
||||
|
||||
except Exception as e:
|
||||
print(f"抓取数据时出错: {traceback.format_exc()}")
|
||||
data["success"] = False
|
||||
return data
|
||||
|
||||
def run(self, country, asin,total_page=2):
|
||||
"""
|
||||
运行亚马逊详情采集任务
|
||||
Args:
|
||||
country: 国家名称（如：英国、德国、法国、西班牙、意大利）
|
||||
asin: 亚马逊商品ASIN码
|
||||
Returns:
|
||||
dict: 包含采集到的数据
|
||||
"""
|
||||
try:
|
||||
# 验证国家是否支持
|
||||
if country not in self.country_info:
|
||||
error_msg = f"不支持的国家: {country}，支持的国家有: {list(self.country_info.keys())}"
|
||||
print(error_msg)
|
||||
show_notification(error_msg, "error")
|
||||
return None
|
||||
|
||||
# 获取国家配置
|
||||
country_config = self.country_info[country]
|
||||
zip_code = country_config["zip_code"]
|
||||
mark = country_config.get("mark")
|
||||
|
||||
# 1. 根据国家和ASIN拼接链接
|
||||
base_url = country_config["url"]
|
||||
# 提取域名部分
|
||||
domain = base_url.split("/dp/")[0]
|
||||
# 拼接新的URL
|
||||
product_url = f"{domain}/dp/{asin}"
|
||||
print(f"正在访问: {product_url}")
|
||||
|
||||
# 打开链接
|
||||
self.tab.get(product_url)
|
||||
time.sleep(3) # 等待页面初步加载
|
||||
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
|
||||
|
||||
self.close_init_popup()
|
||||
# 2. 切换国家/设置邮编
|
||||
print(f"正在检查并设置邮编: {zip_code},标识： {mark}")
|
||||
self._set_zip_code(zip_code, mark)
|
||||
|
||||
# self.tab.wait.doc_loaded(timeout=5, raise_err=False)
|
||||
|
||||
# 3. 抓取数据
|
||||
print("正在抓取商品数据...")
|
||||
data = self._scrape_data()
|
||||
|
||||
# 判断是否采集标题出错
|
||||
new_size = "220,220"
|
||||
pattern = r"\._(?:[A-Z]+)?(\d+)_\."
|
||||
replacement = f"._{new_size}_.jpg"
|
||||
# 使用正则替换
|
||||
image_new_url = re.sub(pattern, lambda m: replacement, data["image_url"])
|
||||
|
||||
iamge_base64 = self.image_to_base64(image_new_url)
|
||||
|
||||
similar_data = []
|
||||
# 4、请求获取插件的数据
|
||||
for page_num in range(total_page):
|
||||
resp_data = self.get_aliprice_data(
|
||||
page=page_num,
|
||||
title = data["title"],
|
||||
domain=domain,
|
||||
category=data["category"],
|
||||
imageBase64 = iamge_base64
|
||||
)
|
||||
self.log(f"Aliprice 扩展数据获取:{resp_data}")
|
||||
if len(resp_data.get("data",[])) > 0:
|
||||
for i in resp_data.get("data"):
|
||||
similar_data.append(i)
|
||||
|
||||
data["similar_data"] = similar_data
|
||||
|
||||
# 添加基本信息
|
||||
data['country'] = country
|
||||
data['asin'] = asin
|
||||
data['url'] = product_url
|
||||
data['timestamp'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
print(f"数据抓取完成: {json.dumps(data)}")
|
||||
return data
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"运行出错: {traceback.format_exc()}"
|
||||
print(error_msg)
|
||||
show_notification(f"采集失败: {str(e)}", "error")
|
||||
return {}
|
||||
|
||||
|
||||
class SimilarAsinTask(TaskBase):
|
||||
mark_name = "相似ASIN采集"
|
||||
|
||||
@staticmethod
|
||||
def group_by_id_prefix(data):
|
||||
"""
|
||||
根据每条数据的 id 字段进行分组：
|
||||
- 如果 id 为 "2_1"，则取 "_" 前面的 "2" 作为分组 key
|
||||
- 如果 id 为 "1"，则分组 key 就是 "1"
|
||||
- 相同 key 的数据归为同一组
|
||||
|
||||
:param data: 原始列表数据
|
||||
:return: 二维列表，按 id 前缀分组
|
||||
"""
|
||||
grouped = defaultdict(list)
|
||||
|
||||
for item in data:
|
||||
item_id = str(item.get("id", ""))
|
||||
group_key = item_id.split("_")[0]
|
||||
grouped[group_key].append(item)
|
||||
|
||||
return list(grouped.values())
|
||||
|
||||
@staticmethod
|
||||
def normalize_groups(data):
|
||||
groups = data.get("groups")
|
||||
if isinstance(groups, list) and len(groups) > 0:
|
||||
return groups
|
||||
|
||||
rows = data.get("rows") or data.get("items") or []
|
||||
if not isinstance(rows, list) or len(rows) == 0:
|
||||
return []
|
||||
|
||||
normalized = []
|
||||
for items in SimilarAsinTask.group_by_id_prefix(rows):
|
||||
first = items[0] if items else {}
|
||||
item_id = str(first.get("id") or first.get("displayId") or "")
|
||||
base_id = item_id.split("_")[0] if item_id else ""
|
||||
normalized.append({
|
||||
"sourceFileKey": first.get("sourceFileKey", ""),
|
||||
"sourceFilename": first.get("sourceFilename", ""),
|
||||
"groupKey": first.get("groupKey") or base_id,
|
||||
"baseId": first.get("baseId") or base_id,
|
||||
"displayId": first.get("displayId") or item_id,
|
||||
"items": items,
|
||||
})
|
||||
return normalized
|
||||
|
||||
def process_task(self, task_data: dict):
|
||||
"""处理审批任务主入口
|
||||
|
||||
Args:
|
||||
task_data: 任务数据
|
||||
"""
|
||||
try:
|
||||
data = task_data.get("data", {})
|
||||
task_id = data.get("taskId")
|
||||
groups = self.normalize_groups(data)
|
||||
|
||||
if not task_id:
|
||||
self.log("任务ID为空，跳过", "WARNING")
|
||||
return
|
||||
|
||||
|
||||
self.log(f"开始处理爬取任务 {task_id}，{len(groups)} 个任务")
|
||||
|
||||
if not groups:
|
||||
self.log("appearance-patent groups/rows is empty, skip", "WARNING")
|
||||
return
|
||||
|
||||
runing_task[task_id] = {
|
||||
"status": "running",
|
||||
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"total_shops": 1,
|
||||
"processed_shops": 0,
|
||||
"total_countries": len(groups) ,
|
||||
"processed_countries": 0,
|
||||
"total_asins": 0,
|
||||
"processed_asins": 0,
|
||||
"success_count": 0,
|
||||
"failed_count": 0,
|
||||
"stop_requested": False
|
||||
}
|
||||
|
||||
# 检查是否收到暂停请求
|
||||
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
|
||||
self.log(f"检测到任务 {task_id} 的暂停请求，停止处理", "WARNING")
|
||||
runing_task[task_id]["status"] = "stopped"
|
||||
return
|
||||
max_retry = 3
|
||||
show_notification(f"开始爬取数据", "info")
|
||||
chrome = ChromeAmzone()
|
||||
try:
|
||||
# 数据整理
|
||||
# new_items = self.group_by_id_prefix(items)
|
||||
result = []
|
||||
for gp_index,gp in enumerate(groups):
|
||||
items = gp.get("items", [])
|
||||
return_data = {}
|
||||
|
||||
for index,value in enumerate(items):
|
||||
|
||||
return_data = {
|
||||
'image_url': "",
|
||||
'title': ""
|
||||
}
|
||||
asin = value.get("asin")
|
||||
country = value.get("country")
|
||||
return_data = None
|
||||
for _ in range(max_retry):
|
||||
try:
|
||||
return_data = chrome.run(country, asin) or {}
|
||||
self.log(f"抓取结果->{return_data}")
|
||||
break
|
||||
except Exception as e:
|
||||
if "与页面的连接已断开" in str(e):
|
||||
chrome = ChromeAmzone()
|
||||
if not isinstance(return_data, dict):
|
||||
return_data = {}
|
||||
if return_data.get("image_url"):
|
||||
break
|
||||
|
||||
group_item = [
|
||||
{
|
||||
"sourceFileKey": i.get("sourceFileKey"),
|
||||
"sourceFilename": i.get("sourceFilename"),
|
||||
"rowToken": i.get("rowToken"),
|
||||
"groupKey": i.get("groupKey"),
|
||||
"id": i.get("id"),
|
||||
"asin": i.get("asin"),
|
||||
"country": i.get("country"),
|
||||
"url": return_data.get("image_url"),
|
||||
"title": return_data.get("title"),
|
||||
}
|
||||
for i in items
|
||||
]
|
||||
# task_id: int, chunkIndex:int,chunkTotal: int, country_code: str, asin: str, status: dict,error:str="",
|
||||
# item_data:dict={},
|
||||
res = {
|
||||
"sourceFileKey": gp.get("sourceFileKey"),
|
||||
"sourceFilename": gp.get("sourceFilename"),
|
||||
"groupKey": gp.get("groupKey"),
|
||||
"baseId": gp.get("baseId"),
|
||||
"displayId": gp.get("displayId"),
|
||||
"items": group_item
|
||||
}
|
||||
print("================")
|
||||
result.append(res)
|
||||
print("================")
|
||||
is_done = gp_index == len(groups)-1
|
||||
if len(result) > 20 or is_done:
|
||||
self.post_result(task_id=task_id,chunkIndex=gp_index+1,chunkTotal=len(groups),
|
||||
asin=asin,item_data=result,is_done=is_done)
|
||||
result = []
|
||||
|
||||
|
||||
if len(result) > 0:
|
||||
is_done = True
|
||||
self.post_result(task_id=task_id, chunkIndex=len(groups), chunkTotal=len(groups),
|
||||
asin=asin, item_data=result, is_done=is_done)
|
||||
|
||||
try:
|
||||
chrome.close()
|
||||
except Exception as e:
|
||||
print("退出浏览器出错",e)
|
||||
# 更新已处理店铺数
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["processed_shops"] += 1
|
||||
except Exception as e:
|
||||
self.log(f"处理店铺 {task_id} 失败: {str(e)}", "ERROR")
|
||||
self.log(traceback.format_exc(), "ERROR")
|
||||
|
||||
# 更新任务状态
|
||||
if task_id in runing_task:
|
||||
if runing_task[task_id].get("stop_requested", False):
|
||||
runing_task[task_id]["status"] = "stopped"
|
||||
self.log(f"任务 {task_id} 已被暂停！")
|
||||
else:
|
||||
runing_task[task_id]["status"] = "completed"
|
||||
self.log(f"任务 {task_id} 处理完成！")
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
|
||||
if task_id:
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["status"] = "failed"
|
||||
runing_task[task_id]["error"] = str(e)
|
||||
|
||||
def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
|
||||
item_data:dict={},
|
||||
is_done: bool = False):
|
||||
"""回传处理结果到API
|
||||
"""
|
||||
|
||||
url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result"
|
||||
|
||||
payload ={
|
||||
"submissionId": f"{int(time.time())}",
|
||||
"chunkIndex": chunkIndex,
|
||||
"chunkTotal": chunkTotal,
|
||||
"error": error,
|
||||
"groups": item_data,
|
||||
"done": is_done
|
||||
}
|
||||
|
||||
max_retries = 3
|
||||
for retry in range(max_retries):
|
||||
try:
|
||||
request_timeout = 300 if is_done else 30
|
||||
print("================【详情采集】=====================")
|
||||
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
|
||||
self.log(f"回传URL: {url}")
|
||||
self.log(f"回传数据: {payload}")
|
||||
response = requests.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
timeout=request_timeout,
|
||||
verify=False
|
||||
)
|
||||
self.log(f"回传结果: {response.text}")
|
||||
data = response.json() if response.text else {}
|
||||
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
|
||||
self.log(f"结果回传成功: {asin} - {item_data}")
|
||||
return
|
||||
else:
|
||||
self.log(f"结果回传失败，状态码: {response.status_code}", "WARNING")
|
||||
print("=====================================")
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"调用API异常: {str(e)}", "ERROR")
|
||||
print("=====================================")
|
||||
|
||||
# 如果还有重试机会，等待后继续
|
||||
if retry < max_retries - 1:
|
||||
time.sleep(2)
|
||||
|
||||
self.log(f"已达到最大重试次数，结果回传最终失败", "ERROR")
|
||||
raise RuntimeError("已达到最大重试次数，结果回传最终失败")
|
||||
|
||||
if __name__ == '__main__':
|
||||
spide = ChromeAmzone()
|
||||
|
||||
resp = spide.run(
|
||||
country="英国", asin="B0F79LCB3X", total_page=2
|
||||
)
|
||||
print(resp)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user