3ca569ed1e
modified: amazon/approve.py modified: amazon/asin_status.py new file: amazon/chrome_base.py modified: amazon/del_brand.py modified: amazon/detail_spider.py new file: "amazon/detail_spider_\345\244\207\344\273\275.py" modified: amazon/main.py modified: amazon/match_action.py modified: amazon/price_match.py new file: amazon/similar_asin.py modified: app.py new file: assets/appearance-patent-Br1dtmol.css new file: assets/appearance-patent-D1DgeYhe.css new file: assets/delete-brand-BfMLVSQU.css new file: assets/patrol-delete-BAzbYtgc.css new file: assets/price-track-Hozgt_em.css new file: assets/product-risk-CbX7bwZi.css new file: assets/query-asin-B4RsOiza.css new file: assets/shop-match-B2HWAgBY.css modified: main.py
506 lines
20 KiB
Python
506 lines
20 KiB
Python
import json
|
||
from pathlib import Path
|
||
import base64
|
||
import time
|
||
import re
|
||
import traceback
|
||
from datetime import datetime
|
||
from collections import defaultdict
|
||
import requests
|
||
from urllib.parse import quote
|
||
|
||
from curl_cffi import requests as requests_frp
|
||
|
||
|
||
|
||
from amazon.tool import show_notification,get_shop_info,remove_special_characters,split_currency_values
|
||
from amazon.amazon_base import TaskBase
|
||
|
||
from amazon.chrome_base import ChromeAmzoneBase
|
||
|
||
from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE
|
||
|
||
|
||
class ChromeAmzone(ChromeAmzoneBase):
|
||
mark_name = "亚马逊相似ASIN"
|
||
|
||
def wait_load_complete(self):
|
||
try:
|
||
for _ in range(3):
|
||
content_area = self.tab.eles('xpath://div[@class="ap-sbi-aside__main"]', timeout=20)
|
||
if len(content_area) > 0:
|
||
pass
|
||
|
||
except Exception as e:
|
||
self.log(f"等待加载完成失败,{e},等待5秒")
|
||
time.sleep(5)
|
||
|
||
def encode_custom(self,input_):
|
||
if not input_ or not isinstance(input_, str):
|
||
return ""
|
||
|
||
transformed = ""
|
||
|
||
# 第一个字符：字符编码 + 字符串长度
|
||
transformed += chr((ord(input_[0]) + len(input_)) & 0xFFFF)
|
||
|
||
# 后续字符：当前字符编码 + 前一个字符编码
|
||
for i in range(1, len(input_)):
|
||
current_code = ord(input_[i])
|
||
prev_code = ord(input_[i - 1])
|
||
transformed += chr((current_code + prev_code) & 0xFFFF)
|
||
|
||
# 对应 JavaScript 的 encodeURIComponent
|
||
encoded = quote(transformed, safe="-_.!~*'()")
|
||
|
||
# 对应 replace(/[!'()*]/g, ...)
|
||
for ch in ["!", "'", "(", ")", "*"]:
|
||
encoded = encoded.replace(ch, "%" + format(ord(ch), "x"))
|
||
return encoded
|
||
|
||
def image_to_base64(self,image_source: str) -> str:
|
||
"""
|
||
将图片链接（URL 或本地文件路径）转换为 Base64 编码的字符串。
|
||
"""
|
||
if image_source.startswith(('http://', 'https://')):
|
||
response = requests.get(image_source, timeout=10)
|
||
response.raise_for_status() # 非 2xx 状态码将抛出异常
|
||
image_data = response.content
|
||
else:
|
||
path = Path(image_source)
|
||
if not path.is_file():
|
||
raise ValueError(f"本地文件不存在: {image_source}")
|
||
with open(path, 'rb') as f:
|
||
image_data = f.read()
|
||
base64_str = base64.b64encode(image_data).decode('utf-8')
|
||
return base64_str
|
||
|
||
def get_aliprice_data(self,page=1,title="",domain="",category="",imageBase64=""):
|
||
"""
|
||
"fishkeeper Quick Aquarium Siphon Pump Gravel Cleaner - 256GPH Adjustable Powerful Fish Tank Vacuum Gravel Cleaning Kit for Aquarium Water Changer, Sand Cleaner, Dirt Removal : Amazon.co.uk: Pet Supplies"
|
||
标题 :
|
||
|
||
"""
|
||
headers = {
|
||
"accept": "application/json, text/plain, */*",
|
||
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||
"browser": "chrome",
|
||
"cache-control": "no-cache",
|
||
"channel": "chrome_offline",
|
||
"content-type": "application/json;charset=UTF-8",
|
||
"ext-id": "10100",
|
||
"ext_id": "10100",
|
||
"origin": "chrome-extension://ephkdklmdkaakeleplfpahjphaokcllh",
|
||
"platform": "1688",
|
||
"pragma": "no-cache",
|
||
"priority": "u=1, i",
|
||
"sec-fetch-dest": "empty",
|
||
"sec-fetch-mode": "cors",
|
||
"sec-fetch-site": "none",
|
||
"sec-fetch-storage-access": "active",
|
||
"version": "3.7.4",
|
||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
|
||
}
|
||
cookie = {
|
||
"e-info": "[{\"e-name\":\"1688\",\"adid\":\"100\",\"version\":\"3.7.4\",\"ext_id\":\"10100\"}]",
|
||
"language": "chinese",
|
||
"province_code": "Guangdong",
|
||
"_ct": "czg",
|
||
"_e_ct": "czg",
|
||
"is_reto": "1",
|
||
"crossborder": "1",
|
||
"agent": "0",
|
||
"first_view": "1",
|
||
"currency": "USD",
|
||
"is_coo": "1",
|
||
"plugin": "chrome",
|
||
"plugin_ext": "100,129",
|
||
"m-info": "[{\"platform\":\"1688\",\"version\":\"3.7.4\",\"browser\":\"chrome\",\"m\":\"uni\",\"t\":1777902599578}]"
|
||
}
|
||
url = "https://api.aliprice.com/index.php/chrome/items/imageAnalysis"
|
||
itemTitle = f"{title} : {domain}: {category}"
|
||
|
||
params = {"page": page, "size": 20, "website": "1688_lite2", "language": "zh-CN", "currency": "USD", "from": "",
|
||
"itemTitle": itemTitle, "domain": domain }
|
||
params['sign'] = self.encode_custom(json.dumps(params, ensure_ascii=False))
|
||
data = {
|
||
"imageBase64" : imageBase64
|
||
}
|
||
response = requests.post(url, headers=headers, params=params, json=data, impersonate="chrome101",
|
||
cookies=cookie)
|
||
response.encoding = "utf-8"
|
||
data = response.json()
|
||
return data
|
||
|
||
def _scrape_data(self):
|
||
"""
|
||
抓取商品数据
|
||
|
||
Returns:
|
||
dict: 抓取到的数据
|
||
"""
|
||
data = {
|
||
'image_url': "",
|
||
'title': "",
|
||
'category' : '',
|
||
'success' : True
|
||
}
|
||
|
||
try:
|
||
# 等待页面加载
|
||
# time.sleep(3)
|
||
title_ele = self.tab.ele('xpath://h1[@id="title"]',timeout=30)
|
||
title = title_ele.text
|
||
data["title"] = title
|
||
imge_ele = self.tab.ele('xpath://div[@id="imgTagWrapperId"]//img',timeout=20)
|
||
image_url = imge_ele.attr("src")
|
||
data["image_url"] = image_url
|
||
category = self.tab.eles('xpath://div[@id="wayfinding-breadcrumbs_feature_div"]//span[@class="a-list-item"]//a[@class="a-link-normal a-color-tertiary"]',timeout=20)
|
||
if len(category) > 0:
|
||
data["category"] = category[0].text
|
||
return data
|
||
|
||
except Exception as e:
|
||
print(f"抓取数据时出错: {traceback.format_exc()}")
|
||
data["success"] = False
|
||
return data
|
||
|
||
def run(self, country, asin,total_page=2):
|
||
"""
|
||
运行亚马逊详情采集任务
|
||
Args:
|
||
country: 国家名称（如：英国、德国、法国、西班牙、意大利）
|
||
asin: 亚马逊商品ASIN码
|
||
Returns:
|
||
dict: 包含采集到的数据
|
||
"""
|
||
try:
|
||
# 验证国家是否支持
|
||
if country not in self.country_info:
|
||
error_msg = f"不支持的国家: {country}，支持的国家有: {list(self.country_info.keys())}"
|
||
print(error_msg)
|
||
show_notification(error_msg, "error")
|
||
return None
|
||
|
||
# 获取国家配置
|
||
country_config = self.country_info[country]
|
||
zip_code = country_config["zip_code"]
|
||
mark = country_config.get("mark")
|
||
|
||
# 1. 根据国家和ASIN拼接链接
|
||
base_url = country_config["url"]
|
||
# 提取域名部分
|
||
domain = base_url.split("/dp/")[0]
|
||
# 拼接新的URL
|
||
product_url = f"{domain}/dp/{asin}"
|
||
print(f"正在访问: {product_url}")
|
||
|
||
# 打开链接
|
||
self.tab.get(product_url)
|
||
time.sleep(3) # 等待页面初步加载
|
||
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
|
||
|
||
self.close_init_popup()
|
||
# 2. 切换国家/设置邮编
|
||
print(f"正在检查并设置邮编: {zip_code},标识： {mark}")
|
||
self._set_zip_code(zip_code, mark)
|
||
|
||
# self.tab.wait.doc_loaded(timeout=5, raise_err=False)
|
||
|
||
# 3. 抓取数据
|
||
print("正在抓取商品数据...")
|
||
data = self._scrape_data()
|
||
|
||
# 判断是否采集标题出错
|
||
new_size = "220,220"
|
||
pattern = r"\._(?:[A-Z]+)?(\d+)_\."
|
||
replacement = f"._{new_size}_.jpg"
|
||
# 使用正则替换
|
||
image_new_url = re.sub(pattern, lambda m: replacement, data["image_url"])
|
||
|
||
iamge_base64 = self.image_to_base64(image_new_url)
|
||
|
||
similar_data = []
|
||
# 4、请求获取插件的数据
|
||
for page_num in range(total_page):
|
||
resp_data = self.get_aliprice_data(
|
||
page=page_num,
|
||
title = data["title"],
|
||
domain=domain,
|
||
category=data["category"],
|
||
imageBase64 = iamge_base64
|
||
)
|
||
self.log(f"Aliprice 扩展数据获取:{resp_data}")
|
||
if len(resp_data.get("data",[])) > 0:
|
||
for i in resp_data.get("data"):
|
||
similar_data.append(i)
|
||
|
||
data["similar_data"] = similar_data
|
||
|
||
# 添加基本信息
|
||
data['country'] = country
|
||
data['asin'] = asin
|
||
data['url'] = product_url
|
||
data['timestamp'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||
|
||
print(f"数据抓取完成: {json.dumps(data)}")
|
||
return data
|
||
|
||
except Exception as e:
|
||
error_msg = f"运行出错: {traceback.format_exc()}"
|
||
print(error_msg)
|
||
show_notification(f"采集失败: {str(e)}", "error")
|
||
return {}
|
||
|
||
|
||
class SimilarAsinTask(TaskBase):
|
||
mark_name = "相似ASIN采集"
|
||
|
||
@staticmethod
|
||
def group_by_id_prefix(data):
|
||
"""
|
||
根据每条数据的 id 字段进行分组：
|
||
- 如果 id 为 "2_1"，则取 "_" 前面的 "2" 作为分组 key
|
||
- 如果 id 为 "1"，则分组 key 就是 "1"
|
||
- 相同 key 的数据归为同一组
|
||
|
||
:param data: 原始列表数据
|
||
:return: 二维列表，按 id 前缀分组
|
||
"""
|
||
grouped = defaultdict(list)
|
||
|
||
for item in data:
|
||
item_id = str(item.get("id", ""))
|
||
group_key = item_id.split("_")[0]
|
||
grouped[group_key].append(item)
|
||
|
||
return list(grouped.values())
|
||
|
||
@staticmethod
|
||
def normalize_groups(data):
|
||
groups = data.get("groups")
|
||
if isinstance(groups, list) and len(groups) > 0:
|
||
return groups
|
||
|
||
rows = data.get("rows") or data.get("items") or []
|
||
if not isinstance(rows, list) or len(rows) == 0:
|
||
return []
|
||
|
||
normalized = []
|
||
for items in SimilarAsinTask.group_by_id_prefix(rows):
|
||
first = items[0] if items else {}
|
||
item_id = str(first.get("id") or first.get("displayId") or "")
|
||
base_id = item_id.split("_")[0] if item_id else ""
|
||
normalized.append({
|
||
"sourceFileKey": first.get("sourceFileKey", ""),
|
||
"sourceFilename": first.get("sourceFilename", ""),
|
||
"groupKey": first.get("groupKey") or base_id,
|
||
"baseId": first.get("baseId") or base_id,
|
||
"displayId": first.get("displayId") or item_id,
|
||
"items": items,
|
||
})
|
||
return normalized
|
||
|
||
def process_task(self, task_data: dict):
|
||
"""处理审批任务主入口
|
||
|
||
Args:
|
||
task_data: 任务数据
|
||
"""
|
||
try:
|
||
data = task_data.get("data", {})
|
||
task_id = data.get("taskId")
|
||
groups = self.normalize_groups(data)
|
||
|
||
if not task_id:
|
||
self.log("任务ID为空，跳过", "WARNING")
|
||
return
|
||
|
||
|
||
self.log(f"开始处理爬取任务 {task_id}，{len(groups)} 个任务")
|
||
|
||
if not groups:
|
||
self.log("appearance-patent groups/rows is empty, skip", "WARNING")
|
||
return
|
||
|
||
runing_task[task_id] = {
|
||
"status": "running",
|
||
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
"total_shops": 1,
|
||
"processed_shops": 0,
|
||
"total_countries": len(groups) ,
|
||
"processed_countries": 0,
|
||
"total_asins": 0,
|
||
"processed_asins": 0,
|
||
"success_count": 0,
|
||
"failed_count": 0,
|
||
"stop_requested": False
|
||
}
|
||
|
||
# 检查是否收到暂停请求
|
||
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
|
||
self.log(f"检测到任务 {task_id} 的暂停请求，停止处理", "WARNING")
|
||
runing_task[task_id]["status"] = "stopped"
|
||
return
|
||
max_retry = 3
|
||
show_notification(f"开始爬取数据", "info")
|
||
chrome = ChromeAmzone()
|
||
try:
|
||
# 数据整理
|
||
# new_items = self.group_by_id_prefix(items)
|
||
result = []
|
||
for gp_index,gp in enumerate(groups):
|
||
items = gp.get("items", [])
|
||
return_data = {}
|
||
|
||
for index,value in enumerate(items):
|
||
|
||
return_data = {
|
||
'image_url': "",
|
||
'title': ""
|
||
}
|
||
asin = value.get("asin")
|
||
country = value.get("country")
|
||
return_data = None
|
||
for _ in range(max_retry):
|
||
try:
|
||
return_data = chrome.run(country, asin) or {}
|
||
self.log(f"抓取结果->{return_data}")
|
||
break
|
||
except Exception as e:
|
||
if "与页面的连接已断开" in str(e):
|
||
chrome = ChromeAmzone()
|
||
if not isinstance(return_data, dict):
|
||
return_data = {}
|
||
if return_data.get("image_url"):
|
||
break
|
||
|
||
group_item = [
|
||
{
|
||
"sourceFileKey": i.get("sourceFileKey"),
|
||
"sourceFilename": i.get("sourceFilename"),
|
||
"rowToken": i.get("rowToken"),
|
||
"groupKey": i.get("groupKey"),
|
||
"id": i.get("id"),
|
||
"asin": i.get("asin"),
|
||
"country": i.get("country"),
|
||
"url": return_data.get("image_url"),
|
||
"title": return_data.get("title"),
|
||
}
|
||
for i in items
|
||
]
|
||
# task_id: int, chunkIndex:int,chunkTotal: int, country_code: str, asin: str, status: dict,error:str="",
|
||
# item_data:dict={},
|
||
res = {
|
||
"sourceFileKey": gp.get("sourceFileKey"),
|
||
"sourceFilename": gp.get("sourceFilename"),
|
||
"groupKey": gp.get("groupKey"),
|
||
"baseId": gp.get("baseId"),
|
||
"displayId": gp.get("displayId"),
|
||
"items": group_item
|
||
}
|
||
print("================")
|
||
result.append(res)
|
||
print("================")
|
||
is_done = gp_index == len(groups)-1
|
||
if len(result) > 20 or is_done:
|
||
self.post_result(task_id=task_id,chunkIndex=gp_index+1,chunkTotal=len(groups),
|
||
asin=asin,item_data=result,is_done=is_done)
|
||
result = []
|
||
|
||
|
||
if len(result) > 0:
|
||
is_done = True
|
||
self.post_result(task_id=task_id, chunkIndex=len(groups), chunkTotal=len(groups),
|
||
asin=asin, item_data=result, is_done=is_done)
|
||
|
||
try:
|
||
chrome.close()
|
||
except Exception as e:
|
||
print("退出浏览器出错",e)
|
||
# 更新已处理店铺数
|
||
if task_id in runing_task:
|
||
runing_task[task_id]["processed_shops"] += 1
|
||
except Exception as e:
|
||
self.log(f"处理店铺 {task_id} 失败: {str(e)}", "ERROR")
|
||
self.log(traceback.format_exc(), "ERROR")
|
||
|
||
# 更新任务状态
|
||
if task_id in runing_task:
|
||
if runing_task[task_id].get("stop_requested", False):
|
||
runing_task[task_id]["status"] = "stopped"
|
||
self.log(f"任务 {task_id} 已被暂停！")
|
||
else:
|
||
runing_task[task_id]["status"] = "completed"
|
||
self.log(f"任务 {task_id} 处理完成！")
|
||
|
||
except Exception as e:
|
||
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
|
||
if task_id:
|
||
if task_id in runing_task:
|
||
runing_task[task_id]["status"] = "failed"
|
||
runing_task[task_id]["error"] = str(e)
|
||
|
||
def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
|
||
item_data:dict={},
|
||
is_done: bool = False):
|
||
"""回传处理结果到API
|
||
"""
|
||
|
||
url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result"
|
||
|
||
payload ={
|
||
"submissionId": f"{int(time.time())}",
|
||
"chunkIndex": chunkIndex,
|
||
"chunkTotal": chunkTotal,
|
||
"error": error,
|
||
"groups": item_data,
|
||
"done": is_done
|
||
}
|
||
|
||
max_retries = 3
|
||
for retry in range(max_retries):
|
||
try:
|
||
request_timeout = 300 if is_done else 30
|
||
print("================【详情采集】=====================")
|
||
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
|
||
self.log(f"回传URL: {url}")
|
||
self.log(f"回传数据: {payload}")
|
||
response = requests.post(
|
||
url,
|
||
json=payload,
|
||
headers={"Content-Type": "application/json"},
|
||
timeout=request_timeout,
|
||
verify=False
|
||
)
|
||
self.log(f"回传结果: {response.text}")
|
||
data = response.json() if response.text else {}
|
||
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
|
||
self.log(f"结果回传成功: {asin} - {item_data}")
|
||
return
|
||
else:
|
||
self.log(f"结果回传失败，状态码: {response.status_code}", "WARNING")
|
||
print("=====================================")
|
||
|
||
except Exception as e:
|
||
self.log(f"调用API异常: {str(e)}", "ERROR")
|
||
print("=====================================")
|
||
|
||
# 如果还有重试机会，等待后继续
|
||
if retry < max_retries - 1:
|
||
time.sleep(2)
|
||
|
||
self.log(f"已达到最大重试次数，结果回传最终失败", "ERROR")
|
||
raise RuntimeError("已达到最大重试次数，结果回传最终失败")
|
||
|
||
if __name__ == '__main__':
|
||
spide = ChromeAmzone()
|
||
|
||
resp = spide.run(
|
||
country="英国", asin="B0F79LCB3X", total_page=2
|
||
)
|
||
print(resp)
|
||
|
||
|
||
|