super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new file: amazon/amazon_base.py

Browse Source 
modified:   amazon/approve.py
	modified:   amazon/asin_status.py
	new file:   amazon/chrome_base.py
	modified:   amazon/del_brand.py
	modified:   amazon/detail_spider.py
	new file:   "amazon/detail_spider_\345\244\207\344\273\275.py"
	modified:   amazon/main.py
	modified:   amazon/match_action.py
	modified:   amazon/price_match.py
	new file:   amazon/similar_asin.py
	modified:   app.py
	new file:   assets/appearance-patent-Br1dtmol.css
	new file:   assets/appearance-patent-D1DgeYhe.css
	new file:   assets/delete-brand-BfMLVSQU.css
	new file:   assets/patrol-delete-BAzbYtgc.css
	new file:   assets/price-track-Hozgt_em.css
	new file:   assets/product-risk-CbX7bwZi.css
	new file:   assets/query-asin-B4RsOiza.css
	new file:   assets/shop-match-B2HWAgBY.css
	modified:   main.py
This commit is contained in:
铭坤
2026-05-04 19:15:50 +08:00
parent 147b324658
commit 3ca569ed1e
13 changed files with 11073 additions and 2970 deletions
Download Patch File Download Diff File Expand all files Collapse all files

298
app/amazon/detail_spider.py
View File

@@ -1,88 +1,23 @@
import json
import sys
import os
import io
# sys.stdout.reconfigure(encoding='utf-8')
import time
import re
import traceback
from datetime import datetime
from DrissionPage import Chromium, ChromiumOptions
from collections import defaultdict
import requests
from amazon.del_brand import AmamzonBase, kill_process
from amazon.tool import show_notification,get_shop_info,remove_special_characters,split_currency_values
from amazon.amazon_base import TaskBase
from amazon.chrome_base import ChromeAmzoneBase
from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE
class ChromeAmzone:
class ChromeAmzone(ChromeAmzoneBase):
mark_name = "亚马逊详情采集"
country_info = {
"英国": {
"url": "https://www.amazon.co.uk/dp/B0CJ8SNXXV",
"zip_code": "SW1A 1AA",
"mark": "SW1A 1"
},
"德国": {
"url": "https://www.amazon.de/dp/B0CC8CW9G2?th=1",
"zip_code": "10115"
},
"法国": {
"url": "https://www.amazon.fr/dp/B0FRG1MJ8H?th=1",
"zip_code": "75001"
},
"西班牙": {
"url": "https://www.amazon.es/dp/B08ZXVNYNN",
"zip_code": "28001"
},
"意大利": {
"url": "https://www.amazon.it/dp/B0D1P17T2Q",
"zip_code": "20121"
}
}
def __init__(self):
"""
杀死当前谷歌浏览器进程，并使用 drissionpage 启动谷歌浏览器，使用系统安装的浏览器默认用户文件夹
"""
# 杀死现有的Chrome进程
print("正在关闭现有的chromium浏览器进程...")
os.system('taskkill /f /t /im chrome.exe')
time.sleep(2)
print("正在启动chromium浏览器...")
# 配置浏览器选项
co = ChromiumOptions()
user_data_path = os.path.join(base_dir, "user_data", "chrome_data")
if not os.path.exists(user_data_path):
os.makedirs(user_data_path, exist_ok=True)
co.set_user_data_path(user_data_path)
co.set_local_port(port=19897)
self.browser = Chromium(co)
self.tab = self.browser.latest_tab
print("Chrome浏览器启动成功")
def close_init_popup(self):
"""
关闭所有的初始化弹窗
"""
self.tab.wait.doc_loaded(timeout=60, raise_err=True)
footbar = self.tab.eles('xpath://footer[@class="el-dialog__footer"]', timeout=5)
if len(footbar) > 0:
do_not_remind = footbar[0].eles('xpath:.//input[@class="el-checkbox__original"]')
if len(do_not_remind) > 0:
do_not_remind[0].check()
resume_immediately = footbar[0].eles('xpath:.//button')
if len(resume_immediately) > 0:
resume_immediately[0].click()
self.tab.wait.doc_loaded(timeout=60, raise_err=True)
accept_btn = self.tab.eles('xpath://input[@id="sp-cc-accept"]', timeout=5)
if len(accept_btn) > 0:
accept_btn[0].click()
def run(self, country, asin):
"""
@@ -99,7 +34,7 @@ class ChromeAmzone:
# 验证国家是否支持
if country not in self.country_info:
error_msg = f"不支持的国家: {country}，支持的国家有: {list(self.country_info.keys())}"
print(error_msg)
self.log(error_msg)
show_notification(error_msg, "error")
return None
@@ -114,7 +49,7 @@ class ChromeAmzone:
domain = base_url.split("/dp/")[0]
# 拼接新的URL
product_url = f"{domain}/dp/{asin}"
print(f"正在访问: {product_url}")
self.log(f"正在访问: {product_url}")
# 打开链接
self.tab.get(product_url)
@@ -123,13 +58,13 @@ class ChromeAmzone:
self.close_init_popup()
# 2. 切换国家/设置邮编
print(f"正在检查并设置邮编: {zip_code},标识： {mark}")
self.log(f"正在检查并设置邮编: {zip_code},标识： {mark}")
self._set_zip_code(zip_code, mark)
# self.tab.wait.doc_loaded(timeout=5, raise_err=False)
# 3. 抓取数据
print("正在抓取商品数据...")
self.log("正在抓取商品数据...")
data = self._scrape_data()
# 添加基本信息
@@ -138,100 +73,15 @@ class ChromeAmzone:
data['url'] = product_url
data['timestamp'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f"数据抓取完成: {json.dumps(data)}")
self.log(f"数据抓取完成: {json.dumps(data)}")
return data
except Exception as e:
error_msg = f"运行出错: {traceback.format_exc()}"
print(error_msg)
self.log(error_msg)
show_notification(f"采集失败: {str(e)}", "error")
return {}
def _set_zip_code(self, zip_code, mark=None):
"""
设置邮编
Args:
zip_code: 目标邮编
"""
try:
# 检查当前邮编
zip_display = self.tab.ele('xpath://div[@id="glow-ingress-block"]', timeout=10)
if zip_display:
current_text = zip_display.text
# print(f"当前地址信息: {current_text}")
# 先检查标识
if mark is not None and mark in current_text:
print(f"邮编检测到标识: {mark}，无需修改")
return True
# 检查是否已经包含目标邮编
if zip_code in current_text:
print(f"邮编已经设置为: {zip_code}，无需修改")
return True
# 需要设置邮编
print(f"正在设置邮编为: {zip_code}")
# 点击地址选择按钮
location_link = self.tab.ele('xpath://a[@id="nav-global-location-popover-link"]', timeout=10)
if not location_link:
print("找不到地址设置按钮")
return False
location_link.click()
time.sleep(1)
# 等待邮编输入框出现
zip_input = self.tab.ele('xpath://input[@id="GLUXZipUpdateInput"]', timeout=10)
if not zip_input:
print("找不到邮编输入框")
return False
# 输入邮编
zip_input.input(zip_code, clear=True)
time.sleep(0.5)
# 点击提交按钮
submit_btn = self.tab.ele('xpath://input[@aria-labelledby="GLUXZipUpdate-announce"]', timeout=10)
if not submit_btn:
print("找不到提交按钮")
return False
submit_btn.click()
continue_btn = self.tab.eles('xpath://div[@class="a-popover-footer"]//input[@id="GLUXConfirmClose"]',
timeout=10)
if len(continue_btn) > 0:
continue_btn[0].click()
# 等待提交按钮消失（表示请求已发送）
print("等待邮编更新...")
time.sleep(2)
# 等待页面加载完成
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
time.sleep(2)
# 验证邮编是否设置成功
zip_display_after = self.tab.ele('xpath://div[@id="glow-ingress-block"]', timeout=10)
if zip_display_after:
updated_text = zip_display_after.text
# print(f"更新后的地址信息: {updated_text}")
if zip_code in updated_text:
print(f"邮编设置成功: {zip_code}")
return True
else:
# print(f"邮编设置可能失败，当前显示: {updated_text}")
return False
return True
except Exception as e:
print(f"设置邮编时出错: {traceback.format_exc()}")
return False
def _scrape_data(self):
"""
抓取商品数据
@@ -256,39 +106,16 @@ class ChromeAmzone:
return data
except Exception as e:
print(f"抓取数据时出错: {traceback.format_exc()}")
self.log(f"抓取数据时出错: {traceback.format_exc()}")
return data
def close(self):
"""关闭浏览器"""
try:
if self.browser:
self.browser.quit()
print("浏览器已关闭")
except Exception as e:
print(f"关闭浏览器时出错: {str(e)}")
class SpiderTask:
mark_name = "亚马逊采集"
def __init__(self, user_info: dict = None):
"""初始化审批任务处理器
Args:
user_info: 用户信息字典，包含 company, username, password
"""
self.user_info = user_info or {}
self.running = True
def log(self, message: str, level: str = "INFO"):
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [PriceTask] [{level}] {message}")
class SpiderTask(TaskBase):
task_name = "亚马逊采集"
@staticmethod
def group_by_id_prefix(data):
def group_by_id_prefix(data):
"""
根据每条数据的 id 字段进行分组：
- 如果 id 为 "2_1"，则取 "_" 前面的 "2" 作为分组 key
@@ -304,35 +131,35 @@ class SpiderTask:
item_id = str(item.get("id", ""))
group_key = item_id.split("_")[0]
grouped[group_key].append(item)
return list(grouped.values())
@staticmethod
def normalize_groups(data):
groups = data.get("groups")
if isinstance(groups, list) and len(groups) > 0:
return groups
rows = data.get("rows") or data.get("items") or []
if not isinstance(rows, list) or len(rows) == 0:
return []
normalized = []
for items in SpiderTask.group_by_id_prefix(rows):
first = items[0] if items else {}
item_id = str(first.get("id") or first.get("displayId") or "")
base_id = item_id.split("_")[0] if item_id else ""
normalized.append({
"sourceFileKey": first.get("sourceFileKey", ""),
"sourceFilename": first.get("sourceFilename", ""),
"groupKey": first.get("groupKey") or base_id,
"baseId": first.get("baseId") or base_id,
"displayId": first.get("displayId") or item_id,
"items": items,
})
return normalized
def process_task(self, task_data: dict):
return list(grouped.values())
@staticmethod
def normalize_groups(data):
groups = data.get("groups")
if isinstance(groups, list) and len(groups) > 0:
return groups
rows = data.get("rows") or data.get("items") or []
if not isinstance(rows, list) or len(rows) == 0:
return []
normalized = []
for items in SpiderTask.group_by_id_prefix(rows):
first = items[0] if items else {}
item_id = str(first.get("id") or first.get("displayId") or "")
base_id = item_id.split("_")[0] if item_id else ""
normalized.append({
"sourceFileKey": first.get("sourceFileKey", ""),
"sourceFilename": first.get("sourceFilename", ""),
"groupKey": first.get("groupKey") or base_id,
"baseId": first.get("baseId") or base_id,
"displayId": first.get("displayId") or item_id,
"items": items,
})
return normalized
def process_task(self, task_data: dict):
"""处理审批任务主入口
Args:
@@ -341,7 +168,7 @@ class SpiderTask:
try:
data = task_data.get("data", {})
task_id = data.get("taskId")
groups = self.normalize_groups(data)
groups = self.normalize_groups(data)
# 用于测试
limit = data.get("limit", None)
@@ -353,11 +180,10 @@ class SpiderTask:
self.log(f"开始处理爬取任务 {task_id}，{len(groups)} 个任务")
if not groups:
self.log("appearance-patent groups/rows is empty, skip", "WARNING")
return
from config import runing_task
if not groups:
self.log("appearance-patent groups/rows is empty, skip", "WARNING")
return
runing_task[task_id] = {
"status": "running",
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
@@ -383,12 +209,12 @@ class SpiderTask:
try:
# 数据整理
# new_items = self.group_by_id_prefix(items)
result = []
for gp_index,gp in enumerate(groups):
items = gp.get("items", [])
return_data = {}
for index,value in enumerate(items):
result = []
for gp_index,gp in enumerate(groups):
items = gp.get("items", [])
return_data = {}
for index,value in enumerate(items):
return_data = {
'image_url': "",
@@ -396,18 +222,17 @@ class SpiderTask:
}
asin = value.get("asin")
country = value.get("country")
return_data = None
for _ in range(max_retry):
try:
return_data = chrome.run(country, asin) or {}
return_data = chrome.run(country, asin) or {}
self.log(f"抓取结果->{return_data}")
break
except Exception as e:
if "与页面的连接已断开" in str(e):
chrome = ChromeAmzone()
if not isinstance(return_data, dict):
return_data = {}
if return_data.get("image_url"):
if not isinstance(return_data, dict):
return_data = {}
if return_data.get("image_url"):
break
group_item = [
@@ -457,7 +282,6 @@ class SpiderTask:
if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1
except Exception as e:
import traceback
self.log(f"处理店铺 {task_id} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
@@ -471,10 +295,8 @@ class SpiderTask:
self.log(f"任务 {task_id} 处理完成！")
except Exception as e:
import traceback
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id:
from config import runing_task
if task_id in runing_task:
runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e)
@@ -488,7 +310,7 @@ class SpiderTask:
url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result"
payload ={
"submissionId": f"{int(time.time())}",
"submissionId": f"{task_id}",
"chunkIndex": chunkIndex,
"chunkTotal": chunkTotal,
"error": error,
@@ -8413,4 +8235,4 @@ if __name__ == '__main__':
]
# spide.process_task(task_data)
res = SpiderTask.group_by_id_prefix(task_data)
print(res)
print(res)