new file: amazon/amazon_base.py

modified:   amazon/approve.py
	modified:   amazon/asin_status.py
	new file:   amazon/chrome_base.py
	modified:   amazon/del_brand.py
	modified:   amazon/detail_spider.py
	new file:   "amazon/detail_spider_\345\244\207\344\273\275.py"
	modified:   amazon/main.py
	modified:   amazon/match_action.py
	modified:   amazon/price_match.py
	new file:   amazon/similar_asin.py
	modified:   app.py
	new file:   assets/appearance-patent-Br1dtmol.css
	new file:   assets/appearance-patent-D1DgeYhe.css
	new file:   assets/delete-brand-BfMLVSQU.css
	new file:   assets/patrol-delete-BAzbYtgc.css
	new file:   assets/price-track-Hozgt_em.css
	new file:   assets/product-risk-CbX7bwZi.css
	new file:   assets/query-asin-B4RsOiza.css
	new file:   assets/shop-match-B2HWAgBY.css
	modified:   main.py
This commit is contained in:
铭坤
2026-05-04 19:15:50 +08:00
parent 147b324658
commit 3ca569ed1e
13 changed files with 11073 additions and 2970 deletions

847
app/amazon/amazon_base.py Normal file
View File

@@ -0,0 +1,847 @@
import traceback
import winreg
import subprocess
import time
import uuid
import requests
import json
import os
import re
from datetime import datetime
from DrissionPage import Chromium,ChromiumOptions
from DrissionPage.common import By
from DrissionPage._pages.chromium_tab import ChromiumTab
from typing import Literal,Dict,Any,TypeVar,Type
from amazon.tool import get_shop_info,show_notification
from config import runing_shop
# 定义类型变量,用于泛型类型注解
T = TypeVar('T', bound='AmamzonBase')
def kill_process(version: Literal["v5", "v6"]):
"""杀紫鸟客户端进程(独立函数版本)"""
driver = ZiniaoDriver({})
driver.kill_process(version)
class ZiniaoDriver:
"""紫鸟浏览器自动化驱动类"""
def __init__(self, user_info: dict, socket_port: int = 19890):
"""
初始化紫鸟浏览器驱动
Args:
user_info: 用户信息字典,包含 company, username, password
socket_port: 客户端通信端口,默认 19890
"""
self.user_info = user_info
self.socket_port = socket_port
self.client_path = None
self.browser = None
self.tab: ChromiumTab = None
self.store_id = None
self.url = None #用于记录当前链接,实现断点续传
def get_zinaio_exe(self, protocol_name: str = "superbrowser"):
"""
获取紫鸟安装目录
Args:
protocol_name: 协议名称,默认 "superbrowser"
Returns:
exe_path: 紫鸟浏览器可执行文件路径
"""
try:
key_path = rf"SOFTWARE\Classes\{protocol_name}\shell\open\command"
key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_path)
command, _ = winreg.QueryValueEx(key, "")
winreg.CloseKey(key)
if isinstance(command, str):
sub = "ziniao.exe"
exe_path = command[0: command.find(sub) + len(sub) + 1]
else:
exe_path = command[0]
return exe_path
except FileNotFoundError:
try:
key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key_path)
command, _ = winreg.QueryValueEx(key, "")
winreg.CloseKey(key)
if isinstance(command, str):
sub = "ziniao.exe"
exe_path = command[0: command.find(sub) + len(sub) + 1]
else:
exe_path = command[0]
return exe_path
except FileNotFoundError:
return None
def update_core(self):
"""
下载所有内核打开店铺前调用需客户端版本5.285.7以上
因为http有超时时间所以这个action适合循环调用直到返回成功
"""
data = {
"action": "updateCore",
"requestId": str(uuid.uuid4()),
}
data.update(self.user_info)
while True:
url = f'http://127.0.0.1:{self.socket_port}'
response = requests.post(url, json.dumps(data).encode('utf-8'), timeout=120)
result = response.json()
print(result)
if result is None:
print("等待客户端启动...")
time.sleep(2)
continue
if result.get("statusCode") is None or result.get("statusCode") == -10003:
print("当前版本不支持此接口,请升级客户端")
return
elif result.get("statusCode") == 0:
print("更新内核完成")
return
else:
print(f"等待更新内核: {json.dumps(result)}")
time.sleep(2)
def kill_process(self, version: Literal["v5", "v6"]):
"""
杀紫鸟客户端进程
Args:
version: 客户端版本
"""
if version == "v5":
process_name = 'SuperBrowser.exe'
os.system('taskkill /f /t /im ' + "starter.exe")
else:
process_name = 'ziniao.exe'
os.system('taskkill /f /t /im ' + process_name)
time.sleep(3)
def get_browser_list(self) -> list:
"""
获取浏览器列表
Returns:
list: 浏览器列表
"""
request_id = str(uuid.uuid4())
data = {
"action": "getBrowserList",
"requestId": request_id
}
data.update(self.user_info)
url = f'http://127.0.0.1:{self.socket_port}'
response = requests.post(url, json.dumps(data).encode('utf-8'), timeout=120)
r = response.json()
if str(r.get("statusCode")) == "0":
print(r)
return r.get("browserList")
elif str(r.get("statusCode")) == "-10003":
print(f"【get_browser_list】登录失败 {json.dumps(r, ensure_ascii=False)}")
# exit()
else:
print(f"【get_browser_list】失败 {json.dumps(r, ensure_ascii=False)} ")
# exit()
def open_store(self, store_info, isWebDriverReadOnlyMode=0, isprivacy=0,
isHeadless=0, cookieTypeSave=0, jsInfo=""):
"""
打开店铺
Args:
store_info: 店铺信息browserId 或 browserOauth
isWebDriverReadOnlyMode: 是否只读模式,默认 0
isprivacy: 隐私模式,默认 0
isHeadless: 无头模式,默认 0
cookieTypeSave: cookie保存类型默认 0
jsInfo: 注入的JS信息默认 ""
Returns:
dict: 返回结果
"""
request_id = str(uuid.uuid4())
data = {
"action": "startBrowser",
"isWaitPluginUpdate": 0,
"isHeadless": isHeadless,
"requestId": request_id,
"isWebDriverReadOnlyMode": isWebDriverReadOnlyMode,
"cookieTypeLoad": 0,
"cookieTypeSave": cookieTypeSave,
"runMode": "1",
"isLoadUserPlugin": True,
"pluginIdType": 1,
"privacyMode": isprivacy
}
data.update(self.user_info)
if store_info.isdigit():
data["browserId"] = store_info
else:
data["browserOauth"] = store_info
if len(str(jsInfo)) > 2:
data["injectJsInfo"] = json.dumps(jsInfo)
url = f'http://127.0.0.1:{self.socket_port}'
response = requests.post(url, json.dumps(data).encode('utf-8'), timeout=120)
r = response.json()
if str(r.get("statusCode")) == "0":
return r
elif str(r.get("statusCode")) == "-10003":
raise RuntimeError(f"【open_store】登录失败 {json.dumps(r, ensure_ascii=False)}")
# exit()
else:
raise RuntimeError(f"【open_store】失败 {json.dumps(r, ensure_ascii=False)} ")
# exit()
def get_browser(self, port) -> Chromium:
"""
获取浏览器实例
Args:
port: 调试端口
Returns:
Chromium: DrissionPage浏览器实例
"""
co = ChromiumOptions()
# 设置不加载图片、静音
co.set_local_port(port)
co.no_imgs(True).mute(True)
browser = Chromium(co)
return browser
def start_client(self):
"""启动紫鸟客户端"""
# 检查端口是否已经启动
try:
url = f'http://127.0.0.1:{self.socket_port}'
response = requests.get(url, timeout=2)
print(f"端口 {self.socket_port} 已经启动,跳过启动客户端操作")
return
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
# 端口未启动,继续执行启动操作
print(f"端口 {self.socket_port} 未启动,开始启动客户端")
self.kill_process('v6')
time.sleep(5)
self.client_path = self.get_zinaio_exe("superbrowserv6").strip('"')
print(self.client_path)
cmd = [self.client_path, '--run_type=web_driver', '--ipc_type=http',
'--port=' + str(self.socket_port)]
print(" ".join(cmd))
# 最大重试次数
max_retries = 3
for retry_count in range(max_retries):
print(f"{retry_count + 1} 次尝试启动客户端...")
# 启动进程
subprocess.Popen(cmd)
# 循环检测10秒每0.5秒检测一次
start_check_time = time.time()
client_started = False
while time.time() - start_check_time < 10:
try:
response = requests.get(url, timeout=2)
print(f"客户端启动成功!(第 {retry_count + 1} 次尝试)")
client_started = True
break
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
# 端口还未启动,继续等待
time.sleep(0.5)
if client_started:
time.sleep(5) # 等待客户端完全启动
# 更新内核
self.update_core()
return
else:
print(f"{retry_count + 1} 次尝试启动失败10秒内未检测到客户端启动")
# 超过最大重试次数,抛出异常
raise RuntimeError(f"客户端启动失败:重试 {max_retries} 次后仍未成功启动")
def open_shop(self, shop_name: str):
"""
打开指定店铺
Args:
shop_name: 店铺名称
Returns:
Chromium or str: 成功返回浏览器实例,失败返回错误信息
"""
print("=============打开指定店铺================")
# 启动客户端
self.start_client()
# 获取店铺列表
shop_ls = self.get_browser_list()
self.store_id = None
print(shop_ls)
for shop in shop_ls:
if shop.get("browserName") == shop_name:
self.store_id = shop.get('browserOauth')
break
if not self.store_id:
print("店铺不存在")
return "店铺不存在"
# 打开店铺
ret_json = self.open_store(self.store_id)
print(ret_json)
self.store_id = ret_json.get("browserOauth")
if self.store_id is None:
self.store_id = ret_json.get("browserId")
# 获取drissionpage浏览器会话
self.browser = self.get_browser(ret_json.get('debuggingPort'))
ip_check_url = ret_json.get("ipDetectionPage")
if not ip_check_url:
print("ip检测页地址为空请升级紫鸟浏览器到最新版")
print(f"=====关闭店铺:{shop_name}=====")
self.close_store(self.store_id)
# exit()
raise RuntimeError("没有IP检测地址为了店铺安全不打开店铺")
ip_usable = self.open_ip_check(self.browser, ip_check_url)
if ip_usable:
print("ip检测通过打开店铺平台主页")
self.open_launcher_page(ret_json.get("launcherPage"), self.browser)
else:
print("IP检测不通过")
raise RuntimeError("IP检测不通过可能是因为网络环境变化导致的为了店铺安全不打开店铺")
return self.browser
def close_store(self, browser_oauth=None):
"""
关闭店铺
Args:
browser_oauth: 店铺OAuth标识如果不提供则使用当前打开的店铺
Returns:
dict: 返回结果
"""
if browser_oauth is None:
browser_oauth = self.store_id
request_id = str(uuid.uuid4())
data = {
"action": "stopBrowser",
"requestId": request_id,
"duplicate": 0,
"browserOauth": browser_oauth
}
data.update(self.user_info)
url = f'http://127.0.0.1:{self.socket_port}'
response = requests.post(url, json.dumps(data).encode('utf-8'), timeout=120)
r = response.json()
if str(r.get("statusCode")) == "0":
return r
elif str(r.get("statusCode")) == "-10003":
raise RuntimeError(f"【close_store】登录失败 {json.dumps(r, ensure_ascii=False)}")
# exit()
else:
raise RuntimeError(f"【close_store】失败 {json.dumps(r, ensure_ascii=False)} ")
# exit()
def open_launcher_page(self, launcher_page: str, browser: Chromium = None):
"""
打开启动页面
Args:
launcher_page: 要打开的页面URL
browser: 浏览器实例,如果不提供则使用当前浏览器实例
"""
if browser is None:
browser = self.browser
tab = browser.new_tab(url=launcher_page)
self.tab = tab
return tab
def open_ip_check(self, browser: Chromium, ip_check_url: str):
"""
打开ip检测页检测ip是否正常
:param browser: drissionpage浏览器会话
:param ip_check_url ip检测页地址
:return 检测结果
"""
try:
tab = browser.latest_tab
tab.get(ip_check_url)
success_button = tab.ele((By.XPATH, '//button[contains(@class, "styles_btn--success")]'),
timeout=60) # 等待查找元素60秒
if success_button:
print("ip检测成功")
return True
else:
print("ip检测超时")
return False
except Exception as e:
print("ip检测异常:" + traceback.format_exc())
return False
class AmamzonBase(ZiniaoDriver):
"""亚马逊操作基类,包含一些通用方法"""
mark_name = "操作基类"
def log(self, message: str, level: str = "INFO"):
"""日志输出
Args:
message: 日志消息
level: 日志级别
"""
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [{self.mark_name}] [{level}] {message}")
except Exception as e:
print(f"输出出错,{e}")
def SwitchingCountries(self, country_name: str):
"""
切换国家
操作:
1、//div[@class="dropdown-account-switcher-header-label"]/span[last()] 获取此元素文本,判断当前国家,如果与目标国家相同则不操作,否则执行下一步
2、点击 //div[@class="dropdown-account-switcher-header-label"] 打开下拉框
3、点击 //div[@class="dropdown-account-switcher-list-item"] 第一个展开国家列表
4、点击 //div[@class="dropdown-account-switcher-list-item dropdown-account-switcher-list-item-indented" and @title="国家名"] 切换到目标国家
5、等待页面加载完成判断国家是否切换成功成功则返回True否则返回False
Args:
country_name: 目标国家名称
Returns:
bool: 切换成功返回True失败返回False
"""
try:
print("=============切换国家================")
if self.browser is None:
print("浏览器实例不存在,请先打开店铺")
return False
# 获取当前标签页
tab = self.tab
tab.wait.doc_loaded(timeout=120, raise_err=False)
# 步骤1获取当前国家名称
self.log(f"正在检查当前国家...")
current_country_ele = tab.ele('xpath://div[@class="dropdown-account-switcher-header-label"]/span[last()]',
timeout=20)
if current_country_ele:
current_country = current_country_ele.text.strip()
self.log(f"当前国家:{current_country}")
# 判断是否与目标国家相同
if current_country == country_name:
self.log(f"当前已经是目标国家 {country_name},无需切换")
return True
else:
self.log("无法获取当前国家信息")
return False
# 步骤2点击打开下拉框
self.log(f"正在打开国家切换下拉框...")
dropdown_header = tab.ele('xpath://div[@class="dropdown-account-switcher-header-label"]', timeout=10)
if not dropdown_header:
self.log("找不到国家切换下拉框")
return False
dropdown_header.click()
time.sleep(1) # 等待下拉框展开
# 步骤3点击第一个展开国家列表
self.log(f"正在展开国家列表...")
first_item = tab.ele('xpath://div[@class="dropdown-account-switcher-list-item"]', timeout=10)
if not first_item:
self.log("找不到国家列表项")
return False
first_item.click()
time.sleep(1) # 等待国家列表展开
# 步骤4点击目标国家
self.log(f"正在切换到国家:{country_name}")
target_country_xpath = f'//div[@class="dropdown-account-switcher-list-item dropdown-account-switcher-list-item-indented" and @title="{country_name}"]'
target_country = tab.ele(f'xpath:{target_country_xpath}', timeout=10)
if not target_country:
self.log(f"找不到目标国家:{country_name}")
return False
target_country.click()
# 步骤5等待页面加载完成并验证切换结果
self.log(f"等待页面加载...")
# time.sleep(3) # 等待页面加载
self.tab.wait.doc_loaded()
# 再次检查当前国家
new_country_ele = tab.ele('xpath://div[@class="dropdown-account-switcher-header-label"]/span[last()]',
timeout=10)
if new_country_ele:
new_country = new_country_ele.text.strip()
if new_country == country_name:
self.log(f"国家切换成功:{new_country}")
return True
else:
self.log(f"国家切换失败,当前国家:{new_country},目标国家:{country_name}")
return False
else:
self.log("无法验证切换结果")
return False
except Exception as e:
self.log(f"切换国家时发生异常:{traceback.format_exc()}")
return False
def need_login(self):
"""
判断是否需要登录,部分国家可能需要登录后才能切换国家
处理流程:
"""
time.sleep(3) # 等待页面可能的登录元素加载,避免跳转等等
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
need_login_ele = self.tab.eles('xpath://h1[@class="a-spacing-small"]|//span[contains(text(),"登录")]', timeout=5)
if len(need_login_ele) > 0:
self.log("检测到需要登录元素")
return True
return False
def login(self, password, username=""):
try:
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
for _ in range(4):
switch_account = self.tab.eles(
'xpath://div[@data-test-id="switchableAccounts"]//div[@class="a-fixed-left-grid"]', timeout=5)
if len(switch_account) > 0:
switch_account[0].click()
time.sleep(1)
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
pwd_input = self.tab.eles('xpath://input[@type="password"]', timeout=5)
if len(pwd_input) > 0:
pwd_input[0].input(password, clear=True)
submit_btn = self.tab.eles('xpath://input[@id="signInSubmit"]', timeout=5)
if len(submit_btn) > 0:
submit_btn[0].click()
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
send_code = self.tab.eles('xpath://span[@id="auth-send-code" and contains(string(.),"发送一次性密码")]',
timeout=5)
self.log(f"发送一次性密码:{len(send_code)}")
if len(send_code) > 0:
self.log("检测到 发送一次性密码")
send_code[0].click()
time.sleep(1)
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
else:
for j in range(5):
opt_code_input = self.tab.eles('xpath://input[@name="otpCode"]', timeout=30)
self.log(f"验证码输入框:{len(opt_code_input)}")
if len(opt_code_input) > 0:
opt_code_input[0].wait.displayed(timeout=10, raise_err=False)
for _ in range(30):
if opt_code_input[0].value is not None and opt_code_input[0].value.strip() != "":
print("检测到验证码输入完成")
submit_btn = self.tab.eles('xpath://input[@id="auth-signin-button"]', timeout=10)
if len(submit_btn) > 0:
submit_btn[0].click()
self.tab.wait.doc_loaded(timeout=20, raise_err=False)
# return True
error_mes = self.tab.eles('xpath://div[@id="auth-error-message-box"]',
timeout=10)
if len(error_mes) > 0:
self.log("验证码输入错误")
self.log(f"验证码输入错误提示:{error_mes[0].text}")
self.tab.refresh()
else:
self.log("登入成功")
return True
time.sleep(1)
else:
break
submit_btn = self.tab.eles('xpath://input[@id="auth-signin-button"]', timeout=10)
if len(submit_btn) > 0:
submit_btn[0].click()
except Exception as e:
self.log(f"登录过程中发生异常:{traceback.format_exc()}")
return False
def SwitchPage(self):
"""
切换至 管理所有库存页面
1、等待 //navigation-favorites-bar[@class="hydrated"] 出现
"""
navigation = self.tab.ele('xpath://navigation-favorites-bar[@class="hydrated"]')
navigation.wait.displayed(raise_err=False)
page_btn = navigation.sr('xpath://internal-fav-bar-links[@data-internal="navigation"]').sr(
'xpath://a[@data-page-id="ezdpc-gui-inventory-mons"]')
page_btn.wait.displayed(raise_err=False)
page_btn.click(timeout=5)
self.tab.wait.doc_loaded()
# 等待搜索框出现
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group')
search_region.wait.displayed(raise_err=False, timeout=60)
def search(self,filter_type="ApprovalRequired"):
sku_ls = []
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
drop_down = self.tab.ele('xpath://div[contains(@class,"VolusListingStatusDropDown-module__verticalContainer")]//kat-dropdown')
drop_down.wait.displayed(raise_err=False)
drop_down.wait.enabled(raise_err=False)
time.sleep(0.6)
drop_down.click()
# //kat-option[@value="SearchSuppressed"]
xp = f'xpath://kat-option[@value="{filter_type}"]'
self.log(f"正在寻找筛选条件 {filter_type}xpath: {xp}")
approval_required = self.tab.eles(xp,timeout=5)
if len(approval_required) == 0:
self.log(f"没有需要{filter_type}选项】没有需要{filter_type}的商品了")
return sku_ls # "没有需要审批的商品了"
else:
approval_required = approval_required[0]
approval_required.wait.displayed(raise_err=False)
approval_required.click()
approval_required_text = approval_required.text
self.log(f"已选择筛选条件: {approval_required_text}")
count = re.findall(r'\d+', approval_required_text)
if count:
count = int(count[0])
self.log(f"待审批的商品数量: {count}")
if count <= 0:
self.log(f"没有需要{filter_type}的商品了")
return sku_ls #"没有需要审批的商品了"
for _ in range(3):
# 等待加载完成
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]")
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=3)
if len(sku_ls) > 0:
break
approval_required.click()
return sku_ls
class TaskBase:
task_name = "任务基类"
country_info = {
"DE": "德国",
"FR": "法国",
"ES": "西班牙",
"IT": "意大利",
"UK": "英国"
}
def __init__(self, user_info: dict = None):
"""初始化审批任务处理器
Args:
user_info: 用户信息字典,包含 company, username, password
"""
self.user_info = user_info or {}
self.running = True
def log(self, message: str, level: str = "INFO"):
"""日志输出
Args:
message: 日志消息
level: 日志级别
"""
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [{self.task_name}] [{level}] {message}")
except Exception as e:
print(f"输出出错,{e}")
def process_task(self, task_data: dict):
pass
def open_shop(self, cls: Type[T], max_retries: int, company_name: str, shop_name: str, iskill: bool = False) -> T:
error_info = ""
driver = None
for retry in range(max_retries):
try:
self.log(f"尝试打开店铺 {shop_name} (第 {retry + 1}/{max_retries} 次)")
if iskill:
self.log("重试前先杀掉浏览器进程...")
kill_process("v6")
kill_process("v5")
time.sleep(2)
# 组装用户信息并创建驱动
user_info = {
**self.user_info,
"company": company_name
}
driver = cls(user_info)
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name}")
else:
self.log(f"打开店铺失败: {browser}", "WARNING")
driver = None
continue
# 判断是否需要登录
need_login = driver.need_login()
print("【是否需要登录】:", need_login)
if need_login:
self.log(f"店铺 {shop_name} 需要登录,正在登录...")
# 获取店铺凭证
response = get_shop_info(shop_name)
print("【获取店铺凭证返回】:", response.text)
shop_data = response.json()
if not shop_data:
mes = f"获取店铺凭证失败,响应数据: {shop_data.get('message', '未知错误')}"
self.log(mes, "ERROR")
show_notification(mes, "ERROR")
continue
password = shop_data["data"]["password"]
login_success = driver.login(password)
if login_success:
self.log(f"店铺 {shop_name} 登录成功,正在重新打开店铺...")
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name} 登录后")
break
else:
self.log(f"登录后打开店铺失败: {browser}", "WARNING")
driver = None
else:
self.log(f"店铺 {shop_name} 登录失败", "WARNING")
driver = None
else:
break
except Exception as e:
import traceback
self.log(f"打开店铺异常: {traceback.format_exc()}", "INFO")
driver = None
error_info = str(e)
time.sleep(10)
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(3)
# 检查是否成功打开
if not driver or not browser or browser == "店铺不存在":
error_msg = f"店铺 {shop_name} 打开失败,已重试 {max_retries} 次,跳过该店铺,{error_info}"
self.log(error_msg, "ERROR")
# 从执行列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
return driver
return driver
def process_shop(self, shop_data: Dict[str, Any], task_id: int):
pass
def action_init(self, driver, country_name, shop_name, risk_listing_filter=None):
"""切换到指定国际 -> 页面 -》 筛选好"""
max_retries = 5
switch_success = False
switch_success_pg = False
search_success = False
sku_ls = []
for retry in range(max_retries):
try:
self.log(f"尝试切换到国家 {country_name} (第 {retry + 1}/{max_retries} 次)")
if retry > 2:
# 刷新不行就重新打开店铺
self.log("重试前重新打开店铺...")
try:
driver.close_store()
time.sleep(3)
driver.open_shop(shop_name)
except Exception as e:
self.log(f"关闭重新打开店铺: {str(e)}", "WARNING")
# 如果不是第一次尝试,先刷新页面
if retry > 0:
self.log("重试前刷新页面...")
try:
driver.tab.refresh()
time.sleep(3)
except Exception as e:
self.log(f"刷新页面失败: {str(e)}", "WARNING")
switch_success = driver.SwitchingCountries(country_name)
if switch_success:
self.log(f"成功切换到国家 {country_name}")
# 切换到库存管理页面
driver.SwitchPage()
self.log(f"已切换到库存管理页面")
time.sleep(3)
switch_success_pg = True
if risk_listing_filter is None:
self.log(f"risk_listing_filter 为空,不筛选, {risk_listing_filter}")
search_success = True
break
sku_ls = driver.search(filter_type=risk_listing_filter)
search_success = True
self.log(f"国家 {country_name} 搜索出 {len(sku_ls)} 商品,开始处理...")
break
else:
self.log(f"切换到国家 {country_name} 失败", "WARNING")
except Exception as e:
import traceback
self.log(f"切换国家 {country_name} 异常: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
return switch_success, switch_success_pg, search_success, sku_ls

File diff suppressed because it is too large Load Diff

View File

@@ -1,42 +1,17 @@
import json
import sys
import os
import io
# sys.stdout.reconfigure(encoding='utf-8')
import time import time
import re
import traceback import traceback
from datetime import datetime from datetime import datetime
from DrissionPage import Chromium, ChromiumOptions
import requests import requests
from amazon.del_brand import AmamzonBase, kill_process from amazon.amazon_base import AmamzonBase, kill_process,TaskBase
from amazon.tool import show_notification, get_shop_info, remove_special_characters, split_currency_values from amazon.tool import show_notification
from config import runing_task, runing_shop, base_dir, DELETE_BRAND_API_BASE from config import runing_task, runing_shop, DELETE_BRAND_API_BASE
class AmzonePStatus(AmamzonBase): class AmzonePStatus(AmamzonBase):
mark_name = "状态查询" mark_name = "状态查询"
def SwitchPage(self):
"""
切换至 管理所有库存页面
1、等待 //navigation-favorites-bar[@class="hydrated"] 出现
"""
navigation = self.tab.ele('xpath://navigation-favorites-bar[@class="hydrated"]')
navigation.wait.displayed(raise_err=False)
page_btn = navigation.sr('xpath://internal-fav-bar-links[@data-internal="navigation"]').sr(
'xpath://a[@data-page-id="ezdpc-gui-inventory-mons"]')
page_btn.wait.displayed(raise_err=False)
page_btn.click(timeout=5)
self.tab.wait.doc_loaded()
# 等待搜索框出现
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group')
search_region.wait.displayed(raise_err=False, timeout=60)
def search_asin(self, asin): def search_asin(self, asin):
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group') search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group')
search_region.wait.displayed(raise_err=False) search_region.wait.displayed(raise_err=False)
@@ -81,14 +56,8 @@ class AmzonePStatus(AmamzonBase):
self.browser.close_tabs(close_tab) self.browser.close_tabs(close_tab)
def run_page_action(self, appoint_asin: str = None): def run_page_action(self, appoint_asin: str = None):
""" self.log(f"==================={self.mark_name}=======================")
self.log("开始执行...")
miniprice_info :
{
"B0DSJGJBWV" : "19.81" # asin : 最低价
}
"""
print(f"{self.mark_name}】,开始执行")
num = 0 num = 0
retry_num = 0 retry_num = 0
already_asin = set() already_asin = set()
@@ -174,29 +143,8 @@ class AmzonePStatus(AmamzonBase):
self.tab.wait.doc_loaded(raise_err=False, timeout=120) self.tab.wait.doc_loaded(raise_err=False, timeout=120)
class StatusTask: class StatusTask(TaskBase):
country_info = { task_name = "状态查询-TASK"
"DE": "德国",
"FR": "法国",
"ES": "西班牙",
"IT": "意大利",
"UK": "英国"
}
def __init__(self, user_info: dict = None):
"""初始化审批任务处理器
Args:
user_info: 用户信息字典,包含 company, username, password
"""
self.user_info = user_info or {}
self.running = True
def log(self, message: str, level: str = "INFO"):
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [PriceTask] [{level}] {message}")
def process_task(self, task_data: dict): def process_task(self, task_data: dict):
"""处理审批任务主入口 """处理审批任务主入口
@@ -218,21 +166,12 @@ class StatusTask:
# 用于测试 # 用于测试
limit = data.get("limit", None) limit = data.get("limit", None)
if not task_id: if not task_id or not queryAsins:
self.log("任务ID为空跳过", "WARNING") self.log("任务ID / queryAsins列表 为空,跳过", "WARNING")
return
# if not items:
# self.log("店铺列表为空,跳过", "WARNING")
# return
if not queryAsins:
self.log("queryAsins 列表为空,跳过", "WARNING")
return return
self.log(f"开始处理审批任务 {task_id},共 1 个店铺,{len(queryAsins)} 个国家") self.log(f"开始处理审批任务 {task_id},共 1 个店铺,{len(queryAsins)} 个国家")
from config import runing_task
runing_task[task_id] = { runing_task[task_id] = {
"status": "running", "status": "running",
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
@@ -265,7 +204,6 @@ class StatusTask:
if task_id in runing_task: if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1 runing_task[task_id]["processed_shops"] += 1
except Exception as e: except Exception as e:
import traceback
self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR") self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR") self.log(traceback.format_exc(), "ERROR")
@@ -279,96 +217,12 @@ class StatusTask:
self.log(f"任务 {task_id} 处理完成!") self.log(f"任务 {task_id} 处理完成!")
except Exception as e: except Exception as e:
import traceback
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR") self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id: if task_id:
from config import runing_task
if task_id in runing_task: if task_id in runing_task:
runing_task[task_id]["status"] = "failed" runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e) runing_task[task_id]["error"] = str(e)
def open_shop(self, max_retries, company_name, shop_name, iskill=False):
error_info = ""
driver = None
for retry in range(max_retries):
try:
self.log(f"尝试打开店铺 {shop_name} (第 {retry + 1}/{max_retries} 次)")
if iskill:
self.log("重试前先杀掉浏览器进程...")
kill_process("v6")
kill_process("v5")
time.sleep(2)
# 组装用户信息并创建驱动
user_info = {
**self.user_info,
"company": company_name
}
driver = AmzonePStatus(user_info)
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name}")
else:
self.log(f"打开店铺失败: {browser}", "WARNING")
driver = None
continue
# 判断是否需要登录
need_login = driver.need_login()
print("【是否需要登录】:", need_login)
if need_login:
self.log(f"店铺 {shop_name} 需要登录,正在登录...")
# 获取店铺凭证
response = get_shop_info(shop_name)
print("【获取店铺凭证返回】:", response.text)
shop_data = response.json()
if not shop_data:
mes = f"获取店铺凭证失败,响应数据: {shop_data.get('message', '未知错误')}"
self.log(mes, "ERROR")
show_notification(mes, "ERROR")
continue
password = shop_data["data"]["password"]
login_success = driver.login(password)
if login_success:
self.log(f"店铺 {shop_name} 登录成功,正在重新打开店铺...")
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name} 登录后")
break
else:
self.log(f"登录后打开店铺失败: {browser}", "WARNING")
driver = None
else:
self.log(f"店铺 {shop_name} 登录失败", "WARNING")
driver = None
else:
break
except Exception as e:
import traceback
self.log(f"打开店铺异常: {traceback.format_exc()}", "INFO")
driver = None
error_info = str(e)
time.sleep(10)
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(3)
# 检查是否成功打开
if not driver or not browser or browser == "店铺不存在":
error_msg = f"店铺 {shop_name} 打开失败,已重试 {max_retries} 次,跳过该店铺,{error_info}"
self.log(error_msg, "ERROR")
# 从执行列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
return driver
return driver
def process_shop(self, shop_item: dict, country_codes: list, task_id: int, def process_shop(self, shop_item: dict, country_codes: list, task_id: int,
user_id=None, stage_index=None, final_stage: bool = True, limit: str = None): user_id=None, stage_index=None, final_stage: bool = True, limit: str = None):
"""处理单个店铺 """处理单个店铺
@@ -382,7 +236,6 @@ class StatusTask:
shop_name = shop_item.get("shopName", "未知店铺") shop_name = shop_item.get("shopName", "未知店铺")
company_name = shop_item.get("companyName", "") company_name = shop_item.get("companyName", "")
if not company_name: if not company_name:
self.log(f"店铺 {shop_name} 的公司名称为空,跳过", "WARNING") self.log(f"店铺 {shop_name} 的公司名称为空,跳过", "WARNING")
return return
@@ -403,7 +256,7 @@ class StatusTask:
country_code = value.get("country") country_code = value.get("country")
all_asin = value.get("asins") all_asin = value.get("asins")
# 打开店铺 # 打开店铺
driver = self.open_shop(max_retries=max_retries, company_name=company_name, driver = self.open_shop(cls=AmzonePStatus,max_retries=max_retries, company_name=company_name,
shop_name=shop_name, iskill=iskill) shop_name=shop_name, iskill=iskill)
if driver is None: if driver is None:
self.log(f"任务 {task_id} 启动店铺失败,结束任务", "ERROR") self.log(f"任务 {task_id} 启动店铺失败,结束任务", "ERROR")
@@ -419,7 +272,6 @@ class StatusTask:
try: try:
self.process_country(driver, country_code, task_id, shop_name,all_asin=all_asin ) self.process_country(driver, country_code, task_id, shop_name,all_asin=all_asin )
except Exception as e: except Exception as e:
import traceback
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR") self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR") self.log(traceback.format_exc(), "ERROR")
if "与页面的连接已断开" in str(e): if "与页面的连接已断开" in str(e):
@@ -450,8 +302,7 @@ class StatusTask:
self.log(f"店铺 {shop_name} 已从执行列表中移除") self.log(f"店铺 {shop_name} 已从执行列表中移除")
def process_country(self, driver: AmzonePStatus, country_code: str, task_id: int, shop_name: str, def process_country(self, driver: AmzonePStatus, country_code: str, task_id: int, shop_name: str,
all_asin:list, all_asin:list,limit: str = None):
limit: str = None):
"""处理单个国家的审批任务 """处理单个国家的审批任务
Args: Args:
@@ -505,7 +356,6 @@ class StatusTask:
self.log(f"切换到国家 {country_name} 失败", "WARNING") self.log(f"切换到国家 {country_name} 失败", "WARNING")
except Exception as e: except Exception as e:
import traceback
self.log(f"切换国家 {country_name} 异常: {str(e)}", "ERROR") self.log(f"切换国家 {country_name} 异常: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR") self.log(traceback.format_exc(), "ERROR")
@@ -527,7 +377,6 @@ class StatusTask:
driver.SwitchPage() driver.SwitchPage()
self.log(f"已切换到库存管理页面") self.log(f"已切换到库存管理页面")
except Exception as e: except Exception as e:
import traceback
self.log(f"切换页面失败: {str(e)}", "ERROR") self.log(f"切换页面失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR") self.log(traceback.format_exc(), "ERROR")
@@ -677,6 +526,7 @@ class StatusTask:
raise RuntimeError("已达到最大重试次数,结果回传最终失败") raise RuntimeError("已达到最大重试次数,结果回传最终失败")
if __name__ == '__main__': if __name__ == '__main__':
# 使用示例 # 使用示例
user_info = { user_info = {

189
app/amazon/chrome_base.py Normal file
View File

@@ -0,0 +1,189 @@
import json
import sys
import os
import io
import time
import traceback
from datetime import datetime
from DrissionPage import Chromium, ChromiumOptions
from collections import defaultdict
from config import base_dir
from amazon.tool import get_shop_info,show_notification
class ChromeAmzoneBase:
mark_name = "谷歌浏览器基类"
country_info = {
"英国": {
"url": "https://www.amazon.co.uk/dp/B0CJ8SNXXV",
"zip_code": "SW1A 1AA",
"mark": "SW1A 1"
},
"德国": {
"url": "https://www.amazon.de/dp/B0CC8CW9G2?th=1",
"zip_code": "10115"
},
"法国": {
"url": "https://www.amazon.fr/dp/B0FRG1MJ8H?th=1",
"zip_code": "75001"
},
"西班牙": {
"url": "https://www.amazon.es/dp/B08ZXVNYNN",
"zip_code": "28001"
},
"意大利": {
"url": "https://www.amazon.it/dp/B0D1P17T2Q",
"zip_code": "20121"
}
}
def __init__(self):
"""
杀死当前谷歌浏览器进程,并使用 drissionpage 启动谷歌浏览器,使用系统安装的浏览器默认用户文件夹
"""
print("正在关闭现有的chromium浏览器进程...")
os.system('taskkill /f /t /im chrome.exe')
time.sleep(2)
print("正在启动chromium浏览器...")
# 配置浏览器选项
co = ChromiumOptions()
user_data_path = os.path.join(base_dir, "user_data", "chrome_data")
if not os.path.exists(user_data_path):
os.makedirs(user_data_path, exist_ok=True)
co.set_user_data_path(user_data_path)
co.set_local_port(port=19897)
self.browser = Chromium(co)
self.tab = self.browser.latest_tab
print("Chrome浏览器启动成功")
def log(self, message: str, level: str = "INFO"):
"""日志输出
Args:
message: 日志消息
level: 日志级别
"""
try:
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [{self.mark_name}] [{level}] {message}")
except Exception as e:
print(f"输出出错,{e}")
def close_init_popup(self):
"""
关闭所有的初始化弹窗
"""
self.tab.wait.doc_loaded(timeout=60, raise_err=True)
footbar = self.tab.eles('xpath://footer[@class="el-dialog__footer"]', timeout=5)
if len(footbar) > 0:
do_not_remind = footbar[0].eles('xpath:.//input[@class="el-checkbox__original"]')
if len(do_not_remind) > 0:
do_not_remind[0].check()
resume_immediately = footbar[0].eles('xpath:.//button')
if len(resume_immediately) > 0:
resume_immediately[0].click()
self.tab.wait.doc_loaded(timeout=60, raise_err=True)
accept_btn = self.tab.eles('xpath://input[@id="sp-cc-accept"]', timeout=5)
if len(accept_btn) > 0:
accept_btn[0].click()
def _set_zip_code(self, zip_code, mark=None):
"""
设置邮编
Args:
zip_code: 目标邮编
"""
try:
# 检查当前邮编
zip_display = self.tab.ele('xpath://div[@id="glow-ingress-block"]', timeout=10)
if zip_display:
current_text = zip_display.text
# print(f"当前地址信息: {current_text}")
# 先检查标识
if mark is not None and mark in current_text:
print(f"邮编检测到标识: {mark},无需修改")
return True
# 检查是否已经包含目标邮编
if zip_code in current_text:
print(f"邮编已经设置为: {zip_code},无需修改")
return True
# 需要设置邮编
print(f"正在设置邮编为: {zip_code}")
# 点击地址选择按钮
location_link = self.tab.ele('xpath://a[@id="nav-global-location-popover-link"]', timeout=10)
if not location_link:
print("找不到地址设置按钮")
return False
location_link.click()
time.sleep(1)
# 等待邮编输入框出现
zip_input = self.tab.ele('xpath://input[@id="GLUXZipUpdateInput"]', timeout=10)
if not zip_input:
print("找不到邮编输入框")
return False
# 输入邮编
zip_input.input(zip_code, clear=True)
time.sleep(0.5)
# 点击提交按钮
submit_btn = self.tab.ele('xpath://input[@aria-labelledby="GLUXZipUpdate-announce"]', timeout=10)
if not submit_btn:
print("找不到提交按钮")
return False
submit_btn.click()
continue_btn = self.tab.eles('xpath://div[@class="a-popover-footer"]//input[@id="GLUXConfirmClose"]',
timeout=10)
if len(continue_btn) > 0:
continue_btn[0].click()
# 等待提交按钮消失(表示请求已发送)
print("等待邮编更新...")
time.sleep(2)
# 等待页面加载完成
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
time.sleep(2)
# 验证邮编是否设置成功
zip_display_after = self.tab.ele('xpath://div[@id="glow-ingress-block"]', timeout=10)
if zip_display_after:
updated_text = zip_display_after.text
# print(f"更新后的地址信息: {updated_text}")
if zip_code in updated_text:
print(f"邮编设置成功: {zip_code}")
return True
else:
# print(f"邮编设置可能失败,当前显示: {updated_text}")
return False
return True
except Exception as e:
print(f"设置邮编时出错: {traceback.format_exc()}")
return False
def close(self):
"""关闭浏览器"""
try:
if self.browser:
self.browser.quit()
print("浏览器已关闭")
except Exception as e:
print(f"关闭浏览器时出错: {str(e)}")

File diff suppressed because it is too large Load Diff

View File

@@ -1,88 +1,23 @@
import json import json
import sys
import os import os
import io
# sys.stdout.reconfigure(encoding='utf-8')
import time import time
import re
import traceback import traceback
from datetime import datetime from datetime import datetime
from DrissionPage import Chromium, ChromiumOptions from DrissionPage import Chromium, ChromiumOptions
from collections import defaultdict from collections import defaultdict
import requests import requests
from amazon.del_brand import AmamzonBase, kill_process
from amazon.tool import show_notification,get_shop_info,remove_special_characters,split_currency_values from amazon.tool import show_notification,get_shop_info,remove_special_characters,split_currency_values
from amazon.amazon_base import TaskBase
from amazon.chrome_base import ChromeAmzoneBase
from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE
class ChromeAmzone: class ChromeAmzone(ChromeAmzoneBase):
mark_name = "亚马逊详情采集" mark_name = "亚马逊详情采集"
country_info = {
"英国": {
"url": "https://www.amazon.co.uk/dp/B0CJ8SNXXV",
"zip_code": "SW1A 1AA",
"mark": "SW1A 1"
},
"德国": {
"url": "https://www.amazon.de/dp/B0CC8CW9G2?th=1",
"zip_code": "10115"
},
"法国": {
"url": "https://www.amazon.fr/dp/B0FRG1MJ8H?th=1",
"zip_code": "75001"
},
"西班牙": {
"url": "https://www.amazon.es/dp/B08ZXVNYNN",
"zip_code": "28001"
},
"意大利": {
"url": "https://www.amazon.it/dp/B0D1P17T2Q",
"zip_code": "20121"
}
}
def __init__(self):
"""
杀死当前谷歌浏览器进程,并使用 drissionpage 启动谷歌浏览器,使用系统安装的浏览器默认用户文件夹
"""
# 杀死现有的Chrome进程
print("正在关闭现有的chromium浏览器进程...")
os.system('taskkill /f /t /im chrome.exe')
time.sleep(2)
print("正在启动chromium浏览器...")
# 配置浏览器选项
co = ChromiumOptions()
user_data_path = os.path.join(base_dir, "user_data", "chrome_data")
if not os.path.exists(user_data_path):
os.makedirs(user_data_path, exist_ok=True)
co.set_user_data_path(user_data_path)
co.set_local_port(port=19897)
self.browser = Chromium(co)
self.tab = self.browser.latest_tab
print("Chrome浏览器启动成功")
def close_init_popup(self):
"""
关闭所有的初始化弹窗
"""
self.tab.wait.doc_loaded(timeout=60, raise_err=True)
footbar = self.tab.eles('xpath://footer[@class="el-dialog__footer"]', timeout=5)
if len(footbar) > 0:
do_not_remind = footbar[0].eles('xpath:.//input[@class="el-checkbox__original"]')
if len(do_not_remind) > 0:
do_not_remind[0].check()
resume_immediately = footbar[0].eles('xpath:.//button')
if len(resume_immediately) > 0:
resume_immediately[0].click()
self.tab.wait.doc_loaded(timeout=60, raise_err=True)
accept_btn = self.tab.eles('xpath://input[@id="sp-cc-accept"]', timeout=5)
if len(accept_btn) > 0:
accept_btn[0].click()
def run(self, country, asin): def run(self, country, asin):
""" """
@@ -99,7 +34,7 @@ class ChromeAmzone:
# 验证国家是否支持 # 验证国家是否支持
if country not in self.country_info: if country not in self.country_info:
error_msg = f"不支持的国家: {country},支持的国家有: {list(self.country_info.keys())}" error_msg = f"不支持的国家: {country},支持的国家有: {list(self.country_info.keys())}"
print(error_msg) self.log(error_msg)
show_notification(error_msg, "error") show_notification(error_msg, "error")
return None return None
@@ -114,7 +49,7 @@ class ChromeAmzone:
domain = base_url.split("/dp/")[0] domain = base_url.split("/dp/")[0]
# 拼接新的URL # 拼接新的URL
product_url = f"{domain}/dp/{asin}" product_url = f"{domain}/dp/{asin}"
print(f"正在访问: {product_url}") self.log(f"正在访问: {product_url}")
# 打开链接 # 打开链接
self.tab.get(product_url) self.tab.get(product_url)
@@ -123,13 +58,13 @@ class ChromeAmzone:
self.close_init_popup() self.close_init_popup()
# 2. 切换国家/设置邮编 # 2. 切换国家/设置邮编
print(f"正在检查并设置邮编: {zip_code},标识: {mark}") self.log(f"正在检查并设置邮编: {zip_code},标识: {mark}")
self._set_zip_code(zip_code, mark) self._set_zip_code(zip_code, mark)
# self.tab.wait.doc_loaded(timeout=5, raise_err=False) # self.tab.wait.doc_loaded(timeout=5, raise_err=False)
# 3. 抓取数据 # 3. 抓取数据
print("正在抓取商品数据...") self.log("正在抓取商品数据...")
data = self._scrape_data() data = self._scrape_data()
# 添加基本信息 # 添加基本信息
@@ -138,100 +73,15 @@ class ChromeAmzone:
data['url'] = product_url data['url'] = product_url
data['timestamp'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S') data['timestamp'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f"数据抓取完成: {json.dumps(data)}") self.log(f"数据抓取完成: {json.dumps(data)}")
return data return data
except Exception as e: except Exception as e:
error_msg = f"运行出错: {traceback.format_exc()}" error_msg = f"运行出错: {traceback.format_exc()}"
print(error_msg) self.log(error_msg)
show_notification(f"采集失败: {str(e)}", "error") show_notification(f"采集失败: {str(e)}", "error")
return {} return {}
def _set_zip_code(self, zip_code, mark=None):
"""
设置邮编
Args:
zip_code: 目标邮编
"""
try:
# 检查当前邮编
zip_display = self.tab.ele('xpath://div[@id="glow-ingress-block"]', timeout=10)
if zip_display:
current_text = zip_display.text
# print(f"当前地址信息: {current_text}")
# 先检查标识
if mark is not None and mark in current_text:
print(f"邮编检测到标识: {mark},无需修改")
return True
# 检查是否已经包含目标邮编
if zip_code in current_text:
print(f"邮编已经设置为: {zip_code},无需修改")
return True
# 需要设置邮编
print(f"正在设置邮编为: {zip_code}")
# 点击地址选择按钮
location_link = self.tab.ele('xpath://a[@id="nav-global-location-popover-link"]', timeout=10)
if not location_link:
print("找不到地址设置按钮")
return False
location_link.click()
time.sleep(1)
# 等待邮编输入框出现
zip_input = self.tab.ele('xpath://input[@id="GLUXZipUpdateInput"]', timeout=10)
if not zip_input:
print("找不到邮编输入框")
return False
# 输入邮编
zip_input.input(zip_code, clear=True)
time.sleep(0.5)
# 点击提交按钮
submit_btn = self.tab.ele('xpath://input[@aria-labelledby="GLUXZipUpdate-announce"]', timeout=10)
if not submit_btn:
print("找不到提交按钮")
return False
submit_btn.click()
continue_btn = self.tab.eles('xpath://div[@class="a-popover-footer"]//input[@id="GLUXConfirmClose"]',
timeout=10)
if len(continue_btn) > 0:
continue_btn[0].click()
# 等待提交按钮消失(表示请求已发送)
print("等待邮编更新...")
time.sleep(2)
# 等待页面加载完成
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
time.sleep(2)
# 验证邮编是否设置成功
zip_display_after = self.tab.ele('xpath://div[@id="glow-ingress-block"]', timeout=10)
if zip_display_after:
updated_text = zip_display_after.text
# print(f"更新后的地址信息: {updated_text}")
if zip_code in updated_text:
print(f"邮编设置成功: {zip_code}")
return True
else:
# print(f"邮编设置可能失败,当前显示: {updated_text}")
return False
return True
except Exception as e:
print(f"设置邮编时出错: {traceback.format_exc()}")
return False
def _scrape_data(self): def _scrape_data(self):
""" """
抓取商品数据 抓取商品数据
@@ -256,39 +106,16 @@ class ChromeAmzone:
return data return data
except Exception as e: except Exception as e:
print(f"抓取数据时出错: {traceback.format_exc()}") self.log(f"抓取数据时出错: {traceback.format_exc()}")
return data return data
def close(self):
"""关闭浏览器"""
try:
if self.browser:
self.browser.quit()
print("浏览器已关闭")
except Exception as e:
print(f"关闭浏览器时出错: {str(e)}")
class SpiderTask: class SpiderTask(TaskBase):
mark_name = "亚马逊采集" task_name = "亚马逊采集"
def __init__(self, user_info: dict = None):
"""初始化审批任务处理器
Args:
user_info: 用户信息字典,包含 company, username, password
"""
self.user_info = user_info or {}
self.running = True
def log(self, message: str, level: str = "INFO"):
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [PriceTask] [{level}] {message}")
@staticmethod @staticmethod
def group_by_id_prefix(data): def group_by_id_prefix(data):
""" """
根据每条数据的 id 字段进行分组: 根据每条数据的 id 字段进行分组:
- 如果 id 为 "2_1",则取 "_" 前面的 "2" 作为分组 key - 如果 id 为 "2_1",则取 "_" 前面的 "2" 作为分组 key
@@ -304,35 +131,35 @@ class SpiderTask:
item_id = str(item.get("id", "")) item_id = str(item.get("id", ""))
group_key = item_id.split("_")[0] group_key = item_id.split("_")[0]
grouped[group_key].append(item) grouped[group_key].append(item)
return list(grouped.values()) return list(grouped.values())
@staticmethod @staticmethod
def normalize_groups(data): def normalize_groups(data):
groups = data.get("groups") groups = data.get("groups")
if isinstance(groups, list) and len(groups) > 0: if isinstance(groups, list) and len(groups) > 0:
return groups return groups
rows = data.get("rows") or data.get("items") or [] rows = data.get("rows") or data.get("items") or []
if not isinstance(rows, list) or len(rows) == 0: if not isinstance(rows, list) or len(rows) == 0:
return [] return []
normalized = [] normalized = []
for items in SpiderTask.group_by_id_prefix(rows): for items in SpiderTask.group_by_id_prefix(rows):
first = items[0] if items else {} first = items[0] if items else {}
item_id = str(first.get("id") or first.get("displayId") or "") item_id = str(first.get("id") or first.get("displayId") or "")
base_id = item_id.split("_")[0] if item_id else "" base_id = item_id.split("_")[0] if item_id else ""
normalized.append({ normalized.append({
"sourceFileKey": first.get("sourceFileKey", ""), "sourceFileKey": first.get("sourceFileKey", ""),
"sourceFilename": first.get("sourceFilename", ""), "sourceFilename": first.get("sourceFilename", ""),
"groupKey": first.get("groupKey") or base_id, "groupKey": first.get("groupKey") or base_id,
"baseId": first.get("baseId") or base_id, "baseId": first.get("baseId") or base_id,
"displayId": first.get("displayId") or item_id, "displayId": first.get("displayId") or item_id,
"items": items, "items": items,
}) })
return normalized return normalized
def process_task(self, task_data: dict): def process_task(self, task_data: dict):
"""处理审批任务主入口 """处理审批任务主入口
Args: Args:
@@ -341,7 +168,7 @@ class SpiderTask:
try: try:
data = task_data.get("data", {}) data = task_data.get("data", {})
task_id = data.get("taskId") task_id = data.get("taskId")
groups = self.normalize_groups(data) groups = self.normalize_groups(data)
# 用于测试 # 用于测试
limit = data.get("limit", None) limit = data.get("limit", None)
@@ -353,11 +180,10 @@ class SpiderTask:
self.log(f"开始处理爬取任务 {task_id}{len(groups)} 个任务") self.log(f"开始处理爬取任务 {task_id}{len(groups)} 个任务")
if not groups: if not groups:
self.log("appearance-patent groups/rows is empty, skip", "WARNING") self.log("appearance-patent groups/rows is empty, skip", "WARNING")
return return
from config import runing_task
runing_task[task_id] = { runing_task[task_id] = {
"status": "running", "status": "running",
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
@@ -383,12 +209,12 @@ class SpiderTask:
try: try:
# 数据整理 # 数据整理
# new_items = self.group_by_id_prefix(items) # new_items = self.group_by_id_prefix(items)
result = [] result = []
for gp_index,gp in enumerate(groups): for gp_index,gp in enumerate(groups):
items = gp.get("items", []) items = gp.get("items", [])
return_data = {} return_data = {}
for index,value in enumerate(items): for index,value in enumerate(items):
return_data = { return_data = {
'image_url': "", 'image_url': "",
@@ -396,18 +222,17 @@ class SpiderTask:
} }
asin = value.get("asin") asin = value.get("asin")
country = value.get("country") country = value.get("country")
return_data = None
for _ in range(max_retry): for _ in range(max_retry):
try: try:
return_data = chrome.run(country, asin) or {} return_data = chrome.run(country, asin) or {}
self.log(f"抓取结果->{return_data}") self.log(f"抓取结果->{return_data}")
break break
except Exception as e: except Exception as e:
if "与页面的连接已断开" in str(e): if "与页面的连接已断开" in str(e):
chrome = ChromeAmzone() chrome = ChromeAmzone()
if not isinstance(return_data, dict): if not isinstance(return_data, dict):
return_data = {} return_data = {}
if return_data.get("image_url"): if return_data.get("image_url"):
break break
group_item = [ group_item = [
@@ -457,7 +282,6 @@ class SpiderTask:
if task_id in runing_task: if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1 runing_task[task_id]["processed_shops"] += 1
except Exception as e: except Exception as e:
import traceback
self.log(f"处理店铺 {task_id} 失败: {str(e)}", "ERROR") self.log(f"处理店铺 {task_id} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR") self.log(traceback.format_exc(), "ERROR")
@@ -471,10 +295,8 @@ class SpiderTask:
self.log(f"任务 {task_id} 处理完成!") self.log(f"任务 {task_id} 处理完成!")
except Exception as e: except Exception as e:
import traceback
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR") self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id: if task_id:
from config import runing_task
if task_id in runing_task: if task_id in runing_task:
runing_task[task_id]["status"] = "failed" runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e) runing_task[task_id]["error"] = str(e)
@@ -488,7 +310,7 @@ class SpiderTask:
url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result" url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result"
payload ={ payload ={
"submissionId": f"{int(time.time())}", "submissionId": f"{task_id}",
"chunkIndex": chunkIndex, "chunkIndex": chunkIndex,
"chunkTotal": chunkTotal, "chunkTotal": chunkTotal,
"error": error, "error": error,
@@ -8413,4 +8235,4 @@ if __name__ == '__main__':
] ]
# spide.process_task(task_data) # spide.process_task(task_data)
res = SpiderTask.group_by_id_prefix(task_data) res = SpiderTask.group_by_id_prefix(task_data)
print(res) print(res)

File diff suppressed because it is too large Load Diff

View File

@@ -1,12 +1,11 @@
import time import traceback
import traceback import requests
import requests import threading
import threading from datetime import datetime
from datetime import datetime
from typing import Dict, Any, List from typing import Dict, Any, List
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
from config import JSON_TASK_QUEUE, runing_task, runing_shop, DELETE_BRAND_API_BASE, ZN_COMPANY, ZN_USERNAME, ZN_PASSWORD from config import JSON_TASK_QUEUE, runing_task, runing_shop, DELETE_BRAND_API_BASE, ZN_COMPANY, ZN_USERNAME, ZN_PASSWORD
from amazon.del_brand import AmazoneDriver, kill_process from amazon.del_brand import AmazoneDriver, DelbramdTask
from amazon.approve import ApproveTask from amazon.approve import ApproveTask
from amazon.match_action import MatchTak from amazon.match_action import MatchTak
from amazon.price_match import PriceTask from amazon.price_match import PriceTask
@@ -14,8 +13,7 @@ from amazon.asin_status import StatusTask
from amazon.patrol_delete import PatrolDeleteTask from amazon.patrol_delete import PatrolDeleteTask
from amazon.detail_spider import SpiderTask from amazon.detail_spider import SpiderTask
from amazon.amazon_base import kill_process
from amazon.tool import get_shop_info,show_notification
class TaskMonitor: class TaskMonitor:
@@ -32,10 +30,10 @@ class TaskMonitor:
self.chunk_index = 1 # 当前处理的分块索引 self.chunk_index = 1 # 当前处理的分块索引
self.max_workers = 5 # 线程池最大线程数 self.max_workers = 5 # 线程池最大线程数
self.executor = None # 线程池执行器 self.executor = None # 线程池执行器
self.serial_task_locks = { self.serial_task_locks = {
"product-risk-resolve-run": threading.Lock(), "product-risk-resolve-run": threading.Lock(),
} }
# 在提交新任务前杀掉旧进程(确保环境干净) # 在提交新任务前杀掉旧进程(确保环境干净)
kill_process("v6") kill_process("v6")
@@ -60,6 +58,7 @@ class TaskMonitor:
futures = [] # 保存所有提交的任务Future对象 futures = [] # 保存所有提交的任务Future对象
task_type_info = { task_type_info = {
"delete-brand-run" : "删除品牌",
"product-risk-resolve-run" : "产品风险审批", "product-risk-resolve-run" : "产品风险审批",
"shop-match-run" : "匹配价格", "shop-match-run" : "匹配价格",
"price-track-run" : "跟价", "price-track-run" : "跟价",
@@ -70,20 +69,13 @@ class TaskMonitor:
try: try:
while self.running: while self.running:
try: try:
# 使用较长超时时间等待任务,减少空等待异常
# 超时后继续循环检查self.running状态避免卡死 # 超时后继续循环检查self.running状态避免卡死
task_data = JSON_TASK_QUEUE.get(block=True, timeout=30) task_data = JSON_TASK_QUEUE.get(block=True, timeout=30)
# 检查任务类型 # 检查任务类型
task_type = task_data.get("type", "") task_type = task_data.get("type", "")
if task_type == "delete-brand-run": if task_type in task_type_info:
# 提交删除品牌任务到线程池
self.log(f"接收到【删除品牌】任务,提交到线程池处理...")
future = self.executor.submit(self._process_task_wrapper, task_data)
futures.append(future)
elif task_type in task_type_info:
# 提交产品风险审批任务到线程池 # 提交产品风险审批任务到线程池
self.log(f"接收到任务,提交到线程池处理...") self.log(f"接收到任务,提交到线程池处理...")
future = self.executor.submit(self._process_approve_task_wrapper, task_data, task_type_info[task_type]) future = self.executor.submit(self._process_approve_task_wrapper, task_data, task_type_info[task_type])
@@ -116,23 +108,23 @@ class TaskMonitor:
task_data: 任务数据 task_data: 任务数据
""" """
try: try:
self.log(f"线程 {id(task_data)} 开始处理删除品牌任务...") self.log(f"线程 {id(task_data)} 开始处理任务...")
self.process_task(task_data) self.process_task(task_data)
self.log(f"线程 {id(task_data)} 删除品牌任务处理完成") self.log(f"线程 {id(task_data)} 任务处理完成")
except Exception as e: except Exception as e:
self.log(f"线程 {id(task_data)} 删除品牌任务处理异常: {traceback.format_exc()}", "ERROR") self.log(f"线程 {id(task_data)} 删除品牌任务处理异常: {traceback.format_exc()}", "ERROR")
def _process_approve_task_wrapper(self, task_data: Dict[str, Any],TASK_TYPE:str): def _process_approve_task_wrapper(self, task_data: Dict[str, Any],TASK_TYPE:str):
task_type = task_data.get("type", "") task_type = task_data.get("type", "")
task_lock = self.serial_task_locks.get(task_type) task_lock = self.serial_task_locks.get(task_type)
if task_lock is not None: if task_lock is not None:
self.log(f"线程 {id(task_data)} 等待 {task_type} 串行锁...") self.log(f"线程 {id(task_data)} 等待 {task_type} 串行锁...")
with task_lock: with task_lock:
self.log(f"线程 {id(task_data)} 获取 {task_type} 串行锁,开始处理...") self.log(f"线程 {id(task_data)} 获取 {task_type} 串行锁,开始处理...")
return self._process_approve_task_unlocked(task_data, TASK_TYPE) return self._process_approve_task_unlocked(task_data, TASK_TYPE)
return self._process_approve_task_unlocked(task_data, TASK_TYPE) return self._process_approve_task_unlocked(task_data, TASK_TYPE)
def _process_approve_task_unlocked(self, task_data: Dict[str, Any],TASK_TYPE:str): def _process_approve_task_unlocked(self, task_data: Dict[str, Any],TASK_TYPE:str):
"""审批任务处理包装器(用于线程池调用) """审批任务处理包装器(用于线程池调用)
Args: Args:
@@ -140,6 +132,7 @@ class TaskMonitor:
""" """
try: try:
TASK_INFO = { TASK_INFO = {
"删除品牌": DelbramdTask ,
"产品风险审批" : ApproveTask, "产品风险审批" : ApproveTask,
"匹配价格" : MatchTak, "匹配价格" : MatchTak,
"跟价" : PriceTask, "跟价" : PriceTask,
@@ -152,540 +145,10 @@ class TaskMonitor:
TASK_CLS = TASK_INFO[TASK_TYPE] # 根据任务类型选择处理类默认为ApproveTask TASK_CLS = TASK_INFO[TASK_TYPE] # 根据任务类型选择处理类默认为ApproveTask
approve_task = TASK_CLS(user_info=self.user_info) approve_task = TASK_CLS(user_info=self.user_info)
approve_task.process_task(task_data) approve_task.process_task(task_data)
self.log(f"线程 {id(task_data)} 产品风险审批任务处理完成") self.log(f"线程 {id(task_data)} {TASK_TYPE} 任务处理完成")
except Exception as e: except Exception as e:
self.log(f"线程 {id(task_data)} 产品风险审批任务处理异常: {traceback.format_exc()}", "ERROR") self.log(f"线程 {id(task_data)} {TASK_TYPE} 任务处理异常: {traceback.format_exc()}", "ERROR")
def process_task(self, task_data: Dict[str, Any]):
"""处理单个任务
Args:
task_data: 任务数据
"""
try:
data = task_data.get("data", {})
task_id = data.get("taskId")
items = data.get("items", [])
if not task_id:
self.log("任务ID为空跳过", "WARNING")
return
# 初始化任务状态
self.update_task_status(
task_id,
status="running",
start_time=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
total_shops=len(items),
processed_shops=0,
total_asins=sum(shop.get("totalRows", 0) for shop in items),
processed_asins=0,
success_count=0,
failed_count=0,
stop_requested=False # 暂停请求标志
)
self.log(f"开始处理任务 {task_id},共 {len(items)} 个店铺")
# 遍历处理每个店铺
for idx, shop_data in enumerate(items, 1):
shop_name = shop_data.get("shopName", "未知店铺")
self.log(f"[{idx}/{len(items)}] 开始处理店铺: {shop_name}")
try:
self.process_shop(shop_data, task_id)
# 更新已处理店铺数
if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1
except Exception as e:
self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 检查任务最终状态
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.update_task_status(task_id, status="stopped")
self.log(f"任务 {task_id} 已被暂停!")
else:
self.update_task_status(task_id, status="completed")
self.log(f"任务 {task_id} 处理完成!")
except Exception as e:
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id:
self.update_task_status(task_id, status="failed", error=str(e))
def process_shop(self, shop_data: Dict[str, Any], task_id: int):
"""处理单个店铺(包含重试逻辑)
Args:
shop_data: 店铺数据
task_id: 任务ID
"""
shop_name = shop_data.get("shopName", "未知店铺")
result_id = shop_data.get("resultId")
countries = shop_data.get("countries", [])
company_name = shop_data.get("companyName", "未知公司")
if not countries:
self.log(f"店铺 {shop_name} 没有国家数据,跳过", "WARNING")
return
# 更新当前处理的店铺
self.update_task_status(task_id, current_shop=shop_name)
# 将店铺添加到正在执行中的店铺列表
start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
runing_shop[shop_name] = start_time
self.log(f"账号:{company_name},店铺 {shop_name} 已添加到执行列表,开始时间: {start_time}")
# 店铺打开重试最多3次
driver = None
browser = None
max_retries = 3
for retry in range(max_retries):
try:
self.log(f"尝试打开店铺 {shop_name} (第 {retry + 1}/{max_retries} 次)")
# 如果不是第一次尝试,先杀进程
# if retry > 0:
# self.log("重试前先杀掉浏览器进程...")
# kill_process("v6")
# kill_process("v5")
# time.sleep(2)
# 创建驱动并打开店铺
user_info = {
**self.user_info,
"company": company_name
}
driver = AmazoneDriver(user_info)
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name}")
break
else:
self.log(f"打开店铺失败: {browser}", "WARNING")
driver = None
# 判断是否需要登录
driver.tab.wait.doc_loaded(timeout=30, raise_err=False) # 等待页面加载,避免过早判断登录状态
need_login = driver.need_login()
print("【是否需要登录】:",need_login)
if need_login:
self.log(f"店铺 {shop_name} 需要登录,正在登录...")
# 获取店铺凭证
response = get_shop_info(shop_name)
print("【获取店铺凭证返回】:",response.text)
shop_data = response.json()
if not shop_data:
mes = f"获取店铺凭证失败,响应数据: {shop_data.get('message', '未知错误')}"
self.log(mes, "ERROR")
show_notification(mes, "ERROR")
continue
password = shop_data["data"]["password"]
login_success = driver.login(password)
if login_success:
self.log(f"店铺 {shop_name} 登录成功,正在重新打开店铺...")
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name} 登录后")
break
else:
self.log(f"登录后打开店铺失败: {browser}", "WARNING")
driver = None
else:
self.log(f"店铺 {shop_name} 登录失败", "WARNING")
driver = None
except Exception as e:
self.log(f"打开店铺异常: {str(e)}", "ERROR")
driver = None
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(3)
# 检查是否成功打开
if not driver or not browser or browser == "店铺不存在":
self.log(f"店铺 {shop_name} 打开失败,已重试 {max_retries} 次,跳过该店铺", "ERROR")
return
try:
# 处理每个国家
chunk_index =1
for country_data in countries:
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理国家", "WARNING")
break # 跳出循环进入finally关闭店铺
try:
chunk_index = self.process_country(driver, country_data, task_id, result_id, shop_data,chunk_index)
except Exception as e:
country_name = country_data.get("country", "未知")
self.log(f"处理国家 {country_name} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
finally:
# 关闭店铺
try:
if driver:
self.log(f"关闭店铺 {shop_name}")
driver.close_store()
time.sleep(2)
except Exception as e:
self.log(f"关闭店铺失败: {str(e)}", "WARNING")
# 从正在执行中的店铺列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
self.log(f"店铺 {shop_name} 已从执行列表中移除")
def process_country(self, driver: AmazoneDriver, country_data: Dict[str, Any],
task_id: int, result_id: int, shop_data: Dict[str, Any],chunk_index:int):
"""处理单个国家的所有ASIN
Args:
driver: 亚马逊驱动实例
country_data: 国家数据
task_id: 任务ID
result_id: 结果ID
shop_data: 店铺数据
"""
country = country_data.get("country", "未知")
items = country_data.get("items", [])
self.log(f"开始处理国家: {country},共 {len(items)} 个ASIN")
self.update_task_status(task_id, current_country=country)
# 切换国家最多重试3次
max_retries = 3
switch_success = False
for retry in range(max_retries):
try:
self.log(f"尝试切换到国家 {country} (第 {retry + 1}/{max_retries} 次)")
# 如果不是第一次尝试,先刷新页面
if retry > 0:
self.log("重试前刷新页面...")
try:
driver.tab.refresh()
time.sleep(3)
except Exception as e:
self.log(f"刷新页面失败: {str(e)}", "WARNING")
switch_success = driver.SwitchingCountries(country)
if switch_success:
self.log(f"成功切换到国家 {country}")
break
else:
self.log(f"切换到国家 {country} 失败", "WARNING")
except Exception as e:
self.log(f"切换国家 {country} 异常: {str(e)}", "ERROR")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
# 如果切换失败回传该国家所有ASIN为失败状态
if not switch_success:
self.log(f"切换到国家 {country} 失败,已重试 {max_retries}将所有ASIN标记为失败", "ERROR")
chunk_index = self._report_all_asins_failed(country, items, task_id, shop_data,chunk_index)
return chunk_index
# 切换到库存管理页面
try:
driver.SwitchPage()
self.log(f"已切换到库存管理页面")
except Exception as e:
self.log(f"切换页面失败: {str(e)}", "ERROR")
# 切换页面失败也回传所有ASIN为失败
chunk_index = self._report_all_asins_failed(country, items, task_id, shop_data,chunk_index)
return chunk_index
# 处理每个ASIN
file_key = shop_data.get("fileKey", "")
source_filename = shop_data.get("sourceFilename", "")
total_rows = shop_data.get("totalRows", 0)
for idx, asin_item in enumerate(items, 1):
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理", "WARNING")
return # 返回到上层,会触发店铺关闭
try:
self.log(f"[{idx}/{len(items)}] 处理ASIN: {asin_item.get('asin', '')}")
self.process_asin(driver, asin_item, country, task_id, result_id,
file_key, source_filename, total_rows,chunk_index)
except Exception as e:
asin = asin_item.get("asin", "未知")
self.log(f"处理ASIN {asin} 失败: {str(e)}", "ERROR")
# 继续处理下一个ASIN
chunk_index += 1
return chunk_index
def process_asin(self, driver: AmazoneDriver, asin_item: Dict[str, Any],
country: str, task_id: int, result_id: int, file_key: str,
source_filename: str, total_rows: int,chunk_index:int):
"""处理单个ASIN并回传结果
Args:
driver: 亚马逊驱动实例
asin_item: ASIN数据项
country: 国家名称
task_id: 任务ID
result_id: 结果ID
file_key: 文件KEY
source_filename: 源文件名
total_rows: 总行数
chunk_index: 当前处理索引
"""
asin = asin_item.get("asin", "")
row_index = asin_item.get("rowIndex", 0)
# 更新当前处理的ASIN
self.update_task_status(task_id, current_asin=asin)
status = "失败"
max_retries = 3 # 最多重试3次
for retry in range(max_retries):
try:
self.log(f"处理ASIN {asin} (第 {retry + 1}/{max_retries} 次)")
# 如果不是第一次尝试,先刷新页面
if retry > 0:
self.log("重试前刷新页面...")
try:
driver.tab.refresh()
time.sleep(3)
except Exception as e:
self.log(f"刷新页面失败: {str(e)}", "WARNING")
# 搜索ASIN
sku_ls = driver.search(asin=asin)
self.log(f"搜索到 {len(sku_ls)} 个SKU")
if len(sku_ls) == 0:
status = "查询不到"
self.log(f"ASIN {asin} 未找到商品", "WARNING")
break # 查询不到商品,无需重试
else:
# 删除所有找到的SKU如果任何一个失败则重新开始整个流程
success_count = 0
all_success = True # 标记是否所有SKU都删除成功
total_sku_count = len(sku_ls)
for sku in sku_ls:
try:
suc = driver.del_action(sku)
if suc:
success_count += 1
self.log(f"SKU 删除成功 ({success_count}/{total_sku_count})")
else:
self.log(f"SKU 删除失败", "WARNING")
all_success = False
break # 任何一个失败,退出循环,准备重试整个流程
except Exception as e:
self.log(f"删除SKU异常: {str(e)}", "ERROR")
all_success = False
break # 发生异常,退出循环,准备重试整个流程
# 判断删除结果
if all_success and success_count == total_sku_count:
status = "成功"
self.log(f"ASIN {asin} 所有SKU删除成功 ({success_count}/{total_sku_count})")
# 更新成功计数
if task_id in runing_task:
runing_task[task_id]["success_count"] += 1
break # 全部成功,跳出重试循环
else:
# 有失败的SKU
if retry < max_retries - 1:
self.log(f"有SKU删除失败准备重试整个流程... ({retry + 1}/{max_retries})")
time.sleep(2)
continue # 继续下一次重试
else:
# 所有重试都用完了
if success_count > 0:
status = "部分成功"
self.log(f"ASIN {asin} 部分SKU删除成功 ({success_count}/{total_sku_count})", "WARNING")
if task_id in runing_task:
runing_task[task_id]["success_count"] += 1
else:
status = "失败"
self.log(f"ASIN {asin} 所有SKU删除失败", "ERROR")
if task_id in runing_task:
runing_task[task_id]["failed_count"] += 1
except Exception as e:
status = "删除异常"
self.log(f"处理ASIN {asin} 异常: {str(e)}", "ERROR")
# 如果还有重试机会,继续重试
if retry < max_retries - 1:
self.log(f"发生异常,准备重试... ({retry + 1}/{max_retries})")
time.sleep(2)
else:
# 所有重试都失败了,更新失败计数
if task_id in runing_task:
runing_task[task_id]["failed_count"] += 1
# 更新已处理ASIN计数
if task_id in runing_task:
runing_task[task_id]["processed_asins"] += 1
# 回传结果到API
try:
payload = {
"submissionId": "",
"files": [{
"fileKey": file_key,
"sourceFilename": source_filename,
"chunkIndex": chunk_index,
"chunkTotal": total_rows,
"processedRows": chunk_index,
"totalRows": total_rows,
"currentCountry": country,
"currentAsin": asin,
"countries": [{
"country": country,
"items": [{
"asin": asin,
"status": status
}]
}]
}]
}
self.post_result(task_id, payload)
self.log(f"ASIN {asin} 结果已回传,状态: {status}")
except Exception as e:
self.log(f"回传结果失败: {str(e)}", "ERROR")
def _report_all_asins_failed(self, country: str, items: List[Dict[str, Any]],
task_id: int, shop_data: Dict[str, Any],chunk_index:int):
"""将国家下所有ASIN标记为失败并回传
Args:
country: 国家名称
items: ASIN列表
task_id: 任务ID
shop_data: 店铺数据
"""
self.log(f"开始回传国家 {country} 下的 {len(items)} 个ASIN失败状态")
file_key = shop_data.get("fileKey", "")
source_filename = shop_data.get("sourceFilename", "")
total_rows = shop_data.get("totalRows", 0)
for asin_item in items:
asin = asin_item.get("asin", "")
try:
# 更新失败计数
if task_id in runing_task:
runing_task[task_id]["failed_count"] += 1
runing_task[task_id]["processed_asins"] += 1
# 回传失败状态
payload = {
"submissionId": "",
"files": [{
"fileKey": file_key,
"sourceFilename": source_filename,
"chunkIndex": chunk_index,
"chunkTotal": total_rows,
"processedRows": chunk_index,
"totalRows": total_rows,
"currentCountry": country,
"currentAsin": asin,
"countries": [{
"country": country,
"items": [{
"asin": asin,
"status": "失败"
}]
}]
}]
}
self.post_result(task_id, payload)
chunk_index += 1
self.log(f"ASIN {asin} 失败状态已回传")
except Exception as e:
self.log(f"回传ASIN {asin} 失败状态时出错: {str(e)}", "ERROR")
return chunk_index
def post_result(self, task_id: int, payload: Dict[str, Any]):
"""回传结果到API带重试机制
Args:
task_id: 任务ID
payload: 结果数据
"""
url = f"{DELETE_BRAND_API_BASE}/api/delete-brand/tasks/{task_id}/result"
max_retries = 3 # 最多重试3次
for retry in range(max_retries):
try:
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False # 忽略SSL证书验证
)
print("【结果提交】:",payload)
print("【结果提交返回】:",response.text)
if response.status_code == 200:
self.log(f"结果回传成功: {url}")
return # 成功后直接返回,不再重试
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
# 如果还有重试机会,继续重试
if retry < max_retries - 1:
self.log(f"准备重试... ({retry + 1}/{max_retries})")
time.sleep(2) # 等待2秒后重试
else:
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
# 如果还有重试机会,继续重试
if retry < max_retries - 1:
self.log(f"发生异常,准备重试... ({retry + 1}/{max_retries})")
time.sleep(2) # 等待2秒后重试
else:
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
def update_task_status(self, task_id: int, **kwargs):
"""更新任务状态
Args:
task_id: 任务ID
**kwargs: 要更新的字段
"""
if task_id not in runing_task:
runing_task[task_id] = {}
runing_task[task_id].update(kwargs)
def stop(self): def stop(self):
"""停止监控""" """停止监控"""
self.running = False self.running = False

View File

@@ -1,82 +1,24 @@
import time import time
import re import re
import traceback import traceback
from DrissionPage._pages.chromium_tab import ChromiumTab
from config import runing_task, runing_shop from config import runing_task, runing_shop,DELETE_BRAND_API_BASE
from datetime import datetime from datetime import datetime
from amazon.del_brand import AmamzonBase, kill_process from amazon.amazon_base import AmamzonBase, kill_process,TaskBase
from amazon.tool import show_notification
from amazon.tool import show_notification,get_shop_info import requests
class AmzoneMatchAction(AmamzonBase): class AmzoneMatchAction(AmamzonBase):
mark_name = "匹配价格" mark_name = "匹配价格"
def SwitchPage(self): def __init__(self, user_info: dict, socket_port: int = 19890):
""" super().__init__(user_info,socket_port)
切换至 管理所有库存页面 self.already_asin = set()
1、等待 //navigation-favorites-bar[@class="hydrated"] 出现
"""
navigation = self.tab.ele('xpath://navigation-favorites-bar[@class="hydrated"]')
navigation.wait.displayed(raise_err=False)
page_btn = navigation.sr('xpath://internal-fav-bar-links[@data-internal="navigation"]').sr(
'xpath://a[@data-page-id="ezdpc-gui-inventory-mons"]')
page_btn.wait.displayed(raise_err=False)
page_btn.click(timeout=5)
self.tab.wait.doc_loaded() def reset_already_asin(self):
# 等待搜索框出现 self.already_asin = set()
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group') self.log("already_asin 已重置")
search_region.wait.displayed(raise_err=False)
def search(self,filter_type="ApprovalRequired"):
sku_ls = []
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
drop_down = self.tab.ele('xpath://div[contains(@class,"VolusListingStatusDropDown-module__verticalContainer")]//kat-dropdown')
drop_down.wait.displayed(raise_err=False)
drop_down.wait.enabled(raise_err=False)
time.sleep(0.6)
drop_down.click()
# //kat-option[@value="SearchSuppressed"]
xp = f'xpath://kat-option[@value="{filter_type}"]'
print(f"{self.mark_name}】正在寻找筛选条件 {filter_type}xpath: {xp}")
approval_required = self.tab.eles(xp,timeout=5)
if len(approval_required) == 0:
print(f"{self.mark_name}】没有需要{filter_type}选项】没有需要{filter_type}的商品了")
return sku_ls # "没有需要审批的商品了"
else:
approval_required = approval_required[0]
approval_required.wait.displayed(raise_err=False)
approval_required.click()
approval_required_text = approval_required.text
print(f"{self.mark_name}】已选择筛选条件: {approval_required_text}")
count = re.findall(r'\d+', approval_required_text)
if count:
count = int(count[0])
print(f"{self.mark_name}】待审批的商品数量: {count}")
if count <= 0:
print(f"{self.mark_name}】没有需要{filter_type}的商品了")
return sku_ls #"没有需要审批的商品了"
for _ in range(3):
# 等待加载完成
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]")
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=3)
if len(sku_ls) > 0:
break
approval_required.click()
return sku_ls
def wait_loaded(self): def wait_loaded(self):
# 等待加载完成 # 等待加载完成
@@ -86,13 +28,13 @@ class AmzoneMatchAction(AmamzonBase):
timeout=3) timeout=3)
load_ele.wait.deleted(timeout=5, raise_err=False) load_ele.wait.deleted(timeout=5, raise_err=False)
except Exception as e: except Exception as e:
print(f"{self.mark_name}】等待加载中消失出错", e) print(f"{self.mark_name}】等待加载中消失出错", e)
def run_page_action(self): def run_page_action(self):
print(f"{self.mark_name}】开始执行") self.log(f"==================={self.mark_name}=======================")
self.log("开始执行...")
num = 0 num = 0
retry_num = 0 retry_num = 0
already_asin = set()
get_page_faild = 0 get_page_faild = 0
while retry_num < 3: # 最多重试3次 while retry_num < 3: # 最多重试3次
@@ -110,39 +52,50 @@ class AmzoneMatchAction(AmamzonBase):
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5) page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) > 0: if len(page_pamel) > 0:
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
# 测试
# if current_page > 1:
# break
# 总页数 # 总页数
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]') total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
if len(total_page) > 0: if len(total_page) > 0:
total_page = total_page[-1].text total_page = total_page[-1].text
else: else:
total_page = 0 total_page = 0
print(f"{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}") self.log(f"{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}")
get_page_faild = 0 get_page_faild = 0
except Exception as e: except Exception as e:
print(f"{self.mark_name}】获取页码失败", e) self.log(f"{self.mark_name}】获取页码失败", e)
get_page_faild+= 1 get_page_faild+= 1
if get_page_faild > 2: if get_page_faild > 2:
show_notification(f"{self.mark_name}】获取页码失败超3次停止任务") show_notification(f"{self.mark_name}】获取页码失败超3次停止任务")
break break
# 保存当前URL用于失败重试时访问
try:
self.url = self.tab.url
self.log(f"获取当前的链接:{self.url}")
except Exception as e:
self.url = None
self.log(f"获取当前的链接失败:{e}")
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10) sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
print(f"{self.mark_name}】获取到 {len(sku_ls)}") self.log(f"{self.mark_name}】获取到 {len(sku_ls)}")
# for sku_ele in sku_ls[0:2]: # for sku_ele in sku_ls[0:2]:
for sku_ele in sku_ls: for sku_ele in sku_ls:
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]') # solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]')
asin = sku_ele.ele('xpath:.//div[contains(@class,"JanusSplitBox-module__container")]//div[contains(@class,"JanusSplitBox-module__panel--") and contains(string(.),"ASIN")]/..//div[last()]',timeout=3).text asin = sku_ele.ele('xpath:.//div[contains(@class,"JanusSplitBox-module__container")]//div[contains(@class,"JanusSplitBox-module__panel--") and contains(string(.),"ASIN")]/..//div[last()]',timeout=3).text
print(f"{self.mark_name}】ASIN {asin} 找到....") self.log(f"{self.mark_name}】ASIN {asin} 找到....")
if asin in already_asin: if asin in self.already_asin:
print(f"{self.mark_name}{asin} 已经处理过了,跳过") self.log(f"{self.mark_name}{asin} 已经处理过了,跳过")
continue continue
price_match = sku_ele.eles('xpath:.//div[@data-test-id="FeaturedOfferPrice"]//a[text()="匹配"]',timeout=2) price_match = sku_ele.eles('xpath:.//div[@data-test-id="FeaturedOfferPrice"]//a[text()="匹配"]',timeout=2)
if len(price_match) == 0: if len(price_match) == 0:
print(f"{self.mark_name}{asin},没有推荐价格匹配按钮") self.log(f"{self.mark_name}{asin},没有推荐价格匹配按钮")
yield (asin,"无需处理") yield (asin,"无需处理")
already_asin.add(asin) self.already_asin.add(asin)
continue continue
try: try:
price_match[0].click() price_match[0].click()
@@ -154,14 +107,14 @@ class AmzoneMatchAction(AmamzonBase):
save_all_btn.wait.deleted(timeout=10, raise_err=False) save_all_btn.wait.deleted(timeout=10, raise_err=False)
yield (asin,"处理完成") yield (asin,"处理完成")
already_asin.add(asin) self.already_asin.add(asin)
except Exception as e: except Exception as e:
print(f"{asin} 处理失败", e) print(f"{asin} 处理失败", e)
yield (asin,"处理失败") yield (asin,"处理失败")
already_asin.add(asin) self.already_asin.add(asin)
continue continue
# 判断是否存在需要翻页的情况 # 判断是否存在需要翻页的情况
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5) page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) == 0: if len(page_pamel) == 0:
@@ -172,56 +125,21 @@ class AmzoneMatchAction(AmamzonBase):
break break
next_page_btn.click() next_page_btn.click()
num += 1 num += 1
print(f"{self.mark_name}】【程序计算】正在翻页,已翻 {num} 页...") self.log(f"{self.mark_name}】【程序计算】正在翻页,已翻 {num} 页...")
already_asin = set()
except Exception as e: except Exception as e:
print(f"{self.mark_name}】处理匹配操作异常", e) self.log(f"{self.mark_name}】处理匹配操作异常", e)
traceback.print_exc() traceback.print_exc()
retry_num += 1 retry_num += 1
self.tab.refresh() self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False,timeout=120) self.tab.wait.doc_loaded(raise_err=False,timeout=120)
class MatchTak:
country_info = {
"DE": "德国",
"FR": "法国",
"ES": "西班牙",
"IT": "意大利",
"UK": "英国"
}
def __init__(self, user_info: dict = None):
"""初始化审批任务处理器
Args:
user_info: 用户信息字典,包含 company, username, password
"""
self.user_info = user_info or {}
self.running = True
def log(self, message: str, level: str = "INFO"):
"""日志输出
Args:
message: 日志消息
level: 日志级别
"""
from datetime import datetime
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [MatchTak] [{level}] {message}")
class MatchTak(TaskBase):
task_name = "匹配价格-TASK"
def process_task(self, task_data: dict): def process_task(self, task_data: dict):
"""处理审批任务主入口 """处理审批任务主入口
Args: Args:
task_data: 任务数据 task_data: 任务数据
""" """
@@ -238,21 +156,12 @@ class MatchTak:
# 用于测试 # 用于测试
limit = data.get("limit",None) limit = data.get("limit",None)
if not task_id: if not task_id or not items or not country_codes:
self.log("任务ID为空跳过", "WARNING") self.log(f"任务ID为空{task_id}/店铺列表为空{items}/国家列表为空{country_codes},跳过", "WARNING")
return return
if not items:
self.log("店铺列表为空,跳过", "WARNING")
return
if not country_codes:
self.log("国家列表为空,跳过", "WARNING")
return
self.log(f"开始处理审批任务 {task_id},共 {len(items)} 个店铺,{len(country_codes)} 个国家") self.log(f"开始处理审批任务 {task_id},共 {len(items)} 个店铺,{len(country_codes)} 个国家")
from config import runing_task
runing_task[task_id] = { runing_task[task_id] = {
"status": "running", "status": "running",
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
@@ -287,7 +196,6 @@ class MatchTak:
if task_id in runing_task: if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1 runing_task[task_id]["processed_shops"] += 1
except Exception as e: except Exception as e:
import traceback
self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR") self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR") self.log(traceback.format_exc(), "ERROR")
@@ -301,98 +209,13 @@ class MatchTak:
self.log(f"任务 {task_id} 处理完成!") self.log(f"任务 {task_id} 处理完成!")
except Exception as e: except Exception as e:
import traceback
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR") self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id: if task_id:
from config import runing_task
if task_id in runing_task: if task_id in runing_task:
runing_task[task_id]["status"] = "failed" runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e) runing_task[task_id]["error"] = str(e)
def open_shop(self,max_retries,company_name,shop_name,iskill=False): def process_shop(self, shop_item: dict, country_codes: list, task_id: int, risk_listing_filter: str,
error_info = ""
driver = None
for retry in range(max_retries):
try:
self.log(f"尝试打开店铺 {shop_name} (第 {retry + 1}/{max_retries} 次)")
if iskill:
self.log("重试前先杀掉浏览器进程...")
kill_process("v6")
kill_process("v5")
time.sleep(2)
# 组装用户信息并创建驱动
user_info = {
**self.user_info,
"company": company_name
}
driver = AmzoneMatchAction(user_info)
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name}")
else:
self.log(f"打开店铺失败: {browser}", "WARNING")
driver = None
continue
# 判断是否需要登录
need_login = driver.need_login()
print("【是否需要登录】:", need_login)
if need_login:
self.log(f"店铺 {shop_name} 需要登录,正在登录...")
# 获取店铺凭证
response = get_shop_info(shop_name)
print("【获取店铺凭证返回】:", response.text)
shop_data = response.json()
if not shop_data:
mes = f"获取店铺凭证失败,响应数据: {shop_data.get('message', '未知错误')}"
self.log(mes, "ERROR")
show_notification(mes, "ERROR")
continue
password = shop_data["data"]["password"]
login_success = driver.login(password)
if login_success:
self.log(f"店铺 {shop_name} 登录成功,正在重新打开店铺...")
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name} 登录后")
break
else:
self.log(f"登录后打开店铺失败: {browser}", "WARNING")
driver = None
else:
self.log(f"店铺 {shop_name} 登录失败", "WARNING")
driver = None
else:
break
except Exception as e:
import traceback
self.log(f"打开店铺异常: {traceback.format_exc()}", "INFO")
driver = None
error_info = str(e)
time.sleep(10)
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(3)
# 检查是否成功打开
if not driver or not browser or browser == "店铺不存在":
error_msg = f"店铺 {shop_name} 打开失败,已重试 {max_retries} 次,跳过该店铺,{error_info}"
self.log(error_msg, "ERROR")
# 从执行列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
return driver
return driver
# def process_shop(self, shop_item: dict, country_codes: list, task_id: int, risk_listing_filter: str):
def process_shop(self, shop_item: dict, country_codes: list, task_id: int, risk_listing_filter: str,
user_id=None, stage_index=None, final_stage: bool = True,limit:str=None): user_id=None, stage_index=None, final_stage: bool = True,limit:str=None):
"""处理单个店铺 """处理单个店铺
@@ -408,7 +231,6 @@ class MatchTak:
if not company_name: if not company_name:
self.log(f"店铺 {shop_name} 的公司名称为空,跳过", "WARNING") self.log(f"店铺 {shop_name} 的公司名称为空,跳过", "WARNING")
return return
if task_id in runing_task: if task_id in runing_task:
runing_task[task_id]["current_shop"] = shop_name runing_task[task_id]["current_shop"] = shop_name
@@ -431,7 +253,7 @@ class MatchTak:
break break
# 打开店铺 # 打开店铺
driver = self.open_shop(max_retries=max_retries, company_name=company_name, driver = self.open_shop(cls=AmzoneMatchAction,max_retries=max_retries, company_name=company_name,
shop_name=shop_name, iskill=iskill) shop_name=shop_name, iskill=iskill)
if driver is None: if driver is None:
self.log(f"任务 {task_id} 启动店铺失败,结束任务", "ERROR") self.log(f"任务 {task_id} 启动店铺失败,结束任务", "ERROR")
@@ -439,18 +261,25 @@ class MatchTak:
self.post_result(task_id, shop_name, country_code, "", "", is_done=True) self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
return return
try: current_url = None
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter,limit) for _ in range(max_retries):
except Exception as e: try:
import traceback self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter,limit,
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR") target_url=current_url)
self.log(traceback.format_exc(), "ERROR") driver.reset_already_asin()
if "与页面的连接已断开" in str(e): driver.close_store()
iskill = True break
except Exception as e:
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
if "与页面的连接已断开" in str(e):
iskill = True
current_url = driver.url
# 更新已处理国家数 # 更新已处理国家数
if task_id in runing_task: if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1 runing_task[task_id]["processed_countries"] += 1
self.log(f"{task_id}任务处理完成") self.log(f"{task_id}任务处理完成")
# 最后回传,标记完成 # 最后回传,标记完成
try: try:
@@ -467,83 +296,11 @@ class MatchTak:
if shop_name in runing_shop: if shop_name in runing_shop:
del runing_shop[shop_name] del runing_shop[shop_name]
self.log(f"店铺 {shop_name} 已从执行列表中移除") self.log(f"店铺 {shop_name} 已从执行列表中移除")
def action_init(self,driver,country_name,shop_name,risk_listing_filter):
"""切换到指定国际 -> 页面 -》 筛选好"""
max_retries = 5
switch_success = False
switch_success_pg = False
search_success = False
sku_ls = []
for retry in range(max_retries):
try:
self.log(f"尝试切换到国家 {country_name} (第 {retry + 1}/{max_retries} 次)")
if retry > 2:
# 刷新不行就重新打开店铺
self.log("重试前重新打开店铺...")
try:
driver.close_store()
time.sleep(3)
driver.open_shop(shop_name)
except Exception as e:
self.log(f"关闭重新打开店铺: {str(e)}", "WARNING")
# 如果不是第一次尝试,先刷新页面
if retry > 0:
self.log("重试前刷新页面...")
try:
driver.tab.refresh()
time.sleep(3)
except Exception as e:
self.log(f"刷新页面失败: {str(e)}", "WARNING")
switch_success = driver.SwitchingCountries(country_name) def process_country(self, driver, country_code, task_id, shop_name, risk_listing_filter,limit=None,target_url=None):
if switch_success:
self.log(f"成功切换到国家 {country_name}")
# 切换到库存管理页面
driver.SwitchPage()
self.log(f"已切换到库存管理页面")
time.sleep(3)
switch_success_pg = True
sku_ls = driver.search(filter_type=risk_listing_filter)
search_success = True
self.log(f"国家 {country_name} 搜索出 {len(sku_ls)} 商品,开始处理...")
break
else:
self.log(f"切换到国家 {country_name} 失败", "WARNING")
except Exception as e:
import traceback
self.log(f"切换国家 {country_name} 异常: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
# 如果切换失败,直接返回
# if not switch_success or not switch_success_pg or not search_success:
# error_message = f"切换到国家({switch_success})/库存页面({switch_success_pg})/搜索筛选到指定选项({search_success}) {country_name} 失败,已重试 {max_retries} 次,跳过该国家"
# self.log(error_message, "ERROR")
# if task_id in runing_task:
# runing_task[task_id]["processed_countries"] += 1
return switch_success,switch_success_pg,search_success,sku_ls
def process_country(self, driver: AmzoneMatchAction, country_code: str, task_id: int, shop_name: str, risk_listing_filter: str,limit:str=None):
"""处理单个国家的审批任务 """处理单个国家的审批任务
Args:
driver: AmzoneApprove驱动实例
country_code: 国家代码(如 UK, DE, FR 等)
task_id: 任务ID
shop_name: 店铺名称
risk_listing_filter: 风险商品筛选条件
""" """
from config import runing_task
# 转换国家代码为中文名称 # 转换国家代码为中文名称
country_name = self.country_info.get(country_code, country_code) country_name = self.country_info.get(country_code, country_code)
info_mes = f"开始处理国家: {country_name} ({country_code})" info_mes = f"开始处理国家: {country_name} ({country_code})"
@@ -554,91 +311,7 @@ class MatchTak:
if task_id in runing_task: if task_id in runing_task:
runing_task[task_id]["current_country"] = country_name runing_task[task_id]["current_country"] = country_name
# 切换国家最多重试3次
# max_retries = 3
# switch_success = False
#
# for retry in range(max_retries):
# try:
# self.log(f"尝试切换到国家 {country_name} (第 {retry + 1}/{max_retries} 次)")
# if retry > 1:
# # 刷新不行就重新打开店铺
# self.log("重试前重新打开店铺...")
# try:
# driver.close_store()
# time.sleep(3)
# driver.open_shop(shop_name)
# except Exception as e:
# self.log(f"关闭重新打开店铺: {str(e)}", "WARNING")
#
# # 如果不是第一次尝试,先刷新页面
# if retry > 0:
# self.log("重试前刷新页面...")
# try:
# driver.tab.refresh()
# time.sleep(3)
# except Exception as e:
# self.log(f"刷新页面失败: {str(e)}", "WARNING")
#
# switch_success = driver.SwitchingCountries(country_name)
# if switch_success:
# self.log(f"成功切换到国家 {country_name}")
# break
# else:
# self.log(f"切换到国家 {country_name} 失败", "WARNING")
#
# except Exception as e:
# import traceback
# self.log(f"切换国家 {country_name} 异常: {str(e)}", "ERROR")
# self.log(traceback.format_exc(), "ERROR")
#
# # 如果还有重试机会,等待后继续
# if retry < max_retries - 1:
# time.sleep(2)
#
# # 如果切换失败,直接返回
# if not switch_success:
# error_message = f"切换到国家 {country_name} 失败,已重试 {max_retries} 次,跳过该国家"
# self.log(error_message, "ERROR")
# if task_id in runing_task:
# runing_task[task_id]["processed_countries"] += 1
# return
#
# # 切换到库存管理页面
# try:
# driver.SwitchPage()
# self.log(f"已切换到库存管理页面")
# except Exception as e:
# import traceback
# self.log(f"切换页面失败: {str(e)}", "ERROR")
# self.log(traceback.format_exc(), "ERROR")
# if task_id in runing_task:
# runing_task[task_id]["processed_countries"] += 1
# return
#
# # 搜索需要审批的商品最多重试3次
# sku_ls = []
# for retry in range(max_retries):
# try:
# self.log(f"尝试搜索匹配操作商品 (第 {retry + 1}/{max_retries} 次)")
# sku_ls = driver.search(filter_type=risk_listing_filter)
# break
# except Exception as e:
# self.log(f"搜索商品异常: {str(e)}", "ERROR")
# if retry < max_retries - 1:
# try:
# driver.tab.refresh()
# time.sleep(3)
# except Exception as refresh_error:
# self.log(f"刷新页面失败: {str(refresh_error)}", "WARNING")
#
# # 如果没有需要审批的商品,直接返回
# if len(sku_ls) == 0:
# self.log(f"国家 {country_name} 没有搜索出的商品")
# if task_id in runing_task:
# runing_task[task_id]["processed_countries"] += 1
# return
#
max_retries = 5 max_retries = 5
switch_success, switch_success_pg, search_success, sku_ls = self.action_init(driver, country_name, shop_name, switch_success, switch_success_pg, search_success, sku_ls = self.action_init(driver, country_name, shop_name,
risk_listing_filter) risk_listing_filter)
@@ -658,7 +331,13 @@ class MatchTak:
return return
self.log(f"国家 {country_name} 搜索出 {len(sku_ls)} 商品,开始处理...") self.log(f"国家 {country_name} 搜索出 {len(sku_ls)} 商品,开始处理...")
if target_url is not None: # 从失败的链接继续
self.log(f"开始访问:{target_url}")
driver.tab.get(url=target_url)
driver.tab.wait.doc_loaded(timeout=60, raise_err=False)
result = []
# 处理所有需要审批的商品通过yield获取结果 # 处理所有需要审批的商品通过yield获取结果
for asin, status in driver.run_page_action(): for asin, status in driver.run_page_action():
# 检查是否收到暂停请求 # 检查是否收到暂停请求
@@ -675,19 +354,22 @@ class MatchTak:
runing_task[task_id]["failed_count"] += 1 runing_task[task_id]["failed_count"] += 1
# 回传结果到API result.append({
try: "asin": asin,
self.post_result(task_id, shop_name, country_code, asin, status) "status": status,
except Exception as e: "done": False
self.log(f"回传结果失败: {str(e)}", "ERROR") })
if len(result) > 20:
self.post_result_batch(task_id, shop_name, country_code,result)
result = []
if len(result) > 0:
self.post_result_batch(task_id, shop_name, country_code, result)
self.log(f"国家 {country_name} 处理完成") self.log(f"国家 {country_name} 处理完成")
def post_stage_finished(self, task_id: int, user_id, stage_index): def post_stage_finished(self, task_id: int, user_id, stage_index):
import requests
from config import DELETE_BRAND_API_BASE
if user_id in (None, "", 0): if user_id in (None, "", 0):
raise ValueError("user_id is required for stage completion callback") raise ValueError("user_id is required for stage completion callback")
@@ -733,9 +415,7 @@ class MatchTak:
asin: ASIN asin: ASIN
status: 处理状态 status: 处理状态
""" """
import requests
from config import DELETE_BRAND_API_BASE
url = f"{DELETE_BRAND_API_BASE}/api/shop-match/tasks/{task_id}/result" url = f"{DELETE_BRAND_API_BASE}/api/shop-match/tasks/{task_id}/result"
payload = { payload = {
@@ -789,7 +469,66 @@ class MatchTak:
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR") self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
raise RuntimeError("已达到最大重试次数,结果回传最终失败") raise RuntimeError("已达到最大重试次数,结果回传最终失败")
def post_result_batch(self, task_id: int, shop_name: str, country_code: str, country_data:list,
is_done: bool = False):
"""回传处理结果到API
Args:
task_id: 任务ID
shop_name: 店铺名称
country_code: 国家代码
asin: ASIN
status: 处理状态
"""
url = f"{DELETE_BRAND_API_BASE}/api/shop-match/tasks/{task_id}/result"
payload = {
"shops": [
{
"error": "",
"countries": {
country_code: country_data
},
"shopName": shop_name
}
]
}
max_retries = 3
for retry in range(max_retries):
try:
print("================【匹配】=====================")
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
self.log(f"回传数据: {payload}")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False
)
self.log(f"回传结果: {response.text}")
data = response.json() if response.text else {}
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
self.log(f"结果回传成功")
return
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
print("=====================================")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
print("=====================================")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
raise RuntimeError("已达到最大重试次数,结果回传最终失败")
if __name__ == "__main__": if __name__ == "__main__":

File diff suppressed because one or more lines are too long

505
app/amazon/similar_asin.py Normal file
View File

@@ -0,0 +1,505 @@
import json
from pathlib import Path
import base64
import time
import re
import traceback
from datetime import datetime
from collections import defaultdict
import requests
from urllib.parse import quote
from curl_cffi import requests as requests_frp
from amazon.tool import show_notification,get_shop_info,remove_special_characters,split_currency_values
from amazon.amazon_base import TaskBase
from amazon.chrome_base import ChromeAmzoneBase
from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE
class ChromeAmzone(ChromeAmzoneBase):
mark_name = "亚马逊相似ASIN"
def wait_load_complete(self):
try:
for _ in range(3):
content_area = self.tab.eles('xpath://div[@class="ap-sbi-aside__main"]', timeout=20)
if len(content_area) > 0:
pass
except Exception as e:
self.log(f"等待加载完成失败,{e},等待5秒")
time.sleep(5)
def encode_custom(self,input_):
if not input_ or not isinstance(input_, str):
return ""
transformed = ""
# 第一个字符:字符编码 + 字符串长度
transformed += chr((ord(input_[0]) + len(input_)) & 0xFFFF)
# 后续字符:当前字符编码 + 前一个字符编码
for i in range(1, len(input_)):
current_code = ord(input_[i])
prev_code = ord(input_[i - 1])
transformed += chr((current_code + prev_code) & 0xFFFF)
# 对应 JavaScript 的 encodeURIComponent
encoded = quote(transformed, safe="-_.!~*'()")
# 对应 replace(/[!'()*]/g, ...)
for ch in ["!", "'", "(", ")", "*"]:
encoded = encoded.replace(ch, "%" + format(ord(ch), "x"))
return encoded
def image_to_base64(self,image_source: str) -> str:
"""
将图片链接URL 或本地文件路径)转换为 Base64 编码的字符串。
"""
if image_source.startswith(('http://', 'https://')):
response = requests.get(image_source, timeout=10)
response.raise_for_status() # 非 2xx 状态码将抛出异常
image_data = response.content
else:
path = Path(image_source)
if not path.is_file():
raise ValueError(f"本地文件不存在: {image_source}")
with open(path, 'rb') as f:
image_data = f.read()
base64_str = base64.b64encode(image_data).decode('utf-8')
return base64_str
def get_aliprice_data(self,page=1,title="",domain="",category="",imageBase64=""):
"""
"fishkeeper Quick Aquarium Siphon Pump Gravel Cleaner - 256GPH Adjustable Powerful Fish Tank Vacuum Gravel Cleaning Kit for Aquarium Water Changer, Sand Cleaner, Dirt Removal : Amazon.co.uk: Pet Supplies"
标题 :
"""
headers = {
"accept": "application/json, text/plain, */*",
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
"browser": "chrome",
"cache-control": "no-cache",
"channel": "chrome_offline",
"content-type": "application/json;charset=UTF-8",
"ext-id": "10100",
"ext_id": "10100",
"origin": "chrome-extension://ephkdklmdkaakeleplfpahjphaokcllh",
"platform": "1688",
"pragma": "no-cache",
"priority": "u=1, i",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "none",
"sec-fetch-storage-access": "active",
"version": "3.7.4",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
}
cookie = {
"e-info": "[{\"e-name\":\"1688\",\"adid\":\"100\",\"version\":\"3.7.4\",\"ext_id\":\"10100\"}]",
"language": "chinese",
"province_code": "Guangdong",
"_ct": "czg",
"_e_ct": "czg",
"is_reto": "1",
"crossborder": "1",
"agent": "0",
"first_view": "1",
"currency": "USD",
"is_coo": "1",
"plugin": "chrome",
"plugin_ext": "100,129",
"m-info": "[{\"platform\":\"1688\",\"version\":\"3.7.4\",\"browser\":\"chrome\",\"m\":\"uni\",\"t\":1777902599578}]"
}
url = "https://api.aliprice.com/index.php/chrome/items/imageAnalysis"
itemTitle = f"{title} : {domain}: {category}"
params = {"page": page, "size": 20, "website": "1688_lite2", "language": "zh-CN", "currency": "USD", "from": "",
"itemTitle": itemTitle, "domain": domain }
params['sign'] = self.encode_custom(json.dumps(params, ensure_ascii=False))
data = {
"imageBase64" : imageBase64
}
response = requests.post(url, headers=headers, params=params, json=data, impersonate="chrome101",
cookies=cookie)
response.encoding = "utf-8"
data = response.json()
return data
def _scrape_data(self):
"""
抓取商品数据
Returns:
dict: 抓取到的数据
"""
data = {
'image_url': "",
'title': "",
'category' : '',
'success' : True
}
try:
# 等待页面加载
# time.sleep(3)
title_ele = self.tab.ele('xpath://h1[@id="title"]',timeout=30)
title = title_ele.text
data["title"] = title
imge_ele = self.tab.ele('xpath://div[@id="imgTagWrapperId"]//img',timeout=20)
image_url = imge_ele.attr("src")
data["image_url"] = image_url
category = self.tab.eles('xpath://div[@id="wayfinding-breadcrumbs_feature_div"]//span[@class="a-list-item"]//a[@class="a-link-normal a-color-tertiary"]',timeout=20)
if len(category) > 0:
data["category"] = category[0].text
return data
except Exception as e:
print(f"抓取数据时出错: {traceback.format_exc()}")
data["success"] = False
return data
def run(self, country, asin,total_page=2):
"""
运行亚马逊详情采集任务
Args:
country: 国家名称(如:英国、德国、法国、西班牙、意大利)
asin: 亚马逊商品ASIN码
Returns:
dict: 包含采集到的数据
"""
try:
# 验证国家是否支持
if country not in self.country_info:
error_msg = f"不支持的国家: {country},支持的国家有: {list(self.country_info.keys())}"
print(error_msg)
show_notification(error_msg, "error")
return None
# 获取国家配置
country_config = self.country_info[country]
zip_code = country_config["zip_code"]
mark = country_config.get("mark")
# 1. 根据国家和ASIN拼接链接
base_url = country_config["url"]
# 提取域名部分
domain = base_url.split("/dp/")[0]
# 拼接新的URL
product_url = f"{domain}/dp/{asin}"
print(f"正在访问: {product_url}")
# 打开链接
self.tab.get(product_url)
time.sleep(3) # 等待页面初步加载
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
self.close_init_popup()
# 2. 切换国家/设置邮编
print(f"正在检查并设置邮编: {zip_code},标识: {mark}")
self._set_zip_code(zip_code, mark)
# self.tab.wait.doc_loaded(timeout=5, raise_err=False)
# 3. 抓取数据
print("正在抓取商品数据...")
data = self._scrape_data()
# 判断是否采集标题出错
new_size = "220,220"
pattern = r"\._(?:[A-Z]+)?(\d+)_\."
replacement = f"._{new_size}_.jpg"
# 使用正则替换
image_new_url = re.sub(pattern, lambda m: replacement, data["image_url"])
iamge_base64 = self.image_to_base64(image_new_url)
similar_data = []
# 4、请求获取插件的数据
for page_num in range(total_page):
resp_data = self.get_aliprice_data(
page=page_num,
title = data["title"],
domain=domain,
category=data["category"],
imageBase64 = iamge_base64
)
self.log(f"Aliprice 扩展数据获取:{resp_data}")
if len(resp_data.get("data",[])) > 0:
for i in resp_data.get("data"):
similar_data.append(i)
data["similar_data"] = similar_data
# 添加基本信息
data['country'] = country
data['asin'] = asin
data['url'] = product_url
data['timestamp'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f"数据抓取完成: {json.dumps(data)}")
return data
except Exception as e:
error_msg = f"运行出错: {traceback.format_exc()}"
print(error_msg)
show_notification(f"采集失败: {str(e)}", "error")
return {}
class SimilarAsinTask(TaskBase):
mark_name = "相似ASIN采集"
@staticmethod
def group_by_id_prefix(data):
"""
根据每条数据的 id 字段进行分组:
- 如果 id 为 "2_1",则取 "_" 前面的 "2" 作为分组 key
- 如果 id 为 "1",则分组 key 就是 "1"
- 相同 key 的数据归为同一组
:param data: 原始列表数据
:return: 二维列表,按 id 前缀分组
"""
grouped = defaultdict(list)
for item in data:
item_id = str(item.get("id", ""))
group_key = item_id.split("_")[0]
grouped[group_key].append(item)
return list(grouped.values())
@staticmethod
def normalize_groups(data):
groups = data.get("groups")
if isinstance(groups, list) and len(groups) > 0:
return groups
rows = data.get("rows") or data.get("items") or []
if not isinstance(rows, list) or len(rows) == 0:
return []
normalized = []
for items in SimilarAsinTask.group_by_id_prefix(rows):
first = items[0] if items else {}
item_id = str(first.get("id") or first.get("displayId") or "")
base_id = item_id.split("_")[0] if item_id else ""
normalized.append({
"sourceFileKey": first.get("sourceFileKey", ""),
"sourceFilename": first.get("sourceFilename", ""),
"groupKey": first.get("groupKey") or base_id,
"baseId": first.get("baseId") or base_id,
"displayId": first.get("displayId") or item_id,
"items": items,
})
return normalized
def process_task(self, task_data: dict):
"""处理审批任务主入口
Args:
task_data: 任务数据
"""
try:
data = task_data.get("data", {})
task_id = data.get("taskId")
groups = self.normalize_groups(data)
if not task_id:
self.log("任务ID为空跳过", "WARNING")
return
self.log(f"开始处理爬取任务 {task_id}{len(groups)} 个任务")
if not groups:
self.log("appearance-patent groups/rows is empty, skip", "WARNING")
return
runing_task[task_id] = {
"status": "running",
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"total_shops": 1,
"processed_shops": 0,
"total_countries": len(groups) ,
"processed_countries": 0,
"total_asins": 0,
"processed_asins": 0,
"success_count": 0,
"failed_count": 0,
"stop_requested": False
}
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理", "WARNING")
runing_task[task_id]["status"] = "stopped"
return
max_retry = 3
show_notification(f"开始爬取数据", "info")
chrome = ChromeAmzone()
try:
# 数据整理
# new_items = self.group_by_id_prefix(items)
result = []
for gp_index,gp in enumerate(groups):
items = gp.get("items", [])
return_data = {}
for index,value in enumerate(items):
return_data = {
'image_url': "",
'title': ""
}
asin = value.get("asin")
country = value.get("country")
return_data = None
for _ in range(max_retry):
try:
return_data = chrome.run(country, asin) or {}
self.log(f"抓取结果->{return_data}")
break
except Exception as e:
if "与页面的连接已断开" in str(e):
chrome = ChromeAmzone()
if not isinstance(return_data, dict):
return_data = {}
if return_data.get("image_url"):
break
group_item = [
{
"sourceFileKey": i.get("sourceFileKey"),
"sourceFilename": i.get("sourceFilename"),
"rowToken": i.get("rowToken"),
"groupKey": i.get("groupKey"),
"id": i.get("id"),
"asin": i.get("asin"),
"country": i.get("country"),
"url": return_data.get("image_url"),
"title": return_data.get("title"),
}
for i in items
]
# task_id: int, chunkIndex:int,chunkTotal: int, country_code: str, asin: str, status: dict,error:str="",
# item_data:dict={},
res = {
"sourceFileKey": gp.get("sourceFileKey"),
"sourceFilename": gp.get("sourceFilename"),
"groupKey": gp.get("groupKey"),
"baseId": gp.get("baseId"),
"displayId": gp.get("displayId"),
"items": group_item
}
print("================")
result.append(res)
print("================")
is_done = gp_index == len(groups)-1
if len(result) > 20 or is_done:
self.post_result(task_id=task_id,chunkIndex=gp_index+1,chunkTotal=len(groups),
asin=asin,item_data=result,is_done=is_done)
result = []
if len(result) > 0:
is_done = True
self.post_result(task_id=task_id, chunkIndex=len(groups), chunkTotal=len(groups),
asin=asin, item_data=result, is_done=is_done)
try:
chrome.close()
except Exception as e:
print("退出浏览器出错",e)
# 更新已处理店铺数
if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1
except Exception as e:
self.log(f"处理店铺 {task_id} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 更新任务状态
if task_id in runing_task:
if runing_task[task_id].get("stop_requested", False):
runing_task[task_id]["status"] = "stopped"
self.log(f"任务 {task_id} 已被暂停!")
else:
runing_task[task_id]["status"] = "completed"
self.log(f"任务 {task_id} 处理完成!")
except Exception as e:
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id:
if task_id in runing_task:
runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e)
def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
item_data:dict={},
is_done: bool = False):
"""回传处理结果到API
"""
url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result"
payload ={
"submissionId": f"{int(time.time())}",
"chunkIndex": chunkIndex,
"chunkTotal": chunkTotal,
"error": error,
"groups": item_data,
"done": is_done
}
max_retries = 3
for retry in range(max_retries):
try:
request_timeout = 300 if is_done else 30
print("================【详情采集】=====================")
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
self.log(f"回传数据: {payload}")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=request_timeout,
verify=False
)
self.log(f"回传结果: {response.text}")
data = response.json() if response.text else {}
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
self.log(f"结果回传成功: {asin} - {item_data}")
return
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
print("=====================================")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
print("=====================================")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
raise RuntimeError("已达到最大重试次数,结果回传最终失败")
if __name__ == '__main__':
spide = ChromeAmzone()
resp = spide.run(
country="英国", asin="B0F79LCB3X", total_page=2
)
print(resp)

View File

@@ -4,7 +4,9 @@
""" """
import os import os
import secrets import secrets
from datetime import timedelta import logging
from datetime import timedelta, datetime
from logging.handlers import RotatingFileHandler
from flask import Flask from flask import Flask
from flask_cors import CORS from flask_cors import CORS
@@ -16,6 +18,52 @@ from blueprints.admin import admin_bp
from blueprints.image import image_bp from blueprints.image import image_bp
from blueprints.brand import brand_bp from blueprints.brand import brand_bp
from blueprints.communication import communication_bp from blueprints.communication import communication_bp
from config import cache_path, debug
def setup_flask_logging(app):
"""配置Flask访问日志输出到独立的日志文件"""
api_cache_path = os.path.join(cache_path, "api_logs")
os.makedirs(api_cache_path, exist_ok=True)
# 只在非debug模式下配置文件日志
if not debug:
# 获取当前日期作为日志文件名的一部分
today = datetime.now().strftime("%Y_%m_%d")
log_file = os.path.join(api_cache_path, f'api_{today}.log')
# 创建文件处理器使用RotatingFileHandler避免日志文件过大
file_handler = RotatingFileHandler(
log_file,
maxBytes=10 * 1024 * 1024, # 10MB
backupCount=5,
encoding='utf-8'
)
# 设置日志格式
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'
)
file_handler.setFormatter(formatter)
file_handler.setLevel(logging.INFO)
# 配置Flask应用日志
app.logger.addHandler(file_handler)
app.logger.setLevel(logging.INFO)
# 配置Werkzeug日志HTTP访问日志
werkzeug_logger = logging.getLogger('werkzeug')
werkzeug_logger.addHandler(file_handler)
werkzeug_logger.setLevel(logging.INFO)
print(f"API 日志已配置,输出到: {log_file}")
else:
print("Debug模式下API 日志输出到控制台")
def create_app(): def create_app():
app = Flask(__name__, template_folder=BASE_DIR, static_folder=BASE_DIR) app = Flask(__name__, template_folder=BASE_DIR, static_folder=BASE_DIR)
frontend_origin = os.environ.get('FRONTEND_ORIGIN', '*') frontend_origin = os.environ.get('FRONTEND_ORIGIN', '*')
@@ -39,6 +87,9 @@ def create_app():
app.register_blueprint(image_bp) app.register_blueprint(image_bp)
app.register_blueprint(brand_bp) app.register_blueprint(brand_bp)
app.register_blueprint(communication_bp) app.register_blueprint(communication_bp)
# 配置Flask访问日志
setup_flask_logging(app)
return app return app

View File

@@ -1,6 +1,3 @@
"""
基于 pywebview 实现的跨平台 UI需先登录方可使用
"""
from config import cache_path,debug,version,JAVA_API_BASE,JSON_TASK_QUEUE,runing_task,runing_shop from config import cache_path,debug,version,JAVA_API_BASE,JSON_TASK_QUEUE,runing_task,runing_shop
import datetime import datetime
import json import json
@@ -21,7 +18,8 @@ import requests
import subprocess import subprocess
from amazon.del_brand import kill_process from amazon.del_brand import kill_process
from app import run_app
from generate_api import generate
with open("version.txt","w",encoding="utf-8") as file: with open("version.txt","w",encoding="utf-8") as file:
@@ -31,7 +29,6 @@ print(f"""
======================================================== ========================================================
版本: {version} 版本: {version}
======================================================== ========================================================
""") """)
@@ -43,7 +40,6 @@ PORT = 5123
def generate_images(params): def generate_images(params):
"""供前端调用的生成图片接口""" """供前端调用的生成图片接口"""
from generate_api import generate
return generate(params) return generate(params)
@@ -67,7 +63,8 @@ class WindowAPI:
finally: finally:
time.sleep(0.5) time.sleep(0.5)
os._exit(0) os._exit(0)
print("===============程序退出================")
# 先销毁窗口,让用户看到立即响应 # 先销毁窗口,让用户看到立即响应
try: try:
self._window.destroy() self._window.destroy()
@@ -368,7 +365,6 @@ class WindowAPI:
def start_flask(): def start_flask():
from app import run_app
run_app(host='127.0.0.1', port=PORT) run_app(host='127.0.0.1', port=PORT)
@@ -395,11 +391,11 @@ def on_window_closing():
# 给一点时间让清理完成,然后强制退出 # 给一点时间让清理完成,然后强制退出
time.sleep(0.5) time.sleep(0.5)
os._exit(0) os._exit(0)
print("=================程序退出=====================")
# 立即在后台线程执行清理,不阻塞窗口关闭 # 立即在后台线程执行清理,不阻塞窗口关闭
cleanup_thread = threading.Thread(target=cleanup_and_exit, daemon=True) cleanup_thread = threading.Thread(target=cleanup_and_exit, daemon=True)
cleanup_thread.start() cleanup_thread.start()
# 立即返回,让窗口快速关闭,用户不会感觉卡顿 # 立即返回,让窗口快速关闭,用户不会感觉卡顿
return True return True