Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin
This commit is contained in:
1
app/.env
1
app/.env
@@ -1,3 +1,4 @@
|
|||||||
|
base_url=http://8.136.19.173:15124
|
||||||
workflow_id=7608812635877900322
|
workflow_id=7608812635877900322
|
||||||
mysql_host=8.136.19.173
|
mysql_host=8.136.19.173
|
||||||
mysql_user=aiimage
|
mysql_user=aiimage
|
||||||
|
|||||||
@@ -352,6 +352,8 @@ class AmzoneApprove(AmamzonBase):
|
|||||||
print(f"没有获取到 ASIN 商品,重试{retry_num}/{max_retry_num}")
|
print(f"没有获取到 ASIN 商品,重试{retry_num}/{max_retry_num}")
|
||||||
self.tab.refresh()
|
self.tab.refresh()
|
||||||
self.tab.wait.doc_loaded(raise_err=False, timeout=120)
|
self.tab.wait.doc_loaded(raise_err=False, timeout=120)
|
||||||
|
if retry_num == max_retry_num:
|
||||||
|
raise RuntimeError(f"与页面的连接已断开,没有获取到 ASIN 商品,重试{retry_num}/{max_retry_num}")
|
||||||
continue
|
continue
|
||||||
# for sku_ele in sku_ls[0:2]:
|
# for sku_ele in sku_ls[0:2]:
|
||||||
for sku_ele in sku_ls:
|
for sku_ele in sku_ls:
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ class ChromeAmzoneBase:
|
|||||||
print("正在启动chromium浏览器...")
|
print("正在启动chromium浏览器...")
|
||||||
# 配置浏览器选项
|
# 配置浏览器选项
|
||||||
co = ChromiumOptions()
|
co = ChromiumOptions()
|
||||||
|
co.no_imgs(True)
|
||||||
user_data_path = os.path.join(base_dir, "user_data", "chrome_data")
|
user_data_path = os.path.join(base_dir, "user_data", "chrome_data")
|
||||||
if not os.path.exists(user_data_path):
|
if not os.path.exists(user_data_path):
|
||||||
os.makedirs(user_data_path, exist_ok=True)
|
os.makedirs(user_data_path, exist_ok=True)
|
||||||
@@ -80,6 +81,11 @@ class ChromeAmzoneBase:
|
|||||||
关闭所有的初始化弹窗
|
关闭所有的初始化弹窗
|
||||||
"""
|
"""
|
||||||
self.tab.wait.doc_loaded(timeout=60, raise_err=True)
|
self.tab.wait.doc_loaded(timeout=60, raise_err=True)
|
||||||
|
continue_shipping = self.tab.eles('xpath://button[@alt="Continue shopping"]',timeout=3)
|
||||||
|
if len(continue_shipping) > 0:
|
||||||
|
continue_shipping[0].click()
|
||||||
|
time.sleep(0.5)
|
||||||
|
self.tab.wait.doc_loaded(timeout=60, raise_err=True)
|
||||||
footbar = self.tab.eles('xpath://footer[@class="el-dialog__footer"]', timeout=5)
|
footbar = self.tab.eles('xpath://footer[@class="el-dialog__footer"]', timeout=5)
|
||||||
if len(footbar) > 0:
|
if len(footbar) > 0:
|
||||||
do_not_remind = footbar[0].eles('xpath:.//input[@class="el-checkbox__original"]')
|
do_not_remind = footbar[0].eles('xpath:.//input[@class="el-checkbox__original"]')
|
||||||
|
|||||||
@@ -113,11 +113,6 @@ class ChromeAmzone(ChromeAmzoneBase):
|
|||||||
|
|
||||||
class SpiderTask(TaskBase):
|
class SpiderTask(TaskBase):
|
||||||
task_name = "亚马逊采集"
|
task_name = "亚马逊采集"
|
||||||
mark_name = "亚马逊采集"
|
|
||||||
|
|
||||||
def __init__(self, user_info: dict = None):
|
|
||||||
super().__init__(user_info)
|
|
||||||
self.result_api_module = "appearance-patent"
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def group_by_id_prefix(data):
|
def group_by_id_prefix(data):
|
||||||
@@ -170,12 +165,10 @@ class SpiderTask(TaskBase):
|
|||||||
Args:
|
Args:
|
||||||
task_data: 任务数据
|
task_data: 任务数据
|
||||||
"""
|
"""
|
||||||
task_id = None
|
|
||||||
try:
|
try:
|
||||||
data = task_data.get("data", {})
|
data = task_data.get("data", {})
|
||||||
task_id = data.get("taskId")
|
task_id = data.get("taskId")
|
||||||
groups = self.normalize_groups(data)
|
groups = self.normalize_groups(data)
|
||||||
self.result_api_module = "similar-asin" if task_data.get("type") == "similar-asin-run" else "appearance-patent"
|
|
||||||
|
|
||||||
# 用于测试
|
# 用于测试
|
||||||
limit = data.get("limit", None)
|
limit = data.get("limit", None)
|
||||||
@@ -309,19 +302,19 @@ class SpiderTask(TaskBase):
|
|||||||
runing_task[task_id]["error"] = str(e)
|
runing_task[task_id]["error"] = str(e)
|
||||||
|
|
||||||
def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
|
def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
|
||||||
item_data=None,
|
item_data:dict={},
|
||||||
is_done: bool = False):
|
is_done: bool = False):
|
||||||
"""回传处理结果到API
|
"""回传处理结果到API
|
||||||
"""
|
"""
|
||||||
|
|
||||||
url = f"{DELETE_BRAND_API_BASE}/api/{self.result_api_module}/tasks/{task_id}/result"
|
url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result"
|
||||||
|
|
||||||
payload ={
|
payload ={
|
||||||
"submissionId": f"{task_id}",
|
"submissionId": f"{task_id}",
|
||||||
"chunkIndex": chunkIndex,
|
"chunkIndex": chunkIndex,
|
||||||
"chunkTotal": chunkTotal,
|
"chunkTotal": chunkTotal,
|
||||||
"error": error,
|
"error": error,
|
||||||
"groups": item_data or [],
|
"groups": item_data,
|
||||||
"done": is_done
|
"done": is_done
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -335,8 +328,8 @@ class SpiderTask(TaskBase):
|
|||||||
self.log(f"回传数据: {payload}")
|
self.log(f"回传数据: {payload}")
|
||||||
response = requests.post(
|
response = requests.post(
|
||||||
url,
|
url,
|
||||||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
json=payload,
|
||||||
headers={"Content-Type": "application/json; charset=utf-8"},
|
headers={"Content-Type": "application/json"},
|
||||||
timeout=request_timeout,
|
timeout=request_timeout,
|
||||||
verify=False
|
verify=False
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from amazon.price_match import PriceTask
|
|||||||
from amazon.asin_status import StatusTask
|
from amazon.asin_status import StatusTask
|
||||||
from amazon.patrol_delete import PatrolDeleteTask
|
from amazon.patrol_delete import PatrolDeleteTask
|
||||||
from amazon.detail_spider import SpiderTask
|
from amazon.detail_spider import SpiderTask
|
||||||
|
from amazon.similar_asin import SimilarAsinTask
|
||||||
from amazon.amazon_base import kill_process
|
from amazon.amazon_base import kill_process
|
||||||
|
|
||||||
|
|
||||||
@@ -65,7 +65,7 @@ class TaskMonitor:
|
|||||||
"query-asin-run" : "状态查询",
|
"query-asin-run" : "状态查询",
|
||||||
"patrol-delete-run" : "巡店删除",
|
"patrol-delete-run" : "巡店删除",
|
||||||
"appearance-patent-run" : "亚马逊采集",
|
"appearance-patent-run" : "亚马逊采集",
|
||||||
"similar-asin-run" : "亚马逊采集",
|
"similar-asin-run" : "相似ASIN"
|
||||||
}
|
}
|
||||||
try:
|
try:
|
||||||
while self.running:
|
while self.running:
|
||||||
@@ -140,6 +140,7 @@ class TaskMonitor:
|
|||||||
"状态查询" : StatusTask,
|
"状态查询" : StatusTask,
|
||||||
"巡店删除" : PatrolDeleteTask,
|
"巡店删除" : PatrolDeleteTask,
|
||||||
"亚马逊采集" : SpiderTask,
|
"亚马逊采集" : SpiderTask,
|
||||||
|
"相似ASIN" : SimilarAsinTask
|
||||||
}
|
}
|
||||||
self.log(f"线程 {id(task_data)} 开始处理产品风险审批任务...")
|
self.log(f"线程 {id(task_data)} 开始处理产品风险审批任务...")
|
||||||
# 创建ApproveTask实例并处理任务
|
# 创建ApproveTask实例并处理任务
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ class RepricingLogic:
|
|||||||
# 0.8 到 10.5 之间全部是在减1
|
# 0.8 到 10.5 之间全部是在减1
|
||||||
return rank1_price - (diff + 1.0)
|
return rank1_price - (diff + 1.0)
|
||||||
else:
|
else:
|
||||||
return None # 相差10.5以上,跳过
|
return rank1_price - 10.5 # 相差10.5以上,跳过
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def situation_2_own_buybox(my_price, rank2_price):
|
def situation_2_own_buybox(my_price, rank2_price):
|
||||||
@@ -582,6 +582,8 @@ class AmzonePriceMatch(AmamzonBase):
|
|||||||
self.log(f"没有获取到 ASIN 商品,重试{retry_num}/{max_retry_num}")
|
self.log(f"没有获取到 ASIN 商品,重试{retry_num}/{max_retry_num}")
|
||||||
self.tab.refresh()
|
self.tab.refresh()
|
||||||
self.tab.wait.doc_loaded(raise_err=False, timeout=120)
|
self.tab.wait.doc_loaded(raise_err=False, timeout=120)
|
||||||
|
if retry_num == max_retry_num:
|
||||||
|
raise RuntimeError(f"与页面的连接已断开,没有获取到 ASIN 商品,重试{retry_num}/{max_retry_num}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# for sku_ele in sku_ls[0:2]:
|
# for sku_ele in sku_ls[0:2]:
|
||||||
@@ -977,7 +979,7 @@ class PriceTask(TaskBase):
|
|||||||
country_name = self.country_info.get(country_code, country_code)
|
country_name = self.country_info.get(country_code, country_code)
|
||||||
info_mes = f"开始处理国家: {country_name} ({country_code}),需要跳过的asin {skip_asin}"
|
info_mes = f"开始处理国家: {country_name} ({country_code}),需要跳过的asin {skip_asin}"
|
||||||
self.log(info_mes)
|
self.log(info_mes)
|
||||||
show_notification(info_mes, "info")
|
show_notification(f"开始处理国家:{country_name}", "info")
|
||||||
|
|
||||||
# 更新当前处理的国家
|
# 更新当前处理的国家
|
||||||
if task_id in runing_task:
|
if task_id in runing_task:
|
||||||
@@ -1216,7 +1218,7 @@ class PriceTask(TaskBase):
|
|||||||
max_retries = 3
|
max_retries = 3
|
||||||
for retry in range(max_retries):
|
for retry in range(max_retries):
|
||||||
try:
|
try:
|
||||||
print("================【跟价】=====================")
|
self.log("================【跟价】=====================")
|
||||||
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
|
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
|
||||||
self.log(f"回传URL: {url}")
|
self.log(f"回传URL: {url}")
|
||||||
self.log(f"回传数据: {payload}")
|
self.log(f"回传数据: {payload}")
|
||||||
@@ -1234,11 +1236,11 @@ class PriceTask(TaskBase):
|
|||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
|
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
|
||||||
print("=====================================")
|
self.log("=====================================")
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.log(f"调用API异常: {str(e)}", "ERROR")
|
self.log(f"调用API异常: {str(e)}", "ERROR")
|
||||||
print("=====================================")
|
self.log("=====================================")
|
||||||
|
|
||||||
# 如果还有重试机会,等待后继续
|
# 如果还有重试机会,等待后继续
|
||||||
if retry < max_retries - 1:
|
if retry < max_retries - 1:
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
# encoding utf-8
|
||||||
|
|
||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import base64
|
import base64
|
||||||
@@ -20,6 +22,17 @@ from amazon.chrome_base import ChromeAmzoneBase
|
|||||||
|
|
||||||
from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE
|
from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE
|
||||||
|
|
||||||
|
try:
|
||||||
|
from config import proxy_url as CONFIG_PROXY_URL, proxy_mode as CONFIG_PROXY_MODE
|
||||||
|
except ImportError:
|
||||||
|
CONFIG_PROXY_URL = None
|
||||||
|
CONFIG_PROXY_MODE = 1
|
||||||
|
|
||||||
|
# Forbidden 后 3 分钟内统一使用代理:记录代理生效截止时间与当前代理
|
||||||
|
_FORBIDDEN_PROXY_UNTIL = 0.0
|
||||||
|
_FORBIDDEN_PROXY_DICT = None
|
||||||
|
_FORBIDDEN_PROXY_MINUTES = 0.5
|
||||||
|
|
||||||
|
|
||||||
class ChromeAmzone(ChromeAmzoneBase):
|
class ChromeAmzone(ChromeAmzoneBase):
|
||||||
mark_name = "亚马逊相似ASIN"
|
mark_name = "亚马逊相似ASIN"
|
||||||
@@ -81,6 +94,8 @@ class ChromeAmzone(ChromeAmzoneBase):
|
|||||||
标题 :
|
标题 :
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
global _FORBIDDEN_PROXY_UNTIL, _FORBIDDEN_PROXY_DICT
|
||||||
|
|
||||||
headers = {
|
headers = {
|
||||||
"accept": "application/json, text/plain, */*",
|
"accept": "application/json, text/plain, */*",
|
||||||
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
|
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||||
@@ -126,11 +141,93 @@ class ChromeAmzone(ChromeAmzoneBase):
|
|||||||
data = {
|
data = {
|
||||||
"imageBase64" : imageBase64
|
"imageBase64" : imageBase64
|
||||||
}
|
}
|
||||||
response = requests.post(url, headers=headers, params=params, json=data, impersonate="chrome101",
|
|
||||||
cookies=cookie)
|
proxies = None
|
||||||
|
|
||||||
|
# 若 3 分钟内曾出现 Forbidden,则直接使用当时保存的代理
|
||||||
|
now = time.time()
|
||||||
|
if now < _FORBIDDEN_PROXY_UNTIL and _FORBIDDEN_PROXY_DICT:
|
||||||
|
proxies = _FORBIDDEN_PROXY_DICT
|
||||||
|
print("处于 Forbidden 代理窗口内,直接使用代理", proxies)
|
||||||
|
|
||||||
|
# 发送第一次请求
|
||||||
|
try:
|
||||||
|
response = requests_frp.post(url, headers=headers, params=params, json=data, impersonate="chrome101",
|
||||||
|
cookies=cookie, proxies=proxies)
|
||||||
response.encoding = "utf-8"
|
response.encoding = "utf-8"
|
||||||
data = response.json()
|
|
||||||
return data
|
# 检查响应状态码和数据有效性
|
||||||
|
need_retry = False
|
||||||
|
if response.status_code != 200:
|
||||||
|
print(f"请求失败,状态码: {response.status_code}")
|
||||||
|
need_retry = True
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
result_data = response.json()
|
||||||
|
# 检查是否获取到有效数据
|
||||||
|
if not result_data or not isinstance(result_data, dict):
|
||||||
|
print("返回数据为空或格式不正确")
|
||||||
|
need_retry = True
|
||||||
|
elif "Forbidden" in response.text or "Too Many Requests" in response.text:
|
||||||
|
print("返回Forbidden或Too Many Requests")
|
||||||
|
need_retry = True
|
||||||
|
except Exception as e:
|
||||||
|
print(f"解析响应JSON失败: {e}")
|
||||||
|
need_retry = True
|
||||||
|
|
||||||
|
# 如果需要重试且配置了代理URL
|
||||||
|
if need_retry and CONFIG_PROXY_URL and not proxies:
|
||||||
|
try:
|
||||||
|
proxy_resp = requests.get(CONFIG_PROXY_URL, timeout=10)
|
||||||
|
print("代理请求结果->:", proxy_resp.text)
|
||||||
|
|
||||||
|
# 模式 2:账号密码代理,接口返回 JSON
|
||||||
|
if CONFIG_PROXY_MODE == 2:
|
||||||
|
resp_json = proxy_resp.json()
|
||||||
|
proxy_list = resp_json.get("data", {}).get("list") or []
|
||||||
|
first_item = proxy_list[0] if proxy_list else None
|
||||||
|
if first_item:
|
||||||
|
ip = first_item.get("ip")
|
||||||
|
port = first_item.get("port")
|
||||||
|
account = first_item.get("account")
|
||||||
|
password = first_item.get("password")
|
||||||
|
if ip and port and account and password:
|
||||||
|
auth_proxy = f"{account}:{password}@{ip}:{port}"
|
||||||
|
print("获取到账号密码代理:", auth_proxy)
|
||||||
|
proxies = {
|
||||||
|
"http": f"http://{auth_proxy}",
|
||||||
|
"https": f"http://{auth_proxy}",
|
||||||
|
}
|
||||||
|
# 默认模式 1:普通 IP:port 文本
|
||||||
|
if CONFIG_PROXY_MODE != 2:
|
||||||
|
proxy_ip = (proxy_resp.text or "").strip()
|
||||||
|
if proxy_ip:
|
||||||
|
proxies = {
|
||||||
|
"http": f"http://{proxy_ip}",
|
||||||
|
"https": f"http://{proxy_ip}",
|
||||||
|
}
|
||||||
|
|
||||||
|
if proxies:
|
||||||
|
print("使用代理重试请求")
|
||||||
|
response = requests_frp.post(url, headers=headers, params=params, json=data,
|
||||||
|
impersonate="chrome101", cookies=cookie, proxies=proxies)
|
||||||
|
response.encoding = "utf-8"
|
||||||
|
print("代理重试结果状态码:", response.status_code)
|
||||||
|
|
||||||
|
# 记录 3 分钟内都使用该代理
|
||||||
|
_FORBIDDEN_PROXY_UNTIL = now + _FORBIDDEN_PROXY_MINUTES * 60
|
||||||
|
_FORBIDDEN_PROXY_DICT = proxies
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print("获取代理或重试失败:", e)
|
||||||
|
|
||||||
|
# 返回最终结果
|
||||||
|
result_data = response.json()
|
||||||
|
return result_data
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"get_aliprice_data 请求异常: {e}")
|
||||||
|
return {}
|
||||||
|
|
||||||
def _scrape_data(self):
|
def _scrape_data(self):
|
||||||
"""
|
"""
|
||||||
@@ -439,6 +536,8 @@ class SimilarAsinTask(TaskBase):
|
|||||||
"country": i.get("country"),
|
"country": i.get("country"),
|
||||||
"url": return_data.get("image_url"),
|
"url": return_data.get("image_url"),
|
||||||
"title": return_data.get("title"),
|
"title": return_data.get("title"),
|
||||||
|
"done": False,
|
||||||
|
"urls" : [i.get("ori_picture") for i in return_data.get("similar_data")]
|
||||||
}
|
}
|
||||||
for i in items
|
for i in items
|
||||||
]
|
]
|
||||||
@@ -452,6 +551,7 @@ class SimilarAsinTask(TaskBase):
|
|||||||
"displayId": gp.get("displayId"),
|
"displayId": gp.get("displayId"),
|
||||||
"items": group_item
|
"items": group_item
|
||||||
}
|
}
|
||||||
|
|
||||||
print("================")
|
print("================")
|
||||||
result.append(res)
|
result.append(res)
|
||||||
print("================")
|
print("================")
|
||||||
@@ -503,7 +603,7 @@ class SimilarAsinTask(TaskBase):
|
|||||||
url = f"{DELETE_BRAND_API_BASE}/api/similar-asin/tasks/{task_id}/result"
|
url = f"{DELETE_BRAND_API_BASE}/api/similar-asin/tasks/{task_id}/result"
|
||||||
|
|
||||||
payload ={
|
payload ={
|
||||||
"submissionId": f"{int(time.time())}",
|
"submissionId": f"{task_id}",
|
||||||
"chunkIndex": chunkIndex,
|
"chunkIndex": chunkIndex,
|
||||||
"chunkTotal": chunkTotal,
|
"chunkTotal": chunkTotal,
|
||||||
"error": error,
|
"error": error,
|
||||||
@@ -521,8 +621,8 @@ class SimilarAsinTask(TaskBase):
|
|||||||
self.log(f"回传数据: {payload}")
|
self.log(f"回传数据: {payload}")
|
||||||
response = requests.post(
|
response = requests.post(
|
||||||
url,
|
url,
|
||||||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
json=payload,
|
||||||
headers={"Content-Type": "application/json; charset=utf-8"},
|
headers={"Content-Type": "application/json"},
|
||||||
timeout=request_timeout,
|
timeout=request_timeout,
|
||||||
verify=False
|
verify=False
|
||||||
)
|
)
|
||||||
@@ -553,6 +653,263 @@ if __name__ == '__main__':
|
|||||||
country="英国", asin="B0F79LCB3X", total_page=2
|
country="英国", asin="B0F79LCB3X", total_page=2
|
||||||
)
|
)
|
||||||
print(resp)
|
print(resp)
|
||||||
|
# task_data = {
|
||||||
|
# "type": "similar-asin-run",
|
||||||
|
# "ts": 1778038688824,
|
||||||
|
# "data": {
|
||||||
|
# "taskId": 6850,
|
||||||
|
# "api_key": "sk-EMcDFg36zCeWUtCbRzKUbrRZyNeC6M4KBhY6fAVcNP7GG4xI",
|
||||||
|
# "groups": [
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "groupKey": "similar-asin:6850",
|
||||||
|
# "baseId": "",
|
||||||
|
# "displayId": "1",
|
||||||
|
# "items": [
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::2",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::1@2",
|
||||||
|
# "id": "1",
|
||||||
|
# "asin": "B0D792ND9V",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "12.29",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::3",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||||
|
# "id": "2_1",
|
||||||
|
# "asin": "B0D14L34S7",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "14.88",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::4",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||||
|
# "id": "2_2",
|
||||||
|
# "asin": "B0D14N8FJY",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "15.36",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::5",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||||
|
# "id": "2_3",
|
||||||
|
# "asin": "B0D14NBL5D",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "14.87",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::6",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||||
|
# "id": "2_4",
|
||||||
|
# "asin": "B0D14M5X4J",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "14.41",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::7",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
|
||||||
|
# "id": "3_1",
|
||||||
|
# "asin": "B0CWQFZMGY",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "17.05",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::8",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
|
||||||
|
# "id": "3_2",
|
||||||
|
# "asin": "B0CX8D68L3",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "16.46",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::9",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::4@9",
|
||||||
|
# "id": "4_1",
|
||||||
|
# "asin": "B0CX87XXWJ",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "17.54",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::10",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::5@10",
|
||||||
|
# "id": "5_1",
|
||||||
|
# "asin": "B0CR3PMR8L",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "16.74",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# }
|
||||||
|
# ]
|
||||||
|
# }
|
||||||
|
# ],
|
||||||
|
# "rows": [
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::2",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::1@2",
|
||||||
|
# "id": "1",
|
||||||
|
# "asin": "B0D792ND9V",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "12.29",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::3",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||||
|
# "id": "2_1",
|
||||||
|
# "asin": "B0D14L34S7",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "14.88",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::4",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||||
|
# "id": "2_2",
|
||||||
|
# "asin": "B0D14N8FJY",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "15.36",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::5",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||||
|
# "id": "2_3",
|
||||||
|
# "asin": "B0D14NBL5D",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "14.87",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::6",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||||
|
# "id": "2_4",
|
||||||
|
# "asin": "B0D14M5X4J",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "14.41",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::7",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
|
||||||
|
# "id": "3_1",
|
||||||
|
# "asin": "B0CWQFZMGY",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "17.05",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::8",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
|
||||||
|
# "id": "3_2",
|
||||||
|
# "asin": "B0CX8D68L3",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "16.46",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::9",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::4@9",
|
||||||
|
# "id": "4_1",
|
||||||
|
# "asin": "B0CX87XXWJ",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "17.54",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||||
|
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||||
|
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::10",
|
||||||
|
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::5@10",
|
||||||
|
# "id": "5_1",
|
||||||
|
# "asin": "B0CR3PMR8L",
|
||||||
|
# "country": "英国",
|
||||||
|
# "price": "16.74",
|
||||||
|
# "url": "",
|
||||||
|
# "title": "",
|
||||||
|
# "target_urls": []
|
||||||
|
# }
|
||||||
|
# ]
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
# sim_asin = SimilarAsinTask({})
|
||||||
|
# sim_asin.process_task(task_data)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -124,7 +124,6 @@ def remove_special_characters(text: str) -> str:
|
|||||||
return cleaned
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def split_currency_values(currency_str: str) -> tuple[float, float]:
|
def split_currency_values(currency_str: str) -> tuple[float, float]:
|
||||||
"""
|
"""
|
||||||
将包含两个货币值的字符串拆分成两个浮点数。
|
将包含两个货币值的字符串拆分成两个浮点数。
|
||||||
|
|||||||
@@ -21,8 +21,7 @@ mysql_database = os.getenv("mysql_database","aiimage")
|
|||||||
proxy_url = os.getenv("proxy_url")
|
proxy_url = os.getenv("proxy_url")
|
||||||
proxy_mode = int(os.getenv("proxy_mode",1))
|
proxy_mode = int(os.getenv("proxy_mode",1))
|
||||||
|
|
||||||
_client_name = os.getenv("client_name", "").strip()
|
client_name=os.getenv("client_name") + ".exe"
|
||||||
client_name = f"{_client_name}.exe" if _client_name else "client.exe"
|
|
||||||
JAVA_API_BASE = os.getenv("java_api_base", "http://127.0.0.1:18080")
|
JAVA_API_BASE = os.getenv("java_api_base", "http://127.0.0.1:18080")
|
||||||
|
|
||||||
# 紫鸟浏览器配置
|
# 紫鸟浏览器配置
|
||||||
@@ -54,8 +53,8 @@ os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret
|
|||||||
os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6"
|
os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6"
|
||||||
|
|
||||||
|
|
||||||
debug = os.getenv("debug", "false").strip().lower() in ("1", "true", "yes", "on")
|
debug = True
|
||||||
version = "1.0.56"
|
version = "1.0.13"
|
||||||
APP_UPDATE_URL = f"{_base_url}/api/version/latest" # 检测更新接口地址,返回 { "file_url": "...", "version": "x.x.x" }
|
APP_UPDATE_URL = f"{_base_url}/api/version/latest" # 检测更新接口地址,返回 { "file_url": "...", "version": "x.x.x" }
|
||||||
os.environ['APP_VERSION'] = version
|
os.environ['APP_VERSION'] = version
|
||||||
os.environ['APP_UPDATE_URL'] = APP_UPDATE_URL
|
os.environ['APP_UPDATE_URL'] = APP_UPDATE_URL
|
||||||
@@ -64,6 +63,9 @@ os.environ['APP_UPDATE_URL'] = APP_UPDATE_URL
|
|||||||
# 队列
|
# 队列
|
||||||
JSON_TASK_QUEUE = Queue()
|
JSON_TASK_QUEUE = Queue()
|
||||||
|
|
||||||
|
|
||||||
|
base_dir = os.getcwd()
|
||||||
|
|
||||||
#删除品牌,正在执行中的任务
|
#删除品牌,正在执行中的任务
|
||||||
runing_task = {
|
runing_task = {
|
||||||
|
|
||||||
|
|||||||
82
app/requirements.txt
Normal file
82
app/requirements.txt
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
alibabacloud-oss-v2==1.2.4
|
||||||
|
altgraph==0.17.5
|
||||||
|
annotated-types==0.7.0
|
||||||
|
anyio==4.12.1
|
||||||
|
Authlib==1.6.8
|
||||||
|
blinker==1.9.0
|
||||||
|
bottle==0.13.4
|
||||||
|
certifi==2026.1.4
|
||||||
|
cffi==2.0.0
|
||||||
|
charset-normalizer==3.4.4
|
||||||
|
click==8.1.8
|
||||||
|
clr_loader==0.2.10
|
||||||
|
colorama==0.4.6
|
||||||
|
cozepy==0.20.0
|
||||||
|
crcmod-plus==2.3.1
|
||||||
|
cryptography==41.0.0
|
||||||
|
cssselect==1.3.0
|
||||||
|
DataRecorder==3.6.2
|
||||||
|
distro==1.9.0
|
||||||
|
DownloadKit==2.0.7
|
||||||
|
DrissionPage==4.1.1.2
|
||||||
|
et_xmlfile==2.0.0
|
||||||
|
exceptiongroup==1.3.1
|
||||||
|
filelock==3.19.1
|
||||||
|
Flask==3.1.3
|
||||||
|
flask-cors==6.0.2
|
||||||
|
h11==0.16.0
|
||||||
|
httpcore==1.0.9
|
||||||
|
httpx==0.28.1
|
||||||
|
idna==3.11
|
||||||
|
importlib_metadata==8.7.1
|
||||||
|
itsdangerous==2.2.0
|
||||||
|
Jinja2==3.1.6
|
||||||
|
lxml==6.0.2
|
||||||
|
MarkupSafe==3.0.3
|
||||||
|
MouseInfo==0.1.3
|
||||||
|
Nuitka==2.8.6
|
||||||
|
numpy==1.25.2
|
||||||
|
openpyxl==3.1.5
|
||||||
|
ordered-set==4.1.0
|
||||||
|
packaging==26.0
|
||||||
|
pandas==2.3.3
|
||||||
|
pefile==2024.8.26
|
||||||
|
pillow==11.3.0
|
||||||
|
proxy_tools==0.1.0
|
||||||
|
psutil==7.2.2
|
||||||
|
PyAutoGUI==0.9.54
|
||||||
|
pycparser==2.23
|
||||||
|
pycryptodome==3.23.0
|
||||||
|
pydantic==2.12.5
|
||||||
|
pydantic_core==2.41.5
|
||||||
|
PyGetWindow==0.0.9
|
||||||
|
pyinstaller==6.19.0
|
||||||
|
pyinstaller-hooks-contrib==2026.3
|
||||||
|
PyMsgBox==2.0.1
|
||||||
|
PyMySQL==1.1.2
|
||||||
|
pyperclip==1.11.0
|
||||||
|
PyQt5==5.15.11
|
||||||
|
PyQt5-Qt5==5.15.2
|
||||||
|
PyQt5_sip==12.17.1
|
||||||
|
PyRect==0.2.0
|
||||||
|
PyScreeze==1.0.1
|
||||||
|
python-dateutil==2.9.0.post0
|
||||||
|
python-dotenv==1.2.1
|
||||||
|
pythonnet==3.0.5
|
||||||
|
pytweening==1.2.0
|
||||||
|
pytz==2026.1.post1
|
||||||
|
pywebview==6.1
|
||||||
|
pywin32-ctypes==0.2.3
|
||||||
|
requests==2.32.5
|
||||||
|
requests-file==3.0.1
|
||||||
|
six==1.17.0
|
||||||
|
tldextract==5.3.0
|
||||||
|
typing-inspection==0.4.2
|
||||||
|
typing_extensions==4.15.0
|
||||||
|
tzdata==2025.3
|
||||||
|
urllib3==2.6.3
|
||||||
|
websocket-client==1.9.0
|
||||||
|
websockets==14.2
|
||||||
|
Werkzeug==3.1.6
|
||||||
|
zipp==3.23.0
|
||||||
|
zstandard==0.25.0
|
||||||
Reference in New Issue
Block a user