This commit is contained in:
super
2026-05-07 16:22:42 +08:00
513 changed files with 7379 additions and 151 deletions

View File

@@ -1,3 +1,5 @@
# encoding utf-8
import json
from pathlib import Path
import base64
@@ -20,6 +22,17 @@ from amazon.chrome_base import ChromeAmzoneBase
from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE
try:
from config import proxy_url as CONFIG_PROXY_URL, proxy_mode as CONFIG_PROXY_MODE
except ImportError:
CONFIG_PROXY_URL = None
CONFIG_PROXY_MODE = 1
# Forbidden 后 3 分钟内统一使用代理:记录代理生效截止时间与当前代理
_FORBIDDEN_PROXY_UNTIL = 0.0
_FORBIDDEN_PROXY_DICT = None
_FORBIDDEN_PROXY_MINUTES = 0.5
class ChromeAmzone(ChromeAmzoneBase):
mark_name = "亚马逊相似ASIN"
@@ -81,6 +94,8 @@ class ChromeAmzone(ChromeAmzoneBase):
标题 :
"""
global _FORBIDDEN_PROXY_UNTIL, _FORBIDDEN_PROXY_DICT
headers = {
"accept": "application/json, text/plain, */*",
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
@@ -126,11 +141,93 @@ class ChromeAmzone(ChromeAmzoneBase):
data = {
"imageBase64" : imageBase64
}
response = requests.post(url, headers=headers, params=params, json=data, impersonate="chrome101",
cookies=cookie)
response.encoding = "utf-8"
data = response.json()
return data
proxies = None
# 若 3 分钟内曾出现 Forbidden则直接使用当时保存的代理
now = time.time()
if now < _FORBIDDEN_PROXY_UNTIL and _FORBIDDEN_PROXY_DICT:
proxies = _FORBIDDEN_PROXY_DICT
print("处于 Forbidden 代理窗口内,直接使用代理", proxies)
# 发送第一次请求
try:
response = requests_frp.post(url, headers=headers, params=params, json=data, impersonate="chrome101",
cookies=cookie, proxies=proxies)
response.encoding = "utf-8"
# 检查响应状态码和数据有效性
need_retry = False
if response.status_code != 200:
print(f"请求失败,状态码: {response.status_code}")
need_retry = True
else:
try:
result_data = response.json()
# 检查是否获取到有效数据
if not result_data or not isinstance(result_data, dict):
print("返回数据为空或格式不正确")
need_retry = True
elif "Forbidden" in response.text or "Too Many Requests" in response.text:
print("返回Forbidden或Too Many Requests")
need_retry = True
except Exception as e:
print(f"解析响应JSON失败: {e}")
need_retry = True
# 如果需要重试且配置了代理URL
if need_retry and CONFIG_PROXY_URL and not proxies:
try:
proxy_resp = requests.get(CONFIG_PROXY_URL, timeout=10)
print("代理请求结果->", proxy_resp.text)
# 模式 2账号密码代理接口返回 JSON
if CONFIG_PROXY_MODE == 2:
resp_json = proxy_resp.json()
proxy_list = resp_json.get("data", {}).get("list") or []
first_item = proxy_list[0] if proxy_list else None
if first_item:
ip = first_item.get("ip")
port = first_item.get("port")
account = first_item.get("account")
password = first_item.get("password")
if ip and port and account and password:
auth_proxy = f"{account}:{password}@{ip}:{port}"
print("获取到账号密码代理:", auth_proxy)
proxies = {
"http": f"http://{auth_proxy}",
"https": f"http://{auth_proxy}",
}
# 默认模式 1普通 IP:port 文本
if CONFIG_PROXY_MODE != 2:
proxy_ip = (proxy_resp.text or "").strip()
if proxy_ip:
proxies = {
"http": f"http://{proxy_ip}",
"https": f"http://{proxy_ip}",
}
if proxies:
print("使用代理重试请求")
response = requests_frp.post(url, headers=headers, params=params, json=data,
impersonate="chrome101", cookies=cookie, proxies=proxies)
response.encoding = "utf-8"
print("代理重试结果状态码:", response.status_code)
# 记录 3 分钟内都使用该代理
_FORBIDDEN_PROXY_UNTIL = now + _FORBIDDEN_PROXY_MINUTES * 60
_FORBIDDEN_PROXY_DICT = proxies
except Exception as e:
print("获取代理或重试失败:", e)
# 返回最终结果
result_data = response.json()
return result_data
except Exception as e:
print(f"get_aliprice_data 请求异常: {e}")
return {}
def _scrape_data(self):
"""
@@ -276,8 +373,8 @@ class SimilarAsinTask(TaskBase):
return list(grouped.values())
@staticmethod
def normalize_groups(data):
@staticmethod
def normalize_groups(data):
groups = data.get("groups")
if isinstance(groups, list) and len(groups) > 0:
return groups
@@ -299,71 +396,71 @@ class SimilarAsinTask(TaskBase):
"displayId": first.get("displayId") or item_id,
"items": items,
})
return normalized
def fetch_parsed_payload(self, task_id, user_id=1):
url = f"{DELETE_BRAND_API_BASE}/api/similar-asin/tasks/{task_id}/parsed-payload"
response = requests.get(url, params={"user_id": user_id}, timeout=300, verify=False)
response.raise_for_status()
payload = response.json() if response.text else {}
if isinstance(payload, dict) and payload.get("success"):
return payload.get("data") or {}
raise RuntimeError(f"获取货源查询解析载荷失败: {payload}")
return normalized
@staticmethod
def _count_payload_rows(data):
rows = data.get("rows") or data.get("items") or []
if isinstance(rows, list) and rows:
return len(rows)
groups = data.get("groups") or []
if not isinstance(groups, list):
return 0
total = 0
for group in groups:
if isinstance(group, dict) and isinstance(group.get("items"), list):
total += len(group.get("items"))
return total
def _merge_parsed_payload(self, data, parsed_payload):
payload_rows = parsed_payload.get("allItems") or parsed_payload.get("items") or []
merged = {
**data,
"groups": parsed_payload.get("groups") or [],
"rows": payload_rows,
}
if parsed_payload.get("aiPrompt"):
merged["prompt"] = parsed_payload.get("aiPrompt")
if parsed_payload.get("apiKey") and not merged.get("api_key"):
merged["api_key"] = parsed_payload.get("apiKey")
return merged
def process_task(self, task_data: dict):
def fetch_parsed_payload(self, task_id, user_id=1):
url = f"{DELETE_BRAND_API_BASE}/api/similar-asin/tasks/{task_id}/parsed-payload"
response = requests.get(url, params={"user_id": user_id}, timeout=300, verify=False)
response.raise_for_status()
payload = response.json() if response.text else {}
if isinstance(payload, dict) and payload.get("success"):
return payload.get("data") or {}
raise RuntimeError(f"获取货源查询解析载荷失败: {payload}")
@staticmethod
def _count_payload_rows(data):
rows = data.get("rows") or data.get("items") or []
if isinstance(rows, list) and rows:
return len(rows)
groups = data.get("groups") or []
if not isinstance(groups, list):
return 0
total = 0
for group in groups:
if isinstance(group, dict) and isinstance(group.get("items"), list):
total += len(group.get("items"))
return total
def _merge_parsed_payload(self, data, parsed_payload):
payload_rows = parsed_payload.get("allItems") or parsed_payload.get("items") or []
merged = {
**data,
"groups": parsed_payload.get("groups") or [],
"rows": payload_rows,
}
if parsed_payload.get("aiPrompt"):
merged["prompt"] = parsed_payload.get("aiPrompt")
if parsed_payload.get("apiKey") and not merged.get("api_key"):
merged["api_key"] = parsed_payload.get("apiKey")
return merged
def process_task(self, task_data: dict):
"""处理审批任务主入口
Args:
task_data: 任务数据
"""
try:
data = task_data.get("data", {})
task_id = data.get("taskId")
if task_id:
queued_rows = self._count_payload_rows(data)
try:
parsed_payload = self.fetch_parsed_payload(task_id, data.get("user_id") or data.get("userId") or 1)
data = self._merge_parsed_payload(data, parsed_payload)
self.log(
f"similar asin task {task_id} loaded full parsed payload from Java, queuedRows={queued_rows}, fullRows={self._count_payload_rows(data)}"
)
except Exception as e:
if not queued_rows:
raise
self.log(
f"similar asin task {task_id} failed to load full parsed payload, fallback to queued rows={queued_rows}: {e}",
"WARNING"
)
groups = self.normalize_groups(data)
data = task_data.get("data", {})
task_id = data.get("taskId")
if task_id:
queued_rows = self._count_payload_rows(data)
try:
parsed_payload = self.fetch_parsed_payload(task_id, data.get("user_id") or data.get("userId") or 1)
data = self._merge_parsed_payload(data, parsed_payload)
self.log(
f"similar asin task {task_id} loaded full parsed payload from Java, queuedRows={queued_rows}, fullRows={self._count_payload_rows(data)}"
)
except Exception as e:
if not queued_rows:
raise
self.log(
f"similar asin task {task_id} failed to load full parsed payload, fallback to queued rows={queued_rows}: {e}",
"WARNING"
)
groups = self.normalize_groups(data)
if not task_id:
self.log("任务ID为空跳过", "WARNING")
@@ -439,6 +536,8 @@ class SimilarAsinTask(TaskBase):
"country": i.get("country"),
"url": return_data.get("image_url"),
"title": return_data.get("title"),
"done": False,
"urls" : [i.get("ori_picture") for i in return_data.get("similar_data")]
}
for i in items
]
@@ -452,6 +551,7 @@ class SimilarAsinTask(TaskBase):
"displayId": gp.get("displayId"),
"items": group_item
}
print("================")
result.append(res)
print("================")
@@ -500,10 +600,10 @@ class SimilarAsinTask(TaskBase):
"""回传处理结果到API
"""
url = f"{DELETE_BRAND_API_BASE}/api/similar-asin/tasks/{task_id}/result"
url = f"{DELETE_BRAND_API_BASE}/api/similar-asin/tasks/{task_id}/result"
payload ={
"submissionId": f"{int(time.time())}",
"submissionId": f"{task_id}",
"chunkIndex": chunkIndex,
"chunkTotal": chunkTotal,
"error": error,
@@ -519,13 +619,13 @@ class SimilarAsinTask(TaskBase):
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
self.log(f"回传数据: {payload}")
response = requests.post(
url,
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
headers={"Content-Type": "application/json; charset=utf-8"},
timeout=request_timeout,
verify=False
)
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=request_timeout,
verify=False
)
self.log(f"回传结果: {response.text}")
data = response.json() if response.text else {}
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
@@ -553,6 +653,263 @@ if __name__ == '__main__':
country="英国", asin="B0F79LCB3X", total_page=2
)
print(resp)
# task_data = {
# "type": "similar-asin-run",
# "ts": 1778038688824,
# "data": {
# "taskId": 6850,
# "api_key": "sk-EMcDFg36zCeWUtCbRzKUbrRZyNeC6M4KBhY6fAVcNP7GG4xI",
# "groups": [
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "groupKey": "similar-asin:6850",
# "baseId": "",
# "displayId": "1",
# "items": [
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::2",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::1@2",
# "id": "1",
# "asin": "B0D792ND9V",
# "country": "英国",
# "price": "12.29",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::3",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_1",
# "asin": "B0D14L34S7",
# "country": "英国",
# "price": "14.88",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::4",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_2",
# "asin": "B0D14N8FJY",
# "country": "英国",
# "price": "15.36",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::5",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_3",
# "asin": "B0D14NBL5D",
# "country": "英国",
# "price": "14.87",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::6",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_4",
# "asin": "B0D14M5X4J",
# "country": "英国",
# "price": "14.41",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::7",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
# "id": "3_1",
# "asin": "B0CWQFZMGY",
# "country": "英国",
# "price": "17.05",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::8",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
# "id": "3_2",
# "asin": "B0CX8D68L3",
# "country": "英国",
# "price": "16.46",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::9",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::4@9",
# "id": "4_1",
# "asin": "B0CX87XXWJ",
# "country": "英国",
# "price": "17.54",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::10",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::5@10",
# "id": "5_1",
# "asin": "B0CR3PMR8L",
# "country": "英国",
# "price": "16.74",
# "url": "",
# "title": "",
# "target_urls": []
# }
# ]
# }
# ],
# "rows": [
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::2",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::1@2",
# "id": "1",
# "asin": "B0D792ND9V",
# "country": "英国",
# "price": "12.29",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::3",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_1",
# "asin": "B0D14L34S7",
# "country": "英国",
# "price": "14.88",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::4",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_2",
# "asin": "B0D14N8FJY",
# "country": "英国",
# "price": "15.36",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::5",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_3",
# "asin": "B0D14NBL5D",
# "country": "英国",
# "price": "14.87",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::6",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
# "id": "2_4",
# "asin": "B0D14M5X4J",
# "country": "英国",
# "price": "14.41",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::7",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
# "id": "3_1",
# "asin": "B0CWQFZMGY",
# "country": "英国",
# "price": "17.05",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::8",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
# "id": "3_2",
# "asin": "B0CX8D68L3",
# "country": "英国",
# "price": "16.46",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::9",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::4@9",
# "id": "4_1",
# "asin": "B0CX87XXWJ",
# "country": "英国",
# "price": "17.54",
# "url": "",
# "title": "",
# "target_urls": []
# },
# {
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
# "sourceFilename": "17(1) - 副本1.xlsx",
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::10",
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::5@10",
# "id": "5_1",
# "asin": "B0CR3PMR8L",
# "country": "英国",
# "price": "16.74",
# "url": "",
# "title": "",
# "target_urls": []
# }
# ]
# }
# }
# sim_asin = SimilarAsinTask({})
# sim_asin.process_task(task_data)