modified: .env
modified: amazon/approve.py modified: amazon/chrome_base.py modified: amazon/detail_spider.py modified: amazon/main.py modified: amazon/price_match.py modified: amazon/similar_asin.py modified: amazon/tool.py new file: amazon/user_data/chrome_data/BrowserMetrics-spare.pma new file: amazon/user_data/chrome_data/BrowserMetrics/BrowserMetrics-69FAEF07-558.pma
This commit is contained in:
铭坤
@@ -1,3 +1,5 @@
|
||||
# encoding utf-8
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import base64
|
||||
@@ -20,6 +22,17 @@ from amazon.chrome_base import ChromeAmzoneBase
|
||||
|
||||
from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE
|
||||
|
||||
try:
|
||||
from config import proxy_url as CONFIG_PROXY_URL, proxy_mode as CONFIG_PROXY_MODE
|
||||
except ImportError:
|
||||
CONFIG_PROXY_URL = None
|
||||
CONFIG_PROXY_MODE = 1
|
||||
|
||||
# Forbidden 后 3 分钟内统一使用代理：记录代理生效截止时间与当前代理
|
||||
_FORBIDDEN_PROXY_UNTIL = 0.0
|
||||
_FORBIDDEN_PROXY_DICT = None
|
||||
_FORBIDDEN_PROXY_MINUTES = 0.5
|
||||
|
||||
|
||||
class ChromeAmzone(ChromeAmzoneBase):
|
||||
mark_name = "亚马逊相似ASIN"
|
||||
@@ -81,6 +94,8 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
标题 :
|
||||
|
||||
"""
|
||||
global _FORBIDDEN_PROXY_UNTIL, _FORBIDDEN_PROXY_DICT
|
||||
|
||||
headers = {
|
||||
"accept": "application/json, text/plain, */*",
|
||||
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
@@ -126,11 +141,93 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
data = {
|
||||
"imageBase64" : imageBase64
|
||||
}
|
||||
response = requests.post(url, headers=headers, params=params, json=data, impersonate="chrome101",
|
||||
cookies=cookie)
|
||||
response.encoding = "utf-8"
|
||||
data = response.json()
|
||||
return data
|
||||
|
||||
proxies = None
|
||||
|
||||
# 若 3 分钟内曾出现 Forbidden，则直接使用当时保存的代理
|
||||
now = time.time()
|
||||
if now < _FORBIDDEN_PROXY_UNTIL and _FORBIDDEN_PROXY_DICT:
|
||||
proxies = _FORBIDDEN_PROXY_DICT
|
||||
print("处于 Forbidden 代理窗口内，直接使用代理", proxies)
|
||||
|
||||
# 发送第一次请求
|
||||
try:
|
||||
response = requests_frp.post(url, headers=headers, params=params, json=data, impersonate="chrome101",
|
||||
cookies=cookie, proxies=proxies)
|
||||
response.encoding = "utf-8"
|
||||
|
||||
# 检查响应状态码和数据有效性
|
||||
need_retry = False
|
||||
if response.status_code != 200:
|
||||
print(f"请求失败，状态码: {response.status_code}")
|
||||
need_retry = True
|
||||
else:
|
||||
try:
|
||||
result_data = response.json()
|
||||
# 检查是否获取到有效数据
|
||||
if not result_data or not isinstance(result_data, dict):
|
||||
print("返回数据为空或格式不正确")
|
||||
need_retry = True
|
||||
elif "Forbidden" in response.text or "Too Many Requests" in response.text:
|
||||
print("返回Forbidden或Too Many Requests")
|
||||
need_retry = True
|
||||
except Exception as e:
|
||||
print(f"解析响应JSON失败: {e}")
|
||||
need_retry = True
|
||||
|
||||
# 如果需要重试且配置了代理URL
|
||||
if need_retry and CONFIG_PROXY_URL and not proxies:
|
||||
try:
|
||||
proxy_resp = requests.get(CONFIG_PROXY_URL, timeout=10)
|
||||
print("代理请求结果->：", proxy_resp.text)
|
||||
|
||||
# 模式 2：账号密码代理，接口返回 JSON
|
||||
if CONFIG_PROXY_MODE == 2:
|
||||
resp_json = proxy_resp.json()
|
||||
proxy_list = resp_json.get("data", {}).get("list") or []
|
||||
first_item = proxy_list[0] if proxy_list else None
|
||||
if first_item:
|
||||
ip = first_item.get("ip")
|
||||
port = first_item.get("port")
|
||||
account = first_item.get("account")
|
||||
password = first_item.get("password")
|
||||
if ip and port and account and password:
|
||||
auth_proxy = f"{account}:{password}@{ip}:{port}"
|
||||
print("获取到账号密码代理：", auth_proxy)
|
||||
proxies = {
|
||||
"http": f"http://{auth_proxy}",
|
||||
"https": f"http://{auth_proxy}",
|
||||
}
|
||||
# 默认模式 1：普通 IP:port 文本
|
||||
if CONFIG_PROXY_MODE != 2:
|
||||
proxy_ip = (proxy_resp.text or "").strip()
|
||||
if proxy_ip:
|
||||
proxies = {
|
||||
"http": f"http://{proxy_ip}",
|
||||
"https": f"http://{proxy_ip}",
|
||||
}
|
||||
|
||||
if proxies:
|
||||
print("使用代理重试请求")
|
||||
response = requests_frp.post(url, headers=headers, params=params, json=data,
|
||||
impersonate="chrome101", cookies=cookie, proxies=proxies)
|
||||
response.encoding = "utf-8"
|
||||
print("代理重试结果状态码:", response.status_code)
|
||||
|
||||
# 记录 3 分钟内都使用该代理
|
||||
_FORBIDDEN_PROXY_UNTIL = now + _FORBIDDEN_PROXY_MINUTES * 60
|
||||
_FORBIDDEN_PROXY_DICT = proxies
|
||||
|
||||
except Exception as e:
|
||||
print("获取代理或重试失败:", e)
|
||||
|
||||
# 返回最终结果
|
||||
result_data = response.json()
|
||||
return result_data
|
||||
|
||||
except Exception as e:
|
||||
print(f"get_aliprice_data 请求异常: {e}")
|
||||
return {}
|
||||
|
||||
def _scrape_data(self):
|
||||
"""
|
||||
@@ -386,6 +483,8 @@ class SimilarAsinTask(TaskBase):
|
||||
"country": i.get("country"),
|
||||
"url": return_data.get("image_url"),
|
||||
"title": return_data.get("title"),
|
||||
"done": False,
|
||||
"urls" : [i.get("ori_picture") for i in return_data.get("similar_data")]
|
||||
}
|
||||
for i in items
|
||||
]
|
||||
@@ -399,6 +498,7 @@ class SimilarAsinTask(TaskBase):
|
||||
"displayId": gp.get("displayId"),
|
||||
"items": group_item
|
||||
}
|
||||
|
||||
print("================")
|
||||
result.append(res)
|
||||
print("================")
|
||||
@@ -447,10 +547,10 @@ class SimilarAsinTask(TaskBase):
|
||||
"""回传处理结果到API
|
||||
"""
|
||||
|
||||
url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result"
|
||||
url = f"{DELETE_BRAND_API_BASE}/api/similar-asin/tasks/{task_id}/result"
|
||||
|
||||
payload ={
|
||||
"submissionId": f"{int(time.time())}",
|
||||
"submissionId": f"{task_id}",
|
||||
"chunkIndex": chunkIndex,
|
||||
"chunkTotal": chunkTotal,
|
||||
"error": error,
|
||||
@@ -466,13 +566,13 @@ class SimilarAsinTask(TaskBase):
|
||||
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
|
||||
self.log(f"回传URL: {url}")
|
||||
self.log(f"回传数据: {payload}")
|
||||
response = requests.post(
|
||||
url,
|
||||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json; charset=utf-8"},
|
||||
timeout=request_timeout,
|
||||
verify=False
|
||||
)
|
||||
response = requests.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
timeout=request_timeout,
|
||||
verify=False
|
||||
)
|
||||
self.log(f"回传结果: {response.text}")
|
||||
data = response.json() if response.text else {}
|
||||
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
|
||||
@@ -500,6 +600,263 @@ if __name__ == '__main__':
|
||||
country="英国", asin="B0F79LCB3X", total_page=2
|
||||
)
|
||||
print(resp)
|
||||
# task_data = {
|
||||
# "type": "similar-asin-run",
|
||||
# "ts": 1778038688824,
|
||||
# "data": {
|
||||
# "taskId": 6850,
|
||||
# "api_key": "sk-EMcDFg36zCeWUtCbRzKUbrRZyNeC6M4KBhY6fAVcNP7GG4xI",
|
||||
# "groups": [
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "groupKey": "similar-asin:6850",
|
||||
# "baseId": "",
|
||||
# "displayId": "1",
|
||||
# "items": [
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::2",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::1@2",
|
||||
# "id": "1",
|
||||
# "asin": "B0D792ND9V",
|
||||
# "country": "英国",
|
||||
# "price": "12.29",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::3",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||
# "id": "2_1",
|
||||
# "asin": "B0D14L34S7",
|
||||
# "country": "英国",
|
||||
# "price": "14.88",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::4",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||
# "id": "2_2",
|
||||
# "asin": "B0D14N8FJY",
|
||||
# "country": "英国",
|
||||
# "price": "15.36",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::5",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||
# "id": "2_3",
|
||||
# "asin": "B0D14NBL5D",
|
||||
# "country": "英国",
|
||||
# "price": "14.87",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::6",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||
# "id": "2_4",
|
||||
# "asin": "B0D14M5X4J",
|
||||
# "country": "英国",
|
||||
# "price": "14.41",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::7",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
|
||||
# "id": "3_1",
|
||||
# "asin": "B0CWQFZMGY",
|
||||
# "country": "英国",
|
||||
# "price": "17.05",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::8",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
|
||||
# "id": "3_2",
|
||||
# "asin": "B0CX8D68L3",
|
||||
# "country": "英国",
|
||||
# "price": "16.46",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::9",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::4@9",
|
||||
# "id": "4_1",
|
||||
# "asin": "B0CX87XXWJ",
|
||||
# "country": "英国",
|
||||
# "price": "17.54",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::10",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::5@10",
|
||||
# "id": "5_1",
|
||||
# "asin": "B0CR3PMR8L",
|
||||
# "country": "英国",
|
||||
# "price": "16.74",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# }
|
||||
# ]
|
||||
# }
|
||||
# ],
|
||||
# "rows": [
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::2",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::1@2",
|
||||
# "id": "1",
|
||||
# "asin": "B0D792ND9V",
|
||||
# "country": "英国",
|
||||
# "price": "12.29",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::3",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||
# "id": "2_1",
|
||||
# "asin": "B0D14L34S7",
|
||||
# "country": "英国",
|
||||
# "price": "14.88",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::4",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||
# "id": "2_2",
|
||||
# "asin": "B0D14N8FJY",
|
||||
# "country": "英国",
|
||||
# "price": "15.36",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::5",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||
# "id": "2_3",
|
||||
# "asin": "B0D14NBL5D",
|
||||
# "country": "英国",
|
||||
# "price": "14.87",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::6",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::2@3",
|
||||
# "id": "2_4",
|
||||
# "asin": "B0D14M5X4J",
|
||||
# "country": "英国",
|
||||
# "price": "14.41",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::7",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
|
||||
# "id": "3_1",
|
||||
# "asin": "B0CWQFZMGY",
|
||||
# "country": "英国",
|
||||
# "price": "17.05",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::8",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::3@7",
|
||||
# "id": "3_2",
|
||||
# "asin": "B0CX8D68L3",
|
||||
# "country": "英国",
|
||||
# "price": "16.46",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::9",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::4@9",
|
||||
# "id": "4_1",
|
||||
# "asin": "B0CX87XXWJ",
|
||||
# "country": "英国",
|
||||
# "price": "17.54",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# },
|
||||
# {
|
||||
# "sourceFileKey": "cdfa437df62b40a0bde5682a05c1f6a3",
|
||||
# "sourceFilename": "17(1) - 副本1.xlsx",
|
||||
# "rowToken": "cdfa437df62b40a0bde5682a05c1f6a3::row::10",
|
||||
# "groupKey": "cdfa437df62b40a0bde5682a05c1f6a3::5@10",
|
||||
# "id": "5_1",
|
||||
# "asin": "B0CR3PMR8L",
|
||||
# "country": "英国",
|
||||
# "price": "16.74",
|
||||
# "url": "",
|
||||
# "title": "",
|
||||
# "target_urls": []
|
||||
# }
|
||||
# ]
|
||||
# }
|
||||
# }
|
||||
# sim_asin = SimilarAsinTask({})
|
||||
# sim_asin.process_task(task_data)
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user