super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加rufts

Browse Source
This commit is contained in:
super
2026-04-27 18:45:33 +08:00
parent 052bb31aea
commit 5105bf7049
35 changed files with 2098 additions and 248 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/.env
+2 -2
View File
@@ -13,8 +13,8 @@ client_name=ShuFuAI
# java_api_base=http://47.111.163.154:18080
# java_api_base=http://127.0.0.1:18080
java_api_base=http://8.136.19.173:18080
java_api_base=http://127.0.0.1:18080
# java_api_base=http://8.136.19.173:18080
# java_api_base=http://121.196.149.225:18080
app/amazon/__pycache__/approve.cpython-312.pyc
BIN
View File
Binary file not shown.
app/amazon/__pycache__/asin_status.cpython-312.pyc
BIN
View File
Binary file not shown.
app/amazon/__pycache__/detail_spider.cpython-312.pyc
BIN
View File
Binary file not shown.
app/amazon/__pycache__/main.cpython-312.pyc
BIN
View File
Binary file not shown.
app/amazon/__pycache__/match_action.cpython-312.pyc
BIN
View File
Binary file not shown.
app/amazon/__pycache__/price_match.cpython-312.pyc
BIN
View File
Binary file not shown.
app/amazon/detail_spider.py
+14 -12
View File
@@ -390,19 +390,20 @@ class SpiderTask:
runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e)
def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
item_data:dict={},
is_done: bool = False):
def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
item_data:dict={},
is_done: bool = False):
"""回传处理结果到API
"""
url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result"
payload ={
"submissionId": f"{int(time.time())}",
"chunkIndex": chunkIndex,
"chunkTotal": chunkTotal,
"error": error,
submission_id = f"appearance-patent:{task_id}"
payload ={
"submissionId": submission_id,
"chunkIndex": chunkIndex,
"chunkTotal": chunkTotal,
"error": error,
"items": [
item_data
],
@@ -410,8 +411,9 @@ class SpiderTask:
}
max_retries = 3
for retry in range(max_retries):
try:
for retry in range(max_retries):
try:
request_timeout = 300 if is_done else 30
print("================【详情采集】=====================")
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
@@ -420,7 +422,7 @@ class SpiderTask:
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
timeout=request_timeout,
verify=False
)
self.log(f"回传结果: {response.text}")
@@ -6904,4 +6906,4 @@ if __name__ == '__main__':
},
"code": None
}
spide.process_task(task_data)
spide.process_task(task_data)
app/assets/convert.js
+1 -1
View File
File diff suppressed because one or more lines are too long
app/assets/dedupe.js
+1 -1
View File
File diff suppressed because one or more lines are too long
app/assets/delete-brand.js
-4
View File
File diff suppressed because one or more lines are too long
app/assets/split.js
+1 -1
View File
File diff suppressed because one or more lines are too long
app/new_web_source/convert.html
+14 -17
View File
@@ -1,19 +1,16 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>格式转换 - 数富AI</title>
<script type="module" crossorigin src="/assets/convert.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-C66x_2Dh.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BZije8D7.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/convert-BKSNvX8i.css">
</head>
<body>
<div id="app"></div>
</body>
</html>
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>格式转换 - 数富AI</title>
<script type="module" crossorigin src="/assets/convert.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-C66x_2Dh.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BZije8D7.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/convert-BKSNvX8i.css">
</head>
<body>
<div id="app"></div>
</body>
</html>
app/new_web_source/dedupe.html
+14 -17
View File
@@ -1,19 +1,16 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据去重 - 数富AI</title>
<script type="module" crossorigin src="/assets/dedupe.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-C66x_2Dh.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BZije8D7.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-DNlVfFj-.css">
</head>
<body>
<div id="app"></div>
</body>
</html>
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据去重 - 数富AI</title>
<script type="module" crossorigin src="/assets/dedupe.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-C66x_2Dh.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BZije8D7.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-DNlVfFj-.css">
</head>
<body>
<div id="app"></div>
</body>
</html>
app/new_web_source/delete-brand.html
+14 -17
View File
@@ -1,19 +1,16 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>删除品牌 - 数富AI</title>
<script type="module" crossorigin src="/assets/delete-brand.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-C66x_2Dh.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BZije8D7.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/delete-brand-BP3XWKAC.css">
</head>
<body>
<div id="app"></div>
</body>
</html>
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>删除品牌 - 数富AI</title>
<script type="module" crossorigin src="/assets/delete-brand.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-C66x_2Dh.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BZije8D7.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/delete-brand-BP3XWKAC.css">
</head>
<body>
<div id="app"></div>
</body>
</html>
app/new_web_source/split.html
+14 -17
View File
@@ -1,19 +1,16 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据拆分 - 数富AI</title>
<script type="module" crossorigin src="/assets/split.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-C66x_2Dh.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BZije8D7.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/split-BoBVrLdC.css">
</head>
<body>
<div id="app"></div>
</body>
</html>
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据拆分 - 数富AI</title>
<script type="module" crossorigin src="/assets/split.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-C66x_2Dh.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BZije8D7.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/split-BoBVrLdC.css">
</head>
<body>
<div id="app"></div>
</body>
</html>