super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完善多店铺开启多窗口、外观进度条等

Browse Source
This commit is contained in:
super
2026-05-01 21:09:54 +08:00
parent 4419c5bacd
commit e1cca219b8
41 changed files with 964 additions and 17737 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app/.env
View File

@@ -14,5 +14,6 @@ client_name=ShuFuAI
# java_api_base=http://47.111.163.154:18080
java_api_base=http://127.0.0.1:18080
# java_api_base=http://8.136.19.173:18080
# java_api_base=http://121.196.149.225:18080

26
app/amazon/main.py
View File

@@ -1,7 +1,8 @@
import time
import traceback
import requests
from datetime import datetime
import time
import traceback
import requests
import threading
from datetime import datetime
from typing import Dict, Any, List
from concurrent.futures import ThreadPoolExecutor, as_completed
from config import JSON_TASK_QUEUE, runing_task, runing_shop, DELETE_BRAND_API_BASE, ZN_COMPANY, ZN_USERNAME, ZN_PASSWORD
@@ -31,7 +32,10 @@ class TaskMonitor:
self.chunk_index = 1 # 当前处理的分块索引
self.max_workers = 5 # 线程池最大线程数
self.executor = None # 线程池执行器
self.executor = None # 线程池执行器
self.serial_task_locks = {
"product-risk-resolve-run": threading.Lock(),
}
# 在提交新任务前杀掉旧进程（确保环境干净）
kill_process("v6")
@@ -118,7 +122,17 @@ class TaskMonitor:
except Exception as e:
self.log(f"线程 {id(task_data)} 删除品牌任务处理异常: {traceback.format_exc()}", "ERROR")
def _process_approve_task_wrapper(self, task_data: Dict[str, Any],TASK_TYPE:str):
def _process_approve_task_wrapper(self, task_data: Dict[str, Any],TASK_TYPE:str):
task_type = task_data.get("type", "")
task_lock = self.serial_task_locks.get(task_type)
if task_lock is not None:
self.log(f"线程 {id(task_data)} 等待 {task_type} 串行锁...")
with task_lock:
self.log(f"线程 {id(task_data)} 获取 {task_type} 串行锁，开始处理...")
return self._process_approve_task_unlocked(task_data, TASK_TYPE)
return self._process_approve_task_unlocked(task_data, TASK_TYPE)
def _process_approve_task_unlocked(self, task_data: Dict[str, Any],TASK_TYPE:str):
"""审批任务处理包装器（用于线程池调用）
Args:

64
app/amazon/patrol_delete.py
View File

@@ -234,12 +234,14 @@ class InventoryManage(AmamzonBase):
country: str,
max_delete_operations: Optional[int] = None,
trace_id: str = "",
delete_conditions: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
"""循环删除非白名单分类商品，直到当前国家没有可删项。"""
trace_id = trace_id or _new_trace_id("del")
logger.info(
f"trace_id={trace_id} 开始批量删除非白名单商品: "
f"shop={shop_name}, country={country}, max_delete_operations={max_delete_operations}"
f"shop={shop_name}, country={country}, max_delete_operations={max_delete_operations}, "
f"delete_conditions={self._format_delete_conditions(delete_conditions)}"
)
all_results: List[Dict[str, Any]] = []
delete_counts: Dict[str, int] = {}
@@ -249,7 +251,10 @@ class InventoryManage(AmamzonBase):
stop_reason = "no-candidates"
latest_status_rows = self._read_listing_status_rows_for_delete(shop_name, country, trace_id=trace_id)
latest_delete_candidates = self._build_deletable_status_rows(latest_status_rows)
latest_delete_candidates = self._build_deletable_status_rows(
latest_status_rows,
delete_conditions=delete_conditions,
)
if not latest_delete_candidates:
logger.info(f"trace_id={trace_id} 未找到可删除商品分类: shop={shop_name}, country={country}")
final_status_rows = latest_status_rows
@@ -783,7 +788,12 @@ class InventoryManage(AmamzonBase):
]
@classmethod
def _build_deletable_status_rows(cls, rows: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
def _build_deletable_status_rows(
cls,
rows: Iterable[Dict[str, Any]],
delete_conditions: Optional[List[Dict[str, Any]]] = None,
) -> List[Dict[str, Any]]:
condition_texts = cls._normalize_delete_condition_texts(delete_conditions)
candidates = []
for row in rows:
status = cls._normalize_option_text(row.get("status") or "")
@@ -795,6 +805,8 @@ class InventoryManage(AmamzonBase):
canonical_status = cls._canonical_listing_status(status)
if not quantity_text or quantity <= 0 or canonical_status in LISTING_STATUS_DELETE_WHITELIST:
continue
if condition_texts and not cls._matches_delete_conditions(status, condition_texts):
continue
candidates.append(
{
@@ -805,6 +817,39 @@ class InventoryManage(AmamzonBase):
)
return candidates
@classmethod
def _normalize_delete_condition_texts(
cls,
delete_conditions: Optional[List[Dict[str, Any]]],
) -> List[str]:
texts = []
for condition in delete_conditions or []:
if isinstance(condition, dict):
raw_text = condition.get("conditionText") or condition.get("condition_text") or condition.get("text")
else:
raw_text = condition
text = cls._normalize_option_text(str(raw_text or "")).lower()
if text:
texts.append(text)
return list(dict.fromkeys(texts))
@classmethod
def _matches_delete_conditions(cls, status: str, condition_texts: List[str]) -> bool:
normalized_status = cls._normalize_option_text(status).lower()
canonical_status = cls._canonical_listing_status(status).lower()
return any(
condition in normalized_status
or condition in canonical_status
or normalized_status in condition
or canonical_status in condition
for condition in condition_texts
)
@classmethod
def _format_delete_conditions(cls, delete_conditions: Optional[List[Dict[str, Any]]]) -> str:
texts = cls._normalize_delete_condition_texts(delete_conditions)
return ", ".join(texts) if texts else "default"
@staticmethod
def _format_deletable_candidates(candidates: Iterable[Dict[str, Any]]) -> str:
summary = []
@@ -1299,6 +1344,7 @@ class PatrolDeleteTask:
items = data.get("items", [])
country_sections = data.get("country_sections", [])
cart_ratios = data.get("cart_ratios", [])
delete_conditions = data.get("delete_conditions") or data.get("deleteConditions") or []
if not task_id:
self.log("任务ID为空，跳过", "WARNING")
@@ -1321,10 +1367,14 @@ class PatrolDeleteTask:
"failed_count": 0,
"failed_countries": 0,
"deleted_listings_count": 0,
"delete_conditions": delete_conditions,
"stop_requested": False,
}
self.log(f"开始处理巡店删除任务 {task_id}，共 {len(items)} 个店铺，{total_countries} 个国家")
self.log(
f"开始处理巡店删除任务 {task_id}，共 {len(items)} 个店铺，{total_countries} 个国家，"
f"删除条件 {len(delete_conditions)} 条"
)
for index, shop_item in enumerate(items, 1):
if runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求，停止处理", "WARNING")
@@ -1333,7 +1383,7 @@ class PatrolDeleteTask:
shop_name = shop_item.get("shopName", "未知店铺")
self.log(f"[{index}/{len(items)}] 开始处理店铺: {shop_name}")
self.process_shop(task_id, shop_item, country_sections, cart_ratios)
self.process_shop(task_id, shop_item, country_sections, cart_ratios, delete_conditions)
runing_task[task_id]["processed_shops"] += 1
if runing_task[task_id].get("stop_requested", False):
@@ -1359,6 +1409,7 @@ class PatrolDeleteTask:
shop_item: Dict[str, Any],
country_sections: List[Dict[str, Any]],
cart_ratios: List[Dict[str, Any]],
delete_conditions: Optional[List[Dict[str, Any]]] = None,
):
"""处理单个店铺。"""
from config import runing_shop, runing_task
@@ -1412,6 +1463,7 @@ class PatrolDeleteTask:
task_id=task_id,
shop_name=shop_name,
country_name=country_name,
delete_conditions=delete_conditions,
)
country_results.append(country_result)
aggregated_sections[country_name] = country_result["countrySection"]
@@ -1522,6 +1574,7 @@ class PatrolDeleteTask:
task_id: int,
shop_name: str,
country_name: str,
delete_conditions: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
from config import runing_task
@@ -1543,6 +1596,7 @@ class PatrolDeleteTask:
shop_name,
country_name,
trace_id=trace_id,
delete_conditions=delete_conditions,
)
latest_status_rows = delete_result.get("statusRows") or []
self.log(

4
app/assets/appearance-patent.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/convert.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/dedupe.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/delete-brand.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/patrol-delete.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/price-track.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/product-risk.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/query-asin.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/shop-match.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/split.js
View File

File diff suppressed because one or more lines are too long

16
app/blueprints/main.py
View File

@@ -168,6 +168,7 @@ def serve_assets(filename):
@main_bp.route('/new_web_source/<path:filename>')
@login_required
def serve_new_web_source(filename):
"""提供 static 目录及子目录下的静态文件访问。"""
filepath = os.path.normpath(os.path.join("new_web_source", filename))
@@ -175,6 +176,21 @@ def serve_new_web_source(filename):
file_abs = os.path.abspath(filepath)
if not file_abs.startswith(static_abs) or not os.path.isfile(file_abs):
return '', 404
if filename.lower().endswith('.html'):
with open(file_abs, 'rb') as f:
content = f.read().decode('utf-8', errors='replace')
raw_uid = session.get('user_id')
uid_value = str(int(raw_uid)) if str(raw_uid or '').isdigit() else '""'
uid_script = (
"\n<script>"
f"window.localStorage.setItem('uid', {uid_value});"
"</script>\n"
)
if '</head>' in content:
content = content.replace('</head>', uid_script + '</head>', 1)
else:
content = uid_script + content
return render_template_string(content)
return send_file(file_abs, as_attachment=False)
@main_bp.route('/logo.jpg', methods=['GET'])

4
app/new_web_source/convert.html
View File

@@ -5,8 +5,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>格式转换 - 数富AI</title>
<script type="module" crossorigin src="/assets/convert.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-ruApoCdL.js">
<link rel="modulepreload" crossorigin href="/assets/brand-qwC923-0.js">
<link rel="modulepreload" crossorigin href="/assets/pywebview-BtdhG28Q.js">
<link rel="modulepreload" crossorigin href="/assets/brand-Lp7YWffz.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/convert-BKSNvX8i.css">
</head>

4
app/new_web_source/dedupe.html
View File

@@ -5,8 +5,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据去重 - 数富AI</title>
<script type="module" crossorigin src="/assets/dedupe.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-ruApoCdL.js">
<link rel="modulepreload" crossorigin href="/assets/brand-qwC923-0.js">
<link rel="modulepreload" crossorigin href="/assets/pywebview-BtdhG28Q.js">
<link rel="modulepreload" crossorigin href="/assets/brand-Lp7YWffz.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-DNlVfFj-.css">
</head>

4
app/new_web_source/delete-brand.html
View File

@@ -5,8 +5,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>删除品牌 - 数富AI</title>
<script type="module" crossorigin src="/assets/delete-brand.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-ruApoCdL.js">
<link rel="modulepreload" crossorigin href="/assets/brand-qwC923-0.js">
<link rel="modulepreload" crossorigin href="/assets/pywebview-BtdhG28Q.js">
<link rel="modulepreload" crossorigin href="/assets/brand-Lp7YWffz.js">
<link rel="modulepreload" crossorigin href="/assets/categorized-timers-JPA-olTr.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/delete-brand-BfMLVSQU.css">

4
app/new_web_source/split.html
View File

@@ -5,8 +5,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据拆分 - 数富AI</title>
<script type="module" crossorigin src="/assets/split.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-ruApoCdL.js">
<link rel="modulepreload" crossorigin href="/assets/brand-qwC923-0.js">
<link rel="modulepreload" crossorigin href="/assets/pywebview-BtdhG28Q.js">
<link rel="modulepreload" crossorigin href="/assets/brand-Lp7YWffz.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/split-BoBVrLdC.css">
</head>