super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

品牌多文件chunk修复

Browse Source
This commit is contained in:
super
2026-03-30 21:06:10 +08:00
parent eeeed6d7b1
commit 1c1acbbe7d
2 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
app/blueprints/__pycache__/brand.cpython-312.pyc
View File

Binary file not shown.

12
app/blueprints/brand.py
View File

@@ -242,9 +242,11 @@ def _background_brand_task(task_id,data):
fileUrl = file_data.get("fileUrl") fileUrl = file_data.get("fileUrl")
if not brand_ls: if not brand_ls:
continue continue
for i in range(0, len(brand_ls), 5): file_chunks = [brand_ls[i:i + 5] for i in range(0, len(brand_ls), 5)]
chunk = brand_ls[i:i + 5] chunk_total = len(file_chunks)
tasks.append((chunk, fileUrl)) total_lines = len(brand_ls)
for chunk_index, chunk in enumerate(file_chunks, start=1):
tasks.append((chunk, fileUrl, total_lines, chunk_total, chunk_index))
if tasks: if tasks:
max_workers = min(8, len(tasks)) max_workers = min(8, len(tasks))
@@ -253,8 +255,8 @@ def _background_brand_task(task_id,data):
_push_task_event(task_id, {'status': 'cancelled'}) _push_task_event(task_id, {'status': 'cancelled'})
return return
futures = [ futures = [
executor.submit(_run_brand_single, task_id, chunk, file_url, strategy,len(brand_ls),len(tasks),chunkIndex+1) executor.submit(_run_brand_single, task_id, chunk, file_url, strategy, total_lines, chunk_total, chunk_index)
for chunkIndex,(chunk, file_url) in enumerate(tasks) for chunk, file_url, total_lines, chunk_total, chunk_index in tasks
] ]
for future in as_completed(futures): for future in as_completed(futures):
if _is_task_cancelled(task_id): if _is_task_cancelled(task_id):