super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

品牌多文件chunk修复

Browse Source
This commit is contained in:
super
2026-03-30 21:06:10 +08:00
parent 9845ab8f2a
commit d9487b6885
2 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
app/blueprints/__pycache__/brand.cpython-312.pyc
View File

Binary file not shown.

12
app/blueprints/brand.py
View File

@@ -242,9 +242,11 @@ def _background_brand_task(task_id,data):
fileUrl = file_data.get("fileUrl")
if not brand_ls:
continue
for i in range(0, len(brand_ls), 5):
chunk = brand_ls[i:i + 5]
tasks.append((chunk, fileUrl))
file_chunks = [brand_ls[i:i + 5] for i in range(0, len(brand_ls), 5)]
chunk_total = len(file_chunks)
total_lines = len(brand_ls)
for chunk_index, chunk in enumerate(file_chunks, start=1):
tasks.append((chunk, fileUrl, total_lines, chunk_total, chunk_index))
if tasks:
max_workers = min(8, len(tasks))
@@ -253,8 +255,8 @@ def _background_brand_task(task_id,data):
_push_task_event(task_id, {'status': 'cancelled'})
return
futures = [
executor.submit(_run_brand_single, task_id, chunk, file_url, strategy,len(brand_ls),len(tasks),chunkIndex+1)
for chunkIndex,(chunk, file_url) in enumerate(tasks)
executor.submit(_run_brand_single, task_id, chunk, file_url, strategy, total_lines, chunk_total, chunk_index)
for chunk, file_url, total_lines, chunk_total, chunk_index in tasks
]
for future in as_completed(futures):
if _is_task_cancelled(task_id):