Compare commits
9 Commits
1e087c1aae
...
origin-syn
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
080625567b | ||
|
|
266c0f17c1 | ||
|
|
225d13fb6e | ||
|
|
2ed1250604 | ||
|
|
ca4a2cd07a | ||
|
|
73ac9187a6 | ||
|
|
7503e3fa8b | ||
|
|
532438faba | ||
|
|
a1376b51b0 |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -43,6 +43,7 @@ MANIFEST
|
||||
.installed.cfg
|
||||
|
||||
# Build / packaging
|
||||
app.zip
|
||||
build/
|
||||
dist/
|
||||
develop-eggs/
|
||||
@@ -108,4 +109,4 @@ xlsx/
|
||||
OPS_REDIS_MYSQL_OPTIMIZATION_NOTES.md
|
||||
架构.md
|
||||
*ts.%
|
||||
.omc
|
||||
.omc
|
||||
|
||||
7818
2026_05_29.log
Normal file
7818
2026_05_29.log
Normal file
File diff suppressed because one or more lines are too long
6
app/.env
6
app/.env
@@ -11,9 +11,9 @@ zn_username=%E8%87%AA%E5%8A%A8%E5%8C%96_Robot
|
||||
|
||||
client_name=ShuFuAI
|
||||
|
||||
# java_api_base=http://47.111.163.154:18080
|
||||
java_api_base=http://127.0.0.1:18080
|
||||
# java_api_base=http://121.196.149.225:18080
|
||||
# java_api_base=http://api.aishufu.top:18080/
|
||||
java_api_base=http://api.aishufu.top:18080/
|
||||
# java_api_base=http://api.aishufu.top:18080/
|
||||
|
||||
# 与 Java 后端共享的 JWT 签名密钥,必须与 backend-java 的 AIIMAGE_JWT_SECRET 完全一致
|
||||
AIIMAGE_JWT_SECRET=please-change-this-secret-please-rotate-at-least-32-bytes
|
||||
|
||||
@@ -9,7 +9,7 @@ proxy_mode=2
|
||||
client_name=ShuFuAI
|
||||
|
||||
|
||||
java_api_base=http://127.0.0.1:18080
|
||||
# java_api_base=http://47.111.163.154:18080
|
||||
java_api_base=http://api.aishufu.top:18080/
|
||||
# java_api_base=http://api.aishufu.top:18080/
|
||||
|
||||
|
||||
|
||||
@@ -11,9 +11,9 @@ zn_username=%E8%87%AA%E5%8A%A8%E5%8C%96_Robot
|
||||
client_name=ShuFuAI
|
||||
|
||||
|
||||
java_api_base=http://47.111.163.154:18080
|
||||
# java_api_base=http://127.0.0.1:18080
|
||||
# java_api_base=http://47.111.163.154:18080
|
||||
java_api_base=http://api.aishufu.top:18080/
|
||||
# java_api_base=http://api.aishufu.top:18080/
|
||||
# java_api_base=http://api.aishufu.top:18080/
|
||||
|
||||
# 与 Java 后端共享的 JWT 签名密钥,必须与 backend-java 的 AIIMAGE_JWT_SECRET 完全一致
|
||||
AIIMAGE_JWT_SECRET=please-change-this-secret-please-rotate-at-least-32-bytes
|
||||
|
||||
@@ -417,8 +417,8 @@ class AmamzonBase(ZiniaoDriver):
|
||||
"""
|
||||
try:
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
if level == "ERROR":
|
||||
show_notification(message, "error")
|
||||
# if level == "ERROR":
|
||||
# show_notification(message, "error")
|
||||
print(f"[{timestamp}] [{self.mark_name}] [{level}] {message}")
|
||||
except Exception as e:
|
||||
print(f"输出出错,{e}")
|
||||
@@ -689,8 +689,8 @@ class TaskBase:
|
||||
"""
|
||||
try:
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
if level == "ERROR":
|
||||
show_notification(message, "error")
|
||||
# if level == "ERROR":
|
||||
# show_notification(message, "error")
|
||||
print(f"[{timestamp}] [{self.task_name}] [{level}] {message}")
|
||||
except Exception as e:
|
||||
print(f"输出出错,{e}")
|
||||
@@ -739,7 +739,7 @@ class TaskBase:
|
||||
if not shop_data:
|
||||
mes = f"获取店铺凭证失败,响应数据: {shop_data.get('message', '未知错误')}"
|
||||
self.log(mes, "ERROR")
|
||||
show_notification(mes, "ERROR")
|
||||
# show_notification(mes, "ERROR")
|
||||
continue
|
||||
|
||||
password = shop_data["data"]["password"]
|
||||
|
||||
@@ -308,8 +308,9 @@ class AmzoneApprove(AmamzonBase):
|
||||
self.log("开始执行...")
|
||||
num = 0
|
||||
retry_num = 0
|
||||
|
||||
max_retry_num = 5
|
||||
total_page = 0
|
||||
current_page = 0
|
||||
while retry_num < max_retry_num: # 最多重试3次
|
||||
# if num > 3: #测试
|
||||
# return
|
||||
@@ -325,13 +326,13 @@ class AmzoneApprove(AmamzonBase):
|
||||
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
|
||||
if len(page_pamel) > 0:
|
||||
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
|
||||
current_page = int(current_page.strip())
|
||||
|
||||
# 总页数
|
||||
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
|
||||
if len(total_page) > 0:
|
||||
total_page = total_page[-1].text
|
||||
else:
|
||||
total_page = 0
|
||||
total_page = int(total_page[-1].text.strip())
|
||||
|
||||
self.log(f"当前页码: {current_page} / 总页数: {total_page}")
|
||||
except Exception as e:
|
||||
self.log(f"获取页码失败:{e}")
|
||||
@@ -543,6 +544,14 @@ class AmzoneApprove(AmamzonBase):
|
||||
self.tab.refresh()
|
||||
self.tab.wait.doc_loaded(raise_err=False,timeout=120)
|
||||
|
||||
#检查页数是否相等,不相等则继续
|
||||
self.log(f"开始检查页数,当前页数 {current_page} / {total_page}")
|
||||
if total_page != 0 and current_page!= 0 and current_page < total_page:
|
||||
self.log(f"检查到页数还未完成,重启浏览器继续")
|
||||
raise RuntimeError(f"与页面的连接已断开,检查到页数还未完成,重启浏览器继续")
|
||||
|
||||
|
||||
|
||||
class ApproveTask(TaskBase):
|
||||
"""审批任务处理类:负责处理产品风险审批任务"""
|
||||
task_name = "产品风险审批-TASK"
|
||||
@@ -674,12 +683,12 @@ class ApproveTask(TaskBase):
|
||||
return
|
||||
|
||||
current_url = None
|
||||
max_retries = 200 #只要没有完成,一直重试
|
||||
for _ in range(max_retries):
|
||||
try:
|
||||
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter,target_url=current_url)
|
||||
driver.reset_already_asin()
|
||||
driver.close_store()
|
||||
|
||||
break
|
||||
except Exception as e:
|
||||
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR")
|
||||
@@ -777,7 +786,7 @@ class ApproveTask(TaskBase):
|
||||
}
|
||||
result.append(country_data)
|
||||
|
||||
if len(result) > 20:
|
||||
if len(result) > 10:
|
||||
self.post_result_batch(task_id, shop_name, country_code, result)
|
||||
result = []
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from datetime import datetime
|
||||
from DrissionPage import Chromium, ChromiumOptions
|
||||
from collections import defaultdict
|
||||
|
||||
from config import base_dir
|
||||
from config import base_dir,debug
|
||||
from amazon.tool import get_shop_info,show_notification
|
||||
|
||||
|
||||
@@ -46,7 +46,8 @@ class ChromeAmzoneBase:
|
||||
杀死当前谷歌浏览器进程,并使用 drissionpage 启动谷歌浏览器,使用系统安装的浏览器默认用户文件夹
|
||||
"""
|
||||
print("正在关闭现有的chromium浏览器进程...")
|
||||
os.system('taskkill /f /t /im chrome.exe')
|
||||
if not debug:
|
||||
os.system('taskkill /f /t /im chrome.exe')
|
||||
time.sleep(2)
|
||||
|
||||
print("正在启动chromium浏览器...")
|
||||
@@ -70,8 +71,8 @@ class ChromeAmzoneBase:
|
||||
"""
|
||||
try:
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
if level == "ERROR":
|
||||
show_notification(message, "error")
|
||||
# if level == "ERROR":
|
||||
# show_notification(message, "error")
|
||||
print(f"[{timestamp}] [{self.mark_name}] [{level}] {message}")
|
||||
except Exception as e:
|
||||
print(f"输出出错,{e}")
|
||||
@@ -192,4 +193,14 @@ class ChromeAmzoneBase:
|
||||
self.browser.quit()
|
||||
print("浏览器已关闭")
|
||||
except Exception as e:
|
||||
print(f"关闭浏览器时出错: {str(e)}")
|
||||
print(f"关闭浏览器时出错: {str(e)}")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from amazon.amazon_base import TaskBase
|
||||
from amazon.chrome_base import ChromeAmzoneBase
|
||||
|
||||
|
||||
from config import runing_task, runing_shop,base_dir,DELETE_BRAND_API_BASE
|
||||
from config import runing_task,runing_shop,base_dir,DELETE_BRAND_API_BASE
|
||||
|
||||
|
||||
class ChromeAmzone(ChromeAmzoneBase):
|
||||
@@ -79,7 +79,8 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
except Exception as e:
|
||||
error_msg = f"运行出错: {traceback.format_exc()}"
|
||||
self.log(error_msg)
|
||||
show_notification(f"采集失败: {str(e)}", "error")
|
||||
# show_notification(f"采集失败: {str(e)}", "error")
|
||||
raise RuntimeError(error_msg)
|
||||
return {}
|
||||
|
||||
def _scrape_data(self):
|
||||
@@ -91,7 +92,8 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
"""
|
||||
data = {
|
||||
'image_url': "",
|
||||
'title': ""
|
||||
'title': "",
|
||||
"sku" : ""
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -100,9 +102,41 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
title_ele = self.tab.ele('xpath://h1[@id="title"]',timeout=30)
|
||||
title = title_ele.text
|
||||
data["title"] = title
|
||||
sku_ele_ls = self.tab.eles('xpath://ul[@class="a-unordered-list a-vertical a-spacing-mini"]', timeout=20)
|
||||
if len(sku_ele_ls) > 0:
|
||||
data["sku"] = sku_ele_ls[0].text
|
||||
imge_ele = self.tab.ele('xpath://div[@id="imgTagWrapperId"]//img',timeout=20)
|
||||
image_url = imge_ele.attr("src")
|
||||
|
||||
image_url = ""
|
||||
min_image_url = ""
|
||||
data_a_dynamic_image = imge_ele.attr("data-a-dynamic-image")
|
||||
if data_a_dynamic_image:
|
||||
dynamic_image_json = json.loads(data_a_dynamic_image)
|
||||
self.log(f"图片信息:{dynamic_image_json}")
|
||||
max_area = 0
|
||||
min_area = 0
|
||||
|
||||
for url, (width, height) in dynamic_image_json.items():
|
||||
area = width * height
|
||||
if area > max_area:
|
||||
max_area = area
|
||||
image_url = url
|
||||
if min_area == 0:
|
||||
min_area = area
|
||||
min_image_url = url
|
||||
if area < min_area:
|
||||
min_area = area
|
||||
min_image_url = url
|
||||
|
||||
if not image_url:
|
||||
image_url = imge_ele.attr("src")
|
||||
|
||||
if not min_image_url:
|
||||
min_image_url = imge_ele.attr("src")
|
||||
|
||||
data["image_url"] = image_url
|
||||
data["min_image_url"] = min_image_url
|
||||
|
||||
return data
|
||||
|
||||
except Exception as e:
|
||||
@@ -246,6 +280,7 @@ class SpiderTask(TaskBase):
|
||||
"country": i.get("country"),
|
||||
"url": return_data.get("image_url"),
|
||||
"title": return_data.get("title"),
|
||||
"sku": return_data.get("sku")
|
||||
}
|
||||
for i in items
|
||||
]
|
||||
@@ -263,7 +298,7 @@ class SpiderTask(TaskBase):
|
||||
result.append(res)
|
||||
print("================")
|
||||
is_done = gp_index == len(groups)-1
|
||||
if len(result) > 20 or is_done:
|
||||
if len(result) > 10 or is_done:
|
||||
self.post_result(task_id=task_id,chunkIndex=gp_index+1,chunkTotal=len(groups),
|
||||
asin=asin,item_data=result,is_done=is_done)
|
||||
result = []
|
||||
|
||||
@@ -35,9 +35,12 @@ class AmzoneMatchAction(AmamzonBase):
|
||||
self.log("开始执行...")
|
||||
num = 0
|
||||
retry_num = 0
|
||||
get_page_faild = 0
|
||||
|
||||
while retry_num < 3: # 最多重试3次
|
||||
max_retry_num = 5
|
||||
total_page = 0
|
||||
current_page = 0
|
||||
|
||||
while retry_num < max_retry_num:
|
||||
# if num > 3: #测试
|
||||
# return
|
||||
# 等待加载完成
|
||||
@@ -45,6 +48,7 @@ class AmzoneMatchAction(AmamzonBase):
|
||||
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
|
||||
if len(load_ele) > 0:
|
||||
load_ele[0].wait.deleted(timeout=3, raise_err=False)
|
||||
|
||||
time.sleep(0.5)
|
||||
|
||||
# 获取当前页码
|
||||
@@ -52,23 +56,17 @@ class AmzoneMatchAction(AmamzonBase):
|
||||
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
|
||||
if len(page_pamel) > 0:
|
||||
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
|
||||
current_page = int(current_page.strip())
|
||||
# 测试
|
||||
# if current_page > 1:
|
||||
# break
|
||||
# 总页数
|
||||
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
|
||||
if len(total_page) > 0:
|
||||
total_page = total_page[-1].text
|
||||
else:
|
||||
total_page = 0
|
||||
total_page = int(total_page[-1].text.strip())
|
||||
self.log(f"【{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}")
|
||||
get_page_faild = 0
|
||||
except Exception as e:
|
||||
self.log(f"【{self.mark_name}】获取页码失败", e)
|
||||
get_page_faild+= 1
|
||||
if get_page_faild > 2:
|
||||
show_notification(f"【{self.mark_name}】获取页码失败超3次停止任务!")
|
||||
break
|
||||
|
||||
# 保存当前URL,用于失败重试时访问
|
||||
try:
|
||||
@@ -81,6 +79,15 @@ class AmzoneMatchAction(AmamzonBase):
|
||||
|
||||
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
|
||||
self.log(f"【{self.mark_name}】获取到 {len(sku_ls)}")
|
||||
if len(sku_ls) == 0:
|
||||
retry_num += 1
|
||||
self.log(f"没有获取到SKU列表,重试{retry_num}/{max_retry_num}")
|
||||
self.tab.refresh()
|
||||
self.tab.wait.doc_loaded(raise_err=False, timeout=120)
|
||||
if retry_num == max_retry_num:
|
||||
raise RuntimeError(f"与页面的连接已断开,没有获取到SKU列表,重试{retry_num}/{max_retry_num}")
|
||||
continue
|
||||
|
||||
# for sku_ele in sku_ls[0:2]:
|
||||
for sku_ele in sku_ls:
|
||||
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]')
|
||||
@@ -140,6 +147,12 @@ class AmzoneMatchAction(AmamzonBase):
|
||||
self.tab.refresh()
|
||||
self.tab.wait.doc_loaded(raise_err=False,timeout=120)
|
||||
|
||||
#检查页数是否相等,不相等则继续
|
||||
self.log(f"开始检查页数,当前页数 {current_page} / {total_page}")
|
||||
if total_page != 0 and current_page!= 0 and current_page < total_page:
|
||||
self.log(f"检查到页数还未完成,重启浏览器继续")
|
||||
raise RuntimeError(f"与页面的连接已断开,检查到页数还未完成,重启浏览器继续")
|
||||
|
||||
|
||||
class MatchTak(TaskBase):
|
||||
task_name = "匹配价格-TASK"
|
||||
@@ -261,7 +274,7 @@ class MatchTak(TaskBase):
|
||||
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理国家", "WARNING")
|
||||
break
|
||||
|
||||
skip_asin = skip_asins_by_country.get(country_code)
|
||||
skip_asin = skip_asins_by_country.get(country_code,[])
|
||||
if skipAsinDetailsByCountry :
|
||||
skipAsinDetails = {i.get("asin"):i.get("minimumPrice") for i in skipAsinDetailsByCountry.get(country_code)}
|
||||
else:
|
||||
@@ -276,6 +289,7 @@ class MatchTak(TaskBase):
|
||||
return
|
||||
|
||||
current_url = None
|
||||
max_retries = 200
|
||||
for _ in range(max_retries):
|
||||
try:
|
||||
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter,limit,
|
||||
@@ -366,8 +380,8 @@ class MatchTak(TaskBase):
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["current_asin"] = asin
|
||||
runing_task[task_id]["processed_asins"] += 1
|
||||
|
||||
runing_task[task_id]["failed_count"] += 1
|
||||
|
||||
minimumPrice = ""
|
||||
if "有最低价跳过" in status:
|
||||
minimumPrice = skipAsinDetails.get(asin)
|
||||
@@ -378,7 +392,7 @@ class MatchTak(TaskBase):
|
||||
"done": False
|
||||
})
|
||||
|
||||
if len(result) > 20:
|
||||
if len(result) > 10:
|
||||
self.post_result_batch(task_id, shop_name, country_code,result)
|
||||
result = []
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -10,9 +10,10 @@ from datetime import datetime
|
||||
from collections import defaultdict
|
||||
import requests
|
||||
from urllib.parse import quote
|
||||
|
||||
import os
|
||||
from curl_cffi import requests as requests_frp
|
||||
|
||||
from config import base_dir
|
||||
|
||||
|
||||
from amazon.tool import show_notification,get_shop_info,remove_special_characters,split_currency_values
|
||||
@@ -76,7 +77,9 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
将图片链接(URL 或本地文件路径)转换为 Base64 编码的字符串。
|
||||
"""
|
||||
if image_source.startswith(('http://', 'https://')):
|
||||
response = requests.get(image_source, timeout=10)
|
||||
response = requests.get(image_source, timeout=10,headers={
|
||||
"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0 18444"
|
||||
})
|
||||
response.raise_for_status() # 非 2xx 状态码将抛出异常
|
||||
image_data = response.content
|
||||
else:
|
||||
@@ -153,7 +156,7 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
# 发送第一次请求
|
||||
try:
|
||||
response = requests_frp.post(url, headers=headers, params=params, json=data, impersonate="chrome101",
|
||||
cookies=cookie, proxies=proxies)
|
||||
cookies=cookie, proxies=proxies,verify=False)
|
||||
response.encoding = "utf-8"
|
||||
|
||||
# 检查响应状态码和数据有效性
|
||||
@@ -176,6 +179,7 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
need_retry = True
|
||||
|
||||
# 如果需要重试且配置了代理URL
|
||||
# need_retry = True
|
||||
if need_retry and CONFIG_PROXY_URL and not proxies:
|
||||
try:
|
||||
proxy_resp = requests.get(CONFIG_PROXY_URL, timeout=10)
|
||||
@@ -204,7 +208,7 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
if proxy_ip:
|
||||
proxies = {
|
||||
"http": f"http://{proxy_ip}",
|
||||
"https": f"http://{proxy_ip}",
|
||||
"https": f"https://{proxy_ip}",
|
||||
}
|
||||
|
||||
if proxies:
|
||||
@@ -251,8 +255,37 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
title = title_ele.text
|
||||
data["title"] = title
|
||||
imge_ele = self.tab.ele('xpath://div[@id="imgTagWrapperId"]//img',timeout=20)
|
||||
image_url = imge_ele.attr("src")
|
||||
|
||||
image_url = ""
|
||||
min_image_url = ""
|
||||
data_a_dynamic_image = imge_ele.attr("data-a-dynamic-image")
|
||||
if data_a_dynamic_image:
|
||||
dynamic_image_json = json.loads(data_a_dynamic_image)
|
||||
self.log(f"图片信息:{dynamic_image_json}")
|
||||
max_area = 0
|
||||
min_area = 0
|
||||
|
||||
for url, (width, height) in dynamic_image_json.items():
|
||||
area = width * height
|
||||
if area > max_area:
|
||||
max_area = area
|
||||
image_url = url
|
||||
if min_area == 0:
|
||||
min_area = area
|
||||
min_image_url = url
|
||||
if area < min_area:
|
||||
min_area = area
|
||||
min_image_url = url
|
||||
|
||||
if not image_url:
|
||||
image_url = imge_ele.attr("src")
|
||||
|
||||
if not min_image_url:
|
||||
min_image_url = imge_ele.attr("src")
|
||||
|
||||
|
||||
data["image_url"] = image_url
|
||||
data["min_image_url"] = min_image_url
|
||||
|
||||
# sku
|
||||
sku_ele_ls = self.tab.eles('xpath://ul[@class="a-unordered-list a-vertical a-spacing-mini"]',timeout=20)
|
||||
@@ -277,12 +310,13 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
Returns:
|
||||
dict: 包含采集到的数据
|
||||
"""
|
||||
return_data = {}
|
||||
try:
|
||||
# 验证国家是否支持
|
||||
if country not in self.country_info:
|
||||
error_msg = f"不支持的国家: {country},支持的国家有: {list(self.country_info.keys())}"
|
||||
print(error_msg)
|
||||
show_notification(error_msg, "error")
|
||||
# show_notification(error_msg, "error")
|
||||
return None
|
||||
|
||||
# 获取国家配置
|
||||
@@ -314,29 +348,35 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
self.log("正在抓取商品数据...")
|
||||
data = self._scrape_data()
|
||||
|
||||
return_data.update(data)
|
||||
|
||||
# 判断是否采集标题出错
|
||||
new_size = "220,220"
|
||||
pattern = r"\._(?:[A-Z]+)?(\d+)_\."
|
||||
replacement = f"._{new_size}_.jpg"
|
||||
# replacement = f"._{new_size}_.jpg"
|
||||
# 使用正则替换
|
||||
image_new_url = re.sub(pattern, lambda m: replacement, data["image_url"])
|
||||
# image_new_url = re.sub(pattern, lambda m: replacement, data["image_url"])
|
||||
image_new_url = data["min_image_url"]
|
||||
|
||||
print(image_new_url)
|
||||
|
||||
iamge_base64 = self.image_to_base64(image_new_url)
|
||||
|
||||
similar_data = []
|
||||
# 4、请求获取插件的数据
|
||||
for page_num in range(total_page):
|
||||
resp_data = self.get_aliprice_data(
|
||||
page=page_num,
|
||||
title = data["title"],
|
||||
domain=domain,
|
||||
category=data["category"],
|
||||
imageBase64 = iamge_base64
|
||||
)
|
||||
self.log(f"Aliprice 扩展数据获取:{str(resp_data)[0:200]}")
|
||||
if len(resp_data.get("data",[])) > 0:
|
||||
for i in resp_data.get("data"):
|
||||
similar_data.append(i)
|
||||
if image_new_url:
|
||||
for page_num in range(total_page):
|
||||
resp_data = self.get_aliprice_data(
|
||||
page=page_num+1,
|
||||
title = data["title"],
|
||||
domain=domain,
|
||||
category=data["category"],
|
||||
imageBase64 = iamge_base64
|
||||
)
|
||||
self.log(f"Aliprice 扩展数据获取:{str(resp_data)[0:200]}")
|
||||
if len(resp_data.get("data",[])) > 0:
|
||||
for i in resp_data.get("data"):
|
||||
similar_data.append(i)
|
||||
|
||||
data["similar_data"] = similar_data
|
||||
|
||||
@@ -346,14 +386,17 @@ class ChromeAmzone(ChromeAmzoneBase):
|
||||
data['url'] = product_url
|
||||
data['timestamp'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
print(f"数据抓取完成: {json.dumps(data)}")
|
||||
return data
|
||||
print(f"数据抓取完成: {json.dumps(return_data)}")
|
||||
return_data.update(data)
|
||||
|
||||
return return_data
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"运行出错: {traceback.format_exc()}"
|
||||
print(error_msg)
|
||||
show_notification(f"采集失败: {str(e)}", "error")
|
||||
return {}
|
||||
# show_notification(f"采集失败: {str(e)}", "error")
|
||||
raise RuntimeError(f"{traceback.format_exc()}")
|
||||
return return_data
|
||||
|
||||
|
||||
class SimilarAsinTask(TaskBase):
|
||||
@@ -478,10 +521,9 @@ class SimilarAsinTask(TaskBase):
|
||||
result = []
|
||||
for gp_index,gp in enumerate(groups):
|
||||
items = gp.get("items", [])
|
||||
return_data = {}
|
||||
|
||||
group_item = []
|
||||
for index,value in enumerate(items):
|
||||
|
||||
print(value)
|
||||
return_data = {
|
||||
'image_url': "",
|
||||
'title': "",
|
||||
@@ -494,37 +536,36 @@ class SimilarAsinTask(TaskBase):
|
||||
country = value.get("country")
|
||||
for _ in range(max_retry):
|
||||
try:
|
||||
return_data = chrome.run(country, asin,total_page=2) or {}
|
||||
return_data = chrome.run(country, asin,total_page=1) or {}
|
||||
self.log(f"抓取结果->{return_data}")
|
||||
break
|
||||
except Exception as e:
|
||||
# if "与页面的连接已断开" in str(e):
|
||||
chrome = ChromeAmzone()
|
||||
self.log(f"{asin}抓取数据报错,{e}")
|
||||
if not isinstance(return_data, dict):
|
||||
return_data = {}
|
||||
if return_data.get("image_url"):
|
||||
break
|
||||
# if not isinstance(return_data, dict):
|
||||
# return_data = {}
|
||||
# if return_data.get("image_url"):
|
||||
# break
|
||||
|
||||
group_item.append({
|
||||
"sourceFileKey": value.get("sourceFileKey",""),
|
||||
"sourceFilename": value.get("sourceFilename",""),
|
||||
"rowToken": value.get("rowToken",""),
|
||||
"groupKey": value.get("groupKey",""),
|
||||
"id": value.get("values",{}).get("id",""),
|
||||
"asin": value.get("asin",""),
|
||||
"sku" : return_data.get("sku",""),
|
||||
"country": value.get("country",""),
|
||||
"url": return_data.get("image_url",""),
|
||||
"title": return_data.get("title",""),
|
||||
"done": False,
|
||||
"urls" : [i.get("ori_picture") for i in return_data.get("similar_data",[])[:16]]
|
||||
})
|
||||
# task_id: int, chunkIndex:int,chunkTotal: int, country_code: str, asin: str, status: dict,error:str="",
|
||||
# item_data:dict={},
|
||||
|
||||
|
||||
group_item = [
|
||||
{
|
||||
"sourceFileKey": i.get("sourceFileKey"),
|
||||
"sourceFilename": i.get("sourceFilename"),
|
||||
"rowToken": i.get("rowToken"),
|
||||
"groupKey": i.get("groupKey"),
|
||||
"id": i.get("id"),
|
||||
"asin": i.get("asin"),
|
||||
"sku" : return_data.get("sku"),
|
||||
"country": i.get("country"),
|
||||
"url": return_data.get("image_url"),
|
||||
"title": return_data.get("title"),
|
||||
"done": False,
|
||||
"urls" : [i.get("ori_picture") for i in return_data.get("similar_data")]
|
||||
}
|
||||
for i in items
|
||||
]
|
||||
# task_id: int, chunkIndex:int,chunkTotal: int, country_code: str, asin: str, status: dict,error:str="",
|
||||
# item_data:dict={},
|
||||
res = {
|
||||
"sourceFileKey": gp.get("sourceFileKey"),
|
||||
"sourceFilename": gp.get("sourceFilename"),
|
||||
@@ -629,7 +670,7 @@ class SimilarAsinTask(TaskBase):
|
||||
time.sleep(2)
|
||||
|
||||
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
|
||||
raise RuntimeError("已达到最大重试次数,结果回传最终失败")
|
||||
# raise RuntimeError("已达到最大重试次数,结果回传最终失败")
|
||||
|
||||
if __name__ == '__main__':
|
||||
spide = ChromeAmzone()
|
||||
|
||||
1
app/assets/appearance-patent-BoTBnU7J.css
Normal file
1
app/assets/appearance-patent-BoTBnU7J.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/assets/brand-COze15GJ.js
Normal file
1
app/assets/brand-COze15GJ.js
Normal file
@@ -0,0 +1 @@
|
||||
import{bN as r}from"./java-modules-B8c-YG5x.js";const n="";function s(e){return r(`${n}/api/brand/expand-folder-recursive`,{folder:e})}export{s as e};
|
||||
@@ -1 +0,0 @@
|
||||
import{b8 as r}from"./pywebview-3bFwh9Cg.js";const n="";function s(e){return r(`${n}/api/brand/expand-folder-recursive`,{folder:e})}export{s as e};
|
||||
1
app/assets/collect-data-DPGAQszU.css
Normal file
1
app/assets/collect-data-DPGAQszU.css
Normal file
File diff suppressed because one or more lines are too long
1
app/assets/collect-data.js
Normal file
1
app/assets/collect-data.js
Normal file
File diff suppressed because one or more lines are too long
1
app/assets/convert-6oxZNMye.css
Normal file
1
app/assets/convert-6oxZNMye.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/assets/dedupe-DpfPYQDv.css
Normal file
1
app/assets/dedupe-DpfPYQDv.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/assets/delete-brand-DZ3x5hOX.css
Normal file
1
app/assets/delete-brand-DZ3x5hOX.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/assets/patrol-delete-Cqhg2qXm.css
Normal file
1
app/assets/patrol-delete-Cqhg2qXm.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/assets/product-risk-BQIMXTON.css
Normal file
1
app/assets/product-risk-BQIMXTON.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/assets/query-asin-CECO5lUB.css
Normal file
1
app/assets/query-asin-CECO5lUB.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
.module-page[data-v-49f8fcb3]{min-height:100vh;background:#1a1a1a}.main-content[data-v-49f8fcb3]{display:flex;height:calc(100vh - 56px);min-height:calc(100vh - 56px)}.left-panel[data-v-49f8fcb3]{width:400px;background:#1e1e1e;padding:20px;overflow-y:auto;border-right:1px solid #2a2a2a}.right-panel[data-v-49f8fcb3]{flex:1;min-width:0;background:#1a1a1a;display:flex;flex-direction:column}.section-title[data-v-49f8fcb3],.subsection-title[data-v-49f8fcb3]{font-size:13px;color:#bbb;margin-bottom:10px}.upload-zone[data-v-49f8fcb3]{border:1px dashed #3a3a3a;border-radius:10px;padding:18px;background:#252525;margin-bottom:18px}.hint[data-v-49f8fcb3],.loading-msg[data-v-49f8fcb3],.files[data-v-49f8fcb3],.muted[data-v-49f8fcb3]{color:#888;font-size:12px;line-height:1.5}.link[data-v-49f8fcb3]{color:#6ea8fe;text-decoration:none}.link[data-v-49f8fcb3]:hover{color:#9fc5ff}.btns[data-v-49f8fcb3],.run-row[data-v-49f8fcb3]{display:flex;gap:10px;flex-wrap:wrap}.opt-btn[data-v-49f8fcb3],.btn-run[data-v-49f8fcb3],.btn-delete[data-v-49f8fcb3],.download[data-v-49f8fcb3]{border:none;cursor:pointer;border-radius:7px}.opt-btn[data-v-49f8fcb3]{padding:8px 14px;color:#ccc;background:#2a2a2a;border:1px solid #3a3a3a}.btn-run[data-v-49f8fcb3]{padding:10px 18px;color:#fff;background:#3498db;font-weight:600}.btn-queue[data-v-49f8fcb3]{background:#27ae60}.btn-run[data-v-49f8fcb3]:disabled{opacity:.55;cursor:not-allowed}.selected-files[data-v-49f8fcb3]{margin-top:14px;color:#999;font-size:12px;word-break:break-all}.selected-files span[data-v-49f8fcb3]{display:block;margin:4px 0}.parse-card[data-v-49f8fcb3],.queue-payload[data-v-49f8fcb3]{margin-top:14px;padding:12px;border:1px solid #2a2a2a;border-radius:8px;background:#202020;color:#b8c1cc;font-size:12px}.queue-payload[data-v-49f8fcb3]{max-height:220px;overflow:auto;color:#8fd3ff;white-space:pre-wrap}.panel-header[data-v-49f8fcb3]{padding:16px 20px;border-bottom:1px solid #2a2a2a;font-size:15px;font-weight:600;color:#ddd}.task-list-wrap[data-v-49f8fcb3]{flex:1;padding:16px 20px;overflow:auto}.clean-result-summary[data-v-49f8fcb3]{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px;margin-bottom:16px}.summary-card[data-v-49f8fcb3]{padding:14px 16px;border:1px solid #2a2a2a;border-radius:8px;background:#1e1e1e}.summary-card strong[data-v-49f8fcb3]{display:block;margin-top:8px;color:#eaf4ff;font-size:22px}.summary-label[data-v-49f8fcb3]{color:#8d8d8d;font-size:12px}.result-list-wrap[data-v-49f8fcb3]{border:1px solid #2a2a2a;border-radius:8px;background:#1e1e1e;min-height:180px;margin:0 0 16px}.result-list-header[data-v-49f8fcb3]{display:flex;justify-content:space-between;padding:12px 16px;border-bottom:1px solid #2a2a2a;color:#ddd;font-size:14px}.empty-tasks[data-v-49f8fcb3]{color:#666;font-size:13px;padding:18px;text-align:center}.task-list[data-v-49f8fcb3]{list-style:none;margin:0;padding:12px}.task-item[data-v-49f8fcb3]{display:flex;align-items:flex-start;justify-content:space-between;gap:12px;padding:12px 14px;border:1px solid #2a2a2a;border-radius:8px;margin-bottom:8px;background:#222}.left[data-v-49f8fcb3]{flex:1;min-width:0}.id[data-v-49f8fcb3]{color:#e0e0e0;font-size:13px;font-weight:600}.task-right[data-v-49f8fcb3]{display:flex;gap:8px;align-items:center;flex-wrap:wrap}.status[data-v-49f8fcb3]{padding:4px 10px;border-radius:6px;font-size:12px}.status.success[data-v-49f8fcb3]{background:#2ecc712e;color:#2ecc71}.status.failed[data-v-49f8fcb3]{background:#e74c3c2e;color:#ff6b6b}.status.running[data-v-49f8fcb3]{background:#3498db2e;color:#3498db}.status.pending[data-v-49f8fcb3]{background:#95a5a62e;color:#bdc3c7}.result-hint[data-v-49f8fcb3]{margin-top:6px;color:#e0b96d}.file-progress[data-v-49f8fcb3]{margin-top:8px;max-width:520px}.file-progress-meta[data-v-49f8fcb3]{display:flex;justify-content:space-between;gap:12px;color:#d8c278;font-size:12px}.file-progress-track[data-v-49f8fcb3]{margin-top:5px;height:8px;border-radius:999px;overflow:hidden;background:#303030;border:1px solid #3b3b3b}.file-progress-bar[data-v-49f8fcb3]{height:100%;border-radius:inherit;background:linear-gradient(90deg,#4aa3ff,#f0c75e);transition:width .25s ease}.file-progress-count[data-v-49f8fcb3]{margin-top:4px;color:#858585;font-size:11px}.download[data-v-49f8fcb3]{padding:6px 10px;color:#d6ecff;background:#3498db2e}.btn-delete[data-v-49f8fcb3]{padding:6px 10px;color:#ff8f8f;background:#e74c3c1f}@media(max-width:1100px){.main-content[data-v-49f8fcb3]{flex-direction:column;height:auto}.left-panel[data-v-49f8fcb3]{width:100%;border-right:none;border-bottom:1px solid #2a2a2a}.clean-result-summary[data-v-49f8fcb3]{grid-template-columns:repeat(2,minmax(0,1fr))}}
|
||||
1
app/assets/similar-asin-Bl4LsbT8.css
Normal file
1
app/assets/similar-asin-Bl4LsbT8.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/assets/split-CR1PtDQE.css
Normal file
1
app/assets/split-CR1PtDQE.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/assets/zh-cn-CscLo6na.js
Normal file
1
app/assets/zh-cn-CscLo6na.js
Normal file
File diff suppressed because one or more lines are too long
@@ -28,6 +28,7 @@ def api_auth_sync():
|
||||
"""前端拿到 Java 返回的 JWT 后调用,把 token 写进 Python 同源 cookie。"""
|
||||
data = request.get_json(silent=True) or {}
|
||||
token = (data.get('token') or '').strip()
|
||||
|
||||
if not token:
|
||||
return jsonify({'success': False, 'error': '缺少 token'}), 400
|
||||
payload, reason = parse_token_with_reason(token)
|
||||
|
||||
@@ -22,7 +22,7 @@ proxy_url = os.getenv("proxy_url")
|
||||
proxy_mode = int(os.getenv("proxy_mode",1))
|
||||
|
||||
client_name=os.getenv("client_name") + ".exe"
|
||||
JAVA_API_BASE = os.getenv("java_api_base", "http://127.0.0.1:18080")
|
||||
JAVA_API_BASE = os.getenv("java_api_base", "http://api.aishufu.top:18080/").rstrip("/")
|
||||
|
||||
# 紫鸟浏览器配置
|
||||
from urllib.parse import unquote
|
||||
@@ -32,7 +32,7 @@ ZN_USERNAME = unquote(ZN_USERNAME)
|
||||
ZN_PASSWORD = os.getenv("zn_password", "#20zsg25")
|
||||
|
||||
# 删除品牌API配置
|
||||
DELETE_BRAND_API_BASE = os.getenv("DELETE_BRAND_API_BASE", JAVA_API_BASE)
|
||||
DELETE_BRAND_API_BASE = os.getenv("DELETE_BRAND_API_BASE", JAVA_API_BASE).rstrip("/")
|
||||
|
||||
|
||||
cache_path = "./user_data"
|
||||
|
||||
@@ -4,6 +4,7 @@ JWT 工具:解析 Java 端签发的 token、从请求中提取 token。
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from flask import request
|
||||
@@ -25,6 +26,14 @@ except ImportError as _e:
|
||||
_DEFAULT_SECRET = "please-change-this-secret-please-rotate-at-least-32-bytes"
|
||||
COOKIE_NAME = os.getenv("AIIMAGE_AUTH_COOKIE_NAME", "aiimage_token")
|
||||
|
||||
# 桌面端与 Java 服务器时钟可能漂移(笔记本休眠、用户手动改时间、跨时区等),
|
||||
# 给 JWT exp/nbf 校验留出容差,避免 /api/auth/sync 因为时间不同步而 401。
|
||||
# 默认 5 分钟,可通过 AIIMAGE_JWT_LEEWAY_SECONDS 覆盖。
|
||||
try:
|
||||
_JWT_LEEWAY_SECONDS = int(os.getenv("AIIMAGE_JWT_LEEWAY_SECONDS", "300"))
|
||||
except ValueError:
|
||||
_JWT_LEEWAY_SECONDS = 300
|
||||
|
||||
|
||||
def _signing_key() -> bytes:
|
||||
"""与 Java JwtService.signingKey 保持一致:UTF-8 字节,不足 32 字节右侧补 0。"""
|
||||
@@ -35,6 +44,26 @@ def _signing_key() -> bytes:
|
||||
return key_bytes
|
||||
|
||||
|
||||
def _describe_clock_skew(token: str) -> str:
|
||||
"""解析 token 里的 iat/exp,与本地时钟比较,返回 ' iat=.. exp=.. now=.. skew=..s' 字符串。
|
||||
仅供日志使用,不做安全决策;解析失败返回空串,不影响主流程。"""
|
||||
if pyjwt is None:
|
||||
return ""
|
||||
try:
|
||||
unverified = pyjwt.decode(token, options={"verify_signature": False, "verify_exp": False, "verify_iat": False, "verify_nbf": False})
|
||||
except Exception:
|
||||
return ""
|
||||
iat = unverified.get("iat")
|
||||
exp = unverified.get("exp")
|
||||
now = int(time.time())
|
||||
parts = [f"now={now}"]
|
||||
if isinstance(iat, (int, float)):
|
||||
parts.append(f"iat={int(iat)} skew_iat={now - int(iat)}s")
|
||||
if isinstance(exp, (int, float)):
|
||||
parts.append(f"exp={int(exp)} skew_exp={now - int(exp)}s")
|
||||
return ", " + " ".join(parts)
|
||||
|
||||
|
||||
def parse_token_with_reason(token: str) -> Tuple[Optional[dict], Optional[str]]:
|
||||
"""解析 JWT,返回 (payload, error_reason)。
|
||||
payload 命中时 error_reason 为 None;失败时 payload 为 None,error_reason 描述根因。"""
|
||||
@@ -49,10 +78,17 @@ def parse_token_with_reason(token: str) -> Tuple[Optional[dict], Optional[str]]:
|
||||
return None, f"无法解析 token header: {type(e).__name__}: {e}"
|
||||
alg = header.get("alg") or "?"
|
||||
try:
|
||||
payload = pyjwt.decode(token, _signing_key(), algorithms=["HS256", "HS384", "HS512"])
|
||||
payload = pyjwt.decode(
|
||||
token,
|
||||
_signing_key(),
|
||||
algorithms=["HS256", "HS384", "HS512"],
|
||||
leeway=_JWT_LEEWAY_SECONDS,
|
||||
)
|
||||
except Exception as e:
|
||||
# 常见:ExpiredSignatureError / InvalidSignatureError / DecodeError / InvalidAlgorithmError
|
||||
return None, f"{type(e).__name__}: {e} (token alg={alg})"
|
||||
# 把 token 内 exp/iat 与本地时钟一起打出来,定位"时间漂移"类失败更直接
|
||||
skew_info = _describe_clock_skew(token)
|
||||
return None, f"{type(e).__name__}: {e} (token alg={alg}{skew_info})"
|
||||
sub = payload.get("sub")
|
||||
try:
|
||||
user_id = int(sub) if sub is not None else None
|
||||
|
||||
128
app/main.py
128
app/main.py
@@ -4,6 +4,8 @@ import json
|
||||
import sys
|
||||
import shutil
|
||||
import os
|
||||
import uuid
|
||||
import ctypes
|
||||
os.makedirs(cache_path,exist_ok=True)
|
||||
if not debug:
|
||||
today = datetime.datetime.now().strftime("%Y_%m_%d")
|
||||
@@ -186,17 +188,7 @@ class WindowAPI:
|
||||
if not result:
|
||||
return {'success': False, 'error': '用户取消'}
|
||||
path = result[0] if isinstance(result, (list, tuple)) else result
|
||||
try:
|
||||
import requests
|
||||
resp = requests.get(url.strip(), timeout=120, stream=True)
|
||||
resp.raise_for_status()
|
||||
with open(path, 'wb') as f:
|
||||
for chunk in resp.iter_content(chunk_size=65536):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
return {'success': True, 'path': path}
|
||||
except Exception as e:
|
||||
return {'success': False, 'error': str(e)}
|
||||
return self._download_url_to_path(url, path)
|
||||
|
||||
def save_file_from_url_new(self, url, default_filename='download.bin'):
|
||||
"""弹窗选择保存位置,从 url 下载文件并保存。根据文件后缀动态设置保存类型。"""
|
||||
@@ -230,18 +222,124 @@ class WindowAPI:
|
||||
return {'success': False, 'error': '用户取消'}
|
||||
|
||||
path = result[0] if isinstance(result, (list, tuple)) else result
|
||||
return self._download_url_to_path(url, path)
|
||||
|
||||
def save_file_from_url_with_progress(self, url, default_filename='download.bin', download_id=''):
|
||||
"""弹窗选择保存位置,从 url 下载文件并通过前端事件上报下载进度。"""
|
||||
print("调用到save_file_from_url_with_progress方法")
|
||||
if not url or not url.strip():
|
||||
return {'success': False, 'error': '下载地址为空'}
|
||||
|
||||
filename = str(default_filename or 'download.bin').strip() or 'download.bin'
|
||||
ext = os.path.splitext(filename)[1].lower()
|
||||
if ext == '.zip':
|
||||
file_types = ('ZIP 压缩包 (*.zip)', '所有文件 (*.*)')
|
||||
elif ext in ('.xlsx', '.xls'):
|
||||
file_types = ('Excel 文件 (*.xlsx;*.xls)', '所有文件 (*.*)')
|
||||
elif ext == '.csv':
|
||||
file_types = ('CSV 文件 (*.csv)', '所有文件 (*.*)')
|
||||
elif ext == '.txt':
|
||||
file_types = ('文本文件 (*.txt)', '所有文件 (*.*)')
|
||||
elif ext == '.json':
|
||||
file_types = ('JSON 文件 (*.json)', '所有文件 (*.*)')
|
||||
else:
|
||||
file_types = ('所有文件 (*.*)',)
|
||||
|
||||
result = self._window.create_file_dialog(
|
||||
webview.SAVE_DIALOG,
|
||||
save_filename=filename,
|
||||
file_types=file_types
|
||||
)
|
||||
if not result:
|
||||
return {'success': False, 'error': '用户取消'}
|
||||
|
||||
path = result[0] if isinstance(result, (list, tuple)) else result
|
||||
progress_id = str(download_id or uuid.uuid4())
|
||||
return self._download_url_to_path(url, path, progress_id)
|
||||
|
||||
def _download_url_to_path(self, url, path, download_id=''):
|
||||
"""先下载到临时文件,完整下载后再替换为最终文件,避免用户打开半成品。"""
|
||||
final_path = os.path.abspath(path)
|
||||
final_dir = os.path.dirname(final_path) or os.getcwd()
|
||||
final_name = os.path.basename(final_path)
|
||||
temp_path = os.path.join(final_dir, f".{final_name}.crdownload")
|
||||
last_progress_emit = 0.0
|
||||
last_progress_percent = -1
|
||||
try:
|
||||
import requests
|
||||
os.makedirs(final_dir, exist_ok=True)
|
||||
if os.path.exists(temp_path):
|
||||
os.remove(temp_path)
|
||||
resp = requests.get(url.strip(), timeout=120, stream=True)
|
||||
resp.raise_for_status()
|
||||
with open(path, 'wb') as f:
|
||||
total = int(resp.headers.get('Content-Length') or 0)
|
||||
downloaded = 0
|
||||
self._emit_download_progress(download_id, 'running', final_path, downloaded, total)
|
||||
with open(temp_path, 'wb') as f:
|
||||
self._hide_file(temp_path)
|
||||
for chunk in resp.iter_content(chunk_size=65536):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
return {'success': True, 'path': path}
|
||||
downloaded += len(chunk)
|
||||
percent = int(downloaded * 100 / total) if total else 0
|
||||
now_time = time.time()
|
||||
if percent != last_progress_percent or now_time - last_progress_emit >= 0.2:
|
||||
self._emit_download_progress(download_id, 'running', final_path, downloaded, total)
|
||||
last_progress_emit = now_time
|
||||
last_progress_percent = percent
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
self._show_file(temp_path)
|
||||
os.replace(temp_path, final_path)
|
||||
self._emit_download_progress(download_id, 'success', final_path, downloaded, total)
|
||||
return {'success': True, 'path': final_path}
|
||||
except Exception as e:
|
||||
try:
|
||||
if os.path.exists(temp_path):
|
||||
os.remove(temp_path)
|
||||
except Exception:
|
||||
pass
|
||||
self._emit_download_progress(download_id, 'failed', final_path, 0, 0, str(e))
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
def _emit_download_progress(self, download_id, status, path, downloaded, total, error=''):
|
||||
if not download_id:
|
||||
return
|
||||
payload = {
|
||||
'id': download_id,
|
||||
'status': status,
|
||||
'path': path,
|
||||
'downloaded': downloaded,
|
||||
'total': total,
|
||||
'percent': round(downloaded * 100 / total, 1) if total else 0,
|
||||
'error': error,
|
||||
}
|
||||
script = (
|
||||
"window.dispatchEvent(new CustomEvent('pywebview-download-progress', "
|
||||
f"{{ detail: {json.dumps(payload, ensure_ascii=False)} }}));"
|
||||
)
|
||||
try:
|
||||
self._window.evaluate_js(script)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _hide_file(self, path):
|
||||
"""Windows 下隐藏临时下载文件,减少用户误打开半成品的机会。"""
|
||||
if os.name != 'nt':
|
||||
return
|
||||
try:
|
||||
ctypes.windll.kernel32.SetFileAttributesW(path, 0x02)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _show_file(self, path):
|
||||
"""恢复普通文件属性,避免最终保存文件被隐藏。"""
|
||||
if os.name != 'nt':
|
||||
return
|
||||
try:
|
||||
ctypes.windll.kernel32.SetFileAttributesW(path, 0x80)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def save_template_xlsx(self):
|
||||
"""弹窗选择保存位置,将品牌文档格式模板 xlsx 保存到用户选择的位置。"""
|
||||
template_name = '品牌文档格式_模板.xlsx'
|
||||
@@ -440,7 +538,7 @@ def main():
|
||||
|
||||
api = WindowAPI(window)
|
||||
window.expose(api.close, api.minimize, api.maximize, api.toggle_maximize, generate_images, api.save_image, api.save_image_to_folder, api.select_folder, api.select_brand_xlsx_files, api.select_brand_folder,
|
||||
api.save_file_from_url, api.save_template_xlsx,api.save_template_zip,api.upload_file_to_java,api.open_external_url,api.enqueue_json,api.get_detail_del,api.save_file_from_url_new,api.get_device_id)
|
||||
api.save_file_from_url, api.save_template_xlsx,api.save_template_zip,api.upload_file_to_java,api.open_external_url,api.enqueue_json,api.get_detail_del,api.save_file_from_url_new,api.save_file_from_url_with_progress,api.get_device_id)
|
||||
webview.start(
|
||||
debug=True,
|
||||
storage_path=cache_path,
|
||||
|
||||
@@ -5,10 +5,10 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>格式转换 - 数富AI</title>
|
||||
<script type="module" crossorigin src="/assets/convert.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-3bFwh9Cg.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/brand-D__LhhPM.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-B7UKcPVJ.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/convert-BKSNvX8i.css">
|
||||
<link rel="modulepreload" crossorigin href="/assets/java-modules-B8c-YG5x.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/brand-COze15GJ.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/java-modules-l5anrOZ2.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/convert-6oxZNMye.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
|
||||
@@ -5,10 +5,10 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>数据去重 - 数富AI</title>
|
||||
<script type="module" crossorigin src="/assets/dedupe.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-3bFwh9Cg.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/brand-D__LhhPM.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-B7UKcPVJ.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/dedupe-DNlVfFj-.css">
|
||||
<link rel="modulepreload" crossorigin href="/assets/java-modules-B8c-YG5x.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/brand-COze15GJ.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/java-modules-l5anrOZ2.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/dedupe-DpfPYQDv.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
|
||||
@@ -5,11 +5,11 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>删除品牌 - 数富AI</title>
|
||||
<script type="module" crossorigin src="/assets/delete-brand.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-3bFwh9Cg.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/brand-D__LhhPM.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/java-modules-B8c-YG5x.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/brand-COze15GJ.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/categorized-timers-JPA-olTr.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-B7UKcPVJ.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/delete-brand-BJod9K65.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/java-modules-l5anrOZ2.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/delete-brand-DZ3x5hOX.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
|
||||
@@ -5,10 +5,10 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>数据拆分 - 数富AI</title>
|
||||
<script type="module" crossorigin src="/assets/split.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-3bFwh9Cg.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/brand-D__LhhPM.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-B7UKcPVJ.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/split-BoBVrLdC.css">
|
||||
<link rel="modulepreload" crossorigin href="/assets/java-modules-B8c-YG5x.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/brand-COze15GJ.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/java-modules-l5anrOZ2.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/split-CR1PtDQE.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
|
||||
@@ -9,12 +9,12 @@ import java.util.function.Predicate;
|
||||
|
||||
public final class FailedStatusRowFilter {
|
||||
|
||||
private static final String STATUS_HEADER_CN = "\u72b6\u6001";
|
||||
private static final String FAILED_STATUS_ERROR = "\u9519\u8bef";
|
||||
private static final String FAILED_STATUS_FAILED_CN = "\u5931\u8d25";
|
||||
private static final String STATUS_HEADER_CN = "状态";
|
||||
private static final String FAILED_STATUS_ERROR = "错误";
|
||||
private static final String FAILED_STATUS_FAILED_CN = "失败";
|
||||
private static final String FAILED_STATUS_FAILED_EN = "failed";
|
||||
private static final String NO_MATCHED_ROWS_MESSAGE =
|
||||
"\u72b6\u6001\u5217\u8fc7\u6ee4\u540e\u672a\u627e\u5230 \u9519\u8bef/\u5931\u8d25/FAILED \u6570\u636e";
|
||||
"状态列过滤后未找到 错误/失败/FAILED 数据";
|
||||
|
||||
private static final Set<String> STATUS_HEADER_ALIASES = Set.of(
|
||||
normalizeHeader(STATUS_HEADER_CN),
|
||||
@@ -56,8 +56,8 @@ public final class FailedStatusRowFilter {
|
||||
|
||||
/**
|
||||
* 暴露表头归一化逻辑给其它模块复用,避免每个模块各自维护一份规则。
|
||||
* 处理:去 BOM、全角空格转半角、首尾 trim、连续空白合一、转小写,
|
||||
* 并剥除常见分隔符(含中文全角括号、方括号、冒号)。
|
||||
* 处理:去 BOM、全角空格转半角、首尾 trim、连续空白合并、转小写,
|
||||
* 并剔除常见分隔符(含中文全角括号、方括号、冒号)。
|
||||
*/
|
||||
public static String canonicalizeHeader(String value) {
|
||||
return normalizeHeader(value);
|
||||
@@ -92,8 +92,8 @@ public final class FailedStatusRowFilter {
|
||||
}
|
||||
|
||||
private static String normalizeHeader(String value) {
|
||||
// 扩展:兼容中文全角括号 ()、中文方括号 【】、中文冒号 :,避免
|
||||
// "状态(结果)"、"标题维度(商标)" 等全角括号表头被识别失败。
|
||||
// 兼容中文全角括号()、中文方括号【】、中文冒号:,
|
||||
// 避免“状态(结果)”“标题维度(商标)”等表头识别失败。
|
||||
return normalizeValue(value).replaceAll("[\\s_\\-()\\[\\]{}::()【】]+", "");
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,14 @@ public class SimilarAsinProperties {
|
||||
private String cozeToken = "";
|
||||
private List<CozeCredential> cozeCredentials = new ArrayList<>();
|
||||
private int cozeCredentialStripeSize = 0;
|
||||
private int cozeBatchSize = 10;
|
||||
/**
|
||||
* P0-1:单次提交 Coze 工作流的 row 数量。
|
||||
* 历史值 10,在含 puzzle 多图行的场景下频繁触发 720712008
|
||||
* "node executed out of limit: 1000"。降到 3 以避免节点上限被打爆。
|
||||
* 出现持续 720712008 时还会被 P1-1 滑窗自适应再降到 1。
|
||||
* 不影响 AppearancePatentProperties 的同名值。
|
||||
*/
|
||||
private int cozeBatchSize = 3;
|
||||
private int cozeConnectTimeoutMillis = 10000;
|
||||
private int cozeReadTimeoutMillis = 60000;
|
||||
private int cozePollIntervalMillis = 30000;
|
||||
@@ -24,6 +31,54 @@ public class SimilarAsinProperties {
|
||||
private int staleTimeoutMinutes = 30;
|
||||
private String staleFinalizeCron = "0 */2 * * * *";
|
||||
|
||||
/**
|
||||
* 同一 credential 两次提交之间的最小间隔(毫秒)。
|
||||
* 历史值硬编码 30000(持锁 sleep),导致单凭证仅 2 batch/分钟。
|
||||
* 几千行任务场景下成为提交吞吐瓶颈,下调到 5000ms 并改为锁外冷却。
|
||||
* 出现 Coze 限流加重时可通过 AIIMAGE_SIMILAR_ASIN_COZE_SUBMIT_MIN_INTERVAL_MILLIS 调高。
|
||||
*/
|
||||
private long cozeSubmitMinIntervalMillis = 5000L;
|
||||
|
||||
/**
|
||||
* 末尾零头 batch 的强制 flush 阈值(分钟):当不足 cozeBatchSize 的零头 row
|
||||
* 长时间挂着(Python 慢回传)时触发提交。
|
||||
* 任务级实测:345 行 / 4h 总耗时中,约 2-3 小时是 batch 永远凑不满 batchSize 在等下一波回传,
|
||||
* 把阈值从 15 调到 1:最多 60s 后 1-2 行也强制提交,让 Coze 提交侧持续进票,
|
||||
* 总耗时降到与 Python 回传节奏接近。配合 cozeBatchSize=3、cozeSubmitMinIntervalMillis=5000,
|
||||
* 实际不会触发 Coze 限流。出现限流加重再调回 5/10。
|
||||
*/
|
||||
private int cozeFlushPendingMinutes = 1;
|
||||
|
||||
/**
|
||||
* 同 batch retry + split retry 共享的最大重试次数。原硬编码 5。
|
||||
*/
|
||||
private int cozeSubmitMaxRetryCount = 5;
|
||||
|
||||
/**
|
||||
* 图片嵌入下载线程池大小。原 SimilarAsinImageEmbedder.DOWNLOAD_POOL_SIZE = 8。
|
||||
* P2-10:1000+ 行 ×3 列图片场景下,pool=16 仍是 assemble 阶段瓶颈(实测下载 244s/918s),
|
||||
* 提到 32 配合 retry=2、global deadline 显著拉低尾延迟;
|
||||
* 受 2GB 堆约束,单图缩略图维持 300KB 以内,整体内存峰值 ≈ 32 * 300KB ≈ 10MB。
|
||||
*/
|
||||
private int imageDownloadPoolSize = 32;
|
||||
|
||||
/**
|
||||
* 单张图片下载超时(秒)。
|
||||
* P2-10:放宽到 8 + retry=1 在快源(aiproxy/m.media-amazon)下没问题,
|
||||
* 但慢源(cbu01.alicdn)会一直挂 8s 才进入 retry,整体串行时间放大。
|
||||
* 调到 5s + retry=2,让慢源更早重试新连接,单图最坏耗时 ≈ 5s * (1+2) = 15s。
|
||||
*/
|
||||
private int imageDownloadTimeoutSeconds = 5;
|
||||
|
||||
/**
|
||||
* assemble 阶段 taskImageCache 的字节上限。
|
||||
* 默认 256MB:5000 行 × 3 列 × 平均 100KB = 1.5GB 远超 2GB 堆,
|
||||
* 用 BoundedImageCache 按字节累计 LRU 淘汰避免爆堆。
|
||||
* 由于 embed() 写完即 remove(),活跃图片字节通常 ≤ 100MB,仅在极端 prefetch 领先场景才会触发淘汰。
|
||||
* 出现淘汰过频影响命中率时可上调到 512MB;2GB 堆约束下不建议超过 768MB。
|
||||
*/
|
||||
private long imageCacheMaxBytes = 256L * 1024L * 1024L;
|
||||
|
||||
/**
|
||||
* 是否在 Coze 请求 parameters 中附带 api_key 字段。
|
||||
* 默认 true:线上 Coze 工作流将该字段视为必填,缺失会得到 4000
|
||||
@@ -41,6 +96,31 @@ public class SimilarAsinProperties {
|
||||
*/
|
||||
private boolean cozeUseLegacyItemFieldOrder = false;
|
||||
|
||||
/**
|
||||
* 是否启用 P0-3 merge 增量缓冲:每个 batch DONE 时仅缓冲 cozeRows,
|
||||
* 不立即合并到 chunk;finalize 前一次性按 chunkScopeHash 分组合并,
|
||||
* 把 OSS chunk 读写从 1000+ 次降到 chunk 数量级。
|
||||
* 仅作用于"正常 poll DONE"路径;失败 batch / 单 batch 任务 / 其他
|
||||
* 11 个 mergeCozeRowsIntoChunk 调用点保留原立即 merge 行为。
|
||||
* 出现问题时可通过 AIIMAGE_SIMILAR_ASIN_COZE_RESULT_BUFFER_ENABLED=false
|
||||
* 一键回滚到老路径。
|
||||
*/
|
||||
private boolean cozeResultBufferEnabled = true;
|
||||
|
||||
/**
|
||||
* P0-4:单 credential 抢 Coze 提交锁的最长等待时间(毫秒)。
|
||||
* 原硬编码 1000ms,在高并发 split retry 时大量抛 "Coze submit throttle lock timeout"
|
||||
* 并把整批行 markFailed。应与 cozeSubmitMinIntervalMillis(5000ms)保持 1.5-2 倍关系,
|
||||
* 默认 10000ms 给抢锁更多时间。
|
||||
*/
|
||||
private long cozeSubmitLockWaitMillis = 10000L;
|
||||
|
||||
/**
|
||||
* P0-4:抢 Coze 提交锁失败后下次重试间隔(毫秒)。
|
||||
* 原硬编码 500ms,会在指数退避算法中作为基础值(500/1000/2000/4000ms 上限 4000)。
|
||||
*/
|
||||
private long cozeSubmitLockRetryDelayMillis = 500L;
|
||||
|
||||
@Data
|
||||
public static class CozeCredential {
|
||||
private String name;
|
||||
|
||||
@@ -355,13 +355,14 @@ public class AppearancePatentCozeClient {
|
||||
List<String> rowIds = rows.stream().map(row -> nonBlank(row.getId(), "")).toList();
|
||||
List<String> asins = rows.stream().map(row -> nonBlank(row.getAsin(), "")).toList();
|
||||
List<String> countries = rows.stream().map(row -> nonBlank(row.getCountry(), "")).toList();
|
||||
List<String> skus = rows.stream().map(row -> nonBlank(row.getSku(), "")).toList();
|
||||
List<String> titles = rows.stream().map(row -> nonBlank(row.getTitle(), row.getAsin())).toList();
|
||||
List<String> urls = rows.stream().map(row -> nonBlank(row.getUrl(), "")).toList();
|
||||
|
||||
Map<String, Object> parameters = new LinkedHashMap<>();
|
||||
parameters.put("title_list", titles);
|
||||
parameters.put("url_list", urls);
|
||||
parameters.put("items", buildItemObjects(rows, groupKeys, rowIds, asins, countries, titles, urls));
|
||||
parameters.put("items", buildItemObjects(rows, groupKeys, rowIds, asins, countries, skus, titles, urls));
|
||||
// 前端没填 prompt 就一律不向 Coze 透传该字段,避免无关默认提示词污染工作流。
|
||||
if (prompt != null && !prompt.isBlank()) {
|
||||
parameters.put("prompt", prompt);
|
||||
@@ -403,6 +404,7 @@ public class AppearancePatentCozeClient {
|
||||
List<String> rowIds,
|
||||
List<String> asins,
|
||||
List<String> countries,
|
||||
List<String> skus,
|
||||
List<String> titles,
|
||||
List<String> urls) {
|
||||
List<Map<String, Object>> items = new ArrayList<>(rows.size());
|
||||
@@ -412,6 +414,7 @@ public class AppearancePatentCozeClient {
|
||||
item.put("row_id", rowIds.get(i));
|
||||
item.put("asin", asins.get(i));
|
||||
item.put("country", countries.get(i));
|
||||
item.put("sku", skus.get(i));
|
||||
item.put("title", titles.get(i));
|
||||
item.put("url", urls.get(i));
|
||||
items.add(item);
|
||||
@@ -465,7 +468,9 @@ public class AppearancePatentCozeClient {
|
||||
text(firstNonNull(node.get("patent_reason"),
|
||||
firstNonNull(firstNonNull(node.get("patentReason"), node.get("patent reason")),
|
||||
firstNonNull(itemNode.get("patent_reason"),
|
||||
firstNonNull(itemNode.get("patentReason"), itemNode.get("patent reason"))))))
|
||||
firstNonNull(itemNode.get("patentReason"), itemNode.get("patent reason")))))),
|
||||
text(firstNonNull(firstNonNull(node.get("score"), node.get("Score")),
|
||||
firstNonNull(itemNode.get("score"), itemNode.get("Score"))))
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -570,6 +575,7 @@ public class AppearancePatentCozeClient {
|
||||
row.setTitleReason(result.titleReason());
|
||||
row.setAppearanceReason(result.appearanceReason());
|
||||
row.setPatentReason(result.patentReason());
|
||||
row.setScore(result.score());
|
||||
}
|
||||
|
||||
private RestClient restClient() {
|
||||
@@ -588,6 +594,7 @@ public class AppearancePatentCozeClient {
|
||||
row.setId(source.getId());
|
||||
row.setAsin(source.getAsin());
|
||||
row.setCountry(source.getCountry());
|
||||
row.setSku(source.getSku());
|
||||
row.setUrl(source.getUrl());
|
||||
row.setTitle(source.getTitle());
|
||||
row.setError(source.getError());
|
||||
@@ -600,13 +607,14 @@ public class AppearancePatentCozeClient {
|
||||
row.setTitleReason(source.getTitleReason());
|
||||
row.setAppearanceReason(source.getAppearanceReason());
|
||||
row.setPatentReason(source.getPatentReason());
|
||||
row.setScore(source.getScore());
|
||||
return row;
|
||||
}
|
||||
|
||||
private AppearancePatentResultRowDto markFailed(AppearancePatentResultRowDto row, String failureMessage) {
|
||||
String reviewMessage = failureMessage == null || failureMessage.isBlank()
|
||||
? "Coze 检测失败,待人工复核"
|
||||
: "Coze 检测失败,待人工复核:" + failureMessage;
|
||||
? "Coze 检测失败"
|
||||
: "Coze 检测失败:" + failureMessage;
|
||||
if (row.getError() == null || row.getError().isBlank()) {
|
||||
row.setError(failureMessage);
|
||||
}
|
||||
@@ -797,7 +805,9 @@ public class AppearancePatentCozeClient {
|
||||
|| item.has("appearanceReason")
|
||||
|| item.has("patent_reason")
|
||||
|| item.has("patentReason")
|
||||
|| item.has("patent reason"));
|
||||
|| item.has("patent reason")
|
||||
|| item.has("score")
|
||||
|| item.has("Score"));
|
||||
}
|
||||
|
||||
private boolean isSuccessfulWorkflowStatus(String status) {
|
||||
@@ -1054,7 +1064,8 @@ public class AppearancePatentCozeClient {
|
||||
String status,
|
||||
String titleReason,
|
||||
String appearanceReason,
|
||||
String patentReason
|
||||
String patentReason,
|
||||
String score
|
||||
) {
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,10 @@ public class AppearancePatentResultRowDto {
|
||||
@Schema(description = "站点或国家。来自 Excel 的国家列,例如英国、德国、法国。", example = "英国")
|
||||
private String country;
|
||||
|
||||
@Schema(description = "商品 SKU。Java 调用 Coze 时会放入 items[].sku。", example = "SKU-001")
|
||||
@JsonAlias({"SKU", "sellerSku", "seller_sku", "merchantSku", "merchant_sku", "商品SKU", "商品 sku", "库存SKU"})
|
||||
private String sku;
|
||||
|
||||
@Schema(description = "商品主图或待检测图片 URL。Java 调用 Coze 时会放入 url_list。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg")
|
||||
@JsonAlias({
|
||||
"imageUrl", "image_url", "imgUrl", "img_url", "pictureUrl", "picture_url",
|
||||
@@ -78,6 +82,10 @@ public class AppearancePatentResultRowDto {
|
||||
@Schema(description = "Coze patent reason", accessMode = Schema.AccessMode.READ_ONLY)
|
||||
private String patentReason;
|
||||
|
||||
@JsonAlias({"score", "Score", "评分"})
|
||||
@Schema(description = "Coze 回流的评分(外观维度),透传到最终 xlsx 第一张 sheet 的“评分”列。", accessMode = Schema.AccessMode.READ_ONLY)
|
||||
private String score;
|
||||
|
||||
/**
|
||||
* 仅内存生命周期标记,标识当前 status 是 markFailed 时合成出来的(而不是用户/Python 真实回传)。
|
||||
* 不入库、不参与 chunk 序列化(@JsonIgnore),用于导出 / 重新上传判定时区分"系统合成 FAILED"与"用户真正失败",
|
||||
|
||||
@@ -36,6 +36,9 @@ public class AppearancePatentParsedRowVo {
|
||||
@Schema(description = "价格。", example = "12.99")
|
||||
private String price;
|
||||
|
||||
@Schema(description = "商品 SKU。Java 调用 Coze 时会放入 items[].sku。", example = "SKU-001")
|
||||
private String sku;
|
||||
|
||||
@Schema(description = "商品图片 URL 或商品 URL,供 Coze 检测使用。", example = "https://webstatic.aiproxy.vip/output/demo.jpg")
|
||||
private String url;
|
||||
|
||||
|
||||
@@ -126,9 +126,9 @@ public class AppearancePatentTaskService {
|
||||
"价格",
|
||||
"标题维度(商标)",
|
||||
"外观维度(外观设计专利)",
|
||||
"专利维度(发明/实用新型专利)",
|
||||
"结论",
|
||||
"状态"
|
||||
"状态",
|
||||
"评分"
|
||||
);
|
||||
|
||||
private final LocalFileStorageService localFileStorageService;
|
||||
@@ -1254,6 +1254,7 @@ public class AppearancePatentTaskService {
|
||||
row.setId(sibling.getDisplayId());
|
||||
row.setAsin(sibling.getAsin());
|
||||
row.setCountry(sibling.getCountry());
|
||||
row.setSku(firstNonBlank(representative.getSku(), sibling.getSku()));
|
||||
row.setUrl(firstNonBlank(representative.getUrl(), sibling.getUrl()));
|
||||
row.setTitle(firstNonBlank(representative.getTitle(), sibling.getTitle()));
|
||||
row.setError(representative.getError());
|
||||
@@ -1265,6 +1266,7 @@ public class AppearancePatentTaskService {
|
||||
row.setTitleReason(representative.getTitleReason());
|
||||
row.setAppearanceReason(representative.getAppearanceReason());
|
||||
row.setPatentReason(representative.getPatentReason());
|
||||
row.setScore(representative.getScore());
|
||||
row.setDone(representative.getDone());
|
||||
return row;
|
||||
}
|
||||
@@ -2176,8 +2178,8 @@ public class AppearancePatentTaskService {
|
||||
|| normalized.contains("execution limit")
|
||||
|| normalized.contains("720712008")
|
||||
|| normalized.contains("720701002")
|
||||
|| normalized.contains("\u5de5\u4f5c\u6d41\u8282\u70b9\u6267\u884c\u8d85\u9650")
|
||||
|| normalized.contains("\u8c03\u7528\u8d85\u65f6");
|
||||
|| normalized.contains("工作流节点执行超限")
|
||||
|| normalized.contains("调用超时");
|
||||
}
|
||||
|
||||
private boolean isRetryableCozeFailure(String failureMessage) {
|
||||
@@ -2195,10 +2197,10 @@ public class AppearancePatentTaskService {
|
||||
|| normalized.contains("720712008")
|
||||
|| normalized.contains("720701002")
|
||||
|| normalized.contains("plugin limit")
|
||||
|| normalized.contains("\u9650\u6d41")
|
||||
|| normalized.contains("\u7a0d\u540e\u91cd\u8bd5")
|
||||
|| normalized.contains("\u5de5\u4f5c\u6d41\u8282\u70b9\u6267\u884c\u8d85\u9650")
|
||||
|| normalized.contains("\u8c03\u7528\u8d85\u65f6");
|
||||
|| normalized.contains("限流")
|
||||
|| normalized.contains("稍后重试")
|
||||
|| normalized.contains("工作流节点执行超限")
|
||||
|| normalized.contains("调用超时");
|
||||
}
|
||||
|
||||
private void updateCozeStateRunning(TaskScopeStateEntity state, String error) {
|
||||
@@ -3213,6 +3215,7 @@ public class AppearancePatentTaskService {
|
||||
List<String> resultHeaders = new ArrayList<>(RESULT_HEADERS);
|
||||
resultHeaders.add(4, "标题");
|
||||
resultHeaders.add(5, "图片链接");
|
||||
resultHeaders.add(6, "sku");
|
||||
Row header = sheet.createRow(0);
|
||||
for (int i = 0; i < resultHeaders.size(); i++) {
|
||||
Cell cell = header.createCell(i);
|
||||
@@ -3236,11 +3239,12 @@ public class AppearancePatentTaskService {
|
||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getPrice(), ""));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getTitle(), "") : firstNonBlank(resultRow.getTitle(), parsedRow.getTitle()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getUrl(), "") : firstNonBlank(resultRow.getUrl(), parsedRow.getUrl()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getSku(), "") : firstNonBlank(resultRow.getSku(), parsedRow.getSku()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? missingReason : userFacingCozeCellValue(resultRow, resultRow.getTitleRisk()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? missingReason : userFacingCozeCellValue(resultRow, resultRow.getAppearanceRisk()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? missingReason : userFacingCozeCellValue(resultRow, resultRow.getPatentRisk()));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? "未送检" : userFacingConclusion(resultRow));
|
||||
row.createCell(col).setCellValue(resultRow == null ? "" : userFacingStatus(resultRow));
|
||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingStatus(resultRow));
|
||||
row.createCell(col).setCellValue(resultRow == null ? "" : firstNonBlank(resultRow.getScore(), ""));
|
||||
}
|
||||
writeReasonSheet(workbook, headerStyle, rowsToWrite, resultMap);
|
||||
workbook.write(fos);
|
||||
@@ -3376,6 +3380,8 @@ public class AppearancePatentTaskService {
|
||||
int asinCol = findRequiredHeader(headerMap, "asin");
|
||||
int countryCol = findRequiredHeader(headerMap, "国家", "country");
|
||||
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
|
||||
int skuCol = findOptionalHeaderExact(headerMap,
|
||||
"sku", "seller sku", "seller_sku", "merchant sku", "merchant_sku", "商品sku", "商品 sku", "库存sku");
|
||||
int urlCol = findOptionalHeaderExact(headerMap,
|
||||
"url", "rul", "link", "image", "img", "pic", "picture",
|
||||
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
|
||||
@@ -3423,6 +3429,7 @@ public class AppearancePatentTaskService {
|
||||
vo.setAsin(asin);
|
||||
vo.setCountry(country);
|
||||
vo.setPrice(priceCol >= 0 ? cell(row, priceCol, formatter) : "");
|
||||
vo.setSku(skuCol >= 0 ? cell(row, skuCol, formatter) : "");
|
||||
vo.setUrl(urlCol >= 0 ? cell(row, urlCol, formatter) : "");
|
||||
vo.setTitle(titleCol >= 0 ? cell(row, titleCol, formatter) : "");
|
||||
parsedRows.add(new ParsedAppearanceRow(vo, statusCol >= 0 ? cell(row, statusCol, formatter) : ""));
|
||||
@@ -3467,9 +3474,11 @@ public class AppearancePatentTaskService {
|
||||
for (List<AppearancePatentParsedRowVo> siblings : rowsByBaseId.values()) {
|
||||
String title = "";
|
||||
String url = "";
|
||||
String sku = "";
|
||||
for (AppearancePatentParsedRowVo sibling : siblings) {
|
||||
title = firstNonBlank(title, sibling.getTitle());
|
||||
url = firstNonBlank(url, sibling.getUrl());
|
||||
sku = firstNonBlank(sku, sibling.getSku());
|
||||
}
|
||||
for (AppearancePatentParsedRowVo sibling : siblings) {
|
||||
if (normalize(sibling.getTitle()).isBlank()) {
|
||||
@@ -3478,6 +3487,9 @@ public class AppearancePatentTaskService {
|
||||
if (normalize(sibling.getUrl()).isBlank()) {
|
||||
sibling.setUrl(url);
|
||||
}
|
||||
if (normalize(sibling.getSku()).isBlank()) {
|
||||
sibling.setSku(sku);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4041,11 +4053,12 @@ public class AppearancePatentTaskService {
|
||||
if (!normalizedValue.isBlank() && !isTechnicalCozeFailure(normalizedValue)) {
|
||||
return value;
|
||||
}
|
||||
// coze 技术性失败:有错误信息则放入错误信息,没有则留空
|
||||
if (row != null && isTechnicalCozeFailure(row.getError())) {
|
||||
return "待人工复核";
|
||||
return firstNonBlank(row.getError(), "");
|
||||
}
|
||||
if (row != null && isFailedCozeStatusValue(row.getStatus())) {
|
||||
return "待人工复核";
|
||||
return firstNonBlank(row.getError(), "");
|
||||
}
|
||||
return firstNonBlank(value, "");
|
||||
}
|
||||
@@ -4058,11 +4071,12 @@ public class AppearancePatentTaskService {
|
||||
if (!conclusion.isBlank() && !isTechnicalCozeFailure(conclusion)) {
|
||||
return row.getConclusion();
|
||||
}
|
||||
// coze 技术性失败:有错误信息则放入错误信息,没有则留空
|
||||
if (isTechnicalCozeFailure(row.getError())) {
|
||||
return "待人工复核";
|
||||
return firstNonBlank(row.getError(), "");
|
||||
}
|
||||
if (isFailedCozeStatusValue(row.getStatus())) {
|
||||
return "待人工复核";
|
||||
return firstNonBlank(row.getError(), "");
|
||||
}
|
||||
return firstNonBlank(row.getConclusion(), "");
|
||||
}
|
||||
@@ -4070,8 +4084,8 @@ public class AppearancePatentTaskService {
|
||||
/**
|
||||
* 导出最终 xlsx 时的状态展示:
|
||||
* - 如果 status 是 markFailed 合成的(failureSyntheticStatus=true),
|
||||
* 且 row 实际并没有任何风险维度结果,导出层降级为"待人工复核",避免用户原样
|
||||
* 重新上传时该行被反复识别为失败行重新触发 Coze。
|
||||
* 且 row 实际并没有任何风险维度结果,导出层展示错误信息(有则放入,无则留空),
|
||||
* 避免用户原样重新上传时该行被反复识别为失败行重新触发 Coze。
|
||||
* - 真正业务侧失败 / Python 真实回传 FAILED 仍按原值显示。
|
||||
*/
|
||||
private String userFacingStatus(AppearancePatentResultRowDto row) {
|
||||
@@ -4082,7 +4096,8 @@ public class AppearancePatentTaskService {
|
||||
if (!row.isFailureSyntheticStatus()) {
|
||||
return original;
|
||||
}
|
||||
return "待人工复核";
|
||||
// 合成状态:有错误信息则放入错误信息,没有则留空
|
||||
return firstNonBlank(row.getError(), "");
|
||||
}
|
||||
|
||||
private boolean isTechnicalCozeFailure(String value) {
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
package com.nanri.aiimage.modules.collectdata.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataParseRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataTaskBatchRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemsPageVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataParseVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/collect-data")
|
||||
@Tag(name = "采集数据", description = "采集数据任务接口。前端上传 Excel 后由 Java 解析并落库;Python 端按页拉取明细数据进行采集。")
|
||||
public class CollectDataController {
|
||||
|
||||
private final CollectDataService service;
|
||||
|
||||
@PostMapping("/parse")
|
||||
@Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件,按行入库到 biz_collect_data_item,并保存任务筛选条件。任务初始状态为 PENDING。")
|
||||
public ApiResponse<CollectDataParseVo> parse(@Valid @RequestBody CollectDataParseRequest request) {
|
||||
return ApiResponse.success(service.parseAndCreateTask(request));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/activate")
|
||||
@Operation(summary = "激活任务", description = "前端推送 Python 队列前调用,将任务状态从 PENDING 修改为 RUNNING,表示已交给 Python 处理。")
|
||||
public ApiResponse<Void> activate(
|
||||
@Parameter(description = "采集任务 ID", required = true, example = "9001")
|
||||
@PathVariable Long taskId,
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
service.activateTask(taskId, userId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@GetMapping("/tasks/{taskId}/items")
|
||||
@Operation(summary = "分页获取任务明细数据", description = "供 Python 端拉取,默认每页 50 条;返回任务关联的筛选条件,便于 Python 端按筛选条件采集。")
|
||||
public ApiResponse<CollectDataItemsPageVo> items(
|
||||
@Parameter(description = "采集任务 ID", required = true, example = "9001")
|
||||
@PathVariable Long taskId,
|
||||
@Parameter(description = "用户 ID,可选;传入时会校验任务归属")
|
||||
@RequestParam(value = "user_id", required = false) Long userId,
|
||||
@Parameter(description = "页码,从 1 开始", example = "1")
|
||||
@RequestParam(value = "page", required = false, defaultValue = "1") Integer page,
|
||||
@Parameter(description = "每页条数,默认 50,最大 200", example = "50")
|
||||
@RequestParam(value = "page_size", required = false, defaultValue = "50") Integer pageSize) {
|
||||
return ApiResponse.success(service.getItemsPage(taskId, userId, page, pageSize));
|
||||
}
|
||||
|
||||
@GetMapping("/dashboard")
|
||||
@Operation(summary = "查询采集任务总览")
|
||||
public ApiResponse<CollectDataDashboardVo> dashboard(
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
return ApiResponse.success(service.dashboard(userId));
|
||||
}
|
||||
|
||||
@GetMapping("/history")
|
||||
@Operation(summary = "查询采集任务历史")
|
||||
public ApiResponse<CollectDataHistoryVo> history(
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId,
|
||||
@Parameter(description = "返回数量上限,默认 50,最大 100", example = "50")
|
||||
@RequestParam(value = "limit", required = false, defaultValue = "50") Integer limit) {
|
||||
return ApiResponse.success(service.history(userId, limit));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量查询任务进度")
|
||||
public ApiResponse<CollectDataTaskBatchVo> progressBatch(@Valid @RequestBody CollectDataTaskBatchRequest request) {
|
||||
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
||||
}
|
||||
|
||||
@DeleteMapping("/tasks/{taskId}")
|
||||
@Operation(summary = "删除任务", description = "删除任务及其明细行、关联结果记录。")
|
||||
public ApiResponse<Void> deleteTask(
|
||||
@Parameter(description = "采集任务 ID", required = true, example = "9001")
|
||||
@PathVariable Long taskId,
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
service.deleteTask(taskId, userId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@DeleteMapping("/history/{resultId}")
|
||||
@Operation(summary = "删除历史记录")
|
||||
public ApiResponse<Void> deleteHistory(
|
||||
@Parameter(description = "结果记录 ID", required = true, example = "1001")
|
||||
@PathVariable Long resultId,
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
service.deleteHistory(resultId, userId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@GetMapping("/country-preference")
|
||||
@Operation(summary = "查询国家处理顺序", description = "返回当前用户保存的国家代码列表,顺序即处理顺序。")
|
||||
public ApiResponse<CollectDataCountryPreferenceVo> getCountryPreference(
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
return ApiResponse.success(service.getCountryPreference(userId));
|
||||
}
|
||||
|
||||
@PutMapping("/country-preference")
|
||||
@Operation(summary = "保存国家处理顺序", description = "保存当前用户的国家勾选与处理顺序。")
|
||||
public ApiResponse<CollectDataCountryPreferenceVo> saveCountryPreference(
|
||||
@Valid @RequestBody CollectDataCountryPreferenceSaveRequest request) {
|
||||
return ApiResponse.success(service.saveCountryPreference(request));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.collectdata.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataCountryPrefEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface CollectDataCountryPrefMapper extends BaseMapper<CollectDataCountryPrefEntity> {
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.nanri.aiimage.modules.collectdata.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import org.apache.ibatis.annotations.Insert;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface CollectDataItemMapper extends BaseMapper<CollectDataItemEntity> {
|
||||
|
||||
@Insert("""
|
||||
<script>
|
||||
INSERT INTO biz_collect_data_item
|
||||
(task_id, row_index, source_file_key, source_filename, keyword, status_value, extra_json, created_at)
|
||||
VALUES
|
||||
<foreach collection="rows" item="row" separator=",">
|
||||
(#{row.taskId}, #{row.rowIndex}, #{row.sourceFileKey}, #{row.sourceFilename},
|
||||
#{row.keyword}, #{row.statusValue}, #{row.extraJson}, #{row.createdAt})
|
||||
</foreach>
|
||||
</script>
|
||||
""")
|
||||
int insertBatch(@Param("rows") List<CollectDataItemEntity> rows);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "保存采集数据五国处理顺序:至少 1 个、最多 5 个,代码须为 DE、UK、FR、IT、ES,且不能重复")
|
||||
public class CollectDataCountryPreferenceSaveRequest {
|
||||
|
||||
@NotNull(message = "user_id 不能为空")
|
||||
@JsonProperty("user_id")
|
||||
@Schema(description = "当前用户 ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
|
||||
private Long userId;
|
||||
|
||||
@NotEmpty(message = "country_codes 不能为空")
|
||||
@Size(max = 5, message = "country_codes 最多 5 项")
|
||||
@JsonProperty("country_codes")
|
||||
@Schema(description = "国家代码列表(DE、UK、FR、IT、ES),顺序即处理顺序", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<String> countryCodes = new ArrayList<>();
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import lombok.Data;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据筛选条件")
|
||||
public class CollectDataFiltersDto {
|
||||
|
||||
@Schema(description = "金额(手动输入),可为空", example = "59.99")
|
||||
private BigDecimal amount;
|
||||
|
||||
@JsonAlias({"min_amount"})
|
||||
@Schema(description = "最小金额,可为空", example = "10.00")
|
||||
private BigDecimal minAmount;
|
||||
|
||||
@JsonAlias({"max_amount"})
|
||||
@Schema(description = "最大金额,可为空", example = "59.99")
|
||||
private BigDecimal maxAmount;
|
||||
|
||||
@Schema(description = "是否启用排名条件")
|
||||
private Boolean rank;
|
||||
|
||||
@Schema(description = "是否包含 FBA 商品")
|
||||
private Boolean fba;
|
||||
|
||||
@Schema(description = "是否包含 FBM 商品")
|
||||
private Boolean fbm;
|
||||
|
||||
@Schema(description = "国家代码列表,按用户拖拽顺序保存", example = "[\"DE\",\"UK\"]")
|
||||
private List<String> countryCodes = new ArrayList<>();
|
||||
|
||||
@JsonAlias({"country_code"})
|
||||
@Schema(description = "单选国家代码,采集数据页面当前只选择一个国家", example = "DE")
|
||||
private String countryCode;
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据解析请求:提交筛选条件和上传后的 Excel 文件,由后端解析并落库。")
|
||||
public class CollectDataParseRequest {
|
||||
|
||||
@JsonProperty("user_id")
|
||||
@NotNull
|
||||
@Schema(description = "当前用户 ID", example = "1", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private Long userId;
|
||||
|
||||
@NotEmpty
|
||||
@Schema(description = "已上传的 Excel 文件列表", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<CollectDataSourceFileDto> files;
|
||||
|
||||
@JsonProperty("task_type")
|
||||
@Schema(description = "任务类型,前端可自定义传入;为空时默认 collect-data", example = "collect-data")
|
||||
private String taskType;
|
||||
|
||||
@Valid
|
||||
@Schema(description = "筛选条件")
|
||||
private CollectDataFiltersDto filters;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据源文件信息")
|
||||
public class CollectDataSourceFileDto {
|
||||
@Schema(description = "上传接口返回的临时文件 key", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String fileKey;
|
||||
|
||||
@Schema(description = "原始文件名")
|
||||
private String originalFilename;
|
||||
|
||||
@Schema(description = "相对目录路径")
|
||||
private String relativePath;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "批量查询采集数据任务进度的请求")
|
||||
public class CollectDataTaskBatchRequest {
|
||||
@NotEmpty
|
||||
@Schema(description = "任务 ID 列表", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<Long> taskIds;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("biz_collect_data_country_pref")
|
||||
public class CollectDataCountryPrefEntity {
|
||||
|
||||
@TableId(type = IdType.INPUT)
|
||||
private Long userId;
|
||||
|
||||
@TableField("country_codes_json")
|
||||
private String countryCodesJson;
|
||||
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("biz_collect_data_item")
|
||||
public class CollectDataItemEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private Long taskId;
|
||||
private Integer rowIndex;
|
||||
private String sourceFileKey;
|
||||
private String sourceFilename;
|
||||
private String keyword;
|
||||
private String statusValue;
|
||||
private String extraJson;
|
||||
private LocalDateTime createdAt;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据用户五国处理偏好:按顺序排列的国家代码列表,如 DE、UK")
|
||||
public class CollectDataCountryPreferenceVo {
|
||||
|
||||
@JsonProperty("country_codes")
|
||||
@Schema(description = "已选国家代码,顺序即处理顺序;未持久化时返回默认 DE -> UK -> FR -> IT -> ES")
|
||||
private List<String> countryCodes = new ArrayList<>();
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据总览统计")
|
||||
public class CollectDataDashboardVo {
|
||||
@Schema(description = "运行中任务数量")
|
||||
private Long pendingTaskCount;
|
||||
@Schema(description = "已结束任务数量")
|
||||
private Long processedTaskCount;
|
||||
@Schema(description = "成功任务数量")
|
||||
private Long successTaskCount;
|
||||
@Schema(description = "失败任务数量")
|
||||
private Long failedTaskCount;
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataFiltersDto;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据历史/任务记录")
|
||||
public class CollectDataHistoryItemVo {
|
||||
@Schema(description = "结果记录 ID")
|
||||
private Long resultId;
|
||||
|
||||
@Schema(description = "任务 ID")
|
||||
private Long taskId;
|
||||
|
||||
@Schema(description = "任务编号")
|
||||
private String taskNo;
|
||||
|
||||
@Schema(description = "源文件聚合展示名")
|
||||
private String sourceFilename;
|
||||
|
||||
@Schema(description = "结果文件名")
|
||||
private String resultFilename;
|
||||
|
||||
@Schema(description = "下载地址")
|
||||
private String downloadUrl;
|
||||
|
||||
@Schema(description = "任务状态:PENDING / RUNNING / SUCCESS / FAILED")
|
||||
private String taskStatus;
|
||||
|
||||
@Schema(description = "结果是否成功")
|
||||
private Boolean success;
|
||||
|
||||
@Schema(description = "错误信息")
|
||||
private String error;
|
||||
|
||||
@Schema(description = "落库行数")
|
||||
private Integer rowCount;
|
||||
|
||||
@Schema(description = "记录创建时间")
|
||||
private String createdAt;
|
||||
|
||||
@Schema(description = "任务开始时间")
|
||||
private String startedAt;
|
||||
|
||||
@Schema(description = "任务结束时间")
|
||||
private String finishedAt;
|
||||
|
||||
@Schema(description = "任务类型")
|
||||
private String taskType;
|
||||
|
||||
@Schema(description = "解析时保存的筛选条件")
|
||||
private CollectDataFiltersDto filters;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据历史列表")
|
||||
public class CollectDataHistoryVo {
|
||||
@Schema(description = "历史记录列表")
|
||||
private List<CollectDataHistoryItemVo> items = new ArrayList<>();
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据明细行")
|
||||
public class CollectDataItemVo {
|
||||
@Schema(description = "数据库主键")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "行号,从 1 开始")
|
||||
private Integer rowIndex;
|
||||
|
||||
@Schema(description = "来源文件 key")
|
||||
private String sourceFileKey;
|
||||
|
||||
@Schema(description = "来源文件名")
|
||||
private String sourceFilename;
|
||||
|
||||
@Schema(description = "关键词列内容")
|
||||
private String keyword;
|
||||
|
||||
@Schema(description = "状态列内容")
|
||||
private String statusValue;
|
||||
|
||||
@Schema(description = "其它列内容(列名 -> 字符串值)")
|
||||
private Map<String, String> extra = new LinkedHashMap<>();
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataFiltersDto;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据明细分页结果,默认每页 50 条,供 Python 拉取")
|
||||
public class CollectDataItemsPageVo {
|
||||
@Schema(description = "任务 ID")
|
||||
private Long taskId;
|
||||
|
||||
@Schema(description = "任务编号")
|
||||
private String taskNo;
|
||||
|
||||
@Schema(description = "任务类型")
|
||||
private String taskType;
|
||||
|
||||
@Schema(description = "任务状态")
|
||||
private String taskStatus;
|
||||
|
||||
@Schema(description = "页码,从 1 开始")
|
||||
private Integer page;
|
||||
|
||||
@Schema(description = "每页条数")
|
||||
private Integer pageSize;
|
||||
|
||||
@Schema(description = "本页条数")
|
||||
private Integer count;
|
||||
|
||||
@Schema(description = "总条数")
|
||||
private Long total;
|
||||
|
||||
@Schema(description = "总页数")
|
||||
private Integer totalPages;
|
||||
|
||||
@Schema(description = "本任务关联的筛选条件")
|
||||
private CollectDataFiltersDto filters;
|
||||
|
||||
@Schema(description = "明细列表")
|
||||
private List<CollectDataItemVo> items = new ArrayList<>();
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据解析结果")
|
||||
public class CollectDataParseVo {
|
||||
|
||||
@Schema(description = "新创建的任务 ID", example = "9001")
|
||||
private Long taskId;
|
||||
|
||||
@Schema(description = "任务编号")
|
||||
private String taskNo;
|
||||
|
||||
@Schema(description = "聚合源文件展示名", example = "采集前数据.xlsx 等 1 个文件")
|
||||
private String sourceFilename;
|
||||
|
||||
@Schema(description = "源文件数量")
|
||||
private Integer sourceFileCount;
|
||||
|
||||
@Schema(description = "Excel 中检测到的非空数据总行数")
|
||||
private Integer totalRows;
|
||||
|
||||
@Schema(description = "落库的有效行数")
|
||||
private Integer acceptedRows;
|
||||
|
||||
@Schema(description = "因缺少必要字段被丢弃的行数")
|
||||
private Integer droppedRows;
|
||||
|
||||
@Schema(description = "推荐分页大小,便于 Python 拉取分页数据", example = "50")
|
||||
private Integer pageSize;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "批量任务进度返回")
|
||||
public class CollectDataTaskBatchVo {
|
||||
@Schema(description = "任务详情列表")
|
||||
private List<CollectDataTaskDetailVo> items = new ArrayList<>();
|
||||
|
||||
@Schema(description = "未找到的任务 ID")
|
||||
private List<Long> missingTaskIds = new ArrayList<>();
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据任务进度详情")
|
||||
public class CollectDataTaskDetailVo {
|
||||
@Schema(description = "任务摘要")
|
||||
private CollectDataTaskSummaryVo task;
|
||||
|
||||
@Schema(description = "结果/历史记录")
|
||||
private List<CollectDataHistoryItemVo> items = new ArrayList<>();
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据任务摘要")
|
||||
public class CollectDataTaskSummaryVo {
|
||||
@Schema(description = "任务 ID")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "任务编号")
|
||||
private String taskNo;
|
||||
|
||||
@Schema(description = "任务状态")
|
||||
private String status;
|
||||
|
||||
@Schema(description = "错误信息")
|
||||
private String errorMessage;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
private String createdAt;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
private String updatedAt;
|
||||
|
||||
@Schema(description = "结束时间")
|
||||
private String finishedAt;
|
||||
}
|
||||
@@ -0,0 +1,732 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataFiltersDto;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataParseRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSourceFileDto;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataCountryPrefEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryItemVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemsPageVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataParseVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class CollectDataService {
|
||||
|
||||
public static final String MODULE_TYPE = "COLLECT_DATA";
|
||||
public static final int DEFAULT_PAGE_SIZE = 50;
|
||||
|
||||
/**
|
||||
* 默认国家偏好顺序:DE -> UK -> FR -> IT -> ES,与前端保持一致。
|
||||
*/
|
||||
public static final List<String> DEFAULT_COUNTRY_PREFERENCE_ORDER = List.of("DE", "UK", "FR", "IT", "ES");
|
||||
|
||||
private static final String STATUS_PENDING = "PENDING";
|
||||
private static final String STATUS_RUNNING = "RUNNING";
|
||||
private static final String STATUS_SUCCESS = "SUCCESS";
|
||||
private static final String STATUS_FAILED = "FAILED";
|
||||
|
||||
private static final String DEFAULT_TASK_TYPE = "collect-data";
|
||||
private static final int ITEM_INSERT_BATCH_SIZE = 500;
|
||||
|
||||
private static final List<String> KEYWORD_HEADER_ALIASES = List.of("关键词", "keyword", "key word");
|
||||
private static final List<String> STATUS_HEADER_ALIASES = List.of("状态", "status");
|
||||
|
||||
private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||
private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
|
||||
|
||||
private final LocalFileStorageService localFileStorageService;
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final CollectDataItemMapper collectDataItemMapper;
|
||||
private final CollectDataCountryPrefMapper collectDataCountryPrefMapper;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TransactionTemplate transactionTemplate;
|
||||
|
||||
public CollectDataParseVo parseAndCreateTask(CollectDataParseRequest request) {
|
||||
long startedAt = System.currentTimeMillis();
|
||||
if (request == null || request.getUserId() == null || request.getUserId() <= 0) {
|
||||
throw new BusinessException("user_id 不合法");
|
||||
}
|
||||
if (request.getFiles() == null || request.getFiles().isEmpty()) {
|
||||
throw new BusinessException("请先上传 Excel 文件");
|
||||
}
|
||||
List<CollectDataSourceFileDto> sources = request.getFiles().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(f -> f.getFileKey() != null && !f.getFileKey().isBlank())
|
||||
.toList();
|
||||
if (sources.isEmpty()) {
|
||||
throw new BusinessException("请先上传 Excel 文件");
|
||||
}
|
||||
String requestedTaskType = normalize(request.getTaskType());
|
||||
final String taskType = requestedTaskType.isBlank() ? DEFAULT_TASK_TYPE : requestedTaskType;
|
||||
CollectDataFiltersDto filters = request.getFilters() == null ? new CollectDataFiltersDto() : request.getFilters();
|
||||
|
||||
List<ParsedRow> parsedRows = new ArrayList<>();
|
||||
int totalRows = 0;
|
||||
int droppedRows = 0;
|
||||
for (CollectDataSourceFileDto source : sources) {
|
||||
File input = localFileStorageService.findLocalSourceFile(source.getFileKey());
|
||||
if (input == null || !input.exists()) {
|
||||
throw new BusinessException("源文件不存在");
|
||||
}
|
||||
ParsedWorkbook parsed = parseWorkbook(input, source);
|
||||
totalRows += parsed.totalRows();
|
||||
droppedRows += parsed.droppedRows();
|
||||
parsedRows.addAll(parsed.rows());
|
||||
}
|
||||
if (parsedRows.isEmpty()) {
|
||||
throw new BusinessException("未解析到有效数据行");
|
||||
}
|
||||
|
||||
long parsedAt = System.currentTimeMillis();
|
||||
String aggregateFilename = buildAggregateSourceFilenameLabel(sources);
|
||||
PersistedParse persisted = transactionTemplate.execute(status ->
|
||||
persistParsedTask(request, sources, taskType, filters, parsedRows, aggregateFilename));
|
||||
if (persisted == null) {
|
||||
throw new BusinessException("创建采集任务失败");
|
||||
}
|
||||
long persistedAt = System.currentTimeMillis();
|
||||
log.info("[collect-data] parseAndCreateTask userId={} files={} totalRows={} acceptedRows={} droppedRows={} parseMs={} persistMs={} totalMs={}",
|
||||
request.getUserId(), sources.size(), totalRows, parsedRows.size(), droppedRows,
|
||||
parsedAt - startedAt, persistedAt - parsedAt, persistedAt - startedAt);
|
||||
|
||||
CollectDataParseVo vo = new CollectDataParseVo();
|
||||
vo.setTaskId(persisted.task().getId());
|
||||
vo.setTaskNo(persisted.task().getTaskNo());
|
||||
vo.setSourceFilename(aggregateFilename);
|
||||
vo.setSourceFileCount(sources.size());
|
||||
vo.setTotalRows(totalRows);
|
||||
vo.setAcceptedRows(parsedRows.size());
|
||||
vo.setDroppedRows(droppedRows);
|
||||
vo.setPageSize(DEFAULT_PAGE_SIZE);
|
||||
return vo;
|
||||
}
|
||||
|
||||
private PersistedParse persistParsedTask(CollectDataParseRequest request,
|
||||
List<CollectDataSourceFileDto> sources,
|
||||
String taskType,
|
||||
CollectDataFiltersDto filters,
|
||||
List<ParsedRow> parsedRows,
|
||||
String aggregateFilename) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType(MODULE_TYPE);
|
||||
task.setTaskMode("PYTHON_QUEUE");
|
||||
task.setStatus(STATUS_PENDING);
|
||||
task.setSourceFileCount(sources.size());
|
||||
task.setSuccessFileCount(0);
|
||||
task.setFailedFileCount(0);
|
||||
task.setCreatedBy("user:" + request.getUserId());
|
||||
task.setUserId(request.getUserId());
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
try {
|
||||
Map<String, Object> requestPayload = new LinkedHashMap<>();
|
||||
requestPayload.put("userId", request.getUserId());
|
||||
requestPayload.put("taskType", taskType);
|
||||
requestPayload.put("filters", filters);
|
||||
requestPayload.put("files", sources);
|
||||
task.setRequestJson(objectMapper.writeValueAsString(requestPayload));
|
||||
task.setResultJson("{}");
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("序列化任务信息失败");
|
||||
}
|
||||
fileTaskMapper.insert(task);
|
||||
|
||||
int rowIndex = 0;
|
||||
LocalDateTime itemCreatedAt = LocalDateTime.now();
|
||||
List<CollectDataItemEntity> itemBatch = new ArrayList<>(Math.min(parsedRows.size(), ITEM_INSERT_BATCH_SIZE));
|
||||
for (ParsedRow parsedRow : parsedRows) {
|
||||
rowIndex++;
|
||||
CollectDataItemEntity entity = new CollectDataItemEntity();
|
||||
entity.setTaskId(task.getId());
|
||||
entity.setRowIndex(rowIndex);
|
||||
entity.setSourceFileKey(parsedRow.sourceFileKey());
|
||||
entity.setSourceFilename(parsedRow.sourceFilename());
|
||||
entity.setKeyword(parsedRow.keyword());
|
||||
entity.setStatusValue(parsedRow.statusValue());
|
||||
try {
|
||||
entity.setExtraJson(objectMapper.writeValueAsString(parsedRow.extra()));
|
||||
} catch (Exception ex) {
|
||||
entity.setExtraJson("{}");
|
||||
}
|
||||
entity.setCreatedAt(itemCreatedAt);
|
||||
itemBatch.add(entity);
|
||||
if (itemBatch.size() >= ITEM_INSERT_BATCH_SIZE) {
|
||||
collectDataItemMapper.insertBatch(itemBatch);
|
||||
itemBatch.clear();
|
||||
}
|
||||
}
|
||||
if (!itemBatch.isEmpty()) {
|
||||
collectDataItemMapper.insertBatch(itemBatch);
|
||||
}
|
||||
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setTaskId(task.getId());
|
||||
result.setModuleType(MODULE_TYPE);
|
||||
result.setSourceFilename(aggregateFilename);
|
||||
result.setSourceFileUrl(buildAggregateSourceFileUrl(sources));
|
||||
result.setRowCount(parsedRows.size());
|
||||
result.setUserId(request.getUserId());
|
||||
result.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(result);
|
||||
|
||||
return new PersistedParse(task, result);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void activateTask(Long taskId, Long userId) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
|
||||
throw new BusinessException("任务已结束");
|
||||
}
|
||||
task.setStatus(STATUS_RUNNING);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
|
||||
public CollectDataDashboardVo dashboard(Long userId) {
|
||||
CollectDataDashboardVo vo = new CollectDataDashboardVo();
|
||||
vo.setPendingTaskCount(countTask(userId, STATUS_RUNNING));
|
||||
vo.setSuccessTaskCount(countTask(userId, STATUS_SUCCESS));
|
||||
vo.setFailedTaskCount(countTask(userId, STATUS_FAILED));
|
||||
long processed = (vo.getSuccessTaskCount() == null ? 0 : vo.getSuccessTaskCount())
|
||||
+ (vo.getFailedTaskCount() == null ? 0 : vo.getFailedTaskCount());
|
||||
vo.setProcessedTaskCount(processed);
|
||||
return vo;
|
||||
}
|
||||
|
||||
public CollectDataHistoryVo history(Long userId, Integer limit) {
|
||||
CollectDataHistoryVo vo = new CollectDataHistoryVo();
|
||||
if (userId == null || userId <= 0) {
|
||||
return vo;
|
||||
}
|
||||
int safeLimit = Math.max(1, Math.min(limit == null ? 50 : limit, 100));
|
||||
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileResultEntity::getUserId, userId)
|
||||
.orderByDesc(FileResultEntity::getCreatedAt)
|
||||
.last("limit " + safeLimit));
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
List<Long> taskIds = rows.stream().map(FileResultEntity::getTaskId).filter(Objects::nonNull).distinct().toList();
|
||||
Map<Long, FileTaskEntity> taskMap = loadTaskMap(taskIds);
|
||||
for (FileResultEntity row : rows) {
|
||||
FileTaskEntity task = row.getTaskId() == null ? null : taskMap.get(row.getTaskId());
|
||||
vo.getItems().add(toHistoryItem(row, task));
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
public CollectDataTaskBatchVo progressBatch(List<Long> taskIds) {
|
||||
CollectDataTaskBatchVo vo = new CollectDataTaskBatchVo();
|
||||
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
|
||||
.filter(id -> id != null && id > 0)
|
||||
.distinct()
|
||||
.toList();
|
||||
if (normalizedIds.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
Map<Long, FileTaskEntity> taskMap = loadTaskMap(normalizedIds);
|
||||
List<FileResultEntity> resultRows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||
.in(FileResultEntity::getTaskId, normalizedIds)
|
||||
.orderByDesc(FileResultEntity::getCreatedAt));
|
||||
Map<Long, FileResultEntity> resultByTaskId = new LinkedHashMap<>();
|
||||
if (resultRows != null) {
|
||||
for (FileResultEntity row : resultRows) {
|
||||
if (row.getTaskId() != null) {
|
||||
resultByTaskId.putIfAbsent(row.getTaskId(), row);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (Long taskId : normalizedIds) {
|
||||
FileTaskEntity task = taskMap.get(taskId);
|
||||
if (task == null) {
|
||||
vo.getMissingTaskIds().add(taskId);
|
||||
continue;
|
||||
}
|
||||
CollectDataTaskDetailVo detail = new CollectDataTaskDetailVo();
|
||||
detail.setTask(toTaskSummary(task));
|
||||
FileResultEntity row = resultByTaskId.get(taskId);
|
||||
if (row != null) {
|
||||
detail.getItems().add(toHistoryItem(row, task));
|
||||
}
|
||||
vo.getItems().add(detail);
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
public CollectDataItemsPageVo getItemsPage(Long taskId, Long userId, Integer page, Integer pageSize) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
int safePage = page == null || page <= 0 ? 1 : page;
|
||||
int safePageSize = pageSize == null || pageSize <= 0 ? DEFAULT_PAGE_SIZE : Math.min(pageSize, 200);
|
||||
|
||||
Long total = collectDataItemMapper.selectCount(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.eq(CollectDataItemEntity::getTaskId, taskId));
|
||||
long totalRows = total == null ? 0L : total;
|
||||
int totalPages = totalRows == 0 ? 0 : (int) ((totalRows + safePageSize - 1) / safePageSize);
|
||||
|
||||
int offset = (safePage - 1) * safePageSize;
|
||||
List<CollectDataItemEntity> rows = collectDataItemMapper.selectList(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.eq(CollectDataItemEntity::getTaskId, taskId)
|
||||
.orderByAsc(CollectDataItemEntity::getRowIndex)
|
||||
.last("limit " + safePageSize + " offset " + Math.max(offset, 0)));
|
||||
|
||||
CollectDataItemsPageVo vo = new CollectDataItemsPageVo();
|
||||
vo.setTaskId(task.getId());
|
||||
vo.setTaskNo(task.getTaskNo());
|
||||
vo.setTaskStatus(task.getStatus());
|
||||
vo.setPage(safePage);
|
||||
vo.setPageSize(safePageSize);
|
||||
vo.setTotal(totalRows);
|
||||
vo.setTotalPages(totalPages);
|
||||
vo.setFilters(extractFiltersFromRequest(task));
|
||||
vo.setTaskType(extractTaskTypeFromRequest(task));
|
||||
|
||||
List<CollectDataItemVo> items = new ArrayList<>();
|
||||
if (rows != null) {
|
||||
for (CollectDataItemEntity row : rows) {
|
||||
items.add(toItemVo(row));
|
||||
}
|
||||
}
|
||||
vo.setItems(items);
|
||||
vo.setCount(items.size());
|
||||
return vo;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteTask(Long taskId, Long userId) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.eq(CollectDataItemEntity::getTaskId, task.getId()));
|
||||
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, task.getId())
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
||||
fileTaskMapper.deleteById(task.getId());
|
||||
}
|
||||
|
||||
public void deleteHistory(Long resultId, Long userId) {
|
||||
FileResultEntity row = fileResultMapper.selectById(resultId);
|
||||
if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
fileResultMapper.deleteById(resultId);
|
||||
}
|
||||
|
||||
public CollectDataCountryPreferenceVo getCountryPreference(Long userId) {
|
||||
if (userId == null || userId <= 0) {
|
||||
throw new BusinessException("user_id 不合法");
|
||||
}
|
||||
CollectDataCountryPrefEntity row = collectDataCountryPrefMapper.selectById(userId);
|
||||
CollectDataCountryPreferenceVo vo = new CollectDataCountryPreferenceVo();
|
||||
if (row == null || row.getCountryCodesJson() == null || row.getCountryCodesJson().isBlank()) {
|
||||
vo.getCountryCodes().addAll(DEFAULT_COUNTRY_PREFERENCE_ORDER);
|
||||
return vo;
|
||||
}
|
||||
try {
|
||||
List<String> parsed = objectMapper.readValue(row.getCountryCodesJson(), new TypeReference<List<String>>() {
|
||||
});
|
||||
vo.getCountryCodes().addAll(sanitizeStoredCountryCodes(parsed));
|
||||
} catch (Exception ex) {
|
||||
vo.getCountryCodes().addAll(DEFAULT_COUNTRY_PREFERENCE_ORDER);
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public CollectDataCountryPreferenceVo saveCountryPreference(CollectDataCountryPreferenceSaveRequest request) {
|
||||
if (request == null || request.getUserId() == null || request.getUserId() <= 0) {
|
||||
throw new BusinessException("user_id 不合法");
|
||||
}
|
||||
List<String> normalized = validateCountryCodesForSave(request.getCountryCodes());
|
||||
String json;
|
||||
try {
|
||||
json = objectMapper.writeValueAsString(normalized);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("保存国家偏好失败");
|
||||
}
|
||||
CollectDataCountryPrefEntity row = collectDataCountryPrefMapper.selectById(request.getUserId());
|
||||
if (row == null) {
|
||||
row = new CollectDataCountryPrefEntity();
|
||||
row.setUserId(request.getUserId());
|
||||
row.setCountryCodesJson(json);
|
||||
collectDataCountryPrefMapper.insert(row);
|
||||
} else {
|
||||
row.setCountryCodesJson(json);
|
||||
collectDataCountryPrefMapper.updateById(row);
|
||||
}
|
||||
CollectDataCountryPreferenceVo vo = new CollectDataCountryPreferenceVo();
|
||||
vo.getCountryCodes().addAll(normalized);
|
||||
return vo;
|
||||
}
|
||||
|
||||
private static List<String> sanitizeStoredCountryCodes(List<String> raw) {
|
||||
if (raw == null || raw.isEmpty()) {
|
||||
return new ArrayList<>(DEFAULT_COUNTRY_PREFERENCE_ORDER);
|
||||
}
|
||||
LinkedHashSet<String> seen = new LinkedHashSet<>();
|
||||
List<String> out = new ArrayList<>();
|
||||
for (String code : raw) {
|
||||
if (code == null || code.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
String upper = code.trim().toUpperCase(Locale.ROOT);
|
||||
if (!DEFAULT_COUNTRY_PREFERENCE_ORDER.contains(upper)) {
|
||||
continue;
|
||||
}
|
||||
if (seen.add(upper)) {
|
||||
out.add(upper);
|
||||
}
|
||||
}
|
||||
return out.isEmpty() ? new ArrayList<>(DEFAULT_COUNTRY_PREFERENCE_ORDER) : out;
|
||||
}
|
||||
|
||||
private static List<String> validateCountryCodesForSave(List<String> raw) {
|
||||
if (raw == null || raw.isEmpty()) {
|
||||
throw new BusinessException("country_codes 至少选择 1 个国家");
|
||||
}
|
||||
LinkedHashSet<String> seen = new LinkedHashSet<>();
|
||||
List<String> out = new ArrayList<>();
|
||||
for (String code : raw) {
|
||||
if (code == null || code.isBlank()) {
|
||||
throw new BusinessException("country_codes 含空项");
|
||||
}
|
||||
String upper = code.trim().toUpperCase(Locale.ROOT);
|
||||
if (!DEFAULT_COUNTRY_PREFERENCE_ORDER.contains(upper)) {
|
||||
throw new BusinessException("非法国家代码: " + code);
|
||||
}
|
||||
if (!seen.add(upper)) {
|
||||
throw new BusinessException("country_codes 存在重复: " + upper);
|
||||
}
|
||||
out.add(upper);
|
||||
}
|
||||
if (out.size() > DEFAULT_COUNTRY_PREFERENCE_ORDER.size()) {
|
||||
throw new BusinessException("country_codes 最多 5 项");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private FileTaskEntity requireTask(Long taskId, Long userId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
if (userId != null && userId > 0 && !Objects.equals(userId, task.getUserId())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
private long countTask(Long userId, String status) {
|
||||
if (userId == null || userId <= 0) {
|
||||
return 0L;
|
||||
}
|
||||
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getUserId, userId)
|
||||
.eq(FileTaskEntity::getStatus, status));
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
private Map<Long, FileTaskEntity> loadTaskMap(List<Long> taskIds) {
|
||||
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return taskMap;
|
||||
}
|
||||
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, taskIds));
|
||||
if (tasks != null) {
|
||||
for (FileTaskEntity task : tasks) {
|
||||
if (task != null && MODULE_TYPE.equals(task.getModuleType())) {
|
||||
taskMap.put(task.getId(), task);
|
||||
}
|
||||
}
|
||||
}
|
||||
return taskMap;
|
||||
}
|
||||
|
||||
private CollectDataHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task) {
|
||||
CollectDataHistoryItemVo vo = new CollectDataHistoryItemVo();
|
||||
vo.setResultId(row.getId());
|
||||
vo.setTaskId(row.getTaskId());
|
||||
vo.setSourceFilename(row.getSourceFilename());
|
||||
vo.setResultFilename(row.getResultFilename());
|
||||
vo.setRowCount(row.getRowCount());
|
||||
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
|
||||
vo.setError(row.getErrorMessage());
|
||||
vo.setCreatedAt(formatTime(row.getCreatedAt()));
|
||||
if (task != null) {
|
||||
vo.setTaskNo(task.getTaskNo());
|
||||
vo.setTaskStatus(task.getStatus());
|
||||
vo.setStartedAt(formatTime(task.getCreatedAt()));
|
||||
vo.setFinishedAt(formatTime(task.getFinishedAt()));
|
||||
vo.setTaskType(extractTaskTypeFromRequest(task));
|
||||
vo.setFilters(extractFiltersFromRequest(task));
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
private CollectDataTaskSummaryVo toTaskSummary(FileTaskEntity task) {
|
||||
CollectDataTaskSummaryVo vo = new CollectDataTaskSummaryVo();
|
||||
vo.setId(task.getId());
|
||||
vo.setTaskNo(task.getTaskNo());
|
||||
vo.setStatus(task.getStatus());
|
||||
vo.setErrorMessage(task.getErrorMessage());
|
||||
vo.setCreatedAt(formatTime(task.getCreatedAt()));
|
||||
vo.setUpdatedAt(formatTime(task.getUpdatedAt()));
|
||||
vo.setFinishedAt(formatTime(task.getFinishedAt()));
|
||||
return vo;
|
||||
}
|
||||
|
||||
private CollectDataItemVo toItemVo(CollectDataItemEntity row) {
|
||||
CollectDataItemVo vo = new CollectDataItemVo();
|
||||
vo.setId(row.getId());
|
||||
vo.setRowIndex(row.getRowIndex());
|
||||
vo.setSourceFileKey(row.getSourceFileKey());
|
||||
vo.setSourceFilename(row.getSourceFilename());
|
||||
vo.setKeyword(row.getKeyword());
|
||||
vo.setStatusValue(row.getStatusValue());
|
||||
if (row.getExtraJson() != null && !row.getExtraJson().isBlank()) {
|
||||
try {
|
||||
Map<String, String> extra = objectMapper.readValue(row.getExtraJson(), new TypeReference<>() {
|
||||
});
|
||||
if (extra != null) {
|
||||
vo.getExtra().putAll(extra);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] parse extra json failed rowId={} err={}", row.getId(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
private CollectDataFiltersDto extractFiltersFromRequest(FileTaskEntity task) {
|
||||
if (task == null || task.getRequestJson() == null || task.getRequestJson().isBlank()) {
|
||||
return new CollectDataFiltersDto();
|
||||
}
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(task.getRequestJson());
|
||||
JsonNode filtersNode = root.get("filters");
|
||||
if (filtersNode == null || filtersNode.isNull()) {
|
||||
return new CollectDataFiltersDto();
|
||||
}
|
||||
return objectMapper.treeToValue(filtersNode, CollectDataFiltersDto.class);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] parse request filters failed taskId={} err={}", task.getId(), ex.getMessage());
|
||||
return new CollectDataFiltersDto();
|
||||
}
|
||||
}
|
||||
|
||||
private String extractTaskTypeFromRequest(FileTaskEntity task) {
|
||||
if (task == null || task.getRequestJson() == null || task.getRequestJson().isBlank()) {
|
||||
return DEFAULT_TASK_TYPE;
|
||||
}
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(task.getRequestJson());
|
||||
JsonNode taskTypeNode = root.get("taskType");
|
||||
String value = taskTypeNode == null || taskTypeNode.isNull() ? null : taskTypeNode.asText(null);
|
||||
return value == null || value.isBlank() ? DEFAULT_TASK_TYPE : value;
|
||||
} catch (Exception ex) {
|
||||
return DEFAULT_TASK_TYPE;
|
||||
}
|
||||
}
|
||||
|
||||
private String buildAggregateSourceFilenameLabel(List<CollectDataSourceFileDto> sources) {
|
||||
if (sources == null || sources.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
String first = firstNonBlank(sources.get(0).getOriginalFilename(), sources.get(0).getFileKey());
|
||||
if (sources.size() == 1) {
|
||||
return first;
|
||||
}
|
||||
return first + " 等 " + sources.size() + " 个文件";
|
||||
}
|
||||
|
||||
private String buildAggregateSourceFileUrl(List<CollectDataSourceFileDto> sources) {
|
||||
if (sources == null || sources.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = 0; i < sources.size(); i++) {
|
||||
if (i > 0) {
|
||||
sb.append('|');
|
||||
}
|
||||
sb.append(sources.get(i).getFileKey());
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private ParsedWorkbook parseWorkbook(File input, CollectDataSourceFileDto source) {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row header = sheet.getRow(0);
|
||||
if (header == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
|
||||
List<String> headers = new ArrayList<>();
|
||||
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||
String value = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||
headers.add(value.isBlank() ? "列" + (i + 1) : value);
|
||||
}
|
||||
|
||||
int keywordCol = findHeaderIndex(headers, KEYWORD_HEADER_ALIASES);
|
||||
int statusCol = findHeaderIndex(headers, STATUS_HEADER_ALIASES);
|
||||
|
||||
List<ParsedRow> rows = new ArrayList<>();
|
||||
int totalRows = 0;
|
||||
int droppedRows = 0;
|
||||
String filename = firstNonBlank(source.getOriginalFilename(), input.getName());
|
||||
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
|
||||
Row row = sheet.getRow(i);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
Map<String, String> extra = new LinkedHashMap<>();
|
||||
String keyword = "";
|
||||
String statusValue = "";
|
||||
boolean nonEmpty = false;
|
||||
for (int c = 0; c < headers.size(); c++) {
|
||||
Cell cell = row.getCell(c);
|
||||
String value = normalize(cell == null ? "" : formatter.formatCellValue(cell));
|
||||
if (!value.isBlank()) {
|
||||
nonEmpty = true;
|
||||
}
|
||||
if (c == keywordCol) {
|
||||
keyword = value;
|
||||
} else if (c == statusCol) {
|
||||
statusValue = value;
|
||||
} else {
|
||||
extra.put(headers.get(c), value);
|
||||
}
|
||||
}
|
||||
if (!nonEmpty) {
|
||||
continue;
|
||||
}
|
||||
totalRows++;
|
||||
if (keywordCol >= 0 && keyword.isBlank()) {
|
||||
droppedRows++;
|
||||
continue;
|
||||
}
|
||||
rows.add(new ParsedRow(source.getFileKey(), filename, keyword, statusValue, extra));
|
||||
}
|
||||
return new ParsedWorkbook(totalRows, droppedRows, rows);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] parse workbook failed file={} err={}", input, ex.getMessage());
|
||||
throw new BusinessException("解析 Excel 失败");
|
||||
}
|
||||
}
|
||||
|
||||
private int findHeaderIndex(List<String> headers, List<String> aliases) {
|
||||
if (headers == null || headers.isEmpty() || aliases == null || aliases.isEmpty()) {
|
||||
return -1;
|
||||
}
|
||||
for (int i = 0; i < headers.size(); i++) {
|
||||
String lower = headers.get(i).toLowerCase(Locale.ROOT);
|
||||
for (String alias : aliases) {
|
||||
if (lower.contains(alias.toLowerCase(Locale.ROOT))) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private String formatTime(LocalDateTime time) {
|
||||
return time == null ? null : time.format(TIME_FORMATTER);
|
||||
}
|
||||
|
||||
private String firstNonBlank(String... values) {
|
||||
if (values == null) {
|
||||
return "";
|
||||
}
|
||||
for (String value : values) {
|
||||
if (value != null && !value.isBlank()) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private String normalize(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
String normalized = value.replace(String.valueOf((char) 0xFEFF), "")
|
||||
.replace((char) 0x3000, ' ')
|
||||
.replace("\r\n", " ")
|
||||
.replace("\r", " ")
|
||||
.replace("\n", " ")
|
||||
.replace("\t", " ")
|
||||
.trim();
|
||||
return WHITESPACE_PATTERN.matcher(normalized).replaceAll(" ");
|
||||
}
|
||||
|
||||
private record ParsedRow(String sourceFileKey,
|
||||
String sourceFilename,
|
||||
String keyword,
|
||||
String statusValue,
|
||||
Map<String, String> extra) {
|
||||
}
|
||||
|
||||
private record ParsedWorkbook(int totalRows, int droppedRows, List<ParsedRow> rows) {
|
||||
}
|
||||
|
||||
private record PersistedParse(FileTaskEntity task, FileResultEntity result) {
|
||||
}
|
||||
}
|
||||
@@ -152,13 +152,13 @@ public class DeleteBrandRunController {
|
||||
@Operation(summary = "Download one delete-brand result")
|
||||
public void downloadResult(
|
||||
@PathVariable Long resultId,
|
||||
@Parameter(name = "user_id", description = "褰撳墠鐧诲綍鐢ㄦ埛 ID", required = true, in = ParameterIn.QUERY)
|
||||
@Parameter(name = "user_id", description = "当前登录用户 ID", required = true, in = ParameterIn.QUERY)
|
||||
@RequestParam("user_id") Long userId,
|
||||
jakarta.servlet.http.HttpServletResponse response) {
|
||||
String url = deleteBrandRunService.resolveResultDownloadUrl(resultId, userId);
|
||||
String filename = deleteBrandRunService.resolveResultDownloadFilename(resultId, userId);
|
||||
if (url == null || url.isBlank()) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "缁撴灉鏃犲彲涓嬭浇");
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "结果无可下载");
|
||||
}
|
||||
if (filename == null || filename.isBlank()) {
|
||||
String rawPath = URI.create(url).getPath();
|
||||
@@ -177,7 +177,7 @@ public class DeleteBrandRunController {
|
||||
response.getOutputStream().flush();
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "涓嬭浇澶辫触");
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -628,7 +628,7 @@ public class DeleteBrandStaleTaskService {
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_QUERY_ASIN)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.set(FileTaskEntity::getStatus, "FAILED")
|
||||
.set(FileTaskEntity::getErrorMessage, "闀挎椂闂存湭鏀跺埌 Python 缁撴灉鍥炰紶锛屼换鍔″凡鑷姩澶辫触")
|
||||
.set(FileTaskEntity::getErrorMessage, "长时间未收到 Python 结果回传,任务已自动失败")
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package com.nanri.aiimage.modules.invalidasin.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataCreateRequest;
|
||||
import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataUpdateRequest;
|
||||
import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataItemVo;
|
||||
import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataPageVo;
|
||||
import com.nanri.aiimage.modules.invalidasin.service.InvalidAsinDataService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.media.Content;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponses;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/admin/invalid-asin-data")
|
||||
@Tag(name = "不符合ASIN数据", description = "维护不符合ASIN数据列表,支持增删改查。")
|
||||
public class InvalidAsinDataController {
|
||||
|
||||
private final InvalidAsinDataService invalidAsinDataService;
|
||||
|
||||
@GetMapping
|
||||
@Operation(summary = "分页查询不符合ASIN数据", description = "分页查询不符合ASIN数据,支持按 ASIN/品牌 模糊搜索。")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功", content = @Content(schema = @Schema(implementation = InvalidAsinDataPageVo.class)))
|
||||
})
|
||||
public ApiResponse<InvalidAsinDataPageVo> page(
|
||||
@Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page,
|
||||
@Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize,
|
||||
@Parameter(description = "模糊搜索关键字") @RequestParam(required = false) String keyword) {
|
||||
return ApiResponse.success(invalidAsinDataService.page(page, pageSize, keyword));
|
||||
}
|
||||
|
||||
@PostMapping
|
||||
@Operation(summary = "新增不符合ASIN数据", description = "新增一条不符合ASIN数据。")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "创建成功", content = @Content(schema = @Schema(implementation = InvalidAsinDataItemVo.class))),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "参数不合法或数据重复")
|
||||
})
|
||||
public ApiResponse<InvalidAsinDataItemVo> create(@Valid @RequestBody InvalidAsinDataCreateRequest request) {
|
||||
return ApiResponse.success("创建成功", invalidAsinDataService.create(request));
|
||||
}
|
||||
|
||||
@PutMapping("/{id}")
|
||||
@Operation(summary = "更新不符合ASIN数据", description = "按 ID 更新一条不符合ASIN数据。")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "更新成功", content = @Content(schema = @Schema(implementation = InvalidAsinDataItemVo.class))),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "参数不合法或数据重复"),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "数据不存在")
|
||||
})
|
||||
public ApiResponse<InvalidAsinDataItemVo> update(
|
||||
@Parameter(description = "主键ID", required = true) @PathVariable Long id,
|
||||
@Valid @RequestBody InvalidAsinDataUpdateRequest request) {
|
||||
return ApiResponse.success("更新成功", invalidAsinDataService.update(id, request));
|
||||
}
|
||||
|
||||
@DeleteMapping("/{id}")
|
||||
@Operation(summary = "删除不符合ASIN数据", description = "按 ID 删除一条不符合ASIN数据。")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功"),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "数据不存在")
|
||||
})
|
||||
public ApiResponse<Void> delete(@Parameter(description = "主键ID", required = true) @PathVariable Long id) {
|
||||
invalidAsinDataService.delete(id);
|
||||
return ApiResponse.success("删除成功", null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.invalidasin.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface InvalidAsinDataMapper extends BaseMapper<InvalidAsinDataEntity> {
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package com.nanri.aiimage.modules.invalidasin.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "不符合ASIN数据新增请求")
|
||||
public class InvalidAsinDataCreateRequest {
|
||||
|
||||
@NotBlank(message = "ASIN 不能为空")
|
||||
@Size(max = 128, message = "ASIN 长度不能超过128个字符")
|
||||
@Schema(description = "ASIN 值", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String dataValue;
|
||||
|
||||
@Size(max = 128, message = "品牌长度不能超过128个字符")
|
||||
@Schema(description = "品牌名称")
|
||||
private String brand;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package com.nanri.aiimage.modules.invalidasin.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "不符合ASIN数据更新请求")
|
||||
public class InvalidAsinDataUpdateRequest {
|
||||
|
||||
@NotBlank(message = "ASIN 不能为空")
|
||||
@Size(max = 128, message = "ASIN 长度不能超过128个字符")
|
||||
@Schema(description = "ASIN 值", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String dataValue;
|
||||
|
||||
@Size(max = 128, message = "品牌长度不能超过128个字符")
|
||||
@Schema(description = "品牌名称")
|
||||
private String brand;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package com.nanri.aiimage.modules.invalidasin.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("biz_invalid_asin_data")
|
||||
public class InvalidAsinDataEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private String dataValue;
|
||||
private String brand;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.nanri.aiimage.modules.invalidasin.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@Schema(description = "不符合ASIN数据项")
|
||||
public class InvalidAsinDataItemVo {
|
||||
|
||||
@Schema(description = "主键ID")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "ASIN 值")
|
||||
private String dataValue;
|
||||
|
||||
@Schema(description = "品牌名称")
|
||||
private String brand;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
private LocalDateTime createdAt;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.nanri.aiimage.modules.invalidasin.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "不符合ASIN数据分页结果")
|
||||
public class InvalidAsinDataPageVo {
|
||||
|
||||
@Schema(description = "列表")
|
||||
private List<InvalidAsinDataItemVo> items;
|
||||
|
||||
@Schema(description = "总数")
|
||||
private Long total;
|
||||
|
||||
@Schema(description = "页码")
|
||||
private Long page;
|
||||
|
||||
@Schema(description = "每页数量")
|
||||
private Long pageSize;
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package com.nanri.aiimage.modules.invalidasin.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
|
||||
import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataCreateRequest;
|
||||
import com.nanri.aiimage.modules.invalidasin.model.dto.InvalidAsinDataUpdateRequest;
|
||||
import com.nanri.aiimage.modules.invalidasin.model.entity.InvalidAsinDataEntity;
|
||||
import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataItemVo;
|
||||
import com.nanri.aiimage.modules.invalidasin.model.vo.InvalidAsinDataPageVo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class InvalidAsinDataService {
|
||||
|
||||
private final InvalidAsinDataMapper invalidAsinDataMapper;
|
||||
|
||||
public InvalidAsinDataPageVo page(long page, long pageSize, String keyword) {
|
||||
long safePage = Math.max(page, 1);
|
||||
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
|
||||
String safeKeyword = keyword == null ? "" : keyword.trim();
|
||||
LambdaQueryWrapper<InvalidAsinDataEntity> query = new LambdaQueryWrapper<InvalidAsinDataEntity>()
|
||||
.and(!safeKeyword.isEmpty(), wrapper -> wrapper
|
||||
.like(InvalidAsinDataEntity::getDataValue, safeKeyword)
|
||||
.or()
|
||||
.like(InvalidAsinDataEntity::getBrand, safeKeyword))
|
||||
.orderByDesc(InvalidAsinDataEntity::getId);
|
||||
Long total = invalidAsinDataMapper.selectCount(query);
|
||||
List<InvalidAsinDataItemVo> items = invalidAsinDataMapper
|
||||
.selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize))
|
||||
.stream()
|
||||
.map(this::toItemVo)
|
||||
.toList();
|
||||
InvalidAsinDataPageVo vo = new InvalidAsinDataPageVo();
|
||||
vo.setItems(items);
|
||||
vo.setTotal(total);
|
||||
vo.setPage(safePage);
|
||||
vo.setPageSize(safePageSize);
|
||||
return vo;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public InvalidAsinDataItemVo create(InvalidAsinDataCreateRequest request) {
|
||||
String dataValue = normalizeRequired(request.getDataValue(), "ASIN 不能为空");
|
||||
String brand = normalizeOptional(request.getBrand());
|
||||
ensureUnique(dataValue, null);
|
||||
InvalidAsinDataEntity entity = new InvalidAsinDataEntity();
|
||||
entity.setDataValue(dataValue);
|
||||
entity.setBrand(brand);
|
||||
invalidAsinDataMapper.insert(entity);
|
||||
return toItemVo(getById(entity.getId()));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public InvalidAsinDataItemVo update(Long id, InvalidAsinDataUpdateRequest request) {
|
||||
InvalidAsinDataEntity entity = getById(id);
|
||||
String dataValue = normalizeRequired(request.getDataValue(), "ASIN 不能为空");
|
||||
String brand = normalizeOptional(request.getBrand());
|
||||
ensureUnique(dataValue, id);
|
||||
entity.setDataValue(dataValue);
|
||||
entity.setBrand(brand);
|
||||
invalidAsinDataMapper.updateById(entity);
|
||||
return toItemVo(getById(id));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void delete(Long id) {
|
||||
InvalidAsinDataEntity entity = getById(id);
|
||||
invalidAsinDataMapper.deleteById(entity.getId());
|
||||
}
|
||||
|
||||
private InvalidAsinDataEntity getById(Long id) {
|
||||
InvalidAsinDataEntity entity = invalidAsinDataMapper.selectById(id);
|
||||
if (entity == null) {
|
||||
throw new BusinessException("数据不存在");
|
||||
}
|
||||
return entity;
|
||||
}
|
||||
|
||||
private void ensureUnique(String dataValue, Long excludeId) {
|
||||
LambdaQueryWrapper<InvalidAsinDataEntity> query = new LambdaQueryWrapper<InvalidAsinDataEntity>()
|
||||
.eq(InvalidAsinDataEntity::getDataValue, dataValue)
|
||||
.last("LIMIT 1");
|
||||
InvalidAsinDataEntity exists = invalidAsinDataMapper.selectOne(query);
|
||||
if (exists != null && (excludeId == null || !exists.getId().equals(excludeId))) {
|
||||
throw new BusinessException("ASIN 已存在");
|
||||
}
|
||||
}
|
||||
|
||||
private String normalizeRequired(String value, String message) {
|
||||
String normalized = normalizeText(value);
|
||||
if (normalized.isEmpty()) {
|
||||
throw new BusinessException(message);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private String normalizeOptional(String value) {
|
||||
String normalized = normalizeText(value);
|
||||
return normalized.isEmpty() ? null : normalized;
|
||||
}
|
||||
|
||||
private String normalizeText(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
return value.replace("", "")
|
||||
.replace(" ", " ")
|
||||
.replace("\r\n", " ")
|
||||
.replace("\r", " ")
|
||||
.replace("\n", " ")
|
||||
.replace("\t", " ")
|
||||
.trim()
|
||||
.replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
private InvalidAsinDataItemVo toItemVo(InvalidAsinDataEntity entity) {
|
||||
InvalidAsinDataItemVo vo = new InvalidAsinDataItemVo();
|
||||
vo.setId(entity.getId());
|
||||
vo.setDataValue(entity.getDataValue());
|
||||
vo.setBrand(entity.getBrand());
|
||||
vo.setCreatedAt(entity.getCreatedAt());
|
||||
return vo;
|
||||
}
|
||||
}
|
||||
@@ -1044,7 +1044,7 @@ public class PatrolDeleteTaskService {
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "浠诲姟姝e湪澶勭悊涓紝璇风◢鍚庡啀璇?");
|
||||
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ public class PermissionMenuSchemaInitializer {
|
||||
new DefaultAdminMenu("用户管理", "admin_users", "users", 10),
|
||||
new DefaultAdminMenu("栏目权限配置", "admin_columns", "columns", 20),
|
||||
new DefaultAdminMenu("数据去重总数据", "admin_dedupe_total_data", "dedupe-total-data", 30),
|
||||
new DefaultAdminMenu("不符合ASIN数据", "admin_invalid_asin_data", "invalid-asin-data", 35),
|
||||
new DefaultAdminMenu("店铺密钥管理", "admin_shop_keys", "shop-keys", 40),
|
||||
new DefaultAdminMenu("店铺管理", "admin_shop_manage", "shop-manage", 50),
|
||||
new DefaultAdminMenu("跳过跟价ASIN", "admin_skip_price_asin", "skip-price-asin", 60),
|
||||
|
||||
@@ -1391,7 +1391,7 @@ public class PriceTrackTaskService {
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "浠诲姟姝e湪澶勭悊涓紝璇风◢鍚庡啀璇?");
|
||||
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
@@ -970,7 +970,7 @@ public class ProductRiskTaskService {
|
||||
private TaskDistributedLockService.LockHandle acquireTaskLockOrThrow(Long taskId) {
|
||||
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(taskId);
|
||||
if (lockHandle == null) {
|
||||
throw new BusinessException(40901, "浠诲姟姝e湪澶勭悊涓紝璇风◢鍚庡啀璇?");
|
||||
throw new BusinessException(40901, "任务正在处理中,请稍后再试");
|
||||
}
|
||||
return lockHandle;
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user