Compare commits
27 Commits
46d91fd8ca
...
ef0e0df0ac
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ef0e0df0ac | ||
|
|
025ca6d4fd | ||
| 951a353881 | |||
|
|
eb04caccf1 | ||
|
|
cd84def61d | ||
|
|
d8098b0378 | ||
|
|
62d30ec190 | ||
|
|
d0d3c6ee67 | ||
|
|
21b6b5b270 | ||
|
|
365050b890 | ||
|
|
73392a9b83 | ||
|
|
c47c03fde4 | ||
|
|
86fd9475e4 | ||
|
|
2d0d1d3461 | ||
|
|
ccaec4a984 | ||
|
|
169ba7edb6 | ||
|
|
0327d1cc51 | ||
|
|
e02ddab599 | ||
|
|
336338cff5 | ||
|
|
acff31e652 | ||
|
|
ad304a6880 | ||
|
|
cb34788e53 | ||
|
|
13b0ffb5d8 | ||
|
|
c9947c4ac8 | ||
|
|
afffb7aad2 | ||
|
|
9720453ac4 | ||
|
|
1ff6d5220e |
1
app/.env
1
app/.env
@@ -12,6 +12,7 @@ zn_username=%E8%87%AA%E5%8A%A8%E5%8C%96_Robot
|
||||
client_name=ShuFuAI
|
||||
|
||||
|
||||
# java_api_base=http://47.111.163.154:18080
|
||||
java_api_base=http://127.0.0.1:18080
|
||||
# java_api_base=http://8.136.19.173:18080
|
||||
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
app/amazon/__pycache__/approve.cpython-312.pyc
Normal file
BIN
app/amazon/__pycache__/approve.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/amazon/__pycache__/approve.cpython-39.pyc
Normal file
BIN
app/amazon/__pycache__/approve.cpython-39.pyc
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
app/amazon/__pycache__/match_action.cpython-312.pyc
Normal file
BIN
app/amazon/__pycache__/match_action.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/amazon/__pycache__/match_action.cpython-39.pyc
Normal file
BIN
app/amazon/__pycache__/match_action.cpython-39.pyc
Normal file
Binary file not shown.
BIN
app/amazon/__pycache__/tool.cpython-312.pyc
Normal file
BIN
app/amazon/__pycache__/tool.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/amazon/__pycache__/tool.cpython-39.pyc
Normal file
BIN
app/amazon/__pycache__/tool.cpython-39.pyc
Normal file
Binary file not shown.
999
app/amazon/approve - 副本.py
Normal file
999
app/amazon/approve - 副本.py
Normal file
@@ -0,0 +1,999 @@
|
||||
import time
|
||||
import re
|
||||
import traceback
|
||||
from DrissionPage._pages.chromium_tab import ChromiumTab
|
||||
|
||||
from config import runing_task, runing_shop
|
||||
from datetime import datetime
|
||||
from amazon.del_brand import AmamzonBase, kill_process
|
||||
|
||||
# 导入 webview 用于前端通知
|
||||
try:
|
||||
import webview
|
||||
except ImportError:
|
||||
webview = None
|
||||
|
||||
class AmzoneApprove(AmamzonBase):
|
||||
|
||||
def SwitchPage(self):
|
||||
"""
|
||||
切换至 管理所有库存页面
|
||||
1、等待 //navigation-favorites-bar[@class="hydrated"] 出现
|
||||
"""
|
||||
navigation = self.tab.ele('xpath://navigation-favorites-bar[@class="hydrated"]')
|
||||
navigation.wait.displayed(raise_err=False)
|
||||
page_btn = navigation.sr('xpath://internal-fav-bar-links[@data-internal="navigation"]').sr(
|
||||
'xpath://a[@data-page-id="ezdpc-gui-inventory-mons"]')
|
||||
page_btn.wait.displayed(raise_err=False)
|
||||
page_btn.click(timeout=5)
|
||||
|
||||
self.tab.wait.doc_loaded()
|
||||
# 等待搜索框出现
|
||||
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group')
|
||||
search_region.wait.displayed(raise_err=False)
|
||||
|
||||
|
||||
def search(self,filter_type="ApprovalRequired"):
|
||||
sku_ls = []
|
||||
|
||||
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
|
||||
if len(load_ele) > 0:
|
||||
load_ele[0].wait.deleted(timeout=3, raise_err=False)
|
||||
time.sleep(0.5)
|
||||
|
||||
drop_down = self.tab.ele('xpath://div[contains(@class,"VolusListingStatusDropDown-module__verticalContainer")]//kat-dropdown')
|
||||
drop_down.wait.displayed(raise_err=False)
|
||||
drop_down.wait.enabled(raise_err=False)
|
||||
time.sleep(0.6)
|
||||
drop_down.click()
|
||||
# //kat-option[@value="SearchSuppressed"]
|
||||
xp = f'xpath://kat-option[@value="{filter_type}"]'
|
||||
print(f"正在寻找筛选条件 {filter_type},xpath: {xp}")
|
||||
approval_required = self.tab.eles(xp,timeout=5)
|
||||
if len(approval_required) == 0:
|
||||
print(f"【没有需要{filter_type}选项】没有需要{filter_type}的商品了")
|
||||
return sku_ls # "没有需要审批的商品了"
|
||||
else:
|
||||
approval_required = approval_required[0]
|
||||
approval_required.wait.displayed(raise_err=False)
|
||||
approval_required.click()
|
||||
|
||||
approval_required_text = approval_required.text
|
||||
print(f"已选择筛选条件: {approval_required_text}")
|
||||
|
||||
count = re.findall(r'\d+', approval_required_text)
|
||||
if count:
|
||||
count = int(count[0])
|
||||
print(f"待审批的商品数量: {count}")
|
||||
if count <= 0:
|
||||
print(f"没有需要{filter_type}的商品了")
|
||||
return sku_ls #"没有需要审批的商品了"
|
||||
asin = "B0C7KT6ZZN"
|
||||
search_input = self.tab.ele("xpath://kat-input[contains(@class,'SearchBox-module__searchInput')]").sr(
|
||||
'xpath://span[@class="container"]//input[@part="input"]')
|
||||
search_input.input(asin, clear=True)
|
||||
search_btn = self.tab.ele("xpath://kat-icon[@name='search']")
|
||||
search_btn.click()
|
||||
|
||||
for _ in range(3):
|
||||
# 等待加载完成
|
||||
load_ele = self.tab.ele("xpath://div[contains(@class,'Loader-module__loader')]")
|
||||
load_ele.wait.deleted(timeout=3, raise_err=False)
|
||||
time.sleep(0.5)
|
||||
search_input = self.tab.ele("xpath://kat-input[contains(@class,'SearchBox-module__searchInput')]").sr(
|
||||
'xpath://span[@class="container"]//input[@part="input"]')
|
||||
search_input.input(asin, clear=True)
|
||||
search_btn = self.tab.ele("xpath://kat-icon[@name='search']")
|
||||
search_btn.click()
|
||||
|
||||
load_ele = self.tab.ele("xpath://div[contains(@class,'Loader-module__loader')]")
|
||||
load_ele.wait.deleted(timeout=3, raise_err=False)
|
||||
time.sleep(0.5)
|
||||
|
||||
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=3)
|
||||
if len(sku_ls) > 0:
|
||||
break
|
||||
approval_required.click()
|
||||
|
||||
return sku_ls
|
||||
|
||||
def wait_loaded(self):
|
||||
# 等待加载完成
|
||||
try:
|
||||
load_ele = self.tab.ele(
|
||||
'xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//div[contains(@class,"Loader-module__loader")]|//kat-panel[@data-testid="kat-panel-ActionPanelContent"]/div[@data-f1-component]//div[contains(@class,"==")]/div[contains(@class,"==")]/span',
|
||||
timeout=3)
|
||||
load_ele.wait.deleted(timeout=5, raise_err=False)
|
||||
except Exception as e:
|
||||
print("等待加载中消失出错", e)
|
||||
|
||||
def clear_tab(self):
|
||||
all_tab = self.browser.get_tabs(title="亚马逊")
|
||||
close_tab = []
|
||||
for tab in all_tab:
|
||||
tab_id = tab if isinstance(tab,str) else tab.tab_id
|
||||
if self.tab.tab_id == tab:
|
||||
continue
|
||||
close_tab.append(tab)
|
||||
print("需要关闭的标签页",close_tab)
|
||||
print("当前操作的tab_id",self.tab.tab_id)
|
||||
self.browser.close_tabs(close_tab)
|
||||
|
||||
def handle_repair_product(self,tab:ChromiumTab):
|
||||
def wait_loaded():
|
||||
# 等待加载完成
|
||||
try:
|
||||
load_ele = tab.ele(
|
||||
'xpath://div[@class="contentWrapper"]//div[contains(@class,"==")]/span[not(node())]',
|
||||
timeout=3)
|
||||
load_ele.wait.deleted(timeout=5, raise_err=False)
|
||||
except Exception as e:
|
||||
print("等待加载中消失出错", e)
|
||||
|
||||
tab.wait.doc_loaded(timeout=30)
|
||||
submit_btn_ls = tab.eles('xpath://div[@id="ahd-product-policies-table"]/div//div[@data-testid="nextStepsMetricWrapper"]//kat-button[@label="提交"]',timeout=10)
|
||||
if len(submit_btn_ls) == 0:
|
||||
print("未找到提交按钮,可能页面未加载完成或者页面结构发生变化")
|
||||
return
|
||||
# todo 遍历
|
||||
for index in range(len(submit_btn_ls)):
|
||||
if index > len(submit_btn_ls)-1:
|
||||
print("提交按钮数量发生变化,停止处理")
|
||||
break
|
||||
|
||||
submit_btn = submit_btn_ls[index]
|
||||
|
||||
# 关闭按钮
|
||||
close_x_btn = tab.eles('xpath://div[@class="flyoutPanelContent"]/span',timeout=5)
|
||||
if len(close_x_btn) > 0:
|
||||
close_x_btn[0].wait.displayed(timeout=5, raise_err=False)
|
||||
close_x_btn[0].click()
|
||||
|
||||
submit_btn.wait.enabled(timeout=5, raise_err=False)
|
||||
submit_btn.click()
|
||||
time.sleep(0.5)
|
||||
self.wait_loaded()
|
||||
|
||||
wait_loaded()
|
||||
|
||||
need_input = tab.eles('xpath://div[@class="contentWrapper"]//kat-input[@data-testid="kat-input-dew:ump_epr_resgitration_number_title"]',timeout=5)
|
||||
if len(need_input) > 0:
|
||||
print("需要输入注册号,不符合操作要求,跳过...")
|
||||
continue
|
||||
|
||||
target_title = tab.eles('xpath://div[@class="contentWrapper"]//section//h4[text()="警告和安全信息"]')
|
||||
if len(target_title) > 0:
|
||||
not_start = tab.eles('xpath://div[@class="contentWrapper"]//div[@aria-label="安全证明"]//kat-label[@text="未开始"]')
|
||||
if len(not_start) == 0:
|
||||
print("安全证明-不是未开始状态,不需要操作")
|
||||
continue
|
||||
|
||||
not_start.wait.displayed(timeout=5, raise_err=False)
|
||||
not_start.click()
|
||||
|
||||
wait_loaded()
|
||||
|
||||
check = tab.ele('xpath://div[@class="contentWrapper"]//kat-checkbox[@name="value"]')
|
||||
check.wait.displayed(timeout=5, raise_err=False)
|
||||
check.click()
|
||||
|
||||
save_btn = tab.ele('xpath://div[@class="contentWrapper"]//kat-button[@variant="primary"]')
|
||||
save_btn.wait.enabled(timeout=5, raise_err=False)
|
||||
save_btn.click()
|
||||
|
||||
wait_loaded()
|
||||
|
||||
close_btn = tab.ele('xpath://div[@class="contentWrapper"]//kat-button[@label="关闭"]',timeout=10)
|
||||
close_btn.wait.displayed(timeout=5, raise_err=False)
|
||||
close_btn.click()
|
||||
|
||||
save_btn_ls = tab.eles('xpath://div[@class="contentWrapper"]//kat-button[@variant="primary"]',timeout=5)
|
||||
if len(save_btn_ls) > 0:
|
||||
option_selection_ls = tab.eles('xpath://div[@class="contentWrapper"]//div[@role="option"]',timeout=5)
|
||||
for option in option_selection_ls:
|
||||
option.click()
|
||||
time.sleep(0.5)
|
||||
|
||||
save_btn_ls[0].wait.enabled(timeout=5, raise_err=False)
|
||||
save_btn_ls[0].click()
|
||||
|
||||
wait_loaded()
|
||||
|
||||
close_btn = tab.ele('xpath://div[@class="contentWrapper"]//kat-button[@label="关闭"]',timeout=10)
|
||||
close_btn.wait.displayed(timeout=5, raise_err=False)
|
||||
close_btn.wait.enabled(timeout=5, raise_err=False)
|
||||
close_btn.click()
|
||||
|
||||
submit_btn_ls = tab.eles('xpath://div[@id="ahd-product-policies-table"]/div//div[@data-testid="nextStepsMetricWrapper"]//kat-button[@label="提交"]',timeout=10)
|
||||
|
||||
print("修复商品信息处理完成!")
|
||||
tab.close()
|
||||
|
||||
def run_page_action(self):
|
||||
print("开始执行")
|
||||
num = 0
|
||||
retry_num = 0
|
||||
already_asin = set()
|
||||
|
||||
while retry_num < 3: # 最多重试3次
|
||||
# if num > 3: #测试
|
||||
# return
|
||||
# 等待加载完成
|
||||
try:
|
||||
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
|
||||
if len(load_ele) > 0:
|
||||
load_ele[0].wait.deleted(timeout=3, raise_err=False)
|
||||
time.sleep(0.5)
|
||||
|
||||
# 获取当前页码
|
||||
try:
|
||||
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
|
||||
if len(page_pamel) > 0:
|
||||
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
|
||||
|
||||
# 总页数
|
||||
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
|
||||
if len(total_page) > 0:
|
||||
total_page = total_page[-1].text
|
||||
else:
|
||||
total_page = 0
|
||||
print(f"当前页码: {current_page} / 总页数: {total_page}")
|
||||
except Exception as e:
|
||||
print("获取页码失败", e)
|
||||
|
||||
|
||||
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
|
||||
print(f"获取到 {len(sku_ls)}")
|
||||
# for sku_ele in sku_ls[0:2]:
|
||||
for sku_ele in sku_ls:
|
||||
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]')
|
||||
|
||||
asin = sku_ele.ele('xpath:.//div[contains(@class,"JanusSplitBox-module__container")]//div[contains(@class,"JanusSplitBox-module__panel--") and contains(string(.),"ASIN")]/..//div[last()]').text
|
||||
print(f"ASIN {asin} 存在问题,正在点击解决...")
|
||||
if asin in already_asin:
|
||||
print(f"{asin} 已经处理过了,跳过")
|
||||
continue
|
||||
solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]|.//kat-link[@label="修复被禁止显示的商品"]|.//kat-link[@label="解决商品移除风险"]')
|
||||
if len(solve_problem) == 0:
|
||||
print(f"{asin},没有处理入口,不处理")
|
||||
yield (asin,"没有处理入口,不处理")
|
||||
already_asin.add(asin)
|
||||
continue
|
||||
|
||||
solve_problem[0].click()
|
||||
|
||||
action_panel = self.tab.ele('xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]')
|
||||
action_panel.wait.displayed(timeout=5, raise_err=False)
|
||||
|
||||
self.wait_loaded()
|
||||
|
||||
try:
|
||||
# 如果存在“请求批准”按钮,则直接返回跳过
|
||||
request_approval_btn = self.tab.eles('xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//div[@data-testid="section-header" and contains(string(.),"移除")]',timeout=5)
|
||||
if len(request_approval_btn) > 0:
|
||||
print(f"ASIN {asin} 存在请求批准按钮,不需要处理,跳过...")
|
||||
panel_close_btn = self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]',
|
||||
timeout=5).sr('xpath:.//button[@class="close"]')
|
||||
panel_close_btn.click()
|
||||
|
||||
action_panel.wait.deleted(timeout=3, raise_err=False)
|
||||
yield (asin,"请求批准")
|
||||
|
||||
# 无需采取任何操作
|
||||
not_operate = self.tab.eles('xpath://kat-alert[contains(@description,"如果您之前已提交更改,则这些更改当前正在处理中") and not(@dismissed)]',timeout=3)
|
||||
if len(not_operate) > 0:
|
||||
print(f"ASIN {asin} 无需采取任何操作,跳过...")
|
||||
yield (asin,"无需操作")
|
||||
already_asin.add(asin)
|
||||
continue
|
||||
|
||||
# 解决商品信息违规问题 按钮
|
||||
kat_box_ls = action_panel.eles('xpath:.//kat-box[@variant="white"]',timeout=5)
|
||||
# if len(kat_box_ls) == 0:
|
||||
# print(f"ASIN {asin} 的操作面板中未找到选项,跳过...")
|
||||
# continue
|
||||
print(f"ASIN {asin}需要处理的有:{len(kat_box_ls)}个问题.")
|
||||
# 解决商品信息违规问题 按钮,有多少个都要处理
|
||||
for i in range(len(kat_box_ls)):
|
||||
print(f"开始处理第{i}个问题")
|
||||
if len(kat_box_ls) <= i:
|
||||
print(f"ASIN {asin} 的操作面板中选项数量发生变化,停止处理...")
|
||||
break
|
||||
kat_box_ls[i].click()
|
||||
time.sleep(0.5)
|
||||
self.wait_loaded()
|
||||
|
||||
# TODO 增加 “出现警告和安全信息时候下滑” 的情况
|
||||
safe_handle_ls = self.tab.eles('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//div[@aria-label="安全证明"]//kat-label[@text="未开始"]',
|
||||
timeout=5)
|
||||
if len(safe_handle_ls) > 0:
|
||||
print("存在安全证明,正在处理...")
|
||||
safe_handle_ls[0].click()
|
||||
time.sleep(0.5)
|
||||
self.wait_loaded()
|
||||
check_box = self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//kat-checkbox')
|
||||
check_box.click()
|
||||
else:
|
||||
problem_selection_ls = self.tab.eles(
|
||||
'xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//div[@data-testid="registry-list"]//div[@data-testid="registry"]',
|
||||
timeout=10)
|
||||
print("待选择数量",len(problem_selection_ls))
|
||||
for problem_selection in problem_selection_ls:
|
||||
problem_selection.click()
|
||||
print("点击选择完成")
|
||||
time.sleep(0.5)
|
||||
|
||||
# 点击保存
|
||||
confirm_btn = self.tab.ele('xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//kat-button[@variant="primary"]')
|
||||
confirm_btn.wait.displayed(timeout=3, raise_err=False)
|
||||
confirm_btn.click()
|
||||
|
||||
self.wait_loaded()
|
||||
|
||||
# 等待关闭按钮出现
|
||||
try:
|
||||
close_btn = self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//kat-button[@label="关闭"]',
|
||||
timeout=10)
|
||||
close_btn.click()
|
||||
except Exception as e:
|
||||
print("点击关闭按钮失败",e)
|
||||
|
||||
kat_box_ls = action_panel.eles('xpath:.//kat-box[@variant="white"]', timeout=5)
|
||||
|
||||
# 修复商品信息
|
||||
repair_product = action_panel.eles('xpath:.//kat-button[@class="action-button"]',timeout=5)
|
||||
if len(repair_product) > 0:
|
||||
print(f"ASIN {asin} 存在修复商品信息按钮,正在点击...")
|
||||
self.clear_tab()
|
||||
repair_product[0].click()
|
||||
time.sleep(1)
|
||||
# 获取最新的tab
|
||||
new_tab = self.browser.latest_tab
|
||||
if isinstance(new_tab, str):
|
||||
new_tab = self.browser.get_tab(id_or_num=new_tab)
|
||||
|
||||
self.handle_repair_product(new_tab)
|
||||
except Exception as e:
|
||||
print("【asin】:",asin,"处理失败",traceback.format_exc())
|
||||
yield (asin,"处理失败")
|
||||
already_asin.add(asin)
|
||||
continue
|
||||
|
||||
yield (asin,"处理完成")
|
||||
already_asin.add(asin)
|
||||
|
||||
try:
|
||||
# 全部操作完成,关闭
|
||||
panel_close_btn= self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]',
|
||||
timeout=5).sr('xpath:.//button[@class="close"]')
|
||||
panel_close_btn.click()
|
||||
|
||||
action_panel.wait.deleted(timeout=3, raise_err=False)
|
||||
|
||||
except Exception as e:
|
||||
print("关闭操作面板失败", e)
|
||||
|
||||
# 判断是否存在需要翻页的情况
|
||||
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
|
||||
if len(page_pamel) == 0:
|
||||
break
|
||||
next_page_btn = page_pamel[0].sr('xpath:.//span[@part="pagination-nav-right"]')
|
||||
class_str = next_page_btn.attr('class')
|
||||
if "end" in class_str:
|
||||
break
|
||||
next_page_btn.click()
|
||||
num += 1
|
||||
print(f"【程序计算】正在翻页,已翻 {num} 页...")
|
||||
|
||||
already_asin = set()
|
||||
|
||||
except Exception as e:
|
||||
print("处理审批操作异常", e)
|
||||
traceback.print_exc()
|
||||
retry_num += 1
|
||||
self.tab.refresh()
|
||||
self.tab.wait.doc_loaded(raise_err=False,timeout=120)
|
||||
|
||||
|
||||
|
||||
|
||||
class ApproveTask:
|
||||
"""审批任务处理类:负责处理产品风险审批任务"""
|
||||
|
||||
country_info = {
|
||||
"DE": "德国",
|
||||
"FR": "法国",
|
||||
"ES": "西班牙",
|
||||
"IT": "意大利",
|
||||
"UK": "英国"
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def show_notification(message: str, message_type: str = "error"):
|
||||
"""显示 pywebview 顶层通知(5秒后自动消失)
|
||||
|
||||
Args:
|
||||
message: 通知消息
|
||||
message_type: 消息类型 (success/warning/error/info)
|
||||
"""
|
||||
if webview and webview.windows:
|
||||
try:
|
||||
# 转义单引号,防止 JavaScript 语法错误
|
||||
safe_message = message.replace("'", "\\'").replace('"', '\\"').replace('\n', '\\n')
|
||||
|
||||
# 设置通知样式颜色
|
||||
color_map = {
|
||||
'success': '#67C23A',
|
||||
'warning': '#E6A23C',
|
||||
'error': '#F56C6C',
|
||||
'info': '#909399'
|
||||
}
|
||||
bg_color = color_map.get(message_type, '#F56C6C')
|
||||
|
||||
# 构建前端通知的 JavaScript 代码 - 创建原生 HTML 通知
|
||||
js_code = f"""
|
||||
(function() {{
|
||||
// 移除已存在的通知
|
||||
var existingNotif = document.getElementById('pywebview-notification');
|
||||
if (existingNotif) {{
|
||||
existingNotif.remove();
|
||||
}}
|
||||
|
||||
// 创建通知容器
|
||||
var notif = document.createElement('div');
|
||||
notif.id = 'pywebview-notification';
|
||||
notif.style.cssText = 'position: fixed; top: 20px; left: 50%; transform: translateX(-50%); ' +
|
||||
'background: {bg_color}; color: white; padding: 12px 20px; border-radius: 4px; ' +
|
||||
'box-shadow: 0 2px 12px rgba(0,0,0,0.3); z-index: 99999; font-size: 14px; ' +
|
||||
'max-width: 600px; word-wrap: break-word; display: flex; align-items: center; gap: 10px;';
|
||||
|
||||
// 添加消息内容
|
||||
var msgSpan = document.createElement('span');
|
||||
msgSpan.textContent = '{safe_message}';
|
||||
notif.appendChild(msgSpan);
|
||||
|
||||
// 添加关闭按钮
|
||||
var closeBtn = document.createElement('span');
|
||||
closeBtn.innerHTML = '×';
|
||||
closeBtn.style.cssText = 'cursor: pointer; font-size: 18px; font-weight: bold; margin-left: 10px;';
|
||||
closeBtn.onclick = function() {{ notif.remove(); }};
|
||||
notif.appendChild(closeBtn);
|
||||
|
||||
// 添加到页面
|
||||
document.body.appendChild(notif);
|
||||
|
||||
// 5秒后自动消失
|
||||
setTimeout(function() {{
|
||||
if (notif && notif.parentNode) {{
|
||||
notif.style.transition = 'opacity 0.3s';
|
||||
notif.style.opacity = '0';
|
||||
setTimeout(function() {{ notif.remove(); }}, 300);
|
||||
}}
|
||||
}}, 5000);
|
||||
}})();
|
||||
"""
|
||||
# 在第一个窗口中执行 JavaScript
|
||||
webview.windows[0].evaluate_js(js_code)
|
||||
except Exception as e:
|
||||
print(f"显示通知失败: {str(e)}")
|
||||
|
||||
"""
|
||||
处理流程:
|
||||
1、解析任务json,提取对应的companyName 组组装user_info实例化 AmzoneApprove
|
||||
2、打开对应的店铺、遍历所有的国家站点处理
|
||||
3、切换至指定国家,切换至 管理所有库存页面
|
||||
4、调用search方法,查询出是否存在需要审批的商品,如果没有则结束;如果有则进入处理流程
|
||||
5、进入处理流程后,调用run_page_action方法(注意是通过yield 返回数据的)
|
||||
|
||||
任务json示例:
|
||||
{
|
||||
"type": "product-risk-resolve-run",
|
||||
"ts": 1775395996876,
|
||||
"data": {
|
||||
"taskId": 925,
|
||||
"items": [
|
||||
{
|
||||
"matched": true,
|
||||
"platform": "亚马逊",
|
||||
"shopName": "魏振峰",
|
||||
"shopId": "27543917795757",
|
||||
"companyName": "rongchuang123",
|
||||
"openStoreUrl": null,
|
||||
"matchedUserId": 17543915345493,
|
||||
"matchStatus": "MATCHED",
|
||||
"matchMessage": null
|
||||
}
|
||||
],
|
||||
"country_codes": [
|
||||
"UK",
|
||||
"DE",
|
||||
"FR",
|
||||
"ES"
|
||||
]
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
def __init__(self, user_info: dict = None):
|
||||
"""初始化审批任务处理器
|
||||
|
||||
Args:
|
||||
user_info: 用户信息字典,包含 company, username, password
|
||||
"""
|
||||
self.user_info = user_info or {}
|
||||
self.running = True
|
||||
|
||||
def log(self, message: str, level: str = "INFO"):
|
||||
"""日志输出
|
||||
|
||||
Args:
|
||||
message: 日志消息
|
||||
level: 日志级别
|
||||
"""
|
||||
from datetime import datetime
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
if level == "ERROR":
|
||||
self.show_notification(message, "error")
|
||||
print(f"[{timestamp}] [ApproveTask] [{level}] {message}")
|
||||
|
||||
def process_task(self, task_data: dict):
|
||||
"""处理审批任务主入口
|
||||
|
||||
Args:
|
||||
task_data: 任务数据
|
||||
"""
|
||||
try:
|
||||
data = task_data.get("data", {})
|
||||
task_id = data.get("taskId")
|
||||
items = data.get("items", [])
|
||||
country_codes = data.get("country_codes", [])
|
||||
risk_listing_filter = data.get("risk_listing_filter", "")
|
||||
|
||||
if not task_id:
|
||||
self.log("任务ID为空,跳过", "WARNING")
|
||||
return
|
||||
|
||||
if not items:
|
||||
self.log("店铺列表为空,跳过", "WARNING")
|
||||
return
|
||||
|
||||
if not country_codes:
|
||||
self.log("国家列表为空,跳过", "WARNING")
|
||||
return
|
||||
|
||||
self.log(f"开始处理审批任务 {task_id},共 {len(items)} 个店铺,{len(country_codes)} 个国家")
|
||||
|
||||
from config import runing_task
|
||||
runing_task[task_id] = {
|
||||
"status": "running",
|
||||
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"total_shops": len(items),
|
||||
"processed_shops": 0,
|
||||
"total_countries": len(country_codes) * len(items),
|
||||
"processed_countries": 0,
|
||||
"total_asins": 0,
|
||||
"processed_asins": 0,
|
||||
"success_count": 0,
|
||||
"failed_count": 0,
|
||||
"stop_requested": False
|
||||
}
|
||||
|
||||
# 遍历处理每个店铺
|
||||
for idx, shop_item in enumerate(items, 1):
|
||||
# 检查是否收到暂停请求
|
||||
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
|
||||
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理", "WARNING")
|
||||
runing_task[task_id]["status"] = "stopped"
|
||||
return
|
||||
|
||||
shop_name = shop_item.get("shopName", "未知店铺")
|
||||
self.log(f"[{idx}/{len(items)}] 开始处理店铺: {shop_name}")
|
||||
self.show_notification(f"开始处理店铺: {shop_name}", "info")
|
||||
|
||||
try:
|
||||
self.process_shop(shop_item, country_codes, task_id,risk_listing_filter)
|
||||
# 更新已处理店铺数
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["processed_shops"] += 1
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR")
|
||||
self.log(traceback.format_exc(), "ERROR")
|
||||
|
||||
# 更新任务状态
|
||||
if task_id in runing_task:
|
||||
if runing_task[task_id].get("stop_requested", False):
|
||||
runing_task[task_id]["status"] = "stopped"
|
||||
self.log(f"任务 {task_id} 已被暂停!")
|
||||
else:
|
||||
runing_task[task_id]["status"] = "completed"
|
||||
self.log(f"任务 {task_id} 处理完成!")
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
|
||||
if task_id:
|
||||
from config import runing_task
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["status"] = "failed"
|
||||
runing_task[task_id]["error"] = str(e)
|
||||
|
||||
def process_shop(self, shop_item: dict, country_codes: list, task_id: int, risk_listing_filter: str):
|
||||
"""处理单个店铺
|
||||
|
||||
Args:
|
||||
shop_item: 店铺信息
|
||||
country_codes: 国家代码列表
|
||||
task_id: 任务ID
|
||||
risk_listing_filter: 风险商品筛选条件
|
||||
"""
|
||||
shop_name = shop_item.get("shopName", "未知店铺")
|
||||
company_name = shop_item.get("companyName", "")
|
||||
|
||||
if not company_name:
|
||||
self.log(f"店铺 {shop_name} 的公司名称为空,跳过", "WARNING")
|
||||
return
|
||||
|
||||
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["current_shop"] = shop_name
|
||||
|
||||
# 将店铺添加到正在执行中的店铺列表
|
||||
start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
runing_shop[shop_name] = start_time
|
||||
self.log(f"店铺 {shop_name} 已添加到执行列表,账号: {company_name},开始时间: {start_time}")
|
||||
|
||||
# 店铺打开重试最多3次
|
||||
driver = None
|
||||
max_retries = 3
|
||||
|
||||
error_info = ""
|
||||
for retry in range(max_retries):
|
||||
try:
|
||||
self.log(f"尝试打开店铺 {shop_name} (第 {retry + 1}/{max_retries} 次)")
|
||||
|
||||
# 如果不是第一次尝试,先杀进程
|
||||
# if retry > 0:
|
||||
# self.log("重试前先杀掉浏览器进程...")
|
||||
# kill_process("v6")
|
||||
# kill_process("v5")
|
||||
# time.sleep(2)
|
||||
|
||||
# 组装用户信息并创建驱动
|
||||
user_info = {
|
||||
**self.user_info,
|
||||
"company": company_name
|
||||
}
|
||||
driver = AmzoneApprove(user_info)
|
||||
browser = driver.open_shop(shop_name)
|
||||
|
||||
if browser and browser != "店铺不存在":
|
||||
self.log(f"成功打开店铺 {shop_name}")
|
||||
break
|
||||
else:
|
||||
self.log(f"打开店铺失败: {browser}", "WARNING")
|
||||
driver = None
|
||||
|
||||
# 判断是否需要登录
|
||||
need_login = driver.need_login()
|
||||
if need_login:
|
||||
self.log(f"店铺 {shop_name} 需要登录,正在登录...")
|
||||
password = ""
|
||||
|
||||
login_success = driver.login(password)
|
||||
if login_success:
|
||||
self.log(f"店铺 {shop_name} 登录成功,正在重新打开店铺...")
|
||||
browser = driver.open_shop(shop_name)
|
||||
if browser and browser != "店铺不存在":
|
||||
self.log(f"成功打开店铺 {shop_name} 登录后")
|
||||
break
|
||||
else:
|
||||
self.log(f"登录后打开店铺失败: {browser}", "WARNING")
|
||||
driver = None
|
||||
else:
|
||||
self.log(f"店铺 {shop_name} 登录失败", "WARNING")
|
||||
driver = None
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"打开店铺异常: {traceback.format_exc()}", "INFO")
|
||||
driver = None
|
||||
error_info = str(e)
|
||||
time.sleep(10)
|
||||
|
||||
# 如果还有重试机会,等待后继续
|
||||
if retry < max_retries - 1:
|
||||
time.sleep(3)
|
||||
|
||||
# 检查是否成功打开
|
||||
if not driver or not browser or browser == "店铺不存在":
|
||||
error_msg = f"店铺 {shop_name} 打开失败,已重试 {max_retries} 次,跳过该店铺,{error_info}"
|
||||
self.log(error_msg, "ERROR")
|
||||
# 从执行列表中移除
|
||||
if shop_name in runing_shop:
|
||||
del runing_shop[shop_name]
|
||||
return
|
||||
|
||||
try:
|
||||
# 处理每个国家
|
||||
for country_code in country_codes:
|
||||
# 检查是否收到暂停请求
|
||||
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
|
||||
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理国家", "WARNING")
|
||||
break
|
||||
|
||||
try:
|
||||
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter)
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR")
|
||||
self.log(traceback.format_exc(), "ERROR")
|
||||
# 最后回传,标记完成
|
||||
try:
|
||||
self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
|
||||
except Exception as e:
|
||||
self.log(f"回传结果失败: {str(e)}", "ERROR")
|
||||
finally:
|
||||
# 关闭店铺
|
||||
try:
|
||||
if driver:
|
||||
self.log(f"关闭店铺 {shop_name}")
|
||||
driver.close_store()
|
||||
time.sleep(2)
|
||||
except Exception as e:
|
||||
self.log(f"关闭店铺失败: {str(e)}", "WARNING")
|
||||
|
||||
# 从正在执行中的店铺列表中移除
|
||||
if shop_name in runing_shop:
|
||||
del runing_shop[shop_name]
|
||||
self.log(f"店铺 {shop_name} 已从执行列表中移除")
|
||||
|
||||
def process_country(self, driver: AmzoneApprove, country_code: str, task_id: int, shop_name: str, risk_listing_filter: str):
|
||||
"""处理单个国家的审批任务
|
||||
|
||||
Args:
|
||||
driver: AmzoneApprove驱动实例
|
||||
country_code: 国家代码(如 UK, DE, FR 等)
|
||||
task_id: 任务ID
|
||||
shop_name: 店铺名称
|
||||
risk_listing_filter: 风险商品筛选条件
|
||||
"""
|
||||
from config import runing_task
|
||||
|
||||
# 转换国家代码为中文名称
|
||||
country_name = self.country_info.get(country_code, country_code)
|
||||
info_mes = f"开始处理国家: {country_name} ({country_code})"
|
||||
self.log(info_mes)
|
||||
self.show_notification(info_mes, "info")
|
||||
|
||||
# 更新当前处理的国家
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["current_country"] = country_name
|
||||
|
||||
# 切换国家,最多重试3次
|
||||
max_retries = 3
|
||||
switch_success = False
|
||||
|
||||
for retry in range(max_retries):
|
||||
try:
|
||||
self.log(f"尝试切换到国家 {country_name} (第 {retry + 1}/{max_retries} 次)")
|
||||
|
||||
# 如果不是第一次尝试,先刷新页面
|
||||
if retry > 0:
|
||||
self.log("重试前刷新页面...")
|
||||
try:
|
||||
driver.tab.refresh()
|
||||
time.sleep(3)
|
||||
except Exception as e:
|
||||
self.log(f"刷新页面失败: {str(e)}", "WARNING")
|
||||
|
||||
switch_success = driver.SwitchingCountries(country_name)
|
||||
if switch_success:
|
||||
self.log(f"成功切换到国家 {country_name}")
|
||||
break
|
||||
else:
|
||||
self.log(f"切换到国家 {country_name} 失败", "WARNING")
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"切换国家 {country_name} 异常: {str(e)}", "ERROR")
|
||||
self.log(traceback.format_exc(), "ERROR")
|
||||
|
||||
# 如果还有重试机会,等待后继续
|
||||
if retry < max_retries - 1:
|
||||
time.sleep(2)
|
||||
|
||||
# 如果切换失败,直接返回
|
||||
if not switch_success:
|
||||
error_message = f"切换到国家 {country_name} 失败,已重试 {max_retries} 次,跳过该国家"
|
||||
self.log(error_message, "ERROR")
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["processed_countries"] += 1
|
||||
return
|
||||
|
||||
# 切换到库存管理页面
|
||||
try:
|
||||
driver.SwitchPage()
|
||||
self.log(f"已切换到库存管理页面")
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"切换页面失败: {str(e)}", "ERROR")
|
||||
self.log(traceback.format_exc(), "ERROR")
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["processed_countries"] += 1
|
||||
return
|
||||
|
||||
# 搜索需要审批的商品,最多重试3次
|
||||
sku_ls = []
|
||||
for retry in range(max_retries):
|
||||
try:
|
||||
self.log(f"尝试搜索需要审批的商品 (第 {retry + 1}/{max_retries} 次)")
|
||||
sku_ls = driver.search(filter_type=risk_listing_filter)
|
||||
break
|
||||
except Exception as e:
|
||||
self.log(f"搜索商品异常: {str(e)}", "ERROR")
|
||||
if retry < max_retries - 1:
|
||||
try:
|
||||
driver.tab.refresh()
|
||||
time.sleep(3)
|
||||
except Exception as refresh_error:
|
||||
self.log(f"刷新页面失败: {str(refresh_error)}", "WARNING")
|
||||
|
||||
# 如果没有需要审批的商品,直接返回
|
||||
if len(sku_ls) == 0:
|
||||
self.log(f"国家 {country_name} 没有需要审批的商品")
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["processed_countries"] += 1
|
||||
return
|
||||
|
||||
self.log(f"国家 {country_name} 有 {len(sku_ls)} 个需要审批的商品,开始处理...")
|
||||
|
||||
# 处理所有需要审批的商品(通过yield获取结果)
|
||||
try:
|
||||
for asin, status in driver.run_page_action():
|
||||
# 检查是否收到暂停请求
|
||||
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
|
||||
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理ASIN", "WARNING")
|
||||
break
|
||||
|
||||
self.log(f"ASIN {asin} 处理结果: {status}")
|
||||
|
||||
# 更新任务状态
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["current_asin"] = asin
|
||||
runing_task[task_id]["processed_asins"] += 1
|
||||
|
||||
if status == "处理完成":
|
||||
runing_task[task_id]["success_count"] += 1
|
||||
elif status == "请求批准":
|
||||
# 请求批准的商品也算作成功(因为不需要处理)
|
||||
runing_task[task_id]["success_count"] += 1
|
||||
else:
|
||||
runing_task[task_id]["failed_count"] += 1
|
||||
|
||||
# 回传结果到API
|
||||
try:
|
||||
self.post_result(task_id, shop_name, country_code, asin, status)
|
||||
except Exception as e:
|
||||
self.log(f"回传结果失败: {str(e)}", "ERROR")
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"处理审批商品异常: {str(e)}", "ERROR")
|
||||
self.log(traceback.format_exc(), "ERROR")
|
||||
|
||||
# 更新已处理国家数
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["processed_countries"] += 1
|
||||
|
||||
self.log(f"国家 {country_name} 处理完成")
|
||||
|
||||
def post_result(self, task_id: int, shop_name: str, country_code: str, asin: str, status: str,is_done: bool = False):
|
||||
"""回传处理结果到API
|
||||
|
||||
Args:
|
||||
task_id: 任务ID
|
||||
shop_name: 店铺名称
|
||||
country_code: 国家代码
|
||||
asin: ASIN
|
||||
status: 处理状态
|
||||
"""
|
||||
import requests
|
||||
from config import DELETE_BRAND_API_BASE
|
||||
|
||||
url = f"{DELETE_BRAND_API_BASE}/api/product-risk-resolve/tasks/{task_id}/result"
|
||||
if status == "请求批准":
|
||||
country_data = {
|
||||
"status": "",
|
||||
"shopName": shop_name,
|
||||
"productAsinSku": "",
|
||||
"done": is_done,
|
||||
"removeAsin": asin,
|
||||
"removeStatus": status
|
||||
}
|
||||
else:
|
||||
country_data = {
|
||||
"status": status,
|
||||
"shopName": shop_name,
|
||||
"productAsinSku": asin,
|
||||
"done": is_done,
|
||||
"removeAsin": "",
|
||||
"removeStatus": ""
|
||||
}
|
||||
|
||||
payload = {
|
||||
"shops": [
|
||||
{
|
||||
"error": "",
|
||||
"countries": {
|
||||
country_code : [ country_data ]
|
||||
},
|
||||
"shopName": shop_name
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
max_retries = 3
|
||||
for retry in range(max_retries):
|
||||
try:
|
||||
print("=====================================")
|
||||
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
|
||||
self.log(f"回传URL: {url}")
|
||||
self.log(f"回传数据: {payload}")
|
||||
response = requests.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
timeout=30,
|
||||
verify=False
|
||||
)
|
||||
self.log(f"回传结果: {response.text}")
|
||||
|
||||
if response.status_code == 200:
|
||||
self.log(f"结果回传成功: {asin} - {status}")
|
||||
return
|
||||
else:
|
||||
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
|
||||
print("=====================================")
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"调用API异常: {str(e)}", "ERROR")
|
||||
print("=====================================")
|
||||
|
||||
# 如果还有重试机会,等待后继续
|
||||
if retry < max_retries - 1:
|
||||
time.sleep(2)
|
||||
|
||||
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
user_info = {
|
||||
"company": "rongchuang123",
|
||||
"username": "自动化_Robot",
|
||||
"password": "#20zsg25"
|
||||
}
|
||||
shop_name = "魏振峰"
|
||||
country = "西班牙"
|
||||
kill_process('v6')
|
||||
driver = AmzoneApprove(user_info)
|
||||
browser = driver.open_shop(shop_name)
|
||||
sw_suc = driver.SwitchingCountries(country)
|
||||
driver.SwitchPage()
|
||||
risk_listing_filter = "Active"
|
||||
for _ in range(3):
|
||||
try:
|
||||
sku_ls = driver.search(filter_type=risk_listing_filter)
|
||||
break
|
||||
except Exception as e:
|
||||
print(e)
|
||||
driver.tab.refresh()
|
||||
if len(sku_ls) > 0:
|
||||
print("有数据,开始操作")
|
||||
for asin, status in driver.run_page_action():
|
||||
print(f"ASIN {asin} 的处理结果: {status}")
|
||||
print("已完成操作")
|
||||
|
||||
# for i in range(0):
|
||||
# print(i)
|
||||
1018
app/amazon/approve.py
Normal file
1018
app/amazon/approve.py
Normal file
File diff suppressed because it is too large
Load Diff
0
app/amazon/backend_api.py
Normal file
0
app/amazon/backend_api.py
Normal file
@@ -10,6 +10,7 @@ import os
|
||||
from typing import Literal
|
||||
from DrissionPage import Chromium
|
||||
from DrissionPage.common import By
|
||||
from DrissionPage._pages.chromium_tab import ChromiumTab
|
||||
|
||||
|
||||
def kill_process(version: Literal["v5", "v6"]):
|
||||
@@ -33,7 +34,7 @@ class ZiniaoDriver:
|
||||
self.socket_port = socket_port
|
||||
self.client_path = None
|
||||
self.browser = None
|
||||
self.tab = None
|
||||
self.tab: ChromiumTab = None
|
||||
self.store_id = None
|
||||
|
||||
def get_zinaio_exe(self, protocol_name: str = "superbrowser"):
|
||||
@@ -52,7 +53,8 @@ class ZiniaoDriver:
|
||||
command, _ = winreg.QueryValueEx(key, "")
|
||||
winreg.CloseKey(key)
|
||||
if isinstance(command, str):
|
||||
exe_path = command.split()[0]
|
||||
sub = "ziniao.exe"
|
||||
exe_path = command[0: command.find(sub)+len(sub)+1]
|
||||
else:
|
||||
exe_path = command[0]
|
||||
return exe_path
|
||||
@@ -62,7 +64,8 @@ class ZiniaoDriver:
|
||||
command, _ = winreg.QueryValueEx(key, "")
|
||||
winreg.CloseKey(key)
|
||||
if isinstance(command, str):
|
||||
exe_path = command.split()[0]
|
||||
sub = "ziniao.exe"
|
||||
exe_path = command[0: command.find(sub)+len(sub)+1]
|
||||
else:
|
||||
exe_path = command[0]
|
||||
return exe_path
|
||||
@@ -107,6 +110,7 @@ class ZiniaoDriver:
|
||||
"""
|
||||
if version == "v5":
|
||||
process_name = 'SuperBrowser.exe'
|
||||
os.system('taskkill /f /t /im ' + "starter.exe")
|
||||
else:
|
||||
process_name = 'ziniao.exe'
|
||||
os.system('taskkill /f /t /im ' + process_name)
|
||||
@@ -133,11 +137,11 @@ class ZiniaoDriver:
|
||||
print(r)
|
||||
return r.get("browserList")
|
||||
elif str(r.get("statusCode")) == "-10003":
|
||||
print(f"login Err {json.dumps(r, ensure_ascii=False)}")
|
||||
exit()
|
||||
print(f"【get_browser_list】登录失败 {json.dumps(r, ensure_ascii=False)}")
|
||||
# exit()
|
||||
else:
|
||||
print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
|
||||
exit()
|
||||
print(f"【get_browser_list】失败 {json.dumps(r, ensure_ascii=False)} ")
|
||||
# exit()
|
||||
|
||||
def open_store(self, store_info, isWebDriverReadOnlyMode=0, isprivacy=0,
|
||||
isHeadless=0, cookieTypeSave=0, jsInfo=""):
|
||||
@@ -185,11 +189,11 @@ class ZiniaoDriver:
|
||||
if str(r.get("statusCode")) == "0":
|
||||
return r
|
||||
elif str(r.get("statusCode")) == "-10003":
|
||||
print(f"login Err {json.dumps(r, ensure_ascii=False)}")
|
||||
exit()
|
||||
raise RuntimeError(f"【open_store】登录失败 {json.dumps(r, ensure_ascii=False)}")
|
||||
# exit()
|
||||
else:
|
||||
print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
|
||||
exit()
|
||||
raise RuntimeError(f"【open_store】失败 {json.dumps(r, ensure_ascii=False)} ")
|
||||
# exit()
|
||||
|
||||
def get_browser(self, port) -> Chromium:
|
||||
"""
|
||||
@@ -206,13 +210,58 @@ class ZiniaoDriver:
|
||||
|
||||
def start_client(self):
|
||||
"""启动紫鸟客户端"""
|
||||
self.client_path = self.get_zinaio_exe("superbrowser").strip('"')
|
||||
|
||||
# 检查端口是否已经启动
|
||||
try:
|
||||
url = f'http://127.0.0.1:{self.socket_port}'
|
||||
response = requests.get(url, timeout=2)
|
||||
print(f"端口 {self.socket_port} 已经启动,跳过启动客户端操作")
|
||||
return
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
# 端口未启动,继续执行启动操作
|
||||
print(f"端口 {self.socket_port} 未启动,开始启动客户端")
|
||||
|
||||
self.kill_process('v6')
|
||||
time.sleep(5)
|
||||
self.client_path = self.get_zinaio_exe("superbrowserv6").strip('"')
|
||||
print(self.client_path)
|
||||
cmd = [self.client_path, '--run_type=web_driver', '--ipc_type=http',
|
||||
'--port=' + str(self.socket_port)]
|
||||
print(" ".join(cmd))
|
||||
subprocess.Popen(cmd)
|
||||
time.sleep(5)
|
||||
|
||||
# 最大重试次数
|
||||
max_retries = 3
|
||||
|
||||
for retry_count in range(max_retries):
|
||||
print(f"第 {retry_count + 1} 次尝试启动客户端...")
|
||||
|
||||
# 启动进程
|
||||
subprocess.Popen(cmd)
|
||||
|
||||
# 循环检测10秒,每0.5秒检测一次
|
||||
start_check_time = time.time()
|
||||
client_started = False
|
||||
|
||||
while time.time() - start_check_time < 10:
|
||||
try:
|
||||
response = requests.get(url, timeout=2)
|
||||
print(f"客户端启动成功!(第 {retry_count + 1} 次尝试)")
|
||||
client_started = True
|
||||
break
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
# 端口还未启动,继续等待
|
||||
time.sleep(0.5)
|
||||
|
||||
if client_started:
|
||||
time.sleep(5) # 等待客户端完全启动
|
||||
# 更新内核
|
||||
self.update_core()
|
||||
return
|
||||
else:
|
||||
print(f"第 {retry_count + 1} 次尝试启动失败,10秒内未检测到客户端启动")
|
||||
|
||||
# 超过最大重试次数,抛出异常
|
||||
raise RuntimeError(f"客户端启动失败:重试 {max_retries} 次后仍未成功启动")
|
||||
|
||||
def open_shop(self, shop_name: str):
|
||||
"""
|
||||
@@ -227,8 +276,7 @@ class ZiniaoDriver:
|
||||
# 启动客户端
|
||||
self.start_client()
|
||||
|
||||
# 更新内核
|
||||
self.update_core()
|
||||
|
||||
|
||||
# 获取店铺列表
|
||||
shop_ls = self.get_browser_list()
|
||||
@@ -258,7 +306,8 @@ class ZiniaoDriver:
|
||||
print("ip检测页地址为空,请升级紫鸟浏览器到最新版")
|
||||
print(f"=====关闭店铺:{shop_name}=====")
|
||||
self.close_store(self.store_id)
|
||||
exit()
|
||||
# exit()
|
||||
raise RuntimeError("没有IP检测地址,为了店铺安全不打开店铺")
|
||||
ip_usable = self.open_ip_check(self.browser, ip_check_url)
|
||||
if ip_usable:
|
||||
print("ip检测通过,打开店铺平台主页")
|
||||
@@ -295,11 +344,11 @@ class ZiniaoDriver:
|
||||
if str(r.get("statusCode")) == "0":
|
||||
return r
|
||||
elif str(r.get("statusCode")) == "-10003":
|
||||
print(f"login Err {json.dumps(r, ensure_ascii=False)}")
|
||||
exit()
|
||||
raise RuntimeError(f"【close_store】登录失败 {json.dumps(r, ensure_ascii=False)}")
|
||||
# exit()
|
||||
else:
|
||||
print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
|
||||
exit()
|
||||
raise RuntimeError(f"【close_store】失败: {json.dumps(r, ensure_ascii=False)} ")
|
||||
# exit()
|
||||
|
||||
def open_launcher_page(self, launcher_page: str, browser: Chromium = None):
|
||||
"""
|
||||
@@ -339,9 +388,9 @@ class ZiniaoDriver:
|
||||
return False
|
||||
|
||||
|
||||
class AmazoneDriver(ZiniaoDriver):
|
||||
"""亚马逊专用驱动类,继承自ZiniaoDriver"""
|
||||
|
||||
class AmamzonBase(ZiniaoDriver):
|
||||
"""亚马逊操作基类,包含一些通用方法"""
|
||||
|
||||
def SwitchingCountries(self, country_name: str):
|
||||
"""
|
||||
切换国家
|
||||
@@ -365,11 +414,12 @@ class AmazoneDriver(ZiniaoDriver):
|
||||
|
||||
# 获取当前标签页
|
||||
tab = self.tab
|
||||
tab.wait.doc_loaded(timeout=120,raise_err=False)
|
||||
|
||||
# 步骤1:获取当前国家名称
|
||||
print(f"正在检查当前国家...")
|
||||
current_country_ele = tab.ele('xpath://div[@class="dropdown-account-switcher-header-label"]/span[last()]',
|
||||
timeout=10)
|
||||
timeout=20)
|
||||
if current_country_ele:
|
||||
current_country = current_country_ele.text.strip()
|
||||
print(f"当前国家:{current_country}")
|
||||
@@ -433,6 +483,49 @@ class AmazoneDriver(ZiniaoDriver):
|
||||
print(f"切换国家时发生异常:{traceback.format_exc()}")
|
||||
return False
|
||||
|
||||
def need_login(self):
|
||||
"""
|
||||
判断是否需要登录,部分国家可能需要登录后才能切换国家
|
||||
处理流程:
|
||||
|
||||
"""
|
||||
time.sleep(3) # 等待页面可能的登录元素加载,避免跳转等等
|
||||
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
|
||||
need_login_ele = self.tab.eles('xpath://h1[@class="a-spacing-small"]|//span[contains(text(),"登录")]',timeout=5)
|
||||
if len(need_login_ele) >0:
|
||||
print("检测到需要登录元素")
|
||||
return True
|
||||
return False
|
||||
|
||||
def login(self,password,username=""):
|
||||
try:
|
||||
pwd_input = self.tab.eles('xpath://input[@type="password"]',timeout=10)
|
||||
if len(pwd_input) > 0:
|
||||
pwd_input[0].input(password,clear=True)
|
||||
submit_btn = self.tab.eles('xpath://input[@id="signInSubmit"]',timeout=10)
|
||||
if len(submit_btn) > 0:
|
||||
submit_btn[0].click()
|
||||
self.tab.wait.doc_loaded()
|
||||
|
||||
opt_code_input = self.tab.ele('xpath://input[@name="otpCode"]',timeout=60)
|
||||
opt_code_input.wait.displayed(timeout=10,raise_err=False)
|
||||
for _ in range(30):
|
||||
if opt_code_input.value is not None and opt_code_input.value.strip() != "":
|
||||
print("检测到验证码输入完成")
|
||||
submit_btn = self.tab.ele('xpath://input[@id="auth-signin-button"]',timeout=10)
|
||||
submit_btn.click()
|
||||
self.tab.wait.doc_loaded()
|
||||
return True
|
||||
time.sleep(1)
|
||||
submit_btn = self.tab.ele('xpath://input[@id="auth-signin-button"]',timeout=10)
|
||||
submit_btn.click()
|
||||
except Exception as e:
|
||||
print("登录过程中发生异常:" + traceback.format_exc())
|
||||
return False
|
||||
|
||||
class AmazoneDriver(AmamzonBase):
|
||||
"""亚马逊专用驱动类,继承自ZiniaoDriver"""
|
||||
|
||||
def SwitchPage(self):
|
||||
"""
|
||||
切换至 管理所有库存页面
|
||||
|
||||
@@ -6,6 +6,11 @@ from typing import Dict, Any, List
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from config import JSON_TASK_QUEUE, runing_task, runing_shop, DELETE_BRAND_API_BASE, ZN_COMPANY, ZN_USERNAME, ZN_PASSWORD
|
||||
from amazon.del_brand import AmazoneDriver, kill_process
|
||||
from amazon.approve import ApproveTask
|
||||
from amazon.match_action import MatchTak
|
||||
|
||||
|
||||
from amazon.tool import get_shop_info,show_notification
|
||||
|
||||
|
||||
class TaskMonitor:
|
||||
@@ -26,6 +31,7 @@ class TaskMonitor:
|
||||
|
||||
# 在提交新任务前杀掉旧进程(确保环境干净)
|
||||
kill_process("v6")
|
||||
kill_process("v5")
|
||||
|
||||
def log(self, message: str, level: str = "INFO"):
|
||||
"""日志输出
|
||||
@@ -45,6 +51,10 @@ class TaskMonitor:
|
||||
self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
|
||||
futures = [] # 保存所有提交的任务Future对象
|
||||
|
||||
task_type_info = {
|
||||
"product-risk-resolve-run" : "产品风险审批",
|
||||
"shop-match-run" : "匹配价格"
|
||||
}
|
||||
try:
|
||||
while self.running:
|
||||
try:
|
||||
@@ -54,21 +64,29 @@ class TaskMonitor:
|
||||
|
||||
# 检查任务类型
|
||||
task_type = task_data.get("type", "")
|
||||
if task_type != "delete-brand-run":
|
||||
|
||||
if task_type == "delete-brand-run":
|
||||
# 提交删除品牌任务到线程池
|
||||
self.log(f"接收到【删除品牌】任务,提交到线程池处理...")
|
||||
future = self.executor.submit(self._process_task_wrapper, task_data)
|
||||
futures.append(future)
|
||||
|
||||
elif task_type in task_type_info:
|
||||
# 提交产品风险审批任务到线程池
|
||||
self.log(f"接收到任务,提交到线程池处理...")
|
||||
future = self.executor.submit(self._process_approve_task_wrapper, task_data, task_type_info[task_type])
|
||||
futures.append(future)
|
||||
|
||||
else:
|
||||
self.log(f"未知任务类型: {task_type},跳过", "WARNING")
|
||||
continue
|
||||
|
||||
# 提交任务到线程池
|
||||
self.log(f"接收到删除品牌任务,提交到线程池处理...")
|
||||
future = self.executor.submit(self._process_task_wrapper, task_data)
|
||||
futures.append(future)
|
||||
|
||||
# 清理已完成的future对象,避免内存累积
|
||||
futures = [f for f in futures if not f.done()]
|
||||
|
||||
except Exception as e:
|
||||
# 静默处理队列为空的超时,只记录真正的异常
|
||||
if "Empty" not in str(e):
|
||||
if str(e) and "Empty" not in str(e):
|
||||
self.log(f"任务监控异常: {str(e)}", "ERROR")
|
||||
# 队列为空时不需要额外等待,直接继续循环
|
||||
|
||||
@@ -86,11 +104,31 @@ class TaskMonitor:
|
||||
task_data: 任务数据
|
||||
"""
|
||||
try:
|
||||
self.log(f"线程 {id(task_data)} 开始处理任务...")
|
||||
self.log(f"线程 {id(task_data)} 开始处理删除品牌任务...")
|
||||
self.process_task(task_data)
|
||||
self.log(f"线程 {id(task_data)} 任务处理完成")
|
||||
self.log(f"线程 {id(task_data)} 删除品牌任务处理完成")
|
||||
except Exception as e:
|
||||
self.log(f"线程 {id(task_data)} 任务处理异常: {traceback.format_exc()}", "ERROR")
|
||||
self.log(f"线程 {id(task_data)} 删除品牌任务处理异常: {traceback.format_exc()}", "ERROR")
|
||||
|
||||
def _process_approve_task_wrapper(self, task_data: Dict[str, Any],TASK_TYPE:str):
|
||||
"""审批任务处理包装器(用于线程池调用)
|
||||
|
||||
Args:
|
||||
task_data: 任务数据
|
||||
"""
|
||||
try:
|
||||
TASK_INFO = {
|
||||
"产品风险审批" : ApproveTask,
|
||||
"匹配价格" : MatchTak
|
||||
}
|
||||
self.log(f"线程 {id(task_data)} 开始处理产品风险审批任务...")
|
||||
# 创建ApproveTask实例并处理任务
|
||||
TASK_CLS = TASK_INFO[TASK_TYPE] # 根据任务类型选择处理类,默认为ApproveTask
|
||||
approve_task = TASK_CLS(user_info=self.user_info)
|
||||
approve_task.process_task(task_data)
|
||||
self.log(f"线程 {id(task_data)} 产品风险审批任务处理完成")
|
||||
except Exception as e:
|
||||
self.log(f"线程 {id(task_data)} 产品风险审批任务处理异常: {traceback.format_exc()}", "ERROR")
|
||||
|
||||
|
||||
def process_task(self, task_data: Dict[str, Any]):
|
||||
@@ -161,6 +199,7 @@ class TaskMonitor:
|
||||
shop_name = shop_data.get("shopName", "未知店铺")
|
||||
result_id = shop_data.get("resultId")
|
||||
countries = shop_data.get("countries", [])
|
||||
company_name = shop_data.get("companyName", "未知公司")
|
||||
|
||||
if not countries:
|
||||
self.log(f"店铺 {shop_name} 没有国家数据,跳过", "WARNING")
|
||||
@@ -172,7 +211,7 @@ class TaskMonitor:
|
||||
# 将店铺添加到正在执行中的店铺列表
|
||||
start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
runing_shop[shop_name] = start_time
|
||||
self.log(f"店铺 {shop_name} 已添加到执行列表,开始时间: {start_time}")
|
||||
self.log(f"账号:{company_name},店铺 {shop_name} 已添加到执行列表,开始时间: {start_time}")
|
||||
|
||||
# 店铺打开重试最多3次
|
||||
driver = None
|
||||
@@ -184,13 +223,18 @@ class TaskMonitor:
|
||||
self.log(f"尝试打开店铺 {shop_name} (第 {retry + 1}/{max_retries} 次)")
|
||||
|
||||
# 如果不是第一次尝试,先杀进程
|
||||
if retry > 0:
|
||||
self.log("重试前先杀掉浏览器进程...")
|
||||
kill_process("v6")
|
||||
time.sleep(2)
|
||||
# if retry > 0:
|
||||
# self.log("重试前先杀掉浏览器进程...")
|
||||
# kill_process("v6")
|
||||
# kill_process("v5")
|
||||
# time.sleep(2)
|
||||
|
||||
# 创建驱动并打开店铺
|
||||
driver = AmazoneDriver(self.user_info)
|
||||
user_info = {
|
||||
**self.user_info,
|
||||
"company": company_name
|
||||
}
|
||||
driver = AmazoneDriver(user_info)
|
||||
browser = driver.open_shop(shop_name)
|
||||
|
||||
if browser and browser != "店铺不存在":
|
||||
@@ -199,6 +243,39 @@ class TaskMonitor:
|
||||
else:
|
||||
self.log(f"打开店铺失败: {browser}", "WARNING")
|
||||
driver = None
|
||||
|
||||
# 判断是否需要登录
|
||||
driver.tab.wait.doc_loaded(timeout=30, raise_err=False) # 等待页面加载,避免过早判断登录状态
|
||||
need_login = driver.need_login()
|
||||
print("【是否需要登录】:",need_login)
|
||||
if need_login:
|
||||
self.log(f"店铺 {shop_name} 需要登录,正在登录...")
|
||||
# 获取店铺凭证
|
||||
response = get_shop_info(shop_name)
|
||||
print("【获取店铺凭证返回】:",response.text)
|
||||
shop_data = response.json()
|
||||
if not shop_data:
|
||||
mes = f"获取店铺凭证失败,响应数据: {shop_data.get('message', '未知错误')}"
|
||||
self.log(mes, "ERROR")
|
||||
show_notification(mes, "ERROR")
|
||||
continue
|
||||
|
||||
password = shop_data["data"]["password"]
|
||||
|
||||
login_success = driver.login(password)
|
||||
if login_success:
|
||||
self.log(f"店铺 {shop_name} 登录成功,正在重新打开店铺...")
|
||||
browser = driver.open_shop(shop_name)
|
||||
if browser and browser != "店铺不存在":
|
||||
self.log(f"成功打开店铺 {shop_name} 登录后")
|
||||
break
|
||||
else:
|
||||
self.log(f"登录后打开店铺失败: {browser}", "WARNING")
|
||||
driver = None
|
||||
else:
|
||||
self.log(f"店铺 {shop_name} 登录失败", "WARNING")
|
||||
driver = None
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"打开店铺异常: {str(e)}", "ERROR")
|
||||
@@ -549,6 +626,7 @@ class TaskMonitor:
|
||||
verify=False # 忽略SSL证书验证
|
||||
)
|
||||
print("【结果提交】:",payload)
|
||||
print("【结果提交返回】:",response.text)
|
||||
|
||||
if response.status_code == 200:
|
||||
self.log(f"结果回传成功: {url}")
|
||||
|
||||
717
app/amazon/match_action.py
Normal file
717
app/amazon/match_action.py
Normal file
@@ -0,0 +1,717 @@
|
||||
import time
|
||||
import re
|
||||
import traceback
|
||||
from DrissionPage._pages.chromium_tab import ChromiumTab
|
||||
|
||||
from config import runing_task, runing_shop
|
||||
from datetime import datetime
|
||||
from amazon.del_brand import AmamzonBase, kill_process
|
||||
|
||||
from amazon.tool import show_notification,get_shop_info
|
||||
|
||||
|
||||
class AmzoneMatchAction(AmamzonBase):
|
||||
mark_name = "匹配价格"
|
||||
|
||||
def SwitchPage(self):
|
||||
"""
|
||||
切换至 管理所有库存页面
|
||||
1、等待 //navigation-favorites-bar[@class="hydrated"] 出现
|
||||
"""
|
||||
navigation = self.tab.ele('xpath://navigation-favorites-bar[@class="hydrated"]')
|
||||
navigation.wait.displayed(raise_err=False)
|
||||
page_btn = navigation.sr('xpath://internal-fav-bar-links[@data-internal="navigation"]').sr(
|
||||
'xpath://a[@data-page-id="ezdpc-gui-inventory-mons"]')
|
||||
page_btn.wait.displayed(raise_err=False)
|
||||
page_btn.click(timeout=5)
|
||||
|
||||
self.tab.wait.doc_loaded()
|
||||
# 等待搜索框出现
|
||||
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group')
|
||||
search_region.wait.displayed(raise_err=False)
|
||||
|
||||
def search(self,filter_type="ApprovalRequired"):
|
||||
sku_ls = []
|
||||
|
||||
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
|
||||
if len(load_ele) > 0:
|
||||
load_ele[0].wait.deleted(timeout=3, raise_err=False)
|
||||
time.sleep(0.5)
|
||||
|
||||
drop_down = self.tab.ele('xpath://div[contains(@class,"VolusListingStatusDropDown-module__verticalContainer")]//kat-dropdown')
|
||||
drop_down.wait.displayed(raise_err=False)
|
||||
drop_down.wait.enabled(raise_err=False)
|
||||
time.sleep(0.6)
|
||||
drop_down.click()
|
||||
# //kat-option[@value="SearchSuppressed"]
|
||||
xp = f'xpath://kat-option[@value="{filter_type}"]'
|
||||
print(f"【{self.mark_name}】正在寻找筛选条件 {filter_type},xpath: {xp}")
|
||||
approval_required = self.tab.eles(xp,timeout=5)
|
||||
if len(approval_required) == 0:
|
||||
print(f"【{self.mark_name}】没有需要{filter_type}选项】没有需要{filter_type}的商品了")
|
||||
return sku_ls # "没有需要审批的商品了"
|
||||
else:
|
||||
approval_required = approval_required[0]
|
||||
approval_required.wait.displayed(raise_err=False)
|
||||
approval_required.click()
|
||||
|
||||
approval_required_text = approval_required.text
|
||||
print(f"【{self.mark_name}】已选择筛选条件: {approval_required_text}")
|
||||
|
||||
count = re.findall(r'\d+', approval_required_text)
|
||||
if count:
|
||||
count = int(count[0])
|
||||
print(f"【{self.mark_name}】待审批的商品数量: {count}")
|
||||
if count <= 0:
|
||||
print(f"【{self.mark_name}】没有需要{filter_type}的商品了")
|
||||
return sku_ls #"没有需要审批的商品了"
|
||||
for _ in range(3):
|
||||
# 等待加载完成
|
||||
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]")
|
||||
if len(load_ele) > 0:
|
||||
load_ele[0].wait.deleted(timeout=3, raise_err=False)
|
||||
time.sleep(0.5)
|
||||
|
||||
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=3)
|
||||
if len(sku_ls) > 0:
|
||||
break
|
||||
approval_required.click()
|
||||
return sku_ls
|
||||
|
||||
def wait_loaded(self):
|
||||
# 等待加载完成
|
||||
try:
|
||||
load_ele = self.tab.ele(
|
||||
'xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//div[contains(@class,"Loader-module__loader")]|//kat-panel[@data-testid="kat-panel-ActionPanelContent"]/div[@data-f1-component]//div[contains(@class,"==")]/div[contains(@class,"==")]/span',
|
||||
timeout=3)
|
||||
load_ele.wait.deleted(timeout=5, raise_err=False)
|
||||
except Exception as e:
|
||||
print(f"【{self.mark_name}】等待加载中消失出错", e)
|
||||
|
||||
def run_page_action(self):
|
||||
print(f"【{self.mark_name}】开始执行")
|
||||
num = 0
|
||||
retry_num = 0
|
||||
already_asin = set()
|
||||
|
||||
while retry_num < 3: # 最多重试3次
|
||||
# if num > 3: #测试
|
||||
# return
|
||||
# 等待加载完成
|
||||
try:
|
||||
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
|
||||
if len(load_ele) > 0:
|
||||
load_ele[0].wait.deleted(timeout=3, raise_err=False)
|
||||
time.sleep(0.5)
|
||||
|
||||
# 获取当前页码
|
||||
try:
|
||||
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
|
||||
if len(page_pamel) > 0:
|
||||
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
|
||||
|
||||
# 总页数
|
||||
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
|
||||
if len(total_page) > 0:
|
||||
total_page = total_page[-1].text
|
||||
else:
|
||||
total_page = 0
|
||||
print(f"【{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}")
|
||||
except Exception as e:
|
||||
print(f"【{self.mark_name}】获取页码失败", e)
|
||||
|
||||
|
||||
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
|
||||
print(f"【{self.mark_name}】获取到 {len(sku_ls)}")
|
||||
# for sku_ele in sku_ls[0:2]:
|
||||
for sku_ele in sku_ls:
|
||||
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]')
|
||||
|
||||
asin = sku_ele.ele('xpath:.//div[contains(@class,"JanusSplitBox-module__container")]//div[contains(@class,"JanusSplitBox-module__panel--") and contains(string(.),"ASIN")]/..//div[last()]',timeout=10).text
|
||||
print(f"【{self.mark_name}】ASIN {asin} 找到....")
|
||||
if asin in already_asin:
|
||||
print(f"【{self.mark_name}】{asin} 已经处理过了,跳过")
|
||||
continue
|
||||
|
||||
price_match = sku_ele.eles('xpath:.//div[@data-test-id="FeaturedOfferPrice"]//a[text()="匹配"]',timeout=5)
|
||||
if len(price_match) == 0:
|
||||
print(f"【{self.mark_name}】{asin},没有推荐价格匹配按钮")
|
||||
yield (asin,"无需处理")
|
||||
already_asin.add(asin)
|
||||
continue
|
||||
try:
|
||||
price_match[0].click()
|
||||
|
||||
save_all_btn = self.tab.ele('xpath://kat-button[@label="保存所有"]',timeout=10)
|
||||
save_all_btn.wait.displayed(timeout=10, raise_err=False)
|
||||
save_all_btn.click()
|
||||
|
||||
save_all_btn.wait.deleted(timeout=10, raise_err=False)
|
||||
|
||||
yield (asin,"处理完成")
|
||||
already_asin.add(asin)
|
||||
except Exception as e:
|
||||
print(f"{asin} 处理失败", e)
|
||||
yield (asin,"处理失败")
|
||||
already_asin.add(asin)
|
||||
continue
|
||||
|
||||
|
||||
# 判断是否存在需要翻页的情况
|
||||
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
|
||||
if len(page_pamel) == 0:
|
||||
break
|
||||
next_page_btn = page_pamel[0].sr('xpath:.//span[@part="pagination-nav-right"]')
|
||||
class_str = next_page_btn.attr('class')
|
||||
if "end" in class_str:
|
||||
break
|
||||
next_page_btn.click()
|
||||
num += 1
|
||||
print(f"【{self.mark_name}】【程序计算】正在翻页,已翻 {num} 页...")
|
||||
|
||||
already_asin = set()
|
||||
|
||||
except Exception as e:
|
||||
print(f"【{self.mark_name}】处理匹配操作异常", e)
|
||||
traceback.print_exc()
|
||||
retry_num += 1
|
||||
self.tab.refresh()
|
||||
self.tab.wait.doc_loaded(raise_err=False,timeout=120)
|
||||
|
||||
|
||||
|
||||
class MatchTak:
|
||||
|
||||
country_info = {
|
||||
"DE": "德国",
|
||||
"FR": "法国",
|
||||
"ES": "西班牙",
|
||||
"IT": "意大利",
|
||||
"UK": "英国"
|
||||
}
|
||||
|
||||
|
||||
def __init__(self, user_info: dict = None):
|
||||
"""初始化审批任务处理器
|
||||
|
||||
Args:
|
||||
user_info: 用户信息字典,包含 company, username, password
|
||||
"""
|
||||
self.user_info = user_info or {}
|
||||
self.running = True
|
||||
|
||||
def log(self, message: str, level: str = "INFO"):
|
||||
"""日志输出
|
||||
|
||||
Args:
|
||||
message: 日志消息
|
||||
level: 日志级别
|
||||
"""
|
||||
from datetime import datetime
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
if level == "ERROR":
|
||||
show_notification(message, "error")
|
||||
print(f"[{timestamp}] [MatchTak] [{level}] {message}")
|
||||
|
||||
|
||||
def process_task(self, task_data: dict):
|
||||
"""处理审批任务主入口
|
||||
|
||||
Args:
|
||||
task_data: 任务数据
|
||||
"""
|
||||
try:
|
||||
data = task_data.get("data", {})
|
||||
task_id = data.get("taskId")
|
||||
items = data.get("items", [])
|
||||
country_codes = data.get("country_codes", [])
|
||||
risk_listing_filter = data.get("risk_listing_filter", "All")
|
||||
user_id = data.get("user_id")
|
||||
stage_index = data.get("stage_index")
|
||||
final_stage = bool(data.get("final_stage", True))
|
||||
|
||||
if not task_id:
|
||||
self.log("任务ID为空,跳过", "WARNING")
|
||||
return
|
||||
|
||||
if not items:
|
||||
self.log("店铺列表为空,跳过", "WARNING")
|
||||
return
|
||||
|
||||
if not country_codes:
|
||||
self.log("国家列表为空,跳过", "WARNING")
|
||||
return
|
||||
|
||||
self.log(f"开始处理审批任务 {task_id},共 {len(items)} 个店铺,{len(country_codes)} 个国家")
|
||||
|
||||
from config import runing_task
|
||||
runing_task[task_id] = {
|
||||
"status": "running",
|
||||
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"total_shops": len(items),
|
||||
"processed_shops": 0,
|
||||
"total_countries": len(country_codes) * len(items),
|
||||
"processed_countries": 0,
|
||||
"total_asins": 0,
|
||||
"processed_asins": 0,
|
||||
"success_count": 0,
|
||||
"failed_count": 0,
|
||||
"stop_requested": False
|
||||
}
|
||||
|
||||
# 遍历处理每个店铺
|
||||
for idx, shop_item in enumerate(items, 1):
|
||||
# 检查是否收到暂停请求
|
||||
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
|
||||
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理", "WARNING")
|
||||
runing_task[task_id]["status"] = "stopped"
|
||||
return
|
||||
|
||||
shop_name = shop_item.get("shopName", "未知店铺")
|
||||
self.log(f"[{idx}/{len(items)}] 开始处理店铺: {shop_name}")
|
||||
show_notification(f"开始处理店铺: {shop_name}", "info")
|
||||
|
||||
try:
|
||||
self.process_shop(shop_item, country_codes, task_id, risk_listing_filter, user_id, stage_index, final_stage)
|
||||
|
||||
# self.process_shop(shop_item, country_codes, task_id,risk_listing_filter)
|
||||
# 更新已处理店铺数
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["processed_shops"] += 1
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR")
|
||||
self.log(traceback.format_exc(), "ERROR")
|
||||
|
||||
# 更新任务状态
|
||||
if task_id in runing_task:
|
||||
if runing_task[task_id].get("stop_requested", False):
|
||||
runing_task[task_id]["status"] = "stopped"
|
||||
self.log(f"任务 {task_id} 已被暂停!")
|
||||
else:
|
||||
runing_task[task_id]["status"] = "completed"
|
||||
self.log(f"任务 {task_id} 处理完成!")
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
|
||||
if task_id:
|
||||
from config import runing_task
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["status"] = "failed"
|
||||
runing_task[task_id]["error"] = str(e)
|
||||
|
||||
# def process_shop(self, shop_item: dict, country_codes: list, task_id: int, risk_listing_filter: str):
|
||||
def process_shop(self, shop_item: dict, country_codes: list, task_id: int, risk_listing_filter: str, user_id=None, stage_index=None, final_stage: bool = True):
|
||||
"""处理单个店铺
|
||||
|
||||
Args:
|
||||
shop_item: 店铺信息
|
||||
country_codes: 国家代码列表
|
||||
task_id: 任务ID
|
||||
risk_listing_filter: 风险商品筛选条件
|
||||
"""
|
||||
shop_name = shop_item.get("shopName", "未知店铺")
|
||||
company_name = shop_item.get("companyName", "")
|
||||
|
||||
if not company_name:
|
||||
self.log(f"店铺 {shop_name} 的公司名称为空,跳过", "WARNING")
|
||||
return
|
||||
|
||||
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["current_shop"] = shop_name
|
||||
|
||||
# 将店铺添加到正在执行中的店铺列表
|
||||
start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
runing_shop[shop_name] = start_time
|
||||
self.log(f"店铺 {shop_name} 已添加到执行列表,账号: {company_name},开始时间: {start_time}")
|
||||
|
||||
# 店铺打开重试最多3次
|
||||
driver = None
|
||||
max_retries = 3
|
||||
|
||||
error_info = ""
|
||||
for retry in range(max_retries):
|
||||
try:
|
||||
self.log(f"尝试打开店铺 {shop_name} (第 {retry + 1}/{max_retries} 次)")
|
||||
|
||||
# 如果不是第一次尝试,先杀进程
|
||||
# if retry > 0:
|
||||
# self.log("重试前先杀掉浏览器进程...")
|
||||
# kill_process("v6")
|
||||
# kill_process("v5")
|
||||
# time.sleep(2)
|
||||
|
||||
# 组装用户信息并创建驱动
|
||||
user_info = {
|
||||
**self.user_info,
|
||||
"company": company_name
|
||||
}
|
||||
driver = AmzoneMatchAction(user_info)
|
||||
browser = driver.open_shop(shop_name)
|
||||
|
||||
if browser and browser != "店铺不存在":
|
||||
self.log(f"成功打开店铺 {shop_name}")
|
||||
else:
|
||||
self.log(f"打开店铺失败: {browser}", "WARNING")
|
||||
driver = None
|
||||
continue
|
||||
|
||||
# 判断是否需要登录
|
||||
need_login = driver.need_login()
|
||||
print("【是否需要登录】:",need_login)
|
||||
if need_login:
|
||||
self.log(f"店铺 {shop_name} 需要登录,正在登录...")
|
||||
# 获取店铺凭证
|
||||
response = get_shop_info(shop_name)
|
||||
print("【获取店铺凭证返回】:",response.text)
|
||||
shop_data = response.json()
|
||||
if not shop_data:
|
||||
mes = f"获取店铺凭证失败,响应数据: {shop_data.get('message', '未知错误')}"
|
||||
self.log(mes, "ERROR")
|
||||
show_notification(mes, "ERROR")
|
||||
continue
|
||||
|
||||
password = shop_data["data"]["password"]
|
||||
|
||||
login_success = driver.login(password)
|
||||
if login_success:
|
||||
self.log(f"店铺 {shop_name} 登录成功,正在重新打开店铺...")
|
||||
browser = driver.open_shop(shop_name)
|
||||
if browser and browser != "店铺不存在":
|
||||
self.log(f"成功打开店铺 {shop_name} 登录后")
|
||||
break
|
||||
else:
|
||||
self.log(f"登录后打开店铺失败: {browser}", "WARNING")
|
||||
driver = None
|
||||
else:
|
||||
self.log(f"店铺 {shop_name} 登录失败", "WARNING")
|
||||
driver = None
|
||||
else:
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"打开店铺异常: {traceback.format_exc()}", "INFO")
|
||||
driver = None
|
||||
error_info = str(e)
|
||||
time.sleep(10)
|
||||
|
||||
# 如果还有重试机会,等待后继续
|
||||
if retry < max_retries - 1:
|
||||
time.sleep(3)
|
||||
|
||||
# 检查是否成功打开
|
||||
if not driver or not browser or browser == "店铺不存在":
|
||||
error_msg = f"店铺 {shop_name} 打开失败,已重试 {max_retries} 次,跳过该店铺,{error_info}"
|
||||
self.log(error_msg, "ERROR")
|
||||
# 从执行列表中移除
|
||||
if shop_name in runing_shop:
|
||||
del runing_shop[shop_name]
|
||||
return
|
||||
|
||||
try:
|
||||
# 处理每个国家
|
||||
for country_code in country_codes:
|
||||
# 检查是否收到暂停请求
|
||||
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
|
||||
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理国家", "WARNING")
|
||||
break
|
||||
|
||||
try:
|
||||
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter)
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR")
|
||||
self.log(traceback.format_exc(), "ERROR")
|
||||
# 最后回传,标记完成
|
||||
try:
|
||||
# self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
|
||||
if final_stage:
|
||||
self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
|
||||
else:
|
||||
self.post_stage_finished(task_id, user_id, stage_index)
|
||||
except Exception as e:
|
||||
self.log(f"回传结果失败: {str(e)}", "ERROR")
|
||||
finally:
|
||||
# 关闭店铺
|
||||
try:
|
||||
if driver:
|
||||
self.log(f"关闭店铺 {shop_name}")
|
||||
driver.close_store()
|
||||
time.sleep(2)
|
||||
except Exception as e:
|
||||
self.log(f"关闭店铺失败: {str(e)}", "WARNING")
|
||||
|
||||
# 从正在执行中的店铺列表中移除
|
||||
if shop_name in runing_shop:
|
||||
del runing_shop[shop_name]
|
||||
self.log(f"店铺 {shop_name} 已从执行列表中移除")
|
||||
|
||||
def process_country(self, driver: AmzoneMatchAction, country_code: str, task_id: int, shop_name: str, risk_listing_filter: str):
|
||||
"""处理单个国家的审批任务
|
||||
|
||||
Args:
|
||||
driver: AmzoneApprove驱动实例
|
||||
country_code: 国家代码(如 UK, DE, FR 等)
|
||||
task_id: 任务ID
|
||||
shop_name: 店铺名称
|
||||
risk_listing_filter: 风险商品筛选条件
|
||||
"""
|
||||
from config import runing_task
|
||||
|
||||
# 转换国家代码为中文名称
|
||||
country_name = self.country_info.get(country_code, country_code)
|
||||
info_mes = f"开始处理国家: {country_name} ({country_code})"
|
||||
self.log(info_mes)
|
||||
show_notification(info_mes, "info")
|
||||
|
||||
# 更新当前处理的国家
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["current_country"] = country_name
|
||||
|
||||
# 切换国家,最多重试3次
|
||||
max_retries = 3
|
||||
switch_success = False
|
||||
|
||||
for retry in range(max_retries):
|
||||
try:
|
||||
self.log(f"尝试切换到国家 {country_name} (第 {retry + 1}/{max_retries} 次)")
|
||||
|
||||
# 如果不是第一次尝试,先刷新页面
|
||||
if retry > 0:
|
||||
self.log("重试前刷新页面...")
|
||||
try:
|
||||
driver.tab.refresh()
|
||||
time.sleep(3)
|
||||
except Exception as e:
|
||||
self.log(f"刷新页面失败: {str(e)}", "WARNING")
|
||||
|
||||
switch_success = driver.SwitchingCountries(country_name)
|
||||
if switch_success:
|
||||
self.log(f"成功切换到国家 {country_name}")
|
||||
break
|
||||
else:
|
||||
self.log(f"切换到国家 {country_name} 失败", "WARNING")
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"切换国家 {country_name} 异常: {str(e)}", "ERROR")
|
||||
self.log(traceback.format_exc(), "ERROR")
|
||||
|
||||
# 如果还有重试机会,等待后继续
|
||||
if retry < max_retries - 1:
|
||||
time.sleep(2)
|
||||
|
||||
# 如果切换失败,直接返回
|
||||
if not switch_success:
|
||||
error_message = f"切换到国家 {country_name} 失败,已重试 {max_retries} 次,跳过该国家"
|
||||
self.log(error_message, "ERROR")
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["processed_countries"] += 1
|
||||
return
|
||||
|
||||
# 切换到库存管理页面
|
||||
try:
|
||||
driver.SwitchPage()
|
||||
self.log(f"已切换到库存管理页面")
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"切换页面失败: {str(e)}", "ERROR")
|
||||
self.log(traceback.format_exc(), "ERROR")
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["processed_countries"] += 1
|
||||
return
|
||||
|
||||
# 搜索需要审批的商品,最多重试3次
|
||||
sku_ls = []
|
||||
for retry in range(max_retries):
|
||||
try:
|
||||
self.log(f"尝试搜索匹配操作商品 (第 {retry + 1}/{max_retries} 次)")
|
||||
sku_ls = driver.search(filter_type=risk_listing_filter)
|
||||
break
|
||||
except Exception as e:
|
||||
self.log(f"搜索商品异常: {str(e)}", "ERROR")
|
||||
if retry < max_retries - 1:
|
||||
try:
|
||||
driver.tab.refresh()
|
||||
time.sleep(3)
|
||||
except Exception as refresh_error:
|
||||
self.log(f"刷新页面失败: {str(refresh_error)}", "WARNING")
|
||||
|
||||
# 如果没有需要审批的商品,直接返回
|
||||
if len(sku_ls) == 0:
|
||||
self.log(f"国家 {country_name} 没有搜索出的商品")
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["processed_countries"] += 1
|
||||
return
|
||||
|
||||
self.log(f"国家 {country_name} 搜索出 {len(sku_ls)} 商品,开始处理...")
|
||||
|
||||
# 处理所有需要审批的商品(通过yield获取结果)
|
||||
try:
|
||||
for asin, status in driver.run_page_action():
|
||||
# 检查是否收到暂停请求
|
||||
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
|
||||
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理ASIN", "WARNING")
|
||||
break
|
||||
|
||||
self.log(f"ASIN {asin} 处理结果: {status}")
|
||||
|
||||
# 更新任务状态
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["current_asin"] = asin
|
||||
runing_task[task_id]["processed_asins"] += 1
|
||||
|
||||
runing_task[task_id]["failed_count"] += 1
|
||||
|
||||
# 回传结果到API
|
||||
try:
|
||||
self.post_result(task_id, shop_name, country_code, asin, status)
|
||||
except Exception as e:
|
||||
self.log(f"回传结果失败: {str(e)}", "ERROR")
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self.log(f"处理审批商品异常: {str(e)}", "ERROR")
|
||||
self.log(traceback.format_exc(), "ERROR")
|
||||
|
||||
# 更新已处理国家数
|
||||
if task_id in runing_task:
|
||||
runing_task[task_id]["processed_countries"] += 1
|
||||
|
||||
self.log(f"国家 {country_name} 处理完成")
|
||||
def post_stage_finished(self, task_id: int, user_id, stage_index):
|
||||
import requests
|
||||
from config import DELETE_BRAND_API_BASE
|
||||
|
||||
if user_id in (None, "", 0):
|
||||
raise ValueError("user_id is required for stage completion callback")
|
||||
if stage_index is None:
|
||||
raise ValueError("stage_index is required for stage completion callback")
|
||||
|
||||
url = f"{DELETE_BRAND_API_BASE}/api/shop-match/tasks/{task_id}/stage-finished"
|
||||
payload = {"stage_index": stage_index}
|
||||
params = {"user_id": user_id}
|
||||
|
||||
max_retries = 3
|
||||
for retry in range(max_retries):
|
||||
try:
|
||||
self.log(f"Attempting stage completion callback ({retry + 1}/{max_retries})")
|
||||
response = requests.post(
|
||||
url,
|
||||
params=params,
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
timeout=30,
|
||||
verify=False,
|
||||
)
|
||||
self.log(f"Stage completion callback response: {response.text}")
|
||||
data = response.json() if response.text else {}
|
||||
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
|
||||
self.log(f"Stage completion callback succeeded: task={task_id}, stage={stage_index}")
|
||||
return
|
||||
self.log(f"Stage completion callback failed, status={response.status_code}", "WARNING")
|
||||
except Exception as e:
|
||||
self.log(f"Stage completion callback exception: {str(e)}", "ERROR")
|
||||
if retry < max_retries - 1:
|
||||
time.sleep(2)
|
||||
|
||||
raise RuntimeError(f"Stage completion callback failed after retries: task={task_id}, stage={stage_index}")
|
||||
|
||||
def post_result(self, task_id: int, shop_name: str, country_code: str, asin: str, status: str,is_done: bool = False):
|
||||
"""回传处理结果到API
|
||||
|
||||
Args:
|
||||
task_id: 任务ID
|
||||
shop_name: 店铺名称
|
||||
country_code: 国家代码
|
||||
asin: ASIN
|
||||
status: 处理状态
|
||||
"""
|
||||
import requests
|
||||
from config import DELETE_BRAND_API_BASE
|
||||
|
||||
url = f"{DELETE_BRAND_API_BASE}/api/shop-match/tasks/{task_id}/result"
|
||||
|
||||
payload = {
|
||||
"shops": [
|
||||
{
|
||||
"error": "",
|
||||
"countries": {
|
||||
country_code: [
|
||||
{
|
||||
"asin": asin,
|
||||
"status": status,
|
||||
"done": is_done
|
||||
}
|
||||
]
|
||||
},
|
||||
"shopName": shop_name
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
max_retries = 3
|
||||
for retry in range(max_retries):
|
||||
try:
|
||||
print("================【匹配】=====================")
|
||||
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
|
||||
self.log(f"回传URL: {url}")
|
||||
self.log(f"回传数据: {payload}")
|
||||
response = requests.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
timeout=30,
|
||||
verify=False
|
||||
)
|
||||
self.log(f"回传结果: {response.text}")
|
||||
data = response.json() if response.text else {}
|
||||
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
|
||||
self.log(f"结果回传成功: {asin} - {status}")
|
||||
return
|
||||
else:
|
||||
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
|
||||
print("=====================================")
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"调用API异常: {str(e)}", "ERROR")
|
||||
print("=====================================")
|
||||
|
||||
# 如果还有重试机会,等待后继续
|
||||
if retry < max_retries - 1:
|
||||
time.sleep(2)
|
||||
|
||||
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
|
||||
raise RuntimeError("已达到最大重试次数,结果回传最终失败")
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
user_info = {
|
||||
"company": "rongchuang123",
|
||||
"username": "自动化_Robot",
|
||||
"password": "#20zsg25"
|
||||
}
|
||||
shop_name = "魏振峰"
|
||||
country = "德国"
|
||||
kill_process('v6')
|
||||
driver = AmzoneMatchAction(user_info)
|
||||
browser = driver.open_shop(shop_name)
|
||||
sw_suc = driver.SwitchingCountries(country)
|
||||
driver.SwitchPage()
|
||||
risk_listing_filter = "All"
|
||||
for _ in range(3):
|
||||
try:
|
||||
sku_ls = driver.search(filter_type=risk_listing_filter)
|
||||
break
|
||||
except Exception as e:
|
||||
print(e)
|
||||
driver.tab.refresh()
|
||||
if len(sku_ls) > 0:
|
||||
print("有数据,开始操作")
|
||||
for asin, status in driver.run_page_action():
|
||||
print(f"ASIN {asin} 的处理结果: {status}")
|
||||
print("已完成操作")
|
||||
116
app/amazon/tool.py
Normal file
116
app/amazon/tool.py
Normal file
@@ -0,0 +1,116 @@
|
||||
# 导入 webview 用于前端通知
|
||||
try:
|
||||
import webview
|
||||
except ImportError:
|
||||
webview = None
|
||||
|
||||
|
||||
import requests
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
def show_notification(message: str, message_type: str = "error"):
|
||||
"""显示 pywebview 顶层通知(5秒后自动消失)
|
||||
|
||||
Args:
|
||||
message: 通知消息
|
||||
message_type: 消息类型 (success/warning/error/info)
|
||||
"""
|
||||
if webview and webview.windows:
|
||||
try:
|
||||
# 转义单引号,防止 JavaScript 语法错误
|
||||
safe_message = message.replace("'", "\\'").replace('"', '\\"').replace('\n', '\\n')
|
||||
|
||||
# 设置通知样式颜色
|
||||
color_map = {
|
||||
'success': '#67C23A',
|
||||
'warning': '#E6A23C',
|
||||
'error': '#F56C6C',
|
||||
'info': '#909399'
|
||||
}
|
||||
bg_color = color_map.get(message_type, '#F56C6C')
|
||||
|
||||
# 构建前端通知的 JavaScript 代码 - 创建原生 HTML 通知
|
||||
js_code = f"""
|
||||
(function() {{
|
||||
// 移除已存在的通知
|
||||
var existingNotif = document.getElementById('pywebview-notification');
|
||||
if (existingNotif) {{
|
||||
existingNotif.remove();
|
||||
}}
|
||||
|
||||
// 创建通知容器
|
||||
var notif = document.createElement('div');
|
||||
notif.id = 'pywebview-notification';
|
||||
notif.style.cssText = 'position: fixed; top: 20px; left: 50%; transform: translateX(-50%); ' +
|
||||
'background: {bg_color}; color: white; padding: 12px 20px; border-radius: 4px; ' +
|
||||
'box-shadow: 0 2px 12px rgba(0,0,0,0.3); z-index: 99999; font-size: 14px; ' +
|
||||
'max-width: 600px; word-wrap: break-word; display: flex; align-items: center; gap: 10px;';
|
||||
|
||||
// 添加消息内容
|
||||
var msgSpan = document.createElement('span');
|
||||
msgSpan.textContent = '{safe_message}';
|
||||
notif.appendChild(msgSpan);
|
||||
|
||||
// 添加关闭按钮
|
||||
var closeBtn = document.createElement('span');
|
||||
closeBtn.innerHTML = '×';
|
||||
closeBtn.style.cssText = 'cursor: pointer; font-size: 18px; font-weight: bold; margin-left: 10px;';
|
||||
closeBtn.onclick = function() {{ notif.remove(); }};
|
||||
notif.appendChild(closeBtn);
|
||||
|
||||
// 添加到页面
|
||||
document.body.appendChild(notif);
|
||||
|
||||
// 5秒后自动消失
|
||||
setTimeout(function() {{
|
||||
if (notif && notif.parentNode) {{
|
||||
notif.style.transition = 'opacity 0.3s';
|
||||
notif.style.opacity = '0';
|
||||
setTimeout(function() {{ notif.remove(); }}, 300);
|
||||
}}
|
||||
}}, 5000);
|
||||
}})();
|
||||
"""
|
||||
# 在第一个窗口中执行 JavaScript
|
||||
webview.windows[0].evaluate_js(js_code)
|
||||
except Exception as e:
|
||||
print(f"显示通知失败: {str(e)}")
|
||||
|
||||
|
||||
|
||||
|
||||
def get_shop_info(shop_name: str, base_url: str = "http://8.136.19.173:18080") -> requests.Response:
|
||||
"""
|
||||
调用商铺凭证接口
|
||||
:param shop_name: 商铺名称,将自动进行 URL 编码
|
||||
:param base_url: API 基础地址,默认从 curl 中提取
|
||||
:return: requests.Response 对象
|
||||
"""
|
||||
url = f"{base_url}/api/admin/shop-manages/credential"
|
||||
params = {"shopName": shop_name}
|
||||
|
||||
headers = {
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"Pragma": "no-cache",
|
||||
"Referer": "http://8.136.19.173:18080/doc.html",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
|
||||
"X-Internal-Token": "59c6691199917a2095827b5188502029",
|
||||
}
|
||||
|
||||
response = requests.get(
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
verify=False,
|
||||
timeout=30
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -110,7 +110,10 @@ def proxy(path):
|
||||
if key.lower() in ['host', 'content-length', 'connection']:
|
||||
continue
|
||||
headers[key] = value
|
||||
ignore_url = [f"{JAVA_API_BASE}/api/delete-brand/tasks/batch",f"{JAVA_API_BASE}/api/delete-brand/history"]
|
||||
ignore_url = [f"{JAVA_API_BASE}/api/delete-brand/tasks/batch",
|
||||
f"{JAVA_API_BASE}/api/delete-brand/history",
|
||||
f"{JAVA_API_BASE}/api/product-risk-resolve/tasks/batch",
|
||||
]
|
||||
if target_url not in ignore_url:
|
||||
try:
|
||||
print("=============================")
|
||||
|
||||
115
app/main.py
115
app/main.py
@@ -11,7 +11,7 @@ from amazon.main import TaskMonitor
|
||||
os.makedirs(cache_path,exist_ok=True)
|
||||
if not debug:
|
||||
today = datetime.datetime.now().strftime("%Y_%m_%d")
|
||||
sys.stdout = open(os.path.join(cache_path,f'{today}.log'), 'a',buffering=1)
|
||||
sys.stdout = open(os.path.join(cache_path,f'{today}.log'), 'a',buffering=1,encoding='utf-8')
|
||||
sys.stderr = sys.stdout
|
||||
|
||||
import webview
|
||||
@@ -19,6 +19,10 @@ import threading
|
||||
import time
|
||||
import requests
|
||||
import subprocess
|
||||
import atexit
|
||||
|
||||
from amazon.del_brand import kill_process
|
||||
|
||||
|
||||
|
||||
with open("version.txt","w",encoding="utf-8") as file:
|
||||
@@ -36,6 +40,21 @@ print(f"""
|
||||
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
APP_URL = "http://127.0.0.1:5123"
|
||||
PORT = 5123
|
||||
_shutdown_started = False
|
||||
|
||||
|
||||
def cleanup_before_exit():
|
||||
global _shutdown_started
|
||||
if _shutdown_started:
|
||||
return
|
||||
_shutdown_started = True
|
||||
try:
|
||||
kill_process('v6')
|
||||
except Exception as e:
|
||||
print(f"【退出前】关闭紫鸟浏览器进程异常: {str(e)}")
|
||||
|
||||
|
||||
atexit.register(cleanup_before_exit)
|
||||
|
||||
|
||||
def generate_images(params):
|
||||
@@ -54,8 +73,26 @@ class WindowAPI:
|
||||
window.events.restored += lambda _: setattr(self, '_is_maximized', False)
|
||||
|
||||
def close(self):
|
||||
sys.exit(1)
|
||||
self._window.destroy()
|
||||
"""关闭窗口,异步执行清理逻辑"""
|
||||
def cleanup_and_exit():
|
||||
"""后台清理线程"""
|
||||
try:
|
||||
kill_process('v6')
|
||||
except Exception as e:
|
||||
print(f"【退出前】关闭紫鸟浏览器进程异常: {str(e)}")
|
||||
finally:
|
||||
time.sleep(0.5)
|
||||
os._exit(0)
|
||||
|
||||
# 先销毁窗口,让用户看到立即响应
|
||||
try:
|
||||
self._window.destroy()
|
||||
except:
|
||||
pass
|
||||
|
||||
# 在后台线程执行清理,不阻塞
|
||||
cleanup_thread = threading.Thread(target=cleanup_and_exit, daemon=True)
|
||||
cleanup_thread.start()
|
||||
|
||||
def minimize(self):
|
||||
self._window.minimize()
|
||||
@@ -156,6 +193,7 @@ class WindowAPI:
|
||||
|
||||
def save_file_from_url(self, url, default_filename='download.zip'):
|
||||
"""弹窗选择保存位置,从 url 下载文件并保存。用于品牌任务结果 zip 等。"""
|
||||
print("调用到save_file_from_url方法")
|
||||
if not url or not url.strip():
|
||||
return {'success': False, 'error': '下载地址为空'}
|
||||
result = self._window.create_file_dialog(
|
||||
@@ -178,6 +216,50 @@ class WindowAPI:
|
||||
except Exception as e:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
def save_file_from_url_new(self, url, default_filename='download.bin'):
|
||||
"""弹窗选择保存位置,从 url 下载文件并保存。根据文件后缀动态设置保存类型。"""
|
||||
print("调用到save_file_from_url_new方法")
|
||||
if not url or not url.strip():
|
||||
return {'success': False, 'error': '下载地址为空'}
|
||||
|
||||
filename = str(default_filename or 'download.bin').strip() or 'download.bin'
|
||||
ext = os.path.splitext(filename)[1].lower()
|
||||
|
||||
# 根据默认文件名后缀动态设置文件类型,避免固定为 zip 造成误导
|
||||
if ext == '.zip':
|
||||
file_types = ('ZIP 压缩包 (*.zip)', '所有文件 (*.*)')
|
||||
elif ext in ('.xlsx', '.xls'):
|
||||
file_types = ('Excel 文件 (*.xlsx;*.xls)', '所有文件 (*.*)')
|
||||
elif ext == '.csv':
|
||||
file_types = ('CSV 文件 (*.csv)', '所有文件 (*.*)')
|
||||
elif ext == '.txt':
|
||||
file_types = ('文本文件 (*.txt)', '所有文件 (*.*)')
|
||||
elif ext == '.json':
|
||||
file_types = ('JSON 文件 (*.json)', '所有文件 (*.*)')
|
||||
else:
|
||||
file_types = ('所有文件 (*.*)',)
|
||||
|
||||
result = self._window.create_file_dialog(
|
||||
webview.SAVE_DIALOG,
|
||||
save_filename=filename,
|
||||
file_types=file_types
|
||||
)
|
||||
if not result:
|
||||
return {'success': False, 'error': '用户取消'}
|
||||
|
||||
path = result[0] if isinstance(result, (list, tuple)) else result
|
||||
try:
|
||||
import requests
|
||||
resp = requests.get(url.strip(), timeout=120, stream=True)
|
||||
resp.raise_for_status()
|
||||
with open(path, 'wb') as f:
|
||||
for chunk in resp.iter_content(chunk_size=65536):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
return {'success': True, 'path': path}
|
||||
except Exception as e:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
def save_template_xlsx(self):
|
||||
"""弹窗选择保存位置,将品牌文档格式模板 xlsx 保存到用户选择的位置。"""
|
||||
template_name = '品牌文档格式_模板.xlsx'
|
||||
@@ -313,6 +395,27 @@ def start_task_monitor():
|
||||
print(f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] 任务监控线程异常退出: {traceback.format_exc()}")
|
||||
|
||||
|
||||
def on_window_closing():
|
||||
"""窗口关闭事件处理器,异步执行清理逻辑,避免阻塞UI"""
|
||||
def cleanup_and_exit():
|
||||
"""后台清理线程"""
|
||||
try:
|
||||
kill_process('v6')
|
||||
except Exception as e:
|
||||
print(f"【退出前】关闭紫鸟浏览器进程异常: {str(e)}")
|
||||
finally:
|
||||
# 给一点时间让清理完成,然后强制退出
|
||||
time.sleep(0.5)
|
||||
os._exit(0)
|
||||
|
||||
# 立即在后台线程执行清理,不阻塞窗口关闭
|
||||
cleanup_thread = threading.Thread(target=cleanup_and_exit, daemon=True)
|
||||
cleanup_thread.start()
|
||||
|
||||
# 立即返回,让窗口快速关闭,用户不会感觉卡顿
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
if not os.path.exists(cache_path):
|
||||
os.makedirs(cache_path,exist_ok=True)
|
||||
@@ -339,9 +442,13 @@ def main():
|
||||
frameless=False,
|
||||
easy_drag=True
|
||||
)
|
||||
|
||||
# 为窗口关闭事件添加处理器
|
||||
window.events.closing += on_window_closing
|
||||
|
||||
api = WindowAPI(window)
|
||||
window.expose(api.close, api.minimize, api.maximize, api.toggle_maximize, generate_images, api.save_image, api.save_image_to_folder, api.select_folder, api.select_brand_xlsx_files, api.select_brand_folder,
|
||||
api.save_file_from_url, api.save_template_xlsx,api.save_template_zip,api.upload_file_to_java,api.open_external_url,api.enqueue_json,api.get_detail_del)
|
||||
api.save_file_from_url, api.save_template_xlsx,api.save_template_zip,api.upload_file_to_java,api.open_external_url,api.enqueue_json,api.get_detail_del,api.save_file_from_url_new)
|
||||
webview.start(
|
||||
debug=True,
|
||||
storage_path=cache_path,
|
||||
|
||||
@@ -5,9 +5,10 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>格式转换 - 数富AI</title>
|
||||
<script type="module" crossorigin src="/assets/convert.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-Dee3nQjE.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-MkjZQlBu.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/convert-Cfr3mUjs.css">
|
||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-D4gpiFjY.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/brand-CXHemZwJ.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/convert-7wWJ02Tw.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
|
||||
@@ -5,9 +5,10 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>数据去重 - 数富AI</title>
|
||||
<script type="module" crossorigin src="/assets/dedupe.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-Dee3nQjE.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-MkjZQlBu.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/dedupe-CS18ls2z.css">
|
||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-D4gpiFjY.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/brand-CXHemZwJ.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/dedupe-BpNHwt51.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
|
||||
@@ -5,9 +5,10 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>删除品牌 - 数富AI</title>
|
||||
<script type="module" crossorigin src="/assets/delete-brand.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-Dee3nQjE.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-MkjZQlBu.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/delete-brand-_jGFWTAn.css">
|
||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-D4gpiFjY.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/brand-CXHemZwJ.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/delete-brand-CdxGWtvK.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
|
||||
@@ -5,9 +5,10 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>数据拆分 - 数富AI</title>
|
||||
<script type="module" crossorigin src="/assets/split.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-Dee3nQjE.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-MkjZQlBu.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/split-mwLUJT1K.css">
|
||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-D4gpiFjY.js">
|
||||
<link rel="modulepreload" crossorigin href="/assets/brand-CXHemZwJ.js">
|
||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
|
||||
<link rel="stylesheet" crossorigin href="/assets/split-CRUIYKS6.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
|
||||
1071
app/web_source/brand - 副本.html
Normal file
1071
app/web_source/brand - 副本.html
Normal file
File diff suppressed because it is too large
Load Diff
@@ -14,128 +14,122 @@
|
||||
height: 100vh;
|
||||
font-weight: bold;
|
||||
}
|
||||
.top-bar {
|
||||
height: 56px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 0 16px;
|
||||
border-bottom: 1px solid #2a2a2a;
|
||||
}
|
||||
.logo-area {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
.top-bar .app-name { font-size: 18px; font-weight: 600; color: #fff; }
|
||||
.btn-home {
|
||||
font-size: 13px;
|
||||
color: #999;
|
||||
text-decoration: none;
|
||||
padding: 8px 12px;
|
||||
border-radius: 6px;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
.btn-home:hover { color: #fff; background: #2a2a2a; }
|
||||
|
||||
.nav-tabs {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.nav-tab-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.nav-dropdown {
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.nav-dropdown-trigger {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 8px;
|
||||
min-width: 112px;
|
||||
height: 38px;
|
||||
padding: 0 16px;
|
||||
border-radius: 10px;
|
||||
border: 1px solid transparent;
|
||||
color: #999;
|
||||
background: transparent;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.nav-dropdown:hover .nav-dropdown-trigger,
|
||||
.nav-dropdown.active .nav-dropdown-trigger {
|
||||
color: #fff;
|
||||
background: #2a2a2a;
|
||||
}
|
||||
|
||||
.nav-dropdown.active .nav-dropdown-trigger {
|
||||
color: #3498db;
|
||||
background: rgba(52, 152, 219, 0.2);
|
||||
border-color: rgba(52, 152, 219, 0.3);
|
||||
}
|
||||
|
||||
.nav-dropdown-arrow {
|
||||
font-size: 11px;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.nav-dropdown-menu {
|
||||
position: absolute;
|
||||
top: 100%;
|
||||
left: 0;
|
||||
min-width: 220px;
|
||||
padding: 8px;
|
||||
border: 1px solid #2a2a2a;
|
||||
border-radius: 12px;
|
||||
background: #171717;
|
||||
box-shadow: 0 12px 30px rgba(0, 0, 0, 0.35);
|
||||
display: none;
|
||||
z-index: 20;
|
||||
}
|
||||
|
||||
.nav-dropdown:hover .nav-dropdown-menu {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.nav-dropdown-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
min-height: 38px;
|
||||
padding: 0 12px;
|
||||
border-radius: 8px;
|
||||
color: #b8b8b8;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
text-decoration: none;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.nav-dropdown-item:hover {
|
||||
color: #fff;
|
||||
background: #2a2a2a;
|
||||
}
|
||||
|
||||
.nav-dropdown-item.active {
|
||||
color: #3498db;
|
||||
background: rgba(52, 152, 219, 0.2);
|
||||
}
|
||||
.top-right {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
.top-bar {
|
||||
min-height: 88px;
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: 20px;
|
||||
padding: 10px 24px 12px;
|
||||
border-bottom: 1px solid #2a2a2a;
|
||||
}
|
||||
.logo-area {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
min-width: 0;
|
||||
padding-top: 10px;
|
||||
}
|
||||
.top-bar .app-name { font-size: 18px; font-weight: 600; color: #fff; }
|
||||
.btn-home {
|
||||
font-size: 13px;
|
||||
color: #d2d2d2;
|
||||
text-decoration: none;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.btn-home:hover { color: #fff; }
|
||||
|
||||
.nav-tabs {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
display: flex;
|
||||
align-items: stretch;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.nav-tab-group {
|
||||
display: flex;
|
||||
align-items: stretch;
|
||||
justify-content: center;
|
||||
flex: 1;
|
||||
gap: 0;
|
||||
}
|
||||
|
||||
.nav-section {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
padding: 0 18px;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.nav-section + .nav-section::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
left: 0;
|
||||
top: 14px;
|
||||
bottom: 8px;
|
||||
width: 1px;
|
||||
background: rgba(255, 255, 255, 0.14);
|
||||
}
|
||||
|
||||
.nav-section-title {
|
||||
margin-bottom: 10px;
|
||||
text-align: center;
|
||||
color: #e8dfcf;
|
||||
font-size: 16px;
|
||||
font-weight: 800;
|
||||
letter-spacing: 0.08em;
|
||||
}
|
||||
|
||||
.nav-section-items {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
justify-content: center;
|
||||
padding: 10px 14px;
|
||||
border-radius: 14px;
|
||||
background: linear-gradient(180deg, rgba(83, 74, 67, 0.92) 0%, rgba(72, 65, 59, 0.96) 100%);
|
||||
box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.06);
|
||||
}
|
||||
|
||||
.nav-item {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-height: 28px;
|
||||
padding: 0 2px;
|
||||
color: #f3eee4;
|
||||
font-size: 14px;
|
||||
font-weight: 700;
|
||||
text-decoration: none;
|
||||
border-bottom: 2px solid transparent;
|
||||
transition: color 0.18s ease, border-color 0.18s ease, opacity 0.18s ease;
|
||||
background: transparent;
|
||||
border-left: none;
|
||||
border-right: none;
|
||||
border-top: none;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.nav-item:hover {
|
||||
color: #fff;
|
||||
border-bottom-color: rgba(255, 255, 255, 0.68);
|
||||
}
|
||||
|
||||
.nav-item.active {
|
||||
color: #fff;
|
||||
border-bottom-color: #8dc4ff;
|
||||
}
|
||||
|
||||
.nav-item.disabled {
|
||||
color: rgba(255, 255, 255, 0.78);
|
||||
opacity: 0.82;
|
||||
cursor: default;
|
||||
}
|
||||
.top-right {
|
||||
width: 60px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.main-content {
|
||||
display: flex;
|
||||
@@ -395,31 +389,40 @@
|
||||
<span class="app-name">数富AI-亚马逊</span>
|
||||
<a href="/home" class="btn-home">返回首页</a>
|
||||
</div>
|
||||
<nav class="nav-tabs">
|
||||
<div class="nav-tab-group">
|
||||
<div class="nav-dropdown active">
|
||||
<button type="button" class="nav-dropdown-trigger">
|
||||
<span>前端工具</span>
|
||||
<span class="nav-dropdown-arrow">▾</span>
|
||||
</button>
|
||||
<div class="nav-dropdown-menu">
|
||||
<button type="button" class="nav-dropdown-item active" data-panel="brandCheck">品牌检测</button>
|
||||
<a class="nav-dropdown-item" href="/new_web_source/dedupe.html">数据去重</a>
|
||||
<a class="nav-dropdown-item" href="/new_web_source/split.html">数据拆分</a>
|
||||
<a class="nav-dropdown-item" href="/new_web_source/convert.html">格式转换</a>
|
||||
</div>
|
||||
</div>
|
||||
<div class="nav-dropdown">
|
||||
<button type="button" class="nav-dropdown-trigger">
|
||||
<span>运营工具</span>
|
||||
<span class="nav-dropdown-arrow">▾</span>
|
||||
</button>
|
||||
<div class="nav-dropdown-menu">
|
||||
<a class="nav-dropdown-item" href="/new_web_source/delete-brand.html">删除指定ASIN和品牌</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
<nav class="nav-tabs">
|
||||
<div class="nav-tab-group">
|
||||
<div class="nav-section">
|
||||
<div class="nav-section-title">前端工具</div>
|
||||
<div class="nav-section-items">
|
||||
<button type="button" class="nav-item active" data-panel="brandCheck">品牌检测</button>
|
||||
<span class="nav-item disabled">采集数据</span>
|
||||
<span class="nav-item disabled">变体分析</span>
|
||||
<a class="nav-item" href="/new_web_source/dedupe.html">数据去重</a>
|
||||
<a class="nav-item" href="/new_web_source/split.html">数据拆分</a>
|
||||
<a class="nav-item" href="/new_web_source/convert.html">格式转换</a>
|
||||
</div>
|
||||
</div>
|
||||
<div class="nav-section">
|
||||
<div class="nav-section-title">运营工具</div>
|
||||
<div class="nav-section-items">
|
||||
<a class="nav-item" href="/new_web_source/delete-brand.html">删除ASIN</a>
|
||||
<a class="nav-item" href="/new_web_source/product-risk.html">商品风险解决</a>
|
||||
<a class="nav-item" href="/new_web_source/shop-match.html">定时匹配</a>
|
||||
<span class="nav-item disabled">跟价</span>
|
||||
<span class="nav-item disabled">巡店删除</span>
|
||||
<span class="nav-item disabled">取款</span>
|
||||
<span class="nav-item disabled">店铺状态查询</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="nav-section">
|
||||
<div class="nav-section-title">后勤工具</div>
|
||||
<div class="nav-section-items">
|
||||
<span class="nav-item disabled">采购</span>
|
||||
<span class="nav-item disabled">ERP</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
<div class="top-right"></div>
|
||||
</header>
|
||||
|
||||
@@ -547,11 +550,11 @@
|
||||
}
|
||||
|
||||
// 顶部栏目切换:点击当前页菜单项时显示对应 data-panel 的 tab-panel
|
||||
document.querySelectorAll('.nav-dropdown-item[data-panel]').forEach(function(tab) {
|
||||
document.querySelectorAll('.nav-item[data-panel]').forEach(function(tab) {
|
||||
tab.addEventListener('click', function() {
|
||||
var panelId = this.getAttribute('data-panel');
|
||||
if (!panelId) return;
|
||||
document.querySelectorAll('.nav-dropdown-item[data-panel]').forEach(function(t) { t.classList.remove('active'); });
|
||||
document.querySelectorAll('.nav-item[data-panel]').forEach(function(t) { t.classList.remove('active'); });
|
||||
document.querySelectorAll('.tab-panel').forEach(function(p) {
|
||||
p.classList.toggle('active', p.getAttribute('data-panel') === panelId);
|
||||
});
|
||||
|
||||
Binary file not shown.
BIN
backend-java/aiimage-backend.log.2026-04-16.0.gz
Normal file
BIN
backend-java/aiimage-backend.log.2026-04-16.0.gz
Normal file
Binary file not shown.
@@ -61,6 +61,11 @@
|
||||
<artifactId>easyexcel</artifactId>
|
||||
<version>${easyexcel.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml</artifactId>
|
||||
<version>5.2.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.aliyun.oss</groupId>
|
||||
<artifactId>aliyun-sdk-oss</artifactId>
|
||||
|
||||
Binary file not shown.
@@ -1,13 +1,11 @@
|
||||
package com.nanri.aiimage.common.api;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Schema(description = "统一响应体")
|
||||
public class ApiResponse<T> {
|
||||
|
||||
@@ -20,6 +18,20 @@ public class ApiResponse<T> {
|
||||
@Schema(description = "响应数据")
|
||||
private T data;
|
||||
|
||||
@Schema(description = "业务错误码,成功时可为空")
|
||||
private Integer code;
|
||||
|
||||
public ApiResponse(boolean success, String message, T data) {
|
||||
this(success, message, data, null);
|
||||
}
|
||||
|
||||
public ApiResponse(boolean success, String message, T data, Integer code) {
|
||||
this.success = success;
|
||||
this.message = message;
|
||||
this.data = data;
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
public static <T> ApiResponse<T> success(T data) {
|
||||
return new ApiResponse<>(true, "操作成功", data);
|
||||
}
|
||||
@@ -31,4 +43,8 @@ public class ApiResponse<T> {
|
||||
public static <T> ApiResponse<T> fail(String message) {
|
||||
return new ApiResponse<>(false, message, null);
|
||||
}
|
||||
|
||||
public static <T> ApiResponse<T> fail(Integer code, String message) {
|
||||
return new ApiResponse<>(false, message, null, code);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,18 @@ package com.nanri.aiimage.common.exception;
|
||||
|
||||
public class BusinessException extends RuntimeException {
|
||||
|
||||
private final Integer code;
|
||||
|
||||
public BusinessException(String message) {
|
||||
this(null, message);
|
||||
}
|
||||
|
||||
public BusinessException(Integer code, String message) {
|
||||
super(message);
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
public Integer getCode() {
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,9 @@ public class GlobalExceptionHandler {
|
||||
|
||||
@ExceptionHandler(BusinessException.class)
|
||||
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
|
||||
return ApiResponse.fail(ex.getMessage());
|
||||
return ex.getCode() == null
|
||||
? ApiResponse.fail(ex.getMessage())
|
||||
: ApiResponse.fail(ex.getCode(), ex.getMessage());
|
||||
}
|
||||
|
||||
@ExceptionHandler(MethodArgumentNotValidException.class)
|
||||
@@ -32,6 +34,6 @@ public class GlobalExceptionHandler {
|
||||
@ExceptionHandler(Exception.class)
|
||||
public ApiResponse<Void> handleException(Exception ex) {
|
||||
log.error("Unhandled exception", ex);
|
||||
return ApiResponse.fail("服务异常:" + ex.getMessage());
|
||||
return ApiResponse.fail("服务异常: " + ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package com.nanri.aiimage.common.security;
|
||||
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.Base64;
|
||||
|
||||
@Service
|
||||
public class ShopCredentialCryptoService {
|
||||
|
||||
@Value("${aiimage.security.shop-credential-key:change-me-shop-credential-key}")
|
||||
private String rawKey;
|
||||
|
||||
private SecretKeySpec keySpec;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
try {
|
||||
MessageDigest sha256 = MessageDigest.getInstance("SHA-256");
|
||||
byte[] full = sha256.digest(rawKey.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] key16 = new byte[16];
|
||||
System.arraycopy(full, 0, key16, 0, 16);
|
||||
this.keySpec = new SecretKeySpec(key16, "AES");
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("初始化店铺凭据加密器失败");
|
||||
}
|
||||
}
|
||||
|
||||
public String encrypt(String plainText) {
|
||||
try {
|
||||
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
|
||||
cipher.init(Cipher.ENCRYPT_MODE, keySpec);
|
||||
byte[] encrypted = cipher.doFinal((plainText == null ? "" : plainText).getBytes(StandardCharsets.UTF_8));
|
||||
return Base64.getEncoder().encodeToString(encrypted);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("店铺凭据加密失败");
|
||||
}
|
||||
}
|
||||
|
||||
public String decrypt(String cipherText) {
|
||||
try {
|
||||
if (cipherText == null || cipherText.isBlank()) {
|
||||
return "";
|
||||
}
|
||||
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
|
||||
cipher.init(Cipher.DECRYPT_MODE, keySpec);
|
||||
byte[] decrypted = cipher.doFinal(Base64.getDecoder().decode(cipherText));
|
||||
return new String(decrypted, StandardCharsets.UTF_8);
|
||||
} catch (Exception ex) {
|
||||
// 兼容历史数据:旧记录可能是明文或使用旧密钥加密,回退原文避免中断自动化流程。
|
||||
return cipherText == null ? "" : cipherText;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package com.nanri.aiimage.common.service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.script.DefaultRedisScript;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
import java.util.UUID;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class DistributedJobLockService {
|
||||
|
||||
private static final DefaultRedisScript<Long> RELEASE_SCRIPT = new DefaultRedisScript<>(
|
||||
"if redis.call('get', KEYS[1]) == ARGV[1] then "
|
||||
+ "return redis.call('del', KEYS[1]) "
|
||||
+ "else return 0 end",
|
||||
Long.class
|
||||
);
|
||||
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
private final String ownerPrefix;
|
||||
|
||||
public DistributedJobLockService(StringRedisTemplate stringRedisTemplate) {
|
||||
this.stringRedisTemplate = stringRedisTemplate;
|
||||
this.ownerPrefix = resolveOwnerPrefix();
|
||||
}
|
||||
|
||||
public LockHandle tryLock(String jobName, Duration ttl) {
|
||||
if (jobName == null || jobName.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
Duration actualTtl = (ttl == null || ttl.isNegative() || ttl.isZero()) ? Duration.ofMinutes(10) : ttl;
|
||||
String key = buildLockKey(jobName);
|
||||
String token = ownerPrefix + ":" + UUID.randomUUID();
|
||||
Boolean locked = stringRedisTemplate.opsForValue().setIfAbsent(key, token, actualTtl);
|
||||
if (!Boolean.TRUE.equals(locked)) {
|
||||
return null;
|
||||
}
|
||||
return new LockHandle(key, token, jobName);
|
||||
}
|
||||
|
||||
public final class LockHandle implements AutoCloseable {
|
||||
|
||||
private final String key;
|
||||
private final String token;
|
||||
private final String jobName;
|
||||
private boolean released;
|
||||
|
||||
private LockHandle(String key, String token, String jobName) {
|
||||
this.key = key;
|
||||
this.token = token;
|
||||
this.jobName = jobName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (released) {
|
||||
return;
|
||||
}
|
||||
released = true;
|
||||
try {
|
||||
stringRedisTemplate.execute(RELEASE_SCRIPT, Collections.singletonList(key), token);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[job-lock] release failed jobName={} key={} msg={}", jobName, key, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String buildLockKey(String jobName) {
|
||||
return "job-lock:" + jobName;
|
||||
}
|
||||
|
||||
private String resolveOwnerPrefix() {
|
||||
try {
|
||||
return InetAddress.getLocalHost().getHostName();
|
||||
} catch (Exception ignored) {
|
||||
return "unknown-host";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package com.nanri.aiimage.common.util;
|
||||
|
||||
import com.alibaba.excel.EasyExcel;
|
||||
import com.alibaba.excel.context.AnalysisContext;
|
||||
import com.alibaba.excel.event.AnalysisEventListener;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public final class ExcelStreamReader {
|
||||
|
||||
private ExcelStreamReader() {
|
||||
}
|
||||
|
||||
public static void readFirstSheet(File file, SheetRowHandler handler) throws IOException {
|
||||
try (InputStream inputStream = new FileInputStream(file)) {
|
||||
readFirstSheet(inputStream, handler);
|
||||
}
|
||||
}
|
||||
|
||||
public static void readFirstSheet(InputStream inputStream, SheetRowHandler handler) throws IOException {
|
||||
runRead(inputStream, false, handler);
|
||||
}
|
||||
|
||||
public static void readAllSheets(File file, SheetRowHandler handler) throws IOException {
|
||||
try (InputStream inputStream = new FileInputStream(file)) {
|
||||
readAllSheets(inputStream, handler);
|
||||
}
|
||||
}
|
||||
|
||||
public static void readAllSheets(InputStream inputStream, SheetRowHandler handler) throws IOException {
|
||||
runRead(inputStream, true, handler);
|
||||
}
|
||||
|
||||
private static void runRead(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler) throws IOException {
|
||||
StreamingListener listener = new StreamingListener(handler);
|
||||
try {
|
||||
if (readAllSheets) {
|
||||
EasyExcel.read(inputStream, listener).headRowNumber(1).doReadAll();
|
||||
} else {
|
||||
EasyExcel.read(inputStream, listener).sheet(0).headRowNumber(1).doRead();
|
||||
}
|
||||
} catch (WrappedRuntimeException ex) {
|
||||
if (ex.getCause() instanceof IOException ioException) {
|
||||
throw ioException;
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
public interface SheetRowHandler {
|
||||
default void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) throws Exception {
|
||||
}
|
||||
|
||||
void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception;
|
||||
}
|
||||
|
||||
private static final class StreamingListener extends AnalysisEventListener<Map<Integer, String>> {
|
||||
|
||||
private final SheetRowHandler handler;
|
||||
private Map<Integer, String> currentHeaderMap = Map.of();
|
||||
|
||||
private StreamingListener(SheetRowHandler handler) {
|
||||
this.handler = handler;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
|
||||
Map<Integer, String> normalizedHeadMap = new LinkedHashMap<>();
|
||||
for (Map.Entry<Integer, String> entry : headMap.entrySet()) {
|
||||
normalizedHeadMap.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
currentHeaderMap = normalizedHeadMap;
|
||||
try {
|
||||
handler.onHeader(sheetName(context), sheetNo(context), currentHeaderMap);
|
||||
} catch (RuntimeException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw new WrappedRuntimeException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void invoke(Map<Integer, String> data, AnalysisContext context) {
|
||||
Map<Integer, String> normalizedRow = new LinkedHashMap<>();
|
||||
for (Map.Entry<Integer, String> entry : data.entrySet()) {
|
||||
normalizedRow.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
try {
|
||||
handler.onRow(
|
||||
sheetName(context),
|
||||
sheetNo(context),
|
||||
context.readRowHolder().getRowIndex(),
|
||||
currentHeaderMap,
|
||||
normalizedRow
|
||||
);
|
||||
} catch (RuntimeException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
throw new WrappedRuntimeException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doAfterAllAnalysed(AnalysisContext context) {
|
||||
}
|
||||
|
||||
private String sheetName(AnalysisContext context) {
|
||||
return context.readSheetHolder() == null ? "" : context.readSheetHolder().getSheetName();
|
||||
}
|
||||
|
||||
private Integer sheetNo(AnalysisContext context) {
|
||||
return context.readSheetHolder() == null ? null : context.readSheetHolder().getSheetNo();
|
||||
}
|
||||
}
|
||||
|
||||
private static final class WrappedRuntimeException extends RuntimeException {
|
||||
private WrappedRuntimeException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,10 +7,29 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
@ConfigurationProperties(prefix = "aiimage.delete-brand-progress")
|
||||
public class DeleteBrandProgressProperties {
|
||||
private long heartbeatTimeoutMinutes = 15;
|
||||
private String staleCheckCron = "0 */2 * * * *";
|
||||
private String staleCheckCron = "*/30 * * * * *";
|
||||
|
||||
/**
|
||||
* 定时补偿 finalize:用于“分片其实已齐,但最后一次提交断开导致没触发 finalize”的场景。
|
||||
*/
|
||||
private String finalizeCheckCron = "30 */2 * * * *";
|
||||
|
||||
/**
|
||||
* 商品风险(PRODUCT_RISK_RESOLVE)RUNNING 超时自动收尾/失败,按分钟计算;
|
||||
* 与删除品牌共用同一定时调度。
|
||||
*/
|
||||
private long productRiskStaleTimeoutMinutes = 20;
|
||||
private long productRiskInitialTimeoutMinutes = 20;
|
||||
|
||||
/**
|
||||
* Price track RUNNING timeout auto-finalize/fail threshold in minutes.
|
||||
*/
|
||||
private long priceTrackStaleTimeoutMinutes = 20;
|
||||
private long priceTrackInitialTimeoutMinutes = 20;
|
||||
|
||||
/**
|
||||
* Shop match RUNNING timeout auto-finalize/fail threshold in minutes.
|
||||
*/
|
||||
private long shopMatchStaleTimeoutMinutes = 20;
|
||||
private long shopMatchInitialTimeoutMinutes = 20;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.net.InetAddress;
|
||||
|
||||
@Component
|
||||
public class InstanceMetadata {
|
||||
|
||||
private final String instanceId;
|
||||
private final String hostname;
|
||||
|
||||
public InstanceMetadata(@Value("${aiimage.instance-id:}") String configuredInstanceId) {
|
||||
this.hostname = resolveHostname();
|
||||
this.instanceId = configuredInstanceId == null || configuredInstanceId.isBlank()
|
||||
? this.hostname
|
||||
: configuredInstanceId.trim();
|
||||
}
|
||||
|
||||
public String getInstanceId() {
|
||||
return instanceId;
|
||||
}
|
||||
|
||||
public String getHostname() {
|
||||
return hostname;
|
||||
}
|
||||
|
||||
private static String resolveHostname() {
|
||||
try {
|
||||
return InetAddress.getLocalHost().getHostName();
|
||||
} catch (Exception ignored) {
|
||||
String envHostname = System.getenv("HOSTNAME");
|
||||
return (envHostname == null || envHostname.isBlank()) ? "unknown-host" : envHostname;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,5 +11,5 @@ import java.util.List;
|
||||
public class ModuleCleanupProperties {
|
||||
private boolean enabled = true;
|
||||
private String cron = "0 0 0 * * *";
|
||||
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND"));
|
||||
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK"));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import jakarta.servlet.FilterChain;
|
||||
import jakarta.servlet.ServletException;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.filter.OncePerRequestFilter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Optional;
|
||||
|
||||
@Component
|
||||
public class RequestTraceFilter extends OncePerRequestFilter {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(RequestTraceFilter.class);
|
||||
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
|
||||
public RequestTraceFilter(InstanceMetadata instanceMetadata) {
|
||||
this.instanceMetadata = instanceMetadata;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
|
||||
throws ServletException, IOException {
|
||||
long start = System.currentTimeMillis();
|
||||
String remoteAddr = firstNonBlank(
|
||||
request.getHeader("X-Forwarded-For"),
|
||||
request.getHeader("X-Real-IP"),
|
||||
request.getRemoteAddr()
|
||||
);
|
||||
String forwardedProto = request.getHeader("X-Forwarded-Proto");
|
||||
String forwardedHost = firstNonBlank(request.getHeader("X-Forwarded-Host"), request.getHeader("Host"));
|
||||
String forwardedPort = request.getHeader("X-Forwarded-Port");
|
||||
String requestId = firstNonBlank(
|
||||
request.getHeader("X-Request-Id"),
|
||||
request.getHeader("X-Amzn-Trace-Id"),
|
||||
request.getHeader("Traceparent")
|
||||
);
|
||||
|
||||
response.setHeader("X-AIIMAGE-Instance", instanceMetadata.getInstanceId());
|
||||
response.setHeader("X-AIIMAGE-Host", instanceMetadata.getHostname());
|
||||
|
||||
try {
|
||||
filterChain.doFilter(request, response);
|
||||
} finally {
|
||||
long costMs = System.currentTimeMillis() - start;
|
||||
log.info(
|
||||
"request-trace instance={} host={} method={} uri={} status={} remote={} forwardedHost={} forwardedProto={} forwardedPort={} requestId={} userAgent={} costMs={}",
|
||||
instanceMetadata.getInstanceId(),
|
||||
instanceMetadata.getHostname(),
|
||||
request.getMethod(),
|
||||
request.getRequestURI(),
|
||||
response.getStatus(),
|
||||
remoteAddr,
|
||||
blankToDash(forwardedHost),
|
||||
blankToDash(forwardedProto),
|
||||
blankToDash(forwardedPort),
|
||||
blankToDash(requestId),
|
||||
blankToDash(request.getHeader("User-Agent")),
|
||||
costMs
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private static String firstNonBlank(String... values) {
|
||||
for (String value : values) {
|
||||
if (value != null && !value.isBlank()) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static String blankToDash(String value) {
|
||||
return Optional.ofNullable(value).filter(v -> !v.isBlank()).orElse("-");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.TaskScheduler;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
|
||||
|
||||
@Configuration
|
||||
@Slf4j
|
||||
public class SchedulingConfig {
|
||||
|
||||
@Bean
|
||||
public TaskScheduler taskScheduler() {
|
||||
ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
|
||||
scheduler.setPoolSize(4);
|
||||
scheduler.setThreadNamePrefix("aiimage-scheduling-");
|
||||
scheduler.setWaitForTasksToCompleteOnShutdown(true);
|
||||
scheduler.setAwaitTerminationSeconds(30);
|
||||
scheduler.setErrorHandler(ex -> log.warn("[scheduling] task execution failed: {}", ex.getMessage(), ex));
|
||||
scheduler.initialize();
|
||||
return scheduler;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.nanri.aiimage.modules.brand.model.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class BrandFileAggregateCacheDto {
|
||||
private String fileUrl;
|
||||
private String originalFilename;
|
||||
private String relativePath;
|
||||
private String mainSheetName;
|
||||
private Integer chunkTotal;
|
||||
private Integer totalLines;
|
||||
private Integer receivedChunkCount = 0;
|
||||
private Integer processedLineCount = 0;
|
||||
private Boolean completed = false;
|
||||
private List<BrandInvalidBrandDto> invalidBrands = new ArrayList<>();
|
||||
private List<String> queryFailedBrands = new ArrayList<>();
|
||||
}
|
||||
@@ -5,10 +5,16 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.BrandProgressProperties;
|
||||
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
@@ -17,18 +23,26 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class BrandTaskProgressCacheService {
|
||||
|
||||
public static final String PHASE_CRAWLING = "crawling";
|
||||
public static final String PHASE_ASSEMBLING = "assembling";
|
||||
public static final String PHASE_UPLOADING = "uploading";
|
||||
public static final String PHASE_FAILED = "failed";
|
||||
private static final Duration FINALIZE_LOCK_TTL = Duration.ofMinutes(10);
|
||||
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
private final BrandProgressProperties brandProgressProperties;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public BrandTaskProgressCacheService(StringRedisTemplate stringRedisTemplate,
|
||||
BrandProgressProperties brandProgressProperties,
|
||||
ObjectMapper objectMapper) {
|
||||
this.stringRedisTemplate = stringRedisTemplate;
|
||||
this.brandProgressProperties = brandProgressProperties;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
public void saveProgressFromResult(Long taskId,
|
||||
String fileUrl,
|
||||
int fileIndex,
|
||||
@@ -51,7 +65,7 @@ public class BrandTaskProgressCacheService {
|
||||
values.put("updated_at", now);
|
||||
values.put("last_heartbeat_at", now);
|
||||
stringRedisTemplate.opsForHash().putAll(key, values);
|
||||
stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getTtlHours()));
|
||||
stringRedisTemplate.expire(key, ttl());
|
||||
}
|
||||
|
||||
public void updatePhase(Long taskId, String phase, int finishedFiles, int fileTotal) {
|
||||
@@ -64,7 +78,7 @@ public class BrandTaskProgressCacheService {
|
||||
values.put("updated_at", now);
|
||||
values.put("last_heartbeat_at", now);
|
||||
stringRedisTemplate.opsForHash().putAll(key, values);
|
||||
stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getTtlHours()));
|
||||
stringRedisTemplate.expire(key, ttl());
|
||||
}
|
||||
|
||||
public void markFailed(Long taskId, String message) {
|
||||
@@ -84,85 +98,324 @@ public class BrandTaskProgressCacheService {
|
||||
|
||||
public void saveParsedPayload(Long taskId, Object payload) {
|
||||
try {
|
||||
stringRedisTemplate.opsForValue().set(buildPayloadKey(taskId), objectMapper.writeValueAsString(payload), Duration.ofHours(brandProgressProperties.getTtlHours()));
|
||||
Files.createDirectories(buildTaskDir(taskId));
|
||||
Files.writeString(
|
||||
buildPayloadFile(taskId),
|
||||
objectMapper.writeValueAsString(payload),
|
||||
StandardOpenOption.CREATE,
|
||||
StandardOpenOption.TRUNCATE_EXISTING,
|
||||
StandardOpenOption.WRITE
|
||||
);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("暂存品牌原始数据失败");
|
||||
throw new BusinessException("鏆傚瓨鍝佺墝鍘熷鏁版嵁澶辫触");
|
||||
}
|
||||
}
|
||||
|
||||
public <T> T getParsedPayload(Long taskId, TypeReference<T> typeReference) {
|
||||
String raw = stringRedisTemplate.opsForValue().get(buildPayloadKey(taskId));
|
||||
if (raw == null || raw.isBlank()) {
|
||||
Path payloadFile = buildPayloadFile(taskId);
|
||||
if (!Files.isRegularFile(payloadFile)) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return objectMapper.readValue(raw, typeReference);
|
||||
return objectMapper.readValue(Files.readString(payloadFile), typeReference);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取品牌原始数据失败");
|
||||
throw new BusinessException("璇诲彇鍝佺墝鍘熷鏁版嵁澶辫触");
|
||||
}
|
||||
}
|
||||
|
||||
public ChunkStoreResult storeChunk(Long taskId, BrandCrawlResultFileDto file) {
|
||||
validateChunk(file);
|
||||
String fileUrl = normalizeFileUrl(file.getFileUrl());
|
||||
String chunkDigest = digestChunk(file);
|
||||
String digestField = String.valueOf(file.getChunkIndex());
|
||||
String digestKey = buildChunkDigestKey(taskId, fileUrl);
|
||||
Object existingDigest = stringRedisTemplate.opsForHash().get(digestKey, digestField);
|
||||
if (existingDigest instanceof String existing && !existing.isBlank()) {
|
||||
refreshFileKeys(taskId, fileUrl);
|
||||
if (existing.equals(chunkDigest)) {
|
||||
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl);
|
||||
int finishedFiles = countCompletedFiles(taskId);
|
||||
return new ChunkStoreResult(false, false, finishedFiles, aggregate);
|
||||
}
|
||||
throw new BusinessException("鍚屼竴鍒嗙墖閲嶅鎻愪氦浣嗗唴瀹逛笉涓€鑷? " + fileUrl + "#" + file.getChunkIndex());
|
||||
}
|
||||
|
||||
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl);
|
||||
if (aggregate == null) {
|
||||
aggregate = createAggregate(file);
|
||||
} else {
|
||||
ensureChunkConsistency(aggregate, file);
|
||||
}
|
||||
mergeAggregate(aggregate, file);
|
||||
|
||||
stringRedisTemplate.opsForSet().add(buildChunkReceiptKey(taskId, fileUrl), digestField);
|
||||
Long receivedSize = stringRedisTemplate.opsForSet().size(buildChunkReceiptKey(taskId, fileUrl));
|
||||
int receivedCount = receivedSize == null ? 0 : receivedSize.intValue();
|
||||
if (receivedCount <= 0) {
|
||||
receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
|
||||
}
|
||||
aggregate.setReceivedChunkCount(Math.max(receivedCount, aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount()));
|
||||
boolean fileCompleted = aggregate.getChunkTotal() != null
|
||||
&& aggregate.getChunkTotal() > 0
|
||||
&& aggregate.getReceivedChunkCount() >= aggregate.getChunkTotal();
|
||||
aggregate.setCompleted(fileCompleted);
|
||||
saveFileAggregate(taskId, aggregate);
|
||||
stringRedisTemplate.opsForHash().put(digestKey, digestField, chunkDigest);
|
||||
refreshFileKeys(taskId, fileUrl);
|
||||
|
||||
boolean newlyCompleted = false;
|
||||
int finishedFiles = countCompletedFiles(taskId);
|
||||
if (fileCompleted) {
|
||||
Long added = stringRedisTemplate.opsForSet().add(buildCompletedFilesKey(taskId), fileUrl);
|
||||
refreshTaskKey(buildCompletedFilesKey(taskId));
|
||||
newlyCompleted = added != null && added > 0;
|
||||
if (newlyCompleted) {
|
||||
finishedFiles = countCompletedFiles(taskId);
|
||||
}
|
||||
}
|
||||
return new ChunkStoreResult(true, newlyCompleted, finishedFiles, aggregate);
|
||||
}
|
||||
|
||||
public BrandFileAggregateCacheDto getFileAggregate(Long taskId, String fileUrl) {
|
||||
String normalizedFileUrl = normalizeFileUrl(fileUrl);
|
||||
Object raw = stringRedisTemplate.opsForHash().get(buildFileAggregateKey(taskId), normalizedFileUrl);
|
||||
if (!(raw instanceof String json) || json.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return objectMapper.readValue(json, BrandFileAggregateCacheDto.class);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("璇诲彇鍝佺墝鏂囦欢鑱氬悎缂撳瓨澶辫触");
|
||||
}
|
||||
}
|
||||
|
||||
public Map<String, BrandFileAggregateCacheDto> getAllFileAggregates(Long taskId) {
|
||||
Map<Object, Object> stored = stringRedisTemplate.opsForHash().entries(buildFileAggregateKey(taskId));
|
||||
Map<String, BrandFileAggregateCacheDto> result = new LinkedHashMap<>();
|
||||
for (Map.Entry<Object, Object> entry : stored.entrySet()) {
|
||||
if (!(entry.getKey() instanceof String fileUrl) || !(entry.getValue() instanceof String json) || json.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
result.put(fileUrl, objectMapper.readValue(json, BrandFileAggregateCacheDto.class));
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public int countCompletedFiles(Long taskId) {
|
||||
Long count = stringRedisTemplate.opsForSet().size(buildCompletedFilesKey(taskId));
|
||||
return count == null ? 0 : count.intValue();
|
||||
}
|
||||
|
||||
public boolean acquireFinalizeLock(Long taskId) {
|
||||
Boolean ok = stringRedisTemplate.opsForValue().setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
|
||||
return Boolean.TRUE.equals(ok);
|
||||
}
|
||||
|
||||
public void releaseFinalizeLock(Long taskId) {
|
||||
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
|
||||
}
|
||||
|
||||
public void delete(Long taskId) {
|
||||
stringRedisTemplate.delete(buildKey(taskId));
|
||||
stringRedisTemplate.delete(buildResultKey(taskId));
|
||||
stringRedisTemplate.delete(buildPayloadKey(taskId));
|
||||
stringRedisTemplate.delete(buildFileAggregateKey(taskId));
|
||||
stringRedisTemplate.delete(buildCompletedFilesKey(taskId));
|
||||
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
|
||||
stringRedisTemplate.delete(buildLegacyResultKey(taskId));
|
||||
deleteTaskDirQuietly(taskId);
|
||||
}
|
||||
|
||||
public void mergeResultChunks(Long taskId, List<BrandCrawlResultFileDto> incomingFiles) {
|
||||
String resultKey = buildResultKey(taskId);
|
||||
for (BrandCrawlResultFileDto file : incomingFiles) {
|
||||
if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0 || file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) {
|
||||
throw new BusinessException("分片参数不合法");
|
||||
}
|
||||
String field = buildChunkField(file.getFileUrl(), file.getChunkIndex());
|
||||
try {
|
||||
stringRedisTemplate.opsForHash().put(resultKey, field, objectMapper.writeValueAsString(file));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("暂存结果分片失败");
|
||||
}
|
||||
}
|
||||
stringRedisTemplate.expire(resultKey, Duration.ofHours(brandProgressProperties.getTtlHours()));
|
||||
}
|
||||
|
||||
public Map<String, List<BrandCrawlResultFileDto>> groupResultChunksByFile(Long taskId) {
|
||||
Map<Object, Object> stored = stringRedisTemplate.opsForHash().entries(buildResultKey(taskId));
|
||||
Map<String, List<BrandCrawlResultFileDto>> grouped = new LinkedHashMap<>();
|
||||
for (Object value : stored.values()) {
|
||||
if (!(value instanceof String raw) || raw.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
BrandCrawlResultFileDto file = objectMapper.readValue(raw, BrandCrawlResultFileDto.class);
|
||||
grouped.computeIfAbsent(file.getFileUrl(), ignored -> new ArrayList<>()).add(file);
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
grouped.values().forEach(list -> list.sort(java.util.Comparator.comparing(BrandCrawlResultFileDto::getChunkIndex)));
|
||||
return grouped;
|
||||
}
|
||||
|
||||
public void clearResultChunks(Long taskId) {
|
||||
stringRedisTemplate.delete(buildResultKey(taskId));
|
||||
public void deleteFileState(Long taskId, String fileUrl) {
|
||||
String normalizedFileUrl = normalizeFileUrl(fileUrl);
|
||||
stringRedisTemplate.opsForHash().delete(buildFileAggregateKey(taskId), normalizedFileUrl);
|
||||
stringRedisTemplate.delete(buildChunkReceiptKey(taskId, normalizedFileUrl));
|
||||
stringRedisTemplate.delete(buildChunkDigestKey(taskId, normalizedFileUrl));
|
||||
}
|
||||
|
||||
public String buildKey(Long taskId) {
|
||||
return "brand:task:progress:" + taskId;
|
||||
}
|
||||
|
||||
private String buildResultKey(Long taskId) {
|
||||
public long getHeartbeatTimeoutMinutes() {
|
||||
return brandProgressProperties.getHeartbeatTimeoutMinutes();
|
||||
}
|
||||
|
||||
private void saveFileAggregate(Long taskId, BrandFileAggregateCacheDto aggregate) {
|
||||
try {
|
||||
stringRedisTemplate.opsForHash().put(buildFileAggregateKey(taskId), normalizeFileUrl(aggregate.getFileUrl()), objectMapper.writeValueAsString(aggregate));
|
||||
refreshTaskKey(buildFileAggregateKey(taskId));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("鏆傚瓨鍝佺墝鏂囦欢鑱氬悎缁撴灉澶辫触");
|
||||
}
|
||||
}
|
||||
|
||||
private void validateChunk(BrandCrawlResultFileDto file) {
|
||||
if (file == null) {
|
||||
throw new BusinessException("鍒嗙墖涓嶈兘涓虹┖");
|
||||
}
|
||||
if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0 || file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) {
|
||||
throw new BusinessException("鍒嗙墖鍙傛暟涓嶅悎娉?");
|
||||
}
|
||||
if (file.getFileUrl() == null || file.getFileUrl().isBlank()) {
|
||||
throw new BusinessException("fileUrl 涓嶈兘涓虹┖");
|
||||
}
|
||||
}
|
||||
|
||||
private BrandFileAggregateCacheDto createAggregate(BrandCrawlResultFileDto file) {
|
||||
BrandFileAggregateCacheDto aggregate = new BrandFileAggregateCacheDto();
|
||||
aggregate.setFileUrl(normalizeFileUrl(file.getFileUrl()));
|
||||
aggregate.setOriginalFilename(file.getOriginalFilename());
|
||||
aggregate.setRelativePath(file.getRelativePath());
|
||||
aggregate.setMainSheetName(file.getMainSheetName());
|
||||
aggregate.setChunkTotal(file.getChunkTotal());
|
||||
aggregate.setTotalLines(safePositive(file.getTotalLines()));
|
||||
aggregate.setReceivedChunkCount(0);
|
||||
aggregate.setProcessedLineCount(0);
|
||||
aggregate.setCompleted(false);
|
||||
aggregate.setInvalidBrands(new ArrayList<>());
|
||||
aggregate.setQueryFailedBrands(new ArrayList<>());
|
||||
return aggregate;
|
||||
}
|
||||
|
||||
private void ensureChunkConsistency(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
|
||||
if (aggregate.getChunkTotal() != null && file.getChunkTotal() != null && !aggregate.getChunkTotal().equals(file.getChunkTotal())) {
|
||||
throw new BusinessException("鍚屼竴鏂囦欢鐨?chunkTotal 涓嶄竴鑷? " + aggregate.getFileUrl());
|
||||
}
|
||||
if (aggregate.getChunkTotal() == null) {
|
||||
aggregate.setChunkTotal(file.getChunkTotal());
|
||||
}
|
||||
int incomingTotalLines = safePositive(file.getTotalLines());
|
||||
if (incomingTotalLines > 0) {
|
||||
aggregate.setTotalLines(Math.max(safePositive(aggregate.getTotalLines()), incomingTotalLines));
|
||||
}
|
||||
if ((aggregate.getOriginalFilename() == null || aggregate.getOriginalFilename().isBlank()) && file.getOriginalFilename() != null && !file.getOriginalFilename().isBlank()) {
|
||||
aggregate.setOriginalFilename(file.getOriginalFilename());
|
||||
}
|
||||
if ((aggregate.getRelativePath() == null || aggregate.getRelativePath().isBlank()) && file.getRelativePath() != null && !file.getRelativePath().isBlank()) {
|
||||
aggregate.setRelativePath(file.getRelativePath());
|
||||
}
|
||||
if ((aggregate.getMainSheetName() == null || aggregate.getMainSheetName().isBlank()) && file.getMainSheetName() != null && !file.getMainSheetName().isBlank()) {
|
||||
aggregate.setMainSheetName(file.getMainSheetName());
|
||||
}
|
||||
}
|
||||
|
||||
private void mergeAggregate(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
|
||||
int processed = safePositive(aggregate.getProcessedLineCount())
|
||||
+ sizeOf(file.getKeptRows())
|
||||
+ sizeOf(file.getInvalidBrands())
|
||||
+ sizeOf(file.getQueryFailedBrands());
|
||||
aggregate.setProcessedLineCount(processed);
|
||||
if (file.getInvalidBrands() != null && !file.getInvalidBrands().isEmpty()) {
|
||||
aggregate.getInvalidBrands().addAll(file.getInvalidBrands());
|
||||
}
|
||||
if (file.getQueryFailedBrands() != null && !file.getQueryFailedBrands().isEmpty()) {
|
||||
aggregate.getQueryFailedBrands().addAll(file.getQueryFailedBrands());
|
||||
}
|
||||
}
|
||||
|
||||
private int sizeOf(List<?> list) {
|
||||
return list == null ? 0 : list.size();
|
||||
}
|
||||
|
||||
private int safePositive(Integer value) {
|
||||
return value == null || value <= 0 ? 0 : value;
|
||||
}
|
||||
|
||||
private String digestChunk(BrandCrawlResultFileDto file) {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest(objectMapper.writeValueAsString(file).getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("璁$畻鍝佺墝缁撴灉鍒嗙墖鎽樿澶辫触");
|
||||
}
|
||||
}
|
||||
|
||||
private void refreshFileKeys(Long taskId, String fileUrl) {
|
||||
refreshTaskKey(buildChunkReceiptKey(taskId, fileUrl));
|
||||
refreshTaskKey(buildChunkDigestKey(taskId, fileUrl));
|
||||
refreshTaskKey(buildFileAggregateKey(taskId));
|
||||
}
|
||||
|
||||
private void refreshTaskKey(String key) {
|
||||
stringRedisTemplate.expire(key, ttl());
|
||||
}
|
||||
|
||||
private Duration ttl() {
|
||||
return Duration.ofHours(brandProgressProperties.getTtlHours());
|
||||
}
|
||||
|
||||
private Path buildTaskDir(Long taskId) {
|
||||
return Path.of(System.getProperty("java.io.tmpdir"), "brand-task-cache", String.valueOf(taskId));
|
||||
}
|
||||
|
||||
private Path buildPayloadFile(Long taskId) {
|
||||
return buildTaskDir(taskId).resolve("parsed-payload.json");
|
||||
}
|
||||
|
||||
private void deleteTaskDirQuietly(Long taskId) {
|
||||
Path taskDir = buildTaskDir(taskId);
|
||||
if (!Files.exists(taskDir)) {
|
||||
return;
|
||||
}
|
||||
try (var walk = Files.walk(taskDir)) {
|
||||
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
|
||||
try {
|
||||
Files.deleteIfExists(path);
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
});
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
private String buildFileAggregateKey(Long taskId) {
|
||||
return "brand:task:file-aggregate:" + taskId;
|
||||
}
|
||||
|
||||
private String buildChunkReceiptKey(Long taskId, String fileUrl) {
|
||||
return "brand:task:file-chunks:" + taskId + ":" + hashFileUrl(fileUrl);
|
||||
}
|
||||
|
||||
private String buildChunkDigestKey(Long taskId, String fileUrl) {
|
||||
return "brand:task:file-chunk-digests:" + taskId + ":" + hashFileUrl(fileUrl);
|
||||
}
|
||||
|
||||
private String buildCompletedFilesKey(Long taskId) {
|
||||
return "brand:task:completed-files:" + taskId;
|
||||
}
|
||||
|
||||
private String buildFinalizeLockKey(Long taskId) {
|
||||
return "brand:task:finalizing:" + taskId;
|
||||
}
|
||||
|
||||
private String buildLegacyResultKey(Long taskId) {
|
||||
return "brand:task:result-chunks:" + taskId;
|
||||
}
|
||||
|
||||
private String buildChunkField(String fileUrl, Integer chunkIndex) {
|
||||
return fileUrl + "#" + chunkIndex;
|
||||
private String normalizeFileUrl(String fileUrl) {
|
||||
return fileUrl == null ? "" : fileUrl.trim();
|
||||
}
|
||||
|
||||
private String buildPayloadKey(Long taskId) {
|
||||
return "brand:task:parsed-payload:" + taskId;
|
||||
}
|
||||
|
||||
public long getHeartbeatTimeoutMinutes() {
|
||||
return brandProgressProperties.getHeartbeatTimeoutMinutes();
|
||||
private String hashFileUrl(String fileUrl) {
|
||||
String normalized = normalizeFileUrl(fileUrl);
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest(normalized.getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("鐢熸垚鏂囦欢缂撳瓨閿け璐?");
|
||||
}
|
||||
}
|
||||
|
||||
private String normalizePhase(String phase) {
|
||||
@@ -179,4 +432,7 @@ public class BrandTaskProgressCacheService {
|
||||
private String blankToEmpty(String value) {
|
||||
return value == null ? "" : value.trim();
|
||||
}
|
||||
|
||||
public record ChunkStoreResult(boolean stored, boolean newlyCompletedFile, int finishedFiles, BrandFileAggregateCacheDto aggregate) {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,11 +7,13 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.BrandProgressProperties;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
|
||||
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultRequest;
|
||||
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
|
||||
import com.nanri.aiimage.modules.brand.model.dto.BrandInvalidBrandDto;
|
||||
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
|
||||
import com.nanri.aiimage.modules.brand.model.dto.BrandQueryFailedDto;
|
||||
@@ -30,6 +32,7 @@ import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskItemVo;
|
||||
import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskListVo;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
@@ -47,6 +50,7 @@ import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
@@ -63,9 +67,11 @@ import java.util.zip.ZipOutputStream;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class BrandTaskService {
|
||||
|
||||
private static final DateTimeFormatter DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
|
||||
private static final Duration STALE_CHECK_LOCK_TTL = Duration.ofMinutes(10);
|
||||
private static final String STATUS_PENDING = "pending";
|
||||
private static final String STATUS_RUNNING = "running";
|
||||
private static final String STATUS_SUCCESS = "success";
|
||||
@@ -77,6 +83,7 @@ public class BrandTaskService {
|
||||
private final StorageProperties storageProperties;
|
||||
private final BrandProgressProperties brandProgressProperties;
|
||||
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public BrandTaskCreateVo createTask(Long userId, BrandTaskCreateRequest request) {
|
||||
@@ -205,18 +212,19 @@ public class BrandTaskService {
|
||||
}
|
||||
|
||||
public void submitCrawlResult(Long taskId, BrandCrawlResultRequest request) {
|
||||
BrandCrawlTaskEntity task = requireTask(taskId);
|
||||
if (STATUS_CANCELLED.equalsIgnoreCase(blankToDefault(task.getStatus(), STATUS_PENDING))) {
|
||||
throw new BusinessException("任务已取消");
|
||||
}
|
||||
long startedAt = System.currentTimeMillis();
|
||||
BrandCrawlTaskEntity task = requireActiveTask(taskId);
|
||||
List<BrandSourceFileDto> sourceFiles = parseSourceFiles(task.getFilePaths());
|
||||
if (sourceFiles.isEmpty()) {
|
||||
throw new BusinessException("任务没有源文件");
|
||||
}
|
||||
|
||||
Map<String, BrandSourceFileDto> sourceByUrl = new LinkedHashMap<>();
|
||||
for (BrandSourceFileDto file : sourceFiles) {
|
||||
Map<String, Integer> sourceIndexByUrl = new LinkedHashMap<>();
|
||||
for (int i = 0; i < sourceFiles.size(); i++) {
|
||||
BrandSourceFileDto file = sourceFiles.get(i);
|
||||
sourceByUrl.put(file.getFileUrl(), file);
|
||||
sourceIndexByUrl.put(file.getFileUrl(), i + 1);
|
||||
}
|
||||
List<BrandParsedFileCacheDto> cachedFiles = brandTaskProgressCacheService.getParsedPayload(taskId,
|
||||
new TypeReference<List<BrandParsedFileCacheDto>>() {
|
||||
@@ -236,82 +244,67 @@ public class BrandTaskService {
|
||||
ensureNoDuplicateResultFiles(resultFiles);
|
||||
|
||||
int totalCount = sourceFiles.size();
|
||||
int started = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
|
||||
.set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
||||
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, null));
|
||||
if (started == 0) {
|
||||
throw new BusinessException("任务已取消");
|
||||
markTaskRunning(taskId, totalCount);
|
||||
|
||||
log.info("[brand-submit] taskId={} chunks={} files={} thread={}",
|
||||
taskId,
|
||||
resultFiles.size(),
|
||||
totalCount,
|
||||
Thread.currentThread().getName());
|
||||
|
||||
int finishedCount = brandTaskProgressCacheService.countCompletedFiles(taskId);
|
||||
for (BrandCrawlResultFileDto resultFile : resultFiles) {
|
||||
String fileUrl = blankToNull(resultFile.getFileUrl());
|
||||
if (fileUrl == null) {
|
||||
throw new BusinessException("fileUrl 不能为空");
|
||||
}
|
||||
BrandSourceFileDto sourceFile = sourceByUrl.get(fileUrl);
|
||||
if (sourceFile == null) {
|
||||
throw new BusinessException("存在未知 fileUrl: " + fileUrl);
|
||||
}
|
||||
BrandParsedFileCacheDto cachedFile = cachedByUrl.get(fileUrl);
|
||||
if (cachedFile == null) {
|
||||
throw new BusinessException("缺少原始缓存数据: " + fileUrl);
|
||||
}
|
||||
int totalLines = cachedFile.getRows() == null ? 0 : cachedFile.getRows().size();
|
||||
if (resultFile.getTotalLines() == null || resultFile.getTotalLines() <= 0) {
|
||||
resultFile.setTotalLines(totalLines);
|
||||
} else if (totalLines > 0 && resultFile.getTotalLines() > totalLines) {
|
||||
throw new BusinessException("totalLines 超过原始文件行数: " + fileUrl);
|
||||
}
|
||||
BrandTaskProgressCacheService.ChunkStoreResult storeResult = brandTaskProgressCacheService.storeChunk(taskId, resultFile);
|
||||
finishedCount = Math.max(finishedCount, storeResult.finishedFiles());
|
||||
log.info("[brand-submit] taskId={} fileUrl={} chunk={}/{} stored={} newlyCompletedFile={} finishedFiles={}",
|
||||
taskId,
|
||||
fileUrl,
|
||||
resultFile.getChunkIndex(),
|
||||
resultFile.getChunkTotal(),
|
||||
storeResult.stored(),
|
||||
storeResult.newlyCompletedFile(),
|
||||
finishedCount);
|
||||
BrandFileAggregateCacheDto aggregate = storeResult.aggregate();
|
||||
String fileName = blankToDefault(sourceFile.getOriginalFilename(), sourceFile.getFileUrl());
|
||||
int fileIndex = sourceIndexByUrl.getOrDefault(fileUrl, 0);
|
||||
int currentLine = aggregate == null ? 0 : defaultInteger(aggregate.getProcessedLineCount());
|
||||
int currentTotalLines = aggregate == null ? 0 : Math.max(defaultInteger(aggregate.getTotalLines()), currentLine);
|
||||
brandTaskProgressCacheService.saveProgressFromResult(taskId,
|
||||
fileUrl,
|
||||
fileIndex,
|
||||
totalCount,
|
||||
fileName,
|
||||
currentLine,
|
||||
currentTotalLines,
|
||||
finishedCount);
|
||||
updateTaskProgress(taskId, finishedCount, totalCount);
|
||||
}
|
||||
|
||||
brandTaskProgressCacheService.mergeResultChunks(taskId, resultFiles);
|
||||
Map<String, List<BrandCrawlResultFileDto>> groupedResults = brandTaskProgressCacheService.groupResultChunksByFile(taskId);
|
||||
int finishedCount = countCompletedFiles(groupedResults);
|
||||
updateResultDrivenProgress(taskId, sourceFiles, sourceByUrl, groupedResults, finishedCount, totalCount);
|
||||
updateTaskProgress(taskId, finishedCount, totalCount);
|
||||
|
||||
if (finishedCount < totalCount) {
|
||||
return;
|
||||
}
|
||||
|
||||
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_ASSEMBLING, finishedCount, totalCount);
|
||||
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "brand-result", String.valueOf(taskId)));
|
||||
List<OutputEntry> outputEntries = new ArrayList<>();
|
||||
|
||||
try {
|
||||
for (List<BrandCrawlResultFileDto> fileChunks : groupedResults.values()) {
|
||||
BrandCrawlResultFileDto resultFile = mergeChunks(fileChunks);
|
||||
BrandSourceFileDto sourceFile = sourceByUrl.get(resultFile.getFileUrl());
|
||||
if (sourceFile == null) {
|
||||
throw new BusinessException("存在未知 fileUrl: " + resultFile.getFileUrl());
|
||||
}
|
||||
BrandParsedFileCacheDto cachedFile = cachedByUrl.get(resultFile.getFileUrl());
|
||||
if (cachedFile == null) {
|
||||
throw new BusinessException("缺少原始缓存数据: " + resultFile.getFileUrl());
|
||||
}
|
||||
File sourceLocalFile = resolveSourceFile(sourceFile);
|
||||
String originalFilename = resolveOriginalFilename(sourceFile, sourceLocalFile);
|
||||
File outputFile = buildNamedOutputFile(outputDir, buildResultFilename(originalFilename));
|
||||
writeBrandWorkbook(outputFile, request.getStrategy(), cachedFile, resultFile);
|
||||
outputEntries.add(new OutputEntry(
|
||||
sourceLocalFile,
|
||||
originalFilename,
|
||||
outputFile,
|
||||
originalFilename));
|
||||
}
|
||||
|
||||
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_UPLOADING, finishedCount, totalCount);
|
||||
Map<String, Object> resultPaths = buildAndUploadResult(taskId, outputEntries);
|
||||
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
|
||||
.set(BrandCrawlTaskEntity::getStatus, STATUS_SUCCESS)
|
||||
.set(BrandCrawlTaskEntity::getResultPaths, JSONUtil.toJsonStr(resultPaths))
|
||||
.set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount)
|
||||
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, null));
|
||||
if (updated == 0) {
|
||||
throw new BusinessException("任务已取消");
|
||||
}
|
||||
brandTaskProgressCacheService.delete(taskId);
|
||||
} catch (Exception ex) {
|
||||
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
|
||||
.set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount)
|
||||
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage()));
|
||||
brandTaskProgressCacheService.markFailed(taskId, ex.getMessage());
|
||||
if (ex instanceof BusinessException businessException) {
|
||||
throw businessException;
|
||||
}
|
||||
throw new BusinessException(ex.getMessage());
|
||||
}
|
||||
tryFinalizeTask(taskId, request.getStrategy(), sourceFiles, cachedByUrl, finishedCount, totalCount);
|
||||
log.info("[brand-submit] taskId={} finishedFiles={}/{} elapsedMs={} thread={}",
|
||||
taskId,
|
||||
finishedCount,
|
||||
totalCount,
|
||||
System.currentTimeMillis() - startedAt,
|
||||
Thread.currentThread().getName());
|
||||
}
|
||||
|
||||
public void cancelTask(Long taskId) {
|
||||
@@ -334,6 +327,7 @@ public class BrandTaskService {
|
||||
if (deleted == 0) {
|
||||
throw new BusinessException("任务不存在或正在执行中无法删除");
|
||||
}
|
||||
brandTaskProgressCacheService.delete(taskId);
|
||||
}
|
||||
|
||||
public String resolveDownloadUrl(Long taskId) {
|
||||
@@ -498,6 +492,139 @@ public class BrandTaskService {
|
||||
return Objects.equals(taskType, 1) ? 1 : 2;
|
||||
}
|
||||
|
||||
private void markTaskRunning(Long taskId, int totalCount) {
|
||||
int started = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
|
||||
.set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
||||
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, null));
|
||||
if (started == 0) {
|
||||
throw new BusinessException("任务已取消");
|
||||
}
|
||||
}
|
||||
|
||||
private BrandCrawlTaskEntity requireActiveTask(Long taskId) {
|
||||
BrandCrawlTaskEntity task = requireTask(taskId);
|
||||
String status = blankToDefault(task.getStatus(), STATUS_PENDING);
|
||||
if (STATUS_CANCELLED.equalsIgnoreCase(status)) {
|
||||
throw new BusinessException("任务已取消");
|
||||
}
|
||||
if (STATUS_SUCCESS.equalsIgnoreCase(status) || STATUS_FAILED.equalsIgnoreCase(status)) {
|
||||
throw new BusinessException("任务已结束,不能继续提交结果");
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
private void tryFinalizeTask(Long taskId,
|
||||
String strategy,
|
||||
List<BrandSourceFileDto> sourceFiles,
|
||||
Map<String, BrandParsedFileCacheDto> cachedByUrl,
|
||||
int finishedCount,
|
||||
int totalCount) {
|
||||
if (finishedCount < totalCount) {
|
||||
return;
|
||||
}
|
||||
log.info("[brand-finalize] taskId={} all files completed, trying finalize lock", taskId);
|
||||
if (!brandTaskProgressCacheService.acquireFinalizeLock(taskId)) {
|
||||
log.info("[brand-finalize] taskId={} finalize lock busy, skip", taskId);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
finalizeTask(taskId, strategy, sourceFiles, cachedByUrl, totalCount);
|
||||
} finally {
|
||||
brandTaskProgressCacheService.releaseFinalizeLock(taskId);
|
||||
}
|
||||
}
|
||||
|
||||
private void finalizeTask(Long taskId,
|
||||
String strategy,
|
||||
List<BrandSourceFileDto> sourceFiles,
|
||||
Map<String, BrandParsedFileCacheDto> cachedByUrl,
|
||||
int totalCount) {
|
||||
long startedAt = System.currentTimeMillis();
|
||||
BrandCrawlTaskEntity freshTask = requireTask(taskId);
|
||||
String freshStatus = blankToDefault(freshTask.getStatus(), STATUS_PENDING);
|
||||
if (STATUS_CANCELLED.equalsIgnoreCase(freshStatus)) {
|
||||
throw new BusinessException("任务已取消");
|
||||
}
|
||||
if (STATUS_SUCCESS.equalsIgnoreCase(freshStatus)) {
|
||||
return;
|
||||
}
|
||||
if (STATUS_FAILED.equalsIgnoreCase(freshStatus)) {
|
||||
throw new BusinessException("任务已结束,不能继续组装结果");
|
||||
}
|
||||
Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskProgressCacheService.getAllFileAggregates(taskId);
|
||||
log.info("[brand-finalize] taskId={} aggregateFiles={} expectedFiles={} thread={}",
|
||||
taskId,
|
||||
aggregates.size(),
|
||||
totalCount,
|
||||
Thread.currentThread().getName());
|
||||
if (aggregates.size() < totalCount) {
|
||||
return;
|
||||
}
|
||||
for (BrandSourceFileDto sourceFile : sourceFiles) {
|
||||
BrandFileAggregateCacheDto aggregate = aggregates.get(sourceFile.getFileUrl());
|
||||
if (aggregate == null || !Boolean.TRUE.equals(aggregate.getCompleted())) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_ASSEMBLING, totalCount, totalCount);
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "brand-result", String.valueOf(taskId)));
|
||||
List<OutputEntry> outputEntries = new ArrayList<>();
|
||||
try {
|
||||
for (BrandSourceFileDto sourceFile : sourceFiles) {
|
||||
BrandFileAggregateCacheDto aggregate = aggregates.get(sourceFile.getFileUrl());
|
||||
BrandParsedFileCacheDto cachedFile = cachedByUrl.get(sourceFile.getFileUrl());
|
||||
if (cachedFile == null) {
|
||||
throw new BusinessException("缺少原始缓存数据: " + sourceFile.getFileUrl());
|
||||
}
|
||||
File sourceLocalFile = resolveSourceFile(sourceFile);
|
||||
String originalFilename = resolveOriginalFilename(sourceFile, sourceLocalFile);
|
||||
File outputFile = buildNamedOutputFile(outputDir, buildResultFilename(originalFilename));
|
||||
writeBrandWorkbook(outputFile, strategy, cachedFile, aggregate);
|
||||
outputEntries.add(new OutputEntry(sourceLocalFile, originalFilename, outputFile, originalFilename));
|
||||
}
|
||||
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_UPLOADING, totalCount, totalCount);
|
||||
Map<String, Object> resultPaths = buildAndUploadResult(taskId, outputEntries);
|
||||
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
|
||||
.set(BrandCrawlTaskEntity::getStatus, STATUS_SUCCESS)
|
||||
.set(BrandCrawlTaskEntity::getResultPaths, JSONUtil.toJsonStr(resultPaths))
|
||||
.set(BrandCrawlTaskEntity::getProgressCurrent, totalCount)
|
||||
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, null));
|
||||
if (updated == 0) {
|
||||
throw new BusinessException("任务已取消");
|
||||
}
|
||||
for (BrandSourceFileDto sourceFile : sourceFiles) {
|
||||
brandTaskProgressCacheService.deleteFileState(taskId, sourceFile.getFileUrl());
|
||||
}
|
||||
brandTaskProgressCacheService.delete(taskId);
|
||||
log.info("[brand-finalize] taskId={} finalized success files={} elapsedMs={}",
|
||||
taskId,
|
||||
totalCount,
|
||||
System.currentTimeMillis() - startedAt);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[brand-finalize] taskId={} finalize failed msg={}", taskId, ex.getMessage());
|
||||
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
|
||||
.set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED)
|
||||
.set(BrandCrawlTaskEntity::getProgressCurrent, brandTaskProgressCacheService.countCompletedFiles(taskId))
|
||||
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage()));
|
||||
brandTaskProgressCacheService.markFailed(taskId, ex.getMessage());
|
||||
if (ex instanceof BusinessException businessException) {
|
||||
throw businessException;
|
||||
}
|
||||
throw new BusinessException(ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureNoDuplicateResultFiles(List<BrandCrawlResultFileDto> resultFiles) {
|
||||
Set<String> seen = new LinkedHashSet<>();
|
||||
for (BrandCrawlResultFileDto resultFile : resultFiles) {
|
||||
@@ -505,8 +632,12 @@ public class BrandTaskService {
|
||||
if (fileUrl == null) {
|
||||
throw new BusinessException("fileUrl 不能为空");
|
||||
}
|
||||
if (!seen.add(fileUrl)) {
|
||||
throw new BusinessException("存在重复 fileUrl: " + fileUrl);
|
||||
if (resultFile.getChunkIndex() == null || resultFile.getChunkIndex() <= 0) {
|
||||
throw new BusinessException("chunkIndex 不合法");
|
||||
}
|
||||
String chunkKey = fileUrl + "#" + resultFile.getChunkIndex();
|
||||
if (!seen.add(chunkKey)) {
|
||||
throw new BusinessException("存在重复分片: " + chunkKey);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -522,81 +653,6 @@ public class BrandTaskService {
|
||||
.set(BrandCrawlTaskEntity::getErrorMessage, null));
|
||||
}
|
||||
|
||||
private void updateResultDrivenProgress(Long taskId,
|
||||
List<BrandSourceFileDto> sourceFiles,
|
||||
Map<String, BrandSourceFileDto> sourceByUrl,
|
||||
Map<String, List<BrandCrawlResultFileDto>> groupedResults,
|
||||
int finishedCount,
|
||||
int totalCount) {
|
||||
if (groupedResults.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String currentFileUrl = null;
|
||||
List<BrandCrawlResultFileDto> currentChunks = null;
|
||||
int currentFileIndex = 0;
|
||||
for (int i = sourceFiles.size() - 1; i >= 0; i--) {
|
||||
BrandSourceFileDto sourceFile = sourceFiles.get(i);
|
||||
List<BrandCrawlResultFileDto> chunks = groupedResults.get(sourceFile.getFileUrl());
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
currentFileUrl = sourceFile.getFileUrl();
|
||||
currentChunks = chunks;
|
||||
currentFileIndex = i + 1;
|
||||
break;
|
||||
}
|
||||
if (currentFileUrl == null || currentChunks == null) {
|
||||
return;
|
||||
}
|
||||
BrandSourceFileDto sourceFile = sourceByUrl.get(currentFileUrl);
|
||||
String fileName = sourceFile == null ? "" : blankToDefault(sourceFile.getOriginalFilename(), sourceFile.getFileUrl());
|
||||
int currentLine = countProcessedLines(currentChunks);
|
||||
int totalLines = Math.max(inferTotalLines(currentChunks), currentLine);
|
||||
brandTaskProgressCacheService.saveProgressFromResult(taskId,
|
||||
currentFileUrl,
|
||||
currentFileIndex,
|
||||
totalCount,
|
||||
fileName,
|
||||
currentLine,
|
||||
totalLines,
|
||||
finishedCount);
|
||||
}
|
||||
|
||||
private int countProcessedLines(List<BrandCrawlResultFileDto> chunks) {
|
||||
int currentLine = 0;
|
||||
for (BrandCrawlResultFileDto chunk : chunks) {
|
||||
if (chunk.getKeptRows() != null) {
|
||||
currentLine += chunk.getKeptRows().size();
|
||||
}
|
||||
if (chunk.getInvalidBrands() != null) {
|
||||
currentLine += chunk.getInvalidBrands().size();
|
||||
}
|
||||
if (chunk.getQueryFailedBrands() != null) {
|
||||
currentLine += chunk.getQueryFailedBrands().size();
|
||||
}
|
||||
}
|
||||
return currentLine;
|
||||
}
|
||||
|
||||
private int inferTotalLines(List<BrandCrawlResultFileDto> chunks) {
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
BrandCrawlResultFileDto last = chunks.get(chunks.size() - 1);
|
||||
if (last.getTotalLines() != null && last.getTotalLines() > 0) {
|
||||
return last.getTotalLines();
|
||||
}
|
||||
if (last.getChunkTotal() == null || last.getChunkTotal() <= 0) {
|
||||
return countProcessedLines(chunks);
|
||||
}
|
||||
int processed = countProcessedLines(chunks);
|
||||
if (Objects.equals(chunks.size(), last.getChunkTotal())) {
|
||||
return processed;
|
||||
}
|
||||
int averagePerChunk = Math.max(processed / Math.max(chunks.size(), 1), 1);
|
||||
return averagePerChunk * last.getChunkTotal();
|
||||
}
|
||||
|
||||
private String buildDesc(String strategy, List<BrandSourceFileDto> files) {
|
||||
String joined = files.stream()
|
||||
.map(file -> blankToDefault(file.getOriginalFilename(), file.getFileUrl()))
|
||||
@@ -769,7 +825,7 @@ public class BrandTaskService {
|
||||
private void writeBrandWorkbook(File outputFile,
|
||||
String strategy,
|
||||
BrandParsedFileCacheDto cachedFile,
|
||||
BrandCrawlResultFileDto resultFile) throws IOException {
|
||||
BrandFileAggregateCacheDto resultFile) throws IOException {
|
||||
String actualStrategy = normalizeStrategy(strategy);
|
||||
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
|
||||
String mainSheetName = blankToDefault(cachedFile.getSheetName(), "Sheet1");
|
||||
@@ -986,52 +1042,6 @@ public class BrandTaskService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
private int countCompletedFiles(Map<String, List<BrandCrawlResultFileDto>> groupedResults) {
|
||||
int count = 0;
|
||||
for (List<BrandCrawlResultFileDto> chunks : groupedResults.values()) {
|
||||
if (!chunks.isEmpty() && Objects.equals(chunks.size(), chunks.get(0).getChunkTotal())) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private BrandCrawlResultFileDto mergeChunks(List<BrandCrawlResultFileDto> chunks) {
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
throw new BusinessException("结果分片为空");
|
||||
}
|
||||
BrandCrawlResultFileDto first = chunks.get(0);
|
||||
if (!Objects.equals(chunks.size(), first.getChunkTotal())) {
|
||||
throw new BusinessException("文件结果分片未完成");
|
||||
}
|
||||
BrandCrawlResultFileDto merged = new BrandCrawlResultFileDto();
|
||||
merged.setFileUrl(first.getFileUrl());
|
||||
merged.setOriginalFilename(first.getOriginalFilename());
|
||||
merged.setRelativePath(first.getRelativePath());
|
||||
merged.setMainSheetName(first.getMainSheetName());
|
||||
merged.setChunkIndex(first.getChunkIndex());
|
||||
merged.setChunkTotal(first.getChunkTotal());
|
||||
merged.setTotalLines(first.getTotalLines());
|
||||
List<String> keptRows = new ArrayList<>();
|
||||
List<BrandInvalidBrandDto> invalidBrands = new ArrayList<>();
|
||||
List<String> queryFailedBrands = new ArrayList<>();
|
||||
for (BrandCrawlResultFileDto chunk : chunks) {
|
||||
if (chunk.getKeptRows() != null) {
|
||||
keptRows.addAll(chunk.getKeptRows());
|
||||
}
|
||||
if (chunk.getInvalidBrands() != null) {
|
||||
invalidBrands.addAll(chunk.getInvalidBrands());
|
||||
}
|
||||
if (chunk.getQueryFailedBrands() != null) {
|
||||
queryFailedBrands.addAll(chunk.getQueryFailedBrands());
|
||||
}
|
||||
}
|
||||
merged.setKeptRows(keptRows);
|
||||
merged.setInvalidBrands(invalidBrands);
|
||||
merged.setQueryFailedBrands(queryFailedBrands);
|
||||
return merged;
|
||||
}
|
||||
|
||||
private Integer parseInteger(Object value) {
|
||||
if (value == null) {
|
||||
return 0;
|
||||
@@ -1046,6 +1056,12 @@ public class BrandTaskService {
|
||||
@Transactional
|
||||
@org.springframework.scheduling.annotation.Scheduled(cron = "${aiimage.brand-progress.stale-check-cron:0 */2 * * * *}")
|
||||
public void failStaleRunningTasks() {
|
||||
DistributedJobLockService.LockHandle lockHandle = distributedJobLockService.tryLock("brand:stale-check", STALE_CHECK_LOCK_TTL);
|
||||
if (lockHandle == null) {
|
||||
log.info("[brand-stale-check] skip because another instance holds the distributed lock");
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(brandProgressProperties.getHeartbeatTimeoutMinutes());
|
||||
List<BrandCrawlTaskEntity> runningTasks = brandCrawlTaskMapper.selectList(new LambdaQueryWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
|
||||
@@ -1070,6 +1086,7 @@ public class BrandTaskService {
|
||||
continue;
|
||||
}
|
||||
failStaleRunningTask(task.getId(), "前端长时间无响应,任务已自动失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ public class ConvertRunController {
|
||||
}
|
||||
|
||||
@GetMapping("/results/{resultId}/download")
|
||||
@Operation(summary = "下载格式转换结果", description = "根据结果记录 ID 下载当前用户的转换结果文件。桌面端前端可继续通过 save_file_from_url 选择保存位置。")
|
||||
@Operation(summary = "下载格式转换结果", description = "根据结果记录 ID 下载当前用户的转换结果文件。桌面端前端可继续通过 save_file_from_url_new 选择保存位置。")
|
||||
@ApiResponses({
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "返回结果文件流"),
|
||||
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "结果文件不存在")
|
||||
|
||||
@@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.util.ExcelStreamReader;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.modules.convert.model.dto.ConvertRunRequest;
|
||||
import com.nanri.aiimage.modules.convert.model.dto.UploadedSourceFileDto;
|
||||
@@ -19,15 +20,10 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -242,14 +238,30 @@ public class ConvertRunService {
|
||||
|
||||
private List<GeneratedConvertFile> generateOutputFiles(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
|
||||
List<String> outputFilenames = resolveOutputFilenames(templateEntity);
|
||||
List<String> rows = buildTxtRows(inputFile, templateEntity);
|
||||
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
|
||||
|
||||
List<GeneratedConvertFile> generatedFiles = new ArrayList<>();
|
||||
Map<String, BufferedWriter> writers = new LinkedHashMap<>();
|
||||
for (String outputFilename : outputFilenames) {
|
||||
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
|
||||
FileUtil.writeLines(rows, outputFile, StandardCharsets.UTF_8);
|
||||
generatedFiles.add(new GeneratedConvertFile(outputFilename, outputFile));
|
||||
writers.put(outputFilename, Files.newBufferedWriter(outputFile.toPath(), StandardCharsets.UTF_8));
|
||||
}
|
||||
try {
|
||||
streamTxtRowsToOutputs(inputFile, templateEntity, writers);
|
||||
} finally {
|
||||
IOException closeException = null;
|
||||
for (BufferedWriter writer : writers.values()) {
|
||||
try {
|
||||
writer.close();
|
||||
} catch (IOException ex) {
|
||||
if (closeException == null) {
|
||||
closeException = ex;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (closeException != null) {
|
||||
throw closeException;
|
||||
}
|
||||
}
|
||||
return generatedFiles;
|
||||
}
|
||||
@@ -266,7 +278,9 @@ public class ConvertRunService {
|
||||
return List.of(outputFilename);
|
||||
}
|
||||
|
||||
private List<String> buildTxtRows(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
|
||||
private void streamTxtRowsToOutputs(File inputFile,
|
||||
ConvertTemplateEntity templateEntity,
|
||||
Map<String, BufferedWriter> writers) throws IOException {
|
||||
List<String> requiredColumns = readStringList(templateEntity.getRequiredSourceColumnsJson());
|
||||
List<String> headerColumns = readStringList(templateEntity.getHeaderColumnsJson());
|
||||
Map<String, String> fieldMapping = readStringMap(templateEntity.getFieldMappingJson());
|
||||
@@ -275,65 +289,60 @@ public class ConvertRunService {
|
||||
int blankHeaderRowsAfterSchema = templateEntity.getBlankHeaderRowsAfterSchema() == null
|
||||
? 0
|
||||
: templateEntity.getBlankHeaderRowsAfterSchema();
|
||||
long batchId = System.currentTimeMillis();
|
||||
int[] rowIndex = new int[]{1};
|
||||
boolean[] headerRead = new boolean[]{false};
|
||||
Map<String, Integer>[] headerMapHolder = new Map[]{null};
|
||||
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile);
|
||||
Workbook workbook = WorkbookFactory.create(fis)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
throw new BusinessException("Excel header row is empty.");
|
||||
}
|
||||
|
||||
Map<String, Integer> headerMap = new LinkedHashMap<>();
|
||||
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
|
||||
String value = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i)));
|
||||
if (!value.isBlank() && !headerMap.containsKey(value)) {
|
||||
headerMap.put(value, i);
|
||||
ExcelStreamReader.readFirstSheet(inputFile, new ExcelStreamReader.SheetRowHandler() {
|
||||
@Override
|
||||
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) throws Exception {
|
||||
Map<String, Integer> resolvedHeaderMap = new LinkedHashMap<>();
|
||||
for (Map.Entry<Integer, String> entry : headerMap.entrySet()) {
|
||||
String value = normalizeCellText(entry.getValue());
|
||||
if (!value.isBlank() && !resolvedHeaderMap.containsKey(value)) {
|
||||
resolvedHeaderMap.put(value, entry.getKey());
|
||||
}
|
||||
}
|
||||
List<String> missing = requiredColumns.stream()
|
||||
.filter(column -> !resolvedHeaderMap.containsKey(column))
|
||||
.toList();
|
||||
if (!missing.isEmpty()) {
|
||||
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
|
||||
}
|
||||
headerRead[0] = true;
|
||||
headerMapHolder[0] = resolvedHeaderMap;
|
||||
for (String line : preambleLines) {
|
||||
writeLineToAll(writers, line);
|
||||
}
|
||||
writeLineToAll(writers, String.join("\t", headerColumns));
|
||||
for (int i = 0; i < blankHeaderRowsAfterSchema; i++) {
|
||||
writeLineToAll(writers, "\t".repeat(Math.max(0, headerColumns.size() - 1)));
|
||||
}
|
||||
}
|
||||
|
||||
List<String> missing = requiredColumns.stream()
|
||||
.filter(column -> !headerMap.containsKey(column))
|
||||
.toList();
|
||||
if (!missing.isEmpty()) {
|
||||
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
|
||||
}
|
||||
|
||||
List<String> rows = new ArrayList<>();
|
||||
rows.addAll(preambleLines);
|
||||
rows.add(String.join("\t", headerColumns));
|
||||
for (int i = 0; i < blankHeaderRowsAfterSchema; i++) {
|
||||
rows.add("\t".repeat(Math.max(0, headerColumns.size() - 1)));
|
||||
}
|
||||
|
||||
long batchId = System.currentTimeMillis();
|
||||
int rowIndex = 1;
|
||||
int asinIndex = headerMap.getOrDefault("ASIN", -1);
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
continue;
|
||||
@Override
|
||||
public void onRow(String sheetName, Integer sheetNo, int excelRowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception {
|
||||
Map<String, Integer> resolvedHeaderMap = headerMapHolder[0];
|
||||
if (resolvedHeaderMap == null) {
|
||||
throw new BusinessException("Excel header row is empty.");
|
||||
}
|
||||
|
||||
String asin = asinIndex >= 0 ? normalizeCellText(formatter.formatCellValue(row.getCell(asinIndex))) : "";
|
||||
int asinIndex = resolvedHeaderMap.getOrDefault("ASIN", -1);
|
||||
String asin = asinIndex >= 0 ? normalizeCellText(rowMap.get(asinIndex)) : "";
|
||||
if (asin.isBlank()) {
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
|
||||
List<String> lineValues = new ArrayList<>();
|
||||
for (String column : headerColumns) {
|
||||
if ("sku".equals(column)) {
|
||||
lineValues.add(batchId + "-" + rowIndex);
|
||||
lineValues.add(batchId + "-" + rowIndex[0]);
|
||||
continue;
|
||||
}
|
||||
if (fieldMapping.containsKey(column)) {
|
||||
String sourceColumn = fieldMapping.get(column);
|
||||
Integer sourceIndex = headerMap.get(sourceColumn);
|
||||
String value = sourceIndex == null
|
||||
? ""
|
||||
: normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
|
||||
lineValues.add(value);
|
||||
Integer sourceIndex = resolvedHeaderMap.get(sourceColumn);
|
||||
lineValues.add(sourceIndex == null ? "" : normalizeCellText(rowMap.get(sourceIndex)));
|
||||
continue;
|
||||
}
|
||||
if (defaults.containsKey(column)) {
|
||||
@@ -343,10 +352,19 @@ public class ConvertRunService {
|
||||
lineValues.add("");
|
||||
}
|
||||
|
||||
rows.add(String.join("\t", lineValues));
|
||||
rowIndex++;
|
||||
writeLineToAll(writers, String.join("\t", lineValues));
|
||||
rowIndex[0]++;
|
||||
}
|
||||
return rows;
|
||||
});
|
||||
if (!headerRead[0]) {
|
||||
throw new BusinessException("Excel header row is empty.");
|
||||
}
|
||||
}
|
||||
|
||||
private void writeLineToAll(Map<String, BufferedWriter> writers, String line) throws IOException {
|
||||
for (BufferedWriter writer : writers.values()) {
|
||||
writer.write(line);
|
||||
writer.newLine();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package com.nanri.aiimage.modules.debug.controller;
|
||||
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestHeader;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/debug")
|
||||
public class DebugRequestInfoController {
|
||||
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
|
||||
public DebugRequestInfoController(InstanceMetadata instanceMetadata) {
|
||||
this.instanceMetadata = instanceMetadata;
|
||||
}
|
||||
|
||||
@GetMapping("/request-info")
|
||||
public Map<String, Object> requestInfo(
|
||||
HttpServletRequest request,
|
||||
@RequestHeader(value = "X-Forwarded-For", required = false) String forwardedFor,
|
||||
@RequestHeader(value = "X-Real-IP", required = false) String realIp,
|
||||
@RequestHeader(value = "X-Forwarded-Proto", required = false) String forwardedProto,
|
||||
@RequestHeader(value = "X-Forwarded-Host", required = false) String forwardedHost,
|
||||
@RequestHeader(value = "X-Forwarded-Port", required = false) String forwardedPort,
|
||||
@RequestHeader(value = "X-Request-Id", required = false) String requestId,
|
||||
@RequestHeader(value = "X-Amzn-Trace-Id", required = false) String amazonTraceId,
|
||||
@RequestHeader(value = "Traceparent", required = false) String traceparent
|
||||
) {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("instanceId", instanceMetadata.getInstanceId());
|
||||
result.put("hostname", instanceMetadata.getHostname());
|
||||
result.put("serverName", request.getServerName());
|
||||
result.put("serverPort", request.getServerPort());
|
||||
result.put("method", request.getMethod());
|
||||
result.put("uri", request.getRequestURI());
|
||||
result.put("remoteAddr", request.getRemoteAddr());
|
||||
result.put("xForwardedFor", forwardedFor);
|
||||
result.put("xRealIp", realIp);
|
||||
result.put("xForwardedProto", forwardedProto);
|
||||
result.put("xForwardedHost", forwardedHost);
|
||||
result.put("xForwardedPort", forwardedPort);
|
||||
result.put("xRequestId", requestId);
|
||||
result.put("xAmznTraceId", amazonTraceId);
|
||||
result.put("traceparent", traceparent);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,7 @@ import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -232,7 +232,7 @@ public class DedupeRunService {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(inputFile);
|
||||
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
|
||||
XSSFWorkbook outputWorkbook = new XSSFWorkbook()) {
|
||||
SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row headerRow = sheet.getRow(0);
|
||||
if (headerRow == null) {
|
||||
@@ -275,7 +275,6 @@ public class DedupeRunService {
|
||||
// The preliminary global scan for mainIdHasSubIdMap was removed.
|
||||
// We now determine subset existence contextually (per group) during the main loop.
|
||||
}
|
||||
List<Row> candidateRows = new ArrayList<>();
|
||||
Set<String> candidateAsinValues = new HashSet<>();
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
@@ -288,7 +287,6 @@ public class DedupeRunService {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
candidateRows.add(row);
|
||||
if (asinColumnIndex != null) {
|
||||
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
|
||||
if (!asinValue.isBlank()) {
|
||||
@@ -300,7 +298,17 @@ public class DedupeRunService {
|
||||
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
|
||||
Set<String> writtenAsinValues = new HashSet<>();
|
||||
int outputRowIndex = 1;
|
||||
for (Row row : candidateRows) {
|
||||
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||
Row row = sheet.getRow(rowNum);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
if (idColumnIndex != null) {
|
||||
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
|
||||
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (asinColumnIndex != null) {
|
||||
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
|
||||
if (!asinValue.isBlank()) {
|
||||
@@ -324,6 +332,7 @@ public class DedupeRunService {
|
||||
try (FileOutputStream fos = new FileOutputStream(outputFile)) {
|
||||
outputWorkbook.write(fos);
|
||||
}
|
||||
outputWorkbook.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,9 @@ import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
@@ -114,9 +116,9 @@ public class DedupeTotalDataService {
|
||||
importProgressMap.put(importId, progress);
|
||||
|
||||
try {
|
||||
byte[] fileBytes = file.getBytes();
|
||||
File tempFile = saveMultipartToTempFile(file);
|
||||
String filename = file.getOriginalFilename();
|
||||
Thread.ofVirtual().start(() -> runImportTask(importId, fileBytes, filename));
|
||||
Thread.ofVirtual().start(() -> runImportTask(importId, tempFile, filename));
|
||||
} catch (Exception e) {
|
||||
importProgressMap.remove(importId);
|
||||
throw new BusinessException("读取上传文件失败");
|
||||
@@ -150,9 +152,9 @@ public class DedupeTotalDataService {
|
||||
deleteImportProgressMap.put(importId, progress);
|
||||
|
||||
try {
|
||||
byte[] fileBytes = file.getBytes();
|
||||
File tempFile = saveMultipartToTempFile(file);
|
||||
String filename = file.getOriginalFilename();
|
||||
Thread.ofVirtual().start(() -> runDeleteImportTask(importId, fileBytes, filename));
|
||||
Thread.ofVirtual().start(() -> runDeleteImportTask(importId, tempFile, filename));
|
||||
} catch (Exception e) {
|
||||
deleteImportProgressMap.remove(importId);
|
||||
throw new BusinessException("读取上传文件失败");
|
||||
@@ -211,6 +213,74 @@ public class DedupeTotalDataService {
|
||||
}
|
||||
}
|
||||
|
||||
private void runDeleteImportTask(String importId, File tempFile, String filename) {
|
||||
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
|
||||
if (progress == null) {
|
||||
return;
|
||||
}
|
||||
progress.setStatus("running");
|
||||
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
|
||||
DedupeTotalDataImportVo result = deleteFromExcelInternal(inputStream, filename, progress);
|
||||
progress.setTotalRows(result.getTotalRows());
|
||||
progress.setAsinCount(result.getAsinCount());
|
||||
progress.setInsertedCount(result.getInsertedCount());
|
||||
progress.setSkippedCount(result.getSkippedCount());
|
||||
progress.setProcessedRows(result.getTotalRows());
|
||||
progress.setStatus("success");
|
||||
} catch (Exception e) {
|
||||
progress.setStatus("failed");
|
||||
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "鍒犻櫎 Excel 鍖归厤鏁版嵁澶辫触");
|
||||
} finally {
|
||||
deleteQuietly(tempFile);
|
||||
}
|
||||
}
|
||||
|
||||
private void runImportTask(String importId, File tempFile, String filename) {
|
||||
DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId);
|
||||
if (progress == null) {
|
||||
return;
|
||||
}
|
||||
progress.setStatus("running");
|
||||
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
|
||||
DedupeTotalDataImportVo result = importFromExcelInternal(inputStream, filename, progress);
|
||||
progress.setTotalRows(result.getTotalRows());
|
||||
progress.setAsinCount(result.getAsinCount());
|
||||
progress.setInsertedCount(result.getInsertedCount());
|
||||
progress.setSkippedCount(result.getSkippedCount());
|
||||
progress.setProcessedRows(result.getTotalRows());
|
||||
progress.setStatus("success");
|
||||
} catch (Exception e) {
|
||||
progress.setStatus("failed");
|
||||
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "瀵煎叆 Excel 澶辫触");
|
||||
} finally {
|
||||
deleteQuietly(tempFile);
|
||||
}
|
||||
}
|
||||
|
||||
private File saveMultipartToTempFile(MultipartFile file) throws Exception {
|
||||
String suffix = ".xlsx";
|
||||
String name = file.getOriginalFilename();
|
||||
if (name != null) {
|
||||
int idx = name.lastIndexOf('.');
|
||||
if (idx >= 0) {
|
||||
suffix = name.substring(idx);
|
||||
}
|
||||
}
|
||||
File tempFile = File.createTempFile("dedupe-total-data-", suffix);
|
||||
file.transferTo(tempFile);
|
||||
return tempFile;
|
||||
}
|
||||
|
||||
private void deleteQuietly(File file) {
|
||||
if (file == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Files.deleteIfExists(file.toPath());
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public DedupeTotalDataImportVo importFromExcel(MultipartFile file) {
|
||||
if (file == null || file.isEmpty()) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.nanri.aiimage.modules.deletebrand.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandRunRequest;
|
||||
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandTaskBatchRequest;
|
||||
@@ -84,6 +85,12 @@ public class DeleteBrandRunController {
|
||||
return ApiResponse.success(deleteBrandRunService.getTaskDetails(request.getTaskIds()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量获取删除品牌任务进度摘要", description = "仅返回任务基础状态和行进度摘要,用于前端轮询降载。")
|
||||
public ApiResponse<DeleteBrandTaskBatchVo> getTaskProgressBatch(@Valid @RequestBody DeleteBrandTaskBatchRequest request) {
|
||||
return ApiResponse.success(deleteBrandRunService.getTaskProgress(request.getTaskIds()));
|
||||
}
|
||||
|
||||
@GetMapping("/tasks/{taskId}/deletion-status")
|
||||
@Operation(summary = "获取删除品牌任务删除状态", description = "供 Python/插件按 taskId 查询该删除品牌任务是否已被前端删空。")
|
||||
public ApiResponse<DeleteBrandTaskDeletionStatusVo> getTaskDeletionStatus(
|
||||
@@ -97,13 +104,22 @@ public class DeleteBrandRunController {
|
||||
@Operation(summary = "提交删除品牌处理结果", description = "插件/前端分片回传处理结果;后端累计分片并推进进度,全部完成后再执行最终组装与落库。")
|
||||
public ApiResponse<Void> submitResult(
|
||||
@PathVariable Long taskId,
|
||||
@Valid @RequestBody DeleteBrandSubmitResultRequest request) {
|
||||
deleteBrandRunService.submitResult(taskId, request);
|
||||
return ApiResponse.success(null);
|
||||
@Valid @RequestBody DeleteBrandSubmitResultRequest request,
|
||||
jakarta.servlet.http.HttpServletResponse response) {
|
||||
try {
|
||||
deleteBrandRunService.submitResult(taskId, request);
|
||||
return ApiResponse.success(null);
|
||||
} catch (BusinessException ex) {
|
||||
response.setStatus(HttpStatus.BAD_REQUEST.value());
|
||||
return ApiResponse.fail(ex.getMessage());
|
||||
} catch (Exception ex) {
|
||||
response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
|
||||
return ApiResponse.fail("服务异常:" + ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping("/tasks/{taskId}/download")
|
||||
@Operation(summary = "下载删除品牌任务结果", description = "返回任务结果文件流,便于桌面端通过 save_file_from_url 直接保存。")
|
||||
@Operation(summary = "下载删除品牌任务结果", description = "返回任务结果文件流,便于桌面端通过 save_file_from_url_new 直接保存。")
|
||||
public void download(@PathVariable Long taskId, jakarta.servlet.http.HttpServletResponse response) {
|
||||
String url = deleteBrandRunService.resolveTaskDownloadUrl(taskId);
|
||||
String filename = deleteBrandRunService.resolveTaskDownloadFilename(taskId);
|
||||
|
||||
@@ -12,6 +12,7 @@ public class DeleteBrandParsedFileCacheDto {
|
||||
private String fileKey;
|
||||
private String sourceFilename;
|
||||
private String shopName;
|
||||
private String companyName;
|
||||
private String shopId;
|
||||
private String platform;
|
||||
private String openStoreUrl;
|
||||
|
||||
@@ -21,6 +21,9 @@ public class DeleteBrandResultItemVo {
|
||||
@Schema(description = "按 Excel 文件名解析出的店铺名")
|
||||
private String shopName;
|
||||
|
||||
@Schema(description = "紫鸟公司名称")
|
||||
private String companyName;
|
||||
|
||||
@Schema(description = "是否匹配到紫鸟店铺")
|
||||
private boolean matched;
|
||||
|
||||
|
||||
@@ -127,12 +127,13 @@ public class DeleteBrandRunService {
|
||||
ZiniaoShopMatchResultVo matchResult = normalizedShopName.isBlank()
|
||||
? ziniaoShopSwitchService.emptyMatchResult()
|
||||
: storeMatchByShopName.computeIfAbsent(normalizedShopName,
|
||||
ignored -> ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName()));
|
||||
ignored -> ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName(), false));
|
||||
item.setMatchStatus(matchResult.getMatchStatus());
|
||||
item.setMatchMessage(matchResult.getMatchMessage());
|
||||
item.setMatched(matchResult.isMatched());
|
||||
if (matchResult.isMatched()) {
|
||||
item.setShopId(matchResult.getShopId());
|
||||
item.setCompanyName(matchResult.getCompanyName());
|
||||
item.setPlatform(matchResult.getPlatform());
|
||||
item.setOpenStoreUrl(matchResult.getOpenStoreUrl());
|
||||
}
|
||||
@@ -153,6 +154,7 @@ public class DeleteBrandRunService {
|
||||
cacheDto.setSourceFilename(sourceFile.getOriginalFilename());
|
||||
cacheDto.setShopName(item.getShopName());
|
||||
cacheDto.setShopId(item.getShopId());
|
||||
cacheDto.setCompanyName(item.getCompanyName());
|
||||
cacheDto.setPlatform(item.getPlatform());
|
||||
cacheDto.setOpenStoreUrl(item.getOpenStoreUrl());
|
||||
cacheDto.setTotalRows(parsed.totalRows());
|
||||
@@ -204,7 +206,7 @@ public class DeleteBrandRunService {
|
||||
|
||||
task.setSuccessFileCount(parsedSuccessCount);
|
||||
task.setFailedFileCount(parsedFailedCount);
|
||||
task.setResultJson(JSONUtil.toJsonStr(items));
|
||||
task.setResultJson(toCompactTaskResultJsonForDb(items));
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setSourceFileCount(processableFileCount);
|
||||
if (processableFileCount <= 0) {
|
||||
@@ -295,6 +297,7 @@ public class DeleteBrandRunService {
|
||||
}
|
||||
|
||||
item.setShopId(candidate.getShopId());
|
||||
item.setCompanyName(candidate.getCompanyName());
|
||||
item.setPlatform(candidate.getPlatform());
|
||||
item.setOpenStoreUrl(candidate.getOpenStoreUrl());
|
||||
|
||||
@@ -541,10 +544,87 @@ public class DeleteBrandRunService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
public com.nanri.aiimage.modules.deletebrand.model.vo.DeleteBrandTaskBatchVo getTaskProgress(java.util.List<Long> taskIds) {
|
||||
com.nanri.aiimage.modules.deletebrand.model.vo.DeleteBrandTaskBatchVo vo = new com.nanri.aiimage.modules.deletebrand.model.vo.DeleteBrandTaskBatchVo();
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
|
||||
java.util.List<Long> normalizedTaskIds = taskIds.stream()
|
||||
.filter(id -> id != null && id > 0)
|
||||
.distinct()
|
||||
.limit(50)
|
||||
.toList();
|
||||
if (normalizedTaskIds.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
|
||||
java.util.Map<Long, FileTaskEntity> cachedTasks = deleteBrandTaskCacheService.getTaskCacheBatch(normalizedTaskIds);
|
||||
java.util.List<Long> missingTaskIds = new java.util.ArrayList<>();
|
||||
for (Long taskId : normalizedTaskIds) {
|
||||
if (!cachedTasks.containsKey(taskId)) {
|
||||
missingTaskIds.add(taskId);
|
||||
}
|
||||
}
|
||||
|
||||
java.util.Map<Long, FileTaskEntity> taskById = new java.util.LinkedHashMap<>(cachedTasks);
|
||||
if (!missingTaskIds.isEmpty()) {
|
||||
java.util.List<FileTaskEntity> dbTasks = fileTaskMapper.selectBatchIds(missingTaskIds).stream()
|
||||
.filter(task -> task != null && MODULE_TYPE.equals(task.getModuleType()))
|
||||
.toList();
|
||||
for (FileTaskEntity dbTask : dbTasks) {
|
||||
taskById.put(dbTask.getId(), dbTask);
|
||||
if ("RUNNING".equals(dbTask.getStatus())) {
|
||||
deleteBrandTaskCacheService.saveTaskCache(dbTask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
java.util.Map<Long, java.util.Map<Object, Object>> progressByTaskId = deleteBrandTaskCacheService.getProgressBatch(normalizedTaskIds);
|
||||
|
||||
for (Long taskId : normalizedTaskIds) {
|
||||
FileTaskEntity task = taskById.get(taskId);
|
||||
if (task == null) {
|
||||
vo.getMissingTaskIds().add(taskId);
|
||||
continue;
|
||||
}
|
||||
vo.getItems().add(buildTaskProgressDetail(task, progressByTaskId.get(taskId)));
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
private DeleteBrandTaskDetailVo buildTaskDetail(FileTaskEntity task) {
|
||||
return buildTaskDetail(task, null);
|
||||
}
|
||||
|
||||
private DeleteBrandTaskDetailVo buildTaskProgressDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) {
|
||||
task = refreshIndexedMatchesIfNeeded(task);
|
||||
DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo();
|
||||
taskVo.setId(task.getId());
|
||||
taskVo.setTaskNo(task.getTaskNo());
|
||||
taskVo.setStatus(task.getStatus());
|
||||
taskVo.setSourceFileCount(task.getSourceFileCount());
|
||||
taskVo.setSuccessFileCount(task.getSuccessFileCount());
|
||||
taskVo.setFailedFileCount(task.getFailedFileCount());
|
||||
taskVo.setErrorMessage(task.getErrorMessage());
|
||||
taskVo.setCreatedAt(task.getCreatedAt());
|
||||
taskVo.setUpdatedAt(task.getUpdatedAt());
|
||||
taskVo.setFinishedAt(task.getFinishedAt());
|
||||
if ("SUCCESS".equals(task.getStatus())) {
|
||||
taskVo.setDownloadUrl(resolveTaskDownloadUrl(task.getId()));
|
||||
taskVo.setDownloadFilename(resolveTaskDownloadFilename(task.getId()));
|
||||
} else {
|
||||
taskVo.setDownloadUrl(null);
|
||||
taskVo.setDownloadFilename(null);
|
||||
}
|
||||
|
||||
DeleteBrandTaskDetailVo detail = new DeleteBrandTaskDetailVo();
|
||||
detail.setTask(taskVo);
|
||||
detail.setLine_progress(buildLineProgress(task.getId(), progress));
|
||||
detail.setFileProgress(java.util.List.of());
|
||||
return detail;
|
||||
}
|
||||
|
||||
private DeleteBrandTaskDetailVo buildTaskDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) {
|
||||
task = refreshIndexedMatchesIfNeeded(task);
|
||||
DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo();
|
||||
@@ -711,11 +791,12 @@ public class DeleteBrandRunService {
|
||||
unmatchedCount++;
|
||||
continue;
|
||||
}
|
||||
ZiniaoShopMatchResultVo refreshed = ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName());
|
||||
ZiniaoShopMatchResultVo refreshed = ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName(), false);
|
||||
if (refreshed != null) {
|
||||
String oldStatus = item.getMatchStatus();
|
||||
boolean oldMatched = item.isMatched();
|
||||
String oldShopId = item.getShopId();
|
||||
String oldCompanyName = item.getCompanyName();
|
||||
String oldPlatform = item.getPlatform();
|
||||
String oldOpenStoreUrl = item.getOpenStoreUrl();
|
||||
String oldMessage = item.getMatchMessage();
|
||||
@@ -724,12 +805,14 @@ public class DeleteBrandRunService {
|
||||
item.setMatchStatus(refreshed.getMatchStatus());
|
||||
item.setMatchMessage(refreshed.getMatchMessage());
|
||||
item.setShopId(refreshed.getShopId());
|
||||
item.setCompanyName(refreshed.getCompanyName());
|
||||
item.setPlatform(refreshed.getPlatform());
|
||||
item.setOpenStoreUrl(refreshed.getOpenStoreUrl());
|
||||
|
||||
if (!java.util.Objects.equals(oldStatus, item.getMatchStatus())
|
||||
|| oldMatched != item.isMatched()
|
||||
|| !java.util.Objects.equals(oldShopId, item.getShopId())
|
||||
|| !java.util.Objects.equals(oldCompanyName, item.getCompanyName())
|
||||
|| !java.util.Objects.equals(oldPlatform, item.getPlatform())
|
||||
|| !java.util.Objects.equals(oldOpenStoreUrl, item.getOpenStoreUrl())
|
||||
|| !java.util.Objects.equals(oldMessage, item.getMatchMessage())) {
|
||||
@@ -745,7 +828,7 @@ public class DeleteBrandRunService {
|
||||
if (!changed) {
|
||||
return task;
|
||||
}
|
||||
task.setResultJson(JSONUtil.toJsonStr(items));
|
||||
task.setResultJson(toCompactTaskResultJsonForDb(items));
|
||||
task.setSourceFileCount(matchedCount);
|
||||
task.setFailedFileCount(unmatchedCount);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
@@ -984,6 +1067,26 @@ public class DeleteBrandRunService {
|
||||
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = loadMergedChunks(taskId);
|
||||
int finishedFiles = countCompletedFiles(mergedByFile);
|
||||
|
||||
if (finishedFiles < expectedFiles) {
|
||||
Map<String, String> progress = new LinkedHashMap<>();
|
||||
progress.put("finished_files", String.valueOf(finishedFiles));
|
||||
// 定时补偿只负责“再试一次 finalize”,不能把缺片任务重新续命,
|
||||
// 否则 stale-check 永远看不到超时任务。
|
||||
if (!fromCompensation) {
|
||||
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
|
||||
}
|
||||
deleteBrandTaskCacheService.saveProgress(taskId, progress);
|
||||
|
||||
task.setSuccessFileCount(finishedFiles);
|
||||
if (!fromCompensation) {
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
}
|
||||
fileTaskMapper.updateById(task);
|
||||
|
||||
log.info("[DeleteBrand] tryFinalizeTask skipping -> finishedFiles ({}) < expectedFiles ({})", finishedFiles, expectedFiles);
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, String> progress = new LinkedHashMap<>();
|
||||
progress.put("finished_files", String.valueOf(finishedFiles));
|
||||
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
|
||||
@@ -995,11 +1098,6 @@ public class DeleteBrandRunService {
|
||||
task.setSuccessFileCount(finishedFiles);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
|
||||
if (finishedFiles < expectedFiles) {
|
||||
log.info("[DeleteBrand] tryFinalizeTask skipping -> finishedFiles ({}) < expectedFiles ({})", finishedFiles, expectedFiles);
|
||||
return;
|
||||
}
|
||||
|
||||
log.info("[DeleteBrand] tryFinalizeTask calling finalizeTask -> finishedFiles: {}, expectedFiles: {}", finishedFiles, expectedFiles);
|
||||
|
||||
@@ -1120,6 +1218,7 @@ public class DeleteBrandRunService {
|
||||
item.setFileKey(parsedFile.getFileKey());
|
||||
item.setSourceFilename(parsedFile.getSourceFilename());
|
||||
item.setShopName(parsedFile.getShopName());
|
||||
item.setCompanyName(parsedFile.getCompanyName());
|
||||
item.setOutputFilename(outputFile.getName());
|
||||
item.setDownloadUrl(ossStorageService.getPublicUrl(objectKey));
|
||||
item.setTotalRows(parsedFile.getTotalRows());
|
||||
@@ -1140,7 +1239,7 @@ public class DeleteBrandRunService {
|
||||
task.setStatus("SUCCESS");
|
||||
task.setSuccessFileCount(successCount);
|
||||
task.setErrorMessage(null);
|
||||
task.setResultJson(JSONUtil.toJsonStr(finalItems));
|
||||
task.setResultJson(toCompactTaskResultJsonForDb(finalItems));
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
task.setFinishedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
@@ -1351,6 +1450,33 @@ public class DeleteBrandRunService {
|
||||
return normalized.isBlank() ? defaultValue : normalized;
|
||||
}
|
||||
|
||||
/**
|
||||
* 落库用 result_json:去掉国家分组、预览行等大字段,保留匹配摘要与计数。
|
||||
* 索引常命中后单次任务 JSON 体积会陡增,易触发 max_allowed_packet / 更新失败,进而前端 unwrap 报「请求失败」。
|
||||
*/
|
||||
private String toCompactTaskResultJsonForDb(List<DeleteBrandResultItemVo> items) {
|
||||
if (items == null || items.isEmpty()) {
|
||||
return JSONUtil.toJsonStr(List.of());
|
||||
}
|
||||
try {
|
||||
List<DeleteBrandResultItemVo> clone = objectMapper.readValue(
|
||||
objectMapper.writeValueAsString(items),
|
||||
new TypeReference<List<DeleteBrandResultItemVo>>() {
|
||||
});
|
||||
for (DeleteBrandResultItemVo vo : clone) {
|
||||
if (vo == null) {
|
||||
continue;
|
||||
}
|
||||
vo.setCountries(new ArrayList<>());
|
||||
vo.setPreviewRows(new ArrayList<>());
|
||||
}
|
||||
return JSONUtil.toJsonStr(clone);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[delete-brand] compact task result json failed, msg={}", ex.getMessage());
|
||||
return JSONUtil.toJsonStr(items);
|
||||
}
|
||||
}
|
||||
|
||||
private record ParsedDeleteBrandFile(int totalRows,
|
||||
List<DeleteBrandCountryGroupVo> countries,
|
||||
List<DeleteBrandPreviewRowVo> previewRows) {
|
||||
|
||||
@@ -1,36 +1,103 @@
|
||||
package com.nanri.aiimage.modules.deletebrand.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.DeleteBrandProgressProperties;
|
||||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService;
|
||||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
|
||||
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService;
|
||||
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.FileTime;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class DeleteBrandStaleTaskService {
|
||||
|
||||
private static final String MODULE_TYPE = "DELETE_BRAND";
|
||||
private static final String MODULE_TYPE_DELETE_BRAND = "DELETE_BRAND";
|
||||
private static final String MODULE_TYPE_PRODUCT_RISK = "PRODUCT_RISK_RESOLVE";
|
||||
private static final String MODULE_TYPE_PRICE_TRACK = "PRICE_TRACK";
|
||||
private static final String MODULE_TYPE_SHOP_MATCH = "SHOP_MATCH";
|
||||
private static final Duration STALE_CHECK_LOCK_TTL = Duration.ofMinutes(10);
|
||||
private static final Duration FINALIZE_CHECK_LOCK_TTL = Duration.ofMinutes(10);
|
||||
private static final Duration TEMP_DIR_CLEANUP_LOCK_TTL = Duration.ofMinutes(10);
|
||||
private static final Duration TEMP_DIR_RETENTION = Duration.ofHours(48);
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
||||
private final DeleteBrandRunService deleteBrandRunService;
|
||||
private final ProductRiskTaskService productRiskTaskService;
|
||||
private final ProductRiskTaskCacheService productRiskTaskCacheService;
|
||||
private final PriceTrackTaskService priceTrackTaskService;
|
||||
private final PriceTrackTaskCacheService priceTrackTaskCacheService;
|
||||
private final ShopMatchTaskService shopMatchTaskService;
|
||||
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
|
||||
private final DeleteBrandProgressProperties deleteBrandProgressProperties;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
@Scheduled(cron = "${aiimage.delete-brand-progress.stale-check-cron:0 */2 * * * *}")
|
||||
public void failStaleRunningTasks() {
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
distributedJobLockService.tryLock("delete-brand:stale-check", STALE_CHECK_LOCK_TTL);
|
||||
if (lockHandle == null) {
|
||||
log.info("[stale-check] skip because another instance holds the distributed lock");
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
long startedAt = System.currentTimeMillis();
|
||||
log.info("[stale-check] scan started thread={}", Thread.currentThread().getName());
|
||||
failStaleDeleteBrandTasks();
|
||||
ProductRiskStaleCheckStats stats = failStaleProductRiskResolveTasks();
|
||||
PriceTrackStaleCheckStats priceTrackStats = failStalePriceTrackTasks();
|
||||
ShopMatchStaleCheckStats shopMatchStats = failStaleShopMatchTasks();
|
||||
log.info("[stale-check] product-risk summary scanned={} finalized={} failed={} skipped={} elapsedMs={} thread={}",
|
||||
stats.scannedTaskCount,
|
||||
stats.finalizedTaskCount,
|
||||
stats.failedTaskCount,
|
||||
stats.skippedTaskCount,
|
||||
System.currentTimeMillis() - startedAt,
|
||||
Thread.currentThread().getName());
|
||||
log.info("[stale-check] price-track summary scanned={} finalized={} failed={} skipped={} elapsedMs={} thread={}",
|
||||
priceTrackStats.scannedTaskCount,
|
||||
priceTrackStats.finalizedTaskCount,
|
||||
priceTrackStats.failedTaskCount,
|
||||
priceTrackStats.skippedTaskCount,
|
||||
System.currentTimeMillis() - startedAt,
|
||||
Thread.currentThread().getName());
|
||||
log.info("[stale-check] shop-match summary scanned={} finalized={} failed={} skipped={} elapsedMs={} thread={}",
|
||||
shopMatchStats.scannedTaskCount,
|
||||
shopMatchStats.finalizedTaskCount,
|
||||
shopMatchStats.failedTaskCount,
|
||||
shopMatchStats.skippedTaskCount,
|
||||
System.currentTimeMillis() - startedAt,
|
||||
Thread.currentThread().getName());
|
||||
}
|
||||
}
|
||||
|
||||
private void failStaleDeleteBrandTasks() {
|
||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(deleteBrandProgressProperties.getHeartbeatTimeoutMinutes());
|
||||
|
||||
// 先按 DB updatedAt 粗筛,避免全表扫描
|
||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||
.last("limit 200"));
|
||||
@@ -57,26 +124,21 @@ public class DeleteBrandStaleTaskService {
|
||||
}
|
||||
}
|
||||
|
||||
LocalDateTime lastActivityTime;
|
||||
// 只有当 has_progress 为 true 且超过 15 分钟没心跳,才认为是异常卡死
|
||||
if (lastHeartbeatAt > 0 && isActive) {
|
||||
// 有心跳,用最后心跳时间对比 15 分钟 (threshold 是 now - 15)
|
||||
lastActivityTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastHeartbeatAt), ZoneId.systemDefault());
|
||||
LocalDateTime lastActivityTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastHeartbeatAt), ZoneId.systemDefault());
|
||||
if (lastActivityTime.isAfter(threshold)) {
|
||||
continue; // 没超时
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
// 没有心跳,或者当前处在 has_progress: false (等待用户点击下一个任务推送的静默期)
|
||||
// 静默期和排队期的宽限期一样,设为 12 个小时
|
||||
LocalDateTime queuedThreshold = LocalDateTime.now().minusHours(12);
|
||||
if (task.getCreatedAt() != null && task.getCreatedAt().isAfter(queuedThreshold)) {
|
||||
continue; // 等待下个队列推送中,不要杀它!
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
int updated = fileTaskMapper.update(null, new com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper<FileTaskEntity>()
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.set(FileTaskEntity::getStatus, "FAILED")
|
||||
.set(FileTaskEntity::getErrorMessage, "结果回传长时间无响应,任务已自动失败")
|
||||
@@ -89,20 +151,300 @@ public class DeleteBrandStaleTaskService {
|
||||
}
|
||||
}
|
||||
|
||||
private ProductRiskStaleCheckStats failStaleProductRiskResolveTasks() {
|
||||
ProductRiskStaleCheckStats stats = new ProductRiskStaleCheckStats();
|
||||
long minutes = Math.max(1L, deleteBrandProgressProperties.getProductRiskStaleTimeoutMinutes());
|
||||
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getProductRiskInitialTimeoutMinutes());
|
||||
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
||||
long nowMillis = System.currentTimeMillis();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LocalDateTime threshold = now.minusMinutes(minutes);
|
||||
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
|
||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PRODUCT_RISK)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||
.last("limit 200"));
|
||||
stats.scannedTaskCount = runningTasks.size();
|
||||
if (runningTasks.isEmpty()) {
|
||||
return stats;
|
||||
}
|
||||
log.info("[stale-check] product-risk candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
|
||||
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
|
||||
for (FileTaskEntity task : runningTasks) {
|
||||
boolean hasUploadedPayload = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
||||
long lastPayloadHeartbeatMillis = productRiskTaskCacheService.getTaskHeartbeatMillis(task.getId());
|
||||
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
||||
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
|
||||
boolean hasStartedProgress = hasUploadedPayload || hasFinishedRows;
|
||||
if (lastPayloadHeartbeatMillis > 0L && nowMillis - lastPayloadHeartbeatMillis < staleTimeoutMillis) {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[stale-check] product-risk skip recent-task-heartbeat taskId={} lastPayloadHeartbeatMillis={} timeoutMinutes={} updatedAt={}",
|
||||
task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt());
|
||||
continue;
|
||||
}
|
||||
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}",
|
||||
task.getId(), task.getCreatedAt(), initialThreshold);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
if (productRiskTaskService.tryFinalizeTask(task.getId(), true)) {
|
||||
stats.finalizedTaskCount++;
|
||||
log.info("[stale-check] product-risk finalized taskId={}", task.getId());
|
||||
continue;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[stale-check] product-risk finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
|
||||
}
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PRODUCT_RISK)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.set(FileTaskEntity::getStatus, "FAILED")
|
||||
.set(FileTaskEntity::getErrorMessage, "长时间未收到 Python 结果回传,任务已自动失败")
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
stats.failedTaskCount++;
|
||||
log.warn("[stale-check] product-risk failed taskId={} reason=timeout", task.getId());
|
||||
} else {
|
||||
stats.skippedTaskCount++;
|
||||
}
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
private PriceTrackStaleCheckStats failStalePriceTrackTasks() {
|
||||
PriceTrackStaleCheckStats stats = new PriceTrackStaleCheckStats();
|
||||
long minutes = Math.max(1L, deleteBrandProgressProperties.getPriceTrackStaleTimeoutMinutes());
|
||||
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getPriceTrackInitialTimeoutMinutes());
|
||||
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
||||
long nowMillis = System.currentTimeMillis();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LocalDateTime threshold = now.minusMinutes(minutes);
|
||||
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
|
||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PRICE_TRACK)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||
.last("limit 200"));
|
||||
stats.scannedTaskCount = runningTasks.size();
|
||||
if (runningTasks.isEmpty()) {
|
||||
return stats;
|
||||
}
|
||||
log.info("[stale-check] price-track candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
|
||||
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
|
||||
for (FileTaskEntity task : runningTasks) {
|
||||
long lastHeartbeatMillis = priceTrackTaskCacheService.getTaskHeartbeatMillis(task.getId());
|
||||
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
||||
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
|
||||
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows;
|
||||
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
|
||||
stats.skippedTaskCount++;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
if (priceTrackTaskService.tryFinalizeTask(task.getId(), true)) {
|
||||
stats.finalizedTaskCount++;
|
||||
log.info("[stale-check] price-track finalized taskId={}", task.getId());
|
||||
continue;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[stale-check] price-track finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
|
||||
}
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PRICE_TRACK)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.set(FileTaskEntity::getStatus, "FAILED")
|
||||
.set(FileTaskEntity::getErrorMessage, "长时间未收到 Python 结果回传,任务已自动失败")
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
stats.failedTaskCount++;
|
||||
priceTrackTaskCacheService.deleteTaskCache(task.getId());
|
||||
log.warn("[stale-check] price-track failed taskId={} reason=timeout", task.getId());
|
||||
} else {
|
||||
stats.skippedTaskCount++;
|
||||
}
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
private ShopMatchStaleCheckStats failStaleShopMatchTasks() {
|
||||
ShopMatchStaleCheckStats stats = new ShopMatchStaleCheckStats();
|
||||
long minutes = Math.max(1L, deleteBrandProgressProperties.getShopMatchStaleTimeoutMinutes());
|
||||
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getShopMatchInitialTimeoutMinutes());
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LocalDateTime threshold = now.minusMinutes(minutes);
|
||||
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
|
||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_SHOP_MATCH)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||
.last("limit 200"));
|
||||
stats.scannedTaskCount = runningTasks.size();
|
||||
if (runningTasks.isEmpty()) {
|
||||
return stats;
|
||||
}
|
||||
log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
|
||||
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
|
||||
for (FileTaskEntity task : runningTasks) {
|
||||
boolean hasUploadedPayload = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
||||
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
||||
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
|
||||
boolean hasStartedProgress = hasUploadedPayload || hasFinishedRows;
|
||||
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
|
||||
stats.skippedTaskCount++;
|
||||
log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}",
|
||||
task.getId(), task.getCreatedAt(), initialThreshold);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
if (shopMatchTaskService.tryFinalizeTask(task.getId(), true)) {
|
||||
stats.finalizedTaskCount++;
|
||||
log.info("[stale-check] shop-match finalized taskId={}", task.getId());
|
||||
continue;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[stale-check] shop-match finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
|
||||
}
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_SHOP_MATCH)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.set(FileTaskEntity::getStatus, "FAILED")
|
||||
.set(FileTaskEntity::getErrorMessage, "长时间未收到 Python 结果回传,任务已自动失败")
|
||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||
if (updated > 0) {
|
||||
stats.failedTaskCount++;
|
||||
shopMatchTaskCacheService.deleteTaskCache(task.getId());
|
||||
log.warn("[stale-check] shop-match failed taskId={} reason=timeout", task.getId());
|
||||
} else {
|
||||
stats.skippedTaskCount++;
|
||||
}
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
@Scheduled(cron = "${aiimage.delete-brand-progress.finalize-check-cron:30 */2 * * * *}")
|
||||
public void finalizeCompletedRunningTasks() {
|
||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||
.last("limit 100"));
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
distributedJobLockService.tryLock("delete-brand:finalize-check", FINALIZE_CHECK_LOCK_TTL);
|
||||
if (lockHandle == null) {
|
||||
log.info("[finalize-check] skip because another instance holds the distributed lock");
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
|
||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||
.last("limit 100"));
|
||||
|
||||
for (FileTaskEntity task : runningTasks) {
|
||||
try {
|
||||
deleteBrandRunService.tryFinalizeTask(task.getId(), true);
|
||||
} catch (Exception ignored) {
|
||||
// 定时补偿不影响主流程;下次调度或人工重试时再处理
|
||||
for (FileTaskEntity task : runningTasks) {
|
||||
try {
|
||||
deleteBrandRunService.tryFinalizeTask(task.getId(), true);
|
||||
} catch (Exception ignored) {
|
||||
// Keep compensation best-effort; the next schedule can retry.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(cron = "${aiimage.temp-dir.cleanup-cron:15 */30 * * * *}")
|
||||
public void cleanupExpiredTempDirs() {
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
distributedJobLockService.tryLock("temp-dir:cleanup", TEMP_DIR_CLEANUP_LOCK_TTL);
|
||||
if (lockHandle == null) {
|
||||
log.info("[temp-dir-cleanup] skip because another instance holds the distributed lock");
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
Instant cutoff = Instant.now().minus(TEMP_DIR_RETENTION);
|
||||
List<String> dirNames = List.of(
|
||||
"shop-match-cache",
|
||||
"brand-task-cache",
|
||||
"delete-brand-task-cache",
|
||||
"shop-match-result",
|
||||
"delete-brand-result",
|
||||
"product-risk-result",
|
||||
"price-track-result"
|
||||
);
|
||||
for (String dirName : dirNames) {
|
||||
cleanupTempRoot(dirName, cutoff);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanupTempRoot(String dirName, Instant cutoff) {
|
||||
Path root = Path.of(System.getProperty("java.io.tmpdir"), dirName);
|
||||
if (!Files.isDirectory(root)) {
|
||||
return;
|
||||
}
|
||||
try (var stream = Files.list(root)) {
|
||||
for (Path child : stream.toList()) {
|
||||
try {
|
||||
FileTime lastModified = Files.getLastModifiedTime(child);
|
||||
if (lastModified.toInstant().isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
deleteRecursively(child);
|
||||
log.info("[temp-dir-cleanup] deleted expired temp path dir={} path={}", dirName, child);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[temp-dir-cleanup] delete failed dir={} path={} msg={}", dirName, child, ex.getMessage());
|
||||
}
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
log.warn("[temp-dir-cleanup] scan failed dir={} msg={}", dirName, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteRecursively(Path path) throws IOException {
|
||||
if (!Files.exists(path)) {
|
||||
return;
|
||||
}
|
||||
try (var walk = Files.walk(path)) {
|
||||
walk.sorted(java.util.Comparator.reverseOrder()).forEach(item -> {
|
||||
try {
|
||||
Files.deleteIfExists(item);
|
||||
} catch (IOException ex) {
|
||||
throw new RuntimeException(ex);
|
||||
}
|
||||
});
|
||||
} catch (RuntimeException ex) {
|
||||
if (ex.getCause() instanceof IOException ioException) {
|
||||
throw ioException;
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
private static final class ProductRiskStaleCheckStats {
|
||||
private int scannedTaskCount;
|
||||
private int finalizedTaskCount;
|
||||
private int failedTaskCount;
|
||||
private int skippedTaskCount;
|
||||
}
|
||||
|
||||
private static final class ShopMatchStaleCheckStats {
|
||||
private int scannedTaskCount;
|
||||
private int finalizedTaskCount;
|
||||
private int failedTaskCount;
|
||||
private int skippedTaskCount;
|
||||
}
|
||||
|
||||
private static final class PriceTrackStaleCheckStats {
|
||||
private int scannedTaskCount;
|
||||
private int finalizedTaskCount;
|
||||
private int failedTaskCount;
|
||||
private int skippedTaskCount;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,9 @@ import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@@ -28,10 +31,17 @@ public class DeleteBrandTaskCacheService {
|
||||
|
||||
public void saveParsedPayload(Long taskId, Object payload) {
|
||||
try {
|
||||
stringRedisTemplate.opsForValue().set(buildPayloadKey(taskId), objectMapper.writeValueAsString(payload), Duration.ofHours(PAYLOAD_TTL_HOURS));
|
||||
Files.createDirectories(buildTaskDir(taskId));
|
||||
Files.writeString(
|
||||
buildPayloadFile(taskId),
|
||||
objectMapper.writeValueAsString(payload),
|
||||
StandardOpenOption.CREATE,
|
||||
StandardOpenOption.TRUNCATE_EXISTING,
|
||||
StandardOpenOption.WRITE
|
||||
);
|
||||
parsedPayloadLocalCache.put(taskId, new LocalParsedPayloadCacheEntry(System.currentTimeMillis(), payload));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("暂存删除品牌原始数据失败");
|
||||
throw new BusinessException("鏆傚瓨鍒犻櫎鍝佺墝鍘熷鏁版嵁澶辫触");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,21 +56,21 @@ public class DeleteBrandTaskCacheService {
|
||||
try {
|
||||
return objectMapper.convertValue(cached.value(), typeReference);
|
||||
} catch (Exception ignored) {
|
||||
// fallthrough to redis
|
||||
// fall through to file
|
||||
}
|
||||
}
|
||||
|
||||
String raw = stringRedisTemplate.opsForValue().get(buildPayloadKey(taskId));
|
||||
if (raw == null || raw.isBlank()) {
|
||||
Path payloadFile = buildPayloadFile(taskId);
|
||||
if (!Files.isRegularFile(payloadFile)) {
|
||||
parsedPayloadLocalCache.remove(taskId);
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
T parsed = objectMapper.readValue(raw, typeReference);
|
||||
T parsed = objectMapper.readValue(Files.readString(payloadFile), typeReference);
|
||||
parsedPayloadLocalCache.put(taskId, new LocalParsedPayloadCacheEntry(now, parsed));
|
||||
return parsed;
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取删除品牌原始数据失败");
|
||||
throw new BusinessException("璇诲彇鍒犻櫎鍝佺墝鍘熷鏁版嵁澶辫触");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,17 +124,17 @@ public class DeleteBrandTaskCacheService {
|
||||
|
||||
public boolean hasSameResultChunk(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
|
||||
if (chunkIndex == null || chunkIndex <= 0) {
|
||||
throw new BusinessException("chunkIndex 不合法");
|
||||
throw new BusinessException("chunkIndex 涓嶅悎娉?");
|
||||
}
|
||||
if (fileIdentity == null || fileIdentity.isBlank()) {
|
||||
throw new BusinessException("fileIdentity 不能为空");
|
||||
throw new BusinessException("fileIdentity 涓嶈兘涓虹┖");
|
||||
}
|
||||
String field = fileIdentity.trim() + "#" + chunkIndex;
|
||||
String chunkJson;
|
||||
try {
|
||||
chunkJson = objectMapper.writeValueAsString(chunk);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取删除品牌结果分片失败");
|
||||
throw new BusinessException("璇诲彇鍒犻櫎鍝佺墝缁撴灉鍒嗙墖澶辫触");
|
||||
}
|
||||
Object existing = stringRedisTemplate.opsForHash().get(buildResultChunksKey(taskId), field);
|
||||
if (chunkJson.equals(existing)) {
|
||||
@@ -137,17 +147,17 @@ public class DeleteBrandTaskCacheService {
|
||||
public boolean mergeResultChunk(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
|
||||
String resultKey = buildResultChunksKey(taskId);
|
||||
if (chunkIndex == null || chunkIndex <= 0) {
|
||||
throw new BusinessException("chunkIndex 不合法");
|
||||
throw new BusinessException("chunkIndex 涓嶅悎娉?");
|
||||
}
|
||||
if (fileIdentity == null || fileIdentity.isBlank()) {
|
||||
throw new BusinessException("fileIdentity 不能为空");
|
||||
throw new BusinessException("fileIdentity 涓嶈兘涓虹┖");
|
||||
}
|
||||
String field = fileIdentity.trim() + "#" + chunkIndex;
|
||||
String chunkJson;
|
||||
try {
|
||||
chunkJson = objectMapper.writeValueAsString(chunk);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("暂存删除品牌结果分片失败");
|
||||
throw new BusinessException("鏆傚瓨鍒犻櫎鍝佺墝缁撴灉鍒嗙墖澶辫触");
|
||||
}
|
||||
Object existing = stringRedisTemplate.opsForHash().get(resultKey, field);
|
||||
if (chunkJson.equals(existing)) {
|
||||
@@ -178,14 +188,16 @@ public class DeleteBrandTaskCacheService {
|
||||
|
||||
public void delete(Long taskId) {
|
||||
parsedPayloadLocalCache.remove(taskId);
|
||||
stringRedisTemplate.delete(buildPayloadKey(taskId));
|
||||
stringRedisTemplate.delete(buildProgressKey(taskId));
|
||||
stringRedisTemplate.delete(buildResultChunksKey(taskId));
|
||||
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
||||
deleteTaskDirQuietly(taskId);
|
||||
}
|
||||
|
||||
public void saveTaskCache(com.nanri.aiimage.modules.task.model.entity.FileTaskEntity task) {
|
||||
if (task == null || task.getId() == null) return;
|
||||
if (task == null || task.getId() == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
stringRedisTemplate.opsForValue().set(buildTaskEntityKey(task.getId()), objectMapper.writeValueAsString(task), Duration.ofHours(PAYLOAD_TTL_HOURS));
|
||||
} catch (Exception ignored) {
|
||||
@@ -194,10 +206,14 @@ public class DeleteBrandTaskCacheService {
|
||||
|
||||
public java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> getTaskCacheBatch(java.util.List<Long> taskIds) {
|
||||
java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> result = new java.util.LinkedHashMap<>();
|
||||
if (taskIds == null || taskIds.isEmpty()) return result;
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
|
||||
java.util.List<Long> normalized = taskIds.stream().filter(id -> id != null && id > 0).distinct().toList();
|
||||
if (normalized.isEmpty()) return result;
|
||||
if (normalized.isEmpty()) {
|
||||
return result;
|
||||
}
|
||||
|
||||
java.util.List<String> keys = normalized.stream().map(this::buildTaskEntityKey).toList();
|
||||
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
|
||||
@@ -215,6 +231,30 @@ public class DeleteBrandTaskCacheService {
|
||||
return result;
|
||||
}
|
||||
|
||||
private Path buildTaskDir(Long taskId) {
|
||||
return Path.of(System.getProperty("java.io.tmpdir"), "delete-brand-task-cache", String.valueOf(taskId));
|
||||
}
|
||||
|
||||
private Path buildPayloadFile(Long taskId) {
|
||||
return buildTaskDir(taskId).resolve("parsed-payload.json");
|
||||
}
|
||||
|
||||
private void deleteTaskDirQuietly(Long taskId) {
|
||||
Path taskDir = buildTaskDir(taskId);
|
||||
if (!Files.exists(taskDir)) {
|
||||
return;
|
||||
}
|
||||
try (var walk = Files.walk(taskDir)) {
|
||||
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
|
||||
try {
|
||||
Files.deleteIfExists(path);
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
});
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
private String buildTaskEntityKey(Long taskId) {
|
||||
return "delete-brand:task:entity:" + taskId;
|
||||
}
|
||||
@@ -225,10 +265,6 @@ public class DeleteBrandTaskCacheService {
|
||||
}
|
||||
}
|
||||
|
||||
private String buildPayloadKey(Long taskId) {
|
||||
return "delete-brand:task:parsed-payload:" + taskId;
|
||||
}
|
||||
|
||||
private String buildProgressKey(Long taskId) {
|
||||
return "delete-brand:task:progress:" + taskId;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package com.nanri.aiimage.modules.permission.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuCreateRequest;
|
||||
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuUpdateRequest;
|
||||
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
|
||||
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
|
||||
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
|
||||
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/admin")
|
||||
@Tag(name = "菜单权限管理", description = "统一维护软件端与后台管理端的菜单权限配置")
|
||||
public class PermissionMenuController {
|
||||
|
||||
private final PermissionMenuService permissionMenuService;
|
||||
|
||||
@GetMapping("/permission-menus")
|
||||
@Operation(summary = "查询菜单权限列表")
|
||||
public ApiResponse<List<PermissionMenuItemVo>> listMenus(
|
||||
@Parameter(description = "菜单类型: app/admin") @RequestParam(required = false) String menuType) {
|
||||
return ApiResponse.success(permissionMenuService.list(menuType));
|
||||
}
|
||||
|
||||
@PostMapping("/permission-menus")
|
||||
@Operation(summary = "新增菜单权限项")
|
||||
public ApiResponse<PermissionMenuItemVo> createMenu(@Valid @RequestBody PermissionMenuCreateRequest request) {
|
||||
return ApiResponse.success("创建成功", permissionMenuService.create(request));
|
||||
}
|
||||
|
||||
@PutMapping("/permission-menus/{id}")
|
||||
@Operation(summary = "编辑菜单权限项")
|
||||
public ApiResponse<PermissionMenuItemVo> updateMenu(@PathVariable Long id,
|
||||
@Valid @RequestBody PermissionMenuUpdateRequest request) {
|
||||
return ApiResponse.success("更新成功", permissionMenuService.update(id, request));
|
||||
}
|
||||
|
||||
@DeleteMapping("/permission-menus/{id}")
|
||||
@Operation(summary = "删除菜单权限项")
|
||||
public ApiResponse<Void> deleteMenu(@PathVariable Long id) {
|
||||
permissionMenuService.delete(id);
|
||||
return ApiResponse.success("删除成功", null);
|
||||
}
|
||||
|
||||
@GetMapping("/permission-users/{userId}/columns")
|
||||
@Operation(summary = "查询用户菜单权限 ID 列表")
|
||||
public ApiResponse<UserColumnIdsVo> getUserColumnIds(@PathVariable Long userId,
|
||||
@RequestParam(required = false) String menuType) {
|
||||
return ApiResponse.success(permissionMenuService.getUserColumnIds(userId, menuType));
|
||||
}
|
||||
|
||||
@PutMapping("/permission-users/{userId}/columns")
|
||||
@Operation(summary = "更新用户菜单权限")
|
||||
public ApiResponse<Void> updateUserColumnIds(@PathVariable Long userId,
|
||||
@RequestBody(required = false) UserColumnPermissionUpdateRequest request) {
|
||||
permissionMenuService.updateUserColumnPermissions(userId, request);
|
||||
return ApiResponse.success("保存成功", null);
|
||||
}
|
||||
|
||||
@GetMapping("/permission-users/{userId}/column-permissions")
|
||||
@Operation(summary = "查询用户菜单权限详情")
|
||||
public ApiResponse<List<PermissionMenuItemVo>> getUserColumnPermissions(@PathVariable Long userId,
|
||||
@RequestParam(required = false) String menuType) {
|
||||
return ApiResponse.success(permissionMenuService.getUserColumnPermissions(userId, menuType));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.permission.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface AdminUserMapper extends BaseMapper<AdminUserEntity> {
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.permission.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.permission.model.entity.PermissionMenuEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface PermissionMenuMapper extends BaseMapper<PermissionMenuEntity> {
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.permission.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface UserColumnPermissionMapper extends BaseMapper<UserColumnPermissionEntity> {
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.nanri.aiimage.modules.permission.model.dto;
|
||||
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class PermissionMenuCreateRequest {
|
||||
|
||||
@NotBlank(message = "菜单名称不能为空")
|
||||
private String name;
|
||||
|
||||
@NotBlank(message = "菜单标识不能为空")
|
||||
private String columnKey;
|
||||
|
||||
@NotBlank(message = "菜单类型不能为空")
|
||||
private String menuType;
|
||||
|
||||
@NotBlank(message = "菜单路由不能为空")
|
||||
private String routePath;
|
||||
|
||||
private Integer sortOrder;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.nanri.aiimage.modules.permission.model.dto;
|
||||
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class PermissionMenuUpdateRequest {
|
||||
|
||||
@NotBlank(message = "菜单名称不能为空")
|
||||
private String name;
|
||||
|
||||
@NotBlank(message = "菜单标识不能为空")
|
||||
private String columnKey;
|
||||
|
||||
@NotBlank(message = "菜单类型不能为空")
|
||||
private String menuType;
|
||||
|
||||
@NotBlank(message = "菜单路由不能为空")
|
||||
private String routePath;
|
||||
|
||||
private Integer sortOrder;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package com.nanri.aiimage.modules.permission.model.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class UserColumnPermissionUpdateRequest {
|
||||
|
||||
private List<Long> columnIds;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package com.nanri.aiimage.modules.permission.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@TableName("users")
|
||||
public class AdminUserEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private String username;
|
||||
private String role;
|
||||
@TableField("created_by_id")
|
||||
private Long createdById;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.nanri.aiimage.modules.permission.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("columns")
|
||||
public class PermissionMenuEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private String name;
|
||||
private String columnKey;
|
||||
private String menuType;
|
||||
private String routePath;
|
||||
private Integer sortOrder;
|
||||
private LocalDateTime createdAt;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.nanri.aiimage.modules.permission.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@TableName("user_column_permission")
|
||||
public class UserColumnPermissionEntity {
|
||||
|
||||
@TableField("user_id")
|
||||
private Long userId;
|
||||
|
||||
@TableField("column_id")
|
||||
private Long columnId;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.permission.model.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
public class PermissionMenuItemVo {
|
||||
|
||||
private Long id;
|
||||
private String name;
|
||||
private String columnKey;
|
||||
private String menuType;
|
||||
private String routePath;
|
||||
private Integer sortOrder;
|
||||
private LocalDateTime createdAt;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package com.nanri.aiimage.modules.permission.model.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class UserColumnIdsVo {
|
||||
|
||||
private List<Long> columnIds;
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package com.nanri.aiimage.modules.permission.service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class PermissionMenuSchemaInitializer {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
|
||||
private static final List<DefaultAdminMenu> DEFAULT_ADMIN_MENUS = List.of(
|
||||
new DefaultAdminMenu("用户管理", "admin_users", "users", 10),
|
||||
new DefaultAdminMenu("栏目权限配置", "admin_columns", "columns", 20),
|
||||
new DefaultAdminMenu("数据去重总数据", "admin_dedupe_total_data", "dedupe-total-data", 30),
|
||||
new DefaultAdminMenu("店铺密钥管理", "admin_shop_keys", "shop-keys", 40),
|
||||
new DefaultAdminMenu("店铺管理", "admin_shop_manage", "shop-manage", 50),
|
||||
new DefaultAdminMenu("跳过跟价ASIN", "admin_skip_price_asin", "skip-price-asin", 60),
|
||||
new DefaultAdminMenu("查看生成记录", "admin_history", "history", 70),
|
||||
new DefaultAdminMenu("版本管理", "admin_version", "version", 80)
|
||||
);
|
||||
|
||||
@PostConstruct
|
||||
public void initialize() {
|
||||
executeQuietly("""
|
||||
CREATE TABLE IF NOT EXISTS columns (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
name VARCHAR(128) NOT NULL COMMENT '菜单标题',
|
||||
column_key VARCHAR(64) NOT NULL COMMENT '菜单唯一标识',
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_column_key (column_key)
|
||||
)
|
||||
""");
|
||||
executeQuietly("ALTER TABLE columns ADD COLUMN menu_type VARCHAR(20) NOT NULL DEFAULT 'app' COMMENT '菜单类型: app/admin' AFTER column_key");
|
||||
executeQuietly("ALTER TABLE columns ADD COLUMN route_path VARCHAR(255) NOT NULL DEFAULT '' COMMENT '菜单路由或页面标识' AFTER menu_type");
|
||||
executeQuietly("ALTER TABLE columns ADD COLUMN sort_order INT NOT NULL DEFAULT 0 COMMENT '菜单排序' AFTER route_path");
|
||||
executeQuietly("UPDATE columns SET menu_type = 'app' WHERE menu_type IS NULL OR menu_type = ''");
|
||||
executeQuietly("UPDATE columns SET sort_order = id WHERE sort_order IS NULL OR sort_order = 0");
|
||||
executeQuietly("ALTER TABLE columns ADD UNIQUE KEY uk_menu_type_route_path (menu_type, route_path)");
|
||||
ensureDefaultAdminMenus();
|
||||
executeQuietly("""
|
||||
CREATE TABLE IF NOT EXISTS user_column_permission (
|
||||
user_id INT NOT NULL,
|
||||
column_id INT NOT NULL,
|
||||
PRIMARY KEY (user_id, column_id),
|
||||
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (column_id) REFERENCES columns(id) ON DELETE CASCADE
|
||||
)
|
||||
""");
|
||||
}
|
||||
|
||||
private void ensureDefaultAdminMenus() {
|
||||
for (DefaultAdminMenu menu : DEFAULT_ADMIN_MENUS) {
|
||||
executeQuietly("""
|
||||
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
|
||||
SELECT '%s', '%s', 'admin', '%s', %d
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM columns WHERE column_key = '%s'
|
||||
)
|
||||
""".formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(), menu.columnKey()));
|
||||
executeQuietly("""
|
||||
UPDATE columns
|
||||
SET sort_order = %d
|
||||
WHERE column_key = '%s' AND (sort_order IS NULL OR sort_order = 0)
|
||||
""".formatted(menu.sortOrder(), menu.columnKey()));
|
||||
}
|
||||
}
|
||||
|
||||
private void executeQuietly(String sql) {
|
||||
try {
|
||||
jdbcTemplate.execute(sql);
|
||||
} catch (Exception ex) {
|
||||
log.debug("[permission-menu] schema init skipped: {}", ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private record DefaultAdminMenu(String name, String columnKey, String routePath, int sortOrder) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
package com.nanri.aiimage.modules.permission.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
|
||||
import com.nanri.aiimage.modules.permission.mapper.PermissionMenuMapper;
|
||||
import com.nanri.aiimage.modules.permission.mapper.UserColumnPermissionMapper;
|
||||
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuCreateRequest;
|
||||
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuUpdateRequest;
|
||||
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
|
||||
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
|
||||
import com.nanri.aiimage.modules.permission.model.entity.PermissionMenuEntity;
|
||||
import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEntity;
|
||||
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
|
||||
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PermissionMenuService {
|
||||
|
||||
public static final String MENU_TYPE_APP = "app";
|
||||
public static final String MENU_TYPE_ADMIN = "admin";
|
||||
|
||||
private final PermissionMenuMapper permissionMenuMapper;
|
||||
private final UserColumnPermissionMapper userColumnPermissionMapper;
|
||||
private final AdminUserMapper adminUserMapper;
|
||||
|
||||
public List<PermissionMenuItemVo> list(String menuType) {
|
||||
LambdaQueryWrapper<PermissionMenuEntity> query = new LambdaQueryWrapper<PermissionMenuEntity>()
|
||||
.eq(isValidMenuType(menuType), PermissionMenuEntity::getMenuType, normalizeMenuType(menuType))
|
||||
.orderByAsc(PermissionMenuEntity::getSortOrder)
|
||||
.orderByAsc(PermissionMenuEntity::getId);
|
||||
return permissionMenuMapper.selectList(query).stream()
|
||||
.map(this::toItemVo)
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public PermissionMenuItemVo create(PermissionMenuCreateRequest request) {
|
||||
String name = normalizeRequired(request.getName(), "菜单名称不能为空");
|
||||
String columnKey = normalizeRequired(request.getColumnKey(), "菜单标识不能为空");
|
||||
String menuType = normalizeMenuTypeRequired(request.getMenuType());
|
||||
String routePath = normalizeRequired(request.getRoutePath(), "菜单路由不能为空");
|
||||
ensureUniqueColumnKey(columnKey, null);
|
||||
ensureUniqueRoutePath(menuType, routePath, null);
|
||||
|
||||
PermissionMenuEntity entity = new PermissionMenuEntity();
|
||||
entity.setName(name);
|
||||
entity.setColumnKey(columnKey);
|
||||
entity.setMenuType(menuType);
|
||||
entity.setRoutePath(routePath);
|
||||
entity.setSortOrder(resolveSortOrder(request.getSortOrder(), null));
|
||||
permissionMenuMapper.insert(entity);
|
||||
return toItemVo(getMenuById(entity.getId()));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public PermissionMenuItemVo update(Long id, PermissionMenuUpdateRequest request) {
|
||||
PermissionMenuEntity entity = getMenuById(id);
|
||||
String name = normalizeRequired(request.getName(), "菜单名称不能为空");
|
||||
String columnKey = normalizeRequired(request.getColumnKey(), "菜单标识不能为空");
|
||||
String menuType = normalizeMenuTypeRequired(request.getMenuType());
|
||||
String routePath = normalizeRequired(request.getRoutePath(), "菜单路由不能为空");
|
||||
ensureUniqueColumnKey(columnKey, id);
|
||||
ensureUniqueRoutePath(menuType, routePath, id);
|
||||
|
||||
entity.setName(name);
|
||||
entity.setColumnKey(columnKey);
|
||||
entity.setMenuType(menuType);
|
||||
entity.setRoutePath(routePath);
|
||||
entity.setSortOrder(resolveSortOrder(request.getSortOrder(), id));
|
||||
permissionMenuMapper.updateById(entity);
|
||||
return toItemVo(getMenuById(id));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void delete(Long id) {
|
||||
PermissionMenuEntity entity = getMenuById(id);
|
||||
userColumnPermissionMapper.delete(new LambdaUpdateWrapper<UserColumnPermissionEntity>()
|
||||
.eq(UserColumnPermissionEntity::getColumnId, entity.getId()));
|
||||
permissionMenuMapper.deleteById(entity.getId());
|
||||
}
|
||||
|
||||
public UserColumnIdsVo getUserColumnIds(Long userId, String menuType) {
|
||||
AdminUserEntity user = getUserById(userId);
|
||||
List<Long> columnIds;
|
||||
if (isSuperAdmin(user)) {
|
||||
columnIds = list(menuType).stream().map(PermissionMenuItemVo::getId).toList();
|
||||
} else {
|
||||
List<Long> assignedIds = userColumnPermissionMapper.selectList(
|
||||
new LambdaQueryWrapper<UserColumnPermissionEntity>()
|
||||
.eq(UserColumnPermissionEntity::getUserId, userId))
|
||||
.stream()
|
||||
.map(UserColumnPermissionEntity::getColumnId)
|
||||
.distinct()
|
||||
.toList();
|
||||
if (assignedIds.isEmpty()) {
|
||||
columnIds = List.of();
|
||||
} else if (isValidMenuType(menuType)) {
|
||||
Set<Long> allowedIds = list(menuType).stream()
|
||||
.map(PermissionMenuItemVo::getId)
|
||||
.collect(Collectors.toSet());
|
||||
columnIds = assignedIds.stream().filter(allowedIds::contains).toList();
|
||||
} else {
|
||||
columnIds = assignedIds;
|
||||
}
|
||||
}
|
||||
UserColumnIdsVo vo = new UserColumnIdsVo();
|
||||
vo.setColumnIds(columnIds);
|
||||
return vo;
|
||||
}
|
||||
|
||||
public List<PermissionMenuItemVo> getUserColumnPermissions(Long userId, String menuType) {
|
||||
AdminUserEntity user = getUserById(userId);
|
||||
if (isSuperAdmin(user)) {
|
||||
return list(menuType);
|
||||
}
|
||||
List<Long> assignedIds = userColumnPermissionMapper.selectList(
|
||||
new LambdaQueryWrapper<UserColumnPermissionEntity>()
|
||||
.eq(UserColumnPermissionEntity::getUserId, userId))
|
||||
.stream()
|
||||
.map(UserColumnPermissionEntity::getColumnId)
|
||||
.distinct()
|
||||
.toList();
|
||||
if (assignedIds.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<PermissionMenuEntity> menus = permissionMenuMapper.selectBatchIds(assignedIds);
|
||||
Map<Long, PermissionMenuEntity> menuMap = menus.stream()
|
||||
.collect(Collectors.toMap(PermissionMenuEntity::getId, Function.identity()));
|
||||
String safeMenuType = normalizeMenuType(menuType);
|
||||
return assignedIds.stream()
|
||||
.map(menuMap::get)
|
||||
.filter(menu -> menu != null)
|
||||
.filter(menu -> safeMenuType == null || safeMenuType.equals(menu.getMenuType()))
|
||||
.sorted(Comparator
|
||||
.comparing(PermissionMenuEntity::getSortOrder, Comparator.nullsLast(Integer::compareTo))
|
||||
.thenComparing(PermissionMenuEntity::getId, Comparator.nullsLast(Long::compareTo)))
|
||||
.map(this::toItemVo)
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void updateUserColumnPermissions(Long userId, UserColumnPermissionUpdateRequest request) {
|
||||
getUserById(userId);
|
||||
List<Long> requestedIds = request == null ? List.of() : normalizeColumnIds(request.getColumnIds());
|
||||
if (!requestedIds.isEmpty()) {
|
||||
Long validCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper<PermissionMenuEntity>()
|
||||
.in(PermissionMenuEntity::getId, requestedIds));
|
||||
if (validCount == null || validCount != requestedIds.size()) {
|
||||
throw new BusinessException("存在无效的菜单权限项");
|
||||
}
|
||||
}
|
||||
|
||||
userColumnPermissionMapper.delete(new LambdaUpdateWrapper<UserColumnPermissionEntity>()
|
||||
.eq(UserColumnPermissionEntity::getUserId, userId));
|
||||
for (Long columnId : requestedIds) {
|
||||
UserColumnPermissionEntity entity = new UserColumnPermissionEntity();
|
||||
entity.setUserId(userId);
|
||||
entity.setColumnId(columnId);
|
||||
userColumnPermissionMapper.insert(entity);
|
||||
}
|
||||
}
|
||||
|
||||
private PermissionMenuEntity getMenuById(Long id) {
|
||||
PermissionMenuEntity entity = permissionMenuMapper.selectById(id);
|
||||
if (entity == null) {
|
||||
throw new BusinessException("菜单权限项不存在");
|
||||
}
|
||||
return entity;
|
||||
}
|
||||
|
||||
private AdminUserEntity getUserById(Long userId) {
|
||||
AdminUserEntity entity = adminUserMapper.selectById(userId);
|
||||
if (entity == null) {
|
||||
throw new BusinessException("用户不存在");
|
||||
}
|
||||
return entity;
|
||||
}
|
||||
|
||||
private boolean isSuperAdmin(AdminUserEntity user) {
|
||||
return user != null && "super_admin".equalsIgnoreCase(normalizeRequired(user.getRole(), ""));
|
||||
}
|
||||
|
||||
private void ensureUniqueColumnKey(String columnKey, Long excludeId) {
|
||||
PermissionMenuEntity exists = permissionMenuMapper.selectOne(new LambdaQueryWrapper<PermissionMenuEntity>()
|
||||
.eq(PermissionMenuEntity::getColumnKey, columnKey)
|
||||
.last("LIMIT 1"));
|
||||
if (exists != null && (excludeId == null || !excludeId.equals(exists.getId()))) {
|
||||
throw new BusinessException("菜单标识已存在");
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureUniqueRoutePath(String menuType, String routePath, Long excludeId) {
|
||||
PermissionMenuEntity exists = permissionMenuMapper.selectOne(new LambdaQueryWrapper<PermissionMenuEntity>()
|
||||
.eq(PermissionMenuEntity::getMenuType, menuType)
|
||||
.eq(PermissionMenuEntity::getRoutePath, routePath)
|
||||
.last("LIMIT 1"));
|
||||
if (exists != null && (excludeId == null || !excludeId.equals(exists.getId()))) {
|
||||
throw new BusinessException("同一菜单类型下的菜单路由已存在");
|
||||
}
|
||||
}
|
||||
|
||||
private String normalizeRequired(String value, String message) {
|
||||
String normalized = value == null ? "" : value.trim();
|
||||
if (normalized.isEmpty() && !message.isEmpty()) {
|
||||
throw new BusinessException(message);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private boolean isValidMenuType(String menuType) {
|
||||
return normalizeMenuType(menuType) != null;
|
||||
}
|
||||
|
||||
private String normalizeMenuTypeRequired(String menuType) {
|
||||
String normalized = normalizeMenuType(menuType);
|
||||
if (normalized == null) {
|
||||
throw new BusinessException("菜单类型仅支持 app 或 admin");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private String normalizeMenuType(String menuType) {
|
||||
String normalized = menuType == null ? "" : menuType.trim().toLowerCase(Locale.ROOT);
|
||||
if (normalized.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
if (MENU_TYPE_APP.equals(normalized) || MENU_TYPE_ADMIN.equals(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private List<Long> normalizeColumnIds(List<Long> columnIds) {
|
||||
if (columnIds == null || columnIds.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
Set<Long> uniqueIds = new LinkedHashSet<>();
|
||||
for (Long columnId : columnIds) {
|
||||
if (columnId != null && columnId > 0) {
|
||||
uniqueIds.add(columnId);
|
||||
}
|
||||
}
|
||||
return new ArrayList<>(uniqueIds);
|
||||
}
|
||||
|
||||
private PermissionMenuItemVo toItemVo(PermissionMenuEntity entity) {
|
||||
PermissionMenuItemVo vo = new PermissionMenuItemVo();
|
||||
vo.setId(entity.getId());
|
||||
vo.setName(entity.getName());
|
||||
vo.setColumnKey(entity.getColumnKey());
|
||||
vo.setMenuType(entity.getMenuType());
|
||||
vo.setRoutePath(entity.getRoutePath());
|
||||
vo.setSortOrder(entity.getSortOrder());
|
||||
vo.setCreatedAt(entity.getCreatedAt());
|
||||
return vo;
|
||||
}
|
||||
|
||||
private Integer resolveSortOrder(Integer sortOrder, Long excludeId) {
|
||||
if (sortOrder != null) {
|
||||
return sortOrder;
|
||||
}
|
||||
PermissionMenuEntity tail = permissionMenuMapper.selectOne(new LambdaQueryWrapper<PermissionMenuEntity>()
|
||||
.ne(excludeId != null, PermissionMenuEntity::getId, excludeId)
|
||||
.orderByDesc(PermissionMenuEntity::getSortOrder)
|
||||
.orderByDesc(PermissionMenuEntity::getId)
|
||||
.last("LIMIT 1"));
|
||||
int currentMax = tail == null || tail.getSortOrder() == null ? 0 : tail.getSortOrder();
|
||||
return currentMax + 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCandidateAddRequest;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCountryPreferenceSaveRequest;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCreateTaskRequest;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackMatchShopsRequest;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackTaskBatchRequest;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackCandidateVo;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackCreateTaskVo;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackDashboardVo;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackHistoryVo;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackMatchShopsVo;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackPendingDeleteVo;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.enums.ParameterIn;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/price-track")
|
||||
@Tag(
|
||||
name = "跟价",
|
||||
description = "亚马逊跟价处理模块,包含备选店铺、紫鸟匹配、任务创建、Python 回传、任务轮询和结果下载接口。查询类接口需携带 query 参数 user_id。"
|
||||
)
|
||||
public class PriceTrackController {
|
||||
|
||||
private final PriceTrackService priceTrackService;
|
||||
private final PriceTrackTaskService priceTrackTaskService;
|
||||
|
||||
@GetMapping("/candidates")
|
||||
@Operation(summary = "查询备选店铺列表", description = "返回当前用户在跟价模块中保存的备选店铺。")
|
||||
public ApiResponse<List<PriceTrackCandidateVo>> listCandidates(
|
||||
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
return ApiResponse.success(priceTrackService.listCandidates(userId));
|
||||
}
|
||||
|
||||
@PostMapping("/candidates")
|
||||
@Operation(summary = "新增备选店铺", description = "新增一条备选店铺记录,供后续匹配和创建任务使用。")
|
||||
public ApiResponse<PriceTrackCandidateVo> addCandidate(@Valid @RequestBody PriceTrackCandidateAddRequest request) {
|
||||
return ApiResponse.success(priceTrackService.addCandidate(request));
|
||||
}
|
||||
|
||||
@DeleteMapping("/candidates/{id}")
|
||||
@Operation(summary = "删除备选店铺", description = "删除一条备选店铺记录,记录必须属于当前 user_id。")
|
||||
public ApiResponse<Void> deleteCandidate(
|
||||
@Parameter(description = "备选店铺记录主键", example = "10") @PathVariable Long id,
|
||||
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
priceTrackService.deleteCandidate(userId, id);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@GetMapping("/country-preference")
|
||||
@Operation(summary = "查询国家处理顺序", description = "返回当前用户保存的国家代码列表,顺序即跟价任务处理顺序。")
|
||||
public ApiResponse<PriceTrackCountryPreferenceVo> getCountryPreference(
|
||||
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
return ApiResponse.success(priceTrackService.getCountryPreference(userId));
|
||||
}
|
||||
|
||||
@PutMapping("/country-preference")
|
||||
@Operation(summary = "保存国家处理顺序", description = "保存当前用户在跟价模块中的国家勾选和处理顺序。")
|
||||
public ApiResponse<PriceTrackCountryPreferenceVo> saveCountryPreference(
|
||||
@Valid @RequestBody PriceTrackCountryPreferenceSaveRequest request) {
|
||||
return ApiResponse.success(priceTrackService.saveCountryPreference(request));
|
||||
}
|
||||
|
||||
@PostMapping("/match-shops")
|
||||
@Operation(summary = "批量匹配店铺", description = "按店铺名查询紫鸟索引,返回 shopId、platform、companyName、matchStatus 等信息。")
|
||||
public ApiResponse<PriceTrackMatchShopsVo> matchShops(@Valid @RequestBody PriceTrackMatchShopsRequest request) {
|
||||
return ApiResponse.success(priceTrackService.matchShops(request));
|
||||
}
|
||||
|
||||
@GetMapping("/dashboard")
|
||||
@Operation(summary = "统计看板", description = "返回备选店铺数、已结束任务数、成功任务数和失败任务数。")
|
||||
public ApiResponse<PriceTrackDashboardVo> dashboard(
|
||||
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
return ApiResponse.success(priceTrackTaskService.dashboard(userId));
|
||||
}
|
||||
|
||||
@GetMapping("/history")
|
||||
@Operation(summary = "处理记录列表", description = "返回当前用户跟价模块下各店铺结果记录,包含运行中和历史记录。")
|
||||
public ApiResponse<PriceTrackHistoryVo> history(
|
||||
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
return ApiResponse.success(priceTrackTaskService.listHistory(userId));
|
||||
}
|
||||
|
||||
@DeleteMapping("/history/{resultId}")
|
||||
@Operation(summary = "删除单条结果记录", description = "删除一条 biz_file_result 结果记录,并同步重算父任务状态。")
|
||||
public ApiResponse<Void> deleteHistory(
|
||||
@Parameter(description = "结果行主键 biz_file_result.id", example = "1001") @PathVariable Long resultId,
|
||||
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
priceTrackTaskService.deleteHistory(resultId, userId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@PostMapping("/tasks")
|
||||
@Operation(summary = "创建跟价任务", description = "落库 RUNNING 任务和各店铺占位结果,返回 taskId 和初始 items 快照。")
|
||||
public ApiResponse<PriceTrackCreateTaskVo> createTask(@Valid @RequestBody PriceTrackCreateTaskRequest request) {
|
||||
return ApiResponse.success(priceTrackTaskService.createTask(request));
|
||||
}
|
||||
|
||||
@DeleteMapping("/tasks/{taskId}")
|
||||
@Operation(summary = "删除整条任务", description = "删除该任务以及其下全部结果行,RUNNING、SUCCESS、FAILED 均可删除。")
|
||||
public ApiResponse<Void> deleteTask(
|
||||
@Parameter(description = "任务主键 biz_file_task.id", example = "200") @PathVariable Long taskId,
|
||||
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
priceTrackTaskService.deleteTask(taskId, userId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@DeleteMapping("/pending-shop-result")
|
||||
@Operation(summary = "按店铺名删除运行中结果", description = "仅在 RUNNING 任务下按规范化店名删除一条 biz_file_result,并同步重算任务状态。")
|
||||
public ApiResponse<PriceTrackPendingDeleteVo> deletePendingShopResult(
|
||||
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
|
||||
@RequestParam("user_id") Long userId,
|
||||
@Parameter(name = "shop_name", description = "店铺名称", required = true, in = ParameterIn.QUERY, example = "某某店铺")
|
||||
@RequestParam("shop_name") String shopName) {
|
||||
return ApiResponse.success(priceTrackTaskService.deletePendingShopResult(userId, shopName));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/batch")
|
||||
@Operation(summary = "批量查询任务详情", description = "按 taskIds 返回任务概要和各店铺结果,供前端查询使用。")
|
||||
public ApiResponse<PriceTrackTaskBatchVo> tasksBatch(@Valid @RequestBody PriceTrackTaskBatchRequest request) {
|
||||
return ApiResponse.success(priceTrackTaskService.getTaskDetailsBatch(request.getTaskIds()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量查询任务进度摘要", description = "仅返回任务状态、时间和错误等轻量信息,供前端轮询降载。")
|
||||
public ApiResponse<PriceTrackTaskBatchVo> taskProgressBatch(@Valid @RequestBody PriceTrackTaskBatchRequest request) {
|
||||
return ApiResponse.success(priceTrackTaskService.getTaskProgressBatch(request.getTaskIds()));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/result")
|
||||
@Operation(summary = "Python 回传处理结果", description = "同一 taskId 可多次回传。成功时按 countries 生成结果文件,所有店铺进入终态后任务会变为 SUCCESS 或 FAILED。")
|
||||
public ApiResponse<Void> submitResult(
|
||||
@Parameter(description = "任务主键,必须处于 RUNNING 状态", example = "200") @PathVariable Long taskId,
|
||||
@Valid @RequestBody PriceTrackSubmitResultRequest request) {
|
||||
priceTrackTaskService.submitResult(taskId, request);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@GetMapping("/results/{resultId}/download")
|
||||
@Operation(summary = "下载结果文件", description = "按结果记录下载后端生成的跟价 Excel 文件。")
|
||||
public void downloadResult(
|
||||
@Parameter(description = "结果记录主键 biz_file_result.id", example = "1001") @PathVariable Long resultId,
|
||||
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
|
||||
@RequestParam("user_id") Long userId,
|
||||
jakarta.servlet.http.HttpServletResponse response) {
|
||||
String url = priceTrackTaskService.resolveResultDownloadUrl(resultId, userId);
|
||||
String filename = priceTrackTaskService.resolveResultDownloadFilename(resultId, userId);
|
||||
if (url == null || url.isBlank()) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
|
||||
}
|
||||
try {
|
||||
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
|
||||
response.setContentType("application/octet-stream");
|
||||
response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
|
||||
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
|
||||
try (InputStream in = URI.create(url).toURL().openStream()) {
|
||||
byte[] buffer = new byte[65536];
|
||||
int read;
|
||||
while ((read = in.read(buffer)) != -1) {
|
||||
response.getOutputStream().write(buffer, 0, read);
|
||||
}
|
||||
response.getOutputStream().flush();
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.entity.PriceTrackCountryPrefEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface PriceTrackCountryPrefMapper extends BaseMapper<PriceTrackCountryPrefEntity> {
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.pricetrack.model.entity.PriceTrackShopCandidateEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface PriceTrackShopCandidateMapper extends BaseMapper<PriceTrackShopCandidateEntity> {
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "新增备选店铺请求体:写入用户维度的待处理店铺列表,供前端勾选与匹配")
|
||||
public class PriceTrackCandidateAddRequest {
|
||||
@NotNull(message = "user_id不能为空")
|
||||
@Schema(description = "当前用户 ID,数据按用户隔离", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
|
||||
private Long userId;
|
||||
|
||||
@NotBlank(message = "店铺名称不能为空")
|
||||
@Schema(description = "店铺名称,入库前会先做规范化和去重", requiredMode = Schema.RequiredMode.REQUIRED, example = "测试店铺A")
|
||||
private String shopName;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "保存用户五国处理顺序:代码仅支持 DE、UK、FR、IT、ES,且不能重复")
|
||||
public class PriceTrackCountryPreferenceSaveRequest {
|
||||
@NotNull(message = "user_id不能为空")
|
||||
@Schema(description = "当前用户 ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
|
||||
private Long userId;
|
||||
|
||||
@NotEmpty(message = "国家列表不能为空")
|
||||
@Schema(description = "国家代码列表,顺序即任务处理顺序", requiredMode = Schema.RequiredMode.REQUIRED, example = "[\"DE\",\"UK\",\"FR\"]")
|
||||
private List<String> countryCodes;
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.dto;
|
||||
|
||||
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackMatchShopsVo;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "创建跟价任务请求体:在推送 Python 队列前调用,用于落库任务和店铺结果占位记录")
|
||||
public class PriceTrackCreateTaskRequest {
|
||||
@NotNull(message = "user_id不能为空")
|
||||
@Schema(description = "任务所属用户 ID,会写入 biz_file_task.user_id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
|
||||
private Long userId;
|
||||
|
||||
@Schema(description = "是否启用按商品状态处理模式")
|
||||
private boolean statusMode;
|
||||
|
||||
@Schema(description = "是否启用按指定 ASIN 文件处理模式")
|
||||
private boolean asinMode;
|
||||
|
||||
@NotEmpty(message = "店铺列表不能为空")
|
||||
@Valid
|
||||
@Schema(description = "已匹配的店铺列表,每项都应为 matched=true;服务端会按规范化店名去重", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<PriceTrackMatchShopsVo.PriceTrackShopQueueItem> items;
|
||||
|
||||
@Schema(description = "ASIN 文件路径列表;asinMode=true 时由前端上传或选择后传入", example = "[\"/tmp/a.xlsx\",\"/tmp/b.csv\"]")
|
||||
private List<String> asinFiles;
|
||||
|
||||
@NotEmpty(message = "国家列表不能为空")
|
||||
@Schema(description = "国家代码列表,顺序即跟价处理顺序", requiredMode = Schema.RequiredMode.REQUIRED, example = "[\"DE\",\"UK\",\"FR\"]")
|
||||
private List<String> countryCodes;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "Price track match shops request")
|
||||
public class PriceTrackMatchShopsRequest {
|
||||
|
||||
@NotNull(message = "userId is required")
|
||||
@Schema(description = "Current user id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
|
||||
private Long userId;
|
||||
|
||||
@NotEmpty(message = "shopNames cannot be empty")
|
||||
@Schema(description = "Shop names to match", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<String> shopNames;
|
||||
|
||||
@Schema(description = "Selected ASIN file local paths")
|
||||
private List<String> asinFiles;
|
||||
|
||||
@Schema(description = "Selected country codes")
|
||||
private List<String> countryCodes;
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@Schema(description = "Python submit payload for price track task results")
|
||||
public class PriceTrackSubmitResultRequest {
|
||||
@Schema(description = "Shop result list", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<ShopResult> shops;
|
||||
|
||||
@Data
|
||||
@Schema(description = "Per-shop submit result")
|
||||
public static class ShopResult {
|
||||
@Schema(description = "Shop name used to match task rows", example = "Test Shop A")
|
||||
private String shopName;
|
||||
|
||||
@Schema(description = "Rows grouped by country code, for example DE/UK/FR/IT/ES")
|
||||
private Map<String, List<AsinResult>> countries;
|
||||
|
||||
@Schema(description = "Failure message")
|
||||
private String error;
|
||||
|
||||
@Schema(description = "Whether the shop finished successfully")
|
||||
private Boolean success;
|
||||
}
|
||||
|
||||
@Data
|
||||
@Schema(description = "Per-ASIN price track result row")
|
||||
public static class AsinResult {
|
||||
@Schema(description = "Shop mall name", example = "Amazon.de")
|
||||
private String shopMallName;
|
||||
|
||||
@Schema(description = "ASIN", example = "B0ABCDE123")
|
||||
private String asin;
|
||||
|
||||
@Schema(description = "Price", example = "19.99")
|
||||
private String price;
|
||||
|
||||
@Schema(description = "Recommended price", example = "18.99")
|
||||
private String recommendedPrice;
|
||||
|
||||
@Schema(description = "Lowest price", example = "18.50")
|
||||
private String minimumPrice;
|
||||
|
||||
@Schema(description = "First place", example = "Competitor A")
|
||||
private String firstPlace;
|
||||
|
||||
@Schema(description = "Second place", example = "Competitor B")
|
||||
private String secondPlace;
|
||||
|
||||
@Schema(description = "Buy box shop name", example = "Shop A")
|
||||
private String cartShopName;
|
||||
|
||||
@Schema(description = "Price change status", example = "UPDATED")
|
||||
private String priceChangeStatus;
|
||||
|
||||
@Schema(description = "Modify count", example = "2")
|
||||
private String modifyCount;
|
||||
|
||||
@Schema(description = "Status", example = "SUCCESS")
|
||||
private String status;
|
||||
|
||||
@Schema(description = "Legacy current price", example = "19.99")
|
||||
private String currentPrice;
|
||||
|
||||
@Schema(description = "Legacy competitor price", example = "18.50")
|
||||
private String competitorPrice;
|
||||
|
||||
@Schema(description = "Legacy action", example = "LOWER_PRICE")
|
||||
private String action;
|
||||
|
||||
@Schema(description = "Legacy done flag")
|
||||
private Boolean done;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "批量查询任务详情请求体:一次拉取多个 taskId 的概要和下属店铺结果,供轮询使用")
|
||||
public class PriceTrackTaskBatchRequest {
|
||||
@NotEmpty(message = "taskIds不能为空")
|
||||
@Schema(description = "待查询的任务主键列表,无效或非本模块 taskId 会出现在 missingTaskIds", requiredMode = Schema.RequiredMode.REQUIRED, example = "[200,201]")
|
||||
private List<Long> taskIds = new ArrayList<>();
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@TableName("biz_price_track_country_pref")
|
||||
public class PriceTrackCountryPrefEntity {
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long userId;
|
||||
|
||||
@TableField("country_codes_json")
|
||||
private String countryCodesJson;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("biz_price_track_shop_candidate")
|
||||
public class PriceTrackShopCandidateEntity {
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private Long userId;
|
||||
private String shopName;
|
||||
private LocalDateTime createdAt;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import lombok.experimental.Accessors;
|
||||
|
||||
@Data
|
||||
@Accessors(chain = true)
|
||||
@Schema(description = "备选店铺列表单项")
|
||||
public class PriceTrackCandidateVo {
|
||||
@Schema(description = "记录主键,删除备选店铺时作为路径参数", example = "10")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "店铺名称", example = "测试店铺A")
|
||||
private String shopName;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "用户五国处理偏好:按顺序返回国家代码列表,例如 DE、UK、FR、IT、ES")
|
||||
public class PriceTrackCountryPreferenceVo {
|
||||
@Schema(description = "当前用户 ID", example = "1")
|
||||
private Long userId;
|
||||
|
||||
@Schema(description = "已选国家代码列表;若用户未保存,则服务端返回默认顺序 DE、UK、FR、IT、ES")
|
||||
private List<String> countryCodes;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@Schema(description = "price track create task response")
|
||||
public class PriceTrackCreateTaskVo {
|
||||
@Schema(description = "task id", example = "200")
|
||||
private Long taskId;
|
||||
|
||||
@Schema(description = "initial task items")
|
||||
private List<PriceTrackResultItemVo> items;
|
||||
|
||||
@Schema(description = "all skip asins grouped by country")
|
||||
private Map<String, List<String>> skipAsinsByCountry;
|
||||
|
||||
@Schema(description = "parsed asin rows by country")
|
||||
private Map<String, List<Map<String, String>>> asinRowsByCountry;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
import lombok.experimental.Accessors;
|
||||
|
||||
@Data
|
||||
@Accessors(chain = true)
|
||||
@Schema(description = "跟价模块统计看板,按当前用户维度汇总")
|
||||
public class PriceTrackDashboardVo {
|
||||
@Schema(description = "备选店铺数量")
|
||||
private Long candidateCount;
|
||||
|
||||
@Schema(description = "已结束任务数,状态为 SUCCESS 或 FAILED")
|
||||
private Long processedTaskCount;
|
||||
|
||||
@Schema(description = "成功结束的任务数")
|
||||
private Long successTaskCount;
|
||||
|
||||
@Schema(description = "失败结束的任务数")
|
||||
private Long failedTaskCount;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "跟价结果列表响应:items 中每项对应一个店铺结果行")
|
||||
public class PriceTrackHistoryVo {
|
||||
@Schema(description = "结果列表,包含运行中和历史记录")
|
||||
private List<PriceTrackResultItemVo> items;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@Schema(description = "Price track shop match response")
|
||||
public class PriceTrackMatchShopsVo {
|
||||
|
||||
@Schema(description = "Match results in the same order as requested shop names")
|
||||
private List<PriceTrackShopQueueItem> items = new ArrayList<>();
|
||||
|
||||
@Schema(description = "All skip ASINs grouped by country code")
|
||||
private Map<String, List<String>> skipAsinsByCountry;
|
||||
|
||||
@Schema(description = "Parsed asin rows grouped by country code")
|
||||
private Map<String, List<Map<String, String>>> asinRowsByCountry;
|
||||
|
||||
@Data
|
||||
@Schema(description = "Single matched shop item")
|
||||
public static class PriceTrackShopQueueItem {
|
||||
|
||||
@Schema(description = "Shop name", example = "Demo Shop")
|
||||
private String shopName;
|
||||
|
||||
@Schema(description = "Matched shop id")
|
||||
private Object shopId;
|
||||
|
||||
@Schema(description = "Platform", example = "Amazon")
|
||||
private String platform;
|
||||
|
||||
@Schema(description = "Company name")
|
||||
private String companyName;
|
||||
|
||||
@Schema(description = "Whether the shop matched")
|
||||
private Boolean matched;
|
||||
|
||||
@Schema(description = "Match status, such as MATCHED or PENDING")
|
||||
private String matchStatus;
|
||||
|
||||
@Schema(description = "Readable match message")
|
||||
private String matchMessage;
|
||||
|
||||
@Schema(description = "All skip ASINs grouped by country code")
|
||||
private Map<String, List<String>> skipAsins;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package com.nanri.aiimage.modules.pricetrack.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "按店铺名删除运行中结果的响应")
|
||||
public class PriceTrackPendingDeleteVo {
|
||||
@Schema(description = "是否实际删除到一条运行中的结果记录")
|
||||
private boolean removed;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user