27 Commits

Author SHA1 Message Date
super
ef0e0df0ac 完成匹配、跟价、权限部分 2026-04-17 12:52:16 +08:00
super
025ca6d4fd 后台管理跳过asin、后台相关数据做数据隔离 2026-04-14 10:04:50 +08:00
951a353881 提交任务更新 2026-04-13 20:07:00 +08:00
super
eb04caccf1 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-13 13:02:45 +08:00
super
cd84def61d 提交工作更新 2026-04-13 13:02:41 +08:00
铭坤
d8098b0378 modified: app/amazon/__pycache__/del_brand.cpython-39.pyc
modified:   app/amazon/__pycache__/match_action.cpython-39.pyc
	modified:   app/amazon/del_brand.py
	modified:   app/amazon/match_action.py
	modified:   app/assets/delete-brand.js
	modified:   app/new_web_source/delete-brand.html
	new file:   "app/web_source/brand - \345\211\257\346\234\254.html"
2026-04-12 20:52:26 +08:00
super
62d30ec190 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-12 17:15:54 +08:00
super
d0d3c6ee67 完成菜单权限添加和软件菜单改造 2026-04-12 17:15:15 +08:00
铭坤
21b6b5b270 modified: app/amazon/__pycache__/approve.cpython-39.pyc
modified:   app/amazon/__pycache__/del_brand.cpython-39.pyc
	modified:   app/amazon/__pycache__/main.cpython-39.pyc
	modified:   app/amazon/__pycache__/match_action.cpython-39.pyc
	modified:   app/amazon/__pycache__/tool.cpython-39.pyc
	modified:   app/amazon/approve.py
	modified:   app/amazon/del_brand.py
	modified:   app/amazon/main.py
	modified:   app/amazon/match_action.py
	modified:   app/amazon/tool.py
	modified:   app/assets/delete-brand.js
	modified:   app/new_web_source/delete-brand.html
2026-04-11 23:00:55 +08:00
super
365050b890 提交bug修复 后台权限管理 2026-04-11 22:25:14 +08:00
super
73392a9b83 修复BUG,完善匹配店铺 2026-04-11 00:57:34 +08:00
super
c47c03fde4 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-10 19:35:08 +08:00
super
86fd9475e4 匹配回收,bug修复 2026-04-10 19:33:58 +08:00
铭坤
2d0d1d3461 modified: app/amazon/__pycache__/approve.cpython-39.pyc
modified:   app/amazon/__pycache__/del_brand.cpython-39.pyc
	modified:   app/amazon/__pycache__/main.cpython-39.pyc
	new file:   app/amazon/__pycache__/match_action.cpython-39.pyc
	new file:   app/amazon/__pycache__/tool.cpython-39.pyc
	new file:   "app/amazon/approve - \345\211\257\346\234\254.py"
	modified:   app/amazon/approve.py
	modified:   app/amazon/del_brand.py
	modified:   app/amazon/main.py
	new file:   app/amazon/match_action.py
	new file:   app/amazon/tool.py
	modified:   app/main.py
	modified:   app/web_source/brand.html
2026-04-10 16:51:52 +08:00
super
ccaec4a984 完善后台管理店铺搜索 2026-04-08 21:08:52 +08:00
super
169ba7edb6 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-08 16:14:30 +08:00
super
0327d1cc51 修复接口内容缺失 2026-04-08 16:14:26 +08:00
铭坤
e02ddab599 modified: amazon/__pycache__/approve.cpython-39.pyc
modified:   amazon/__pycache__/del_brand.cpython-39.pyc
	modified:   amazon/__pycache__/main.cpython-39.pyc
	modified:   amazon/approve.py
	new file:   amazon/backend_api.py
	modified:   amazon/del_brand.py
	modified:   amazon/main.py
	modified:   blueprints/main.py
	modified:   main.py
2026-04-08 16:03:48 +08:00
super
336338cff5 完善后台接口字段 2026-04-07 17:57:03 +08:00
super
acff31e652 Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-07 14:50:09 +08:00
super
ad304a6880 完成店铺增删改查 2026-04-07 14:50:06 +08:00
铭坤
cb34788e53 modified: "web_source/brand-\346\227\247.html"
modified:   web_source/brand.html
	modified:   web_source/templates_backup/brand.html
2026-04-07 14:10:33 +08:00
铭坤
13b0ffb5d8 new file: amazon/__pycache__/approve.cpython-39.pyc
modified:   amazon/__pycache__/del_brand.cpython-39.pyc
	modified:   amazon/__pycache__/main.cpython-39.pyc
	new file:   amazon/approve.py
	modified:   amazon/del_brand.py
	modified:   amazon/main.py
	modified:   main.py
2026-04-07 14:09:42 +08:00
super
c9947c4ac8 更改下载方法名 2026-04-06 22:26:44 +08:00
super
afffb7aad2 更改下载类型处理 2026-04-06 16:58:58 +08:00
super
9720453ac4 更新商品风险处理 2026-04-05 22:04:21 +08:00
super
1ff6d5220e 新增删除增加公司名 2026-04-04 00:10:08 +08:00
251 changed files with 939206 additions and 2081 deletions

View File

@@ -12,6 +12,7 @@ zn_username=%E8%87%AA%E5%8A%A8%E5%8C%96_Robot
client_name=ShuFuAI
# java_api_base=http://47.111.163.154:18080
java_api_base=http://127.0.0.1:18080
# java_api_base=http://8.136.19.173:18080

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@@ -0,0 +1,999 @@
import time
import re
import traceback
from DrissionPage._pages.chromium_tab import ChromiumTab
from config import runing_task, runing_shop
from datetime import datetime
from amazon.del_brand import AmamzonBase, kill_process
# 导入 webview 用于前端通知
try:
import webview
except ImportError:
webview = None
class AmzoneApprove(AmamzonBase):
def SwitchPage(self):
"""
切换至 管理所有库存页面
1、等待 //navigation-favorites-bar[@class="hydrated"] 出现
"""
navigation = self.tab.ele('xpath://navigation-favorites-bar[@class="hydrated"]')
navigation.wait.displayed(raise_err=False)
page_btn = navigation.sr('xpath://internal-fav-bar-links[@data-internal="navigation"]').sr(
'xpath://a[@data-page-id="ezdpc-gui-inventory-mons"]')
page_btn.wait.displayed(raise_err=False)
page_btn.click(timeout=5)
self.tab.wait.doc_loaded()
# 等待搜索框出现
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group')
search_region.wait.displayed(raise_err=False)
def search(self,filter_type="ApprovalRequired"):
sku_ls = []
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
drop_down = self.tab.ele('xpath://div[contains(@class,"VolusListingStatusDropDown-module__verticalContainer")]//kat-dropdown')
drop_down.wait.displayed(raise_err=False)
drop_down.wait.enabled(raise_err=False)
time.sleep(0.6)
drop_down.click()
# //kat-option[@value="SearchSuppressed"]
xp = f'xpath://kat-option[@value="{filter_type}"]'
print(f"正在寻找筛选条件 {filter_type}xpath: {xp}")
approval_required = self.tab.eles(xp,timeout=5)
if len(approval_required) == 0:
print(f"【没有需要{filter_type}选项】没有需要{filter_type}的商品了")
return sku_ls # "没有需要审批的商品了"
else:
approval_required = approval_required[0]
approval_required.wait.displayed(raise_err=False)
approval_required.click()
approval_required_text = approval_required.text
print(f"已选择筛选条件: {approval_required_text}")
count = re.findall(r'\d+', approval_required_text)
if count:
count = int(count[0])
print(f"待审批的商品数量: {count}")
if count <= 0:
print(f"没有需要{filter_type}的商品了")
return sku_ls #"没有需要审批的商品了"
asin = "B0C7KT6ZZN"
search_input = self.tab.ele("xpath://kat-input[contains(@class,'SearchBox-module__searchInput')]").sr(
'xpath://span[@class="container"]//input[@part="input"]')
search_input.input(asin, clear=True)
search_btn = self.tab.ele("xpath://kat-icon[@name='search']")
search_btn.click()
for _ in range(3):
# 等待加载完成
load_ele = self.tab.ele("xpath://div[contains(@class,'Loader-module__loader')]")
load_ele.wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
search_input = self.tab.ele("xpath://kat-input[contains(@class,'SearchBox-module__searchInput')]").sr(
'xpath://span[@class="container"]//input[@part="input"]')
search_input.input(asin, clear=True)
search_btn = self.tab.ele("xpath://kat-icon[@name='search']")
search_btn.click()
load_ele = self.tab.ele("xpath://div[contains(@class,'Loader-module__loader')]")
load_ele.wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=3)
if len(sku_ls) > 0:
break
approval_required.click()
return sku_ls
def wait_loaded(self):
# 等待加载完成
try:
load_ele = self.tab.ele(
'xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//div[contains(@class,"Loader-module__loader")]|//kat-panel[@data-testid="kat-panel-ActionPanelContent"]/div[@data-f1-component]//div[contains(@class,"==")]/div[contains(@class,"==")]/span',
timeout=3)
load_ele.wait.deleted(timeout=5, raise_err=False)
except Exception as e:
print("等待加载中消失出错", e)
def clear_tab(self):
all_tab = self.browser.get_tabs(title="亚马逊")
close_tab = []
for tab in all_tab:
tab_id = tab if isinstance(tab,str) else tab.tab_id
if self.tab.tab_id == tab:
continue
close_tab.append(tab)
print("需要关闭的标签页",close_tab)
print("当前操作的tab_id",self.tab.tab_id)
self.browser.close_tabs(close_tab)
def handle_repair_product(self,tab:ChromiumTab):
def wait_loaded():
# 等待加载完成
try:
load_ele = tab.ele(
'xpath://div[@class="contentWrapper"]//div[contains(@class,"==")]/span[not(node())]',
timeout=3)
load_ele.wait.deleted(timeout=5, raise_err=False)
except Exception as e:
print("等待加载中消失出错", e)
tab.wait.doc_loaded(timeout=30)
submit_btn_ls = tab.eles('xpath://div[@id="ahd-product-policies-table"]/div//div[@data-testid="nextStepsMetricWrapper"]//kat-button[@label="提交"]',timeout=10)
if len(submit_btn_ls) == 0:
print("未找到提交按钮,可能页面未加载完成或者页面结构发生变化")
return
# todo 遍历
for index in range(len(submit_btn_ls)):
if index > len(submit_btn_ls)-1:
print("提交按钮数量发生变化,停止处理")
break
submit_btn = submit_btn_ls[index]
# 关闭按钮
close_x_btn = tab.eles('xpath://div[@class="flyoutPanelContent"]/span',timeout=5)
if len(close_x_btn) > 0:
close_x_btn[0].wait.displayed(timeout=5, raise_err=False)
close_x_btn[0].click()
submit_btn.wait.enabled(timeout=5, raise_err=False)
submit_btn.click()
time.sleep(0.5)
self.wait_loaded()
wait_loaded()
need_input = tab.eles('xpath://div[@class="contentWrapper"]//kat-input[@data-testid="kat-input-dew:ump_epr_resgitration_number_title"]',timeout=5)
if len(need_input) > 0:
print("需要输入注册号,不符合操作要求,跳过...")
continue
target_title = tab.eles('xpath://div[@class="contentWrapper"]//section//h4[text()="警告和安全信息"]')
if len(target_title) > 0:
not_start = tab.eles('xpath://div[@class="contentWrapper"]//div[@aria-label="安全证明"]//kat-label[@text="未开始"]')
if len(not_start) == 0:
print("安全证明-不是未开始状态,不需要操作")
continue
not_start.wait.displayed(timeout=5, raise_err=False)
not_start.click()
wait_loaded()
check = tab.ele('xpath://div[@class="contentWrapper"]//kat-checkbox[@name="value"]')
check.wait.displayed(timeout=5, raise_err=False)
check.click()
save_btn = tab.ele('xpath://div[@class="contentWrapper"]//kat-button[@variant="primary"]')
save_btn.wait.enabled(timeout=5, raise_err=False)
save_btn.click()
wait_loaded()
close_btn = tab.ele('xpath://div[@class="contentWrapper"]//kat-button[@label="关闭"]',timeout=10)
close_btn.wait.displayed(timeout=5, raise_err=False)
close_btn.click()
save_btn_ls = tab.eles('xpath://div[@class="contentWrapper"]//kat-button[@variant="primary"]',timeout=5)
if len(save_btn_ls) > 0:
option_selection_ls = tab.eles('xpath://div[@class="contentWrapper"]//div[@role="option"]',timeout=5)
for option in option_selection_ls:
option.click()
time.sleep(0.5)
save_btn_ls[0].wait.enabled(timeout=5, raise_err=False)
save_btn_ls[0].click()
wait_loaded()
close_btn = tab.ele('xpath://div[@class="contentWrapper"]//kat-button[@label="关闭"]',timeout=10)
close_btn.wait.displayed(timeout=5, raise_err=False)
close_btn.wait.enabled(timeout=5, raise_err=False)
close_btn.click()
submit_btn_ls = tab.eles('xpath://div[@id="ahd-product-policies-table"]/div//div[@data-testid="nextStepsMetricWrapper"]//kat-button[@label="提交"]',timeout=10)
print("修复商品信息处理完成!")
tab.close()
def run_page_action(self):
print("开始执行")
num = 0
retry_num = 0
already_asin = set()
while retry_num < 3: # 最多重试3次
# if num > 3: #测试
# return
# 等待加载完成
try:
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
# 获取当前页码
try:
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) > 0:
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
# 总页数
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
if len(total_page) > 0:
total_page = total_page[-1].text
else:
total_page = 0
print(f"当前页码: {current_page} / 总页数: {total_page}")
except Exception as e:
print("获取页码失败", e)
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
print(f"获取到 {len(sku_ls)}")
# for sku_ele in sku_ls[0:2]:
for sku_ele in sku_ls:
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]')
asin = sku_ele.ele('xpath:.//div[contains(@class,"JanusSplitBox-module__container")]//div[contains(@class,"JanusSplitBox-module__panel--") and contains(string(.),"ASIN")]/..//div[last()]').text
print(f"ASIN {asin} 存在问题,正在点击解决...")
if asin in already_asin:
print(f"{asin} 已经处理过了,跳过")
continue
solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]|.//kat-link[@label="修复被禁止显示的商品"]|.//kat-link[@label="解决商品移除风险"]')
if len(solve_problem) == 0:
print(f"{asin},没有处理入口,不处理")
yield (asin,"没有处理入口,不处理")
already_asin.add(asin)
continue
solve_problem[0].click()
action_panel = self.tab.ele('xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]')
action_panel.wait.displayed(timeout=5, raise_err=False)
self.wait_loaded()
try:
# 如果存在“请求批准”按钮,则直接返回跳过
request_approval_btn = self.tab.eles('xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//div[@data-testid="section-header" and contains(string(.),"移除")]',timeout=5)
if len(request_approval_btn) > 0:
print(f"ASIN {asin} 存在请求批准按钮,不需要处理,跳过...")
panel_close_btn = self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]',
timeout=5).sr('xpath:.//button[@class="close"]')
panel_close_btn.click()
action_panel.wait.deleted(timeout=3, raise_err=False)
yield (asin,"请求批准")
# 无需采取任何操作
not_operate = self.tab.eles('xpath://kat-alert[contains(@description,"如果您之前已提交更改,则这些更改当前正在处理中") and not(@dismissed)]',timeout=3)
if len(not_operate) > 0:
print(f"ASIN {asin} 无需采取任何操作,跳过...")
yield (asin,"无需操作")
already_asin.add(asin)
continue
# 解决商品信息违规问题 按钮
kat_box_ls = action_panel.eles('xpath:.//kat-box[@variant="white"]',timeout=5)
# if len(kat_box_ls) == 0:
# print(f"ASIN {asin} 的操作面板中未找到选项,跳过...")
# continue
print(f"ASIN {asin}需要处理的有:{len(kat_box_ls)}个问题.")
# 解决商品信息违规问题 按钮,有多少个都要处理
for i in range(len(kat_box_ls)):
print(f"开始处理第{i}个问题")
if len(kat_box_ls) <= i:
print(f"ASIN {asin} 的操作面板中选项数量发生变化,停止处理...")
break
kat_box_ls[i].click()
time.sleep(0.5)
self.wait_loaded()
# TODO 增加 “出现警告和安全信息时候下滑” 的情况
safe_handle_ls = self.tab.eles('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//div[@aria-label="安全证明"]//kat-label[@text="未开始"]',
timeout=5)
if len(safe_handle_ls) > 0:
print("存在安全证明,正在处理...")
safe_handle_ls[0].click()
time.sleep(0.5)
self.wait_loaded()
check_box = self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//kat-checkbox')
check_box.click()
else:
problem_selection_ls = self.tab.eles(
'xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//div[@data-testid="registry-list"]//div[@data-testid="registry"]',
timeout=10)
print("待选择数量",len(problem_selection_ls))
for problem_selection in problem_selection_ls:
problem_selection.click()
print("点击选择完成")
time.sleep(0.5)
# 点击保存
confirm_btn = self.tab.ele('xpath://kat-panel-wrapper[@data-testid="kat-panel-wrapper-ActionPanelContent"]//kat-button[@variant="primary"]')
confirm_btn.wait.displayed(timeout=3, raise_err=False)
confirm_btn.click()
self.wait_loaded()
# 等待关闭按钮出现
try:
close_btn = self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//kat-button[@label="关闭"]',
timeout=10)
close_btn.click()
except Exception as e:
print("点击关闭按钮失败",e)
kat_box_ls = action_panel.eles('xpath:.//kat-box[@variant="white"]', timeout=5)
# 修复商品信息
repair_product = action_panel.eles('xpath:.//kat-button[@class="action-button"]',timeout=5)
if len(repair_product) > 0:
print(f"ASIN {asin} 存在修复商品信息按钮,正在点击...")
self.clear_tab()
repair_product[0].click()
time.sleep(1)
# 获取最新的tab
new_tab = self.browser.latest_tab
if isinstance(new_tab, str):
new_tab = self.browser.get_tab(id_or_num=new_tab)
self.handle_repair_product(new_tab)
except Exception as e:
print("【asin】",asin,"处理失败",traceback.format_exc())
yield (asin,"处理失败")
already_asin.add(asin)
continue
yield (asin,"处理完成")
already_asin.add(asin)
try:
# 全部操作完成,关闭
panel_close_btn= self.tab.ele('xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]',
timeout=5).sr('xpath:.//button[@class="close"]')
panel_close_btn.click()
action_panel.wait.deleted(timeout=3, raise_err=False)
except Exception as e:
print("关闭操作面板失败", e)
# 判断是否存在需要翻页的情况
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) == 0:
break
next_page_btn = page_pamel[0].sr('xpath:.//span[@part="pagination-nav-right"]')
class_str = next_page_btn.attr('class')
if "end" in class_str:
break
next_page_btn.click()
num += 1
print(f"【程序计算】正在翻页,已翻 {num} 页...")
already_asin = set()
except Exception as e:
print("处理审批操作异常", e)
traceback.print_exc()
retry_num += 1
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False,timeout=120)
class ApproveTask:
"""审批任务处理类:负责处理产品风险审批任务"""
country_info = {
"DE": "德国",
"FR": "法国",
"ES": "西班牙",
"IT": "意大利",
"UK": "英国"
}
@staticmethod
def show_notification(message: str, message_type: str = "error"):
"""显示 pywebview 顶层通知5秒后自动消失
Args:
message: 通知消息
message_type: 消息类型 (success/warning/error/info)
"""
if webview and webview.windows:
try:
# 转义单引号,防止 JavaScript 语法错误
safe_message = message.replace("'", "\\'").replace('"', '\\"').replace('\n', '\\n')
# 设置通知样式颜色
color_map = {
'success': '#67C23A',
'warning': '#E6A23C',
'error': '#F56C6C',
'info': '#909399'
}
bg_color = color_map.get(message_type, '#F56C6C')
# 构建前端通知的 JavaScript 代码 - 创建原生 HTML 通知
js_code = f"""
(function() {{
// 移除已存在的通知
var existingNotif = document.getElementById('pywebview-notification');
if (existingNotif) {{
existingNotif.remove();
}}
// 创建通知容器
var notif = document.createElement('div');
notif.id = 'pywebview-notification';
notif.style.cssText = 'position: fixed; top: 20px; left: 50%; transform: translateX(-50%); ' +
'background: {bg_color}; color: white; padding: 12px 20px; border-radius: 4px; ' +
'box-shadow: 0 2px 12px rgba(0,0,0,0.3); z-index: 99999; font-size: 14px; ' +
'max-width: 600px; word-wrap: break-word; display: flex; align-items: center; gap: 10px;';
// 添加消息内容
var msgSpan = document.createElement('span');
msgSpan.textContent = '{safe_message}';
notif.appendChild(msgSpan);
// 添加关闭按钮
var closeBtn = document.createElement('span');
closeBtn.innerHTML = '×';
closeBtn.style.cssText = 'cursor: pointer; font-size: 18px; font-weight: bold; margin-left: 10px;';
closeBtn.onclick = function() {{ notif.remove(); }};
notif.appendChild(closeBtn);
// 添加到页面
document.body.appendChild(notif);
// 5秒后自动消失
setTimeout(function() {{
if (notif && notif.parentNode) {{
notif.style.transition = 'opacity 0.3s';
notif.style.opacity = '0';
setTimeout(function() {{ notif.remove(); }}, 300);
}}
}}, 5000);
}})();
"""
# 在第一个窗口中执行 JavaScript
webview.windows[0].evaluate_js(js_code)
except Exception as e:
print(f"显示通知失败: {str(e)}")
"""
处理流程:
1、解析任务json,提取对应的companyName 组组装user_info实例化 AmzoneApprove
2、打开对应的店铺、遍历所有的国家站点处理
3、切换至指定国家切换至 管理所有库存页面
4、调用search方法查询出是否存在需要审批的商品如果没有则结束如果有则进入处理流程
5、进入处理流程后调用run_page_action方法(注意是通过yield 返回数据的)
任务json示例
{
"type": "product-risk-resolve-run",
"ts": 1775395996876,
"data": {
"taskId": 925,
"items": [
{
"matched": true,
"platform": "亚马逊",
"shopName": "魏振峰",
"shopId": "27543917795757",
"companyName": "rongchuang123",
"openStoreUrl": null,
"matchedUserId": 17543915345493,
"matchStatus": "MATCHED",
"matchMessage": null
}
],
"country_codes": [
"UK",
"DE",
"FR",
"ES"
]
}
}
"""
def __init__(self, user_info: dict = None):
"""初始化审批任务处理器
Args:
user_info: 用户信息字典,包含 company, username, password
"""
self.user_info = user_info or {}
self.running = True
def log(self, message: str, level: str = "INFO"):
"""日志输出
Args:
message: 日志消息
level: 日志级别
"""
from datetime import datetime
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
self.show_notification(message, "error")
print(f"[{timestamp}] [ApproveTask] [{level}] {message}")
def process_task(self, task_data: dict):
"""处理审批任务主入口
Args:
task_data: 任务数据
"""
try:
data = task_data.get("data", {})
task_id = data.get("taskId")
items = data.get("items", [])
country_codes = data.get("country_codes", [])
risk_listing_filter = data.get("risk_listing_filter", "")
if not task_id:
self.log("任务ID为空跳过", "WARNING")
return
if not items:
self.log("店铺列表为空,跳过", "WARNING")
return
if not country_codes:
self.log("国家列表为空,跳过", "WARNING")
return
self.log(f"开始处理审批任务 {task_id},共 {len(items)} 个店铺,{len(country_codes)} 个国家")
from config import runing_task
runing_task[task_id] = {
"status": "running",
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"total_shops": len(items),
"processed_shops": 0,
"total_countries": len(country_codes) * len(items),
"processed_countries": 0,
"total_asins": 0,
"processed_asins": 0,
"success_count": 0,
"failed_count": 0,
"stop_requested": False
}
# 遍历处理每个店铺
for idx, shop_item in enumerate(items, 1):
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理", "WARNING")
runing_task[task_id]["status"] = "stopped"
return
shop_name = shop_item.get("shopName", "未知店铺")
self.log(f"[{idx}/{len(items)}] 开始处理店铺: {shop_name}")
self.show_notification(f"开始处理店铺: {shop_name}", "info")
try:
self.process_shop(shop_item, country_codes, task_id,risk_listing_filter)
# 更新已处理店铺数
if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1
except Exception as e:
import traceback
self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 更新任务状态
if task_id in runing_task:
if runing_task[task_id].get("stop_requested", False):
runing_task[task_id]["status"] = "stopped"
self.log(f"任务 {task_id} 已被暂停!")
else:
runing_task[task_id]["status"] = "completed"
self.log(f"任务 {task_id} 处理完成!")
except Exception as e:
import traceback
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id:
from config import runing_task
if task_id in runing_task:
runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e)
def process_shop(self, shop_item: dict, country_codes: list, task_id: int, risk_listing_filter: str):
"""处理单个店铺
Args:
shop_item: 店铺信息
country_codes: 国家代码列表
task_id: 任务ID
risk_listing_filter: 风险商品筛选条件
"""
shop_name = shop_item.get("shopName", "未知店铺")
company_name = shop_item.get("companyName", "")
if not company_name:
self.log(f"店铺 {shop_name} 的公司名称为空,跳过", "WARNING")
return
if task_id in runing_task:
runing_task[task_id]["current_shop"] = shop_name
# 将店铺添加到正在执行中的店铺列表
start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
runing_shop[shop_name] = start_time
self.log(f"店铺 {shop_name} 已添加到执行列表,账号: {company_name},开始时间: {start_time}")
# 店铺打开重试最多3次
driver = None
max_retries = 3
error_info = ""
for retry in range(max_retries):
try:
self.log(f"尝试打开店铺 {shop_name} (第 {retry + 1}/{max_retries} 次)")
# 如果不是第一次尝试,先杀进程
# if retry > 0:
# self.log("重试前先杀掉浏览器进程...")
# kill_process("v6")
# kill_process("v5")
# time.sleep(2)
# 组装用户信息并创建驱动
user_info = {
**self.user_info,
"company": company_name
}
driver = AmzoneApprove(user_info)
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name}")
break
else:
self.log(f"打开店铺失败: {browser}", "WARNING")
driver = None
# 判断是否需要登录
need_login = driver.need_login()
if need_login:
self.log(f"店铺 {shop_name} 需要登录,正在登录...")
password = ""
login_success = driver.login(password)
if login_success:
self.log(f"店铺 {shop_name} 登录成功,正在重新打开店铺...")
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name} 登录后")
break
else:
self.log(f"登录后打开店铺失败: {browser}", "WARNING")
driver = None
else:
self.log(f"店铺 {shop_name} 登录失败", "WARNING")
driver = None
except Exception as e:
import traceback
self.log(f"打开店铺异常: {traceback.format_exc()}", "INFO")
driver = None
error_info = str(e)
time.sleep(10)
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(3)
# 检查是否成功打开
if not driver or not browser or browser == "店铺不存在":
error_msg = f"店铺 {shop_name} 打开失败,已重试 {max_retries} 次,跳过该店铺,{error_info}"
self.log(error_msg, "ERROR")
# 从执行列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
return
try:
# 处理每个国家
for country_code in country_codes:
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理国家", "WARNING")
break
try:
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter)
except Exception as e:
import traceback
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 最后回传,标记完成
try:
self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
except Exception as e:
self.log(f"回传结果失败: {str(e)}", "ERROR")
finally:
# 关闭店铺
try:
if driver:
self.log(f"关闭店铺 {shop_name}")
driver.close_store()
time.sleep(2)
except Exception as e:
self.log(f"关闭店铺失败: {str(e)}", "WARNING")
# 从正在执行中的店铺列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
self.log(f"店铺 {shop_name} 已从执行列表中移除")
def process_country(self, driver: AmzoneApprove, country_code: str, task_id: int, shop_name: str, risk_listing_filter: str):
"""处理单个国家的审批任务
Args:
driver: AmzoneApprove驱动实例
country_code: 国家代码(如 UK, DE, FR 等)
task_id: 任务ID
shop_name: 店铺名称
risk_listing_filter: 风险商品筛选条件
"""
from config import runing_task
# 转换国家代码为中文名称
country_name = self.country_info.get(country_code, country_code)
info_mes = f"开始处理国家: {country_name} ({country_code})"
self.log(info_mes)
self.show_notification(info_mes, "info")
# 更新当前处理的国家
if task_id in runing_task:
runing_task[task_id]["current_country"] = country_name
# 切换国家最多重试3次
max_retries = 3
switch_success = False
for retry in range(max_retries):
try:
self.log(f"尝试切换到国家 {country_name} (第 {retry + 1}/{max_retries} 次)")
# 如果不是第一次尝试,先刷新页面
if retry > 0:
self.log("重试前刷新页面...")
try:
driver.tab.refresh()
time.sleep(3)
except Exception as e:
self.log(f"刷新页面失败: {str(e)}", "WARNING")
switch_success = driver.SwitchingCountries(country_name)
if switch_success:
self.log(f"成功切换到国家 {country_name}")
break
else:
self.log(f"切换到国家 {country_name} 失败", "WARNING")
except Exception as e:
import traceback
self.log(f"切换国家 {country_name} 异常: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
# 如果切换失败,直接返回
if not switch_success:
error_message = f"切换到国家 {country_name} 失败,已重试 {max_retries} 次,跳过该国家"
self.log(error_message, "ERROR")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
# 切换到库存管理页面
try:
driver.SwitchPage()
self.log(f"已切换到库存管理页面")
except Exception as e:
import traceback
self.log(f"切换页面失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
# 搜索需要审批的商品最多重试3次
sku_ls = []
for retry in range(max_retries):
try:
self.log(f"尝试搜索需要审批的商品 (第 {retry + 1}/{max_retries} 次)")
sku_ls = driver.search(filter_type=risk_listing_filter)
break
except Exception as e:
self.log(f"搜索商品异常: {str(e)}", "ERROR")
if retry < max_retries - 1:
try:
driver.tab.refresh()
time.sleep(3)
except Exception as refresh_error:
self.log(f"刷新页面失败: {str(refresh_error)}", "WARNING")
# 如果没有需要审批的商品,直接返回
if len(sku_ls) == 0:
self.log(f"国家 {country_name} 没有需要审批的商品")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
self.log(f"国家 {country_name}{len(sku_ls)} 个需要审批的商品,开始处理...")
# 处理所有需要审批的商品通过yield获取结果
try:
for asin, status in driver.run_page_action():
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求停止处理ASIN", "WARNING")
break
self.log(f"ASIN {asin} 处理结果: {status}")
# 更新任务状态
if task_id in runing_task:
runing_task[task_id]["current_asin"] = asin
runing_task[task_id]["processed_asins"] += 1
if status == "处理完成":
runing_task[task_id]["success_count"] += 1
elif status == "请求批准":
# 请求批准的商品也算作成功(因为不需要处理)
runing_task[task_id]["success_count"] += 1
else:
runing_task[task_id]["failed_count"] += 1
# 回传结果到API
try:
self.post_result(task_id, shop_name, country_code, asin, status)
except Exception as e:
self.log(f"回传结果失败: {str(e)}", "ERROR")
except Exception as e:
import traceback
self.log(f"处理审批商品异常: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 更新已处理国家数
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
self.log(f"国家 {country_name} 处理完成")
def post_result(self, task_id: int, shop_name: str, country_code: str, asin: str, status: str,is_done: bool = False):
"""回传处理结果到API
Args:
task_id: 任务ID
shop_name: 店铺名称
country_code: 国家代码
asin: ASIN
status: 处理状态
"""
import requests
from config import DELETE_BRAND_API_BASE
url = f"{DELETE_BRAND_API_BASE}/api/product-risk-resolve/tasks/{task_id}/result"
if status == "请求批准":
country_data = {
"status": "",
"shopName": shop_name,
"productAsinSku": "",
"done": is_done,
"removeAsin": asin,
"removeStatus": status
}
else:
country_data = {
"status": status,
"shopName": shop_name,
"productAsinSku": asin,
"done": is_done,
"removeAsin": "",
"removeStatus": ""
}
payload = {
"shops": [
{
"error": "",
"countries": {
country_code : [ country_data ]
},
"shopName": shop_name
}
]
}
max_retries = 3
for retry in range(max_retries):
try:
print("=====================================")
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
self.log(f"回传数据: {payload}")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False
)
self.log(f"回传结果: {response.text}")
if response.status_code == 200:
self.log(f"结果回传成功: {asin} - {status}")
return
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
print("=====================================")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
print("=====================================")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
if __name__ == '__main__':
user_info = {
"company": "rongchuang123",
"username": "自动化_Robot",
"password": "#20zsg25"
}
shop_name = "魏振峰"
country = "西班牙"
kill_process('v6')
driver = AmzoneApprove(user_info)
browser = driver.open_shop(shop_name)
sw_suc = driver.SwitchingCountries(country)
driver.SwitchPage()
risk_listing_filter = "Active"
for _ in range(3):
try:
sku_ls = driver.search(filter_type=risk_listing_filter)
break
except Exception as e:
print(e)
driver.tab.refresh()
if len(sku_ls) > 0:
print("有数据,开始操作")
for asin, status in driver.run_page_action():
print(f"ASIN {asin} 的处理结果: {status}")
print("已完成操作")
# for i in range(0):
# print(i)

1018
app/amazon/approve.py Normal file

File diff suppressed because it is too large Load Diff

View File

View File

@@ -10,6 +10,7 @@ import os
from typing import Literal
from DrissionPage import Chromium
from DrissionPage.common import By
from DrissionPage._pages.chromium_tab import ChromiumTab
def kill_process(version: Literal["v5", "v6"]):
@@ -33,7 +34,7 @@ class ZiniaoDriver:
self.socket_port = socket_port
self.client_path = None
self.browser = None
self.tab = None
self.tab: ChromiumTab = None
self.store_id = None
def get_zinaio_exe(self, protocol_name: str = "superbrowser"):
@@ -52,7 +53,8 @@ class ZiniaoDriver:
command, _ = winreg.QueryValueEx(key, "")
winreg.CloseKey(key)
if isinstance(command, str):
exe_path = command.split()[0]
sub = "ziniao.exe"
exe_path = command[0: command.find(sub)+len(sub)+1]
else:
exe_path = command[0]
return exe_path
@@ -62,7 +64,8 @@ class ZiniaoDriver:
command, _ = winreg.QueryValueEx(key, "")
winreg.CloseKey(key)
if isinstance(command, str):
exe_path = command.split()[0]
sub = "ziniao.exe"
exe_path = command[0: command.find(sub)+len(sub)+1]
else:
exe_path = command[0]
return exe_path
@@ -107,6 +110,7 @@ class ZiniaoDriver:
"""
if version == "v5":
process_name = 'SuperBrowser.exe'
os.system('taskkill /f /t /im ' + "starter.exe")
else:
process_name = 'ziniao.exe'
os.system('taskkill /f /t /im ' + process_name)
@@ -133,11 +137,11 @@ class ZiniaoDriver:
print(r)
return r.get("browserList")
elif str(r.get("statusCode")) == "-10003":
print(f"login Err {json.dumps(r, ensure_ascii=False)}")
exit()
print(f"【get_browser_list】登录失败 {json.dumps(r, ensure_ascii=False)}")
# exit()
else:
print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
exit()
print(f"【get_browser_list】失败 {json.dumps(r, ensure_ascii=False)} ")
# exit()
def open_store(self, store_info, isWebDriverReadOnlyMode=0, isprivacy=0,
isHeadless=0, cookieTypeSave=0, jsInfo=""):
@@ -185,11 +189,11 @@ class ZiniaoDriver:
if str(r.get("statusCode")) == "0":
return r
elif str(r.get("statusCode")) == "-10003":
print(f"login Err {json.dumps(r, ensure_ascii=False)}")
exit()
raise RuntimeError(f"【open_store】登录失败 {json.dumps(r, ensure_ascii=False)}")
# exit()
else:
print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
exit()
raise RuntimeError(f"【open_store】失败 {json.dumps(r, ensure_ascii=False)} ")
# exit()
def get_browser(self, port) -> Chromium:
"""
@@ -206,13 +210,58 @@ class ZiniaoDriver:
def start_client(self):
"""启动紫鸟客户端"""
self.client_path = self.get_zinaio_exe("superbrowser").strip('"')
# 检查端口是否已经启动
try:
url = f'http://127.0.0.1:{self.socket_port}'
response = requests.get(url, timeout=2)
print(f"端口 {self.socket_port} 已经启动,跳过启动客户端操作")
return
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
# 端口未启动,继续执行启动操作
print(f"端口 {self.socket_port} 未启动,开始启动客户端")
self.kill_process('v6')
time.sleep(5)
self.client_path = self.get_zinaio_exe("superbrowserv6").strip('"')
print(self.client_path)
cmd = [self.client_path, '--run_type=web_driver', '--ipc_type=http',
'--port=' + str(self.socket_port)]
print(" ".join(cmd))
subprocess.Popen(cmd)
time.sleep(5)
# 最大重试次数
max_retries = 3
for retry_count in range(max_retries):
print(f"{retry_count + 1} 次尝试启动客户端...")
# 启动进程
subprocess.Popen(cmd)
# 循环检测10秒每0.5秒检测一次
start_check_time = time.time()
client_started = False
while time.time() - start_check_time < 10:
try:
response = requests.get(url, timeout=2)
print(f"客户端启动成功!(第 {retry_count + 1} 次尝试)")
client_started = True
break
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
# 端口还未启动,继续等待
time.sleep(0.5)
if client_started:
time.sleep(5) # 等待客户端完全启动
# 更新内核
self.update_core()
return
else:
print(f"{retry_count + 1} 次尝试启动失败10秒内未检测到客户端启动")
# 超过最大重试次数,抛出异常
raise RuntimeError(f"客户端启动失败:重试 {max_retries} 次后仍未成功启动")
def open_shop(self, shop_name: str):
"""
@@ -227,8 +276,7 @@ class ZiniaoDriver:
# 启动客户端
self.start_client()
# 更新内核
self.update_core()
# 获取店铺列表
shop_ls = self.get_browser_list()
@@ -258,7 +306,8 @@ class ZiniaoDriver:
print("ip检测页地址为空请升级紫鸟浏览器到最新版")
print(f"=====关闭店铺:{shop_name}=====")
self.close_store(self.store_id)
exit()
# exit()
raise RuntimeError("没有IP检测地址为了店铺安全不打开店铺")
ip_usable = self.open_ip_check(self.browser, ip_check_url)
if ip_usable:
print("ip检测通过打开店铺平台主页")
@@ -295,11 +344,11 @@ class ZiniaoDriver:
if str(r.get("statusCode")) == "0":
return r
elif str(r.get("statusCode")) == "-10003":
print(f"login Err {json.dumps(r, ensure_ascii=False)}")
exit()
raise RuntimeError(f"【close_store】登录失败 {json.dumps(r, ensure_ascii=False)}")
# exit()
else:
print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
exit()
raise RuntimeError(f"【close_store】失败 {json.dumps(r, ensure_ascii=False)} ")
# exit()
def open_launcher_page(self, launcher_page: str, browser: Chromium = None):
"""
@@ -339,9 +388,9 @@ class ZiniaoDriver:
return False
class AmazoneDriver(ZiniaoDriver):
"""亚马逊专用驱动类继承自ZiniaoDriver"""
class AmamzonBase(ZiniaoDriver):
"""亚马逊操作基类,包含一些通用方法"""
def SwitchingCountries(self, country_name: str):
"""
切换国家
@@ -365,11 +414,12 @@ class AmazoneDriver(ZiniaoDriver):
# 获取当前标签页
tab = self.tab
tab.wait.doc_loaded(timeout=120,raise_err=False)
# 步骤1获取当前国家名称
print(f"正在检查当前国家...")
current_country_ele = tab.ele('xpath://div[@class="dropdown-account-switcher-header-label"]/span[last()]',
timeout=10)
timeout=20)
if current_country_ele:
current_country = current_country_ele.text.strip()
print(f"当前国家:{current_country}")
@@ -433,6 +483,49 @@ class AmazoneDriver(ZiniaoDriver):
print(f"切换国家时发生异常:{traceback.format_exc()}")
return False
def need_login(self):
"""
判断是否需要登录,部分国家可能需要登录后才能切换国家
处理流程:
"""
time.sleep(3) # 等待页面可能的登录元素加载,避免跳转等等
self.tab.wait.doc_loaded(timeout=30, raise_err=False)
need_login_ele = self.tab.eles('xpath://h1[@class="a-spacing-small"]|//span[contains(text(),"登录")]',timeout=5)
if len(need_login_ele) >0:
print("检测到需要登录元素")
return True
return False
def login(self,password,username=""):
try:
pwd_input = self.tab.eles('xpath://input[@type="password"]',timeout=10)
if len(pwd_input) > 0:
pwd_input[0].input(password,clear=True)
submit_btn = self.tab.eles('xpath://input[@id="signInSubmit"]',timeout=10)
if len(submit_btn) > 0:
submit_btn[0].click()
self.tab.wait.doc_loaded()
opt_code_input = self.tab.ele('xpath://input[@name="otpCode"]',timeout=60)
opt_code_input.wait.displayed(timeout=10,raise_err=False)
for _ in range(30):
if opt_code_input.value is not None and opt_code_input.value.strip() != "":
print("检测到验证码输入完成")
submit_btn = self.tab.ele('xpath://input[@id="auth-signin-button"]',timeout=10)
submit_btn.click()
self.tab.wait.doc_loaded()
return True
time.sleep(1)
submit_btn = self.tab.ele('xpath://input[@id="auth-signin-button"]',timeout=10)
submit_btn.click()
except Exception as e:
print("登录过程中发生异常:" + traceback.format_exc())
return False
class AmazoneDriver(AmamzonBase):
"""亚马逊专用驱动类继承自ZiniaoDriver"""
def SwitchPage(self):
"""
切换至 管理所有库存页面

View File

@@ -6,6 +6,11 @@ from typing import Dict, Any, List
from concurrent.futures import ThreadPoolExecutor, as_completed
from config import JSON_TASK_QUEUE, runing_task, runing_shop, DELETE_BRAND_API_BASE, ZN_COMPANY, ZN_USERNAME, ZN_PASSWORD
from amazon.del_brand import AmazoneDriver, kill_process
from amazon.approve import ApproveTask
from amazon.match_action import MatchTak
from amazon.tool import get_shop_info,show_notification
class TaskMonitor:
@@ -26,6 +31,7 @@ class TaskMonitor:
# 在提交新任务前杀掉旧进程(确保环境干净)
kill_process("v6")
kill_process("v5")
def log(self, message: str, level: str = "INFO"):
"""日志输出
@@ -45,6 +51,10 @@ class TaskMonitor:
self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
futures = [] # 保存所有提交的任务Future对象
task_type_info = {
"product-risk-resolve-run" : "产品风险审批",
"shop-match-run" : "匹配价格"
}
try:
while self.running:
try:
@@ -54,21 +64,29 @@ class TaskMonitor:
# 检查任务类型
task_type = task_data.get("type", "")
if task_type != "delete-brand-run":
if task_type == "delete-brand-run":
# 提交删除品牌任务到线程池
self.log(f"接收到【删除品牌】任务,提交到线程池处理...")
future = self.executor.submit(self._process_task_wrapper, task_data)
futures.append(future)
elif task_type in task_type_info:
# 提交产品风险审批任务到线程池
self.log(f"接收到任务,提交到线程池处理...")
future = self.executor.submit(self._process_approve_task_wrapper, task_data, task_type_info[task_type])
futures.append(future)
else:
self.log(f"未知任务类型: {task_type},跳过", "WARNING")
continue
# 提交任务到线程池
self.log(f"接收到删除品牌任务,提交到线程池处理...")
future = self.executor.submit(self._process_task_wrapper, task_data)
futures.append(future)
# 清理已完成的future对象避免内存累积
futures = [f for f in futures if not f.done()]
except Exception as e:
# 静默处理队列为空的超时,只记录真正的异常
if "Empty" not in str(e):
if str(e) and "Empty" not in str(e):
self.log(f"任务监控异常: {str(e)}", "ERROR")
# 队列为空时不需要额外等待,直接继续循环
@@ -86,11 +104,31 @@ class TaskMonitor:
task_data: 任务数据
"""
try:
self.log(f"线程 {id(task_data)} 开始处理任务...")
self.log(f"线程 {id(task_data)} 开始处理删除品牌任务...")
self.process_task(task_data)
self.log(f"线程 {id(task_data)} 任务处理完成")
self.log(f"线程 {id(task_data)} 删除品牌任务处理完成")
except Exception as e:
self.log(f"线程 {id(task_data)} 任务处理异常: {traceback.format_exc()}", "ERROR")
self.log(f"线程 {id(task_data)} 删除品牌任务处理异常: {traceback.format_exc()}", "ERROR")
def _process_approve_task_wrapper(self, task_data: Dict[str, Any],TASK_TYPE:str):
"""审批任务处理包装器(用于线程池调用)
Args:
task_data: 任务数据
"""
try:
TASK_INFO = {
"产品风险审批" : ApproveTask,
"匹配价格" : MatchTak
}
self.log(f"线程 {id(task_data)} 开始处理产品风险审批任务...")
# 创建ApproveTask实例并处理任务
TASK_CLS = TASK_INFO[TASK_TYPE] # 根据任务类型选择处理类默认为ApproveTask
approve_task = TASK_CLS(user_info=self.user_info)
approve_task.process_task(task_data)
self.log(f"线程 {id(task_data)} 产品风险审批任务处理完成")
except Exception as e:
self.log(f"线程 {id(task_data)} 产品风险审批任务处理异常: {traceback.format_exc()}", "ERROR")
def process_task(self, task_data: Dict[str, Any]):
@@ -161,6 +199,7 @@ class TaskMonitor:
shop_name = shop_data.get("shopName", "未知店铺")
result_id = shop_data.get("resultId")
countries = shop_data.get("countries", [])
company_name = shop_data.get("companyName", "未知公司")
if not countries:
self.log(f"店铺 {shop_name} 没有国家数据,跳过", "WARNING")
@@ -172,7 +211,7 @@ class TaskMonitor:
# 将店铺添加到正在执行中的店铺列表
start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
runing_shop[shop_name] = start_time
self.log(f"店铺 {shop_name} 已添加到执行列表,开始时间: {start_time}")
self.log(f"账号:{company_name},店铺 {shop_name} 已添加到执行列表,开始时间: {start_time}")
# 店铺打开重试最多3次
driver = None
@@ -184,13 +223,18 @@ class TaskMonitor:
self.log(f"尝试打开店铺 {shop_name} (第 {retry + 1}/{max_retries} 次)")
# 如果不是第一次尝试,先杀进程
if retry > 0:
self.log("重试前先杀掉浏览器进程...")
kill_process("v6")
time.sleep(2)
# if retry > 0:
# self.log("重试前先杀掉浏览器进程...")
# kill_process("v6")
# kill_process("v5")
# time.sleep(2)
# 创建驱动并打开店铺
driver = AmazoneDriver(self.user_info)
user_info = {
**self.user_info,
"company": company_name
}
driver = AmazoneDriver(user_info)
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
@@ -199,6 +243,39 @@ class TaskMonitor:
else:
self.log(f"打开店铺失败: {browser}", "WARNING")
driver = None
# 判断是否需要登录
driver.tab.wait.doc_loaded(timeout=30, raise_err=False) # 等待页面加载,避免过早判断登录状态
need_login = driver.need_login()
print("【是否需要登录】:",need_login)
if need_login:
self.log(f"店铺 {shop_name} 需要登录,正在登录...")
# 获取店铺凭证
response = get_shop_info(shop_name)
print("【获取店铺凭证返回】:",response.text)
shop_data = response.json()
if not shop_data:
mes = f"获取店铺凭证失败,响应数据: {shop_data.get('message', '未知错误')}"
self.log(mes, "ERROR")
show_notification(mes, "ERROR")
continue
password = shop_data["data"]["password"]
login_success = driver.login(password)
if login_success:
self.log(f"店铺 {shop_name} 登录成功,正在重新打开店铺...")
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name} 登录后")
break
else:
self.log(f"登录后打开店铺失败: {browser}", "WARNING")
driver = None
else:
self.log(f"店铺 {shop_name} 登录失败", "WARNING")
driver = None
except Exception as e:
self.log(f"打开店铺异常: {str(e)}", "ERROR")
@@ -549,6 +626,7 @@ class TaskMonitor:
verify=False # 忽略SSL证书验证
)
print("【结果提交】:",payload)
print("【结果提交返回】:",response.text)
if response.status_code == 200:
self.log(f"结果回传成功: {url}")

717
app/amazon/match_action.py Normal file
View File

@@ -0,0 +1,717 @@
import time
import re
import traceback
from DrissionPage._pages.chromium_tab import ChromiumTab
from config import runing_task, runing_shop
from datetime import datetime
from amazon.del_brand import AmamzonBase, kill_process
from amazon.tool import show_notification,get_shop_info
class AmzoneMatchAction(AmamzonBase):
mark_name = "匹配价格"
def SwitchPage(self):
"""
切换至 管理所有库存页面
1、等待 //navigation-favorites-bar[@class="hydrated"] 出现
"""
navigation = self.tab.ele('xpath://navigation-favorites-bar[@class="hydrated"]')
navigation.wait.displayed(raise_err=False)
page_btn = navigation.sr('xpath://internal-fav-bar-links[@data-internal="navigation"]').sr(
'xpath://a[@data-page-id="ezdpc-gui-inventory-mons"]')
page_btn.wait.displayed(raise_err=False)
page_btn.click(timeout=5)
self.tab.wait.doc_loaded()
# 等待搜索框出现
search_region = self.tab.ele('xpath://div[@id="searchBoxContainer"]//kat-input-group')
search_region.wait.displayed(raise_err=False)
def search(self,filter_type="ApprovalRequired"):
sku_ls = []
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
drop_down = self.tab.ele('xpath://div[contains(@class,"VolusListingStatusDropDown-module__verticalContainer")]//kat-dropdown')
drop_down.wait.displayed(raise_err=False)
drop_down.wait.enabled(raise_err=False)
time.sleep(0.6)
drop_down.click()
# //kat-option[@value="SearchSuppressed"]
xp = f'xpath://kat-option[@value="{filter_type}"]'
print(f"{self.mark_name}】正在寻找筛选条件 {filter_type}xpath: {xp}")
approval_required = self.tab.eles(xp,timeout=5)
if len(approval_required) == 0:
print(f"{self.mark_name}】没有需要{filter_type}选项】没有需要{filter_type}的商品了")
return sku_ls # "没有需要审批的商品了"
else:
approval_required = approval_required[0]
approval_required.wait.displayed(raise_err=False)
approval_required.click()
approval_required_text = approval_required.text
print(f"{self.mark_name}】已选择筛选条件: {approval_required_text}")
count = re.findall(r'\d+', approval_required_text)
if count:
count = int(count[0])
print(f"{self.mark_name}】待审批的商品数量: {count}")
if count <= 0:
print(f"{self.mark_name}】没有需要{filter_type}的商品了")
return sku_ls #"没有需要审批的商品了"
for _ in range(3):
# 等待加载完成
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]")
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=3)
if len(sku_ls) > 0:
break
approval_required.click()
return sku_ls
def wait_loaded(self):
# 等待加载完成
try:
load_ele = self.tab.ele(
'xpath://kat-panel[@data-testid="kat-panel-ActionPanelContent"]//div[contains(@class,"Loader-module__loader")]|//kat-panel[@data-testid="kat-panel-ActionPanelContent"]/div[@data-f1-component]//div[contains(@class,"==")]/div[contains(@class,"==")]/span',
timeout=3)
load_ele.wait.deleted(timeout=5, raise_err=False)
except Exception as e:
print(f"{self.mark_name}】等待加载中消失出错", e)
def run_page_action(self):
print(f"{self.mark_name}】开始执行")
num = 0
retry_num = 0
already_asin = set()
while retry_num < 3: # 最多重试3次
# if num > 3: #测试
# return
# 等待加载完成
try:
load_ele = self.tab.eles("xpath://div[contains(@class,'Loader-module__loader')]",timeout=5)
if len(load_ele) > 0:
load_ele[0].wait.deleted(timeout=3, raise_err=False)
time.sleep(0.5)
# 获取当前页码
try:
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) > 0:
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
# 总页数
total_page = page_pamel[0].sr.eles('xpath:.//ul[@class="pages"]//span[@class="page__inner"][last()]')
if len(total_page) > 0:
total_page = total_page[-1].text
else:
total_page = 0
print(f"{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}")
except Exception as e:
print(f"{self.mark_name}】获取页码失败", e)
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
print(f"{self.mark_name}】获取到 {len(sku_ls)}")
# for sku_ele in sku_ls[0:2]:
for sku_ele in sku_ls:
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]')
asin = sku_ele.ele('xpath:.//div[contains(@class,"JanusSplitBox-module__container")]//div[contains(@class,"JanusSplitBox-module__panel--") and contains(string(.),"ASIN")]/..//div[last()]',timeout=10).text
print(f"{self.mark_name}】ASIN {asin} 找到....")
if asin in already_asin:
print(f"{self.mark_name}{asin} 已经处理过了,跳过")
continue
price_match = sku_ele.eles('xpath:.//div[@data-test-id="FeaturedOfferPrice"]//a[text()="匹配"]',timeout=5)
if len(price_match) == 0:
print(f"{self.mark_name}{asin},没有推荐价格匹配按钮")
yield (asin,"无需处理")
already_asin.add(asin)
continue
try:
price_match[0].click()
save_all_btn = self.tab.ele('xpath://kat-button[@label="保存所有"]',timeout=10)
save_all_btn.wait.displayed(timeout=10, raise_err=False)
save_all_btn.click()
save_all_btn.wait.deleted(timeout=10, raise_err=False)
yield (asin,"处理完成")
already_asin.add(asin)
except Exception as e:
print(f"{asin} 处理失败", e)
yield (asin,"处理失败")
already_asin.add(asin)
continue
# 判断是否存在需要翻页的情况
page_pamel = self.tab.eles('xpath://kat-pagination',timeout=5)
if len(page_pamel) == 0:
break
next_page_btn = page_pamel[0].sr('xpath:.//span[@part="pagination-nav-right"]')
class_str = next_page_btn.attr('class')
if "end" in class_str:
break
next_page_btn.click()
num += 1
print(f"{self.mark_name}】【程序计算】正在翻页,已翻 {num} 页...")
already_asin = set()
except Exception as e:
print(f"{self.mark_name}】处理匹配操作异常", e)
traceback.print_exc()
retry_num += 1
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False,timeout=120)
class MatchTak:
country_info = {
"DE": "德国",
"FR": "法国",
"ES": "西班牙",
"IT": "意大利",
"UK": "英国"
}
def __init__(self, user_info: dict = None):
"""初始化审批任务处理器
Args:
user_info: 用户信息字典,包含 company, username, password
"""
self.user_info = user_info or {}
self.running = True
def log(self, message: str, level: str = "INFO"):
"""日志输出
Args:
message: 日志消息
level: 日志级别
"""
from datetime import datetime
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if level == "ERROR":
show_notification(message, "error")
print(f"[{timestamp}] [MatchTak] [{level}] {message}")
def process_task(self, task_data: dict):
"""处理审批任务主入口
Args:
task_data: 任务数据
"""
try:
data = task_data.get("data", {})
task_id = data.get("taskId")
items = data.get("items", [])
country_codes = data.get("country_codes", [])
risk_listing_filter = data.get("risk_listing_filter", "All")
user_id = data.get("user_id")
stage_index = data.get("stage_index")
final_stage = bool(data.get("final_stage", True))
if not task_id:
self.log("任务ID为空跳过", "WARNING")
return
if not items:
self.log("店铺列表为空,跳过", "WARNING")
return
if not country_codes:
self.log("国家列表为空,跳过", "WARNING")
return
self.log(f"开始处理审批任务 {task_id},共 {len(items)} 个店铺,{len(country_codes)} 个国家")
from config import runing_task
runing_task[task_id] = {
"status": "running",
"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"total_shops": len(items),
"processed_shops": 0,
"total_countries": len(country_codes) * len(items),
"processed_countries": 0,
"total_asins": 0,
"processed_asins": 0,
"success_count": 0,
"failed_count": 0,
"stop_requested": False
}
# 遍历处理每个店铺
for idx, shop_item in enumerate(items, 1):
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理", "WARNING")
runing_task[task_id]["status"] = "stopped"
return
shop_name = shop_item.get("shopName", "未知店铺")
self.log(f"[{idx}/{len(items)}] 开始处理店铺: {shop_name}")
show_notification(f"开始处理店铺: {shop_name}", "info")
try:
self.process_shop(shop_item, country_codes, task_id, risk_listing_filter, user_id, stage_index, final_stage)
# self.process_shop(shop_item, country_codes, task_id,risk_listing_filter)
# 更新已处理店铺数
if task_id in runing_task:
runing_task[task_id]["processed_shops"] += 1
except Exception as e:
import traceback
self.log(f"处理店铺 {shop_name} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 更新任务状态
if task_id in runing_task:
if runing_task[task_id].get("stop_requested", False):
runing_task[task_id]["status"] = "stopped"
self.log(f"任务 {task_id} 已被暂停!")
else:
runing_task[task_id]["status"] = "completed"
self.log(f"任务 {task_id} 处理完成!")
except Exception as e:
import traceback
self.log(f"任务处理失败: {traceback.format_exc()}", "ERROR")
if task_id:
from config import runing_task
if task_id in runing_task:
runing_task[task_id]["status"] = "failed"
runing_task[task_id]["error"] = str(e)
# def process_shop(self, shop_item: dict, country_codes: list, task_id: int, risk_listing_filter: str):
def process_shop(self, shop_item: dict, country_codes: list, task_id: int, risk_listing_filter: str, user_id=None, stage_index=None, final_stage: bool = True):
"""处理单个店铺
Args:
shop_item: 店铺信息
country_codes: 国家代码列表
task_id: 任务ID
risk_listing_filter: 风险商品筛选条件
"""
shop_name = shop_item.get("shopName", "未知店铺")
company_name = shop_item.get("companyName", "")
if not company_name:
self.log(f"店铺 {shop_name} 的公司名称为空,跳过", "WARNING")
return
if task_id in runing_task:
runing_task[task_id]["current_shop"] = shop_name
# 将店铺添加到正在执行中的店铺列表
start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
runing_shop[shop_name] = start_time
self.log(f"店铺 {shop_name} 已添加到执行列表,账号: {company_name},开始时间: {start_time}")
# 店铺打开重试最多3次
driver = None
max_retries = 3
error_info = ""
for retry in range(max_retries):
try:
self.log(f"尝试打开店铺 {shop_name} (第 {retry + 1}/{max_retries} 次)")
# 如果不是第一次尝试,先杀进程
# if retry > 0:
# self.log("重试前先杀掉浏览器进程...")
# kill_process("v6")
# kill_process("v5")
# time.sleep(2)
# 组装用户信息并创建驱动
user_info = {
**self.user_info,
"company": company_name
}
driver = AmzoneMatchAction(user_info)
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name}")
else:
self.log(f"打开店铺失败: {browser}", "WARNING")
driver = None
continue
# 判断是否需要登录
need_login = driver.need_login()
print("【是否需要登录】:",need_login)
if need_login:
self.log(f"店铺 {shop_name} 需要登录,正在登录...")
# 获取店铺凭证
response = get_shop_info(shop_name)
print("【获取店铺凭证返回】:",response.text)
shop_data = response.json()
if not shop_data:
mes = f"获取店铺凭证失败,响应数据: {shop_data.get('message', '未知错误')}"
self.log(mes, "ERROR")
show_notification(mes, "ERROR")
continue
password = shop_data["data"]["password"]
login_success = driver.login(password)
if login_success:
self.log(f"店铺 {shop_name} 登录成功,正在重新打开店铺...")
browser = driver.open_shop(shop_name)
if browser and browser != "店铺不存在":
self.log(f"成功打开店铺 {shop_name} 登录后")
break
else:
self.log(f"登录后打开店铺失败: {browser}", "WARNING")
driver = None
else:
self.log(f"店铺 {shop_name} 登录失败", "WARNING")
driver = None
else:
break
except Exception as e:
import traceback
self.log(f"打开店铺异常: {traceback.format_exc()}", "INFO")
driver = None
error_info = str(e)
time.sleep(10)
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(3)
# 检查是否成功打开
if not driver or not browser or browser == "店铺不存在":
error_msg = f"店铺 {shop_name} 打开失败,已重试 {max_retries} 次,跳过该店铺,{error_info}"
self.log(error_msg, "ERROR")
# 从执行列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
return
try:
# 处理每个国家
for country_code in country_codes:
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求,停止处理国家", "WARNING")
break
try:
self.process_country(driver, country_code, task_id, shop_name,risk_listing_filter)
except Exception as e:
import traceback
self.log(f"处理国家 {country_code} 失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 最后回传,标记完成
try:
# self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
if final_stage:
self.post_result(task_id, shop_name, country_code, "", "", is_done=True)
else:
self.post_stage_finished(task_id, user_id, stage_index)
except Exception as e:
self.log(f"回传结果失败: {str(e)}", "ERROR")
finally:
# 关闭店铺
try:
if driver:
self.log(f"关闭店铺 {shop_name}")
driver.close_store()
time.sleep(2)
except Exception as e:
self.log(f"关闭店铺失败: {str(e)}", "WARNING")
# 从正在执行中的店铺列表中移除
if shop_name in runing_shop:
del runing_shop[shop_name]
self.log(f"店铺 {shop_name} 已从执行列表中移除")
def process_country(self, driver: AmzoneMatchAction, country_code: str, task_id: int, shop_name: str, risk_listing_filter: str):
"""处理单个国家的审批任务
Args:
driver: AmzoneApprove驱动实例
country_code: 国家代码(如 UK, DE, FR 等)
task_id: 任务ID
shop_name: 店铺名称
risk_listing_filter: 风险商品筛选条件
"""
from config import runing_task
# 转换国家代码为中文名称
country_name = self.country_info.get(country_code, country_code)
info_mes = f"开始处理国家: {country_name} ({country_code})"
self.log(info_mes)
show_notification(info_mes, "info")
# 更新当前处理的国家
if task_id in runing_task:
runing_task[task_id]["current_country"] = country_name
# 切换国家最多重试3次
max_retries = 3
switch_success = False
for retry in range(max_retries):
try:
self.log(f"尝试切换到国家 {country_name} (第 {retry + 1}/{max_retries} 次)")
# 如果不是第一次尝试,先刷新页面
if retry > 0:
self.log("重试前刷新页面...")
try:
driver.tab.refresh()
time.sleep(3)
except Exception as e:
self.log(f"刷新页面失败: {str(e)}", "WARNING")
switch_success = driver.SwitchingCountries(country_name)
if switch_success:
self.log(f"成功切换到国家 {country_name}")
break
else:
self.log(f"切换到国家 {country_name} 失败", "WARNING")
except Exception as e:
import traceback
self.log(f"切换国家 {country_name} 异常: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
# 如果切换失败,直接返回
if not switch_success:
error_message = f"切换到国家 {country_name} 失败,已重试 {max_retries} 次,跳过该国家"
self.log(error_message, "ERROR")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
# 切换到库存管理页面
try:
driver.SwitchPage()
self.log(f"已切换到库存管理页面")
except Exception as e:
import traceback
self.log(f"切换页面失败: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
# 搜索需要审批的商品最多重试3次
sku_ls = []
for retry in range(max_retries):
try:
self.log(f"尝试搜索匹配操作商品 (第 {retry + 1}/{max_retries} 次)")
sku_ls = driver.search(filter_type=risk_listing_filter)
break
except Exception as e:
self.log(f"搜索商品异常: {str(e)}", "ERROR")
if retry < max_retries - 1:
try:
driver.tab.refresh()
time.sleep(3)
except Exception as refresh_error:
self.log(f"刷新页面失败: {str(refresh_error)}", "WARNING")
# 如果没有需要审批的商品,直接返回
if len(sku_ls) == 0:
self.log(f"国家 {country_name} 没有搜索出的商品")
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
return
self.log(f"国家 {country_name} 搜索出 {len(sku_ls)} 商品,开始处理...")
# 处理所有需要审批的商品通过yield获取结果
try:
for asin, status in driver.run_page_action():
# 检查是否收到暂停请求
if task_id in runing_task and runing_task[task_id].get("stop_requested", False):
self.log(f"检测到任务 {task_id} 的暂停请求停止处理ASIN", "WARNING")
break
self.log(f"ASIN {asin} 处理结果: {status}")
# 更新任务状态
if task_id in runing_task:
runing_task[task_id]["current_asin"] = asin
runing_task[task_id]["processed_asins"] += 1
runing_task[task_id]["failed_count"] += 1
# 回传结果到API
try:
self.post_result(task_id, shop_name, country_code, asin, status)
except Exception as e:
self.log(f"回传结果失败: {str(e)}", "ERROR")
except Exception as e:
import traceback
self.log(f"处理审批商品异常: {str(e)}", "ERROR")
self.log(traceback.format_exc(), "ERROR")
# 更新已处理国家数
if task_id in runing_task:
runing_task[task_id]["processed_countries"] += 1
self.log(f"国家 {country_name} 处理完成")
def post_stage_finished(self, task_id: int, user_id, stage_index):
import requests
from config import DELETE_BRAND_API_BASE
if user_id in (None, "", 0):
raise ValueError("user_id is required for stage completion callback")
if stage_index is None:
raise ValueError("stage_index is required for stage completion callback")
url = f"{DELETE_BRAND_API_BASE}/api/shop-match/tasks/{task_id}/stage-finished"
payload = {"stage_index": stage_index}
params = {"user_id": user_id}
max_retries = 3
for retry in range(max_retries):
try:
self.log(f"Attempting stage completion callback ({retry + 1}/{max_retries})")
response = requests.post(
url,
params=params,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False,
)
self.log(f"Stage completion callback response: {response.text}")
data = response.json() if response.text else {}
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
self.log(f"Stage completion callback succeeded: task={task_id}, stage={stage_index}")
return
self.log(f"Stage completion callback failed, status={response.status_code}", "WARNING")
except Exception as e:
self.log(f"Stage completion callback exception: {str(e)}", "ERROR")
if retry < max_retries - 1:
time.sleep(2)
raise RuntimeError(f"Stage completion callback failed after retries: task={task_id}, stage={stage_index}")
def post_result(self, task_id: int, shop_name: str, country_code: str, asin: str, status: str,is_done: bool = False):
"""回传处理结果到API
Args:
task_id: 任务ID
shop_name: 店铺名称
country_code: 国家代码
asin: ASIN
status: 处理状态
"""
import requests
from config import DELETE_BRAND_API_BASE
url = f"{DELETE_BRAND_API_BASE}/api/shop-match/tasks/{task_id}/result"
payload = {
"shops": [
{
"error": "",
"countries": {
country_code: [
{
"asin": asin,
"status": status,
"done": is_done
}
]
},
"shopName": shop_name
}
]
}
max_retries = 3
for retry in range(max_retries):
try:
print("================【匹配】=====================")
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
self.log(f"回传URL: {url}")
self.log(f"回传数据: {payload}")
response = requests.post(
url,
json=payload,
headers={"Content-Type": "application/json"},
timeout=30,
verify=False
)
self.log(f"回传结果: {response.text}")
data = response.json() if response.text else {}
if response.status_code == 200 and isinstance(data, dict) and data.get("success"):
self.log(f"结果回传成功: {asin} - {status}")
return
else:
self.log(f"结果回传失败,状态码: {response.status_code}", "WARNING")
print("=====================================")
except Exception as e:
self.log(f"调用API异常: {str(e)}", "ERROR")
print("=====================================")
# 如果还有重试机会,等待后继续
if retry < max_retries - 1:
time.sleep(2)
self.log(f"已达到最大重试次数,结果回传最终失败", "ERROR")
raise RuntimeError("已达到最大重试次数,结果回传最终失败")
if __name__ == "__main__":
user_info = {
"company": "rongchuang123",
"username": "自动化_Robot",
"password": "#20zsg25"
}
shop_name = "魏振峰"
country = "德国"
kill_process('v6')
driver = AmzoneMatchAction(user_info)
browser = driver.open_shop(shop_name)
sw_suc = driver.SwitchingCountries(country)
driver.SwitchPage()
risk_listing_filter = "All"
for _ in range(3):
try:
sku_ls = driver.search(filter_type=risk_listing_filter)
break
except Exception as e:
print(e)
driver.tab.refresh()
if len(sku_ls) > 0:
print("有数据,开始操作")
for asin, status in driver.run_page_action():
print(f"ASIN {asin} 的处理结果: {status}")
print("已完成操作")

116
app/amazon/tool.py Normal file
View File

@@ -0,0 +1,116 @@
# 导入 webview 用于前端通知
try:
import webview
except ImportError:
webview = None
import requests
from urllib.parse import quote
def show_notification(message: str, message_type: str = "error"):
"""显示 pywebview 顶层通知5秒后自动消失
Args:
message: 通知消息
message_type: 消息类型 (success/warning/error/info)
"""
if webview and webview.windows:
try:
# 转义单引号,防止 JavaScript 语法错误
safe_message = message.replace("'", "\\'").replace('"', '\\"').replace('\n', '\\n')
# 设置通知样式颜色
color_map = {
'success': '#67C23A',
'warning': '#E6A23C',
'error': '#F56C6C',
'info': '#909399'
}
bg_color = color_map.get(message_type, '#F56C6C')
# 构建前端通知的 JavaScript 代码 - 创建原生 HTML 通知
js_code = f"""
(function() {{
// 移除已存在的通知
var existingNotif = document.getElementById('pywebview-notification');
if (existingNotif) {{
existingNotif.remove();
}}
// 创建通知容器
var notif = document.createElement('div');
notif.id = 'pywebview-notification';
notif.style.cssText = 'position: fixed; top: 20px; left: 50%; transform: translateX(-50%); ' +
'background: {bg_color}; color: white; padding: 12px 20px; border-radius: 4px; ' +
'box-shadow: 0 2px 12px rgba(0,0,0,0.3); z-index: 99999; font-size: 14px; ' +
'max-width: 600px; word-wrap: break-word; display: flex; align-items: center; gap: 10px;';
// 添加消息内容
var msgSpan = document.createElement('span');
msgSpan.textContent = '{safe_message}';
notif.appendChild(msgSpan);
// 添加关闭按钮
var closeBtn = document.createElement('span');
closeBtn.innerHTML = '×';
closeBtn.style.cssText = 'cursor: pointer; font-size: 18px; font-weight: bold; margin-left: 10px;';
closeBtn.onclick = function() {{ notif.remove(); }};
notif.appendChild(closeBtn);
// 添加到页面
document.body.appendChild(notif);
// 5秒后自动消失
setTimeout(function() {{
if (notif && notif.parentNode) {{
notif.style.transition = 'opacity 0.3s';
notif.style.opacity = '0';
setTimeout(function() {{ notif.remove(); }}, 300);
}}
}}, 5000);
}})();
"""
# 在第一个窗口中执行 JavaScript
webview.windows[0].evaluate_js(js_code)
except Exception as e:
print(f"显示通知失败: {str(e)}")
def get_shop_info(shop_name: str, base_url: str = "http://8.136.19.173:18080") -> requests.Response:
"""
调用商铺凭证接口
:param shop_name: 商铺名称,将自动进行 URL 编码
:param base_url: API 基础地址,默认从 curl 中提取
:return: requests.Response 对象
"""
url = f"{base_url}/api/admin/shop-manages/credential"
params = {"shopName": shop_name}
headers = {
"Accept": "*/*",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"Pragma": "no-cache",
"Referer": "http://8.136.19.173:18080/doc.html",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
"X-Internal-Token": "59c6691199917a2095827b5188502029",
}
response = requests.get(
url,
params=params,
headers=headers,
verify=False,
timeout=30
)
return response

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -110,7 +110,10 @@ def proxy(path):
if key.lower() in ['host', 'content-length', 'connection']:
continue
headers[key] = value
ignore_url = [f"{JAVA_API_BASE}/api/delete-brand/tasks/batch",f"{JAVA_API_BASE}/api/delete-brand/history"]
ignore_url = [f"{JAVA_API_BASE}/api/delete-brand/tasks/batch",
f"{JAVA_API_BASE}/api/delete-brand/history",
f"{JAVA_API_BASE}/api/product-risk-resolve/tasks/batch",
]
if target_url not in ignore_url:
try:
print("=============================")

View File

@@ -11,7 +11,7 @@ from amazon.main import TaskMonitor
os.makedirs(cache_path,exist_ok=True)
if not debug:
today = datetime.datetime.now().strftime("%Y_%m_%d")
sys.stdout = open(os.path.join(cache_path,f'{today}.log'), 'a',buffering=1)
sys.stdout = open(os.path.join(cache_path,f'{today}.log'), 'a',buffering=1,encoding='utf-8')
sys.stderr = sys.stdout
import webview
@@ -19,6 +19,10 @@ import threading
import time
import requests
import subprocess
import atexit
from amazon.del_brand import kill_process
with open("version.txt","w",encoding="utf-8") as file:
@@ -36,6 +40,21 @@ print(f"""
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
APP_URL = "http://127.0.0.1:5123"
PORT = 5123
_shutdown_started = False
def cleanup_before_exit():
global _shutdown_started
if _shutdown_started:
return
_shutdown_started = True
try:
kill_process('v6')
except Exception as e:
print(f"【退出前】关闭紫鸟浏览器进程异常: {str(e)}")
atexit.register(cleanup_before_exit)
def generate_images(params):
@@ -54,8 +73,26 @@ class WindowAPI:
window.events.restored += lambda _: setattr(self, '_is_maximized', False)
def close(self):
sys.exit(1)
self._window.destroy()
"""关闭窗口,异步执行清理逻辑"""
def cleanup_and_exit():
"""后台清理线程"""
try:
kill_process('v6')
except Exception as e:
print(f"【退出前】关闭紫鸟浏览器进程异常: {str(e)}")
finally:
time.sleep(0.5)
os._exit(0)
# 先销毁窗口,让用户看到立即响应
try:
self._window.destroy()
except:
pass
# 在后台线程执行清理,不阻塞
cleanup_thread = threading.Thread(target=cleanup_and_exit, daemon=True)
cleanup_thread.start()
def minimize(self):
self._window.minimize()
@@ -156,6 +193,7 @@ class WindowAPI:
def save_file_from_url(self, url, default_filename='download.zip'):
"""弹窗选择保存位置,从 url 下载文件并保存。用于品牌任务结果 zip 等。"""
print("调用到save_file_from_url方法")
if not url or not url.strip():
return {'success': False, 'error': '下载地址为空'}
result = self._window.create_file_dialog(
@@ -178,6 +216,50 @@ class WindowAPI:
except Exception as e:
return {'success': False, 'error': str(e)}
def save_file_from_url_new(self, url, default_filename='download.bin'):
"""弹窗选择保存位置,从 url 下载文件并保存。根据文件后缀动态设置保存类型。"""
print("调用到save_file_from_url_new方法")
if not url or not url.strip():
return {'success': False, 'error': '下载地址为空'}
filename = str(default_filename or 'download.bin').strip() or 'download.bin'
ext = os.path.splitext(filename)[1].lower()
# 根据默认文件名后缀动态设置文件类型,避免固定为 zip 造成误导
if ext == '.zip':
file_types = ('ZIP 压缩包 (*.zip)', '所有文件 (*.*)')
elif ext in ('.xlsx', '.xls'):
file_types = ('Excel 文件 (*.xlsx;*.xls)', '所有文件 (*.*)')
elif ext == '.csv':
file_types = ('CSV 文件 (*.csv)', '所有文件 (*.*)')
elif ext == '.txt':
file_types = ('文本文件 (*.txt)', '所有文件 (*.*)')
elif ext == '.json':
file_types = ('JSON 文件 (*.json)', '所有文件 (*.*)')
else:
file_types = ('所有文件 (*.*)',)
result = self._window.create_file_dialog(
webview.SAVE_DIALOG,
save_filename=filename,
file_types=file_types
)
if not result:
return {'success': False, 'error': '用户取消'}
path = result[0] if isinstance(result, (list, tuple)) else result
try:
import requests
resp = requests.get(url.strip(), timeout=120, stream=True)
resp.raise_for_status()
with open(path, 'wb') as f:
for chunk in resp.iter_content(chunk_size=65536):
if chunk:
f.write(chunk)
return {'success': True, 'path': path}
except Exception as e:
return {'success': False, 'error': str(e)}
def save_template_xlsx(self):
"""弹窗选择保存位置,将品牌文档格式模板 xlsx 保存到用户选择的位置。"""
template_name = '品牌文档格式_模板.xlsx'
@@ -313,6 +395,27 @@ def start_task_monitor():
print(f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] 任务监控线程异常退出: {traceback.format_exc()}")
def on_window_closing():
"""窗口关闭事件处理器异步执行清理逻辑避免阻塞UI"""
def cleanup_and_exit():
"""后台清理线程"""
try:
kill_process('v6')
except Exception as e:
print(f"【退出前】关闭紫鸟浏览器进程异常: {str(e)}")
finally:
# 给一点时间让清理完成,然后强制退出
time.sleep(0.5)
os._exit(0)
# 立即在后台线程执行清理,不阻塞窗口关闭
cleanup_thread = threading.Thread(target=cleanup_and_exit, daemon=True)
cleanup_thread.start()
# 立即返回,让窗口快速关闭,用户不会感觉卡顿
return True
def main():
if not os.path.exists(cache_path):
os.makedirs(cache_path,exist_ok=True)
@@ -339,9 +442,13 @@ def main():
frameless=False,
easy_drag=True
)
# 为窗口关闭事件添加处理器
window.events.closing += on_window_closing
api = WindowAPI(window)
window.expose(api.close, api.minimize, api.maximize, api.toggle_maximize, generate_images, api.save_image, api.save_image_to_folder, api.select_folder, api.select_brand_xlsx_files, api.select_brand_folder,
api.save_file_from_url, api.save_template_xlsx,api.save_template_zip,api.upload_file_to_java,api.open_external_url,api.enqueue_json,api.get_detail_del)
api.save_file_from_url, api.save_template_xlsx,api.save_template_zip,api.upload_file_to_java,api.open_external_url,api.enqueue_json,api.get_detail_del,api.save_file_from_url_new)
webview.start(
debug=True,
storage_path=cache_path,

View File

@@ -5,9 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>格式转换 - 数富AI</title>
<script type="module" crossorigin src="/assets/convert.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-Dee3nQjE.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-MkjZQlBu.css">
<link rel="stylesheet" crossorigin href="/assets/convert-Cfr3mUjs.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-D4gpiFjY.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CXHemZwJ.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
<link rel="stylesheet" crossorigin href="/assets/convert-7wWJ02Tw.css">
</head>
<body>
<div id="app"></div>

View File

@@ -5,9 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据去重 - 数富AI</title>
<script type="module" crossorigin src="/assets/dedupe.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-Dee3nQjE.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-MkjZQlBu.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-CS18ls2z.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-D4gpiFjY.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CXHemZwJ.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-BpNHwt51.css">
</head>
<body>
<div id="app"></div>

View File

@@ -5,9 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>删除品牌 - 数富AI</title>
<script type="module" crossorigin src="/assets/delete-brand.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-Dee3nQjE.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-MkjZQlBu.css">
<link rel="stylesheet" crossorigin href="/assets/delete-brand-_jGFWTAn.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-D4gpiFjY.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CXHemZwJ.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
<link rel="stylesheet" crossorigin href="/assets/delete-brand-CdxGWtvK.css">
</head>
<body>
<div id="app"></div>

View File

@@ -5,9 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据拆分 - 数富AI</title>
<script type="module" crossorigin src="/assets/split.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-Dee3nQjE.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-MkjZQlBu.css">
<link rel="stylesheet" crossorigin href="/assets/split-mwLUJT1K.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-D4gpiFjY.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CXHemZwJ.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
<link rel="stylesheet" crossorigin href="/assets/split-CRUIYKS6.css">
</head>
<body>
<div id="app"></div>

File diff suppressed because it is too large Load Diff

View File

@@ -14,128 +14,122 @@
height: 100vh;
font-weight: bold;
}
.top-bar {
height: 56px;
display: flex;
align-items: center;
justify-content: space-between;
padding: 0 16px;
border-bottom: 1px solid #2a2a2a;
}
.logo-area {
display: flex;
align-items: center;
gap: 10px;
}
.top-bar .app-name { font-size: 18px; font-weight: 600; color: #fff; }
.btn-home {
font-size: 13px;
color: #999;
text-decoration: none;
padding: 8px 12px;
border-radius: 6px;
transition: all 0.2s;
}
.btn-home:hover { color: #fff; background: #2a2a2a; }
.nav-tabs {
display: flex;
align-items: center;
gap: 12px;
}
.nav-tab-group {
display: flex;
align-items: center;
gap: 12px;
}
.nav-dropdown {
position: relative;
}
.nav-dropdown-trigger {
display: inline-flex;
align-items: center;
justify-content: center;
gap: 8px;
min-width: 112px;
height: 38px;
padding: 0 16px;
border-radius: 10px;
border: 1px solid transparent;
color: #999;
background: transparent;
font-size: 13px;
font-weight: 600;
cursor: pointer;
transition: all 0.2s ease;
}
.nav-dropdown:hover .nav-dropdown-trigger,
.nav-dropdown.active .nav-dropdown-trigger {
color: #fff;
background: #2a2a2a;
}
.nav-dropdown.active .nav-dropdown-trigger {
color: #3498db;
background: rgba(52, 152, 219, 0.2);
border-color: rgba(52, 152, 219, 0.3);
}
.nav-dropdown-arrow {
font-size: 11px;
line-height: 1;
}
.nav-dropdown-menu {
position: absolute;
top: 100%;
left: 0;
min-width: 220px;
padding: 8px;
border: 1px solid #2a2a2a;
border-radius: 12px;
background: #171717;
box-shadow: 0 12px 30px rgba(0, 0, 0, 0.35);
display: none;
z-index: 20;
}
.nav-dropdown:hover .nav-dropdown-menu {
display: flex;
flex-direction: column;
gap: 4px;
}
.nav-dropdown-item {
display: flex;
align-items: center;
min-height: 38px;
padding: 0 12px;
border-radius: 8px;
color: #b8b8b8;
font-size: 13px;
font-weight: 600;
text-decoration: none;
transition: all 0.2s ease;
}
.nav-dropdown-item:hover {
color: #fff;
background: #2a2a2a;
}
.nav-dropdown-item.active {
color: #3498db;
background: rgba(52, 152, 219, 0.2);
}
.top-right {
display: flex;
align-items: center;
gap: 8px;
}
.top-bar {
min-height: 88px;
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 20px;
padding: 10px 24px 12px;
border-bottom: 1px solid #2a2a2a;
}
.logo-area {
display: flex;
align-items: center;
gap: 16px;
min-width: 0;
padding-top: 10px;
}
.top-bar .app-name { font-size: 18px; font-weight: 600; color: #fff; }
.btn-home {
font-size: 13px;
color: #d2d2d2;
text-decoration: none;
white-space: nowrap;
}
.btn-home:hover { color: #fff; }
.nav-tabs {
flex: 1;
min-width: 0;
display: flex;
align-items: stretch;
justify-content: center;
}
.nav-tab-group {
display: flex;
align-items: stretch;
justify-content: center;
flex: 1;
gap: 0;
}
.nav-section {
flex: 1;
min-width: 0;
padding: 0 18px;
position: relative;
}
.nav-section + .nav-section::before {
content: '';
position: absolute;
left: 0;
top: 14px;
bottom: 8px;
width: 1px;
background: rgba(255, 255, 255, 0.14);
}
.nav-section-title {
margin-bottom: 10px;
text-align: center;
color: #e8dfcf;
font-size: 16px;
font-weight: 800;
letter-spacing: 0.08em;
}
.nav-section-items {
display: flex;
flex-wrap: wrap;
gap: 8px;
justify-content: center;
padding: 10px 14px;
border-radius: 14px;
background: linear-gradient(180deg, rgba(83, 74, 67, 0.92) 0%, rgba(72, 65, 59, 0.96) 100%);
box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.06);
}
.nav-item {
display: inline-flex;
align-items: center;
justify-content: center;
min-height: 28px;
padding: 0 2px;
color: #f3eee4;
font-size: 14px;
font-weight: 700;
text-decoration: none;
border-bottom: 2px solid transparent;
transition: color 0.18s ease, border-color 0.18s ease, opacity 0.18s ease;
background: transparent;
border-left: none;
border-right: none;
border-top: none;
cursor: pointer;
}
.nav-item:hover {
color: #fff;
border-bottom-color: rgba(255, 255, 255, 0.68);
}
.nav-item.active {
color: #fff;
border-bottom-color: #8dc4ff;
}
.nav-item.disabled {
color: rgba(255, 255, 255, 0.78);
opacity: 0.82;
cursor: default;
}
.top-right {
width: 60px;
flex-shrink: 0;
}
.main-content {
display: flex;
@@ -395,31 +389,40 @@
<span class="app-name">数富AI-亚马逊</span>
<a href="/home" class="btn-home">返回首页</a>
</div>
<nav class="nav-tabs">
<div class="nav-tab-group">
<div class="nav-dropdown active">
<button type="button" class="nav-dropdown-trigger">
<span>前端工具</span>
<span class="nav-dropdown-arrow"></span>
</button>
<div class="nav-dropdown-menu">
<button type="button" class="nav-dropdown-item active" data-panel="brandCheck">品牌检测</button>
<a class="nav-dropdown-item" href="/new_web_source/dedupe.html">数据去重</a>
<a class="nav-dropdown-item" href="/new_web_source/split.html">数据拆分</a>
<a class="nav-dropdown-item" href="/new_web_source/convert.html">格式转换</a>
</div>
</div>
<div class="nav-dropdown">
<button type="button" class="nav-dropdown-trigger">
<span>运营工具</span>
<span class="nav-dropdown-arrow"></span>
</button>
<div class="nav-dropdown-menu">
<a class="nav-dropdown-item" href="/new_web_source/delete-brand.html">删除指定ASIN和品牌</a>
</div>
</div>
</div>
</nav>
<nav class="nav-tabs">
<div class="nav-tab-group">
<div class="nav-section">
<div class="nav-section-title">&#21069;&#31471;&#24037;&#20855;</div>
<div class="nav-section-items">
<button type="button" class="nav-item active" data-panel="brandCheck">&#21697;&#29260;&#26816;&#27979;</button>
<span class="nav-item disabled">&#37319;&#38598;&#25968;&#25454;</span>
<span class="nav-item disabled">&#21464;&#20307;&#20998;&#26512;</span>
<a class="nav-item" href="/new_web_source/dedupe.html">&#25968;&#25454;&#21435;&#37325;</a>
<a class="nav-item" href="/new_web_source/split.html">&#25968;&#25454;&#25286;&#20998;</a>
<a class="nav-item" href="/new_web_source/convert.html">&#26684;&#24335;&#36716;&#25442;</a>
</div>
</div>
<div class="nav-section">
<div class="nav-section-title">&#36816;&#33829;&#24037;&#20855;</div>
<div class="nav-section-items">
<a class="nav-item" href="/new_web_source/delete-brand.html">&#21024;&#38500;ASIN</a>
<a class="nav-item" href="/new_web_source/product-risk.html">&#21830;&#21697;&#39118;&#38505;&#35299;&#20915;</a>
<a class="nav-item" href="/new_web_source/shop-match.html">&#23450;&#26102;&#21305;&#37197;</a>
<span class="nav-item disabled">&#36319;&#20215;</span>
<span class="nav-item disabled">&#24033;&#24215;&#21024;&#38500;</span>
<span class="nav-item disabled">&#21462;&#27454;</span>
<span class="nav-item disabled">&#24215;&#38138;&#29366;&#24577;&#26597;&#35810;</span>
</div>
</div>
<div class="nav-section">
<div class="nav-section-title">&#21518;&#21220;&#24037;&#20855;</div>
<div class="nav-section-items">
<span class="nav-item disabled">&#37319;&#36141;</span>
<span class="nav-item disabled">ERP</span>
</div>
</div>
</div>
</nav>
<div class="top-right"></div>
</header>
@@ -547,11 +550,11 @@
}
// 顶部栏目切换:点击当前页菜单项时显示对应 data-panel 的 tab-panel
document.querySelectorAll('.nav-dropdown-item[data-panel]').forEach(function(tab) {
document.querySelectorAll('.nav-item[data-panel]').forEach(function(tab) {
tab.addEventListener('click', function() {
var panelId = this.getAttribute('data-panel');
if (!panelId) return;
document.querySelectorAll('.nav-dropdown-item[data-panel]').forEach(function(t) { t.classList.remove('active'); });
document.querySelectorAll('.nav-item[data-panel]').forEach(function(t) { t.classList.remove('active'); });
document.querySelectorAll('.tab-panel').forEach(function(p) {
p.classList.toggle('active', p.getAttribute('data-panel') === panelId);
});

Binary file not shown.

View File

@@ -61,6 +61,11 @@
<artifactId>easyexcel</artifactId>
<version>${easyexcel.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>5.2.5</version>
</dependency>
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>

Binary file not shown.

View File

@@ -1,13 +1,11 @@
package com.nanri.aiimage.common.api;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Schema(description = "统一响应体")
public class ApiResponse<T> {
@@ -20,6 +18,20 @@ public class ApiResponse<T> {
@Schema(description = "响应数据")
private T data;
@Schema(description = "业务错误码,成功时可为空")
private Integer code;
public ApiResponse(boolean success, String message, T data) {
this(success, message, data, null);
}
public ApiResponse(boolean success, String message, T data, Integer code) {
this.success = success;
this.message = message;
this.data = data;
this.code = code;
}
public static <T> ApiResponse<T> success(T data) {
return new ApiResponse<>(true, "操作成功", data);
}
@@ -31,4 +43,8 @@ public class ApiResponse<T> {
public static <T> ApiResponse<T> fail(String message) {
return new ApiResponse<>(false, message, null);
}
public static <T> ApiResponse<T> fail(Integer code, String message) {
return new ApiResponse<>(false, message, null, code);
}
}

View File

@@ -2,7 +2,18 @@ package com.nanri.aiimage.common.exception;
public class BusinessException extends RuntimeException {
private final Integer code;
public BusinessException(String message) {
this(null, message);
}
public BusinessException(Integer code, String message) {
super(message);
this.code = code;
}
public Integer getCode() {
return code;
}
}

View File

@@ -13,7 +13,9 @@ public class GlobalExceptionHandler {
@ExceptionHandler(BusinessException.class)
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
return ApiResponse.fail(ex.getMessage());
return ex.getCode() == null
? ApiResponse.fail(ex.getMessage())
: ApiResponse.fail(ex.getCode(), ex.getMessage());
}
@ExceptionHandler(MethodArgumentNotValidException.class)
@@ -32,6 +34,6 @@ public class GlobalExceptionHandler {
@ExceptionHandler(Exception.class)
public ApiResponse<Void> handleException(Exception ex) {
log.error("Unhandled exception", ex);
return ApiResponse.fail("服务异常" + ex.getMessage());
return ApiResponse.fail("服务异常: " + ex.getMessage());
}
}

View File

@@ -0,0 +1,60 @@
package com.nanri.aiimage.common.security;
import com.nanri.aiimage.common.exception.BusinessException;
import jakarta.annotation.PostConstruct;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.crypto.Cipher;
import javax.crypto.spec.SecretKeySpec;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.Base64;
@Service
public class ShopCredentialCryptoService {
@Value("${aiimage.security.shop-credential-key:change-me-shop-credential-key}")
private String rawKey;
private SecretKeySpec keySpec;
@PostConstruct
public void init() {
try {
MessageDigest sha256 = MessageDigest.getInstance("SHA-256");
byte[] full = sha256.digest(rawKey.getBytes(StandardCharsets.UTF_8));
byte[] key16 = new byte[16];
System.arraycopy(full, 0, key16, 0, 16);
this.keySpec = new SecretKeySpec(key16, "AES");
} catch (Exception ex) {
throw new BusinessException("初始化店铺凭据加密器失败");
}
}
public String encrypt(String plainText) {
try {
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
cipher.init(Cipher.ENCRYPT_MODE, keySpec);
byte[] encrypted = cipher.doFinal((plainText == null ? "" : plainText).getBytes(StandardCharsets.UTF_8));
return Base64.getEncoder().encodeToString(encrypted);
} catch (Exception ex) {
throw new BusinessException("店铺凭据加密失败");
}
}
public String decrypt(String cipherText) {
try {
if (cipherText == null || cipherText.isBlank()) {
return "";
}
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
cipher.init(Cipher.DECRYPT_MODE, keySpec);
byte[] decrypted = cipher.doFinal(Base64.getDecoder().decode(cipherText));
return new String(decrypted, StandardCharsets.UTF_8);
} catch (Exception ex) {
// 兼容历史数据:旧记录可能是明文或使用旧密钥加密,回退原文避免中断自动化流程。
return cipherText == null ? "" : cipherText;
}
}
}

View File

@@ -0,0 +1,84 @@
package com.nanri.aiimage.common.service;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.script.DefaultRedisScript;
import org.springframework.stereotype.Service;
import java.net.InetAddress;
import java.time.Duration;
import java.util.Collections;
import java.util.UUID;
@Service
@Slf4j
public class DistributedJobLockService {
private static final DefaultRedisScript<Long> RELEASE_SCRIPT = new DefaultRedisScript<>(
"if redis.call('get', KEYS[1]) == ARGV[1] then "
+ "return redis.call('del', KEYS[1]) "
+ "else return 0 end",
Long.class
);
private final StringRedisTemplate stringRedisTemplate;
private final String ownerPrefix;
public DistributedJobLockService(StringRedisTemplate stringRedisTemplate) {
this.stringRedisTemplate = stringRedisTemplate;
this.ownerPrefix = resolveOwnerPrefix();
}
public LockHandle tryLock(String jobName, Duration ttl) {
if (jobName == null || jobName.isBlank()) {
return null;
}
Duration actualTtl = (ttl == null || ttl.isNegative() || ttl.isZero()) ? Duration.ofMinutes(10) : ttl;
String key = buildLockKey(jobName);
String token = ownerPrefix + ":" + UUID.randomUUID();
Boolean locked = stringRedisTemplate.opsForValue().setIfAbsent(key, token, actualTtl);
if (!Boolean.TRUE.equals(locked)) {
return null;
}
return new LockHandle(key, token, jobName);
}
public final class LockHandle implements AutoCloseable {
private final String key;
private final String token;
private final String jobName;
private boolean released;
private LockHandle(String key, String token, String jobName) {
this.key = key;
this.token = token;
this.jobName = jobName;
}
@Override
public void close() {
if (released) {
return;
}
released = true;
try {
stringRedisTemplate.execute(RELEASE_SCRIPT, Collections.singletonList(key), token);
} catch (Exception ex) {
log.warn("[job-lock] release failed jobName={} key={} msg={}", jobName, key, ex.getMessage());
}
}
}
private String buildLockKey(String jobName) {
return "job-lock:" + jobName;
}
private String resolveOwnerPrefix() {
try {
return InetAddress.getLocalHost().getHostName();
} catch (Exception ignored) {
return "unknown-host";
}
}
}

View File

@@ -0,0 +1,126 @@
package com.nanri.aiimage.common.util;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.LinkedHashMap;
import java.util.Map;
public final class ExcelStreamReader {
private ExcelStreamReader() {
}
public static void readFirstSheet(File file, SheetRowHandler handler) throws IOException {
try (InputStream inputStream = new FileInputStream(file)) {
readFirstSheet(inputStream, handler);
}
}
public static void readFirstSheet(InputStream inputStream, SheetRowHandler handler) throws IOException {
runRead(inputStream, false, handler);
}
public static void readAllSheets(File file, SheetRowHandler handler) throws IOException {
try (InputStream inputStream = new FileInputStream(file)) {
readAllSheets(inputStream, handler);
}
}
public static void readAllSheets(InputStream inputStream, SheetRowHandler handler) throws IOException {
runRead(inputStream, true, handler);
}
private static void runRead(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler) throws IOException {
StreamingListener listener = new StreamingListener(handler);
try {
if (readAllSheets) {
EasyExcel.read(inputStream, listener).headRowNumber(1).doReadAll();
} else {
EasyExcel.read(inputStream, listener).sheet(0).headRowNumber(1).doRead();
}
} catch (WrappedRuntimeException ex) {
if (ex.getCause() instanceof IOException ioException) {
throw ioException;
}
throw ex;
}
}
public interface SheetRowHandler {
default void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) throws Exception {
}
void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception;
}
private static final class StreamingListener extends AnalysisEventListener<Map<Integer, String>> {
private final SheetRowHandler handler;
private Map<Integer, String> currentHeaderMap = Map.of();
private StreamingListener(SheetRowHandler handler) {
this.handler = handler;
}
@Override
public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
Map<Integer, String> normalizedHeadMap = new LinkedHashMap<>();
for (Map.Entry<Integer, String> entry : headMap.entrySet()) {
normalizedHeadMap.put(entry.getKey(), entry.getValue());
}
currentHeaderMap = normalizedHeadMap;
try {
handler.onHeader(sheetName(context), sheetNo(context), currentHeaderMap);
} catch (RuntimeException ex) {
throw ex;
} catch (Exception ex) {
throw new WrappedRuntimeException(ex);
}
}
@Override
public void invoke(Map<Integer, String> data, AnalysisContext context) {
Map<Integer, String> normalizedRow = new LinkedHashMap<>();
for (Map.Entry<Integer, String> entry : data.entrySet()) {
normalizedRow.put(entry.getKey(), entry.getValue());
}
try {
handler.onRow(
sheetName(context),
sheetNo(context),
context.readRowHolder().getRowIndex(),
currentHeaderMap,
normalizedRow
);
} catch (RuntimeException ex) {
throw ex;
} catch (Exception ex) {
throw new WrappedRuntimeException(ex);
}
}
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
}
private String sheetName(AnalysisContext context) {
return context.readSheetHolder() == null ? "" : context.readSheetHolder().getSheetName();
}
private Integer sheetNo(AnalysisContext context) {
return context.readSheetHolder() == null ? null : context.readSheetHolder().getSheetNo();
}
}
private static final class WrappedRuntimeException extends RuntimeException {
private WrappedRuntimeException(Throwable cause) {
super(cause);
}
}
}

View File

@@ -7,10 +7,29 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "aiimage.delete-brand-progress")
public class DeleteBrandProgressProperties {
private long heartbeatTimeoutMinutes = 15;
private String staleCheckCron = "0 */2 * * * *";
private String staleCheckCron = "*/30 * * * * *";
/**
* 定时补偿 finalize用于“分片其实已齐但最后一次提交断开导致没触发 finalize”的场景。
*/
private String finalizeCheckCron = "30 */2 * * * *";
/**
* 商品风险PRODUCT_RISK_RESOLVERUNNING 超时自动收尾/失败,按分钟计算;
* 与删除品牌共用同一定时调度。
*/
private long productRiskStaleTimeoutMinutes = 20;
private long productRiskInitialTimeoutMinutes = 20;
/**
* Price track RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long priceTrackStaleTimeoutMinutes = 20;
private long priceTrackInitialTimeoutMinutes = 20;
/**
* Shop match RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long shopMatchStaleTimeoutMinutes = 20;
private long shopMatchInitialTimeoutMinutes = 20;
}

View File

@@ -0,0 +1,37 @@
package com.nanri.aiimage.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.net.InetAddress;
@Component
public class InstanceMetadata {
private final String instanceId;
private final String hostname;
public InstanceMetadata(@Value("${aiimage.instance-id:}") String configuredInstanceId) {
this.hostname = resolveHostname();
this.instanceId = configuredInstanceId == null || configuredInstanceId.isBlank()
? this.hostname
: configuredInstanceId.trim();
}
public String getInstanceId() {
return instanceId;
}
public String getHostname() {
return hostname;
}
private static String resolveHostname() {
try {
return InetAddress.getLocalHost().getHostName();
} catch (Exception ignored) {
String envHostname = System.getenv("HOSTNAME");
return (envHostname == null || envHostname.isBlank()) ? "unknown-host" : envHostname;
}
}
}

View File

@@ -11,5 +11,5 @@ import java.util.List;
public class ModuleCleanupProperties {
private boolean enabled = true;
private String cron = "0 0 0 * * *";
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND"));
private List<String> moduleTypes = new ArrayList<>(List.of("DEDUPE", "SPLIT", "CONVERT", "DELETE_BRAND", "PRODUCT_RISK_RESOLVE", "PRICE_TRACK"));
}

View File

@@ -0,0 +1,81 @@
package com.nanri.aiimage.config;
import jakarta.servlet.FilterChain;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import org.springframework.web.filter.OncePerRequestFilter;
import java.io.IOException;
import java.util.Optional;
@Component
public class RequestTraceFilter extends OncePerRequestFilter {
private static final Logger log = LoggerFactory.getLogger(RequestTraceFilter.class);
private final InstanceMetadata instanceMetadata;
public RequestTraceFilter(InstanceMetadata instanceMetadata) {
this.instanceMetadata = instanceMetadata;
}
@Override
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
throws ServletException, IOException {
long start = System.currentTimeMillis();
String remoteAddr = firstNonBlank(
request.getHeader("X-Forwarded-For"),
request.getHeader("X-Real-IP"),
request.getRemoteAddr()
);
String forwardedProto = request.getHeader("X-Forwarded-Proto");
String forwardedHost = firstNonBlank(request.getHeader("X-Forwarded-Host"), request.getHeader("Host"));
String forwardedPort = request.getHeader("X-Forwarded-Port");
String requestId = firstNonBlank(
request.getHeader("X-Request-Id"),
request.getHeader("X-Amzn-Trace-Id"),
request.getHeader("Traceparent")
);
response.setHeader("X-AIIMAGE-Instance", instanceMetadata.getInstanceId());
response.setHeader("X-AIIMAGE-Host", instanceMetadata.getHostname());
try {
filterChain.doFilter(request, response);
} finally {
long costMs = System.currentTimeMillis() - start;
log.info(
"request-trace instance={} host={} method={} uri={} status={} remote={} forwardedHost={} forwardedProto={} forwardedPort={} requestId={} userAgent={} costMs={}",
instanceMetadata.getInstanceId(),
instanceMetadata.getHostname(),
request.getMethod(),
request.getRequestURI(),
response.getStatus(),
remoteAddr,
blankToDash(forwardedHost),
blankToDash(forwardedProto),
blankToDash(forwardedPort),
blankToDash(requestId),
blankToDash(request.getHeader("User-Agent")),
costMs
);
}
}
private static String firstNonBlank(String... values) {
for (String value : values) {
if (value != null && !value.isBlank()) {
return value;
}
}
return null;
}
private static String blankToDash(String value) {
return Optional.ofNullable(value).filter(v -> !v.isBlank()).orElse("-");
}
}

View File

@@ -0,0 +1,24 @@
package com.nanri.aiimage.config;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
@Configuration
@Slf4j
public class SchedulingConfig {
@Bean
public TaskScheduler taskScheduler() {
ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
scheduler.setPoolSize(4);
scheduler.setThreadNamePrefix("aiimage-scheduling-");
scheduler.setWaitForTasksToCompleteOnShutdown(true);
scheduler.setAwaitTerminationSeconds(30);
scheduler.setErrorHandler(ex -> log.warn("[scheduling] task execution failed: {}", ex.getMessage(), ex));
scheduler.initialize();
return scheduler;
}
}

View File

@@ -0,0 +1,21 @@
package com.nanri.aiimage.modules.brand.model.dto;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class BrandFileAggregateCacheDto {
private String fileUrl;
private String originalFilename;
private String relativePath;
private String mainSheetName;
private Integer chunkTotal;
private Integer totalLines;
private Integer receivedChunkCount = 0;
private Integer processedLineCount = 0;
private Boolean completed = false;
private List<BrandInvalidBrandDto> invalidBrands = new ArrayList<>();
private List<String> queryFailedBrands = new ArrayList<>();
}

View File

@@ -5,10 +5,16 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.BrandProgressProperties;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
import lombok.RequiredArgsConstructor;
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
@@ -17,18 +23,26 @@ import java.util.List;
import java.util.Map;
@Service
@RequiredArgsConstructor
public class BrandTaskProgressCacheService {
public static final String PHASE_CRAWLING = "crawling";
public static final String PHASE_ASSEMBLING = "assembling";
public static final String PHASE_UPLOADING = "uploading";
public static final String PHASE_FAILED = "failed";
private static final Duration FINALIZE_LOCK_TTL = Duration.ofMinutes(10);
private final StringRedisTemplate stringRedisTemplate;
private final BrandProgressProperties brandProgressProperties;
private final ObjectMapper objectMapper;
public BrandTaskProgressCacheService(StringRedisTemplate stringRedisTemplate,
BrandProgressProperties brandProgressProperties,
ObjectMapper objectMapper) {
this.stringRedisTemplate = stringRedisTemplate;
this.brandProgressProperties = brandProgressProperties;
this.objectMapper = objectMapper;
}
public void saveProgressFromResult(Long taskId,
String fileUrl,
int fileIndex,
@@ -51,7 +65,7 @@ public class BrandTaskProgressCacheService {
values.put("updated_at", now);
values.put("last_heartbeat_at", now);
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getTtlHours()));
stringRedisTemplate.expire(key, ttl());
}
public void updatePhase(Long taskId, String phase, int finishedFiles, int fileTotal) {
@@ -64,7 +78,7 @@ public class BrandTaskProgressCacheService {
values.put("updated_at", now);
values.put("last_heartbeat_at", now);
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getTtlHours()));
stringRedisTemplate.expire(key, ttl());
}
public void markFailed(Long taskId, String message) {
@@ -84,85 +98,324 @@ public class BrandTaskProgressCacheService {
public void saveParsedPayload(Long taskId, Object payload) {
try {
stringRedisTemplate.opsForValue().set(buildPayloadKey(taskId), objectMapper.writeValueAsString(payload), Duration.ofHours(brandProgressProperties.getTtlHours()));
Files.createDirectories(buildTaskDir(taskId));
Files.writeString(
buildPayloadFile(taskId),
objectMapper.writeValueAsString(payload),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE
);
} catch (Exception ex) {
throw new BusinessException("暂存品牌原始数据失败");
throw new BusinessException("鏆傚瓨鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
public <T> T getParsedPayload(Long taskId, TypeReference<T> typeReference) {
String raw = stringRedisTemplate.opsForValue().get(buildPayloadKey(taskId));
if (raw == null || raw.isBlank()) {
Path payloadFile = buildPayloadFile(taskId);
if (!Files.isRegularFile(payloadFile)) {
return null;
}
try {
return objectMapper.readValue(raw, typeReference);
return objectMapper.readValue(Files.readString(payloadFile), typeReference);
} catch (Exception ex) {
throw new BusinessException("读取品牌原始数据失败");
throw new BusinessException("璇诲彇鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
public ChunkStoreResult storeChunk(Long taskId, BrandCrawlResultFileDto file) {
validateChunk(file);
String fileUrl = normalizeFileUrl(file.getFileUrl());
String chunkDigest = digestChunk(file);
String digestField = String.valueOf(file.getChunkIndex());
String digestKey = buildChunkDigestKey(taskId, fileUrl);
Object existingDigest = stringRedisTemplate.opsForHash().get(digestKey, digestField);
if (existingDigest instanceof String existing && !existing.isBlank()) {
refreshFileKeys(taskId, fileUrl);
if (existing.equals(chunkDigest)) {
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl);
int finishedFiles = countCompletedFiles(taskId);
return new ChunkStoreResult(false, false, finishedFiles, aggregate);
}
throw new BusinessException("鍚屼竴鍒嗙墖閲嶅鎻愪氦浣嗗唴瀹逛笉涓€鑷? " + fileUrl + "#" + file.getChunkIndex());
}
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl);
if (aggregate == null) {
aggregate = createAggregate(file);
} else {
ensureChunkConsistency(aggregate, file);
}
mergeAggregate(aggregate, file);
stringRedisTemplate.opsForSet().add(buildChunkReceiptKey(taskId, fileUrl), digestField);
Long receivedSize = stringRedisTemplate.opsForSet().size(buildChunkReceiptKey(taskId, fileUrl));
int receivedCount = receivedSize == null ? 0 : receivedSize.intValue();
if (receivedCount <= 0) {
receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
}
aggregate.setReceivedChunkCount(Math.max(receivedCount, aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount()));
boolean fileCompleted = aggregate.getChunkTotal() != null
&& aggregate.getChunkTotal() > 0
&& aggregate.getReceivedChunkCount() >= aggregate.getChunkTotal();
aggregate.setCompleted(fileCompleted);
saveFileAggregate(taskId, aggregate);
stringRedisTemplate.opsForHash().put(digestKey, digestField, chunkDigest);
refreshFileKeys(taskId, fileUrl);
boolean newlyCompleted = false;
int finishedFiles = countCompletedFiles(taskId);
if (fileCompleted) {
Long added = stringRedisTemplate.opsForSet().add(buildCompletedFilesKey(taskId), fileUrl);
refreshTaskKey(buildCompletedFilesKey(taskId));
newlyCompleted = added != null && added > 0;
if (newlyCompleted) {
finishedFiles = countCompletedFiles(taskId);
}
}
return new ChunkStoreResult(true, newlyCompleted, finishedFiles, aggregate);
}
public BrandFileAggregateCacheDto getFileAggregate(Long taskId, String fileUrl) {
String normalizedFileUrl = normalizeFileUrl(fileUrl);
Object raw = stringRedisTemplate.opsForHash().get(buildFileAggregateKey(taskId), normalizedFileUrl);
if (!(raw instanceof String json) || json.isBlank()) {
return null;
}
try {
return objectMapper.readValue(json, BrandFileAggregateCacheDto.class);
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍝佺墝鏂囦欢鑱氬悎缂撳瓨澶辫触");
}
}
public Map<String, BrandFileAggregateCacheDto> getAllFileAggregates(Long taskId) {
Map<Object, Object> stored = stringRedisTemplate.opsForHash().entries(buildFileAggregateKey(taskId));
Map<String, BrandFileAggregateCacheDto> result = new LinkedHashMap<>();
for (Map.Entry<Object, Object> entry : stored.entrySet()) {
if (!(entry.getKey() instanceof String fileUrl) || !(entry.getValue() instanceof String json) || json.isBlank()) {
continue;
}
try {
result.put(fileUrl, objectMapper.readValue(json, BrandFileAggregateCacheDto.class));
} catch (Exception ignored) {
}
}
return result;
}
public int countCompletedFiles(Long taskId) {
Long count = stringRedisTemplate.opsForSet().size(buildCompletedFilesKey(taskId));
return count == null ? 0 : count.intValue();
}
public boolean acquireFinalizeLock(Long taskId) {
Boolean ok = stringRedisTemplate.opsForValue().setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
return Boolean.TRUE.equals(ok);
}
public void releaseFinalizeLock(Long taskId) {
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
}
public void delete(Long taskId) {
stringRedisTemplate.delete(buildKey(taskId));
stringRedisTemplate.delete(buildResultKey(taskId));
stringRedisTemplate.delete(buildPayloadKey(taskId));
stringRedisTemplate.delete(buildFileAggregateKey(taskId));
stringRedisTemplate.delete(buildCompletedFilesKey(taskId));
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
stringRedisTemplate.delete(buildLegacyResultKey(taskId));
deleteTaskDirQuietly(taskId);
}
public void mergeResultChunks(Long taskId, List<BrandCrawlResultFileDto> incomingFiles) {
String resultKey = buildResultKey(taskId);
for (BrandCrawlResultFileDto file : incomingFiles) {
if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0 || file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) {
throw new BusinessException("分片参数不合法");
}
String field = buildChunkField(file.getFileUrl(), file.getChunkIndex());
try {
stringRedisTemplate.opsForHash().put(resultKey, field, objectMapper.writeValueAsString(file));
} catch (Exception ex) {
throw new BusinessException("暂存结果分片失败");
}
}
stringRedisTemplate.expire(resultKey, Duration.ofHours(brandProgressProperties.getTtlHours()));
}
public Map<String, List<BrandCrawlResultFileDto>> groupResultChunksByFile(Long taskId) {
Map<Object, Object> stored = stringRedisTemplate.opsForHash().entries(buildResultKey(taskId));
Map<String, List<BrandCrawlResultFileDto>> grouped = new LinkedHashMap<>();
for (Object value : stored.values()) {
if (!(value instanceof String raw) || raw.isBlank()) {
continue;
}
try {
BrandCrawlResultFileDto file = objectMapper.readValue(raw, BrandCrawlResultFileDto.class);
grouped.computeIfAbsent(file.getFileUrl(), ignored -> new ArrayList<>()).add(file);
} catch (Exception ignored) {
}
}
grouped.values().forEach(list -> list.sort(java.util.Comparator.comparing(BrandCrawlResultFileDto::getChunkIndex)));
return grouped;
}
public void clearResultChunks(Long taskId) {
stringRedisTemplate.delete(buildResultKey(taskId));
public void deleteFileState(Long taskId, String fileUrl) {
String normalizedFileUrl = normalizeFileUrl(fileUrl);
stringRedisTemplate.opsForHash().delete(buildFileAggregateKey(taskId), normalizedFileUrl);
stringRedisTemplate.delete(buildChunkReceiptKey(taskId, normalizedFileUrl));
stringRedisTemplate.delete(buildChunkDigestKey(taskId, normalizedFileUrl));
}
public String buildKey(Long taskId) {
return "brand:task:progress:" + taskId;
}
private String buildResultKey(Long taskId) {
public long getHeartbeatTimeoutMinutes() {
return brandProgressProperties.getHeartbeatTimeoutMinutes();
}
private void saveFileAggregate(Long taskId, BrandFileAggregateCacheDto aggregate) {
try {
stringRedisTemplate.opsForHash().put(buildFileAggregateKey(taskId), normalizeFileUrl(aggregate.getFileUrl()), objectMapper.writeValueAsString(aggregate));
refreshTaskKey(buildFileAggregateKey(taskId));
} catch (Exception ex) {
throw new BusinessException("鏆傚瓨鍝佺墝鏂囦欢鑱氬悎缁撴灉澶辫触");
}
}
private void validateChunk(BrandCrawlResultFileDto file) {
if (file == null) {
throw new BusinessException("鍒嗙墖涓嶈兘涓虹┖");
}
if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0 || file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) {
throw new BusinessException("鍒嗙墖鍙傛暟涓嶅悎娉?");
}
if (file.getFileUrl() == null || file.getFileUrl().isBlank()) {
throw new BusinessException("fileUrl 涓嶈兘涓虹┖");
}
}
private BrandFileAggregateCacheDto createAggregate(BrandCrawlResultFileDto file) {
BrandFileAggregateCacheDto aggregate = new BrandFileAggregateCacheDto();
aggregate.setFileUrl(normalizeFileUrl(file.getFileUrl()));
aggregate.setOriginalFilename(file.getOriginalFilename());
aggregate.setRelativePath(file.getRelativePath());
aggregate.setMainSheetName(file.getMainSheetName());
aggregate.setChunkTotal(file.getChunkTotal());
aggregate.setTotalLines(safePositive(file.getTotalLines()));
aggregate.setReceivedChunkCount(0);
aggregate.setProcessedLineCount(0);
aggregate.setCompleted(false);
aggregate.setInvalidBrands(new ArrayList<>());
aggregate.setQueryFailedBrands(new ArrayList<>());
return aggregate;
}
private void ensureChunkConsistency(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
if (aggregate.getChunkTotal() != null && file.getChunkTotal() != null && !aggregate.getChunkTotal().equals(file.getChunkTotal())) {
throw new BusinessException("鍚屼竴鏂囦欢鐨?chunkTotal 涓嶄竴鑷? " + aggregate.getFileUrl());
}
if (aggregate.getChunkTotal() == null) {
aggregate.setChunkTotal(file.getChunkTotal());
}
int incomingTotalLines = safePositive(file.getTotalLines());
if (incomingTotalLines > 0) {
aggregate.setTotalLines(Math.max(safePositive(aggregate.getTotalLines()), incomingTotalLines));
}
if ((aggregate.getOriginalFilename() == null || aggregate.getOriginalFilename().isBlank()) && file.getOriginalFilename() != null && !file.getOriginalFilename().isBlank()) {
aggregate.setOriginalFilename(file.getOriginalFilename());
}
if ((aggregate.getRelativePath() == null || aggregate.getRelativePath().isBlank()) && file.getRelativePath() != null && !file.getRelativePath().isBlank()) {
aggregate.setRelativePath(file.getRelativePath());
}
if ((aggregate.getMainSheetName() == null || aggregate.getMainSheetName().isBlank()) && file.getMainSheetName() != null && !file.getMainSheetName().isBlank()) {
aggregate.setMainSheetName(file.getMainSheetName());
}
}
private void mergeAggregate(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
int processed = safePositive(aggregate.getProcessedLineCount())
+ sizeOf(file.getKeptRows())
+ sizeOf(file.getInvalidBrands())
+ sizeOf(file.getQueryFailedBrands());
aggregate.setProcessedLineCount(processed);
if (file.getInvalidBrands() != null && !file.getInvalidBrands().isEmpty()) {
aggregate.getInvalidBrands().addAll(file.getInvalidBrands());
}
if (file.getQueryFailedBrands() != null && !file.getQueryFailedBrands().isEmpty()) {
aggregate.getQueryFailedBrands().addAll(file.getQueryFailedBrands());
}
}
private int sizeOf(List<?> list) {
return list == null ? 0 : list.size();
}
private int safePositive(Integer value) {
return value == null || value <= 0 ? 0 : value;
}
private String digestChunk(BrandCrawlResultFileDto file) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(objectMapper.writeValueAsString(file).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new BusinessException("璁$畻鍝佺墝缁撴灉鍒嗙墖鎽樿澶辫触");
}
}
private void refreshFileKeys(Long taskId, String fileUrl) {
refreshTaskKey(buildChunkReceiptKey(taskId, fileUrl));
refreshTaskKey(buildChunkDigestKey(taskId, fileUrl));
refreshTaskKey(buildFileAggregateKey(taskId));
}
private void refreshTaskKey(String key) {
stringRedisTemplate.expire(key, ttl());
}
private Duration ttl() {
return Duration.ofHours(brandProgressProperties.getTtlHours());
}
private Path buildTaskDir(Long taskId) {
return Path.of(System.getProperty("java.io.tmpdir"), "brand-task-cache", String.valueOf(taskId));
}
private Path buildPayloadFile(Long taskId) {
return buildTaskDir(taskId).resolve("parsed-payload.json");
}
private void deleteTaskDirQuietly(Long taskId) {
Path taskDir = buildTaskDir(taskId);
if (!Files.exists(taskDir)) {
return;
}
try (var walk = Files.walk(taskDir)) {
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (IOException ignored) {
}
});
} catch (IOException ignored) {
}
}
private String buildFileAggregateKey(Long taskId) {
return "brand:task:file-aggregate:" + taskId;
}
private String buildChunkReceiptKey(Long taskId, String fileUrl) {
return "brand:task:file-chunks:" + taskId + ":" + hashFileUrl(fileUrl);
}
private String buildChunkDigestKey(Long taskId, String fileUrl) {
return "brand:task:file-chunk-digests:" + taskId + ":" + hashFileUrl(fileUrl);
}
private String buildCompletedFilesKey(Long taskId) {
return "brand:task:completed-files:" + taskId;
}
private String buildFinalizeLockKey(Long taskId) {
return "brand:task:finalizing:" + taskId;
}
private String buildLegacyResultKey(Long taskId) {
return "brand:task:result-chunks:" + taskId;
}
private String buildChunkField(String fileUrl, Integer chunkIndex) {
return fileUrl + "#" + chunkIndex;
private String normalizeFileUrl(String fileUrl) {
return fileUrl == null ? "" : fileUrl.trim();
}
private String buildPayloadKey(Long taskId) {
return "brand:task:parsed-payload:" + taskId;
}
public long getHeartbeatTimeoutMinutes() {
return brandProgressProperties.getHeartbeatTimeoutMinutes();
private String hashFileUrl(String fileUrl) {
String normalized = normalizeFileUrl(fileUrl);
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalized.getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new BusinessException("鐢熸垚鏂囦欢缂撳瓨閿け璐?");
}
}
private String normalizePhase(String phase) {
@@ -179,4 +432,7 @@ public class BrandTaskProgressCacheService {
private String blankToEmpty(String value) {
return value == null ? "" : value.trim();
}
public record ChunkStoreResult(boolean stored, boolean newlyCompletedFile, int finishedFiles, BrandFileAggregateCacheDto aggregate) {
}
}

View File

@@ -7,11 +7,13 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.BrandProgressProperties;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultRequest;
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandInvalidBrandDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandQueryFailedDto;
@@ -30,6 +32,7 @@ import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskItemVo;
import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskListVo;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
@@ -47,6 +50,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.Files;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
@@ -63,9 +67,11 @@ import java.util.zip.ZipOutputStream;
@Service
@RequiredArgsConstructor
@Slf4j
public class BrandTaskService {
private static final DateTimeFormatter DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
private static final Duration STALE_CHECK_LOCK_TTL = Duration.ofMinutes(10);
private static final String STATUS_PENDING = "pending";
private static final String STATUS_RUNNING = "running";
private static final String STATUS_SUCCESS = "success";
@@ -77,6 +83,7 @@ public class BrandTaskService {
private final StorageProperties storageProperties;
private final BrandProgressProperties brandProgressProperties;
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
private final DistributedJobLockService distributedJobLockService;
private final ObjectMapper objectMapper;
public BrandTaskCreateVo createTask(Long userId, BrandTaskCreateRequest request) {
@@ -205,18 +212,19 @@ public class BrandTaskService {
}
public void submitCrawlResult(Long taskId, BrandCrawlResultRequest request) {
BrandCrawlTaskEntity task = requireTask(taskId);
if (STATUS_CANCELLED.equalsIgnoreCase(blankToDefault(task.getStatus(), STATUS_PENDING))) {
throw new BusinessException("任务已取消");
}
long startedAt = System.currentTimeMillis();
BrandCrawlTaskEntity task = requireActiveTask(taskId);
List<BrandSourceFileDto> sourceFiles = parseSourceFiles(task.getFilePaths());
if (sourceFiles.isEmpty()) {
throw new BusinessException("任务没有源文件");
}
Map<String, BrandSourceFileDto> sourceByUrl = new LinkedHashMap<>();
for (BrandSourceFileDto file : sourceFiles) {
Map<String, Integer> sourceIndexByUrl = new LinkedHashMap<>();
for (int i = 0; i < sourceFiles.size(); i++) {
BrandSourceFileDto file = sourceFiles.get(i);
sourceByUrl.put(file.getFileUrl(), file);
sourceIndexByUrl.put(file.getFileUrl(), i + 1);
}
List<BrandParsedFileCacheDto> cachedFiles = brandTaskProgressCacheService.getParsedPayload(taskId,
new TypeReference<List<BrandParsedFileCacheDto>>() {
@@ -236,82 +244,67 @@ public class BrandTaskService {
ensureNoDuplicateResultFiles(resultFiles);
int totalCount = sourceFiles.size();
int started = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
.set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now())
.set(BrandCrawlTaskEntity::getErrorMessage, null));
if (started == 0) {
throw new BusinessException("任务已取消");
markTaskRunning(taskId, totalCount);
log.info("[brand-submit] taskId={} chunks={} files={} thread={}",
taskId,
resultFiles.size(),
totalCount,
Thread.currentThread().getName());
int finishedCount = brandTaskProgressCacheService.countCompletedFiles(taskId);
for (BrandCrawlResultFileDto resultFile : resultFiles) {
String fileUrl = blankToNull(resultFile.getFileUrl());
if (fileUrl == null) {
throw new BusinessException("fileUrl 不能为空");
}
BrandSourceFileDto sourceFile = sourceByUrl.get(fileUrl);
if (sourceFile == null) {
throw new BusinessException("存在未知 fileUrl: " + fileUrl);
}
BrandParsedFileCacheDto cachedFile = cachedByUrl.get(fileUrl);
if (cachedFile == null) {
throw new BusinessException("缺少原始缓存数据: " + fileUrl);
}
int totalLines = cachedFile.getRows() == null ? 0 : cachedFile.getRows().size();
if (resultFile.getTotalLines() == null || resultFile.getTotalLines() <= 0) {
resultFile.setTotalLines(totalLines);
} else if (totalLines > 0 && resultFile.getTotalLines() > totalLines) {
throw new BusinessException("totalLines 超过原始文件行数: " + fileUrl);
}
BrandTaskProgressCacheService.ChunkStoreResult storeResult = brandTaskProgressCacheService.storeChunk(taskId, resultFile);
finishedCount = Math.max(finishedCount, storeResult.finishedFiles());
log.info("[brand-submit] taskId={} fileUrl={} chunk={}/{} stored={} newlyCompletedFile={} finishedFiles={}",
taskId,
fileUrl,
resultFile.getChunkIndex(),
resultFile.getChunkTotal(),
storeResult.stored(),
storeResult.newlyCompletedFile(),
finishedCount);
BrandFileAggregateCacheDto aggregate = storeResult.aggregate();
String fileName = blankToDefault(sourceFile.getOriginalFilename(), sourceFile.getFileUrl());
int fileIndex = sourceIndexByUrl.getOrDefault(fileUrl, 0);
int currentLine = aggregate == null ? 0 : defaultInteger(aggregate.getProcessedLineCount());
int currentTotalLines = aggregate == null ? 0 : Math.max(defaultInteger(aggregate.getTotalLines()), currentLine);
brandTaskProgressCacheService.saveProgressFromResult(taskId,
fileUrl,
fileIndex,
totalCount,
fileName,
currentLine,
currentTotalLines,
finishedCount);
updateTaskProgress(taskId, finishedCount, totalCount);
}
brandTaskProgressCacheService.mergeResultChunks(taskId, resultFiles);
Map<String, List<BrandCrawlResultFileDto>> groupedResults = brandTaskProgressCacheService.groupResultChunksByFile(taskId);
int finishedCount = countCompletedFiles(groupedResults);
updateResultDrivenProgress(taskId, sourceFiles, sourceByUrl, groupedResults, finishedCount, totalCount);
updateTaskProgress(taskId, finishedCount, totalCount);
if (finishedCount < totalCount) {
return;
}
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_ASSEMBLING, finishedCount, totalCount);
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "brand-result", String.valueOf(taskId)));
List<OutputEntry> outputEntries = new ArrayList<>();
try {
for (List<BrandCrawlResultFileDto> fileChunks : groupedResults.values()) {
BrandCrawlResultFileDto resultFile = mergeChunks(fileChunks);
BrandSourceFileDto sourceFile = sourceByUrl.get(resultFile.getFileUrl());
if (sourceFile == null) {
throw new BusinessException("存在未知 fileUrl: " + resultFile.getFileUrl());
}
BrandParsedFileCacheDto cachedFile = cachedByUrl.get(resultFile.getFileUrl());
if (cachedFile == null) {
throw new BusinessException("缺少原始缓存数据: " + resultFile.getFileUrl());
}
File sourceLocalFile = resolveSourceFile(sourceFile);
String originalFilename = resolveOriginalFilename(sourceFile, sourceLocalFile);
File outputFile = buildNamedOutputFile(outputDir, buildResultFilename(originalFilename));
writeBrandWorkbook(outputFile, request.getStrategy(), cachedFile, resultFile);
outputEntries.add(new OutputEntry(
sourceLocalFile,
originalFilename,
outputFile,
originalFilename));
}
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_UPLOADING, finishedCount, totalCount);
Map<String, Object> resultPaths = buildAndUploadResult(taskId, outputEntries);
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
.set(BrandCrawlTaskEntity::getStatus, STATUS_SUCCESS)
.set(BrandCrawlTaskEntity::getResultPaths, JSONUtil.toJsonStr(resultPaths))
.set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount)
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
.set(BrandCrawlTaskEntity::getErrorMessage, null));
if (updated == 0) {
throw new BusinessException("任务已取消");
}
brandTaskProgressCacheService.delete(taskId);
} catch (Exception ex) {
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
.set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED)
.set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount)
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
.set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage()));
brandTaskProgressCacheService.markFailed(taskId, ex.getMessage());
if (ex instanceof BusinessException businessException) {
throw businessException;
}
throw new BusinessException(ex.getMessage());
}
tryFinalizeTask(taskId, request.getStrategy(), sourceFiles, cachedByUrl, finishedCount, totalCount);
log.info("[brand-submit] taskId={} finishedFiles={}/{} elapsedMs={} thread={}",
taskId,
finishedCount,
totalCount,
System.currentTimeMillis() - startedAt,
Thread.currentThread().getName());
}
public void cancelTask(Long taskId) {
@@ -334,6 +327,7 @@ public class BrandTaskService {
if (deleted == 0) {
throw new BusinessException("任务不存在或正在执行中无法删除");
}
brandTaskProgressCacheService.delete(taskId);
}
public String resolveDownloadUrl(Long taskId) {
@@ -498,6 +492,139 @@ public class BrandTaskService {
return Objects.equals(taskType, 1) ? 1 : 2;
}
private void markTaskRunning(Long taskId, int totalCount) {
int started = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
.set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now())
.set(BrandCrawlTaskEntity::getErrorMessage, null));
if (started == 0) {
throw new BusinessException("任务已取消");
}
}
private BrandCrawlTaskEntity requireActiveTask(Long taskId) {
BrandCrawlTaskEntity task = requireTask(taskId);
String status = blankToDefault(task.getStatus(), STATUS_PENDING);
if (STATUS_CANCELLED.equalsIgnoreCase(status)) {
throw new BusinessException("任务已取消");
}
if (STATUS_SUCCESS.equalsIgnoreCase(status) || STATUS_FAILED.equalsIgnoreCase(status)) {
throw new BusinessException("任务已结束,不能继续提交结果");
}
return task;
}
private void tryFinalizeTask(Long taskId,
String strategy,
List<BrandSourceFileDto> sourceFiles,
Map<String, BrandParsedFileCacheDto> cachedByUrl,
int finishedCount,
int totalCount) {
if (finishedCount < totalCount) {
return;
}
log.info("[brand-finalize] taskId={} all files completed, trying finalize lock", taskId);
if (!brandTaskProgressCacheService.acquireFinalizeLock(taskId)) {
log.info("[brand-finalize] taskId={} finalize lock busy, skip", taskId);
return;
}
try {
finalizeTask(taskId, strategy, sourceFiles, cachedByUrl, totalCount);
} finally {
brandTaskProgressCacheService.releaseFinalizeLock(taskId);
}
}
private void finalizeTask(Long taskId,
String strategy,
List<BrandSourceFileDto> sourceFiles,
Map<String, BrandParsedFileCacheDto> cachedByUrl,
int totalCount) {
long startedAt = System.currentTimeMillis();
BrandCrawlTaskEntity freshTask = requireTask(taskId);
String freshStatus = blankToDefault(freshTask.getStatus(), STATUS_PENDING);
if (STATUS_CANCELLED.equalsIgnoreCase(freshStatus)) {
throw new BusinessException("任务已取消");
}
if (STATUS_SUCCESS.equalsIgnoreCase(freshStatus)) {
return;
}
if (STATUS_FAILED.equalsIgnoreCase(freshStatus)) {
throw new BusinessException("任务已结束,不能继续组装结果");
}
Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskProgressCacheService.getAllFileAggregates(taskId);
log.info("[brand-finalize] taskId={} aggregateFiles={} expectedFiles={} thread={}",
taskId,
aggregates.size(),
totalCount,
Thread.currentThread().getName());
if (aggregates.size() < totalCount) {
return;
}
for (BrandSourceFileDto sourceFile : sourceFiles) {
BrandFileAggregateCacheDto aggregate = aggregates.get(sourceFile.getFileUrl());
if (aggregate == null || !Boolean.TRUE.equals(aggregate.getCompleted())) {
return;
}
}
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_ASSEMBLING, totalCount, totalCount);
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "brand-result", String.valueOf(taskId)));
List<OutputEntry> outputEntries = new ArrayList<>();
try {
for (BrandSourceFileDto sourceFile : sourceFiles) {
BrandFileAggregateCacheDto aggregate = aggregates.get(sourceFile.getFileUrl());
BrandParsedFileCacheDto cachedFile = cachedByUrl.get(sourceFile.getFileUrl());
if (cachedFile == null) {
throw new BusinessException("缺少原始缓存数据: " + sourceFile.getFileUrl());
}
File sourceLocalFile = resolveSourceFile(sourceFile);
String originalFilename = resolveOriginalFilename(sourceFile, sourceLocalFile);
File outputFile = buildNamedOutputFile(outputDir, buildResultFilename(originalFilename));
writeBrandWorkbook(outputFile, strategy, cachedFile, aggregate);
outputEntries.add(new OutputEntry(sourceLocalFile, originalFilename, outputFile, originalFilename));
}
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_UPLOADING, totalCount, totalCount);
Map<String, Object> resultPaths = buildAndUploadResult(taskId, outputEntries);
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
.set(BrandCrawlTaskEntity::getStatus, STATUS_SUCCESS)
.set(BrandCrawlTaskEntity::getResultPaths, JSONUtil.toJsonStr(resultPaths))
.set(BrandCrawlTaskEntity::getProgressCurrent, totalCount)
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
.set(BrandCrawlTaskEntity::getErrorMessage, null));
if (updated == 0) {
throw new BusinessException("任务已取消");
}
for (BrandSourceFileDto sourceFile : sourceFiles) {
brandTaskProgressCacheService.deleteFileState(taskId, sourceFile.getFileUrl());
}
brandTaskProgressCacheService.delete(taskId);
log.info("[brand-finalize] taskId={} finalized success files={} elapsedMs={}",
taskId,
totalCount,
System.currentTimeMillis() - startedAt);
} catch (Exception ex) {
log.warn("[brand-finalize] taskId={} finalize failed msg={}", taskId, ex.getMessage());
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
.set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED)
.set(BrandCrawlTaskEntity::getProgressCurrent, brandTaskProgressCacheService.countCompletedFiles(taskId))
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
.set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage()));
brandTaskProgressCacheService.markFailed(taskId, ex.getMessage());
if (ex instanceof BusinessException businessException) {
throw businessException;
}
throw new BusinessException(ex.getMessage());
}
}
private void ensureNoDuplicateResultFiles(List<BrandCrawlResultFileDto> resultFiles) {
Set<String> seen = new LinkedHashSet<>();
for (BrandCrawlResultFileDto resultFile : resultFiles) {
@@ -505,8 +632,12 @@ public class BrandTaskService {
if (fileUrl == null) {
throw new BusinessException("fileUrl 不能为空");
}
if (!seen.add(fileUrl)) {
throw new BusinessException("存在重复 fileUrl: " + fileUrl);
if (resultFile.getChunkIndex() == null || resultFile.getChunkIndex() <= 0) {
throw new BusinessException("chunkIndex 不合法");
}
String chunkKey = fileUrl + "#" + resultFile.getChunkIndex();
if (!seen.add(chunkKey)) {
throw new BusinessException("存在重复分片: " + chunkKey);
}
}
}
@@ -522,81 +653,6 @@ public class BrandTaskService {
.set(BrandCrawlTaskEntity::getErrorMessage, null));
}
private void updateResultDrivenProgress(Long taskId,
List<BrandSourceFileDto> sourceFiles,
Map<String, BrandSourceFileDto> sourceByUrl,
Map<String, List<BrandCrawlResultFileDto>> groupedResults,
int finishedCount,
int totalCount) {
if (groupedResults.isEmpty()) {
return;
}
String currentFileUrl = null;
List<BrandCrawlResultFileDto> currentChunks = null;
int currentFileIndex = 0;
for (int i = sourceFiles.size() - 1; i >= 0; i--) {
BrandSourceFileDto sourceFile = sourceFiles.get(i);
List<BrandCrawlResultFileDto> chunks = groupedResults.get(sourceFile.getFileUrl());
if (chunks == null || chunks.isEmpty()) {
continue;
}
currentFileUrl = sourceFile.getFileUrl();
currentChunks = chunks;
currentFileIndex = i + 1;
break;
}
if (currentFileUrl == null || currentChunks == null) {
return;
}
BrandSourceFileDto sourceFile = sourceByUrl.get(currentFileUrl);
String fileName = sourceFile == null ? "" : blankToDefault(sourceFile.getOriginalFilename(), sourceFile.getFileUrl());
int currentLine = countProcessedLines(currentChunks);
int totalLines = Math.max(inferTotalLines(currentChunks), currentLine);
brandTaskProgressCacheService.saveProgressFromResult(taskId,
currentFileUrl,
currentFileIndex,
totalCount,
fileName,
currentLine,
totalLines,
finishedCount);
}
private int countProcessedLines(List<BrandCrawlResultFileDto> chunks) {
int currentLine = 0;
for (BrandCrawlResultFileDto chunk : chunks) {
if (chunk.getKeptRows() != null) {
currentLine += chunk.getKeptRows().size();
}
if (chunk.getInvalidBrands() != null) {
currentLine += chunk.getInvalidBrands().size();
}
if (chunk.getQueryFailedBrands() != null) {
currentLine += chunk.getQueryFailedBrands().size();
}
}
return currentLine;
}
private int inferTotalLines(List<BrandCrawlResultFileDto> chunks) {
if (chunks == null || chunks.isEmpty()) {
return 0;
}
BrandCrawlResultFileDto last = chunks.get(chunks.size() - 1);
if (last.getTotalLines() != null && last.getTotalLines() > 0) {
return last.getTotalLines();
}
if (last.getChunkTotal() == null || last.getChunkTotal() <= 0) {
return countProcessedLines(chunks);
}
int processed = countProcessedLines(chunks);
if (Objects.equals(chunks.size(), last.getChunkTotal())) {
return processed;
}
int averagePerChunk = Math.max(processed / Math.max(chunks.size(), 1), 1);
return averagePerChunk * last.getChunkTotal();
}
private String buildDesc(String strategy, List<BrandSourceFileDto> files) {
String joined = files.stream()
.map(file -> blankToDefault(file.getOriginalFilename(), file.getFileUrl()))
@@ -769,7 +825,7 @@ public class BrandTaskService {
private void writeBrandWorkbook(File outputFile,
String strategy,
BrandParsedFileCacheDto cachedFile,
BrandCrawlResultFileDto resultFile) throws IOException {
BrandFileAggregateCacheDto resultFile) throws IOException {
String actualStrategy = normalizeStrategy(strategy);
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
String mainSheetName = blankToDefault(cachedFile.getSheetName(), "Sheet1");
@@ -986,52 +1042,6 @@ public class BrandTaskService {
return vo;
}
private int countCompletedFiles(Map<String, List<BrandCrawlResultFileDto>> groupedResults) {
int count = 0;
for (List<BrandCrawlResultFileDto> chunks : groupedResults.values()) {
if (!chunks.isEmpty() && Objects.equals(chunks.size(), chunks.get(0).getChunkTotal())) {
count++;
}
}
return count;
}
private BrandCrawlResultFileDto mergeChunks(List<BrandCrawlResultFileDto> chunks) {
if (chunks == null || chunks.isEmpty()) {
throw new BusinessException("结果分片为空");
}
BrandCrawlResultFileDto first = chunks.get(0);
if (!Objects.equals(chunks.size(), first.getChunkTotal())) {
throw new BusinessException("文件结果分片未完成");
}
BrandCrawlResultFileDto merged = new BrandCrawlResultFileDto();
merged.setFileUrl(first.getFileUrl());
merged.setOriginalFilename(first.getOriginalFilename());
merged.setRelativePath(first.getRelativePath());
merged.setMainSheetName(first.getMainSheetName());
merged.setChunkIndex(first.getChunkIndex());
merged.setChunkTotal(first.getChunkTotal());
merged.setTotalLines(first.getTotalLines());
List<String> keptRows = new ArrayList<>();
List<BrandInvalidBrandDto> invalidBrands = new ArrayList<>();
List<String> queryFailedBrands = new ArrayList<>();
for (BrandCrawlResultFileDto chunk : chunks) {
if (chunk.getKeptRows() != null) {
keptRows.addAll(chunk.getKeptRows());
}
if (chunk.getInvalidBrands() != null) {
invalidBrands.addAll(chunk.getInvalidBrands());
}
if (chunk.getQueryFailedBrands() != null) {
queryFailedBrands.addAll(chunk.getQueryFailedBrands());
}
}
merged.setKeptRows(keptRows);
merged.setInvalidBrands(invalidBrands);
merged.setQueryFailedBrands(queryFailedBrands);
return merged;
}
private Integer parseInteger(Object value) {
if (value == null) {
return 0;
@@ -1046,6 +1056,12 @@ public class BrandTaskService {
@Transactional
@org.springframework.scheduling.annotation.Scheduled(cron = "${aiimage.brand-progress.stale-check-cron:0 */2 * * * *}")
public void failStaleRunningTasks() {
DistributedJobLockService.LockHandle lockHandle = distributedJobLockService.tryLock("brand:stale-check", STALE_CHECK_LOCK_TTL);
if (lockHandle == null) {
log.info("[brand-stale-check] skip because another instance holds the distributed lock");
return;
}
try (lockHandle) {
LocalDateTime threshold = LocalDateTime.now().minusMinutes(brandProgressProperties.getHeartbeatTimeoutMinutes());
List<BrandCrawlTaskEntity> runningTasks = brandCrawlTaskMapper.selectList(new LambdaQueryWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
@@ -1070,6 +1086,7 @@ public class BrandTaskService {
continue;
}
failStaleRunningTask(task.getId(), "前端长时间无响应,任务已自动失败");
}
}
}

View File

@@ -95,7 +95,7 @@ public class ConvertRunController {
}
@GetMapping("/results/{resultId}/download")
@Operation(summary = "下载格式转换结果", description = "根据结果记录 ID 下载当前用户的转换结果文件。桌面端前端可继续通过 save_file_from_url 选择保存位置。")
@Operation(summary = "下载格式转换结果", description = "根据结果记录 ID 下载当前用户的转换结果文件。桌面端前端可继续通过 save_file_from_url_new 选择保存位置。")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "返回结果文件流"),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "结果文件不存在")

View File

@@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.convert.model.dto.ConvertRunRequest;
import com.nanri.aiimage.modules.convert.model.dto.UploadedSourceFileDto;
@@ -19,15 +20,10 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.springframework.stereotype.Service;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -242,14 +238,30 @@ public class ConvertRunService {
private List<GeneratedConvertFile> generateOutputFiles(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
List<String> outputFilenames = resolveOutputFilenames(templateEntity);
List<String> rows = buildTxtRows(inputFile, templateEntity);
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
List<GeneratedConvertFile> generatedFiles = new ArrayList<>();
Map<String, BufferedWriter> writers = new LinkedHashMap<>();
for (String outputFilename : outputFilenames) {
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
FileUtil.writeLines(rows, outputFile, StandardCharsets.UTF_8);
generatedFiles.add(new GeneratedConvertFile(outputFilename, outputFile));
writers.put(outputFilename, Files.newBufferedWriter(outputFile.toPath(), StandardCharsets.UTF_8));
}
try {
streamTxtRowsToOutputs(inputFile, templateEntity, writers);
} finally {
IOException closeException = null;
for (BufferedWriter writer : writers.values()) {
try {
writer.close();
} catch (IOException ex) {
if (closeException == null) {
closeException = ex;
}
}
}
if (closeException != null) {
throw closeException;
}
}
return generatedFiles;
}
@@ -266,7 +278,9 @@ public class ConvertRunService {
return List.of(outputFilename);
}
private List<String> buildTxtRows(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
private void streamTxtRowsToOutputs(File inputFile,
ConvertTemplateEntity templateEntity,
Map<String, BufferedWriter> writers) throws IOException {
List<String> requiredColumns = readStringList(templateEntity.getRequiredSourceColumnsJson());
List<String> headerColumns = readStringList(templateEntity.getHeaderColumnsJson());
Map<String, String> fieldMapping = readStringMap(templateEntity.getFieldMappingJson());
@@ -275,65 +289,60 @@ public class ConvertRunService {
int blankHeaderRowsAfterSchema = templateEntity.getBlankHeaderRowsAfterSchema() == null
? 0
: templateEntity.getBlankHeaderRowsAfterSchema();
long batchId = System.currentTimeMillis();
int[] rowIndex = new int[]{1};
boolean[] headerRead = new boolean[]{false};
Map<String, Integer>[] headerMapHolder = new Map[]{null};
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel header row is empty.");
}
Map<String, Integer> headerMap = new LinkedHashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
String value = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i)));
if (!value.isBlank() && !headerMap.containsKey(value)) {
headerMap.put(value, i);
ExcelStreamReader.readFirstSheet(inputFile, new ExcelStreamReader.SheetRowHandler() {
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) throws Exception {
Map<String, Integer> resolvedHeaderMap = new LinkedHashMap<>();
for (Map.Entry<Integer, String> entry : headerMap.entrySet()) {
String value = normalizeCellText(entry.getValue());
if (!value.isBlank() && !resolvedHeaderMap.containsKey(value)) {
resolvedHeaderMap.put(value, entry.getKey());
}
}
List<String> missing = requiredColumns.stream()
.filter(column -> !resolvedHeaderMap.containsKey(column))
.toList();
if (!missing.isEmpty()) {
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
}
headerRead[0] = true;
headerMapHolder[0] = resolvedHeaderMap;
for (String line : preambleLines) {
writeLineToAll(writers, line);
}
writeLineToAll(writers, String.join("\t", headerColumns));
for (int i = 0; i < blankHeaderRowsAfterSchema; i++) {
writeLineToAll(writers, "\t".repeat(Math.max(0, headerColumns.size() - 1)));
}
}
List<String> missing = requiredColumns.stream()
.filter(column -> !headerMap.containsKey(column))
.toList();
if (!missing.isEmpty()) {
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
}
List<String> rows = new ArrayList<>();
rows.addAll(preambleLines);
rows.add(String.join("\t", headerColumns));
for (int i = 0; i < blankHeaderRowsAfterSchema; i++) {
rows.add("\t".repeat(Math.max(0, headerColumns.size() - 1)));
}
long batchId = System.currentTimeMillis();
int rowIndex = 1;
int asinIndex = headerMap.getOrDefault("ASIN", -1);
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
@Override
public void onRow(String sheetName, Integer sheetNo, int excelRowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception {
Map<String, Integer> resolvedHeaderMap = headerMapHolder[0];
if (resolvedHeaderMap == null) {
throw new BusinessException("Excel header row is empty.");
}
String asin = asinIndex >= 0 ? normalizeCellText(formatter.formatCellValue(row.getCell(asinIndex))) : "";
int asinIndex = resolvedHeaderMap.getOrDefault("ASIN", -1);
String asin = asinIndex >= 0 ? normalizeCellText(rowMap.get(asinIndex)) : "";
if (asin.isBlank()) {
continue;
return;
}
List<String> lineValues = new ArrayList<>();
for (String column : headerColumns) {
if ("sku".equals(column)) {
lineValues.add(batchId + "-" + rowIndex);
lineValues.add(batchId + "-" + rowIndex[0]);
continue;
}
if (fieldMapping.containsKey(column)) {
String sourceColumn = fieldMapping.get(column);
Integer sourceIndex = headerMap.get(sourceColumn);
String value = sourceIndex == null
? ""
: normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
lineValues.add(value);
Integer sourceIndex = resolvedHeaderMap.get(sourceColumn);
lineValues.add(sourceIndex == null ? "" : normalizeCellText(rowMap.get(sourceIndex)));
continue;
}
if (defaults.containsKey(column)) {
@@ -343,10 +352,19 @@ public class ConvertRunService {
lineValues.add("");
}
rows.add(String.join("\t", lineValues));
rowIndex++;
writeLineToAll(writers, String.join("\t", lineValues));
rowIndex[0]++;
}
return rows;
});
if (!headerRead[0]) {
throw new BusinessException("Excel header row is empty.");
}
}
private void writeLineToAll(Map<String, BufferedWriter> writers, String line) throws IOException {
for (BufferedWriter writer : writers.values()) {
writer.write(line);
writer.newLine();
}
}

View File

@@ -0,0 +1,53 @@
package com.nanri.aiimage.modules.debug.controller;
import com.nanri.aiimage.config.InstanceMetadata;
import jakarta.servlet.http.HttpServletRequest;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.LinkedHashMap;
import java.util.Map;
@RestController
@RequestMapping("/debug")
public class DebugRequestInfoController {
private final InstanceMetadata instanceMetadata;
public DebugRequestInfoController(InstanceMetadata instanceMetadata) {
this.instanceMetadata = instanceMetadata;
}
@GetMapping("/request-info")
public Map<String, Object> requestInfo(
HttpServletRequest request,
@RequestHeader(value = "X-Forwarded-For", required = false) String forwardedFor,
@RequestHeader(value = "X-Real-IP", required = false) String realIp,
@RequestHeader(value = "X-Forwarded-Proto", required = false) String forwardedProto,
@RequestHeader(value = "X-Forwarded-Host", required = false) String forwardedHost,
@RequestHeader(value = "X-Forwarded-Port", required = false) String forwardedPort,
@RequestHeader(value = "X-Request-Id", required = false) String requestId,
@RequestHeader(value = "X-Amzn-Trace-Id", required = false) String amazonTraceId,
@RequestHeader(value = "Traceparent", required = false) String traceparent
) {
Map<String, Object> result = new LinkedHashMap<>();
result.put("instanceId", instanceMetadata.getInstanceId());
result.put("hostname", instanceMetadata.getHostname());
result.put("serverName", request.getServerName());
result.put("serverPort", request.getServerPort());
result.put("method", request.getMethod());
result.put("uri", request.getRequestURI());
result.put("remoteAddr", request.getRemoteAddr());
result.put("xForwardedFor", forwardedFor);
result.put("xRealIp", realIp);
result.put("xForwardedProto", forwardedProto);
result.put("xForwardedHost", forwardedHost);
result.put("xForwardedPort", forwardedPort);
result.put("xRequestId", requestId);
result.put("xAmznTraceId", amazonTraceId);
result.put("traceparent", traceparent);
return result;
}
}

View File

@@ -20,7 +20,7 @@ import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
@@ -232,7 +232,7 @@ public class DedupeRunService {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
XSSFWorkbook outputWorkbook = new XSSFWorkbook()) {
SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
@@ -275,7 +275,6 @@ public class DedupeRunService {
// The preliminary global scan for mainIdHasSubIdMap was removed.
// We now determine subset existence contextually (per group) during the main loop.
}
List<Row> candidateRows = new ArrayList<>();
Set<String> candidateAsinValues = new HashSet<>();
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
@@ -288,7 +287,6 @@ public class DedupeRunService {
continue;
}
}
candidateRows.add(row);
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank()) {
@@ -300,7 +298,17 @@ public class DedupeRunService {
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
Set<String> writtenAsinValues = new HashSet<>();
int outputRowIndex = 1;
for (Row row : candidateRows) {
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
if (idColumnIndex != null) {
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) {
continue;
}
}
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank()) {
@@ -324,6 +332,7 @@ public class DedupeRunService {
try (FileOutputStream fos = new FileOutputStream(outputFile)) {
outputWorkbook.write(fos);
}
outputWorkbook.dispose();
}
}

View File

@@ -23,7 +23,9 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -114,9 +116,9 @@ public class DedupeTotalDataService {
importProgressMap.put(importId, progress);
try {
byte[] fileBytes = file.getBytes();
File tempFile = saveMultipartToTempFile(file);
String filename = file.getOriginalFilename();
Thread.ofVirtual().start(() -> runImportTask(importId, fileBytes, filename));
Thread.ofVirtual().start(() -> runImportTask(importId, tempFile, filename));
} catch (Exception e) {
importProgressMap.remove(importId);
throw new BusinessException("读取上传文件失败");
@@ -150,9 +152,9 @@ public class DedupeTotalDataService {
deleteImportProgressMap.put(importId, progress);
try {
byte[] fileBytes = file.getBytes();
File tempFile = saveMultipartToTempFile(file);
String filename = file.getOriginalFilename();
Thread.ofVirtual().start(() -> runDeleteImportTask(importId, fileBytes, filename));
Thread.ofVirtual().start(() -> runDeleteImportTask(importId, tempFile, filename));
} catch (Exception e) {
deleteImportProgressMap.remove(importId);
throw new BusinessException("读取上传文件失败");
@@ -211,6 +213,74 @@ public class DedupeTotalDataService {
}
}
private void runDeleteImportTask(String importId, File tempFile, String filename) {
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
if (progress == null) {
return;
}
progress.setStatus("running");
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
DedupeTotalDataImportVo result = deleteFromExcelInternal(inputStream, filename, progress);
progress.setTotalRows(result.getTotalRows());
progress.setAsinCount(result.getAsinCount());
progress.setInsertedCount(result.getInsertedCount());
progress.setSkippedCount(result.getSkippedCount());
progress.setProcessedRows(result.getTotalRows());
progress.setStatus("success");
} catch (Exception e) {
progress.setStatus("failed");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "鍒犻櫎 Excel 鍖归厤鏁版嵁澶辫触");
} finally {
deleteQuietly(tempFile);
}
}
private void runImportTask(String importId, File tempFile, String filename) {
DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId);
if (progress == null) {
return;
}
progress.setStatus("running");
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
DedupeTotalDataImportVo result = importFromExcelInternal(inputStream, filename, progress);
progress.setTotalRows(result.getTotalRows());
progress.setAsinCount(result.getAsinCount());
progress.setInsertedCount(result.getInsertedCount());
progress.setSkippedCount(result.getSkippedCount());
progress.setProcessedRows(result.getTotalRows());
progress.setStatus("success");
} catch (Exception e) {
progress.setStatus("failed");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "瀵煎叆 Excel 澶辫触");
} finally {
deleteQuietly(tempFile);
}
}
private File saveMultipartToTempFile(MultipartFile file) throws Exception {
String suffix = ".xlsx";
String name = file.getOriginalFilename();
if (name != null) {
int idx = name.lastIndexOf('.');
if (idx >= 0) {
suffix = name.substring(idx);
}
}
File tempFile = File.createTempFile("dedupe-total-data-", suffix);
file.transferTo(tempFile);
return tempFile;
}
private void deleteQuietly(File file) {
if (file == null) {
return;
}
try {
Files.deleteIfExists(file.toPath());
} catch (Exception ignored) {
}
}
@Transactional
public DedupeTotalDataImportVo importFromExcel(MultipartFile file) {
if (file == null || file.isEmpty()) {

View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.deletebrand.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandRunRequest;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandSubmitResultRequest;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandTaskBatchRequest;
@@ -84,6 +85,12 @@ public class DeleteBrandRunController {
return ApiResponse.success(deleteBrandRunService.getTaskDetails(request.getTaskIds()));
}
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量获取删除品牌任务进度摘要", description = "仅返回任务基础状态和行进度摘要,用于前端轮询降载。")
public ApiResponse<DeleteBrandTaskBatchVo> getTaskProgressBatch(@Valid @RequestBody DeleteBrandTaskBatchRequest request) {
return ApiResponse.success(deleteBrandRunService.getTaskProgress(request.getTaskIds()));
}
@GetMapping("/tasks/{taskId}/deletion-status")
@Operation(summary = "获取删除品牌任务删除状态", description = "供 Python/插件按 taskId 查询该删除品牌任务是否已被前端删空。")
public ApiResponse<DeleteBrandTaskDeletionStatusVo> getTaskDeletionStatus(
@@ -97,13 +104,22 @@ public class DeleteBrandRunController {
@Operation(summary = "提交删除品牌处理结果", description = "插件/前端分片回传处理结果;后端累计分片并推进进度,全部完成后再执行最终组装与落库。")
public ApiResponse<Void> submitResult(
@PathVariable Long taskId,
@Valid @RequestBody DeleteBrandSubmitResultRequest request) {
deleteBrandRunService.submitResult(taskId, request);
return ApiResponse.success(null);
@Valid @RequestBody DeleteBrandSubmitResultRequest request,
jakarta.servlet.http.HttpServletResponse response) {
try {
deleteBrandRunService.submitResult(taskId, request);
return ApiResponse.success(null);
} catch (BusinessException ex) {
response.setStatus(HttpStatus.BAD_REQUEST.value());
return ApiResponse.fail(ex.getMessage());
} catch (Exception ex) {
response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
return ApiResponse.fail("服务异常:" + ex.getMessage());
}
}
@GetMapping("/tasks/{taskId}/download")
@Operation(summary = "下载删除品牌任务结果", description = "返回任务结果文件流,便于桌面端通过 save_file_from_url 直接保存。")
@Operation(summary = "下载删除品牌任务结果", description = "返回任务结果文件流,便于桌面端通过 save_file_from_url_new 直接保存。")
public void download(@PathVariable Long taskId, jakarta.servlet.http.HttpServletResponse response) {
String url = deleteBrandRunService.resolveTaskDownloadUrl(taskId);
String filename = deleteBrandRunService.resolveTaskDownloadFilename(taskId);

View File

@@ -12,6 +12,7 @@ public class DeleteBrandParsedFileCacheDto {
private String fileKey;
private String sourceFilename;
private String shopName;
private String companyName;
private String shopId;
private String platform;
private String openStoreUrl;

View File

@@ -21,6 +21,9 @@ public class DeleteBrandResultItemVo {
@Schema(description = "按 Excel 文件名解析出的店铺名")
private String shopName;
@Schema(description = "紫鸟公司名称")
private String companyName;
@Schema(description = "是否匹配到紫鸟店铺")
private boolean matched;

View File

@@ -127,12 +127,13 @@ public class DeleteBrandRunService {
ZiniaoShopMatchResultVo matchResult = normalizedShopName.isBlank()
? ziniaoShopSwitchService.emptyMatchResult()
: storeMatchByShopName.computeIfAbsent(normalizedShopName,
ignored -> ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName()));
ignored -> ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName(), false));
item.setMatchStatus(matchResult.getMatchStatus());
item.setMatchMessage(matchResult.getMatchMessage());
item.setMatched(matchResult.isMatched());
if (matchResult.isMatched()) {
item.setShopId(matchResult.getShopId());
item.setCompanyName(matchResult.getCompanyName());
item.setPlatform(matchResult.getPlatform());
item.setOpenStoreUrl(matchResult.getOpenStoreUrl());
}
@@ -153,6 +154,7 @@ public class DeleteBrandRunService {
cacheDto.setSourceFilename(sourceFile.getOriginalFilename());
cacheDto.setShopName(item.getShopName());
cacheDto.setShopId(item.getShopId());
cacheDto.setCompanyName(item.getCompanyName());
cacheDto.setPlatform(item.getPlatform());
cacheDto.setOpenStoreUrl(item.getOpenStoreUrl());
cacheDto.setTotalRows(parsed.totalRows());
@@ -204,7 +206,7 @@ public class DeleteBrandRunService {
task.setSuccessFileCount(parsedSuccessCount);
task.setFailedFileCount(parsedFailedCount);
task.setResultJson(JSONUtil.toJsonStr(items));
task.setResultJson(toCompactTaskResultJsonForDb(items));
task.setUpdatedAt(LocalDateTime.now());
task.setSourceFileCount(processableFileCount);
if (processableFileCount <= 0) {
@@ -295,6 +297,7 @@ public class DeleteBrandRunService {
}
item.setShopId(candidate.getShopId());
item.setCompanyName(candidate.getCompanyName());
item.setPlatform(candidate.getPlatform());
item.setOpenStoreUrl(candidate.getOpenStoreUrl());
@@ -541,10 +544,87 @@ public class DeleteBrandRunService {
return vo;
}
public com.nanri.aiimage.modules.deletebrand.model.vo.DeleteBrandTaskBatchVo getTaskProgress(java.util.List<Long> taskIds) {
com.nanri.aiimage.modules.deletebrand.model.vo.DeleteBrandTaskBatchVo vo = new com.nanri.aiimage.modules.deletebrand.model.vo.DeleteBrandTaskBatchVo();
if (taskIds == null || taskIds.isEmpty()) {
return vo;
}
java.util.List<Long> normalizedTaskIds = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.limit(50)
.toList();
if (normalizedTaskIds.isEmpty()) {
return vo;
}
java.util.Map<Long, FileTaskEntity> cachedTasks = deleteBrandTaskCacheService.getTaskCacheBatch(normalizedTaskIds);
java.util.List<Long> missingTaskIds = new java.util.ArrayList<>();
for (Long taskId : normalizedTaskIds) {
if (!cachedTasks.containsKey(taskId)) {
missingTaskIds.add(taskId);
}
}
java.util.Map<Long, FileTaskEntity> taskById = new java.util.LinkedHashMap<>(cachedTasks);
if (!missingTaskIds.isEmpty()) {
java.util.List<FileTaskEntity> dbTasks = fileTaskMapper.selectBatchIds(missingTaskIds).stream()
.filter(task -> task != null && MODULE_TYPE.equals(task.getModuleType()))
.toList();
for (FileTaskEntity dbTask : dbTasks) {
taskById.put(dbTask.getId(), dbTask);
if ("RUNNING".equals(dbTask.getStatus())) {
deleteBrandTaskCacheService.saveTaskCache(dbTask);
}
}
}
java.util.Map<Long, java.util.Map<Object, Object>> progressByTaskId = deleteBrandTaskCacheService.getProgressBatch(normalizedTaskIds);
for (Long taskId : normalizedTaskIds) {
FileTaskEntity task = taskById.get(taskId);
if (task == null) {
vo.getMissingTaskIds().add(taskId);
continue;
}
vo.getItems().add(buildTaskProgressDetail(task, progressByTaskId.get(taskId)));
}
return vo;
}
private DeleteBrandTaskDetailVo buildTaskDetail(FileTaskEntity task) {
return buildTaskDetail(task, null);
}
private DeleteBrandTaskDetailVo buildTaskProgressDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) {
task = refreshIndexedMatchesIfNeeded(task);
DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo();
taskVo.setId(task.getId());
taskVo.setTaskNo(task.getTaskNo());
taskVo.setStatus(task.getStatus());
taskVo.setSourceFileCount(task.getSourceFileCount());
taskVo.setSuccessFileCount(task.getSuccessFileCount());
taskVo.setFailedFileCount(task.getFailedFileCount());
taskVo.setErrorMessage(task.getErrorMessage());
taskVo.setCreatedAt(task.getCreatedAt());
taskVo.setUpdatedAt(task.getUpdatedAt());
taskVo.setFinishedAt(task.getFinishedAt());
if ("SUCCESS".equals(task.getStatus())) {
taskVo.setDownloadUrl(resolveTaskDownloadUrl(task.getId()));
taskVo.setDownloadFilename(resolveTaskDownloadFilename(task.getId()));
} else {
taskVo.setDownloadUrl(null);
taskVo.setDownloadFilename(null);
}
DeleteBrandTaskDetailVo detail = new DeleteBrandTaskDetailVo();
detail.setTask(taskVo);
detail.setLine_progress(buildLineProgress(task.getId(), progress));
detail.setFileProgress(java.util.List.of());
return detail;
}
private DeleteBrandTaskDetailVo buildTaskDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) {
task = refreshIndexedMatchesIfNeeded(task);
DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo();
@@ -711,11 +791,12 @@ public class DeleteBrandRunService {
unmatchedCount++;
continue;
}
ZiniaoShopMatchResultVo refreshed = ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName());
ZiniaoShopMatchResultVo refreshed = ziniaoShopSwitchService.findIndexedStoreByName(item.getShopName(), false);
if (refreshed != null) {
String oldStatus = item.getMatchStatus();
boolean oldMatched = item.isMatched();
String oldShopId = item.getShopId();
String oldCompanyName = item.getCompanyName();
String oldPlatform = item.getPlatform();
String oldOpenStoreUrl = item.getOpenStoreUrl();
String oldMessage = item.getMatchMessage();
@@ -724,12 +805,14 @@ public class DeleteBrandRunService {
item.setMatchStatus(refreshed.getMatchStatus());
item.setMatchMessage(refreshed.getMatchMessage());
item.setShopId(refreshed.getShopId());
item.setCompanyName(refreshed.getCompanyName());
item.setPlatform(refreshed.getPlatform());
item.setOpenStoreUrl(refreshed.getOpenStoreUrl());
if (!java.util.Objects.equals(oldStatus, item.getMatchStatus())
|| oldMatched != item.isMatched()
|| !java.util.Objects.equals(oldShopId, item.getShopId())
|| !java.util.Objects.equals(oldCompanyName, item.getCompanyName())
|| !java.util.Objects.equals(oldPlatform, item.getPlatform())
|| !java.util.Objects.equals(oldOpenStoreUrl, item.getOpenStoreUrl())
|| !java.util.Objects.equals(oldMessage, item.getMatchMessage())) {
@@ -745,7 +828,7 @@ public class DeleteBrandRunService {
if (!changed) {
return task;
}
task.setResultJson(JSONUtil.toJsonStr(items));
task.setResultJson(toCompactTaskResultJsonForDb(items));
task.setSourceFileCount(matchedCount);
task.setFailedFileCount(unmatchedCount);
task.setUpdatedAt(LocalDateTime.now());
@@ -984,6 +1067,26 @@ public class DeleteBrandRunService {
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = loadMergedChunks(taskId);
int finishedFiles = countCompletedFiles(mergedByFile);
if (finishedFiles < expectedFiles) {
Map<String, String> progress = new LinkedHashMap<>();
progress.put("finished_files", String.valueOf(finishedFiles));
// 定时补偿只负责“再试一次 finalize”不能把缺片任务重新续命
// 否则 stale-check 永远看不到超时任务。
if (!fromCompensation) {
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
}
deleteBrandTaskCacheService.saveProgress(taskId, progress);
task.setSuccessFileCount(finishedFiles);
if (!fromCompensation) {
task.setUpdatedAt(LocalDateTime.now());
}
fileTaskMapper.updateById(task);
log.info("[DeleteBrand] tryFinalizeTask skipping -> finishedFiles ({}) < expectedFiles ({})", finishedFiles, expectedFiles);
return;
}
Map<String, String> progress = new LinkedHashMap<>();
progress.put("finished_files", String.valueOf(finishedFiles));
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
@@ -995,11 +1098,6 @@ public class DeleteBrandRunService {
task.setSuccessFileCount(finishedFiles);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
if (finishedFiles < expectedFiles) {
log.info("[DeleteBrand] tryFinalizeTask skipping -> finishedFiles ({}) < expectedFiles ({})", finishedFiles, expectedFiles);
return;
}
log.info("[DeleteBrand] tryFinalizeTask calling finalizeTask -> finishedFiles: {}, expectedFiles: {}", finishedFiles, expectedFiles);
@@ -1120,6 +1218,7 @@ public class DeleteBrandRunService {
item.setFileKey(parsedFile.getFileKey());
item.setSourceFilename(parsedFile.getSourceFilename());
item.setShopName(parsedFile.getShopName());
item.setCompanyName(parsedFile.getCompanyName());
item.setOutputFilename(outputFile.getName());
item.setDownloadUrl(ossStorageService.getPublicUrl(objectKey));
item.setTotalRows(parsedFile.getTotalRows());
@@ -1140,7 +1239,7 @@ public class DeleteBrandRunService {
task.setStatus("SUCCESS");
task.setSuccessFileCount(successCount);
task.setErrorMessage(null);
task.setResultJson(JSONUtil.toJsonStr(finalItems));
task.setResultJson(toCompactTaskResultJsonForDb(finalItems));
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
@@ -1351,6 +1450,33 @@ public class DeleteBrandRunService {
return normalized.isBlank() ? defaultValue : normalized;
}
/**
* 落库用 result_json去掉国家分组、预览行等大字段保留匹配摘要与计数。
* 索引常命中后单次任务 JSON 体积会陡增,易触发 max_allowed_packet / 更新失败,进而前端 unwrap 报「请求失败」。
*/
private String toCompactTaskResultJsonForDb(List<DeleteBrandResultItemVo> items) {
if (items == null || items.isEmpty()) {
return JSONUtil.toJsonStr(List.of());
}
try {
List<DeleteBrandResultItemVo> clone = objectMapper.readValue(
objectMapper.writeValueAsString(items),
new TypeReference<List<DeleteBrandResultItemVo>>() {
});
for (DeleteBrandResultItemVo vo : clone) {
if (vo == null) {
continue;
}
vo.setCountries(new ArrayList<>());
vo.setPreviewRows(new ArrayList<>());
}
return JSONUtil.toJsonStr(clone);
} catch (Exception ex) {
log.warn("[delete-brand] compact task result json failed, msg={}", ex.getMessage());
return JSONUtil.toJsonStr(items);
}
}
private record ParsedDeleteBrandFile(int totalRows,
List<DeleteBrandCountryGroupVo> countries,
List<DeleteBrandPreviewRowVo> previewRows) {

View File

@@ -1,36 +1,103 @@
package com.nanri.aiimage.modules.deletebrand.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.DeleteBrandProgressProperties;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileTime;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.List;
import java.util.Map;
@Service
@RequiredArgsConstructor
@Slf4j
public class DeleteBrandStaleTaskService {
private static final String MODULE_TYPE = "DELETE_BRAND";
private static final String MODULE_TYPE_DELETE_BRAND = "DELETE_BRAND";
private static final String MODULE_TYPE_PRODUCT_RISK = "PRODUCT_RISK_RESOLVE";
private static final String MODULE_TYPE_PRICE_TRACK = "PRICE_TRACK";
private static final String MODULE_TYPE_SHOP_MATCH = "SHOP_MATCH";
private static final Duration STALE_CHECK_LOCK_TTL = Duration.ofMinutes(10);
private static final Duration FINALIZE_CHECK_LOCK_TTL = Duration.ofMinutes(10);
private static final Duration TEMP_DIR_CLEANUP_LOCK_TTL = Duration.ofMinutes(10);
private static final Duration TEMP_DIR_RETENTION = Duration.ofHours(48);
private final FileTaskMapper fileTaskMapper;
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
private final DeleteBrandRunService deleteBrandRunService;
private final ProductRiskTaskService productRiskTaskService;
private final ProductRiskTaskCacheService productRiskTaskCacheService;
private final PriceTrackTaskService priceTrackTaskService;
private final PriceTrackTaskCacheService priceTrackTaskCacheService;
private final ShopMatchTaskService shopMatchTaskService;
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
private final DeleteBrandProgressProperties deleteBrandProgressProperties;
private final DistributedJobLockService distributedJobLockService;
@Scheduled(cron = "${aiimage.delete-brand-progress.stale-check-cron:0 */2 * * * *}")
public void failStaleRunningTasks() {
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("delete-brand:stale-check", STALE_CHECK_LOCK_TTL);
if (lockHandle == null) {
log.info("[stale-check] skip because another instance holds the distributed lock");
return;
}
try (lockHandle) {
long startedAt = System.currentTimeMillis();
log.info("[stale-check] scan started thread={}", Thread.currentThread().getName());
failStaleDeleteBrandTasks();
ProductRiskStaleCheckStats stats = failStaleProductRiskResolveTasks();
PriceTrackStaleCheckStats priceTrackStats = failStalePriceTrackTasks();
ShopMatchStaleCheckStats shopMatchStats = failStaleShopMatchTasks();
log.info("[stale-check] product-risk summary scanned={} finalized={} failed={} skipped={} elapsedMs={} thread={}",
stats.scannedTaskCount,
stats.finalizedTaskCount,
stats.failedTaskCount,
stats.skippedTaskCount,
System.currentTimeMillis() - startedAt,
Thread.currentThread().getName());
log.info("[stale-check] price-track summary scanned={} finalized={} failed={} skipped={} elapsedMs={} thread={}",
priceTrackStats.scannedTaskCount,
priceTrackStats.finalizedTaskCount,
priceTrackStats.failedTaskCount,
priceTrackStats.skippedTaskCount,
System.currentTimeMillis() - startedAt,
Thread.currentThread().getName());
log.info("[stale-check] shop-match summary scanned={} finalized={} failed={} skipped={} elapsedMs={} thread={}",
shopMatchStats.scannedTaskCount,
shopMatchStats.finalizedTaskCount,
shopMatchStats.failedTaskCount,
shopMatchStats.skippedTaskCount,
System.currentTimeMillis() - startedAt,
Thread.currentThread().getName());
}
}
private void failStaleDeleteBrandTasks() {
LocalDateTime threshold = LocalDateTime.now().minusMinutes(deleteBrandProgressProperties.getHeartbeatTimeoutMinutes());
// 先按 DB updatedAt 粗筛,避免全表扫描
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
.eq(FileTaskEntity::getStatus, "RUNNING")
.lt(FileTaskEntity::getUpdatedAt, threshold)
.last("limit 200"));
@@ -57,26 +124,21 @@ public class DeleteBrandStaleTaskService {
}
}
LocalDateTime lastActivityTime;
// 只有当 has_progress 为 true 且超过 15 分钟没心跳,才认为是异常卡死
if (lastHeartbeatAt > 0 && isActive) {
// 有心跳,用最后心跳时间对比 15 分钟 (threshold 是 now - 15)
lastActivityTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastHeartbeatAt), ZoneId.systemDefault());
LocalDateTime lastActivityTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastHeartbeatAt), ZoneId.systemDefault());
if (lastActivityTime.isAfter(threshold)) {
continue; // 没超时
continue;
}
} else {
// 没有心跳,或者当前处在 has_progress: false (等待用户点击下一个任务推送的静默期)
// 静默期和排队期的宽限期一样,设为 12 个小时
LocalDateTime queuedThreshold = LocalDateTime.now().minusHours(12);
if (task.getCreatedAt() != null && task.getCreatedAt().isAfter(queuedThreshold)) {
continue; // 等待下个队列推送中,不要杀它!
continue;
}
}
int updated = fileTaskMapper.update(null, new com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper<FileTaskEntity>()
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
.eq(FileTaskEntity::getStatus, "RUNNING")
.set(FileTaskEntity::getStatus, "FAILED")
.set(FileTaskEntity::getErrorMessage, "结果回传长时间无响应,任务已自动失败")
@@ -89,20 +151,300 @@ public class DeleteBrandStaleTaskService {
}
}
private ProductRiskStaleCheckStats failStaleProductRiskResolveTasks() {
ProductRiskStaleCheckStats stats = new ProductRiskStaleCheckStats();
long minutes = Math.max(1L, deleteBrandProgressProperties.getProductRiskStaleTimeoutMinutes());
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getProductRiskInitialTimeoutMinutes());
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
long nowMillis = System.currentTimeMillis();
LocalDateTime now = LocalDateTime.now();
LocalDateTime threshold = now.minusMinutes(minutes);
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PRODUCT_RISK)
.eq(FileTaskEntity::getStatus, "RUNNING")
.lt(FileTaskEntity::getUpdatedAt, threshold)
.last("limit 200"));
stats.scannedTaskCount = runningTasks.size();
if (runningTasks.isEmpty()) {
return stats;
}
log.info("[stale-check] product-risk candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
for (FileTaskEntity task : runningTasks) {
boolean hasUploadedPayload = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId());
long lastPayloadHeartbeatMillis = productRiskTaskCacheService.getTaskHeartbeatMillis(task.getId());
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = hasUploadedPayload || hasFinishedRows;
if (lastPayloadHeartbeatMillis > 0L && nowMillis - lastPayloadHeartbeatMillis < staleTimeoutMillis) {
stats.skippedTaskCount++;
log.info("[stale-check] product-risk skip recent-task-heartbeat taskId={} lastPayloadHeartbeatMillis={} timeoutMinutes={} updatedAt={}",
task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt());
continue;
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++;
log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}",
task.getId(), task.getCreatedAt(), initialThreshold);
continue;
}
try {
if (productRiskTaskService.tryFinalizeTask(task.getId(), true)) {
stats.finalizedTaskCount++;
log.info("[stale-check] product-risk finalized taskId={}", task.getId());
continue;
}
} catch (Exception ex) {
log.warn("[stale-check] product-risk finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
}
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PRODUCT_RISK)
.eq(FileTaskEntity::getStatus, "RUNNING")
.set(FileTaskEntity::getStatus, "FAILED")
.set(FileTaskEntity::getErrorMessage, "长时间未收到 Python 结果回传,任务已自动失败")
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
if (updated > 0) {
stats.failedTaskCount++;
log.warn("[stale-check] product-risk failed taskId={} reason=timeout", task.getId());
} else {
stats.skippedTaskCount++;
}
}
return stats;
}
private PriceTrackStaleCheckStats failStalePriceTrackTasks() {
PriceTrackStaleCheckStats stats = new PriceTrackStaleCheckStats();
long minutes = Math.max(1L, deleteBrandProgressProperties.getPriceTrackStaleTimeoutMinutes());
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getPriceTrackInitialTimeoutMinutes());
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
long nowMillis = System.currentTimeMillis();
LocalDateTime now = LocalDateTime.now();
LocalDateTime threshold = now.minusMinutes(minutes);
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PRICE_TRACK)
.eq(FileTaskEntity::getStatus, "RUNNING")
.lt(FileTaskEntity::getUpdatedAt, threshold)
.last("limit 200"));
stats.scannedTaskCount = runningTasks.size();
if (runningTasks.isEmpty()) {
return stats;
}
log.info("[stale-check] price-track candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
for (FileTaskEntity task : runningTasks) {
long lastHeartbeatMillis = priceTrackTaskCacheService.getTaskHeartbeatMillis(task.getId());
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows;
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
stats.skippedTaskCount++;
continue;
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++;
continue;
}
try {
if (priceTrackTaskService.tryFinalizeTask(task.getId(), true)) {
stats.finalizedTaskCount++;
log.info("[stale-check] price-track finalized taskId={}", task.getId());
continue;
}
} catch (Exception ex) {
log.warn("[stale-check] price-track finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
}
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PRICE_TRACK)
.eq(FileTaskEntity::getStatus, "RUNNING")
.set(FileTaskEntity::getStatus, "FAILED")
.set(FileTaskEntity::getErrorMessage, "长时间未收到 Python 结果回传,任务已自动失败")
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
if (updated > 0) {
stats.failedTaskCount++;
priceTrackTaskCacheService.deleteTaskCache(task.getId());
log.warn("[stale-check] price-track failed taskId={} reason=timeout", task.getId());
} else {
stats.skippedTaskCount++;
}
}
return stats;
}
private ShopMatchStaleCheckStats failStaleShopMatchTasks() {
ShopMatchStaleCheckStats stats = new ShopMatchStaleCheckStats();
long minutes = Math.max(1L, deleteBrandProgressProperties.getShopMatchStaleTimeoutMinutes());
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getShopMatchInitialTimeoutMinutes());
LocalDateTime now = LocalDateTime.now();
LocalDateTime threshold = now.minusMinutes(minutes);
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_SHOP_MATCH)
.eq(FileTaskEntity::getStatus, "RUNNING")
.lt(FileTaskEntity::getUpdatedAt, threshold)
.last("limit 200"));
stats.scannedTaskCount = runningTasks.size();
if (runningTasks.isEmpty()) {
return stats;
}
log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
for (FileTaskEntity task : runningTasks) {
boolean hasUploadedPayload = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = hasUploadedPayload || hasFinishedRows;
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++;
log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}",
task.getId(), task.getCreatedAt(), initialThreshold);
continue;
}
try {
if (shopMatchTaskService.tryFinalizeTask(task.getId(), true)) {
stats.finalizedTaskCount++;
log.info("[stale-check] shop-match finalized taskId={}", task.getId());
continue;
}
} catch (Exception ex) {
log.warn("[stale-check] shop-match finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
}
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_SHOP_MATCH)
.eq(FileTaskEntity::getStatus, "RUNNING")
.set(FileTaskEntity::getStatus, "FAILED")
.set(FileTaskEntity::getErrorMessage, "长时间未收到 Python 结果回传,任务已自动失败")
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
if (updated > 0) {
stats.failedTaskCount++;
shopMatchTaskCacheService.deleteTaskCache(task.getId());
log.warn("[stale-check] shop-match failed taskId={} reason=timeout", task.getId());
} else {
stats.skippedTaskCount++;
}
}
return stats;
}
@Scheduled(cron = "${aiimage.delete-brand-progress.finalize-check-cron:30 */2 * * * *}")
public void finalizeCompletedRunningTasks() {
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getStatus, "RUNNING")
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit 100"));
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("delete-brand:finalize-check", FINALIZE_CHECK_LOCK_TTL);
if (lockHandle == null) {
log.info("[finalize-check] skip because another instance holds the distributed lock");
return;
}
try (lockHandle) {
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
.eq(FileTaskEntity::getStatus, "RUNNING")
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit 100"));
for (FileTaskEntity task : runningTasks) {
try {
deleteBrandRunService.tryFinalizeTask(task.getId(), true);
} catch (Exception ignored) {
// 定时补偿不影响主流程;下次调度或人工重试时再处理
for (FileTaskEntity task : runningTasks) {
try {
deleteBrandRunService.tryFinalizeTask(task.getId(), true);
} catch (Exception ignored) {
// Keep compensation best-effort; the next schedule can retry.
}
}
}
}
@Scheduled(cron = "${aiimage.temp-dir.cleanup-cron:15 */30 * * * *}")
public void cleanupExpiredTempDirs() {
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("temp-dir:cleanup", TEMP_DIR_CLEANUP_LOCK_TTL);
if (lockHandle == null) {
log.info("[temp-dir-cleanup] skip because another instance holds the distributed lock");
return;
}
try (lockHandle) {
Instant cutoff = Instant.now().minus(TEMP_DIR_RETENTION);
List<String> dirNames = List.of(
"shop-match-cache",
"brand-task-cache",
"delete-brand-task-cache",
"shop-match-result",
"delete-brand-result",
"product-risk-result",
"price-track-result"
);
for (String dirName : dirNames) {
cleanupTempRoot(dirName, cutoff);
}
}
}
private void cleanupTempRoot(String dirName, Instant cutoff) {
Path root = Path.of(System.getProperty("java.io.tmpdir"), dirName);
if (!Files.isDirectory(root)) {
return;
}
try (var stream = Files.list(root)) {
for (Path child : stream.toList()) {
try {
FileTime lastModified = Files.getLastModifiedTime(child);
if (lastModified.toInstant().isAfter(cutoff)) {
continue;
}
deleteRecursively(child);
log.info("[temp-dir-cleanup] deleted expired temp path dir={} path={}", dirName, child);
} catch (Exception ex) {
log.warn("[temp-dir-cleanup] delete failed dir={} path={} msg={}", dirName, child, ex.getMessage());
}
}
} catch (IOException ex) {
log.warn("[temp-dir-cleanup] scan failed dir={} msg={}", dirName, ex.getMessage());
}
}
private void deleteRecursively(Path path) throws IOException {
if (!Files.exists(path)) {
return;
}
try (var walk = Files.walk(path)) {
walk.sorted(java.util.Comparator.reverseOrder()).forEach(item -> {
try {
Files.deleteIfExists(item);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
});
} catch (RuntimeException ex) {
if (ex.getCause() instanceof IOException ioException) {
throw ioException;
}
throw ex;
}
}
private static final class ProductRiskStaleCheckStats {
private int scannedTaskCount;
private int finalizedTaskCount;
private int failedTaskCount;
private int skippedTaskCount;
}
private static final class ShopMatchStaleCheckStats {
private int scannedTaskCount;
private int finalizedTaskCount;
private int failedTaskCount;
private int skippedTaskCount;
}
private static final class PriceTrackStaleCheckStats {
private int scannedTaskCount;
private int finalizedTaskCount;
private int failedTaskCount;
private int skippedTaskCount;
}
}

View File

@@ -7,6 +7,9 @@ import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.Duration;
import java.util.concurrent.ConcurrentHashMap;
@@ -28,10 +31,17 @@ public class DeleteBrandTaskCacheService {
public void saveParsedPayload(Long taskId, Object payload) {
try {
stringRedisTemplate.opsForValue().set(buildPayloadKey(taskId), objectMapper.writeValueAsString(payload), Duration.ofHours(PAYLOAD_TTL_HOURS));
Files.createDirectories(buildTaskDir(taskId));
Files.writeString(
buildPayloadFile(taskId),
objectMapper.writeValueAsString(payload),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE
);
parsedPayloadLocalCache.put(taskId, new LocalParsedPayloadCacheEntry(System.currentTimeMillis(), payload));
} catch (Exception ex) {
throw new BusinessException("暂存删除品牌原始数据失败");
throw new BusinessException("鏆傚瓨鍒犻櫎鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
@@ -46,21 +56,21 @@ public class DeleteBrandTaskCacheService {
try {
return objectMapper.convertValue(cached.value(), typeReference);
} catch (Exception ignored) {
// fallthrough to redis
// fall through to file
}
}
String raw = stringRedisTemplate.opsForValue().get(buildPayloadKey(taskId));
if (raw == null || raw.isBlank()) {
Path payloadFile = buildPayloadFile(taskId);
if (!Files.isRegularFile(payloadFile)) {
parsedPayloadLocalCache.remove(taskId);
return null;
}
try {
T parsed = objectMapper.readValue(raw, typeReference);
T parsed = objectMapper.readValue(Files.readString(payloadFile), typeReference);
parsedPayloadLocalCache.put(taskId, new LocalParsedPayloadCacheEntry(now, parsed));
return parsed;
} catch (Exception ex) {
throw new BusinessException("读取删除品牌原始数据失败");
throw new BusinessException("璇诲彇鍒犻櫎鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
@@ -114,17 +124,17 @@ public class DeleteBrandTaskCacheService {
public boolean hasSameResultChunk(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
if (chunkIndex == null || chunkIndex <= 0) {
throw new BusinessException("chunkIndex 不合法");
throw new BusinessException("chunkIndex 涓嶅悎娉?");
}
if (fileIdentity == null || fileIdentity.isBlank()) {
throw new BusinessException("fileIdentity 不能为空");
throw new BusinessException("fileIdentity 涓嶈兘涓虹┖");
}
String field = fileIdentity.trim() + "#" + chunkIndex;
String chunkJson;
try {
chunkJson = objectMapper.writeValueAsString(chunk);
} catch (Exception ex) {
throw new BusinessException("读取删除品牌结果分片失败");
throw new BusinessException("璇诲彇鍒犻櫎鍝佺墝缁撴灉鍒嗙墖澶辫触");
}
Object existing = stringRedisTemplate.opsForHash().get(buildResultChunksKey(taskId), field);
if (chunkJson.equals(existing)) {
@@ -137,17 +147,17 @@ public class DeleteBrandTaskCacheService {
public boolean mergeResultChunk(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
String resultKey = buildResultChunksKey(taskId);
if (chunkIndex == null || chunkIndex <= 0) {
throw new BusinessException("chunkIndex 不合法");
throw new BusinessException("chunkIndex 涓嶅悎娉?");
}
if (fileIdentity == null || fileIdentity.isBlank()) {
throw new BusinessException("fileIdentity 不能为空");
throw new BusinessException("fileIdentity 涓嶈兘涓虹┖");
}
String field = fileIdentity.trim() + "#" + chunkIndex;
String chunkJson;
try {
chunkJson = objectMapper.writeValueAsString(chunk);
} catch (Exception ex) {
throw new BusinessException("暂存删除品牌结果分片失败");
throw new BusinessException("鏆傚瓨鍒犻櫎鍝佺墝缁撴灉鍒嗙墖澶辫触");
}
Object existing = stringRedisTemplate.opsForHash().get(resultKey, field);
if (chunkJson.equals(existing)) {
@@ -178,14 +188,16 @@ public class DeleteBrandTaskCacheService {
public void delete(Long taskId) {
parsedPayloadLocalCache.remove(taskId);
stringRedisTemplate.delete(buildPayloadKey(taskId));
stringRedisTemplate.delete(buildProgressKey(taskId));
stringRedisTemplate.delete(buildResultChunksKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
deleteTaskDirQuietly(taskId);
}
public void saveTaskCache(com.nanri.aiimage.modules.task.model.entity.FileTaskEntity task) {
if (task == null || task.getId() == null) return;
if (task == null || task.getId() == null) {
return;
}
try {
stringRedisTemplate.opsForValue().set(buildTaskEntityKey(task.getId()), objectMapper.writeValueAsString(task), Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ignored) {
@@ -194,10 +206,14 @@ public class DeleteBrandTaskCacheService {
public java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> getTaskCacheBatch(java.util.List<Long> taskIds) {
java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> result = new java.util.LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) return result;
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
java.util.List<Long> normalized = taskIds.stream().filter(id -> id != null && id > 0).distinct().toList();
if (normalized.isEmpty()) return result;
if (normalized.isEmpty()) {
return result;
}
java.util.List<String> keys = normalized.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
@@ -215,6 +231,30 @@ public class DeleteBrandTaskCacheService {
return result;
}
private Path buildTaskDir(Long taskId) {
return Path.of(System.getProperty("java.io.tmpdir"), "delete-brand-task-cache", String.valueOf(taskId));
}
private Path buildPayloadFile(Long taskId) {
return buildTaskDir(taskId).resolve("parsed-payload.json");
}
private void deleteTaskDirQuietly(Long taskId) {
Path taskDir = buildTaskDir(taskId);
if (!Files.exists(taskDir)) {
return;
}
try (var walk = Files.walk(taskDir)) {
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (Exception ignored) {
}
});
} catch (Exception ignored) {
}
}
private String buildTaskEntityKey(Long taskId) {
return "delete-brand:task:entity:" + taskId;
}
@@ -225,10 +265,6 @@ public class DeleteBrandTaskCacheService {
}
}
private String buildPayloadKey(Long taskId) {
return "delete-brand:task:parsed-payload:" + taskId;
}
private String buildProgressKey(Long taskId) {
return "delete-brand:task:progress:" + taskId;
}

View File

@@ -0,0 +1,83 @@
package com.nanri.aiimage.modules.permission.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuCreateRequest;
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuUpdateRequest;
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/admin")
@Tag(name = "菜单权限管理", description = "统一维护软件端与后台管理端的菜单权限配置")
public class PermissionMenuController {
private final PermissionMenuService permissionMenuService;
@GetMapping("/permission-menus")
@Operation(summary = "查询菜单权限列表")
public ApiResponse<List<PermissionMenuItemVo>> listMenus(
@Parameter(description = "菜单类型: app/admin") @RequestParam(required = false) String menuType) {
return ApiResponse.success(permissionMenuService.list(menuType));
}
@PostMapping("/permission-menus")
@Operation(summary = "新增菜单权限项")
public ApiResponse<PermissionMenuItemVo> createMenu(@Valid @RequestBody PermissionMenuCreateRequest request) {
return ApiResponse.success("创建成功", permissionMenuService.create(request));
}
@PutMapping("/permission-menus/{id}")
@Operation(summary = "编辑菜单权限项")
public ApiResponse<PermissionMenuItemVo> updateMenu(@PathVariable Long id,
@Valid @RequestBody PermissionMenuUpdateRequest request) {
return ApiResponse.success("更新成功", permissionMenuService.update(id, request));
}
@DeleteMapping("/permission-menus/{id}")
@Operation(summary = "删除菜单权限项")
public ApiResponse<Void> deleteMenu(@PathVariable Long id) {
permissionMenuService.delete(id);
return ApiResponse.success("删除成功", null);
}
@GetMapping("/permission-users/{userId}/columns")
@Operation(summary = "查询用户菜单权限 ID 列表")
public ApiResponse<UserColumnIdsVo> getUserColumnIds(@PathVariable Long userId,
@RequestParam(required = false) String menuType) {
return ApiResponse.success(permissionMenuService.getUserColumnIds(userId, menuType));
}
@PutMapping("/permission-users/{userId}/columns")
@Operation(summary = "更新用户菜单权限")
public ApiResponse<Void> updateUserColumnIds(@PathVariable Long userId,
@RequestBody(required = false) UserColumnPermissionUpdateRequest request) {
permissionMenuService.updateUserColumnPermissions(userId, request);
return ApiResponse.success("保存成功", null);
}
@GetMapping("/permission-users/{userId}/column-permissions")
@Operation(summary = "查询用户菜单权限详情")
public ApiResponse<List<PermissionMenuItemVo>> getUserColumnPermissions(@PathVariable Long userId,
@RequestParam(required = false) String menuType) {
return ApiResponse.success(permissionMenuService.getUserColumnPermissions(userId, menuType));
}
}

View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.permission.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface AdminUserMapper extends BaseMapper<AdminUserEntity> {
}

View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.permission.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.permission.model.entity.PermissionMenuEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface PermissionMenuMapper extends BaseMapper<PermissionMenuEntity> {
}

View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.permission.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface UserColumnPermissionMapper extends BaseMapper<UserColumnPermissionEntity> {
}

View File

@@ -0,0 +1,22 @@
package com.nanri.aiimage.modules.permission.model.dto;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
@Data
public class PermissionMenuCreateRequest {
@NotBlank(message = "菜单名称不能为空")
private String name;
@NotBlank(message = "菜单标识不能为空")
private String columnKey;
@NotBlank(message = "菜单类型不能为空")
private String menuType;
@NotBlank(message = "菜单路由不能为空")
private String routePath;
private Integer sortOrder;
}

View File

@@ -0,0 +1,22 @@
package com.nanri.aiimage.modules.permission.model.dto;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
@Data
public class PermissionMenuUpdateRequest {
@NotBlank(message = "菜单名称不能为空")
private String name;
@NotBlank(message = "菜单标识不能为空")
private String columnKey;
@NotBlank(message = "菜单类型不能为空")
private String menuType;
@NotBlank(message = "菜单路由不能为空")
private String routePath;
private Integer sortOrder;
}

View File

@@ -0,0 +1,11 @@
package com.nanri.aiimage.modules.permission.model.dto;
import lombok.Data;
import java.util.List;
@Data
public class UserColumnPermissionUpdateRequest {
private List<Long> columnIds;
}

View File

@@ -0,0 +1,19 @@
package com.nanri.aiimage.modules.permission.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
@Data
@TableName("users")
public class AdminUserEntity {
@TableId(type = IdType.AUTO)
private Long id;
private String username;
private String role;
@TableField("created_by_id")
private Long createdById;
}

View File

@@ -0,0 +1,22 @@
package com.nanri.aiimage.modules.permission.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("columns")
public class PermissionMenuEntity {
@TableId(type = IdType.AUTO)
private Long id;
private String name;
private String columnKey;
private String menuType;
private String routePath;
private Integer sortOrder;
private LocalDateTime createdAt;
}

View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.permission.model.entity;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
@Data
@TableName("user_column_permission")
public class UserColumnPermissionEntity {
@TableField("user_id")
private Long userId;
@TableField("column_id")
private Long columnId;
}

View File

@@ -0,0 +1,17 @@
package com.nanri.aiimage.modules.permission.model.vo;
import lombok.Data;
import java.time.LocalDateTime;
@Data
public class PermissionMenuItemVo {
private Long id;
private String name;
private String columnKey;
private String menuType;
private String routePath;
private Integer sortOrder;
private LocalDateTime createdAt;
}

View File

@@ -0,0 +1,11 @@
package com.nanri.aiimage.modules.permission.model.vo;
import lombok.Data;
import java.util.List;
@Data
public class UserColumnIdsVo {
private List<Long> columnIds;
}

View File

@@ -0,0 +1,85 @@
package com.nanri.aiimage.modules.permission.service;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@RequiredArgsConstructor
@Slf4j
public class PermissionMenuSchemaInitializer {
private final JdbcTemplate jdbcTemplate;
private static final List<DefaultAdminMenu> DEFAULT_ADMIN_MENUS = List.of(
new DefaultAdminMenu("用户管理", "admin_users", "users", 10),
new DefaultAdminMenu("栏目权限配置", "admin_columns", "columns", 20),
new DefaultAdminMenu("数据去重总数据", "admin_dedupe_total_data", "dedupe-total-data", 30),
new DefaultAdminMenu("店铺密钥管理", "admin_shop_keys", "shop-keys", 40),
new DefaultAdminMenu("店铺管理", "admin_shop_manage", "shop-manage", 50),
new DefaultAdminMenu("跳过跟价ASIN", "admin_skip_price_asin", "skip-price-asin", 60),
new DefaultAdminMenu("查看生成记录", "admin_history", "history", 70),
new DefaultAdminMenu("版本管理", "admin_version", "version", 80)
);
@PostConstruct
public void initialize() {
executeQuietly("""
CREATE TABLE IF NOT EXISTS columns (
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(128) NOT NULL COMMENT '菜单标题',
column_key VARCHAR(64) NOT NULL COMMENT '菜单唯一标识',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE KEY uk_column_key (column_key)
)
""");
executeQuietly("ALTER TABLE columns ADD COLUMN menu_type VARCHAR(20) NOT NULL DEFAULT 'app' COMMENT '菜单类型: app/admin' AFTER column_key");
executeQuietly("ALTER TABLE columns ADD COLUMN route_path VARCHAR(255) NOT NULL DEFAULT '' COMMENT '菜单路由或页面标识' AFTER menu_type");
executeQuietly("ALTER TABLE columns ADD COLUMN sort_order INT NOT NULL DEFAULT 0 COMMENT '菜单排序' AFTER route_path");
executeQuietly("UPDATE columns SET menu_type = 'app' WHERE menu_type IS NULL OR menu_type = ''");
executeQuietly("UPDATE columns SET sort_order = id WHERE sort_order IS NULL OR sort_order = 0");
executeQuietly("ALTER TABLE columns ADD UNIQUE KEY uk_menu_type_route_path (menu_type, route_path)");
ensureDefaultAdminMenus();
executeQuietly("""
CREATE TABLE IF NOT EXISTS user_column_permission (
user_id INT NOT NULL,
column_id INT NOT NULL,
PRIMARY KEY (user_id, column_id),
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
FOREIGN KEY (column_id) REFERENCES columns(id) ON DELETE CASCADE
)
""");
}
private void ensureDefaultAdminMenus() {
for (DefaultAdminMenu menu : DEFAULT_ADMIN_MENUS) {
executeQuietly("""
INSERT INTO columns (name, column_key, menu_type, route_path, sort_order)
SELECT '%s', '%s', 'admin', '%s', %d
WHERE NOT EXISTS (
SELECT 1 FROM columns WHERE column_key = '%s'
)
""".formatted(menu.name(), menu.columnKey(), menu.routePath(), menu.sortOrder(), menu.columnKey()));
executeQuietly("""
UPDATE columns
SET sort_order = %d
WHERE column_key = '%s' AND (sort_order IS NULL OR sort_order = 0)
""".formatted(menu.sortOrder(), menu.columnKey()));
}
}
private void executeQuietly(String sql) {
try {
jdbcTemplate.execute(sql);
} catch (Exception ex) {
log.debug("[permission-menu] schema init skipped: {}", ex.getMessage());
}
}
private record DefaultAdminMenu(String name, String columnKey, String routePath, int sortOrder) {
}
}

View File

@@ -0,0 +1,286 @@
package com.nanri.aiimage.modules.permission.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.mapper.PermissionMenuMapper;
import com.nanri.aiimage.modules.permission.mapper.UserColumnPermissionMapper;
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuCreateRequest;
import com.nanri.aiimage.modules.permission.model.dto.PermissionMenuUpdateRequest;
import com.nanri.aiimage.modules.permission.model.dto.UserColumnPermissionUpdateRequest;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.model.entity.PermissionMenuEntity;
import com.nanri.aiimage.modules.permission.model.entity.UserColumnPermissionEntity;
import com.nanri.aiimage.modules.permission.model.vo.PermissionMenuItemVo;
import com.nanri.aiimage.modules.permission.model.vo.UserColumnIdsVo;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class PermissionMenuService {
public static final String MENU_TYPE_APP = "app";
public static final String MENU_TYPE_ADMIN = "admin";
private final PermissionMenuMapper permissionMenuMapper;
private final UserColumnPermissionMapper userColumnPermissionMapper;
private final AdminUserMapper adminUserMapper;
public List<PermissionMenuItemVo> list(String menuType) {
LambdaQueryWrapper<PermissionMenuEntity> query = new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(isValidMenuType(menuType), PermissionMenuEntity::getMenuType, normalizeMenuType(menuType))
.orderByAsc(PermissionMenuEntity::getSortOrder)
.orderByAsc(PermissionMenuEntity::getId);
return permissionMenuMapper.selectList(query).stream()
.map(this::toItemVo)
.toList();
}
@Transactional
public PermissionMenuItemVo create(PermissionMenuCreateRequest request) {
String name = normalizeRequired(request.getName(), "菜单名称不能为空");
String columnKey = normalizeRequired(request.getColumnKey(), "菜单标识不能为空");
String menuType = normalizeMenuTypeRequired(request.getMenuType());
String routePath = normalizeRequired(request.getRoutePath(), "菜单路由不能为空");
ensureUniqueColumnKey(columnKey, null);
ensureUniqueRoutePath(menuType, routePath, null);
PermissionMenuEntity entity = new PermissionMenuEntity();
entity.setName(name);
entity.setColumnKey(columnKey);
entity.setMenuType(menuType);
entity.setRoutePath(routePath);
entity.setSortOrder(resolveSortOrder(request.getSortOrder(), null));
permissionMenuMapper.insert(entity);
return toItemVo(getMenuById(entity.getId()));
}
@Transactional
public PermissionMenuItemVo update(Long id, PermissionMenuUpdateRequest request) {
PermissionMenuEntity entity = getMenuById(id);
String name = normalizeRequired(request.getName(), "菜单名称不能为空");
String columnKey = normalizeRequired(request.getColumnKey(), "菜单标识不能为空");
String menuType = normalizeMenuTypeRequired(request.getMenuType());
String routePath = normalizeRequired(request.getRoutePath(), "菜单路由不能为空");
ensureUniqueColumnKey(columnKey, id);
ensureUniqueRoutePath(menuType, routePath, id);
entity.setName(name);
entity.setColumnKey(columnKey);
entity.setMenuType(menuType);
entity.setRoutePath(routePath);
entity.setSortOrder(resolveSortOrder(request.getSortOrder(), id));
permissionMenuMapper.updateById(entity);
return toItemVo(getMenuById(id));
}
@Transactional
public void delete(Long id) {
PermissionMenuEntity entity = getMenuById(id);
userColumnPermissionMapper.delete(new LambdaUpdateWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getColumnId, entity.getId()));
permissionMenuMapper.deleteById(entity.getId());
}
public UserColumnIdsVo getUserColumnIds(Long userId, String menuType) {
AdminUserEntity user = getUserById(userId);
List<Long> columnIds;
if (isSuperAdmin(user)) {
columnIds = list(menuType).stream().map(PermissionMenuItemVo::getId).toList();
} else {
List<Long> assignedIds = userColumnPermissionMapper.selectList(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId))
.stream()
.map(UserColumnPermissionEntity::getColumnId)
.distinct()
.toList();
if (assignedIds.isEmpty()) {
columnIds = List.of();
} else if (isValidMenuType(menuType)) {
Set<Long> allowedIds = list(menuType).stream()
.map(PermissionMenuItemVo::getId)
.collect(Collectors.toSet());
columnIds = assignedIds.stream().filter(allowedIds::contains).toList();
} else {
columnIds = assignedIds;
}
}
UserColumnIdsVo vo = new UserColumnIdsVo();
vo.setColumnIds(columnIds);
return vo;
}
public List<PermissionMenuItemVo> getUserColumnPermissions(Long userId, String menuType) {
AdminUserEntity user = getUserById(userId);
if (isSuperAdmin(user)) {
return list(menuType);
}
List<Long> assignedIds = userColumnPermissionMapper.selectList(
new LambdaQueryWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId))
.stream()
.map(UserColumnPermissionEntity::getColumnId)
.distinct()
.toList();
if (assignedIds.isEmpty()) {
return List.of();
}
List<PermissionMenuEntity> menus = permissionMenuMapper.selectBatchIds(assignedIds);
Map<Long, PermissionMenuEntity> menuMap = menus.stream()
.collect(Collectors.toMap(PermissionMenuEntity::getId, Function.identity()));
String safeMenuType = normalizeMenuType(menuType);
return assignedIds.stream()
.map(menuMap::get)
.filter(menu -> menu != null)
.filter(menu -> safeMenuType == null || safeMenuType.equals(menu.getMenuType()))
.sorted(Comparator
.comparing(PermissionMenuEntity::getSortOrder, Comparator.nullsLast(Integer::compareTo))
.thenComparing(PermissionMenuEntity::getId, Comparator.nullsLast(Long::compareTo)))
.map(this::toItemVo)
.toList();
}
@Transactional
public void updateUserColumnPermissions(Long userId, UserColumnPermissionUpdateRequest request) {
getUserById(userId);
List<Long> requestedIds = request == null ? List.of() : normalizeColumnIds(request.getColumnIds());
if (!requestedIds.isEmpty()) {
Long validCount = permissionMenuMapper.selectCount(new LambdaQueryWrapper<PermissionMenuEntity>()
.in(PermissionMenuEntity::getId, requestedIds));
if (validCount == null || validCount != requestedIds.size()) {
throw new BusinessException("存在无效的菜单权限项");
}
}
userColumnPermissionMapper.delete(new LambdaUpdateWrapper<UserColumnPermissionEntity>()
.eq(UserColumnPermissionEntity::getUserId, userId));
for (Long columnId : requestedIds) {
UserColumnPermissionEntity entity = new UserColumnPermissionEntity();
entity.setUserId(userId);
entity.setColumnId(columnId);
userColumnPermissionMapper.insert(entity);
}
}
private PermissionMenuEntity getMenuById(Long id) {
PermissionMenuEntity entity = permissionMenuMapper.selectById(id);
if (entity == null) {
throw new BusinessException("菜单权限项不存在");
}
return entity;
}
private AdminUserEntity getUserById(Long userId) {
AdminUserEntity entity = adminUserMapper.selectById(userId);
if (entity == null) {
throw new BusinessException("用户不存在");
}
return entity;
}
private boolean isSuperAdmin(AdminUserEntity user) {
return user != null && "super_admin".equalsIgnoreCase(normalizeRequired(user.getRole(), ""));
}
private void ensureUniqueColumnKey(String columnKey, Long excludeId) {
PermissionMenuEntity exists = permissionMenuMapper.selectOne(new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(PermissionMenuEntity::getColumnKey, columnKey)
.last("LIMIT 1"));
if (exists != null && (excludeId == null || !excludeId.equals(exists.getId()))) {
throw new BusinessException("菜单标识已存在");
}
}
private void ensureUniqueRoutePath(String menuType, String routePath, Long excludeId) {
PermissionMenuEntity exists = permissionMenuMapper.selectOne(new LambdaQueryWrapper<PermissionMenuEntity>()
.eq(PermissionMenuEntity::getMenuType, menuType)
.eq(PermissionMenuEntity::getRoutePath, routePath)
.last("LIMIT 1"));
if (exists != null && (excludeId == null || !excludeId.equals(exists.getId()))) {
throw new BusinessException("同一菜单类型下的菜单路由已存在");
}
}
private String normalizeRequired(String value, String message) {
String normalized = value == null ? "" : value.trim();
if (normalized.isEmpty() && !message.isEmpty()) {
throw new BusinessException(message);
}
return normalized;
}
private boolean isValidMenuType(String menuType) {
return normalizeMenuType(menuType) != null;
}
private String normalizeMenuTypeRequired(String menuType) {
String normalized = normalizeMenuType(menuType);
if (normalized == null) {
throw new BusinessException("菜单类型仅支持 app 或 admin");
}
return normalized;
}
private String normalizeMenuType(String menuType) {
String normalized = menuType == null ? "" : menuType.trim().toLowerCase(Locale.ROOT);
if (normalized.isEmpty()) {
return null;
}
if (MENU_TYPE_APP.equals(normalized) || MENU_TYPE_ADMIN.equals(normalized)) {
return normalized;
}
return null;
}
private List<Long> normalizeColumnIds(List<Long> columnIds) {
if (columnIds == null || columnIds.isEmpty()) {
return List.of();
}
Set<Long> uniqueIds = new LinkedHashSet<>();
for (Long columnId : columnIds) {
if (columnId != null && columnId > 0) {
uniqueIds.add(columnId);
}
}
return new ArrayList<>(uniqueIds);
}
private PermissionMenuItemVo toItemVo(PermissionMenuEntity entity) {
PermissionMenuItemVo vo = new PermissionMenuItemVo();
vo.setId(entity.getId());
vo.setName(entity.getName());
vo.setColumnKey(entity.getColumnKey());
vo.setMenuType(entity.getMenuType());
vo.setRoutePath(entity.getRoutePath());
vo.setSortOrder(entity.getSortOrder());
vo.setCreatedAt(entity.getCreatedAt());
return vo;
}
private Integer resolveSortOrder(Integer sortOrder, Long excludeId) {
if (sortOrder != null) {
return sortOrder;
}
PermissionMenuEntity tail = permissionMenuMapper.selectOne(new LambdaQueryWrapper<PermissionMenuEntity>()
.ne(excludeId != null, PermissionMenuEntity::getId, excludeId)
.orderByDesc(PermissionMenuEntity::getSortOrder)
.orderByDesc(PermissionMenuEntity::getId)
.last("LIMIT 1"));
int currentMax = tail == null || tail.getSortOrder() == null ? 0 : tail.getSortOrder();
return currentMax + 1;
}
}

View File

@@ -0,0 +1,204 @@
package com.nanri.aiimage.modules.pricetrack.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCandidateAddRequest;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCountryPreferenceSaveRequest;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackCreateTaskRequest;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackMatchShopsRequest;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequest;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackTaskBatchRequest;
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackCandidateVo;
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackCountryPreferenceVo;
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackCreateTaskVo;
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackDashboardVo;
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackHistoryVo;
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackMatchShopsVo;
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackPendingDeleteVo;
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackTaskBatchVo;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackService;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream;
import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.List;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/price-track")
@Tag(
name = "跟价",
description = "亚马逊跟价处理模块包含备选店铺、紫鸟匹配、任务创建、Python 回传、任务轮询和结果下载接口。查询类接口需携带 query 参数 user_id。"
)
public class PriceTrackController {
private final PriceTrackService priceTrackService;
private final PriceTrackTaskService priceTrackTaskService;
@GetMapping("/candidates")
@Operation(summary = "查询备选店铺列表", description = "返回当前用户在跟价模块中保存的备选店铺。")
public ApiResponse<List<PriceTrackCandidateVo>> listCandidates(
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(priceTrackService.listCandidates(userId));
}
@PostMapping("/candidates")
@Operation(summary = "新增备选店铺", description = "新增一条备选店铺记录,供后续匹配和创建任务使用。")
public ApiResponse<PriceTrackCandidateVo> addCandidate(@Valid @RequestBody PriceTrackCandidateAddRequest request) {
return ApiResponse.success(priceTrackService.addCandidate(request));
}
@DeleteMapping("/candidates/{id}")
@Operation(summary = "删除备选店铺", description = "删除一条备选店铺记录,记录必须属于当前 user_id。")
public ApiResponse<Void> deleteCandidate(
@Parameter(description = "备选店铺记录主键", example = "10") @PathVariable Long id,
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
priceTrackService.deleteCandidate(userId, id);
return ApiResponse.success(null);
}
@GetMapping("/country-preference")
@Operation(summary = "查询国家处理顺序", description = "返回当前用户保存的国家代码列表,顺序即跟价任务处理顺序。")
public ApiResponse<PriceTrackCountryPreferenceVo> getCountryPreference(
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(priceTrackService.getCountryPreference(userId));
}
@PutMapping("/country-preference")
@Operation(summary = "保存国家处理顺序", description = "保存当前用户在跟价模块中的国家勾选和处理顺序。")
public ApiResponse<PriceTrackCountryPreferenceVo> saveCountryPreference(
@Valid @RequestBody PriceTrackCountryPreferenceSaveRequest request) {
return ApiResponse.success(priceTrackService.saveCountryPreference(request));
}
@PostMapping("/match-shops")
@Operation(summary = "批量匹配店铺", description = "按店铺名查询紫鸟索引,返回 shopId、platform、companyName、matchStatus 等信息。")
public ApiResponse<PriceTrackMatchShopsVo> matchShops(@Valid @RequestBody PriceTrackMatchShopsRequest request) {
return ApiResponse.success(priceTrackService.matchShops(request));
}
@GetMapping("/dashboard")
@Operation(summary = "统计看板", description = "返回备选店铺数、已结束任务数、成功任务数和失败任务数。")
public ApiResponse<PriceTrackDashboardVo> dashboard(
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(priceTrackTaskService.dashboard(userId));
}
@GetMapping("/history")
@Operation(summary = "处理记录列表", description = "返回当前用户跟价模块下各店铺结果记录,包含运行中和历史记录。")
public ApiResponse<PriceTrackHistoryVo> history(
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(priceTrackTaskService.listHistory(userId));
}
@DeleteMapping("/history/{resultId}")
@Operation(summary = "删除单条结果记录", description = "删除一条 biz_file_result 结果记录,并同步重算父任务状态。")
public ApiResponse<Void> deleteHistory(
@Parameter(description = "结果行主键 biz_file_result.id", example = "1001") @PathVariable Long resultId,
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
priceTrackTaskService.deleteHistory(resultId, userId);
return ApiResponse.success(null);
}
@PostMapping("/tasks")
@Operation(summary = "创建跟价任务", description = "落库 RUNNING 任务和各店铺占位结果,返回 taskId 和初始 items 快照。")
public ApiResponse<PriceTrackCreateTaskVo> createTask(@Valid @RequestBody PriceTrackCreateTaskRequest request) {
return ApiResponse.success(priceTrackTaskService.createTask(request));
}
@DeleteMapping("/tasks/{taskId}")
@Operation(summary = "删除整条任务", description = "删除该任务以及其下全部结果行RUNNING、SUCCESS、FAILED 均可删除。")
public ApiResponse<Void> deleteTask(
@Parameter(description = "任务主键 biz_file_task.id", example = "200") @PathVariable Long taskId,
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId) {
priceTrackTaskService.deleteTask(taskId, userId);
return ApiResponse.success(null);
}
@DeleteMapping("/pending-shop-result")
@Operation(summary = "按店铺名删除运行中结果", description = "仅在 RUNNING 任务下按规范化店名删除一条 biz_file_result并同步重算任务状态。")
public ApiResponse<PriceTrackPendingDeleteVo> deletePendingShopResult(
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId,
@Parameter(name = "shop_name", description = "店铺名称", required = true, in = ParameterIn.QUERY, example = "某某店铺")
@RequestParam("shop_name") String shopName) {
return ApiResponse.success(priceTrackTaskService.deletePendingShopResult(userId, shopName));
}
@PostMapping("/tasks/batch")
@Operation(summary = "批量查询任务详情", description = "按 taskIds 返回任务概要和各店铺结果,供前端查询使用。")
public ApiResponse<PriceTrackTaskBatchVo> tasksBatch(@Valid @RequestBody PriceTrackTaskBatchRequest request) {
return ApiResponse.success(priceTrackTaskService.getTaskDetailsBatch(request.getTaskIds()));
}
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询任务进度摘要", description = "仅返回任务状态、时间和错误等轻量信息,供前端轮询降载。")
public ApiResponse<PriceTrackTaskBatchVo> taskProgressBatch(@Valid @RequestBody PriceTrackTaskBatchRequest request) {
return ApiResponse.success(priceTrackTaskService.getTaskProgressBatch(request.getTaskIds()));
}
@PostMapping("/tasks/{taskId}/result")
@Operation(summary = "Python 回传处理结果", description = "同一 taskId 可多次回传。成功时按 countries 生成结果文件,所有店铺进入终态后任务会变为 SUCCESS 或 FAILED。")
public ApiResponse<Void> submitResult(
@Parameter(description = "任务主键,必须处于 RUNNING 状态", example = "200") @PathVariable Long taskId,
@Valid @RequestBody PriceTrackSubmitResultRequest request) {
priceTrackTaskService.submitResult(taskId, request);
return ApiResponse.success(null);
}
@GetMapping("/results/{resultId}/download")
@Operation(summary = "下载结果文件", description = "按结果记录下载后端生成的跟价 Excel 文件。")
public void downloadResult(
@Parameter(description = "结果记录主键 biz_file_result.id", example = "1001") @PathVariable Long resultId,
@Parameter(name = "user_id", description = "当前用户 ID", required = true, in = ParameterIn.QUERY, example = "1")
@RequestParam("user_id") Long userId,
jakarta.servlet.http.HttpServletResponse response) {
String url = priceTrackTaskService.resolveResultDownloadUrl(resultId, userId);
String filename = priceTrackTaskService.resolveResultDownloadFilename(resultId, userId);
if (url == null || url.isBlank()) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
}
try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536];
int read;
while ((read = in.read(buffer)) != -1) {
response.getOutputStream().write(buffer, 0, read);
}
response.getOutputStream().flush();
}
} catch (Exception ex) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
}
}
}

View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.pricetrack.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.pricetrack.model.entity.PriceTrackCountryPrefEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface PriceTrackCountryPrefMapper extends BaseMapper<PriceTrackCountryPrefEntity> {
}

View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.pricetrack.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.pricetrack.model.entity.PriceTrackShopCandidateEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface PriceTrackShopCandidateMapper extends BaseMapper<PriceTrackShopCandidateEntity> {
}

View File

@@ -0,0 +1,18 @@
package com.nanri.aiimage.modules.pricetrack.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
@Data
@Schema(description = "新增备选店铺请求体:写入用户维度的待处理店铺列表,供前端勾选与匹配")
public class PriceTrackCandidateAddRequest {
@NotNull(message = "user_id不能为空")
@Schema(description = "当前用户 ID数据按用户隔离", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
private Long userId;
@NotBlank(message = "店铺名称不能为空")
@Schema(description = "店铺名称,入库前会先做规范化和去重", requiredMode = Schema.RequiredMode.REQUIRED, example = "测试店铺A")
private String shopName;
}

View File

@@ -0,0 +1,20 @@
package com.nanri.aiimage.modules.pricetrack.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "保存用户五国处理顺序:代码仅支持 DE、UK、FR、IT、ES且不能重复")
public class PriceTrackCountryPreferenceSaveRequest {
@NotNull(message = "user_id不能为空")
@Schema(description = "当前用户 ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
private Long userId;
@NotEmpty(message = "国家列表不能为空")
@Schema(description = "国家代码列表,顺序即任务处理顺序", requiredMode = Schema.RequiredMode.REQUIRED, example = "[\"DE\",\"UK\",\"FR\"]")
private List<String> countryCodes;
}

View File

@@ -0,0 +1,36 @@
package com.nanri.aiimage.modules.pricetrack.model.dto;
import com.nanri.aiimage.modules.pricetrack.model.vo.PriceTrackMatchShopsVo;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "创建跟价任务请求体:在推送 Python 队列前调用,用于落库任务和店铺结果占位记录")
public class PriceTrackCreateTaskRequest {
@NotNull(message = "user_id不能为空")
@Schema(description = "任务所属用户 ID会写入 biz_file_task.user_id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
private Long userId;
@Schema(description = "是否启用按商品状态处理模式")
private boolean statusMode;
@Schema(description = "是否启用按指定 ASIN 文件处理模式")
private boolean asinMode;
@NotEmpty(message = "店铺列表不能为空")
@Valid
@Schema(description = "已匹配的店铺列表,每项都应为 matched=true服务端会按规范化店名去重", requiredMode = Schema.RequiredMode.REQUIRED)
private List<PriceTrackMatchShopsVo.PriceTrackShopQueueItem> items;
@Schema(description = "ASIN 文件路径列表asinMode=true 时由前端上传或选择后传入", example = "[\"/tmp/a.xlsx\",\"/tmp/b.csv\"]")
private List<String> asinFiles;
@NotEmpty(message = "国家列表不能为空")
@Schema(description = "国家代码列表,顺序即跟价处理顺序", requiredMode = Schema.RequiredMode.REQUIRED, example = "[\"DE\",\"UK\",\"FR\"]")
private List<String> countryCodes;
}

View File

@@ -0,0 +1,27 @@
package com.nanri.aiimage.modules.pricetrack.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "Price track match shops request")
public class PriceTrackMatchShopsRequest {
@NotNull(message = "userId is required")
@Schema(description = "Current user id", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
private Long userId;
@NotEmpty(message = "shopNames cannot be empty")
@Schema(description = "Shop names to match", requiredMode = Schema.RequiredMode.REQUIRED)
private List<String> shopNames;
@Schema(description = "Selected ASIN file local paths")
private List<String> asinFiles;
@Schema(description = "Selected country codes")
private List<String> countryCodes;
}

View File

@@ -0,0 +1,79 @@
package com.nanri.aiimage.modules.pricetrack.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
import java.util.Map;
@Data
@Schema(description = "Python submit payload for price track task results")
public class PriceTrackSubmitResultRequest {
@Schema(description = "Shop result list", requiredMode = Schema.RequiredMode.REQUIRED)
private List<ShopResult> shops;
@Data
@Schema(description = "Per-shop submit result")
public static class ShopResult {
@Schema(description = "Shop name used to match task rows", example = "Test Shop A")
private String shopName;
@Schema(description = "Rows grouped by country code, for example DE/UK/FR/IT/ES")
private Map<String, List<AsinResult>> countries;
@Schema(description = "Failure message")
private String error;
@Schema(description = "Whether the shop finished successfully")
private Boolean success;
}
@Data
@Schema(description = "Per-ASIN price track result row")
public static class AsinResult {
@Schema(description = "Shop mall name", example = "Amazon.de")
private String shopMallName;
@Schema(description = "ASIN", example = "B0ABCDE123")
private String asin;
@Schema(description = "Price", example = "19.99")
private String price;
@Schema(description = "Recommended price", example = "18.99")
private String recommendedPrice;
@Schema(description = "Lowest price", example = "18.50")
private String minimumPrice;
@Schema(description = "First place", example = "Competitor A")
private String firstPlace;
@Schema(description = "Second place", example = "Competitor B")
private String secondPlace;
@Schema(description = "Buy box shop name", example = "Shop A")
private String cartShopName;
@Schema(description = "Price change status", example = "UPDATED")
private String priceChangeStatus;
@Schema(description = "Modify count", example = "2")
private String modifyCount;
@Schema(description = "Status", example = "SUCCESS")
private String status;
@Schema(description = "Legacy current price", example = "19.99")
private String currentPrice;
@Schema(description = "Legacy competitor price", example = "18.50")
private String competitorPrice;
@Schema(description = "Legacy action", example = "LOWER_PRICE")
private String action;
@Schema(description = "Legacy done flag")
private Boolean done;
}
}

View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.pricetrack.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "批量查询任务详情请求体:一次拉取多个 taskId 的概要和下属店铺结果,供轮询使用")
public class PriceTrackTaskBatchRequest {
@NotEmpty(message = "taskIds不能为空")
@Schema(description = "待查询的任务主键列表,无效或非本模块 taskId 会出现在 missingTaskIds", requiredMode = Schema.RequiredMode.REQUIRED, example = "[200,201]")
private List<Long> taskIds = new ArrayList<>();
}

View File

@@ -0,0 +1,17 @@
package com.nanri.aiimage.modules.pricetrack.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
@Data
@TableName("biz_price_track_country_pref")
public class PriceTrackCountryPrefEntity {
@TableId(type = IdType.AUTO)
private Long userId;
@TableField("country_codes_json")
private String countryCodesJson;
}

View File

@@ -0,0 +1,17 @@
package com.nanri.aiimage.modules.pricetrack.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_price_track_shop_candidate")
public class PriceTrackShopCandidateEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long userId;
private String shopName;
private LocalDateTime createdAt;
}

View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.pricetrack.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import lombok.experimental.Accessors;
@Data
@Accessors(chain = true)
@Schema(description = "备选店铺列表单项")
public class PriceTrackCandidateVo {
@Schema(description = "记录主键,删除备选店铺时作为路径参数", example = "10")
private Long id;
@Schema(description = "店铺名称", example = "测试店铺A")
private String shopName;
}

View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.pricetrack.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "用户五国处理偏好:按顺序返回国家代码列表,例如 DE、UK、FR、IT、ES")
public class PriceTrackCountryPreferenceVo {
@Schema(description = "当前用户 ID", example = "1")
private Long userId;
@Schema(description = "已选国家代码列表;若用户未保存,则服务端返回默认顺序 DE、UK、FR、IT、ES")
private List<String> countryCodes;
}

View File

@@ -0,0 +1,23 @@
package com.nanri.aiimage.modules.pricetrack.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
import java.util.Map;
@Data
@Schema(description = "price track create task response")
public class PriceTrackCreateTaskVo {
@Schema(description = "task id", example = "200")
private Long taskId;
@Schema(description = "initial task items")
private List<PriceTrackResultItemVo> items;
@Schema(description = "all skip asins grouped by country")
private Map<String, List<String>> skipAsinsByCountry;
@Schema(description = "parsed asin rows by country")
private Map<String, List<Map<String, String>>> asinRowsByCountry;
}

View File

@@ -0,0 +1,22 @@
package com.nanri.aiimage.modules.pricetrack.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import lombok.experimental.Accessors;
@Data
@Accessors(chain = true)
@Schema(description = "跟价模块统计看板,按当前用户维度汇总")
public class PriceTrackDashboardVo {
@Schema(description = "备选店铺数量")
private Long candidateCount;
@Schema(description = "已结束任务数,状态为 SUCCESS 或 FAILED")
private Long processedTaskCount;
@Schema(description = "成功结束的任务数")
private Long successTaskCount;
@Schema(description = "失败结束的任务数")
private Long failedTaskCount;
}

View File

@@ -0,0 +1,13 @@
package com.nanri.aiimage.modules.pricetrack.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "跟价结果列表响应items 中每项对应一个店铺结果行")
public class PriceTrackHistoryVo {
@Schema(description = "结果列表,包含运行中和历史记录")
private List<PriceTrackResultItemVo> items;
}

View File

@@ -0,0 +1,51 @@
package com.nanri.aiimage.modules.pricetrack.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Data
@Schema(description = "Price track shop match response")
public class PriceTrackMatchShopsVo {
@Schema(description = "Match results in the same order as requested shop names")
private List<PriceTrackShopQueueItem> items = new ArrayList<>();
@Schema(description = "All skip ASINs grouped by country code")
private Map<String, List<String>> skipAsinsByCountry;
@Schema(description = "Parsed asin rows grouped by country code")
private Map<String, List<Map<String, String>>> asinRowsByCountry;
@Data
@Schema(description = "Single matched shop item")
public static class PriceTrackShopQueueItem {
@Schema(description = "Shop name", example = "Demo Shop")
private String shopName;
@Schema(description = "Matched shop id")
private Object shopId;
@Schema(description = "Platform", example = "Amazon")
private String platform;
@Schema(description = "Company name")
private String companyName;
@Schema(description = "Whether the shop matched")
private Boolean matched;
@Schema(description = "Match status, such as MATCHED or PENDING")
private String matchStatus;
@Schema(description = "Readable match message")
private String matchMessage;
@Schema(description = "All skip ASINs grouped by country code")
private Map<String, List<String>> skipAsins;
}
}

View File

@@ -0,0 +1,11 @@
package com.nanri.aiimage.modules.pricetrack.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "按店铺名删除运行中结果的响应")
public class PriceTrackPendingDeleteVo {
@Schema(description = "是否实际删除到一条运行中的结果记录")
private boolean removed;
}

Some files were not shown because too many files have changed in this diff Show More