8 Commits

Author SHA1 Message Date
铭坤
4f8cbc3e38 new file: update/PyQt5/Qt/.keep_dir.txt
new file:   update/PyQt5/Qt5/.keep_dir.txt
	new file:   update/PyQt5/QtCore.pyd
	new file:   update/PyQt5/QtGui.pyd
	new file:   update/PyQt5/QtWidgets.pyd
	new file:   update/PyQt5/qt-plugins/iconengines/qsvgicon.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qgif.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qicns.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qico.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qjpeg.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qsvg.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qtga.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qtiff.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qwbmp.dll
	new file:   update/PyQt5/qt-plugins/imageformats/qwebp.dll
	new file:   update/PyQt5/qt-plugins/mediaservice/dsengine.dll
	new file:   update/PyQt5/qt-plugins/mediaservice/qtmedia_audioengine.dll
	new file:   update/PyQt5/qt-plugins/mediaservice/wmfengine.dll
	new file:   update/PyQt5/qt-plugins/platforms/qminimal.dll
	new file:   update/PyQt5/qt-plugins/platforms/qoffscreen.dll
	new file:   update/PyQt5/qt-plugins/platforms/qwebgl.dll
	new file:   update/PyQt5/qt-plugins/platforms/qwindows.dll
	new file:   update/PyQt5/qt-plugins/platformthemes/qxdgdesktopportal.dll
	new file:   update/PyQt5/qt-plugins/printsupport/windowsprintersupport.dll
	new file:   update/PyQt5/qt-plugins/styles/qwindowsvistastyle.dll
	new file:   update/PyQt5/sip.pyd
	new file:   update/_bz2.pyd
	new file:   update/_ctypes.pyd
	new file:   update/_decimal.pyd
	new file:   update/_elementtree.pyd
	new file:   update/_hashlib.pyd
	new file:   update/_hashlib.zip
	new file:   update/_lzma.pyd
	new file:   update/_socket.pyd
	new file:   update/libcrypto-1_1-x64.dll
	new file:   update/libeay32.dll
	new file:   update/libffi-7.dll
	new file:   update/msvcp140.dll
	new file:   update/msvcp140_1.dll
	new file:   update/psutil/_psutil_windows.pyd
	new file:   update/pyexpat.pyd
	new file:   update/python3.dll
	new file:   update/python39.dll
	new file:   update/qt5core.dll
	new file:   update/qt5dbus.dll
	new file:   update/qt5gui.dll
	new file:   update/qt5multimedia.dll
	new file:   update/qt5network.dll
	new file:   update/qt5printsupport.dll
	new file:   update/qt5qml.dll
	new file:   update/qt5qmlmodels.dll
	new file:   update/qt5quick.dll
	new file:   update/qt5svg.dll
	new file:   update/qt5websockets.dll
	new file:   update/qt5widgets.dll
	new file:   update/select.pyd
	new file:   update/ssleay32.dll
	new file:   update/unicodedata.pyd
	new file:   update/update.exe
	new file:   update/vcruntime140.dll
	new file:   update/vcruntime140_1.dll
2026-04-19 15:39:46 +08:00
铭坤
3ffbb2b004 modified: ali_oss.py
modified:   config.py
	modified:   main.py
	new file:   winsrc.bat
2026-04-19 15:38:09 +08:00
super
34ed15a9bd Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-19 15:06:13 +08:00
super
4b6295dd44 改造后台权限和APP权限 2026-04-19 15:06:09 +08:00
supernijia
01116d5607 删除文件 desktop 2026-04-18 16:23:13 +00:00
铭坤
07904a29aa modified: amazon/match_action.py
modified:   amazon/price_match.py
	modified:   config.py
2026-04-19 00:18:28 +08:00
铭坤
8d683a791d Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin 2026-04-19 00:18:02 +08:00
super
ef0e0df0ac 完成匹配、跟价、权限部分 2026-04-17 12:52:16 +08:00
179 changed files with 10440 additions and 2949 deletions

View File

@@ -14,7 +14,6 @@ def upload_file(file_content: bytes, key: str):
cfg.credentials_provider = credentials_provider
cfg.region = region
cfg.endpoint = endpoint
cfg.retry_max_attempts = 3
client = oss.Client(cfg)
result = client.put_object(

View File

@@ -93,6 +93,7 @@ class AmzoneMatchAction(AmamzonBase):
num = 0
retry_num = 0
already_asin = set()
get_page_faild = 0
while retry_num < 3: # 最多重试3次
# if num > 3: #测试
@@ -117,9 +118,13 @@ class AmzoneMatchAction(AmamzonBase):
else:
total_page = 0
print(f"{self.mark_name}】当前页码: {current_page} / 总页数: {total_page}")
get_page_faild = 0
except Exception as e:
print(f"{self.mark_name}】获取页码失败", e)
get_page_faild+= 1
if get_page_faild > 2:
show_notification(f"{self.mark_name}】获取页码失败超3次停止任务")
break
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
print(f"{self.mark_name}】获取到 {len(sku_ls)}")

View File

@@ -1,14 +1,17 @@
import time
import re
import traceback
from config import runing_task, runing_shop
from datetime import datetime
from amazon.del_brand import AmamzonBase, kill_process
from DrissionPage import Chromium, ChromiumOptions
from amazon.del_brand import AmamzonBase, kill_process
from amazon.tool import show_notification,get_shop_info
from config import runing_task, runing_shop,base_dir
class ChromeAmzone:
mark_name = "亚马逊详情采集"
country_info = {
"英国": {
@@ -43,21 +46,46 @@ class ChromeAmzone:
os.system('taskkill /f /t /im chrome.exe')
time.sleep(2)
# 使用 DrissionPage 启动浏览器(使用默认用户数据)
from DrissionPage import Chromium, ChromiumOptions
print("正在启动Chrome浏览器...")
# sellersprite_plug_path = os.path.join(base_dir,"app_resource","sellersprite-extension-mv3")
sellersprite_plug_path = os.path.join("D:\\私单交付\\maixiang_AI\\","app_resource","sellersprite-extension-mv3")
print(sellersprite_plug_path)
# 配置浏览器选项
co = ChromiumOptions()
# co.use_system_user_path(on_off=True)
co.set_user_data_path(r"D:\私单交付\maixiang_AI\user_data\chrome_data")
co.set_local_port(port=19890)
co.add_extension(sellersprite_plug_path)
# co.set_argument('--disable-features=DisableLoadExtensionCommandLineSwitch')
# co.set_argument('--load-extension',sellersprite_plug_path)
co.set_browser_path(r'D:\私单交付\maixiang_AI\app_resource\chrome-win\chrome.exe')
# 使用系统默认的用户数据目录
# co.set_user_data_path(r'C:\Users\{}\AppData\Local\Google\Chrome\User Data'.format(os.getenv('USERNAME')))
self.browser = Chromium(co)
self.tab = self.browser.latest_tab
print("Chrome浏览器启动成功")
def accept_cookie(self):
pass
def close_init_popup(self):
"""
关闭所有的初始化弹窗
"""
self.tab.wait.doc_loaded(timeout=60,raise_err=True)
footbar = self.tab.eles('xpath://footer[@class="el-dialog__footer"]',timeout=5)
if len(footbar) > 0:
do_not_remind = footbar[0].eles('xpath:.//input[@class="el-checkbox__original"]')
if len(do_not_remind) > 0:
do_not_remind[0].check()
resume_immediately = footbar[0].eles('xpath:.//button')
if len(resume_immediately) > 0:
resume_immediately[0].click()
self.tab.wait.doc_loaded(timeout=60,raise_err=True)
accept_btn = self.tab.eles('xpath://input[@id="sp-cc-accept"]',timeout=10)
if len(accept_btn) > 0:
accept_btn[0].click()
def run(self, country, asin):
"""
@@ -98,6 +126,9 @@ class ChromeAmzone:
# 2. 切换国家/设置邮编
print(f"正在检查并设置邮编: {zip_code}")
self._set_zip_code(zip_code)
self.close_init_popup()
# 3. 抓取数据
print("正在抓取商品数据...")
@@ -317,14 +348,14 @@ if __name__ == '__main__':
print(f"采集成功: {result1}")
# 示例2采集德国站点的商品信息
print("\n示例2采集德国站点商品")
result2 = chrome.run(country="德国", asin="B0CC8CW9G2")
if result2:
print(f"采集成功: {result2}")
# print("\n示例2采集德国站点商品")
# result2 = chrome.run(country="德国", asin="B0CC8CW9G2")
# if result2:
# print(f"采集成功: {result2}")
# 关闭浏览器
print("\n正在关闭浏览器...")
chrome.close()
# # 关闭浏览器
# print("\n正在关闭浏览器...")
# chrome.close()
except Exception as e:
print(f"运行出错: {traceback.format_exc()}")

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -267,7 +267,8 @@ def admin_delete_user(uid):
@login_required
def admin_user_column_permissions(uid):
"""获取指定用户的栏目权限列表:当前用户只能查自己,管理员可查任意用户。超级管理员返回全部栏目。"""
current_uid = session.get('user_id')
current_uid = session.get('user_id')
menu_type = (request.args.get('menu_type') or '').strip().lower()
if current_uid != uid:
role, _ = _get_current_admin_role()
if not role:
@@ -277,28 +278,44 @@ def admin_user_column_permissions(uid):
with conn.cursor() as cur:
cur.execute("SELECT role FROM users WHERE id = %s", (uid,))
user_row = cur.fetchone()
if user_row and (user_row.get('role') or '').strip() == 'super_admin':
cur.execute("""
SELECT id, name, column_key, created_at FROM columns ORDER BY id
""")
rows = cur.fetchall()
else:
cur.execute("""
SELECT c.id, c.name, c.column_key, c.created_at
FROM columns c
INNER JOIN user_column_permission ucp ON ucp.column_id = c.id
WHERE ucp.user_id = %s
ORDER BY c.id
""", (uid,))
rows = cur.fetchall()
valid_menu_type = menu_type if menu_type in ('app', 'admin') else ''
if user_row and (user_row.get('role') or '').strip() == 'super_admin':
sql = """
SELECT id, name, column_key, menu_type, route_path, sort_order, created_at
FROM columns
"""
params = []
if valid_menu_type:
sql += " WHERE menu_type = %s"
params.append(valid_menu_type)
sql += " ORDER BY sort_order ASC, id ASC"
cur.execute(sql, tuple(params))
rows = cur.fetchall()
else:
sql = """
SELECT c.id, c.name, c.column_key, c.menu_type, c.route_path, c.sort_order, c.created_at
FROM columns c
INNER JOIN user_column_permission ucp ON ucp.column_id = c.id
WHERE ucp.user_id = %s
"""
params = [uid]
if valid_menu_type:
sql += " AND c.menu_type = %s"
params.append(valid_menu_type)
sql += " ORDER BY c.sort_order ASC, c.id ASC"
cur.execute(sql, tuple(params))
rows = cur.fetchall()
conn.close()
items = [
{
'id': r['id'],
'name': r['name'],
'column_key': r['column_key'],
'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r.get('created_at') else '',
}
{
'id': r['id'],
'name': r['name'],
'column_key': r['column_key'],
'menu_type': r.get('menu_type') or 'app',
'route_path': r.get('route_path') or '',
'sort_order': r.get('sort_order') or 0,
'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r.get('created_at') else '',
}
for r in rows
]
return jsonify({'success': True, 'items': items})

View File

@@ -50,10 +50,10 @@ def wb():
return _render_html('index.html')
@main_bp.route('/brand')
@login_required
def brand_page():
return _render_html('brand.html')
@main_bp.route('/brand')
@login_required
def brand_page():
return _render_html('brand.html', user_id=session.get('user_id'))
@@ -161,4 +161,4 @@ def proxy(path):
return Response("无法连接到后端服务", status=502)
except Exception as e:
print(f"代理请求失败: {e}")
return Response(f"代理错误: {str(e)}", status=500)
return Response(f"代理错误: {str(e)}", status=500)

View File

@@ -49,8 +49,8 @@ os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret
os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6"
debug = True
version = "1.0.13"
debug = False
version = "1.0.56"
APP_UPDATE_URL = f"{_base_url}/api/version/latest" # 检测更新接口地址,返回 { "file_url": "...", "version": "x.x.x" }
os.environ['APP_VERSION'] = version
os.environ['APP_UPDATE_URL'] = APP_UPDATE_URL

View File

@@ -434,7 +434,7 @@ def main():
window.expose(api.close, api.minimize, api.maximize, api.toggle_maximize, generate_images, api.save_image, api.save_image_to_folder, api.select_folder, api.select_brand_xlsx_files, api.select_brand_folder,
api.save_file_from_url, api.save_template_xlsx,api.save_template_zip,api.upload_file_to_java,api.open_external_url,api.enqueue_json,api.get_detail_del,api.save_file_from_url_new)
webview.start(
debug=True,
debug=False,
storage_path=cache_path,
private_mode=False
)

View File

@@ -5,9 +5,9 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>格式转换 - 数富AI</title>
<script type="module" crossorigin src="/assets/convert.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-DiP0HdY6.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BATjYfEL.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-ClWy2SbE.js">
<link rel="modulepreload" crossorigin href="/assets/brand-DBpLCgAF.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-hj_EXwAP.css">
<link rel="stylesheet" crossorigin href="/assets/convert-7wWJ02Tw.css">
</head>
<body>

View File

@@ -5,9 +5,9 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据去重 - 数富AI</title>
<script type="module" crossorigin src="/assets/dedupe.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-DiP0HdY6.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BATjYfEL.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-ClWy2SbE.js">
<link rel="modulepreload" crossorigin href="/assets/brand-DBpLCgAF.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-hj_EXwAP.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-BpNHwt51.css">
</head>
<body>

View File

@@ -5,10 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>删除品牌 - 数富AI</title>
<script type="module" crossorigin src="/assets/delete-brand.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-DiP0HdY6.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BATjYfEL.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
<link rel="stylesheet" crossorigin href="/assets/delete-brand-DfRLfrtm.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-ClWy2SbE.js">
<link rel="modulepreload" crossorigin href="/assets/brand-DBpLCgAF.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-hj_EXwAP.css">
<link rel="stylesheet" crossorigin href="/assets/delete-brand-CdxGWtvK.css">
</head>
<body>
<div id="app"></div>

View File

@@ -5,9 +5,9 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据拆分 - 数富AI</title>
<script type="module" crossorigin src="/assets/split.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-DiP0HdY6.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BATjYfEL.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-ClWy2SbE.js">
<link rel="modulepreload" crossorigin href="/assets/brand-DBpLCgAF.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-hj_EXwAP.css">
<link rel="stylesheet" crossorigin href="/assets/split-CRUIYKS6.css">
</head>
<body>

View File

View File

BIN
app/update/PyQt5/QtCore.pyd Normal file

Binary file not shown.

BIN
app/update/PyQt5/QtGui.pyd Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
app/update/PyQt5/sip.pyd Normal file

Binary file not shown.

BIN
app/update/_bz2.pyd Normal file

Binary file not shown.

BIN
app/update/_ctypes.pyd Normal file

Binary file not shown.

BIN
app/update/_decimal.pyd Normal file

Binary file not shown.

BIN
app/update/_elementtree.pyd Normal file

Binary file not shown.

BIN
app/update/_hashlib.pyd Normal file

Binary file not shown.

BIN
app/update/_hashlib.zip Normal file

Binary file not shown.

BIN
app/update/_lzma.pyd Normal file

Binary file not shown.

BIN
app/update/_socket.pyd Normal file

Binary file not shown.

Binary file not shown.

BIN
app/update/libeay32.dll Normal file

Binary file not shown.

BIN
app/update/libffi-7.dll Normal file

Binary file not shown.

BIN
app/update/msvcp140.dll Normal file

Binary file not shown.

BIN
app/update/msvcp140_1.dll Normal file

Binary file not shown.

Binary file not shown.

BIN
app/update/pyexpat.pyd Normal file

Binary file not shown.

BIN
app/update/python3.dll Normal file

Binary file not shown.

BIN
app/update/python39.dll Normal file

Binary file not shown.

BIN
app/update/qt5core.dll Normal file

Binary file not shown.

BIN
app/update/qt5dbus.dll Normal file

Binary file not shown.

BIN
app/update/qt5gui.dll Normal file

Binary file not shown.

Binary file not shown.

BIN
app/update/qt5network.dll Normal file

Binary file not shown.

Binary file not shown.

BIN
app/update/qt5qml.dll Normal file

Binary file not shown.

BIN
app/update/qt5qmlmodels.dll Normal file

Binary file not shown.

BIN
app/update/qt5quick.dll Normal file

Binary file not shown.

BIN
app/update/qt5svg.dll Normal file

Binary file not shown.

Binary file not shown.

BIN
app/update/qt5widgets.dll Normal file

Binary file not shown.

BIN
app/update/select.pyd Normal file

Binary file not shown.

BIN
app/update/ssleay32.dll Normal file

Binary file not shown.

BIN
app/update/unicodedata.pyd Normal file

Binary file not shown.

BIN
app/update/update.exe Normal file

Binary file not shown.

BIN
app/update/vcruntime140.dll Normal file

Binary file not shown.

Binary file not shown.

View File

@@ -62,6 +62,10 @@
position: relative;
}
body.nav-permissions-loading .nav-section[data-column-key] {
display: none;
}
.nav-section + .nav-section::before {
content: '';
position: absolute;
@@ -383,7 +387,7 @@
.toast.show { opacity: 1; }
</style>
</head>
<body>
<body data-user-id="{{ user_id or '' }}" class="nav-permissions-loading">
<header class="top-bar">
<div class="logo-area">
<span class="app-name">数富AI-亚马逊</span>
@@ -391,33 +395,33 @@
</div>
<nav class="nav-tabs">
<div class="nav-tab-group">
<div class="nav-section">
<div class="nav-section-title">&#21069;&#31471;&#24037;&#20855;</div>
<div class="nav-section" data-column-key="brand_front_tools">
<div class="nav-section-title">前端工具</div>
<div class="nav-section-items">
<button type="button" class="nav-item active" data-panel="brandCheck">&#21697;&#29260;&#26816;&#27979;</button>
<span class="nav-item disabled">&#37319;&#38598;&#25968;&#25454;</span>
<span class="nav-item disabled">&#21464;&#20307;&#20998;&#26512;</span>
<a class="nav-item" href="/new_web_source/dedupe.html">&#25968;&#25454;&#21435;&#37325;</a>
<a class="nav-item" href="/new_web_source/split.html">&#25968;&#25454;&#25286;&#20998;</a>
<a class="nav-item" href="/new_web_source/convert.html">&#26684;&#24335;&#36716;&#25442;</a>
<span class="nav-item disabled">采集数据</span>
<span class="nav-item disabled">变体分析</span>
<button type="button" class="nav-item active" data-panel="brandCheck">品牌检测</button>
<a class="nav-item" href="/new_web_source/dedupe.html">数据去重</a>
<a class="nav-item" href="/new_web_source/split.html">数据拆分</a>
<a class="nav-item" href="/new_web_source/convert.html">格式转换</a>
</div>
</div>
<div class="nav-section">
<div class="nav-section-title">&#36816;&#33829;&#24037;&#20855;</div>
<div class="nav-section" data-column-key="brand_operation_tools">
<div class="nav-section-title">运营工具</div>
<div class="nav-section-items">
<a class="nav-item" href="/new_web_source/delete-brand.html">&#21024;&#38500;ASIN</a>
<a class="nav-item" href="/new_web_source/product-risk.html">&#21830;&#21697;&#39118;&#38505;&#35299;&#20915;</a>
<a class="nav-item" href="/new_web_source/shop-match.html">&#23450;&#26102;&#21305;&#37197;</a>
<span class="nav-item disabled">&#36319;&#20215;</span>
<span class="nav-item disabled">&#24033;&#24215;&#21024;&#38500;</span>
<span class="nav-item disabled">&#21462;&#27454;</span>
<span class="nav-item disabled">&#24215;&#38138;&#29366;&#24577;&#26597;&#35810;</span>
<a class="nav-item" href="/new_web_source/delete-brand.html">删除ASIN</a>
<a class="nav-item" href="/new_web_source/product-risk.html">商品风险解决</a>
<a class="nav-item" href="/new_web_source/shop-match.html">定时匹配</a>
<a class="nav-item" href="/new_web_source/price-track.html">跟价</a>
<span class="nav-item disabled">巡店删除</span>
<span class="nav-item disabled">取款</span>
<span class="nav-item disabled">店铺状态查询</span>
</div>
</div>
<div class="nav-section">
<div class="nav-section-title">&#21518;&#21220;&#24037;&#20855;</div>
<div class="nav-section" data-column-key="brand_logistics_tools">
<div class="nav-section-title">后勤工具</div>
<div class="nav-section-items">
<span class="nav-item disabled">&#37319;&#36141;</span>
<span class="nav-item disabled">采购</span>
<span class="nav-item disabled">ERP</span>
</div>
</div>
@@ -535,9 +539,13 @@
<script>
(function() {
var selectedPaths = [];
var cachedTasks = [];
var api = window.pywebview && window.pywebview.api;
var selectedPaths = [];
var cachedTasks = [];
var api = window.pywebview && window.pywebview.api;
function getAppPermissionCacheKey(uid) {
return 'app_column_permissions:' + String(uid || '');
}
function getUid() {
return localStorage.getItem('uid') || '';
@@ -550,20 +558,80 @@
}
// 顶部栏目切换:点击当前页菜单项时显示对应 data-panel 的 tab-panel
document.querySelectorAll('.nav-item[data-panel]').forEach(function(tab) {
tab.addEventListener('click', function() {
var panelId = this.getAttribute('data-panel');
document.querySelectorAll('.nav-item[data-panel]').forEach(function(tab) {
tab.addEventListener('click', function() {
var panelId = this.getAttribute('data-panel');
if (!panelId) return;
document.querySelectorAll('.nav-item[data-panel]').forEach(function(t) { t.classList.remove('active'); });
document.querySelectorAll('.tab-panel').forEach(function(p) {
p.classList.toggle('active', p.getAttribute('data-panel') === panelId);
});
this.classList.add('active');
});
});
function showToast(msg) {
var el = document.getElementById('toast');
this.classList.add('active');
});
});
(function applyNavSectionPermissions() {
function finishNavPermissionLoading() {
document.body.classList.remove('nav-permissions-loading');
}
function showAllNavSections() {
document.querySelectorAll('.nav-section[data-column-key]').forEach(function(section) {
section.style.display = '';
});
}
var uid = document.body.getAttribute('data-user-id');
if (!uid) {
showAllNavSections();
finishNavPermissionLoading();
return;
}
localStorage.setItem('uid', uid);
var cacheKey = getAppPermissionCacheKey(uid);
try {
var cachedItems = JSON.parse(localStorage.getItem(cacheKey) || 'null');
if (Array.isArray(cachedItems)) {
showAllNavSections();
var cachedKeys = cachedItems.map(function(item) {
return (item.column_key || '').toLowerCase();
});
document.querySelectorAll('.nav-section[data-column-key]').forEach(function(section) {
var key = (section.getAttribute('data-column-key') || '').toLowerCase();
section.style.display = cachedKeys.indexOf(key) >= 0 ? '' : 'none';
});
finishNavPermissionLoading();
return;
}
} catch (e) {}
fetch('/api/admin/user/' + uid + '/column-permissions?menu_type=app', { credentials: 'same-origin' })
.then(function(r) { return r.json(); })
.then(function(res) {
showAllNavSections();
if (!res || !res.success || !Array.isArray(res.items)) {
finishNavPermissionLoading();
return;
}
try {
localStorage.setItem(cacheKey, JSON.stringify(res.items));
} catch (e) {}
var allowedKeys = res.items.map(function(item) {
return (item.column_key || '').toLowerCase();
});
document.querySelectorAll('.nav-section[data-column-key]').forEach(function(section) {
var key = (section.getAttribute('data-column-key') || '').toLowerCase();
section.style.display = allowedKeys.indexOf(key) >= 0 ? '' : 'none';
});
finishNavPermissionLoading();
})
.catch(function() {
showAllNavSections();
finishNavPermissionLoading();
});
})();
function showToast(msg) {
var el = document.getElementById('toast');
el.textContent = msg;
el.classList.add('show');
setTimeout(function() { el.classList.remove('show'); }, 2500);

View File

@@ -217,17 +217,24 @@
<div class="toast" id="toast"></div>
<script>
function getAppPermissionCacheKey(uid) {
return 'app_column_permissions:' + String(uid || '');
}
// 权限校验:根据 column-permissions 接口按 column_key 显示入口
(function() {
localStorage.setItem("uid",{{ user_id }})
var uid = document.body.getAttribute('data-user-id');
if (!uid) return;
var cacheKey = getAppPermissionCacheKey(uid);
var baseUrl = window.location.origin;
var apiUrl = baseUrl + '/api/admin/user/' + uid + '/column-permissions';
fetch(apiUrl, { credentials: 'same-origin' })
.then(function(r) { return r.json(); })
.then(function(res) {
if (!res || !res.success || !Array.isArray(res.items)) return;
try {
localStorage.setItem(cacheKey, JSON.stringify(res.items));
} catch (e) {}
var allowedKeys = res.items.map(function(item) { return (item.column_key || '').toLowerCase(); });
document.querySelectorAll('.entrances .entrance-btn[data-column-key]').forEach(function(btn) {
var key = (btn.getAttribute('data-column-key') || '').toLowerCase();
@@ -251,6 +258,10 @@
function handleLogout() {
try{
localStorage.removeItem('maixiang_api_key');
var uid = document.body.getAttribute('data-user-id');
if (uid) {
localStorage.removeItem(getAppPermissionCacheKey(uid));
}
}catch (e) {
console.log(e)
}

View File

@@ -117,6 +117,21 @@
</form>
</div>
<script>
function clearAppPermissionCaches() {
try {
var keysToRemove = [];
for (var i = 0; i < localStorage.length; i++) {
var key = localStorage.key(i);
if (key && key.indexOf('app_column_permissions:') === 0) {
keysToRemove.push(key);
}
}
keysToRemove.forEach(function(key) {
localStorage.removeItem(key);
});
} catch (e) {}
}
(function() {
fetch('/api/auth/check', { credentials: 'same-origin' })
.then(function(r) { return r.json(); })
@@ -142,6 +157,7 @@
.then(function(r) { return r.json(); })
.then(function(res) {
if (res.success) {
clearAppPermissionCaches();
window.location.href = res.redirect || '/home';
} else {
var errEl = document.querySelector('.error-msg');

38
app/winsrc.bat Normal file
View File

@@ -0,0 +1,38 @@
@echo off
chcp 65001 >nul
setlocal enabledelayedexpansion
echo 正在执行 PowerShell 命令解除文件锁定...
powershell -Command "Get-ChildItem -Path \"%cd%\" -Recurse | Unblock-File"
if %errorlevel% equ 0 (
echo 文件解锁完成
) else (
echo 文件解锁失败,请检查 PowerShell 权限
)
echo.
echo 检查 ShuFuAi.exe.config 文件...
if exist "ShuFuAi.exe.config" (
echo ShuFuAi.exe.config 已存在,跳过创建
) else (
echo 正在创建 ShuFuAi.exe.config...
(
echo ^<?xml version="1.0" encoding="utf-8" ?^>
echo ^<configuration^>
echo ^<runtime^>
echo ^<loadFromRemoteSources enabled="true"/^>
echo ^</runtime^>
echo ^</configuration^>
) > "ShuFuAi.exe.config"
if exist "ShuFuAi.exe.config" (
echo ShuFuAi.exe.config 创建成功
) else (
echo ShuFuAi.exe.config 创建失败,请检查权限
)
)
echo.
echo 操作完成!
pause

Binary file not shown.

Binary file not shown.

View File

@@ -1,13 +1,11 @@
package com.nanri.aiimage.common.api;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Schema(description = "统一响应体")
public class ApiResponse<T> {
@@ -20,6 +18,20 @@ public class ApiResponse<T> {
@Schema(description = "响应数据")
private T data;
@Schema(description = "业务错误码,成功时可为空")
private Integer code;
public ApiResponse(boolean success, String message, T data) {
this(success, message, data, null);
}
public ApiResponse(boolean success, String message, T data, Integer code) {
this.success = success;
this.message = message;
this.data = data;
this.code = code;
}
public static <T> ApiResponse<T> success(T data) {
return new ApiResponse<>(true, "操作成功", data);
}
@@ -31,4 +43,8 @@ public class ApiResponse<T> {
public static <T> ApiResponse<T> fail(String message) {
return new ApiResponse<>(false, message, null);
}
public static <T> ApiResponse<T> fail(Integer code, String message) {
return new ApiResponse<>(false, message, null, code);
}
}

View File

@@ -2,7 +2,18 @@ package com.nanri.aiimage.common.exception;
public class BusinessException extends RuntimeException {
private final Integer code;
public BusinessException(String message) {
this(null, message);
}
public BusinessException(Integer code, String message) {
super(message);
this.code = code;
}
public Integer getCode() {
return code;
}
}

View File

@@ -13,7 +13,9 @@ public class GlobalExceptionHandler {
@ExceptionHandler(BusinessException.class)
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
return ApiResponse.fail(ex.getMessage());
return ex.getCode() == null
? ApiResponse.fail(ex.getMessage())
: ApiResponse.fail(ex.getCode(), ex.getMessage());
}
@ExceptionHandler(MethodArgumentNotValidException.class)

View File

@@ -0,0 +1,126 @@
package com.nanri.aiimage.common.util;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.LinkedHashMap;
import java.util.Map;
public final class ExcelStreamReader {
private ExcelStreamReader() {
}
public static void readFirstSheet(File file, SheetRowHandler handler) throws IOException {
try (InputStream inputStream = new FileInputStream(file)) {
readFirstSheet(inputStream, handler);
}
}
public static void readFirstSheet(InputStream inputStream, SheetRowHandler handler) throws IOException {
runRead(inputStream, false, handler);
}
public static void readAllSheets(File file, SheetRowHandler handler) throws IOException {
try (InputStream inputStream = new FileInputStream(file)) {
readAllSheets(inputStream, handler);
}
}
public static void readAllSheets(InputStream inputStream, SheetRowHandler handler) throws IOException {
runRead(inputStream, true, handler);
}
private static void runRead(InputStream inputStream, boolean readAllSheets, SheetRowHandler handler) throws IOException {
StreamingListener listener = new StreamingListener(handler);
try {
if (readAllSheets) {
EasyExcel.read(inputStream, listener).headRowNumber(1).doReadAll();
} else {
EasyExcel.read(inputStream, listener).sheet(0).headRowNumber(1).doRead();
}
} catch (WrappedRuntimeException ex) {
if (ex.getCause() instanceof IOException ioException) {
throw ioException;
}
throw ex;
}
}
public interface SheetRowHandler {
default void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) throws Exception {
}
void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception;
}
private static final class StreamingListener extends AnalysisEventListener<Map<Integer, String>> {
private final SheetRowHandler handler;
private Map<Integer, String> currentHeaderMap = Map.of();
private StreamingListener(SheetRowHandler handler) {
this.handler = handler;
}
@Override
public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
Map<Integer, String> normalizedHeadMap = new LinkedHashMap<>();
for (Map.Entry<Integer, String> entry : headMap.entrySet()) {
normalizedHeadMap.put(entry.getKey(), entry.getValue());
}
currentHeaderMap = normalizedHeadMap;
try {
handler.onHeader(sheetName(context), sheetNo(context), currentHeaderMap);
} catch (RuntimeException ex) {
throw ex;
} catch (Exception ex) {
throw new WrappedRuntimeException(ex);
}
}
@Override
public void invoke(Map<Integer, String> data, AnalysisContext context) {
Map<Integer, String> normalizedRow = new LinkedHashMap<>();
for (Map.Entry<Integer, String> entry : data.entrySet()) {
normalizedRow.put(entry.getKey(), entry.getValue());
}
try {
handler.onRow(
sheetName(context),
sheetNo(context),
context.readRowHolder().getRowIndex(),
currentHeaderMap,
normalizedRow
);
} catch (RuntimeException ex) {
throw ex;
} catch (Exception ex) {
throw new WrappedRuntimeException(ex);
}
}
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
}
private String sheetName(AnalysisContext context) {
return context.readSheetHolder() == null ? "" : context.readSheetHolder().getSheetName();
}
private Integer sheetNo(AnalysisContext context) {
return context.readSheetHolder() == null ? null : context.readSheetHolder().getSheetNo();
}
}
private static final class WrappedRuntimeException extends RuntimeException {
private WrappedRuntimeException(Throwable cause) {
super(cause);
}
}
}

View File

@@ -18,18 +18,18 @@ public class DeleteBrandProgressProperties {
* 商品风险PRODUCT_RISK_RESOLVERUNNING 超时自动收尾/失败,按分钟计算;
* 与删除品牌共用同一定时调度。
*/
private long productRiskStaleTimeoutMinutes = 10;
private long productRiskInitialTimeoutMinutes = 10;
private long productRiskStaleTimeoutMinutes = 20;
private long productRiskInitialTimeoutMinutes = 20;
/**
* Price track RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long priceTrackStaleTimeoutMinutes = 10;
private long priceTrackInitialTimeoutMinutes = 10;
private long priceTrackStaleTimeoutMinutes = 20;
private long priceTrackInitialTimeoutMinutes = 20;
/**
* Shop match RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long shopMatchStaleTimeoutMinutes = 10;
private long shopMatchInitialTimeoutMinutes = 10;
private long shopMatchStaleTimeoutMinutes = 20;
private long shopMatchInitialTimeoutMinutes = 20;
}

View File

@@ -9,7 +9,11 @@ import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.time.Duration;
import java.time.Instant;
@@ -94,21 +98,28 @@ public class BrandTaskProgressCacheService {
public void saveParsedPayload(Long taskId, Object payload) {
try {
stringRedisTemplate.opsForValue().set(buildPayloadKey(taskId), objectMapper.writeValueAsString(payload), ttl());
Files.createDirectories(buildTaskDir(taskId));
Files.writeString(
buildPayloadFile(taskId),
objectMapper.writeValueAsString(payload),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE
);
} catch (Exception ex) {
throw new BusinessException("暂存品牌原始数据失败");
throw new BusinessException("鏆傚瓨鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
public <T> T getParsedPayload(Long taskId, TypeReference<T> typeReference) {
String raw = stringRedisTemplate.opsForValue().get(buildPayloadKey(taskId));
if (raw == null || raw.isBlank()) {
Path payloadFile = buildPayloadFile(taskId);
if (!Files.isRegularFile(payloadFile)) {
return null;
}
try {
return objectMapper.readValue(raw, typeReference);
return objectMapper.readValue(Files.readString(payloadFile), typeReference);
} catch (Exception ex) {
throw new BusinessException("读取品牌原始数据失败");
throw new BusinessException("璇诲彇鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
@@ -126,7 +137,7 @@ public class BrandTaskProgressCacheService {
int finishedFiles = countCompletedFiles(taskId);
return new ChunkStoreResult(false, false, finishedFiles, aggregate);
}
throw new BusinessException("同一分片重复提交但内容不一致: " + fileUrl + "#" + file.getChunkIndex());
throw new BusinessException("鍚屼竴鍒嗙墖閲嶅鎻愪氦浣嗗唴瀹逛笉涓€鑷? " + fileUrl + "#" + file.getChunkIndex());
}
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl);
@@ -174,7 +185,7 @@ public class BrandTaskProgressCacheService {
try {
return objectMapper.readValue(json, BrandFileAggregateCacheDto.class);
} catch (Exception ex) {
throw new BusinessException("读取品牌文件聚合缓存失败");
throw new BusinessException("璇诲彇鍝佺墝鏂囦欢鑱氬悎缂撳瓨澶辫触");
}
}
@@ -209,11 +220,11 @@ public class BrandTaskProgressCacheService {
public void delete(Long taskId) {
stringRedisTemplate.delete(buildKey(taskId));
stringRedisTemplate.delete(buildPayloadKey(taskId));
stringRedisTemplate.delete(buildFileAggregateKey(taskId));
stringRedisTemplate.delete(buildCompletedFilesKey(taskId));
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
stringRedisTemplate.delete(buildLegacyResultKey(taskId));
deleteTaskDirQuietly(taskId);
}
public void deleteFileState(Long taskId, String fileUrl) {
@@ -236,19 +247,19 @@ public class BrandTaskProgressCacheService {
stringRedisTemplate.opsForHash().put(buildFileAggregateKey(taskId), normalizeFileUrl(aggregate.getFileUrl()), objectMapper.writeValueAsString(aggregate));
refreshTaskKey(buildFileAggregateKey(taskId));
} catch (Exception ex) {
throw new BusinessException("暂存品牌文件聚合结果失败");
throw new BusinessException("鏆傚瓨鍝佺墝鏂囦欢鑱氬悎缁撴灉澶辫触");
}
}
private void validateChunk(BrandCrawlResultFileDto file) {
if (file == null) {
throw new BusinessException("分片不能为空");
throw new BusinessException("鍒嗙墖涓嶈兘涓虹┖");
}
if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0 || file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) {
throw new BusinessException("分片参数不合法");
throw new BusinessException("鍒嗙墖鍙傛暟涓嶅悎娉?");
}
if (file.getFileUrl() == null || file.getFileUrl().isBlank()) {
throw new BusinessException("fileUrl 不能为空");
throw new BusinessException("fileUrl 涓嶈兘涓虹┖");
}
}
@@ -270,7 +281,7 @@ public class BrandTaskProgressCacheService {
private void ensureChunkConsistency(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
if (aggregate.getChunkTotal() != null && file.getChunkTotal() != null && !aggregate.getChunkTotal().equals(file.getChunkTotal())) {
throw new BusinessException("同一文件的 chunkTotal 不一致: " + aggregate.getFileUrl());
throw new BusinessException("鍚屼竴鏂囦欢鐨?chunkTotal 涓嶄竴鑷? " + aggregate.getFileUrl());
}
if (aggregate.getChunkTotal() == null) {
aggregate.setChunkTotal(file.getChunkTotal());
@@ -322,7 +333,7 @@ public class BrandTaskProgressCacheService {
}
return sb.toString();
} catch (Exception ex) {
throw new BusinessException("计算品牌结果分片摘要失败");
throw new BusinessException("璁$畻鍝佺墝缁撴灉鍒嗙墖鎽樿澶辫触");
}
}
@@ -340,8 +351,28 @@ public class BrandTaskProgressCacheService {
return Duration.ofHours(brandProgressProperties.getTtlHours());
}
private String buildPayloadKey(Long taskId) {
return "brand:task:parsed-payload:" + taskId;
private Path buildTaskDir(Long taskId) {
return Path.of(System.getProperty("java.io.tmpdir"), "brand-task-cache", String.valueOf(taskId));
}
private Path buildPayloadFile(Long taskId) {
return buildTaskDir(taskId).resolve("parsed-payload.json");
}
private void deleteTaskDirQuietly(Long taskId) {
Path taskDir = buildTaskDir(taskId);
if (!Files.exists(taskDir)) {
return;
}
try (var walk = Files.walk(taskDir)) {
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (IOException ignored) {
}
});
} catch (IOException ignored) {
}
}
private String buildFileAggregateKey(Long taskId) {
@@ -383,7 +414,7 @@ public class BrandTaskProgressCacheService {
}
return sb.toString();
} catch (Exception ex) {
throw new BusinessException("生成文件缓存键失败");
throw new BusinessException("鐢熸垚鏂囦欢缂撳瓨閿け璐?");
}
}

View File

@@ -327,6 +327,7 @@ public class BrandTaskService {
if (deleted == 0) {
throw new BusinessException("任务不存在或正在执行中无法删除");
}
brandTaskProgressCacheService.delete(taskId);
}
public String resolveDownloadUrl(Long taskId) {

View File

@@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.convert.model.dto.ConvertRunRequest;
import com.nanri.aiimage.modules.convert.model.dto.UploadedSourceFileDto;
@@ -19,15 +20,10 @@ import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import lombok.RequiredArgsConstructor;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.springframework.stereotype.Service;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -242,14 +238,30 @@ public class ConvertRunService {
private List<GeneratedConvertFile> generateOutputFiles(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
List<String> outputFilenames = resolveOutputFilenames(templateEntity);
List<String> rows = buildTxtRows(inputFile, templateEntity);
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
List<GeneratedConvertFile> generatedFiles = new ArrayList<>();
Map<String, BufferedWriter> writers = new LinkedHashMap<>();
for (String outputFilename : outputFilenames) {
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
FileUtil.writeLines(rows, outputFile, StandardCharsets.UTF_8);
generatedFiles.add(new GeneratedConvertFile(outputFilename, outputFile));
writers.put(outputFilename, Files.newBufferedWriter(outputFile.toPath(), StandardCharsets.UTF_8));
}
try {
streamTxtRowsToOutputs(inputFile, templateEntity, writers);
} finally {
IOException closeException = null;
for (BufferedWriter writer : writers.values()) {
try {
writer.close();
} catch (IOException ex) {
if (closeException == null) {
closeException = ex;
}
}
}
if (closeException != null) {
throw closeException;
}
}
return generatedFiles;
}
@@ -266,7 +278,9 @@ public class ConvertRunService {
return List.of(outputFilename);
}
private List<String> buildTxtRows(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
private void streamTxtRowsToOutputs(File inputFile,
ConvertTemplateEntity templateEntity,
Map<String, BufferedWriter> writers) throws IOException {
List<String> requiredColumns = readStringList(templateEntity.getRequiredSourceColumnsJson());
List<String> headerColumns = readStringList(templateEntity.getHeaderColumnsJson());
Map<String, String> fieldMapping = readStringMap(templateEntity.getFieldMappingJson());
@@ -275,65 +289,60 @@ public class ConvertRunService {
int blankHeaderRowsAfterSchema = templateEntity.getBlankHeaderRowsAfterSchema() == null
? 0
: templateEntity.getBlankHeaderRowsAfterSchema();
long batchId = System.currentTimeMillis();
int[] rowIndex = new int[]{1};
boolean[] headerRead = new boolean[]{false};
Map<String, Integer>[] headerMapHolder = new Map[]{null};
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel header row is empty.");
}
Map<String, Integer> headerMap = new LinkedHashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
String value = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i)));
if (!value.isBlank() && !headerMap.containsKey(value)) {
headerMap.put(value, i);
ExcelStreamReader.readFirstSheet(inputFile, new ExcelStreamReader.SheetRowHandler() {
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) throws Exception {
Map<String, Integer> resolvedHeaderMap = new LinkedHashMap<>();
for (Map.Entry<Integer, String> entry : headerMap.entrySet()) {
String value = normalizeCellText(entry.getValue());
if (!value.isBlank() && !resolvedHeaderMap.containsKey(value)) {
resolvedHeaderMap.put(value, entry.getKey());
}
}
List<String> missing = requiredColumns.stream()
.filter(column -> !resolvedHeaderMap.containsKey(column))
.toList();
if (!missing.isEmpty()) {
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
}
headerRead[0] = true;
headerMapHolder[0] = resolvedHeaderMap;
for (String line : preambleLines) {
writeLineToAll(writers, line);
}
writeLineToAll(writers, String.join("\t", headerColumns));
for (int i = 0; i < blankHeaderRowsAfterSchema; i++) {
writeLineToAll(writers, "\t".repeat(Math.max(0, headerColumns.size() - 1)));
}
}
List<String> missing = requiredColumns.stream()
.filter(column -> !headerMap.containsKey(column))
.toList();
if (!missing.isEmpty()) {
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
}
List<String> rows = new ArrayList<>();
rows.addAll(preambleLines);
rows.add(String.join("\t", headerColumns));
for (int i = 0; i < blankHeaderRowsAfterSchema; i++) {
rows.add("\t".repeat(Math.max(0, headerColumns.size() - 1)));
}
long batchId = System.currentTimeMillis();
int rowIndex = 1;
int asinIndex = headerMap.getOrDefault("ASIN", -1);
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
@Override
public void onRow(String sheetName, Integer sheetNo, int excelRowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception {
Map<String, Integer> resolvedHeaderMap = headerMapHolder[0];
if (resolvedHeaderMap == null) {
throw new BusinessException("Excel header row is empty.");
}
String asin = asinIndex >= 0 ? normalizeCellText(formatter.formatCellValue(row.getCell(asinIndex))) : "";
int asinIndex = resolvedHeaderMap.getOrDefault("ASIN", -1);
String asin = asinIndex >= 0 ? normalizeCellText(rowMap.get(asinIndex)) : "";
if (asin.isBlank()) {
continue;
return;
}
List<String> lineValues = new ArrayList<>();
for (String column : headerColumns) {
if ("sku".equals(column)) {
lineValues.add(batchId + "-" + rowIndex);
lineValues.add(batchId + "-" + rowIndex[0]);
continue;
}
if (fieldMapping.containsKey(column)) {
String sourceColumn = fieldMapping.get(column);
Integer sourceIndex = headerMap.get(sourceColumn);
String value = sourceIndex == null
? ""
: normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
lineValues.add(value);
Integer sourceIndex = resolvedHeaderMap.get(sourceColumn);
lineValues.add(sourceIndex == null ? "" : normalizeCellText(rowMap.get(sourceIndex)));
continue;
}
if (defaults.containsKey(column)) {
@@ -343,10 +352,19 @@ public class ConvertRunService {
lineValues.add("");
}
rows.add(String.join("\t", lineValues));
rowIndex++;
writeLineToAll(writers, String.join("\t", lineValues));
rowIndex[0]++;
}
return rows;
});
if (!headerRead[0]) {
throw new BusinessException("Excel header row is empty.");
}
}
private void writeLineToAll(Map<String, BufferedWriter> writers, String line) throws IOException {
for (BufferedWriter writer : writers.values()) {
writer.write(line);
writer.newLine();
}
}

View File

@@ -20,7 +20,7 @@ import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
@@ -232,7 +232,7 @@ public class DedupeRunService {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
XSSFWorkbook outputWorkbook = new XSSFWorkbook()) {
SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
@@ -275,7 +275,6 @@ public class DedupeRunService {
// The preliminary global scan for mainIdHasSubIdMap was removed.
// We now determine subset existence contextually (per group) during the main loop.
}
List<Row> candidateRows = new ArrayList<>();
Set<String> candidateAsinValues = new HashSet<>();
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
@@ -288,7 +287,6 @@ public class DedupeRunService {
continue;
}
}
candidateRows.add(row);
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank()) {
@@ -300,7 +298,17 @@ public class DedupeRunService {
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
Set<String> writtenAsinValues = new HashSet<>();
int outputRowIndex = 1;
for (Row row : candidateRows) {
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
if (idColumnIndex != null) {
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, sheet, rowNum, idColumnIndex, formatter)) {
continue;
}
}
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank()) {
@@ -324,6 +332,7 @@ public class DedupeRunService {
try (FileOutputStream fos = new FileOutputStream(outputFile)) {
outputWorkbook.write(fos);
}
outputWorkbook.dispose();
}
}

View File

@@ -23,7 +23,9 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -114,9 +116,9 @@ public class DedupeTotalDataService {
importProgressMap.put(importId, progress);
try {
byte[] fileBytes = file.getBytes();
File tempFile = saveMultipartToTempFile(file);
String filename = file.getOriginalFilename();
Thread.ofVirtual().start(() -> runImportTask(importId, fileBytes, filename));
Thread.ofVirtual().start(() -> runImportTask(importId, tempFile, filename));
} catch (Exception e) {
importProgressMap.remove(importId);
throw new BusinessException("读取上传文件失败");
@@ -150,9 +152,9 @@ public class DedupeTotalDataService {
deleteImportProgressMap.put(importId, progress);
try {
byte[] fileBytes = file.getBytes();
File tempFile = saveMultipartToTempFile(file);
String filename = file.getOriginalFilename();
Thread.ofVirtual().start(() -> runDeleteImportTask(importId, fileBytes, filename));
Thread.ofVirtual().start(() -> runDeleteImportTask(importId, tempFile, filename));
} catch (Exception e) {
deleteImportProgressMap.remove(importId);
throw new BusinessException("读取上传文件失败");
@@ -211,6 +213,74 @@ public class DedupeTotalDataService {
}
}
private void runDeleteImportTask(String importId, File tempFile, String filename) {
DedupeTotalDataImportProgressVo progress = deleteImportProgressMap.get(importId);
if (progress == null) {
return;
}
progress.setStatus("running");
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
DedupeTotalDataImportVo result = deleteFromExcelInternal(inputStream, filename, progress);
progress.setTotalRows(result.getTotalRows());
progress.setAsinCount(result.getAsinCount());
progress.setInsertedCount(result.getInsertedCount());
progress.setSkippedCount(result.getSkippedCount());
progress.setProcessedRows(result.getTotalRows());
progress.setStatus("success");
} catch (Exception e) {
progress.setStatus("failed");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "鍒犻櫎 Excel 鍖归厤鏁版嵁澶辫触");
} finally {
deleteQuietly(tempFile);
}
}
private void runImportTask(String importId, File tempFile, String filename) {
DedupeTotalDataImportProgressVo progress = importProgressMap.get(importId);
if (progress == null) {
return;
}
progress.setStatus("running");
try (InputStream inputStream = Files.newInputStream(tempFile.toPath())) {
DedupeTotalDataImportVo result = importFromExcelInternal(inputStream, filename, progress);
progress.setTotalRows(result.getTotalRows());
progress.setAsinCount(result.getAsinCount());
progress.setInsertedCount(result.getInsertedCount());
progress.setSkippedCount(result.getSkippedCount());
progress.setProcessedRows(result.getTotalRows());
progress.setStatus("success");
} catch (Exception e) {
progress.setStatus("failed");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "瀵煎叆 Excel 澶辫触");
} finally {
deleteQuietly(tempFile);
}
}
private File saveMultipartToTempFile(MultipartFile file) throws Exception {
String suffix = ".xlsx";
String name = file.getOriginalFilename();
if (name != null) {
int idx = name.lastIndexOf('.');
if (idx >= 0) {
suffix = name.substring(idx);
}
}
File tempFile = File.createTempFile("dedupe-total-data-", suffix);
file.transferTo(tempFile);
return tempFile;
}
private void deleteQuietly(File file) {
if (file == null) {
return;
}
try {
Files.deleteIfExists(file.toPath());
} catch (Exception ignored) {
}
}
@Transactional
public DedupeTotalDataImportVo importFromExcel(MultipartFile file) {
if (file == null || file.isEmpty()) {

View File

@@ -85,6 +85,12 @@ public class DeleteBrandRunController {
return ApiResponse.success(deleteBrandRunService.getTaskDetails(request.getTaskIds()));
}
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量获取删除品牌任务进度摘要", description = "仅返回任务基础状态和行进度摘要,用于前端轮询降载。")
public ApiResponse<DeleteBrandTaskBatchVo> getTaskProgressBatch(@Valid @RequestBody DeleteBrandTaskBatchRequest request) {
return ApiResponse.success(deleteBrandRunService.getTaskProgress(request.getTaskIds()));
}
@GetMapping("/tasks/{taskId}/deletion-status")
@Operation(summary = "获取删除品牌任务删除状态", description = "供 Python/插件按 taskId 查询该删除品牌任务是否已被前端删空。")
public ApiResponse<DeleteBrandTaskDeletionStatusVo> getTaskDeletionStatus(

View File

@@ -544,10 +544,87 @@ public class DeleteBrandRunService {
return vo;
}
public com.nanri.aiimage.modules.deletebrand.model.vo.DeleteBrandTaskBatchVo getTaskProgress(java.util.List<Long> taskIds) {
com.nanri.aiimage.modules.deletebrand.model.vo.DeleteBrandTaskBatchVo vo = new com.nanri.aiimage.modules.deletebrand.model.vo.DeleteBrandTaskBatchVo();
if (taskIds == null || taskIds.isEmpty()) {
return vo;
}
java.util.List<Long> normalizedTaskIds = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.limit(50)
.toList();
if (normalizedTaskIds.isEmpty()) {
return vo;
}
java.util.Map<Long, FileTaskEntity> cachedTasks = deleteBrandTaskCacheService.getTaskCacheBatch(normalizedTaskIds);
java.util.List<Long> missingTaskIds = new java.util.ArrayList<>();
for (Long taskId : normalizedTaskIds) {
if (!cachedTasks.containsKey(taskId)) {
missingTaskIds.add(taskId);
}
}
java.util.Map<Long, FileTaskEntity> taskById = new java.util.LinkedHashMap<>(cachedTasks);
if (!missingTaskIds.isEmpty()) {
java.util.List<FileTaskEntity> dbTasks = fileTaskMapper.selectBatchIds(missingTaskIds).stream()
.filter(task -> task != null && MODULE_TYPE.equals(task.getModuleType()))
.toList();
for (FileTaskEntity dbTask : dbTasks) {
taskById.put(dbTask.getId(), dbTask);
if ("RUNNING".equals(dbTask.getStatus())) {
deleteBrandTaskCacheService.saveTaskCache(dbTask);
}
}
}
java.util.Map<Long, java.util.Map<Object, Object>> progressByTaskId = deleteBrandTaskCacheService.getProgressBatch(normalizedTaskIds);
for (Long taskId : normalizedTaskIds) {
FileTaskEntity task = taskById.get(taskId);
if (task == null) {
vo.getMissingTaskIds().add(taskId);
continue;
}
vo.getItems().add(buildTaskProgressDetail(task, progressByTaskId.get(taskId)));
}
return vo;
}
private DeleteBrandTaskDetailVo buildTaskDetail(FileTaskEntity task) {
return buildTaskDetail(task, null);
}
private DeleteBrandTaskDetailVo buildTaskProgressDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) {
task = refreshIndexedMatchesIfNeeded(task);
DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo();
taskVo.setId(task.getId());
taskVo.setTaskNo(task.getTaskNo());
taskVo.setStatus(task.getStatus());
taskVo.setSourceFileCount(task.getSourceFileCount());
taskVo.setSuccessFileCount(task.getSuccessFileCount());
taskVo.setFailedFileCount(task.getFailedFileCount());
taskVo.setErrorMessage(task.getErrorMessage());
taskVo.setCreatedAt(task.getCreatedAt());
taskVo.setUpdatedAt(task.getUpdatedAt());
taskVo.setFinishedAt(task.getFinishedAt());
if ("SUCCESS".equals(task.getStatus())) {
taskVo.setDownloadUrl(resolveTaskDownloadUrl(task.getId()));
taskVo.setDownloadFilename(resolveTaskDownloadFilename(task.getId()));
} else {
taskVo.setDownloadUrl(null);
taskVo.setDownloadFilename(null);
}
DeleteBrandTaskDetailVo detail = new DeleteBrandTaskDetailVo();
detail.setTask(taskVo);
detail.setLine_progress(buildLineProgress(task.getId(), progress));
detail.setFileProgress(java.util.List.of());
return detail;
}
private DeleteBrandTaskDetailVo buildTaskDetail(FileTaskEntity task, java.util.Map<Object, Object> progress) {
task = refreshIndexedMatchesIfNeeded(task);
DeleteBrandTaskItemVo taskVo = new DeleteBrandTaskItemVo();

Some files were not shown because too many files have changed in this diff Show More